Validate strings as UTF-8 during parsing

.envrc

@@ -1,2 +0,0 @@
-source_env ..
-use nix

lock.json

@@ -1,5 +1,17 @@
 {
   "depends": [
+    {
+      "method": "fetchzip",
+      "packages": [
+        "bigints"
+      ],
+      "path": "/nix/store/jvrm392g8adfsgf36prgwkbyd7vh5jsw-source",
+      "ref": "20231006",
+      "rev": "86ea14d31eea9275e1408ca34e6bfe9c99989a96",
+      "sha256": "15pcpmnk1bnw3k8769rjzcpg00nahyrypwbxs88jnwr4aczp99j4",
+      "srcDir": "src",
+      "url": "https://github.com/ehmry/nim-bigints/archive/86ea14d31eea9275e1408ca34e6bfe9c99989a96.tar.gz"
+    },
     {
       "method": "fetchzip",
       "packages": [

preserves.nimble

@@ -1,6 +1,6 @@
 # Package
 
-version = "20231220"
+version = "20231224"
 author        = "Emery Hemingway"
 description   = "data model and serialization format"
 license       = "Unlicense"
@@ -11,4 +11,4 @@ bin           = @["preserves/preserves_schema_nim", "preserves/private/preserves
 
 # Dependencies
 
-requires "nim >= 2.0.0", "compiler >= 1.4.8", "https://github.com/zevv/npeg.git >= 1.2.1"
+requires "nim >= 2.0.0", "compiler >= 1.4.8", "https://github.com/zevv/npeg.git >= 1.2.1", "https://github.com/ehmry/nim-bigints.git >= 20231006"

shell.nix

@@ -1,5 +0,0 @@
-{ pkgs ? import <nixpkgs> { } }:
-pkgs.buildNimPackage {
-  name = "dummy";
-  lockFile = ./lock.json;
-}

src/preserves.nim

@@ -2,12 +2,12 @@
 # SPDX-License-Identifier: Unlicense
 
 import std/[options, sets, sequtils, strutils, tables, typetraits]
-import ./preserves/private/macros
-
 from std/algorithm import sort
 from std/json import escapeJson, escapeJsonUnquoted
-import ./preserves/private/[encoding, decoding, dot, parsing, texts, values]
 
+import bigints
+
+import ./preserves/private/[encoding, decoding, dot, macros, parsing, texts, values]
 export encoding, decoding, parsing, texts, values
 
 when defined(tracePreserves):
@@ -37,6 +37,12 @@ proc cannonicalize*[E](pr: var Preserve[E]) =
   else:
     discard
 
+proc getInt*(pr: Preserve): int =
+  case pr.kind
+  of pkRegister: result = pr.register
+  of pkBigInt: result = pr.bigint.toInt[int].get
+  else: raise newException(ValueError, "Preserves value is not a integer")
+
 proc getOrDefault(pr: Preserve; key: Preserve): Preserve =
   ## Retrieves the value of `pr[key]` if `pr` is a dictionary containing `key`
   ## or returns the `#f` Preserves value.
@@ -83,9 +89,9 @@ func step*(pr, idx: Preserve): Option[Preserve] =
       if k == idx:
         result = some(v)
         break
-  elif (pr.isRecord or pr.isSequence) and idx.isInteger:
-    let i = int idx.int
-    if i < pr.len:
+  elif (pr.isRecord or pr.isSequence):
+    var i: int
+    if i.fromPreserve(pr) and i < pr.len:
       result = some(pr[i])
 
 func step*(pr: Preserve; path: varargs[Preserve]): Option[Preserve] =
@@ -206,12 +212,16 @@ func isFloat*(pr: Preserve): bool {.inline.} = pr.kind == pkFloat
 func isDouble*(pr: Preserve): bool {.inline.} = pr.kind == pkDouble
   ## Check if ``pr`` is a Preserve double.
 
-func isInteger*(pr: Preserve): bool {.inline.} = pr.kind == pkSignedInteger
+func isInteger*(pr: Preserve): bool {.inline.} =
   ## Check if ``pr`` is a Preserve integer.
+  pr.kind == pkRegister or pr.kind == pkBigInt
 
 func isInteger*(pr: Preserve; i: SomeInteger): bool {.inline.} =
   ## Check if ``pr`` is a Preserve integer equivalent to `i`.
-  pr.kind == pkSignedInteger and pr.int == BiggestInt(i)
+  case pr.kind
+  of pkRegister: pr.register == i.int
+  of pkBigInt: pr.int == i.initBigInt
+  else: false
 
 func isString*(pr: Preserve): bool {.inline.} = pr.kind == pkString
   ## Check if ``pr`` is a Preserve text string.
@@ -374,14 +384,21 @@ proc toPreserve*[T](x: T; E = void): Preserve[E] {.gcsafe.} =
     for xf in fields(x):
       result.sequence.add(toPreserve(xf, E))
   elif T is Ordinal:
-    result = Preserve[E](kind: pkSignedInteger, int: x.ord.BiggestInt)
+    var n = ord x
+    if n <= high(int):
+      result = Preserve[E](kind: pkRegister, register: n.int)
+    else:
+      result = Preserve[E](kind: pkBigInt, bigint: n.initBigInt)
   elif T is ptr | ref:
     if system.`==`(x, nil): result = toSymbol("null", E)
     else: result = toPreserve(x[], E)
   elif T is string:
     result = Preserve[E](kind: pkString, string: x)
   elif T is SomeInteger:
-    result = Preserve[E](kind: pkSignedInteger, int: x.BiggestInt)
+    if x <= high(int):
+      result = Preserve[E](kind: pkRegister, register: x.int)
+    else:
+      result = Preserve[E](kind: pkBigInt, bigint: x.initBigInt)
   elif T is Symbol:
     result = Preserve[E](kind: pkSymbol, symbol: x)
   elif T is distinct:
@@ -512,9 +529,14 @@ proc fromPreserve*[T, E](v: var T; pr: Preserve[E]): bool {.gcsafe.} =
       v = pr.bool
       result = true
   elif T is SomeInteger:
-    if pr.kind == pkSignedInteger:
-      v = T(pr.int)
-      result = true
+    if pr.kind == pkRegister:
+      result = pr.register.T < high(T)
+      if result:
+        v = T pr.register
+    elif pr.kind == pkBigInt:
+      var o = toInt[T](pr.bigint)
+      result = o.isSome
+      if result: v = o.get
   elif T is seq[byte]:
     if pr.kind == pkByteString:
       v = pr.bytes
@@ -542,9 +564,14 @@ proc fromPreserve*[T, E](v: var T; pr: Preserve[E]): bool {.gcsafe.} =
       result = true
     else: discard
   elif T is Ordinal | SomeInteger:
-    if pr.kind == pkSignedInteger:
-      v = (T)pr.int
-      result = true
+    if pr.kind == pkRegister:
+      v = T(pr.register)
+      result = int(v) == T(pr.register)
+    elif pr.kind == pkBigInt:
+      var o = pr.bigint.toInt[T]
+      if o.isSome:
+        v = get o
+        result = true
   elif T is string:
     if pr.kind == pkString:
       v = pr.string
@@ -731,7 +758,9 @@ proc apply*[E](result: var Preserve[E]; op: proc(_: var Preserve[E]) {.gcsafe.})
   proc recurse(result: var Preserve[E]) = apply(result, op)
   op(result)
   case result.kind
-  of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol, pkEmbedded: discard
+  of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt,
+      pkString, pkByteString, pkSymbol, pkEmbedded:
+    discard
   of pkRecord:
     apply(result.record, recurse)
   of pkSequence:
@@ -754,7 +783,8 @@ proc mapEmbeds*(pr: sink Preserve[void]; E: typedesc): Preserve[E] =
       raise newException(ValueError, "failed to convert " & $E & " from " & $pr)
   else:
     case pr.kind
-    of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol:
+    of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt,
+        pkString, pkByteString, pkSymbol:
       result = cast[Preserve[E]](pr)
     of pkRecord:
       result = Preserve[E](kind: pr.kind)
@@ -801,7 +831,8 @@ proc mapEmbeds*[A, B](pr: sink Preserve[A]; op: proc (v: A): B): Preserve[B] =
     result = embed op(e)
   else:
     case pr.kind
-    of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol:
+    of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt,
+        pkString, pkByteString, pkSymbol:
       result = cast[Preserve[B]](pr)
     of pkRecord:
       result = Preserve[B](kind: pr.kind)
@@ -827,7 +858,8 @@ proc contract*[E](pr: sink Preserve[E]; op: proc (v: E): Preserve[void] {.gcsafe
   ## Convert `Preserve[E]` to `Preserve[void]` using an `E → Preserve[void]` procedure.
   if not pr.embedded:
     case pr.kind
-    of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol:
+    of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt,
+        pkString, pkByteString, pkSymbol:
       result = cast[Preserve[void]](pr)
     of pkRecord:
       result = Preserve[void](kind: pr.kind)
@@ -855,7 +887,8 @@ proc expand*[E](pr: sink Preserve[void]; op: proc (v: Preserve[void]): Preserve[
     result = op(pr)
   else:
     case pr.kind
-    of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol:
+    of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt,
+        pkString, pkByteString, pkSymbol:
       result = cast[Preserve[E]](pr)
     of pkRecord:
       result = Preserve[E](kind: pr.kind)

src/preserves/Tupfile

@@ -5,3 +5,4 @@ NIM_FLAGS += --path:$(TUP_CWD)/..
 DOT_FILES = ../../Document.dot ../../Schema.dot
 : schemac.nim |> !nim_bin |> $(BIN_DIR)/%B | $(DOT_FILES) $(BIN_DIR)/<%B>
 : foreach $(DOT_FILES) |> dot -Tsvg -LO %f > %o |> ../../%B-Grammer-Graph.svg
+: jsonhooks.nim |> !nim_run |>

src/preserves/jsonhooks.nim

@@ -7,11 +7,11 @@ import ../preserves
 proc toPreserveHook*(js: JsonNode; E: typedesc): Preserve[E] =
   case js.kind
   of JString:
-    result = Preserve[E](kind: pkString, string: js.str)
+    result = js.str.toPreserve(E)
   of JInt:
-    result = Preserve[E](kind: pkSignedInteger, int: js.num)
+    result = js.num.toPreserve(E)
   of JFloat:
-    result = Preserve[E](kind: pkDouble, double: js.fnum)
+    result = js.fnum.toPreserve(E)
   of JBool:
     result = case js.bval
       of false: toSymbol("false", E)
@@ -28,7 +28,7 @@ proc toPreserveHook*(js: JsonNode; E: typedesc): Preserve[E] =
     for i, e in js.elems:
       result.sequence[i] = toPreserveHook(e, E)
 
-proc fromPreserveHook*[E](js: var JsonNode; prs: Preserve[E]): bool =
+proc fromPreserveHook*[E](js: var JsonNode; pr: Preserve[E]): bool =
   runnableExamples:
     import std/json
     var js = JsonNode()
@@ -36,19 +36,19 @@ proc fromPreserveHook*[E](js: var JsonNode; prs: Preserve[E]): bool =
     assert fromPreserveHook(js, pr)
     fromJsonHook(pr, js)
     js = toJsonHook(pr)
-  case prs.kind:
+  case pr.kind:
   of pkBoolean:
-    js = newJBool(prs.bool)
+    js = newJBool(pr.bool)
   of pkFloat:
-    js = newJFloat(prs.float)
+    js = newJFloat(pr.float)
   of pkDouble:
-    js = newJFloat(prs.double)
-  of pkSignedInteger:
-    js = newJInt(prs.int)
+    js = newJFloat(pr.double)
+  of pkRegister:
+    js = newJInt(pr.register)
   of pkString:
-    js = newJString(prs.string)
+    js = newJString(pr.string)
   of pkSymbol:
-    case prs.symbol.string
+    case pr.symbol.string
     of "false":
       js = newJBool(false)
     of "true":
@@ -59,21 +59,21 @@ proc fromPreserveHook*[E](js: var JsonNode; prs: Preserve[E]): bool =
       return false
   of pkSequence:
     js = newJArray()
-    js.elems.setLen(prs.sequence.len)
-    for i, val in prs.sequence:
+    js.elems.setLen(pr.sequence.len)
+    for i, val in pr.sequence:
       if not fromPreserveHook(js.elems[i], val):
         return false
   of pkSet:
     js = newJArray()
-    js.elems.setLen(prs.set.len)
+    js.elems.setLen(pr.set.len)
     var i: int
-    for val in prs.set:
+    for val in pr.set:
       if not fromPreserveHook(js.elems[i], val):
         return false
       inc i
   of pkDictionary:
     js = newJObject()
-    for (key, val) in prs.dict.items:
+    for (key, val) in pr.dict.items:
       case key.kind
       of pkSymbol:
         var jsVal: JsonNode

src/preserves/pegs.nim

@@ -9,8 +9,9 @@ when defined(nimHasUsed): {.used.}
 
 grammar "Preserves":
 
-  ws <- *(' ' | '\t' | '\r' | '\n' )
+  ws <- *{ ' ', '\t', '\r', '\n' }
   commas <- *(ws * ',') * ws
+  delimiter <- { ' ', '\t', '\r', '\n', '<', '>', '[', ']', '{', '}', '#', ':', '"', '|', '@', ';', ',' } | !1
 
   Document <- Value * ws * !1
 
@@ -31,18 +32,18 @@ grammar "Preserves":
 
   Set <- "#{" * *(commas * Value) * commas * '}'
 
-  Boolean <- "#f" | "#t"
+  Boolean <- '#' * {'f', 't'} * &delimiter
 
-  nat <- '0' | (Digit-'0') * *Digit
-  int <- ?'-' * nat
+  nat <- +Digit
+  int <- ?('-'|'+') * nat
   frac <- '.' * +Digit
   exp <- 'e' * ?('-'|'+') * +Digit
   flt <- int * ((frac * exp) | frac | exp)
 
-  Float <- >flt * 'f'
-  Double <- flt
+  Float <- >flt * {'f','F'} * &delimiter
+  Double <- flt * &delimiter
 
-  SignedInteger <- int
+  SignedInteger <- int * &delimiter
 
   char <- unescaped | '|' | (escape * (escaped | '"' | ('u' * Xdigit[4])))
   String <- '"' * >(*char) * '"'
@@ -56,12 +57,12 @@ grammar "Preserves":
   binchar <- binunescaped | (escape * (escaped | '"' | ('x' * Xdigit[2])))
   binunescaped <- {' '..'!', '#'..'[', ']'..'~'}
 
-  symchar <- (utf8.any - { 0..127, '\\', '|' }) | (escape * (escaped | ('u' * Xdigit[4]))) | "\\|"
+  symchar <- (utf8.any - {'\\', '|'}) | (escape * (escaped | ('u' * Xdigit[4]))) | "\\|"
   QuotedSymbol <- '|' * >(*symchar) * '|'
   sympunct <- {'~', '!', '$', '%', '^', '&', '*', '?', '_', '=', '+', '-', '/', '.'}
   symuchar <- utf8.any - { 0..127 }
   SymbolOrNumber <- >(+(Alpha | Digit | sympunct | symuchar))
-  Symbol <- QuotedSymbol | SymbolOrNumber
+  Symbol <- QuotedSymbol | (SymbolOrNumber * &delimiter)
 
   Embedded <- "#!" * Value
 

src/preserves/private/decoding.nim

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Unlicense
 
 import std/[endians, streams, strutils]
+import bigints
 import ./values
 
 proc readVarint(s: Stream): uint =
@@ -29,31 +30,56 @@ proc decodePreserves*(s: Stream; E = void): Preserve[E] =
     result = decodePreserves(s, E)
     result.embedded = true
   of 0x87:
-    let n = s.readUint8()
+    var N: int
+    let n = int s.readUint8()
     case n
     of 4:
-      when system.cpuEndian == bigEndian:
-        result = Preserve[E](kind: pkFloat, float: s.readFloat32())
-      else:
-        result = Preserve[E](kind: pkFloat)
-        var be = s.readFloat32()
-        swapEndian32(result.float.addr, be.addr)
+      result = Preserve[E](kind: pkFloat)
+      var buf: uint32
+      N = s.readData(addr buf, sizeof(buf))
+      bigEndian32(addr result.float, addr buf)
     of 8:
-      when system.cpuEndian == bigEndian:
-        result = Preserve[E](kind: pkDouble, double: s.readFloat64())
-      else:
-        result = Preserve[E](kind: pkDouble)
-        var be = s.readFloat64()
-        swapEndian64(result.double.addr, be.addr)
+      result = Preserve[E](kind: pkDouble)
+      var buf: uint64
+      N = s.readData(addr buf, sizeof(buf))
+      bigEndian64(addr result.double, addr buf)
     else:
       raise newException(IOError, "unhandled IEEE754 value of " & $n & " bytes")
-  of 0xb1:
-    var data = newString(s.readVarint())
-    if data.len > 0:
-      let n = s.readData(unsafeAddr data[0], data.len)
-      if n != data.len:
+    if N != n: raise newException(IOError, "short read")
+  of 0xb0:
+    var n = int s.readVarint()
+    if n <= sizeof(int):
+      result = Preserve[E](kind: pkRegister)
+      if n > 0:
+        var
+          buf: array[sizeof(int), byte]
+          off = buf.len - n
+        if s.readData(addr buf[off], n) != n:
+          raise newException(IOError, "short read")
+        if off > 0:
+          var fill: uint8 = if (buf[off] and 0x80) == 0x80'u8: 0xff else: 0x00'u8
+          for i in 0..<off: buf[i] = fill
+        when buf.len == 4:
+          bigEndian32(addr result.register, addr buf[0])
+        elif buf.len == 8:
+          bigEndian64(addr result.register, addr buf[0])
+        else: {.error: "int size " & $buf.len & " not supported here".}
+    else:
+      result = Preserve[E](kind: pkBigInt)
+      var buf = newSeq[byte](n)
+      if s.readData(addr buf[0], buf.len) != n:
+        raise newException(IOError, "short read")
+      if (buf[0] and 0x80) == 0x80:
+        for i, b in buf: buf[i] = not b
+        result.bigint.fromBytes(buf, bigEndian)
+        result.bigint = -(result.bigint.succ)
+      else:
+        result.bigint.fromBytes(buf, bigEndian)
+  of 0xb1:
+    result = Preserve[E](kind: pkString, string: newString(s.readVarint()))
+    if result.string.len > 0:
+      if s.readData(addr result.string[0], result.string.len) != result.string.len:
         raise newException(IOError, "short read")
-    result = Preserve[E](kind: pkString, string: data)
   of 0xb2:
     var data = newSeq[byte](s.readVarint())
     if data.len > 0:
@@ -90,16 +116,6 @@ proc decodePreserves*(s: Stream; E = void): Preserve[E] =
     while s.peekUint8() != endMarker:
       result[decodePreserves(s, E)] = decodePreserves(s, E)
     discard s.readUint8()
-  of 0xb0:
-    var len = s.readVarint()
-    result = Preserve[E](kind: pkSignedInteger)
-    if len > 0:
-      if (s.peekUint8() and 0x80) == 0x80:
-        result.int = BiggestInt -1
-      while len > 0:
-        result.int = (result.int shl 8) + s.readUint8().BiggestInt
-        dec(len)
-
   of endMarker:
     raise newException(ValueError, "invalid Preserves stream")
   else:

src/preserves/private/encoding.nim

@@ -1,7 +1,8 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[endians, options, sets, sequtils, streams, tables, typetraits]
+import std/[endians, streams]
+import bigints
 import ./values
 
 proc writeVarint(s: Stream; n: Natural) =
@@ -35,25 +36,44 @@ proc write*[E](str: Stream; pr: Preserve[E]) =
       var be: float64
       swapEndian64(be.addr, pr.double.unsafeAddr)
       str.write(be)
-  of pkSignedInteger:
-    if pr.int == 0:
-      str.write("\xb0\x00")
+  of pkRegister:
+    if pr.register == 0: str.write("\xb0\x00")
     else:
-      var bitCount = 1'u8
-      if pr.int < 0:
-        while ((not pr.int) shr bitCount) != 0:
-          inc(bitCount)
+      const bufLen = sizeof(int)
+      var buf: array[bufLen, byte]
+      when bufLen == 4: bigEndian32(addr buf[0], addr pr.register)
+      elif bufLen == 8: bigEndian64(addr buf[0], addr pr.register)
+      else: {.error: "int size " & $bufLen & " not supported here".}
+      if buf[0] != 0x00 and buf[0] != 0xff:
+        str.write(cast[string](buf)) # dumbass hex conversion
       else:
-        while (pr.int shr bitCount) != 0:
-          inc(bitCount)
-      var byteCount = (bitCount + 8) div 8
-      str.write(0xb0'u8)
-      str.writeVarint(byteCount)
-      proc write(n: uint8; i: BiggestInt) =
-        if n > 1:
-          write(n.pred, i shr 8)
-        str.write(i.uint8)
-      write(byteCount, pr.int)
+        var start = 0
+        while start < buf.high and buf[0] == buf[succ start]: inc start
+        if start < buf.high and (buf[succ start] and 0x80) == (buf[0] and 0x80): inc start
+        str.write('\xb0')
+        str.write(uint8(bufLen - start))
+        str.write(cast[string](buf[start..<bufLen]))
+  of pkBigInt:
+    if pr.bigint.isZero: str.write("\xb0\x00")
+    elif pr.bigint.isNegative:
+      var buf = pr.bigint.succ.toBytes(bigEndian)
+      for i, b in buf: buf[i] = not b
+      str.write('\xb0')
+      if (buf[0] and 0x80) != 0x80:
+        str.writeVarint(buf.len.succ)
+        str.write('\xff')
+      else:
+        str.writeVarint(buf.len)
+      str.write(cast[string](buf))
+    else:
+      var buf = pr.bigint.toBytes(bigEndian)
+      str.write('\xb0')
+      if (buf[0] and 0x80) != 0:
+        str.writeVarint(buf.len.succ)
+        str.write('\x00')
+      else:
+        str.writeVarint(buf.len)
+      str.write(cast[string](buf))
   of pkString:
     str.write(0xb1'u8)
     str.writeVarint(pr.string.len)

src/preserves/private/parsing.nim

@@ -1,10 +1,11 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[base64, parseutils, strutils, unicode]
+import std/[base64, options, parseutils, strutils, unicode]
 from std/sequtils import insert
 
-import npeg
+import bigints, npeg
+
 import ../pegs
 import ./decoding, ./values
 
@@ -37,11 +38,30 @@ template unescape*(buf: var string; capture: string) =
       of 't': add(buf, char 0x09)
       of '"': add(buf, char 0x22)
       of 'u':
-        var r: int32
+        var short: uint16
         inc(i)
-        discard parseHex(capture, r, i, 4)
+        discard parseHex(capture, short, i, 4)
         inc(i, 3)
-        add(buf, Rune r)
+        if (short shr 15) == 0:
+          add(buf, Rune(short).toUtf8)
+        elif (short shr 10) == 0b110110:
+          if i+6 >= capture.len:
+            raise newException(ValueError, "Invalid UTF-16 surrogate pair")
+          var rune = uint32(short shl 10) + 0x10000
+          validate(capture[i+1] == '\\')
+          validate(capture[i+2] == 'u')
+          inc(i, 3)
+          discard parseHex(capture, short, i, 4)
+          if (short shr 10) != 0b110111:
+            raise newException(ValueError, "Invalid UTF-16 surrogate pair")
+          inc(i, 3)
+          rune = rune or (short and 0b1111111111)
+          #add(buf, Rune(rune).toUTF8)
+          let j = buf.len
+          buf.setLen(buf.len+4)
+          rune.Rune.fastToUTF8Copy(buf, j, false)
+        else:
+          raise newException(ValueError, "Invalid UTF-16 escape sequence " & capture)
       else:
         validate(false)
     else:
@@ -79,7 +99,7 @@ proc pushHexNibble[T](result: var T; c: char) =
     of '0'..'9': T(ord(c) - ord('0'))
     of 'a'..'f': T(ord(c) - ord('a') + 10)
     of 'A'..'F': T(ord(c) - ord('A') + 10)
-    else: 0
+    else: return
   result = (result shl 4) or n
 
 proc parsePreserves*(text: string): Preserve[void] =
@@ -157,11 +177,19 @@ proc parsePreserves*(text: string): Preserve[void] =
       pushStack Value(kind: pkDouble, double: cast[float64](reg))
 
     Preserves.SignedInteger <- Preserves.SignedInteger:
-      pushStack Value(kind: pkSignedInteger, int: parseInt($0))
+      var
+        big = initBigInt($0)
+        small = toInt[int](big)
+      if small.isSome:
+        pushStack Value(kind: pkRegister, register: small.get)
+      else:
+        pushStack Value(kind: pkBigInt, bigint: big)
 
     Preserves.String <- Preserves.String:
       var v = Value(kind: pkString, string: newStringOfCap(len($1)))
       unescape(v.string, $1)
+      if validateUtf8(v.string) != -1:
+        raise newException(ValueError, "Preserves text contains an invalid UTF-8 sequence")
       pushStack v
 
     Preserves.charByteString <- Preserves.charByteString:
@@ -176,7 +204,9 @@ proc parsePreserves*(text: string): Preserve[void] =
       pushStack Value(kind: pkByteString, bytes: cast[seq[byte]](base64.decode(joinWhitespace($1))))
 
     Preserves.Symbol <- Preserves.Symbol:
-      pushStack Value(kind: pkSymbol, symbol: Symbol $1)
+      var buf = newStringOfCap(len($1))
+      unescape(buf, $1)
+      pushStack Value(kind: pkSymbol, symbol: Symbol buf)
 
     Preserves.Embedded <- Preserves.Embedded:
       var v = stack.pop.value

src/preserves/private/texts.nim

@@ -1,40 +1,47 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[base64, json, options, sets, sequtils, streams, strutils, tables, typetraits]
+import std/[base64, endians, math, sequtils, streams, strutils, unicode]
+import bigints
 import ./values
 
-proc `$`*(s: Symbol): string =
-  let sym = string s
-  if sym.len > 0 and sym[0] in {'A'..'z'} and not sym.anyIt(char(it) in { '\x00'..'\x19', '"', '\\', '|' }):
-    result = sym
-  else:
-    result = newStringOfCap(sym.len shl 1)
-    result.add('|')
-    for c in sym:
-      case c
-      of '\\':
-        result.add("\\\\")
-      of '/':
-        result.add("\\/")
-      of '\x08':
-        result.add("\\b")
-      of '\x0c':
-        result.add("\\f")
-      of '\x0a':
-        result.add("\\n")
-      of '\x0d':
-        result.add("\\r")
-      of '\x09':
-        result.add("\\t")
-      of '|':
-        result.add("\\|")
-      else:
-        result.add(c)
-    result.add('|')
+const hexAlphabet = "0123456789abcdef"
 
 type TextMode* = enum textPreserves, textJson
 
+template writeEscaped(stream: Stream; text: string; delim: char) =
+  const escaped = { '"', '\\', '\b', '\f', '\n', '\r', '\t' }
+  var
+    i: int
+    c: char
+  while i < text.len:
+    c = text[i]
+    case c
+    of delim:
+      write(stream, '\\')
+      write(stream, delim)
+    of '\\': write(stream, "\\\\")
+    of '\b': write(stream, "\\b")
+    of '\f': write(stream, "\\f")
+    of '\n': write(stream, "\\n")
+    of '\r': write(stream, "\\r")
+    of '\t': write(stream, "\\t")
+    of { '\x00'..'\x1f', '\x7f' } - escaped:
+      # do not use \x__ notation because
+      # it is a subset of \u____.
+      write(stream, "\\u00")
+      write(stream, c.uint8.toHex(2))
+    else: write(stream, c)
+    inc i
+
+proc writeSymbol(stream: Stream; sym: string) =
+  if sym.len > 0 and sym[0] in {'A'..'z'} and not sym.anyIt(char(it) in { '\x00'..'\x19', '"', '\\', '|' }):
+    write(stream, sym)
+  else:
+    write(stream, '|')
+    writeEscaped(stream, sym, '|')
+    write(stream, '|')
+
 proc writeText*[E](stream: Stream; pr: Preserve[E]; mode = textPreserves) =
   ## Encode Preserves to a `Stream` as text.
   if pr.embedded: write(stream, "#!")
@@ -44,14 +51,38 @@ proc writeText*[E](stream: Stream; pr: Preserve[E]; mode = textPreserves) =
     of false: write(stream, "#f")
     of true: write(stream, "#t")
   of pkFloat:
-    write(stream, $pr.float)
-    write(stream, 'f')
+    case pr.float.classify:
+    of fcNormal, fcZero, fcNegZero:
+      write(stream, $pr.float)
+      write(stream, 'f')
+    else:
+      var buf: array[4, byte]
+      bigEndian32(addr buf[0], addr pr.float)
+      write(stream, "#xf\"")
+      for b in buf:
+        write(stream, hexAlphabet[b shr 4])
+        write(stream, hexAlphabet[b and 0xf])
+      write(stream, '"')
   of pkDouble:
-    write(stream, $pr.double)
-  of pkSignedInteger:
-    write(stream, $pr.int)
+    case pr.double.classify:
+    of fcNormal, fcZero, fcNegZero:
+      write(stream, $pr.double)
+    else:
+      var buf: array[8, byte]
+      bigEndian64(addr buf[0], addr pr.double)
+      write(stream, "#xd\"")
+      for b in buf:
+        write(stream, hexAlphabet[b shr 4])
+        write(stream, hexAlphabet[b and 0xf])
+      write(stream, '"')
+  of pkRegister:
+    write(stream, $pr.register)
+  of pkBigInt:
+    write(stream, $pr.bigint)
   of pkString:
-    write(stream, escapeJson(pr.string))
+    write(stream, '"')
+    writeEscaped(stream, pr.string, '"')
+    write(stream, '"')
   of pkByteString:
     if pr.bytes.allIt(char(it) in {' '..'!', '#'..'~'}):
       write(stream, "#\"")
@@ -63,39 +94,13 @@ proc writeText*[E](stream: Stream; pr: Preserve[E]; mode = textPreserves) =
         write(stream, base64.encode(pr.bytes))
         write(stream, ']')
       else:
-        const alphabet = "0123456789abcdef"
         write(stream, "#x\"")
         for b in pr.bytes:
-          write(stream, alphabet[int(b shr 4)])
-          write(stream, alphabet[int(b and 0xf)])
+          write(stream, hexAlphabet[b.int shr 4])
+          write(stream, hexAlphabet[b.int and 0xf])
         write(stream, '"')
   of pkSymbol:
-    let sym = pr.symbol.string
-    if sym.len > 0 and sym[0] in {'A'..'z'} and not sym.anyIt(char(it) in { '\x00'..'\x19', '"', '\\', '|' }):
-      write(stream, sym)
-    else:
-      write(stream, '|')
-      for c in sym:
-        case c
-        of '\\':
-          write(stream, "\\\\")
-        of '/':
-          write(stream, "\\/")
-        of '\x08':
-          write(stream, "\\b")
-        of '\x0c':
-          write(stream, "\\f")
-        of '\x0a':
-          write(stream, "\\n")
-        of '\x0d':
-          write(stream, "\\r")
-        of '\x09':
-          write(stream, "\\t")
-        of '|':
-          write(stream, "\\|")
-        else:
-          write(stream, c)
-      write(stream, '|')
+    writeSymbol(stream, pr.symbol.string)
   of pkRecord:
     assert(pr.record.len > 0)
     write(stream, '<')
@@ -153,6 +158,11 @@ proc writeText*[E](stream: Stream; pr: Preserve[E]; mode = textPreserves) =
     else:
       write(stream, "…")
 
+proc `$`*(sym: Symbol): string =
+  var stream = newStringStream()
+  writeSymbol(stream, sym.string)
+  result = move stream.data
+
 proc `$`*[E](pr: Preserve[E]): string =
   ## Generate the textual representation of ``pr``.
   var stream = newStringStream()

src/preserves/private/values.nim

@@ -1,15 +1,17 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[hashes, options, sets, sequtils, tables]
+import std/[hashes, math, options, sets, sequtils, tables]
+
+import bigints
 
 type
   PreserveKind* = enum
-    pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol,
+    pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt, pkString, pkByteString, pkSymbol,
         pkRecord, pkSequence, pkSet, pkDictionary, pkEmbedded
 
 const
-  atomKinds* = {pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol}
+  atomKinds* = {pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt, pkString, pkByteString, pkSymbol}
   compoundKinds* = {pkRecord, pkSequence, pkSet, pkDictionary}
 
 type Symbol* = distinct string
@@ -29,8 +31,10 @@ type
       float*: float32
     of pkDouble:
       double*: float64
-    of pkSignedInteger:
-      int*: BiggestInt
+    of pkRegister:
+      register*: int
+    of pkBigInt:
+      bigint*: BigInt
     of pkString:
       string*: string
     of pkByteString:
@@ -52,6 +56,11 @@ type
 
   DictEntry*[E] = tuple[key: Preserve[E], val: Preserve[E]]
 
+func `===`[T: SomeFloat](a, b: T): bool =
+  ## Compare where Nan == NaN.
+  let class = a.classify
+  (class == b.classify) and ((class notin {fcNormal,fcSubnormal}) or (a == b))
+
 func `==`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
   ## Check `x` and `y` for equivalence.
   if x.kind == y.kind and x.embedded == y.embedded:
@@ -59,11 +68,13 @@ func `==`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
     of pkBoolean:
       result = x.bool == y.bool
     of pkFloat:
-      result = x.float == y.float
+      result = x.float === y.float
     of pkDouble:
-      result = x.double == y.double
-    of pkSignedInteger:
-      result = x.int == y.int
+      result = x.double === y.double
+    of pkRegister:
+      result = x.register == y.register
+    of pkBigInt:
+      result = x.bigint == y.bigint
     of pkString:
       result = x.string == y.string
     of pkByteString:
@@ -118,8 +129,10 @@ proc `<`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
       result = x.float < y.float
     of pkDouble:
       result = x.double < y.double
-    of pkSignedInteger:
-      result = x.int < y.int
+    of pkRegister:
+      result = x.register < y.register
+    of pkBigInt:
+      result = x.bigint < y.bigint
     of pkString:
       result = x.string < y.string
     of pkByteString:
@@ -172,8 +185,10 @@ proc hash*[E](pr: Preserve[E]): Hash =
     h = h !& hash(pr.float)
   of pkDouble:
     h = h !& hash(pr.double)
-  of pkSignedInteger:
-    h = h !& hash(pr.int)
+  of pkRegister:
+    h = h !& hash(pr.register)
+  of pkBigInt:
+    h = h !& hash(pr.bigint)
   of pkString:
     h = h !& hash(pr.string)
   of pkByteString:

src/preserves/sprypreserves.nim

@@ -48,8 +48,10 @@ proc toSpry(pr: Preserve[void], spry: Interpreter): Node =
       result = newValue(pr.float)
     of pkDouble:
       result = newValue(pr.double)
-    of pkSignedInteger:
-      result = newValue(int pr.int)
+    of pkRegister:
+      result = newValue(pr.register)
+    of pkBigInt:
+      raiseAssert "Arbitrary sized integers not supported by Spry implementation"
     of pkString:
       result = newValue(pr.string)
     of pkByteString:

tests/test_parser.nim.cfg

@@ -1 +0,0 @@
-threads:off