Split pkSignedInteger into pkRegister and pkBigInt

This commit is contained in:
Emery Hemingway 2023-12-22 17:55:23 +02:00
parent 37043a03bf
commit cedf25d1c3
11 changed files with 195 additions and 74 deletions

View file

@ -1,5 +1,17 @@
{
"depends": [
{
"method": "fetchzip",
"packages": [
"bigints"
],
"path": "/nix/store/jvrm392g8adfsgf36prgwkbyd7vh5jsw-source",
"ref": "20231006",
"rev": "86ea14d31eea9275e1408ca34e6bfe9c99989a96",
"sha256": "15pcpmnk1bnw3k8769rjzcpg00nahyrypwbxs88jnwr4aczp99j4",
"srcDir": "src",
"url": "https://github.com/ehmry/nim-bigints/archive/86ea14d31eea9275e1408ca34e6bfe9c99989a96.tar.gz"
},
{
"method": "fetchzip",
"packages": [

View file

@ -1,6 +1,6 @@
# Package
version = "20231220"
version = "20231222"
author = "Emery Hemingway"
description = "data model and serialization format"
license = "Unlicense"
@ -11,4 +11,4 @@ bin = @["preserves/preserves_schema_nim", "preserves/private/preserves
# Dependencies
requires "nim >= 2.0.0", "compiler >= 1.4.8", "https://github.com/zevv/npeg.git >= 1.2.1"
requires "nim >= 2.0.0", "compiler >= 1.4.8", "https://github.com/zevv/npeg.git >= 1.2.1", "https://github.com/ehmry/nim-bigints.git >= 20231006"

View file

@ -2,11 +2,10 @@
# SPDX-License-Identifier: Unlicense
import std/[options, sets, sequtils, strutils, tables, typetraits]
import ./preserves/private/macros
from std/algorithm import sort
from std/json import escapeJson, escapeJsonUnquoted
import ./preserves/private/[encoding, decoding, dot, parsing, texts, values]
import bigints
import ./preserves/private/[encoding, decoding, dot, macros, parsing, texts, values]
export encoding, decoding, parsing, texts, values
@ -37,6 +36,12 @@ proc cannonicalize*[E](pr: var Preserve[E]) =
else:
discard
proc toInt*(pr: Preserve): Option[int] =
case pr.kind
of pkRegister: result = some pr.register
of pkBigInt: result = toInt[int](pr.bigint)
else: discard
proc getOrDefault(pr: Preserve; key: Preserve): Preserve =
## Retrieves the value of `pr[key]` if `pr` is a dictionary containing `key`
## or returns the `#f` Preserves value.
@ -61,12 +66,17 @@ proc pop*(pr: var Preserve; key: Preserve; val: var Preserve): bool =
proc `[]`*(pr, key: Preserve): Preserve {.deprecated: "use step instead".} =
## Select a value by `key` from `pr`.
## Works for sequences, records, and dictionaries.
if pr.isDictionary:
case pr.kind
of pkDictionary:
for (k, v) in pr.dict.items:
if k == key: return v
raise newException(KeyError, "value not in Preserves dictionary")
elif (pr.isRecord or pr.isSequence) and key.isInteger:
result = pr[int key.int]
of pkRecord, pkSequence:
let idx = key.toInt
if idx.isSome:
result = pr[get idx]
else:
raise newException(ValueError, "invalid Preserves index")
else:
raise newException(ValueError, "invalid Preserves indexing")
@ -78,15 +88,19 @@ func step*(pr, idx: Preserve): Option[Preserve] =
assert step(parsePreserves("""<foo 1 2>"""), 1.toPreserve) == some(2.toPreserve)
assert step(parsePreserves("""{ foo: 1 bar: 2}"""), "foo".toSymbol) == some(1.toPreserve)
assert step(parsePreserves("""[ ]"""), 1.toPreserve) == none(Preserve[void])
if pr.isDictionary:
case pr.kind
of pkDictionary:
for (k, v) in pr.dict.items:
if k == idx:
result = some(v)
break
elif (pr.isRecord or pr.isSequence) and idx.isInteger:
let i = int idx.int
if i < pr.len:
result = some(pr[i])
of pkRecord, pkSequence:
var i = idx.toInt
if i.isSome:
var i = get i
if i < pr.len:
result = some(pr[i])
else: discard
func step*(pr: Preserve; path: varargs[Preserve]): Option[Preserve] =
## Step into `pr` by indexes at `path`.
@ -206,12 +220,16 @@ func isFloat*(pr: Preserve): bool {.inline.} = pr.kind == pkFloat
func isDouble*(pr: Preserve): bool {.inline.} = pr.kind == pkDouble
## Check if ``pr`` is a Preserve double.
func isInteger*(pr: Preserve): bool {.inline.} = pr.kind == pkSignedInteger
func isInteger*(pr: Preserve): bool {.inline.} =
## Check if ``pr`` is a Preserve integer.
pr.kind == pkRegister or pr.kind == pkBigInt
func isInteger*(pr: Preserve; i: SomeInteger): bool {.inline.} =
## Check if ``pr`` is a Preserve integer equivalent to `i`.
pr.kind == pkSignedInteger and pr.int == BiggestInt(i)
case pr.kind
of pkRegister: pr.register == i.int
of pkBigInt: pr.int == i.initBigInt
else: false
func isString*(pr: Preserve): bool {.inline.} = pr.kind == pkString
## Check if ``pr`` is a Preserve text string.
@ -374,14 +392,16 @@ proc toPreserve*[T](x: T; E = void): Preserve[E] {.gcsafe.} =
for xf in fields(x):
result.sequence.add(toPreserve(xf, E))
elif T is Ordinal:
result = Preserve[E](kind: pkSignedInteger, int: x.ord.BiggestInt)
result = Preserve[E](kind: pkRegister, register: x.ord)
assert result.register.T == x
elif T is ptr | ref:
if system.`==`(x, nil): result = toSymbol("null", E)
else: result = toPreserve(x[], E)
elif T is string:
result = Preserve[E](kind: pkString, string: x)
elif T is SomeInteger:
result = Preserve[E](kind: pkSignedInteger, int: x.BiggestInt)
result = Preserve[E](kind: pkRegister, register: x.int)
assert result.register.T == x
elif T is Symbol:
result = Preserve[E](kind: pkSymbol, symbol: x)
elif T is distinct:
@ -512,8 +532,8 @@ proc fromPreserve*[T, E](v: var T; pr: Preserve[E]): bool {.gcsafe.} =
v = pr.bool
result = true
elif T is SomeInteger:
if pr.kind == pkSignedInteger:
v = T(pr.int)
if pr.kind == pkRegister:
v = T(pr.register)
result = true
elif T is seq[byte]:
if pr.kind == pkByteString:
@ -542,9 +562,15 @@ proc fromPreserve*[T, E](v: var T; pr: Preserve[E]): bool {.gcsafe.} =
result = true
else: discard
elif T is Ordinal | SomeInteger:
if pr.kind == pkSignedInteger:
v = (T)pr.int
case pr.kind
of pkRegister:
v = (T)pr.register
result = true
of pkBigInt:
var o = toInt[T](pr.bigint)
result = o.isSome
if result: v = get o
else: discard
elif T is string:
if pr.kind == pkString:
v = pr.string
@ -731,7 +757,9 @@ proc apply*[E](result: var Preserve[E]; op: proc(_: var Preserve[E]) {.gcsafe.})
proc recurse(result: var Preserve[E]) = apply(result, op)
op(result)
case result.kind
of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol, pkEmbedded: discard
of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt,
pkString, pkByteString, pkSymbol, pkEmbedded:
discard
of pkRecord:
apply(result.record, recurse)
of pkSequence:
@ -754,7 +782,8 @@ proc mapEmbeds*(pr: sink Preserve[void]; E: typedesc): Preserve[E] =
raise newException(ValueError, "failed to convert " & $E & " from " & $pr)
else:
case pr.kind
of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol:
of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt,
pkString, pkByteString, pkSymbol:
result = cast[Preserve[E]](pr)
of pkRecord:
result = Preserve[E](kind: pr.kind)
@ -801,7 +830,8 @@ proc mapEmbeds*[A, B](pr: sink Preserve[A]; op: proc (v: A): B): Preserve[B] =
result = embed op(e)
else:
case pr.kind
of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol:
of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt,
pkString, pkByteString, pkSymbol:
result = cast[Preserve[B]](pr)
of pkRecord:
result = Preserve[B](kind: pr.kind)
@ -827,7 +857,8 @@ proc contract*[E](pr: sink Preserve[E]; op: proc (v: E): Preserve[void] {.gcsafe
## Convert `Preserve[E]` to `Preserve[void]` using an `E → Preserve[void]` procedure.
if not pr.embedded:
case pr.kind
of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol:
of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt,
pkString, pkByteString, pkSymbol:
result = cast[Preserve[void]](pr)
of pkRecord:
result = Preserve[void](kind: pr.kind)
@ -855,7 +886,8 @@ proc expand*[E](pr: sink Preserve[void]; op: proc (v: Preserve[void]): Preserve[
result = op(pr)
else:
case pr.kind
of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol:
of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt,
pkString, pkByteString, pkSymbol:
result = cast[Preserve[E]](pr)
of pkRecord:
result = Preserve[E](kind: pr.kind)

View file

@ -9,7 +9,7 @@ proc toPreserveHook*(js: JsonNode; E: typedesc): Preserve[E] =
of JString:
result = Preserve[E](kind: pkString, string: js.str)
of JInt:
result = Preserve[E](kind: pkSignedInteger, int: js.num)
result = Preserve[E](kind: pkRegister, register: js.num)
of JFloat:
result = Preserve[E](kind: pkDouble, double: js.fnum)
of JBool:
@ -43,8 +43,8 @@ proc fromPreserveHook*[E](js: var JsonNode; prs: Preserve[E]): bool =
js = newJFloat(prs.float)
of pkDouble:
js = newJFloat(prs.double)
of pkSignedInteger:
js = newJInt(prs.int)
of pkRegister:
js = newJInt(prs.register)
of pkString:
js = newJString(prs.string)
of pkSymbol:

View file

@ -2,6 +2,7 @@
# SPDX-License-Identifier: Unlicense
import std/[endians, streams, strutils]
import bigints
import ./values
proc readVarint(s: Stream): uint =
@ -47,6 +48,35 @@ proc decodePreserves*(s: Stream; E = void): Preserve[E] =
swapEndian64(result.double.addr, be.addr)
else:
raise newException(IOError, "unhandled IEEE754 value of " & $n & " bytes")
of 0xb0:
var n = int s.readVarint()
if n <= sizeof(int):
result = Preserve[E](kind: pkRegister)
if n > 0:
var
buf: array[sizeof(int), byte]
off = buf.len - n
if s.readData(addr buf[off], n) != n:
raise newException(IOError, "short read")
if off > 0:
var fill: uint8 = if (buf[off] and 0x80) == 0x80'u8: 0xff else: 0x00'u8
for i in 0..<off: buf[i] = fill
when buf.len == 4:
bigEndian32(addr result.register, addr buf[0])
elif buf.len == 8:
bigEndian64(addr result.register, addr buf[0])
else: {.error: "int size " & $buf.len & " not supported here".}
else:
result = Preserve[E](kind: pkBigInt)
var buf = newSeq[byte](n)
if s.readData(addr buf[0], buf.len) != n:
raise newException(IOError, "short read")
if (buf[0] and 0x80) == 0x80:
for i, b in buf: buf[i] = not b
result.bigint.fromBytes(buf, bigEndian)
result.bigint = -(result.bigint.succ)
else:
result.bigint.fromBytes(buf, bigEndian)
of 0xb1:
var data = newString(s.readVarint())
if data.len > 0:
@ -90,16 +120,6 @@ proc decodePreserves*(s: Stream; E = void): Preserve[E] =
while s.peekUint8() != endMarker:
result[decodePreserves(s, E)] = decodePreserves(s, E)
discard s.readUint8()
of 0xb0:
var len = s.readVarint()
result = Preserve[E](kind: pkSignedInteger)
if len > 0:
if (s.peekUint8() and 0x80) == 0x80:
result.int = BiggestInt -1
while len > 0:
result.int = (result.int shl 8) + s.readUint8().BiggestInt
dec(len)
of endMarker:
raise newException(ValueError, "invalid Preserves stream")
else:

View file

@ -1,7 +1,8 @@
# SPDX-FileCopyrightText: ☭ Emery Hemingway
# SPDX-License-Identifier: Unlicense
import std/[endians, options, sets, sequtils, streams, tables, typetraits]
import std/[endians, streams]
import bigints
import ./values
proc writeVarint(s: Stream; n: Natural) =
@ -35,25 +36,44 @@ proc write*[E](str: Stream; pr: Preserve[E]) =
var be: float64
swapEndian64(be.addr, pr.double.unsafeAddr)
str.write(be)
of pkSignedInteger:
if pr.int == 0:
str.write("\xb0\x00")
of pkRegister:
if pr.register == 0: str.write("\xb0\x00")
else:
var bitCount = 1'u8
if pr.int < 0:
while ((not pr.int) shr bitCount) != 0:
inc(bitCount)
const bufLen = sizeof(int)
var buf: array[bufLen, byte]
when bufLen == 4: bigEndian32(addr buf[0], addr pr.register)
elif bufLen == 8: bigEndian64(addr buf[0], addr pr.register)
else: {.error: "int size " & $bufLen & " not supported here".}
if buf[0] != 0x00 and buf[0] != 0xff:
str.write(cast[string](buf)) # dumbass hex conversion
else:
while (pr.int shr bitCount) != 0:
inc(bitCount)
var byteCount = (bitCount + 8) div 8
str.write(0xb0'u8)
str.writeVarint(byteCount)
proc write(n: uint8; i: BiggestInt) =
if n > 1:
write(n.pred, i shr 8)
str.write(i.uint8)
write(byteCount, pr.int)
var start = 0
while start < buf.high and buf[0] == buf[succ start]: inc start
if start < buf.high and (buf[succ start] and 0x80) == (buf[0] and 0x80): inc start
str.write('\xb0')
str.write(uint8(bufLen - start))
str.write(cast[string](buf[start..<bufLen]))
of pkBigInt:
if pr.bigint.isZero: str.write("\xb0\x00")
elif pr.bigint.isNegative:
var buf = pr.bigint.succ.toBytes(bigEndian)
for i, b in buf: buf[i] = not b
str.write('\xb0')
if (buf[0] and 0x80) != 0x80:
str.writeVarint(buf.len.succ)
str.write('\xff')
else:
str.writeVarint(buf.len)
str.write(cast[string](buf))
else:
var buf = pr.bigint.toBytes(bigEndian)
str.write('\xb0')
if (buf[0] and 0x80) != 0:
str.writeVarint(buf.len.succ)
str.write('\x00')
else:
str.writeVarint(buf.len)
str.write(cast[string](buf))
of pkString:
str.write(0xb1'u8)
str.writeVarint(pr.string.len)

View file

@ -1,10 +1,11 @@
# SPDX-FileCopyrightText: ☭ Emery Hemingway
# SPDX-License-Identifier: Unlicense
import std/[base64, parseutils, strutils, unicode]
import std/[base64, options, parseutils, strutils, unicode]
from std/sequtils import insert
import npeg
import bigints, npeg
import ../pegs
import ./decoding, ./values
@ -157,7 +158,13 @@ proc parsePreserves*(text: string): Preserve[void] =
pushStack Value(kind: pkDouble, double: cast[float64](reg))
Preserves.SignedInteger <- Preserves.SignedInteger:
pushStack Value(kind: pkSignedInteger, int: parseInt($0))
var
big = initBigInt($0)
small = toInt[int](big)
if small.isSome:
pushStack Value(kind: pkRegister, register: small.get)
else:
pushStack Value(kind: pkBigInt, bigint: big)
Preserves.String <- Preserves.String:
var v = Value(kind: pkString, string: newStringOfCap(len($1)))

View file

@ -48,8 +48,10 @@ proc writeText*[E](stream: Stream; pr: Preserve[E]; mode = textPreserves) =
write(stream, 'f')
of pkDouble:
write(stream, $pr.double)
of pkSignedInteger:
write(stream, $pr.int)
of pkRegister:
write(stream, $pr.register)
of pkBigInt:
write(stream, $pr.bigint)
of pkString:
write(stream, escapeJson(pr.string))
of pkByteString:

View file

@ -3,13 +3,15 @@
import std/[hashes, options, sets, sequtils, tables]
import bigints
type
PreserveKind* = enum
pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol,
pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt, pkString, pkByteString, pkSymbol,
pkRecord, pkSequence, pkSet, pkDictionary, pkEmbedded
const
atomKinds* = {pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol}
atomKinds* = {pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt, pkString, pkByteString, pkSymbol}
compoundKinds* = {pkRecord, pkSequence, pkSet, pkDictionary}
type Symbol* = distinct string
@ -29,8 +31,10 @@ type
float*: float32
of pkDouble:
double*: float64
of pkSignedInteger:
int*: BiggestInt
of pkRegister:
register*: int
of pkBigInt:
bigint*: BigInt
of pkString:
string*: string
of pkByteString:
@ -62,8 +66,10 @@ func `==`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
result = x.float == y.float
of pkDouble:
result = x.double == y.double
of pkSignedInteger:
result = x.int == y.int
of pkRegister:
result = x.register == y.register
of pkBigInt:
result = x.bigint == y.bigint
of pkString:
result = x.string == y.string
of pkByteString:
@ -118,8 +124,10 @@ proc `<`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
result = x.float < y.float
of pkDouble:
result = x.double < y.double
of pkSignedInteger:
result = x.int < y.int
of pkRegister:
result = x.register < y.register
of pkBigInt:
result = x.bigint < y.bigint
of pkString:
result = x.string < y.string
of pkByteString:
@ -172,8 +180,10 @@ proc hash*[E](pr: Preserve[E]): Hash =
h = h !& hash(pr.float)
of pkDouble:
h = h !& hash(pr.double)
of pkSignedInteger:
h = h !& hash(pr.int)
of pkRegister:
h = h !& hash(pr.register)
of pkBigInt:
h = h !& hash(pr.bigint)
of pkString:
h = h !& hash(pr.string)
of pkByteString:

View file

@ -48,8 +48,10 @@ proc toSpry(pr: Preserve[void], spry: Interpreter): Node =
result = newValue(pr.float)
of pkDouble:
result = newValue(pr.double)
of pkSignedInteger:
result = newValue(int pr.int)
of pkRegister:
result = newValue(pr.register)
of pkBigInt:
raiseAssert "Arbitrary sized integers not supported by Spry implementation"
of pkString:
result = newValue(pr.string)
of pkByteString:

16
tests/test_step.nim Normal file
View file

@ -0,0 +1,16 @@
# SPDX-FileCopyrightText: ☭ Emery Hemingway
# SPDX-License-Identifier: Unlicense
import std/[options, sequtils, unittest]
import preserves
suite "step":
var data = parsePreserves """
<foo "bar" [ 0.0 {a: #f, "b": #t } ] >
"""
var o = some data
for i in [1.toPreserve, 1.toPreserve, "b".toPreserve]:
test $i:
o = step(get o, i)
check o.isSome