diff --git a/lock.json b/lock.json index 30a8aa3..77cc6cb 100644 --- a/lock.json +++ b/lock.json @@ -1,5 +1,17 @@ { "depends": [ + { + "method": "fetchzip", + "packages": [ + "bigints" + ], + "path": "/nix/store/jvrm392g8adfsgf36prgwkbyd7vh5jsw-source", + "ref": "20231006", + "rev": "86ea14d31eea9275e1408ca34e6bfe9c99989a96", + "sha256": "15pcpmnk1bnw3k8769rjzcpg00nahyrypwbxs88jnwr4aczp99j4", + "srcDir": "src", + "url": "https://github.com/ehmry/nim-bigints/archive/86ea14d31eea9275e1408ca34e6bfe9c99989a96.tar.gz" + }, { "method": "fetchzip", "packages": [ diff --git a/preserves.nimble b/preserves.nimble index 8a15e08..d008e2f 100644 --- a/preserves.nimble +++ b/preserves.nimble @@ -1,6 +1,6 @@ # Package -version = "20231220" +version = "20231222" author = "Emery Hemingway" description = "data model and serialization format" license = "Unlicense" @@ -11,4 +11,4 @@ bin = @["preserves/preserves_schema_nim", "preserves/private/preserves # Dependencies -requires "nim >= 2.0.0", "compiler >= 1.4.8", "https://github.com/zevv/npeg.git >= 1.2.1" +requires "nim >= 2.0.0", "compiler >= 1.4.8", "https://github.com/zevv/npeg.git >= 1.2.1", "https://github.com/ehmry/nim-bigints.git >= 20231006" diff --git a/src/preserves.nim b/src/preserves.nim index 8654275..e22ce72 100644 --- a/src/preserves.nim +++ b/src/preserves.nim @@ -2,11 +2,10 @@ # SPDX-License-Identifier: Unlicense import std/[options, sets, sequtils, strutils, tables, typetraits] -import ./preserves/private/macros - from std/algorithm import sort from std/json import escapeJson, escapeJsonUnquoted -import ./preserves/private/[encoding, decoding, dot, parsing, texts, values] +import bigints +import ./preserves/private/[encoding, decoding, dot, macros, parsing, texts, values] export encoding, decoding, parsing, texts, values @@ -37,6 +36,12 @@ proc cannonicalize*[E](pr: var Preserve[E]) = else: discard +proc toInt*(pr: Preserve): Option[int] = + case pr.kind + of pkRegister: result = some pr.register + of pkBigInt: result = toInt[int](pr.bigint) + else: discard + proc getOrDefault(pr: Preserve; key: Preserve): Preserve = ## Retrieves the value of `pr[key]` if `pr` is a dictionary containing `key` ## or returns the `#f` Preserves value. @@ -61,12 +66,17 @@ proc pop*(pr: var Preserve; key: Preserve; val: var Preserve): bool = proc `[]`*(pr, key: Preserve): Preserve {.deprecated: "use step instead".} = ## Select a value by `key` from `pr`. ## Works for sequences, records, and dictionaries. - if pr.isDictionary: + case pr.kind + of pkDictionary: for (k, v) in pr.dict.items: if k == key: return v raise newException(KeyError, "value not in Preserves dictionary") - elif (pr.isRecord or pr.isSequence) and key.isInteger: - result = pr[int key.int] + of pkRecord, pkSequence: + let idx = key.toInt + if idx.isSome: + result = pr[get idx] + else: + raise newException(ValueError, "invalid Preserves index") else: raise newException(ValueError, "invalid Preserves indexing") @@ -78,15 +88,19 @@ func step*(pr, idx: Preserve): Option[Preserve] = assert step(parsePreserves(""""""), 1.toPreserve) == some(2.toPreserve) assert step(parsePreserves("""{ foo: 1 bar: 2}"""), "foo".toSymbol) == some(1.toPreserve) assert step(parsePreserves("""[ ]"""), 1.toPreserve) == none(Preserve[void]) - if pr.isDictionary: + case pr.kind + of pkDictionary: for (k, v) in pr.dict.items: if k == idx: result = some(v) break - elif (pr.isRecord or pr.isSequence) and idx.isInteger: - let i = int idx.int - if i < pr.len: - result = some(pr[i]) + of pkRecord, pkSequence: + var i = idx.toInt + if i.isSome: + var i = get i + if i < pr.len: + result = some(pr[i]) + else: discard func step*(pr: Preserve; path: varargs[Preserve]): Option[Preserve] = ## Step into `pr` by indexes at `path`. @@ -206,12 +220,16 @@ func isFloat*(pr: Preserve): bool {.inline.} = pr.kind == pkFloat func isDouble*(pr: Preserve): bool {.inline.} = pr.kind == pkDouble ## Check if ``pr`` is a Preserve double. -func isInteger*(pr: Preserve): bool {.inline.} = pr.kind == pkSignedInteger +func isInteger*(pr: Preserve): bool {.inline.} = ## Check if ``pr`` is a Preserve integer. + pr.kind == pkRegister or pr.kind == pkBigInt func isInteger*(pr: Preserve; i: SomeInteger): bool {.inline.} = ## Check if ``pr`` is a Preserve integer equivalent to `i`. - pr.kind == pkSignedInteger and pr.int == BiggestInt(i) + case pr.kind + of pkRegister: pr.register == i.int + of pkBigInt: pr.int == i.initBigInt + else: false func isString*(pr: Preserve): bool {.inline.} = pr.kind == pkString ## Check if ``pr`` is a Preserve text string. @@ -374,14 +392,16 @@ proc toPreserve*[T](x: T; E = void): Preserve[E] {.gcsafe.} = for xf in fields(x): result.sequence.add(toPreserve(xf, E)) elif T is Ordinal: - result = Preserve[E](kind: pkSignedInteger, int: x.ord.BiggestInt) + result = Preserve[E](kind: pkRegister, register: x.ord) + assert result.register.T == x elif T is ptr | ref: if system.`==`(x, nil): result = toSymbol("null", E) else: result = toPreserve(x[], E) elif T is string: result = Preserve[E](kind: pkString, string: x) elif T is SomeInteger: - result = Preserve[E](kind: pkSignedInteger, int: x.BiggestInt) + result = Preserve[E](kind: pkRegister, register: x.int) + assert result.register.T == x elif T is Symbol: result = Preserve[E](kind: pkSymbol, symbol: x) elif T is distinct: @@ -512,8 +532,8 @@ proc fromPreserve*[T, E](v: var T; pr: Preserve[E]): bool {.gcsafe.} = v = pr.bool result = true elif T is SomeInteger: - if pr.kind == pkSignedInteger: - v = T(pr.int) + if pr.kind == pkRegister: + v = T(pr.register) result = true elif T is seq[byte]: if pr.kind == pkByteString: @@ -542,9 +562,15 @@ proc fromPreserve*[T, E](v: var T; pr: Preserve[E]): bool {.gcsafe.} = result = true else: discard elif T is Ordinal | SomeInteger: - if pr.kind == pkSignedInteger: - v = (T)pr.int + case pr.kind + of pkRegister: + v = (T)pr.register result = true + of pkBigInt: + var o = toInt[T](pr.bigint) + result = o.isSome + if result: v = get o + else: discard elif T is string: if pr.kind == pkString: v = pr.string @@ -731,7 +757,9 @@ proc apply*[E](result: var Preserve[E]; op: proc(_: var Preserve[E]) {.gcsafe.}) proc recurse(result: var Preserve[E]) = apply(result, op) op(result) case result.kind - of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol, pkEmbedded: discard + of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt, + pkString, pkByteString, pkSymbol, pkEmbedded: + discard of pkRecord: apply(result.record, recurse) of pkSequence: @@ -754,7 +782,8 @@ proc mapEmbeds*(pr: sink Preserve[void]; E: typedesc): Preserve[E] = raise newException(ValueError, "failed to convert " & $E & " from " & $pr) else: case pr.kind - of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol: + of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt, + pkString, pkByteString, pkSymbol: result = cast[Preserve[E]](pr) of pkRecord: result = Preserve[E](kind: pr.kind) @@ -801,7 +830,8 @@ proc mapEmbeds*[A, B](pr: sink Preserve[A]; op: proc (v: A): B): Preserve[B] = result = embed op(e) else: case pr.kind - of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol: + of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt, + pkString, pkByteString, pkSymbol: result = cast[Preserve[B]](pr) of pkRecord: result = Preserve[B](kind: pr.kind) @@ -827,7 +857,8 @@ proc contract*[E](pr: sink Preserve[E]; op: proc (v: E): Preserve[void] {.gcsafe ## Convert `Preserve[E]` to `Preserve[void]` using an `E → Preserve[void]` procedure. if not pr.embedded: case pr.kind - of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol: + of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt, + pkString, pkByteString, pkSymbol: result = cast[Preserve[void]](pr) of pkRecord: result = Preserve[void](kind: pr.kind) @@ -855,7 +886,8 @@ proc expand*[E](pr: sink Preserve[void]; op: proc (v: Preserve[void]): Preserve[ result = op(pr) else: case pr.kind - of pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol: + of pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt, + pkString, pkByteString, pkSymbol: result = cast[Preserve[E]](pr) of pkRecord: result = Preserve[E](kind: pr.kind) diff --git a/src/preserves/jsonhooks.nim b/src/preserves/jsonhooks.nim index b85de83..803ace8 100644 --- a/src/preserves/jsonhooks.nim +++ b/src/preserves/jsonhooks.nim @@ -9,7 +9,7 @@ proc toPreserveHook*(js: JsonNode; E: typedesc): Preserve[E] = of JString: result = Preserve[E](kind: pkString, string: js.str) of JInt: - result = Preserve[E](kind: pkSignedInteger, int: js.num) + result = Preserve[E](kind: pkRegister, register: js.num) of JFloat: result = Preserve[E](kind: pkDouble, double: js.fnum) of JBool: @@ -43,8 +43,8 @@ proc fromPreserveHook*[E](js: var JsonNode; prs: Preserve[E]): bool = js = newJFloat(prs.float) of pkDouble: js = newJFloat(prs.double) - of pkSignedInteger: - js = newJInt(prs.int) + of pkRegister: + js = newJInt(prs.register) of pkString: js = newJString(prs.string) of pkSymbol: diff --git a/src/preserves/private/decoding.nim b/src/preserves/private/decoding.nim index 0c855e4..99dd6c6 100644 --- a/src/preserves/private/decoding.nim +++ b/src/preserves/private/decoding.nim @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Unlicense import std/[endians, streams, strutils] +import bigints import ./values proc readVarint(s: Stream): uint = @@ -47,6 +48,35 @@ proc decodePreserves*(s: Stream; E = void): Preserve[E] = swapEndian64(result.double.addr, be.addr) else: raise newException(IOError, "unhandled IEEE754 value of " & $n & " bytes") + of 0xb0: + var n = int s.readVarint() + if n <= sizeof(int): + result = Preserve[E](kind: pkRegister) + if n > 0: + var + buf: array[sizeof(int), byte] + off = buf.len - n + if s.readData(addr buf[off], n) != n: + raise newException(IOError, "short read") + if off > 0: + var fill: uint8 = if (buf[off] and 0x80) == 0x80'u8: 0xff else: 0x00'u8 + for i in 0.. 0: @@ -90,16 +120,6 @@ proc decodePreserves*(s: Stream; E = void): Preserve[E] = while s.peekUint8() != endMarker: result[decodePreserves(s, E)] = decodePreserves(s, E) discard s.readUint8() - of 0xb0: - var len = s.readVarint() - result = Preserve[E](kind: pkSignedInteger) - if len > 0: - if (s.peekUint8() and 0x80) == 0x80: - result.int = BiggestInt -1 - while len > 0: - result.int = (result.int shl 8) + s.readUint8().BiggestInt - dec(len) - of endMarker: raise newException(ValueError, "invalid Preserves stream") else: diff --git a/src/preserves/private/encoding.nim b/src/preserves/private/encoding.nim index f6838eb..365b3ac 100644 --- a/src/preserves/private/encoding.nim +++ b/src/preserves/private/encoding.nim @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: ☭ Emery Hemingway # SPDX-License-Identifier: Unlicense -import std/[endians, options, sets, sequtils, streams, tables, typetraits] +import std/[endians, streams] +import bigints import ./values proc writeVarint(s: Stream; n: Natural) = @@ -35,25 +36,44 @@ proc write*[E](str: Stream; pr: Preserve[E]) = var be: float64 swapEndian64(be.addr, pr.double.unsafeAddr) str.write(be) - of pkSignedInteger: - if pr.int == 0: - str.write("\xb0\x00") + of pkRegister: + if pr.register == 0: str.write("\xb0\x00") else: - var bitCount = 1'u8 - if pr.int < 0: - while ((not pr.int) shr bitCount) != 0: - inc(bitCount) + const bufLen = sizeof(int) + var buf: array[bufLen, byte] + when bufLen == 4: bigEndian32(addr buf[0], addr pr.register) + elif bufLen == 8: bigEndian64(addr buf[0], addr pr.register) + else: {.error: "int size " & $bufLen & " not supported here".} + if buf[0] != 0x00 and buf[0] != 0xff: + str.write(cast[string](buf)) # dumbass hex conversion else: - while (pr.int shr bitCount) != 0: - inc(bitCount) - var byteCount = (bitCount + 8) div 8 - str.write(0xb0'u8) - str.writeVarint(byteCount) - proc write(n: uint8; i: BiggestInt) = - if n > 1: - write(n.pred, i shr 8) - str.write(i.uint8) - write(byteCount, pr.int) + var start = 0 + while start < buf.high and buf[0] == buf[succ start]: inc start + if start < buf.high and (buf[succ start] and 0x80) == (buf[0] and 0x80): inc start + str.write('\xb0') + str.write(uint8(bufLen - start)) + str.write(cast[string](buf[start.. + """ + + var o = some data + for i in [1.toPreserve, 1.toPreserve, "b".toPreserve]: + test $i: + o = step(get o, i) + check o.isSome