From a2dc5becc0596d52ab205d869b7c167c0b562fb4 Mon Sep 17 00:00:00 2001 From: Emery Hemingway Date: Thu, 19 Oct 2023 18:05:23 +0100 Subject: [PATCH] Update binary encoding to Preserves-0.990 --- preserves.nimble | 2 +- src/preserves.nim | 75 +++++++++++++++++++++---------------------- tests/test_parser.nim | 20 ++++++------ 3 files changed, 47 insertions(+), 50 deletions(-) diff --git a/preserves.nimble b/preserves.nimble index b576b1c..7bbfb96 100644 --- a/preserves.nimble +++ b/preserves.nimble @@ -1,6 +1,6 @@ # Package -version = "20230914" +version = "20231019" author = "Emery Hemingway" description = "data model and serialization format" license = "Unlicense" diff --git a/src/preserves.nim b/src/preserves.nim index a1f1d05..744147b 100644 --- a/src/preserves.nim +++ b/src/preserves.nim @@ -552,7 +552,7 @@ proc write*[E](str: Stream; pr: Preserve[E]) = of false: str.write(0x80'u8) of true: str.write(0x81'u8) of pkFloat: - str.write(0x82'u8) + str.write("\x87\x04") when system.cpuEndian == bigEndian: str.write(pr.float) else: @@ -560,7 +560,7 @@ proc write*[E](str: Stream; pr: Preserve[E]) = swapEndian32(be.addr, pr.float.unsafeAddr) str.write(be) of pkDouble: - str.write(0x83'u8) + str.write("\x87\x08") when system.cpuEndian == bigEndian: str.write(pr.double) else: @@ -568,8 +568,8 @@ proc write*[E](str: Stream; pr: Preserve[E]) = swapEndian64(be.addr, pr.double.unsafeAddr) str.write(be) of pkSignedInteger: - if (-3 <= pr.int) and (pr.int <= 12): - str.write(0x90'i8 or int8(if pr.int < 0: pr.int + 16 else: pr.int)) + if pr.int == 0: + str.write("\xb0\x00") else: var bitCount = 1'u8 if pr.int < 0: @@ -579,11 +579,12 @@ proc write*[E](str: Stream; pr: Preserve[E]) = while (pr.int shr bitCount) != 0: inc(bitCount) var byteCount = (bitCount + 8) div 8 - str.write(0xa0'u8 or (byteCount - 1)) + str.write(0xb0'u8) + str.writeVarint(byteCount) proc write(n: uint8; i: BiggestInt) = - if n > 0: + if n > 1: write(n.pred, i shr 8) - str.write(i.uint8) + str.write(i.uint8) write(byteCount, pr.int) of pkString: str.write(0xb1'u8) @@ -639,20 +640,6 @@ proc decodePreserves*(s: Stream; E = void): Preserve[E] = case tag of 0x80: result = Preserve[E](kind: pkBoolean, bool: false) of 0x81: result = Preserve[E](kind: pkBoolean, bool: true) - of 0x82: - when system.cpuEndian == bigEndian: - result = Preserve[E](kind: pkFloat, float: s.readFloat32()) - else: - result = Preserve[E](kind: pkFloat) - var be = s.readFloat32() - swapEndian32(result.float.addr, be.addr) - of 0x83: - when system.cpuEndian == bigEndian: - result = Preserve[E](kind: pkDouble, double: s.readFloat64()) - else: - result = Preserve[E](kind: pkDouble) - var be = s.readFloat64() - swapEndian64(result.double.addr, be.addr) of 0x85: discard decodePreserves(s, E) while s.peekUint8() == 0x85: @@ -661,6 +648,25 @@ proc decodePreserves*(s: Stream; E = void): Preserve[E] = of 0x86: result = decodePreserves(s, E) result.embedded = true + of 0x87: + let n = s.readUint8() + case n + of 4: + when system.cpuEndian == bigEndian: + result = Preserve[E](kind: pkFloat, float: s.readFloat32()) + else: + result = Preserve[E](kind: pkFloat) + var be = s.readFloat32() + swapEndian32(result.float.addr, be.addr) + of 8: + when system.cpuEndian == bigEndian: + result = Preserve[E](kind: pkDouble, double: s.readFloat64()) + else: + result = Preserve[E](kind: pkDouble) + var be = s.readFloat64() + swapEndian64(result.double.addr, be.addr) + else: + raise newException(IOError, "unhandled IEEE754 value of " & $n & " bytes") of 0xb1: var data = newString(s.readVarint()) if data.len > 0: @@ -705,28 +711,19 @@ proc decodePreserves*(s: Stream; E = void): Preserve[E] = result[decodePreserves(s, E)] = decodePreserves(s, E) discard s.readUint8() of 0xb0: - let len = s.readVarint() + var len = s.readVarint() result = Preserve[E](kind: pkSignedInteger) - for _ in 1..len: - result.int = (result.int shl 8) + s.readUint8().BiggestInt + if len > 0: + if (s.peekUint8() and 0x80) == 0x80: + result.int = BiggestInt -1 + while len > 0: + result.int = (result.int shl 8) + s.readUint8().BiggestInt + dec(len) + of endMarker: raise newException(ValueError, "invalid Preserves stream") else: - case 0xf0 and tag - of 0x90: - var n = tag.BiggestInt - result = Preserve[E](kind: pkSignedInteger, - int: n - (if n > 0x9c: 0xa0 else: 0x90)) - of 0xa0: - let len = (tag.int and 0x0f) + 1 - if len > 8: raise newException(ValueError, - "numbers wider than 64-bits not supported by this Preserves implementation") - result = Preserve[E](kind: pkSignedInteger, int: s.readUint8().BiggestInt) - if (result.int and 0x80) != 0: result.int.dec(0x100) - for i in 1..>""", "\xB4\xB3\x07capture\xB4\xB3\x07discard\x84\x84"), -("""[1 2 3 4]""", "\xB5\x91\x92\x93\x94\x84"), -("""[-2 -1 0 1]""", "\xB5\x9E\x9F\x90\x91\x84"), +("""[1 2 3 4]""", "\xB5\xB0\x01\x01\xB0\x01\x02\xB0\x01\x03\xB0\x01\x04\x84"), +("""[-2 -1 0 1]""", "\xB5\xB0\x01\xFE\xB0\x01\xFF\xB0\x00\xB0\x01\x01\x84"), (""""hello"""", "\xB1\x05hello"), ("""" \"hello\" """", "\xB1\x09 \"hello\" "), ("""["a" b #"c" [] #{} #t #f]""", "\xB5\xB1\x01a\xB3\x01b\xB2\x01c\xB5\x84\xB6\x84\x81\x80\x84"), -("""-257""", "\xA1\xFE\xFF"), -("""-1""", "\x9F"), -("""0""", "\x90"), -("""1""", "\x91"), -("""255""", "\xA1\x00\xFF"), -("""1.0f""", "\x82\x3F\x80\x00\x00"), -("""1.0""", "\x83\x3F\xF0\x00\x00\x00\x00\x00\x00"), -("""-1.202e300""", "\x83\xFE\x3C\xB7\xB7\x59\xBF\x04\x26"), +("""-257""", "\xB0\x02\xFE\xFF"), +("""-1""", "\xB0\x01\xFF"), +("""0""", "\xB0\x00"), +("""1""", "\xB0\x01\x01"), +("""255""", "\xB0\x02\x00\xFF"), +("""1.0f""", "\x87\x04\x3F\x80\x00\x00"), +("""1.0""", "\x87\x08\x3F\xF0\x00\x00\x00\x00\x00\x00"), +("""-1.202e300""", "\x87\x08\xFE\x3C\xB7\xB7\x59\xBF\x04\x26"), ("""#=#x"B4B30763617074757265B4B307646973636172648484"""", "\xB4\xB3\x07capture\xB4\xB3\x07discard\x84\x84"), ("""#f""", "\x80") ]