diff --git a/implementations/javascript/packages/core/src/bytes.ts b/implementations/javascript/packages/core/src/bytes.ts index 89fd2a1..3f09e8a 100644 --- a/implementations/javascript/packages/core/src/bytes.ts +++ b/implementations/javascript/packages/core/src/bytes.ts @@ -53,13 +53,17 @@ export class Bytes implements Preservable, PreserveWritable { static fromHex(s: string): Bytes { if (s.length & 1) throw new Error("Cannot decode odd-length hexadecimal string"); + const result = new Bytes(s.length >> 1); + Bytes._raw_fromHexInto(s, result._view); + return result; + } + + static _raw_fromHexInto(s: string, target: Uint8Array): void { const len = s.length >> 1; - const result = new Bytes(len); for (let i = 0; i < len; i++) { - result._view[i] = + target[i] = (unhexDigit(s.charCodeAt(i << 1)) << 4) | unhexDigit(s.charCodeAt((i << 1) + 1)); } - return result; } static fromIO(io: string | BytesLike): string | Bytes { @@ -135,11 +139,11 @@ export class Bytes implements Preservable, PreserveWritable { return Bytes.isBytes(v) ? v : void 0; } - toHex(): string { + toHex(digit = hexDigit): string { var nibbles = []; for (let i = 0; i < this.length; i++) { - nibbles.push(hexDigit(this._view[i] >> 4)); - nibbles.push(hexDigit(this._view[i] & 15)); + nibbles.push(digit(this._view[i] >> 4)); + nibbles.push(digit(this._view[i] & 15)); } return nibbles.join(''); } diff --git a/implementations/javascript/packages/core/src/decoder.ts b/implementations/javascript/packages/core/src/decoder.ts index f552aa8..56eb106 100644 --- a/implementations/javascript/packages/core/src/decoder.ts +++ b/implementations/javascript/packages/core/src/decoder.ts @@ -4,7 +4,7 @@ import { Tag } from "./constants"; import { Set, Dictionary } from "./dictionary"; import { DoubleFloat, SingleFloat } from "./float"; import { Record } from "./record"; -import { Bytes, BytesLike, underlying } from "./bytes"; +import { Bytes, BytesLike, underlying, hexDigit } from "./bytes"; import { Value } from "./values"; import { is } from "./is"; import { embed, GenericEmbedded, Embedded, EmbeddedTypeDecode } from "./embedded"; @@ -34,7 +34,7 @@ export interface TypedDecoder { nextFloat(): SingleFloat | undefined; nextDouble(): DoubleFloat | undefined; nextEmbedded(): Embedded | undefined; - nextSignedInteger(): number | undefined; + nextSignedInteger(): number | bigint | undefined; nextString(): string | undefined; nextByteString(): Bytes | undefined; nextSymbol(): symbol | undefined; @@ -130,15 +130,42 @@ export class DecoderState { return (this.nextbyte() === Tag.End) || (this.index--, false); } - nextint(n: number): number { - // TODO: Bignums :-/ + nextint(n: number): number | bigint { + const start = this.index; if (n === 0) return 0; + if (n > 7) return this.nextbigint(n); + if (n === 7) { + const highByte = this.packet[this.index]; + if ((highByte >= 0x20) && (highByte < 0xe0)) { + return this.nextbigint(n); + } + // if highByte is 0xe0, we still might have a value + // equal to (Number.MIN_SAFE_INTEGER-1). + } let acc = this.nextbyte(); if (acc & 0x80) acc -= 256; for (let i = 1; i < n; i++) acc = (acc * 256) + this.nextbyte(); + if (!Number.isSafeInteger(acc)) { + this.index = start; + return this.nextbigint(n); + } return acc; } + nextbigint(n: number): bigint { + if (n === 0) return BigInt(0); + const bs = Bytes.from(this.nextbytes(n)); + if (bs.get(0) >= 128) { + // negative + const hex = bs.toHex(d => hexDigit(15 - d)); + return ~BigInt('0x' + hex); + } else { + // (strictly) positive + const hex = bs.toHex(); + return BigInt('0x' + hex); + } + } + wrap(v: Value): Value { return this.includeAnnotations ? new Annotated(v) : v; } @@ -306,7 +333,7 @@ export class Decoder implements TypedDecoder { }); } - nextSignedInteger(): number | undefined { + nextSignedInteger(): number | bigint | undefined { return this.skipAnnotations((reset) => { switch (this.state.nextbyte()) { case Tag.SignedInteger: return this.state.nextint(this.state.varint()); diff --git a/implementations/javascript/packages/core/src/encoder.ts b/implementations/javascript/packages/core/src/encoder.ts index 92b9a8c..c4942dc 100644 --- a/implementations/javascript/packages/core/src/encoder.ts +++ b/implementations/javascript/packages/core/src/encoder.ts @@ -1,5 +1,5 @@ import { Tag } from "./constants"; -import { Bytes } from "./bytes"; +import { Bytes, unhexDigit } from "./bytes"; import { Value } from "./values"; import { EncodeError } from "./codec"; import { Record, Tuple } from "./record"; @@ -122,6 +122,13 @@ export class EncoderState { this.index += bs.length; } + claimbytes(count: number) { + this.makeroom(count); + const view = new Uint8Array(this.view.buffer, this.index, count); + this.index += count; + return view; + } + varint(v: number) { while (v >= 128) { this.emitbyte((v % 128) + 128); @@ -130,8 +137,9 @@ export class EncoderState { this.emitbyte(v); } - encodeint(v: number) { - // TODO: Bignums :-/ + encodeint(v: number | bigint) { + if (typeof v === 'bigint') return this.encodebigint(v); + this.emitbyte(Tag.SignedInteger); if (v === 0) { @@ -153,6 +161,37 @@ export class EncoderState { enc(bytecount, v); } + encodebigint(v: bigint) { + this.emitbyte(Tag.SignedInteger); + + let hex: string; + if (v > 0) { + hex = v.toString(16); + if (hex.length & 1) { + hex = '0' + hex; + } else if (unhexDigit(hex.charCodeAt(0)) >= 8) { + hex = '00' + hex; + } + } else if (v < 0) { + const negatedHex = (~v).toString(16); + hex = ''; + for (let i = 0; i < negatedHex.length; i++) { + hex = hex + 'fedcba9876543210'[unhexDigit(negatedHex.charCodeAt(i))]; + } + if (hex.length & 1) { + hex = 'f' + hex; + } else if (unhexDigit(hex.charCodeAt(0)) < 8) { + hex = 'ff' + hex; + } + } else { + this.emitbyte(0); + return; + } + + this.varint(hex.length >> 1); + Bytes._raw_fromHexInto(hex, this.claimbytes(hex.length >> 1)); + } + encodebytes(tag: Tag, bs: Uint8Array) { this.emitbyte(tag); this.varint(bs.length); @@ -219,7 +258,7 @@ export class Encoder { else if (typeof v === 'boolean') { this.state.emitbyte(v ? Tag.True : Tag.False); } - else if (typeof v === 'number') { + else if (typeof v === 'number' || typeof v === 'bigint') { this.state.encodeint(v); } else if (typeof v === 'string') { diff --git a/implementations/javascript/packages/core/src/fold.ts b/implementations/javascript/packages/core/src/fold.ts index fe04412..d65b024 100644 --- a/implementations/javascript/packages/core/src/fold.ts +++ b/implementations/javascript/packages/core/src/fold.ts @@ -28,7 +28,7 @@ export interface FoldMethods { boolean(b: boolean): R; single(f: number): R; double(f: number): R; - integer(i: number): R; + integer(i: number | bigint): R; string(s: string): R; bytes(b: Bytes): R; symbol(s: symbol): R; @@ -47,7 +47,7 @@ export class VoidFold implements FoldMethods { boolean(b: boolean): void {} single(f: number): void {} double(f: number): void {} - integer(i: number): void {} + integer(i: number | bigint): void {} string(s: string): void {} bytes(b: Bytes): void {} symbol(s: symbol): void {} @@ -79,7 +79,7 @@ export abstract class ValueFold implements FoldMethods> { double(f: number): Value { return Double(f); } - integer(i: number): Value { + integer(i: number | bigint): Value { return i; } string(s: string): Value { @@ -138,6 +138,8 @@ export function valueClass(v: Value): ValueClass { } else { return ValueClass.SignedInteger; } + case 'bigint': + return ValueClass.SignedInteger; case 'string': return ValueClass.String; case 'symbol': @@ -181,6 +183,8 @@ export function fold(v: Value, o: FoldMethods): R { } else { return o.integer(v); } + case 'bigint': + return o.integer(v); case 'string': return o.string(v); case 'symbol': diff --git a/implementations/javascript/packages/core/src/fromjs.ts b/implementations/javascript/packages/core/src/fromjs.ts index c676152..7e3c1d2 100644 --- a/implementations/javascript/packages/core/src/fromjs.ts +++ b/implementations/javascript/packages/core/src/fromjs.ts @@ -12,6 +12,7 @@ export function fromJS(x: any): Value { throw new TypeError("Refusing to autoconvert non-integer number to Single or Double"); } // FALL THROUGH + case 'bigint': case 'string': case 'symbol': case 'boolean': @@ -19,7 +20,6 @@ export function fromJS(x: any): Value { case 'undefined': case 'function': - case 'bigint': break; case 'object': diff --git a/implementations/javascript/packages/core/src/is.ts b/implementations/javascript/packages/core/src/is.ts index 03551da..355f59c 100644 --- a/implementations/javascript/packages/core/src/is.ts +++ b/implementations/javascript/packages/core/src/is.ts @@ -12,7 +12,13 @@ export function is(a: any, b: any): boolean { if (isAnnotated(a)) a = a.item; if (isAnnotated(b)) b = b.item; if (Object.is(a, b)) return true; - if (typeof a !== typeof b) return false; + if (typeof a !== typeof b) { + if ((typeof a === 'number' && typeof b === 'bigint') || + (typeof a === 'bigint' && typeof b === 'number')) { + return a == b; + } + return false; + } if (typeof a === 'object') { if (a === null || b === null) return false; if ('equals' in a && typeof a.equals === 'function') return a.equals(b, is); diff --git a/implementations/javascript/packages/core/src/merge.ts b/implementations/javascript/packages/core/src/merge.ts index fa8b215..052374f 100644 --- a/implementations/javascript/packages/core/src/merge.ts +++ b/implementations/javascript/packages/core/src/merge.ts @@ -7,6 +7,7 @@ import { Set, Dictionary } from "./dictionary"; import { Annotated } from "./annotated"; import { unannotate } from "./strip"; import { embed, isEmbedded, Embedded } from "./embedded"; +import { isCompound } from "./compound"; export function merge( mergeEmbeddeds: (a: T, b: T) => T | undefined, @@ -18,7 +19,17 @@ export function merge( } function walk(a: Value, b: Value): Value { - if (a === b) return a; + if (a === b) { + // Shortcut for merges of trivially identical values. + return a; + } + if (!isCompound(a) && !isCompound(b)) { + // Don't do expensive recursive comparisons for compounds. + if (is(a, b)) { + // Shortcut for merges of marginally less trivially identical values. + return a; + } + } return fold>(a, { boolean: die, single(_f: number) { return is(a, b) ? a : die(); }, diff --git a/implementations/javascript/packages/core/src/reader.ts b/implementations/javascript/packages/core/src/reader.ts index 50bec7b..b17c86c 100644 --- a/implementations/javascript/packages/core/src/reader.ts +++ b/implementations/javascript/packages/core/src/reader.ts @@ -21,9 +21,8 @@ export interface ReaderOptions extends ReaderStateOptions { embeddedDecode?: EmbeddedTypeDecode; } -type IntOrFloat = 'int' | 'float'; -type Numeric = number | SingleFloat | DoubleFloat; -type IntContinuation = (kind: IntOrFloat, acc: string) => Numeric; +const MAX_SAFE_INTEGERn = BigInt(Number.MAX_SAFE_INTEGER); +const MIN_SAFE_INTEGERn = BigInt(Number.MIN_SAFE_INTEGER); export const NUMBER_RE: RegExp = /^([-+]?\d+)(((\.\d+([eE][-+]?\d+)?)|([eE][-+]?\d+))([fF]?))?$/; // Groups: @@ -174,9 +173,12 @@ export class ReaderState { const m = NUMBER_RE.exec(acc); if (m) { if (m[2] === void 0) { - let v = parseInt(m[1]); - if (Object.is(v, -0)) v = 0; - return v; + let v = BigInt(m[1]); + if (v <= MIN_SAFE_INTEGERn || v >= MAX_SAFE_INTEGERn) { + return v; + } else { + return Number(v); + } } else if (m[7] === '') { return Double(parseFloat(m[1] + m[3])); } else { diff --git a/implementations/javascript/packages/core/src/values.ts b/implementations/javascript/packages/core/src/values.ts index 1746bcb..1a030f5 100644 --- a/implementations/javascript/packages/core/src/values.ts +++ b/implementations/javascript/packages/core/src/values.ts @@ -15,7 +15,7 @@ export type Atom = | boolean | SingleFloat | DoubleFloat - | number + | number | bigint | string | Bytes | symbol; diff --git a/implementations/javascript/packages/core/src/writer.ts b/implementations/javascript/packages/core/src/writer.ts index 8409e5c..93c9d20 100644 --- a/implementations/javascript/packages/core/src/writer.ts +++ b/implementations/javascript/packages/core/src/writer.ts @@ -278,6 +278,7 @@ export class Writer { } break; } + case 'bigint': case 'number': this.state.pieces.push('' + v); break; @@ -328,7 +329,9 @@ export class Writer { } break; default: - throw new Error(`Internal error: unhandled in Preserves Writer.push for ${v}`); + ((_: never) => { + throw new Error(`Internal error: unhandled in Preserves Writer.push for ${v}`); + })(v); } return this; // for chaining } diff --git a/implementations/javascript/packages/core/test/codec.test.ts b/implementations/javascript/packages/core/test/codec.test.ts index 70f821d..9b0291c 100644 --- a/implementations/javascript/packages/core/test/codec.test.ts +++ b/implementations/javascript/packages/core/test/codec.test.ts @@ -184,6 +184,71 @@ describe('encoding and decoding embeddeds', () => { }); }); +describe('integer text parsing', () => { + it('should work for zero', () => { + expect(parse('0')).is(0); + }); + + it('should work for smallish positive integers', () => { + expect(parse('60000')).is(60000); + }); + it('should work for smallish negative integers', () => { + expect(parse('-60000')).is(-60000); + }); + + it('should work for largeish positive integers', () => { + expect(parse('1234567812345678123456781234567')) + .is(BigInt("1234567812345678123456781234567")); + }); + it('should work for largeish negative integers', () => { + expect(parse('-1234567812345678123456781234567')) + .is(BigInt("-1234567812345678123456781234567")); + }); + + it('should work for larger positive integers', () => { + expect(parse('12345678123456781234567812345678')) + .is(BigInt("12345678123456781234567812345678")); + }); + it('should work for larger negative integers', () => { + expect(parse('-12345678123456781234567812345678')) + .is(BigInt("-12345678123456781234567812345678")); + }); +}); + +describe('integer binary encoding', () => { + it('should work for zero integers', () => { + expect(encode(0)).is(Bytes.fromHex('b000')); + }); + it('should work for zero bigints', () => { + expect(encode(BigInt(0))).is(Bytes.fromHex('b000')); + }); + + it('should work for smallish positive integers', () => { + expect(encode(60000)).is(Bytes.fromHex('b00300ea60')); + }); + it('should work for smallish negative integers', () => { + expect(encode(-60000)).is(Bytes.fromHex('b003ff15a0')); + }); + + it('should work for largeish positive integers', () => { + expect(encode(BigInt("1234567812345678123456781234567"))) + .is(Bytes.fromHex('b00d0f951a8f2b4b049d518b923187')); + }); + it('should work for largeish negative integers', () => { + expect(encode(BigInt("-1234567812345678123456781234567"))) + .is(Bytes.fromHex('b00df06ae570d4b4fb62ae746dce79')); + }); + + it('should work for larger positive integers', () => { + expect(encode(BigInt("12345678123456781234567812345678"))) + .is(Bytes.fromHex('b00e009bd30997b0ee2e252f73b5ef4e')); + }); + it('should work for larger negative integers', () => { + expect(encode(BigInt("-12345678123456781234567812345678"))) + .is(Bytes.fromHex('b00eff642cf6684f11d1dad08c4a10b2')); + }); +}); + describe('common test suite', () => { const samples_bin = fs.readFileSync(__dirname + '/../../../../../tests/samples.bin'); const samples = decodeWithAnnotations(samples_bin, { embeddedDecode: genericEmbeddedTypeDecode }); diff --git a/implementations/javascript/packages/core/test/values.test.ts b/implementations/javascript/packages/core/test/values.test.ts index 11ad12d..7a91ba6 100644 --- a/implementations/javascript/packages/core/test/values.test.ts +++ b/implementations/javascript/packages/core/test/values.test.ts @@ -1,4 +1,4 @@ -import { Single, Double, fromJS, Dictionary, IDENTITY_FOLD, fold, mapEmbeddeds, Value, embed } from '../src/index'; +import { Single, Double, fromJS, Dictionary, IDENTITY_FOLD, fold, mapEmbeddeds, Value, embed, preserves } from '../src/index'; import './test-utils'; describe('Single', () => { @@ -41,4 +41,51 @@ describe('fromJS', () => { it('should map integers to themselves', () => { expect(fromJS(1)).toBe(1); }); + + it('should map bigints to themselves', () => { + expect(fromJS(BigInt("12345678123456781234567812345678"))) + .toBe(BigInt("12345678123456781234567812345678"));; + }); +}); + +describe('is()', () => { + it('should compare small integers sensibly', () => { + expect(3).is(3); + expect(3).not.is(4); + }); + it('should compare large integers sensibly', () => { + const a = BigInt("12345678123456781234567812345678"); + const b = BigInt("12345678123456781234567812345679"); + expect(a).is(a); + expect(a).is(BigInt("12345678123456781234567812345678")); + expect(a).not.is(b); + }); + it('should compare mixed integers sensibly', () => { + const a = BigInt("12345678123456781234567812345678"); + const b = BigInt("3"); + const c = BigInt("4"); + expect(3).not.is(a); + expect(a).not.is(3); + expect(3).not.toBe(b); + expect(3).is(b); + expect(b).not.toBe(3); + expect(b).is(3); + expect(3).not.toBe(c); + expect(3).not.is(c); + expect(c).not.toBe(3); + expect(c).not.is(3); + }); +}); + +describe('`preserves` formatter', () => { + it('should format numbers', () => { + expect(preserves`>${3}<`).toBe('>3<'); + }); + it('should format small bigints', () => { + expect(preserves`>${BigInt("3")}<`).toBe('>3<'); + }); + it('should format big bigints', () => { + expect(preserves`>${BigInt("12345678123456781234567812345678")}<`) + .toBe('>12345678123456781234567812345678<'); + }); }); diff --git a/implementations/python/tests/samples.bin b/implementations/python/tests/samples.bin index 70ebf1b..dbb41bf 100644 Binary files a/implementations/python/tests/samples.bin and b/implementations/python/tests/samples.bin differ diff --git a/implementations/python/tests/samples.pr b/implementations/python/tests/samples.pr index 4646594..df8ae0b 100644 --- a/implementations/python/tests/samples.pr +++ b/implementations/python/tests/samples.pr @@ -118,6 +118,8 @@ float14: @"+qNaN" float15: @"-qNaN" float16: @"-qNaN" + int-12345678123456781234567812345678: + int-1234567812345678123456781234567: int-257: int-256: int-255: @@ -146,6 +148,8 @@ int65536: int131072: int2500000000: + int1234567812345678123456781234567: + int12345678123456781234567812345678: int87112285931760246646623899502532662132736: list0: list4: diff --git a/implementations/racket/preserves/preserves/tests/samples.pr b/implementations/racket/preserves/preserves/tests/samples.pr index 4646594..df8ae0b 100644 --- a/implementations/racket/preserves/preserves/tests/samples.pr +++ b/implementations/racket/preserves/preserves/tests/samples.pr @@ -118,6 +118,8 @@ float14: @"+qNaN" float15: @"-qNaN" float16: @"-qNaN" + int-12345678123456781234567812345678: + int-1234567812345678123456781234567: int-257: int-256: int-255: @@ -146,6 +148,8 @@ int65536: int131072: int2500000000: + int1234567812345678123456781234567: + int12345678123456781234567812345678: int87112285931760246646623899502532662132736: list0: list4: diff --git a/tests/samples.bin b/tests/samples.bin index 70ebf1b..dbb41bf 100644 Binary files a/tests/samples.bin and b/tests/samples.bin differ diff --git a/tests/samples.pr b/tests/samples.pr index 4646594..df8ae0b 100644 --- a/tests/samples.pr +++ b/tests/samples.pr @@ -118,6 +118,8 @@ float14: @"+qNaN" float15: @"-qNaN" float16: @"-qNaN" + int-12345678123456781234567812345678: + int-1234567812345678123456781234567: int-257: int-256: int-255: @@ -146,6 +148,8 @@ int65536: int131072: int2500000000: + int1234567812345678123456781234567: + int12345678123456781234567812345678: int87112285931760246646623899502532662132736: list0: list4: