syndicate-lang
/
preserves
mirror of https://gitlab.com/preserves/preserves.git
1
0
Fork 1
Code Releases Activity

Handle pointer type nesting properly

This commit is contained in:
Tony Garnock-Jones 2021-04-22 20:51:48 +02:00
parent 825d208198
commit 9f9514a7e6
9 changed files with 437 additions and 352 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
implementations/javascript/packages/core/src/annotated.ts
View File

@ -71,7 +71,7 @@ export class Annotated<T = DefaultPointer> {
[PreserveOn](encoder: Encoder<T>) { [PreserveOn](encoder: Encoder<T>) {
if (encoder.includeAnnotations) { if (encoder.includeAnnotations) {
for (const a of this.annotations) { for (const a of this.annotations) {
encoder.emitbyte(Tag.Annotation); encoder.state.emitbyte(Tag.Annotation);
encoder.push(a); encoder.push(a);
} }
} }

6
implementations/javascript/packages/core/src/bytes.ts
View File

@ -161,9 +161,9 @@ export class Bytes implements Preservable<never> {
} }
[PreserveOn](encoder: Encoder<never>) { [PreserveOn](encoder: Encoder<never>) {
encoder.emitbyte(Tag.ByteString); encoder.state.emitbyte(Tag.ByteString);
encoder.varint(this.length); encoder.state.varint(this.length);
encoder.emitbytes(this._view); encoder.state.emitbytes(this._view);
} }
get [IsPreservesBytes](): boolean { get [IsPreservesBytes](): boolean {

283
implementations/javascript/packages/core/src/decoder.ts
View File

@ -13,7 +13,7 @@ export interface DecoderOptions {
} }
export interface DecoderPointerOptions<T> extends DecoderOptions { export interface DecoderPointerOptions<T> extends DecoderOptions {
decodePointer?(d: TypedDecoder<T>): T | undefined; decodePointer?(d: TypedDecoder<never>): T;
} }
export interface TypedDecoder<T> { export interface TypedDecoder<T> {
@ -24,7 +24,7 @@ export interface TypedDecoder<T> {
skip(): void; skip(): void;
next(): Value<T>; next(): Value<T>;
withPointerDecoder<S, R>(decodePointer: (d: TypedDecoder<S>) => S | undefined, withPointerDecoder<S, R>(decodePointer: (d: TypedDecoder<T>) => S,
body: (d: TypedDecoder<S>) => R): R; body: (d: TypedDecoder<S>) => R): R;
nextBoolean(): boolean | undefined; nextBoolean(): boolean | undefined;
@ -51,17 +51,16 @@ export function asLiteral<T, E extends Exclude<Value<T>, Annotated<T>>>(
return is(actual, expected) ? expected : void 0; return is(actual, expected) ? expected : void 0;
} }
function _defaultDecodePointer<T>(_d: TypedDecoder<T>): T | undefined { function _defaultDecodePointer<T>(): T {
throw new DecodeError("No decodePointer function supplied"); throw new DecodeError("No decodePointer function supplied");
} }
export class Decoder<T = never> implements TypedDecoder<T> { export class DecoderState {
packet: Uint8Array; packet: Uint8Array;
index = 0; index = 0;
options: DecoderOptions; options: DecoderOptions;
decodePointer: ((d: TypedDecoder<T>) => T | undefined) = _defaultDecodePointer;
constructor(packet: BytesLike = new Uint8Array(0), options: DecoderOptions = {}) { constructor(packet: BytesLike, options: DecoderOptions) {
this.packet = underlying(packet); this.packet = underlying(packet);
this.options = options; this.options = options;
} }
@ -79,6 +78,34 @@ export class Decoder<T = never> implements TypedDecoder<T> {
this.index = 0; this.index = 0;
} }
atEnd(): boolean {
return this.index >= this.packet.length;
}
mark(): number {
return this.index;
}
restoreMark(m: number): void {
this.index = m;
}
shortGuard<R>(body: () => R, short: () => R): R {
if (this.atEnd()) return short();
// ^ important somewhat-common case optimization - avoid the exception
const start = this.mark();
try {
return body();
} catch (e) {
if (ShortPacket.isShortPacket(e)) {
this.restoreMark(start);
return short();
}
throw e;
}
}
nextbyte(): number { nextbyte(): number {
if (this.atEnd()) throw new ShortPacket("Short packet"); if (this.atEnd()) throw new ShortPacket("Short packet");
return this.packet[this.index++]; return this.packet[this.index++];
@ -103,12 +130,6 @@ export class Decoder<T = never> implements TypedDecoder<T> {
return (this.nextbyte() === Tag.End) || (this.index--, false); return (this.nextbyte() === Tag.End) || (this.index--, false);
} }
nextvalues(): Value<T>[] {
const result = [];
while (!this.peekend()) result.push(this.next());
return result;
}
nextint(n: number): number { nextint(n: number): number {
// TODO: Bignums :-/ // TODO: Bignums :-/
if (n === 0) return 0; if (n === 0) return 0;
@ -118,68 +139,6 @@ export class Decoder<T = never> implements TypedDecoder<T> {
return acc; return acc;
} }
wrap(v: Value<T>): Value<T> {
return this.includeAnnotations ? new Annotated(v) : v;
}
static dictionaryFromArray<T>(vs: Value<T>[]): Dictionary<T> {
const d = new Dictionary<T>();
if (vs.length % 2) throw new DecodeError("Missing dictionary value");
for (let i = 0; i < vs.length; i += 2) {
d.set(vs[i], vs[i+1]);
}
return d;
}
unshiftAnnotation(a: Value<T>, v: Annotated<T>) {
if (this.includeAnnotations) {
v.annotations.unshift(a);
}
return v;
}
next(): Value<T> {
const tag = this.nextbyte();
switch (tag) {
case Tag.False: return this.wrap(false);
case Tag.True: return this.wrap(true);
case Tag.Float: return this.wrap(new SingleFloat(this.nextbytes(4).getFloat32(0, false)));
case Tag.Double: return this.wrap(new DoubleFloat(this.nextbytes(8).getFloat64(0, false)));
case Tag.End: throw new DecodeError("Unexpected Compound end marker");
case Tag.Annotation: {
const a = this.next();
const v = this.next() as Annotated<T>;
return this.unshiftAnnotation(a, v);
}
case Tag.Pointer: {
const v = this.decodePointer(this);
if (v === void 0) {
throw new DecodeError("decodePointer function failed");
}
return this.wrap(v);
}
case Tag.SignedInteger: return this.wrap(this.nextint(this.varint()));
case Tag.String: return this.wrap(Bytes.from(this.nextbytes(this.varint())).fromUtf8());
case Tag.ByteString: return this.wrap(Bytes.from(this.nextbytes(this.varint())));
case Tag.Symbol: return this.wrap(Symbol.for(Bytes.from(this.nextbytes(this.varint())).fromUtf8()));
case Tag.Record: {
const vs = this.nextvalues();
if (vs.length === 0) throw new DecodeError("Too few elements in encoded record");
return this.wrap(Record(vs[0], vs.slice(1)));
}
case Tag.Sequence: return this.wrap(this.nextvalues());
case Tag.Set: return this.wrap(new Set(this.nextvalues()));
case Tag.Dictionary: return this.wrap(Decoder.dictionaryFromArray(this.nextvalues()));
default: {
const v = this.nextSmallOrMediumInteger(tag);
if (v === void 0) {
throw new DecodeError("Unsupported Preserves tag: " + tag);
}
return this.wrap(v);
}
}
}
nextSmallOrMediumInteger(tag: number): number | undefined { nextSmallOrMediumInteger(tag: number): number | undefined {
if (tag >= Tag.SmallInteger_lo && tag <= Tag.SmallInteger_lo + 15) { if (tag >= Tag.SmallInteger_lo && tag <= Tag.SmallInteger_lo + 15) {
const v = tag - Tag.SmallInteger_lo; const v = tag - Tag.SmallInteger_lo;
@ -192,36 +151,109 @@ export class Decoder<T = never> implements TypedDecoder<T> {
return void 0; return void 0;
} }
shortGuard<R>(body: () => R, short: () => R): R { wrap<T>(v: Value<T>): Value<T> {
if (this.atEnd()) return short(); return this.includeAnnotations ? new Annotated(v) : v;
// ^ important somewhat-common case optimization - avoid the exception }
const start = this.mark(); unshiftAnnotation<T>(a: Value<T>, v: Annotated<T>): Annotated<T> {
try { if (this.includeAnnotations) {
return body(); v.annotations.unshift(a);
} catch (e) { }
if (ShortPacket.isShortPacket(e)) { return v;
this.restoreMark(start); }
return short(); }
type DecoderFn<T> = (d: DecoderState) => T;
export class Decoder<T = never> implements TypedDecoder<T> {
state: DecoderState;
decodePointer: DecoderFn<T> = _defaultDecodePointer;
constructor(state: DecoderState, decodePointer: DecoderFn<T>);
constructor(packet?: BytesLike, options?: DecoderOptions);
constructor(
packet_or_state: (DecoderState | BytesLike) = new Uint8Array(0),
options_or_decoder?: (DecoderOptions | DecoderFn<T>))
{
if (packet_or_state instanceof DecoderState) {
this.state = packet_or_state;
this.decodePointer = options_or_decoder as DecoderFn<T>;
} else {
this.state = new DecoderState(
packet_or_state,
(options_or_decoder as DecoderOptions) ?? {});
}
}
write(data: BytesLike) {
this.state.write(data);
}
nextvalues(): Value<T>[] {
const result = [];
while (!this.state.peekend()) result.push(this.next());
return result;
}
static dictionaryFromArray<T>(vs: Value<T>[]): Dictionary<T> {
const d = new Dictionary<T>();
if (vs.length % 2) throw new DecodeError("Missing dictionary value");
for (let i = 0; i < vs.length; i += 2) {
d.set(vs[i], vs[i+1]);
}
return d;
}
next(): Value<T> {
const tag = this.state.nextbyte();
switch (tag) {
case Tag.False: return this.state.wrap<T>(false);
case Tag.True: return this.state.wrap<T>(true);
case Tag.Float: return this.state.wrap<T>(new SingleFloat(this.state.nextbytes(4).getFloat32(0, false)));
case Tag.Double: return this.state.wrap<T>(new DoubleFloat(this.state.nextbytes(8).getFloat64(0, false)));
case Tag.End: throw new DecodeError("Unexpected Compound end marker");
case Tag.Annotation: {
const a = this.next();
const v = this.next() as Annotated<T>;
return this.state.unshiftAnnotation(a, v);
}
case Tag.Pointer: return this.state.wrap<T>(this.decodePointer(this.state));
case Tag.SignedInteger: return this.state.wrap<T>(this.state.nextint(this.state.varint()));
case Tag.String: return this.state.wrap<T>(Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8());
case Tag.ByteString: return this.state.wrap<T>(Bytes.from(this.state.nextbytes(this.state.varint())));
case Tag.Symbol: return this.state.wrap<T>(Symbol.for(Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8()));
case Tag.Record: {
const vs = this.nextvalues();
if (vs.length === 0) throw new DecodeError("Too few elements in encoded record");
return this.state.wrap<T>(Record(vs[0], vs.slice(1)));
}
case Tag.Sequence: return this.state.wrap<T>(this.nextvalues());
case Tag.Set: return this.state.wrap<T>(new Set(this.nextvalues()));
case Tag.Dictionary: return this.state.wrap<T>(Decoder.dictionaryFromArray(this.nextvalues()));
default: {
const v = this.state.nextSmallOrMediumInteger(tag);
if (v === void 0) {
throw new DecodeError("Unsupported Preserves tag: " + tag);
}
return this.state.wrap<T>(v);
} }
throw e;
} }
} }
try_next(): Value<T> | undefined { try_next(): Value<T> | undefined {
return this.shortGuard(() => this.next(), () => void 0); return this.state.shortGuard(() => this.next(), () => void 0);
} }
atEnd(): boolean { atEnd(): boolean {
return this.index >= this.packet.length; return this.state.atEnd();
} }
mark(): number { mark(): any {
return this.index; return this.state.mark();
} }
restoreMark(m: number): void { restoreMark(m: any): void {
this.index = m; this.state.restoreMark(m);
} }
skip(): void { skip(): void {
@ -229,39 +261,26 @@ export class Decoder<T = never> implements TypedDecoder<T> {
this.next(); this.next();
} }
replacePointerDecoder<S>(decodePointer: (d: TypedDecoder<S>) => S | undefined): Decoder<S> { pushPointerDecoder<S>(decodePointer: (d: TypedDecoder<T>) => S): Decoder<S> {
const replacement = new Decoder<S>(this.packet, this.options); return new Decoder<S>(this.state, (_s: DecoderState) => decodePointer(this));
replacement.index = this.index;
replacement.decodePointer = decodePointer;
this.packet = new Uint8Array();
this.index = 0;
this.decodePointer = _defaultDecodePointer;
return replacement;
} }
withPointerDecoder<S, R>(decodePointer: (d: TypedDecoder<S>) => S | undefined, withPointerDecoder<S, R>(decodePointer: (d: TypedDecoder<T>) => S,
body: (d: TypedDecoder<S>) => R): R body: (d: TypedDecoder<S>) => R): R
{ {
const oldDecodePointer = this.decodePointer; return body(this.pushPointerDecoder(decodePointer));
const disguised = this as unknown as Decoder<S>;
disguised.decodePointer = decodePointer;
try {
return body(disguised);
} finally {
this.decodePointer = oldDecodePointer;
}
} }
skipAnnotations(): void { skipAnnotations(): void {
if (!this.atEnd() && this.packet[this.index] === Tag.Annotation) { if (!this.state.atEnd() && this.state.packet[this.state.index] === Tag.Annotation) {
this.index++; this.state.index++;
this.skip(); this.skip();
} }
} }
nextBoolean(): boolean | undefined { nextBoolean(): boolean | undefined {
this.skipAnnotations(); this.skipAnnotations();
switch (this.nextbyte()) { switch (this.state.nextbyte()) {
case Tag.False: return false; case Tag.False: return false;
case Tag.True: return true; case Tag.True: return true;
default: return void 0; default: return void 0;
@ -270,58 +289,58 @@ export class Decoder<T = never> implements TypedDecoder<T> {
nextFloat(): SingleFloat | undefined { nextFloat(): SingleFloat | undefined {
this.skipAnnotations(); this.skipAnnotations();
switch (this.nextbyte()) { switch (this.state.nextbyte()) {
case Tag.Float: return new SingleFloat(this.nextbytes(4).getFloat32(0, false)); case Tag.Float: return new SingleFloat(this.state.nextbytes(4).getFloat32(0, false));
default: return void 0; default: return void 0;
} }
} }
nextDouble(): DoubleFloat | undefined { nextDouble(): DoubleFloat | undefined {
this.skipAnnotations(); this.skipAnnotations();
switch (this.nextbyte()) { switch (this.state.nextbyte()) {
case Tag.Double: return new DoubleFloat(this.nextbytes(8).getFloat64(0, false)); case Tag.Double: return new DoubleFloat(this.state.nextbytes(8).getFloat64(0, false));
default: return void 0; default: return void 0;
} }
} }
nextPointer(): T | undefined { nextPointer(): T | undefined {
this.skipAnnotations(); this.skipAnnotations();
switch (this.nextbyte()) { switch (this.state.nextbyte()) {
case Tag.Pointer: return this.decodePointer(this); case Tag.Pointer: return this.decodePointer(this.state);
default: return void 0; default: return void 0;
} }
} }
nextSignedInteger(): number | undefined { nextSignedInteger(): number | undefined {
this.skipAnnotations(); this.skipAnnotations();
const b = this.nextbyte(); const b = this.state.nextbyte();
switch (b) { switch (b) {
case Tag.SignedInteger: return this.nextint(this.varint()); case Tag.SignedInteger: return this.state.nextint(this.state.varint());
default: return this.nextSmallOrMediumInteger(b); default: return this.state.nextSmallOrMediumInteger(b);
} }
} }
nextString(): string | undefined { nextString(): string | undefined {
this.skipAnnotations(); this.skipAnnotations();
switch (this.nextbyte()) { switch (this.state.nextbyte()) {
case Tag.String: return Bytes.from(this.nextbytes(this.varint())).fromUtf8(); case Tag.String: return Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8();
default: return void 0; default: return void 0;
} }
} }
nextByteString(): Bytes | undefined { nextByteString(): Bytes | undefined {
this.skipAnnotations(); this.skipAnnotations();
switch (this.nextbyte()) { switch (this.state.nextbyte()) {
case Tag.ByteString: return Bytes.from(this.nextbytes(this.varint())); case Tag.ByteString: return Bytes.from(this.state.nextbytes(this.state.varint()));
default: return void 0; default: return void 0;
} }
} }
nextSymbol(): symbol | undefined { nextSymbol(): symbol | undefined {
this.skipAnnotations(); this.skipAnnotations();
switch (this.nextbyte()) { switch (this.state.nextbyte()) {
case Tag.Symbol: case Tag.Symbol:
return Symbol.for(Bytes.from(this.nextbytes(this.varint())).fromUtf8()); return Symbol.for(Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8());
default: default:
return void 0; return void 0;
} }
@ -329,31 +348,31 @@ export class Decoder<T = never> implements TypedDecoder<T> {
openRecord(): boolean { openRecord(): boolean {
this.skipAnnotations(); this.skipAnnotations();
return (this.nextbyte() === Tag.Record) || (this.index--, false); return (this.state.nextbyte() === Tag.Record) || (this.state.index--, false);
} }
openSequence(): boolean { openSequence(): boolean {
this.skipAnnotations(); this.skipAnnotations();
return (this.nextbyte() === Tag.Sequence) || (this.index--, false); return (this.state.nextbyte() === Tag.Sequence) || (this.state.index--, false);
} }
openSet(): boolean { openSet(): boolean {
this.skipAnnotations(); this.skipAnnotations();
return (this.nextbyte() === Tag.Set) || (this.index--, false); return (this.state.nextbyte() === Tag.Set) || (this.state.index--, false);
} }
openDictionary(): boolean { openDictionary(): boolean {
this.skipAnnotations(); this.skipAnnotations();
return (this.nextbyte() === Tag.Dictionary) || (this.index--, false); return (this.state.nextbyte() === Tag.Dictionary) || (this.state.index--, false);
} }
closeCompound(): boolean { closeCompound(): boolean {
return this.peekend(); return this.state.peekend();
} }
} }
export function decode<T>(bs: BytesLike, options: DecoderPointerOptions<T> = {}): Value<T> { export function decode<T>(bs: BytesLike, options: DecoderPointerOptions<T> = {}): Value<T> {
return new Decoder<T>(bs, options).withPointerDecoder<T, Value<T>>( return new Decoder<never>(bs, options).withPointerDecoder<T, Value<T>>(
options.decodePointer ?? _defaultDecodePointer, options.decodePointer ?? _defaultDecodePointer,
d => d.next()); d => d.next());
} }

8
implementations/javascript/packages/core/src/dictionary.ts
View File

@ -57,20 +57,20 @@ export class KeyedDictionary<K extends Value<T>, V, T = DefaultPointer> extends
const entries = Array.from(this); const entries = Array.from(this);
const pieces = entries.map<[Bytes, number]>(([k, _v], i) => [canonicalEncode(k), i]); const pieces = entries.map<[Bytes, number]>(([k, _v], i) => [canonicalEncode(k), i]);
pieces.sort((a, b) => Bytes.compare(a[0], b[0])); pieces.sort((a, b) => Bytes.compare(a[0], b[0]));
encoder.emitbyte(Tag.Dictionary); encoder.state.emitbyte(Tag.Dictionary);
pieces.forEach(([_encodedKey, i]) => { pieces.forEach(([_encodedKey, i]) => {
const [k, v] = entries[i]; const [k, v] = entries[i];
encoder.push(k); encoder.push(k);
encoder.push(v as unknown as Value<T>); // Suuuuuuuper unsound encoder.push(v as unknown as Value<T>); // Suuuuuuuper unsound
}); });
encoder.emitbyte(Tag.End); encoder.state.emitbyte(Tag.End);
} else { } else {
encoder.emitbyte(Tag.Dictionary); encoder.state.emitbyte(Tag.Dictionary);
this.forEach((v, k) => { this.forEach((v, k) => {
encoder.push(k); encoder.push(k);
encoder.push(v as unknown as Value<T>); // Suuuuuuuper unsound encoder.push(v as unknown as Value<T>); // Suuuuuuuper unsound
}); });
encoder.emitbyte(Tag.End); encoder.state.emitbyte(Tag.End);
} }
} }
} }

96
implementations/javascript/packages/core/src/encoder.ts
View File

@ -22,7 +22,7 @@ export interface EncoderOptions {
} }
export interface EncoderPointerOptions<T> extends EncoderOptions { export interface EncoderPointerOptions<T> extends EncoderOptions {
encodePointer?: (e: Encoder<T>, v: T) => void; encodePointer?: EncodePointerFunction<T>;
} }
export function asLatin1(bs: Uint8Array): string { export function asLatin1(bs: Uint8Array): string {
@ -33,38 +33,23 @@ function isIterable<T>(v: any): v is Iterable<T> {
return typeof v === 'object' && v !== null && typeof v[Symbol.iterator] === 'function'; return typeof v === 'object' && v !== null && typeof v[Symbol.iterator] === 'function';
} }
function _defaultEncodePointer<T>(e: Encoder<T>, v: T): void { function _defaultEncodePointer<T>(s: EncoderState, v: T): void {
e.push(pointerId(v)); new Encoder(s).push(pointerId(v));
} }
export class Encoder<T> { export class EncoderState {
chunks: Array<Uint8Array>; chunks: Array<Uint8Array>;
view: DataView; view: DataView;
index: number; index: number;
options: EncoderOptions; options: EncoderOptions;
encodePointer: ((e: Encoder<T>, v: T) => void) = _defaultEncodePointer;
constructor(options: EncoderOptions = {}) { constructor(options: EncoderOptions) {
this.chunks = []; this.chunks = [];
this.view = new DataView(new ArrayBuffer(256)); this.view = new DataView(new ArrayBuffer(256));
this.index = 0; this.index = 0;
this.options = options; this.options = options;
} }
withPointerEncoder<S>(encodePointer: (e: Encoder<S>, v: S) => void,
body: (e: Encoder<S>) => void): this
{
const oldEncodePointer = this.encodePointer;
const disguised = this as unknown as Encoder<S>;
disguised.encodePointer = encodePointer;
try {
body(disguised);
return this;
} finally {
this.encodePointer = oldEncodePointer;
}
}
get canonical(): boolean { get canonical(): boolean {
return this.options.canonical ?? true; return this.options.canonical ?? true;
} }
@ -152,11 +137,56 @@ export class Encoder<T> {
this.varint(bs.length); this.varint(bs.length);
this.emitbytes(bs); this.emitbytes(bs);
} }
}
export type EncodePointerFunction<T> = (s: EncoderState, v: T) => void;
export class Encoder<T> {
state: EncoderState;
encodePointer: EncodePointerFunction<T>;
constructor(options: EncoderOptions);
constructor(state: EncoderState, encodePointer?: EncodePointerFunction<T>);
constructor(
state_or_options: (EncoderState | EncoderOptions) = {},
encodePointer?: EncodePointerFunction<T>)
{
if (state_or_options instanceof EncoderState) {
this.state = state_or_options;
this.encodePointer = encodePointer ?? _defaultEncodePointer;
} else {
this.state = new EncoderState(state_or_options);
this.encodePointer = _defaultEncodePointer;
}
}
withPointerEncoder<S>(encodePointer: EncodePointerFunction<S>,
body: (e: Encoder<S>) => void): this
{
body(new Encoder(this.state, encodePointer));
return this;
}
get canonical(): boolean {
return this.state.canonical;
}
get includeAnnotations(): boolean {
return this.state.includeAnnotations;
}
contents(): Bytes {
return this.state.contents();
}
contentsString(): string {
return this.state.contentsString();
}
encodevalues(tag: Tag, items: Iterable<Value<T>>) { encodevalues(tag: Tag, items: Iterable<Value<T>>) {
this.emitbyte(tag); this.state.emitbyte(tag);
for (let i of items) { this.push(i); } for (let i of items) { this.push(i); }
this.emitbyte(Tag.End); this.state.emitbyte(Tag.End);
} }
push(v: Encodable<T>) { push(v: Encodable<T>) {
@ -167,36 +197,36 @@ export class Encoder<T> {
v[PreserveOn](this); v[PreserveOn](this);
} }
else if (typeof v === 'boolean') { else if (typeof v === 'boolean') {
this.emitbyte(v ? Tag.True : Tag.False); this.state.emitbyte(v ? Tag.True : Tag.False);
} }
else if (typeof v === 'number') { else if (typeof v === 'number') {
if (v >= -3 && v <= 12) { if (v >= -3 && v <= 12) {
this.emitbyte(Tag.SmallInteger_lo + ((v + 16) & 0xf)); this.state.emitbyte(Tag.SmallInteger_lo + ((v + 16) & 0xf));
} else { } else {
this.encodeint(v); this.state.encodeint(v);
} }
} }
else if (typeof v === 'string') { else if (typeof v === 'string') {
this.encodebytes(Tag.String, new Bytes(v)._view); this.state.encodebytes(Tag.String, new Bytes(v)._view);
} }
else if (typeof v === 'symbol') { else if (typeof v === 'symbol') {
const key = Symbol.keyFor(v); const key = Symbol.keyFor(v);
if (key === void 0) throw new EncodeError("Cannot preserve non-global Symbol", v); if (key === void 0) throw new EncodeError("Cannot preserve non-global Symbol", v);
this.encodebytes(Tag.Symbol, new Bytes(key)._view); this.state.encodebytes(Tag.Symbol, new Bytes(key)._view);
} }
else if (ArrayBuffer.isView(v)) { else if (ArrayBuffer.isView(v)) {
if (v instanceof Uint8Array) { if (v instanceof Uint8Array) {
this.encodebytes(Tag.ByteString, v); this.state.encodebytes(Tag.ByteString, v);
} else { } else {
const bs = new Uint8Array(v.buffer, v.byteOffset, v.byteLength); const bs = new Uint8Array(v.buffer, v.byteOffset, v.byteLength);
this.encodebytes(Tag.ByteString, bs); this.state.encodebytes(Tag.ByteString, bs);
} }
} }
else if (Record.isRecord<Value<T>, Tuple<Value<T>>, T>(v)) { else if (Record.isRecord<Value<T>, Tuple<Value<T>>, T>(v)) {
this.emitbyte(Tag.Record); this.state.emitbyte(Tag.Record);
this.push(v.label); this.push(v.label);
for (let i of v) { this.push(i); } for (let i of v) { this.push(i); }
this.emitbyte(Tag.End); this.state.emitbyte(Tag.End);
} }
else if (Array.isArray(v)) { else if (Array.isArray(v)) {
this.encodevalues(Tag.Sequence, v); this.encodevalues(Tag.Sequence, v);
@ -205,8 +235,8 @@ export class Encoder<T> {
this.encodevalues(Tag.Sequence, v as Iterable<Value<T>>); this.encodevalues(Tag.Sequence, v as Iterable<Value<T>>);
} }
else { else {
this.emitbyte(Tag.Pointer); this.state.emitbyte(Tag.Pointer);
this.encodePointer(this, v); this.encodePointer(this.state, v);
} }
return this; // for chaining return this; // for chaining
} }

16
implementations/javascript/packages/core/src/float.ts
View File

@ -50,10 +50,10 @@ export class SingleFloat extends Float implements Preservable<never> {
} }
[PreserveOn](encoder: Encoder<never>) { [PreserveOn](encoder: Encoder<never>) {
encoder.emitbyte(Tag.Float); encoder.state.emitbyte(Tag.Float);
encoder.makeroom(4); encoder.state.makeroom(4);
encoder.view.setFloat32(encoder.index, this.value, false); encoder.state.view.setFloat32(encoder.state.index, this.value, false);
encoder.index += 4; encoder.state.index += 4;
} }
get [FloatType](): 'Single' { get [FloatType](): 'Single' {
@ -75,10 +75,10 @@ export class DoubleFloat extends Float implements Preservable<never> {
} }
[PreserveOn](encoder: Encoder<never>) { [PreserveOn](encoder: Encoder<never>) {
encoder.emitbyte(Tag.Double); encoder.state.emitbyte(Tag.Double);
encoder.makeroom(8); encoder.state.makeroom(8);
encoder.view.setFloat64(encoder.index, this.value, false); encoder.state.view.setFloat64(encoder.state.index, this.value, false);
encoder.index += 8; encoder.state.index += 8;
} }
get [FloatType](): 'Double' { get [FloatType](): 'Double' {

10
implementations/javascript/packages/core/src/pointer.ts
View File

@ -1,4 +1,4 @@
import type { Encoder } from "./encoder"; import { Encoder, EncoderState } from "./encoder";
import type { TypedDecoder } from "./decoder"; import type { TypedDecoder } from "./decoder";
import type { Value } from "./values"; import type { Value } from "./values";
@ -24,10 +24,10 @@ export function readDefaultPointer(v: Value<DefaultPointer>): DefaultPointer {
return new DefaultPointer(strip(v)); return new DefaultPointer(strip(v));
} }
export function decodeDefaultPointer(d: TypedDecoder<DefaultPointer>): DefaultPointer { export function decodeDefaultPointer<T>(d: TypedDecoder<T>): DefaultPointer {
return readDefaultPointer(d.next()); return readDefaultPointer(d.withPointerDecoder(decodeDefaultPointer, d => d.next()));
} }
export function encodeDefaultPointer(e: Encoder<DefaultPointer>, w: DefaultPointer): void { export function encodeDefaultPointer(e: EncoderState, w: DefaultPointer): void {
e.push(w.v); new Encoder(e, encodeDefaultPointer).push(w.v);
} }

361
implementations/javascript/packages/core/src/reader.ts
View File

@ -10,25 +10,31 @@ import { Record } from './record';
import { Annotated, newPosition, Position, updatePosition } from './annotated'; import { Annotated, newPosition, Position, updatePosition } from './annotated';
import { Double, DoubleFloat, Single, SingleFloat } from './float'; import { Double, DoubleFloat, Single, SingleFloat } from './float';
import { stringify } from './text'; import { stringify } from './text';
import { decodeDefaultPointer, DefaultPointer, readDefaultPointer } from './pointer';
export interface ReaderOptions<T> { export interface ReaderStateOptions {
includeAnnotations?: boolean; includeAnnotations?: boolean;
decodePointer?: (v: Value<T>) => T;
name?: string | Position; name?: string | Position;
} }
export type DecodePointerFunction<T> = (v: Value<DefaultPointer>) => T;
export interface ReaderOptions<T> extends ReaderStateOptions {
decodePointer?: DecodePointerFunction<T>;
}
type IntOrFloat = 'int' | 'float'; type IntOrFloat = 'int' | 'float';
type Numeric = number | SingleFloat | DoubleFloat; type Numeric = number | SingleFloat | DoubleFloat;
type IntContinuation = (kind: IntOrFloat, acc: string) => Numeric; type IntContinuation = (kind: IntOrFloat, acc: string) => Numeric;
export class Reader<T> { export class ReaderState {
buffer: string; buffer: string;
pos: Position; pos: Position;
index: number; index: number;
discarded = 0; discarded = 0;
options: ReaderOptions<T>; options: ReaderStateOptions;
constructor(buffer: string = '', options: ReaderOptions<T> = {}) { constructor(buffer: string, options: ReaderStateOptions) {
this.buffer = buffer; this.buffer = buffer;
switch (typeof options.name) { switch (typeof options.name) {
case 'undefined': this.pos = newPosition(); break; case 'undefined': this.pos = newPosition(); break;
@ -39,10 +45,18 @@ export class Reader<T> {
this.options = options; this.options = options;
} }
error(message: string, pos: Position): never {
throw new DecodeError(message, { ... pos });
}
get includeAnnotations(): boolean { get includeAnnotations(): boolean {
return this.options.includeAnnotations ?? false; return this.options.includeAnnotations ?? false;
} }
copyPos(): Position {
return { ... this.pos };
}
write(data: string) { write(data: string) {
if (this.atEnd()) { if (this.atEnd()) {
this.buffer = data; this.buffer = data;
@ -53,10 +67,6 @@ export class Reader<T> {
this.index = 0; this.index = 0;
} }
error(message: string, pos: Position): never {
throw new DecodeError(message, { ... pos });
}
atEnd(): boolean { atEnd(): boolean {
return (this.index >= this.buffer.length); return (this.index >= this.buffer.length);
} }
@ -90,129 +100,18 @@ export class Reader<T> {
} }
} }
readCommentLine(): Value<T> { readHex2(): number {
const startPos = { ... this.pos }; const x1 = unhexDigit(this.nextcharcode());
let acc = ''; const x2 = unhexDigit(this.nextcharcode());
while (true) { return (x1 << 4) | x2;
const c = this.nextchar();
if (c === '\n' || c === '\r') {
return this.wrap(acc, startPos);
}
acc = acc + c;
}
} }
wrap(v: Value<T>, pos: Position): Value<T> { readHex4(): number {
if (this.includeAnnotations && !Annotated.isAnnotated(v)) { const x1 = unhexDigit(this.nextcharcode());
v = new Annotated(v, pos); const x2 = unhexDigit(this.nextcharcode());
} const x3 = unhexDigit(this.nextcharcode());
return v; const x4 = unhexDigit(this.nextcharcode());
} return (x1 << 12) | (x2 << 8) | (x3 << 4) | x4;
annotateNextWith(v: Value<T>): Value<T> {
this.skipws();
if (this.atEnd()) {
throw new DecodeError("Trailing annotations and comments are not permitted", this.pos);
}
const u = this.next();
if (this.includeAnnotations) (u as Annotated<T>).annotations.unshift(v);
return u;
}
readToEnd(): Array<Value<T>> {
const acc = [];
while (true) {
this.skipws();
if (this.atEnd()) return acc;
acc.push(this.next());
}
}
next(): Value<T> {
this.skipws();
const startPos = { ... this.pos };
const unwrapped = (() => {
const c = this.nextchar();
switch (c) {
case '-':
return this.readIntpart('-', this.nextchar());
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
return this.readIntpart('', c);
case '"':
return this.readString('"');
case '|':
return Symbol.for(this.readString('|'));
case ';':
return this.annotateNextWith(this.readCommentLine());
case '@':
return this.annotateNextWith(this.next());
case ':':
this.error('Unexpected key/value separator between items', startPos);
case '#': {
const c = this.nextchar();
switch (c) {
case 'f': return false;
case 't': return true;
case '{': return this.seq(new Set<T>(), (v, s) => s.add(v), '}');
case '"': return this.readLiteralBinary();
case 'x':
if (this.nextchar() !== '"') {
this.error('Expected open-quote at start of hex ByteString',
startPos);
}
return this.readHexBinary();
case '[': return this.readBase64Binary();
case '=': {
const bs = unannotate(this.next());
if (!Bytes.isBytes(bs)) this.error('ByteString must follow #=',
startPos);
return decode<T>(bs, {
decodePointer: d => this.options.decodePointer?.(d.next()),
includeAnnotations: this.options.includeAnnotations,
});
}
case '!': {
const d = this.options.decodePointer;
if (d === void 0) {
this.error("No decodePointer function supplied", startPos);
}
return d(this.next());
}
default:
this.error(`Invalid # syntax: ${c}`, startPos);
}
}
case '<': {
const label = this.next();
const fields = this.readSequence('>');
return Record(label, fields);
}
case '[': return this.readSequence(']');
case '{': return this.readDictionary();
case '>': this.error('Unexpected >', startPos);
case ']': this.error('Unexpected ]', startPos);
case '}': this.error('Unexpected }', startPos);
default:
return this.readRawSymbol(c);
}
})();
return this.wrap(unwrapped, startPos);
}
seq<S>(acc: S, update: (v: Value<T>, acc: S) => void, ch: string): S {
while (true) {
this.skipws();
if (this.peek() === ch) {
this.advance();
return acc;
}
update(this.next(), acc);
}
}
readSequence(ch: string): Array<Value<T>> {
return this.seq([] as Array<Value<T>>, (v, acc) => acc.push(v), ch);
} }
readHexBinary(): Bytes { readHexBinary(): Bytes {
@ -227,24 +126,6 @@ export class Reader<T> {
} }
} }
readDictionary(): Dictionary<T> {
return this.seq(new Dictionary<T>(),
(k, acc) => {
this.skipws();
switch (this.peek()) {
case ':':
if (acc.has(k)) this.error(
`Duplicate key: ${stringify(k)}`, this.pos);
this.advance();
acc.set(k, this.next());
break;
default:
this.error('Missing key/value separator', this.pos);
}
},
'}');
}
readBase64Binary(): Bytes { readBase64Binary(): Bytes {
let acc = ''; let acc = '';
while (true) { while (true) {
@ -315,7 +196,7 @@ export class Reader<T> {
} }
} }
readRawSymbol(acc: string): Value<T> { readRawSymbol<T>(acc: string): Value<T> {
while (true) { while (true) {
if (this.atEnd()) break; if (this.atEnd()) break;
const ch = this.peek(); const ch = this.peek();
@ -366,20 +247,6 @@ export class Reader<T> {
} }
} }
readHex2(): number {
const x1 = unhexDigit(this.nextcharcode());
const x2 = unhexDigit(this.nextcharcode());
return (x1 << 4) | x2;
}
readHex4(): number {
const x1 = unhexDigit(this.nextcharcode());
const x2 = unhexDigit(this.nextcharcode());
const x3 = unhexDigit(this.nextcharcode());
const x4 = unhexDigit(this.nextcharcode());
return (x1 << 12) | (x2 << 8) | (x3 << 4) | x4;
}
readString(terminator: string): string { readString(terminator: string): string {
return this.readStringlike(x => x, xs => xs.join(''), terminator, 'u', () => { return this.readStringlike(x => x, xs => xs.join(''), terminator, 'u', () => {
const n1 = this.readHex4(); const n1 = this.readHex4();
@ -410,6 +277,174 @@ export class Reader<T> {
} }
} }
export class Reader<T> {
state: ReaderState;
decodePointer?: DecodePointerFunction<T>;
constructor(state: ReaderState, decodePointer?: DecodePointerFunction<T>);
constructor(buffer: string, options?: ReaderOptions<T>);
constructor(
state_or_buffer: (ReaderState | string) = '',
decodePointer_or_options?: (DecodePointerFunction<T> | ReaderOptions<T>))
{
if (state_or_buffer instanceof ReaderState) {
this.state = state_or_buffer;
this.decodePointer = decodePointer_or_options as DecodePointerFunction<T>;
} else {
const options = (decodePointer_or_options as ReaderOptions<T>) ?? {};
this.state = new ReaderState(state_or_buffer, options);
this.decodePointer = options.decodePointer;
}
}
write(data: string) {
this.state.write(data);
}
readCommentLine(): Value<T> {
const startPos = this.state.copyPos();
let acc = '';
while (true) {
const c = this.state.nextchar();
if (c === '\n' || c === '\r') {
return this.wrap(acc, startPos);
}
acc = acc + c;
}
}
wrap(v: Value<T>, pos: Position): Value<T> {
if (this.state.includeAnnotations && !Annotated.isAnnotated(v)) {
v = new Annotated(v, pos);
}
return v;
}
annotateNextWith(v: Value<T>): Value<T> {
this.state.skipws();
if (this.state.atEnd()) {
throw new DecodeError("Trailing annotations and comments are not permitted",
this.state.pos);
}
const u = this.next();
if (this.state.includeAnnotations) (u as Annotated<T>).annotations.unshift(v);
return u;
}
readToEnd(): Array<Value<T>> {
const acc = [];
while (true) {
this.state.skipws();
if (this.state.atEnd()) return acc;
acc.push(this.next());
}
}
next(): Value<T> {
this.state.skipws();
const startPos = this.state.copyPos();
const unwrapped = ((): Value<T> => {
const c = this.state.nextchar();
switch (c) {
case '-':
return this.state.readIntpart('-', this.state.nextchar());
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
return this.state.readIntpart('', c);
case '"':
return this.state.readString('"');
case '|':
return Symbol.for(this.state.readString('|'));
case ';':
return this.annotateNextWith(this.readCommentLine());
case '@':
return this.annotateNextWith(this.next());
case ':':
this.state.error('Unexpected key/value separator between items', startPos);
case '#': {
const c = this.state.nextchar();
switch (c) {
case 'f': return false;
case 't': return true;
case '{': return this.seq(new Set<T>(), (v, s) => s.add(v), '}');
case '"': return this.state.readLiteralBinary();
case 'x':
if (this.state.nextchar() !== '"') {
this.state.error('Expected open-quote at start of hex ByteString',
startPos);
}
return this.state.readHexBinary();
case '[': return this.state.readBase64Binary();
case '=': {
const bs = unannotate(this.next());
if (!Bytes.isBytes(bs)) this.state.error('ByteString must follow #=',
startPos);
return decode<T>(bs, {
decodePointer: <never>decodeDefaultPointer,
includeAnnotations: this.state.options.includeAnnotations,
});
}
case '!': {
if (this.decodePointer === void 0) {
this.state.error("No decodePointer function supplied", startPos);
}
return this.decodePointer(new Reader(this.state, readDefaultPointer).next());
}
default:
this.state.error(`Invalid # syntax: ${c}`, startPos);
}
}
case '<': {
const label = this.next();
const fields = this.readSequence('>');
return Record(label, fields);
}
case '[': return this.readSequence(']');
case '{': return this.readDictionary();
case '>': this.state.error('Unexpected >', startPos);
case ']': this.state.error('Unexpected ]', startPos);
case '}': this.state.error('Unexpected }', startPos);
default:
return this.state.readRawSymbol(c);
}
})();
return this.wrap(unwrapped, startPos);
}
seq<S>(acc: S, update: (v: Value<T>, acc: S) => void, ch: string): S {
while (true) {
this.state.skipws();
if (this.state.peek() === ch) {
this.state.advance();
return acc;
}
update(this.next(), acc);
}
}
readSequence(ch: string): Array<Value<T>> {
return this.seq([] as Array<Value<T>>, (v, acc) => acc.push(v), ch);
}
readDictionary(): Dictionary<T> {
return this.seq(new Dictionary<T>(),
(k, acc) => {
this.state.skipws();
switch (this.state.peek()) {
case ':':
if (acc.has(k)) this.state.error(
`Duplicate key: ${stringify(k)}`, this.state.pos);
this.state.advance();
acc.set(k, this.next());
break;
default:
this.state.error('Missing key/value separator', this.state.pos);
}
},
'}');
}
}
const BASE64: {[key: string]: number} = {}; const BASE64: {[key: string]: number} = {};
[... 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'].forEach( [... 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'].forEach(
(c, i) => BASE64[c] = i); (c, i) => BASE64[c] = i);

7
implementations/javascript/packages/core/test/codec.test.ts
View File

@ -14,6 +14,7 @@ import {
DefaultPointer, DefaultPointer,
decodeDefaultPointer, decodeDefaultPointer,
encodeDefaultPointer, encodeDefaultPointer,
EncoderState,
} from '../src/index'; } from '../src/index';
const { Tag } = Constants; const { Tag } = Constants;
import './test-utils'; import './test-utils';
@ -108,9 +109,9 @@ describe('encoding and decoding pointers', () => {
expect(encode( expect(encode(
[A, B], [A, B],
{ {
encodePointer(e: Encoder<object>, v: object): void { encodePointer(e: EncoderState, v: object): void {
objects.push(v); objects.push(v);
e.push(objects.length - 1); new Encoder(e, encodeDefaultPointer).push(objects.length - 1);
} }
})).is(Bytes.from([Tag.Sequence, })).is(Bytes.from([Tag.Sequence,
Tag.Pointer, Tag.SmallInteger_lo, Tag.Pointer, Tag.SmallInteger_lo,
@ -128,7 +129,7 @@ describe('encoding and decoding pointers', () => {
Tag.Pointer, Tag.SmallInteger_lo + 1, Tag.Pointer, Tag.SmallInteger_lo + 1,
Tag.End Tag.End
]), { ]), {
decodePointer(d: TypedDecoder<object>): object { decodePointer(d: TypedDecoder<never>): object {
const v = d.next(); const v = d.next();
if (typeof v !== 'number' || v < 0 || v >= objects.length) { if (typeof v !== 'number' || v < 0 || v >= objects.length) {
throw new Error("Unknown pointer target"); throw new Error("Unknown pointer target");