syndicate-lang
/
preserves
mirror of https://gitlab.com/preserves/preserves.git
1
0
Fork 1
Code Releases Activity

Handle pointer type nesting properly

This commit is contained in:
Tony Garnock-Jones 2021-04-22 20:51:48 +02:00
parent 825d208198
commit 9f9514a7e6
9 changed files with 437 additions and 352 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
implementations/javascript/packages/core/src/annotated.ts
View File

@ -71,7 +71,7 @@ export class Annotated<T = DefaultPointer> {
[PreserveOn](encoder: Encoder<T>) {
if (encoder.includeAnnotations) {
for (const a of this.annotations) {
encoder.emitbyte(Tag.Annotation);
encoder.state.emitbyte(Tag.Annotation);
encoder.push(a);
}
}

6
implementations/javascript/packages/core/src/bytes.ts
View File

@ -161,9 +161,9 @@ export class Bytes implements Preservable<never> {
}
[PreserveOn](encoder: Encoder<never>) {
encoder.emitbyte(Tag.ByteString);
encoder.varint(this.length);
encoder.emitbytes(this._view);
encoder.state.emitbyte(Tag.ByteString);
encoder.state.varint(this.length);
encoder.state.emitbytes(this._view);
}
get [IsPreservesBytes](): boolean {

283
implementations/javascript/packages/core/src/decoder.ts
View File

@ -13,7 +13,7 @@ export interface DecoderOptions {
}
export interface DecoderPointerOptions<T> extends DecoderOptions {
decodePointer?(d: TypedDecoder<T>): T | undefined;
decodePointer?(d: TypedDecoder<never>): T;
}
export interface TypedDecoder<T> {
@ -24,7 +24,7 @@ export interface TypedDecoder<T> {
skip(): void;
next(): Value<T>;
withPointerDecoder<S, R>(decodePointer: (d: TypedDecoder<S>) => S | undefined,
withPointerDecoder<S, R>(decodePointer: (d: TypedDecoder<T>) => S,
body: (d: TypedDecoder<S>) => R): R;
nextBoolean(): boolean | undefined;
@ -51,17 +51,16 @@ export function asLiteral<T, E extends Exclude<Value<T>, Annotated<T>>>(
return is(actual, expected) ? expected : void 0;
}
function _defaultDecodePointer<T>(_d: TypedDecoder<T>): T | undefined {
function _defaultDecodePointer<T>(): T {
throw new DecodeError("No decodePointer function supplied");
}
export class Decoder<T = never> implements TypedDecoder<T> {
export class DecoderState {
packet: Uint8Array;
index = 0;
options: DecoderOptions;
decodePointer: ((d: TypedDecoder<T>) => T | undefined) = _defaultDecodePointer;
constructor(packet: BytesLike = new Uint8Array(0), options: DecoderOptions = {}) {
constructor(packet: BytesLike, options: DecoderOptions) {
this.packet = underlying(packet);
this.options = options;
}
@ -79,6 +78,34 @@ export class Decoder<T = never> implements TypedDecoder<T> {
this.index = 0;
}
atEnd(): boolean {
return this.index >= this.packet.length;
}
mark(): number {
return this.index;
}
restoreMark(m: number): void {
this.index = m;
}
shortGuard<R>(body: () => R, short: () => R): R {
if (this.atEnd()) return short();
// ^ important somewhat-common case optimization - avoid the exception
const start = this.mark();
try {
return body();
} catch (e) {
if (ShortPacket.isShortPacket(e)) {
this.restoreMark(start);
return short();
}
throw e;
}
}
nextbyte(): number {
if (this.atEnd()) throw new ShortPacket("Short packet");
return this.packet[this.index++];
@ -103,12 +130,6 @@ export class Decoder<T = never> implements TypedDecoder<T> {
return (this.nextbyte() === Tag.End) || (this.index--, false);
}
nextvalues(): Value<T>[] {
const result = [];
while (!this.peekend()) result.push(this.next());
return result;
}
nextint(n: number): number {
// TODO: Bignums :-/
if (n === 0) return 0;
@ -118,68 +139,6 @@ export class Decoder<T = never> implements TypedDecoder<T> {
return acc;
}
wrap(v: Value<T>): Value<T> {
return this.includeAnnotations ? new Annotated(v) : v;
}
static dictionaryFromArray<T>(vs: Value<T>[]): Dictionary<T> {
const d = new Dictionary<T>();
if (vs.length % 2) throw new DecodeError("Missing dictionary value");
for (let i = 0; i < vs.length; i += 2) {
d.set(vs[i], vs[i+1]);
}
return d;
}
unshiftAnnotation(a: Value<T>, v: Annotated<T>) {
if (this.includeAnnotations) {
v.annotations.unshift(a);
}
return v;
}
next(): Value<T> {
const tag = this.nextbyte();
switch (tag) {
case Tag.False: return this.wrap(false);
case Tag.True: return this.wrap(true);
case Tag.Float: return this.wrap(new SingleFloat(this.nextbytes(4).getFloat32(0, false)));
case Tag.Double: return this.wrap(new DoubleFloat(this.nextbytes(8).getFloat64(0, false)));
case Tag.End: throw new DecodeError("Unexpected Compound end marker");
case Tag.Annotation: {
const a = this.next();
const v = this.next() as Annotated<T>;
return this.unshiftAnnotation(a, v);
}
case Tag.Pointer: {
const v = this.decodePointer(this);
if (v === void 0) {
throw new DecodeError("decodePointer function failed");
}
return this.wrap(v);
}
case Tag.SignedInteger: return this.wrap(this.nextint(this.varint()));
case Tag.String: return this.wrap(Bytes.from(this.nextbytes(this.varint())).fromUtf8());
case Tag.ByteString: return this.wrap(Bytes.from(this.nextbytes(this.varint())));
case Tag.Symbol: return this.wrap(Symbol.for(Bytes.from(this.nextbytes(this.varint())).fromUtf8()));
case Tag.Record: {
const vs = this.nextvalues();
if (vs.length === 0) throw new DecodeError("Too few elements in encoded record");
return this.wrap(Record(vs[0], vs.slice(1)));
}
case Tag.Sequence: return this.wrap(this.nextvalues());
case Tag.Set: return this.wrap(new Set(this.nextvalues()));
case Tag.Dictionary: return this.wrap(Decoder.dictionaryFromArray(this.nextvalues()));
default: {
const v = this.nextSmallOrMediumInteger(tag);
if (v === void 0) {
throw new DecodeError("Unsupported Preserves tag: " + tag);
}
return this.wrap(v);
}
}
}
nextSmallOrMediumInteger(tag: number): number | undefined {
if (tag >= Tag.SmallInteger_lo && tag <= Tag.SmallInteger_lo + 15) {
const v = tag - Tag.SmallInteger_lo;
@ -192,36 +151,109 @@ export class Decoder<T = never> implements TypedDecoder<T> {
return void 0;
}
shortGuard<R>(body: () => R, short: () => R): R {
if (this.atEnd()) return short();
// ^ important somewhat-common case optimization - avoid the exception
wrap<T>(v: Value<T>): Value<T> {
return this.includeAnnotations ? new Annotated(v) : v;
}
const start = this.mark();
try {
return body();
} catch (e) {
if (ShortPacket.isShortPacket(e)) {
this.restoreMark(start);
return short();
unshiftAnnotation<T>(a: Value<T>, v: Annotated<T>): Annotated<T> {
if (this.includeAnnotations) {
v.annotations.unshift(a);
}
return v;
}
}
type DecoderFn<T> = (d: DecoderState) => T;
export class Decoder<T = never> implements TypedDecoder<T> {
state: DecoderState;
decodePointer: DecoderFn<T> = _defaultDecodePointer;
constructor(state: DecoderState, decodePointer: DecoderFn<T>);
constructor(packet?: BytesLike, options?: DecoderOptions);
constructor(
packet_or_state: (DecoderState | BytesLike) = new Uint8Array(0),
options_or_decoder?: (DecoderOptions | DecoderFn<T>))
{
if (packet_or_state instanceof DecoderState) {
this.state = packet_or_state;
this.decodePointer = options_or_decoder as DecoderFn<T>;
} else {
this.state = new DecoderState(
packet_or_state,
(options_or_decoder as DecoderOptions) ?? {});
}
}
write(data: BytesLike) {
this.state.write(data);
}
nextvalues(): Value<T>[] {
const result = [];
while (!this.state.peekend()) result.push(this.next());
return result;
}
static dictionaryFromArray<T>(vs: Value<T>[]): Dictionary<T> {
const d = new Dictionary<T>();
if (vs.length % 2) throw new DecodeError("Missing dictionary value");
for (let i = 0; i < vs.length; i += 2) {
d.set(vs[i], vs[i+1]);
}
return d;
}
next(): Value<T> {
const tag = this.state.nextbyte();
switch (tag) {
case Tag.False: return this.state.wrap<T>(false);
case Tag.True: return this.state.wrap<T>(true);
case Tag.Float: return this.state.wrap<T>(new SingleFloat(this.state.nextbytes(4).getFloat32(0, false)));
case Tag.Double: return this.state.wrap<T>(new DoubleFloat(this.state.nextbytes(8).getFloat64(0, false)));
case Tag.End: throw new DecodeError("Unexpected Compound end marker");
case Tag.Annotation: {
const a = this.next();
const v = this.next() as Annotated<T>;
return this.state.unshiftAnnotation(a, v);
}
case Tag.Pointer: return this.state.wrap<T>(this.decodePointer(this.state));
case Tag.SignedInteger: return this.state.wrap<T>(this.state.nextint(this.state.varint()));
case Tag.String: return this.state.wrap<T>(Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8());
case Tag.ByteString: return this.state.wrap<T>(Bytes.from(this.state.nextbytes(this.state.varint())));
case Tag.Symbol: return this.state.wrap<T>(Symbol.for(Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8()));
case Tag.Record: {
const vs = this.nextvalues();
if (vs.length === 0) throw new DecodeError("Too few elements in encoded record");
return this.state.wrap<T>(Record(vs[0], vs.slice(1)));
}
case Tag.Sequence: return this.state.wrap<T>(this.nextvalues());
case Tag.Set: return this.state.wrap<T>(new Set(this.nextvalues()));
case Tag.Dictionary: return this.state.wrap<T>(Decoder.dictionaryFromArray(this.nextvalues()));
default: {
const v = this.state.nextSmallOrMediumInteger(tag);
if (v === void 0) {
throw new DecodeError("Unsupported Preserves tag: " + tag);
}
return this.state.wrap<T>(v);
}
throw e;
}
}
try_next(): Value<T> | undefined {
return this.shortGuard(() => this.next(), () => void 0);
return this.state.shortGuard(() => this.next(), () => void 0);
}
atEnd(): boolean {
return this.index >= this.packet.length;
return this.state.atEnd();
}
mark(): number {
return this.index;
mark(): any {
return this.state.mark();
}
restoreMark(m: number): void {
this.index = m;
restoreMark(m: any): void {
this.state.restoreMark(m);
}
skip(): void {
@ -229,39 +261,26 @@ export class Decoder<T = never> implements TypedDecoder<T> {
this.next();
}
replacePointerDecoder<S>(decodePointer: (d: TypedDecoder<S>) => S | undefined): Decoder<S> {
const replacement = new Decoder<S>(this.packet, this.options);
replacement.index = this.index;
replacement.decodePointer = decodePointer;
this.packet = new Uint8Array();
this.index = 0;
this.decodePointer = _defaultDecodePointer;
return replacement;
pushPointerDecoder<S>(decodePointer: (d: TypedDecoder<T>) => S): Decoder<S> {
return new Decoder<S>(this.state, (_s: DecoderState) => decodePointer(this));
}
withPointerDecoder<S, R>(decodePointer: (d: TypedDecoder<S>) => S | undefined,
withPointerDecoder<S, R>(decodePointer: (d: TypedDecoder<T>) => S,
body: (d: TypedDecoder<S>) => R): R
{
const oldDecodePointer = this.decodePointer;
const disguised = this as unknown as Decoder<S>;
disguised.decodePointer = decodePointer;
try {
return body(disguised);
} finally {
this.decodePointer = oldDecodePointer;
}
return body(this.pushPointerDecoder(decodePointer));
}
skipAnnotations(): void {
if (!this.atEnd() && this.packet[this.index] === Tag.Annotation) {
this.index++;
if (!this.state.atEnd() && this.state.packet[this.state.index] === Tag.Annotation) {
this.state.index++;
this.skip();
}
}
nextBoolean(): boolean | undefined {
this.skipAnnotations();
switch (this.nextbyte()) {
switch (this.state.nextbyte()) {
case Tag.False: return false;
case Tag.True: return true;
default: return void 0;
@ -270,58 +289,58 @@ export class Decoder<T = never> implements TypedDecoder<T> {
nextFloat(): SingleFloat | undefined {
this.skipAnnotations();
switch (this.nextbyte()) {
case Tag.Float: return new SingleFloat(this.nextbytes(4).getFloat32(0, false));
switch (this.state.nextbyte()) {
case Tag.Float: return new SingleFloat(this.state.nextbytes(4).getFloat32(0, false));
default: return void 0;
}
}
nextDouble(): DoubleFloat | undefined {
this.skipAnnotations();
switch (this.nextbyte()) {
case Tag.Double: return new DoubleFloat(this.nextbytes(8).getFloat64(0, false));
switch (this.state.nextbyte()) {
case Tag.Double: return new DoubleFloat(this.state.nextbytes(8).getFloat64(0, false));
default: return void 0;
}
}
nextPointer(): T | undefined {
this.skipAnnotations();
switch (this.nextbyte()) {
case Tag.Pointer: return this.decodePointer(this);
switch (this.state.nextbyte()) {
case Tag.Pointer: return this.decodePointer(this.state);
default: return void 0;
}
}
nextSignedInteger(): number | undefined {
this.skipAnnotations();
const b = this.nextbyte();
const b = this.state.nextbyte();
switch (b) {
case Tag.SignedInteger: return this.nextint(this.varint());
default: return this.nextSmallOrMediumInteger(b);
case Tag.SignedInteger: return this.state.nextint(this.state.varint());
default: return this.state.nextSmallOrMediumInteger(b);
}
}
nextString(): string | undefined {
this.skipAnnotations();
switch (this.nextbyte()) {
case Tag.String: return Bytes.from(this.nextbytes(this.varint())).fromUtf8();
switch (this.state.nextbyte()) {
case Tag.String: return Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8();
default: return void 0;
}
}
nextByteString(): Bytes | undefined {
this.skipAnnotations();
switch (this.nextbyte()) {
case Tag.ByteString: return Bytes.from(this.nextbytes(this.varint()));
switch (this.state.nextbyte()) {
case Tag.ByteString: return Bytes.from(this.state.nextbytes(this.state.varint()));
default: return void 0;
}
}
nextSymbol(): symbol | undefined {
this.skipAnnotations();
switch (this.nextbyte()) {
switch (this.state.nextbyte()) {
case Tag.Symbol:
return Symbol.for(Bytes.from(this.nextbytes(this.varint())).fromUtf8());
return Symbol.for(Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8());
default:
return void 0;
}
@ -329,31 +348,31 @@ export class Decoder<T = never> implements TypedDecoder<T> {
openRecord(): boolean {
this.skipAnnotations();
return (this.nextbyte() === Tag.Record) || (this.index--, false);
return (this.state.nextbyte() === Tag.Record) || (this.state.index--, false);
}
openSequence(): boolean {
this.skipAnnotations();
return (this.nextbyte() === Tag.Sequence) || (this.index--, false);
return (this.state.nextbyte() === Tag.Sequence) || (this.state.index--, false);
}
openSet(): boolean {
this.skipAnnotations();
return (this.nextbyte() === Tag.Set) || (this.index--, false);
return (this.state.nextbyte() === Tag.Set) || (this.state.index--, false);
}
openDictionary(): boolean {
this.skipAnnotations();
return (this.nextbyte() === Tag.Dictionary) || (this.index--, false);
return (this.state.nextbyte() === Tag.Dictionary) || (this.state.index--, false);
}
closeCompound(): boolean {
return this.peekend();
return this.state.peekend();
}
}
export function decode<T>(bs: BytesLike, options: DecoderPointerOptions<T> = {}): Value<T> {
return new Decoder<T>(bs, options).withPointerDecoder<T, Value<T>>(
return new Decoder<never>(bs, options).withPointerDecoder<T, Value<T>>(
options.decodePointer ?? _defaultDecodePointer,
d => d.next());
}

8
implementations/javascript/packages/core/src/dictionary.ts
View File

@ -57,20 +57,20 @@ export class KeyedDictionary<K extends Value<T>, V, T = DefaultPointer> extends
const entries = Array.from(this);
const pieces = entries.map<[Bytes, number]>(([k, _v], i) => [canonicalEncode(k), i]);
pieces.sort((a, b) => Bytes.compare(a[0], b[0]));
encoder.emitbyte(Tag.Dictionary);
encoder.state.emitbyte(Tag.Dictionary);
pieces.forEach(([_encodedKey, i]) => {
const [k, v] = entries[i];
encoder.push(k);
encoder.push(v as unknown as Value<T>); // Suuuuuuuper unsound
});
encoder.emitbyte(Tag.End);
encoder.state.emitbyte(Tag.End);
} else {
encoder.emitbyte(Tag.Dictionary);
encoder.state.emitbyte(Tag.Dictionary);
this.forEach((v, k) => {
encoder.push(k);
encoder.push(v as unknown as Value<T>); // Suuuuuuuper unsound
});
encoder.emitbyte(Tag.End);
encoder.state.emitbyte(Tag.End);
}
}
}

96
implementations/javascript/packages/core/src/encoder.ts
View File

@ -22,7 +22,7 @@ export interface EncoderOptions {
}
export interface EncoderPointerOptions<T> extends EncoderOptions {
encodePointer?: (e: Encoder<T>, v: T) => void;
encodePointer?: EncodePointerFunction<T>;
}
export function asLatin1(bs: Uint8Array): string {
@ -33,38 +33,23 @@ function isIterable<T>(v: any): v is Iterable<T> {
return typeof v === 'object' && v !== null && typeof v[Symbol.iterator] === 'function';
}
function _defaultEncodePointer<T>(e: Encoder<T>, v: T): void {
e.push(pointerId(v));
function _defaultEncodePointer<T>(s: EncoderState, v: T): void {
new Encoder(s).push(pointerId(v));
}
export class Encoder<T> {
export class EncoderState {
chunks: Array<Uint8Array>;
view: DataView;
index: number;
options: EncoderOptions;
encodePointer: ((e: Encoder<T>, v: T) => void) = _defaultEncodePointer;
constructor(options: EncoderOptions = {}) {
constructor(options: EncoderOptions) {
this.chunks = [];
this.view = new DataView(new ArrayBuffer(256));
this.index = 0;
this.options = options;
}
withPointerEncoder<S>(encodePointer: (e: Encoder<S>, v: S) => void,
body: (e: Encoder<S>) => void): this
{
const oldEncodePointer = this.encodePointer;
const disguised = this as unknown as Encoder<S>;
disguised.encodePointer = encodePointer;
try {
body(disguised);
return this;
} finally {
this.encodePointer = oldEncodePointer;
}
}
get canonical(): boolean {
return this.options.canonical ?? true;
}
@ -152,11 +137,56 @@ export class Encoder<T> {
this.varint(bs.length);
this.emitbytes(bs);
}
}
export type EncodePointerFunction<T> = (s: EncoderState, v: T) => void;
export class Encoder<T> {
state: EncoderState;
encodePointer: EncodePointerFunction<T>;
constructor(options: EncoderOptions);
constructor(state: EncoderState, encodePointer?: EncodePointerFunction<T>);
constructor(
state_or_options: (EncoderState | EncoderOptions) = {},
encodePointer?: EncodePointerFunction<T>)
{
if (state_or_options instanceof EncoderState) {
this.state = state_or_options;
this.encodePointer = encodePointer ?? _defaultEncodePointer;
} else {
this.state = new EncoderState(state_or_options);
this.encodePointer = _defaultEncodePointer;
}
}
withPointerEncoder<S>(encodePointer: EncodePointerFunction<S>,
body: (e: Encoder<S>) => void): this
{
body(new Encoder(this.state, encodePointer));
return this;
}
get canonical(): boolean {
return this.state.canonical;
}
get includeAnnotations(): boolean {
return this.state.includeAnnotations;
}
contents(): Bytes {
return this.state.contents();
}
contentsString(): string {
return this.state.contentsString();
}
encodevalues(tag: Tag, items: Iterable<Value<T>>) {
this.emitbyte(tag);
this.state.emitbyte(tag);
for (let i of items) { this.push(i); }
this.emitbyte(Tag.End);
this.state.emitbyte(Tag.End);
}
push(v: Encodable<T>) {
@ -167,36 +197,36 @@ export class Encoder<T> {
v[PreserveOn](this);
}
else if (typeof v === 'boolean') {
this.emitbyte(v ? Tag.True : Tag.False);
this.state.emitbyte(v ? Tag.True : Tag.False);
}
else if (typeof v === 'number') {
if (v >= -3 && v <= 12) {
this.emitbyte(Tag.SmallInteger_lo + ((v + 16) & 0xf));
this.state.emitbyte(Tag.SmallInteger_lo + ((v + 16) & 0xf));
} else {
this.encodeint(v);
this.state.encodeint(v);
}
}
else if (typeof v === 'string') {
this.encodebytes(Tag.String, new Bytes(v)._view);
this.state.encodebytes(Tag.String, new Bytes(v)._view);
}
else if (typeof v === 'symbol') {
const key = Symbol.keyFor(v);
if (key === void 0) throw new EncodeError("Cannot preserve non-global Symbol", v);
this.encodebytes(Tag.Symbol, new Bytes(key)._view);
this.state.encodebytes(Tag.Symbol, new Bytes(key)._view);
}
else if (ArrayBuffer.isView(v)) {
if (v instanceof Uint8Array) {
this.encodebytes(Tag.ByteString, v);
this.state.encodebytes(Tag.ByteString, v);
} else {
const bs = new Uint8Array(v.buffer, v.byteOffset, v.byteLength);
this.encodebytes(Tag.ByteString, bs);
this.state.encodebytes(Tag.ByteString, bs);
}
}
else if (Record.isRecord<Value<T>, Tuple<Value<T>>, T>(v)) {
this.emitbyte(Tag.Record);
this.state.emitbyte(Tag.Record);
this.push(v.label);
for (let i of v) { this.push(i); }
this.emitbyte(Tag.End);
this.state.emitbyte(Tag.End);
}
else if (Array.isArray(v)) {
this.encodevalues(Tag.Sequence, v);
@ -205,8 +235,8 @@ export class Encoder<T> {
this.encodevalues(Tag.Sequence, v as Iterable<Value<T>>);
}
else {
this.emitbyte(Tag.Pointer);
this.encodePointer(this, v);
this.state.emitbyte(Tag.Pointer);
this.encodePointer(this.state, v);
}
return this; // for chaining
}

16
implementations/javascript/packages/core/src/float.ts
View File

@ -50,10 +50,10 @@ export class SingleFloat extends Float implements Preservable<never> {
}
[PreserveOn](encoder: Encoder<never>) {
encoder.emitbyte(Tag.Float);
encoder.makeroom(4);
encoder.view.setFloat32(encoder.index, this.value, false);
encoder.index += 4;
encoder.state.emitbyte(Tag.Float);
encoder.state.makeroom(4);
encoder.state.view.setFloat32(encoder.state.index, this.value, false);
encoder.state.index += 4;
}
get [FloatType](): 'Single' {
@ -75,10 +75,10 @@ export class DoubleFloat extends Float implements Preservable<never> {
}
[PreserveOn](encoder: Encoder<never>) {
encoder.emitbyte(Tag.Double);
encoder.makeroom(8);
encoder.view.setFloat64(encoder.index, this.value, false);
encoder.index += 8;
encoder.state.emitbyte(Tag.Double);
encoder.state.makeroom(8);
encoder.state.view.setFloat64(encoder.state.index, this.value, false);
encoder.state.index += 8;
}
get [FloatType](): 'Double' {

10
implementations/javascript/packages/core/src/pointer.ts
View File

@ -1,4 +1,4 @@
import type { Encoder } from "./encoder";
import { Encoder, EncoderState } from "./encoder";
import type { TypedDecoder } from "./decoder";
import type { Value } from "./values";
@ -24,10 +24,10 @@ export function readDefaultPointer(v: Value<DefaultPointer>): DefaultPointer {
return new DefaultPointer(strip(v));
}
export function decodeDefaultPointer(d: TypedDecoder<DefaultPointer>): DefaultPointer {
return readDefaultPointer(d.next());
export function decodeDefaultPointer<T>(d: TypedDecoder<T>): DefaultPointer {
return readDefaultPointer(d.withPointerDecoder(decodeDefaultPointer, d => d.next()));
}
export function encodeDefaultPointer(e: Encoder<DefaultPointer>, w: DefaultPointer): void {
e.push(w.v);
export function encodeDefaultPointer(e: EncoderState, w: DefaultPointer): void {
new Encoder(e, encodeDefaultPointer).push(w.v);
}

361
implementations/javascript/packages/core/src/reader.ts
View File

@ -10,25 +10,31 @@ import { Record } from './record';
import { Annotated, newPosition, Position, updatePosition } from './annotated';
import { Double, DoubleFloat, Single, SingleFloat } from './float';
import { stringify } from './text';
import { decodeDefaultPointer, DefaultPointer, readDefaultPointer } from './pointer';
export interface ReaderOptions<T> {
export interface ReaderStateOptions {
includeAnnotations?: boolean;
decodePointer?: (v: Value<T>) => T;
name?: string | Position;
}
export type DecodePointerFunction<T> = (v: Value<DefaultPointer>) => T;
export interface ReaderOptions<T> extends ReaderStateOptions {
decodePointer?: DecodePointerFunction<T>;
}
type IntOrFloat = 'int' | 'float';
type Numeric = number | SingleFloat | DoubleFloat;
type IntContinuation = (kind: IntOrFloat, acc: string) => Numeric;
export class Reader<T> {
export class ReaderState {
buffer: string;
pos: Position;
index: number;
discarded = 0;
options: ReaderOptions<T>;
options: ReaderStateOptions;
constructor(buffer: string = '', options: ReaderOptions<T> = {}) {
constructor(buffer: string, options: ReaderStateOptions) {
this.buffer = buffer;
switch (typeof options.name) {
case 'undefined': this.pos = newPosition(); break;
@ -39,10 +45,18 @@ export class Reader<T> {
this.options = options;
}
error(message: string, pos: Position): never {
throw new DecodeError(message, { ... pos });
}
get includeAnnotations(): boolean {
return this.options.includeAnnotations ?? false;
}
copyPos(): Position {
return { ... this.pos };
}
write(data: string) {
if (this.atEnd()) {
this.buffer = data;
@ -53,10 +67,6 @@ export class Reader<T> {
this.index = 0;
}
error(message: string, pos: Position): never {
throw new DecodeError(message, { ... pos });
}
atEnd(): boolean {
return (this.index >= this.buffer.length);
}
@ -90,129 +100,18 @@ export class Reader<T> {
}
}
readCommentLine(): Value<T> {
const startPos = { ... this.pos };
let acc = '';
while (true) {
const c = this.nextchar();
if (c === '\n' || c === '\r') {
return this.wrap(acc, startPos);
}
acc = acc + c;
}
readHex2(): number {
const x1 = unhexDigit(this.nextcharcode());
const x2 = unhexDigit(this.nextcharcode());
return (x1 << 4) | x2;
}
wrap(v: Value<T>, pos: Position): Value<T> {
if (this.includeAnnotations && !Annotated.isAnnotated(v)) {
v = new Annotated(v, pos);
}
return v;
}
annotateNextWith(v: Value<T>): Value<T> {
this.skipws();
if (this.atEnd()) {
throw new DecodeError("Trailing annotations and comments are not permitted", this.pos);
}
const u = this.next();
if (this.includeAnnotations) (u as Annotated<T>).annotations.unshift(v);
return u;
}
readToEnd(): Array<Value<T>> {
const acc = [];
while (true) {
this.skipws();
if (this.atEnd()) return acc;
acc.push(this.next());
}
}
next(): Value<T> {
this.skipws();
const startPos = { ... this.pos };
const unwrapped = (() => {
const c = this.nextchar();
switch (c) {
case '-':
return this.readIntpart('-', this.nextchar());
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
return this.readIntpart('', c);
case '"':
return this.readString('"');
case '|':
return Symbol.for(this.readString('|'));
case ';':
return this.annotateNextWith(this.readCommentLine());
case '@':
return this.annotateNextWith(this.next());
case ':':
this.error('Unexpected key/value separator between items', startPos);
case '#': {
const c = this.nextchar();
switch (c) {
case 'f': return false;
case 't': return true;
case '{': return this.seq(new Set<T>(), (v, s) => s.add(v), '}');
case '"': return this.readLiteralBinary();
case 'x':
if (this.nextchar() !== '"') {
this.error('Expected open-quote at start of hex ByteString',
startPos);
}
return this.readHexBinary();
case '[': return this.readBase64Binary();
case '=': {
const bs = unannotate(this.next());
if (!Bytes.isBytes(bs)) this.error('ByteString must follow #=',
startPos);
return decode<T>(bs, {
decodePointer: d => this.options.decodePointer?.(d.next()),
includeAnnotations: this.options.includeAnnotations,
});
}
case '!': {
const d = this.options.decodePointer;
if (d === void 0) {
this.error("No decodePointer function supplied", startPos);
}
return d(this.next());
}
default:
this.error(`Invalid # syntax: ${c}`, startPos);
}
}
case '<': {
const label = this.next();
const fields = this.readSequence('>');
return Record(label, fields);
}
case '[': return this.readSequence(']');
case '{': return this.readDictionary();
case '>': this.error('Unexpected >', startPos);
case ']': this.error('Unexpected ]', startPos);
case '}': this.error('Unexpected }', startPos);
default:
return this.readRawSymbol(c);
}
})();
return this.wrap(unwrapped, startPos);
}
seq<S>(acc: S, update: (v: Value<T>, acc: S) => void, ch: string): S {
while (true) {
this.skipws();
if (this.peek() === ch) {
this.advance();
return acc;
}
update(this.next(), acc);
}
}
readSequence(ch: string): Array<Value<T>> {
return this.seq([] as Array<Value<T>>, (v, acc) => acc.push(v), ch);
readHex4(): number {
const x1 = unhexDigit(this.nextcharcode());
const x2 = unhexDigit(this.nextcharcode());
const x3 = unhexDigit(this.nextcharcode());
const x4 = unhexDigit(this.nextcharcode());
return (x1 << 12) | (x2 << 8) | (x3 << 4) | x4;
}
readHexBinary(): Bytes {
@ -227,24 +126,6 @@ export class Reader<T> {
}
}
readDictionary(): Dictionary<T> {
return this.seq(new Dictionary<T>(),
(k, acc) => {
this.skipws();
switch (this.peek()) {
case ':':
if (acc.has(k)) this.error(
`Duplicate key: ${stringify(k)}`, this.pos);
this.advance();
acc.set(k, this.next());
break;
default:
this.error('Missing key/value separator', this.pos);
}
},
'}');
}
readBase64Binary(): Bytes {
let acc = '';
while (true) {
@ -315,7 +196,7 @@ export class Reader<T> {
}
}
readRawSymbol(acc: string): Value<T> {
readRawSymbol<T>(acc: string): Value<T> {
while (true) {
if (this.atEnd()) break;
const ch = this.peek();
@ -366,20 +247,6 @@ export class Reader<T> {
}
}
readHex2(): number {
const x1 = unhexDigit(this.nextcharcode());
const x2 = unhexDigit(this.nextcharcode());
return (x1 << 4) | x2;
}
readHex4(): number {
const x1 = unhexDigit(this.nextcharcode());
const x2 = unhexDigit(this.nextcharcode());
const x3 = unhexDigit(this.nextcharcode());
const x4 = unhexDigit(this.nextcharcode());
return (x1 << 12) | (x2 << 8) | (x3 << 4) | x4;
}
readString(terminator: string): string {
return this.readStringlike(x => x, xs => xs.join(''), terminator, 'u', () => {
const n1 = this.readHex4();
@ -410,6 +277,174 @@ export class Reader<T> {
}
}
export class Reader<T> {
state: ReaderState;
decodePointer?: DecodePointerFunction<T>;
constructor(state: ReaderState, decodePointer?: DecodePointerFunction<T>);
constructor(buffer: string, options?: ReaderOptions<T>);
constructor(
state_or_buffer: (ReaderState | string) = '',
decodePointer_or_options?: (DecodePointerFunction<T> | ReaderOptions<T>))
{
if (state_or_buffer instanceof ReaderState) {
this.state = state_or_buffer;
this.decodePointer = decodePointer_or_options as DecodePointerFunction<T>;
} else {
const options = (decodePointer_or_options as ReaderOptions<T>) ?? {};
this.state = new ReaderState(state_or_buffer, options);
this.decodePointer = options.decodePointer;
}
}
write(data: string) {
this.state.write(data);
}
readCommentLine(): Value<T> {
const startPos = this.state.copyPos();
let acc = '';
while (true) {
const c = this.state.nextchar();
if (c === '\n' || c === '\r') {
return this.wrap(acc, startPos);
}
acc = acc + c;
}
}
wrap(v: Value<T>, pos: Position): Value<T> {
if (this.state.includeAnnotations && !Annotated.isAnnotated(v)) {
v = new Annotated(v, pos);
}
return v;
}
annotateNextWith(v: Value<T>): Value<T> {
this.state.skipws();
if (this.state.atEnd()) {
throw new DecodeError("Trailing annotations and comments are not permitted",
this.state.pos);
}
const u = this.next();
if (this.state.includeAnnotations) (u as Annotated<T>).annotations.unshift(v);
return u;
}
readToEnd(): Array<Value<T>> {
const acc = [];
while (true) {
this.state.skipws();
if (this.state.atEnd()) return acc;
acc.push(this.next());
}
}
next(): Value<T> {
this.state.skipws();
const startPos = this.state.copyPos();
const unwrapped = ((): Value<T> => {
const c = this.state.nextchar();
switch (c) {
case '-':
return this.state.readIntpart('-', this.state.nextchar());
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
return this.state.readIntpart('', c);
case '"':
return this.state.readString('"');
case '|':
return Symbol.for(this.state.readString('|'));
case ';':
return this.annotateNextWith(this.readCommentLine());
case '@':
return this.annotateNextWith(this.next());
case ':':
this.state.error('Unexpected key/value separator between items', startPos);
case '#': {
const c = this.state.nextchar();
switch (c) {
case 'f': return false;
case 't': return true;
case '{': return this.seq(new Set<T>(), (v, s) => s.add(v), '}');
case '"': return this.state.readLiteralBinary();
case 'x':
if (this.state.nextchar() !== '"') {
this.state.error('Expected open-quote at start of hex ByteString',
startPos);
}
return this.state.readHexBinary();
case '[': return this.state.readBase64Binary();
case '=': {
const bs = unannotate(this.next());
if (!Bytes.isBytes(bs)) this.state.error('ByteString must follow #=',
startPos);
return decode<T>(bs, {
decodePointer: <never>decodeDefaultPointer,
includeAnnotations: this.state.options.includeAnnotations,
});
}
case '!': {
if (this.decodePointer === void 0) {
this.state.error("No decodePointer function supplied", startPos);
}
return this.decodePointer(new Reader(this.state, readDefaultPointer).next());
}
default:
this.state.error(`Invalid # syntax: ${c}`, startPos);
}
}
case '<': {
const label = this.next();
const fields = this.readSequence('>');
return Record(label, fields);
}
case '[': return this.readSequence(']');
case '{': return this.readDictionary();
case '>': this.state.error('Unexpected >', startPos);
case ']': this.state.error('Unexpected ]', startPos);
case '}': this.state.error('Unexpected }', startPos);
default:
return this.state.readRawSymbol(c);
}
})();
return this.wrap(unwrapped, startPos);
}
seq<S>(acc: S, update: (v: Value<T>, acc: S) => void, ch: string): S {
while (true) {
this.state.skipws();
if (this.state.peek() === ch) {
this.state.advance();
return acc;
}
update(this.next(), acc);
}
}
readSequence(ch: string): Array<Value<T>> {
return this.seq([] as Array<Value<T>>, (v, acc) => acc.push(v), ch);
}
readDictionary(): Dictionary<T> {
return this.seq(new Dictionary<T>(),
(k, acc) => {
this.state.skipws();
switch (this.state.peek()) {
case ':':
if (acc.has(k)) this.state.error(
`Duplicate key: ${stringify(k)}`, this.state.pos);
this.state.advance();
acc.set(k, this.next());
break;
default:
this.state.error('Missing key/value separator', this.state.pos);
}
},
'}');
}
}
const BASE64: {[key: string]: number} = {};
[... 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'].forEach(
(c, i) => BASE64[c] = i);

7
implementations/javascript/packages/core/test/codec.test.ts
View File

@ -14,6 +14,7 @@ import {
DefaultPointer,
decodeDefaultPointer,
encodeDefaultPointer,
EncoderState,
} from '../src/index';
const { Tag } = Constants;
import './test-utils';
@ -108,9 +109,9 @@ describe('encoding and decoding pointers', () => {
expect(encode(
[A, B],
{
encodePointer(e: Encoder<object>, v: object): void {
encodePointer(e: EncoderState, v: object): void {
objects.push(v);
e.push(objects.length - 1);
new Encoder(e, encodeDefaultPointer).push(objects.length - 1);
}
})).is(Bytes.from([Tag.Sequence,
Tag.Pointer, Tag.SmallInteger_lo,
@ -128,7 +129,7 @@ describe('encoding and decoding pointers', () => {
Tag.Pointer, Tag.SmallInteger_lo + 1,
Tag.End
]), {
decodePointer(d: TypedDecoder<object>): object {
decodePointer(d: TypedDecoder<never>): object {
const v = d.next();
if (typeof v !== 'number' || v < 0 || v >= objects.length) {
throw new Error("Unknown pointer target");