syndicate-lang
/
preserves
mirror of https://gitlab.com/preserves/preserves.git
1
0
Fork 1
Code Releases Activity

Merge branch 'main' into comment-syntax-hash-space

This commit is contained in:
Tony Garnock-Jones 2023-10-31 10:51:36 +01:00
parent 7c0fc8f358 1a2657fe33
commit c053102d07
13 changed files with 270 additions and 476 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

136
implementations/javascript/packages/core/test/codec.test.ts
View File

@ -1,12 +1,13 @@
import {
Value,
Dictionary,
decode, decodeWithAnnotations, encode, encodeWithAnnotations, canonicalEncode,
decode, decodeWithAnnotations, encode, canonicalEncode,
DecodeError, ShortPacket,
Bytes, Record,
annotate,
strip, peel,
preserves,
stringify,
fromJS,
Constants,
Encoder,
@ -192,105 +193,51 @@ describe('common test suite', () => {
}>()(Symbol.for('TestCases'), ['cases']);
type TestCases = ReturnType<typeof TestCases>;
function DS(bs: Bytes) {
return decode(bs, { embeddedDecode: genericEmbeddedTypeDecode });
function encodeBinary(v: Value<GenericEmbedded>): Bytes {
return encode(v, { canonical: true, embeddedEncode: genericEmbeddedTypeEncode });
}
function D(bs: Bytes) {
return decodeWithAnnotations(bs, { embeddedDecode: genericEmbeddedTypeDecode });
function looseEncodeBinary(v: Value<GenericEmbedded>): Bytes {
return encode(v, { canonical: false, includeAnnotations: true, embeddedEncode: genericEmbeddedTypeEncode });
}
function E(v: Value<GenericEmbedded>) {
return encodeWithAnnotations(v, { embeddedEncode: genericEmbeddedTypeEncode });
function annotatedBinary(v: Value<GenericEmbedded>): Bytes {
return encode(v, { canonical: true, includeAnnotations: true, embeddedEncode: genericEmbeddedTypeEncode });
}
function decodeBinary(bs: Bytes): Value<GenericEmbedded> {
return decode(bs, { includeAnnotations: true, embeddedDecode: genericEmbeddedTypeDecode });
}
function encodeText(v: Value<GenericEmbedded>): string {
return stringify(v, { includeAnnotations: true, embeddedWrite: genericEmbeddedTypeEncode });
}
function decodeText(s: string): Value<GenericEmbedded> {
return parse(s, { includeAnnotations: true, embeddedDecode: genericEmbeddedTypeDecode });
}
interface ExpectedValues {
[testName: string]: ({
value: Value<GenericEmbedded>;
} | {
forward: Value<GenericEmbedded>;
back: Value<GenericEmbedded>;
});
}
const expectedValues: ExpectedValues = {
annotation1: { forward: annotate<GenericEmbedded>(9, "abc"),
back: 9 },
annotation2: { forward: annotate<GenericEmbedded>([[], annotate<GenericEmbedded>([], "x")],
"abc",
"def"),
back: [[], []] },
annotation3: { forward: annotate<GenericEmbedded>(5,
annotate<GenericEmbedded>(2, 1),
annotate<GenericEmbedded>(4, 3)),
back: 5 },
annotation5: {
forward: annotate<GenericEmbedded>(
Record<symbol, any>(Symbol.for('R'),
[annotate<GenericEmbedded>(Symbol.for('f'),
Symbol.for('af'))]),
Symbol.for('ar')),
back: Record<Value<GenericEmbedded>, any>(Symbol.for('R'), [Symbol.for('f')])
},
annotation6: {
forward: Record<Value<GenericEmbedded>, any>(
annotate<GenericEmbedded>(Symbol.for('R'),
Symbol.for('ar')),
[annotate<GenericEmbedded>(Symbol.for('f'),
Symbol.for('af'))]),
back: Record<symbol, any>(Symbol.for('R'), [Symbol.for('f')])
},
annotation7: {
forward: annotate<GenericEmbedded>([], Symbol.for('a'), Symbol.for('b'), Symbol.for('c')),
back: []
},
delimiters4: {
forward: [false, annotate<GenericEmbedded>(true, "a line comment")],
back: [false, true],
},
delimiters5: {
forward: [false, annotate<GenericEmbedded>(true, Symbol.for('ann'))],
back: [false, true],
},
list1: {
forward: [1, 2, 3, 4],
back: [1, 2, 3, 4]
},
record2: {
value: Observe(Record(Symbol.for("speak"), [
Discard(),
Capture(Discard())
]))
},
};
type Variety = 'normal' | 'nondeterministic' | 'decode';
type Variety = 'normal' | 'nondeterministic';
function runTestCase(variety: Variety,
tName: string,
binaryForm: Bytes,
annotatedTextForm: Value<GenericEmbedded>)
binary: Bytes,
annotatedValue: Value<GenericEmbedded>)
{
describe(tName, () => {
const textForm = strip(annotatedTextForm);
const {forward, back} = (function () {
const entry = expectedValues[tName] ?? {value: textForm};
if ('value' in entry) {
return {forward: entry.value, back: entry.value};
} else if ('forward' in entry && 'back' in entry) {
return entry;
} else {
throw new Error('Invalid expectedValues entry for ' + tName);
}
})();
it('should match the expected value', () => expect(textForm).is(back));
it('should round-trip', () => expect(DS(E(textForm))).is(back));
it('should go forward', () => expect(DS(E(forward))).is(back));
it('should go back', () => expect(DS(binaryForm)).is(back));
it('should go back with annotations',
() => expect(D(E(annotatedTextForm))).is(annotatedTextForm));
if (variety !== 'decode' && variety !== 'nondeterministic') {
it('should encode correctly', () => expect(E(forward)).is(binaryForm));
it('should encode correctly with annotations',
() => expect(E(annotatedTextForm)).is(binaryForm));
const stripped = strip(annotatedValue);
it('should round-trip, canonically', () =>
expect(decodeBinary(encodeBinary(annotatedValue))).is(stripped));
it('should go back, stripped', () =>
expect(strip(decodeBinary(binary))).is(stripped));
it('should go back', () =>
expect(decodeBinary(binary)).is(annotatedValue));
it('should round-trip, with annotations', () =>
expect(decodeBinary(annotatedBinary(annotatedValue))).is(annotatedValue));
it('should round-trip as text, stripped', () =>
expect(decodeText(encodeText(stripped))).is(stripped));
it('should round-trip as text, with annotations', () =>
expect(decodeText(encodeText(annotatedValue))).is(annotatedValue));
it('should go forward', () =>
expect(annotatedBinary(annotatedValue)).is(binary));
if (variety === 'normal') {
it('should go forward, loosely', () =>
expect(looseEncodeBinary(annotatedValue)).is(binary));
}
});
}
@ -307,13 +254,10 @@ describe('common test suite', () => {
case Symbol.for('NondeterministicTest'):
runTestCase('nondeterministic', tName, strip(t[0]) as Bytes, t[1]);
break;
case Symbol.for('DecodeTest'):
runTestCase('decode', tName, strip(t[0]) as Bytes, t[1]);
break;
case Symbol.for('DecodeError'):
describe(tName, () => {
it('should fail with DecodeError', () => {
expect(() => D(strip(t[0]) as Bytes))
expect(() => decodeBinary(strip(t[0]) as Bytes))
.toThrowFilter(e =>
DecodeError.isDecodeError(e) &&
!ShortPacket.isShortPacket(e));
@ -324,7 +268,7 @@ describe('common test suite', () => {
case Symbol.for('DecodeShort'):
describe(tName, () => {
it('should fail with ShortPacket', () => {
expect(() => D(strip(t[0]) as Bytes))
expect(() => decodeBinary(strip(t[0]) as Bytes))
.toThrowFilter(e => ShortPacket.isShortPacket(e));
});
});

16
implementations/python/preserves/path.py
View File

@ -56,14 +56,16 @@ The result of evaluating it on `testdata` is as follows:
>>> for result in selector.exec(testdata):
... print(stringify(result))
"Individual test cases may be any of the following record types:"
"In each test, let value = strip(annotatedValue),"
" forward = value,"
" back = value,"
"except where test-case-specific values of `forward` and/or `back`"
"are provided by the executing harness, and check the following"
"numbered expectations according to the table above:"
"In each test, let stripped = strip(annotatedValue),"
" encodeBinary(·) produce canonical ordering and no annotations,"
" looseEncodeBinary(·) produce any ordering, but with annotations,"
" annotatedBinary(·) produce canonical ordering, but with annotations,"
" decodeBinary(·) include annotations,"
" encodeText(·) include annotations,"
" decodeText(·) include annotations,"
"and check the following numbered expectations according to the table above:"
"Implementations may vary in their treatment of the difference between expectations"
"13/14 and 16/17, depending on how they wish to treat end-of-stream conditions."
"21/22 and 31/32, depending on how they wish to treat end-of-stream conditions."
```

BIN
implementations/python/tests/samples.bin
View File

Binary file not shown.

64
implementations/python/tests/samples.pr
View File

@ -2,45 +2,43 @@
@<Documentation [
"Individual test cases may be any of the following record types:"
<TestCaseTypes {
Test: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8 9 11}}
NondeterministicTest: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8 10 11}}
DecodeTest: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8}}
ParseError: {fields: [text] expectations: #{12}}
ParseShort: {fields: [text] expectations: #{13}}
ParseEOF: {fields: [text] expectations: #{14}}
DecodeError: {fields: [bytes] expectations: #{15}}
DecodeShort: {fields: [bytes] expectations: #{16}}
DecodeEOF: {fields: [bytes] expectations: #{17}}
Test: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8}}
NondeterministicTest: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7}}
ParseError: {fields: [text] expectations: #{20}}
ParseShort: {fields: [text] expectations: #{21}}
ParseEOF: {fields: [text] expectations: #{22}}
DecodeError: {fields: [bytes] expectations: #{30}}
DecodeShort: {fields: [bytes] expectations: #{31}}
DecodeEOF: {fields: [bytes] expectations: #{32}}
}>
"In each test, let value = strip(annotatedValue),",
" forward = value,",
" back = value,"
"except where test-case-specific values of `forward` and/or `back`",
"are provided by the executing harness, and check the following"
"numbered expectations according to the table above:"
"In each test, let stripped = strip(annotatedValue),"
" encodeBinary(·) produce canonical ordering and no annotations,"
" looseEncodeBinary(·) produce any ordering, but with annotations,"
" annotatedBinary(·) produce canonical ordering, but with annotations,"
" decodeBinary(·) include annotations,"
" encodeText(·) include annotations,"
" decodeText(·) include annotations,"
"and check the following numbered expectations according to the table above:"
<TestCaseExpectations {
1: "value = back"
2: "strip(decodeBinary(encodeBinary(value))) = back"
3: "strip(decodeBinary(encodeBinary(forward))) = back"
4: "strip(decodeBinary(binary)) = back"
5: "decodeBinary(binary) = annotatedValue"
6: "decodeBinary(encodeBinary(annotatedValue)) = annotatedValue"
7: "decodeText(encodeText(value)) = back"
8: "decodeText(encodeText(forward)) = back"
9: "encodeBinary(forward) = binary"
10: "canonicallyOrderedEncodedBinaryWithAnnotations(forward) = binary"
11: "encodeBinary(annotatedValue) = binary"
1: "decodeBinary(encodeBinary(annotatedValue))) = stripped"
2: "strip(decodeBinary(binary)) = stripped"
3: "decodeBinary(binary) = annotatedValue"
4: "decodeBinary(annotatedBinary(annotatedValue)) = annotatedValue"
5: "decodeText(encodeText(stripped)) = stripped"
6: "decodeText(encodeText(annotatedValue)) = annotatedValue"
7: "annotatedBinary(annotatedValue) = binary"
8: "looseEncodeBinary(annotatedValue) = binary"
12: "decodeText(text) fails with a syntax error (NB. never with EOF)"
13: "decodeText(text) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
14: "decodeText(text) fails signalling immediate EOF (NB. never with a syntax error)"
20: "decodeText(text) fails with a syntax error (NB. never with EOF)"
21: "decodeText(text) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
22: "decodeText(text) fails signalling immediate EOF (NB. never with a syntax error)"
15: "decodeBinary(bytes) fails with a syntax error (NB. never with EOF)"
16: "decodeBinary(bytes) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
17: "decodeBinary(bytes) fails signalling immediate EOF (NB. never with a syntax error)"
30: "decodeBinary(bytes) fails with a syntax error (NB. never with EOF)"
31: "decodeBinary(bytes) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
32: "decodeBinary(bytes) fails signalling immediate EOF (NB. never with a syntax error)"
}>
"Implementations may vary in their treatment of the difference between expectations"
"13/14 and 16/17, depending on how they wish to treat end-of-stream conditions."
"21/22 and 31/32, depending on how they wish to treat end-of-stream conditions."
]>
<TestCases {
annotation1: <Test #x"85 B103616263 B00109" @"abc" 9>

136
implementations/python/tests/test_preserves.py
View File

@ -41,11 +41,18 @@ def _varint(v):
e.varint(v)
return e.contents()
def _d(bs):
return decode(bs)
def _e(v):
return encode(v)
def encodeBinary(v):
return encode(v, canonicalize=True)
def looseEncodeBinary(v):
return encode(v, canonicalize=False, include_annotations=True)
def annotatedBinary(v):
return encode(v, canonicalize=True, include_annotations=True)
def decodeBinary(bs):
return decode(bs, include_annotations=True)
def encodeText(v):
return stringify(v)
def decodeText(s):
return parse(s, include_annotations=True)
def _R(k, *args):
return Record(Symbol(k), args)
@ -53,11 +60,11 @@ def _R(k, *args):
class BinaryCodecTests(PreservesTestCase):
def _roundtrip(self, forward, expected, back=None, nondeterministic=False):
if back is None: back = forward
self.assertPreservesEqual(_d(_e(forward)), back)
self.assertPreservesEqual(_d(_e(back)), back)
self.assertPreservesEqual(_d(expected), back)
self.assertPreservesEqual(decode(encode(forward)), back)
self.assertPreservesEqual(decode(encode(back)), back)
self.assertPreservesEqual(decode(expected), back)
if not nondeterministic:
actual = _e(forward)
actual = encode(forward)
self.assertPreservesEqual(actual, expected, '%s != %s' % (_hex(actual), _hex(expected)))
def test_decode_varint(self):
@ -152,13 +159,13 @@ class BinaryCodecTests(PreservesTestCase):
r = r'b5(b5b1016.b0010.84){3}84'
if hasattr(d, 'iteritems'):
# python 2
bs = _e(d.iteritems())
bs = encode(d.iteritems())
self.assertRegexpMatches(_hex(bs), r)
else:
# python 3
bs = _e(d.items())
bs = encode(d.items())
self.assertRegex(_hex(bs), r)
self.assertPreservesEqual(sorted(_d(bs)), [(u'a', 1), (u'b', 2), (u'c', 3)])
self.assertPreservesEqual(sorted(decode(bs)), [(u'a', 1), (u'b', 2), (u'c', 3)])
def test_long_sequence(self):
self._roundtrip((False,) * 14, _buf(0xb5, b'\x80' * 14, 0x84))
@ -226,61 +233,39 @@ def add_method(d, tName, fn):
fn.__name__ = fname
d[fname] = fn
expected_values = {
"annotation1": { "forward": annotate(9, u"abc"), "back": 9 },
"annotation2": { "forward": annotate([[], annotate([], u"x")], u"abc", u"def"), "back": ((), ()) },
"annotation3": { "forward": annotate(5, annotate(2, 1), annotate(4, 3)), "back": 5 },
"annotation5": { "forward": annotate(_R('R', annotate(Symbol('f'), Symbol('af'))),
Symbol('ar')),
"back": _R('R', Symbol('f')) },
"annotation6": { "forward": Record(annotate(Symbol('R'), Symbol('ar')),
[annotate(Symbol('f'), Symbol('af'))]),
"back": _R('R', Symbol('f')) },
"annotation7": { "forward": annotate([], Symbol('a'), Symbol('b'), Symbol('c')),
"back": () },
"delimiters4": { "forward": [False, annotate(True, 'a line comment')], "back": [False, True] },
"delimiters5": { "forward": [False, annotate(True, Symbol('ann'))], "back": [False, True] },
"record2": { "value": _R('observe', _R('speak', _R('discard'), _R('capture', _R('discard')))) },
}
def install_test(d, is_nondet, tName, binary, annotatedValue):
stripped = annotatedValue.strip()
def test_canonical_roundtrip(self):
self.assertPreservesEqual(decodeBinary(encodeBinary(annotatedValue)), stripped)
def test_back_stripped(self):
self.assertPreservesEqual(decodeBinary(binary).strip(), stripped)
def test_back(self):
self.assertPreservesEqual(decodeBinary(binary), annotatedValue)
def test_annotated_roundtrip(self):
self.assertPreservesEqual(decodeBinary(annotatedBinary(annotatedValue)), annotatedValue)
def test_text_roundtrip_stripped(self):
self.assertPreservesEqual(decodeText(encodeText(stripped)), stripped)
def test_text_roundtrip(self):
self.assertPreservesEqual(decodeText(encodeText(annotatedValue)), annotatedValue)
def test_forward(self):
self.assertPreservesEqual(annotatedBinary(annotatedValue), binary)
def test_forward_loose(self):
self.assertPreservesEqual(looseEncodeBinary(annotatedValue), binary)
def get_expected_values(tName, textForm):
entry = expected_values.get(tName, {"value": textForm})
if 'value' in entry:
return { "forward": entry['value'], "back": entry['value'] }
elif 'forward' in entry and 'back' in entry:
return entry
else:
raise Exception('Invalid expected_values entry for ' + tName)
def install_test(d, variant, tName, binaryForm, annotatedTextForm):
textForm = annotatedTextForm.strip()
entry = get_expected_values(tName, textForm)
forward = entry['forward']
back = entry['back']
def test_match_expected(self): self.assertPreservesEqual(textForm, back)
def test_roundtrip(self): self.assertPreservesEqual(self.DS(self.E(textForm)), back)
def test_forward(self): self.assertPreservesEqual(self.DS(self.E(forward)), back)
def test_back(self): self.assertPreservesEqual(self.DS(binaryForm), back)
def test_back_ann(self): self.assertPreservesEqual(self.D(self.E(annotatedTextForm)), annotatedTextForm)
def test_encode(self): self.assertPreservesEqual(self.E(forward), binaryForm)
def test_encode_nondet(self): self.assertPreservesEqual(self.ENONDET(annotatedTextForm), binaryForm)
def test_encode_ann(self): self.assertPreservesEqual(self.E(annotatedTextForm), binaryForm)
add_method(d, tName, test_match_expected)
add_method(d, tName, test_roundtrip)
add_method(d, tName, test_forward)
add_method(d, tName, test_canonical_roundtrip)
add_method(d, tName, test_back_stripped)
add_method(d, tName, test_back)
add_method(d, tName, test_back_ann)
if variant in ['normal']:
add_method(d, tName, test_encode)
if variant in ['nondeterministic']:
add_method(d, tName, test_encode_nondet)
if variant in ['normal', 'nondeterministic']:
add_method(d, tName, test_encode_ann)
add_method(d, tName, test_annotated_roundtrip)
add_method(d, tName, test_text_roundtrip_stripped)
add_method(d, tName, test_text_roundtrip)
add_method(d, tName, test_forward)
if not is_nondet:
add_method(d, tName, test_forward_loose)
def install_exn_test(d, tName, testLambda, check_proc):
def test_exn(self):
try:
testLambda(self)
testLambda()
except:
check_proc(self, sys.exc_info()[1])
return
@ -303,38 +288,21 @@ class CommonTestSuite(PreservesTestCase):
tName = tName0.strip().name
t = t0.peel()
if t.key == Symbol('Test'):
install_test(locals(), 'normal', tName, t[0].strip(), t[1])
install_test(locals(), False, tName, t[0].strip(), t[1])
elif t.key == Symbol('NondeterministicTest'):
install_test(locals(), 'nondeterministic', tName, t[0].strip(), t[1])
elif t.key == Symbol('DecodeTest'):
install_test(locals(), 'decode', tName, t[0].strip(), t[1])
install_test(locals(), True, tName, t[0].strip(), t[1])
elif t.key == Symbol('DecodeError'):
install_exn_test(locals(), tName, lambda self, t=t: self.D(t[0].strip()), expected_err)
install_exn_test(locals(), tName, lambda t=t: decodeBinary(t[0].strip()), expected_err)
elif t.key in [Symbol('DecodeShort'), Symbol('DecodeEOF')]:
install_exn_test(locals(), tName, lambda self, t=t: self.D(t[0].strip()), expected_short)
install_exn_test(locals(), tName, lambda t=t: decodeBinary(t[0].strip()), expected_short)
elif t.key == Symbol('ParseError'):
install_exn_test(locals(), tName, lambda self, t=t: self.R(t[0].strip()), expected_err)
install_exn_test(locals(), tName, lambda t=t: decodeText(t[0].strip()), expected_err)
elif t.key in [Symbol('ParseShort'), Symbol('ParseEOF')]:
install_exn_test(locals(), tName, lambda self, t=t: self.R(t[0].strip()), expected_short)
install_exn_test(locals(), tName, lambda t=t: decodeText(t[0].strip()), expected_short)
pass
else:
raise Exception('Unsupported test kind', t.key)
def R(self, text):
return parse(text)
def DS(self, bs):
return decode(bs, decode_embedded=lambda x: x)
def D(self, bs):
return decode_with_annotations(bs, decode_embedded=lambda x: x)
def E(self, v):
return encode(v, encode_embedded=lambda x: x)
def ENONDET(self, v):
return encode(v, encode_embedded=lambda x: x, canonicalize=True, include_annotations=True)
class RecordTests(PreservesTestCase):
def test_getters(self):
T = Record.makeConstructor('t', 'x y z')

16
implementations/racket/preserves/preserves/main.rkt
View File

@ -13,12 +13,16 @@
(all-from-out "write-binary.rkt")
(all-from-out "write-text.rkt")
has-any-annotations?
detect-preserve-syntax
read-preserve
port->preserves
file->preserves)
(require racket/dict)
(require racket/match)
(require racket/set)
(require (only-in racket/file file->list))
(require (only-in racket/port port->list))
@ -34,6 +38,18 @@
(require "write-binary.rkt")
(require "write-text.rkt")
(define (has-any-annotations? v #:check-embedded? [check-embedded? #t])
(let walk ((v v))
(match v
[(annotated '() _ item) (walk item)]
[(annotated _ _ _) #t]
[(record label fields) (or (walk label) (ormap walk fields))]
[(? list?) (ormap walk v)]
[(? set?) (for/or [(i (in-set v))] (walk i))]
[(? dict?) (for/or [((k v) (in-dict v))] (or (walk k) (walk v)))]
[(embedded v) (and check-embedded? (walk v))]
[_ #f])))
(define (detect-preserve-syntax [in-port (current-input-port)])
(define b (peek-byte in-port))
(cond [(eof-object? b) b]

64
implementations/racket/preserves/preserves/tests/samples.pr
View File

@ -2,45 +2,43 @@
@<Documentation [
"Individual test cases may be any of the following record types:"
<TestCaseTypes {
Test: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8 9 11}}
NondeterministicTest: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8 10 11}}
DecodeTest: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8}}
ParseError: {fields: [text] expectations: #{12}}
ParseShort: {fields: [text] expectations: #{13}}
ParseEOF: {fields: [text] expectations: #{14}}
DecodeError: {fields: [bytes] expectations: #{15}}
DecodeShort: {fields: [bytes] expectations: #{16}}
DecodeEOF: {fields: [bytes] expectations: #{17}}
Test: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8}}
NondeterministicTest: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7}}
ParseError: {fields: [text] expectations: #{20}}
ParseShort: {fields: [text] expectations: #{21}}
ParseEOF: {fields: [text] expectations: #{22}}
DecodeError: {fields: [bytes] expectations: #{30}}
DecodeShort: {fields: [bytes] expectations: #{31}}
DecodeEOF: {fields: [bytes] expectations: #{32}}
}>
"In each test, let value = strip(annotatedValue),",
" forward = value,",
" back = value,"
"except where test-case-specific values of `forward` and/or `back`",
"are provided by the executing harness, and check the following"
"numbered expectations according to the table above:"
"In each test, let stripped = strip(annotatedValue),"
" encodeBinary(·) produce canonical ordering and no annotations,"
" looseEncodeBinary(·) produce any ordering, but with annotations,"
" annotatedBinary(·) produce canonical ordering, but with annotations,"
" decodeBinary(·) include annotations,"
" encodeText(·) include annotations,"
" decodeText(·) include annotations,"
"and check the following numbered expectations according to the table above:"
<TestCaseExpectations {
1: "value = back"
2: "strip(decodeBinary(encodeBinary(value))) = back"
3: "strip(decodeBinary(encodeBinary(forward))) = back"
4: "strip(decodeBinary(binary)) = back"
5: "decodeBinary(binary) = annotatedValue"
6: "decodeBinary(encodeBinary(annotatedValue)) = annotatedValue"
7: "decodeText(encodeText(value)) = back"
8: "decodeText(encodeText(forward)) = back"
9: "encodeBinary(forward) = binary"
10: "canonicallyOrderedEncodedBinaryWithAnnotations(forward) = binary"
11: "encodeBinary(annotatedValue) = binary"
1: "decodeBinary(encodeBinary(annotatedValue))) = stripped"
2: "strip(decodeBinary(binary)) = stripped"
3: "decodeBinary(binary) = annotatedValue"
4: "decodeBinary(annotatedBinary(annotatedValue)) = annotatedValue"
5: "decodeText(encodeText(stripped)) = stripped"
6: "decodeText(encodeText(annotatedValue)) = annotatedValue"
7: "annotatedBinary(annotatedValue) = binary"
8: "looseEncodeBinary(annotatedValue) = binary"
12: "decodeText(text) fails with a syntax error (NB. never with EOF)"
13: "decodeText(text) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
14: "decodeText(text) fails signalling immediate EOF (NB. never with a syntax error)"
20: "decodeText(text) fails with a syntax error (NB. never with EOF)"
21: "decodeText(text) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
22: "decodeText(text) fails signalling immediate EOF (NB. never with a syntax error)"
15: "decodeBinary(bytes) fails with a syntax error (NB. never with EOF)"
16: "decodeBinary(bytes) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
17: "decodeBinary(bytes) fails signalling immediate EOF (NB. never with a syntax error)"
30: "decodeBinary(bytes) fails with a syntax error (NB. never with EOF)"
31: "decodeBinary(bytes) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
32: "decodeBinary(bytes) fails signalling immediate EOF (NB. never with a syntax error)"
}>
"Implementations may vary in their treatment of the difference between expectations"
"13/14 and 16/17, depending on how they wish to treat end-of-stream conditions."
"21/22 and 31/32, depending on how they wish to treat end-of-stream conditions."
]>
<TestCases {
annotation1: <Test #x"85 B103616263 B00109" @"abc" 9>

200
implementations/racket/preserves/preserves/tests/test-main.rkt
View File

@ -9,171 +9,50 @@
(require racket/runtime-path)
(require syntax/srcloc)
(define (d bs #:allow-invalid-prefix? [allow-invalid-prefix? #f])
(define (encodeBinary v)
(preserve->bytes v #:encode-embedded values #:canonicalizing? #t))
(define (looseEncodeBinary v)
(preserve->bytes v #:encode-embedded values #:canonicalizing? #f #:write-annotations? #t))
(define (annotatedBinary v)
(preserve->bytes v #:encode-embedded values #:canonicalizing? #t #:write-annotations? #t))
(define (decodeBinary bs #:allow-invalid-prefix? [allow-invalid-prefix? #f])
(for [(i (in-range 1 (- (bytes-length bs) 1)))]
(define result (bytes->preserve (subbytes bs 0 i)
#:decode-embedded strip-annotations
#:on-short (lambda () 'short) void))
(when (and (not (eq? result 'short))
(not (and allow-invalid-prefix? (void? result))))
(error 'd "~a-byte prefix of ~v does not read as short; result: ~v" i bs result)))
(error 'decodeBinary "~a-byte prefix of ~v does not read as short; result: ~v" i bs result)))
(bytes->preserve bs
#:read-syntax? #t
#:decode-embedded strip-annotations
#:on-short (lambda () 'short)
void))
(define (d-strip bs) (strip-annotations (d bs)))
(define (decodeBinary/strip bs)
(strip-annotations (decodeBinary bs)))
(struct discard () #:prefab)
(struct capture (detail) #:prefab)
(struct observe (specification) #:prefab)
(define (encodeText v)
(preserve->string v #:encode-embedded values))
(define (decodeText s)
(string->preserve s #:read-syntax? #t #:decode-embedded strip-annotations))
(struct speak (who what) #:prefab)
(struct date (year month day) #:prefab)
(struct thing (id) #:prefab)
(struct person thing (name date-of-birth) #:prefab)
(struct titled person (title) #:prefab)
(struct asymmetric (forward back))
(define samples-txt-expected
(hash 'record1 (capture (discard))
'record2 (observe (speak (discard) (capture (discard))))
'list4a '(1 2 3 4)
'list5 '(-2 -1 0 1)
'string3 "hello"
'list6 `("hello" there #"world" () ,(set) #t #f)
'bytes2 #"hello"
'bytes3 #"ABC"
'bytes4 #"ABC"
'bytes5 #"AJN"
'bytes7 #"corymb"
'bytes8 #"corymb"
'bytes9 #"Hi"
'bytes10 #"Hi"
'bytes11 #"Hi"
'value1 #"corymb"
'value2 #t
'value3 #t
'value4 #t
'value5 #t
'value6 (list 1 2 3)
'list0 '()
'dict0 (hash)
'string0 ""
'symbol0 '||
'set0 (set)
'set1 (set 1 2 3)
'set1a (set 1 2 3)
'string4 "abc\u6c34\u6C34\\/\"\b\f\n\r\txyz"
'bytes13 #"abc\x6c\x34\xf0\\/\"\b\f\n\r\txyz"
'string5 "\U0001D11E"
'record1 (capture (discard))
'record2 (observe (speak (discard) (capture (discard))))
'record3 (titled 101 "Blackwell" (date 1821 2 3) "Dr")
'record4 (asymmetric (record 'discard '()) (discard))
'record5 (record 7 '(()))
'record6 (asymmetric (record 'discard '(surprise))
'#s(discard surprise))
'record7 (record "aString" '(3 4))
'record8 (record (discard) '(3 4))
'list7 (list 'abc '|...| 'def)
'dict1 (hash 'a 1
"b" #t
'(1 2 3) #"c"
(hash 'first-name "Elizabeth") (hash 'surname "Blackwell"))
'rfc8259-example1 (hash "Image"
(hash "Width" 800
"Height" 600
"Title" "View from 15th Floor"
"Thumbnail" (hash "Url" "http://www.example.com/image/481989943"
"Height" 125
"Width" 100)
"Animated" 'false
"IDs" (list 116 943 234 38793)))
'rfc8259-example2 (list (hash
"precision" "zip"
"Latitude" 37.7668
"Longitude" -122.3959
"Address" ""
"City" "SAN FRANCISCO"
"State" "CA"
"Zip" "94107"
"Country" "US")
(hash
"precision" "zip"
"Latitude" 37.371991
"Longitude" -122.026020
"Address" ""
"City" "SUNNYVALE"
"State" "CA"
"Zip" "94085"
"Country" "US"))
'annotation1 (asymmetric (annotate 9 "abc") 9)
'annotation2 (asymmetric (annotate (list '() (annotate '() "x")) "abc" "def") '(() ()))
'annotation3 (asymmetric (annotate 5 (annotate 2 1) (annotate 4 3)) 5)
'annotation4 (asymmetric (hash (annotate 'a 'ak) (annotate 1 'av)
(annotate 'b 'bk) (annotate 2 'bv))
(hash 'a 1 'b 2))
'annotation5 (asymmetric (annotate `#s(R ,(annotate 'f 'af)) 'ar) `#s(R f))
'annotation6 (asymmetric (record (annotate 'R 'ar) (list (annotate 'f 'af))) `#s(R f))
'annotation7 (asymmetric (annotate '() 'a 'b 'c) '())
'delimiters4 (asymmetric (list #f (annotate #t "a line comment")) (list #f #t))
'delimiters5 (asymmetric (list #f (annotate #t 'ann)) (list #f #t))
))
(define (run-test-case variety t-name loc binary-form annotated-text-form)
(define text-form (strip-annotations annotated-text-form))
(define-values (forward back can-execute-nondet-with-canonicalization?)
(match (hash-ref samples-txt-expected t-name text-form)
[(asymmetric f b) (values f b #f)] ;; #f because e.g. annotation4 includes annotations
[v (values v v #t)]))
(check-equal? text-form back loc) ;; expectation 1
(check-equal? (d-strip (preserve->bytes #:encode-embedded values text-form))
back
loc) ;; expectation 2
(check-equal? (d-strip (preserve->bytes #:encode-embedded values forward))
back
loc) ;; expectation 3
(check-equal? (d-strip binary-form) back loc) ;; expectation 4
(check-equal? (d binary-form) annotated-text-form loc) ;; expectation 5
(check-equal? (d (preserve->bytes #:encode-embedded values annotated-text-form))
annotated-text-form
loc) ;; expectation 6
(check-equal? (string->preserve #:decode-embedded strip-annotations
(preserve->string #:encode-embedded values text-form))
back
loc) ;; expectation 7
(check-equal? (string->preserve #:decode-embedded strip-annotations
(preserve->string #:encode-embedded values forward))
back
loc) ;; expectation 8
;; similar to 8:
(check-equal? (string->preserve #:decode-embedded strip-annotations
(preserve->string #:encode-embedded values
annotated-text-form)
#:read-syntax? #t)
annotated-text-form
loc)
(when (and (not (memq variety '(decode)))
(or (not (memq variety '(nondeterministic)))
(and can-execute-nondet-with-canonicalization?)))
;; expectations 9 and 10
(check-equal? (preserve->bytes forward
#:encode-embedded values
#:canonicalizing? #t
#:write-annotations? #t)
binary-form
loc))
(unless (memq variety '(decode nondeterministic))
;; expectation 11
(check-equal? (preserve->bytes annotated-text-form
#:encode-embedded values
#:write-annotations? #t)
binary-form
loc)))
(define (run-test-case nondet? t-name loc binary annotatedValue)
(define stripped (strip-annotations annotatedValue))
(let ((roundtripped (decodeBinary (encodeBinary annotatedValue)))) ;; expectation 1
(check-false (has-any-annotations? roundtripped) loc)
(check-equal? (strip-annotations roundtripped) stripped loc))
(check-equal? (decodeBinary/strip binary) stripped loc) ;; expectation 2
(check-equal? (decodeBinary binary) annotatedValue loc) ;; expectation 3
(check-equal? (decodeBinary (annotatedBinary annotatedValue)) annotatedValue loc) ;; expectation 4
(let ((roundtripped (decodeText (encodeText stripped)))) ;; expectation 5
(check-false (has-any-annotations? roundtripped) loc)
(check-equal? (strip-annotations roundtripped) stripped loc))
(check-equal? (decodeText (encodeText annotatedValue)) annotatedValue loc) ;; expectation 6
(check-equal? (annotatedBinary annotatedValue) binary loc) ;; expectation 7
(unless nondet? (check-equal? (looseEncodeBinary annotatedValue) binary loc)) ;; expectation 8
)
(define-runtime-path samples-pr-path "./samples.pr")
(let* ((testfile (call-with-input-file samples-pr-path
@ -189,14 +68,12 @@
(define loc (format "Test case '~a' (~a)" t-name (source-location->string (annotated-srcloc t*))))
(define (fail-test fmt . args)
(fail (format "~a: ~a" loc (apply format fmt args))))
(displayln loc)
(log-debug "~a" loc)
(match (peel-annotations t*)
[`#s(Test ,(strip-annotations binary-form) ,annotated-text-form)
(run-test-case 'normal t-name loc binary-form annotated-text-form)]
[`#s(NondeterministicTest ,(strip-annotations binary-form) ,annotated-text-form)
(run-test-case 'nondeterministic t-name loc binary-form annotated-text-form)]
[`#s(DecodeTest ,(strip-annotations binary-form) ,annotated-text-form)
(run-test-case 'decode t-name loc binary-form annotated-text-form)]
[`#s(Test ,(strip-annotations binary) ,annotatedValue)
(run-test-case #f t-name loc binary annotatedValue)]
[`#s(NondeterministicTest ,(strip-annotations binary) ,annotatedValue)
(run-test-case #t t-name loc binary annotatedValue)]
[`#s(ParseError ,(strip-annotations str))
(with-handlers [(exn:fail:read:eof?
(lambda (e) (fail-test "Unexpected EOF: ~e" e)))
@ -215,10 +92,9 @@
(fail-test "Unexpected success"))]
[(or `#s(DecodeShort ,(strip-annotations bs))
`#s(DecodeEOF ,(strip-annotations bs)))
(check-eq? (d bs) 'short loc)]
(check-eq? (decodeBinary bs) 'short loc)]
[`#s(DecodeError ,(strip-annotations bs))
(check-true (void? (d bs #:allow-invalid-prefix? #t)) loc)]
(check-true (void? (decodeBinary bs #:allow-invalid-prefix? #t)) loc)]
[_
(write-preserve/text t* #:indent #f)
(newline)]))
(fail-test "Unknown test case kind")]))
)

1
implementations/rust/preserves/tests/samples/mod.rs
View File

@ -10,7 +10,6 @@ pub struct TestCases {
pub enum TestCase {
Test(#[serde(with = "serde_bytes")] Vec<u8>, IOValue),
NondeterministicTest(#[serde(with = "serde_bytes")] Vec<u8>, IOValue),
DecodeTest(#[serde(with = "serde_bytes")] Vec<u8>, IOValue),
ParseError(String),
ParseShort(String),
ParseEOF(String),

37
implementations/rust/preserves/tests/samples_tests.rs
View File

@ -124,38 +124,27 @@ fn run() -> io::Result<()> {
let mut d = src.packed_iovalues().configured(true);
let tests: TestCases = deserialize_from_value(&d.next().unwrap().unwrap()).unwrap();
for (Symbol(ref name), ref case) in tests.tests {
for (Symbol(name), case) in tests.tests {
println!("{:?} ==> {:?}", name, case);
match case {
TestCase::Test(ref bin, ref val) => {
TestCase::Test(bin, val) => {
assert_eq!(
&decode_all(&PackedWriter::encode_iovalue(val)?[..])?,
&decode_all(&PackedWriter::encode_iovalue(&val)?[..])?,
&[val.clone()]
);
assert_eq!(&decode_all(&bin[..])?, &[val.clone()]);
assert_eq!(&PackedWriter::encode_iovalue(val)?, bin);
assert_eq!(&PackedWriter::encode_iovalue(&val)?, &bin);
}
TestCase::NondeterministicTest(ref bin, ref val) => {
// The test cases in samples.pr are carefully written
// so that while strictly "nondeterministic", the
// order of keys in encoded dictionaries follows
// Preserves canonical order.
assert_eq!(&PackedWriter::encode_iovalue(val)?, bin);
TestCase::NondeterministicTest(bin, val) => {
assert_eq!(&PackedWriter::encode_iovalue(&val)?, &bin);
assert_eq!(
&decode_all(&PackedWriter::encode_iovalue(val)?[..])?,
&[val.clone()]
);
assert_eq!(&decode_all(&bin[..])?, &[val.clone()]);
}
TestCase::DecodeTest(ref bin, ref val) => {
assert_eq!(
&decode_all(&PackedWriter::encode_iovalue(val)?[..])?,
&decode_all(&PackedWriter::encode_iovalue(&val)?[..])?,
&[val.clone()]
);
assert_eq!(&decode_all(&bin[..])?, &[val.clone()]);
}
TestCase::ParseError(text) => {
match parse_all(text) {
match parse_all(&text) {
Ok(_) => panic!("Unexpected success"),
Err(e) => {
if is_syntax_io_error(&e) {
@ -185,7 +174,7 @@ fn run() -> io::Result<()> {
.next()
.is_none());
}
TestCase::DecodeError(ref bin) => {
TestCase::DecodeError(bin) => {
match decode_all(&bin[..]) {
Ok(_) => panic!("Unexpected success"),
Err(e) => {
@ -197,8 +186,8 @@ fn run() -> io::Result<()> {
}
}
}
TestCase::DecodeShort(ref bin) => {
assert!(if let Err(e) = BytesBinarySource::new(bin)
TestCase::DecodeShort(bin) => {
assert!(if let Err(e) = BytesBinarySource::new(&bin)
.packed_iovalues()
.configured(true)
.next()
@ -209,8 +198,8 @@ fn run() -> io::Result<()> {
false
})
}
TestCase::DecodeEOF(ref bin) => {
assert!(BytesBinarySource::new(bin)
TestCase::DecodeEOF(bin) => {
assert!(BytesBinarySource::new(&bin)
.packed_iovalues()
.configured(true)
.next()

12
preserves.md
View File

@ -75,9 +75,15 @@ lexicographically.
### Symbols.
Programming languages like Lisp and Prolog frequently use string-like
values called *symbols*. Here, a `Symbol` is, like a `String`, a
sequence of Unicode scalar values representing an identifier of some
kind. `Symbol`s are also compared lexicographically by scalar value.
values called *symbols*.[^even-java-has-quasi-symbols] Here, a `Symbol`
is, like a `String`, a sequence of Unicode scalar values representing an
identifier of some kind. `Symbol`s are also compared lexicographically
by scalar value.
[^even-java-has-quasi-symbols]: Even Java has quasi-symbols in the form
of its "interned strings". A Java Preserves implementation might
intern Preserves `Symbol`s while leaving Preserves `String`s
uninterned.
### Booleans.

BIN
tests/samples.bin
View File

Binary file not shown.

64
tests/samples.pr
View File

@ -2,45 +2,43 @@
@<Documentation [
"Individual test cases may be any of the following record types:"
<TestCaseTypes {
Test: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8 9 11}}
NondeterministicTest: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8 10 11}}
DecodeTest: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8}}
ParseError: {fields: [text] expectations: #{12}}
ParseShort: {fields: [text] expectations: #{13}}
ParseEOF: {fields: [text] expectations: #{14}}
DecodeError: {fields: [bytes] expectations: #{15}}
DecodeShort: {fields: [bytes] expectations: #{16}}
DecodeEOF: {fields: [bytes] expectations: #{17}}
Test: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7 8}}
NondeterministicTest: {fields: [binary annotatedValue] expectations: #{1 2 3 4 5 6 7}}
ParseError: {fields: [text] expectations: #{20}}
ParseShort: {fields: [text] expectations: #{21}}
ParseEOF: {fields: [text] expectations: #{22}}
DecodeError: {fields: [bytes] expectations: #{30}}
DecodeShort: {fields: [bytes] expectations: #{31}}
DecodeEOF: {fields: [bytes] expectations: #{32}}
}>
"In each test, let value = strip(annotatedValue),",
" forward = value,",
" back = value,"
"except where test-case-specific values of `forward` and/or `back`",
"are provided by the executing harness, and check the following"
"numbered expectations according to the table above:"
"In each test, let stripped = strip(annotatedValue),"
" encodeBinary(·) produce canonical ordering and no annotations,"
" looseEncodeBinary(·) produce any ordering, but with annotations,"
" annotatedBinary(·) produce canonical ordering, but with annotations,"
" decodeBinary(·) include annotations,"
" encodeText(·) include annotations,"
" decodeText(·) include annotations,"
"and check the following numbered expectations according to the table above:"
<TestCaseExpectations {
1: "value = back"
2: "strip(decodeBinary(encodeBinary(value))) = back"
3: "strip(decodeBinary(encodeBinary(forward))) = back"
4: "strip(decodeBinary(binary)) = back"
5: "decodeBinary(binary) = annotatedValue"
6: "decodeBinary(encodeBinary(annotatedValue)) = annotatedValue"
7: "decodeText(encodeText(value)) = back"
8: "decodeText(encodeText(forward)) = back"
9: "encodeBinary(forward) = binary"
10: "canonicallyOrderedEncodedBinaryWithAnnotations(forward) = binary"
11: "encodeBinary(annotatedValue) = binary"
1: "decodeBinary(encodeBinary(annotatedValue))) = stripped"
2: "strip(decodeBinary(binary)) = stripped"
3: "decodeBinary(binary) = annotatedValue"
4: "decodeBinary(annotatedBinary(annotatedValue)) = annotatedValue"
5: "decodeText(encodeText(stripped)) = stripped"
6: "decodeText(encodeText(annotatedValue)) = annotatedValue"
7: "annotatedBinary(annotatedValue) = binary"
8: "looseEncodeBinary(annotatedValue) = binary"
12: "decodeText(text) fails with a syntax error (NB. never with EOF)"
13: "decodeText(text) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
14: "decodeText(text) fails signalling immediate EOF (NB. never with a syntax error)"
20: "decodeText(text) fails with a syntax error (NB. never with EOF)"
21: "decodeText(text) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
22: "decodeText(text) fails signalling immediate EOF (NB. never with a syntax error)"
15: "decodeBinary(bytes) fails with a syntax error (NB. never with EOF)"
16: "decodeBinary(bytes) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
17: "decodeBinary(bytes) fails signalling immediate EOF (NB. never with a syntax error)"
30: "decodeBinary(bytes) fails with a syntax error (NB. never with EOF)"
31: "decodeBinary(bytes) fails signalling premature EOF after partial parse (NB. never with a syntax error)"
32: "decodeBinary(bytes) fails signalling immediate EOF (NB. never with a syntax error)"
}>
"Implementations may vary in their treatment of the difference between expectations"
"13/14 and 16/17, depending on how they wish to treat end-of-stream conditions."
"21/22 and 31/32, depending on how they wish to treat end-of-stream conditions."
]>
<TestCases {
annotation1: <Test #x"85 B103616263 B00109" @"abc" 9>