From b7a2acf65b7dd4b6849079a635afa1f4ac37de8f Mon Sep 17 00:00:00 2001 From: Tony Garnock-Jones Date: Sat, 14 Oct 2023 21:29:09 +0200 Subject: [PATCH] Update implementations for simplified binary syntax. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As part of this, I noticed a problem when updating the Rust implementation. With the BF..84 design for annotations, if you are wanting to skip annotations—for example, if you're parsing incrementally, SAX-style, ignoring annotations—then with any parenthesised form you have to remember to check for and skip something after reading a term. This is awkward, so I want to revisit the design of annotations and go back to the prefix-form (85..) of the current-mainline design. --- implementations/c/preserves.h | 203 ++++---- implementations/cpp/main.cpp | 16 +- .../cpp/preserves_binary_reader.hpp | 48 +- .../cpp/preserves_binary_writer.hpp | 73 ++- implementations/cpp/preserves_impl.hpp | 3 +- implementations/javascript/package.json | 3 +- .../javascript/packages/core/src/annotated.ts | 9 +- .../javascript/packages/core/src/constants.ts | 13 +- .../javascript/packages/core/src/decoder.ts | 176 ++++--- .../javascript/packages/core/src/encoder.ts | 23 +- .../javascript/packages/core/src/float.ts | 6 +- .../packages/core/test/codec.test.ts | 12 +- .../packages/schema/src/gen/schema.ts | 2 +- implementations/javascript/yarn.lock | 98 +++- .../syndicate-protocols-schema-bundle.bin | 24 +- implementations/python/preserves/binary.py | 82 ++-- implementations/python/preserves/path.prb | 2 +- implementations/python/preserves/path.py | 12 +- implementations/python/preserves/schema.prb | 4 +- implementations/python/preserves/values.py | 16 +- implementations/python/tests/samples.bin | Bin 10057 -> 10541 bytes implementations/python/tests/samples.pr | 187 ++++---- .../python/tests/test_preserves.py | 70 +-- .../racket/preserves/preserves/jelly.rkt | 50 +- .../preserves/preserves/read-binary.rkt | 16 +- .../preserves/preserves/tests/samples.pr | 187 ++++---- .../preserves/preserves/write-binary.rkt | 35 +- implementations/rust/preserves-path/path.bin | 2 +- .../rust/preserves/src/value/boundary.rs | 2 - .../preserves/src/value/packed/constants.rs | 28 +- .../rust/preserves/src/value/packed/reader.rs | 435 +++++++++--------- .../rust/preserves/src/value/packed/writer.rs | 63 ++- .../rust/preserves/src/value/reader.rs | 8 +- path/path.bin | 2 +- python/0.18.1/binary/index.html | 140 +++--- python/0.18.1/index.html | 2 +- python/0.18.1/path/index.html | 12 +- python/0.18.1/search/search_index.json | 2 +- python/0.18.1/sitemap.xml | 22 +- python/0.18.1/values/index.html | 108 ++--- schema/schema.bin | 4 +- tests/samples.bin | Bin 10057 -> 10541 bytes tests/samples.pr | 187 ++++---- 43 files changed, 1172 insertions(+), 1215 deletions(-) diff --git a/implementations/c/preserves.h b/implementations/c/preserves.h index 446b21d..88ce988 100644 --- a/implementations/c/preserves.h +++ b/implementations/c/preserves.h @@ -218,19 +218,11 @@ PRESERVES_INLINE int preserves_pool_alloc_bytes(preserves_pool_t *pool, typedef enum preserves_binary_format_tag { PRESERVES_BINARY_FORMAT_TAG_FALSE = 0x80, PRESERVES_BINARY_FORMAT_TAG_TRUE = 0x81, - PRESERVES_BINARY_FORMAT_TAG_FLOAT = 0x82, - PRESERVES_BINARY_FORMAT_TAG_DOUBLE = 0x83, PRESERVES_BINARY_FORMAT_TAG_END = 0x84, - PRESERVES_BINARY_FORMAT_TAG_ANNOTATION = 0x85, PRESERVES_BINARY_FORMAT_TAG_EMBEDDED = 0x86, + PRESERVES_BINARY_FORMAT_TAG_IEEE754 = 0x87, - PRESERVES_BINARY_FORMAT_TAG_SMALL_INTEGER_LO = 0x90, - PRESERVES_BINARY_FORMAT_TAG_SMALL_INTEGER_HI = 0x9F, - - PRESERVES_BINARY_FORMAT_TAG_MEDIUM_INTEGER_LO = 0xA0, - PRESERVES_BINARY_FORMAT_TAG_MEDIUM_INTEGER_HI = 0xAF, - - PRESERVES_BINARY_FORMAT_TAG_LARGE_INTEGER = 0xB0, + PRESERVES_BINARY_FORMAT_TAG_SIGNED_INTEGER = 0xB0, PRESERVES_BINARY_FORMAT_TAG_STRING = 0xB1, PRESERVES_BINARY_FORMAT_TAG_BYTE_STRING = 0xB2, PRESERVES_BINARY_FORMAT_TAG_SYMBOL = 0xB3, @@ -239,6 +231,8 @@ typedef enum preserves_binary_format_tag { PRESERVES_BINARY_FORMAT_TAG_SEQUENCE = 0xB5, PRESERVES_BINARY_FORMAT_TAG_SET = 0xB6, PRESERVES_BINARY_FORMAT_TAG_DICTIONARY = 0xB7, + + PRESERVES_BINARY_FORMAT_TAG_ANNOTATION = 0xBF, } preserves_binary_format_tag_t; PRESERVES_OUTOFLINE @@ -247,12 +241,10 @@ PRESERVES_OUTOFLINE switch (tag) { case PRESERVES_BINARY_FORMAT_TAG_FALSE: return "FALSE"; case PRESERVES_BINARY_FORMAT_TAG_TRUE: return "TRUE"; - case PRESERVES_BINARY_FORMAT_TAG_FLOAT: return "FLOAT"; - case PRESERVES_BINARY_FORMAT_TAG_DOUBLE: return "DOUBLE"; case PRESERVES_BINARY_FORMAT_TAG_END: return "END"; - case PRESERVES_BINARY_FORMAT_TAG_ANNOTATION: return "ANNOTATION"; case PRESERVES_BINARY_FORMAT_TAG_EMBEDDED: return "EMBEDDED"; - case PRESERVES_BINARY_FORMAT_TAG_LARGE_INTEGER: return "LARGE_INTEGER"; + case PRESERVES_BINARY_FORMAT_TAG_IEEE754: return "IEEE754"; + case PRESERVES_BINARY_FORMAT_TAG_SIGNED_INTEGER: return "SIGNED_INTEGER"; case PRESERVES_BINARY_FORMAT_TAG_STRING: return "STRING"; case PRESERVES_BINARY_FORMAT_TAG_BYTE_STRING: return "BYTE_STRING"; case PRESERVES_BINARY_FORMAT_TAG_SYMBOL: return "SYMBOL"; @@ -260,16 +252,8 @@ PRESERVES_OUTOFLINE case PRESERVES_BINARY_FORMAT_TAG_SEQUENCE: return "SEQUENCE"; case PRESERVES_BINARY_FORMAT_TAG_SET: return "SET"; case PRESERVES_BINARY_FORMAT_TAG_DICTIONARY: return "DICTIONARY"; - default: - if ((tag >= PRESERVES_BINARY_FORMAT_TAG_SMALL_INTEGER_LO) && - (tag <= PRESERVES_BINARY_FORMAT_TAG_SMALL_INTEGER_HI)) { - return "SMALL_INTEGER"; - } else if ((tag >= PRESERVES_BINARY_FORMAT_TAG_MEDIUM_INTEGER_LO) && - (tag <= PRESERVES_BINARY_FORMAT_TAG_MEDIUM_INTEGER_HI)) { - return "MEDIUM_INTEGER"; - } else { - return "UNKNOWN"; - } + case PRESERVES_BINARY_FORMAT_TAG_ANNOTATION: return "ANNOTATION"; + default: return "UNKNOWN"; } }); @@ -328,9 +312,11 @@ typedef enum preserves_error_code { PRESERVES_END_UNEXPECTED_END, PRESERVES_END_DICTIONARY_MISSING_VALUE, PRESERVES_END_RECORD_MISSING_LABEL, + PRESERVES_END_ANNOTATION_TOO_SHORT, PRESERVES_END_VARINT_TOO_BIG, PRESERVES_END_INVALID_UTF8, PRESERVES_END_INVALID_TAG, + PRESERVES_END_INVALID_IEEE754, } preserves_error_code_t; PRESERVES_OUTOFLINE(char const *preserves_error_code_name(preserves_error_code_t code), { @@ -343,9 +329,11 @@ PRESERVES_OUTOFLINE(char const *preserves_error_code_name(preserves_error_code_t case PRESERVES_END_UNEXPECTED_END: return "UNEXPECTED_END"; case PRESERVES_END_DICTIONARY_MISSING_VALUE: return "DICTIONARY_MISSING_VALUE"; case PRESERVES_END_RECORD_MISSING_LABEL: return "RECORD_MISSING_LABEL"; + case PRESERVES_END_ANNOTATION_TOO_SHORT: return "ANNOTATION_TOO_SHORT"; case PRESERVES_END_VARINT_TOO_BIG: return "VARINT_TOO_BIG"; case PRESERVES_END_INVALID_UTF8: return "INVALID_UTF8"; case PRESERVES_END_INVALID_TAG: return "INVALID_TAG"; + case PRESERVES_END_INVALID_IEEE754: return "INVALID_IEEE754"; default: return "UNKNOWN"; } }); @@ -406,23 +394,19 @@ PRESERVES_OUTOFLINE - repr=PRESERVES_ESCAPED -> len, data._unsigned as absolute offset within input to utf-8 bytes that need Symbol-style backslash-escapes interpreted - PRESERVES_RECORD, PRESERVES_SEQUENCE, PRESERVES_SET, PRESERVES_DICTIONARY: + PRESERVES_RECORD, PRESERVES_SEQUENCE, PRESERVES_SET, PRESERVES_DICTIONARY, PRESERVES_ANNOTATION: - repr==PRESERVES_REPR_NONE, - len counts number of items: - PRESERVES_RECORD -> number of fields plus one (for the label) - PRESERVES_SEQUENCE -> number of items - PRESERVES_SET -> number of items - PRESERVES_DICTIONARY -> twice the number of key-value pairs + - PRESERVES_ANNOTATION -> number of annotations plus one (for the underlying item) + - for PRESERVES_ANNOTATION, the annotated item will not be a PRESERVES_ANNOTATION - data._unsigned as relative offset within index to next item, starting from this entry; zero means "no end known" PRESERVES_EMBEDDED: repr==PRESERVES_REPR_NONE, len==0, following item is the embedded value - PRESERVES_ANNOTATION: - - repr=PRESERVES_REPR_NONE, - - len counts number of annotations, - - data._unsigned as relative offset within index to annotated - item, starting from this entry; zero means "no end known" - - the annotated item will not be a PRESERVES_ANNOTATION PRESERVES_END_MARKER: repr==PRESERVES_REPR_NONE, len==0, data._err */ @@ -452,7 +436,6 @@ typedef struct preserves_reader { size_t stack_top; /* ascending empty */ size_t input_pos; /* ascending full */ size_t index_pos; /* ascending empty */ - bool annotation_tag_seen; } preserves_reader_t; typedef struct preserves_reader_result { @@ -472,7 +455,6 @@ PRESERVES_INLINE preserves_reader_t preserves_create_reader(void) { .stack_top = 0, .input_pos = 0, .index_pos = 0, - .annotation_tag_seen = false, }; } @@ -483,7 +465,6 @@ PRESERVES_OUTOFLINE(void preserves_free_reader(preserves_reader_t *r), { r->stack_top = 0; r->input_pos = 0; r->index_pos = 0; - r->annotation_tag_seen = false; }); PRESERVES_IMPLEMENTATION_CHUNK @@ -501,8 +482,8 @@ PRESERVES_IMPLEMENTATION_CHUNK } static inline int _preserves_reader_next(preserves_reader_t *r) { - if (r->input_pos >= r->input.len) return -1; - int result = PRESERVES_ARRAY_ELEMENT(&r->input, uint8_t, r->input_pos); + int result = _preserves_reader_peek(r); + if (result == -1) return -1; r->input_pos++; return result; } @@ -553,11 +534,6 @@ PRESERVES_IMPLEMENTATION_CHUNK return base; } - static inline bool _preserves_reader_in_annotations(preserves_reader_t *r) { - return (r->stack_top > 0) && - (_preserves_reader_stack_top_entry(r)->type == PRESERVES_ANNOTATION); - } - static inline void _preserves_reader_inc_collection_len(preserves_reader_t *r, size_t *count_ptr) { if (r->stack_top > 0) { check_for_embedded: @@ -578,10 +554,6 @@ PRESERVES_IMPLEMENTATION_CHUNK static inline preserves_index_entry_t *_preserves_reader_emit_entry(preserves_reader_t *r, size_t *count_ptr, preserves_index_entry_t e) { - if (!r->annotation_tag_seen && _preserves_reader_in_annotations(r)) { - /* printf("(popping annotation collector)\n"); */ - _preserves_reader_finish_seq(r); - } if (count_ptr != NULL) { _preserves_reader_inc_collection_len(r, count_ptr); } @@ -594,8 +566,6 @@ PRESERVES_IMPLEMENTATION_CHUNK *ix = e; r->index_pos++; - r->annotation_tag_seen = false; - return ix; } @@ -618,19 +588,25 @@ PRESERVES_IMPLEMENTATION_CHUNK } } - static inline int _preserves_reader_varint(preserves_reader_t *r, size_t *v) { + static inline size_t _preserves_reader_varint(preserves_reader_t *r, preserves_error_code_t *code) { unsigned int shift_amount = 0; size_t result = 0; while (true) { int b = _preserves_reader_next(r); - if (b == -1) return -1; + if (b == -1) { + *code = PRESERVES_END_INCOMPLETE_INPUT; + return 0; + } result |= (b & 0x7f) << shift_amount; if (b & 0x80) { shift_amount += 7; - if (shift_amount > ((sizeof(size_t) * 8) - 7)) return -2; + if (shift_amount > ((sizeof(size_t) * 8) - 7)) { + *code = PRESERVES_END_VARINT_TOO_BIG; + return 0; + } } else { - *v = result; - return 0; + *code = PRESERVES_END_NO_ERROR; + return result; } } } @@ -742,12 +718,9 @@ PRESERVES_IMPLEMENTATION_CHUNK size_t *count_ptr, preserves_type_tag_t type, bool should_check_utf8) { - size_t len = 0; - switch (_preserves_reader_varint(r, &len)) { - case -1: return PRESERVES_END_INCOMPLETE_INPUT; - case -2: return PRESERVES_END_VARINT_TOO_BIG; - default: break; - } + preserves_error_code_t varint_err = PRESERVES_END_NO_ERROR; + size_t len = _preserves_reader_varint(r, &varint_err); + if (varint_err != PRESERVES_END_NO_ERROR) return varint_err; size_t starting_pos = r->input_pos; uint8_t *maybe_utf = _preserves_reader_next_bytes(r, len); if (should_check_utf8 && (check_utf8(maybe_utf, len) == -1)) { @@ -784,7 +757,7 @@ PRESERVES_IMPLEMENTATION_CHUNK PRESERVES_INLINE preserves_index_entry_t *preserves_skip_annotations(preserves_index_entry_t *ix) { if (ix == NULL) return NULL; if (ix->type != PRESERVES_ANNOTATION) return ix; - ix += ix->data._unsigned; + ix++; if (ix->type == PRESERVES_ANNOTATION) abort(); return ix; } @@ -799,11 +772,10 @@ PRESERVES_OUTOFLINE /* printf(" %02d: (%5lu) ", i, ip); */ /* preserves_dump_index_entry(stdout, &r->input, _preserves_reader_index_entry(r, ip), true); */ /* } */ - /* printf("pos %lu (%05lx), count %lu, annotation tag seen %d: ", */ + /* printf("pos %lu (%05lx), count %lu: ", */ /* r->input_pos, */ /* r->input_pos, */ - /* count, */ - /* r->annotation_tag_seen); */ + /* count); */ int b = _preserves_reader_next(r); /* printf("tag 0x%02x %s\n", b, preserves_binary_format_tag_name(b)); */ if (b == -1) return _preserves_reader_finish(r, PRESERVES_END_INCOMPLETE_INPUT); @@ -823,36 +795,43 @@ PRESERVES_OUTOFLINE }})); break; - case PRESERVES_BINARY_FORMAT_TAG_FLOAT: { - uint8_t *bs = _preserves_reader_next_bytes(r, 4); + case PRESERVES_BINARY_FORMAT_TAG_IEEE754: { + preserves_error_code_t varint_err = PRESERVES_END_NO_ERROR; + size_t len = _preserves_reader_varint(r, &varint_err); + if (varint_err != PRESERVES_END_NO_ERROR) return _preserves_reader_finish(r, varint_err); + uint8_t *bs = _preserves_reader_next_bytes(r, len); if (bs == NULL) return _preserves_reader_finish(r, PRESERVES_END_INCOMPLETE_INPUT); - uint32_t i; - memcpy(&i, bs, 4); - i = ntohl(i); - float f; - memcpy(&f, &i, 4); - RETURN_ON_FAIL(_preserves_reader_emit_entry(r, &count, (preserves_index_entry_t) { - .type = PRESERVES_FLOAT, .repr = PRESERVES_REPR_NONE, .len = 0, .data = { - ._float = f - }})); - break; - } - - case PRESERVES_BINARY_FORMAT_TAG_DOUBLE: { - uint8_t *bs = _preserves_reader_next_bytes(r, 8); - if (bs == NULL) return _preserves_reader_finish(r, PRESERVES_END_INCOMPLETE_INPUT); - uint32_t lo, hi; - memcpy(&hi, bs, 4); - memcpy(&lo, bs + 4, 4); - lo = ntohl(lo); - hi = ntohl(hi); - uint64_t i = (((uint64_t) hi) << 32) | ((uint64_t) lo); - double f; - memcpy(&f, &i, 8); - RETURN_ON_FAIL(_preserves_reader_emit_entry(r, &count, (preserves_index_entry_t) { - .type = PRESERVES_DOUBLE, .repr = PRESERVES_REPR_NONE, .len = 0, .data = { - ._double = f - }})); + switch (len) { + case 4: { + uint32_t i; + memcpy(&i, bs, 4); + i = ntohl(i); + float f; + memcpy(&f, &i, 4); + RETURN_ON_FAIL(_preserves_reader_emit_entry(r, &count, (preserves_index_entry_t) { + .type = PRESERVES_FLOAT, .repr = PRESERVES_REPR_NONE, .len = 0, .data = { + ._float = f + }})); + break; + } + case 8: { + uint32_t lo, hi; + memcpy(&hi, bs, 4); + memcpy(&lo, bs + 4, 4); + lo = ntohl(lo); + hi = ntohl(hi); + uint64_t i = (((uint64_t) hi) << 32) | ((uint64_t) lo); + double f; + memcpy(&f, &i, 8); + RETURN_ON_FAIL(_preserves_reader_emit_entry(r, &count, (preserves_index_entry_t) { + .type = PRESERVES_DOUBLE, .repr = PRESERVES_REPR_NONE, .len = 0, .data = { + ._double = f + }})); + break; + } + default: + return _preserves_reader_finish(r, PRESERVES_END_INVALID_IEEE754); + } break; } @@ -868,26 +847,19 @@ PRESERVES_OUTOFLINE if ((base->type == PRESERVES_RECORD) && (base->len == 0)) { return _preserves_reader_finish(r, PRESERVES_END_RECORD_MISSING_LABEL); } - break; - - case PRESERVES_BINARY_FORMAT_TAG_ANNOTATION: - if (r->annotation_tag_seen || !_preserves_reader_in_annotations(r)) { - RETURN_ON_FAIL(_preserves_reader_push(r, PRESERVES_ANNOTATION)); + if ((base->type == PRESERVES_ANNOTATION) && (base->len < 2)) { + return _preserves_reader_finish(r, PRESERVES_END_ANNOTATION_TOO_SHORT); } - r->annotation_tag_seen = true; break; case PRESERVES_BINARY_FORMAT_TAG_EMBEDDED: RETURN_ON_FAIL(_preserves_reader_push(r, PRESERVES_EMBEDDED)); break; - case PRESERVES_BINARY_FORMAT_TAG_LARGE_INTEGER: { - size_t len = 0; - switch (_preserves_reader_varint(r, &len)) { - case -1: return _preserves_reader_finish(r, PRESERVES_END_INCOMPLETE_INPUT); - case -2: return _preserves_reader_finish(r, PRESERVES_END_VARINT_TOO_BIG); - default: break; - } + case PRESERVES_BINARY_FORMAT_TAG_SIGNED_INTEGER: { + preserves_error_code_t varint_err = PRESERVES_END_NO_ERROR; + size_t len = _preserves_reader_varint(r, &varint_err); + if (varint_err != PRESERVES_END_NO_ERROR) return _preserves_reader_finish(r, varint_err); if (_preserves_reader_decode_intbytes(r, &count, len) == -1) { return _preserves_reader_finish(r, PRESERVES_END_INCOMPLETE_INPUT); } @@ -931,22 +903,12 @@ PRESERVES_OUTOFLINE RETURN_ON_FAIL(_preserves_reader_push(r, PRESERVES_DICTIONARY)); break; - default: - if ((b >= PRESERVES_BINARY_FORMAT_TAG_SMALL_INTEGER_LO) && - (b <= PRESERVES_BINARY_FORMAT_TAG_SMALL_INTEGER_HI)) { - int64_t value = b - PRESERVES_BINARY_FORMAT_TAG_SMALL_INTEGER_LO; - if (value > 12) value -= 16; - RETURN_ON_FAIL(_preserves_emit_small_int(r, &count, false, value)); - } else if ((b >= PRESERVES_BINARY_FORMAT_TAG_MEDIUM_INTEGER_LO) && - (b <= PRESERVES_BINARY_FORMAT_TAG_MEDIUM_INTEGER_HI)) { - size_t len = (b - PRESERVES_BINARY_FORMAT_TAG_MEDIUM_INTEGER_LO) + 1; - if (_preserves_reader_decode_intbytes(r, &count, len) == -1) { - return _preserves_reader_finish(r, PRESERVES_END_INCOMPLETE_INPUT); - } - } else { - return _preserves_reader_finish(r, PRESERVES_END_INVALID_TAG); - } + case PRESERVES_BINARY_FORMAT_TAG_ANNOTATION: + RETURN_ON_FAIL(_preserves_reader_push(r, PRESERVES_ANNOTATION)); break; + + default: + return _preserves_reader_finish(r, PRESERVES_END_INVALID_TAG); } } @@ -975,7 +937,6 @@ PRESERVES_OUTOFLINE r->stack_top = 0; r->input_pos = 0; r->index_pos = 0; - r->annotation_tag_seen = false; preserves_bytes_move(&r->input, input); return preserves_read_binary_continue(r, count); @@ -1039,11 +1000,11 @@ PRESERVES_IMPLEMENTATION_CHUNK case PRESERVES_SEQUENCE: case PRESERVES_SET: case PRESERVES_DICTIONARY: + case PRESERVES_ANNOTATION: fprintf(f, " skip %lu", i->data._unsigned - 1); break; case PRESERVES_EMBEDDED: - case PRESERVES_ANNOTATION: break; case PRESERVES_END_MARKER: diff --git a/implementations/cpp/main.cpp b/implementations/cpp/main.cpp index cea80e0..bfd0cb6 100644 --- a/implementations/cpp/main.cpp +++ b/implementations/cpp/main.cpp @@ -24,14 +24,14 @@ TEST(Value, Basics) { } TEST(BinaryReader, Negative257) { - istringstream input("\xA1\xFE\xFF"); + istringstream input("\xB0\x02\xFE\xFF"); auto v = BinaryReader<>(input).next(); ASSERT_TRUE(v); ASSERT_EQ(v->to_signed(), -257); } TEST(BinaryReader, Negative127) { - istringstream input("\xA0\x81"); + istringstream input("\xB0\x01\x81"); auto v = BinaryReader<>(input).next(); ASSERT_TRUE(v); ASSERT_EQ(v->to_signed(), -127); @@ -42,9 +42,10 @@ TEST(BinaryWriter, Negative257) { BinaryWriter w(s); w << -257; std::string output(s.str()); - ASSERT_EQ(output[0], char(BinaryTag::MediumInteger_lo) + 1); - ASSERT_EQ(output[1], char(0xFE)); - ASSERT_EQ(output[2], char(0xFF)); + ASSERT_EQ(output[0], char(BinaryTag::SignedInteger)); + ASSERT_EQ(output[1], char(0x02)); + ASSERT_EQ(output[2], char(0xFE)); + ASSERT_EQ(output[3], char(0xFF)); } TEST(BinaryWriter, Negative127) { @@ -52,8 +53,9 @@ TEST(BinaryWriter, Negative127) { BinaryWriter w(s); w << -127; std::string output(s.str()); - ASSERT_EQ(output[0], char(BinaryTag::MediumInteger_lo)); - ASSERT_EQ(output[1], char(0x81)); + ASSERT_EQ(output[0], char(BinaryTag::SignedInteger)); + ASSERT_EQ(output[1], char(0x01)); + ASSERT_EQ(output[2], char(0x81)); } TEST(BinaryReader, ReadSamples) { diff --git a/implementations/cpp/preserves_binary_reader.hpp b/implementations/cpp/preserves_binary_reader.hpp index 5ca5bb4..0ea2107 100644 --- a/implementations/cpp/preserves_binary_reader.hpp +++ b/implementations/cpp/preserves_binary_reader.hpp @@ -95,30 +95,18 @@ namespace Preserves { switch (tag) { case BinaryTag::False: return Value::from_bool(false); case BinaryTag::True: return Value::from_bool(true); - case BinaryTag::Float: return next_float().map(Value::from_float); - case BinaryTag::Double: return next_double().map(Value::from_double); case BinaryTag::End: end_sentinel = true; return boost::none; - case BinaryTag::Annotation: { - std::vector> annotations; - while (true) { - auto ann = next(); - if (!ann) return boost::none; - annotations.push_back(*ann); - if (BinaryTag(i.peek()) != BinaryTag::Annotation) break; - i.get(); - } - auto underlying = next(); - if (!underlying) return boost::none; - return Value(new AnnotatedValue(std::move(annotations), *underlying)); - } case BinaryTag::Embedded: return BinaryReader<>(i).next().map(decodeEmbedded).map(Value::from_embedded); - case BinaryTag::SmallInteger_lo ... BinaryTag::SmallInteger_hi: { - int64_t n = int64_t(tag) - int64_t(BinaryTag::SmallInteger_lo); - return Value::from_int(n <= 12 ? n : n - 16); - } - case BinaryTag::MediumInteger_lo ... BinaryTag::MediumInteger_hi: { - int n = int(tag) - int(BinaryTag::MediumInteger_lo) + 1; + case BinaryTag::Ieee754: return varint(i).flat_map([&](size_t len)-> boost::optional> { + switch (len) { + case 4: return next_float().map(Value::from_float); + case 8: return next_double().map(Value::from_double); + default: return boost::none; + } + }); + case BinaryTag::SignedInteger: return varint(i).flat_map([&](size_t n)-> boost::optional> { + if (n == 0) return Value::from_int(uint64_t(0)); if (n < 9) return next_machineword(n, false); if (n == 9) { // We can handle this with uint64_t if it's unsigned and the first byte is 0. @@ -128,9 +116,6 @@ namespace Preserves { } } return next_bignum(n); - } - case BinaryTag::SignedInteger: return varint(i).flat_map([&](size_t len) { - return next_bignum(len); }); case BinaryTag::String: return varint(i).flat_map([&](size_t len)-> boost::optional> { auto s = std::make_shared>(std::string()); @@ -192,6 +177,21 @@ namespace Preserves { s->values.emplace(*k, *v); } } + + case BinaryTag::Annotation: return next().flat_map([&](auto underlying)-> boost::optional> { + std::vector> annotations; + while (true) { + bool end_rec = false; + auto ann = _next(end_rec); + if (end_rec) { + if (annotations.size() == 0) return boost::none; + return Value(new AnnotatedValue(std::move(annotations), underlying)); + } + if (!ann) return boost::none; + annotations.push_back(*ann); + } + }); + default: return boost::none; } diff --git a/implementations/cpp/preserves_binary_writer.hpp b/implementations/cpp/preserves_binary_writer.hpp index 6feb4a4..4d67684 100644 --- a/implementations/cpp/preserves_binary_writer.hpp +++ b/implementations/cpp/preserves_binary_writer.hpp @@ -11,15 +11,9 @@ namespace Preserves { enum class BinaryTag { False = 0x80, True = 0x81, - Float = 0x82, - Double = 0x83, End = 0x84, - Annotation = 0x85, Embedded = 0x86, - SmallInteger_lo = 0x90, - SmallInteger_hi = 0x9f, - MediumInteger_lo = 0xa0, - MediumInteger_hi = 0xaf, + Ieee754 = 0x87, SignedInteger = 0xb0, String = 0xb1, ByteString = 0xb2, @@ -28,6 +22,7 @@ namespace Preserves { Sequence = 0xb5, Set = 0xb6, Dictionary = 0xb7, + Annotation = 0xbf, }; template @@ -90,12 +85,9 @@ namespace Preserves { } BinaryWriter& bignum(std::vector const& n) { - if (n.size() == 0) return (*this) << BinaryTag::SmallInteger_lo; size_t startOffset = 0; - if (n[0] & 0x80) { - if (n.size() == 1 && int8_t(n[0]) >= -3) { - return put(uint8_t(int(BinaryTag::SmallInteger_lo) + 16 + int8_t(n[0]))); - } else { + if (n.size() > 0) { + if (n[0] & 0x80) { while (true) { if (startOffset == n.size() - 1) break; if (n[startOffset] != 0xff) break; @@ -105,10 +97,6 @@ namespace Preserves { break; } } - } - } else { - if (n.size() == 1 && n[0] <= 12) { - return put(uint8_t(int(BinaryTag::SmallInteger_lo) + n[0])); } else { while (true) { if (startOffset == n.size() - 1) break; @@ -122,12 +110,8 @@ namespace Preserves { } } size_t count = n.size() - startOffset; - if (count > 16) { - (*this) << BinaryTag::SignedInteger; - varint(count); - } else { - put(uint8_t(int(BinaryTag::MediumInteger_lo) + count - 1)); - } + (*this) << BinaryTag::SignedInteger; + varint(count); return write(&n[startOffset], count); } @@ -175,7 +159,8 @@ namespace Preserves { buf[1] = (n >> 16) & 0xff; buf[2] = (n >> 8) & 0xff; buf[3] = (n) & 0xff; - (*this) << BinaryTag::Float; + (*this) << BinaryTag::Ieee754; + put(uint8_t(sizeof(buf))); return write(buf, sizeof(buf)); } @@ -191,12 +176,14 @@ namespace Preserves { buf[5] = (n >> 16) & 0xff; buf[6] = (n >> 8) & 0xff; buf[7] = (n) & 0xff; - (*this) << BinaryTag::Double; + (*this) << BinaryTag::Ieee754; + put(uint8_t(sizeof(buf))); return write(buf, sizeof(buf)); } BinaryWriter& _put_medium_int(uint64_t u, int shift, int extra) { - put(uint8_t(int(BinaryTag::MediumInteger_lo) + (shift >> 3) + extra)); + (*this) << BinaryTag::SignedInteger; + put(uint8_t((shift >> 3) + extra + 1)); if (extra) put(0); while (shift >= 0) { put(uint8_t((u >> shift) & 0xff)); @@ -207,30 +194,19 @@ namespace Preserves { template typename std::enable_if::value, BinaryWriter&>::type operator<<(T t) { - if (std::numeric_limits::is_signed) { + if (t == 0) { + (*this) << BinaryTag::SignedInteger; + put(0); + return *this; + } else if (std::numeric_limits::is_signed) { auto i = static_cast(t); - if (i < 0) { - if (i >= -3) { - return put(uint8_t(int(BinaryTag::SmallInteger_lo) + 16 + i)); - } else { - uint64_t u; - memcpy(&u, &i, sizeof(i)); - return _put_medium_int(u, _shift_for<0xff, 0x80>(u), 0); - } - } else { - if (i <= 12) { - return put(uint8_t(int(BinaryTag::SmallInteger_lo) + i)); - } else { - uint64_t u; - memcpy(&u, &i, sizeof(i)); - return _put_medium_int(u, _shift_for<0x00, 0x00>(u), 0); - } - } + uint64_t u; + memcpy(&u, &i, sizeof(i)); + int shift = i < 0 ? _shift_for<0xff, 0x80>(u) : _shift_for<0x00, 0x00>(u); + return _put_medium_int(u, shift, 0); } else { auto u = static_cast(t); - if (u <= 12) { - return put(uint8_t(int(BinaryTag::SmallInteger_lo) + u)); - } else if ((u & 0x8000000000000000) != 0) { + if ((u & 0x8000000000000000) != 0) { return _put_medium_int(u, 56, 1); } else { return _put_medium_int(u, _shift_for<0x00, 0x00>(u), 0); @@ -259,5 +235,10 @@ namespace Preserves { for (auto& i : vs) { (*this) << i.first << i.second; } return (*this) << BinaryTag::End; } + + template + BinaryWriter& annotated(Value const& underlying, std::vector> const& vs) { + return (*this) << BinaryTag::Annotation << underlying << iterable(vs) << BinaryTag::End; + } }; } diff --git a/implementations/cpp/preserves_impl.hpp b/implementations/cpp/preserves_impl.hpp index ee50d5d..dd64d89 100644 --- a/implementations/cpp/preserves_impl.hpp +++ b/implementations/cpp/preserves_impl.hpp @@ -333,8 +333,7 @@ namespace Preserves { } BinaryWriter& write(BinaryWriter &w) const override { - for (auto& v : anns) { w << BinaryTag::Annotation << v; } - return w << underlying; + return w.annotated(underlying, anns); } }; diff --git a/implementations/javascript/package.json b/implementations/javascript/package.json index 9a2a861..fa61f7a 100644 --- a/implementations/javascript/package.json +++ b/implementations/javascript/package.json @@ -4,11 +4,12 @@ "devDependencies": { "@rollup/plugin-terser": "^0.4", "@types/jest": "^27.0", + "@types/node": "^20.8.5", "jest": "^27.4", "lerna": "^4.0", "rollup": "^3.10", "ts-jest": "^27.0", - "ts-node-dev": "^1.1", + "ts-node-dev": "^2.0.0", "typescript": "^4.9", "typescript-language-server": "^3.0" }, diff --git a/implementations/javascript/packages/core/src/annotated.ts b/implementations/javascript/packages/core/src/annotated.ts index 1555152..c56131a 100644 --- a/implementations/javascript/packages/core/src/annotated.ts +++ b/implementations/javascript/packages/core/src/annotated.ts @@ -72,13 +72,16 @@ export class Annotated implements Preservable, PreserveW } __preserve_on__(encoder: Encoder): void { - if (encoder.includeAnnotations) { + if (encoder.includeAnnotations && this.annotations.length > 0) { + encoder.state.emitbyte(Tag.Annotated); + encoder.push(this.item); for (const a of this.annotations) { - encoder.state.emitbyte(Tag.Annotation); encoder.push(a); } + encoder.state.emitbyte(Tag.End); + } else { + encoder.push(this.item); } - encoder.push(this.item); } __preserve_text_on__(w: Writer): void { diff --git a/implementations/javascript/packages/core/src/constants.ts b/implementations/javascript/packages/core/src/constants.ts index d583885..0a02802 100644 --- a/implementations/javascript/packages/core/src/constants.ts +++ b/implementations/javascript/packages/core/src/constants.ts @@ -1,14 +1,9 @@ export enum Tag { False = 0x80, True, - Float, - Double, - End, - Annotation, - Embedded, - - SmallInteger_lo = 0x90, - MediumInteger_lo = 0xa0, + End = 0x84, + Embedded = 0x86, + Ieee754, SignedInteger = 0xb0, String, @@ -18,4 +13,6 @@ export enum Tag { Sequence, Set, Dictionary, + + Annotated = 0xbf, } diff --git a/implementations/javascript/packages/core/src/decoder.ts b/implementations/javascript/packages/core/src/decoder.ts index 25b35e8..7381bb7 100644 --- a/implementations/javascript/packages/core/src/decoder.ts +++ b/implementations/javascript/packages/core/src/decoder.ts @@ -22,7 +22,7 @@ export interface TypedDecoder { atEnd(): boolean; mark(): any; - restoreMark(m: any): void; + restoreMark(m: any): undefined; skip(): void; next(): Value; @@ -85,8 +85,9 @@ export class DecoderState { return this.index; } - restoreMark(m: number): void { + restoreMark(m: number): undefined { this.index = m; + return void 0; } shortGuard(body: () => R, short: () => R): R { @@ -138,28 +139,9 @@ export class DecoderState { return acc; } - nextSmallOrMediumInteger(tag: number): number | undefined { - if (tag >= Tag.SmallInteger_lo && tag <= Tag.SmallInteger_lo + 15) { - const v = tag - Tag.SmallInteger_lo; - return v > 12 ? v - 16 : v; - } - if (tag >= Tag.MediumInteger_lo && tag <= Tag.MediumInteger_lo + 15) { - const n = tag - Tag.MediumInteger_lo; - return this.nextint(n + 1); - } - return void 0; - } - wrap(v: Value): Value { return this.includeAnnotations ? new Annotated(v) : v; } - - unshiftAnnotation(a: Value, v: Annotated): Annotated { - if (this.includeAnnotations) { - v.annotations.unshift(a); - } - return v; - } } export const neverEmbeddedTypeDecode: EmbeddedTypeDecode = { @@ -216,15 +198,14 @@ export class Decoder implements TypedDecoder { switch (tag) { case Tag.False: return this.state.wrap(false); case Tag.True: return this.state.wrap(true); - case Tag.Float: return this.state.wrap(SingleFloat.fromBytes(this.state.nextbytes(4))); - case Tag.Double: return this.state.wrap(DoubleFloat.fromBytes(this.state.nextbytes(8))); case Tag.End: throw new DecodeError("Unexpected Compound end marker"); - case Tag.Annotation: { - const a = this.next(); - const v = this.next() as Annotated; - return this.state.unshiftAnnotation(a, v); - } case Tag.Embedded: return this.state.wrap(embed(this.embeddedDecode.decode(this.state))); + case Tag.Ieee754: + switch (this.state.varint()) { + case 4: return this.state.wrap(SingleFloat.fromBytes(this.state.nextbytes(4))); + case 8: return this.state.wrap(DoubleFloat.fromBytes(this.state.nextbytes(8))); + default: throw new DecodeError("Invalid IEEE754 size"); + } case Tag.SignedInteger: return this.state.wrap(this.state.nextint(this.state.varint())); case Tag.String: return this.state.wrap(Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8()); case Tag.ByteString: return this.state.wrap(Bytes.from(this.state.nextbytes(this.state.varint()))); @@ -237,13 +218,18 @@ export class Decoder implements TypedDecoder { case Tag.Sequence: return this.state.wrap(this.nextvalues()); case Tag.Set: return this.state.wrap(new Set(this.nextvalues())); case Tag.Dictionary: return this.state.wrap(Decoder.dictionaryFromArray(this.nextvalues())); - default: { - const v = this.state.nextSmallOrMediumInteger(tag); - if (v === void 0) { - throw new DecodeError("Unsupported Preserves tag: " + tag); + case Tag.Annotated: { + const vs = this.nextvalues(); + if (vs.length < 2) throw new DecodeError("Too few items in annotated value"); + if (this.state.includeAnnotations) { + const a = vs.shift()! as Annotated; + a.annotations.push(... vs); + return a; + } else { + return vs[0]; } - return this.state.wrap(v); } + default: throw new DecodeError("Unsupported Preserves tag: " + tag); } } @@ -259,8 +245,8 @@ export class Decoder implements TypedDecoder { return this.state.mark(); } - restoreMark(m: any): void { - this.state.restoreMark(m); + restoreMark(m: any): undefined { + return this.state.restoreMark(m); } skip(): void { @@ -275,99 +261,109 @@ export class Decoder implements TypedDecoder { return body(new Decoder(this.state, embeddedDecode)); } - skipAnnotations(): void { - if (!this.state.atEnd() && this.state.packet[this.state.index] === Tag.Annotation) { + skipAnnotations(f: (reset: () => undefined) => R): R { + const m = this.mark(); + if (!this.state.atEnd() && this.state.packet[this.state.index] === Tag.Annotated) { this.state.index++; - this.skip(); + const result = f(() => this.restoreMark(m)); + while (!this.state.peekend()) this.skip(); + return result; + } else { + return f(() => this.restoreMark(m)); } } nextBoolean(): boolean | undefined { - this.skipAnnotations(); - switch (this.state.nextbyte()) { - case Tag.False: return false; - case Tag.True: return true; - default: return void 0; - } + return this.skipAnnotations((reset) => { + switch (this.state.nextbyte()) { + case Tag.False: return false; + case Tag.True: return true; + default: return reset(); + } + }); } nextFloat(): SingleFloat | undefined { - this.skipAnnotations(); - switch (this.state.nextbyte()) { - case Tag.Float: return SingleFloat.fromBytes(this.state.nextbytes(4)); - default: return void 0; - } + return this.skipAnnotations((reset) => { + if (this.state.nextbyte() !== Tag.Ieee754) return reset(); + if (this.state.nextbyte() !== 4) return reset(); + return SingleFloat.fromBytes(this.state.nextbytes(4)); + }); } nextDouble(): DoubleFloat | undefined { - this.skipAnnotations(); - switch (this.state.nextbyte()) { - case Tag.Double: return DoubleFloat.fromBytes(this.state.nextbytes(8)); - default: return void 0; - } + return this.skipAnnotations((reset) => { + if (this.state.nextbyte() !== Tag.Ieee754) return reset(); + if (this.state.nextbyte() !== 8) return reset(); + return DoubleFloat.fromBytes(this.state.nextbytes(8)); + }); } nextEmbedded(): Embedded | undefined { - this.skipAnnotations(); - switch (this.state.nextbyte()) { - case Tag.Embedded: return embed(this.embeddedDecode.decode(this.state)); - default: return void 0; - } + return this.skipAnnotations((reset) => { + switch (this.state.nextbyte()) { + case Tag.Embedded: return embed(this.embeddedDecode.decode(this.state)); + default: return reset(); + } + }); } nextSignedInteger(): number | undefined { - this.skipAnnotations(); - const b = this.state.nextbyte(); - switch (b) { - case Tag.SignedInteger: return this.state.nextint(this.state.varint()); - default: return this.state.nextSmallOrMediumInteger(b); - } + return this.skipAnnotations((reset) => { + switch (this.state.nextbyte()) { + case Tag.SignedInteger: return this.state.nextint(this.state.varint()); + default: return reset(); + } + }); } nextString(): string | undefined { - this.skipAnnotations(); - switch (this.state.nextbyte()) { - case Tag.String: return Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8(); - default: return void 0; - } + return this.skipAnnotations((reset) => { + switch (this.state.nextbyte()) { + case Tag.String: return Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8(); + default: return reset(); + } + }); } nextByteString(): Bytes | undefined { - this.skipAnnotations(); - switch (this.state.nextbyte()) { - case Tag.ByteString: return Bytes.from(this.state.nextbytes(this.state.varint())); - default: return void 0; - } + return this.skipAnnotations((reset) => { + switch (this.state.nextbyte()) { + case Tag.ByteString: return Bytes.from(this.state.nextbytes(this.state.varint())); + default: return reset(); + } + }); } nextSymbol(): symbol | undefined { - this.skipAnnotations(); - switch (this.state.nextbyte()) { - case Tag.Symbol: - return Symbol.for(Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8()); - default: - return void 0; - } + return this.skipAnnotations((reset) => { + switch (this.state.nextbyte()) { + case Tag.Symbol: + return Symbol.for(Bytes.from(this.state.nextbytes(this.state.varint())).fromUtf8()); + default: + return reset(); + } + }); } openRecord(): boolean { - this.skipAnnotations(); - return (this.state.nextbyte() === Tag.Record) || (this.state.index--, false); + return this.skipAnnotations((reset) => + (this.state.nextbyte() === Tag.Record) || (reset(), false)); } openSequence(): boolean { - this.skipAnnotations(); - return (this.state.nextbyte() === Tag.Sequence) || (this.state.index--, false); + return this.skipAnnotations((reset) => + (this.state.nextbyte() === Tag.Sequence) || (reset(), false)); } openSet(): boolean { - this.skipAnnotations(); - return (this.state.nextbyte() === Tag.Set) || (this.state.index--, false); + return this.skipAnnotations((reset) => + (this.state.nextbyte() === Tag.Set) || (reset(), false)); } openDictionary(): boolean { - this.skipAnnotations(); - return (this.state.nextbyte() === Tag.Dictionary) || (this.state.index--, false); + return this.skipAnnotations((reset) => + (this.state.nextbyte() === Tag.Dictionary) || (reset(), false)); } closeCompound(): boolean { diff --git a/implementations/javascript/packages/core/src/encoder.ts b/implementations/javascript/packages/core/src/encoder.ts index 011fa56..92b9a8c 100644 --- a/implementations/javascript/packages/core/src/encoder.ts +++ b/implementations/javascript/packages/core/src/encoder.ts @@ -132,15 +132,18 @@ export class EncoderState { encodeint(v: number) { // TODO: Bignums :-/ - const plain_bitcount = Math.floor(Math.log2(v > 0 ? v : -(1 + v))) + 1; + this.emitbyte(Tag.SignedInteger); + + if (v === 0) { + this.emitbyte(0); + return; + } + + const plain_bitcount = v === -1 ? 0 : Math.floor(Math.log2(v > 0 ? v : -(1 + v))) + 1; const signed_bitcount = plain_bitcount + 1; const bytecount = (signed_bitcount + 7) >> 3; - if (bytecount <= 16) { - this.emitbyte(Tag.MediumInteger_lo + bytecount - 1); - } else { - this.emitbyte(Tag.SignedInteger); - this.varint(bytecount); - } + this.varint(bytecount); + const enc = (n: number, x: number) => { if (n > 0) { enc(n - 1, Math.floor(x / 256)); @@ -217,11 +220,7 @@ export class Encoder { this.state.emitbyte(v ? Tag.True : Tag.False); } else if (typeof v === 'number') { - if (v >= -3 && v <= 12) { - this.state.emitbyte(Tag.SmallInteger_lo + ((v + 16) & 0xf)); - } else { - this.state.encodeint(v); - } + this.state.encodeint(v); } else if (typeof v === 'string') { this.state.encodebytes(Tag.String, new Bytes(v)._view); diff --git a/implementations/javascript/packages/core/src/float.ts b/implementations/javascript/packages/core/src/float.ts index a95e943..4ef7283 100644 --- a/implementations/javascript/packages/core/src/float.ts +++ b/implementations/javascript/packages/core/src/float.ts @@ -100,7 +100,8 @@ export class SingleFloat extends Float implements Preservable, PreserveWrit } __preserve_on__(encoder: Encoder) { - encoder.state.emitbyte(Tag.Float); + encoder.state.emitbyte(Tag.Ieee754); + encoder.state.emitbyte(4); encoder.state.makeroom(4); this.__w(encoder.state.view, encoder.state.index); encoder.state.index += 4; @@ -143,7 +144,8 @@ export class DoubleFloat extends Float implements Preservable, PreserveWrit } __preserve_on__(encoder: Encoder) { - encoder.state.emitbyte(Tag.Double); + encoder.state.emitbyte(Tag.Ieee754); + encoder.state.emitbyte(8); encoder.state.makeroom(8); encoder.state.view.setFloat64(encoder.state.index, this.value, false); encoder.state.index += 8; diff --git a/implementations/javascript/packages/core/test/codec.test.ts b/implementations/javascript/packages/core/test/codec.test.ts index fe99f9c..bbf5130 100644 --- a/implementations/javascript/packages/core/test/codec.test.ts +++ b/implementations/javascript/packages/core/test/codec.test.ts @@ -80,7 +80,7 @@ describe('reusing buffer space', () => { `b5 b103616161 b7 - b10161 91 + b10161 b00101 84 b1037a7a7a 84`.replace(/\s+/g, '')); @@ -137,7 +137,7 @@ describe('encoding and decoding embeddeds', () => { expect(bs1).is(bs3); }); it('should refuse to decode embeddeds when no function has been supplied', () => { - expect(() => decode(Bytes.from([Tag.Embedded, Tag.SmallInteger_lo]))) + expect(() => decode(Bytes.from([Tag.Embedded, Tag.False]))) .toThrow("Embeddeds not permitted at this point in Preserves document"); }); it('should encode properly', () => { @@ -147,8 +147,8 @@ describe('encoding and decoding embeddeds', () => { const B = embed({b: 2}); expect(encode([A, B], { embeddedEncode: pt })).is( Bytes.from([Tag.Sequence, - Tag.Embedded, Tag.SmallInteger_lo, - Tag.Embedded, Tag.SmallInteger_lo + 1, + Tag.Embedded, Tag.SignedInteger, 0, + Tag.Embedded, Tag.SignedInteger, 1, 1, Tag.End])); expect(objects).toEqual([A.embeddedValue, B.embeddedValue]); }); @@ -161,8 +161,8 @@ describe('encoding and decoding embeddeds', () => { objects.push(Y.embeddedValue); expect(decode(Bytes.from([ Tag.Sequence, - Tag.Embedded, Tag.SmallInteger_lo, - Tag.Embedded, Tag.SmallInteger_lo + 1, + Tag.Embedded, Tag.SignedInteger, 0, + Tag.Embedded, Tag.SignedInteger, 1, 1, Tag.End ]), { embeddedDecode: pt })).is([X, Y]); }); diff --git a/implementations/javascript/packages/schema/src/gen/schema.ts b/implementations/javascript/packages/schema/src/gen/schema.ts index 9f056c0..1a33182 100644 --- a/implementations/javascript/packages/schema/src/gen/schema.ts +++ b/implementations/javascript/packages/schema/src/gen/schema.ts @@ -34,7 +34,7 @@ let __schema: _.Value | null = null; export function _schema() { if (__schema === null) { - __schema = _.decode<_.GenericEmbedded>(_.Bytes.fromHex("b4b306736368656d61b7b30776657273696f6e91b30b646566696e6974696f6e73b7b303526566b4b303726563b4b3036c6974b30372656684b4b3057475706c65b5b4b3056e616d6564b3066d6f64756c65b4b303726566b584b30a4d6f64756c65506174688484b4b3056e616d6564b3046e616d65b4b30461746f6db30653796d626f6c8484848484b30642756e646c65b4b303726563b4b3036c6974b30662756e646c6584b4b3057475706c65b5b4b3056e616d6564b3076d6f64756c6573b4b303726566b584b3074d6f64756c65738484848484b306536368656d61b4b303726563b4b3036c6974b306736368656d6184b4b3057475706c65b5b4b30464696374b7b30776657273696f6eb4b3056e616d6564b30776657273696f6eb4b303726566b584b30756657273696f6e8484b30b646566696e6974696f6e73b4b3056e616d6564b30b646566696e6974696f6e73b4b303726566b584b30b446566696e6974696f6e738484b30c656d62656464656454797065b4b3056e616d6564b30c656d62656464656454797065b4b303726566b584b310456d626564646564547970654e616d6584848484848484b30742696e64696e67b4b303726563b4b3036c6974b3056e616d656484b4b3057475706c65b5b4b3056e616d6564b3046e616d65b4b30461746f6db30653796d626f6c8484b4b3056e616d6564b3077061747465726eb4b303726566b584b30d53696d706c655061747465726e8484848484b3074d6f64756c6573b4b306646963746f66b4b303726566b584b30a4d6f64756c655061746884b4b303726566b584b306536368656d618484b3075061747465726eb4b3026f72b5b5b10d53696d706c655061747465726eb4b303726566b584b30d53696d706c655061747465726e8484b5b10f436f6d706f756e645061747465726eb4b303726566b584b30f436f6d706f756e645061747465726e84848484b30756657273696f6eb4b3036c69749184b30841746f6d4b696e64b4b3026f72b5b5b107426f6f6c65616eb4b3036c6974b307426f6f6c65616e8484b5b105466c6f6174b4b3036c6974b305466c6f61748484b5b106446f75626c65b4b3036c6974b306446f75626c658484b5b10d5369676e6564496e7465676572b4b3036c6974b30d5369676e6564496e74656765728484b5b106537472696e67b4b3036c6974b306537472696e678484b5b10a42797465537472696e67b4b3036c6974b30a42797465537472696e678484b5b10653796d626f6cb4b3036c6974b30653796d626f6c84848484b30a446566696e6974696f6eb4b3026f72b5b5b1026f72b4b303726563b4b3036c6974b3026f7284b4b3057475706c65b5b4b30b7475706c65507265666978b5b4b3056e616d6564b3087061747465726e30b4b303726566b584b3104e616d6564416c7465726e61746976658484b4b3056e616d6564b3087061747465726e31b4b303726566b584b3104e616d6564416c7465726e6174697665848484b4b3056e616d6564b3087061747465726e4eb4b3057365716f66b4b303726566b584b3104e616d6564416c7465726e61746976658484848484848484b5b103616e64b4b303726563b4b3036c6974b303616e6484b4b3057475706c65b5b4b30b7475706c65507265666978b5b4b3056e616d6564b3087061747465726e30b4b303726566b584b30c4e616d65645061747465726e8484b4b3056e616d6564b3087061747465726e31b4b303726566b584b30c4e616d65645061747465726e848484b4b3056e616d6564b3087061747465726e4eb4b3057365716f66b4b303726566b584b30c4e616d65645061747465726e8484848484848484b5b1075061747465726eb4b303726566b584b3075061747465726e84848484b30a4d6f64756c6550617468b4b3057365716f66b4b30461746f6db30653796d626f6c8484b30b446566696e6974696f6e73b4b306646963746f66b4b30461746f6db30653796d626f6c84b4b303726566b584b30a446566696e6974696f6e8484b30c4e616d65645061747465726eb4b3026f72b5b5b1056e616d6564b4b303726566b584b30742696e64696e678484b5b109616e6f6e796d6f7573b4b303726566b584b3075061747465726e84848484b30d53696d706c655061747465726eb4b3026f72b5b5b103616e79b4b3036c6974b303616e798484b5b10461746f6db4b303726563b4b3036c6974b30461746f6d84b4b3057475706c65b5b4b3056e616d6564b30861746f6d4b696e64b4b303726566b584b30841746f6d4b696e64848484848484b5b108656d626564646564b4b303726563b4b3036c6974b308656d62656464656484b4b3057475706c65b5b4b3056e616d6564b309696e74657266616365b4b303726566b584b30d53696d706c655061747465726e848484848484b5b1036c6974b4b303726563b4b3036c6974b3036c697484b4b3057475706c65b5b4b3056e616d6564b30576616c7565b303616e798484848484b5b1057365716f66b4b303726563b4b3036c6974b3057365716f6684b4b3057475706c65b5b4b3056e616d6564b3077061747465726eb4b303726566b584b30d53696d706c655061747465726e848484848484b5b1057365746f66b4b303726563b4b3036c6974b3057365746f6684b4b3057475706c65b5b4b3056e616d6564b3077061747465726eb4b303726566b584b30d53696d706c655061747465726e848484848484b5b106646963746f66b4b303726563b4b3036c6974b306646963746f6684b4b3057475706c65b5b4b3056e616d6564b3036b6579b4b303726566b584b30d53696d706c655061747465726e8484b4b3056e616d6564b30576616c7565b4b303726566b584b30d53696d706c655061747465726e848484848484b5b103526566b4b303726566b584b30352656684848484b30f436f6d706f756e645061747465726eb4b3026f72b5b5b103726563b4b303726563b4b3036c6974b30372656384b4b3057475706c65b5b4b3056e616d6564b3056c6162656cb4b303726566b584b30c4e616d65645061747465726e8484b4b3056e616d6564b3066669656c6473b4b303726566b584b30c4e616d65645061747465726e848484848484b5b1057475706c65b4b303726563b4b3036c6974b3057475706c6584b4b3057475706c65b5b4b3056e616d6564b3087061747465726e73b4b3057365716f66b4b303726566b584b30c4e616d65645061747465726e84848484848484b5b10b7475706c65507265666978b4b303726563b4b3036c6974b30b7475706c6550726566697884b4b3057475706c65b5b4b3056e616d6564b3056669786564b4b3057365716f66b4b303726566b584b30c4e616d65645061747465726e848484b4b3056e616d6564b3087661726961626c65b4b303726566b584b3124e616d656453696d706c655061747465726e848484848484b5b10464696374b4b303726563b4b3036c6974b3046469637484b4b3057475706c65b5b4b3056e616d6564b307656e7472696573b4b303726566b584b31144696374696f6e617279456e74726965738484848484848484b310456d626564646564547970654e616d65b4b3026f72b5b5b10566616c7365b4b3036c6974808484b5b103526566b4b303726566b584b30352656684848484b3104e616d6564416c7465726e6174697665b4b3057475706c65b5b4b3056e616d6564b30c76617269616e744c6162656cb4b30461746f6db306537472696e678484b4b3056e616d6564b3077061747465726eb4b303726566b584b3075061747465726e84848484b31144696374696f6e617279456e7472696573b4b306646963746f66b303616e79b4b303726566b584b3124e616d656453696d706c655061747465726e8484b3124e616d656453696d706c655061747465726eb4b3026f72b5b5b1056e616d6564b4b303726566b584b30742696e64696e678484b5b109616e6f6e796d6f7573b4b303726566b584b30d53696d706c655061747465726e8484848484b30c656d62656464656454797065808484")); + __schema = _.decode<_.GenericEmbedded>(_.Bytes.fromHex("b4b306736368656d61b7b30776657273696f6eb00101b30b646566696e6974696f6e73b7b303526566b4b303726563b4b3036c6974b30372656684b4b3057475706c65b5b4b3056e616d6564b3066d6f64756c65b4b303726566b584b30a4d6f64756c65506174688484b4b3056e616d6564b3046e616d65b4b30461746f6db30653796d626f6c8484848484b30642756e646c65b4b303726563b4b3036c6974b30662756e646c6584b4b3057475706c65b5b4b3056e616d6564b3076d6f64756c6573b4b303726566b584b3074d6f64756c65738484848484b306536368656d61b4b303726563b4b3036c6974b306736368656d6184b4b3057475706c65b5b4b30464696374b7b30776657273696f6eb4b3056e616d6564b30776657273696f6eb4b303726566b584b30756657273696f6e8484b30b646566696e6974696f6e73b4b3056e616d6564b30b646566696e6974696f6e73b4b303726566b584b30b446566696e6974696f6e738484b30c656d62656464656454797065b4b3056e616d6564b30c656d62656464656454797065b4b303726566b584b310456d626564646564547970654e616d6584848484848484b30742696e64696e67b4b303726563b4b3036c6974b3056e616d656484b4b3057475706c65b5b4b3056e616d6564b3046e616d65b4b30461746f6db30653796d626f6c8484b4b3056e616d6564b3077061747465726eb4b303726566b584b30d53696d706c655061747465726e8484848484b3074d6f64756c6573b4b306646963746f66b4b303726566b584b30a4d6f64756c655061746884b4b303726566b584b306536368656d618484b3075061747465726eb4b3026f72b5b5b10d53696d706c655061747465726eb4b303726566b584b30d53696d706c655061747465726e8484b5b10f436f6d706f756e645061747465726eb4b303726566b584b30f436f6d706f756e645061747465726e84848484b30756657273696f6eb4b3036c6974b0010184b30841746f6d4b696e64b4b3026f72b5b5b107426f6f6c65616eb4b3036c6974b307426f6f6c65616e8484b5b105466c6f6174b4b3036c6974b305466c6f61748484b5b106446f75626c65b4b3036c6974b306446f75626c658484b5b10d5369676e6564496e7465676572b4b3036c6974b30d5369676e6564496e74656765728484b5b106537472696e67b4b3036c6974b306537472696e678484b5b10a42797465537472696e67b4b3036c6974b30a42797465537472696e678484b5b10653796d626f6cb4b3036c6974b30653796d626f6c84848484b30a446566696e6974696f6eb4b3026f72b5b5b1026f72b4b303726563b4b3036c6974b3026f7284b4b3057475706c65b5b4b30b7475706c65507265666978b5b4b3056e616d6564b3087061747465726e30b4b303726566b584b3104e616d6564416c7465726e61746976658484b4b3056e616d6564b3087061747465726e31b4b303726566b584b3104e616d6564416c7465726e6174697665848484b4b3056e616d6564b3087061747465726e4eb4b3057365716f66b4b303726566b584b3104e616d6564416c7465726e61746976658484848484848484b5b103616e64b4b303726563b4b3036c6974b303616e6484b4b3057475706c65b5b4b30b7475706c65507265666978b5b4b3056e616d6564b3087061747465726e30b4b303726566b584b30c4e616d65645061747465726e8484b4b3056e616d6564b3087061747465726e31b4b303726566b584b30c4e616d65645061747465726e848484b4b3056e616d6564b3087061747465726e4eb4b3057365716f66b4b303726566b584b30c4e616d65645061747465726e8484848484848484b5b1075061747465726eb4b303726566b584b3075061747465726e84848484b30a4d6f64756c6550617468b4b3057365716f66b4b30461746f6db30653796d626f6c8484b30b446566696e6974696f6e73b4b306646963746f66b4b30461746f6db30653796d626f6c84b4b303726566b584b30a446566696e6974696f6e8484b30c4e616d65645061747465726eb4b3026f72b5b5b1056e616d6564b4b303726566b584b30742696e64696e678484b5b109616e6f6e796d6f7573b4b303726566b584b3075061747465726e84848484b30d53696d706c655061747465726eb4b3026f72b5b5b103616e79b4b3036c6974b303616e798484b5b10461746f6db4b303726563b4b3036c6974b30461746f6d84b4b3057475706c65b5b4b3056e616d6564b30861746f6d4b696e64b4b303726566b584b30841746f6d4b696e64848484848484b5b108656d626564646564b4b303726563b4b3036c6974b308656d62656464656484b4b3057475706c65b5b4b3056e616d6564b309696e74657266616365b4b303726566b584b30d53696d706c655061747465726e848484848484b5b1036c6974b4b303726563b4b3036c6974b3036c697484b4b3057475706c65b5b4b3056e616d6564b30576616c7565b303616e798484848484b5b1057365716f66b4b303726563b4b3036c6974b3057365716f6684b4b3057475706c65b5b4b3056e616d6564b3077061747465726eb4b303726566b584b30d53696d706c655061747465726e848484848484b5b1057365746f66b4b303726563b4b3036c6974b3057365746f6684b4b3057475706c65b5b4b3056e616d6564b3077061747465726eb4b303726566b584b30d53696d706c655061747465726e848484848484b5b106646963746f66b4b303726563b4b3036c6974b306646963746f6684b4b3057475706c65b5b4b3056e616d6564b3036b6579b4b303726566b584b30d53696d706c655061747465726e8484b4b3056e616d6564b30576616c7565b4b303726566b584b30d53696d706c655061747465726e848484848484b5b103526566b4b303726566b584b30352656684848484b30f436f6d706f756e645061747465726eb4b3026f72b5b5b103726563b4b303726563b4b3036c6974b30372656384b4b3057475706c65b5b4b3056e616d6564b3056c6162656cb4b303726566b584b30c4e616d65645061747465726e8484b4b3056e616d6564b3066669656c6473b4b303726566b584b30c4e616d65645061747465726e848484848484b5b1057475706c65b4b303726563b4b3036c6974b3057475706c6584b4b3057475706c65b5b4b3056e616d6564b3087061747465726e73b4b3057365716f66b4b303726566b584b30c4e616d65645061747465726e84848484848484b5b10b7475706c65507265666978b4b303726563b4b3036c6974b30b7475706c6550726566697884b4b3057475706c65b5b4b3056e616d6564b3056669786564b4b3057365716f66b4b303726566b584b30c4e616d65645061747465726e848484b4b3056e616d6564b3087661726961626c65b4b303726566b584b3124e616d656453696d706c655061747465726e848484848484b5b10464696374b4b303726563b4b3036c6974b3046469637484b4b3057475706c65b5b4b3056e616d6564b307656e7472696573b4b303726566b584b31144696374696f6e617279456e74726965738484848484848484b310456d626564646564547970654e616d65b4b3026f72b5b5b10566616c7365b4b3036c6974808484b5b103526566b4b303726566b584b30352656684848484b3104e616d6564416c7465726e6174697665b4b3057475706c65b5b4b3056e616d6564b30c76617269616e744c6162656cb4b30461746f6db306537472696e678484b4b3056e616d6564b3077061747465726eb4b303726566b584b3075061747465726e84848484b31144696374696f6e617279456e7472696573b4b306646963746f66b303616e79b4b303726566b584b3124e616d656453696d706c655061747465726e8484b3124e616d656453696d706c655061747465726eb4b3026f72b5b5b1056e616d6564b4b303726566b584b30742696e64696e678484b5b109616e6f6e796d6f7573b4b303726566b584b30d53696d706c655061747465726e8484848484b30c656d62656464656454797065808484")); }; return __schema; } diff --git a/implementations/javascript/yarn.lock b/implementations/javascript/yarn.lock index 96ab76a..9840f4c 100644 --- a/implementations/javascript/yarn.lock +++ b/implementations/javascript/yarn.lock @@ -291,6 +291,13 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@cspotcode/source-map-support@^0.8.0": + version "0.8.1" + resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" + integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw== + dependencies: + "@jridgewell/trace-mapping" "0.3.9" + "@gar/promisify@^1.0.1": version "1.1.3" resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" @@ -508,6 +515,11 @@ resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== +"@jridgewell/resolve-uri@^3.0.3": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" + integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA== + "@jridgewell/set-array@^1.0.0", "@jridgewell/set-array@^1.0.1": version "1.1.2" resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" @@ -526,6 +538,14 @@ resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz#add4c98d341472a289190b424efbdb096991bb24" integrity sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw== +"@jridgewell/trace-mapping@0.3.9": + version "0.3.9" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9" + integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ== + dependencies: + "@jridgewell/resolve-uri" "^3.0.3" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping@^0.3.9": version "0.3.17" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.17.tgz#793041277af9073b0951a7fe0f0d8c4c98c36985" @@ -1425,6 +1445,26 @@ resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== +"@tsconfig/node10@^1.0.7": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2" + integrity sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA== + +"@tsconfig/node12@^1.0.7": + version "1.0.11" + resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.11.tgz#ee3def1f27d9ed66dac6e46a295cffb0152e058d" + integrity sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag== + +"@tsconfig/node14@^1.0.0": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.3.tgz#e4386316284f00b98435bf40f72f75a09dabf6c1" + integrity sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow== + +"@tsconfig/node16@^1.0.2": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9" + integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA== + "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14": version "7.20.0" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.0.tgz#61bc5a4cae505ce98e1e36c5445e4bee060d8891" @@ -1520,6 +1560,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f" integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA== +"@types/node@^20.8.5": + version "20.8.5" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.5.tgz#13352ae1f80032171616910e8aba2e3e52e57d96" + integrity sha512-SPlobFgbidfIeOYlzXiEjSYeIJiOCthv+9tSQVpvk4PAdIIc+2SmjNVzWXk9t0Y7dl73Zdf+OgXKHX9XtkqUpw== + dependencies: + undici-types "~5.25.1" + "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" @@ -1593,6 +1640,11 @@ acorn-walk@^7.1.1: resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== +acorn-walk@^8.1.1: + version "8.2.0" + resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" + integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== + acorn@^7.1.1: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" @@ -1603,6 +1655,11 @@ acorn@^8.2.4, acorn@^8.5.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== +acorn@^8.4.1: + version "8.10.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.10.0.tgz#8be5b3907a67221a81ab23c7889c4c5526b62ec5" + integrity sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw== + add-stream@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/add-stream/-/add-stream-1.0.0.tgz#6a7990437ca736d5e1288db92bd3266d5f5cb2aa" @@ -5720,7 +5777,7 @@ sort-keys@^4.0.0: dependencies: is-plain-obj "^2.0.0" -source-map-support@^0.5.12, source-map-support@^0.5.17, source-map-support@^0.5.6, source-map-support@~0.5.20: +source-map-support@^0.5.12, source-map-support@^0.5.6, source-map-support@~0.5.20: version "0.5.21" resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== @@ -6150,32 +6207,39 @@ ts-jest@^27.0: semver "7.x" yargs-parser "20.x" -ts-node-dev@^1.1: - version "1.1.8" - resolved "https://registry.yarnpkg.com/ts-node-dev/-/ts-node-dev-1.1.8.tgz#95520d8ab9d45fffa854d6668e2f8f9286241066" - integrity sha512-Q/m3vEwzYwLZKmV6/0VlFxcZzVV/xcgOt+Tx/VjaaRHyiBcFlV0541yrT09QjzzCxlDZ34OzKjrFAynlmtflEg== +ts-node-dev@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ts-node-dev/-/ts-node-dev-2.0.0.tgz#bdd53e17ab3b5d822ef519928dc6b4a7e0f13065" + integrity sha512-ywMrhCfH6M75yftYvrvNarLEY+SUXtUvU8/0Z6llrHQVBx12GiFk5sStF8UdfE/yfzk9IAq7O5EEbTQsxlBI8w== dependencies: chokidar "^3.5.1" dynamic-dedupe "^0.3.0" - minimist "^1.2.5" + minimist "^1.2.6" mkdirp "^1.0.4" resolve "^1.0.0" rimraf "^2.6.1" source-map-support "^0.5.12" tree-kill "^1.2.2" - ts-node "^9.0.0" + ts-node "^10.4.0" tsconfig "^7.0.0" -ts-node@^9.0.0: - version "9.1.1" - resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-9.1.1.tgz#51a9a450a3e959401bda5f004a72d54b936d376d" - integrity sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg== +ts-node@^10.4.0: + version "10.9.1" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b" + integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw== dependencies: + "@cspotcode/source-map-support" "^0.8.0" + "@tsconfig/node10" "^1.0.7" + "@tsconfig/node12" "^1.0.7" + "@tsconfig/node14" "^1.0.0" + "@tsconfig/node16" "^1.0.2" + acorn "^8.4.1" + acorn-walk "^8.1.1" arg "^4.1.0" create-require "^1.1.0" diff "^4.0.1" make-error "^1.1.1" - source-map-support "^0.5.17" + v8-compile-cache-lib "^3.0.1" yn "3.1.1" tsconfig@^7.0.0: @@ -6298,6 +6362,11 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" +undici-types@~5.25.1: + version "5.25.3" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.25.3.tgz#e044115914c85f0bcbb229f346ab739f064998c3" + integrity sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA== + unique-filename@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230" @@ -6372,6 +6441,11 @@ uuid@^3.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== +v8-compile-cache-lib@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" + integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== + v8-to-istanbul@^8.1.0: version "8.1.1" resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-8.1.1.tgz#77b752fd3975e31bbcef938f85e9bd1c7a8d60ed" diff --git a/implementations/python/docs/syndicate-protocols-schema-bundle.bin b/implementations/python/docs/syndicate-protocols-schema-bundle.bin index edad452..3fe50c6 100644 --- a/implementations/python/docs/syndicate-protocols-schema-bundle.bin +++ b/implementations/python/docs/syndicate-protocols-schema-bundle.bin @@ -1,28 +1,28 @@ -´³bundle·µ³tcp„´³schema·³version‘³ definitions·³TcpLocal´³rec´³lit³ tcp-local„´³tupleµ´³named³host´³atom³String„„´³named³port´³atom³ SignedInteger„„„„„³ TcpRemote´³rec´³lit³ -tcp-remote„´³tupleµ´³named³host´³atom³String„„´³named³port´³atom³ SignedInteger„„„„„³ TcpPeerInfo´³rec´³lit³tcp-peer„´³tupleµ´³named³handle´³embedded³any„„´³named³local´³refµ„³TcpLocal„„´³named³remote´³refµ„³ TcpRemote„„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³http„´³schema·³version‘³ definitions·³Chunk´³orµµ±string´³atom³String„„µ±bytes´³atom³ +´³bundle·µ³tcp„´³schema·³version°³ definitions·³TcpLocal´³rec´³lit³ tcp-local„´³tupleµ´³named³host´³atom³String„„´³named³port´³atom³ SignedInteger„„„„„³ TcpRemote´³rec´³lit³ +tcp-remote„´³tupleµ´³named³host´³atom³String„„´³named³port´³atom³ SignedInteger„„„„„³ TcpPeerInfo´³rec´³lit³tcp-peer„´³tupleµ´³named³handle´³embedded³any„„´³named³local´³refµ„³TcpLocal„„´³named³remote´³refµ„³ TcpRemote„„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³http„´³schema·³version°³ definitions·³Chunk´³orµµ±string´³atom³String„„µ±bytes´³atom³ ByteString„„„„³Headers´³dictof´³atom³Symbol„´³atom³String„„³MimeType´³atom³Symbol„³ QueryValue´³orµµ±string´³atom³String„„µ±file´³rec´³lit³file„´³tupleµ´³named³filename´³atom³String„„´³named³headers´³refµ„³Headers„„´³named³body´³atom³ ByteString„„„„„„„„³ HostPattern´³orµµ±host´³atom³String„„µ±any´³lit€„„„„³ HttpBinding´³rec´³lit³ http-bind„´³tupleµ´³named³host´³refµ„³ HostPattern„„´³named³port´³atom³ SignedInteger„„´³named³method´³refµ„³ MethodPattern„„´³named³path´³refµ„³ PathPattern„„´³named³handler´³embedded´³refµ„³ HttpRequest„„„„„„³ HttpContext´³rec´³lit³request„´³tupleµ´³named³req´³refµ„³ HttpRequest„„´³named³res´³embedded´³refµ„³ HttpResponse„„„„„„³ HttpRequest´³rec´³lit³ http-request„´³tupleµ´³named³sequenceNumber´³atom³ SignedInteger„„´³named³host´³atom³String„„´³named³port´³atom³ SignedInteger„„´³named³method´³atom³Symbol„„´³named³path´³seqof´³atom³String„„„´³named³headers´³refµ„³Headers„„´³named³query´³dictof´³atom³Symbol„´³seqof´³refµ„³ QueryValue„„„„´³named³body´³refµ„³ RequestBody„„„„„³ HttpService´³rec´³lit³ http-service„´³tupleµ´³named³host´³refµ„³ HostPattern„„´³named³port´³atom³ SignedInteger„„´³named³method´³refµ„³ MethodPattern„„´³named³path´³refµ„³ PathPattern„„„„„³ PathPattern´³seqof´³refµ„³PathPatternElement„„³ RequestBody´³orµµ±present´³atom³ -ByteString„„µ±absent´³lit€„„„„³ HttpListener´³rec´³lit³ http-listener„´³tupleµ´³named³port´³atom³ SignedInteger„„„„„³ HttpResponse´³orµµ±status´³rec´³lit³status„´³tupleµ´³named³code´³atom³ SignedInteger„„´³named³message´³atom³String„„„„„„µ±header´³rec´³lit³header„´³tupleµ´³named³name´³atom³Symbol„„´³named³value´³atom³String„„„„„„µ±chunk´³rec´³lit³chunk„´³tupleµ´³named³chunk´³refµ„³Chunk„„„„„„µ±done´³rec´³lit³done„´³tupleµ´³named³chunk´³refµ„³Chunk„„„„„„„„³ MethodPattern´³orµµ±any´³lit€„„µ±specific´³atom³Symbol„„„„³PathPatternElement´³orµµ±label´³atom³String„„µ±wildcard´³lit³_„„µ±rest´³lit³...„„„„„³ embeddedType€„„µ³noise„´³schema·³version‘³ definitions·³Packet´³orµµ±complete´³atom³ +ByteString„„µ±absent´³lit€„„„„³ HttpListener´³rec´³lit³ http-listener„´³tupleµ´³named³port´³atom³ SignedInteger„„„„„³ HttpResponse´³orµµ±status´³rec´³lit³status„´³tupleµ´³named³code´³atom³ SignedInteger„„´³named³message´³atom³String„„„„„„µ±header´³rec´³lit³header„´³tupleµ´³named³name´³atom³Symbol„„´³named³value´³atom³String„„„„„„µ±chunk´³rec´³lit³chunk„´³tupleµ´³named³chunk´³refµ„³Chunk„„„„„„µ±done´³rec´³lit³done„´³tupleµ´³named³chunk´³refµ„³Chunk„„„„„„„„³ MethodPattern´³orµµ±any´³lit€„„µ±specific´³atom³Symbol„„„„³PathPatternElement´³orµµ±label´³atom³String„„µ±wildcard´³lit³_„„µ±rest´³lit³...„„„„„³ embeddedType€„„µ³noise„´³schema·³version°³ definitions·³Packet´³orµµ±complete´³atom³ ByteString„„µ± fragmented´³seqof´³atom³ ByteString„„„„„³ NoiseSpec´³andµ´³dict·³key´³named³key´³atom³ ByteString„„³service´³named³service´³refµ„³ServiceSelector„„„„´³named³protocol´³refµ„³ NoiseProtocol„„´³named³ preSharedKeys´³refµ„³NoisePreSharedKeys„„„„³ NoiseProtocol´³orµµ±present´³dict·³protocol´³named³protocol´³atom³String„„„„„µ±invalid´³dict·³protocol´³named³protocol³any„„„„µ±absent´³dict·„„„„„³ NoiseStepType´³lit³noise„³SecretKeyField´³orµµ±present´³dict·³ secretKey´³named³ secretKey´³atom³ ByteString„„„„„µ±invalid´³dict·³ secretKey´³named³ secretKey³any„„„„µ±absent´³dict·„„„„„³DefaultProtocol´³lit±!Noise_NK_25519_ChaChaPoly_BLAKE2s„³NoiseStepDetail´³refµ„³ServiceSelector„³ServiceSelector³any³NoiseServiceSpec´³andµ´³named³base´³refµ„³ NoiseSpec„„´³named³ secretKey´³refµ„³SecretKeyField„„„„³NoisePreSharedKeys´³orµµ±present´³dict·³ preSharedKeys´³named³ preSharedKeys´³seqof´³atom³ -ByteString„„„„„„µ±invalid´³dict·³ preSharedKeys´³named³ preSharedKeys³any„„„„µ±absent´³dict·„„„„„³NoisePathStepDetail´³refµ„³ NoiseSpec„³NoiseDescriptionDetail´³refµ„³NoiseServiceSpec„„³ embeddedType€„„µ³timer„´³schema·³version‘³ definitions·³SetTimer´³rec´³lit³ set-timer„´³tupleµ´³named³label³any„´³named³seconds´³atom³Double„„´³named³kind´³refµ„³ TimerKind„„„„„³ LaterThan´³rec´³lit³ -later-than„´³tupleµ´³named³seconds´³atom³Double„„„„„³ TimerKind´³orµµ±relative´³lit³relative„„µ±absolute´³lit³absolute„„µ±clear´³lit³clear„„„„³ TimerExpired´³rec´³lit³ timer-expired„´³tupleµ´³named³label³any„´³named³seconds´³atom³Double„„„„„„³ embeddedType€„„µ³trace„´³schema·³version‘³ definitions·³Oid³any³Name´³orµµ± anonymous´³rec´³lit³ anonymous„´³tupleµ„„„„µ±named´³rec´³lit³named„´³tupleµ´³named³name³any„„„„„„„³Target´³rec´³lit³entity„´³tupleµ´³named³actor´³refµ„³ActorId„„´³named³facet´³refµ„³FacetId„„´³named³oid´³refµ„³Oid„„„„„³TaskId³any³TurnId³any³ActorId³any³FacetId³any³ TurnCause´³orµµ±turn´³rec´³lit³ caused-by„´³tupleµ´³named³id´³refµ„³TurnId„„„„„„µ±cleanup´³rec´³lit³cleanup„´³tupleµ„„„„µ±linkedTaskRelease´³rec´³lit³linked-task-release„´³tupleµ´³named³id´³refµ„³TaskId„„´³named³reason´³refµ„³LinkedTaskReleaseReason„„„„„„µ±periodicActivation´³rec´³lit³periodic-activation„´³tupleµ´³named³period´³atom³Double„„„„„„µ±delay´³rec´³lit³delay„´³tupleµ´³named³ causingTurn´³refµ„³TurnId„„´³named³amount´³atom³Double„„„„„„µ±external´³rec´³lit³external„´³tupleµ´³named³ description³any„„„„„„„³ TurnEvent´³orµµ±assert´³rec´³lit³assert„´³tupleµ´³named³ assertion´³refµ„³AssertionDescription„„´³named³handle´³refµ³protocol„³Handle„„„„„„µ±retract´³rec´³lit³retract„´³tupleµ´³named³handle´³refµ³protocol„³Handle„„„„„„µ±message´³rec´³lit³message„´³tupleµ´³named³body´³refµ„³AssertionDescription„„„„„„µ±sync´³rec´³lit³sync„´³tupleµ´³named³peer´³refµ„³Target„„„„„„µ± breakLink´³rec´³lit³ +ByteString„„„„„„µ±invalid´³dict·³ preSharedKeys´³named³ preSharedKeys³any„„„„µ±absent´³dict·„„„„„³NoisePathStepDetail´³refµ„³ NoiseSpec„³NoiseDescriptionDetail´³refµ„³NoiseServiceSpec„„³ embeddedType€„„µ³timer„´³schema·³version°³ definitions·³SetTimer´³rec´³lit³ set-timer„´³tupleµ´³named³label³any„´³named³seconds´³atom³Double„„´³named³kind´³refµ„³ TimerKind„„„„„³ LaterThan´³rec´³lit³ +later-than„´³tupleµ´³named³seconds´³atom³Double„„„„„³ TimerKind´³orµµ±relative´³lit³relative„„µ±absolute´³lit³absolute„„µ±clear´³lit³clear„„„„³ TimerExpired´³rec´³lit³ timer-expired„´³tupleµ´³named³label³any„´³named³seconds´³atom³Double„„„„„„³ embeddedType€„„µ³trace„´³schema·³version°³ definitions·³Oid³any³Name´³orµµ± anonymous´³rec´³lit³ anonymous„´³tupleµ„„„„µ±named´³rec´³lit³named„´³tupleµ´³named³name³any„„„„„„„³Target´³rec´³lit³entity„´³tupleµ´³named³actor´³refµ„³ActorId„„´³named³facet´³refµ„³FacetId„„´³named³oid´³refµ„³Oid„„„„„³TaskId³any³TurnId³any³ActorId³any³FacetId³any³ TurnCause´³orµµ±turn´³rec´³lit³ caused-by„´³tupleµ´³named³id´³refµ„³TurnId„„„„„„µ±cleanup´³rec´³lit³cleanup„´³tupleµ„„„„µ±linkedTaskRelease´³rec´³lit³linked-task-release„´³tupleµ´³named³id´³refµ„³TaskId„„´³named³reason´³refµ„³LinkedTaskReleaseReason„„„„„„µ±periodicActivation´³rec´³lit³periodic-activation„´³tupleµ´³named³period´³atom³Double„„„„„„µ±delay´³rec´³lit³delay„´³tupleµ´³named³ causingTurn´³refµ„³TurnId„„´³named³amount´³atom³Double„„„„„„µ±external´³rec´³lit³external„´³tupleµ´³named³ description³any„„„„„„„³ TurnEvent´³orµµ±assert´³rec´³lit³assert„´³tupleµ´³named³ assertion´³refµ„³AssertionDescription„„´³named³handle´³refµ³protocol„³Handle„„„„„„µ±retract´³rec´³lit³retract„´³tupleµ´³named³handle´³refµ³protocol„³Handle„„„„„„µ±message´³rec´³lit³message„´³tupleµ´³named³body´³refµ„³AssertionDescription„„„„„„µ±sync´³rec´³lit³sync„´³tupleµ´³named³peer´³refµ„³Target„„„„„„µ± breakLink´³rec´³lit³ break-link„´³tupleµ´³named³source´³refµ„³ActorId„„´³named³handle´³refµ³protocol„³Handle„„„„„„„„³ ExitStatus´³orµµ±ok´³lit³ok„„µ±Error´³refµ³protocol„³Error„„„„³ TraceEntry´³rec´³lit³trace„´³tupleµ´³named³ timestamp´³atom³Double„„´³named³actor´³refµ„³ActorId„„´³named³item´³refµ„³ActorActivation„„„„„³ActorActivation´³orµµ±start´³rec´³lit³start„´³tupleµ´³named³ actorName´³refµ„³Name„„„„„„µ±turn´³refµ„³TurnDescription„„µ±stop´³rec´³lit³stop„´³tupleµ´³named³status´³refµ„³ ExitStatus„„„„„„„„³FacetStopReason´³orµµ±explicitAction´³lit³explicit-action„„µ±inert´³lit³inert„„µ±parentStopping´³lit³parent-stopping„„µ± actorStopping´³lit³actor-stopping„„„„³TurnDescription´³rec´³lit³turn„´³tupleµ´³named³id´³refµ„³TurnId„„´³named³cause´³refµ„³ TurnCause„„´³named³actions´³seqof´³refµ„³ActionDescription„„„„„„³ActionDescription´³orµµ±dequeue´³rec´³lit³dequeue„´³tupleµ´³named³event´³refµ„³TargetedTurnEvent„„„„„„µ±enqueue´³rec´³lit³enqueue„´³tupleµ´³named³event´³refµ„³TargetedTurnEvent„„„„„„µ±dequeueInternal´³rec´³lit³dequeue-internal„´³tupleµ´³named³event´³refµ„³TargetedTurnEvent„„„„„„µ±enqueueInternal´³rec´³lit³enqueue-internal„´³tupleµ´³named³event´³refµ„³TargetedTurnEvent„„„„„„µ±spawn´³rec´³lit³spawn„´³tupleµ´³named³link´³atom³Boolean„„´³named³id´³refµ„³ActorId„„„„„„µ±link´³rec´³lit³link„´³tupleµ´³named³ parentActor´³refµ„³ActorId„„´³named³ childToParent´³refµ³protocol„³Handle„„´³named³ childActor´³refµ„³ActorId„„´³named³ parentToChild´³refµ³protocol„³Handle„„„„„„µ± facetStart´³rec´³lit³ facet-start„´³tupleµ´³named³path´³seqof´³refµ„³FacetId„„„„„„„µ± facetStop´³rec´³lit³ -facet-stop„´³tupleµ´³named³path´³seqof´³refµ„³FacetId„„„´³named³reason´³refµ„³FacetStopReason„„„„„„µ±linkedTaskStart´³rec´³lit³linked-task-start„´³tupleµ´³named³taskName´³refµ„³Name„„´³named³id´³refµ„³TaskId„„„„„„„„³TargetedTurnEvent´³rec´³lit³event„´³tupleµ´³named³target´³refµ„³Target„„´³named³detail´³refµ„³ TurnEvent„„„„„³AssertionDescription´³orµµ±value´³rec´³lit³value„´³tupleµ´³named³value³any„„„„„µ±opaque´³rec´³lit³opaque„´³tupleµ´³named³ description³any„„„„„„„³LinkedTaskReleaseReason´³orµµ± cancelled´³lit³ cancelled„„µ±normal´³lit³normal„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³stream„´³schema·³version‘³ definitions·³Mode´³orµµ±bytes´³lit³bytes„„µ±lines´³refµ„³LineMode„„µ±packet´³rec´³lit³packet„´³tupleµ´³named³size´³atom³ SignedInteger„„„„„„µ±object´³rec´³lit³object„´³tupleµ´³named³ description³any„„„„„„„³Sink´³orµµ±source´³rec´³lit³source„´³tupleµ´³named³ +facet-stop„´³tupleµ´³named³path´³seqof´³refµ„³FacetId„„„´³named³reason´³refµ„³FacetStopReason„„„„„„µ±linkedTaskStart´³rec´³lit³linked-task-start„´³tupleµ´³named³taskName´³refµ„³Name„„´³named³id´³refµ„³TaskId„„„„„„„„³TargetedTurnEvent´³rec´³lit³event„´³tupleµ´³named³target´³refµ„³Target„„´³named³detail´³refµ„³ TurnEvent„„„„„³AssertionDescription´³orµµ±value´³rec´³lit³value„´³tupleµ´³named³value³any„„„„„µ±opaque´³rec´³lit³opaque„´³tupleµ´³named³ description³any„„„„„„„³LinkedTaskReleaseReason´³orµµ± cancelled´³lit³ cancelled„„µ±normal´³lit³normal„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³stream„´³schema·³version°³ definitions·³Mode´³orµµ±bytes´³lit³bytes„„µ±lines´³refµ„³LineMode„„µ±packet´³rec´³lit³packet„´³tupleµ´³named³size´³atom³ SignedInteger„„„„„„µ±object´³rec´³lit³object„´³tupleµ´³named³ description³any„„„„„„„³Sink´³orµµ±source´³rec´³lit³source„´³tupleµ´³named³ controller´³embedded´³refµ„³Source„„„„„„„µ± StreamError´³refµ„³ StreamError„„µ±data´³rec´³lit³data„´³tupleµ´³named³payload³any„´³named³mode´³refµ„³Mode„„„„„„µ±eof´³rec´³lit³eof„´³tupleµ„„„„„„³Source´³orµµ±sink´³rec´³lit³sink„´³tupleµ´³named³ -controller´³embedded´³refµ„³Sink„„„„„„„µ± StreamError´³refµ„³ StreamError„„µ±credit´³rec´³lit³credit„´³tupleµ´³named³amount´³refµ„³ CreditAmount„„´³named³mode´³refµ„³Mode„„„„„„„„³LineMode´³orµµ±lf´³lit³lf„„µ±crlf´³lit³crlf„„„„³ StreamError´³rec´³lit³error„´³tupleµ´³named³message´³atom³String„„„„„³ CreditAmount´³orµµ±count´³atom³ SignedInteger„„µ± unbounded´³lit³ unbounded„„„„³StreamConnection´³rec´³lit³stream-connection„´³tupleµ´³named³source´³embedded´³refµ„³Source„„„´³named³sink´³embedded´³refµ„³Sink„„„´³named³spec³any„„„„³StreamListenerError´³rec´³lit³stream-listener-error„´³tupleµ´³named³spec³any„´³named³message´³atom³String„„„„„³StreamListenerReady´³rec´³lit³stream-listener-ready„´³tupleµ´³named³spec³any„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³sturdy„´³schema·³version‘³ definitions·³Lit´³rec´³lit³lit„´³tupleµ´³named³value³any„„„„³Oid´³atom³ SignedInteger„³Alts´³rec´³lit³or„´³tupleµ´³named³ alternatives´³seqof´³refµ„³Rewrite„„„„„„³PAnd´³rec´³lit³and„´³tupleµ´³named³patterns´³seqof´³refµ„³Pattern„„„„„„³PNot´³rec´³lit³not„´³tupleµ´³named³pattern´³refµ„³Pattern„„„„„³TRef´³rec´³lit³ref„´³tupleµ´³named³binding´³atom³ SignedInteger„„„„„³PAtom´³orµµ±Boolean´³lit³Boolean„„µ±Float´³lit³Float„„µ±Double´³lit³Double„„µ± SignedInteger´³lit³ SignedInteger„„µ±String´³lit³String„„µ± +controller´³embedded´³refµ„³Sink„„„„„„„µ± StreamError´³refµ„³ StreamError„„µ±credit´³rec´³lit³credit„´³tupleµ´³named³amount´³refµ„³ CreditAmount„„´³named³mode´³refµ„³Mode„„„„„„„„³LineMode´³orµµ±lf´³lit³lf„„µ±crlf´³lit³crlf„„„„³ StreamError´³rec´³lit³error„´³tupleµ´³named³message´³atom³String„„„„„³ CreditAmount´³orµµ±count´³atom³ SignedInteger„„µ± unbounded´³lit³ unbounded„„„„³StreamConnection´³rec´³lit³stream-connection„´³tupleµ´³named³source´³embedded´³refµ„³Source„„„´³named³sink´³embedded´³refµ„³Sink„„„´³named³spec³any„„„„³StreamListenerError´³rec´³lit³stream-listener-error„´³tupleµ´³named³spec³any„´³named³message´³atom³String„„„„„³StreamListenerReady´³rec´³lit³stream-listener-ready„´³tupleµ´³named³spec³any„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³sturdy„´³schema·³version°³ definitions·³Lit´³rec´³lit³lit„´³tupleµ´³named³value³any„„„„³Oid´³atom³ SignedInteger„³Alts´³rec´³lit³or„´³tupleµ´³named³ alternatives´³seqof´³refµ„³Rewrite„„„„„„³PAnd´³rec´³lit³and„´³tupleµ´³named³patterns´³seqof´³refµ„³Pattern„„„„„„³PNot´³rec´³lit³not„´³tupleµ´³named³pattern´³refµ„³Pattern„„„„„³TRef´³rec´³lit³ref„´³tupleµ´³named³binding´³atom³ SignedInteger„„„„„³PAtom´³orµµ±Boolean´³lit³Boolean„„µ±Float´³lit³Float„„µ±Double´³lit³Double„„µ± SignedInteger´³lit³ SignedInteger„„µ±String´³lit³String„„µ± ByteString´³lit³ -ByteString„„µ±Symbol´³lit³Symbol„„„„³PBind´³rec´³lit³bind„´³tupleµ´³named³pattern´³refµ„³Pattern„„„„„³Caveat´³orµµ±Rewrite´³refµ„³Rewrite„„µ±Alts´³refµ„³Alts„„µ±Reject´³refµ„³Reject„„µ±unknown³any„„„³Reject´³rec´³lit³reject„´³tupleµ´³named³pattern´³refµ„³Pattern„„„„„³Pattern´³orµµ±PDiscard´³refµ„³PDiscard„„µ±PAtom´³refµ„³PAtom„„µ± PEmbedded´³refµ„³ PEmbedded„„µ±PBind´³refµ„³PBind„„µ±PAnd´³refµ„³PAnd„„µ±PNot´³refµ„³PNot„„µ±Lit´³refµ„³Lit„„µ± PCompound´³refµ„³ PCompound„„„„³Rewrite´³rec´³lit³rewrite„´³tupleµ´³named³pattern´³refµ„³Pattern„„´³named³template´³refµ„³Template„„„„„³WireRef´³orµµ±mine´³tupleµ´³lit„´³named³oid´³refµ„³Oid„„„„„µ±yours´³ tuplePrefixµ´³lit‘„´³named³oid´³refµ„³Oid„„„´³named³ attenuation´³seqof´³refµ„³Caveat„„„„„„„³PDiscard´³rec´³lit³_„´³tupleµ„„„³Template´³orµµ± +ByteString„„µ±Symbol´³lit³Symbol„„„„³PBind´³rec´³lit³bind„´³tupleµ´³named³pattern´³refµ„³Pattern„„„„„³Caveat´³orµµ±Rewrite´³refµ„³Rewrite„„µ±Alts´³refµ„³Alts„„µ±Reject´³refµ„³Reject„„µ±unknown³any„„„³Reject´³rec´³lit³reject„´³tupleµ´³named³pattern´³refµ„³Pattern„„„„„³Pattern´³orµµ±PDiscard´³refµ„³PDiscard„„µ±PAtom´³refµ„³PAtom„„µ± PEmbedded´³refµ„³ PEmbedded„„µ±PBind´³refµ„³PBind„„µ±PAnd´³refµ„³PAnd„„µ±PNot´³refµ„³PNot„„µ±Lit´³refµ„³Lit„„µ± PCompound´³refµ„³ PCompound„„„„³Rewrite´³rec´³lit³rewrite„´³tupleµ´³named³pattern´³refµ„³Pattern„„´³named³template´³refµ„³Template„„„„„³WireRef´³orµµ±mine´³tupleµ´³lit°„´³named³oid´³refµ„³Oid„„„„„µ±yours´³ tuplePrefixµ´³lit°„´³named³oid´³refµ„³Oid„„„´³named³ attenuation´³seqof´³refµ„³Caveat„„„„„„„³PDiscard´³rec´³lit³_„´³tupleµ„„„³Template´³orµµ± TAttenuate´³refµ„³ TAttenuate„„µ±TRef´³refµ„³TRef„„µ±Lit´³refµ„³Lit„„µ± TCompound´³refµ„³ TCompound„„„„³ PCompound´³orµµ±rec´³rec´³lit³rec„´³tupleµ´³named³label³any„´³named³fields´³seqof´³refµ„³Pattern„„„„„„„µ±arr´³rec´³lit³arr„´³tupleµ´³named³items´³seqof´³refµ„³Pattern„„„„„„„µ±dict´³rec´³lit³dict„´³tupleµ´³named³entries´³dictof³any´³refµ„³Pattern„„„„„„„„„³ PEmbedded´³lit³Embedded„³ SturdyRef´³rec´³lit³ref„´³tupleµ´³named³ parameters´³refµ„³ @@ -32,9 +32,9 @@ ByteString TAttenuate´³rec´³lit³ attenuate„´³tupleµ´³named³template´³refµ„³Template„„´³named³ attenuation´³seqof´³refµ„³Caveat„„„„„„³ CaveatsField´³orµµ±present´³dict·³caveats´³named³caveats´³seqof´³refµ„³Caveat„„„„„„µ±invalid´³dict·³caveats´³named³caveats³any„„„„µ±absent´³dict·„„„„„³SturdyStepType´³lit³ref„³SturdyStepDetail´³refµ„³ Parameters„³SturdyPathStepDetail´³refµ„³ Parameters„³SturdyDescriptionDetail´³dict·³key´³named³key´³atom³ -ByteString„„³oid´³named³oid³any„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³worker„´³schema·³version‘³ definitions·³Instance´³rec´³lit³Instance„´³tupleµ´³named³name´³atom³String„„´³named³argument³any„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³service„´³schema·³version‘³ definitions·³State´³orµµ±started´³lit³started„„µ±ready´³lit³ready„„µ±failed´³lit³failed„„µ±complete´³lit³complete„„µ± userDefined³any„„„³ +ByteString„„³oid´³named³oid³any„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³worker„´³schema·³version°³ definitions·³Instance´³rec´³lit³Instance„´³tupleµ´³named³name´³atom³String„„´³named³argument³any„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³service„´³schema·³version°³ definitions·³State´³orµµ±started´³lit³started„„µ±ready´³lit³ready„„µ±failed´³lit³failed„„µ±complete´³lit³complete„„µ± userDefined³any„„„³ RunService´³rec´³lit³ run-service„´³tupleµ´³named³ serviceName³any„„„„³ ServiceState´³rec´³lit³ service-state„´³tupleµ´³named³ serviceName³any„´³named³state´³refµ„³State„„„„„³ ServiceObject´³rec´³lit³service-object„´³tupleµ´³named³ serviceName³any„´³named³object³any„„„„³RequireService´³rec´³lit³require-service„´³tupleµ´³named³ serviceName³any„„„„³RestartService´³rec´³lit³restart-service„´³tupleµ´³named³ serviceName³any„„„„³ServiceDependency´³rec´³lit³ -depends-on„´³tupleµ´³named³depender³any„´³named³dependee´³refµ„³ ServiceState„„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³protocol„´³schema·³version‘³ definitions·³Oid´³atom³ SignedInteger„³Sync´³rec´³lit³sync„´³tupleµ´³named³peer´³embedded´³lit„„„„„„³Turn´³seqof´³refµ„³ TurnEvent„„³Error´³rec´³lit³error„´³tupleµ´³named³message´³atom³String„„´³named³detail³any„„„„³Event´³orµµ±Assert´³refµ„³Assert„„µ±Retract´³refµ„³Retract„„µ±Message´³refµ„³Message„„µ±Sync´³refµ„³Sync„„„„³Assert´³rec´³lit³assert„´³tupleµ´³named³ assertion´³refµ„³ Assertion„„´³named³handle´³refµ„³Handle„„„„„³Handle´³atom³ SignedInteger„³Packet´³orµµ±Turn´³refµ„³Turn„„µ±Error´³refµ„³Error„„µ± Extension´³refµ„³ Extension„„„„³Message´³rec´³lit³message„´³tupleµ´³named³body´³refµ„³ Assertion„„„„„³Retract´³rec´³lit³retract„´³tupleµ´³named³handle´³refµ„³Handle„„„„„³ Assertion³any³ Extension´³rec´³named³label³any„´³named³fields´³seqof³any„„„³ TurnEvent´³tupleµ´³named³oid´³refµ„³Oid„„´³named³event´³refµ„³Event„„„„„³ embeddedType€„„µ³ dataspace„´³schema·³version‘³ definitions·³Observe´³rec´³lit³Observe„´³tupleµ´³named³pattern´³refµ³dataspacePatterns„³Pattern„„´³named³observer´³embedded³any„„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³ -gatekeeper„´³schema·³version‘³ definitions·³Bind´³rec´³lit³bind„´³tupleµ´³named³ description´³refµ„³ Description„„´³named³target´³embedded³any„„´³named³observer´³refµ„³ BindObserver„„„„„³Step´³rec´³named³stepType´³atom³Symbol„„´³tupleµ´³named³detail³any„„„„³Bound´³orµµ±bound´³rec´³lit³bound„´³tupleµ´³named³pathStep´³refµ„³PathStep„„„„„„µ±Rejected´³refµ„³Rejected„„„„³Route´³rec´³lit³route„´³ tuplePrefixµ´³named³ -transports´³seqof³any„„„´³named³ pathSteps´³seqof´³refµ„³PathStep„„„„„³Resolve´³rec´³lit³resolve„´³tupleµ´³named³step´³refµ„³Step„„´³named³observer´³embedded´³refµ„³Resolved„„„„„„³PathStep´³rec´³named³stepType´³atom³Symbol„„´³tupleµ´³named³detail³any„„„„³Rejected´³rec´³lit³rejected„´³tupleµ´³named³detail³any„„„„³Resolved´³orµµ±accepted´³rec´³lit³accepted„´³tupleµ´³named³responderSession´³embedded³any„„„„„„µ±Rejected´³refµ„³Rejected„„„„³ Description´³rec´³named³stepType´³atom³Symbol„„´³tupleµ´³named³detail³any„„„„³ ResolvePath´³rec´³lit³ resolve-path„´³tupleµ´³named³route´³refµ„³Route„„´³named³addr³any„´³named³control´³embedded´³refµ„³TransportControl„„„´³named³resolved´³refµ„³Resolved„„„„„³ BindObserver´³orµµ±present´³embedded´³refµ„³Bound„„„µ±absent´³lit€„„„„³ForceDisconnect´³rec´³lit³force-disconnect„´³tupleµ„„„³ResolvedPathStep´³rec´³lit³ path-step„´³tupleµ´³named³origin´³embedded´³refµ„³Resolve„„„´³named³pathStep´³refµ„³PathStep„„´³named³resolved´³refµ„³Resolved„„„„„³TransportControl´³refµ„³ForceDisconnect„³TransportConnection´³rec´³lit³connect-transport„´³tupleµ´³named³addr³any„´³named³control´³embedded´³refµ„³TransportControl„„„´³named³resolved´³refµ„³Resolved„„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³transportAddress„´³schema·³version‘³ definitions·³Tcp´³rec´³lit³tcp„´³tupleµ´³named³host´³atom³String„„´³named³port´³atom³ SignedInteger„„„„„³Unix´³rec´³lit³unix„´³tupleµ´³named³path´³atom³String„„„„„³Stdio´³rec´³lit³stdio„´³tupleµ„„„³ WebSocket´³rec´³lit³ws„´³tupleµ´³named³url´³atom³String„„„„„„³ embeddedType€„„µ³dataspacePatterns„´³schema·³version‘³ definitions·³DLit´³rec´³lit³lit„´³tupleµ´³named³value´³refµ„³AnyAtom„„„„„³DBind´³rec´³lit³bind„´³tupleµ´³named³pattern´³refµ„³Pattern„„„„„³AnyAtom´³orµµ±bool´³atom³Boolean„„µ±float´³atom³Float„„µ±double´³atom³Double„„µ±int´³atom³ SignedInteger„„µ±string´³atom³String„„µ±bytes´³atom³ +depends-on„´³tupleµ´³named³depender³any„´³named³dependee´³refµ„³ ServiceState„„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³protocol„´³schema·³version°³ definitions·³Oid´³atom³ SignedInteger„³Sync´³rec´³lit³sync„´³tupleµ´³named³peer´³embedded´³lit„„„„„„³Turn´³seqof´³refµ„³ TurnEvent„„³Error´³rec´³lit³error„´³tupleµ´³named³message´³atom³String„„´³named³detail³any„„„„³Event´³orµµ±Assert´³refµ„³Assert„„µ±Retract´³refµ„³Retract„„µ±Message´³refµ„³Message„„µ±Sync´³refµ„³Sync„„„„³Assert´³rec´³lit³assert„´³tupleµ´³named³ assertion´³refµ„³ Assertion„„´³named³handle´³refµ„³Handle„„„„„³Handle´³atom³ SignedInteger„³Packet´³orµµ±Turn´³refµ„³Turn„„µ±Error´³refµ„³Error„„µ± Extension´³refµ„³ Extension„„„„³Message´³rec´³lit³message„´³tupleµ´³named³body´³refµ„³ Assertion„„„„„³Retract´³rec´³lit³retract„´³tupleµ´³named³handle´³refµ„³Handle„„„„„³ Assertion³any³ Extension´³rec´³named³label³any„´³named³fields´³seqof³any„„„³ TurnEvent´³tupleµ´³named³oid´³refµ„³Oid„„´³named³event´³refµ„³Event„„„„„³ embeddedType€„„µ³ dataspace„´³schema·³version°³ definitions·³Observe´³rec´³lit³Observe„´³tupleµ´³named³pattern´³refµ³dataspacePatterns„³Pattern„„´³named³observer´³embedded³any„„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³ +gatekeeper„´³schema·³version°³ definitions·³Bind´³rec´³lit³bind„´³tupleµ´³named³ description´³refµ„³ Description„„´³named³target´³embedded³any„„´³named³observer´³refµ„³ BindObserver„„„„„³Step´³rec´³named³stepType´³atom³Symbol„„´³tupleµ´³named³detail³any„„„„³Bound´³orµµ±bound´³rec´³lit³bound„´³tupleµ´³named³pathStep´³refµ„³PathStep„„„„„„µ±Rejected´³refµ„³Rejected„„„„³Route´³rec´³lit³route„´³ tuplePrefixµ´³named³ +transports´³seqof³any„„„´³named³ pathSteps´³seqof´³refµ„³PathStep„„„„„³Resolve´³rec´³lit³resolve„´³tupleµ´³named³step´³refµ„³Step„„´³named³observer´³embedded´³refµ„³Resolved„„„„„„³PathStep´³rec´³named³stepType´³atom³Symbol„„´³tupleµ´³named³detail³any„„„„³Rejected´³rec´³lit³rejected„´³tupleµ´³named³detail³any„„„„³Resolved´³orµµ±accepted´³rec´³lit³accepted„´³tupleµ´³named³responderSession´³embedded³any„„„„„„µ±Rejected´³refµ„³Rejected„„„„³ Description´³rec´³named³stepType´³atom³Symbol„„´³tupleµ´³named³detail³any„„„„³ ResolvePath´³rec´³lit³ resolve-path„´³tupleµ´³named³route´³refµ„³Route„„´³named³addr³any„´³named³control´³embedded´³refµ„³TransportControl„„„´³named³resolved´³refµ„³Resolved„„„„„³ BindObserver´³orµµ±present´³embedded´³refµ„³Bound„„„µ±absent´³lit€„„„„³ForceDisconnect´³rec´³lit³force-disconnect„´³tupleµ„„„³ResolvedPathStep´³rec´³lit³ path-step„´³tupleµ´³named³origin´³embedded´³refµ„³Resolve„„„´³named³pathStep´³refµ„³PathStep„„´³named³resolved´³refµ„³Resolved„„„„„³TransportControl´³refµ„³ForceDisconnect„³TransportConnection´³rec´³lit³connect-transport„´³tupleµ´³named³addr³any„´³named³control´³embedded´³refµ„³TransportControl„„„´³named³resolved´³refµ„³Resolved„„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„µ³transportAddress„´³schema·³version°³ definitions·³Tcp´³rec´³lit³tcp„´³tupleµ´³named³host´³atom³String„„´³named³port´³atom³ SignedInteger„„„„„³Unix´³rec´³lit³unix„´³tupleµ´³named³path´³atom³String„„„„„³Stdio´³rec´³lit³stdio„´³tupleµ„„„³ WebSocket´³rec´³lit³ws„´³tupleµ´³named³url´³atom³String„„„„„„³ embeddedType€„„µ³dataspacePatterns„´³schema·³version°³ definitions·³DLit´³rec´³lit³lit„´³tupleµ´³named³value´³refµ„³AnyAtom„„„„„³DBind´³rec´³lit³bind„´³tupleµ´³named³pattern´³refµ„³Pattern„„„„„³AnyAtom´³orµµ±bool´³atom³Boolean„„µ±float´³atom³Float„„µ±double´³atom³Double„„µ±int´³atom³ SignedInteger„„µ±string´³atom³String„„µ±bytes´³atom³ ByteString„„µ±symbol´³atom³Symbol„„µ±embedded´³embedded³any„„„„³Pattern´³orµµ±DDiscard´³refµ„³DDiscard„„µ±DBind´³refµ„³DBind„„µ±DLit´³refµ„³DLit„„µ± DCompound´³refµ„³ DCompound„„„„³DDiscard´³rec´³lit³_„´³tupleµ„„„³ DCompound´³orµµ±rec´³rec´³lit³rec„´³tupleµ´³named³label³any„´³named³fields´³seqof´³refµ„³Pattern„„„„„„„µ±arr´³rec´³lit³arr„´³tupleµ´³named³items´³seqof´³refµ„³Pattern„„„„„„„µ±dict´³rec´³lit³dict„´³tupleµ´³named³entries´³dictof³any´³refµ„³Pattern„„„„„„„„„„³ embeddedType´³refµ³ EntityRef„³Cap„„„„„ \ No newline at end of file diff --git a/implementations/python/preserves/binary.py b/implementations/python/preserves/binary.py index ee0b267..6d291bc 100644 --- a/implementations/python/preserves/binary.py +++ b/implementations/python/preserves/binary.py @@ -45,7 +45,7 @@ class Decoder(BinaryCodec): until a [ShortPacket][preserves.error.ShortPacket] exception is raised: ```python - >>> d = Decoder(b'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84') + >>> d = Decoder(b'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84') >>> d.next() 123 >>> d.next() @@ -63,7 +63,7 @@ class Decoder(BinaryCodec): `None`, which is not in the domain of Preserves `Value`s: ```python - >>> d = Decoder(b'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84') + >>> d = Decoder(b'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84') >>> d.try_next() 123 >>> d.try_next() @@ -79,14 +79,14 @@ class Decoder(BinaryCodec): over all complete values in an input: ```python - >>> d = Decoder(b'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84') + >>> d = Decoder(b'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84') >>> list(d) [123, 'hello', ()] ``` ```python - >>> for v in Decoder(b'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84'): + >>> for v in Decoder(b'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84'): ... print(repr(v)) 123 'hello' @@ -97,7 +97,7 @@ class Decoder(BinaryCodec): Supply `include_annotations=True` to read annotations alongside the annotated values: ```python - >>> d = Decoder(b'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84', include_annotations=True) + >>> d = Decoder(b'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84', include_annotations=True) >>> list(d) [123, 'hello', @#x ()] @@ -107,7 +107,7 @@ class Decoder(BinaryCodec): [extend][preserves.binary.Decoder.extend] to add new input as if comes available: ```python - >>> d = Decoder(b'\\xa0{\\xb1\\x05he') + >>> d = Decoder(b'\\xb0\\x01{\\xb1\\x05he') >>> d.try_next() 123 >>> d.try_next() # returns None because the input is incomplete @@ -181,11 +181,6 @@ class Decoder(BinaryCodec): def wrap(self, v): return Annotated(v) if self.include_annotations else v - def unshift_annotation(self, a, v): - if self.include_annotations: - v.annotations.insert(0, a) - return v - def next(self): """Reads the next complete `Value` from the internal buffer, raising [ShortPacket][preserves.error.ShortPacket] if too few bytes are available, or @@ -195,19 +190,16 @@ class Decoder(BinaryCodec): tag = self.nextbyte() if tag == 0x80: return self.wrap(False) if tag == 0x81: return self.wrap(True) - if tag == 0x82: return self.wrap(Float.from_bytes(self.nextbytes(4))) - if tag == 0x83: return self.wrap(struct.unpack('>d', self.nextbytes(8))[0]) if tag == 0x84: raise DecodeError('Unexpected end-of-stream marker') - if tag == 0x85: - a = self.next() - v = self.next() - return self.unshift_annotation(a, v) if tag == 0x86: if self.decode_embedded is None: raise DecodeError('No decode_embedded function supplied') return self.wrap(Embedded(self.decode_embedded(self.next()))) - if tag >= 0x90 and tag <= 0x9f: return self.wrap(tag - (0xa0 if tag > 0x9c else 0x90)) - if tag >= 0xa0 and tag <= 0xaf: return self.wrap(self.nextint(tag - 0xa0 + 1)) + if tag == 0x87: + count = self.nextbyte() + if count == 4: return self.wrap(Float.from_bytes(self.nextbytes(4))) + if count == 8: return self.wrap(struct.unpack('>d', self.nextbytes(8))[0]) + raise DecodeError('Invalid IEEE754 size') if tag == 0xb0: return self.wrap(self.nextint(self.varint())) if tag == 0xb1: return self.wrap(self.nextbytes(self.varint()).decode('utf-8')) if tag == 0xb2: return self.wrap(self.nextbytes(self.varint())) @@ -219,6 +211,12 @@ class Decoder(BinaryCodec): if tag == 0xb5: return self.wrap(tuple(self.nextvalues())) if tag == 0xb6: return self.wrap(frozenset(self.nextvalues())) if tag == 0xb7: return self.wrap(ImmutableDict.from_kvs(self.nextvalues())) + if tag == 0xbf: + vs = self.nextvalues() + if len(vs) < 2: raise DecodeError('Too few items in encoded annotated value') + if self.include_annotations: + vs[0].annotations.extend(vs[1:]) + return vs[0] raise DecodeError('Invalid tag: ' + hex(tag)) def try_next(self): @@ -265,7 +263,7 @@ class Encoder(BinaryCodec): >>> e.append('hello') >>> e.append(annotate([], Symbol('x'))) >>> e.contents() - b'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84' + b'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84' ``` @@ -325,13 +323,13 @@ class Encoder(BinaryCodec): self.varint(v // 128) def encodeint(self, v): - bitcount = (~v if v < 0 else v).bit_length() + 1 - bytecount = (bitcount + 7) // 8 - if bytecount <= 16: - self.buffer.append(0xa0 + bytecount - 1) + self.buffer.append(0xb0) + if v == 0: + bytecount = 0 else: - self.buffer.append(0xb0) - self.varint(bytecount) + bitcount = (~v if v < 0 else v).bit_length() + 1 + bytecount = (bitcount + 7) // 8 + self.varint(bytecount) def enc(n,x): if n > 0: enc(n-1, x >> 8) @@ -339,30 +337,30 @@ class Encoder(BinaryCodec): enc(bytecount, v) def encodevalues(self, tag, items): - self.buffer.append(0xb0 + tag) + self.buffer.append(tag) for i in items: self.append(i) self.buffer.append(0x84) def encodebytes(self, tag, bs): - self.buffer.append(0xb0 + tag) + self.buffer.append(tag) self.varint(len(bs)) self.buffer.extend(bs) def encodeset(self, v): if not self._canonicalize: - self.encodevalues(6, v) + self.encodevalues(0xb6, v) else: c = Canonicalizer(self._encode_embedded, self.include_annotations) for i in v: c.entry([i]) - c.emit_entries(self, 6) + c.emit_entries(self, 0xb6) def encodedict(self, v): if not self._canonicalize: - self.encodevalues(7, list(dict_kvs(v))) + self.encodevalues(0xb7, list(dict_kvs(v))) else: c = Canonicalizer(self._encode_embedded, self.include_annotations) for (kk, vv) in v.items(): c.entry([kk, vv]) - c.emit_entries(self, 7) + c.emit_entries(self, 0xb7) def append(self, v): """Extend `self.buffer` with an encoding of `v`.""" @@ -374,21 +372,19 @@ class Encoder(BinaryCodec): elif v is True: self.buffer.append(0x81) elif isinstance(v, float): - self.buffer.append(0x83) + self.buffer.append(0x87) + self.buffer.append(8) self.buffer.extend(struct.pack('>d', v)) elif isinstance(v, numbers.Number): - if v >= -3 and v <= 12: - self.buffer.append(0x90 + (v if v >= 0 else v + 16)) - else: - self.encodeint(v) + self.encodeint(v) elif isinstance(v, bytes): - self.encodebytes(2, v) + self.encodebytes(0xb2, v) elif isinstance(v, basestring_): - self.encodebytes(1, v.encode('utf-8')) + self.encodebytes(0xb1, v.encode('utf-8')) elif isinstance(v, list): - self.encodevalues(5, v) + self.encodevalues(0xb5, v) elif isinstance(v, tuple): - self.encodevalues(5, v) + self.encodevalues(0xb5, v) elif isinstance(v, set): self.encodeset(v) elif isinstance(v, frozenset): @@ -403,7 +399,7 @@ class Encoder(BinaryCodec): if i is None: self.cannot_encode(v) else: - self.encodevalues(5, i) + self.encodevalues(0xb5, i) def cannot_encode(self, v): raise TypeError('Cannot preserves-encode: ' + repr(v)) @@ -420,7 +416,7 @@ class Canonicalizer: self.entries.append(entry) def emit_entries(self, outer_encoder, tag): - outer_encoder.buffer.append(0xb0 + tag) + outer_encoder.buffer.append(tag) for e in sorted(self.entries): outer_encoder.buffer.extend(e) outer_encoder.buffer.append(0x84) diff --git a/implementations/python/preserves/path.prb b/implementations/python/preserves/path.prb index e5241a0..5679a63 100644 --- a/implementations/python/preserves/path.prb +++ b/implementations/python/preserves/path.prb @@ -1,4 +1,4 @@ -´³schema·³version‘³ definitions·³Axis´³orµµ±values´³rec´³lit³values„´³tupleµ„„„„µ± descendants´³rec´³lit³ descendants„´³tupleµ„„„„µ±at´³rec´³lit³at„´³tupleµ´³named³key³any„„„„„µ±label´³rec´³lit³label„´³tupleµ„„„„µ±keys´³rec´³lit³keys„´³tupleµ„„„„µ±length´³rec´³lit³length„´³tupleµ„„„„µ± annotations´³rec´³lit³ annotations„´³tupleµ„„„„µ±embedded´³rec´³lit³embedded„´³tupleµ„„„„µ±parse´³rec´³lit³parse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„µ±unparse´³rec´³lit³unparse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„„„³Step´³orµµ±Axis´³refµ„³Axis„„µ±Filter´³refµ„³Filter„„µ±Function´³refµ„³Function„„„„³Filter´³orµµ±nop´³rec´³lit³nop„´³tupleµ„„„„µ±compare´³rec´³lit³compare„´³tupleµ´³named³op´³refµ„³ +´³schema·³version°³ definitions·³Axis´³orµµ±values´³rec´³lit³values„´³tupleµ„„„„µ± descendants´³rec´³lit³ descendants„´³tupleµ„„„„µ±at´³rec´³lit³at„´³tupleµ´³named³key³any„„„„„µ±label´³rec´³lit³label„´³tupleµ„„„„µ±keys´³rec´³lit³keys„´³tupleµ„„„„µ±length´³rec´³lit³length„´³tupleµ„„„„µ± annotations´³rec´³lit³ annotations„´³tupleµ„„„„µ±embedded´³rec´³lit³embedded„´³tupleµ„„„„µ±parse´³rec´³lit³parse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„µ±unparse´³rec´³lit³unparse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„„„³Step´³orµµ±Axis´³refµ„³Axis„„µ±Filter´³refµ„³Filter„„µ±Function´³refµ„³Function„„„„³Filter´³orµµ±nop´³rec´³lit³nop„´³tupleµ„„„„µ±compare´³rec´³lit³compare„´³tupleµ´³named³op´³refµ„³ Comparison„„´³named³literal³any„„„„„µ±regex´³rec´³lit³regex„´³tupleµ´³named³regex´³atom³String„„„„„„µ±test´³rec´³lit³test„´³tupleµ´³named³pred´³refµ„³ Predicate„„„„„„µ±real´³rec´³lit³real„´³tupleµ„„„„µ±int´³rec´³lit³int„´³tupleµ„„„„µ±kind´³rec´³lit³kind„´³tupleµ´³named³kind´³refµ„³ ValueKind„„„„„„„„³Function´³rec´³lit³count„´³tupleµ´³named³selector´³refµ„³Selector„„„„„³Selector´³seqof´³refµ„³Step„„³ Predicate´³orµµ±Selector´³refµ„³Selector„„µ±not´³rec´³lit³not„´³tupleµ´³named³pred´³refµ„³ Predicate„„„„„„µ±or´³rec´³lit³or„´³tupleµ´³named³preds´³seqof´³refµ„³ Predicate„„„„„„„µ±and´³rec´³lit³and„´³tupleµ´³named³preds´³seqof´³refµ„³ Predicate„„„„„„„„„³ ValueKind´³orµµ±Boolean´³lit³Boolean„„µ±Float´³lit³Float„„µ±Double´³lit³Double„„µ± SignedInteger´³lit³ SignedInteger„„µ±String´³lit³String„„µ± ByteString´³lit³ ByteString„„µ±Symbol´³lit³Symbol„„µ±Record´³lit³Record„„µ±Sequence´³lit³Sequence„„µ±Set´³lit³Set„„µ± diff --git a/implementations/python/preserves/path.py b/implementations/python/preserves/path.py index f76386c..f2e2e44 100644 --- a/implementations/python/preserves/path.py +++ b/implementations/python/preserves/path.py @@ -94,14 +94,14 @@ Evaluating the expression against `testdata` yields the following: ... print(stringify(result)) >> >>>> - "Dr">> + "Dr">> > -> +> > -> - 3 4>> -> -> +> + 3 4>> +> +> ``` diff --git a/implementations/python/preserves/schema.prb b/implementations/python/preserves/schema.prb index 408302c..a98ba99 100644 --- a/implementations/python/preserves/schema.prb +++ b/implementations/python/preserves/schema.prb @@ -1,6 +1,6 @@ -´³schema·³version‘³ definitions·³Ref´³rec´³lit³ref„´³tupleµ´³named³module´³refµ„³ +´³schema·³version°³ definitions·³Ref´³rec´³lit³ref„´³tupleµ´³named³module´³refµ„³ ModulePath„„´³named³name´³atom³Symbol„„„„„³Bundle´³rec´³lit³bundle„´³tupleµ´³named³modules´³refµ„³Modules„„„„„³Schema´³rec´³lit³schema„´³tupleµ´³dict·³version´³named³version´³refµ„³Version„„³ definitions´³named³ definitions´³refµ„³ Definitions„„³ embeddedType´³named³ embeddedType´³refµ„³EmbeddedTypeName„„„„„„„³Binding´³rec´³lit³named„´³tupleµ´³named³name´³atom³Symbol„„´³named³pattern´³refµ„³ SimplePattern„„„„„³Modules´³dictof´³refµ„³ -ModulePath„´³refµ„³Schema„„³Pattern´³orµµ± SimplePattern´³refµ„³ SimplePattern„„µ±CompoundPattern´³refµ„³CompoundPattern„„„„³Version´³lit‘„³AtomKind´³orµµ±Boolean´³lit³Boolean„„µ±Float´³lit³Float„„µ±Double´³lit³Double„„µ± SignedInteger´³lit³ SignedInteger„„µ±String´³lit³String„„µ± +ModulePath„´³refµ„³Schema„„³Pattern´³orµµ± SimplePattern´³refµ„³ SimplePattern„„µ±CompoundPattern´³refµ„³CompoundPattern„„„„³Version´³lit°„³AtomKind´³orµµ±Boolean´³lit³Boolean„„µ±Float´³lit³Float„„µ±Double´³lit³Double„„µ± SignedInteger´³lit³ SignedInteger„„µ±String´³lit³String„„µ± ByteString´³lit³ ByteString„„µ±Symbol´³lit³Symbol„„„„³ Definition´³orµµ±or´³rec´³lit³or„´³tupleµ´³ tuplePrefixµ´³named³pattern0´³refµ„³NamedAlternative„„´³named³pattern1´³refµ„³NamedAlternative„„„´³named³patternN´³seqof´³refµ„³NamedAlternative„„„„„„„„µ±and´³rec´³lit³and„´³tupleµ´³ tuplePrefixµ´³named³pattern0´³refµ„³ NamedPattern„„´³named³pattern1´³refµ„³ NamedPattern„„„´³named³patternN´³seqof´³refµ„³ NamedPattern„„„„„„„„µ±Pattern´³refµ„³Pattern„„„„³ diff --git a/implementations/python/preserves/values.py b/implementations/python/preserves/values.py index 31cb587..7d303dc 100644 --- a/implementations/python/preserves/values.py +++ b/implementations/python/preserves/values.py @@ -57,9 +57,9 @@ class Float(object): >>> preserves.parse('3.45') 3.45 >>> preserves.encode(Float(3.45)) - b'\\x82@\\\\\\xcc\\xcd' + b'\\x87\\x04@\\\\\\xcc\\xcd' >>> preserves.encode(3.45) - b'\\x83@\\x0b\\x99\\x99\\x99\\x99\\x99\\x9a' + b'\\x87\\x08@\\x0b\\x99\\x99\\x99\\x99\\x99\\x9a' ``` @@ -123,7 +123,8 @@ class Float(object): return struct.pack('>f', self.value) def __preserve_write_binary__(self, encoder): - encoder.buffer.append(0x82) + encoder.buffer.append(0x87) + encoder.buffer.append(4) encoder.buffer.extend(self.to_bytes()) def __preserve_write_text__(self, formatter): @@ -577,11 +578,14 @@ class Annotated(object): self.item = item def __preserve_write_binary__(self, encoder): - if encoder.include_annotations: + if encoder.include_annotations and self.annotations: + encoder.buffer.append(0xbf) + encoder.append(self.item) for a in self.annotations: - encoder.buffer.append(0x85) encoder.append(a) - encoder.append(self.item) + encoder.buffer.append(0x84) + else: + encoder.append(self.item) def __preserve_write_text__(self, formatter): if formatter.include_annotations: diff --git a/implementations/python/tests/samples.bin b/implementations/python/tests/samples.bin index 0d2ad15eb64f956840bea334ef53692505004815..d47492d1533490e7250e55cedadf3076b92aa744 100644 GIT binary patch literal 10541 zcmd^FTW=f371olXsEcJ=@g)U{BvZRF>`Jt>q^@q7M6zW>MIr~jrD@Y5OYTrynA~M| zm%7*}ECY)K{Q*VM0_hJZP=uqEqL%{BbACi0`p}0y<)IHPkXZd@W_NZ;QFVek0g4jj zaCgpSX3m`X&fFgFhC{ch=U+8E>g|X9mhDgNhW$KiC)4h0_XQ$PkMpu^O~V(t9!+t6 z5_}TKJaC>~b!?OR)NNR{<@r`cXdC%Nh-TVaHb^~6sp8|^aQJP*_2{jd@%(WDAWv?Z7(t1j#k+ZhpU$B`32i(P&Tw&x9%Hd>eqt( zFz>ZonY&asD)%;My&eS5Oi@(j?#>#0K#VJ-JV7c~hSqm%XH)&w;TWQvneJX0*-uP}Qeu~iydwv}ObnH~)XZS4Tn zuDv&+S;)wJFR52t&JkPj@e;Al#;h;wv0M$V<%BuEMqRp-+i={v zxd#wp6MVx(#=RyDR4W{q+s%wE2HIn0{08XU@qMCO$fNQlsW7O*pMKPDbzE&+nmbNn5xPF z^||T0n=*~5-EfAUsGie{rRkYj^u(#_bc4FYuNgL(C$bx2$!mhv=UV{bd2jGiKVn5Ik7L|IzEa^v2ru^MaUgEPgZjd3<@EX4*y)r-na zG4FVYFT_Q@a?bvCrf-}7B&9f`1`ekj@Lx<(@CAz2$A6%3ywp<@12OD(vU}Ux0XrF^ z3!fM8(H`?Fm&tKR6}SrQmAaI}aN03WI~LQXjkwKE0p`4b7F~^s)cA@>ss@KiNpzOZ zF?9aid#%|@X&R34vcv{&%l3^;;@I^q;CfMrJU4i3cywrU>wb@{Sg;s3^zlv%i%eg!F(b5%I@zdM0H`Kram4b7fiSW`%?bEk zrQR~dJkMz~4DWk^g3`R{w90j=>@ob|U|;k}o=~`CQ?>!yYoANL$E5Y-|X?P%*mWx4pNH&8^f$*_m)fs&1%dLu?8n_xjA zt0^rUItsrO(2*B#o=nub@LZCUDI3V!=zf`^r-ZZ^>P zbs!O{ldpTc9al(BBpQz_87ATW4&PR_;E_XVfA}e{1y4EtrQ51=ze@25ocvVcWCtJj zAjNke^zpRYc6vt;IDjU!O6rSk4168$1-y9>3Y~;UZS4_SBnb+V0oB|1Dnc?%XlD){ z@?S(M&Zrp>Ao(D}e~OgG@dFJctT{zV#hEma9{|V?0OS!##S18W0`z|Z^v}Ck^0}sh zg|oY1zG5`}mP_$uT3*F)%^>KVtHc#tpkjHa>``|e;yJHLje99IM`+>NzrK(vT!GN? z?p`?HTYjCI;XsqRo@2MQOo(YXoQubzwuTjco*wJ1EP>J)pWYzuVUGSgu5WN|Wl?Gl`~f5KG#oYk8F0Fw&XIRKlw3$97rXxaGUT zB5=n3;82DVQcinnH_;9wO0mOPO0auzk$L6SfSryf4ASFB9n11ML_AR$6}~d8k8zik z*{W~z(IhsW%f<}KzGvBKgo3}dckxXH;@-JRzq0xO>;L?t;14q8WK8;go4YhH6 z6+v}%F3gUPkBdYRRhL4lfUhDb9D-D#P!LHF4$4VG5>ya_?JB)Wi)M;HD{ zo)ljy%&4FSu#G~a0PqNKqYCIryn#Rza}fpOrzA4O0zB|Y$X5cq1SyaX5wSq`YUQk& zW$;LqnjdsVtXMn=MZ87-6fNgrI(kY1Xe^M#BqbnBAOxfdWWL zDpv(n>w_#3P;JMun=PNOFNUIx#~asouzotR>R`eP5;5?_CS5ofMJdjk-U-!S3D|dI zToKzG1;bve(*oZI6&n|^BzK_ns!#(Wz1Y3qaY$IAKx(ZKW(o%Xp-K@=Mb3e+9|Omb zX+l?6A?doH>j_;!1?c1YMe_x6IvdjZon zgFuKdv72>7c83yWpED>WhaV!f#!Kwy5t~08&g@VC`}E#(zp(!p8^~eCxW~0T;=P^LofrpN-lsE!_R%CKi(>ex`jZZS1PdH}d-J*zp^w zdY4k(SIG`aceKb7LIS408M{2+W%1zO*xU0RLf5eg+(Nk7B3l-%V^!)jaBd{lr75L6 zkPM2$xaAwyF;ndktwsUe#hIz-6VF5P?Ofk)GXLs}{(|tNTdm9&XBG-{(`Yp76v)~e zs6Ns5S#G6atkHdzUAg9A+stogvL9pr?(Yb-zx_30|82hPVW;m}bc0k~r$O`?zecXt z9i05--my%-1_{Cj-k{c64bqvPu)#&!YG62kIP>3)7{z0L@hTaR|ka*8Uf3eYj%~$9i|n7up^y z^;4_KatqT4)vW(2_J`T%t;JPx{pRB8)swR~K1+(KKD^d06A*uR1N; zcehyP&Mh|hCSt4KGUCM^v54n|FoO!D*DkMZ+y1&@u8xF|L3*`;RFWx z{G6^Ai}N!JC4FvosyID6i*K=nK`}E`oGBIYqGMuRN&+ zajFRAlvEwQuMA{E_~*Mn#NTRC?+P^Dj&3{L<{7y0z~A?A-oXLU!|qDivJH1DJRZ#e zsCif3AUGzk35be#$pr>6OUt#j0c>fswLwf6+u8^~S^;QEcfU_JaXQ(p={VSvU3Hk7C1HZS)Wv|;G8TLZ!y?_R(58>$6r8||vI~4L6tE*%v8q-@D)RV%ct6tqK1PT+ z>A2*5uIhdEssYKS>tI0P_Dgw7j2ya2D=nXo92_3nKyqBp!1-6DMp1gl&|)PU!4#$A z=8U*PBgY0YDtyFo4VZIq3E#l!4Kd2jI>iV)9go1}ln}q3!H$WxqlvtFSO&34X+Trl z*;x_;^CGqR1)k_d2wb{!Ny%iH{xq4$7-^|~cCLq%M1qv^9DBK=DiH$;@9bir0sCPG zXGf(wnT7t&EI2WJqbZEM5q^@?FT zHm*)#l6b>4dFL(*gPYRg?^u3qQLU7{i#?Q#u$Mc^>1>zJBE8JXPljwj^~*+g=oK|Mg99 zl2wmZ^K7j!#%_M4qP~_PfT1^6`0_vndKuBz5r?r$#v#PQH43cbea;bwc+;v@;h#3n zk#YNNgHph73Q^dqUYf+M7!F?g?4&+7K}_1jQ9gGF$0jvrgKOS`GkL&nAQdlw-vKN* X)dJvv8VypxO-5XeaBS}~<}3dO%4*Gi literal 10057 zcmd^F&2JmW6&FcSGGoiOY{xB7B$?QaVOOFglKON$9Lbgu6^PwXL91O|fd#w!ErK=0bI+r0WKnrOJ6pD^sVsgU*+z>~+NY zV(I34?|<;&t&eVh;yBf9Z%C#^$4{RO>1nm;yuDd!c_+A2C(t8h6MHhhfblf&cY^;Z zkN?T~$@O@>G)=?&u72JT>{By_88%PzVRe9iyFaW=`nMR^uDZnLCbp2+4T*h|hOSVX znnhJtEnAhJ%qSmY@*IctP3VM5@{H?I-9)1{ndNg%UM7}ps-?qGS2)tm4v>aV!7%3} z_}R;zY&UzZfraEH`BI%>l+MevWRtlWYErlDH23DT1vgU*s_ZGUxJO^~wO@pbE4_wE zE^<{D*((wxOQwN{pbE(^xUNs(5B0PA#jOOA&d7qV-5@u$*2l=Fz%B!p==3 znTHJ5U>mN4E#)O6$TLlLpQ#C8N$5uXwcBna<7@N;{n>`!pKv2)CU1NUCUX>-?&{bK|w1v~A!Y6Xv zHL&7Y_)XQG8TDWc=xH5pQ1J_+}(ixMaeG1sUbt80wSH1B-2&__xQDF zgYIb4N0xW8)8E~US}amU1-84*Z0_!6mAlxOr_wX(VHc?%d;GguVQVels|h;UnMZ2r zv8(`?JTp-&X;dU*FYC@CR}xj{D509fHmPKDjvx&yYM}tP)MbkNY|l{&qz!=vo6O{T zGyN_;Wd?gQL)}E7C5jvmXE1a!W6W{Q3-BWg@C-2G4G=J-jY1lcPo*M}krPzDJg|Bk z2WyG|op^Frb-QoP%kitRci(kfN=8tYmO!y+ohSZ<^9Aw@r-7?!x>-%~RKe}uEu!JLC#dz_6Z#-)uvp#JudY6AzZBzi-9qz zsB*|U>&`mbaEV)*XLOy#^4F^|X{pBTv>KDwt4|eF)3VdLRHQ65uBn$Lc*BgdUTwF^ zrbivqkOJnYra8_NEAYGcNb}Nhg-Az9zMIHLiGsOt2Zzb&s87x%x)7b3ovusS`K1aD$vfxrM-3H-))fbb$}HM!0W2<`CVuBaPxQo0GL?%JPz>ItT^ z!rSlN^GP{Dx_j3r^#`Q*@;A%hc*IOy^v)fhBt+xO9m^dcYFLMfj>KzDjdz$6g~m4S z2k$#x=~oT}&9&9;efsW_DiKFB;aUWjlZVyFJw6=J(>|?+=diuT&3=#@4WjC5`jk*@sfB4L7iwJd1Fui`FXbp>Ht4`PtrLoe(V`Z zC&R=ueBgk>;)Mg5Y_5M`kXzjC3?OLF>jvLVY%RCz;CvuAG&~{+S&l|Q){wDtu%gX% zcw(0?4-5_s3n6$(=NpEF^gBzj3@Bh*W&##KIQUIZN+0{Omo>(V%Vx8atQf9Kk730fSm#+p?;eLsgmTR~?dUim> z1?U^N0=fXE#7e9M7w8=6GRV%m1IXa1nFp=N%`RN5CTiEXdY4_eMWP1eTLNHubm0<- z+8qa%Pw$t4!EZv+b{8Bn!m~pm;#Y~58mo{TU-07FU%MN+wm__Mse}!`MJ~*D6P$mL z3+1RAqz)dqP#%? zb^+K7tH7KRmQO^r2(|^r996pETC!GlSB0`sEK1h%s|`@tt{7$PQiU~Ai!J^~+@NKT zN!TT|c@NFxacCuac5C04yzT z5lHgLO{Syx0{Dq>3=--ySWh=H0bn*d)5yf}?Msetz9x7{)LzLO8(b4F1QVQ~k%5&D5#+R=x1L1tKNZW~(jjOo)itK<`j-VorOH_m~cO$m4!8^@c$PR=+ccW!x z1EA|y2ux3`e7k`Q72a8|i0=Js)pl7b@dE|iDXJyogpThKn^A?hhIk-iN*z4s#M!b)U*u!CSL}_x0ak`Y@@`63E4rx9&)Aq z8SEBV{DkQ|fvD(+n5ga1F$*gO))o_&T)TzEzgq|`V)CA@YpHI-rmtkG7Uj0`^@WH+ zSWIxo8f%xPa1^UgV zwV~B}Rc)!dy>iLIi`%Qg9?IQH3AuW=tI13{o6h+^f}j2xrL|{GYP3QE>R&s1ybpB+AZ+%#Y=GYB+xYR`&{*G zkXq!~fPch|(B-9tFfIq#C)Vj_9}c~~PB%Y9P+VlhSdbb4X8|`7MHJ2hZs_*%XIM8f z#MWtLzsUMLt%TBc&=yY}k@SXfeC-YzS=7I&_sq>9;ba(a^HICp;hyni5?PvK9D(lO zrmh<+i00gU)OFo}{9JZuQ20k;NYm`w_Yb) zvGsnerG73r@gi28I7U1Xyro;^tbS$z>(2tva1RI4l0L1` zG(YeQ?f3m7f|u%oSVKI!`IMl!tbbMj*xBX`exYP9r5EDx7}Ln#&m$ZKOm*5wGJ@wo$_CbIILzdA^7(nN40L=`>Clus_3A z&Bj|s+{~Di5{u#`4L{DV&zKeLU3XSoTW|}+WXwh62~7)^>JtgAu-O?l`NsG2*x+*C zkJ?RIxE}{Y%+`naRqRI@rdyw?bA|GY$yz+V9(I*|<;GmiKd313@w0DrRsef}p1qP9P_ zd=URni&r|;-DW`^&JK*E>Ab|>%>f>pqtRbt>nu4@l&0xAOH7=wYRmRY;f++jm~2bEIGzMUol7Yu%HYf%j0!H&w-+SxFGq_8L{EXY7^uuh_OxUB#PP z>`jsdNwcsWIXi>D66km*j26zB8ap&&+ohwuy>oMOJ%0CkaCXxRv1!`dKb#pE9vSJ+ z`RbRIK>PCB_)EtBR&N_80x78ZM0_2q!9KrMODs9kkL=1?kK@3Ubxe&;l9OjfCy!4| z9Y4*IQ~ZxdEPi~HwT~(a{uE_l$I*V=DC@Slz~bko*tSzRL$%8aU3_)yABV2pojAi% zr;wFB+Lyn(eO~+bbI-FB-l)r}h1AIsm(>ys8#LZGJm5E;nmRW*`PTW-Q{#`@@Mm`m ze_l&O8@~Rh(_f8!*3hu?|2l4`+Ko^i9?E30+2MhaTxMvnFWWyjh%1{z2n_UP2Xa|F XWU%nfiHyDP^F3IjZu&H%AzJ - annotation2: - annotation3: - annotation4: + annotation2: + annotation3: + annotation4: - annotation5: > - annotation6: > + annotation5: > + annotation6: > annotation7: ;Stop reading symbols at @ -- this test has three separate annotations - + bytes2: bytes2a: bytes3: @@ -68,108 +69,108 @@ bytes13: dict0: - dict1: + dict1: dict2: @"Missing close brace" dict2a: @"Missing close brace" dict3: @"Duplicate key" dict4: @"Unexpected close brace" - dict5: @"Missing value" - double0: - double+0: - double-0: - double1: - double2: - double3: + dict5: @"Missing value" + double0: + double+0: + double-0: + double1: + double2: + double3: double4: @"Fewer than 16 digits" double5: @"More than 16 digits" double6: @"Invalid chars" - double7: @"Positive infinity" - double8: @"Negative infinity" - double9: @"-qNaN" - double10: @"-qNaN" - double11: @"+qNaN" - double12: @"+qNaN" + double7: @"Positive infinity" + double8: @"Negative infinity" + double9: @"-qNaN" + double10: @"-qNaN" + double11: @"+qNaN" + double12: @"+qNaN" double13: @"Bad spacing" - double14: @"-sNaN" - double15: @"-sNaN" - double16: @"+sNaN" - double17: @"+sNaN" - float0: - float+0: - float-0: - float1: - float2: + double14: @"-sNaN" + double15: @"-sNaN" + double16: @"+sNaN" + double17: @"+sNaN" + float0: + float+0: + float-0: + float1: + float2: float3: @"Fewer than 8 digits" float4: @"More than 8 digits" float5: @"Invalid chars" - float6: @"Positive infinity" - float7: @"Negative infinity" - float8: @"+sNaN" - float9: @"+sNaN" - float10: @"-sNaN" - float11: @"-sNaN" + float6: @"Positive infinity" + float7: @"Negative infinity" + float8: @"+sNaN" + float9: @"+sNaN" + float10: @"-sNaN" + float11: @"-sNaN" float12: @"Bad spacing" - float13: @"+qNaN" - float14: @"+qNaN" - float15: @"-qNaN" - float16: @"-qNaN" - int-257: - int-256: - int-255: - int-254: - int-129: - int-128: - int-127: - int-4: - int-3: - int-2: - int-1: - int0: - int+0: - int-0: - int1: - int12: - int13: - int127: - int+127: - int128: - int255: - int256: - int32767: - int32768: - int65535: - int65536: - int131072: - int2500000000: + float13: @"+qNaN" + float14: @"+qNaN" + float15: @"-qNaN" + float16: @"-qNaN" + int-257: + int-256: + int-255: + int-254: + int-129: + int-128: + int-127: + int-4: + int-3: + int-2: + int-1: + int0: + int+0: + int-0: + int1: + int12: + int13: + int127: + int+127: + int128: + int255: + int256: + int32767: + int32768: + int65535: + int65536: + int131072: + int2500000000: int87112285931760246646623899502532662132736: list0: - list4: - list4a: - list5: + list4: + list4a: + list5: list6: list7: list8: @"Missing close bracket" list9: @"Unexpected close bracket" list10: @"Missing end byte" - list11: - list12: + list11: + list12: noinput0: @"No input at all" - embed0: - embed1: - embed2: + embed0: + embed1: + embed2: record1: >> record2: , >>>> - record3: "Dr">> + record3: "Dr">> record4: > - record5: > + record5: > record6: > - record7: > - record8: 3 4>> + record7: > + record8: 3 4>> record9: @"Missing record label" "> record10: @"Missing close-angle-bracket" record11: @"Unexpected close-angle-bracket" "> set0: - set1: + set1: set2: @"Missing close brace" set2a: @"Missing close brace" set3: @"Duplicate value" @@ -277,17 +278,17 @@ B7 B1 03 494473 B5 - A0 74 - A1 03 AF - A1 00 EA - A2 00 97 89 + B0 01 74 + B0 02 03 AF + B0 02 00 EA + B0 03 00 97 89 84 B1 05 5469746c65 B1 14 566965772066726f6d203135746820466c6f6f72 B1 05 5769647468 - A1 03 20 + B0 02 03 20 B1 06 486569676874 - A1 02 58 + B0 02 02 58 B1 08 416e696d61746564 B3 05 66616c7365 B1 09 5468756d626e61696c @@ -295,9 +296,9 @@ B1 03 55726c B1 26 687474703a2f2f7777772e6578616d706c652e636f6d2f696d6167652f343831393839393433 B1 05 5769647468 - A0 64 + B0 01 64 B1 06 486569676874 - A0 7D + B0 01 7D 84 84 84" @@ -324,8 +325,8 @@ b1 05 5374617465 b1 02 4341 b1 07 41646472657373 b1 00 b1 07 436f756e747279 b1 02 5553 - b1 08 4c61746974756465 83 4042e226809d4952 - b1 09 4c6f6e676974756465 83 c05e99566cf41f21 + b1 08 4c61746974756465 8708 4042e226809d4952 + b1 09 4c6f6e676974756465 8708 c05e99566cf41f21 b1 09 707265636973696f6e b1 03 7a6970 84 b7 @@ -334,8 +335,8 @@ b1 05 5374617465 b1 02 4341 b1 07 41646472657373 b1 00 b1 07 436f756e747279 b1 02 5553 - b1 08 4c61746974756465 83 4042af9d66adb403 - b1 09 4c6f6e676974756465 83 c05e81aa4fca42af + b1 08 4c61746974756465 8708 4042af9d66adb403 + b1 09 4c6f6e676974756465 8708 c05e81aa4fca42af b1 09 707265636973696f6e b1 03 7a6970 84 84" diff --git a/implementations/python/tests/test_preserves.py b/implementations/python/tests/test_preserves.py index 856caba..51d8baa 100644 --- a/implementations/python/tests/test_preserves.py +++ b/implementations/python/tests/test_preserves.py @@ -79,12 +79,12 @@ class BinaryCodecTests(PreservesTestCase): self.assertPreservesEqual(_varint(1000000000), _buf(128, 148, 235, 220, 3)) def test_simple_seq(self): - self._roundtrip([1,2,3,4], _buf(0xb5, 0x91, 0x92, 0x93, 0x94, 0x84), back=(1,2,3,4)) + self._roundtrip([1,2,3,4], _buf(0xb5, 0xb0, 0x01, 0x01, 0xb0, 0x01, 0x02, 0xb0, 0x01, 0x03, 0xb0, 0x01, 0x04, 0x84), back=(1,2,3,4)) self._roundtrip(iter([1,2,3,4]), - _buf(0xb5, 0x91, 0x92, 0x93, 0x94, 0x84), + _buf(0xb5, 0xb0, 0x01, 0x01, 0xb0, 0x01, 0x02, 0xb0, 0x01, 0x03, 0xb0, 0x01, 0x04, 0x84), back=(1,2,3,4), nondeterministic=True) - self._roundtrip((-2,-1,0,1), _buf(0xb5, 0x9E, 0x9F, 0x90, 0x91, 0x84)) + self._roundtrip((-2,-1,0,1), _buf(0xb5, 0xb0, 0x01, 0xFE, 0xb0, 0x01, 0xFF, 0xb0, 0x00, 0xb0, 0x01, 0x01, 0x84)) def test_str(self): self._roundtrip(u'hello', _buf(0xb1, 0x05, 'hello')) @@ -102,35 +102,35 @@ class BinaryCodecTests(PreservesTestCase): 0x84)) def test_signedinteger(self): - self._roundtrip(-257, _buf(0xa1, 0xFE, 0xFF)) - self._roundtrip(-256, _buf(0xa1, 0xFF, 0x00)) - self._roundtrip(-255, _buf(0xa1, 0xFF, 0x01)) - self._roundtrip(-254, _buf(0xa1, 0xFF, 0x02)) - self._roundtrip(-129, _buf(0xa1, 0xFF, 0x7F)) - self._roundtrip(-128, _buf(0xa0, 0x80)) - self._roundtrip(-127, _buf(0xa0, 0x81)) - self._roundtrip(-4, _buf(0xa0, 0xFC)) - self._roundtrip(-3, _buf(0x9D)) - self._roundtrip(-2, _buf(0x9E)) - self._roundtrip(-1, _buf(0x9F)) - self._roundtrip(0, _buf(0x90)) - self._roundtrip(1, _buf(0x91)) - self._roundtrip(12, _buf(0x9C)) - self._roundtrip(13, _buf(0xa0, 0x0D)) - self._roundtrip(127, _buf(0xa0, 0x7F)) - self._roundtrip(128, _buf(0xa1, 0x00, 0x80)) - self._roundtrip(255, _buf(0xa1, 0x00, 0xFF)) - self._roundtrip(256, _buf(0xa1, 0x01, 0x00)) - self._roundtrip(32767, _buf(0xa1, 0x7F, 0xFF)) - self._roundtrip(32768, _buf(0xa2, 0x00, 0x80, 0x00)) - self._roundtrip(65535, _buf(0xa2, 0x00, 0xFF, 0xFF)) - self._roundtrip(65536, _buf(0xa2, 0x01, 0x00, 0x00)) - self._roundtrip(131072, _buf(0xa2, 0x02, 0x00, 0x00)) + self._roundtrip(-257, _buf(0xb0, 0x02, 0xFE, 0xFF)) + self._roundtrip(-256, _buf(0xb0, 0x02, 0xFF, 0x00)) + self._roundtrip(-255, _buf(0xb0, 0x02, 0xFF, 0x01)) + self._roundtrip(-254, _buf(0xb0, 0x02, 0xFF, 0x02)) + self._roundtrip(-129, _buf(0xb0, 0x02, 0xFF, 0x7F)) + self._roundtrip(-128, _buf(0xb0, 0x01, 0x80)) + self._roundtrip(-127, _buf(0xb0, 0x01, 0x81)) + self._roundtrip(-4, _buf(0xb0, 0x01, 0xFC)) + self._roundtrip(-3, _buf(0xb0, 0x01, 0xFD)) + self._roundtrip(-2, _buf(0xb0, 0x01, 0xFE)) + self._roundtrip(-1, _buf(0xb0, 0x01, 0xFF)) + self._roundtrip(0, _buf(0xb0, 0x00)) + self._roundtrip(1, _buf(0xb0, 0x01, 0x01)) + self._roundtrip(12, _buf(0xb0, 0x01, 0x0C)) + self._roundtrip(13, _buf(0xb0, 0x01, 0x0D)) + self._roundtrip(127, _buf(0xb0, 0x01, 0x7F)) + self._roundtrip(128, _buf(0xb0, 0x02, 0x00, 0x80)) + self._roundtrip(255, _buf(0xb0, 0x02, 0x00, 0xFF)) + self._roundtrip(256, _buf(0xb0, 0x02, 0x01, 0x00)) + self._roundtrip(32767, _buf(0xb0, 0x02, 0x7F, 0xFF)) + self._roundtrip(32768, _buf(0xb0, 0x03, 0x00, 0x80, 0x00)) + self._roundtrip(65535, _buf(0xb0, 0x03, 0x00, 0xFF, 0xFF)) + self._roundtrip(65536, _buf(0xb0, 0x03, 0x01, 0x00, 0x00)) + self._roundtrip(131072, _buf(0xb0, 0x03, 0x02, 0x00, 0x00)) def test_floats(self): - self._roundtrip(Float(1.0), _buf(0x82, 0x3f, 0x80, 0, 0)) - self._roundtrip(1.0, _buf(0x83, 0x3f, 0xf0, 0, 0, 0, 0, 0, 0)) - self._roundtrip(-1.202e300, _buf(0x83, 0xfe, 0x3c, 0xb7, 0xb7, 0x59, 0xbf, 0x04, 0x26)) + self._roundtrip(Float(1.0), _buf(0x87, 0x04, 0x3f, 0x80, 0, 0)) + self._roundtrip(1.0, _buf(0x87, 0x08, 0x3f, 0xf0, 0, 0, 0, 0, 0, 0)) + self._roundtrip(-1.202e300, _buf(0x87, 0x08, 0xfe, 0x3c, 0xb7, 0xb7, 0x59, 0xbf, 0x04, 0x26)) def test_dict(self): self._roundtrip({ Symbol(u'a'): 1, @@ -139,9 +139,9 @@ class BinaryCodecTests(PreservesTestCase): ImmutableDict({ Symbol(u'first-name'): u'Elizabeth', }): { Symbol(u'surname'): u'Blackwell' } }, _buf(0xB7, - 0xb3, 0x01, "a", 0x91, + 0xb3, 0x01, "a", 0xb0, 0x01, 0x01, 0xb1, 0x01, "b", 0x81, - 0xb5, 0x91, 0x92, 0x93, 0x84, 0xb2, 0x01, "c", + 0xb5, 0xb0, 0x01, 0x01, 0xb0, 0x01, 0x02, 0xb0, 0x01, 0x03, 0x84, 0xb2, 0x01, "c", 0xB7, 0xb3, 0x0A, "first-name", 0xb1, 0x09, "Elizabeth", 0x84, 0xB7, 0xb3, 0x07, "surname", 0xb1, 0x09, "Blackwell", 0x84, 0x84), @@ -149,7 +149,7 @@ class BinaryCodecTests(PreservesTestCase): def test_iterator_stream(self): d = {u'a': 1, u'b': 2, u'c': 3} - r = r'b5(b5b1016.9.84){3}84' + r = r'b5(b5b1016.b0010.84){3}84' if hasattr(d, 'iteritems'): # python 2 bs = _e(d.iteritems()) @@ -187,13 +187,13 @@ class BinaryCodecTests(PreservesTestCase): objects.append(p) return len(objects) - 1 self.assertPreservesEqual(encode([Embedded(object()), Embedded(object())], encode_embedded = enc), - b'\xb5\x86\x90\x86\x91\x84') + b'\xb5\x86\xb0\x00\x86\xb0\x01\x01\x84') def test_decode_embedded(self): objects = [123, 234] def dec(v): return objects[v] - self.assertPreservesEqual(decode(b'\xb5\x86\x90\x86\x91\x84', decode_embedded = dec), + self.assertPreservesEqual(decode(b'\xb5\x86\xb0\x00\x86\xb0\x01\x01\x84', decode_embedded = dec), (Embedded(123), Embedded(234))) def load_binary_samples(): diff --git a/implementations/racket/preserves/preserves/jelly.rkt b/implementations/racket/preserves/preserves/jelly.rkt index 6ab8ef1..3884bb2 100644 --- a/implementations/racket/preserves/preserves/jelly.rkt +++ b/implementations/racket/preserves/preserves/jelly.rkt @@ -23,17 +23,11 @@ (match (next-byte) [#x80 #f] [#x81 #t] - [#x82 (bytes->float (next-bytes 4))] - [#x83 (bytes->double (next-bytes 8))] [#x84 '#:end] - [#x85 (let ((a (next))) - (match (next) - [(annotated as i) (annotated (cons a as) i)] - [i (annotated (list a) i)]))] [#x86 (embedded (next))] - [(? (between #x90 #x9C) v) (- v #x90)] - [(? (between #x9D #x9F) v) (- v #xA0)] - [(? (between #xA0 #xAF) v) (next-integer (- v #xA0 -1))] + [#x87 (match (next-byte) + [4 (bytes->float (next-bytes 4))] + [8 (bytes->double (next-bytes 8))])] [#xB0 (next-integer (next-varint))] [#xB1 (bytes->string/utf-8 (next-bytes (next-varint)))] [#xB2 (next-bytes (next-varint))] @@ -41,7 +35,8 @@ [#xB4 (apply (lambda (label . fields) (record label fields)) (next-items))] [#xB5 (next-items)] [#xB6 (list->set (next-items))] - [#xB7 (apply hash (next-items))])) + [#xB7 (apply hash (next-items))] + [#xBF (apply (lambda (underlying . anns) (annotated anns underlying)) (next-items))])) (define (next-items) (match (next) ['#:end '()] [v (cons v (next-items))])) @@ -56,14 +51,13 @@ (define (next-varint) (eof-guard (read-varint in-port))) (define (next-integer n) - (define acc0 (next-byte)) - (define acc (if (< acc0 128) acc0 (- acc0 256))) - (for/fold [(acc acc)] [(n (in-range (- n 1)))] (+ (* acc 256) (next-byte)))) + (cond [(zero? n) 0] + [else (define acc0 (next-byte)) + (define acc (if (< acc0 128) acc0 (- acc0 256))) + (for/fold [(acc acc)] [(n (in-range (- n 1)))] (+ (* acc 256) (next-byte)))])) (next))) -(define ((between lo hi) v) (<= lo v hi)) - (define (read-varint in-port) (let/ec return (let loop () @@ -80,26 +74,19 @@ (match v [#f (write-byte #x80 out-port)] [#t (write-byte #x81 out-port)] - [(float _) (write-byte #x82 out-port) (output-bytes (float->bytes v))] - [(? flonum?) (write-byte #x83 out-port) (output-bytes (double->bytes v))] - - [(annotated as v) - (for [(a (in-list as))] (write-byte #x85 out-port) (output a)) - (output v)] + [(float _) (write-byte #x87 out-port) (write-byte 4 out-port) (output-bytes (float->bytes v))] + [(? flonum?) (write-byte #x87 out-port) (write-byte 8 out-port) (output-bytes (double->bytes v))] [(embedded v) (write-byte #x86 out-port) (output v)] [(? integer?) - (cond [(<= -3 v -1) (write-byte (+ v #xA0) out-port)] - [(<= 0 v 12) (write-byte (+ v #x90) out-port)] - [else (define raw-bit-count (+ (integer-length v) 1)) ;; at least one sign bit - (define byte-count (quotient (+ raw-bit-count 7) 8)) - (if (<= byte-count 16) - (write-byte (+ byte-count #xA0 -1) out-port) - (begin (write-byte #xB0 out-port) - (write-varint byte-count out-port))) - (for [(shift (in-range (* byte-count 8) 0 -8))] - (write-byte (bitwise-bit-field v (- shift 8) shift) out-port))])] + (write-byte #xB0 out-port) + (define byte-count (cond [(zero? v) 0] + [else (define raw-bit-count (+ (integer-length v) 1)) + (quotient (+ raw-bit-count 7) 8)])) + (write-varint byte-count out-port) + (for [(shift (in-range (* byte-count 8) 0 -8))] + (write-byte (bitwise-bit-field v (- shift 8) shift) out-port))] [(? string?) (count-bytes 1 (string->bytes/utf-8 v))] [(? bytes?) (count-bytes 2 v)] @@ -109,6 +96,7 @@ [(? list?) (with-seq 5 (for-each output v))] [(? set?) (with-seq 6 (output-set v))] [(? hash?) (with-seq 7 (output-hash v))] + [(annotated as v) (with-seq 15 (output v) (for-each output as))] [_ (error 'write-preserve/binary "Invalid value: ~v" v)])) diff --git a/implementations/racket/preserves/preserves/read-binary.rkt b/implementations/racket/preserves/preserves/read-binary.rkt index 9459adb..4d9cee0 100644 --- a/implementations/racket/preserves/preserves/read-binary.rkt +++ b/implementations/racket/preserves/preserves/read-binary.rkt @@ -34,8 +34,6 @@ [(? eof-object?) (on-short)] [v v])))) -(define ((between lo hi) v) (<= lo v hi)) - (define (read-preserve/binary [in-port (current-input-port)] #:read-syntax? [read-syntax? #f] #:decode-embedded [decode-embedded0 #f] @@ -71,17 +69,12 @@ (match lead-byte [#x80 #f] [#x81 #t] - [#x82 (bytes->float (next-bytes 4))] - [#x83 (bytes->double (next-bytes 8))] [#x84 '#:end] - [#x85 (let ((a (next))) - (if read-annotations? - (annotate (next) a) - (next)))] [#x86 (embedded (decode-embedded (next)))] - [(? (between #x90 #x9C) v) (- v #x90)] - [(? (between #x9D #x9F) v) (- v #xA0)] - [(? (between #xA0 #xAF) v) (next-integer (- v #xA0 -1))] + [#x87 (match (next-varint) + [4 (bytes->float (next-bytes 4))] + [8 (bytes->double (next-bytes 8))] + [n (return (on-fail "Invalid Preserves IEEE754 size: ~v" n))])] [#xB0 (next-integer (next-varint))] [#xB1 (bytes->string/utf-8 (next-bytes (next-varint)))] [#xB2 (next-bytes (next-varint))] @@ -90,6 +83,7 @@ [#xB5 (next-items)] [#xB6 (list->set (next-items))] [#xB7 (build-dictionary (next-items))] + [#xBF (apply annotate (next-items))] [_ (return (on-fail "Invalid Preserves binary tag: ~v" lead-byte))])) (define (eof-guard v) diff --git a/implementations/racket/preserves/preserves/tests/samples.pr b/implementations/racket/preserves/preserves/tests/samples.pr index a29deb4..5d9d642 100644 --- a/implementations/racket/preserves/preserves/tests/samples.pr +++ b/implementations/racket/preserves/preserves/tests/samples.pr @@ -43,16 +43,17 @@ "13/14 and 16/17, depending on how they wish to treat end-of-stream conditions." ]> - annotation2: - annotation3: - annotation4: + annotation2: + annotation3: + annotation4: - annotation5: > - annotation6: > + annotation5: > + annotation6: > annotation7: ;Stop reading symbols at @ -- this test has three separate annotations - + bytes2: bytes2a: bytes3: @@ -68,108 +69,108 @@ bytes13: dict0: - dict1: + dict1: dict2: @"Missing close brace" dict2a: @"Missing close brace" dict3: @"Duplicate key" dict4: @"Unexpected close brace" - dict5: @"Missing value" - double0: - double+0: - double-0: - double1: - double2: - double3: + dict5: @"Missing value" + double0: + double+0: + double-0: + double1: + double2: + double3: double4: @"Fewer than 16 digits" double5: @"More than 16 digits" double6: @"Invalid chars" - double7: @"Positive infinity" - double8: @"Negative infinity" - double9: @"-qNaN" - double10: @"-qNaN" - double11: @"+qNaN" - double12: @"+qNaN" + double7: @"Positive infinity" + double8: @"Negative infinity" + double9: @"-qNaN" + double10: @"-qNaN" + double11: @"+qNaN" + double12: @"+qNaN" double13: @"Bad spacing" - double14: @"-sNaN" - double15: @"-sNaN" - double16: @"+sNaN" - double17: @"+sNaN" - float0: - float+0: - float-0: - float1: - float2: + double14: @"-sNaN" + double15: @"-sNaN" + double16: @"+sNaN" + double17: @"+sNaN" + float0: + float+0: + float-0: + float1: + float2: float3: @"Fewer than 8 digits" float4: @"More than 8 digits" float5: @"Invalid chars" - float6: @"Positive infinity" - float7: @"Negative infinity" - float8: @"+sNaN" - float9: @"+sNaN" - float10: @"-sNaN" - float11: @"-sNaN" + float6: @"Positive infinity" + float7: @"Negative infinity" + float8: @"+sNaN" + float9: @"+sNaN" + float10: @"-sNaN" + float11: @"-sNaN" float12: @"Bad spacing" - float13: @"+qNaN" - float14: @"+qNaN" - float15: @"-qNaN" - float16: @"-qNaN" - int-257: - int-256: - int-255: - int-254: - int-129: - int-128: - int-127: - int-4: - int-3: - int-2: - int-1: - int0: - int+0: - int-0: - int1: - int12: - int13: - int127: - int+127: - int128: - int255: - int256: - int32767: - int32768: - int65535: - int65536: - int131072: - int2500000000: + float13: @"+qNaN" + float14: @"+qNaN" + float15: @"-qNaN" + float16: @"-qNaN" + int-257: + int-256: + int-255: + int-254: + int-129: + int-128: + int-127: + int-4: + int-3: + int-2: + int-1: + int0: + int+0: + int-0: + int1: + int12: + int13: + int127: + int+127: + int128: + int255: + int256: + int32767: + int32768: + int65535: + int65536: + int131072: + int2500000000: int87112285931760246646623899502532662132736: list0: - list4: - list4a: - list5: + list4: + list4a: + list5: list6: list7: list8: @"Missing close bracket" list9: @"Unexpected close bracket" list10: @"Missing end byte" - list11: - list12: + list11: + list12: noinput0: @"No input at all" - embed0: - embed1: - embed2: + embed0: + embed1: + embed2: record1: >> record2: , >>>> - record3: "Dr">> + record3: "Dr">> record4: > - record5: > + record5: > record6: > - record7: > - record8: 3 4>> + record7: > + record8: 3 4>> record9: @"Missing record label" "> record10: @"Missing close-angle-bracket" record11: @"Unexpected close-angle-bracket" "> set0: - set1: + set1: set2: @"Missing close brace" set2a: @"Missing close brace" set3: @"Duplicate value" @@ -277,17 +278,17 @@ B7 B1 03 494473 B5 - A0 74 - A1 03 AF - A1 00 EA - A2 00 97 89 + B0 01 74 + B0 02 03 AF + B0 02 00 EA + B0 03 00 97 89 84 B1 05 5469746c65 B1 14 566965772066726f6d203135746820466c6f6f72 B1 05 5769647468 - A1 03 20 + B0 02 03 20 B1 06 486569676874 - A1 02 58 + B0 02 02 58 B1 08 416e696d61746564 B3 05 66616c7365 B1 09 5468756d626e61696c @@ -295,9 +296,9 @@ B1 03 55726c B1 26 687474703a2f2f7777772e6578616d706c652e636f6d2f696d6167652f343831393839393433 B1 05 5769647468 - A0 64 + B0 01 64 B1 06 486569676874 - A0 7D + B0 01 7D 84 84 84" @@ -324,8 +325,8 @@ b1 05 5374617465 b1 02 4341 b1 07 41646472657373 b1 00 b1 07 436f756e747279 b1 02 5553 - b1 08 4c61746974756465 83 4042e226809d4952 - b1 09 4c6f6e676974756465 83 c05e99566cf41f21 + b1 08 4c61746974756465 8708 4042e226809d4952 + b1 09 4c6f6e676974756465 8708 c05e99566cf41f21 b1 09 707265636973696f6e b1 03 7a6970 84 b7 @@ -334,8 +335,8 @@ b1 05 5374617465 b1 02 4341 b1 07 41646472657373 b1 00 b1 07 436f756e747279 b1 02 5553 - b1 08 4c61746974756465 83 4042af9d66adb403 - b1 09 4c6f6e676974756465 83 c05e81aa4fca42af + b1 08 4c61746974756465 8708 4042af9d66adb403 + b1 09 4c6f6e676974756465 8708 c05e81aa4fca42af b1 09 707265636973696f6e b1 03 7a6970 84 84" diff --git a/implementations/racket/preserves/preserves/write-binary.rkt b/implementations/racket/preserves/preserves/write-binary.rkt index d84b559..1f548eb 100644 --- a/implementations/racket/preserves/preserves/write-binary.rkt +++ b/implementations/racket/preserves/preserves/write-binary.rkt @@ -88,30 +88,22 @@ [#t (output-byte #x81)] [(float _) - (output-byte #x82) + (output-byte #x87) + (output-byte 4) (output-bytes (float->bytes v))] [(? flonum?) - (output-byte #x83) + (output-byte #x87) + (output-byte 8) (output-bytes (double->bytes v))] - [(annotated as _ v) - (when write-annotations? - (for [(a (in-list as))] - (output-byte #x85) - (output a))) - (output v)] - [(? integer?) - (cond [(<= -3 v -1) (output-byte (+ v #xA0))] - [(<= 0 v 12) (output-byte (+ v #x90))] - [else (define raw-bit-count (+ (integer-length v) 1)) ;; at least one sign bit - (define byte-count (quotient (+ raw-bit-count 7) 8)) - (if (<= byte-count 16) - (output-byte (+ byte-count #xA0 -1)) - (begin (output-byte #xB0) - (output-varint byte-count))) - (for [(shift (in-range (* byte-count 8) 0 -8))] - (output-byte (bitwise-bit-field v (- shift 8) shift)))])] + (output-byte #xB0) + (define byte-count (cond [(zero? v) 0] + [else (define raw-bit-count (+ (integer-length v) 1)) + (quotient (+ raw-bit-count 7) 8)])) + (output-varint byte-count) + (for [(shift (in-range (* byte-count 8) 0 -8))] + (output-byte (bitwise-bit-field v (- shift 8) shift)))] [(? string?) (count-bytes 1 (string->bytes/utf-8 v))] [(? bytes?) (count-bytes 2 v)] @@ -122,6 +114,11 @@ [(? set?) (with-seq 6 (output-set v))] [(? dict?) (with-seq 7 (output-dict v))] + [(annotated as _ v) + (if (and write-annotations? (pair? as)) + (with-seq 15 (output v) (output-all as)) + (output v))] + [(embedded value) (output-byte #x86) (output (encode-embedded value))] diff --git a/implementations/rust/preserves-path/path.bin b/implementations/rust/preserves-path/path.bin index e5241a0..5679a63 100644 --- a/implementations/rust/preserves-path/path.bin +++ b/implementations/rust/preserves-path/path.bin @@ -1,4 +1,4 @@ -´³schema·³version‘³ definitions·³Axis´³orµµ±values´³rec´³lit³values„´³tupleµ„„„„µ± descendants´³rec´³lit³ descendants„´³tupleµ„„„„µ±at´³rec´³lit³at„´³tupleµ´³named³key³any„„„„„µ±label´³rec´³lit³label„´³tupleµ„„„„µ±keys´³rec´³lit³keys„´³tupleµ„„„„µ±length´³rec´³lit³length„´³tupleµ„„„„µ± annotations´³rec´³lit³ annotations„´³tupleµ„„„„µ±embedded´³rec´³lit³embedded„´³tupleµ„„„„µ±parse´³rec´³lit³parse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„µ±unparse´³rec´³lit³unparse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„„„³Step´³orµµ±Axis´³refµ„³Axis„„µ±Filter´³refµ„³Filter„„µ±Function´³refµ„³Function„„„„³Filter´³orµµ±nop´³rec´³lit³nop„´³tupleµ„„„„µ±compare´³rec´³lit³compare„´³tupleµ´³named³op´³refµ„³ +´³schema·³version°³ definitions·³Axis´³orµµ±values´³rec´³lit³values„´³tupleµ„„„„µ± descendants´³rec´³lit³ descendants„´³tupleµ„„„„µ±at´³rec´³lit³at„´³tupleµ´³named³key³any„„„„„µ±label´³rec´³lit³label„´³tupleµ„„„„µ±keys´³rec´³lit³keys„´³tupleµ„„„„µ±length´³rec´³lit³length„´³tupleµ„„„„µ± annotations´³rec´³lit³ annotations„´³tupleµ„„„„µ±embedded´³rec´³lit³embedded„´³tupleµ„„„„µ±parse´³rec´³lit³parse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„µ±unparse´³rec´³lit³unparse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„„„³Step´³orµµ±Axis´³refµ„³Axis„„µ±Filter´³refµ„³Filter„„µ±Function´³refµ„³Function„„„„³Filter´³orµµ±nop´³rec´³lit³nop„´³tupleµ„„„„µ±compare´³rec´³lit³compare„´³tupleµ´³named³op´³refµ„³ Comparison„„´³named³literal³any„„„„„µ±regex´³rec´³lit³regex„´³tupleµ´³named³regex´³atom³String„„„„„„µ±test´³rec´³lit³test„´³tupleµ´³named³pred´³refµ„³ Predicate„„„„„„µ±real´³rec´³lit³real„´³tupleµ„„„„µ±int´³rec´³lit³int„´³tupleµ„„„„µ±kind´³rec´³lit³kind„´³tupleµ´³named³kind´³refµ„³ ValueKind„„„„„„„„³Function´³rec´³lit³count„´³tupleµ´³named³selector´³refµ„³Selector„„„„„³Selector´³seqof´³refµ„³Step„„³ Predicate´³orµµ±Selector´³refµ„³Selector„„µ±not´³rec´³lit³not„´³tupleµ´³named³pred´³refµ„³ Predicate„„„„„„µ±or´³rec´³lit³or„´³tupleµ´³named³preds´³seqof´³refµ„³ Predicate„„„„„„„µ±and´³rec´³lit³and„´³tupleµ´³named³preds´³seqof´³refµ„³ Predicate„„„„„„„„„³ ValueKind´³orµµ±Boolean´³lit³Boolean„„µ±Float´³lit³Float„„µ±Double´³lit³Double„„µ± SignedInteger´³lit³ SignedInteger„„µ±String´³lit³String„„µ± ByteString´³lit³ ByteString„„µ±Symbol´³lit³Symbol„„µ±Record´³lit³Record„„µ±Sequence´³lit³Sequence„„µ±Set´³lit³Set„„µ± diff --git a/implementations/rust/preserves/src/value/boundary.rs b/implementations/rust/preserves/src/value/boundary.rs index 86e85cf..777560e 100644 --- a/implementations/rust/preserves/src/value/boundary.rs +++ b/implementations/rust/preserves/src/value/boundary.rs @@ -6,8 +6,6 @@ pub struct Type { #[derive(Clone, Debug)] pub enum Item { - Annotation, - AnnotatedValue, DictionaryKey, DictionaryValue, RecordField, diff --git a/implementations/rust/preserves/src/value/packed/constants.rs b/implementations/rust/preserves/src/value/packed/constants.rs index 97a9a54..fafb1bd 100644 --- a/implementations/rust/preserves/src/value/packed/constants.rs +++ b/implementations/rust/preserves/src/value/packed/constants.rs @@ -5,13 +5,9 @@ use std::io; pub enum Tag { False, True, - Float, - Double, End, - Annotation, Embedded, - SmallInteger(i8), - MediumInteger(u8), + Ieee754, SignedInteger, String, ByteString, @@ -20,6 +16,7 @@ pub enum Tag { Sequence, Set, Dictionary, + Annotated, } #[derive(Debug, PartialEq, Eq)] @@ -46,14 +43,9 @@ impl TryFrom for Tag { match v { 0x80 => Ok(Self::False), 0x81 => Ok(Self::True), - 0x82 => Ok(Self::Float), - 0x83 => Ok(Self::Double), 0x84 => Ok(Self::End), - 0x85 => Ok(Self::Annotation), 0x86 => Ok(Self::Embedded), - 0x90..=0x9c => Ok(Self::SmallInteger((v - 0x90) as i8)), - 0x9d..=0x9f => Ok(Self::SmallInteger((v - 0x90) as i8 - 16)), - 0xa0..=0xaf => Ok(Self::MediumInteger(v - 0xa0 + 1)), + 0x87 => Ok(Self::Ieee754), 0xb0 => Ok(Self::SignedInteger), 0xb1 => Ok(Self::String), 0xb2 => Ok(Self::ByteString), @@ -62,6 +54,7 @@ impl TryFrom for Tag { 0xb5 => Ok(Self::Sequence), 0xb6 => Ok(Self::Set), 0xb7 => Ok(Self::Dictionary), + 0xbf => Ok(Self::Annotated), _ => Err(InvalidTag(v)), } } @@ -72,19 +65,9 @@ impl From for u8 { match v { Tag::False => 0x80, Tag::True => 0x81, - Tag::Float => 0x82, - Tag::Double => 0x83, Tag::End => 0x84, - Tag::Annotation => 0x85, Tag::Embedded => 0x86, - Tag::SmallInteger(v) => { - if v < 0 { - (v + 16) as u8 + 0x90 - } else { - v as u8 + 0x90 - } - } - Tag::MediumInteger(count) => count - 1 + 0xa0, + Tag::Ieee754 => 0x87, Tag::SignedInteger => 0xb0, Tag::String => 0xb1, Tag::ByteString => 0xb2, @@ -93,6 +76,7 @@ impl From for u8 { Tag::Sequence => 0xb5, Tag::Set => 0xb6, Tag::Dictionary => 0xb7, + Tag::Annotated => 0xbf, } } } diff --git a/implementations/rust/preserves/src/value/packed/reader.rs b/implementations/rust/preserves/src/value/packed/reader.rs index 5ccc193..7e42218 100644 --- a/implementations/rust/preserves/src/value/packed/reader.rs +++ b/implementations/rust/preserves/src/value/packed/reader.rs @@ -115,37 +115,56 @@ impl<'de, 'src, N: NestedValue, Dec: DomainDecode, S: BinarySource< } #[inline(always)] - fn peek_next_nonannotation_tag(&mut self) -> ReaderResult { - loop { - match Tag::try_from(self.peek()?)? { - Tag::Annotation => { - self.skip()?; - self.skip_value()?; + fn try_next_nonannotation(&mut self, f: F) -> ReaderResult + where + F: FnOnce(&mut Self, Tag) -> ReaderResult, + { + let m = self.source.mark()?; + match Tag::try_from(self.peek()?)? { + Tag::Annotated => { + self.skip()?; + match f(self, Tag::try_from(self.peek()?)?) { + Ok(v) => { + self.skip_annotations()?; + Ok(v) + } + Err(e) => { + self.source.restore(&m)?; + Err(e) + } + } + } + other => match f(self, other) { + Ok(v) => Ok(v), + Err(e) => { + self.source.restore(&m)?; + Err(e) } - other => return Ok(other), } } } fn next_atomic(&mut self, expected_tag: Tag, k: ExpectedKind) -> ReaderResult> { - let actual_tag = self.peek_next_nonannotation_tag()?; - if actual_tag == expected_tag { - self.skip()?; - let count = self.varint()?; - Ok(self.readbytes(count)?) - } else { - Err(self.expected(k)) - } + self.try_next_nonannotation(|_, actual_tag| { + if actual_tag == expected_tag { + self.skip()?; + let count = self.varint()?; + Ok(self.readbytes(count)?) + } else { + Err(self.expected(k)) + } + }) } fn next_compound(&mut self, expected_tag: Tag, k: ExpectedKind) -> ReaderResult<()> { - let actual_tag = self.peek_next_nonannotation_tag()?; - if actual_tag == expected_tag { - self.skip()?; - Ok(()) - } else { - Err(self.expected(k)) - } + self.try_next_nonannotation(|_, actual_tag| { + if actual_tag == expected_tag { + self.skip()?; + Ok(()) + } else { + Err(self.expected(k)) + } + }) } #[inline(always)] @@ -203,31 +222,18 @@ impl<'de, 'src, N: NestedValue, Dec: DomainDecode, S: BinarySource< where F: FnOnce(u128) -> Option, { - let tag = self.peek_next_nonannotation_tag()?; - match tag { - Tag::SmallInteger(v) => { - self.skip()?; - if v < 0 { - Err(out_of_range(v)) - } else { - f(v as u128).ok_or_else(|| out_of_range(v)) + self.try_next_nonannotation(|_, tag| { + match tag { + Tag::SignedInteger => { + self.skip()?; + let count = self.varint()?; + let n = &self.read_signed_integer(count)?; + let i = n.try_into().map_err(|_| out_of_range(n))?; + f(i).ok_or_else(|| out_of_range(i)) } + _ => Err(self.expected(ExpectedKind::SignedInteger)), } - Tag::MediumInteger(count) => { - self.skip()?; - let n = &self.read_signed_integer(count.into())?; - let i = n.try_into().map_err(|_| out_of_range(n))?; - f(i).ok_or_else(|| out_of_range(i)) - } - Tag::SignedInteger => { - self.skip()?; - let count = self.varint()?; - let n = &self.read_signed_integer(count)?; - let i = n.try_into().map_err(|_| out_of_range(n))?; - f(i).ok_or_else(|| out_of_range(i)) - } - _ => Err(self.expected(ExpectedKind::SignedInteger)), - } + }) } #[inline(always)] @@ -235,43 +241,23 @@ impl<'de, 'src, N: NestedValue, Dec: DomainDecode, S: BinarySource< where F: FnOnce(i128) -> Option, { - let tag = self.peek_next_nonannotation_tag()?; - match tag { - Tag::SmallInteger(v) => { - self.skip()?; - f(v.into()).ok_or_else(|| out_of_range(v)) + self.try_next_nonannotation(|_, tag| { + match tag { + Tag::SignedInteger => { + self.skip()?; + let count = self.varint()?; + let n = &self.read_signed_integer(count)?; + let i = n.try_into().map_err(|_| out_of_range(n))?; + f(i).ok_or_else(|| out_of_range(i)) + } + _ => Err(self.expected(ExpectedKind::SignedInteger)), } - Tag::MediumInteger(count) => { - self.skip()?; - let n = &self.read_signed_integer(count.into())?; - let i = n.try_into().map_err(|_| out_of_range(n))?; - f(i).ok_or_else(|| out_of_range(i)) - } - Tag::SignedInteger => { - self.skip()?; - let count = self.varint()?; - let n = &self.read_signed_integer(count)?; - let i = n.try_into().map_err(|_| out_of_range(n))?; - f(i).ok_or_else(|| out_of_range(i)) - } - _ => Err(self.expected(ExpectedKind::SignedInteger)), - } - } - - fn gather_annotations(&mut self) -> io::Result> { - let mut annotations = vec![self.demand_next(true)?]; - while Tag::try_from(self.peek()?)? == Tag::Annotation { - self.skip()?; - annotations.push(self.demand_next(true)?); - } - Ok(annotations) + }) } fn skip_annotations(&mut self) -> io::Result<()> { - self.skip_value()?; - while Tag::try_from(self.peek()?)? == Tag::Annotation { - self.skip()?; - self.skip_value()?; + while !self.peekend()? { + self.skip_value()? } Ok(()) } @@ -296,39 +282,22 @@ impl<'de, 'src, N: NestedValue, Dec: DomainDecode, S: BinarySource< Ok(Some(match Tag::try_from(self.read()?)? { Tag::False => N::new(false), Tag::True => N::new(true), - Tag::Float => { - let mut bs = [0; 4]; - self.readbytes_into(&mut bs)?; - Value::from(f32::from_bits(u32::from_be_bytes(bs))).wrap() - } - Tag::Double => { - let mut bs = [0; 8]; - self.readbytes_into(&mut bs)?; - Value::from(f64::from_bits(u64::from_be_bytes(bs))).wrap() - } - Tag::Annotation => { - if read_annotations { - let mut annotations = self.gather_annotations()?; - let (existing_annotations, v) = self.demand_next(read_annotations)?.pieces(); - annotations.extend_from_slice(existing_annotations.slice()); - N::wrap(Annotations::new(Some(annotations)), v) - } else { - self.skip_annotations()?; - self.demand_next(read_annotations)? - } - } Tag::Embedded => Value::Embedded( self.decode_embedded .decode_embedded(self.source, read_annotations)?, - ) - .wrap(), - Tag::SmallInteger(v) => { - // TODO: prebuild these in value.rs - Value::from(v).wrap() - } - Tag::MediumInteger(count) => { - let n = self.read_signed_integer(count.into())?; - Value::SignedInteger(n).wrap() + ).wrap(), + Tag::Ieee754 => match self.varint()? { + 4 => { + let mut bs = [0; 4]; + self.readbytes_into(&mut bs)?; + Value::from(f32::from_bits(u32::from_be_bytes(bs))).wrap() + } + 8 => { + let mut bs = [0; 8]; + self.readbytes_into(&mut bs)?; + Value::from(f64::from_bits(u64::from_be_bytes(bs))).wrap() + } + _ => return Err(io_syntax_error("Invalid IEEE754 size")), } Tag::SignedInteger => { let count = self.varint()?; @@ -383,6 +352,28 @@ impl<'de, 'src, N: NestedValue, Dec: DomainDecode, S: BinarySource< } Value::Dictionary(d).wrap() } + Tag::Annotated => { + match self.next_upto_end(read_annotations)? { + None => return Err(io_syntax_error("Missing underlying annotated value")), + Some(underlying) => { + if read_annotations { + let mut annotations = Vec::new(); + while let Some(v) = self.next_upto_end(read_annotations)? { + annotations.push(v); + } + if annotations.is_empty() { + return Err(io_syntax_error("Too few annotations in annotated value")); + } + let (existing_annotations, v) = underlying.pieces(); + annotations.extend_from_slice(existing_annotations.slice()); + N::wrap(Annotations::new(Some(annotations)), v) + } else { + self.skip_annotations()?; + underlying + } + } + } + } tag @ Tag::End => { return Err(io_syntax_error(&format!("Invalid tag: {:?}", tag))); } @@ -399,17 +390,19 @@ impl<'de, 'src, N: NestedValue, Dec: DomainDecode, S: BinarySource< #[inline(always)] fn open_sequence_or_set(&mut self) -> ReaderResult { - match self.peek_next_nonannotation_tag()? { - Tag::Sequence => { - self.skip()?; - Ok(B::Item::SequenceValue) + self.try_next_nonannotation(|_, tag| { + match tag { + Tag::Sequence => { + self.skip()?; + Ok(B::Item::SequenceValue) + } + Tag::Set => { + self.skip()?; + Ok(B::Item::SetValue) + } + _ => Err(self.expected(ExpectedKind::SequenceOrSet)), } - Tag::Set => { - self.skip()?; - Ok(B::Item::SetValue) - } - _ => Err(self.expected(ExpectedKind::SequenceOrSet)), - } + }) } #[inline(always)] @@ -427,6 +420,14 @@ impl<'de, 'src, N: NestedValue, Dec: DomainDecode, S: BinarySource< self.next_compound(Tag::Dictionary, ExpectedKind::Dictionary) } + #[inline(always)] + fn open_annotations(&mut self, annotations: Option<&mut Vec>) -> ReaderResult<()> { + } + + #[inline(always)] + fn close_annotations(&mut self, annotations: Option<&mut Vec>) -> ReaderResult<()> { + } + #[inline(always)] fn boundary(&mut self, _b: &B::Type) -> ReaderResult<()> { Ok(()) @@ -460,101 +461,76 @@ impl<'de, 'src, N: NestedValue, Dec: DomainDecode, S: BinarySource< } fn next_token(&mut self, read_embedded_annotations: bool) -> io::Result> { - loop { - return Ok(match Tag::try_from(self.peek()?)? { - Tag::Embedded => { - self.skip()?; - Token::Embedded( - self.decode_embedded - .decode_embedded(self.source, read_embedded_annotations)?, - ) - } - Tag::False + Ok(match Tag::try_from(self.peek()?)? { + Tag::Embedded => { + self.skip()?; + Token::Embedded( + self.decode_embedded + .decode_embedded(self.source, read_embedded_annotations)?, + ) + } + Tag::False | Tag::True - | Tag::Float - | Tag::Double - | Tag::SmallInteger(_) - | Tag::MediumInteger(_) + | Tag::Ieee754 | Tag::SignedInteger | Tag::String | Tag::ByteString | Tag::Symbol => Token::Atom(self.demand_next(false)?), - Tag::Record => { - self.skip()?; - Token::Compound(CompoundClass::Record) - } - Tag::Sequence => { - self.skip()?; - Token::Compound(CompoundClass::Sequence) - } - Tag::Set => { - self.skip()?; - Token::Compound(CompoundClass::Set) - } - Tag::Dictionary => { - self.skip()?; - Token::Compound(CompoundClass::Dictionary) - } - - Tag::End => { - self.skip()?; - Token::End - } - - Tag::Annotation => { - self.skip()?; - self.skip_annotations()?; - continue; - } - }); - } - } - - fn next_annotations_and_token(&mut self) -> io::Result<(Vec, Token)> { - match Tag::try_from(self.peek()?)? { - Tag::Annotation => { + Tag::Record => { self.skip()?; - let annotations = self.gather_annotations()?; - Ok((annotations, self.next_token(true)?)) + Token::Compound(CompoundClass::Record) } - _ => Ok((Vec::new(), self.next_token(true)?)), - } + Tag::Sequence => { + self.skip()?; + Token::Compound(CompoundClass::Sequence) + } + Tag::Set => { + self.skip()?; + Token::Compound(CompoundClass::Set) + } + Tag::Dictionary => { + self.skip()?; + Token::Compound(CompoundClass::Dictionary) + } + + Tag::End => { + self.skip()?; + Token::End + } + + Tag::Annotated => Token::AnnotationStart, + }) } #[inline(always)] fn next_boolean(&mut self) -> ReaderResult { - match self.peek_next_nonannotation_tag()? { - Tag::False => { - self.skip()?; - Ok(false) + self.try_next_nonannotation(|_, tag| { + match tag { + Tag::False => { + self.skip()?; + Ok(false) + } + Tag::True => { + self.skip()?; + Ok(true) + } + _ => Err(self.expected(ExpectedKind::Boolean)), } - Tag::True => { - self.skip()?; - Ok(true) - } - _ => Err(self.expected(ExpectedKind::Boolean)), - } + }) } fn next_signedinteger(&mut self) -> ReaderResult { - let tag = self.peek_next_nonannotation_tag()?; - match tag { - Tag::SmallInteger(v) => { - self.skip()?; - Ok(SignedInteger::from(v as i32)) + self.try_next_nonannotation(|_, tag| { + match tag { + Tag::SignedInteger => { + self.skip()?; + let count = self.varint()?; + Ok(self.read_signed_integer(count)?) + } + _ => Err(self.expected(ExpectedKind::SignedInteger)), } - Tag::MediumInteger(count) => { - self.skip()?; - Ok(self.read_signed_integer(count.into())?) - } - Tag::SignedInteger => { - self.skip()?; - let count = self.varint()?; - Ok(self.read_signed_integer(count)?) - } - _ => Err(self.expected(ExpectedKind::SignedInteger)), - } + }) } fn next_i8(&mut self) -> ReaderResult { @@ -590,39 +566,50 @@ impl<'de, 'src, N: NestedValue, Dec: DomainDecode, S: BinarySource< } fn next_f32(&mut self) -> ReaderResult { - match self.peek_next_nonannotation_tag()? { - Tag::Float => { - self.skip()?; - let mut bs = [0; 4]; - self.readbytes_into(&mut bs)?; - Ok(f32::from_bits(u32::from_be_bytes(bs))) + self.try_next_nonannotation(|_, tag| { + if tag == Tag::Ieee754 { + match self.varint()? { + 4 => { + let mut bs = [0; 4]; + self.readbytes_into(&mut bs)?; + Ok(f32::from_bits(u32::from_be_bytes(bs))) + } + 8 => { + self.skip()?; + let mut bs = [0; 8]; + self.readbytes_into(&mut bs)?; + Ok(f64::from_bits(u64::from_be_bytes(bs)) as f32) + } + _ => Err(io_syntax_error("Invalid IEEE754 size"))?, + } + } else { + Err(self.expected(ExpectedKind::Float)) } - Tag::Double => { - self.skip()?; - let mut bs = [0; 8]; - self.readbytes_into(&mut bs)?; - Ok(f64::from_bits(u64::from_be_bytes(bs)) as f32) - } - _ => Err(self.expected(ExpectedKind::Float)), - } + }) } fn next_f64(&mut self) -> ReaderResult { - match self.peek_next_nonannotation_tag()? { - Tag::Float => { - self.skip()?; - let mut bs = [0; 4]; - self.readbytes_into(&mut bs)?; - Ok(f32::from_bits(u32::from_be_bytes(bs)) as f64) + self.try_next_nonannotation(|_, tag| { + if tag == Tag::Ieee754 { + match self.varint()? { + 4 => { + self.skip()?; + let mut bs = [0; 4]; + self.readbytes_into(&mut bs)?; + Ok(f32::from_bits(u32::from_be_bytes(bs)) as f64) + } + 8 => { + self.skip()?; + let mut bs = [0; 8]; + self.readbytes_into(&mut bs)?; + Ok(f64::from_bits(u64::from_be_bytes(bs))) + } + _ => Err(io_syntax_error("Invalid IEEE754 size"))?, + } + } else { + Err(self.expected(ExpectedKind::Double)) } - Tag::Double => { - self.skip()?; - let mut bs = [0; 8]; - self.readbytes_into(&mut bs)?; - Ok(f64::from_bits(u64::from_be_bytes(bs))) - } - _ => Err(self.expected(ExpectedKind::Double)), - } + }) } fn next_str(&mut self) -> ReaderResult> { diff --git a/implementations/rust/preserves/src/value/packed/writer.rs b/implementations/rust/preserves/src/value/packed/writer.rs index 0363501..29aad6a 100644 --- a/implementations/rust/preserves/src/value/packed/writer.rs +++ b/implementations/rust/preserves/src/value/packed/writer.rs @@ -49,13 +49,8 @@ impl PackedWriter { } #[inline(always)] - pub fn write_medium_integer(&mut self, bs: &[u8]) -> io::Result<()> { - let count: u8 = bs.len().try_into().unwrap(); - if !(1..=16).contains(&count) { - panic!("Invalid medium_integer count: {}", count) - } - self.write_byte(Tag::MediumInteger(count).into())?; - self.w().write_all(bs) + pub fn write_integer(&mut self, bs: &[u8]) -> io::Result<()> { + self.write_atom(Tag::SignedInteger, bs) } #[inline(always)] @@ -151,9 +146,6 @@ impl WriteWriter for BinaryOrderWriter { impl CompoundWriter for PackedWriter { #[inline(always)] fn boundary(&mut self, b: &B::Type) -> io::Result<()> { - if let Some(B::Item::Annotation) = b.opening { - self.write_tag(Tag::Annotation)?; - } Ok(()) } } @@ -310,22 +302,21 @@ impl Writer for PackedWriter { #[inline(always)] fn write_f32(&mut self, v: f32) -> io::Result<()> { - self.write_tag(Tag::Float)?; + self.write_tag(Tag::Ieee754)?; + self.write_byte(4)?; self.write_raw_bytes(&u32::to_be_bytes(f32::to_bits(v))) } #[inline(always)] fn write_f64(&mut self, v: f64) -> io::Result<()> { - self.write_tag(Tag::Double)?; + self.write_tag(Tag::Ieee754)?; + self.write_byte(8)?; self.write_raw_bytes(&u64::to_be_bytes(f64::to_bits(v))) } #[inline(always)] fn write_i8(&mut self, v: i8) -> io::Result<()> { - if v >= -3 && v <= 12 { - return self.write_tag(Tag::SmallInteger(v)); - } - self.write_medium_integer(&[v as u8]) + self.write_integer(&[v as u8]) } #[inline(always)] @@ -333,7 +324,7 @@ impl Writer for PackedWriter { if let Ok(w) = v.try_into() { return self.write_i8(w); } - self.write_medium_integer(&[0, v]) + self.write_integer(&[0, v]) } #[inline(always)] @@ -341,7 +332,7 @@ impl Writer for PackedWriter { if let Ok(w) = v.try_into() { return self.write_i8(w); } - self.write_medium_integer(&[(v >> 8) as u8, (v & 255) as u8]) + self.write_integer(&[(v >> 8) as u8, (v & 255) as u8]) } #[inline(always)] @@ -349,7 +340,7 @@ impl Writer for PackedWriter { if let Ok(w) = v.try_into() { return self.write_i16(w); } - self.write_medium_integer(&[0, (v >> 8) as u8, (v & 255) as u8]) + self.write_integer(&[0, (v >> 8) as u8, (v & 255) as u8]) } #[inline(always)] @@ -358,9 +349,9 @@ impl Writer for PackedWriter { return self.write_i16(w); } if fits_in_bytes!(v, 3) { - return self.write_medium_integer(&[(v >> 16) as u8, (v >> 8) as u8, (v & 255) as u8]); + return self.write_integer(&[(v >> 16) as u8, (v >> 8) as u8, (v & 255) as u8]); } - self.write_medium_integer(&[ + self.write_integer(&[ (v >> 24) as u8, (v >> 16) as u8, (v >> 8) as u8, @@ -373,7 +364,7 @@ impl Writer for PackedWriter { if let Ok(w) = v.try_into() { return self.write_i32(w); } - self.write_medium_integer(&[ + self.write_integer(&[ 0, (v >> 24) as u8, (v >> 16) as u8, @@ -388,7 +379,7 @@ impl Writer for PackedWriter { return self.write_i32(w); } if fits_in_bytes!(v, 5) { - return self.write_medium_integer(&[ + return self.write_integer(&[ (v >> 32) as u8, (v >> 24) as u8, (v >> 16) as u8, @@ -397,7 +388,7 @@ impl Writer for PackedWriter { ]); } if fits_in_bytes!(v, 6) { - return self.write_medium_integer(&[ + return self.write_integer(&[ (v >> 40) as u8, (v >> 32) as u8, (v >> 24) as u8, @@ -407,7 +398,7 @@ impl Writer for PackedWriter { ]); } if fits_in_bytes!(v, 7) { - return self.write_medium_integer(&[ + return self.write_integer(&[ (v >> 48) as u8, (v >> 40) as u8, (v >> 32) as u8, @@ -417,7 +408,7 @@ impl Writer for PackedWriter { (v & 255) as u8, ]); } - self.write_medium_integer(&[ + self.write_integer(&[ (v >> 56) as u8, (v >> 48) as u8, (v >> 40) as u8, @@ -434,7 +425,7 @@ impl Writer for PackedWriter { if let Ok(w) = v.try_into() { return self.write_i64(w); } - self.write_medium_integer(&[ + self.write_integer(&[ 0, (v >> 56) as u8, (v >> 48) as u8, @@ -454,27 +445,27 @@ impl Writer for PackedWriter { } let bs: [u8; 16] = v.to_be_bytes(); if fits_in_bytes!(v, 9) { - return self.write_medium_integer(&bs[7..]); + return self.write_integer(&bs[7..]); } if fits_in_bytes!(v, 10) { - return self.write_medium_integer(&bs[6..]); + return self.write_integer(&bs[6..]); } if fits_in_bytes!(v, 11) { - return self.write_medium_integer(&bs[5..]); + return self.write_integer(&bs[5..]); } if fits_in_bytes!(v, 12) { - return self.write_medium_integer(&bs[4..]); + return self.write_integer(&bs[4..]); } if fits_in_bytes!(v, 13) { - return self.write_medium_integer(&bs[3..]); + return self.write_integer(&bs[3..]); } if fits_in_bytes!(v, 14) { - return self.write_medium_integer(&bs[2..]); + return self.write_integer(&bs[2..]); } if fits_in_bytes!(v, 15) { - return self.write_medium_integer(&bs[1..]); + return self.write_integer(&bs[1..]); } - self.write_medium_integer(&bs) + self.write_integer(&bs) } #[inline(always)] @@ -495,7 +486,7 @@ impl Writer for PackedWriter { Some(n) => self.write_i8(n), None => match v.to_i128() { Some(n) => self.write_i128(n), - None => self.write_atom(Tag::SignedInteger, &v.to_signed_bytes_be()), + None => self.write_integer(&v.to_signed_bytes_be()), }, } } diff --git a/implementations/rust/preserves/src/value/reader.rs b/implementations/rust/preserves/src/value/reader.rs index fc4b8dd..c4ffa72 100644 --- a/implementations/rust/preserves/src/value/reader.rs +++ b/implementations/rust/preserves/src/value/reader.rs @@ -40,12 +40,14 @@ pub trait Reader<'de, N: NestedValue> { fn open_embedded(&mut self) -> ReaderResult<()>; fn close_embedded(&mut self) -> ReaderResult<()>; + fn open_annotations(&mut self, annotations: Option<&mut Vec>) -> ReaderResult<()>; + fn close_annotations(&mut self, annotations: Option<&mut Vec>) -> ReaderResult<()>; + type Mark; fn mark(&mut self) -> io::Result; fn restore(&mut self, mark: &Self::Mark) -> io::Result<()>; fn next_token(&mut self, read_embedded_annotations: bool) -> io::Result>; - fn next_annotations_and_token(&mut self) -> io::Result<(Vec, Token)>; //--------------------------------------------------------------------------- @@ -248,10 +250,6 @@ impl<'r, 'de, N: NestedValue, R: Reader<'de, N>> Reader<'de, N> for &'r mut R { fn next_token(&mut self, read_embedded_annotations: bool) -> io::Result> { (*self).next_token(read_embedded_annotations) } - - fn next_annotations_and_token(&mut self) -> io::Result<(Vec, Token)> { - (*self).next_annotations_and_token() - } } pub trait BinarySource<'de>: Sized { diff --git a/path/path.bin b/path/path.bin index e5241a0..5679a63 100644 --- a/path/path.bin +++ b/path/path.bin @@ -1,4 +1,4 @@ -´³schema·³version‘³ definitions·³Axis´³orµµ±values´³rec´³lit³values„´³tupleµ„„„„µ± descendants´³rec´³lit³ descendants„´³tupleµ„„„„µ±at´³rec´³lit³at„´³tupleµ´³named³key³any„„„„„µ±label´³rec´³lit³label„´³tupleµ„„„„µ±keys´³rec´³lit³keys„´³tupleµ„„„„µ±length´³rec´³lit³length„´³tupleµ„„„„µ± annotations´³rec´³lit³ annotations„´³tupleµ„„„„µ±embedded´³rec´³lit³embedded„´³tupleµ„„„„µ±parse´³rec´³lit³parse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„µ±unparse´³rec´³lit³unparse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„„„³Step´³orµµ±Axis´³refµ„³Axis„„µ±Filter´³refµ„³Filter„„µ±Function´³refµ„³Function„„„„³Filter´³orµµ±nop´³rec´³lit³nop„´³tupleµ„„„„µ±compare´³rec´³lit³compare„´³tupleµ´³named³op´³refµ„³ +´³schema·³version°³ definitions·³Axis´³orµµ±values´³rec´³lit³values„´³tupleµ„„„„µ± descendants´³rec´³lit³ descendants„´³tupleµ„„„„µ±at´³rec´³lit³at„´³tupleµ´³named³key³any„„„„„µ±label´³rec´³lit³label„´³tupleµ„„„„µ±keys´³rec´³lit³keys„´³tupleµ„„„„µ±length´³rec´³lit³length„´³tupleµ„„„„µ± annotations´³rec´³lit³ annotations„´³tupleµ„„„„µ±embedded´³rec´³lit³embedded„´³tupleµ„„„„µ±parse´³rec´³lit³parse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„µ±unparse´³rec´³lit³unparse„´³tupleµ´³named³module´³seqof´³atom³Symbol„„„´³named³name´³atom³Symbol„„„„„„„„³Step´³orµµ±Axis´³refµ„³Axis„„µ±Filter´³refµ„³Filter„„µ±Function´³refµ„³Function„„„„³Filter´³orµµ±nop´³rec´³lit³nop„´³tupleµ„„„„µ±compare´³rec´³lit³compare„´³tupleµ´³named³op´³refµ„³ Comparison„„´³named³literal³any„„„„„µ±regex´³rec´³lit³regex„´³tupleµ´³named³regex´³atom³String„„„„„„µ±test´³rec´³lit³test„´³tupleµ´³named³pred´³refµ„³ Predicate„„„„„„µ±real´³rec´³lit³real„´³tupleµ„„„„µ±int´³rec´³lit³int„´³tupleµ„„„„µ±kind´³rec´³lit³kind„´³tupleµ´³named³kind´³refµ„³ ValueKind„„„„„„„„³Function´³rec´³lit³count„´³tupleµ´³named³selector´³refµ„³Selector„„„„„³Selector´³seqof´³refµ„³Step„„³ Predicate´³orµµ±Selector´³refµ„³Selector„„µ±not´³rec´³lit³not„´³tupleµ´³named³pred´³refµ„³ Predicate„„„„„„µ±or´³rec´³lit³or„´³tupleµ´³named³preds´³seqof´³refµ„³ Predicate„„„„„„„µ±and´³rec´³lit³and„´³tupleµ´³named³preds´³seqof´³refµ„³ Predicate„„„„„„„„„³ ValueKind´³orµµ±Boolean´³lit³Boolean„„µ±Float´³lit³Float„„µ±Double´³lit³Double„„µ± SignedInteger´³lit³ SignedInteger„„µ±String´³lit³String„„µ± ByteString´³lit³ ByteString„„µ±Symbol´³lit³Symbol„„µ±Record´³lit³Record„„µ±Sequence´³lit³Sequence„„µ±Set´³lit³Set„„µ± diff --git a/python/0.18.1/binary/index.html b/python/0.18.1/binary/index.html index c2ec92a..81950ce 100644 --- a/python/0.18.1/binary/index.html +++ b/python/0.18.1/binary/index.html @@ -756,7 +756,7 @@ suitable for placing into an next until a ShortPacket exception is raised:

-
>>> d = Decoder(b'\xa0{\xb1\x05hello\x85\xb3\x01x\xb5\x84')
+
>>> d = Decoder(b'\xb0\x01{\xb1\x05hello\xbf\xb5\x84\xb3\x01x\x84')
 >>> d.next()
 123
 >>> d.next()
@@ -770,7 +770,7 @@ until a try_next until it yields
 None, which is not in the domain of Preserves Values:

-
>>> d = Decoder(b'\xa0{\xb1\x05hello\x85\xb3\x01x\xb5\x84')
+
>>> d = Decoder(b'\xb0\x01{\xb1\x05hello\xbf\xb5\x84\xb3\x01x\x84')
 >>> d.try_next()
 123
 >>> d.try_next()
@@ -782,24 +782,24 @@ until a Decoder implements the iterator interface,
 backing it with try_next, so you can simply iterate
 over all complete values in an input:

-
>>> d = Decoder(b'\xa0{\xb1\x05hello\x85\xb3\x01x\xb5\x84')
+
>>> d = Decoder(b'\xb0\x01{\xb1\x05hello\xbf\xb5\x84\xb3\x01x\x84')
 >>> list(d)
 [123, 'hello', ()]
 

-
>>> for v in Decoder(b'\xa0{\xb1\x05hello\x85\xb3\x01x\xb5\x84'):
+
>>> for v in Decoder(b'\xb0\x01{\xb1\x05hello\xbf\xb5\x84\xb3\x01x\x84'):
 ...     print(repr(v))
 123
 'hello'
 ()
 

 
Supply include_annotations=True to read annotations alongside the annotated values:

-
>>> d = Decoder(b'\xa0{\xb1\x05hello\x85\xb3\x01x\xb5\x84', include_annotations=True)
+
>>> d = Decoder(b'\xb0\x01{\xb1\x05hello\xbf\xb5\x84\xb3\x01x\x84', include_annotations=True)
 >>> list(d)
 [123, 'hello', @#x ()]
 

 
If you are incrementally reading from, say, a socket, you can use
 extend to add new input as if comes available:

-
>>> d = Decoder(b'\xa0{\xb1\x05he')
+
>>> d = Decoder(b'\xb0\x01{\xb1\x05he')
 >>> d.try_next()
 123
 >>> d.try_next() # returns None because the input is incomplete
@@ -915,7 +915,12 @@ bytes from the front of self.packet and resetting self.index<
 
       

         Source code in preserves/binary.py
-        
189
+        
184
+185
+186
+187
+188
+189
 190
 191
 192
@@ -946,9 +951,7 @@ bytes from the front of self.packet and resetting self.index<
 217
 218
 219
-220
-221
-222
def next(self):
+220
def next(self):
     """Reads the next complete `Value` from the internal buffer, raising
     [ShortPacket][preserves.error.ShortPacket] if too few bytes are available, or
     [DecodeError][preserves.error.DecodeError] if the input is invalid somehow.
@@ -957,19 +960,16 @@ bytes from the front of self.packet and resetting self.index<
     tag = self.nextbyte()
     if tag == 0x80: return self.wrap(False)
     if tag == 0x81: return self.wrap(True)
-    if tag == 0x82: return self.wrap(Float.from_bytes(self.nextbytes(4)))
-    if tag == 0x83: return self.wrap(struct.unpack('>d', self.nextbytes(8))[0])
     if tag == 0x84: raise DecodeError('Unexpected end-of-stream marker')
-    if tag == 0x85:
-        a = self.next()
-        v = self.next()
-        return self.unshift_annotation(a, v)
     if tag == 0x86:
         if self.decode_embedded is None:
             raise DecodeError('No decode_embedded function supplied')
         return self.wrap(Embedded(self.decode_embedded(self.next())))
-    if tag >= 0x90 and tag <= 0x9f: return self.wrap(tag - (0xa0 if tag > 0x9c else 0x90))
-    if tag >= 0xa0 and tag <= 0xaf: return self.wrap(self.nextint(tag - 0xa0 + 1))
+    if tag == 0x87:
+        count = self.nextbyte()
+        if count == 4: return self.wrap(Float.from_bytes(self.nextbytes(4)))
+        if count == 8: return self.wrap(struct.unpack('>d', self.nextbytes(8))[0])
+        raise DecodeError('Invalid IEEE754 size')
     if tag == 0xb0: return self.wrap(self.nextint(self.varint()))
     if tag == 0xb1: return self.wrap(self.nextbytes(self.varint()).decode('utf-8'))
     if tag == 0xb2: return self.wrap(self.nextbytes(self.varint()))
@@ -981,6 +981,12 @@ bytes from the front of self.packet and resetting self.index<
     if tag == 0xb5: return self.wrap(tuple(self.nextvalues()))
     if tag == 0xb6: return self.wrap(frozenset(self.nextvalues()))
     if tag == 0xb7: return self.wrap(ImmutableDict.from_kvs(self.nextvalues()))
+    if tag == 0xbf:
+        vs = self.nextvalues()
+        if len(vs) < 2: raise DecodeError('Too few items in encoded annotated value')
+        if self.include_annotations:
+            vs[0].annotations.extend(vs[1:])
+        return vs[0]
     raise DecodeError('Invalid tag: ' + hex(tag))
 

       
@@ -1005,15 +1011,15 @@ bytes from the front of self.packet and resetting self.index<
 
       

         Source code in preserves/binary.py
-        
224
+        
222
+223
+224
 225
 226
 227
 228
 229
-230
-231
-232
def try_next(self):
+230
def try_next(self):
     """Like [next][preserves.binary.Decoder.next], but returns `None` instead of raising
     [ShortPacket][preserves.error.ShortPacket]."""
     start = self.index
@@ -1057,7 +1063,7 @@ bytes from the front of self.packet and resetting self.index<
 >>> e.append('hello')
 >>> e.append(annotate([], Symbol('x')))
 >>> e.contents()
-b'\xa0{\xb1\x05hello\x85\xb3\x01x\xb5\x84'
+b'\xb0\x01{\xb1\x05hello\xbf\xb5\x84\xb3\x01x\x84'
 

 
   
Parameters:

@@ -1134,7 +1140,9 @@ annotations. If explicitly True or
 
           

             Source code in preserves/binary.py
-            
294
+            
292
+293
+294
 295
 296
 297
@@ -1143,9 +1151,7 @@ annotations. If explicitly True or
 300
 301
 302
-303
-304
-305
def __init__(self,
+303
def __init__(self,
              encode_embedded=lambda x: x,
              canonicalize=False,
              include_annotations=None):
@@ -1188,7 +1194,9 @@ annotations. If explicitly True or
 
       

         Source code in preserves/binary.py
-        
367
+        
365
+366
+367
 368
 369
 370
@@ -1223,11 +1231,7 @@ annotations. If explicitly True or
 399
 400
 401
-402
-403
-404
-405
-406
def append(self, v):
+402
def append(self, v):
     """Extend `self.buffer` with an encoding of `v`."""
     v = preserve(v)
     if hasattr(v, '__preserve_write_binary__'):
@@ -1237,21 +1241,19 @@ annotations. If explicitly True or
     elif v is True:
         self.buffer.append(0x81)
     elif isinstance(v, float):
-        self.buffer.append(0x83)
+        self.buffer.append(0x87)
+        self.buffer.append(8)
         self.buffer.extend(struct.pack('>d', v))
     elif isinstance(v, numbers.Number):
-        if v >= -3 and v <= 12:
-            self.buffer.append(0x90 + (v if v >= 0 else v + 16))
-        else:
-            self.encodeint(v)
+        self.encodeint(v)
     elif isinstance(v, bytes):
-        self.encodebytes(2, v)
+        self.encodebytes(0xb2, v)
     elif isinstance(v, basestring_):
-        self.encodebytes(1, v.encode('utf-8'))
+        self.encodebytes(0xb1, v.encode('utf-8'))
     elif isinstance(v, list):
-        self.encodevalues(5, v)
+        self.encodevalues(0xb5, v)
     elif isinstance(v, tuple):
-        self.encodevalues(5, v)
+        self.encodevalues(0xb5, v)
     elif isinstance(v, set):
         self.encodeset(v)
     elif isinstance(v, frozenset):
@@ -1266,7 +1268,7 @@ annotations. If explicitly True or
         if i is None:
             self.cannot_encode(v)
         else:
-            self.encodevalues(5, i)
+            self.encodevalues(0xb5, i)
 

       
   

@@ -1289,9 +1291,9 @@ annotations. If explicitly True or
 
       

         Source code in preserves/binary.py
-        
316
-317
-318
def contents(self):
+        
314
+315
+316
def contents(self):
     """Returns a `bytes` constructed from the contents of `self.buffer`."""
     return bytes(self.buffer)
 

@@ -1316,9 +1318,9 @@ annotations. If explicitly True or
 
       

         Source code in preserves/binary.py
-        
307
-308
-309
def reset(self):
+        
305
+306
+307
def reset(self):
     """Clears `self.buffer` to a fresh empty `bytearray`."""
     self.buffer = bytearray()
 

@@ -1353,12 +1355,12 @@ annotations. If explicitly True or
 
       

         Source code in preserves/binary.py
-        
434
-435
-436
-437
-438
-439
def canonicalize(v, **kwargs):
+        
430
+431
+432
+433
+434
+435
def canonicalize(v, **kwargs):
     """As [encode][preserves.binary.encode], but sets `canonicalize=True` in the
     [Encoder][preserves.binary.Encoder] constructor.
 
@@ -1412,16 +1414,16 @@ annotations. If explicitly True or
 
       

         Source code in preserves/binary.py
-        
243
+        
241
+242
+243
 244
 245
 246
 247
 248
 249
-250
-251
-252
def decode(bs, **kwargs):
+250
def decode(bs, **kwargs):
     """Yields the first complete encoded value from `bs`, passing `kwargs` through to the
     [Decoder][preserves.binary.Decoder] constructor. Raises exceptions as per
     [next][preserves.binary.Decoder.next].
@@ -1454,10 +1456,10 @@ annotations. If explicitly True or
 
       

         Source code in preserves/binary.py
-        
254
-255
-256
-257
def decode_with_annotations(bs, **kwargs):
+        
252
+253
+254
+255
def decode_with_annotations(bs, **kwargs):
     """Like [decode][preserves.binary.decode], but supplying `include_annotations=True` to the
     [Decoder][preserves.binary.Decoder] constructor."""
     return Decoder(packet=bs, include_annotations=True, **kwargs).next()
@@ -1484,12 +1486,12 @@ underlying 
         Source code in preserves/binary.py
-        
427
-428
-429
-430
-431
-432
def encode(v, **kwargs):
+        
423
+424
+425
+426
+427
+428
def encode(v, **kwargs):
     """Encode a single `Value` `v` to a byte string. Any supplied `kwargs` are passed on to the
     underlying [Encoder][preserves.binary.Encoder] constructor."""
     e = Encoder(**kwargs)
diff --git a/python/0.18.1/index.html b/python/0.18.1/index.html
index d022a46..dc1dc30 100644
--- a/python/0.18.1/index.html
+++ b/python/0.18.1/index.html
@@ -536,7 +536,7 @@ for their Preserves counterparts. Wrapper objects for March 16, 2023
+      March 17, 2023
       
         

         Created:
diff --git a/python/0.18.1/path/index.html b/python/0.18.1/path/index.html
index 3d4980f..46a72c9 100644
--- a/python/0.18.1/path/index.html
+++ b/python/0.18.1/path/index.html
@@ -725,14 +725,14 @@ annotations (for documentation and other purposes).

 ...     print(stringify(result))
 <Test #[tLMHY2FwdHVyZbSzB2Rpc2NhcmSEhA==] <capture <discard>>>
 <Test #[tLMHb2JzZXJ2ZbSzBXNwZWFrtLMHZGlzY2FyZIS0swdjYXB0dXJltLMHZGlzY2FyZISEhIQ=] <observe <speak <discard> <capture <discard>>>>>
-<Test #[tLWzBnRpdGxlZLMGcGVyc29ukrMFdGhpbmeRhKBlsQlCbGFja3dlbGy0swRkYXRloQcdkpOEsQJEcoQ=] <[titled person 2 thing 1] 101 "Blackwell" <date 1821 2 3> "Dr">>
+<Test #[tLWzBnRpdGxlZLMGcGVyc29usAECswV0aGluZ7ABAYSwAWWxCUJsYWNrd2VsbLSzBGRhdGWwAgcdsAECsAEDhLECRHKE] <[titled person 2 thing 1] 101 "Blackwell" <date 1821 2 3> "Dr">>
 <Test #[tLMHZGlzY2FyZIQ=] <discard>>
-<Test #[tJe1hIQ=] <7 []>>
+<Test #[tLABB7WEhA==] <7 []>>
 <Test #[tLMHZGlzY2FyZLMIc3VycHJpc2WE] <discard surprise>>
-<Test #[tLEHYVN0cmluZ5OUhA==] <"aString" 3 4>>
-<Test #[tLSzB2Rpc2NhcmSEk5SE] <<discard> 3 4>>
-<Test #[hbMCYXK0swFShbMCYWazAWaE] @ar <R @af f>>
-<Test #[tIWzAmFyswFShbMCYWazAWaE] <@ar R @af f>>
+<Test #[tLEHYVN0cmluZ7ABA7ABBIQ=] <"aString" 3 4>>
+<Test #[tLSzB2Rpc2NhcmSEsAEDsAEEhA==] <<discard> 3 4>>
+<Test #[v7SzAVK/swFmswJhZoSEswJhcoQ=] @ar <R @af f>>
+<Test #[tL+zAVKzAmFyhL+zAWazAmFmhIQ=] <@ar R @af f>>
 

 
   
diff --git a/python/0.18.1/search/search_index.json b/python/0.18.1/search/search_index.json
index f28b640..ebb6f1a 100644
--- a/python/0.18.1/search/search_index.json
+++ b/python/0.18.1/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Overview","text":"
pip install preserves\n
 
This package (preserves on pypi.org) implements Preserves for Python 3.x. It provides the core semantics as well as both the human-readable text syntax (a superset of JSON) and machine-oriented binary format (including canonicalization) for Preserves. It also implements Preserves Schema and Preserves Path.
 
     
  • Main package API: preserves
    •  
"},{"location":"#what-is-preserves","title":"What is Preserves?","text":"
Preserves is a data model, with associated serialization formats.
 
It supports records with user-defined labels, embedded references, and the usual suite of atomic and compound data types, including binary data as a distinct type from text strings. Its annotations allow separation of data from metadata such as comments, trace information, and provenance information.
 
Preserves departs from many other data languages in defining how to compare two values. Comparison is based on the data model, not on syntax or on data structures of any particular implementation language.
"},{"location":"#mapping-between-preserves-values-and-python-values","title":"Mapping between Preserves values and Python values","text":"
Preserves Values are categorized in the following way:
 
                      Value = Atom\n                            | Compound\n                            | Embedded\n\n                       Atom = Boolean\n                            | Float\n                            | Double\n                            | SignedInteger\n                            | String\n                            | ByteString\n                            | Symbol\n\n                   Compound = Record\n                            | Sequence\n                            | Set\n                            | Dictionary\n
 
Python's strings, byte strings, integers, booleans, and double-precision floats stand directly for their Preserves counterparts. Wrapper objects for Float and Symbol complete the suite of atomic types.
 
Python's lists and tuples correspond to Preserves Sequences, and dicts and sets to Dictionary and Set values, respectively. Preserves Records are represented by Record objects. Finally, embedded values are represented by Embedded objects.
"},{"location":"api/","title":"The top-level preserves package","text":"
import preserves\n
 
The main package re-exports a subset of the exports of its constituent modules:
 
     
  •  
    From preserves.values:
     
       
    • Annotated
      •  
    • Embedded
      •  
    • Float
      •  
    • ImmutableDict
      •  
    • Record
      •  
    • Symbol
      •  
    • annotate
      •  
    • is_annotated
      •  
    • preserve
      •  
    • strip_annotations
      •  
     
    •  
  •  
    From preserves.error:
     
       
    • DecodeError
      •  
    • EncodeError
      •  
    • ShortPacket
      •  
     
    •  
  •  
    From preserves.binary:
     
       
    • Decoder
      •  
    • Encoder
      •  
    • canonicalize
      •  
    • decode
      •  
    • decode_with_annotations
      •  
    • encode
      •  
     
    •  
  •  
    From preserves.text:
     
       
    • Formatter
      •  
    • Parser
      •  
    • parse
      •  
    • parse_with_annotations
      •  
    • stringify
      •  
     
    •  
  •  
    From preserves.compare:
     
       
    • cmp
      •  
     
    •  
  •  
    From preserves.merge:
     
       
    • merge
      •  
     
    •  
 
It also exports the compare and fold modules themselves, permitting patterns like
 
>>> from preserves import *\n>>> compare.cmp(123, 234)\n-1\n
 
Finally, it provides a few utility aliases for common tasks:
"},{"location":"api/#preserves.dumps","title":"dumps = stringify  module-attribute","text":"
This alias for stringify provides a familiar pythonesque name for converting a Preserves Value to a string.
"},{"location":"api/#preserves.loads","title":"loads = parse  module-attribute","text":"
This alias for parse provides a familiar pythonesque name for converting a string to a Preserves Value.
"},{"location":"binary/","title":"Machine-oriented binary syntax","text":"
The preserves.binary module implements the Preserves machine-oriented binary syntax.
 
The main entry points are functions encode, canonicalize, decode, and decode_with_annotations.
 
>>> encode(Record(Symbol('hi'), []))\nb'\\xb4\\xb3\\x02hi\\x84'\n>>> decode(b'\\xb4\\xb3\\x02hi\\x84')\n#hi()\n
"},{"location":"binary/#preserves.binary.Decoder","title":"Decoder(packet=b'', include_annotations=False, decode_embedded=lambda x: x)","text":"
         Bases: BinaryCodec
 
Implementation of a decoder for the machine-oriented binary Preserves syntax.
 
Parameters:
 Name Type Description Default packet bytes 
initial contents of the input buffer; may subsequently be extended by calling extend.
 b'' include_annotations bool 
if True, wrap each value and subvalue in an Annotated object.
 False decode_embedded 
function accepting a Value and returning a possibly-decoded form of that value suitable for placing into an Embedded object.
 lambda x: x 
Normal usage is to supply a buffer, and keep calling next until a ShortPacket exception is raised:
 
>>> d = Decoder(b'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84')\n>>> d.next()\n123\n>>> d.next()\n'hello'\n>>> d.next()\n()\n>>> d.next()\nTraceback (most recent call last):\n  ...\npreserves.error.ShortPacket: Short packet\n
 
Alternatively, keep calling try_next until it yields None, which is not in the domain of Preserves Values:
 
>>> d = Decoder(b'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84')\n>>> d.try_next()\n123\n>>> d.try_next()\n'hello'\n>>> d.try_next()\n()\n>>> d.try_next()\n
 
For convenience, Decoder implements the iterator interface, backing it with try_next, so you can simply iterate over all complete values in an input:
 
>>> d = Decoder(b'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84')\n>>> list(d)\n[123, 'hello', ()]\n
 
>>> for v in Decoder(b'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84'):\n...     print(repr(v))\n123\n'hello'\n()\n
 
Supply include_annotations=True to read annotations alongside the annotated values:
 
>>> d = Decoder(b'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84', include_annotations=True)\n>>> list(d)\n[123, 'hello', @#x ()]\n
 
If you are incrementally reading from, say, a socket, you can use extend to add new input as if comes available:
 
>>> d = Decoder(b'\\xa0{\\xb1\\x05he')\n>>> d.try_next()\n123\n>>> d.try_next() # returns None because the input is incomplete\n>>> d.extend(b'llo')\n>>> d.try_next()\n'hello'\n>>> d.try_next()\n
 
Attributes:
 Name Type Description packet bytes 
buffered input waiting to be processed
 index int 
read position within packet
 Source code in preserves/binary.py 
def __init__(self, packet=b'', include_annotations=False, decode_embedded=lambda x: x):\n    super(Decoder, self).__init__()\n    self.packet = packet\n    self.index = 0\n    self.include_annotations = include_annotations\n    self.decode_embedded = decode_embedded\n
"},{"location":"binary/#preserves.binary.Decoder.extend","title":"extend(data)","text":"
Appends data to the remaining bytes in self.packet, trimming already-processed bytes from the front of self.packet and resetting self.index to zero.
 Source code in preserves/binary.py 
def extend(self, data):\n\"\"\"Appends `data` to the remaining bytes in `self.packet`, trimming already-processed\n    bytes from the front of `self.packet` and resetting `self.index` to zero.\"\"\"\n    self.packet = self.packet[self.index:] + data\n    self.index = 0\n
"},{"location":"binary/#preserves.binary.Decoder.next","title":"next()","text":"
Reads the next complete Value from the internal buffer, raising ShortPacket if too few bytes are available, or DecodeError if the input is invalid somehow.
 Source code in preserves/binary.py 
def next(self):\n\"\"\"Reads the next complete `Value` from the internal buffer, raising\n    [ShortPacket][preserves.error.ShortPacket] if too few bytes are available, or\n    [DecodeError][preserves.error.DecodeError] if the input is invalid somehow.\n\n    \"\"\"\n    tag = self.nextbyte()\n    if tag == 0x80: return self.wrap(False)\n    if tag == 0x81: return self.wrap(True)\n    if tag == 0x82: return self.wrap(Float.from_bytes(self.nextbytes(4)))\n    if tag == 0x83: return self.wrap(struct.unpack('>d', self.nextbytes(8))[0])\n    if tag == 0x84: raise DecodeError('Unexpected end-of-stream marker')\n    if tag == 0x85:\n        a = self.next()\n        v = self.next()\n        return self.unshift_annotation(a, v)\n    if tag == 0x86:\n        if self.decode_embedded is None:\n            raise DecodeError('No decode_embedded function supplied')\n        return self.wrap(Embedded(self.decode_embedded(self.next())))\n    if tag >= 0x90 and tag <= 0x9f: return self.wrap(tag - (0xa0 if tag > 0x9c else 0x90))\n    if tag >= 0xa0 and tag <= 0xaf: return self.wrap(self.nextint(tag - 0xa0 + 1))\n    if tag == 0xb0: return self.wrap(self.nextint(self.varint()))\n    if tag == 0xb1: return self.wrap(self.nextbytes(self.varint()).decode('utf-8'))\n    if tag == 0xb2: return self.wrap(self.nextbytes(self.varint()))\n    if tag == 0xb3: return self.wrap(Symbol(self.nextbytes(self.varint()).decode('utf-8')))\n    if tag == 0xb4:\n        vs = self.nextvalues()\n        if not vs: raise DecodeError('Too few elements in encoded record')\n        return self.wrap(Record(vs[0], vs[1:]))\n    if tag == 0xb5: return self.wrap(tuple(self.nextvalues()))\n    if tag == 0xb6: return self.wrap(frozenset(self.nextvalues()))\n    if tag == 0xb7: return self.wrap(ImmutableDict.from_kvs(self.nextvalues()))\n    raise DecodeError('Invalid tag: ' + hex(tag))\n
"},{"location":"binary/#preserves.binary.Decoder.try_next","title":"try_next()","text":"
Like next, but returns None instead of raising ShortPacket.
 Source code in preserves/binary.py 
def try_next(self):\n\"\"\"Like [next][preserves.binary.Decoder.next], but returns `None` instead of raising\n    [ShortPacket][preserves.error.ShortPacket].\"\"\"\n    start = self.index\n    try:\n        return self.next()\n    except ShortPacket:\n        self.index = start\n        return None\n
"},{"location":"binary/#preserves.binary.Encoder","title":"Encoder(encode_embedded=lambda x: x, canonicalize=False, include_annotations=None)","text":"
         Bases: BinaryCodec
 
Implementation of an encoder for the machine-oriented binary Preserves syntax.
 
>>> e = Encoder()\n>>> e.append(123)\n>>> e.append('hello')\n>>> e.append(annotate([], Symbol('x')))\n>>> e.contents()\nb'\\xa0{\\xb1\\x05hello\\x85\\xb3\\x01x\\xb5\\x84'\n
 
Parameters:
 Name Type Description Default encode_embedded 
function accepting an Embedded.embeddedValue and returning a Value for serialization.
 lambda x: x canonicalize bool 
if True, ensures the serialized data are in canonical form. This is slightly more work than producing potentially-non-canonical output.
 False include_annotations bool | None 
if None, includes annotations in the output only when canonicalize is False, because canonical serialization of values demands omission of annotations. If explicitly True or False, however, annotations will be included resp. excluded no matter the canonicalize setting. This can be used to get canonical ordering (canonicalize=True) and annotations (include_annotations=True).
 None 
Attributes:
 Name Type Description buffer bytearray 
accumulator for the output of the encoder
 Source code in preserves/binary.py 
def __init__(self,\n             encode_embedded=lambda x: x,\n             canonicalize=False,\n             include_annotations=None):\n    super(Encoder, self).__init__()\n    self.buffer = bytearray()\n    self._encode_embedded = encode_embedded\n    self._canonicalize = canonicalize\n    if include_annotations is None:\n        self.include_annotations = not self._canonicalize\n    else:\n        self.include_annotations = include_annotations\n
"},{"location":"binary/#preserves.binary.Encoder.append","title":"append(v)","text":"
Extend self.buffer with an encoding of v.
 Source code in preserves/binary.py 
def append(self, v):\n\"\"\"Extend `self.buffer` with an encoding of `v`.\"\"\"\n    v = preserve(v)\n    if hasattr(v, '__preserve_write_binary__'):\n        v.__preserve_write_binary__(self)\n    elif v is False:\n        self.buffer.append(0x80)\n    elif v is True:\n        self.buffer.append(0x81)\n    elif isinstance(v, float):\n        self.buffer.append(0x83)\n        self.buffer.extend(struct.pack('>d', v))\n    elif isinstance(v, numbers.Number):\n        if v >= -3 and v <= 12:\n            self.buffer.append(0x90 + (v if v >= 0 else v + 16))\n        else:\n            self.encodeint(v)\n    elif isinstance(v, bytes):\n        self.encodebytes(2, v)\n    elif isinstance(v, basestring_):\n        self.encodebytes(1, v.encode('utf-8'))\n    elif isinstance(v, list):\n        self.encodevalues(5, v)\n    elif isinstance(v, tuple):\n        self.encodevalues(5, v)\n    elif isinstance(v, set):\n        self.encodeset(v)\n    elif isinstance(v, frozenset):\n        self.encodeset(v)\n    elif isinstance(v, dict):\n        self.encodedict(v)\n    else:\n        try:\n            i = iter(v)\n        except TypeError:\n            i = None\n        if i is None:\n            self.cannot_encode(v)\n        else:\n            self.encodevalues(5, i)\n
"},{"location":"binary/#preserves.binary.Encoder.contents","title":"contents()","text":"
Returns a bytes constructed from the contents of self.buffer.
 Source code in preserves/binary.py 
def contents(self):\n\"\"\"Returns a `bytes` constructed from the contents of `self.buffer`.\"\"\"\n    return bytes(self.buffer)\n
"},{"location":"binary/#preserves.binary.Encoder.reset","title":"reset()","text":"
Clears self.buffer to a fresh empty bytearray.
 Source code in preserves/binary.py 
def reset(self):\n\"\"\"Clears `self.buffer` to a fresh empty `bytearray`.\"\"\"\n    self.buffer = bytearray()\n
"},{"location":"binary/#preserves.binary.canonicalize","title":"canonicalize(v, **kwargs)","text":"
As encode, but sets canonicalize=True in the Encoder constructor.
 Source code in preserves/binary.py 
def canonicalize(v, **kwargs):\n\"\"\"As [encode][preserves.binary.encode], but sets `canonicalize=True` in the\n    [Encoder][preserves.binary.Encoder] constructor.\n\n    \"\"\"\n    return encode(v, canonicalize=True, **kwargs)\n
"},{"location":"binary/#preserves.binary.decode","title":"decode(bs, **kwargs)","text":"
Yields the first complete encoded value from bs, passing kwargs through to the Decoder constructor. Raises exceptions as per next.
 
Parameters:
 Name Type Description Default bs bytes 
encoded data to decode
 required Source code in preserves/binary.py 
def decode(bs, **kwargs):\n\"\"\"Yields the first complete encoded value from `bs`, passing `kwargs` through to the\n    [Decoder][preserves.binary.Decoder] constructor. Raises exceptions as per\n    [next][preserves.binary.Decoder.next].\n\n    Args:\n        bs (bytes): encoded data to decode\n\n    \"\"\"\n    return Decoder(packet=bs, **kwargs).next()\n
"},{"location":"binary/#preserves.binary.decode_with_annotations","title":"decode_with_annotations(bs, **kwargs)","text":"
Like decode, but supplying include_annotations=True to the Decoder constructor.
 Source code in preserves/binary.py 
def decode_with_annotations(bs, **kwargs):\n\"\"\"Like [decode][preserves.binary.decode], but supplying `include_annotations=True` to the\n    [Decoder][preserves.binary.Decoder] constructor.\"\"\"\n    return Decoder(packet=bs, include_annotations=True, **kwargs).next()\n
"},{"location":"binary/#preserves.binary.encode","title":"encode(v, **kwargs)","text":"
Encode a single Value v to a byte string. Any supplied kwargs are passed on to the underlying Encoder constructor.
 Source code in preserves/binary.py 
def encode(v, **kwargs):\n\"\"\"Encode a single `Value` `v` to a byte string. Any supplied `kwargs` are passed on to the\n    underlying [Encoder][preserves.binary.Encoder] constructor.\"\"\"\n    e = Encoder(**kwargs)\n    e.append(v)\n    return e.contents()\n
"},{"location":"compare/","title":"Comparing Values","text":"
Preserves specifies a total ordering and an equivalence between terms. The preserves.compare module implements the ordering and equivalence relations.
 
>>> cmp(\"bzz\", \"c\")\n-1\n>>> cmp(True, [])\n-1\n>>> lt(\"bzz\", \"c\")\nTrue\n>>> eq(\"bzz\", \"c\")\nFalse\n
 
Note that the ordering relates more values than Python's built-in ordering:
 
>>> [1, 2, 2] < [1, 2, \"3\"]\nTraceback (most recent call last):\n  ..\nTypeError: '<' not supported between instances of 'int' and 'str'\n\n>>> lt([1, 2, 2], [1, 2, \"3\"])\nTrue\n
"},{"location":"compare/#preserves.compare.cmp","title":"cmp(a, b)","text":"
Returns -1 if a < b, or 0 if a = b, or 1 if a > b according to the Preserves total order.
 Source code in preserves/compare.py 
def cmp(a, b):\n\"\"\"Returns `-1` if `a` < `b`, or `0` if `a` = `b`, or `1` if `a` > `b` according to the\n    [Preserves total order](https://preserves.dev/preserves.html#total-order).\"\"\"\n    return _cmp(preserve(a), preserve(b))\n
"},{"location":"compare/#preserves.compare.eq","title":"eq(a, b)","text":"
Returns True iff a = b according to the Preserves equivalence relation.
 Source code in preserves/compare.py 
def eq(a, b):\n\"\"\"Returns `True` iff `a` = `b` according to the [Preserves equivalence\n    relation](https://preserves.dev/preserves.html#equivalence).\"\"\"\n    return _eq(preserve(a), preserve(b))\n
"},{"location":"compare/#preserves.compare.le","title":"le(a, b)","text":"
Returns True iff a \u2264 b according to the Preserves total order.
 Source code in preserves/compare.py 
def le(a, b):\n\"\"\"Returns `True` iff `a` \u2264 `b` according to the [Preserves total\n    order](https://preserves.dev/preserves.html#total-order).\"\"\"\n    return cmp(a, b) <= 0\n
"},{"location":"compare/#preserves.compare.lt","title":"lt(a, b)","text":"
Returns True iff a < b according to the Preserves total order.
 Source code in preserves/compare.py 
def lt(a, b):\n\"\"\"Returns `True` iff `a` < `b` according to the [Preserves total\n    order](https://preserves.dev/preserves.html#total-order).\"\"\"\n    return cmp(a, b) < 0\n
"},{"location":"compare/#preserves.compare.sorted","title":"sorted(iterable, *, key=lambda x: x, reverse=False)","text":"
Returns a sorted list built from iterable, extracting a sort key using key, and ordering according to the Preserves total order. Directly analogous to the built-in Python sorted routine, except uses the Preserves order instead of Python's less-than relation.
 Source code in preserves/compare.py 
def sorted(iterable, *, key=lambda x: x, reverse=False):\n\"\"\"Returns a sorted list built from `iterable`, extracting a sort key using `key`, and\n    ordering according to the [Preserves total\n    order](https://preserves.dev/preserves.html#total-order). Directly analogous to the\n    [built-in Python `sorted`\n    routine](https://docs.python.org/3/library/functions.html#sorted), except uses the\n    Preserves order instead of Python's less-than relation.\n\n    \"\"\"\n    return _sorted(iterable, key=lambda x: _key(key(x)), reverse=reverse)\n
"},{"location":"compare/#preserves.compare.sorted_items","title":"sorted_items(d)","text":"
Given a dictionary d, yields a list of (key, value) tuples sorted by key.
 Source code in preserves/compare.py 
def sorted_items(d):\n\"\"\"Given a dictionary `d`, yields a list of `(key, value)` tuples sorted by `key`.\"\"\"\n    return sorted(d.items(), key=_item_key)\n
"},{"location":"error/","title":"Codec errors","text":"
The preserves.error module exports various Error subclasses.
"},{"location":"error/#preserves.error.DecodeError","title":"DecodeError","text":"
         Bases: ValueError
 
Raised whenever preserves.binary.Decoder or preserves.text.Parser detect invalid input.
"},{"location":"error/#preserves.error.EncodeError","title":"EncodeError","text":"
         Bases: ValueError
 
Raised whenever preserves.binary.Encoder or preserves.text.Formatter are unable to proceed.
"},{"location":"error/#preserves.error.ShortPacket","title":"ShortPacket","text":"
         Bases: DecodeError
 
Raised whenever preserves.binary.Decoder or preserves.text.Parser discover that they want to read beyond the end of the currently-available input buffer in order to completely read an encoded value.
"},{"location":"fold/","title":"Traversing values","text":"
The preserves.fold module exports various utilities for traversing compound Values.
"},{"location":"fold/#preserves.fold.map_embeddeds","title":"map_embeddeds(f, v)","text":"
Returns an equivalent copy of v, except where each contained Embedded value is replaced by f applied to the Embedded's embeddedValue attribute.
 
>>> map_embeddeds(lambda w: Embedded(f'w={w}'), ['a', Embedded(123), {'z': 6.0}])\n('a', #!'w=123', {'z': 6.0})\n
 Source code in preserves/fold.py 
def map_embeddeds(f, v):\n\"\"\"Returns an [equivalent][preserves.compare.eq] copy of `v`, except where each contained\n    [Embedded][preserves.values.Embedded] value is replaced by `f` applied to the Embedded's\n    `embeddedValue` attribute.\n\n    ```python\n    >>> map_embeddeds(lambda w: Embedded(f'w={w}'), ['a', Embedded(123), {'z': 6.0}])\n    ('a', #!'w=123', {'z': 6.0})\n\n    ```\n    \"\"\"\n    def walk(v):\n        if isinstance(v, Embedded):\n            return f(v.embeddedValue)\n        elif isinstance(v, (list, tuple)):\n            return tuple(walk(w) for w in v)\n        elif isinstance(v, (set, frozenset)):\n            return frozenset(walk(w) for w in v)\n        elif isinstance(v, dict):\n            return ImmutableDict.from_kvs(walk(w) for w in dict_kvs(v))\n        elif isinstance(v, Record):\n            return Record(walk(v.key), walk(v.fields))\n        else:\n            return v\n    return walk(v)\n
"},{"location":"merge/","title":"Merging values","text":"
The preserves.merge module exports various utilities for merging Values.
"},{"location":"merge/#preserves.merge.merge","title":"merge(v0, *vs, merge_embedded=None)","text":"
Repeatedly merges v0 with each element in vs using merge2, returning the final result. The merge_embedded parameter is passed on to merge2.
 Source code in preserves/merge.py 
def merge(v0, *vs, merge_embedded=None):\n\"\"\"Repeatedly merges `v0` with each element in `vs` using [merge2][preserves.merge.merge2],\n    returning the final result. The `merge_embedded` parameter is passed on to merge2.\"\"\"\n    v = v0\n    for vN in vs:\n        v = merge2(v, vN, merge_embedded=merge_embedded)\n    return v\n
"},{"location":"merge/#preserves.merge.merge2","title":"merge2(a, b, merge_embedded=None)","text":"
Merges a and b, returning the result. Raises ValueError if, during the merge, a pair of incompatible values is discovered.
 
If a and b are Embedded objects, their embeddedValues are merged using merge_embedded, and the result is again wrapped in an Embedded object.
 
>>> merge2(123, 234)\nTraceback (most recent call last):\n  ...\nValueError: Cannot merge items\n>>> merge2(123, 123)\n123\n>>> merge2('hi', 0)\nTraceback (most recent call last):\n  ...\nValueError: Cannot merge items\n>>> merge2([1, 2], [1, 2])\n[1, 2]\n>>> merge2([1, 2], [1, 3])\nTraceback (most recent call last):\n  ...\nValueError: Cannot merge items\n>>> merge2({'a': 1, 'b': 2}, {'a': 1, 'c': 3})\n{'a': 1, 'b': 2, 'c': 3}\n>>> merge2({'a': 1, 'b': 2}, {'a': 10, 'c': 3})\nTraceback (most recent call last):\n  ...\nValueError: Cannot merge items\n>>> merge2(Record('a', [1, {'x': 2}]), Record('a', [1, {'y': 3}]))\na(1, {'x': 2, 'y': 3})\n
 Source code in preserves/merge.py 
def merge2(a, b, merge_embedded=None):\n\"\"\"Merges `a` and `b`, returning the result. Raises `ValueError` if, during the merge, a\n    pair of incompatible values is discovered.\n\n    If `a` and `b` are [Embedded][preserves.values.Embedded] objects, their `embeddedValue`s\n    are merged using `merge_embedded`, and the result is again wrapped in an\n    [Embedded][preserves.values.Embedded] object.\n\n    ```python\n    >>> merge2(123, 234)\n    Traceback (most recent call last):\n      ...\n    ValueError: Cannot merge items\n    >>> merge2(123, 123)\n    123\n    >>> merge2('hi', 0)\n    Traceback (most recent call last):\n      ...\n    ValueError: Cannot merge items\n    >>> merge2([1, 2], [1, 2])\n    [1, 2]\n    >>> merge2([1, 2], [1, 3])\n    Traceback (most recent call last):\n      ...\n    ValueError: Cannot merge items\n    >>> merge2({'a': 1, 'b': 2}, {'a': 1, 'c': 3})\n    {'a': 1, 'b': 2, 'c': 3}\n    >>> merge2({'a': 1, 'b': 2}, {'a': 10, 'c': 3})\n    Traceback (most recent call last):\n      ...\n    ValueError: Cannot merge items\n    >>> merge2(Record('a', [1, {'x': 2}]), Record('a', [1, {'y': 3}]))\n    a(1, {'x': 2, 'y': 3})\n\n    ```\n\n    \"\"\"\n    if a == b:\n        return a\n    if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)):\n        return merge_seq(a, b)\n    if isinstance(a, (set, frozenset)) and isinstance(b, (set, frozenset)):\n        _die()\n    if isinstance(a, dict) and isinstance(b, dict):\n        r = {}\n        for (ak, av) in a.items():\n            bv = b.get(ak, None)\n            r[ak] = av if bv is None else merge2(av, bv, merge_embedded=merge_embedded)\n        for (bk, bv) in b.items():\n            if bk not in r:\n                r[bk] = bv\n        return r\n    if isinstance(a, Record) and isinstance(b, Record):\n        return Record(merge2(a.key, b.key, merge_embedded=merge_embedded),\n                      merge_seq(a.fields, b.fields, merge_embedded=merge_embedded))\n    if isinstance(a, Embedded) and isinstance(b, Embedded):\n        m = (merge_embedded or merge_embedded_id)(a.embeddedValue, b.embeddedValue)\n        if m is None: _die()\n        return Embedded(m)\n    _die()\n
"},{"location":"path/","title":"Preserves Path","text":"
The preserves.path module implements Preserves Path.
 
Preserves Path is roughly analogous to XPath, but for Preserves values: just as XPath selects portions of an XML document, a Preserves Path uses path expressions to select portions of a Value.
 
Use parse to compile a path expression, and then use the exec method on the result to apply it to a given input:
 
parse(PATH_EXPRESSION_STRING).exec(PRESERVES_VALUE)\n    -> SEQUENCE_OF_PRESERVES_VALUES\n
"},{"location":"path/#preserves.path--command-line-usage","title":"Command-line usage","text":"
When preserves.path is run as a __main__ module, sys.argv[1] is parsed, interpreted as a path expression, and run against human-readable values read from standard input. Each matching result is passed to stringify and printed to standard output.
"},{"location":"path/#preserves.path--examples","title":"Examples","text":""},{"location":"path/#preserves.path--setup-loading-test-data","title":"Setup: Loading test data","text":"
The following examples use testdata:
 
>>> with open('tests/samples.bin', 'rb') as f:\n...     testdata = decode_with_annotations(f.read())\n
 
Recall that samples.bin contains a binary-syntax form of the human-readable [samples.pr](https://preserves.dev/tests/samples.pr) test data file, intended to exercise most of the features of Preserves. In particular, the rootValue` in the file has a number of annotations (for documentation and other purposes).
"},{"location":"path/#preserves.path--example-1-selecting-string-valued-documentation-annotations","title":"Example 1: Selecting string-valued documentation annotations","text":"
The path expression .annotations ^ Documentation . 0 / string proceeds in five steps:
 
     
  1. .annotations selects each annotation on the root document
    2.  
  2. ^ Documentation retains only those values (each an annotation of the root) that are Records with label equal to the symbol Documentation
    3.  
  3. . 0 moves into the first child (the first field) of each such Record, which in our case is a list of other Values
    4.  
  4. / selects all immediate children of these lists
    5.  
  5. string retains only those values that are strings
    6.  
 
The result of evaluating it on testdata is as follows:
 
>>> selector = parse('.annotations ^ Documentation . 0 / string')\n>>> for result in selector.exec(testdata):\n...     print(stringify(result))\n\"Individual test cases may be any of the following record types:\"\n\"In each test, let value = strip(annotatedValue),\"\n\"                  forward = value,\"\n\"                  back = value,\"\n\"except where test-case-specific values of `forward` and/or `back`\"\n\"are provided by the executing harness, and check the following\"\n\"numbered expectations according to the table above:\"\n\"Implementations may vary in their treatment of the difference between expectations\"\n\"13/14 and 16/17, depending on how they wish to treat end-of-stream conditions.\"\n
"},{"location":"path/#preserves.path--example-2-selecting-tests-with-records-as-their-annotatedvalues","title":"Example 2: Selecting tests with Records as their annotatedValues","text":"
The path expression // [.^ [= Test + = NondeterministicTest]] [. 1 rec] proceeds in three steps:
 
     
  1.  
    // recursively decomposes the input, yielding all direct and indirect descendants of each input value
     
    2.  
  2.  
    [.^ [= Test + = NondeterministicTest]] retains only those inputs (each a descendant of the root) that yield more than zero results when executed against the expression within the brackets:
     
       
    1. .^ selects only labels of values that are Records, filtering by type and transforming in a single step
      2.  
    2. [= Test + = NondeterministicTest] again filters by a path expression:
         
      1. the infix + operator takes the union of matches of its arguments
        2.  
      2. the left-hand argument, = Test selects values (remember, record labels) equal to the symbol Test
        3.  
      3. the right-hand argument = NondeterministicTest selects values equal to NondeterministicTest
        4.  
       
      3.  
     
    The result is thus all Records anywhere inside testdata that have either Test or NondeterministicTest as their labels.
     
    3.  
  3.  
    [. 1 rec] filters these Records by another path expression:
     
       
    1. . 1 selects their second field (fields are numbered from 0)
      2.  
    2. rec retains only values that are Records
      3.  
     
    4.  
 
Evaluating the expression against testdata yields the following:
 
>>> selector = parse('// [.^ [= Test + = NondeterministicTest]] [. 1 rec]')\n>>> for result in selector.exec(testdata):\n...     print(stringify(result))\n<Test #[tLMHY2FwdHVyZbSzB2Rpc2NhcmSEhA==] <capture <discard>>>\n<Test #[tLMHb2JzZXJ2ZbSzBXNwZWFrtLMHZGlzY2FyZIS0swdjYXB0dXJltLMHZGlzY2FyZISEhIQ=] <observe <speak <discard> <capture <discard>>>>>\n<Test #[tLWzBnRpdGxlZLMGcGVyc29ukrMFdGhpbmeRhKBlsQlCbGFja3dlbGy0swRkYXRloQcdkpOEsQJEcoQ=] <[titled person 2 thing 1] 101 \"Blackwell\" <date 1821 2 3> \"Dr\">>\n<Test #[tLMHZGlzY2FyZIQ=] <discard>>\n<Test #[tJe1hIQ=] <7 []>>\n<Test #[tLMHZGlzY2FyZLMIc3VycHJpc2WE] <discard surprise>>\n<Test #[tLEHYVN0cmluZ5OUhA==] <\"aString\" 3 4>>\n<Test #[tLSzB2Rpc2NhcmSEk5SE] <<discard> 3 4>>\n<Test #[hbMCYXK0swFShbMCYWazAWaE] @ar <R @af f>>\n<Test #[tIWzAmFyswFShbMCYWazAWaE] <@ar R @af f>>\n
"},{"location":"path/#preserves.path.Predicate","title":"Predicate = syntax.Predicate  module-attribute","text":"
Schema definition for representing a Preserves Path Predicate.
"},{"location":"path/#preserves.path.Selector","title":"Selector = syntax.Selector  module-attribute","text":"
Schema definition for representing a sequence of Preserves Path Steps.
"},{"location":"path/#preserves.path.syntax","title":"syntax = load_schema_file(pathlib.Path(__file__).parent / 'path.prb').path  module-attribute","text":"
This value is a Python representation of a Preserves Schema definition for the Preserves Path expression language. The language is defined in the file path.prs.
"},{"location":"path/#preserves.path.exec","title":"exec(self, v)","text":"
WARNING: This is not a function: it is a method on Selector, Predicate, and so on.
 
>>> sel = parse('/ [.length gt 1]')\n>>> sel.exec(['', 'a', 'ab', 'abc', 'abcd', 'bcd', 'cd', 'd', ''])\n('ab', 'abc', 'abcd', 'bcd', 'cd')\n
 Source code in preserves/path.py 
@extend(syntax.Function)\ndef exec(self, v):\n\"\"\"WARNING: This is not a *function*: it is a *method* on\n    [Selector][preserves.path.Selector], [Predicate][preserves.path.Predicate], and so on.\n\n    ```python\n    >>> sel = parse('/ [.length gt 1]')\n    >>> sel.exec(['', 'a', 'ab', 'abc', 'abcd', 'bcd', 'cd', 'd', ''])\n    ('ab', 'abc', 'abcd', 'bcd', 'cd')\n\n    ```\n\n    \"\"\"\n    return (len(self.selector.exec(v)),)\n
"},{"location":"path/#preserves.path.parse","title":"parse(s)","text":"
Parse s as a Preserves Path path expression, yielding a Selector object. Selectors (and Predicates etc.) have an exec method defined on them.
 
Raises ValueError if s is not a valid path expression.
 Source code in preserves/path.py 
def parse(s):\n\"\"\"Parse `s` as a Preserves Path path expression, yielding a\n    [Selector][preserves.path.Selector] object. Selectors (and Predicates etc.) have an\n    [exec][preserves.path.exec] method defined on them.\n\n    Raises `ValueError` if `s` is not a valid path expression.\n\n    \"\"\"\n    return parse_selector(Parser(s))\n
"},{"location":"schema/","title":"Preserves Schema","text":"
A Preserves schema connects Preserves Values to host-language data structures. Each definition within a schema can be processed by a compiler to produce
 
     
  •  
    a simple host-language type definition;
     
    •  
  •  
    a partial parsing function from Values to instances of the    produced type; and
     
    •  
  •  
    a total serialization function from instances of the type to    Values.
     
    •  
 
Every parsed Value retains enough information to always be able to be serialized again, and every instance of a host-language data structure contains, by construction, enough information to be successfully serialized.
"},{"location":"schema/#schema-support-in-python","title":"Schema support in Python","text":"
The preserves.schema module implements Preserves Schema for Python.
 
A Schema source file (like this one) is first compiled using preserves-schemac to produce a binary-syntax schema bundle containing schema module definitons (like this one). Python code then loads the bundle, exposing its contents as Namespaces ultimately containing SchemaObjects.
"},{"location":"schema/#preserves.schema--examples","title":"Examples","text":""},{"location":"schema/#preserves.schema--setup-loading-a-schema-bundle","title":"Setup: Loading a schema bundle","text":"
For our running example, we will use schemas associated with the Syndicated Actor Model. (The schema bundle is a copy of this file from the syndicate-protocols repository.)
 
To load a schema bundle, use load_schema_file (or, alternatively, use Compiler directly):
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n>>> type(bundle)\n<class 'preserves.schema.Namespace'>\n
 
The top-level entries in the loaded bundle are schema modules. Let's examine the stream schema module, whose source code indicates that it should contain definitions for Mode, Source, Sink, etc.:
 
>>> bundle.stream                                           # doctest: +ELLIPSIS\n{'Mode': <class 'stream.Mode'>, 'Sink': <class 'stream.Sink'>, ...}\n
"},{"location":"schema/#preserves.schema--example-1-streamstreamlistenererror-a-product-type","title":"Example 1: stream.StreamListenerError, a product type","text":"
Drilling down further, let's consider the definition of StreamListenerError, which appears in the source as
 
StreamListenerError = <stream-listener-error @spec any @message string> .\n
 
This reads, in the Preserves Schema language, as the definition of a simple product type (record, class, object) with two named fields spec and message. Parsing a value into a StreamListenerError will only succeed if it's a record, if the label matches, the second field (message) is a string, and it has exactly two fields.
 
>>> bundle.stream.StreamListenerError\n<class 'stream.StreamListenerError'>\n
 
The StreamListenerError class includes a decode method that analyzes an input value:
 
>>> bundle.stream.StreamListenerError.decode(\n...     parse('<stream-listener-error <xyz> \"an error\">'))\nStreamListenerError {'spec': #xyz(), 'message': 'an error'}\n
 
If invalid input is supplied, decode will raise SchemaDecodeFailed, which includes helpful information for diagnosing the problem (as we will see below, this is especially useful for parsers for sum types):
 
>>> bundle.stream.StreamListenerError.decode(\n...     parse('<i-am-invalid>'))\nTraceback (most recent call last):\n  ...\npreserves.schema.SchemaDecodeFailed: Could not decode i-am-invalid using <class 'stream.StreamListenerError'>\nMost likely reason: in stream.StreamListenerError: <lit stream-listener-error> didn't match i-am-invalid\nFull explanation: \n  in stream.StreamListenerError: <lit stream-listener-error> didn't match i-am-invalid\n
 
Alternatively, the try_decode method catches SchemaDecodeFailed, transforming it into None:
 
>>> bundle.stream.StreamListenerError.try_decode(\n...     parse('<stream-listener-error <xyz> \"an error\">'))\nStreamListenerError {'spec': #xyz(), 'message': 'an error'}\n>>> bundle.stream.StreamListenerError.try_decode(\n...     parse('<i-am-invalid>'))\n
 
The class can also be instantiated directly:
 
>>> err = bundle.stream.StreamListenerError(Record(Symbol('xyz'), []), 'an error')\n>>> err\nStreamListenerError {'spec': #xyz(), 'message': 'an error'}\n
 
The fields and contents of instances can be queried:
 
>>> err.spec\n#xyz()\n>>> err.message\n'an error'\n
 
And finally, instances can of course be serialized and encoded:
 
>>> print(stringify(err))\n<stream-listener-error <xyz> \"an error\">\n>>> canonicalize(err)\nb'\\xb4\\xb3\\x15stream-listener-error\\xb4\\xb3\\x03xyz\\x84\\xb1\\x08an error\\x84'\n
"},{"location":"schema/#preserves.schema--example-2-streammode-a-sum-type","title":"Example 2: stream.Mode, a sum type","text":"
Now let's consider the definition of Mode, which appears in the source as
 
Mode = =bytes / @lines LineMode / <packet @size int> / <object @description any> .\n
 
This reads, in the Preserves Schema language, as an alternation (disjoint union, variant, sum type) of four possible kinds of value: the symbol bytes; a LineMode value; a record with packet as its label and an integer as its only field; or a record with object as its label and any kind of value as its only field. In Python, this becomes:
 
>>> bundle.stream.Mode.bytes\n<class 'stream.Mode.bytes'>\n>>> bundle.stream.Mode.lines\n<class 'stream.Mode.lines'>\n>>> bundle.stream.Mode.packet\n<class 'stream.Mode.packet'>\n>>> bundle.stream.Mode.object\n<class 'stream.Mode.object'>\n
 
As before, Mode includes a decode method that analyzes an input value:
 
>>> bundle.stream.Mode.decode(parse('bytes'))\nMode.bytes()\n>>> bundle.stream.Mode.decode(parse('lf'))\nMode.lines(LineMode.lf())\n>>> bundle.stream.Mode.decode(parse('<packet 123>'))\nMode.packet {'size': 123}\n>>> bundle.stream.Mode.decode(parse('<object \"?\">'))\nMode.object {'description': '?'}\n
 
Invalid input causes SchemaDecodeFailed to be raised:
 
>>> bundle.stream.Mode.decode(parse('<i-am-not-a-valid-mode>'))\nTraceback (most recent call last):\n  ...\npreserves.schema.SchemaDecodeFailed: Could not decode <i-am-not-a-valid-mode> using <class 'stream.Mode'>\nMost likely reason: in stream.LineMode.crlf: <lit crlf> didn't match <i-am-not-a-valid-mode>\nFull explanation: \n  in stream.Mode: matching <i-am-not-a-valid-mode>\n    in stream.Mode.bytes: <lit bytes> didn't match <i-am-not-a-valid-mode>\n    in stream.Mode.lines: <ref [] LineMode> didn't match <i-am-not-a-valid-mode>\n      in stream.LineMode: matching <i-am-not-a-valid-mode>\n        in stream.LineMode.lf: <lit lf> didn't match <i-am-not-a-valid-mode>\n        in stream.LineMode.crlf: <lit crlf> didn't match <i-am-not-a-valid-mode>\n    in stream.Mode.packet: <lit packet> didn't match i-am-not-a-valid-mode\n    in stream.Mode.object: <lit object> didn't match i-am-not-a-valid-mode\n
 
The \"full explanation\" includes details on which parses were attempted, and why they failed.
 
Again, the try_decode method catches SchemaDecodeFailed, transforming it into None:
 
>>> bundle.stream.Mode.try_decode(parse('bytes'))\nMode.bytes()\n>>> bundle.stream.Mode.try_decode(parse('<i-am-not-a-valid-mode>'))\n
 
Direct instantiation is done with the variant classes, not with Mode itself:
 
>>> bundle.stream.Mode.bytes()\nMode.bytes()\n>>> bundle.stream.Mode.lines(bundle.stream.LineMode.lf())\nMode.lines(LineMode.lf())\n>>> bundle.stream.Mode.packet(123)\nMode.packet {'size': 123}\n>>> bundle.stream.Mode.object('?')\nMode.object {'description': '?'}\n
 
Fields and contents can be queried as usual:
 
>>> bundle.stream.Mode.lines(bundle.stream.LineMode.lf()).value\nLineMode.lf()\n>>> bundle.stream.Mode.packet(123).size\n123\n>>> bundle.stream.Mode.object('?').description\n'?'\n
 
And serialization and encoding are also as expected:
 
>>> print(stringify(bundle.stream.Mode.bytes()))\nbytes\n>>> print(stringify(bundle.stream.Mode.lines(bundle.stream.LineMode.lf())))\nlf\n>>> print(stringify(bundle.stream.Mode.packet(123)))\n<packet 123>\n>>> print(stringify(bundle.stream.Mode.object('?')))\n<object \"?\">\n>>> canonicalize(bundle.stream.Mode.object('?'))\nb'\\xb4\\xb3\\x06object\\xb1\\x01?\\x84'\n
 
Finally, the VARIANT attribute of instances allows code to dispatch on what kind of data it is handling at a given moment:
 
>>> bundle.stream.Mode.bytes().VARIANT\n#bytes\n>>> bundle.stream.Mode.lines(bundle.stream.LineMode.lf()).VARIANT\n#lines\n>>> bundle.stream.Mode.packet(123).VARIANT\n#packet\n>>> bundle.stream.Mode.object('?').VARIANT\n#object\n
"},{"location":"schema/#preserves.schema.meta","title":"meta = load_schema_file(__metaschema_filename).schema  module-attribute","text":"
Schema module Namespace corresponding to Preserves Schema's metaschema.
"},{"location":"schema/#preserves.schema.Compiler","title":"Compiler()","text":"
Instances of Compiler populate an initially-empty Namespace by loading and compiling schema bundle files.
 
>>> c = Compiler()\n>>> c.load('docs/syndicate-protocols-schema-bundle.bin')\n>>> type(c.root)\n<class 'preserves.schema.Namespace'>\n
 
Attributes:
 Name Type Description root Namespace 
the root namespace into which top-level schema modules are installed.
 Source code in preserves/schema.py 
def __init__(self):\n    self.root = Namespace(())\n
"},{"location":"schema/#preserves.schema.Compiler.load","title":"load(filename)","text":"
Opens the file at filename, passing the resulting file object to load_filelike.
 Source code in preserves/schema.py 
def load(self, filename):\n\"\"\"Opens the file at `filename`, passing the resulting file object to\n    [load_filelike][preserves.schema.Compiler.load_filelike].\"\"\"\n    filename = pathlib.Path(filename)\n    with open(filename, 'rb') as f:\n        self.load_filelike(f, filename.stem)\n
"},{"location":"schema/#preserves.schema.Compiler.load_filelike","title":"load_filelike(f, module_name=None)","text":"
Reads a meta.Bundle or meta.Schema from the filelike object f, compiling and installing it in self.root. If f contains a bundle, module_name is not used, since the schema modules in the bundle know their own names; if f contains a plain schema module, however, module_name is used directly if it is a string, and if it is None, a suitable module name is computed from the name attribute of f, if it is present. If name is absent in that case, ValueError is raised.
 Source code in preserves/schema.py 
def load_filelike(self, f, module_name=None):\n\"\"\"Reads a `meta.Bundle` or `meta.Schema` from the filelike object `f`, compiling and\n    installing it in `self.root`. If `f` contains a bundle, `module_name` is not used,\n    since the schema modules in the bundle know their own names; if `f` contains a plain\n    schema module, however, `module_name` is used directly if it is a string, and if it is\n    `None`, a suitable module name is computed from the `name` attribute of `f`, if it is\n    present. If `name` is absent in that case, `ValueError` is raised.\n\n    \"\"\"\n    x = Decoder(f.read()).next()\n    if x.key == SCHEMA:\n        if module_name is None:\n            if hasattr(f, 'name'):\n                module_name = pathlib.Path(f.name).stem\n            else:\n                raise ValueError('Cannot load schema module from filelike object without a module_name')\n        self.load_schema((Symbol(module_name),), x)\n    elif x.key == BUNDLE:\n        for (p, s) in x[0].items():\n            self.load_schema(p, s)\n
"},{"location":"schema/#preserves.schema.Definition","title":"Definition(*args, **kwargs)","text":"
         Bases: SchemaObject
 
Subclasses of Definition are used to represent both standalone non-alternation definitions as well as alternatives within an Enumeration.
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n\n>>> bundle.stream.StreamListenerError.FIELD_NAMES\n['spec', 'message']\n>>> bundle.stream.StreamListenerError.SAFE_FIELD_NAMES\n['spec', 'message']\n>>> bundle.stream.StreamListenerError.ENUMERATION is None\nTrue\n\n>>> bundle.stream.Mode.object.FIELD_NAMES\n['description']\n>>> bundle.stream.Mode.object.SAFE_FIELD_NAMES\n['description']\n>>> bundle.stream.Mode.object.ENUMERATION is bundle.stream.Mode\nTrue\n\n>>> bundle.stream.CreditAmount.count.FIELD_NAMES\n[]\n>>> bundle.stream.CreditAmount.count.SAFE_FIELD_NAMES\n[]\n>>> bundle.stream.CreditAmount.count.ENUMERATION is bundle.stream.CreditAmount\nTrue\n\n>>> bundle.stream.CreditAmount.decode(parse('123'))\nCreditAmount.count(123)\n>>> bundle.stream.CreditAmount.count(123)\nCreditAmount.count(123)\n>>> bundle.stream.CreditAmount.count(123).value\n123\n
 Source code in preserves/schema.py 
def __init__(self, *args, **kwargs):\n    self._fields = args\n    if self.SIMPLE:\n        if self.EMPTY:\n            if len(args) != 0:\n                raise TypeError('%s takes no arguments' % (self._constructor_name(),))\n        else:\n            if len(args) != 1:\n                raise TypeError('%s needs exactly one argument' % (self._constructor_name(),))\n            self.value = args[0]\n    else:\n        i = 0\n        for arg in args:\n            if i >= len(self.FIELD_NAMES):\n                raise TypeError('%s given too many positional arguments' % (self._constructor_name(),))\n            setattr(self, self.SAFE_FIELD_NAMES[i], arg)\n            i = i + 1\n        for (argname, arg) in kwargs.items():\n            if hasattr(self, argname):\n                raise TypeError('%s given duplicate attribute: %r' % (self._constructor_name, argname))\n            if argname not in self.SAFE_FIELD_NAMES:\n                raise TypeError('%s given unknown attribute: %r' % (self._constructor_name, argname))\n            setattr(self, argname, arg)\n            i = i + 1\n        if i != len(self.FIELD_NAMES):\n            raise TypeError('%s needs argument(s) %r' % (self._constructor_name(), self.FIELD_NAMES))\n
"},{"location":"schema/#preserves.schema.Definition.ENUMERATION","title":"ENUMERATION = None  class-attribute","text":"
None for standalone top-level definitions with a module; otherwise, an Enumeration subclass representing a top-level alternation definition.
"},{"location":"schema/#preserves.schema.Definition.FIELD_NAMES","title":"FIELD_NAMES = []  class-attribute","text":"
List of strings: names of the fields contained within this definition, if it has named fields at all; otherwise, an empty list, and the definition is a simple wrapper for another value, in which case that value is accessed via the value attribute.
"},{"location":"schema/#preserves.schema.Definition.SAFE_FIELD_NAMES","title":"SAFE_FIELD_NAMES = []  class-attribute","text":"
The list produced by mapping safeattrname over FIELD_NAMES.
"},{"location":"schema/#preserves.schema.Enumeration","title":"Enumeration()","text":"
         Bases: SchemaObject
 
Subclasses of Enumeration represent a group of variant options within a sum type.
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n\n>>> import pprint\n>>> pprint.pprint(bundle.stream.Mode.VARIANTS)\n[(#bytes, <class 'stream.Mode.bytes'>),\n (#lines, <class 'stream.Mode.lines'>),\n (#packet, <class 'stream.Mode.packet'>),\n (#object, <class 'stream.Mode.object'>)]\n\n>>> bundle.stream.Mode.VARIANTS[0][1] is bundle.stream.Mode.bytes\nTrue\n
 Source code in preserves/schema.py 
def __init__(self):\n    raise TypeError('Cannot create instance of Enumeration')\n
"},{"location":"schema/#preserves.schema.Enumeration.VARIANTS","title":"VARIANTS = None  class-attribute","text":"
List of (Symbol, SchemaObject class) tuples representing the possible options within this sum type.
"},{"location":"schema/#preserves.schema.Namespace","title":"Namespace(prefix)","text":"
A Namespace is a dictionary-like object representing a schema module that knows its location in a schema module hierarchy and whose attributes correspond to definitions and submodules within the schema module.
 
Attributes:
 Name Type Description _prefix tuple[Symbol] 
path to this module/Namespace from the root Namespace
 Source code in preserves/schema.py 
def __init__(self, prefix):\n    self._prefix = prefix\n
"},{"location":"schema/#preserves.schema.SchemaDecodeFailed","title":"SchemaDecodeFailed(cls, p, v, failures=None)","text":"
         Bases: ValueError
 
Raised when decode cannot find a way to parse a given input.
 
Attributes:
 Name Type Description cls class 
the SchemaObject subclass attempting the parse
 pattern Value 
the failing pattern, a Value conforming to schema meta.Pattern
 value Value 
the unparseable value
 failures list[SchemaDecodeFailed] 
descriptions of failed paths attempted during the match this failure describes
 Source code in preserves/schema.py 
def __init__(self, cls, p, v, failures=None):\n    super().__init__()\n    self.cls = cls\n    self.pattern = p\n    self.value = v\n    self.failures = [] if failures is None else failures\n
"},{"location":"schema/#preserves.schema.SchemaObject","title":"SchemaObject","text":"
Base class for classes representing grammatical productions in a schema: instances of SchemaObject represent schema definitions. This is an abstract class, as are its subclasses Enumeration and Definition. It is subclasses of those subclasses, automatically produced during schema loading, that are actually instantiated.
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n\n>>> bundle.stream.Mode.mro()[1:-1]\n[<class 'preserves.schema.Enumeration'>, <class 'preserves.schema.SchemaObject'>]\n\n>>> bundle.stream.Mode.packet.mro()[1:-1]\n[<class 'stream.Mode._ALL'>, <class 'preserves.schema.Definition'>, <class 'preserves.schema.SchemaObject'>]\n\n>>> bundle.stream.StreamListenerError.mro()[1:-1]\n[<class 'preserves.schema.Definition'>, <class 'preserves.schema.SchemaObject'>]\n
 
Illustrating the class attributes on SchemaObject subclasses:
 
>>> bundle.stream.Mode.ROOTNS is bundle\nTrue\n\n>>> print(stringify(bundle.stream.Mode.SCHEMA, indent=2))\n<or [\n  [\n    \"bytes\"\n    <lit bytes>\n  ]\n  [\n    \"lines\"\n    <ref [] LineMode>\n  ]\n  [\n    \"packet\"\n    <rec <lit packet> <tuple [<named size <atom SignedInteger>>]>>\n  ]\n  [\n    \"object\"\n    <rec <lit object> <tuple [<named description any>]>>\n  ]\n]>\n\n>>> bundle.stream.Mode.MODULE_PATH\n(#stream,)\n\n>>> bundle.stream.Mode.NAME\n#Mode\n\n>>> bundle.stream.Mode.VARIANT is None\nTrue\n>>> bundle.stream.Mode.packet.VARIANT\n#packet\n
"},{"location":"schema/#preserves.schema.SchemaObject.MODULE_PATH","title":"MODULE_PATH = None  class-attribute","text":"
A sequence (tuple) of Symbols naming the path from the root to the schema module containing this definition.
"},{"location":"schema/#preserves.schema.SchemaObject.NAME","title":"NAME = None  class-attribute","text":"
A Symbol naming this definition within its module.
"},{"location":"schema/#preserves.schema.SchemaObject.ROOTNS","title":"ROOTNS = None  class-attribute","text":"
A Namespace that is the top-level environment for all bundles included in the Compiler run that produced this SchemaObject.
"},{"location":"schema/#preserves.schema.SchemaObject.SCHEMA","title":"SCHEMA = None  class-attribute","text":"
A Value conforming to schema meta.Definition (and thus often to meta.Pattern etc.), interpreted by the SchemaObject machinery to drive parsing, unparsing and so forth.
"},{"location":"schema/#preserves.schema.SchemaObject.VARIANT","title":"VARIANT = None  class-attribute","text":"
None for Definitions (such as bundle.stream.StreamListenerError above) and for overall Enumerations (such as bundle.stream.Mode), or a Symbol for variant definitions contained within an enumeration (such as bundle.stream.Mode.packet).
"},{"location":"schema/#preserves.schema.SchemaObject.__preserve__","title":"__preserve__()","text":"
Called by preserves.values.preserve: unparses the information represented by this instance, using its schema definition, to produce a Preserves Value.
 Source code in preserves/schema.py 
def __preserve__(self):\n\"\"\"Called by [preserves.values.preserve][]: *unparses* the information represented by\n    this instance, using its schema definition, to produce a Preserves `Value`.\"\"\"\n    raise NotImplementedError('Subclass responsibility')\n
"},{"location":"schema/#preserves.schema.SchemaObject.decode","title":"decode(v)  classmethod","text":"
Parses v using the SCHEMA, returning a (sub)instance of SchemaObject or raising SchemaDecodeFailed.
 Source code in preserves/schema.py 
@classmethod\ndef decode(cls, v):\n\"\"\"Parses `v` using the [SCHEMA][preserves.schema.SchemaObject.SCHEMA], returning a\n    (sub)instance of [SchemaObject][preserves.schema.SchemaObject] or raising\n    [SchemaDecodeFailed][preserves.schema.SchemaDecodeFailed].\"\"\"\n    raise NotImplementedError('Subclass responsibility')\n
"},{"location":"schema/#preserves.schema.SchemaObject.try_decode","title":"try_decode(v)  classmethod","text":"
Parses v using the SCHEMA, returning a (sub)instance of SchemaObject or None if parsing failed.
 Source code in preserves/schema.py 
@classmethod\ndef try_decode(cls, v):\n\"\"\"Parses `v` using the [SCHEMA][preserves.schema.SchemaObject.SCHEMA], returning a\n    (sub)instance of [SchemaObject][preserves.schema.SchemaObject] or `None` if parsing\n    failed.\"\"\"\n    try:\n        return cls.decode(v)\n    except SchemaDecodeFailed:\n        return None\n
"},{"location":"schema/#preserves.schema.extend","title":"extend(cls)","text":"
A decorator for function definitions. Useful for adding behaviour to the classes resulting from loading a schema module:
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n\n>>> @extend(bundle.stream.LineMode.lf)\n... def what_am_i(self):\n...     return 'I am a LINEFEED linemode'\n\n>>> @extend(bundle.stream.LineMode.crlf)\n... def what_am_i(self):\n...     return 'I am a CARRIAGE-RETURN-PLUS-LINEFEED linemode'\n\n>>> bundle.stream.LineMode.lf()\nLineMode.lf()\n>>> bundle.stream.LineMode.lf().what_am_i()\n'I am a LINEFEED linemode'\n\n>>> bundle.stream.LineMode.crlf()\nLineMode.crlf()\n>>> bundle.stream.LineMode.crlf().what_am_i()\n'I am a CARRIAGE-RETURN-PLUS-LINEFEED linemode'\n
 Source code in preserves/schema.py 
def extend(cls):\n\"\"\"A decorator for function definitions. Useful for adding *behaviour* to the classes\n    resulting from loading a schema module:\n\n    ```python\n    >>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n\n    >>> @extend(bundle.stream.LineMode.lf)\n    ... def what_am_i(self):\n    ...     return 'I am a LINEFEED linemode'\n\n    >>> @extend(bundle.stream.LineMode.crlf)\n    ... def what_am_i(self):\n    ...     return 'I am a CARRIAGE-RETURN-PLUS-LINEFEED linemode'\n\n    >>> bundle.stream.LineMode.lf()\n    LineMode.lf()\n    >>> bundle.stream.LineMode.lf().what_am_i()\n    'I am a LINEFEED linemode'\n\n    >>> bundle.stream.LineMode.crlf()\n    LineMode.crlf()\n    >>> bundle.stream.LineMode.crlf().what_am_i()\n    'I am a CARRIAGE-RETURN-PLUS-LINEFEED linemode'\n\n    ```\n\n    \"\"\"\n    @wraps(cls)\n    def extender(f):\n        setattr(cls, f.__name__, f)\n        return f\n    return extender\n
"},{"location":"schema/#preserves.schema.load_schema_file","title":"load_schema_file(filename)","text":"
Simple entry point to the compiler: creates a Compiler, calls load on it, and returns its root Namespace.
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n>>> type(bundle)\n<class 'preserves.schema.Namespace'>\n
 Source code in preserves/schema.py 
def load_schema_file(filename):\n\"\"\"Simple entry point to the compiler: creates a [Compiler][preserves.schema.Compiler],\n    calls [load][preserves.schema.Compiler.load] on it, and returns its `root`\n    [Namespace][preserves.schema.Namespace].\n\n    ```python\n    >>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n    >>> type(bundle)\n    <class 'preserves.schema.Namespace'>\n\n    ```\n    \"\"\"\n    c = Compiler()\n    c.load(filename)\n    return c.root\n
"},{"location":"schema/#preserves.schema.safeattrname","title":"safeattrname(k)","text":"
Escapes Python keywords by prepending _; passes all other strings through.
 Source code in preserves/schema.py 
def safeattrname(k):\n\"\"\"Escapes Python keywords by prepending `_`; passes all other strings through.\"\"\"\n    return k + '_' if keyword.iskeyword(k) else k\n
"},{"location":"text/","title":"Human-readable text syntax","text":"
The preserves.text module implements the Preserves human-readable text syntax.
 
The main entry points are functions stringify, parse, and parse_with_annotations.
 
>>> stringify(Record(Symbol('hi'), [1, [2, 3]]))\n'<hi 1 [2 3]>'\n>>> parse('<hi 1 [2 3]>')\n#hi(1, (2, 3))\n
"},{"location":"text/#preserves.text.Formatter","title":"Formatter(format_embedded=lambda x: x, indent=None, with_commas=False, trailing_comma=False, include_annotations=True)","text":"
         Bases: TextCodec
 
Printer (and indenting pretty-printer) for producing human-readable syntax from Preserves Values.
 
>>> f = Formatter()\n>>> f.append({'a': 1, 'b': 2})\n>>> f.append(Record(Symbol('label'), ['field1', ['field2item1', 'field2item2']]))\n>>> print(f.contents())\n{\"a\": 1 \"b\": 2} <label \"field1\" [\"field2item1\" \"field2item2\"]>\n\n>>> f = Formatter(indent=4)\n>>> f.append({'a': 1, 'b': 2})\n>>> f.append(Record(Symbol('label'), ['field1', ['field2item1', 'field2item2']]))\n>>> print(f.contents())\n{\n    \"a\": 1\n    \"b\": 2\n}\n<label \"field1\" [\n    \"field2item1\"\n    \"field2item2\"\n]>\n
 
Parameters:
 Name Type Description Default format_embedded 
function accepting an Embedded.embeddedValue and returning a Value for serialization.
 lambda x: x indent int | None 
None disables indented pretty-printing; otherwise, an int specifies indentation per nesting-level.
 None with_commas bool 
True causes commas to separate sequence and set items and dictionary entries; False omits commas.
 False trailing_comma bool 
True causes a comma to be printed after the final item or entry in a sequence, set or dictionary; False omits this trailing comma
 False include_annotations bool 
True causes annotations to be included in the output; False causes them to be omitted.
 True 
Attributes:
 Name Type Description indent_delta int 
indentation per nesting-level
 chunks list[str] 
fragments of output
 Source code in preserves/text.py 
def __init__(self,\n             format_embedded=lambda x: x,\n             indent=None,\n             with_commas=False,\n             trailing_comma=False,\n             include_annotations=True):\n    super(Formatter, self).__init__()\n    self.indent_delta = 0 if indent is None else indent\n    self.indent_distance = 0\n    self.nesting = 0\n    self.with_commas = with_commas\n    self.trailing_comma = trailing_comma\n    self.chunks = []\n    self._format_embedded = format_embedded\n    self.include_annotations = include_annotations\n
"},{"location":"text/#preserves.text.Formatter.append","title":"append(v)","text":"
Extend self.chunks with at least one chunk, together making up the text representation of v.
 Source code in preserves/text.py 
def append(self, v):\n\"\"\"Extend `self.chunks` with at least one chunk, together making up the text\n    representation of `v`.\"\"\"\n    if self.chunks and self.nesting == 0:\n        self.write_indent_space()\n    try:\n        self.nesting += 1\n        self._append(v)\n    finally:\n        self.nesting -= 1\n
"},{"location":"text/#preserves.text.Formatter.contents","title":"contents()","text":"
Returns a str constructed from the join of the chunks in self.chunks.
 Source code in preserves/text.py 
def contents(self):\n\"\"\"Returns a `str` constructed from the join of the chunks in `self.chunks`.\"\"\"\n    return u''.join(self.chunks)\n
"},{"location":"text/#preserves.text.Formatter.is_indenting","title":"is_indenting()","text":"
Returns True iff this Formatter is in pretty-printing indenting mode.
 Source code in preserves/text.py 
def is_indenting(self):\n\"\"\"Returns `True` iff this [Formatter][preserves.text.Formatter] is in pretty-printing\n    indenting mode.\"\"\"\n    return self.indent_delta > 0\n
"},{"location":"text/#preserves.text.Parser","title":"Parser(input_buffer='', include_annotations=False, parse_embedded=lambda x: x)","text":"
         Bases: TextCodec
 
Parser for the human-readable Preserves text syntax.
 
Parameters:
 Name Type Description Default input_buffer str 
initial contents of the input buffer; may subsequently be extended by calling extend.
 '' include_annotations bool 
if True, wrap each value and subvalue in an Annotated object.
 False parse_embedded 
function accepting a Value and returning a possibly-decoded form of that value suitable for placing into an Embedded object.
 lambda x: x 
Normal usage is to supply input text, and keep calling next until a ShortPacket exception is raised:
 
>>> d = Parser('123 \"hello\" @x []')\n>>> d.next()\n123\n>>> d.next()\n'hello'\n>>> d.next()\n()\n>>> d.next()\nTraceback (most recent call last):\n  ...\npreserves.error.ShortPacket: Short input buffer\n
 
Alternatively, keep calling try_next until it yields None, which is not in the domain of Preserves Values:
 
>>> d = Parser('123 \"hello\" @x []')\n>>> d.try_next()\n123\n>>> d.try_next()\n'hello'\n>>> d.try_next()\n()\n>>> d.try_next()\n
 
For convenience, Parser implements the iterator interface, backing it with try_next, so you can simply iterate over all complete values in an input:
 
>>> d = Parser('123 \"hello\" @x []')\n>>> list(d)\n[123, 'hello', ()]\n
 
>>> for v in Parser('123 \"hello\" @x []'):\n...     print(repr(v))\n123\n'hello'\n()\n
 
Supply include_annotations=True to read annotations alongside the annotated values:
 
>>> d = Parser('123 \"hello\" @x []', include_annotations=True)\n>>> list(d)\n[123, 'hello', @#x ()]\n
 
If you are incrementally reading from, say, a socket, you can use extend to add new input as if comes available:
 
>>> d = Parser('123 \"he')\n>>> d.try_next()\n123\n>>> d.try_next() # returns None because the input is incomplete\n>>> d.extend('llo\"')\n>>> d.try_next()\n'hello'\n>>> d.try_next()\n
 
Attributes:
 Name Type Description input_buffer str 
buffered input waiting to be processed
 index int 
read position within input_buffer
 Source code in preserves/text.py 
def __init__(self, input_buffer=u'', include_annotations=False, parse_embedded=lambda x: x):\n    super(Parser, self).__init__()\n    self.input_buffer = input_buffer\n    self.index = 0\n    self.include_annotations = include_annotations\n    self.parse_embedded = parse_embedded\n
"},{"location":"text/#preserves.text.Parser.extend","title":"extend(text)","text":"
Appends text to the remaining contents of self.input_buffer, trimming already-processed text from the front of self.input_buffer and resetting self.index to zero.
 Source code in preserves/text.py 
def extend(self, text):\n\"\"\"Appends `text` to the remaining contents of `self.input_buffer`, trimming already-processed\n    text from the front of `self.input_buffer` and resetting `self.index` to zero.\"\"\"\n    self.input_buffer = self.input_buffer[self.index:] + text\n    self.index = 0\n
"},{"location":"text/#preserves.text.Parser.next","title":"next()","text":"
Reads the next complete Value from the internal buffer, raising ShortPacket if too few bytes are available, or DecodeError if the input is invalid somehow.
 Source code in preserves/text.py 
def next(self):\n\"\"\"Reads the next complete `Value` from the internal buffer, raising\n    [ShortPacket][preserves.error.ShortPacket] if too few bytes are available, or\n    [DecodeError][preserves.error.DecodeError] if the input is invalid somehow.\n\n    \"\"\"\n    self.skip_whitespace()\n    c = self.peek()\n    if c == '\"':\n        self.skip()\n        return self.wrap(self.read_string('\"'))\n    if c == '|':\n        self.skip()\n        return self.wrap(Symbol(self.read_string('|')))\n    if c in ';@':\n        annotations = self.gather_annotations()\n        v = self.next()\n        if self.include_annotations:\n            v.annotations = annotations + v.annotations\n        return v\n    if c == ':':\n        raise DecodeError('Unexpected key/value separator between items')\n    if c == '#':\n        self.skip()\n        c = self.nextchar()\n        if c == 'f': return self.wrap(False)\n        if c == 't': return self.wrap(True)\n        if c == '{': return self.wrap(frozenset(self.upto('}')))\n        if c == '\"': return self.wrap(self.read_literal_binary())\n        if c == 'x':\n            c = self.nextchar()\n            if c == '\"': return self.wrap(self.read_hex_binary())\n            if c == 'f': return self.wrap(self.read_hex_float(4))\n            if c == 'd': return self.wrap(self.read_hex_float(8))\n            raise DecodeError('Invalid #x syntax')\n        if c == '[': return self.wrap(self.read_base64_binary())\n        if c == '=':\n            old_ann = self.include_annotations\n            self.include_annotations = True\n            bs_val = self.next()\n            self.include_annotations = old_ann\n            if len(bs_val.annotations) > 0:\n                raise DecodeError('Annotations not permitted after #=')\n            bs_val = bs_val.item\n            if not isinstance(bs_val, bytes):\n                raise DecodeError('ByteString must follow #=')\n            return self.wrap(Decoder(bs_val, include_annotations = self.include_annotations).next())\n        if c == '!':\n            if self.parse_embedded is None:\n                raise DecodeError('No parse_embedded function supplied')\n            return self.wrap(Embedded(self.parse_embedded(self.next())))\n        raise DecodeError('Invalid # syntax')\n    if c == '<':\n        self.skip()\n        vs = self.upto('>')\n        if len(vs) == 0:\n            raise DecodeError('Missing record label')\n        return self.wrap(Record(vs[0], vs[1:]))\n    if c == '[':\n        self.skip()\n        return self.wrap(self.upto(']'))\n    if c == '{':\n        self.skip()\n        return self.wrap(self.read_dictionary())\n    if c in '>]}':\n        raise DecodeError('Unexpected ' + c)\n    self.skip()\n    return self.wrap(self.read_raw_symbol_or_number([c]))\n
"},{"location":"text/#preserves.text.Parser.try_next","title":"try_next()","text":"
Like next, but returns None instead of raising ShortPacket.
 Source code in preserves/text.py 
def try_next(self):\n\"\"\"Like [next][preserves.text.Parser.next], but returns `None` instead of raising\n    [ShortPacket][preserves.error.ShortPacket].\"\"\"\n    start = self.index\n    try:\n        return self.next()\n    except ShortPacket:\n        self.index = start\n        return None\n
"},{"location":"text/#preserves.text.parse","title":"parse(text, **kwargs)","text":"
Yields the first complete encoded value from text, passing kwargs through to the Parser constructor. Raises exceptions as per next.
 
Parameters:
 Name Type Description Default text str 
encoded data to decode
 required Source code in preserves/text.py 
def parse(text, **kwargs):\n\"\"\"Yields the first complete encoded value from `text`, passing `kwargs` through to the\n    [Parser][preserves.text.Parser] constructor. Raises exceptions as per\n    [next][preserves.text.Parser.next].\n\n    Args:\n        text (str): encoded data to decode\n\n    \"\"\"\n    return Parser(input_buffer=text, **kwargs).next()\n
"},{"location":"text/#preserves.text.parse_with_annotations","title":"parse_with_annotations(bs, **kwargs)","text":"
Like parse, but supplying include_annotations=True to the Parser constructor.
 Source code in preserves/text.py 
def parse_with_annotations(bs, **kwargs):\n\"\"\"Like [parse][preserves.text.parse], but supplying `include_annotations=True` to the\n    [Parser][preserves.text.Parser] constructor.\"\"\"\n    return Parser(input_buffer=bs, include_annotations=True, **kwargs).next()\n
"},{"location":"text/#preserves.text.stringify","title":"stringify(v, **kwargs)","text":"
Convert a single Value v to a string. Any supplied kwargs are passed on to the underlying Formatter constructor.
 Source code in preserves/text.py 
def stringify(v, **kwargs):\n\"\"\"Convert a single `Value` `v` to a string. Any supplied `kwargs` are passed on to the\n    underlying [Formatter][preserves.text.Formatter] constructor.\"\"\"\n    e = Formatter(**kwargs)\n    e.append(v)\n    return e.contents()\n
"},{"location":"values/","title":"Representations of Values","text":"
Python's strings, byte strings, integers, booleans, and double-precision floats stand directly for their Preserves counterparts. Wrapper objects for Float and Symbol complete the suite of atomic types.
 
Python's lists and tuples correspond to Preserves Sequences, and dicts and sets to Dictionary and Set values, respectively. Preserves Records are represented by Record objects. Finally, embedded values are represented by Embedded objects.
 
The preserves.values module implements the core representations of Preserves Values as Python values.
"},{"location":"values/#preserves.values.Annotated","title":"Annotated(item)","text":"
         Bases: object
 
A Preserves Value along with a sequence of Values annotating it. Compares equal to the underlying Value, ignoring the annotations. See the specification document for more about annotations.
 
>>> import preserves\n>>> a = preserves.parse('''\n... ; A comment\n... [1 2 3]\n... ''', include_annotations=True)\n>>> a\n@' A comment' (1, 2, 3)\n>>> a.item\n(1, 2, 3)\n>>> a.annotations\n[' A comment']\n>>> a == (1, 2, 3)\nTrue\n>>> a == preserves.parse('@xyz [1 2 3]', include_annotations=True)\nTrue\n>>> a[0]\nTraceback (most recent call last):\n  ...\nTypeError: 'Annotated' object is not subscriptable\n>>> a.item[0]\n1\n>>> type(a.item[0])\n<class 'preserves.values.Annotated'>\n>>> a.item[0].annotations\n[]\n>>> print(preserves.stringify(a))\n@\" A comment\" [1 2 3]\n>>> print(preserves.stringify(a, include_annotations=False))\n[1 2 3]\n
 
Attributes:
 Name Type Description item Value 
the underlying annotated Value
 annotations list[Value] 
the annotations attached to self.item
 Source code in preserves/values.py 
def __init__(self, item):\n    self.annotations = []\n    self.item = item\n
"},{"location":"values/#preserves.values.Annotated.peel","title":"peel()","text":"
Calls strip_annotations on self with depth=1.
 Source code in preserves/values.py 
def peel(self):\n\"\"\"Calls [strip_annotations][preserves.values.strip_annotations] on `self` with `depth=1`.\"\"\"\n    return strip_annotations(self, 1)\n
"},{"location":"values/#preserves.values.Annotated.strip","title":"strip(depth=inf)","text":"
Calls strip_annotations on self and depth.
 Source code in preserves/values.py 
def strip(self, depth=inf):\n\"\"\"Calls [strip_annotations][preserves.values.strip_annotations] on `self` and `depth`.\"\"\"\n    return strip_annotations(self, depth)\n
"},{"location":"values/#preserves.values.Embedded","title":"Embedded(embeddedValue)","text":"
Representation of a Preserves Embedded value. For more on the meaning and use of embedded values, see the specification.
 
>>> import io\n>>> e = Embedded(io.StringIO('some text'))\n>>> e                                        # doctest: +ELLIPSIS\n#!<_io.StringIO object at ...>\n>>> e.embeddedValue                          # doctest: +ELLIPSIS\n<_io.StringIO object at ...>\n
 
>>> import preserves\n>>> print(preserves.stringify(Embedded(None)))\nTraceback (most recent call last):\n  ...\nTypeError: Cannot preserves-format: None\n>>> print(preserves.stringify(Embedded(None), format_embedded=lambda x: 'abcdef'))\n#!\"abcdef\"\n
 
Attributes:
 Name Type Description embeddedValue 
any Python value; could be a platform object, could be a representation of a Preserves Value, could be None, could be anything!
 Source code in preserves/values.py 
def __init__(self, embeddedValue):\n    self.embeddedValue = embeddedValue\n
"},{"location":"values/#preserves.values.Float","title":"Float(value)","text":"
         Bases: object
 
Wrapper for treating a Python double-precision floating-point value as a single-precision (32-bit) float, from Preserves' perspective. (Python lacks native single-precision floating point support.)
 
>>> Float(3.45)\nFloat(3.45)\n>>> import preserves\n>>> preserves.stringify(Float(3.45))\n'3.45f'\n>>> preserves.stringify(3.45)\n'3.45'\n>>> preserves.parse('3.45f')\nFloat(3.45)\n>>> preserves.parse('3.45')\n3.45\n>>> preserves.encode(Float(3.45))\nb'\\x82@\\\\\\xcc\\xcd'\n>>> preserves.encode(3.45)\nb'\\x83@\\x0b\\x99\\x99\\x99\\x99\\x99\\x9a'\n
 
Attributes:
 Name Type Description value float 
the double-precision representation of intended single-precision value
 Source code in preserves/values.py 
def __init__(self, value):\n    self.value = value\n
"},{"location":"values/#preserves.values.Float.from_bytes","title":"from_bytes(bs)  staticmethod","text":"
Converts a 4-byte-long byte string to a 32-bit single-precision floating point value wrapped in a Float instance. Takes care to preserve the quiet/signalling bit-pattern of NaN values, unlike its struct.unpack('>f', ...) equivalent.
 
>>> Float.from_bytes(b'\\x7f\\x80\\x00{')\nFloat(nan)\n>>> Float.from_bytes(b'\\x7f\\x80\\x00{').to_bytes()\nb'\\x7f\\x80\\x00{'\n\n>>> struct.unpack('>f', b'\\x7f\\x80\\x00{')[0]\nnan\n>>> Float(struct.unpack('>f', b'\\x7f\\x80\\x00{')[0]).to_bytes()\nb'\\x7f\\xc0\\x00{'\n>>> struct.pack('>f', struct.unpack('>f', b'\\x7f\\x80\\x00{')[0])\nb'\\x7f\\xc0\\x00{'\n
 
(Note well the difference between 7f80007b and 7fc0007b!)
 Source code in preserves/values.py 
@staticmethod\ndef from_bytes(bs):\n\"\"\"Converts a 4-byte-long byte string to a 32-bit single-precision floating point value\n    wrapped in a [Float][preserves.values.Float] instance. Takes care to preserve the\n    quiet/signalling bit-pattern of NaN values, unlike its `struct.unpack('>f', ...)`\n    equivalent.\n\n    ```python\n    >>> Float.from_bytes(b'\\\\x7f\\\\x80\\\\x00{')\n    Float(nan)\n    >>> Float.from_bytes(b'\\\\x7f\\\\x80\\\\x00{').to_bytes()\n    b'\\\\x7f\\\\x80\\\\x00{'\n\n    >>> struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0]\n    nan\n    >>> Float(struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0]).to_bytes()\n    b'\\\\x7f\\\\xc0\\\\x00{'\n    >>> struct.pack('>f', struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0])\n    b'\\\\x7f\\\\xc0\\\\x00{'\n\n    ```\n\n    (Note well the difference between `7f80007b` and `7fc0007b`!)\n\n    \"\"\"\n    vf = struct.unpack('>I', bs)[0]\n    if (vf & 0x7f800000) == 0x7f800000:\n        # NaN or inf. Preserve quiet/signalling bit by manually expanding to double-precision.\n        sign = vf >> 31\n        payload = vf & 0x007fffff\n        dbs = struct.pack('>Q', (sign << 63) | 0x7ff0000000000000 | (payload << 29))\n        return Float(struct.unpack('>d', dbs)[0])\n    else:\n        return Float(struct.unpack('>f', bs)[0])\n
"},{"location":"values/#preserves.values.Float.to_bytes","title":"to_bytes()","text":"
Converts this 32-bit single-precision floating point value to its binary32 format, taking care to preserve the quiet/signalling bit-pattern of NaN values, unlike its struct.pack('>f', ...) equivalent.
 
>>> Float.from_bytes(b'\\x7f\\x80\\x00{')\nFloat(nan)\n>>> Float.from_bytes(b'\\x7f\\x80\\x00{').to_bytes()\nb'\\x7f\\x80\\x00{'\n\n>>> struct.unpack('>f', b'\\x7f\\x80\\x00{')[0]\nnan\n>>> Float(struct.unpack('>f', b'\\x7f\\x80\\x00{')[0]).to_bytes()\nb'\\x7f\\xc0\\x00{'\n>>> struct.pack('>f', struct.unpack('>f', b'\\x7f\\x80\\x00{')[0])\nb'\\x7f\\xc0\\x00{'\n
 
(Note well the difference between 7f80007b and 7fc0007b!)
 Source code in preserves/values.py 
def to_bytes(self):\n\"\"\"Converts this 32-bit single-precision floating point value to its binary32 format,\n    taking care to preserve the quiet/signalling bit-pattern of NaN values, unlike its\n    `struct.pack('>f', ...)` equivalent.\n\n    ```python\n    >>> Float.from_bytes(b'\\\\x7f\\\\x80\\\\x00{')\n    Float(nan)\n    >>> Float.from_bytes(b'\\\\x7f\\\\x80\\\\x00{').to_bytes()\n    b'\\\\x7f\\\\x80\\\\x00{'\n\n    >>> struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0]\n    nan\n    >>> Float(struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0]).to_bytes()\n    b'\\\\x7f\\\\xc0\\\\x00{'\n    >>> struct.pack('>f', struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0])\n    b'\\\\x7f\\\\xc0\\\\x00{'\n\n    ```\n\n    (Note well the difference between `7f80007b` and `7fc0007b`!)\n\n    \"\"\"\n\n    if math.isnan(self.value) or math.isinf(self.value):\n        dbs = struct.pack('>d', self.value)\n        vd = struct.unpack('>Q', dbs)[0]\n        sign = vd >> 63\n        payload = (vd >> 29) & 0x007fffff\n        vf = (sign << 31) | 0x7f800000 | payload\n        return struct.pack('>I', vf)\n    else:\n        return struct.pack('>f', self.value)\n
"},{"location":"values/#preserves.values.ImmutableDict","title":"ImmutableDict(*args, **kwargs)","text":"
         Bases: dict
 
A subclass of Python's built-in dict that overrides methods that could mutate the dictionary, causing them to raise TypeError('Immutable') if called.
 
Implements the __hash__ method, allowing ImmutableDict instances to be used whereever immutable data are permitted; in particular, as keys in other dictionaries.
 
>>> d = ImmutableDict([('a', 1), ('b', 2)])\n>>> d\n{'a': 1, 'b': 2}\n>>> d['c'] = 3\nTraceback (most recent call last):\n  ...\nTypeError: Immutable\n>>> del d['b']\nTraceback (most recent call last):\n  ...\nTypeError: Immutable\n
 Source code in preserves/values.py 
def __init__(self, *args, **kwargs):\n    if hasattr(self, '__hash'): raise TypeError('Immutable')\n    super(ImmutableDict, self).__init__(*args, **kwargs)\n    self.__hash = None\n
"},{"location":"values/#preserves.values.ImmutableDict.from_kvs","title":"from_kvs(kvs)  staticmethod","text":"
Constructs an ImmutableDict from a sequence of alternating keys and values; compare to the ImmutableDict constructor, which takes a sequence of key-value pairs.
 
>>> ImmutableDict.from_kvs(['a', 1, 'b', 2])\n{'a': 1, 'b': 2}\n>>> ImmutableDict.from_kvs(['a', 1, 'b', 2])['c'] = 3\nTraceback (most recent call last):\n  ...\nTypeError: Immutable\n
 Source code in preserves/values.py 
@staticmethod\ndef from_kvs(kvs):\n\"\"\"Constructs an [ImmutableDict][preserves.values.ImmutableDict] from a sequence of\n    alternating keys and values; compare to the\n    [ImmutableDict][preserves.values.ImmutableDict] constructor, which takes a sequence of\n    key-value pairs.\n\n    ```python\n    >>> ImmutableDict.from_kvs(['a', 1, 'b', 2])\n    {'a': 1, 'b': 2}\n    >>> ImmutableDict.from_kvs(['a', 1, 'b', 2])['c'] = 3\n    Traceback (most recent call last):\n      ...\n    TypeError: Immutable\n\n    ```\n\n    \"\"\"\n\n    i = iter(kvs)\n    result = ImmutableDict()\n    result_proxy = super(ImmutableDict, result)\n    try:\n        while True:\n            k = next(i)\n            try:\n                v = next(i)\n            except StopIteration:\n                raise DecodeError(\"Missing dictionary value\")\n            result_proxy.__setitem__(k, v)\n    except StopIteration:\n        pass\n    return result\n
"},{"location":"values/#preserves.values.Record","title":"Record(key, fields)","text":"
         Bases: object
 
Representation of Preserves Records, which are a pair of a label Value and a sequence of field Values.
 
>>> r = Record(Symbol('label'), ['field1', ['field2item1', 'field2item2']])\n>>> r\n#label('field1', ['field2item1', 'field2item2'])\n>>> r.key\n#label\n>>> r.fields\n('field1', ['field2item1', 'field2item2'])\n>>> import preserves\n>>> preserves.stringify(r)\n'<label \"field1\" [\"field2item1\" \"field2item2\"]>'\n>>> r == preserves.parse('<label \"field1\" [\"field2item1\" \"field2item2\"]>')\nTrue\n
 
Parameters:
 Name Type Description Default key Value 
the Record's label
 required fields iterable[Value] 
the fields of the Record
 required 
Attributes:
 Name Type Description key Value 
the Record's label
 fields tuple[Value] 
the fields of the Record
 Source code in preserves/values.py 
def __init__(self, key, fields):\n    self.key = key\n    self.fields = tuple(fields)\n    self.__hash = None\n
"},{"location":"values/#preserves.values.Record.makeBasicConstructor","title":"makeBasicConstructor(label, fieldNames)  staticmethod","text":"
Constructs and returns a \"constructor\" for Records having a certain label and number of fields.
 Deprecated 
Use preserves.schema definitions instead.
 
The \"constructor\" is a callable function that accepts len(fields) arguments and returns a Record with label as its label and the arguments to the constructor as field values.
 
In addition, the \"constructor\" has a constructorInfo attribute holding a RecordConstructorInfo object, an isClassOf attribute holding a unary function that returns True iff its argument is a Record with label label and arity len(fieldNames), and an ensureClassOf attribute that raises an Exception if isClassOf returns false on its argument and returns the argument otherwise.
 
Finally, for each field name f in fieldNames, the \"constructor\" object has an attribute _f that is a unary function that retrieves the f field from the passed in argument.
 
>>> c = Record.makeBasicConstructor(Symbol('date'), 'year month day')\n>>> c(1969, 7, 16)\n#date(1969, 7, 16)\n>>> c.constructorInfo\n#date/3\n>>> c.isClassOf(c(1969, 7, 16))\nTrue\n>>> c.isClassOf(Record(Symbol('date'), [1969, 7, 16]))\nTrue\n>>> c.isClassOf(Record(Symbol('date'), [1969]))\nFalse\n>>> c.ensureClassOf(c(1969, 7, 16))\n#date(1969, 7, 16)\n>>> c.ensureClassOf(Record(Symbol('date'), [1969]))\nTraceback (most recent call last):\n  ...\nTypeError: Record: expected #date/3, got #date(1969)\n>>> c._year(c(1969, 7, 16))\n1969\n>>> c._month(c(1969, 7, 16))\n7\n>>> c._day(c(1969, 7, 16))\n16\n
 
Parameters:
 Name Type Description Default label Value 
Label to use for constructed/matched Records
 required fieldNames tuple[str] | list[str] | str 
Names of the Record's fields
 required Source code in preserves/values.py 
@staticmethod\ndef makeBasicConstructor(label, fieldNames):\n\"\"\"Constructs and returns a \"constructor\" for `Record`s having a certain `label` and\n    number of fields.\n\n    Deprecated:\n       Use [preserves.schema][] definitions instead.\n\n    The \"constructor\" is a callable function that accepts `len(fields)` arguments and\n    returns a [Record][preserves.values.Record] with `label` as its label and the arguments\n    to the constructor as field values.\n\n    In addition, the \"constructor\" has a `constructorInfo` attribute holding a\n    [RecordConstructorInfo][preserves.values.RecordConstructorInfo] object, an `isClassOf`\n    attribute holding a unary function that returns `True` iff its argument is a\n    [Record][preserves.values.Record] with label `label` and arity `len(fieldNames)`, and\n    an `ensureClassOf` attribute that raises an `Exception` if `isClassOf` returns false on\n    its argument and returns the argument otherwise.\n\n    Finally, for each field name `f` in `fieldNames`, the \"constructor\" object has an\n    attribute `_f` that is a unary function that retrieves the `f` field from the passed in\n    argument.\n\n    ```python\n    >>> c = Record.makeBasicConstructor(Symbol('date'), 'year month day')\n    >>> c(1969, 7, 16)\n    #date(1969, 7, 16)\n    >>> c.constructorInfo\n    #date/3\n    >>> c.isClassOf(c(1969, 7, 16))\n    True\n    >>> c.isClassOf(Record(Symbol('date'), [1969, 7, 16]))\n    True\n    >>> c.isClassOf(Record(Symbol('date'), [1969]))\n    False\n    >>> c.ensureClassOf(c(1969, 7, 16))\n    #date(1969, 7, 16)\n    >>> c.ensureClassOf(Record(Symbol('date'), [1969]))\n    Traceback (most recent call last):\n      ...\n    TypeError: Record: expected #date/3, got #date(1969)\n    >>> c._year(c(1969, 7, 16))\n    1969\n    >>> c._month(c(1969, 7, 16))\n    7\n    >>> c._day(c(1969, 7, 16))\n    16\n\n    ```\n\n    Args:\n        label (Value): Label to use for constructed/matched `Record`s\n        fieldNames (tuple[str] | list[str] | str): Names of the `Record`'s fields\n\n    \"\"\"\n    if type(fieldNames) == str:\n        fieldNames = fieldNames.split()\n    arity = len(fieldNames)\n    def ctor(*fields):\n        if len(fields) != arity:\n            raise Exception(\"Record: cannot instantiate %r expecting %d fields with %d fields\"%(\n                label,\n                arity,\n                len(fields)))\n        return Record(label, fields)\n    ctor.constructorInfo = RecordConstructorInfo(label, arity)\n    ctor.isClassOf = lambda v: \\\n                     isinstance(v, Record) and v.key == label and len(v.fields) == arity\n    def ensureClassOf(v):\n        if not ctor.isClassOf(v):\n            raise TypeError(\"Record: expected %r/%d, got %r\" % (label, arity, v))\n        return v\n    ctor.ensureClassOf = ensureClassOf\n    for fieldIndex in range(len(fieldNames)):\n        fieldName = fieldNames[fieldIndex]\n        # Stupid python scoping bites again\n        def getter(fieldIndex):\n            return lambda v: ensureClassOf(v)[fieldIndex]\n        setattr(ctor, '_' + fieldName, getter(fieldIndex))\n    return ctor\n
"},{"location":"values/#preserves.values.Record.makeConstructor","title":"makeConstructor(labelSymbolText, fieldNames)  staticmethod","text":"
Equivalent to Record.makeBasicConstructor(Symbol(labelSymbolText), fieldNames).
 Deprecated 
Use preserves.schema definitions instead.
 Source code in preserves/values.py 
@staticmethod\ndef makeConstructor(labelSymbolText, fieldNames):\n\"\"\"\n    Equivalent to `Record.makeBasicConstructor(Symbol(labelSymbolText), fieldNames)`.\n\n    Deprecated:\n       Use [preserves.schema][] definitions instead.\n    \"\"\"\n    return Record.makeBasicConstructor(Symbol(labelSymbolText), fieldNames)\n
"},{"location":"values/#preserves.values.RecordConstructorInfo","title":"RecordConstructorInfo(key, arity)","text":"
         Bases: object
 
Describes the shape of a Record constructor, namely its label and its arity (field count).
 
>>> RecordConstructorInfo(Symbol('label'), 3)\n#label/3\n
 
Attributes:
 Name Type Description key Value 
the label of matching Records
 arity int 
the number of fields in matching Records
 Source code in preserves/values.py 
def __init__(self, key, arity):\n    self.key = key\n    self.arity = arity\n
"},{"location":"values/#preserves.values.Symbol","title":"Symbol(name)","text":"
         Bases: object
 
Representation of Preserves Symbols.
 
>>> Symbol('xyz')\n#xyz\n>>> Symbol('xyz').name\n'xyz'\n>>> import preserves\n>>> preserves.stringify(Symbol('xyz'))\n'xyz'\n>>> preserves.stringify(Symbol('hello world'))\n'|hello world|'\n>>> preserves.parse('xyz')\n#xyz\n>>> preserves.parse('|hello world|')\n#hello world\n
 
Attributes:
 Name Type Description name str | Symbol 
The symbol's text label. If an existing Symbol is passed in, the existing Symbol's name is used as the name for the new Symbol.
 Source code in preserves/values.py 
def __init__(self, name):\n    self.name = name.name if isinstance(name, Symbol) else name\n
"},{"location":"values/#preserves.values.annotate","title":"annotate(v, *anns)","text":"
Wraps v in an Annotated object, if it isn't already wrapped, and appends each of the anns to the Annotated's annotations sequence. NOTE: Does not recursively ensure that any parts of the argument v are themselves wrapped in Annotated objects!
 
>>> import preserves\n>>> print(preserves.stringify(annotate(123, \"A comment\", \"Another comment\")))\n@\"A comment\" @\"Another comment\" 123\n
 Source code in preserves/values.py 
def annotate(v, *anns):\n\"\"\"Wraps `v` in an [Annotated][preserves.values.Annotated] object, if it isn't already\n    wrapped, and appends each of the `anns` to the [Annotated][preserves.values.Annotated]'s\n    `annotations` sequence. NOTE: Does not recursively ensure that any parts of the argument\n    `v` are themselves wrapped in [Annotated][preserves.values.Annotated] objects!\n\n    ```python\n    >>> import preserves\n    >>> print(preserves.stringify(annotate(123, \"A comment\", \"Another comment\")))\n    @\"A comment\" @\"Another comment\" 123\n\n    ```\n    \"\"\"\n    if not is_annotated(v):\n        v = Annotated(v)\n    for a in anns:\n        v.annotations.append(a)\n    return v\n
"},{"location":"values/#preserves.values.cmp_floats","title":"cmp_floats(a, b)","text":"
Implements the totalOrder predicate defined in section 5.10 of IEEE Std 754-2008.
 Source code in preserves/values.py 
def cmp_floats(a, b):\n\"\"\"Implements the `totalOrder` predicate defined in section 5.10 of [IEEE Std\n    754-2008](https://dx.doi.org/10.1109/IEEESTD.2008.4610935).\n\n    \"\"\"\n    a = float_to_int(a)\n    b = float_to_int(b)\n    if a & 0x8000000000000000: a = a ^ 0x7fffffffffffffff\n    if b & 0x8000000000000000: b = b ^ 0x7fffffffffffffff\n    return a - b\n
"},{"location":"values/#preserves.values.dict_kvs","title":"dict_kvs(d)","text":"
Generator function yielding a sequence of alternating keys and values from d. In some sense the inverse of ImmutableDict.from_kvs.
 
>>> list(dict_kvs({'a': 1, 'b': 2}))\n['a', 1, 'b', 2]\n
 Source code in preserves/values.py 
def dict_kvs(d):\n\"\"\"Generator function yielding a sequence of alternating keys and values from `d`. In some\n    sense the inverse of [ImmutableDict.from_kvs][preserves.values.ImmutableDict.from_kvs].\n\n    ```python\n    >>> list(dict_kvs({'a': 1, 'b': 2}))\n    ['a', 1, 'b', 2]\n\n    ```\n    \"\"\"\n    for k in d:\n        yield k\n        yield d[k]\n
"},{"location":"values/#preserves.values.is_annotated","title":"is_annotated(v)","text":"
True iff v is an instance of Annotated.
 Source code in preserves/values.py 
def is_annotated(v):\n\"\"\"`True` iff `v` is an instance of [Annotated][preserves.values.Annotated].\"\"\"\n    return isinstance(v, Annotated)\n
"},{"location":"values/#preserves.values.preserve","title":"preserve(v)","text":"
Converts v to a representation of a Preserves Value by (repeatedly) setting
 
v = v.__preserve__()\n
 
while v has a __preserve__ method. Parsed Schema values are able to render themselves to their serialized representations this way.
 Source code in preserves/values.py 
def preserve(v):\n\"\"\"Converts `v` to a representation of a Preserves `Value` by (repeatedly) setting\n\n    ```python\n    v = v.__preserve__()\n    ```\n\n    while `v` has a `__preserve__` method. Parsed [Schema][preserves.schema]\n    values are able to render themselves to their serialized representations this way.\n\n    \"\"\"\n    while hasattr(v, '__preserve__'):\n        v = v.__preserve__()\n    return v\n
"},{"location":"values/#preserves.values.strip_annotations","title":"strip_annotations(v, depth=inf)","text":"
Exposes depth layers of raw structure of potentially-Annotated Values. If depth==0 or v is not Annotated, just returns v. Otherwise, descends recursively into the structure of v.item.
 
>>> import preserves\n>>> a = preserves.parse('@\"A comment\" [@a 1 @b 2 @c 3]', include_annotations=True)\n>>> is_annotated(a)\nTrue\n>>> print(preserves.stringify(a))\n@\"A comment\" [@a 1 @b 2 @c 3]\n>>> print(preserves.stringify(strip_annotations(a)))\n[1 2 3]\n>>> print(preserves.stringify(strip_annotations(a, depth=1)))\n[@a 1 @b 2 @c 3]\n
 Source code in preserves/values.py 
def strip_annotations(v, depth=inf):\n\"\"\"Exposes `depth` layers of raw structure of\n    potentially-[Annotated][preserves.values.Annotated] `Value`s. If `depth==0` or `v` is not\n    [Annotated][preserves.values.Annotated], just returns `v`. Otherwise, descends recursively\n    into the structure of `v.item`.\n\n    ```python\n    >>> import preserves\n    >>> a = preserves.parse('@\"A comment\" [@a 1 @b 2 @c 3]', include_annotations=True)\n    >>> is_annotated(a)\n    True\n    >>> print(preserves.stringify(a))\n    @\"A comment\" [@a 1 @b 2 @c 3]\n    >>> print(preserves.stringify(strip_annotations(a)))\n    [1 2 3]\n    >>> print(preserves.stringify(strip_annotations(a, depth=1)))\n    [@a 1 @b 2 @c 3]\n\n    ```\n    \"\"\"\n\n    if depth == 0: return v\n    if not is_annotated(v): return v\n\n    next_depth = depth - 1\n    def walk(v):\n        return strip_annotations(v, next_depth)\n\n    v = v.item\n    if isinstance(v, Record):\n        return Record(strip_annotations(v.key, depth), tuple(walk(f) for f in v.fields))\n    elif isinstance(v, list):\n        return tuple(walk(f) for f in v)\n    elif isinstance(v, tuple):\n        return tuple(walk(f) for f in v)\n    elif isinstance(v, set):\n        return frozenset(walk(f) for f in v)\n    elif isinstance(v, frozenset):\n        return frozenset(walk(f) for f in v)\n    elif isinstance(v, dict):\n        return ImmutableDict.from_kvs(walk(f) for f in dict_kvs(v))\n    elif is_annotated(v):\n        raise ValueError('Improper annotation structure')\n    else:\n        return v\n
"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"separator":"[\\s\\-]+","pipeline":["stopWordFilter"]},"docs":[{"location":"","title":"Overview","text":"
pip install preserves\n
 
This package (preserves on pypi.org) implements Preserves for Python 3.x. It provides the core semantics as well as both the human-readable text syntax (a superset of JSON) and machine-oriented binary format (including canonicalization) for Preserves. It also implements Preserves Schema and Preserves Path.
 
     
  • Main package API: preserves
    •  
"},{"location":"#what-is-preserves","title":"What is Preserves?","text":"
Preserves is a data model, with associated serialization formats.
 
It supports records with user-defined labels, embedded references, and the usual suite of atomic and compound data types, including binary data as a distinct type from text strings. Its annotations allow separation of data from metadata such as comments, trace information, and provenance information.
 
Preserves departs from many other data languages in defining how to compare two values. Comparison is based on the data model, not on syntax or on data structures of any particular implementation language.
"},{"location":"#mapping-between-preserves-values-and-python-values","title":"Mapping between Preserves values and Python values","text":"
Preserves Values are categorized in the following way:
 
                      Value = Atom\n                            | Compound\n                            | Embedded\n\n                       Atom = Boolean\n                            | Float\n                            | Double\n                            | SignedInteger\n                            | String\n                            | ByteString\n                            | Symbol\n\n                   Compound = Record\n                            | Sequence\n                            | Set\n                            | Dictionary\n
 
Python's strings, byte strings, integers, booleans, and double-precision floats stand directly for their Preserves counterparts. Wrapper objects for Float and Symbol complete the suite of atomic types.
 
Python's lists and tuples correspond to Preserves Sequences, and dicts and sets to Dictionary and Set values, respectively. Preserves Records are represented by Record objects. Finally, embedded values are represented by Embedded objects.
"},{"location":"api/","title":"The top-level preserves package","text":"
import preserves\n
 
The main package re-exports a subset of the exports of its constituent modules:
 
     
  •  
    From preserves.values:
     
       
    • Annotated
      •  
    • Embedded
      •  
    • Float
      •  
    • ImmutableDict
      •  
    • Record
      •  
    • Symbol
      •  
    • annotate
      •  
    • is_annotated
      •  
    • preserve
      •  
    • strip_annotations
      •  
     
    •  
  •  
    From preserves.error:
     
       
    • DecodeError
      •  
    • EncodeError
      •  
    • ShortPacket
      •  
     
    •  
  •  
    From preserves.binary:
     
       
    • Decoder
      •  
    • Encoder
      •  
    • canonicalize
      •  
    • decode
      •  
    • decode_with_annotations
      •  
    • encode
      •  
     
    •  
  •  
    From preserves.text:
     
       
    • Formatter
      •  
    • Parser
      •  
    • parse
      •  
    • parse_with_annotations
      •  
    • stringify
      •  
     
    •  
  •  
    From preserves.compare:
     
       
    • cmp
      •  
     
    •  
  •  
    From preserves.merge:
     
       
    • merge
      •  
     
    •  
 
It also exports the compare and fold modules themselves, permitting patterns like
 
>>> from preserves import *\n>>> compare.cmp(123, 234)\n-1\n
 
Finally, it provides a few utility aliases for common tasks:
"},{"location":"api/#preserves.dumps","title":"dumps = stringify  module-attribute","text":"
This alias for stringify provides a familiar pythonesque name for converting a Preserves Value to a string.
"},{"location":"api/#preserves.loads","title":"loads = parse  module-attribute","text":"
This alias for parse provides a familiar pythonesque name for converting a string to a Preserves Value.
"},{"location":"binary/","title":"Machine-oriented binary syntax","text":"
The preserves.binary module implements the Preserves machine-oriented binary syntax.
 
The main entry points are functions encode, canonicalize, decode, and decode_with_annotations.
 
>>> encode(Record(Symbol('hi'), []))\nb'\\xb4\\xb3\\x02hi\\x84'\n>>> decode(b'\\xb4\\xb3\\x02hi\\x84')\n#hi()\n
"},{"location":"binary/#preserves.binary.Decoder","title":"Decoder(packet=b'', include_annotations=False, decode_embedded=lambda x: x)","text":"
         Bases: BinaryCodec
 
Implementation of a decoder for the machine-oriented binary Preserves syntax.
 
Parameters:
 Name Type Description Default packet bytes 
initial contents of the input buffer; may subsequently be extended by calling extend.
 b'' include_annotations bool 
if True, wrap each value and subvalue in an Annotated object.
 False decode_embedded 
function accepting a Value and returning a possibly-decoded form of that value suitable for placing into an Embedded object.
 lambda x: x 
Normal usage is to supply a buffer, and keep calling next until a ShortPacket exception is raised:
 
>>> d = Decoder(b'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84')\n>>> d.next()\n123\n>>> d.next()\n'hello'\n>>> d.next()\n()\n>>> d.next()\nTraceback (most recent call last):\n  ...\npreserves.error.ShortPacket: Short packet\n
 
Alternatively, keep calling try_next until it yields None, which is not in the domain of Preserves Values:
 
>>> d = Decoder(b'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84')\n>>> d.try_next()\n123\n>>> d.try_next()\n'hello'\n>>> d.try_next()\n()\n>>> d.try_next()\n
 
For convenience, Decoder implements the iterator interface, backing it with try_next, so you can simply iterate over all complete values in an input:
 
>>> d = Decoder(b'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84')\n>>> list(d)\n[123, 'hello', ()]\n
 
>>> for v in Decoder(b'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84'):\n...     print(repr(v))\n123\n'hello'\n()\n
 
Supply include_annotations=True to read annotations alongside the annotated values:
 
>>> d = Decoder(b'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84', include_annotations=True)\n>>> list(d)\n[123, 'hello', @#x ()]\n
 
If you are incrementally reading from, say, a socket, you can use extend to add new input as if comes available:
 
>>> d = Decoder(b'\\xb0\\x01{\\xb1\\x05he')\n>>> d.try_next()\n123\n>>> d.try_next() # returns None because the input is incomplete\n>>> d.extend(b'llo')\n>>> d.try_next()\n'hello'\n>>> d.try_next()\n
 
Attributes:
 Name Type Description packet bytes 
buffered input waiting to be processed
 index int 
read position within packet
 Source code in preserves/binary.py 
def __init__(self, packet=b'', include_annotations=False, decode_embedded=lambda x: x):\n    super(Decoder, self).__init__()\n    self.packet = packet\n    self.index = 0\n    self.include_annotations = include_annotations\n    self.decode_embedded = decode_embedded\n
"},{"location":"binary/#preserves.binary.Decoder.extend","title":"extend(data)","text":"
Appends data to the remaining bytes in self.packet, trimming already-processed bytes from the front of self.packet and resetting self.index to zero.
 Source code in preserves/binary.py 
def extend(self, data):\n\"\"\"Appends `data` to the remaining bytes in `self.packet`, trimming already-processed\n    bytes from the front of `self.packet` and resetting `self.index` to zero.\"\"\"\n    self.packet = self.packet[self.index:] + data\n    self.index = 0\n
"},{"location":"binary/#preserves.binary.Decoder.next","title":"next()","text":"
Reads the next complete Value from the internal buffer, raising ShortPacket if too few bytes are available, or DecodeError if the input is invalid somehow.
 Source code in preserves/binary.py 
def next(self):\n\"\"\"Reads the next complete `Value` from the internal buffer, raising\n    [ShortPacket][preserves.error.ShortPacket] if too few bytes are available, or\n    [DecodeError][preserves.error.DecodeError] if the input is invalid somehow.\n\n    \"\"\"\n    tag = self.nextbyte()\n    if tag == 0x80: return self.wrap(False)\n    if tag == 0x81: return self.wrap(True)\n    if tag == 0x84: raise DecodeError('Unexpected end-of-stream marker')\n    if tag == 0x86:\n        if self.decode_embedded is None:\n            raise DecodeError('No decode_embedded function supplied')\n        return self.wrap(Embedded(self.decode_embedded(self.next())))\n    if tag == 0x87:\n        count = self.nextbyte()\n        if count == 4: return self.wrap(Float.from_bytes(self.nextbytes(4)))\n        if count == 8: return self.wrap(struct.unpack('>d', self.nextbytes(8))[0])\n        raise DecodeError('Invalid IEEE754 size')\n    if tag == 0xb0: return self.wrap(self.nextint(self.varint()))\n    if tag == 0xb1: return self.wrap(self.nextbytes(self.varint()).decode('utf-8'))\n    if tag == 0xb2: return self.wrap(self.nextbytes(self.varint()))\n    if tag == 0xb3: return self.wrap(Symbol(self.nextbytes(self.varint()).decode('utf-8')))\n    if tag == 0xb4:\n        vs = self.nextvalues()\n        if not vs: raise DecodeError('Too few elements in encoded record')\n        return self.wrap(Record(vs[0], vs[1:]))\n    if tag == 0xb5: return self.wrap(tuple(self.nextvalues()))\n    if tag == 0xb6: return self.wrap(frozenset(self.nextvalues()))\n    if tag == 0xb7: return self.wrap(ImmutableDict.from_kvs(self.nextvalues()))\n    if tag == 0xbf:\n        vs = self.nextvalues()\n        if len(vs) < 2: raise DecodeError('Too few items in encoded annotated value')\n        if self.include_annotations:\n            vs[0].annotations.extend(vs[1:])\n        return vs[0]\n    raise DecodeError('Invalid tag: ' + hex(tag))\n
"},{"location":"binary/#preserves.binary.Decoder.try_next","title":"try_next()","text":"
Like next, but returns None instead of raising ShortPacket.
 Source code in preserves/binary.py 
def try_next(self):\n\"\"\"Like [next][preserves.binary.Decoder.next], but returns `None` instead of raising\n    [ShortPacket][preserves.error.ShortPacket].\"\"\"\n    start = self.index\n    try:\n        return self.next()\n    except ShortPacket:\n        self.index = start\n        return None\n
"},{"location":"binary/#preserves.binary.Encoder","title":"Encoder(encode_embedded=lambda x: x, canonicalize=False, include_annotations=None)","text":"
         Bases: BinaryCodec
 
Implementation of an encoder for the machine-oriented binary Preserves syntax.
 
>>> e = Encoder()\n>>> e.append(123)\n>>> e.append('hello')\n>>> e.append(annotate([], Symbol('x')))\n>>> e.contents()\nb'\\xb0\\x01{\\xb1\\x05hello\\xbf\\xb5\\x84\\xb3\\x01x\\x84'\n
 
Parameters:
 Name Type Description Default encode_embedded 
function accepting an Embedded.embeddedValue and returning a Value for serialization.
 lambda x: x canonicalize bool 
if True, ensures the serialized data are in canonical form. This is slightly more work than producing potentially-non-canonical output.
 False include_annotations bool | None 
if None, includes annotations in the output only when canonicalize is False, because canonical serialization of values demands omission of annotations. If explicitly True or False, however, annotations will be included resp. excluded no matter the canonicalize setting. This can be used to get canonical ordering (canonicalize=True) and annotations (include_annotations=True).
 None 
Attributes:
 Name Type Description buffer bytearray 
accumulator for the output of the encoder
 Source code in preserves/binary.py 
def __init__(self,\n             encode_embedded=lambda x: x,\n             canonicalize=False,\n             include_annotations=None):\n    super(Encoder, self).__init__()\n    self.buffer = bytearray()\n    self._encode_embedded = encode_embedded\n    self._canonicalize = canonicalize\n    if include_annotations is None:\n        self.include_annotations = not self._canonicalize\n    else:\n        self.include_annotations = include_annotations\n
"},{"location":"binary/#preserves.binary.Encoder.append","title":"append(v)","text":"
Extend self.buffer with an encoding of v.
 Source code in preserves/binary.py 
def append(self, v):\n\"\"\"Extend `self.buffer` with an encoding of `v`.\"\"\"\n    v = preserve(v)\n    if hasattr(v, '__preserve_write_binary__'):\n        v.__preserve_write_binary__(self)\n    elif v is False:\n        self.buffer.append(0x80)\n    elif v is True:\n        self.buffer.append(0x81)\n    elif isinstance(v, float):\n        self.buffer.append(0x87)\n        self.buffer.append(8)\n        self.buffer.extend(struct.pack('>d', v))\n    elif isinstance(v, numbers.Number):\n        self.encodeint(v)\n    elif isinstance(v, bytes):\n        self.encodebytes(0xb2, v)\n    elif isinstance(v, basestring_):\n        self.encodebytes(0xb1, v.encode('utf-8'))\n    elif isinstance(v, list):\n        self.encodevalues(0xb5, v)\n    elif isinstance(v, tuple):\n        self.encodevalues(0xb5, v)\n    elif isinstance(v, set):\n        self.encodeset(v)\n    elif isinstance(v, frozenset):\n        self.encodeset(v)\n    elif isinstance(v, dict):\n        self.encodedict(v)\n    else:\n        try:\n            i = iter(v)\n        except TypeError:\n            i = None\n        if i is None:\n            self.cannot_encode(v)\n        else:\n            self.encodevalues(0xb5, i)\n
"},{"location":"binary/#preserves.binary.Encoder.contents","title":"contents()","text":"
Returns a bytes constructed from the contents of self.buffer.
 Source code in preserves/binary.py 
def contents(self):\n\"\"\"Returns a `bytes` constructed from the contents of `self.buffer`.\"\"\"\n    return bytes(self.buffer)\n
"},{"location":"binary/#preserves.binary.Encoder.reset","title":"reset()","text":"
Clears self.buffer to a fresh empty bytearray.
 Source code in preserves/binary.py 
def reset(self):\n\"\"\"Clears `self.buffer` to a fresh empty `bytearray`.\"\"\"\n    self.buffer = bytearray()\n
"},{"location":"binary/#preserves.binary.canonicalize","title":"canonicalize(v, **kwargs)","text":"
As encode, but sets canonicalize=True in the Encoder constructor.
 Source code in preserves/binary.py 
def canonicalize(v, **kwargs):\n\"\"\"As [encode][preserves.binary.encode], but sets `canonicalize=True` in the\n    [Encoder][preserves.binary.Encoder] constructor.\n\n    \"\"\"\n    return encode(v, canonicalize=True, **kwargs)\n
"},{"location":"binary/#preserves.binary.decode","title":"decode(bs, **kwargs)","text":"
Yields the first complete encoded value from bs, passing kwargs through to the Decoder constructor. Raises exceptions as per next.
 
Parameters:
 Name Type Description Default bs bytes 
encoded data to decode
 required Source code in preserves/binary.py 
def decode(bs, **kwargs):\n\"\"\"Yields the first complete encoded value from `bs`, passing `kwargs` through to the\n    [Decoder][preserves.binary.Decoder] constructor. Raises exceptions as per\n    [next][preserves.binary.Decoder.next].\n\n    Args:\n        bs (bytes): encoded data to decode\n\n    \"\"\"\n    return Decoder(packet=bs, **kwargs).next()\n
"},{"location":"binary/#preserves.binary.decode_with_annotations","title":"decode_with_annotations(bs, **kwargs)","text":"
Like decode, but supplying include_annotations=True to the Decoder constructor.
 Source code in preserves/binary.py 
def decode_with_annotations(bs, **kwargs):\n\"\"\"Like [decode][preserves.binary.decode], but supplying `include_annotations=True` to the\n    [Decoder][preserves.binary.Decoder] constructor.\"\"\"\n    return Decoder(packet=bs, include_annotations=True, **kwargs).next()\n
"},{"location":"binary/#preserves.binary.encode","title":"encode(v, **kwargs)","text":"
Encode a single Value v to a byte string. Any supplied kwargs are passed on to the underlying Encoder constructor.
 Source code in preserves/binary.py 
def encode(v, **kwargs):\n\"\"\"Encode a single `Value` `v` to a byte string. Any supplied `kwargs` are passed on to the\n    underlying [Encoder][preserves.binary.Encoder] constructor.\"\"\"\n    e = Encoder(**kwargs)\n    e.append(v)\n    return e.contents()\n
"},{"location":"compare/","title":"Comparing Values","text":"
Preserves specifies a total ordering and an equivalence between terms. The preserves.compare module implements the ordering and equivalence relations.
 
>>> cmp(\"bzz\", \"c\")\n-1\n>>> cmp(True, [])\n-1\n>>> lt(\"bzz\", \"c\")\nTrue\n>>> eq(\"bzz\", \"c\")\nFalse\n
 
Note that the ordering relates more values than Python's built-in ordering:
 
>>> [1, 2, 2] < [1, 2, \"3\"]\nTraceback (most recent call last):\n  ..\nTypeError: '<' not supported between instances of 'int' and 'str'\n\n>>> lt([1, 2, 2], [1, 2, \"3\"])\nTrue\n
"},{"location":"compare/#preserves.compare.cmp","title":"cmp(a, b)","text":"
Returns -1 if a < b, or 0 if a = b, or 1 if a > b according to the Preserves total order.
 Source code in preserves/compare.py 
def cmp(a, b):\n\"\"\"Returns `-1` if `a` < `b`, or `0` if `a` = `b`, or `1` if `a` > `b` according to the\n    [Preserves total order](https://preserves.dev/preserves.html#total-order).\"\"\"\n    return _cmp(preserve(a), preserve(b))\n
"},{"location":"compare/#preserves.compare.eq","title":"eq(a, b)","text":"
Returns True iff a = b according to the Preserves equivalence relation.
 Source code in preserves/compare.py 
def eq(a, b):\n\"\"\"Returns `True` iff `a` = `b` according to the [Preserves equivalence\n    relation](https://preserves.dev/preserves.html#equivalence).\"\"\"\n    return _eq(preserve(a), preserve(b))\n
"},{"location":"compare/#preserves.compare.le","title":"le(a, b)","text":"
Returns True iff a \u2264 b according to the Preserves total order.
 Source code in preserves/compare.py 
def le(a, b):\n\"\"\"Returns `True` iff `a` \u2264 `b` according to the [Preserves total\n    order](https://preserves.dev/preserves.html#total-order).\"\"\"\n    return cmp(a, b) <= 0\n
"},{"location":"compare/#preserves.compare.lt","title":"lt(a, b)","text":"
Returns True iff a < b according to the Preserves total order.
 Source code in preserves/compare.py 
def lt(a, b):\n\"\"\"Returns `True` iff `a` < `b` according to the [Preserves total\n    order](https://preserves.dev/preserves.html#total-order).\"\"\"\n    return cmp(a, b) < 0\n
"},{"location":"compare/#preserves.compare.sorted","title":"sorted(iterable, *, key=lambda x: x, reverse=False)","text":"
Returns a sorted list built from iterable, extracting a sort key using key, and ordering according to the Preserves total order. Directly analogous to the built-in Python sorted routine, except uses the Preserves order instead of Python's less-than relation.
 Source code in preserves/compare.py 
def sorted(iterable, *, key=lambda x: x, reverse=False):\n\"\"\"Returns a sorted list built from `iterable`, extracting a sort key using `key`, and\n    ordering according to the [Preserves total\n    order](https://preserves.dev/preserves.html#total-order). Directly analogous to the\n    [built-in Python `sorted`\n    routine](https://docs.python.org/3/library/functions.html#sorted), except uses the\n    Preserves order instead of Python's less-than relation.\n\n    \"\"\"\n    return _sorted(iterable, key=lambda x: _key(key(x)), reverse=reverse)\n
"},{"location":"compare/#preserves.compare.sorted_items","title":"sorted_items(d)","text":"
Given a dictionary d, yields a list of (key, value) tuples sorted by key.
 Source code in preserves/compare.py 
def sorted_items(d):\n\"\"\"Given a dictionary `d`, yields a list of `(key, value)` tuples sorted by `key`.\"\"\"\n    return sorted(d.items(), key=_item_key)\n
"},{"location":"error/","title":"Codec errors","text":"
The preserves.error module exports various Error subclasses.
"},{"location":"error/#preserves.error.DecodeError","title":"DecodeError","text":"
         Bases: ValueError
 
Raised whenever preserves.binary.Decoder or preserves.text.Parser detect invalid input.
"},{"location":"error/#preserves.error.EncodeError","title":"EncodeError","text":"
         Bases: ValueError
 
Raised whenever preserves.binary.Encoder or preserves.text.Formatter are unable to proceed.
"},{"location":"error/#preserves.error.ShortPacket","title":"ShortPacket","text":"
         Bases: DecodeError
 
Raised whenever preserves.binary.Decoder or preserves.text.Parser discover that they want to read beyond the end of the currently-available input buffer in order to completely read an encoded value.
"},{"location":"fold/","title":"Traversing values","text":"
The preserves.fold module exports various utilities for traversing compound Values.
"},{"location":"fold/#preserves.fold.map_embeddeds","title":"map_embeddeds(f, v)","text":"
Returns an equivalent copy of v, except where each contained Embedded value is replaced by f applied to the Embedded's embeddedValue attribute.
 
>>> map_embeddeds(lambda w: Embedded(f'w={w}'), ['a', Embedded(123), {'z': 6.0}])\n('a', #!'w=123', {'z': 6.0})\n
 Source code in preserves/fold.py 
def map_embeddeds(f, v):\n\"\"\"Returns an [equivalent][preserves.compare.eq] copy of `v`, except where each contained\n    [Embedded][preserves.values.Embedded] value is replaced by `f` applied to the Embedded's\n    `embeddedValue` attribute.\n\n    ```python\n    >>> map_embeddeds(lambda w: Embedded(f'w={w}'), ['a', Embedded(123), {'z': 6.0}])\n    ('a', #!'w=123', {'z': 6.0})\n\n    ```\n    \"\"\"\n    def walk(v):\n        if isinstance(v, Embedded):\n            return f(v.embeddedValue)\n        elif isinstance(v, (list, tuple)):\n            return tuple(walk(w) for w in v)\n        elif isinstance(v, (set, frozenset)):\n            return frozenset(walk(w) for w in v)\n        elif isinstance(v, dict):\n            return ImmutableDict.from_kvs(walk(w) for w in dict_kvs(v))\n        elif isinstance(v, Record):\n            return Record(walk(v.key), walk(v.fields))\n        else:\n            return v\n    return walk(v)\n
"},{"location":"merge/","title":"Merging values","text":"
The preserves.merge module exports various utilities for merging Values.
"},{"location":"merge/#preserves.merge.merge","title":"merge(v0, *vs, merge_embedded=None)","text":"
Repeatedly merges v0 with each element in vs using merge2, returning the final result. The merge_embedded parameter is passed on to merge2.
 Source code in preserves/merge.py 
def merge(v0, *vs, merge_embedded=None):\n\"\"\"Repeatedly merges `v0` with each element in `vs` using [merge2][preserves.merge.merge2],\n    returning the final result. The `merge_embedded` parameter is passed on to merge2.\"\"\"\n    v = v0\n    for vN in vs:\n        v = merge2(v, vN, merge_embedded=merge_embedded)\n    return v\n
"},{"location":"merge/#preserves.merge.merge2","title":"merge2(a, b, merge_embedded=None)","text":"
Merges a and b, returning the result. Raises ValueError if, during the merge, a pair of incompatible values is discovered.
 
If a and b are Embedded objects, their embeddedValues are merged using merge_embedded, and the result is again wrapped in an Embedded object.
 
>>> merge2(123, 234)\nTraceback (most recent call last):\n  ...\nValueError: Cannot merge items\n>>> merge2(123, 123)\n123\n>>> merge2('hi', 0)\nTraceback (most recent call last):\n  ...\nValueError: Cannot merge items\n>>> merge2([1, 2], [1, 2])\n[1, 2]\n>>> merge2([1, 2], [1, 3])\nTraceback (most recent call last):\n  ...\nValueError: Cannot merge items\n>>> merge2({'a': 1, 'b': 2}, {'a': 1, 'c': 3})\n{'a': 1, 'b': 2, 'c': 3}\n>>> merge2({'a': 1, 'b': 2}, {'a': 10, 'c': 3})\nTraceback (most recent call last):\n  ...\nValueError: Cannot merge items\n>>> merge2(Record('a', [1, {'x': 2}]), Record('a', [1, {'y': 3}]))\na(1, {'x': 2, 'y': 3})\n
 Source code in preserves/merge.py 
def merge2(a, b, merge_embedded=None):\n\"\"\"Merges `a` and `b`, returning the result. Raises `ValueError` if, during the merge, a\n    pair of incompatible values is discovered.\n\n    If `a` and `b` are [Embedded][preserves.values.Embedded] objects, their `embeddedValue`s\n    are merged using `merge_embedded`, and the result is again wrapped in an\n    [Embedded][preserves.values.Embedded] object.\n\n    ```python\n    >>> merge2(123, 234)\n    Traceback (most recent call last):\n      ...\n    ValueError: Cannot merge items\n    >>> merge2(123, 123)\n    123\n    >>> merge2('hi', 0)\n    Traceback (most recent call last):\n      ...\n    ValueError: Cannot merge items\n    >>> merge2([1, 2], [1, 2])\n    [1, 2]\n    >>> merge2([1, 2], [1, 3])\n    Traceback (most recent call last):\n      ...\n    ValueError: Cannot merge items\n    >>> merge2({'a': 1, 'b': 2}, {'a': 1, 'c': 3})\n    {'a': 1, 'b': 2, 'c': 3}\n    >>> merge2({'a': 1, 'b': 2}, {'a': 10, 'c': 3})\n    Traceback (most recent call last):\n      ...\n    ValueError: Cannot merge items\n    >>> merge2(Record('a', [1, {'x': 2}]), Record('a', [1, {'y': 3}]))\n    a(1, {'x': 2, 'y': 3})\n\n    ```\n\n    \"\"\"\n    if a == b:\n        return a\n    if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)):\n        return merge_seq(a, b)\n    if isinstance(a, (set, frozenset)) and isinstance(b, (set, frozenset)):\n        _die()\n    if isinstance(a, dict) and isinstance(b, dict):\n        r = {}\n        for (ak, av) in a.items():\n            bv = b.get(ak, None)\n            r[ak] = av if bv is None else merge2(av, bv, merge_embedded=merge_embedded)\n        for (bk, bv) in b.items():\n            if bk not in r:\n                r[bk] = bv\n        return r\n    if isinstance(a, Record) and isinstance(b, Record):\n        return Record(merge2(a.key, b.key, merge_embedded=merge_embedded),\n                      merge_seq(a.fields, b.fields, merge_embedded=merge_embedded))\n    if isinstance(a, Embedded) and isinstance(b, Embedded):\n        m = (merge_embedded or merge_embedded_id)(a.embeddedValue, b.embeddedValue)\n        if m is None: _die()\n        return Embedded(m)\n    _die()\n
"},{"location":"path/","title":"Preserves Path","text":"
The preserves.path module implements Preserves Path.
 
Preserves Path is roughly analogous to XPath, but for Preserves values: just as XPath selects portions of an XML document, a Preserves Path uses path expressions to select portions of a Value.
 
Use parse to compile a path expression, and then use the exec method on the result to apply it to a given input:
 
parse(PATH_EXPRESSION_STRING).exec(PRESERVES_VALUE)\n    -> SEQUENCE_OF_PRESERVES_VALUES\n
"},{"location":"path/#preserves.path--command-line-usage","title":"Command-line usage","text":"
When preserves.path is run as a __main__ module, sys.argv[1] is parsed, interpreted as a path expression, and run against human-readable values read from standard input. Each matching result is passed to stringify and printed to standard output.
"},{"location":"path/#preserves.path--examples","title":"Examples","text":""},{"location":"path/#preserves.path--setup-loading-test-data","title":"Setup: Loading test data","text":"
The following examples use testdata:
 
>>> with open('tests/samples.bin', 'rb') as f:\n...     testdata = decode_with_annotations(f.read())\n
 
Recall that samples.bin contains a binary-syntax form of the human-readable [samples.pr](https://preserves.dev/tests/samples.pr) test data file, intended to exercise most of the features of Preserves. In particular, the rootValue` in the file has a number of annotations (for documentation and other purposes).
"},{"location":"path/#preserves.path--example-1-selecting-string-valued-documentation-annotations","title":"Example 1: Selecting string-valued documentation annotations","text":"
The path expression .annotations ^ Documentation . 0 / string proceeds in five steps:
 
     
  1. .annotations selects each annotation on the root document
    2.  
  2. ^ Documentation retains only those values (each an annotation of the root) that are Records with label equal to the symbol Documentation
    3.  
  3. . 0 moves into the first child (the first field) of each such Record, which in our case is a list of other Values
    4.  
  4. / selects all immediate children of these lists
    5.  
  5. string retains only those values that are strings
    6.  
 
The result of evaluating it on testdata is as follows:
 
>>> selector = parse('.annotations ^ Documentation . 0 / string')\n>>> for result in selector.exec(testdata):\n...     print(stringify(result))\n\"Individual test cases may be any of the following record types:\"\n\"In each test, let value = strip(annotatedValue),\"\n\"                  forward = value,\"\n\"                  back = value,\"\n\"except where test-case-specific values of `forward` and/or `back`\"\n\"are provided by the executing harness, and check the following\"\n\"numbered expectations according to the table above:\"\n\"Implementations may vary in their treatment of the difference between expectations\"\n\"13/14 and 16/17, depending on how they wish to treat end-of-stream conditions.\"\n
"},{"location":"path/#preserves.path--example-2-selecting-tests-with-records-as-their-annotatedvalues","title":"Example 2: Selecting tests with Records as their annotatedValues","text":"
The path expression // [.^ [= Test + = NondeterministicTest]] [. 1 rec] proceeds in three steps:
 
     
  1.  
    // recursively decomposes the input, yielding all direct and indirect descendants of each input value
     
    2.  
  2.  
    [.^ [= Test + = NondeterministicTest]] retains only those inputs (each a descendant of the root) that yield more than zero results when executed against the expression within the brackets:
     
       
    1. .^ selects only labels of values that are Records, filtering by type and transforming in a single step
      2.  
    2. [= Test + = NondeterministicTest] again filters by a path expression:
         
      1. the infix + operator takes the union of matches of its arguments
        2.  
      2. the left-hand argument, = Test selects values (remember, record labels) equal to the symbol Test
        3.  
      3. the right-hand argument = NondeterministicTest selects values equal to NondeterministicTest
        4.  
       
      3.  
     
    The result is thus all Records anywhere inside testdata that have either Test or NondeterministicTest as their labels.
     
    3.  
  3.  
    [. 1 rec] filters these Records by another path expression:
     
       
    1. . 1 selects their second field (fields are numbered from 0)
      2.  
    2. rec retains only values that are Records
      3.  
     
    4.  
 
Evaluating the expression against testdata yields the following:
 
>>> selector = parse('// [.^ [= Test + = NondeterministicTest]] [. 1 rec]')\n>>> for result in selector.exec(testdata):\n...     print(stringify(result))\n<Test #[tLMHY2FwdHVyZbSzB2Rpc2NhcmSEhA==] <capture <discard>>>\n<Test #[tLMHb2JzZXJ2ZbSzBXNwZWFrtLMHZGlzY2FyZIS0swdjYXB0dXJltLMHZGlzY2FyZISEhIQ=] <observe <speak <discard> <capture <discard>>>>>\n<Test #[tLWzBnRpdGxlZLMGcGVyc29usAECswV0aGluZ7ABAYSwAWWxCUJsYWNrd2VsbLSzBGRhdGWwAgcdsAECsAEDhLECRHKE] <[titled person 2 thing 1] 101 \"Blackwell\" <date 1821 2 3> \"Dr\">>\n<Test #[tLMHZGlzY2FyZIQ=] <discard>>\n<Test #[tLABB7WEhA==] <7 []>>\n<Test #[tLMHZGlzY2FyZLMIc3VycHJpc2WE] <discard surprise>>\n<Test #[tLEHYVN0cmluZ7ABA7ABBIQ=] <\"aString\" 3 4>>\n<Test #[tLSzB2Rpc2NhcmSEsAEDsAEEhA==] <<discard> 3 4>>\n<Test #[v7SzAVK/swFmswJhZoSEswJhcoQ=] @ar <R @af f>>\n<Test #[tL+zAVKzAmFyhL+zAWazAmFmhIQ=] <@ar R @af f>>\n
"},{"location":"path/#preserves.path.Predicate","title":"Predicate = syntax.Predicate  module-attribute","text":"
Schema definition for representing a Preserves Path Predicate.
"},{"location":"path/#preserves.path.Selector","title":"Selector = syntax.Selector  module-attribute","text":"
Schema definition for representing a sequence of Preserves Path Steps.
"},{"location":"path/#preserves.path.syntax","title":"syntax = load_schema_file(pathlib.Path(__file__).parent / 'path.prb').path  module-attribute","text":"
This value is a Python representation of a Preserves Schema definition for the Preserves Path expression language. The language is defined in the file path.prs.
"},{"location":"path/#preserves.path.exec","title":"exec(self, v)","text":"
WARNING: This is not a function: it is a method on Selector, Predicate, and so on.
 
>>> sel = parse('/ [.length gt 1]')\n>>> sel.exec(['', 'a', 'ab', 'abc', 'abcd', 'bcd', 'cd', 'd', ''])\n('ab', 'abc', 'abcd', 'bcd', 'cd')\n
 Source code in preserves/path.py 
@extend(syntax.Function)\ndef exec(self, v):\n\"\"\"WARNING: This is not a *function*: it is a *method* on\n    [Selector][preserves.path.Selector], [Predicate][preserves.path.Predicate], and so on.\n\n    ```python\n    >>> sel = parse('/ [.length gt 1]')\n    >>> sel.exec(['', 'a', 'ab', 'abc', 'abcd', 'bcd', 'cd', 'd', ''])\n    ('ab', 'abc', 'abcd', 'bcd', 'cd')\n\n    ```\n\n    \"\"\"\n    return (len(self.selector.exec(v)),)\n
"},{"location":"path/#preserves.path.parse","title":"parse(s)","text":"
Parse s as a Preserves Path path expression, yielding a Selector object. Selectors (and Predicates etc.) have an exec method defined on them.
 
Raises ValueError if s is not a valid path expression.
 Source code in preserves/path.py 
def parse(s):\n\"\"\"Parse `s` as a Preserves Path path expression, yielding a\n    [Selector][preserves.path.Selector] object. Selectors (and Predicates etc.) have an\n    [exec][preserves.path.exec] method defined on them.\n\n    Raises `ValueError` if `s` is not a valid path expression.\n\n    \"\"\"\n    return parse_selector(Parser(s))\n
"},{"location":"schema/","title":"Preserves Schema","text":"
A Preserves schema connects Preserves Values to host-language data structures. Each definition within a schema can be processed by a compiler to produce
 
     
  •  
    a simple host-language type definition;
     
    •  
  •  
    a partial parsing function from Values to instances of the    produced type; and
     
    •  
  •  
    a total serialization function from instances of the type to    Values.
     
    •  
 
Every parsed Value retains enough information to always be able to be serialized again, and every instance of a host-language data structure contains, by construction, enough information to be successfully serialized.
"},{"location":"schema/#schema-support-in-python","title":"Schema support in Python","text":"
The preserves.schema module implements Preserves Schema for Python.
 
A Schema source file (like this one) is first compiled using preserves-schemac to produce a binary-syntax schema bundle containing schema module definitons (like this one). Python code then loads the bundle, exposing its contents as Namespaces ultimately containing SchemaObjects.
"},{"location":"schema/#preserves.schema--examples","title":"Examples","text":""},{"location":"schema/#preserves.schema--setup-loading-a-schema-bundle","title":"Setup: Loading a schema bundle","text":"
For our running example, we will use schemas associated with the Syndicated Actor Model. (The schema bundle is a copy of this file from the syndicate-protocols repository.)
 
To load a schema bundle, use load_schema_file (or, alternatively, use Compiler directly):
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n>>> type(bundle)\n<class 'preserves.schema.Namespace'>\n
 
The top-level entries in the loaded bundle are schema modules. Let's examine the stream schema module, whose source code indicates that it should contain definitions for Mode, Source, Sink, etc.:
 
>>> bundle.stream                                           # doctest: +ELLIPSIS\n{'Mode': <class 'stream.Mode'>, 'Sink': <class 'stream.Sink'>, ...}\n
"},{"location":"schema/#preserves.schema--example-1-streamstreamlistenererror-a-product-type","title":"Example 1: stream.StreamListenerError, a product type","text":"
Drilling down further, let's consider the definition of StreamListenerError, which appears in the source as
 
StreamListenerError = <stream-listener-error @spec any @message string> .\n
 
This reads, in the Preserves Schema language, as the definition of a simple product type (record, class, object) with two named fields spec and message. Parsing a value into a StreamListenerError will only succeed if it's a record, if the label matches, the second field (message) is a string, and it has exactly two fields.
 
>>> bundle.stream.StreamListenerError\n<class 'stream.StreamListenerError'>\n
 
The StreamListenerError class includes a decode method that analyzes an input value:
 
>>> bundle.stream.StreamListenerError.decode(\n...     parse('<stream-listener-error <xyz> \"an error\">'))\nStreamListenerError {'spec': #xyz(), 'message': 'an error'}\n
 
If invalid input is supplied, decode will raise SchemaDecodeFailed, which includes helpful information for diagnosing the problem (as we will see below, this is especially useful for parsers for sum types):
 
>>> bundle.stream.StreamListenerError.decode(\n...     parse('<i-am-invalid>'))\nTraceback (most recent call last):\n  ...\npreserves.schema.SchemaDecodeFailed: Could not decode i-am-invalid using <class 'stream.StreamListenerError'>\nMost likely reason: in stream.StreamListenerError: <lit stream-listener-error> didn't match i-am-invalid\nFull explanation: \n  in stream.StreamListenerError: <lit stream-listener-error> didn't match i-am-invalid\n
 
Alternatively, the try_decode method catches SchemaDecodeFailed, transforming it into None:
 
>>> bundle.stream.StreamListenerError.try_decode(\n...     parse('<stream-listener-error <xyz> \"an error\">'))\nStreamListenerError {'spec': #xyz(), 'message': 'an error'}\n>>> bundle.stream.StreamListenerError.try_decode(\n...     parse('<i-am-invalid>'))\n
 
The class can also be instantiated directly:
 
>>> err = bundle.stream.StreamListenerError(Record(Symbol('xyz'), []), 'an error')\n>>> err\nStreamListenerError {'spec': #xyz(), 'message': 'an error'}\n
 
The fields and contents of instances can be queried:
 
>>> err.spec\n#xyz()\n>>> err.message\n'an error'\n
 
And finally, instances can of course be serialized and encoded:
 
>>> print(stringify(err))\n<stream-listener-error <xyz> \"an error\">\n>>> canonicalize(err)\nb'\\xb4\\xb3\\x15stream-listener-error\\xb4\\xb3\\x03xyz\\x84\\xb1\\x08an error\\x84'\n
"},{"location":"schema/#preserves.schema--example-2-streammode-a-sum-type","title":"Example 2: stream.Mode, a sum type","text":"
Now let's consider the definition of Mode, which appears in the source as
 
Mode = =bytes / @lines LineMode / <packet @size int> / <object @description any> .\n
 
This reads, in the Preserves Schema language, as an alternation (disjoint union, variant, sum type) of four possible kinds of value: the symbol bytes; a LineMode value; a record with packet as its label and an integer as its only field; or a record with object as its label and any kind of value as its only field. In Python, this becomes:
 
>>> bundle.stream.Mode.bytes\n<class 'stream.Mode.bytes'>\n>>> bundle.stream.Mode.lines\n<class 'stream.Mode.lines'>\n>>> bundle.stream.Mode.packet\n<class 'stream.Mode.packet'>\n>>> bundle.stream.Mode.object\n<class 'stream.Mode.object'>\n
 
As before, Mode includes a decode method that analyzes an input value:
 
>>> bundle.stream.Mode.decode(parse('bytes'))\nMode.bytes()\n>>> bundle.stream.Mode.decode(parse('lf'))\nMode.lines(LineMode.lf())\n>>> bundle.stream.Mode.decode(parse('<packet 123>'))\nMode.packet {'size': 123}\n>>> bundle.stream.Mode.decode(parse('<object \"?\">'))\nMode.object {'description': '?'}\n
 
Invalid input causes SchemaDecodeFailed to be raised:
 
>>> bundle.stream.Mode.decode(parse('<i-am-not-a-valid-mode>'))\nTraceback (most recent call last):\n  ...\npreserves.schema.SchemaDecodeFailed: Could not decode <i-am-not-a-valid-mode> using <class 'stream.Mode'>\nMost likely reason: in stream.LineMode.crlf: <lit crlf> didn't match <i-am-not-a-valid-mode>\nFull explanation: \n  in stream.Mode: matching <i-am-not-a-valid-mode>\n    in stream.Mode.bytes: <lit bytes> didn't match <i-am-not-a-valid-mode>\n    in stream.Mode.lines: <ref [] LineMode> didn't match <i-am-not-a-valid-mode>\n      in stream.LineMode: matching <i-am-not-a-valid-mode>\n        in stream.LineMode.lf: <lit lf> didn't match <i-am-not-a-valid-mode>\n        in stream.LineMode.crlf: <lit crlf> didn't match <i-am-not-a-valid-mode>\n    in stream.Mode.packet: <lit packet> didn't match i-am-not-a-valid-mode\n    in stream.Mode.object: <lit object> didn't match i-am-not-a-valid-mode\n
 
The \"full explanation\" includes details on which parses were attempted, and why they failed.
 
Again, the try_decode method catches SchemaDecodeFailed, transforming it into None:
 
>>> bundle.stream.Mode.try_decode(parse('bytes'))\nMode.bytes()\n>>> bundle.stream.Mode.try_decode(parse('<i-am-not-a-valid-mode>'))\n
 
Direct instantiation is done with the variant classes, not with Mode itself:
 
>>> bundle.stream.Mode.bytes()\nMode.bytes()\n>>> bundle.stream.Mode.lines(bundle.stream.LineMode.lf())\nMode.lines(LineMode.lf())\n>>> bundle.stream.Mode.packet(123)\nMode.packet {'size': 123}\n>>> bundle.stream.Mode.object('?')\nMode.object {'description': '?'}\n
 
Fields and contents can be queried as usual:
 
>>> bundle.stream.Mode.lines(bundle.stream.LineMode.lf()).value\nLineMode.lf()\n>>> bundle.stream.Mode.packet(123).size\n123\n>>> bundle.stream.Mode.object('?').description\n'?'\n
 
And serialization and encoding are also as expected:
 
>>> print(stringify(bundle.stream.Mode.bytes()))\nbytes\n>>> print(stringify(bundle.stream.Mode.lines(bundle.stream.LineMode.lf())))\nlf\n>>> print(stringify(bundle.stream.Mode.packet(123)))\n<packet 123>\n>>> print(stringify(bundle.stream.Mode.object('?')))\n<object \"?\">\n>>> canonicalize(bundle.stream.Mode.object('?'))\nb'\\xb4\\xb3\\x06object\\xb1\\x01?\\x84'\n
 
Finally, the VARIANT attribute of instances allows code to dispatch on what kind of data it is handling at a given moment:
 
>>> bundle.stream.Mode.bytes().VARIANT\n#bytes\n>>> bundle.stream.Mode.lines(bundle.stream.LineMode.lf()).VARIANT\n#lines\n>>> bundle.stream.Mode.packet(123).VARIANT\n#packet\n>>> bundle.stream.Mode.object('?').VARIANT\n#object\n
"},{"location":"schema/#preserves.schema.meta","title":"meta = load_schema_file(__metaschema_filename).schema  module-attribute","text":"
Schema module Namespace corresponding to Preserves Schema's metaschema.
"},{"location":"schema/#preserves.schema.Compiler","title":"Compiler()","text":"
Instances of Compiler populate an initially-empty Namespace by loading and compiling schema bundle files.
 
>>> c = Compiler()\n>>> c.load('docs/syndicate-protocols-schema-bundle.bin')\n>>> type(c.root)\n<class 'preserves.schema.Namespace'>\n
 
Attributes:
 Name Type Description root Namespace 
the root namespace into which top-level schema modules are installed.
 Source code in preserves/schema.py 
def __init__(self):\n    self.root = Namespace(())\n
"},{"location":"schema/#preserves.schema.Compiler.load","title":"load(filename)","text":"
Opens the file at filename, passing the resulting file object to load_filelike.
 Source code in preserves/schema.py 
def load(self, filename):\n\"\"\"Opens the file at `filename`, passing the resulting file object to\n    [load_filelike][preserves.schema.Compiler.load_filelike].\"\"\"\n    filename = pathlib.Path(filename)\n    with open(filename, 'rb') as f:\n        self.load_filelike(f, filename.stem)\n
"},{"location":"schema/#preserves.schema.Compiler.load_filelike","title":"load_filelike(f, module_name=None)","text":"
Reads a meta.Bundle or meta.Schema from the filelike object f, compiling and installing it in self.root. If f contains a bundle, module_name is not used, since the schema modules in the bundle know their own names; if f contains a plain schema module, however, module_name is used directly if it is a string, and if it is None, a suitable module name is computed from the name attribute of f, if it is present. If name is absent in that case, ValueError is raised.
 Source code in preserves/schema.py 
def load_filelike(self, f, module_name=None):\n\"\"\"Reads a `meta.Bundle` or `meta.Schema` from the filelike object `f`, compiling and\n    installing it in `self.root`. If `f` contains a bundle, `module_name` is not used,\n    since the schema modules in the bundle know their own names; if `f` contains a plain\n    schema module, however, `module_name` is used directly if it is a string, and if it is\n    `None`, a suitable module name is computed from the `name` attribute of `f`, if it is\n    present. If `name` is absent in that case, `ValueError` is raised.\n\n    \"\"\"\n    x = Decoder(f.read()).next()\n    if x.key == SCHEMA:\n        if module_name is None:\n            if hasattr(f, 'name'):\n                module_name = pathlib.Path(f.name).stem\n            else:\n                raise ValueError('Cannot load schema module from filelike object without a module_name')\n        self.load_schema((Symbol(module_name),), x)\n    elif x.key == BUNDLE:\n        for (p, s) in x[0].items():\n            self.load_schema(p, s)\n
"},{"location":"schema/#preserves.schema.Definition","title":"Definition(*args, **kwargs)","text":"
         Bases: SchemaObject
 
Subclasses of Definition are used to represent both standalone non-alternation definitions as well as alternatives within an Enumeration.
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n\n>>> bundle.stream.StreamListenerError.FIELD_NAMES\n['spec', 'message']\n>>> bundle.stream.StreamListenerError.SAFE_FIELD_NAMES\n['spec', 'message']\n>>> bundle.stream.StreamListenerError.ENUMERATION is None\nTrue\n\n>>> bundle.stream.Mode.object.FIELD_NAMES\n['description']\n>>> bundle.stream.Mode.object.SAFE_FIELD_NAMES\n['description']\n>>> bundle.stream.Mode.object.ENUMERATION is bundle.stream.Mode\nTrue\n\n>>> bundle.stream.CreditAmount.count.FIELD_NAMES\n[]\n>>> bundle.stream.CreditAmount.count.SAFE_FIELD_NAMES\n[]\n>>> bundle.stream.CreditAmount.count.ENUMERATION is bundle.stream.CreditAmount\nTrue\n\n>>> bundle.stream.CreditAmount.decode(parse('123'))\nCreditAmount.count(123)\n>>> bundle.stream.CreditAmount.count(123)\nCreditAmount.count(123)\n>>> bundle.stream.CreditAmount.count(123).value\n123\n
 Source code in preserves/schema.py 
def __init__(self, *args, **kwargs):\n    self._fields = args\n    if self.SIMPLE:\n        if self.EMPTY:\n            if len(args) != 0:\n                raise TypeError('%s takes no arguments' % (self._constructor_name(),))\n        else:\n            if len(args) != 1:\n                raise TypeError('%s needs exactly one argument' % (self._constructor_name(),))\n            self.value = args[0]\n    else:\n        i = 0\n        for arg in args:\n            if i >= len(self.FIELD_NAMES):\n                raise TypeError('%s given too many positional arguments' % (self._constructor_name(),))\n            setattr(self, self.SAFE_FIELD_NAMES[i], arg)\n            i = i + 1\n        for (argname, arg) in kwargs.items():\n            if hasattr(self, argname):\n                raise TypeError('%s given duplicate attribute: %r' % (self._constructor_name, argname))\n            if argname not in self.SAFE_FIELD_NAMES:\n                raise TypeError('%s given unknown attribute: %r' % (self._constructor_name, argname))\n            setattr(self, argname, arg)\n            i = i + 1\n        if i != len(self.FIELD_NAMES):\n            raise TypeError('%s needs argument(s) %r' % (self._constructor_name(), self.FIELD_NAMES))\n
"},{"location":"schema/#preserves.schema.Definition.ENUMERATION","title":"ENUMERATION = None  class-attribute","text":"
None for standalone top-level definitions with a module; otherwise, an Enumeration subclass representing a top-level alternation definition.
"},{"location":"schema/#preserves.schema.Definition.FIELD_NAMES","title":"FIELD_NAMES = []  class-attribute","text":"
List of strings: names of the fields contained within this definition, if it has named fields at all; otherwise, an empty list, and the definition is a simple wrapper for another value, in which case that value is accessed via the value attribute.
"},{"location":"schema/#preserves.schema.Definition.SAFE_FIELD_NAMES","title":"SAFE_FIELD_NAMES = []  class-attribute","text":"
The list produced by mapping safeattrname over FIELD_NAMES.
"},{"location":"schema/#preserves.schema.Enumeration","title":"Enumeration()","text":"
         Bases: SchemaObject
 
Subclasses of Enumeration represent a group of variant options within a sum type.
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n\n>>> import pprint\n>>> pprint.pprint(bundle.stream.Mode.VARIANTS)\n[(#bytes, <class 'stream.Mode.bytes'>),\n (#lines, <class 'stream.Mode.lines'>),\n (#packet, <class 'stream.Mode.packet'>),\n (#object, <class 'stream.Mode.object'>)]\n\n>>> bundle.stream.Mode.VARIANTS[0][1] is bundle.stream.Mode.bytes\nTrue\n
 Source code in preserves/schema.py 
def __init__(self):\n    raise TypeError('Cannot create instance of Enumeration')\n
"},{"location":"schema/#preserves.schema.Enumeration.VARIANTS","title":"VARIANTS = None  class-attribute","text":"
List of (Symbol, SchemaObject class) tuples representing the possible options within this sum type.
"},{"location":"schema/#preserves.schema.Namespace","title":"Namespace(prefix)","text":"
A Namespace is a dictionary-like object representing a schema module that knows its location in a schema module hierarchy and whose attributes correspond to definitions and submodules within the schema module.
 
Attributes:
 Name Type Description _prefix tuple[Symbol] 
path to this module/Namespace from the root Namespace
 Source code in preserves/schema.py 
def __init__(self, prefix):\n    self._prefix = prefix\n
"},{"location":"schema/#preserves.schema.SchemaDecodeFailed","title":"SchemaDecodeFailed(cls, p, v, failures=None)","text":"
         Bases: ValueError
 
Raised when decode cannot find a way to parse a given input.
 
Attributes:
 Name Type Description cls class 
the SchemaObject subclass attempting the parse
 pattern Value 
the failing pattern, a Value conforming to schema meta.Pattern
 value Value 
the unparseable value
 failures list[SchemaDecodeFailed] 
descriptions of failed paths attempted during the match this failure describes
 Source code in preserves/schema.py 
def __init__(self, cls, p, v, failures=None):\n    super().__init__()\n    self.cls = cls\n    self.pattern = p\n    self.value = v\n    self.failures = [] if failures is None else failures\n
"},{"location":"schema/#preserves.schema.SchemaObject","title":"SchemaObject","text":"
Base class for classes representing grammatical productions in a schema: instances of SchemaObject represent schema definitions. This is an abstract class, as are its subclasses Enumeration and Definition. It is subclasses of those subclasses, automatically produced during schema loading, that are actually instantiated.
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n\n>>> bundle.stream.Mode.mro()[1:-1]\n[<class 'preserves.schema.Enumeration'>, <class 'preserves.schema.SchemaObject'>]\n\n>>> bundle.stream.Mode.packet.mro()[1:-1]\n[<class 'stream.Mode._ALL'>, <class 'preserves.schema.Definition'>, <class 'preserves.schema.SchemaObject'>]\n\n>>> bundle.stream.StreamListenerError.mro()[1:-1]\n[<class 'preserves.schema.Definition'>, <class 'preserves.schema.SchemaObject'>]\n
 
Illustrating the class attributes on SchemaObject subclasses:
 
>>> bundle.stream.Mode.ROOTNS is bundle\nTrue\n\n>>> print(stringify(bundle.stream.Mode.SCHEMA, indent=2))\n<or [\n  [\n    \"bytes\"\n    <lit bytes>\n  ]\n  [\n    \"lines\"\n    <ref [] LineMode>\n  ]\n  [\n    \"packet\"\n    <rec <lit packet> <tuple [<named size <atom SignedInteger>>]>>\n  ]\n  [\n    \"object\"\n    <rec <lit object> <tuple [<named description any>]>>\n  ]\n]>\n\n>>> bundle.stream.Mode.MODULE_PATH\n(#stream,)\n\n>>> bundle.stream.Mode.NAME\n#Mode\n\n>>> bundle.stream.Mode.VARIANT is None\nTrue\n>>> bundle.stream.Mode.packet.VARIANT\n#packet\n
"},{"location":"schema/#preserves.schema.SchemaObject.MODULE_PATH","title":"MODULE_PATH = None  class-attribute","text":"
A sequence (tuple) of Symbols naming the path from the root to the schema module containing this definition.
"},{"location":"schema/#preserves.schema.SchemaObject.NAME","title":"NAME = None  class-attribute","text":"
A Symbol naming this definition within its module.
"},{"location":"schema/#preserves.schema.SchemaObject.ROOTNS","title":"ROOTNS = None  class-attribute","text":"
A Namespace that is the top-level environment for all bundles included in the Compiler run that produced this SchemaObject.
"},{"location":"schema/#preserves.schema.SchemaObject.SCHEMA","title":"SCHEMA = None  class-attribute","text":"
A Value conforming to schema meta.Definition (and thus often to meta.Pattern etc.), interpreted by the SchemaObject machinery to drive parsing, unparsing and so forth.
"},{"location":"schema/#preserves.schema.SchemaObject.VARIANT","title":"VARIANT = None  class-attribute","text":"
None for Definitions (such as bundle.stream.StreamListenerError above) and for overall Enumerations (such as bundle.stream.Mode), or a Symbol for variant definitions contained within an enumeration (such as bundle.stream.Mode.packet).
"},{"location":"schema/#preserves.schema.SchemaObject.__preserve__","title":"__preserve__()","text":"
Called by preserves.values.preserve: unparses the information represented by this instance, using its schema definition, to produce a Preserves Value.
 Source code in preserves/schema.py 
def __preserve__(self):\n\"\"\"Called by [preserves.values.preserve][]: *unparses* the information represented by\n    this instance, using its schema definition, to produce a Preserves `Value`.\"\"\"\n    raise NotImplementedError('Subclass responsibility')\n
"},{"location":"schema/#preserves.schema.SchemaObject.decode","title":"decode(v)  classmethod","text":"
Parses v using the SCHEMA, returning a (sub)instance of SchemaObject or raising SchemaDecodeFailed.
 Source code in preserves/schema.py 
@classmethod\ndef decode(cls, v):\n\"\"\"Parses `v` using the [SCHEMA][preserves.schema.SchemaObject.SCHEMA], returning a\n    (sub)instance of [SchemaObject][preserves.schema.SchemaObject] or raising\n    [SchemaDecodeFailed][preserves.schema.SchemaDecodeFailed].\"\"\"\n    raise NotImplementedError('Subclass responsibility')\n
"},{"location":"schema/#preserves.schema.SchemaObject.try_decode","title":"try_decode(v)  classmethod","text":"
Parses v using the SCHEMA, returning a (sub)instance of SchemaObject or None if parsing failed.
 Source code in preserves/schema.py 
@classmethod\ndef try_decode(cls, v):\n\"\"\"Parses `v` using the [SCHEMA][preserves.schema.SchemaObject.SCHEMA], returning a\n    (sub)instance of [SchemaObject][preserves.schema.SchemaObject] or `None` if parsing\n    failed.\"\"\"\n    try:\n        return cls.decode(v)\n    except SchemaDecodeFailed:\n        return None\n
"},{"location":"schema/#preserves.schema.extend","title":"extend(cls)","text":"
A decorator for function definitions. Useful for adding behaviour to the classes resulting from loading a schema module:
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n\n>>> @extend(bundle.stream.LineMode.lf)\n... def what_am_i(self):\n...     return 'I am a LINEFEED linemode'\n\n>>> @extend(bundle.stream.LineMode.crlf)\n... def what_am_i(self):\n...     return 'I am a CARRIAGE-RETURN-PLUS-LINEFEED linemode'\n\n>>> bundle.stream.LineMode.lf()\nLineMode.lf()\n>>> bundle.stream.LineMode.lf().what_am_i()\n'I am a LINEFEED linemode'\n\n>>> bundle.stream.LineMode.crlf()\nLineMode.crlf()\n>>> bundle.stream.LineMode.crlf().what_am_i()\n'I am a CARRIAGE-RETURN-PLUS-LINEFEED linemode'\n
 Source code in preserves/schema.py 
def extend(cls):\n\"\"\"A decorator for function definitions. Useful for adding *behaviour* to the classes\n    resulting from loading a schema module:\n\n    ```python\n    >>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n\n    >>> @extend(bundle.stream.LineMode.lf)\n    ... def what_am_i(self):\n    ...     return 'I am a LINEFEED linemode'\n\n    >>> @extend(bundle.stream.LineMode.crlf)\n    ... def what_am_i(self):\n    ...     return 'I am a CARRIAGE-RETURN-PLUS-LINEFEED linemode'\n\n    >>> bundle.stream.LineMode.lf()\n    LineMode.lf()\n    >>> bundle.stream.LineMode.lf().what_am_i()\n    'I am a LINEFEED linemode'\n\n    >>> bundle.stream.LineMode.crlf()\n    LineMode.crlf()\n    >>> bundle.stream.LineMode.crlf().what_am_i()\n    'I am a CARRIAGE-RETURN-PLUS-LINEFEED linemode'\n\n    ```\n\n    \"\"\"\n    @wraps(cls)\n    def extender(f):\n        setattr(cls, f.__name__, f)\n        return f\n    return extender\n
"},{"location":"schema/#preserves.schema.load_schema_file","title":"load_schema_file(filename)","text":"
Simple entry point to the compiler: creates a Compiler, calls load on it, and returns its root Namespace.
 
>>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n>>> type(bundle)\n<class 'preserves.schema.Namespace'>\n
 Source code in preserves/schema.py 
def load_schema_file(filename):\n\"\"\"Simple entry point to the compiler: creates a [Compiler][preserves.schema.Compiler],\n    calls [load][preserves.schema.Compiler.load] on it, and returns its `root`\n    [Namespace][preserves.schema.Namespace].\n\n    ```python\n    >>> bundle = load_schema_file('docs/syndicate-protocols-schema-bundle.bin')\n    >>> type(bundle)\n    <class 'preserves.schema.Namespace'>\n\n    ```\n    \"\"\"\n    c = Compiler()\n    c.load(filename)\n    return c.root\n
"},{"location":"schema/#preserves.schema.safeattrname","title":"safeattrname(k)","text":"
Escapes Python keywords by prepending _; passes all other strings through.
 Source code in preserves/schema.py 
def safeattrname(k):\n\"\"\"Escapes Python keywords by prepending `_`; passes all other strings through.\"\"\"\n    return k + '_' if keyword.iskeyword(k) else k\n
"},{"location":"text/","title":"Human-readable text syntax","text":"
The preserves.text module implements the Preserves human-readable text syntax.
 
The main entry points are functions stringify, parse, and parse_with_annotations.
 
>>> stringify(Record(Symbol('hi'), [1, [2, 3]]))\n'<hi 1 [2 3]>'\n>>> parse('<hi 1 [2 3]>')\n#hi(1, (2, 3))\n
"},{"location":"text/#preserves.text.Formatter","title":"Formatter(format_embedded=lambda x: x, indent=None, with_commas=False, trailing_comma=False, include_annotations=True)","text":"
         Bases: TextCodec
 
Printer (and indenting pretty-printer) for producing human-readable syntax from Preserves Values.
 
>>> f = Formatter()\n>>> f.append({'a': 1, 'b': 2})\n>>> f.append(Record(Symbol('label'), ['field1', ['field2item1', 'field2item2']]))\n>>> print(f.contents())\n{\"a\": 1 \"b\": 2} <label \"field1\" [\"field2item1\" \"field2item2\"]>\n\n>>> f = Formatter(indent=4)\n>>> f.append({'a': 1, 'b': 2})\n>>> f.append(Record(Symbol('label'), ['field1', ['field2item1', 'field2item2']]))\n>>> print(f.contents())\n{\n    \"a\": 1\n    \"b\": 2\n}\n<label \"field1\" [\n    \"field2item1\"\n    \"field2item2\"\n]>\n
 
Parameters:
 Name Type Description Default format_embedded 
function accepting an Embedded.embeddedValue and returning a Value for serialization.
 lambda x: x indent int | None 
None disables indented pretty-printing; otherwise, an int specifies indentation per nesting-level.
 None with_commas bool 
True causes commas to separate sequence and set items and dictionary entries; False omits commas.
 False trailing_comma bool 
True causes a comma to be printed after the final item or entry in a sequence, set or dictionary; False omits this trailing comma
 False include_annotations bool 
True causes annotations to be included in the output; False causes them to be omitted.
 True 
Attributes:
 Name Type Description indent_delta int 
indentation per nesting-level
 chunks list[str] 
fragments of output
 Source code in preserves/text.py 
def __init__(self,\n             format_embedded=lambda x: x,\n             indent=None,\n             with_commas=False,\n             trailing_comma=False,\n             include_annotations=True):\n    super(Formatter, self).__init__()\n    self.indent_delta = 0 if indent is None else indent\n    self.indent_distance = 0\n    self.nesting = 0\n    self.with_commas = with_commas\n    self.trailing_comma = trailing_comma\n    self.chunks = []\n    self._format_embedded = format_embedded\n    self.include_annotations = include_annotations\n
"},{"location":"text/#preserves.text.Formatter.append","title":"append(v)","text":"
Extend self.chunks with at least one chunk, together making up the text representation of v.
 Source code in preserves/text.py 
def append(self, v):\n\"\"\"Extend `self.chunks` with at least one chunk, together making up the text\n    representation of `v`.\"\"\"\n    if self.chunks and self.nesting == 0:\n        self.write_indent_space()\n    try:\n        self.nesting += 1\n        self._append(v)\n    finally:\n        self.nesting -= 1\n
"},{"location":"text/#preserves.text.Formatter.contents","title":"contents()","text":"
Returns a str constructed from the join of the chunks in self.chunks.
 Source code in preserves/text.py 
def contents(self):\n\"\"\"Returns a `str` constructed from the join of the chunks in `self.chunks`.\"\"\"\n    return u''.join(self.chunks)\n
"},{"location":"text/#preserves.text.Formatter.is_indenting","title":"is_indenting()","text":"
Returns True iff this Formatter is in pretty-printing indenting mode.
 Source code in preserves/text.py 
def is_indenting(self):\n\"\"\"Returns `True` iff this [Formatter][preserves.text.Formatter] is in pretty-printing\n    indenting mode.\"\"\"\n    return self.indent_delta > 0\n
"},{"location":"text/#preserves.text.Parser","title":"Parser(input_buffer='', include_annotations=False, parse_embedded=lambda x: x)","text":"
         Bases: TextCodec
 
Parser for the human-readable Preserves text syntax.
 
Parameters:
 Name Type Description Default input_buffer str 
initial contents of the input buffer; may subsequently be extended by calling extend.
 '' include_annotations bool 
if True, wrap each value and subvalue in an Annotated object.
 False parse_embedded 
function accepting a Value and returning a possibly-decoded form of that value suitable for placing into an Embedded object.
 lambda x: x 
Normal usage is to supply input text, and keep calling next until a ShortPacket exception is raised:
 
>>> d = Parser('123 \"hello\" @x []')\n>>> d.next()\n123\n>>> d.next()\n'hello'\n>>> d.next()\n()\n>>> d.next()\nTraceback (most recent call last):\n  ...\npreserves.error.ShortPacket: Short input buffer\n
 
Alternatively, keep calling try_next until it yields None, which is not in the domain of Preserves Values:
 
>>> d = Parser('123 \"hello\" @x []')\n>>> d.try_next()\n123\n>>> d.try_next()\n'hello'\n>>> d.try_next()\n()\n>>> d.try_next()\n
 
For convenience, Parser implements the iterator interface, backing it with try_next, so you can simply iterate over all complete values in an input:
 
>>> d = Parser('123 \"hello\" @x []')\n>>> list(d)\n[123, 'hello', ()]\n
 
>>> for v in Parser('123 \"hello\" @x []'):\n...     print(repr(v))\n123\n'hello'\n()\n
 
Supply include_annotations=True to read annotations alongside the annotated values:
 
>>> d = Parser('123 \"hello\" @x []', include_annotations=True)\n>>> list(d)\n[123, 'hello', @#x ()]\n
 
If you are incrementally reading from, say, a socket, you can use extend to add new input as if comes available:
 
>>> d = Parser('123 \"he')\n>>> d.try_next()\n123\n>>> d.try_next() # returns None because the input is incomplete\n>>> d.extend('llo\"')\n>>> d.try_next()\n'hello'\n>>> d.try_next()\n
 
Attributes:
 Name Type Description input_buffer str 
buffered input waiting to be processed
 index int 
read position within input_buffer
 Source code in preserves/text.py 
def __init__(self, input_buffer=u'', include_annotations=False, parse_embedded=lambda x: x):\n    super(Parser, self).__init__()\n    self.input_buffer = input_buffer\n    self.index = 0\n    self.include_annotations = include_annotations\n    self.parse_embedded = parse_embedded\n
"},{"location":"text/#preserves.text.Parser.extend","title":"extend(text)","text":"
Appends text to the remaining contents of self.input_buffer, trimming already-processed text from the front of self.input_buffer and resetting self.index to zero.
 Source code in preserves/text.py 
def extend(self, text):\n\"\"\"Appends `text` to the remaining contents of `self.input_buffer`, trimming already-processed\n    text from the front of `self.input_buffer` and resetting `self.index` to zero.\"\"\"\n    self.input_buffer = self.input_buffer[self.index:] + text\n    self.index = 0\n
"},{"location":"text/#preserves.text.Parser.next","title":"next()","text":"
Reads the next complete Value from the internal buffer, raising ShortPacket if too few bytes are available, or DecodeError if the input is invalid somehow.
 Source code in preserves/text.py 
def next(self):\n\"\"\"Reads the next complete `Value` from the internal buffer, raising\n    [ShortPacket][preserves.error.ShortPacket] if too few bytes are available, or\n    [DecodeError][preserves.error.DecodeError] if the input is invalid somehow.\n\n    \"\"\"\n    self.skip_whitespace()\n    c = self.peek()\n    if c == '\"':\n        self.skip()\n        return self.wrap(self.read_string('\"'))\n    if c == '|':\n        self.skip()\n        return self.wrap(Symbol(self.read_string('|')))\n    if c in ';@':\n        annotations = self.gather_annotations()\n        v = self.next()\n        if self.include_annotations:\n            v.annotations = annotations + v.annotations\n        return v\n    if c == ':':\n        raise DecodeError('Unexpected key/value separator between items')\n    if c == '#':\n        self.skip()\n        c = self.nextchar()\n        if c == 'f': return self.wrap(False)\n        if c == 't': return self.wrap(True)\n        if c == '{': return self.wrap(frozenset(self.upto('}')))\n        if c == '\"': return self.wrap(self.read_literal_binary())\n        if c == 'x':\n            c = self.nextchar()\n            if c == '\"': return self.wrap(self.read_hex_binary())\n            if c == 'f': return self.wrap(self.read_hex_float(4))\n            if c == 'd': return self.wrap(self.read_hex_float(8))\n            raise DecodeError('Invalid #x syntax')\n        if c == '[': return self.wrap(self.read_base64_binary())\n        if c == '=':\n            old_ann = self.include_annotations\n            self.include_annotations = True\n            bs_val = self.next()\n            self.include_annotations = old_ann\n            if len(bs_val.annotations) > 0:\n                raise DecodeError('Annotations not permitted after #=')\n            bs_val = bs_val.item\n            if not isinstance(bs_val, bytes):\n                raise DecodeError('ByteString must follow #=')\n            return self.wrap(Decoder(bs_val, include_annotations = self.include_annotations).next())\n        if c == '!':\n            if self.parse_embedded is None:\n                raise DecodeError('No parse_embedded function supplied')\n            return self.wrap(Embedded(self.parse_embedded(self.next())))\n        raise DecodeError('Invalid # syntax')\n    if c == '<':\n        self.skip()\n        vs = self.upto('>')\n        if len(vs) == 0:\n            raise DecodeError('Missing record label')\n        return self.wrap(Record(vs[0], vs[1:]))\n    if c == '[':\n        self.skip()\n        return self.wrap(self.upto(']'))\n    if c == '{':\n        self.skip()\n        return self.wrap(self.read_dictionary())\n    if c in '>]}':\n        raise DecodeError('Unexpected ' + c)\n    self.skip()\n    return self.wrap(self.read_raw_symbol_or_number([c]))\n
"},{"location":"text/#preserves.text.Parser.try_next","title":"try_next()","text":"
Like next, but returns None instead of raising ShortPacket.
 Source code in preserves/text.py 
def try_next(self):\n\"\"\"Like [next][preserves.text.Parser.next], but returns `None` instead of raising\n    [ShortPacket][preserves.error.ShortPacket].\"\"\"\n    start = self.index\n    try:\n        return self.next()\n    except ShortPacket:\n        self.index = start\n        return None\n
"},{"location":"text/#preserves.text.parse","title":"parse(text, **kwargs)","text":"
Yields the first complete encoded value from text, passing kwargs through to the Parser constructor. Raises exceptions as per next.
 
Parameters:
 Name Type Description Default text str 
encoded data to decode
 required Source code in preserves/text.py 
def parse(text, **kwargs):\n\"\"\"Yields the first complete encoded value from `text`, passing `kwargs` through to the\n    [Parser][preserves.text.Parser] constructor. Raises exceptions as per\n    [next][preserves.text.Parser.next].\n\n    Args:\n        text (str): encoded data to decode\n\n    \"\"\"\n    return Parser(input_buffer=text, **kwargs).next()\n
"},{"location":"text/#preserves.text.parse_with_annotations","title":"parse_with_annotations(bs, **kwargs)","text":"
Like parse, but supplying include_annotations=True to the Parser constructor.
 Source code in preserves/text.py 
def parse_with_annotations(bs, **kwargs):\n\"\"\"Like [parse][preserves.text.parse], but supplying `include_annotations=True` to the\n    [Parser][preserves.text.Parser] constructor.\"\"\"\n    return Parser(input_buffer=bs, include_annotations=True, **kwargs).next()\n
"},{"location":"text/#preserves.text.stringify","title":"stringify(v, **kwargs)","text":"
Convert a single Value v to a string. Any supplied kwargs are passed on to the underlying Formatter constructor.
 Source code in preserves/text.py 
def stringify(v, **kwargs):\n\"\"\"Convert a single `Value` `v` to a string. Any supplied `kwargs` are passed on to the\n    underlying [Formatter][preserves.text.Formatter] constructor.\"\"\"\n    e = Formatter(**kwargs)\n    e.append(v)\n    return e.contents()\n
"},{"location":"values/","title":"Representations of Values","text":"
Python's strings, byte strings, integers, booleans, and double-precision floats stand directly for their Preserves counterparts. Wrapper objects for Float and Symbol complete the suite of atomic types.
 
Python's lists and tuples correspond to Preserves Sequences, and dicts and sets to Dictionary and Set values, respectively. Preserves Records are represented by Record objects. Finally, embedded values are represented by Embedded objects.
 
The preserves.values module implements the core representations of Preserves Values as Python values.
"},{"location":"values/#preserves.values.Annotated","title":"Annotated(item)","text":"
         Bases: object
 
A Preserves Value along with a sequence of Values annotating it. Compares equal to the underlying Value, ignoring the annotations. See the specification document for more about annotations.
 
>>> import preserves\n>>> a = preserves.parse('''\n... ; A comment\n... [1 2 3]\n... ''', include_annotations=True)\n>>> a\n@' A comment' (1, 2, 3)\n>>> a.item\n(1, 2, 3)\n>>> a.annotations\n[' A comment']\n>>> a == (1, 2, 3)\nTrue\n>>> a == preserves.parse('@xyz [1 2 3]', include_annotations=True)\nTrue\n>>> a[0]\nTraceback (most recent call last):\n  ...\nTypeError: 'Annotated' object is not subscriptable\n>>> a.item[0]\n1\n>>> type(a.item[0])\n<class 'preserves.values.Annotated'>\n>>> a.item[0].annotations\n[]\n>>> print(preserves.stringify(a))\n@\" A comment\" [1 2 3]\n>>> print(preserves.stringify(a, include_annotations=False))\n[1 2 3]\n
 
Attributes:
 Name Type Description item Value 
the underlying annotated Value
 annotations list[Value] 
the annotations attached to self.item
 Source code in preserves/values.py 
def __init__(self, item):\n    self.annotations = []\n    self.item = item\n
"},{"location":"values/#preserves.values.Annotated.peel","title":"peel()","text":"
Calls strip_annotations on self with depth=1.
 Source code in preserves/values.py 
def peel(self):\n\"\"\"Calls [strip_annotations][preserves.values.strip_annotations] on `self` with `depth=1`.\"\"\"\n    return strip_annotations(self, 1)\n
"},{"location":"values/#preserves.values.Annotated.strip","title":"strip(depth=inf)","text":"
Calls strip_annotations on self and depth.
 Source code in preserves/values.py 
def strip(self, depth=inf):\n\"\"\"Calls [strip_annotations][preserves.values.strip_annotations] on `self` and `depth`.\"\"\"\n    return strip_annotations(self, depth)\n
"},{"location":"values/#preserves.values.Embedded","title":"Embedded(embeddedValue)","text":"
Representation of a Preserves Embedded value. For more on the meaning and use of embedded values, see the specification.
 
>>> import io\n>>> e = Embedded(io.StringIO('some text'))\n>>> e                                        # doctest: +ELLIPSIS\n#!<_io.StringIO object at ...>\n>>> e.embeddedValue                          # doctest: +ELLIPSIS\n<_io.StringIO object at ...>\n
 
>>> import preserves\n>>> print(preserves.stringify(Embedded(None)))\nTraceback (most recent call last):\n  ...\nTypeError: Cannot preserves-format: None\n>>> print(preserves.stringify(Embedded(None), format_embedded=lambda x: 'abcdef'))\n#!\"abcdef\"\n
 
Attributes:
 Name Type Description embeddedValue 
any Python value; could be a platform object, could be a representation of a Preserves Value, could be None, could be anything!
 Source code in preserves/values.py 
def __init__(self, embeddedValue):\n    self.embeddedValue = embeddedValue\n
"},{"location":"values/#preserves.values.Float","title":"Float(value)","text":"
         Bases: object
 
Wrapper for treating a Python double-precision floating-point value as a single-precision (32-bit) float, from Preserves' perspective. (Python lacks native single-precision floating point support.)
 
>>> Float(3.45)\nFloat(3.45)\n>>> import preserves\n>>> preserves.stringify(Float(3.45))\n'3.45f'\n>>> preserves.stringify(3.45)\n'3.45'\n>>> preserves.parse('3.45f')\nFloat(3.45)\n>>> preserves.parse('3.45')\n3.45\n>>> preserves.encode(Float(3.45))\nb'\\x87\\x04@\\\\\\xcc\\xcd'\n>>> preserves.encode(3.45)\nb'\\x87\\x08@\\x0b\\x99\\x99\\x99\\x99\\x99\\x9a'\n
 
Attributes:
 Name Type Description value float 
the double-precision representation of intended single-precision value
 Source code in preserves/values.py 
def __init__(self, value):\n    self.value = value\n
"},{"location":"values/#preserves.values.Float.from_bytes","title":"from_bytes(bs)  staticmethod","text":"
Converts a 4-byte-long byte string to a 32-bit single-precision floating point value wrapped in a Float instance. Takes care to preserve the quiet/signalling bit-pattern of NaN values, unlike its struct.unpack('>f', ...) equivalent.
 
>>> Float.from_bytes(b'\\x7f\\x80\\x00{')\nFloat(nan)\n>>> Float.from_bytes(b'\\x7f\\x80\\x00{').to_bytes()\nb'\\x7f\\x80\\x00{'\n\n>>> struct.unpack('>f', b'\\x7f\\x80\\x00{')[0]\nnan\n>>> Float(struct.unpack('>f', b'\\x7f\\x80\\x00{')[0]).to_bytes()\nb'\\x7f\\xc0\\x00{'\n>>> struct.pack('>f', struct.unpack('>f', b'\\x7f\\x80\\x00{')[0])\nb'\\x7f\\xc0\\x00{'\n
 
(Note well the difference between 7f80007b and 7fc0007b!)
 Source code in preserves/values.py 
@staticmethod\ndef from_bytes(bs):\n\"\"\"Converts a 4-byte-long byte string to a 32-bit single-precision floating point value\n    wrapped in a [Float][preserves.values.Float] instance. Takes care to preserve the\n    quiet/signalling bit-pattern of NaN values, unlike its `struct.unpack('>f', ...)`\n    equivalent.\n\n    ```python\n    >>> Float.from_bytes(b'\\\\x7f\\\\x80\\\\x00{')\n    Float(nan)\n    >>> Float.from_bytes(b'\\\\x7f\\\\x80\\\\x00{').to_bytes()\n    b'\\\\x7f\\\\x80\\\\x00{'\n\n    >>> struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0]\n    nan\n    >>> Float(struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0]).to_bytes()\n    b'\\\\x7f\\\\xc0\\\\x00{'\n    >>> struct.pack('>f', struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0])\n    b'\\\\x7f\\\\xc0\\\\x00{'\n\n    ```\n\n    (Note well the difference between `7f80007b` and `7fc0007b`!)\n\n    \"\"\"\n    vf = struct.unpack('>I', bs)[0]\n    if (vf & 0x7f800000) == 0x7f800000:\n        # NaN or inf. Preserve quiet/signalling bit by manually expanding to double-precision.\n        sign = vf >> 31\n        payload = vf & 0x007fffff\n        dbs = struct.pack('>Q', (sign << 63) | 0x7ff0000000000000 | (payload << 29))\n        return Float(struct.unpack('>d', dbs)[0])\n    else:\n        return Float(struct.unpack('>f', bs)[0])\n
"},{"location":"values/#preserves.values.Float.to_bytes","title":"to_bytes()","text":"
Converts this 32-bit single-precision floating point value to its binary32 format, taking care to preserve the quiet/signalling bit-pattern of NaN values, unlike its struct.pack('>f', ...) equivalent.
 
>>> Float.from_bytes(b'\\x7f\\x80\\x00{')\nFloat(nan)\n>>> Float.from_bytes(b'\\x7f\\x80\\x00{').to_bytes()\nb'\\x7f\\x80\\x00{'\n\n>>> struct.unpack('>f', b'\\x7f\\x80\\x00{')[0]\nnan\n>>> Float(struct.unpack('>f', b'\\x7f\\x80\\x00{')[0]).to_bytes()\nb'\\x7f\\xc0\\x00{'\n>>> struct.pack('>f', struct.unpack('>f', b'\\x7f\\x80\\x00{')[0])\nb'\\x7f\\xc0\\x00{'\n
 
(Note well the difference between 7f80007b and 7fc0007b!)
 Source code in preserves/values.py 
def to_bytes(self):\n\"\"\"Converts this 32-bit single-precision floating point value to its binary32 format,\n    taking care to preserve the quiet/signalling bit-pattern of NaN values, unlike its\n    `struct.pack('>f', ...)` equivalent.\n\n    ```python\n    >>> Float.from_bytes(b'\\\\x7f\\\\x80\\\\x00{')\n    Float(nan)\n    >>> Float.from_bytes(b'\\\\x7f\\\\x80\\\\x00{').to_bytes()\n    b'\\\\x7f\\\\x80\\\\x00{'\n\n    >>> struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0]\n    nan\n    >>> Float(struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0]).to_bytes()\n    b'\\\\x7f\\\\xc0\\\\x00{'\n    >>> struct.pack('>f', struct.unpack('>f', b'\\\\x7f\\\\x80\\\\x00{')[0])\n    b'\\\\x7f\\\\xc0\\\\x00{'\n\n    ```\n\n    (Note well the difference between `7f80007b` and `7fc0007b`!)\n\n    \"\"\"\n\n    if math.isnan(self.value) or math.isinf(self.value):\n        dbs = struct.pack('>d', self.value)\n        vd = struct.unpack('>Q', dbs)[0]\n        sign = vd >> 63\n        payload = (vd >> 29) & 0x007fffff\n        vf = (sign << 31) | 0x7f800000 | payload\n        return struct.pack('>I', vf)\n    else:\n        return struct.pack('>f', self.value)\n
"},{"location":"values/#preserves.values.ImmutableDict","title":"ImmutableDict(*args, **kwargs)","text":"
         Bases: dict
 
A subclass of Python's built-in dict that overrides methods that could mutate the dictionary, causing them to raise TypeError('Immutable') if called.
 
Implements the __hash__ method, allowing ImmutableDict instances to be used whereever immutable data are permitted; in particular, as keys in other dictionaries.
 
>>> d = ImmutableDict([('a', 1), ('b', 2)])\n>>> d\n{'a': 1, 'b': 2}\n>>> d['c'] = 3\nTraceback (most recent call last):\n  ...\nTypeError: Immutable\n>>> del d['b']\nTraceback (most recent call last):\n  ...\nTypeError: Immutable\n
 Source code in preserves/values.py 
def __init__(self, *args, **kwargs):\n    if hasattr(self, '__hash'): raise TypeError('Immutable')\n    super(ImmutableDict, self).__init__(*args, **kwargs)\n    self.__hash = None\n
"},{"location":"values/#preserves.values.ImmutableDict.from_kvs","title":"from_kvs(kvs)  staticmethod","text":"
Constructs an ImmutableDict from a sequence of alternating keys and values; compare to the ImmutableDict constructor, which takes a sequence of key-value pairs.
 
>>> ImmutableDict.from_kvs(['a', 1, 'b', 2])\n{'a': 1, 'b': 2}\n>>> ImmutableDict.from_kvs(['a', 1, 'b', 2])['c'] = 3\nTraceback (most recent call last):\n  ...\nTypeError: Immutable\n
 Source code in preserves/values.py 
@staticmethod\ndef from_kvs(kvs):\n\"\"\"Constructs an [ImmutableDict][preserves.values.ImmutableDict] from a sequence of\n    alternating keys and values; compare to the\n    [ImmutableDict][preserves.values.ImmutableDict] constructor, which takes a sequence of\n    key-value pairs.\n\n    ```python\n    >>> ImmutableDict.from_kvs(['a', 1, 'b', 2])\n    {'a': 1, 'b': 2}\n    >>> ImmutableDict.from_kvs(['a', 1, 'b', 2])['c'] = 3\n    Traceback (most recent call last):\n      ...\n    TypeError: Immutable\n\n    ```\n\n    \"\"\"\n\n    i = iter(kvs)\n    result = ImmutableDict()\n    result_proxy = super(ImmutableDict, result)\n    try:\n        while True:\n            k = next(i)\n            try:\n                v = next(i)\n            except StopIteration:\n                raise DecodeError(\"Missing dictionary value\")\n            result_proxy.__setitem__(k, v)\n    except StopIteration:\n        pass\n    return result\n
"},{"location":"values/#preserves.values.Record","title":"Record(key, fields)","text":"
         Bases: object
 
Representation of Preserves Records, which are a pair of a label Value and a sequence of field Values.
 
>>> r = Record(Symbol('label'), ['field1', ['field2item1', 'field2item2']])\n>>> r\n#label('field1', ['field2item1', 'field2item2'])\n>>> r.key\n#label\n>>> r.fields\n('field1', ['field2item1', 'field2item2'])\n>>> import preserves\n>>> preserves.stringify(r)\n'<label \"field1\" [\"field2item1\" \"field2item2\"]>'\n>>> r == preserves.parse('<label \"field1\" [\"field2item1\" \"field2item2\"]>')\nTrue\n
 
Parameters:
 Name Type Description Default key Value 
the Record's label
 required fields iterable[Value] 
the fields of the Record
 required 
Attributes:
 Name Type Description key Value 
the Record's label
 fields tuple[Value] 
the fields of the Record
 Source code in preserves/values.py 
def __init__(self, key, fields):\n    self.key = key\n    self.fields = tuple(fields)\n    self.__hash = None\n
"},{"location":"values/#preserves.values.Record.makeBasicConstructor","title":"makeBasicConstructor(label, fieldNames)  staticmethod","text":"
Constructs and returns a \"constructor\" for Records having a certain label and number of fields.
 Deprecated 
Use preserves.schema definitions instead.
 
The \"constructor\" is a callable function that accepts len(fields) arguments and returns a Record with label as its label and the arguments to the constructor as field values.
 
In addition, the \"constructor\" has a constructorInfo attribute holding a RecordConstructorInfo object, an isClassOf attribute holding a unary function that returns True iff its argument is a Record with label label and arity len(fieldNames), and an ensureClassOf attribute that raises an Exception if isClassOf returns false on its argument and returns the argument otherwise.
 
Finally, for each field name f in fieldNames, the \"constructor\" object has an attribute _f that is a unary function that retrieves the f field from the passed in argument.
 
>>> c = Record.makeBasicConstructor(Symbol('date'), 'year month day')\n>>> c(1969, 7, 16)\n#date(1969, 7, 16)\n>>> c.constructorInfo\n#date/3\n>>> c.isClassOf(c(1969, 7, 16))\nTrue\n>>> c.isClassOf(Record(Symbol('date'), [1969, 7, 16]))\nTrue\n>>> c.isClassOf(Record(Symbol('date'), [1969]))\nFalse\n>>> c.ensureClassOf(c(1969, 7, 16))\n#date(1969, 7, 16)\n>>> c.ensureClassOf(Record(Symbol('date'), [1969]))\nTraceback (most recent call last):\n  ...\nTypeError: Record: expected #date/3, got #date(1969)\n>>> c._year(c(1969, 7, 16))\n1969\n>>> c._month(c(1969, 7, 16))\n7\n>>> c._day(c(1969, 7, 16))\n16\n
 
Parameters:
 Name Type Description Default label Value 
Label to use for constructed/matched Records
 required fieldNames tuple[str] | list[str] | str 
Names of the Record's fields
 required Source code in preserves/values.py 
@staticmethod\ndef makeBasicConstructor(label, fieldNames):\n\"\"\"Constructs and returns a \"constructor\" for `Record`s having a certain `label` and\n    number of fields.\n\n    Deprecated:\n       Use [preserves.schema][] definitions instead.\n\n    The \"constructor\" is a callable function that accepts `len(fields)` arguments and\n    returns a [Record][preserves.values.Record] with `label` as its label and the arguments\n    to the constructor as field values.\n\n    In addition, the \"constructor\" has a `constructorInfo` attribute holding a\n    [RecordConstructorInfo][preserves.values.RecordConstructorInfo] object, an `isClassOf`\n    attribute holding a unary function that returns `True` iff its argument is a\n    [Record][preserves.values.Record] with label `label` and arity `len(fieldNames)`, and\n    an `ensureClassOf` attribute that raises an `Exception` if `isClassOf` returns false on\n    its argument and returns the argument otherwise.\n\n    Finally, for each field name `f` in `fieldNames`, the \"constructor\" object has an\n    attribute `_f` that is a unary function that retrieves the `f` field from the passed in\n    argument.\n\n    ```python\n    >>> c = Record.makeBasicConstructor(Symbol('date'), 'year month day')\n    >>> c(1969, 7, 16)\n    #date(1969, 7, 16)\n    >>> c.constructorInfo\n    #date/3\n    >>> c.isClassOf(c(1969, 7, 16))\n    True\n    >>> c.isClassOf(Record(Symbol('date'), [1969, 7, 16]))\n    True\n    >>> c.isClassOf(Record(Symbol('date'), [1969]))\n    False\n    >>> c.ensureClassOf(c(1969, 7, 16))\n    #date(1969, 7, 16)\n    >>> c.ensureClassOf(Record(Symbol('date'), [1969]))\n    Traceback (most recent call last):\n      ...\n    TypeError: Record: expected #date/3, got #date(1969)\n    >>> c._year(c(1969, 7, 16))\n    1969\n    >>> c._month(c(1969, 7, 16))\n    7\n    >>> c._day(c(1969, 7, 16))\n    16\n\n    ```\n\n    Args:\n        label (Value): Label to use for constructed/matched `Record`s\n        fieldNames (tuple[str] | list[str] | str): Names of the `Record`'s fields\n\n    \"\"\"\n    if type(fieldNames) == str:\n        fieldNames = fieldNames.split()\n    arity = len(fieldNames)\n    def ctor(*fields):\n        if len(fields) != arity:\n            raise Exception(\"Record: cannot instantiate %r expecting %d fields with %d fields\"%(\n                label,\n                arity,\n                len(fields)))\n        return Record(label, fields)\n    ctor.constructorInfo = RecordConstructorInfo(label, arity)\n    ctor.isClassOf = lambda v: \\\n                     isinstance(v, Record) and v.key == label and len(v.fields) == arity\n    def ensureClassOf(v):\n        if not ctor.isClassOf(v):\n            raise TypeError(\"Record: expected %r/%d, got %r\" % (label, arity, v))\n        return v\n    ctor.ensureClassOf = ensureClassOf\n    for fieldIndex in range(len(fieldNames)):\n        fieldName = fieldNames[fieldIndex]\n        # Stupid python scoping bites again\n        def getter(fieldIndex):\n            return lambda v: ensureClassOf(v)[fieldIndex]\n        setattr(ctor, '_' + fieldName, getter(fieldIndex))\n    return ctor\n
"},{"location":"values/#preserves.values.Record.makeConstructor","title":"makeConstructor(labelSymbolText, fieldNames)  staticmethod","text":"
Equivalent to Record.makeBasicConstructor(Symbol(labelSymbolText), fieldNames).
 Deprecated 
Use preserves.schema definitions instead.
 Source code in preserves/values.py 
@staticmethod\ndef makeConstructor(labelSymbolText, fieldNames):\n\"\"\"\n    Equivalent to `Record.makeBasicConstructor(Symbol(labelSymbolText), fieldNames)`.\n\n    Deprecated:\n       Use [preserves.schema][] definitions instead.\n    \"\"\"\n    return Record.makeBasicConstructor(Symbol(labelSymbolText), fieldNames)\n
"},{"location":"values/#preserves.values.RecordConstructorInfo","title":"RecordConstructorInfo(key, arity)","text":"
         Bases: object
 
Describes the shape of a Record constructor, namely its label and its arity (field count).
 
>>> RecordConstructorInfo(Symbol('label'), 3)\n#label/3\n
 
Attributes:
 Name Type Description key Value 
the label of matching Records
 arity int 
the number of fields in matching Records
 Source code in preserves/values.py 
def __init__(self, key, arity):\n    self.key = key\n    self.arity = arity\n
"},{"location":"values/#preserves.values.Symbol","title":"Symbol(name)","text":"
         Bases: object
 
Representation of Preserves Symbols.
 
>>> Symbol('xyz')\n#xyz\n>>> Symbol('xyz').name\n'xyz'\n>>> import preserves\n>>> preserves.stringify(Symbol('xyz'))\n'xyz'\n>>> preserves.stringify(Symbol('hello world'))\n'|hello world|'\n>>> preserves.parse('xyz')\n#xyz\n>>> preserves.parse('|hello world|')\n#hello world\n
 
Attributes:
 Name Type Description name str | Symbol 
The symbol's text label. If an existing Symbol is passed in, the existing Symbol's name is used as the name for the new Symbol.
 Source code in preserves/values.py 
def __init__(self, name):\n    self.name = name.name if isinstance(name, Symbol) else name\n
"},{"location":"values/#preserves.values.annotate","title":"annotate(v, *anns)","text":"
Wraps v in an Annotated object, if it isn't already wrapped, and appends each of the anns to the Annotated's annotations sequence. NOTE: Does not recursively ensure that any parts of the argument v are themselves wrapped in Annotated objects!
 
>>> import preserves\n>>> print(preserves.stringify(annotate(123, \"A comment\", \"Another comment\")))\n@\"A comment\" @\"Another comment\" 123\n
 Source code in preserves/values.py 
def annotate(v, *anns):\n\"\"\"Wraps `v` in an [Annotated][preserves.values.Annotated] object, if it isn't already\n    wrapped, and appends each of the `anns` to the [Annotated][preserves.values.Annotated]'s\n    `annotations` sequence. NOTE: Does not recursively ensure that any parts of the argument\n    `v` are themselves wrapped in [Annotated][preserves.values.Annotated] objects!\n\n    ```python\n    >>> import preserves\n    >>> print(preserves.stringify(annotate(123, \"A comment\", \"Another comment\")))\n    @\"A comment\" @\"Another comment\" 123\n\n    ```\n    \"\"\"\n    if not is_annotated(v):\n        v = Annotated(v)\n    for a in anns:\n        v.annotations.append(a)\n    return v\n
"},{"location":"values/#preserves.values.cmp_floats","title":"cmp_floats(a, b)","text":"
Implements the totalOrder predicate defined in section 5.10 of IEEE Std 754-2008.
 Source code in preserves/values.py 
def cmp_floats(a, b):\n\"\"\"Implements the `totalOrder` predicate defined in section 5.10 of [IEEE Std\n    754-2008](https://dx.doi.org/10.1109/IEEESTD.2008.4610935).\n\n    \"\"\"\n    a = float_to_int(a)\n    b = float_to_int(b)\n    if a & 0x8000000000000000: a = a ^ 0x7fffffffffffffff\n    if b & 0x8000000000000000: b = b ^ 0x7fffffffffffffff\n    return a - b\n
"},{"location":"values/#preserves.values.dict_kvs","title":"dict_kvs(d)","text":"
Generator function yielding a sequence of alternating keys and values from d. In some sense the inverse of ImmutableDict.from_kvs.
 
>>> list(dict_kvs({'a': 1, 'b': 2}))\n['a', 1, 'b', 2]\n
 Source code in preserves/values.py 
def dict_kvs(d):\n\"\"\"Generator function yielding a sequence of alternating keys and values from `d`. In some\n    sense the inverse of [ImmutableDict.from_kvs][preserves.values.ImmutableDict.from_kvs].\n\n    ```python\n    >>> list(dict_kvs({'a': 1, 'b': 2}))\n    ['a', 1, 'b', 2]\n\n    ```\n    \"\"\"\n    for k in d:\n        yield k\n        yield d[k]\n
"},{"location":"values/#preserves.values.is_annotated","title":"is_annotated(v)","text":"
True iff v is an instance of Annotated.
 Source code in preserves/values.py 
def is_annotated(v):\n\"\"\"`True` iff `v` is an instance of [Annotated][preserves.values.Annotated].\"\"\"\n    return isinstance(v, Annotated)\n
"},{"location":"values/#preserves.values.preserve","title":"preserve(v)","text":"
Converts v to a representation of a Preserves Value by (repeatedly) setting
 
v = v.__preserve__()\n
 
while v has a __preserve__ method. Parsed Schema values are able to render themselves to their serialized representations this way.
 Source code in preserves/values.py 
def preserve(v):\n\"\"\"Converts `v` to a representation of a Preserves `Value` by (repeatedly) setting\n\n    ```python\n    v = v.__preserve__()\n    ```\n\n    while `v` has a `__preserve__` method. Parsed [Schema][preserves.schema]\n    values are able to render themselves to their serialized representations this way.\n\n    \"\"\"\n    while hasattr(v, '__preserve__'):\n        v = v.__preserve__()\n    return v\n
"},{"location":"values/#preserves.values.strip_annotations","title":"strip_annotations(v, depth=inf)","text":"
Exposes depth layers of raw structure of potentially-Annotated Values. If depth==0 or v is not Annotated, just returns v. Otherwise, descends recursively into the structure of v.item.
 
>>> import preserves\n>>> a = preserves.parse('@\"A comment\" [@a 1 @b 2 @c 3]', include_annotations=True)\n>>> is_annotated(a)\nTrue\n>>> print(preserves.stringify(a))\n@\"A comment\" [@a 1 @b 2 @c 3]\n>>> print(preserves.stringify(strip_annotations(a)))\n[1 2 3]\n>>> print(preserves.stringify(strip_annotations(a, depth=1)))\n[@a 1 @b 2 @c 3]\n
 Source code in preserves/values.py 
def strip_annotations(v, depth=inf):\n\"\"\"Exposes `depth` layers of raw structure of\n    potentially-[Annotated][preserves.values.Annotated] `Value`s. If `depth==0` or `v` is not\n    [Annotated][preserves.values.Annotated], just returns `v`. Otherwise, descends recursively\n    into the structure of `v.item`.\n\n    ```python\n    >>> import preserves\n    >>> a = preserves.parse('@\"A comment\" [@a 1 @b 2 @c 3]', include_annotations=True)\n    >>> is_annotated(a)\n    True\n    >>> print(preserves.stringify(a))\n    @\"A comment\" [@a 1 @b 2 @c 3]\n    >>> print(preserves.stringify(strip_annotations(a)))\n    [1 2 3]\n    >>> print(preserves.stringify(strip_annotations(a, depth=1)))\n    [@a 1 @b 2 @c 3]\n\n    ```\n    \"\"\"\n\n    if depth == 0: return v\n    if not is_annotated(v): return v\n\n    next_depth = depth - 1\n    def walk(v):\n        return strip_annotations(v, next_depth)\n\n    v = v.item\n    if isinstance(v, Record):\n        return Record(strip_annotations(v.key, depth), tuple(walk(f) for f in v.fields))\n    elif isinstance(v, list):\n        return tuple(walk(f) for f in v)\n    elif isinstance(v, tuple):\n        return tuple(walk(f) for f in v)\n    elif isinstance(v, set):\n        return frozenset(walk(f) for f in v)\n    elif isinstance(v, frozenset):\n        return frozenset(walk(f) for f in v)\n    elif isinstance(v, dict):\n        return ImmutableDict.from_kvs(walk(f) for f in dict_kvs(v))\n    elif is_annotated(v):\n        raise ValueError('Improper annotation structure')\n    else:\n        return v\n
"}]}
\ No newline at end of file
diff --git a/python/0.18.1/sitemap.xml b/python/0.18.1/sitemap.xml
index 6913358..a796144 100644
--- a/python/0.18.1/sitemap.xml
+++ b/python/0.18.1/sitemap.xml
@@ -2,57 +2,57 @@
 
     
          https://preserves.dev/python/latest/
-         2023-03-17
+         2023-10-13
          daily
     
     
          https://preserves.dev/python/latest/api/
-         2023-03-17
+         2023-10-13
          daily
     
     
          https://preserves.dev/python/latest/binary/
-         2023-03-17
+         2023-10-13
          daily
     
     
          https://preserves.dev/python/latest/compare/
-         2023-03-17
+         2023-10-13
          daily
     
     
          https://preserves.dev/python/latest/error/
-         2023-03-17
+         2023-10-13
          daily
     
     
          https://preserves.dev/python/latest/fold/
-         2023-03-17
+         2023-10-13
          daily
     
     
          https://preserves.dev/python/latest/merge/
-         2023-03-17
+         2023-10-13
          daily
     
     
          https://preserves.dev/python/latest/path/
-         2023-03-17
+         2023-10-13
          daily
     
     
          https://preserves.dev/python/latest/schema/
-         2023-03-17
+         2023-10-13
          daily
     
     
          https://preserves.dev/python/latest/text/
-         2023-03-17
+         2023-10-13
          daily
     
     
          https://preserves.dev/python/latest/values/
-         2023-03-17
+         2023-10-13
          daily
     
 
\ No newline at end of file
diff --git a/python/0.18.1/values/index.html b/python/0.18.1/values/index.html
index 9954ce6..1207b3f 100644
--- a/python/0.18.1/values/index.html
+++ b/python/0.18.1/values/index.html
@@ -904,9 +904,9 @@ about annotations.

 
           

             Source code in preserves/values.py
-            
575
-576
-577
def __init__(self, item):
+            
576
+577
+578
def __init__(self, item):
     self.annotations = []
     self.item = item
 

@@ -940,9 +940,9 @@ about annotations.

 
       

         Source code in preserves/values.py
-        
598
-599
-600
def peel(self):
+        
602
+603
+604
def peel(self):
     """Calls [strip_annotations][preserves.values.strip_annotations] on `self` with `depth=1`."""
     return strip_annotations(self, 1)
 

@@ -967,9 +967,9 @@ about annotations.

 
       

         Source code in preserves/values.py
-        
594
-595
-596
def strip(self, depth=inf):
+        
598
+599
+600
def strip(self, depth=inf):
     """Calls [strip_annotations][preserves.values.strip_annotations] on `self` and `depth`."""
     return strip_annotations(self, depth)
 

@@ -1040,8 +1040,8 @@ Preserves Value, could be None, could be anything!

 
           

             Source code in preserves/values.py
-            
720
-721
def __init__(self, embeddedValue):
+            
724
+725
def __init__(self, embeddedValue):
     self.embeddedValue = embeddedValue
 

           
@@ -1096,9 +1096,9 @@ single-precision floating point support.)

 >>> preserves.parse('3.45')
 3.45
 >>> preserves.encode(Float(3.45))
-b'\x82@\\\xcc\xcd'
+b'\x87\x04@\\\xcc\xcd'
 >>> preserves.encode(3.45)
-b'\x83@\x0b\x99\x99\x99\x99\x99\x9a'
+b'\x87\x08@\x0b\x99\x99\x99\x99\x99\x9a'
 

 
   
Attributes:

@@ -1178,8 +1178,7 @@ equivalent.

 
       

         Source code in preserves/values.py
-        
135
-136
+        
136
 137
 138
 139
@@ -1211,7 +1210,8 @@ equivalent.

 165
 166
 167
-168
@staticmethod
+168
+169
@staticmethod
 def from_bytes(bs):
     """Converts a 4-byte-long byte string to a 32-bit single-precision floating point value
     wrapped in a [Float][preserves.values.Float] instance. Takes care to preserve the
@@ -1397,10 +1397,10 @@ other dictionaries.

 
           

             Source code in preserves/values.py
-            
461
-462
+            
462
 463
-464
def __init__(self, *args, **kwargs):
+464
+465
def __init__(self, *args, **kwargs):
     if hasattr(self, '__hash'): raise TypeError('Immutable')
     super(ImmutableDict, self).__init__(*args, **kwargs)
     self.__hash = None
@@ -1449,8 +1449,7 @@ key-value pairs.

 
       

         Source code in preserves/values.py
-        
482
-483
+        
483
 484
 485
 486
@@ -1481,7 +1480,8 @@ key-value pairs.

 511
 512
 513
-514
@staticmethod
+514
+515
@staticmethod
 def from_kvs(kvs):
     """Constructs an [ImmutableDict][preserves.values.ImmutableDict] from a sequence of
     alternating keys and values; compare to the
@@ -1622,10 +1622,10 @@ key-value pairs.

 
           

             Source code in preserves/values.py
-            
273
-274
+            
274
 275
-276
def __init__(self, key, fields):
+276
+277
def __init__(self, key, fields):
     self.key = key
     self.fields = tuple(fields)
     self.__hash = None
@@ -1739,8 +1739,7 @@ argument.

 
       

         Source code in preserves/values.py
-        
321
-322
+        
322
 323
 324
 325
@@ -1818,7 +1817,8 @@ argument.

 397
 398
 399
-400
@staticmethod
+400
+401
@staticmethod
 def makeBasicConstructor(label, fieldNames):
     """Constructs and returns a "constructor" for `Record`s having a certain `label` and
     number of fields.
@@ -1928,15 +1928,15 @@ argument.

 
       

         Source code in preserves/values.py
-        
311
-312
+        
312
 313
 314
 315
 316
 317
 318
-319
@staticmethod
+319
+320
@staticmethod
 def makeConstructor(labelSymbolText, fieldNames):
     """
     Equivalent to `Record.makeBasicConstructor(Symbol(labelSymbolText), fieldNames)`.
@@ -2010,9 +2010,9 @@ count).

 
           

             Source code in preserves/values.py
-            
416
-417
-418
def __init__(self, key, arity):
+            
417
+418
+419
def __init__(self, key, arity):
     self.key = key
     self.arity = arity
 

@@ -2093,8 +2093,8 @@ in, the existing Symbol's name is used as the name for
 
           

             Source code in preserves/values.py
-            
202
-203
def __init__(self, name):
+            
203
+204
def __init__(self, name):
     self.name = name.name if isinstance(name, Symbol) else name
 

           
@@ -2143,11 +2143,7 @@ wrapped, and appends each of the anns to the 
         Source code in preserves/values.py
-        
664
-665
-666
-667
-668
+        
668
 669
 670
 671
@@ -2160,7 +2156,11 @@ wrapped, and appends each of the anns to the 678
 679
 680
-681
def annotate(v, *anns):
+681
+682
+683
+684
+685
def annotate(v, *anns):
     """Wraps `v` in an [Annotated][preserves.values.Annotated] object, if it isn't already
     wrapped, and appends each of the `anns` to the [Annotated][preserves.values.Annotated]'s
     `annotations` sequence. NOTE: Does not recursively ensure that any parts of the argument
@@ -2246,8 +2246,7 @@ sense the inverse of 
         Source code in preserves/values.py
-        
516
-517
+        
517
 518
 519
 520
@@ -2258,7 +2257,8 @@ sense the inverse of 525
 526
 527
-528
def dict_kvs(d):
+528
+529
def dict_kvs(d):
     """Generator function yielding a sequence of alternating keys and values from `d`. In some
     sense the inverse of [ImmutableDict.from_kvs][preserves.values.ImmutableDict.from_kvs].
 
@@ -2293,9 +2293,9 @@ sense the inverse of 
         Source code in preserves/values.py
-        
614
-615
-616
def is_annotated(v):
+        
618
+619
+620
def is_annotated(v):
     """`True` iff `v` is an instance of [Annotated][preserves.values.Annotated]."""
     return isinstance(v, Annotated)
 

@@ -2387,11 +2387,7 @@ into the structure of v.item.

 
       

         Source code in preserves/values.py
-        
618
-619
-620
-621
-622
+        
622
 623
 624
 625
@@ -2431,7 +2427,11 @@ into the structure of v.item.

 659
 660
 661
-662
def strip_annotations(v, depth=inf):
+662
+663
+664
+665
+666
def strip_annotations(v, depth=inf):
     """Exposes `depth` layers of raw structure of
     potentially-[Annotated][preserves.values.Annotated] `Value`s. If `depth==0` or `v` is not
     [Annotated][preserves.values.Annotated], just returns `v`. Otherwise, descends recursively
diff --git a/schema/schema.bin b/schema/schema.bin
index 408302c..a98ba99 100644
--- a/schema/schema.bin
+++ b/schema/schema.bin
@@ -1,6 +1,6 @@
-´³schema·³version‘³definitions·³Ref´³rec´³lit³ref„´³tupleµ´³named³module´³refµ„³
+´³schema·³version°³definitions·³Ref´³rec´³lit³ref„´³tupleµ´³named³module´³refµ„³
 ModulePath„„´³named³name´³atom³Symbol„„„„„³Bundle´³rec´³lit³bundle„´³tupleµ´³named³modules´³refµ„³Modules„„„„„³Schema´³rec´³lit³schema„´³tupleµ´³dict·³version´³named³version´³refµ„³Version„„³definitions´³named³definitions´³refµ„³Definitions„„³embeddedType´³named³embeddedType´³refµ„³EmbeddedTypeName„„„„„„„³Binding´³rec´³lit³named„´³tupleµ´³named³name´³atom³Symbol„„´³named³pattern´³refµ„³
SimplePattern„„„„„³Modules´³dictof´³refµ„³
-ModulePath„´³refµ„³Schema„„³Pattern´³orµµ±
SimplePattern´³refµ„³
SimplePattern„„µ±CompoundPattern´³refµ„³CompoundPattern„„„„³Version´³lit‘„³AtomKind´³orµµ±Boolean´³lit³Boolean„„µ±Float´³lit³Float„„µ±Double´³lit³Double„„µ±
SignedInteger´³lit³
SignedInteger„„µ±String´³lit³String„„µ±
+ModulePath„´³refµ„³Schema„„³Pattern´³orµµ±
SimplePattern´³refµ„³
SimplePattern„„µ±CompoundPattern´³refµ„³CompoundPattern„„„„³Version´³lit°„³AtomKind´³orµµ±Boolean´³lit³Boolean„„µ±Float´³lit³Float„„µ±Double´³lit³Double„„µ±
SignedInteger´³lit³
SignedInteger„„µ±String´³lit³String„„µ±
 ByteString´³lit³
 ByteString„„µ±Symbol´³lit³Symbol„„„„³
 Definition´³orµµ±or´³rec´³lit³or„´³tupleµ´³tuplePrefixµ´³named³pattern0´³refµ„³NamedAlternative„„´³named³pattern1´³refµ„³NamedAlternative„„„´³named³patternN´³seqof´³refµ„³NamedAlternative„„„„„„„„µ±and´³rec´³lit³and„´³tupleµ´³tuplePrefixµ´³named³pattern0´³refµ„³NamedPattern„„´³named³pattern1´³refµ„³NamedPattern„„„´³named³patternN´³seqof´³refµ„³NamedPattern„„„„„„„„µ±Pattern´³refµ„³Pattern„„„„³
diff --git a/tests/samples.bin b/tests/samples.bin
index 0d2ad15eb64f956840bea334ef53692505004815..d47492d1533490e7250e55cedadf3076b92aa744 100644
GIT binary patch
literal 10541
zcmd^FTW=f371olXsEcJ=@g)U{BvZRF>`Jt>q^@q7M6zW>MIr~jrD@Y5OYTrynA~M|
zm%7*}ECY)K{Q*VM0_hJZP=uqEqL%{BbACi0`p}0y<)IHPkXZd@W_NZ;QFVek0g4jj
zaCgpSX3m`X&fFgFhC{ch=U+8E>g|X9mhDgNhW$KiC)4h0_XQ$PkMpu^O~V(t9!+t6
z5_}TKJaC>~b!?OR)NNR{<@r`cXdC%Nh-TVaHb^~6sp8|^aQJP*_2{jd@%(WDAWv?Z7(t1j#k+ZhpU$B`32i(P&Tw&x9%Hd>eqt(
zFz>ZonY&asD)%;My&eS5Oi@(j?#>#0K#VJ-JV7c~hSqm%XH)&w;TWQvneJX0*-uP}Qeu~iydwv}ObnH~)XZS4Tn
zuDv&+S;)wJFR52t&JkPj@e;Al#;h;wv0M$V<%BuEMqRp-+i={v
zxd#wp6MVx(#=RyDR4W{q+s%wE2HIn0{08XU@qMCO$fNQlsW7O*pMKPDbzE&+nmbNn5xPF
z^||T0n=*~5-EfAUsGie{rRkYj^u(#_bc4FYuNgL(C$bx2$!mhv=UV{bd2jGiKVn5Ik7L|IzEa^v2ru^MaUgEPgZjd3<@EX4*y)r-na
zG4FVYFT_Q@a?bvCrf-}7B&9f`1`ekj@Lx<(@CAz2$A6%3ywp<@12OD(vU}Ux0XrF^
z3!fM8(H`?Fm&tKR6}SrQmAaI}aN03WI~LQXjkwKE0p`4b7F~^s)cA@>ss@KiNpzOZ
zF?9aid#%|@X&R34vcv{&%l3^;;@I^q;CfMrJU4i3cywrU>wb@{Sg;s3^zlv%i%eg!F(b5%I@zdM0H`Kram4b7fiSW`%?bEk
zrQR~dJkMz~4DWk^g3`R{w90j=>@ob|U|;k}o=~`CQ?>!yYoANL$E5Y-|X?P%*mWx4pNH&8^f$*_m)fs&1%dLu?8n_xjA
zt0^rUItsrO(2*B#o=nub@LZCUDI3V!=zf`^r-ZZ^>P
zbs!O{ldpTc9al(BBpQz_87ATW4&PR_;E_XVfA}e{1y4EtrQ51=ze@25ocvVcWCtJj
zAjNke^zpRYc6vt;IDjU!O6rSk4168$1-y9>3Y~;UZS4_SBnb+V0oB|1Dnc?%XlD){
z@?S(M&Zrp>Ao(D}e~OgG@dFJctT{zV#hEma9{|V?0OS!##S18W0`z|Z^v}Ck^0}sh
zg|oY1zG5`}mP_$uT3*F)%^>KVtHc#tpkjHa>``|e;yJHLje99IM`+>NzrK(vT!GN?
z?p`?HTYjCI;XsqRo@2MQOo(YXoQubzwuTjco*wJ1EP>J)pWYzuVUGSgu5WN|Wl?Gl`~f5KG#oYk8F0Fw&XIRKlw3$97rXxaGUT
zB5=n3;82DVQcinnH_;9wO0mOPO0auzk$L6SfSryf4ASFB9n11ML_AR$6}~d8k8zik
z*{W~z(IhsW%f<}KzGvBKgo3}dckxXH;@-JRzq0xO>;L?t;14q8WK8;go4YhH6
z6+v}%F3gUPkBdYRRhL4lfUhDb9D-D#P!LHF4$4VG5>ya_?JB)Wi)M;HD{
zo)ljy%&4FSu#G~a0PqNKqYCIryn#Rza}fpOrzA4O0zB|Y$X5cq1SyaX5wSq`YUQk&
zW$;LqnjdsVtXMn=MZ87-6fNgrI(kY1Xe^M#BqbnBAOxfdWWL
zDpv(n>w_#3P;JMun=PNOFNUIx#~asouzotR>R`eP5;5?_CS5ofMJdjk-U-!S3D|dI
zToKzG1;bve(*oZI6&n|^BzK_ns!#(Wz1Y3qaY$IAKx(ZKW(o%Xp-K@=Mb3e+9|Omb
zX+l?6A?doH>j_;!1?c1YMe_x6IvdjZon
zgFuKdv72>7c83yWpED>WhaV!f#!Kwy5t~08&g@VC`}E#(zp(!p8^~eCxW~0T;=P^LofrpN-lsE!_R%CKi(>ex`jZZS1PdH}d-J*zp^w
zdY4k(SIG`aceKb7LIS408M{2+W%1zO*xU0RLf5eg+(Nk7B3l-%V^!)jaBd{lr75L6
zkPM2$xaAwyF;ndktwsUe#hIz-6VF5P?Ofk)GXLs}{(|tNTdm9&XBG-{(`Yp76v)~e
zs6Ns5S#G6atkHdzUAg9A+stogvL9pr?(Yb-zx_30|82hPVW;m}bc0k~r$O`?zecXt
z9i05--my%-1_{Cj-k{c64bqvPu)#&!YG62kIP>3)7{z0L@hTaR|ka*8Uf3eYj%~$9i|n7up^y
z^;4_KatqT4)vW(2_J`T%t;JPx{pRB8)swR~K1+(KKD^d06A*uR1N;
zcehyP&Mh|hCSt4KGUCM^v54n|FoO!D*DkMZ+y1&@u8xF|L3*`;RFWx
z{G6^Ai}N!JC4FvosyID6i*K=nK`}E`oGBIYqGMuRN&+
zajFRAlvEwQuMA{E_~*Mn#NTRC?+P^Dj&3{L<{7y0z~A?A-oXLU!|qDivJH1DJRZ#e
zsCif3AUGzk35be#$pr>6OUt#j0c>fswLwf6+u8^~S^;QEcfU_JaXQ(p={VSvU3Hk7C1HZS)Wv|;G8TLZ!y?_R(58>$6r8||vI~4L6tE*%v8q-@D)RV%ct6tqK1PT+
z>A2*5uIhdEssYKS>tI0P_Dgw7j2ya2D=nXo92_3nKyqBp!1-6DMp1gl&|)PU!4#$A
z=8U*PBgY0YDtyFo4VZIq3E#l!4Kd2jI>iV)9go1}ln}q3!H$WxqlvtFSO&34X+Trl
z*;x_;^CGqR1)k_d2wb{!Ny%iH{xq4$7-^|~cCLq%M1qv^9DBK=DiH$;@9bir0sCPG
zXGf(wnT7t&EI2WJqbZEM5q^@?FT
zHm*)#l6b>4dFL(*gPYRg?^u3qQLU7{i#?Q#u$Mc^>1>zJBE8JXPljwj^~*+g=oK|Mg99
zl2wmZ^K7j!#%_M4qP~_PfT1^6`0_vndKuBz5r?r$#v#PQH43cbea;bwc+;v@;h#3n
zk#YNNgHph73Q^dqUYf+M7!F?g?4&+7K}_1jQ9gGF$0jvrgKOS`GkL&nAQdlw-vKN*
X)dJvv8VypxO-5XeaBS}~<}3dO%4*Gi

literal 10057
zcmd^F&2JmW6&FcSGGoiOY{xB7B$?QaVOOFglKON$9Lbgu6^PwXL91O|fd#w!ErK=0bI+r0WKnrOJ6pD^sVsgU*+z>~+NY
zV(I34?|<;&t&eVh;yBf9Z%C#^$4{RO>1nm;yuDd!c_+A2C(t8h6MHhhfblf&cY^;Z
zkN?T~$@O@>G)=?&u72JT>{By_88%PzVRe9iyFaW=`nMR^uDZnLCbp2+4T*h|hOSVX
znnhJtEnAhJ%qSmY@*IctP3VM5@{H?I-9)1{ndNg%UM7}ps-?qGS2)tm4v>aV!7%3}
z_}R;zY&UzZfraEH`BI%>l+MevWRtlWYErlDH23DT1vgU*s_ZGUxJO^~wO@pbE4_wE
zE^<{D*((wxOQwN{pbE(^xUNs(5B0PA#jOOA&d7qV-5@u$*2l=Fz%B!p==3
znTHJ5U>mN4E#)O6$TLlLpQ#C8N$5uXwcBna<7@N;{n>`!pKv2)CU1NUCUX>-?&{bK|w1v~A!Y6Xv
zHL&7Y_)XQG8TDWc=xH5pQ1J_+}(ixMaeG1sUbt80wSH1B-2&__xQDF
zgYIb4N0xW8)8E~US}amU1-84*Z0_!6mAlxOr_wX(VHc?%d;GguVQVels|h;UnMZ2r
zv8(`?JTp-&X;dU*FYC@CR}xj{D509fHmPKDjvx&yYM}tP)MbkNY|l{&qz!=vo6O{T
zGyN_;Wd?gQL)}E7C5jvmXE1a!W6W{Q3-BWg@C-2G4G=J-jY1lcPo*M}krPzDJg|Bk
z2WyG|op^Frb-QoP%kitRci(kfN=8tYmO!y+ohSZ<^9Aw@r-7?!x>-%~RKe}uEu!JLC#dz_6Z#-)uvp#JudY6AzZBzi-9qz
zsB*|U>&`mbaEV)*XLOy#^4F^|X{pBTv>KDwt4|eF)3VdLRHQ65uBn$Lc*BgdUTwF^
zrbivqkOJnYra8_NEAYGcNb}Nhg-Az9zMIHLiGsOt2Zzb&s87x%x)7b3ovusS`K1aD$vfxrM-3H-))fbb$}HM!0W2<`CVuBaPxQo0GL?%JPz>ItT^
z!rSlN^GP{Dx_j3r^#`Q*@;A%hc*IOy^v)fhBt+xO9m^dcYFLMfj>KzDjdz$6g~m4S
z2k$#x=~oT}&9&9;efsW_DiKFB;aUWjlZVyFJw6=J(>|?+=diuT&3=#@4WjC5`jk*@sfB4L7iwJd1Fui`FXbp>Ht4`PtrLoe(V`Z
zC&R=ueBgk>;)Mg5Y_5M`kXzjC3?OLF>jvLVY%RCz;CvuAG&~{+S&l|Q){wDtu%gX%
zcw(0?4-5_s3n6$(=NpEF^gBzj3@Bh*W&##KIQUIZN+0{Omo>(V%Vx8atQf9Kk730fSm#+p?;eLsgmTR~?dUim>
z1?U^N0=fXE#7e9M7w8=6GRV%m1IXa1nFp=N%`RN5CTiEXdY4_eMWP1eTLNHubm0<-
z+8qa%Pw$t4!EZv+b{8Bn!m~pm;#Y~58mo{TU-07FU%MN+wm__Mse}!`MJ~*D6P$mL
z3+1RAqz)dqP#%?
zb^+K7tH7KRmQO^r2(|^r996pETC!GlSB0`sEK1h%s|`@tt{7$PQiU~Ai!J^~+@NKT
zN!TT|c@NFxacCuac5C04yzT
z5lHgLO{Syx0{Dq>3=--ySWh=H0bn*d)5yf}?Msetz9x7{)LzLO8(b4F1QVQ~k%5&D5#+R=x1L1tKNZW~(jjOo)itK<`j-VorOH_m~cO$m4!8^@c$PR=+ccW!x
z1EA|y2ux3`e7k`Q72a8|i0=Js)pl7b@dE|iDXJyogpThKn^A?hhIk-iN*z4s#M!b)U*u!CSL}_x0ak`Y@@`63E4rx9&)Aq
z8SEBV{DkQ|fvD(+n5ga1F$*gO))o_&T)TzEzgq|`V)CA@YpHI-rmtkG7Uj0`^@WH+
zSWIxo8f%xPa1^UgV
zwV~B}Rc)!dy>iLIi`%Qg9?IQH3AuW=tI13{o6h+^f}j2xrL|{GYP3QE>R&s1ybpB+AZ+%#Y=GYB+xYR`&{*G
zkXq!~fPch|(B-9tFfIq#C)Vj_9}c~~PB%Y9P+VlhSdbb4X8|`7MHJ2hZs_*%XIM8f
z#MWtLzsUMLt%TBc&=yY}k@SXfeC-YzS=7I&_sq>9;ba(a^HICp;hyni5?PvK9D(lO
zrmh<+i00gU)OFo}{9JZuQ20k;NYm`w_Yb)
zvGsnerG73r@gi28I7U1Xyro;^tbS$z>(2tva1RI4l0L1`
zG(YeQ?f3m7f|u%oSVKI!`IMl!tbbMj*xBX`exYP9r5EDx7}Ln#&m$ZKOm*5wGJ@wo$_CbIILzdA^7(nN40L=`>Clus_3A
z&Bj|s+{~Di5{u#`4L{DV&zKeLU3XSoTW|}+WXwh62~7)^>JtgAu-O?l`NsG2*x+*C
zkJ?RIxE}{Y%+`naRqRI@rdyw?bA|GY$yz+V9(I*|<;GmiKd313@w0DrRsef}p1qP9P_
zd=URni&r|;-DW`^&JK*E>Ab|>%>f>pqtRbt>nu4@l&0xAOH7=wYRmRY;f++jm~2bEIGzMUol7Yu%HYf%j0!H&w-+SxFGq_8L{EXY7^uuh_OxUB#PP
z>`jsdNwcsWIXi>D66km*j26zB8ap&&+ohwuy>oMOJ%0CkaCXxRv1!`dKb#pE9vSJ+
z`RbRIK>PCB_)EtBR&N_80x78ZM0_2q!9KrMODs9kkL=1?kK@3Ubxe&;l9OjfCy!4|
z9Y4*IQ~ZxdEPi~HwT~(a{uE_l$I*V=DC@Slz~bko*tSzRL$%8aU3_)yABV2pojAi%
zr;wFB+Lyn(eO~+bbI-FB-l)r}h1AIsm(>ys8#LZGJm5E;nmRW*`PTW-Q{#`@@Mm`m
ze_l&O8@~Rh(_f8!*3hu?|2l4`+Ko^i9?E30+2MhaTxMvnFWWyjh%1{z2n_UP2Xa|F
XWU%nfiHyDP^F3IjZu&H%AzJ
 
-  annotation2: 
-  annotation3: 
-  annotation4: 
+  annotation2: 
+  annotation3: 
+  annotation4: 
-  annotation5: >
-  annotation6: >
+  annotation5: >
+  annotation6: >
   annotation7:
   ;Stop reading symbols at @ -- this test has three separate annotations
-  
+  
   bytes2: 
   bytes2a: 
   bytes3: 
@@ -68,108 +69,108 @@
   bytes13: 
 
   dict0: 
-  dict1: 
+  dict1: 
   dict2: @"Missing close brace" 
   dict2a: @"Missing close brace" 
   dict3: @"Duplicate key" 
   dict4: @"Unexpected close brace" 
-  dict5: @"Missing value" 
-  double0: 
-  double+0: 
-  double-0: 
-  double1: 
-  double2: 
-  double3: 
+  dict5: @"Missing value" 
+  double0: 
+  double+0: 
+  double-0: 
+  double1: 
+  double2: 
+  double3: 
   double4: @"Fewer than 16 digits" 
   double5: @"More than 16 digits" 
   double6: @"Invalid chars" 
-  double7: @"Positive infinity" 
-  double8: @"Negative infinity" 
-  double9: @"-qNaN" 
-  double10: @"-qNaN" 
-  double11: @"+qNaN" 
-  double12: @"+qNaN" 
+  double7: @"Positive infinity" 
+  double8: @"Negative infinity" 
+  double9: @"-qNaN" 
+  double10: @"-qNaN" 
+  double11: @"+qNaN" 
+  double12: @"+qNaN" 
   double13: @"Bad spacing" 
-  double14: @"-sNaN" 
-  double15: @"-sNaN" 
-  double16: @"+sNaN" 
-  double17: @"+sNaN" 
-  float0: 
-  float+0: 
-  float-0: 
-  float1: 
-  float2: 
+  double14: @"-sNaN" 
+  double15: @"-sNaN" 
+  double16: @"+sNaN" 
+  double17: @"+sNaN" 
+  float0: 
+  float+0: 
+  float-0: 
+  float1: 
+  float2: 
   float3: @"Fewer than 8 digits" 
   float4: @"More than 8 digits" 
   float5: @"Invalid chars" 
-  float6: @"Positive infinity" 
-  float7: @"Negative infinity" 
-  float8: @"+sNaN" 
-  float9: @"+sNaN" 
-  float10: @"-sNaN" 
-  float11: @"-sNaN" 
+  float6: @"Positive infinity" 
+  float7: @"Negative infinity" 
+  float8: @"+sNaN" 
+  float9: @"+sNaN" 
+  float10: @"-sNaN" 
+  float11: @"-sNaN" 
   float12: @"Bad spacing" 
-  float13: @"+qNaN" 
-  float14: @"+qNaN" 
-  float15: @"-qNaN" 
-  float16: @"-qNaN" 
-  int-257: 
-  int-256: 
-  int-255: 
-  int-254: 
-  int-129: 
-  int-128: 
-  int-127: 
-  int-4: 
-  int-3: 
-  int-2: 
-  int-1: 
-  int0: 
-  int+0: 
-  int-0: 
-  int1: 
-  int12: 
-  int13: 
-  int127: 
-  int+127: 
-  int128: 
-  int255: 
-  int256: 
-  int32767: 
-  int32768: 
-  int65535: 
-  int65536: 
-  int131072: 
-  int2500000000: 
+  float13: @"+qNaN" 
+  float14: @"+qNaN" 
+  float15: @"-qNaN" 
+  float16: @"-qNaN" 
+  int-257: 
+  int-256: 
+  int-255: 
+  int-254: 
+  int-129: 
+  int-128: 
+  int-127: 
+  int-4: 
+  int-3: 
+  int-2: 
+  int-1: 
+  int0: 
+  int+0: 
+  int-0: 
+  int1: 
+  int12: 
+  int13: 
+  int127: 
+  int+127: 
+  int128: 
+  int255: 
+  int256: 
+  int32767: 
+  int32768: 
+  int65535: 
+  int65536: 
+  int131072: 
+  int2500000000: 
   int87112285931760246646623899502532662132736: 
   list0: 
-  list4: 
-  list4a: 
-  list5: 
+  list4: 
+  list4a: 
+  list5: 
   list6: 
   list7: 
   list8: @"Missing close bracket" 
   list9: @"Unexpected close bracket" 
   list10: @"Missing end byte" 
-  list11: 
-  list12: 
+  list11: 
+  list12: 
   noinput0: @"No input at all" 
-  embed0: 
-  embed1: 
-  embed2: 
+  embed0: 
+  embed1: 
+  embed2: 
   record1: >>
   record2: , >>>>
-  record3:  "Dr">>
+  record3:  "Dr">>
   record4: >
-  record5: >
+  record5: >
   record6: >
-  record7: >
-  record8:  3 4>>
+  record7: >
+  record8:  3 4>>
   record9: @"Missing record label" ">
   record10: @"Missing close-angle-bracket" 
   record11: @"Unexpected close-angle-bracket" ">
   set0: 
-  set1: 
+  set1: 
   set2: @"Missing close brace" 
   set2a: @"Missing close brace" 
   set3: @"Duplicate value" 
@@ -277,17 +278,17 @@
                           B7
                             B1 03 494473
                               B5
-                                A0 74
-                                A1 03 AF
-                                A1 00 EA
-                                A2 00 97 89
+                                B0 01 74
+                                B0 02 03 AF
+                                B0 02 00 EA
+                                B0 03 00 97 89
                               84
                             B1 05 5469746c65
                               B1 14 566965772066726f6d203135746820466c6f6f72
                             B1 05 5769647468
-                              A1 03 20
+                              B0 02 03 20
                             B1 06 486569676874
-                              A1 02 58
+                              B0 02 02 58
                             B1 08 416e696d61746564
                               B3 05 66616c7365
                             B1 09 5468756d626e61696c
@@ -295,9 +296,9 @@
                                 B1 03 55726c
                                   B1 26 687474703a2f2f7777772e6578616d706c652e636f6d2f696d6167652f343831393839393433
                                 B1 05 5769647468
-                                  A0 64
+                                  B0 01 64
                                 B1 06 486569676874
-                                  A0 7D
+                                  B0 01 7D
                               84
                           84
                         84"
@@ -324,8 +325,8 @@
                             b1 05 5374617465           b1 02 4341
                             b1 07 41646472657373       b1 00
                             b1 07 436f756e747279       b1 02 5553
-                            b1 08 4c61746974756465     83 4042e226809d4952
-                            b1 09 4c6f6e676974756465   83 c05e99566cf41f21
+                            b1 08 4c61746974756465     8708 4042e226809d4952
+                            b1 09 4c6f6e676974756465   8708 c05e99566cf41f21
                             b1 09 707265636973696f6e   b1 03 7a6970
                           84
                           b7
@@ -334,8 +335,8 @@
                             b1 05 5374617465           b1 02 4341
                             b1 07 41646472657373       b1 00
                             b1 07 436f756e747279       b1 02 5553
-                            b1 08 4c61746974756465     83 4042af9d66adb403
-                            b1 09 4c6f6e676974756465   83 c05e81aa4fca42af
+                            b1 08 4c61746974756465     8708 4042af9d66adb403
+                            b1 09 4c6f6e676974756465   8708 c05e81aa4fca42af
                             b1 09 707265636973696f6e   b1 03 7a6970
                           84
                         84"