Update chat example code

examples/example-simple-chat/package.json

@@ -16,13 +16,13 @@
   "author": "Tony Garnock-Jones <tonyg@leastfixedpoint.com>",
   "license": "GPL-3.0+",
   "dependencies": {
-    "@preserves/core": "0.995",
+    "@preserves/core": "0.995.200",
     "@syndicate-lang/core": "*",
     "@syndicate-lang/html": "*",
     "@syndicate-lang/ws-relay": "*"
   },
   "devDependencies": {
-    "@preserves/schema-cli": "0.995",
+    "@preserves/schema-cli": "0.995.201",
     "@syndicate-lang/ts-plugin": "*",
     "@syndicate-lang/tsc": "*",
     "rollup": "^2.60",

examples/example-simple-chat/src/index.ts

@@ -1,7 +1,7 @@
 /// SPDX-License-Identifier: GPL-3.0-or-later
 /// SPDX-FileCopyrightText: Copyright © 2016-2021 Tony Garnock-Jones <tonyg@leastfixedpoint.com>
 
-import { fromJS, Bytes, Dataspace, Ref, Sturdy, AnyValue, Reader, Schemas, Embedded, stringify } from "@syndicate-lang/core";
+import { fromJS, Bytes, Dataspace, Ref, Sturdy, AnyValue, Reader, Schemas, stringify } from "@syndicate-lang/core";
 import { boot as bootHtml, Anchor, template as html, HtmlFragments, GlobalEvent, UIAttribute, UIChangeableProperty } from "@syndicate-lang/html";
 import { boot as bootWakeDetector, WakeEvent } from "./wake-detector";
 import { boot as bootWsRelay } from "@syndicate-lang/ws-relay";
@@ -46,12 +46,9 @@ function bootChat(ds: Ref) {
                 during G.ResolvePath({
                     "route": route,
                     "addr": $addr,
-                    "control": $control_e: Embedded,
+                    "control": $control: Ref,
-                    "resolved": G.Resolved.accepted($remoteDs_e: Embedded),
+                    "resolved": G.Resolved.accepted($remoteDs: Ref),
                 }) => {
-                    const remoteDs = remoteDs_e.embeddedValue;
-                    const control = control_e.embeddedValue;
-
                     on message WakeEvent() => at control {
                         send message G.ForceDisconnect();
                     }

packages/core/protocols/schema-bundle.bin

@@ -2,9 +2,9 @@
 tcp-remote„´³tupleµ´³named³host´³atom³String„„´³named³port´³atom³
SignedInteger„„„„„³TcpPeerInfo´³rec´³lit³tcp-peer„´³tupleµ´³named³handle´³embedded³any„„´³named³local´³refµ„³TcpLocal„„´³named³remote´³refµ„³	TcpRemote„„„„„„³embeddedType´³refµ³	EntityRef„³Cap„„„µ³http„´³schema·³version°

³definitions·³Chunk´³orµµ±string´³atom³String„„µ±bytes´³atom³
 ByteString„„„„³Headers´³dictof´³atom³Symbol„´³atom³String„„³MimeType´³atom³Symbol„³
 QueryValue´³orµµ±string´³atom³String„„µ±file´³rec´³lit³file„´³tupleµ´³named³filename´³atom³String„„´³named³headers´³refµ„³Headers„„´³named³body´³atom³
-ByteString„„„„„„„„³HostPattern´³orµµ±host´³atom³String„„µ±any´³lit€„„„„³HttpBinding´³rec´³lit³	http-bind„´³tupleµ´³named³host´³refµ„³HostPattern„„´³named³port´³atom³
SignedInteger„„´³named³method´³refµ„³
MethodPattern„„´³named³path´³refµ„³PathPattern„„´³named³handler´³embedded´³refµ„³HttpRequest„„„„„„³HttpContext´³rec´³lit³request„´³tupleµ´³named³req´³refµ„³HttpRequest„„´³named³res´³embedded´³refµ„³HttpResponse„„„„„„³HttpRequest´³rec´³lit³http-request„´³tupleµ´³named³sequenceNumber´³atom³
SignedInteger„„´³named³host´³atom³String„„´³named³port´³atom³
SignedInteger„„´³named³method´³atom³Symbol„„´³named³path´³seqof´³atom³String„„„´³named³headers´³refµ„³Headers„„´³named³query´³dictof´³atom³Symbol„´³seqof´³refµ„³
+ByteString„„„„„„„„³HostPattern´³orµµ±host´³atom³String„„µ±any´³lit€„„„„³HttpBinding´³rec´³lit³	http-bind„´³tupleµ´³named³host´³refµ„³HostPattern„„´³named³port´³atom³
SignedInteger„„´³named³method´³refµ„³
MethodPattern„„´³named³path´³refµ„³PathPattern„„´³named³handler´³embedded´³refµ„³HttpRequest„„„„„„³HttpContext´³rec´³lit³request„´³tupleµ´³named³req´³refµ„³HttpRequest„„´³named³res´³embedded´³refµ„³HttpResponse„„„„„„³HttpRequest´³rec´³lit³http-request„´³tupleµ´³named³sequenceNumber´³atom³
SignedInteger„„´³named³host´³refµ„³RequestHost„„´³named³port´³atom³
SignedInteger„„´³named³method´³atom³Symbol„„´³named³path´³seqof´³atom³String„„„´³named³headers´³refµ„³Headers„„´³named³query´³dictof´³atom³Symbol„´³seqof´³refµ„³
 QueryValue„„„„´³named³body´³refµ„³RequestBody„„„„„³HttpService´³rec´³lit³http-service„´³tupleµ´³named³host´³refµ„³HostPattern„„´³named³port´³atom³
SignedInteger„„´³named³method´³refµ„³
MethodPattern„„´³named³path´³refµ„³PathPattern„„„„„³PathPattern´³seqof´³refµ„³PathPatternElement„„³RequestBody´³orµµ±present´³atom³
-ByteString„„µ±absent´³lit€„„„„³HttpListener´³rec´³lit³
http-listener„´³tupleµ´³named³port´³atom³
SignedInteger„„„„„³HttpResponse´³orµµ±status´³rec´³lit³status„´³tupleµ´³named³code´³atom³
SignedInteger„„´³named³message´³atom³String„„„„„„µ±header´³rec´³lit³header„´³tupleµ´³named³name´³atom³Symbol„„´³named³value´³atom³String„„„„„„µ±chunk´³rec´³lit³chunk„´³tupleµ´³named³chunk´³refµ„³Chunk„„„„„„µ±done´³rec´³lit³done„´³tupleµ´³named³chunk´³refµ„³Chunk„„„„„„„„³
MethodPattern´³orµµ±any´³lit€„„µ±specific´³atom³Symbol„„„„³PathPatternElement´³orµµ±label´³atom³String„„µ±wildcard´³lit³
_„„µ±rest´³lit³...„„„„„³embeddedType€„„µ³noise„´³schema·³version°

³definitions·³Packet´³orµµ±complete´³atom³
+ByteString„„µ±absent´³lit€„„„„³RequestHost´³orµµ±present´³atom³String„„µ±absent´³lit€„„„„³HttpListener´³rec´³lit³
http-listener„´³tupleµ´³named³port´³atom³
SignedInteger„„„„„³HttpResponse´³orµµ±status´³rec´³lit³status„´³tupleµ´³named³code´³atom³
SignedInteger„„´³named³message´³atom³String„„„„„„µ±header´³rec´³lit³header„´³tupleµ´³named³name´³atom³Symbol„„´³named³value´³atom³String„„„„„„µ±chunk´³rec´³lit³chunk„´³tupleµ´³named³chunk´³refµ„³Chunk„„„„„„µ±done´³rec´³lit³done„´³tupleµ´³named³chunk´³refµ„³Chunk„„„„„„„„³
MethodPattern´³orµµ±any´³lit€„„µ±specific´³atom³Symbol„„„„³PathPatternElement´³orµµ±label´³atom³String„„µ±wildcard´³lit³
_„„µ±rest´³lit³...„„„„„³embeddedType€„„µ³noise„´³schema·³version°

³definitions·³Packet´³orµµ±complete´³atom³
 ByteString„„µ±
 fragmented´³seqof´³atom³
 ByteString„„„„„³	Initiator´³rec´³lit³	initiator„´³tupleµ´³named³initiatorSession´³embedded´³refµ„³Packet„„„„„„³	NoiseSpec´³andµ´³dict·³key´³named³key´³atom³
@@ -40,5 +40,5 @@ ByteString
 RunService´³rec´³lit³run-service„´³tupleµ´³named³serviceName³any„„„„³ServiceState´³rec´³lit³
service-state„´³tupleµ´³named³serviceName³any„´³named³state´³refµ„³State„„„„„³
ServiceObject´³rec´³lit³service-object„´³tupleµ´³named³serviceName³any„´³named³object³any„„„„³RequireService´³rec´³lit³require-service„´³tupleµ´³named³serviceName³any„„„„³RestartService´³rec´³lit³restart-service„´³tupleµ´³named³serviceName³any„„„„³ServiceDependency´³rec´³lit³
 depends-on„´³tupleµ´³named³depender³any„´³named³dependee´³refµ„³ServiceState„„„„„„³embeddedType´³refµ³	EntityRef„³Cap„„„µ³protocol„´³schema·³version°

³definitions·³Oid´³atom³
SignedInteger„³Sync´³rec´³lit³
S„´³tupleµ´³named³peer´³embedded´³lit<69>„„„„„„³Turn´³seqof´³refµ„³	TurnEvent„„³Error´³rec´³lit³error„´³tupleµ´³named³message´³atom³String„„´³named³detail³any„„„„³Event´³orµµ±Assert´³refµ„³Assert„„µ±Retract´³refµ„³Retract„„µ±Message´³refµ„³Message„„µ±Sync´³refµ„³Sync„„„„³Assert´³rec´³lit³
A„´³tupleµ´³named³	assertion´³refµ„³	Assertion„„´³named³handle´³refµ„³Handle„„„„„³Handle´³atom³
SignedInteger„³Packet´³orµµ±Turn´³refµ„³Turn„„µ±Error´³refµ„³Error„„µ±	Extension´³refµ„³	Extension„„„„³Message´³rec´³lit³
M„´³tupleµ´³named³body´³refµ„³	Assertion„„„„„³Retract´³rec´³lit³
R„´³tupleµ´³named³handle´³refµ„³Handle„„„„„³	Assertion³any³	Extension´³rec´³named³label³any„´³named³fields´³seqof³any„„„³	TurnEvent´³tupleµ´³named³oid´³refµ„³Oid„„´³named³event´³refµ„³Event„„„„„³embeddedType€„„µ³	dataspace„´³schema·³version°

³definitions·³Observe´³rec´³lit³Observe„´³tupleµ´³named³pattern´³refµ³dataspacePatterns„³Pattern„„´³named³observer´³embedded³any„„„„„„³embeddedType´³refµ³	EntityRef„³Cap„„„µ³
 gatekeeper„´³schema·³version°

³definitions·³Bind´³rec´³lit³bind„´³tupleµ´³named³description´³refµ„³Description„„´³named³target´³embedded³any„„´³named³observer´³refµ„³BindObserver„„„„„³Step´³rec´³named³stepType´³atom³Symbol„„´³tupleµ´³named³detail³any„„„„³Bound´³orµµ±bound´³rec´³lit³bound„´³tupleµ´³named³pathStep´³refµ„³PathStep„„„„„„µ±Rejected´³refµ„³Rejected„„„„³Route´³rec´³lit³route„´³tuplePrefixµ´³named³
-transports´³seqof³any„„„´³named³	pathSteps´³seqof´³refµ„³PathStep„„„„„³Resolve´³rec´³lit³resolve„´³tupleµ´³named³step´³refµ„³Step„„´³named³observer´³embedded´³refµ„³Resolved„„„„„„³PathStep´³rec´³named³stepType´³atom³Symbol„„´³tupleµ´³named³detail³any„„„„³Rejected´³rec´³lit³rejected„´³tupleµ´³named³detail³any„„„„³Resolved´³orµµ±accepted´³rec´³lit³accepted„´³tupleµ´³named³responderSession´³embedded³any„„„„„„µ±Rejected´³refµ„³Rejected„„„„³Description´³rec´³named³stepType´³atom³Symbol„„´³tupleµ´³named³detail³any„„„„³ResolvePath´³rec´³lit³resolve-path„´³tupleµ´³named³route´³refµ„³Route„„´³named³addr³any„´³named³control´³embedded´³refµ„³TransportControl„„„´³named³resolved´³refµ„³Resolved„„„„„³BindObserver´³orµµ±present´³embedded´³refµ„³Bound„„„µ±absent´³lit€„„„„³ForceDisconnect´³rec´³lit³force-disconnect„´³tupleµ„„„³ResolvedPathStep´³rec´³lit³	path-step„´³tupleµ´³named³origin´³embedded´³refµ„³Resolve„„„´³named³pathStep´³refµ„³PathStep„„´³named³resolved´³refµ„³Resolved„„„„„³TransportControl´³refµ„³ForceDisconnect„³TransportConnection´³rec´³lit³connect-transport„´³tupleµ´³named³addr³any„´³named³control´³embedded´³refµ„³TransportControl„„„´³named³resolved´³refµ„³Resolved„„„„„„³embeddedType´³refµ³	EntityRef„³Cap„„„µ³transportAddress„´³schema·³version°

³definitions·³Tcp´³rec´³lit³tcp„´³tupleµ´³named³host´³atom³String„„´³named³port´³atom³
SignedInteger„„„„„³Unix´³rec´³lit³unix„´³tupleµ´³named³path´³atom³String„„„„„³Stdio´³rec´³lit³stdio„´³tupleµ„„„³	WebSocket´³rec´³lit³ws„´³tupleµ´³named³url´³atom³String„„„„„„³embeddedType€„„µ³dataspacePatterns„´³schema·³version°

³definitions·³DLit´³rec´³lit³lit„´³tupleµ´³named³value´³refµ„³AnyAtom„„„„„³DBind´³rec´³lit³bind„´³tupleµ´³named³pattern´³refµ„³Pattern„„„„„³AnyAtom´³orµµ±bool´³atom³Boolean„„µ±double´³atom³Double„„µ±int´³atom³
SignedInteger„„µ±string´³atom³String„„µ±bytes´³atom³
+transports´³seqof³any„„„´³named³	pathSteps´³seqof´³refµ„³PathStep„„„„„³Resolve´³rec´³lit³resolve„´³tupleµ´³named³step´³refµ„³Step„„´³named³observer´³embedded´³refµ„³Resolved„„„„„„³PathStep´³rec´³named³stepType´³atom³Symbol„„´³tupleµ´³named³detail³any„„„„³Rejected´³rec´³lit³rejected„´³tupleµ´³named³detail³any„„„„³Resolved´³orµµ±accepted´³rec´³lit³accepted„´³tupleµ´³named³responderSession´³embedded³any„„„„„„µ±Rejected´³refµ„³Rejected„„„„³Description´³rec´³named³stepType´³atom³Symbol„„´³tupleµ´³named³detail³any„„„„³ResolvePath´³rec´³lit³resolve-path„´³tupleµ´³named³route´³refµ„³Route„„´³named³addr³any„´³named³control´³embedded´³refµ„³TransportControl„„„´³named³resolved´³refµ„³Resolved„„„„„³BindObserver´³orµµ±present´³embedded´³refµ„³Bound„„„µ±absent´³lit€„„„„³ForceDisconnect´³rec´³lit³force-disconnect„´³tupleµ„„„³ResolvedPathStep´³rec´³lit³	path-step„´³tupleµ´³named³origin´³embedded´³refµ„³Resolve„„„´³named³pathStep´³refµ„³PathStep„„´³named³resolved´³refµ„³Resolved„„„„„³TransportControl´³refµ„³ForceDisconnect„³TransportConnection´³rec´³lit³connect-transport„´³tupleµ´³named³addr³any„´³named³control´³embedded´³refµ„³TransportControl„„„´³named³resolved´³refµ„³Resolved„„„„„„³embeddedType´³refµ³	EntityRef„³Cap„„„µ³transportAddress„´³schema·³version°

³definitions·³Tcp´³rec´³lit³tcp„´³tupleµ´³named³host´³atom³String„„´³named³port´³atom³
SignedInteger„„„„„³Unix´³rec´³lit³unix„´³tupleµ´³named³path´³atom³String„„„„„³Stdio´³rec´³lit³stdio„´³tupleµ„„„³	WebSocket´³rec´³lit³ws„´³tupleµ´³named³url´³atom³String„„„„„„³embeddedType€„„µ³dataspacePatterns„´³schema·³version°

³definitions·³AnyAtom´³orµµ±bool´³atom³Boolean„„µ±double´³atom³Double„„µ±int´³atom³
SignedInteger„„µ±string´³atom³String„„µ±bytes´³atom³
-ByteString„„µ±symbol´³atom³Symbol„„µ±embedded´³embedded³any„„„„³Pattern´³orµµ±DDiscard´³refµ„³DDiscard„„µ±DBind´³refµ„³DBind„„µ±DLit´³refµ„³DLit„„µ±	DCompound´³refµ„³	DCompound„„„„³DDiscard´³rec´³lit³
_„´³tupleµ„„„³	DCompound´³orµµ±rec´³rec´³lit³rec„´³tupleµ´³named³label³any„´³named³fields´³seqof´³refµ„³Pattern„„„„„„„µ±arr´³rec´³lit³arr„´³tupleµ´³named³items´³seqof´³refµ„³Pattern„„„„„„„µ±dict´³rec´³lit³dict„´³tupleµ´³named³entries´³dictof³any´³refµ„³Pattern„„„„„„„„„„³embeddedType´³refµ³	EntityRef„³Cap„„„„„
+ByteString„„µ±symbol´³atom³Symbol„„µ±embedded´³embedded³any„„„„³Pattern´³orµµ±discard´³rec´³lit³
_„´³tupleµ„„„„µ±bind´³rec´³lit³bind„´³tupleµ´³named³pattern´³refµ„³Pattern„„„„„„µ±lit´³rec´³lit³lit„´³tupleµ´³named³value´³refµ„³AnyAtom„„„„„„µ±group´³rec´³lit³group„´³tupleµ´³named³type´³refµ„³	GroupType„„´³named³entries´³dictof³any´³refµ„³Pattern„„„„„„„„„³	GroupType´³orµµ±rec´³rec´³lit³rec„´³tupleµ´³named³label³any„„„„„µ±arr´³rec´³lit³arr„´³tupleµ„„„„µ±dict´³rec´³lit³dict„´³tupleµ„„„„„„„³embeddedType´³refµ³	EntityRef„³Cap„„„„„

packages/core/protocols/schemas/dataspacePatterns.prs

@@ -1,15 +1,23 @@
 version 1 .
 embeddedType EntityRef.Cap .
 
-# Dataspace patterns: a sublanguage of attenuation patterns.
+# Dataspace patterns: *almost* a sublanguage of attenuation patterns.
-Pattern = DDiscard / DBind / DLit / DCompound .
+#
+# One key difference is that Dataspace patterns are extensible, in that
+# they ignore fields not mentioned in group patterns.
 
-DDiscard = <_>.
+Pattern =
-DBind = <bind @pattern Pattern>.
+    / @discard <_>
-DLit = <lit @value AnyAtom>.
+    / <bind @pattern Pattern>
-DCompound = <rec @label any @fields [Pattern ...]>
+    / <lit @value AnyAtom>
-          / <arr @items [Pattern ...]>
+    / <group @type GroupType @entries { any: Pattern ...:... }>
-          / <dict @entries { any: Pattern ...:... }> .
+.
+
+GroupType =
+    / <rec @label any>
+    / <arr>
+    / <dict>
+.
 
 AnyAtom =
     / @bool bool

packages/core/protocols/schemas/http.prs

@@ -21,7 +21,7 @@ MethodPattern = @any #f / @specific @"Lowercase" symbol .
 # Assertion in driver DS
 HttpRequest = <http-request
                @sequenceNumber int
-               @host string
+               @host RequestHost
                @port int
                @method @"Lowercase" symbol
                @path [string ...]
@@ -32,13 +32,24 @@ HttpRequest = <http-request
 Headers = {@"Lowercase" symbol: string ...:...} .
 QueryValue = @string string / <file @filename string @headers Headers @body bytes> .
 RequestBody = @present bytes / @absent #f .
+RequestHost = @present string / @absent #f .
 
 # Assertion to handler entity
 HttpContext = <request @req HttpRequest @res #:HttpResponse> .
 
+# HttpResponse protocol. Delivered to the `res` ref in `HttpContext`.
+#
+#     (status | header)* . chunk* . done
+#
+# Done triggers completion of the response and retraction of the frame by the peer. If the
+# HttpBinding responsible for the request is withdrawn mid-way through a response (i.e. when
+# chunked transfer is used and at least one chunk has been sent) the request is abruptly
+# closed; if it is withdrawn at any other moment in the lifetime of the request, a 500 Internal
+# Server Error is send to the client.
+#
 @<TODO "trailers?">
-# Messages
 HttpResponse =
+# Messages.
 / <status @code int @message string>
 / <header @name symbol @value string>
 / <chunk @chunk Chunk>

packages/core/protocols/schemas/noise.prs

@@ -60,7 +60,7 @@ NoisePreSharedKeys = @present { preSharedKeys: [bytes ...] } / @invalid { preSha
 # complete Noise "transport message"; when `complete`, the whole thing is likewise a complete
 # "transport message".
 #
-# Retraction of the `Introduction` ends the session from the initiator-side; retraction of the
+# Retraction of the `Initiator` ends the session from the initiator-side; retraction of the
 # `<accepted ...>` assertion ends the session from the responder-side.
 
 SessionItem = Initiator / Packet .

packages/core/src/runtime/pattern.ts

@@ -1,7 +1,7 @@
 /// SPDX-License-Identifier: GPL-3.0-or-later
 /// SPDX-FileCopyrightText: Copyright © 2016-2024 Tony Garnock-Jones <tonyg@leastfixedpoint.com>
 
-import { canonicalString, KeyedDictionary, is, Record, RecordConstructorInfo, Value, _iterMap, DictionaryMap, Dictionary } from '@preserves/core';
+import { canonicalString, KeyedDictionary, is, Record, RecordConstructorInfo, Value, _iterMap, DictionaryMap, Dictionary, EncodableDictionary } from '@preserves/core';
 import { AnyValue, Ref } from './actor.js';
 import * as P from '../gen/dataspacePatterns.js';
 
@@ -21,7 +21,7 @@ export function classOfValue(v: any): Shape | null {
     }
 }
 
-export function classOfCtor(v: P.DCompound): Shape {
+export function classOfCtor(v: P.GroupType): Shape {
     switch (v._variant) {
         case 'rec':
             return canonicalString(v.label);
@@ -46,44 +46,47 @@ export function step(v: AnyValue, index: AnyValue): AnyValue | undefined {
     }
 }
 
+export type ConstantPositions = {
+    withValues: Array<Path>,
+    requiredToExist: Array<Path>,
+}
+
 export type PatternAnalysis = {
-    constPaths: Array<Path>,
+    constPositions: ConstantPositions,
     constValues: Array<AnyValue>,
     capturePaths: Array<Path>,
 };
 
 export function analysePattern(p: P.Pattern): PatternAnalysis {
     const result: PatternAnalysis = {
-        constPaths: [],
+        constPositions: {
+            withValues: [],
+            requiredToExist: [],
+        },
         constValues: [],
         capturePaths: [],
     };
     const path: Path = [];
 
-    function walkKey(p: P.Pattern, key: AnyValue) {
-        path.push(key);
-        walk(p);
-        path.pop();
-    }
-
     function walk(p: P.Pattern) {
         switch (p._variant) {
-            case 'DCompound':
+            case 'group':
-                switch (p.value._variant) {
+                p.entries.forEach((p, k) => {
-                    case 'rec': p.value.fields.forEach(walkKey); break;
+                    path.push(k);
-                    case 'arr': p.value.items.forEach(walkKey); break;
+                    walk(p);
-                    case 'dict': p.value.entries.forEach(walkKey); break;
+                    path.pop();
-                }
+                });
                 break;
-            case 'DBind':
+            case 'bind':
                 result.capturePaths.push(path.slice());
-                walk(p.value.pattern);
+                walk(p.pattern);
                 break;
-            case 'DDiscard':
+            case 'discard':
+                result.constPositions.requiredToExist.push(path.slice());
                 break;
-            case 'DLit':
+            case 'lit':
-                result.constPaths.push(path.slice());
+                result.constPositions.withValues.push(path.slice());
-                result.constValues.push(P.fromAnyAtom(p.value.value));
+                result.constValues.push(P.fromAnyAtom(p.value));
                 break;
         }
     }
@@ -97,32 +100,20 @@ export function match(p: P.Pattern, v: AnyValue): Array<AnyValue> | false {
 
     function walk(p: P.Pattern, v: AnyValue): boolean {
         switch (p._variant) {
-            case 'DBind': {
+            case 'bind': {
                 captures.push(v);
-                return walk(p.value.pattern, v);
+                return walk(p.pattern, v);
             }
-            case 'DDiscard':
+            case 'discard':
                 return true;
-            case 'DLit':
+            case 'lit':
-                return is(p.value.value, v);
+                return is(p.value, v);
-            case 'DCompound': {
+            case 'group': {
-                const pcls = classOfCtor(p.value);
+                const pcls = classOfCtor(p.type);
                 const vcls = classOfValue(v);
                 if (pcls !== vcls) return false;
-                let items: Array<P.Pattern>;
+                for (const [stepIndex, pp] of p.entries.entries()) {
-                switch (p.value._variant) {
+                    const vv = step(v, stepIndex);
-                    case 'dict':
-                        for (const [stepIndex, pp] of p.value.entries.entries()) {
-                            const vv = step(v, stepIndex);
-                            if (vv === void 0 || !walk(pp, vv)) return false;
-                        }
-                        return true;
-                    case 'rec': items = p.value.fields; break;
-                    case 'arr': items = p.value.items; break;
-                }
-                let index = 0;
-                for (const pp of items) {
-                    const vv = step(v, index++);
                     if (vv === void 0 || !walk(pp, vv)) return false;
                 }
                 return true;
@@ -136,19 +127,14 @@ export function match(p: P.Pattern, v: AnyValue): Array<AnyValue> | false {
 export function isCompletelyConcrete(p: P.Pattern): boolean {
     function walk(p: P.Pattern): boolean {
         switch (p._variant) {
-            case 'DBind': return false;
+            case 'bind': return false;
-            case 'DDiscard': return false;
+            case 'discard': return false;
-            case 'DLit': return true;
+            case 'lit': return true;
-            case 'DCompound': switch (p.value._variant) {
+            case 'group':
-                case 'rec': return p.value.fields.every(isCompletelyConcrete);
+                for (const pp of p.entries.values()) {
-                case 'arr': return p.value.items.every(isCompletelyConcrete);
+                    if (!walk(pp)) return false;
-                case 'dict': {
-                    for (const pp of p.value.entries.values()) {
-                        if (!walk(pp)) return false;
-                    }
-                    return true;
                 }
-            }
+                return true;
         }
     }
     return walk(p);
@@ -157,26 +143,19 @@ export function isCompletelyConcrete(p: P.Pattern): boolean {
 export function withoutCaptures(p: P.Pattern): P.Pattern {
     function walk(p: P.Pattern): P.Pattern {
         switch (p._variant) {
-            case 'DBind': return walk(p.value.pattern);
+            case 'bind': return walk(p.pattern);
-            case 'DDiscard': return p;
+            case 'discard': return p;
-            case 'DLit': return p;
+            case 'lit': return p;
-            case 'DCompound':
+            case 'group': {
-                switch (p.value._variant) {
+                const newEntries = new KeyedDictionary<Ref, Value<Ref>, P.Pattern<Ref>>();
-                    case 'rec':
+                for (const [kk, pp] of p.entries) {
-                        return P.Pattern.DCompound(P.DCompound.rec({
+                    newEntries.set(kk, walk(pp));
-                            label: p.value.label,
-                            fields: p.value.fields.map(walk),
-                        }));
-                    case 'arr':
-                        return P.Pattern.DCompound(P.DCompound.arr(p.value.items.map(walk)));
-                    case 'dict': {
-                        const newDict = new KeyedDictionary<Ref, Value<Ref>, P.Pattern<Ref>>();
-                        for (const [k, v] of p.value.entries) {
-                            newDict.set(k, walk(v));
-                        }
-                        return P.Pattern.DCompound(P.DCompound.dict(newDict));
-                    }
                 }
+                return P.Pattern.group({
+                    type: p.type,
+                    entries: newEntries,
+                });
+            }
         }
     }
     return walk(p);
@@ -186,24 +165,33 @@ export function withoutCaptures(p: P.Pattern): P.Pattern {
 // Constructor helpers
 
 export function bind(p?: P.Pattern): P.Pattern {
-    return P.Pattern.DBind(P.DBind(p ?? _));
+    return P.Pattern.bind(p ?? _);
 }
 
 export function discard(): P.Pattern {
-    return P.Pattern.DDiscard(P.DDiscard());
+    return P.Pattern.discard();
 }
 
 export const _ = discard();
 
+function lit_seq_entries(vs: AnyValue[]): KeyedDictionary<Ref, AnyValue, P.Pattern<Ref>> {
+    const entries = new KeyedDictionary<Ref, AnyValue, P.Pattern<Ref>>();
+    vs.forEach((v, i) => entries.set(i, lit(v)));
+    return entries;
+}
+
 export function lit(v: AnyValue): P.Pattern {
     if (Array.isArray(v)) {
         if ('label' in v) {
-            return P.Pattern.DCompound(P.DCompound.rec({
+            return P.Pattern.group({
-                label: v.label,
+                type: P.GroupType.rec(v.label),
-                fields: v.map(lit),
+                entries: lit_seq_entries(v),
-            }));
+            });
         } else {
-            return P.Pattern.DCompound(P.DCompound.arr(v.map(lit)));
+            return P.Pattern.group({
+                type: P.GroupType.arr(),
+                entries: lit_seq_entries(v),
+            });
         }
     }
 
@@ -211,38 +199,58 @@ export function lit(v: AnyValue): P.Pattern {
     if (vMap) {
         const r = new KeyedDictionary<Ref, AnyValue, P.Pattern>();
         vMap.forEach((val, key) => r.set(key, lit(val)));
-        return P.Pattern.DCompound(P.DCompound.dict(r));
+        return P.Pattern.group({
+            type: P.GroupType.dict(),
+            entries: r,
+        });
     }
 
     if (Set.isSet(v)) {
         throw new Error("Cannot express literal set in pattern");
     }
 
-    return P.Pattern.DLit(P.DLit(P.asAnyAtom(v)));
+    return P.Pattern.lit(P.asAnyAtom(v));
 }
 
 export function drop_lit(p: P.Pattern): AnyValue | null {
     const e = new Error();
+
+    function walkEntries(target: AnyValue[], entries: EncodableDictionary<Ref, AnyValue, P.Pattern<Ref>>): void {
+        let maxKey = -1;
+        for (const key of entries.keys()) {
+            if (typeof key !== 'number') throw e;
+            maxKey = Math.max(maxKey, key);
+        }
+        for (let i = 0; i < maxKey + 1; i++) {
+            const p = entries.get(i);
+            if (p === void 0) throw e;
+            target.push(walk(p));
+        }
+    }
+    
     function walk(p: P.Pattern): AnyValue {
         switch (p._variant) {
-            case 'DCompound':
+            case 'group':
-                switch (p.value._variant) {
+                switch (p.type._variant) {
                     case 'rec': {
                         const v = [] as unknown as Record<AnyValue, AnyValue[], Ref>;
-                        v.label = p.value.label;
+                        v.label = p.type.label;
-                        p.value.fields.forEach(tt => v.push(walk(tt)));
+                        walkEntries(v, p.entries);
+                        return v;
+                    }
+                    case 'arr': {
+                        const v = [] as AnyValue[];
+                        walkEntries(v, p.entries);
                         return v;
                     }
-                    case 'arr':
-                        return p.value.items.map(walk);
                     case 'dict': {
                         const v = new DictionaryMap<Ref, AnyValue>();
-                        p.value.entries.forEach((pp, key) => v.set(key, walk(pp)));
+                        p.entries.forEach((pp, key) => v.set(key, walk(pp)));
                         return v.simplifiedValue();
                     }
                 }
-            case 'DLit':
+            case 'lit':
-                return P.fromAnyAtom(p.value.value);
+                return P.fromAnyAtom(p.value);
             default:
                 throw e;
         }
@@ -256,14 +264,22 @@ export function drop_lit(p: P.Pattern): AnyValue | null {
 }
 
 export function rec(label: AnyValue, ... fields: P.Pattern[]): P.Pattern {
-    return P.Pattern.DCompound(P.DCompound.rec({ label, fields }));
+    return P.Pattern.group({
+        type: P.GroupType.rec(label),
+        entries: new KeyedDictionary<Ref, AnyValue, P.Pattern>(fields.map((p, i) => [i, p])),
+    });
 }
 
 export function arr(... patterns: P.Pattern[]): P.Pattern {
-    return P.Pattern.DCompound(P.DCompound.arr(patterns));
+    return P.Pattern.group({
+        type: P.GroupType.arr(),
+        entries: new KeyedDictionary<Ref, AnyValue, P.Pattern>(patterns.map((p, i) => [i, p])),
+    });
 }
 
 export function dict(... entries: [AnyValue, P.Pattern][]): P.Pattern {
-    return P.Pattern.DCompound(P.DCompound.dict(
+    return P.Pattern.group({
-        new KeyedDictionary<Ref, AnyValue, P.Pattern>(entries)));
+        type: P.GroupType.dict(),
+        entries: new KeyedDictionary<Ref, AnyValue, P.Pattern>(entries),
+    });
 }

packages/core/src/runtime/quasivalue.ts

@@ -2,7 +2,7 @@
 /// SPDX-FileCopyrightText: Copyright © 2021-2024 Tony Garnock-Jones <tonyg@leastfixedpoint.com>
 
 import { AnyValue, Ref } from './actor.js';
-import { Pattern, toPattern } from '../gen/dataspacePatterns.js';
+import { Pattern, toPattern, GroupType, fromGroupType } from '../gen/dataspacePatterns.js';
 import * as P from './pattern.js';
 import { RecordConstructorInfo, is, Record, JsDictionary } from '@preserves/core';
 import { Meta, Type, GenType, SchemaDefinition } from '@preserves/schema';
@@ -17,9 +17,7 @@ export type QuasiValue =
     | { type: 'bind', inner: QuasiValue }
     | { type: 'discard' }
     | { type: 'lit', value: AnyValue }
-    | { type: 'rec', label: AnyValue, items: QuasiValue[] }
+    | { type: 'group', groupType: GroupType, entries: [AnyValue, QuasiValue][] }
-    | { type: 'arr', items: QuasiValue[] }
-    | { type: 'dict', entries: [AnyValue, QuasiValue][] }
     | { type: 'unquote', unquoted: QuasiValue }
 ;
 
@@ -58,7 +56,8 @@ export function rec(label: AnyValue, ... items: QuasiValue[]): QuasiValue {
     if (literals.length === items.length) {
         return lit(Record(label, literals));
     } else {
-        return { type: 'rec', label, items };
+        const entries = items.map((v, i) => [i, v] as [number, QuasiValue]);
+        return { type: 'group', groupType: GroupType.rec(label), entries };
     }
 }
 
@@ -67,12 +66,13 @@ export function arr(... items: QuasiValue[]): QuasiValue {
     if (literals.length === items.length) {
         return lit(literals);
     } else {
-        return { type: 'arr', items };
+        const entries = items.map((v, i) => [i, v] as [number, QuasiValue]);
+        return { type: 'group', groupType: GroupType.arr(), entries };
     }
 }
 
 export function dict(... entries: [AnyValue, QuasiValue][]): QuasiValue {
-    return { type: 'dict', entries };
+    return { type: 'group', groupType: GroupType.dict(), entries };
 }
 
 export function quote(quoted: QuasiValue): QuasiValue {
@@ -80,12 +80,9 @@ export function quote(quoted: QuasiValue): QuasiValue {
         case 'bind': return quote(bind.quasiValue(quoted.inner));
         case 'discard': return quote(discard.quasiValue());
         case 'lit': return rec(Symbol.for('lit'), lit(quoted.value));
-        case 'arr': return rec(Symbol.for('arr'), arr(
+        case 'group':
-            ... quoted.items.map(quote)));
+            return rec(Symbol.for('group'), lit(fromGroupType(quoted.groupType)), dict(
-        case 'rec': return rec(Symbol.for('rec'), lit(quoted.label), arr(
+                ... quoted.entries.map(([k, qq]) => [k, quote(qq)] as [AnyValue, QuasiValue])));
-            ... quoted.items.map(quote)));
-        case 'dict': return rec(Symbol.for('dict'), dict(
-            ... quoted.entries.map(([k, qq]) => [k, quote(qq)] as [AnyValue, QuasiValue])));
         case 'unquote': return quoted.unquoted;
     }
 }
@@ -94,6 +91,20 @@ export function unquote(unquoted: QuasiValue): QuasiValue {
     return { type: 'unquote', unquoted };
 }
 
+function entriesSeq<V>(entries: [AnyValue, V][], defaultValue: V): V[] {
+    let maxKey = -1;
+    const result: V[] = [];
+    for (const [k, q] of entries) {
+        if (typeof k !== 'number') throw new Error("Invalid sequence quasivalue entries key");
+        result[k] = q;
+        maxKey = Math.max(maxKey, k);
+    }
+    for (let i = 0; i < maxKey + 1; i++) {
+        if (result[i] === void 0) result[i] = defaultValue;
+    }
+    return result;
+}
+
 export function ctor(info: QuasiValueConstructorInfo, ... items: QuasiValue[]): QuasiValue {
     if ('constructorInfo' in info) {
         return rec(info.constructorInfo.label, ... items);
@@ -158,7 +169,7 @@ export function ctor(info: QuasiValueConstructorInfo, ... items: QuasiValue[]):
                 if (d.type === 'discard') {
                     return d;
                 }
-                if (d.type !== 'dict') {
+                if (d.type !== 'group' || d.groupType._variant !== 'dict') {
                     throw new Error(`Dictionary argument needed to ${defNameStr}`);
                 }
                 for (const [k, p] of d.entries) {
@@ -217,8 +228,8 @@ export function ctor(info: QuasiValueConstructorInfo, ... items: QuasiValue[]):
         }
 
         function qArr(q: QuasiValue): QuasiValue[] {
-            if (q.type === 'arr') {
+            if (q.type === 'group' && q.groupType._variant === 'arr') {
-                return q.items;
+                return entriesSeq(q.entries, discard());
             } else if (q.type === 'lit' && Array.isArray(q.value)) {
                 return q.value.map(lit);
             } else {
@@ -284,10 +295,12 @@ function walk(q: QuasiValue): Pattern {
         case 'bind': return P.bind(walk(q.inner));
         case 'discard': return P._;
         case 'lit': return P.lit(q.value);
-        case 'arr': return P.arr(... q.items.map(walk));
+        case 'group': switch (q.groupType._variant) {
-        case 'rec': return P.rec(q.label, ... q.items.map(walk));
+            case 'arr': return P.arr(... entriesSeq(q.entries, discard()).map(walk));
-        case 'dict': return P.dict(... q.entries.map(
+            case 'rec': return P.rec(q.groupType.label, ... entriesSeq(q.entries, discard()).map(walk));
-            ([k, qq]) => [k, walk(qq)] as [AnyValue, Pattern]));
+            case 'dict': return P.dict(... q.entries.map(
+                ([k, qq]) => [k, walk(qq)] as [AnyValue, Pattern]));
+        }
         case 'unquote': throw new Error('Unexpected unquote in QuasiValue');
     }
 }

packages/core/src/runtime/skeleton.ts

@@ -6,7 +6,7 @@ import { AnyValue, Assertion, Ref } from './actor.js';
 import { Bag, ChangeDescription } from './bag.js';
 import * as Stack from './stack.js';
 import * as P from '../gen/dataspacePatterns.js';
-import { Path, analysePattern, classOfCtor, classOfValue, step, Shape } from './pattern.js';
+import { Path, analysePattern, classOfCtor, classOfValue, step, Shape, ConstantPositions } from './pattern.js';
 
 enum EventType {
     ADDED = +1,
@@ -31,13 +31,15 @@ export class Index<T> {
     readonly root: Node<T> = new Node(new Continuation(new Set()));
 
     addObserver(pattern: P.Pattern, observer: IndexObserver<T>, parameter: T) {
-        let {constPaths, constValues, capturePaths} = analysePattern(pattern);
+        let {constPositions, constValues, capturePaths} = analysePattern(pattern);
         const continuation = this.root.extend(pattern);
-        let constValMap = continuation.leafMap.get(constPaths);
+        let constValMap = continuation.leafMap.get(constPositions);
         if (!constValMap) {
             constValMap = new KeyedDictionary();
             continuation.cachedAssertions.forEach((a) => {
-                const key = projectPaths(a, constPaths);
+                if (projectPaths(a, constPositions.requiredToExist) === void 0) return;
+                const key = projectPaths(a, constPositions.withValues);
+                if (key === void 0) return;
                 let leaf = constValMap!.get(key);
                 if (!leaf) {
                     leaf = new Leaf();
@@ -45,7 +47,7 @@ export class Index<T> {
                 }
                 leaf.cachedAssertions.add(a);
             });
-            continuation.leafMap.set(constPaths, constValMap);
+            continuation.leafMap.set(constPositions, constValMap);
         }
         let leaf = constValMap.get(constValues);
         if (!leaf) {
@@ -55,8 +57,10 @@ export class Index<T> {
         let observerGroup = leaf.observerGroups.get(capturePaths);
         if (!observerGroup) {
             const cachedCaptures = new Bag<Ref, Array<AnyValue>>();
-            leaf.cachedAssertions.forEach((a) =>
+            leaf.cachedAssertions.forEach((a) => {
-                cachedCaptures._items.update(projectPaths(a, capturePaths), n => n! + 1, 0));
+                const vs = projectPaths(a, capturePaths);
+                if (vs !== void 0) cachedCaptures._items.update(vs, n => n! + 1, 0);
+            });
             observerGroup = new ObserverGroup(cachedCaptures);
             leaf.observerGroups.set(capturePaths, observerGroup);
         }
@@ -65,9 +69,9 @@ export class Index<T> {
     }
 
     removeObserver(pattern: P.Pattern, observer: IndexObserver<T>, parameter: T) {
-        let {constPaths, constValues, capturePaths} = analysePattern(pattern);
+        let {constPositions, constValues, capturePaths} = analysePattern(pattern);
         const continuation = this.root.extend(pattern);
-        let constValMap = continuation.leafMap.get(constPaths);
+        let constValMap = continuation.leafMap.get(constPositions);
         if (!constValMap) return;
         let leaf = constValMap.get(constValues);
         if (!leaf) return;
@@ -82,7 +86,7 @@ export class Index<T> {
             constValMap.delete(constValues);
         }
         if (constValMap.size === 0) {
-            continuation.leafMap.delete(constPaths);
+            continuation.leafMap.delete(constPositions);
         }
     }
 
@@ -168,19 +172,19 @@ class Node<T> {
         function walkNode(node: Node<T>, popCount: number, stepIndex: AnyValue, p: P.Pattern): [number, Node<T>]
         {
             switch (p._variant) {
-                case 'DDiscard':
+                case 'discard':
-                case 'DLit':
+                case 'lit':
                     return [popCount, node];
-                case 'DBind':
+                case 'bind':
-                    return walkNode(node, popCount, stepIndex, p.value.pattern);
+                    return walkNode(node, popCount, stepIndex, p.pattern);
-                case 'DCompound': {
+                case 'group': {
                     const selector: Selector = [popCount, stepIndex];
                     let table = node.edges.get(selector);
                     if (!table) {
                         table = {};
                         node.edges.set(selector, table);
                     }
-                    let cls = classOfCtor(p.value);
+                    let cls = classOfCtor(p.type);
                     let nextNode = table[cls];
                     if (!nextNode) {
                         nextNode = new Node(new Continuation(
@@ -189,16 +193,11 @@ class Node<T> {
                         table[cls] = nextNode;
                     }
                     popCount = 0;
-                    function walkKey(pp: P.Pattern, stepIndex: AnyValue) {
+                    p.entries.forEach((pp, stepIndex) => {
                         path.push(stepIndex);
                         [popCount, nextNode] = walkNode(nextNode, popCount, stepIndex, pp);
                         path.pop();
-                    }
+                    });
-                    switch (p.value._variant) {
-                        case 'rec': p.value.fields.forEach(walkKey); break;
-                        case 'arr': p.value.items.forEach(walkKey); break;
-                        case 'dict': p.value.entries.forEach(walkKey); break;
-                    }
                     return [popCount + 1, nextNode];
                 }
             }
@@ -226,8 +225,10 @@ class Node<T> {
 
         function walkContinuation(continuation: Continuation<T>) {
             m_cont(continuation, outerValue);
-            continuation.leafMap.forEach((constValMap, constPaths) => {
+            continuation.leafMap.forEach((constValMap, constPositions) => {
-                let constValues = projectPaths(outerValue, constPaths);
+                if (projectPaths(outerValue, constPositions.requiredToExist) === void 0) return;
+                let constValues = projectPaths(outerValue, constPositions.withValues);
+                if (constValues === void 0) return;
                 let leaf = constValMap.get(constValues);
                 if (!leaf && operation === EventType.ADDED) {
                     leaf = new Leaf();
@@ -236,12 +237,13 @@ class Node<T> {
                 if (leaf) {
                     m_leaf(leaf, outerValue);
                     leaf.observerGroups.forEach((observerGroup, capturePaths) => {
-                        m_observerGroup(observerGroup, projectPaths(outerValue, capturePaths));
+                        const vs = projectPaths(outerValue, capturePaths);
+                        if (vs !== void 0) m_observerGroup(observerGroup, vs);
                     });
                     if (operation === EventType.REMOVED && leaf.isEmpty()) {
                         constValMap.delete(constValues);
                         if (constValMap.size === 0) {
-                            continuation.leafMap.delete(constPaths);
+                            continuation.leafMap.delete(constPositions);
                         }
                     }
                 }
@@ -267,7 +269,7 @@ class Node<T> {
 
 class Continuation<T> {
     readonly cachedAssertions: Set<Ref>;
-    readonly leafMap: KeyedDictionary<Ref, Array<Path>, KeyedDictionary<Ref, Assertion, Leaf<T>>> = new KeyedDictionary();
+    readonly leafMap: KeyedDictionary<Ref, ConstantPositions, KeyedDictionary<Ref, Assertion, Leaf<T>>> = new KeyedDictionary();
 
     constructor(cachedAssertions: Set<Ref>) {
         this.cachedAssertions = cachedAssertions;
@@ -320,15 +322,21 @@ class ObserverGroup<T> {
     }
 }
 
-// Total by assumption that path is valid for v
+function projectPath(v: AnyValue, path: Path): AnyValue | undefined {
-function projectPath(v: AnyValue, path: Path): AnyValue {
     for (let index of path) {
-        v = step(v, index)!;
+        const next = step(v, index);
+        if (next === void 0) return void 0;
+        v = next;
     }
     return v;
 }
 
-// Total by assumption that paths are valid for v
+function projectPaths(v: AnyValue, paths: Array<Path>): AnyValue[] | undefined {
-function projectPaths(v: AnyValue, paths: Array<Path>): AnyValue[] {
+    const result = [];
-  return paths.map((path) => projectPath(v, path));
+    for (const path of paths) {
+        const w = projectPath(v, path);
+        if (w === void 0) return void 0;
+        result.push(w);
+    }
+    return result;
 }