First run of new compiler output!

packages/core/examples/box-and-client.js

@@ -60,11 +60,11 @@ export function boot(thisFacet) {
   thisFacet.spawn('client', function (thisFacet) {
     thisFacet.addEndpoint(() => {
       let analysis = Skeleton.analyzeAssertion(BoxState(_$));
-      analysis.callback = thisFacet.wrap((thisFacet, evt, vs) => {
+      analysis.callback = thisFacet.wrap((thisFacet, evt, [v]) => {
         if (evt === Skeleton.EventType.ADDED) {
           thisFacet.scheduleScript(() => {
-            // console.log('client sending SetBox', vs[0] + 1);
+            // console.log('client sending SetBox', v + 1);
-            thisFacet.send(SetBox(vs[0] + 1));
+            thisFacet.send(SetBox(v + 1));
           });
         }
       });

packages/core/examples/box-and-client.syndicate.js

@@ -28,13 +28,7 @@ boot {
   spawn named 'box' {
     field this.value = 0;
     assert BoxState(this.value);
-    dataflow {
+    stop on (this.value === N) console.log('terminated box root facet');
-      if (this.value === N) {
-        stop {
-          console.log('terminated box root facet');
-        }
-      }
-    }
     on message SetBox($v) => this.value = v;
   }
 
@@ -46,3 +40,5 @@ boot {
   thisFacet.actor.dataspace.addStopHandler(() =>
     console.timeEnd('box-and-client-' + N.toString()));
 }
+
+new __SYNDICATE__.Ground(__SYNDICATE__bootProc).start();

packages/core/src/compiler/grammar.ts

@@ -1,13 +1,15 @@
 import {
-    Token, Items,
+    TokenType, Token, Items,
     Pattern,
+    foldItems, match, anonymousTemplate as template, commaJoin,
+    startPos,
 
-    scope, bind, seq, alt, upTo, atom, group, exec,
+    scope, bind, seq, alt, upTo, atom, atomString, group, exec,
-    repeat, option, withoutSpace, map, rest, discard,
+    repeat, option, withoutSpace, map, mapm, rest, discard,
-    value,
+    value, succeed, fail, separatedBy, anything,
-
 } from '../syntax/index.js';
 import * as Matcher from '../syntax/matcher.js';
+import { Path, Skeleton } from '../runtime/api.js';
 
 export type Expr = Items;
 export type Statement = Items;
@@ -18,8 +20,8 @@ export const block = (acc?: Items) =>
     ? group('{', discard)
     : group('{', map(rest, items => acc.push(... items)));
 
-export const statementBoundary = alt(atom(';'), Matcher.newline, Matcher.end);
+export const statementBoundary = alt<any>(atom(';'), Matcher.newline, Matcher.end);
-export const exprBoundary = alt(atom(';'), atom(','), group('{', discard), Matcher.end);
+export const exprBoundary = alt<any>(atom(';'), atom(','), group('{', discard), Matcher.end);
 
 export const identifier: Pattern<Identifier> = atom();
 
@@ -28,9 +30,9 @@ export function expr(... extraStops: Pattern<any>[]): Pattern<Expr> {
 }
 
 export function statement(acc: Items): Pattern<void> {
-    return alt(group('{', map(rest, items => acc.push(... items))),
+    return alt<any>(group('{', map(rest, items => acc.push(... items))),
-               withoutSpace(seq(map(upTo(statementBoundary), items => acc.push(... items)),
+                    withoutSpace(seq(map(upTo(statementBoundary), items => acc.push(... items)),
-                                map(statementBoundary, i => i ? acc.push(i) : void 0))));
+                                     map(statementBoundary, i => i ? acc.push(i) : void 0))));
 }
 
 export interface FacetAction {
@@ -135,14 +137,28 @@ export function statementFacetAction(kw: Pattern<any>): Pattern<StatementFacetAc
 // Principal: Facet
 export const dataflowStatement = statementFacetAction(atom('dataflow'));
 
-export interface EventHandlerEndpointStatement extends FacetAction {
+export interface GenericEventEndpointStatement extends StatementFacetAction {
     terminal: boolean;
-    triggerType: 'dataflow' | 'start' | 'stop' | 'asserted' | 'retracted' | 'message';
     isDynamic: boolean;
-    pattern?: Expr;
-    body: Statement;
 }
 
+export interface DataflowEndpointStatement extends GenericEventEndpointStatement {
+    triggerType: 'dataflow';
+    predicate: Expr;
+}
+
+export interface PseudoEventEndpointStatement extends GenericEventEndpointStatement {
+    triggerType: 'start' | 'stop';
+}
+
+export interface AssertionEventEndpointStatement extends GenericEventEndpointStatement {
+    triggerType: 'asserted' | 'retracted' | 'message';
+    pattern: ValuePattern;
+}
+
+export type EventHandlerEndpointStatement =
+    DataflowEndpointStatement | PseudoEventEndpointStatement | AssertionEventEndpointStatement;
+
 // Principal: Facet
 export const eventHandlerEndpointStatement: Pattern<EventHandlerEndpointStatement> =
     facetAction(o => {
@@ -151,19 +167,23 @@ export const eventHandlerEndpointStatement: Pattern<EventHandlerEndpointStatemen
         o.body = [];
         return seq(option(map(atom('stop'), _ => o.terminal = true)),
                    atom('on'),
-                   alt(map(group('(', bind(o, 'pattern', expr())), _ => o.triggerType = 'dataflow'),
+                   alt<any>(seq(map(group('(', bind(o as DataflowEndpointStatement, 'predicate',
-                       seq(bind(o, 'triggerType',
+                                                    expr())),
-                                map(alt(atom('start'), atom('stop')), e => e.text)),
+                                    _ => o.triggerType = 'dataflow'),
-                           option(statement(o.body))),
+                                option(statement(o.body))),
-                       seq(bind(o, 'triggerType',
+                            mapm(seq(bind(o, 'triggerType',
-                                map(alt(atom('asserted'),
+                                          alt(atomString('start'), atomString('stop'))),
-                                        atom('retracted'),
+                                     option(statement(o.body))),
-                                        atom('message')),
+                                 v => o.terminal ? fail : succeed(v)),
-                                    e => e.text)),
+                            seq(bind(o, 'triggerType',
-                           option(map(atom(':snapshot'), _ => o.isDynamic = false)),
+                                     alt(atomString('asserted'),
-                           bind(o, 'pattern', expr(atom('=>'))),
+                                         atomString('retracted'),
-                           alt(seq(atom('=>'), statement(o.body)),
+                                         atomString('message'))),
-                               statementBoundary))));
+                                option(map(atom(':snapshot'), _ => o.isDynamic = false)),
+                                bind(o as AssertionEventEndpointStatement, 'pattern',
+                                     valuePattern(atom('=>'))),
+                                alt<any>(seq(atom('=>'), statement(o.body)),
+                                         statementBoundary))));
     });
 
 export interface TypeDefinitionStatement {
@@ -175,7 +195,7 @@ export interface TypeDefinitionStatement {
 
 // Principal: none
 export const typeDefinitionStatement: Pattern<TypeDefinitionStatement> =
-    scope(o => seq(bind(o, 'expectedUse', map(alt(atom('message'), atom('assertion')), e => e.text)),
+    scope(o => seq(bind(o, 'expectedUse', alt(atomString('message'), atomString('assertion'))),
                    atom('type'),
                    bind(o, 'label', identifier),
                    group('(', bind(o, 'fields', repeat(identifier, { separator: atom(',') }))),
@@ -194,7 +214,7 @@ export const messageSendStatement: Pattern<MessageSendStatement> =
                          statementBoundary));
 
 export interface DuringStatement extends FacetAction {
-    pattern: Expr;
+    pattern: ValuePattern;
     body: Statement;
 }
 
@@ -203,8 +223,9 @@ export const duringStatement: Pattern<DuringStatement> =
     facetAction(o => {
         o.body = [];
         return seq(atom('during'),
-                   bind(o, 'pattern', expr()),
+                   bind(o, 'pattern', valuePattern(atom('=>'))),
-                   statement(o.body));
+                   alt(seq(atom('=>'), statement(o.body)),
+                       statementBoundary));
     });
 
 // Principal: Facet
@@ -219,3 +240,230 @@ export const bootStatement: Pattern<Statement> =
 
 // Principal: Facet
 export const stopStatement = statementFacetAction(atom('stop'));
+
+//---------------------------------------------------------------------------
+// Syntax of patterns over Value, used in endpoints
+
+export interface PCapture {
+    type: 'PCapture',
+    binder: Identifier,
+    inner: ValuePattern,
+}
+
+export interface PDiscard {
+    type: 'PDiscard',
+}
+
+export interface PConstructor {
+    type: 'PConstructor',
+    ctor: Expr,
+    arguments: ValuePattern[],
+}
+
+export interface PConstant {
+    type: 'PConstant',
+    value: Expr,
+}
+
+export interface PArray {
+    type: 'PArray',
+    elements: ValuePattern[],
+}
+
+export type ValuePattern = PCapture | PDiscard | PConstructor | PConstant | PArray;
+
+const pCaptureId: Pattern<Identifier> =
+    mapm(identifier, i => i.text.startsWith('$')
+        ? succeed({ ... i, text: i.text.slice(1) })
+        : fail);
+
+const pDiscard: Pattern<void> = mapm(identifier, i => i.text === '_' ? succeed(void 0) : fail);
+
+function hasCapturesOrDiscards(e: Expr): boolean {
+    return foldItems(e,
+                     t => match(alt<any>(pCaptureId, pDiscard), [t], null) !== null,
+                     (_s, _e, b, _k) => b,
+                     bs => bs.some(b => b));
+}
+
+// $id - capture of discard
+// _ - discard
+//
+// expr(pat, ...) - record ctor
+// $id(pat) - nested capture
+// [pat, ...] - array pat
+//
+// expr(expr, ...) - constant
+// [expr, ...] - constant
+// other - constant
+
+interface RawCall {
+    items: Items;
+    callee: Expr;
+    arguments: Expr[];
+}
+
+function pRawCall(... extraStops: Pattern<any>[]): Pattern<RawCall> {
+    return scope((o: RawCall) => seq(bind(o, 'callee',
+                                          expr(seq(group('(', discard),
+                                                   alt(exprBoundary, ... extraStops)))),
+                                     seq(map(anything({ advance: false }),
+                                             g => o.items = [... o.callee, g]),
+                                         group('(', bind(o, 'arguments',
+                                                         separatedBy(expr(), atom(',')))))));
+}
+
+function isConstant(o: RawCall) {
+    return (!(hasCapturesOrDiscards(o.callee) || o.arguments.some(hasCapturesOrDiscards)));
+}
+
+export function valuePattern(... extraStops: Pattern<any>[]): Pattern<ValuePattern> {
+    return alt<ValuePattern>(
+        scope<PCapture>(o => {
+            o.type = 'PCapture';
+            o.inner = { type: 'PDiscard' };
+            return bind(o, 'binder', pCaptureId);
+        }),
+        scope(o => map(pDiscard, _ => o.type = 'PDiscard')),
+        mapm<RawCall, ValuePattern>(
+            pRawCall(... extraStops),
+            o => {
+                if (isConstant(o)) {
+                    return succeed({ type: 'PConstant', value: o.items });
+                } else if (hasCapturesOrDiscards(o.callee)) {
+                    const r = match(pCaptureId, o.callee, null);
+                    if (r !== null && o.arguments.length === 1)
+                    {
+                        const argPat = match(valuePattern(), o.arguments[0], null);
+                        if (argPat === null) return fail;
+                        return succeed({
+                            type: 'PCapture',
+                            inner: argPat,
+                            binder: r
+                        });
+                    } else {
+                        return fail;
+                    }
+                } else {
+                    const argPats = o.arguments.map(a => match(valuePattern(), a, null));
+                    if (argPats.some(p => p === null)) return fail;
+                    return succeed({
+                        type: 'PConstructor',
+                        ctor: o.callee,
+                        arguments: argPats as ValuePattern[]
+                    });
+                }
+            }),
+        map(expr(), e => ({ type: 'PConstant', value: e }))
+    );
+}
+
+export function patternText(p: ValuePattern): Items {
+    switch (p.type) {
+        case 'PDiscard': return template`_`;
+        case 'PConstant': return p.value;
+        case 'PCapture':
+            {
+                const binder = { ... p.binder, text: '$' + p.binder.text };
+                if (p.inner.type === 'PDiscard') {
+                    return [binder];
+                } else {
+                    return template`${[binder]}(${patternText(p.inner)})`;
+                }
+            }
+        case 'PArray': return template`[${commaJoin(p.elements.map(patternText))}]`;
+        case 'PConstructor': return template`${p.ctor}(${commaJoin(p.arguments.map(patternText))})`;
+    }
+}
+
+export interface StaticAnalysis {
+    skeleton: Expr;
+    constPaths: Path[];
+    constVals: Expr[];
+    capturePaths: Path[];
+    captureIds: Identifier[];
+    assertion: Expr;
+}
+
+const eDiscard: Expr = template`(__SYNDICATE__.Discard._instance)`;
+const eCapture = (e: Expr): Expr => template`(__SYNDICATE__.Capture(${e}))`;
+
+export function compilePattern(pattern: ValuePattern): StaticAnalysis {
+    const constPaths: Path[] = [];
+    const constVals: Expr[] = [];
+    const capturePaths: Path[] = [];
+    const captureIds: Identifier[] = [];
+
+    const currentPath: Path = [];
+
+    function walk(pattern: ValuePattern): [Skeleton<Expr>, Expr] {
+        switch (pattern.type) {
+            case 'PDiscard':
+                return [null, eDiscard];
+            case 'PCapture': {
+                capturePaths.push(currentPath.slice());
+                captureIds.push(pattern.binder);
+                const [s, a] = walk(pattern.inner);
+                return [s, eCapture(a)];
+            }
+            case 'PConstant':
+                constVals.push(pattern.value);
+                return [null, pattern.value];
+            case 'PConstructor': {
+                const skel: Skeleton<Expr> = {
+                    shape: template`__SYNDICATE__.Skeleton.constructorInfoSignature((${pattern.ctor}).constructorInfo)`,
+                    members: [],
+                };
+                const assertionArgs: Expr[] = [];
+                pattern.arguments.forEach((argPat, i) => {
+                    currentPath.push(i);
+                    const [s, a] = walk(argPat);
+                    skel.members.push(s);
+                    assertionArgs.push(a);
+                    currentPath.pop();
+                });
+                return [skel, template`(${pattern.ctor}(${commaJoin(assertionArgs)}))`];
+            }
+            case 'PArray': {
+                const skel: Skeleton<Expr> = {
+                    shape: [ {
+                        start: startPos(null),
+                        end: startPos(null),
+                        type: TokenType.STRING,
+                        text: JSON.stringify(pattern.elements.length.toString()),
+                    } ],
+                    members: []
+                };
+                const elements: Expr[] = [];
+                pattern.elements.forEach((elemPat, i) => {
+                    currentPath.push(i);
+                    const [s, a] = walk(elemPat);
+                    skel.members.push(s);
+                    elements.push(a);
+                    currentPath.pop();
+                });
+                return [skel, template`[${commaJoin(elements)}]`];
+            }
+        }
+    }
+
+    const [skeletonStructure, assertion] = walk(pattern);
+    const skeleton = renderSkeleton(skeletonStructure);
+
+    return {
+        skeleton,
+        constPaths,
+        constVals,
+        capturePaths,
+        captureIds,
+        assertion,
+    };
+}
+
+function renderSkeleton(skel: Skeleton<Expr>): Expr {
+    if (skel === null) {
+        return template`null`;
+    } else {
+        return template`({shape:${skel.shape}, members: [${commaJoin(skel.members.map(renderSkeleton))}]})`;
+    }
+}

packages/core/src/compiler/main.ts

@@ -1,9 +1,18 @@
 import fs from 'fs';
 import * as S from '../syntax/index.js';
-import { ArrayList, Substitution } from '../syntax/index.js';
+import { Substitution } from '../syntax/index.js';
 import * as G from './grammar.js';
 import { BootProc } from './internals.js';
 
+export function stripShebang(items: S.Items): S.Items {
+    if ((items.length > 0) &&
+        S.isToken(items[0]) &&
+        items[0].text.startsWith('#!')) {
+        while (items.length > 0 && !S.isTokenType(items[0], S.TokenType.NEWLINE)) items.shift();
+    }
+    return items;
+}
+
 export function main(argv: string[]) {
     let [ inputFilename ] = argv.slice(2);
     inputFilename = inputFilename ?? '/dev/stdin';
@@ -11,12 +20,12 @@ export function main(argv: string[]) {
 
     const scanner = new S.StringScanner(S.startPos(inputFilename), source);
     const reader = new S.LaxReader(scanner);
-    let tree = reader.readToEnd();
+    let tree = stripShebang(reader.readToEnd());
     let macro = new S.Templates();
 
     tree = macro.template()`import * as __SYNDICATE__ from '@syndicate/core';\n${tree}`;
 
-    let passNumber = 1;
+    let passNumber = 0;
     let expansionNeeded = true;
     function expand<T>(p: S.Pattern<T>, f: (t: T) => S.Items) {
         tree = S.replace(tree, p, t => {
@@ -33,10 +42,22 @@ export function main(argv: string[]) {
         expand(p, t => macro.template()`${receiverFor(t)}${f(t)}`);
     }
 
+    function terminalWrap(isTerminal: boolean, body: G.Statement): G.Statement {
+        if (isTerminal) {
+            return macro.template()`thisFacet._stop(function (thisFacet) {${body}})`
+        } else {
+            return body;
+        }
+    }
+
     while (expansionNeeded) {
-        if (passNumber >= 128) {
+        if (++passNumber >= 128) {
             throw new Error(`Too many compiler passes (${passNumber})!`);
         }
+
+        // console.log(`\n\n\n======================================== PASS ${passNumber}\n`);
+        // console.log(S.itemText(tree, { color: true, missing: '\x1b[41m□\x1b[0m' }));
+
         expansionNeeded = false;
         expandFacetAction(
             G.spawn,
@@ -44,7 +65,7 @@ export function main(argv: string[]) {
                 let proc = macro.template()`function (thisFacet) {${s.bootProcBody}}`;
                 if (s.isDataspace) proc = macro.template()`__SYNDICATE__.inNestedDataspace(${proc})`;
                 let assertions = (s.initialAssertions.length > 0)
-                    ? macro.template()`, new __SYNDICATE__.Set([${S.joinItems(s.initialAssertions, ', ')}])`
+                    ? macro.template()`, new __SYNDICATE__.Set([${S.commaJoin(s.initialAssertions)}])`
                     : ``;
                 return macro.template()`_spawn(${s.name ?? 'null'}, ${proc}${assertions});`;
             });
@@ -61,18 +82,76 @@ export function main(argv: string[]) {
             });
         expandFacetAction(
             G.assertionEndpointStatement,
-            s => macro.template()`addEndpoint(thisFacet => (${s.test ?? 'true'}) && (${s.template}), ${''+s.isDynamic});`);
+            s => {
+                if (s.test == void 0) {
+                    return macro.template()`addEndpoint(thisFacet => ({ assertion: ${s.template}, analysis: null }));`;
+                } else {
+                    return macro.template()`addEndpoint(thisFacet => (${s.test ?? 'true'})
+                                              ? ({ assertion: ${s.template}, analysis: null })
+                                              : ({ assertion: void 0, analysis: null }), ${''+s.isDynamic});`;
+                }
+            });
         expandFacetAction(
             G.dataflowStatement,
             s => macro.template()`addDataflow(function (thisFacet) {${s.body}});`);
         expandFacetAction(
             G.eventHandlerEndpointStatement,
             s => {
-                return macro.template()`EVENTHANDLER[${`${s.terminal}/${s.isDynamic}`}][${s.triggerType}][${s.pattern ?? []}][${s.body}]`;
+                switch (s.triggerType) {
+                    case 'dataflow':
+                        return macro.template()`withSelfDo(function (thisFacet) { dataflow { if (${s.predicate}) { ${terminalWrap(s.terminal, s.body)} } } });`;
+
+                    case 'start':
+                    case 'stop': {
+                        const m = s.triggerType === 'start' ? 'addStartScript' : 'addStopScript';
+                        return macro.template()`${m}(function (thisFacet) {${s.body}});`;
+                    }
+
+                    case 'asserted':
+                    case 'retracted':
+                    case 'message': {
+                        const sa = G.compilePattern(s.pattern);
+                        const expectedEvt = ({
+                            'asserted': 'ADDED',
+                            'retracted': 'REMOVED',
+                            'message': 'MESSAGE',
+                        })[s.triggerType];
+                        return macro.template()`addEndpoint(thisFacet => ({
+  assertion: __SYNDICATE__.Observe(${sa.assertion}),
+  analysis: {
+    skeleton: ${sa.skeleton},
+    constPaths: ${JSON.stringify(sa.constPaths)},
+    constVals: [${S.commaJoin(sa.constVals)}],
+    capturePaths: ${JSON.stringify(sa.capturePaths)},
+    callback: thisFacet.wrap((thisFacet, __Evt, [${S.commaJoin(sa.captureIds.map(i=>[i]))}]) => {
+      if (__Evt === __SYNDICATE__.Skeleton.EventType.${expectedEvt}) {
+        thisFacet.scheduleScript(() => {${terminalWrap(s.terminal, s.body)}});
+      }
+    })
+  }
+}), ${'' + s.isDynamic});`;
+                    }
+                }
             });
         expandFacetAction(
             G.duringStatement,
-            s => macro.template()`DURING[${s.pattern}][${s.body}]`);
+            s => {
+                // TODO: spawn during
+                const sa = G.compilePattern(s.pattern);
+                return macro.template()`withSelfDo(function (thisFacet) {
+  const _Facets = new __SYNDICATE__.Dictionary();
+  on asserted ${G.patternText(s.pattern)} => react {
+    _Facets.set([${S.commaJoin(sa.captureIds.map(t=>[t]))}], thisFacet);
+    dataflow void 0; // TODO: horrible hack to keep the facet alive if no other endpoints
+    ${s.body}
+  }
+  on retracted ${G.patternText(s.pattern)} => {
+    const _Key = [${S.commaJoin(sa.captureIds.map(t=>[t]))}];
+    _Facets.get(_Key)._stop();
+    _Facets.delete(_Key);
+  }
+});`;
+            });
         expand(
             G.typeDefinitionStatement,
             s => {
@@ -94,7 +173,8 @@ export function main(argv: string[]) {
             s => macro.template()`_stop(function (thisFacet) {${s.body}});`);
     }
 
-    console.log(S.itemText(tree, { color: true, missing: '\x1b[41m□\x1b[0m' }));
+    // console.log(`\n\n\n======================================== FINAL OUTPUT\n`);
+    console.log(S.itemText(tree));
 
     const cw = new S.CodeWriter(inputFilename);
     cw.emit(tree);

packages/core/src/index.ts

@@ -21,6 +21,7 @@ export * from 'preserves';
 export * from './runtime/randomid.js';
 export * from './runtime/assertions.js';
 export * from './runtime/bag.js';
+export * as API from './runtime/api.js';
 export * as Skeleton from './runtime/skeleton.js';
 export * from './runtime/dataspace.js';
 export * from './runtime/ground.js';

packages/core/src/runtime/api.ts

@@ -0,0 +1,3 @@
+export type NonEmptySkeleton<Shape> = { shape: Shape, members: Skeleton<Shape>[] };
+export type Skeleton<Shape> = null | NonEmptySkeleton<Shape>;
+export type Path = Array<number>;

packages/core/src/runtime/dataspace.ts

@@ -89,7 +89,7 @@ export abstract class Dataspace {
         return this.ground().backgroundTask();
     }
 
-    runTasks() { // TODO: rename?
+    runTasks(): boolean { // TODO: rename?
         this.runPendingTasks();
         this.performPendingActions();
         return this.runnable.length > 0 || this.pendingTurns.length > 0;
@@ -684,6 +684,10 @@ export class Facet {
     addChildFacet(bootProc: Script<void>) {
         this.actor.addFacet(this, bootProc, true);
     }
+
+    withSelfDo(t: Script<void>) {
+        t(this);
+    }
 }
 
 export class Endpoint {

packages/core/src/runtime/relay.ts

@@ -71,7 +71,6 @@ export class NestedDataspace extends Dataspace {
                         constPaths: h.constPaths,
                         constVals: h.constVals.map(v => (v as Record)[0]),
                         capturePaths: h.capturePaths.map(p => p.slice(1)),
-                        assertion,
                         callback
                     }
                     : {
@@ -79,7 +78,6 @@ export class NestedDataspace extends Dataspace {
                         constPaths: h.constPaths.map(p => p.slice(1)),
                         constVals: h.constVals,
                         capturePaths: h.capturePaths.map(p => p.slice(1)),
-                        assertion,
                         callback
                     });
             return { assertion, analysis };

packages/core/src/runtime/skeleton.ts

@@ -17,13 +17,15 @@
 //---------------------------------------------------------------------------
 
 import { IdentitySet } from './idcoll.js';
-import { is, Value, Record, Set, Dictionary, canonicalString, preserves } from 'preserves';
+import { is, Value, Record, Set, Dictionary, canonicalString, preserves, RecordConstructorInfo } from 'preserves';
 
 import { Bag, ChangeDescription } from './bag.js';
 import { Discard, Capture, Observe } from './assertions.js';
 
 import * as Stack from './stack.js';
 
+import { Path, NonEmptySkeleton, Skeleton } from './api.js';
+
 export enum EventType {
     ADDED = +1,
     REMOVED = -1,
@@ -33,15 +35,12 @@ export enum EventType {
 export type HandlerCallback = (eventType: EventType, bindings: Array<Value>) => void;
 
 export type Shape = string;
-export type NonEmptySkeleton = { shape: Shape, members: Skeleton[] };
+
-export type Skeleton = null | NonEmptySkeleton;
-export type Path = Array<number>;
 export interface Analysis {
-    skeleton: Skeleton;
+    skeleton: Skeleton<Shape>;
     constPaths: Array<Path>;
     constVals: Array<Value>;
     capturePaths: Array<Path>;
-    assertion: Value;
     callback?: HandlerCallback;
 }
 
@@ -159,13 +158,13 @@ class Node {
         this.continuation = continuation;
     }
 
-    extend(skeleton: Skeleton): Continuation {
+    extend(skeleton: Skeleton<Shape>): Continuation {
         const path: Path = [];
 
         function walkNode(node: Node,
                           popCount: number,
                           index: number,
-                          skeleton: Skeleton): [number, Node]
+                          skeleton: Skeleton<Shape>): [number, Node]
         {
             if (skeleton === null) {
                 return [popCount, node];
@@ -280,10 +279,13 @@ class Handler {
     }
 }
 
-function classOf(v: any): string {
+export function constructorInfoSignature(ci: RecordConstructorInfo): string {
+    return canonicalString(ci.label) + '/' + ci.arity;
+}
+
+function classOf(v: any): Shape {
     if (Record.isRecord(v)) {
-        const ci = v.getConstructorInfo();
+        return constructorInfoSignature(v.getConstructorInfo());
-        return canonicalString(ci.label) + '/' + ci.arity;
     } else if (Array.isArray(v)) {
         return '' + v.length;
     } else {
@@ -312,7 +314,7 @@ export function analyzeAssertion(a: Value): Analysis {
     const capturePaths: Path[] = [];
     const path: Path = [];
 
-    function walk(a: Value): Skeleton {
+    function walk(a: Value): Skeleton<Shape> {
         if (Capture.isClassOf(a)) {
             // NB. isUnrestricted relies on the specific order that
             // capturePaths is computed here.
@@ -329,7 +331,7 @@ export function analyzeAssertion(a: Value): Analysis {
             let aa = a as Array<Value>;
             // ^ We know this is safe because it's either Record or Array
             let arity = aa.length;
-            let result: NonEmptySkeleton = { shape: cls, members: [] };
+            let result: NonEmptySkeleton<Shape> = { shape: cls, members: [] };
             path.push(0);
             for (let i = 0; i < arity; i++) {
                 path[path.length - 1] = i;
@@ -346,7 +348,7 @@ export function analyzeAssertion(a: Value): Analysis {
 
     let skeleton = walk(a);
 
-    return { skeleton, constPaths, constVals, capturePaths, assertion: Observe(a) };
+    return { skeleton, constPaths, constVals, capturePaths };
 }
 
 export function match(p: Value, v: Value): Array<Value> | false {

packages/core/src/syntax/matcher.ts

@@ -8,6 +8,13 @@ import { List, ArrayList, atEnd, notAtEnd } from './list.js';
 export type PatternResult<T> = [T, List<Item>] | null;
 export type Pattern<T> = (i: List<Item>) => PatternResult<T>;
 
+export function match<T,F>(p: Pattern<T>, items: Items, failure: F): T | F {
+    const r = p(new ArrayList(items));
+    if (r === null) return failure;
+    if (notAtEnd(r[1])) return failure;
+    return r[0];
+}
+
 export const noItems = new ArrayList<Item>([]);
 
 export const fail: Pattern<never> = _i => null;
@@ -15,7 +22,7 @@ export function succeed<T>(t: T): Pattern<T> { return i => [t, i]; }
 
 export const discard: Pattern<void> = _i => [void 0, noItems];
 export const rest: Pattern<Items> = i => [i.toArray(), noItems];
-export const end: Pattern<void> = i => atEnd(i) ? [void 0, noItems] : null;
+export const end: Pattern<void> = i => atEnd(skipSpace(i)) ? [void 0, noItems] : null;
 export const pos: Pattern<Pos> = i =>
     notAtEnd(i)
     ? [isGroup(i.item) ? i.item.start.start : i.item.start, i]
@@ -55,7 +62,7 @@ export function seq(... patterns: Pattern<any>[]): Pattern<void> {
     };
 }
 
-export function alt(... alts: Pattern<any>[]): Pattern<any> {
+export function alt<T>(... alts: Pattern<T>[]): Pattern<T> {
     return i => {
         for (const a of alts) {
             const r = a(i);
@@ -65,7 +72,7 @@ export function alt(... alts: Pattern<any>[]): Pattern<any> {
     };
 }
 
-export function scope<I, T extends I, R>(pf: (scope: T) => Pattern<R>): Pattern<T> {
+export function scope<T>(pf: (scope: T) => Pattern<any>): Pattern<T> {
     return i => {
         const scope = Object.create(null);
         const r = pf(scope)(i);
@@ -107,8 +114,17 @@ export function map<T, R>(p: Pattern<T>, f: (t: T) => R): Pattern<R> {
     };
 }
 
+export function mapm<T, R>(p: Pattern<T>, f: (t: T) => Pattern<R>): Pattern<R> {
+    return i => {
+        const r = p(i);
+        if (r === null) return null;
+        return f(r[0])(r[1]);
+    };
+}
+
 export interface ItemOptions {
     skipSpace?: boolean, // default: true
+    advance?: boolean, // default: true
 }
 
 export interface GroupOptions extends ItemOptions {
@@ -127,18 +143,22 @@ export function group<T>(opener: string, items: Pattern<T>, options: GroupOption
         const r = items(new ArrayList(i.item.items));
         if (r === null) return null;
         if (!atEnd(r[1])) return null;
-        return [r[0], i.next];
+        return [r[0], (options.advance ?? true) ? i.next : i];
     };
 }
 
-export function atom(text?: string | undefined, options: TokenOptions = {}): Pattern<Token> {
+export function atomString<T extends string>(text: T, options: TokenOptions = {}): Pattern<T> {
+    return map(atom(text, options), t => text);
+}
+
+export function atom(text?: string, options: TokenOptions = {}): Pattern<Token> {
     return i => {
         if (options.skipSpace ?? true) i = skipSpace(i);
         if (!notAtEnd(i)) return null;
         if (!isToken(i.item)) return null;
         if (i.item.type !== (options.tokenType ?? TokenType.ATOM)) return null;
         if (text !== void 0 && i.item.text !== text) return null;
-        return [i.item, i.next];
+        return [i.item, (options.advance ?? true) ? i.next : i];
     }
 }
 
@@ -146,7 +166,7 @@ export function anything(options: ItemOptions = {}): Pattern<Item> {
     return i => {
         if (options.skipSpace ?? true) i = skipSpace(i);
         if (!notAtEnd(i)) return null;
-        return [i.item, i.next];
+        return [i.item, (options.advance ?? true) ? i.next : i];
     };
 }
 
@@ -164,6 +184,29 @@ export function upTo(p: Pattern<any>): Pattern<Items> {
     };
 }
 
+export function separatedBy<T>(itemPattern: Pattern<T>, separator: Pattern<any>): Pattern<T[]> {
+    return i => {
+        const acc: T[] = [];
+        if (end(i) !== null) return [acc, noItems];
+        while (true) {
+            {
+                const r = itemPattern(i);
+                if (r === null) return null;
+                acc.push(r[0]);
+                i = r[1];
+            }
+            {
+                const r = separator(i);
+                if (r === null) {
+                    if (end(i) !== null) return [acc, noItems];
+                    return null;
+                }
+                i = r[1];
+            }
+        }
+    };
+}
+
 export interface RepeatOptions {
     min?: number;
     max?: number;

packages/core/src/syntax/reader.ts

@@ -1,5 +1,6 @@
 import { TokenType, Token, Group, Item, Items } from './tokens.js';
-import { Scanner } from './scanner.js';
+import { Pos, startPos } from './position.js';
+import { Scanner, StringScanner } from './scanner.js';
 
 function matchingParen(c: string): string | null {
     switch (c) {
@@ -122,3 +123,17 @@ export class LaxReader implements IterableIterator<Item> {
         }
     }
 }
+
+export interface LaxReadOptions {
+    start?: Pos,
+    name?: string,
+    extraDelimiters?: string,
+}
+
+export function laxRead(source: string, options: LaxReadOptions = {}): Items {
+    const start = options.start ?? startPos(options.name ?? null);
+    const scanner = new StringScanner(start, source);
+    if (options.extraDelimiters) scanner.addDelimiters(options.extraDelimiters);
+    const reader = new LaxReader(scanner);
+    return reader.readToEnd();
+}

packages/core/src/syntax/scanner.ts

@@ -8,7 +8,7 @@ export abstract class Scanner implements IterableIterator<Token> {
     delimiters = ' \t\n\r\'"`.,;()[]{}/';
 
     constructor(pos: Pos) {
-        this.pos = pos;
+        this.pos = { ... pos };
     }
 
     [Symbol.iterator](): IterableIterator<Token> {

packages/core/src/syntax/template.ts

@@ -1,7 +1,6 @@
 import { Items, TokenType } from './tokens.js';
 import { Pos, startPos } from './position.js';
-import { StringScanner } from './scanner.js';
+import { laxRead } from './reader.js';
-import { LaxReader } from './reader.js';
 import * as M from './matcher.js';
 
 const substPat = M.scope((o: { pos: Pos }) =>
@@ -11,7 +10,7 @@ const substPat = M.scope((o: { pos: Pos }) =>
 export type Substitution = Items | string;
 
 function toItems(s: Substitution, pos: Pos): Items {
-    return typeof s === 'string' ? [{ type: TokenType.ATOM, text: s, start: pos, end: pos }] : s;
+    return typeof s === 'string' ? laxRead(s) : s;
 }
 
 export class Templates {
@@ -32,10 +31,10 @@ export class Templates {
                 }
                 this.sources[start.name] = source;
             }
-            const reader = new LaxReader(new StringScanner(start, source));
-            reader.scanner.addDelimiters('$');
             let i = 0;
-            return M.replace(reader.readToEnd(), substPat, sub => toItems(vars[i++], sub.pos));
+            return M.replace(laxRead(source, { start, extraDelimiters: '$' }),
+                             substPat,
+                             sub => toItems(vars[i++], sub.pos));
         };
     }
 
@@ -54,6 +53,12 @@ export function joinItems(itemss: Items[], separator0: Substitution): Items {
     return acc;
 }
 
+export function commaJoin(itemss: Items[]): Items {
+    return joinItems(itemss, ', ');
+}
+
+export const anonymousTemplate = (new Templates()).template();
+
 // const lib = new Templates();
 // const t = (o: {xs: Items}) => lib.template('testTemplate')`YOYOYOYO ${o.xs}><`;
 // console.log(t({xs: lib.template()`hello there`}));

packages/core/src/syntax/tokens.ts

@@ -60,12 +60,25 @@ export type ItemTextOptions = {
     color?: boolean,
 };
 
-export function itemText(i: Items, options: ItemTextOptions = {}): string {
+export function foldItems<T>(i: Items,
-    const walkItems = (i: Items): string => i.map(walk).join('');
+                             fToken: (t: Token) => T,
-    const walk = (i: Item): string => {
+                             fGroup: (start: Token, end: Token | null, t: T, k: (t: Token) => T) => T,
+                             fItems: (ts: T[]) => T): T
+{
+    const walk = (i: Item): T => {
         if (isGroup(i)) {
-            return walk(i.start) + walkItems(i.items) + (i.end ? walk(i.end) : options.missing ?? '');
+            return fGroup(i.start, i.end, fItems(i.items.map(walk)), walk);
         } else {
+            return fToken(i);
+        }
+    };
+    return fItems(i.map(walk));
+}
+
+export function itemText(items: Items, options: ItemTextOptions = {}): string {
+    return foldItems(
+        items,
+        i => {
             if (options.color ?? false) {
                 switch (i.type) {
                     case TokenType.SPACE:
@@ -79,7 +92,7 @@ export function itemText(i: Items, options: ItemTextOptions = {}): string {
             } else {
                 return i.text;
             }
-        }
+        },
-    };
+        (start, end, inner, k) => k(start) + inner + (end ? k(end) : options.missing ?? ''),
-    return walkItems(i);
+        strs => strs.join(''));
 }

todo/syntax-playground/src/avahipublish.js

@@ -10,6 +10,7 @@ spawn named 'test' {
 
   during M.Discovered(M.Service($name, '_syndicate+testing._tcp'),
                       $host, $port, _, $addr, "IPv4", $ifName)
+    =>
   {
     on start { this.count++; console.log('+', name, host, port, addr, ifName); }
     on stop  { this.count--; console.log('-', name, host, port, addr, ifName); }

todo/syntax-playground/src/chatclient.js

@@ -24,18 +24,18 @@ const stdin = genUuid('stdin');
 const stdout = genUuid('stdout');
 spawn named 'stdioServer' {
   during Observe(S.Stream(stdin, S.Readable()))
-    spawn S.readableStreamBehaviour(stdin, process.stdin);
+    => spawn S.readableStreamBehaviour(stdin, process.stdin);
   during Observe(S.Stream(stdout, S.Writable()))
-    spawn S.writableStreamBehaviour(stdout, process.stdout);
+    => spawn S.writableStreamBehaviour(stdout, process.stdout);
 }
 
 spawn named 'chatclient' {
   const id = genUuid('tcpconn');
   assert S.Stream(id, S.Outgoing(S.TcpAddress('localhost', 5999)));
-  stop on message S.Stream(id, S.Rejected($err)) {
+  stop on message S.Stream(id, S.Rejected($err)) => {
     console.error('Connection rejected', err);
   }
-  stop on message S.Stream(id, S.Accepted()) {
+  stop on message S.Stream(id, S.Accepted()) => {
     react {
       stop on retracted S.Stream(id, S.Duplex());
       stop on retracted S.Stream(stdin, S.Readable());
@@ -44,10 +44,10 @@ spawn named 'chatclient' {
       assert S.Stream(stdin, S.BackPressure(id));
       assert S.Stream(id, S.BackPressure(stdout));
 
-      on message S.Stream(stdin, S.Line($line)) {
+      on message S.Stream(stdin, S.Line($line)) => {
         send S.Stream(id, S.Push(line.fromUtf8() + '\n', false));
       }
-      on message S.Stream(id, S.Line($line)) {
+      on message S.Stream(id, S.Line($line)) => {
         send S.Stream(stdout, S.Push(line.fromUtf8() + '\n', false));
       }
     }

todo/syntax-playground/src/index.js

@@ -44,9 +44,9 @@ spawn named 'view' {
     return <td>{text}</td>;
   }
 
-  on message SetSortColumn($c) { this.orderColumn = c; }
+  on message SetSortColumn($c) => this.orderColumn = c;
 
-  during Person($id, $firstName, $lastName, $address, $age) {
+  during Person($id, $firstName, $lastName, $address, $age) => {
     assert ui.context(id)
       .html('table#the-table tbody',
             <tr>{[id, firstName, lastName, address, age].map(cell)}</tr>,
@@ -55,7 +55,7 @@ spawn named 'view' {
 }
 
 spawn named 'controller' {
-  on message UI.GlobalEvent('table#the-table th', 'click', $e) {
+  on message UI.GlobalEvent('table#the-table th', 'click', $e) => {
     send SetSortColumn(JSON.parse(e.target.dataset.column));
   }
 }
@@ -64,7 +64,7 @@ spawn named 'alerter' {
   let ui = new UI.Anchor();
   assert ui.html('#extra', <button>Click me</button>);
 
-  on message UI.UIEvent(ui.fragmentId, '.', 'click', $e) {
+  on message UI.UIEvent(ui.fragmentId, '.', 'click', $e) => {
     alert("Hello!");
   }
 }