From 690ac12cc031906dfc8db40d38da90eb8a18ae27 Mon Sep 17 00:00:00 2001 From: Tony Garnock-Jones Date: Mon, 25 Jan 2021 22:16:52 +0100 Subject: [PATCH] Many fixes to compiler; watchable syndicate-tsc --- packages/compiler/src/compiler/codegen.ts | 100 +++++-- packages/compiler/src/compiler/grammar.ts | 35 ++- packages/compiler/src/syntax/matcher.ts | 2 +- packages/compiler/src/syntax/template.ts | 28 +- packages/core/src/runtime/dataspace.ts | 14 +- packages/ts-plugin/src/index.ts | 91 +++++- packages/tsc/src/tsc.ts | 337 ++++++++++++++-------- 7 files changed, 420 insertions(+), 187 deletions(-) diff --git a/packages/compiler/src/compiler/codegen.ts b/packages/compiler/src/compiler/codegen.ts index 93d95fe..49536f8 100644 --- a/packages/compiler/src/compiler/codegen.ts +++ b/packages/compiler/src/compiler/codegen.ts @@ -1,6 +1,6 @@ import { isToken, isTokenType, replace, commaJoin, startPos, fixPos, joinItems, - anonymousTemplate, laxRead, itemText, + laxRead, itemText, match, Items, Pattern, Templates, Substitution, TokenType, SourceMap, CodeWriter, TemplateFunction, Token, SpanIndex, @@ -16,6 +16,7 @@ import { compilePattern, patternText, + instantiatePatternToPattern, } from './grammar.js'; import { BootProc, @@ -64,6 +65,7 @@ export class ExpansionContext { hasBootProc: boolean = false; readonly typescript: boolean; _collectedFields: FacetFields | null = null; + nextIdNumber = 0; constructor(moduleType: ModuleType, typescript: boolean) @@ -73,8 +75,8 @@ export class ExpansionContext { this.typescript = typescript; } - argDecl(name: Substitution, type: Substitution): Substitution { - return this.typescript ? anonymousTemplate`${name}: ${type}` : name; + quasiRandomId(): string { + return '__SYNDICATE__id_' + (this.nextIdNumber++); } get collectedFields(): FacetFields { @@ -110,10 +112,16 @@ function facetFieldObjectType(t: TemplateFunction, fs: FacetFields): Substitutio return t`{${commaJoin(fs.map(formatBinder))}}`; } -function binderTypeGuard(t: TemplateFunction): (binder: Binder) => Items { - return (binder) => { +function binderTypeGuard(t: TemplateFunction): (binder: Binder, index: number) => Items { + return (binder, index) => { + if (binder.id.text[0] === '_') { + return t`${`/* Ignoring underscore-prefixed binder ${binder.id.text} */`}`; + } + + const raw = t`__vs[${''+index}]`; + const bind = t`const ${[binder.id]} = ${raw};`; if (binder.type === void 0) { - return t`${`/* ${binder.id.text} is a plain Value */`}`; + return bind; } else { const typeText = itemText(binder.type); switch (typeText) { @@ -121,7 +129,9 @@ function binderTypeGuard(t: TemplateFunction): (binder: Binder) => Items { case 'string': case 'number': case 'symbol': - return t`if (typeof (${[binder.id]}) !== ${JSON.stringify(typeText)}) return;\n`; + return t`if (typeof (${raw}) !== ${JSON.stringify(typeText)}) return;\n${bind}`; + case 'any': + return bind; default: throw new Error(`Unhandled binding type: ${JSON.stringify(typeText)}`); } @@ -130,7 +140,7 @@ function binderTypeGuard(t: TemplateFunction): (binder: Binder) => Items { } export function expand(tree: Items, ctx: ExpansionContext): Items { - const macro = new Templates(); + const macro = new Templates(undefined, { extraDelimiters: ':' }); function terminalWrap(t: TemplateFunction, isTerminal: boolean, body: Statement): Statement { if (isTerminal) { @@ -152,25 +162,59 @@ export function expand(tree: Items, ctx: ExpansionContext): Items { const maybeWalk = (tree?: Items) : Items | undefined => (tree === void 0) ? tree : walk(tree); xf(ctx.parser.duringStatement, (s, t) => { - // TODO: spawn during - const sa = compilePattern(s.pattern); - const body = ctx.withCollectedFields(s.facetFields, () => walk(s.body)); - return t`withSelfDo(function (thisFacet) { - const _Facets = new __SYNDICATE__.Dictionary(); - on asserted ${patternText(s.pattern)} => react { - _Facets.set([${commaJoin(sa.captureBinders.map(t=>[t.id]))}], thisFacet); - dataflow void 0; // TODO: horrible hack to keep the facet alive if no other endpoints - ${body} - } - on retracted ${patternText(s.pattern)} => { - const _Key = [${commaJoin(sa.captureBinders.map(t=>[t.id]))}]; - _Facets.get(_Key)._stop(); - _Facets.delete(_Key); - } - });`; + let spawn0 = match(ctx.parser.spawn, s.body, null); + if (spawn0 !== null) { + const spawn = spawn0; + const id = ctx.quasiRandomId(); + const instantiated = patternText(instantiatePatternToPattern(s.pattern)); + return t`on asserted ${patternText(s.pattern)} => { + const ${id} = __SYNDICATE__.genUuid(); + const ${id}_inst = __SYNDICATE__.Instance(${id}); + react { + stop on asserted ${id}_inst => react { + stop on retracted ${id}_inst; + stop on retracted :snapshot ${instantiated}; + } + stop on retracted :snapshot ${instantiated} => react { + stop on asserted ${id}_inst; + } + } + spawn + ${spawn.isDataspace ? 'dataspace' : []} + ${spawn.name === void 0 ? [] : t`named ${spawn.name}`} + :asserting ${id}_inst + ${joinItems(spawn.initialAssertions.map(e => t`:asserting ${e}`), ' ')} + ${joinItems(spawn.parentBinders.map((b, i) => { + const init = spawn.parentInits[i]; + return t`:let ${[b.id]}${b.type === void 0 ? [] : t`: ${b.type}`} = ${init}`; + }), ' ')} + { + assert ${id}_inst; + stop on retracted __SYNDICATE__.Observe(${id}_inst); + ${spawn.body} + } + }`; + } else { + const sa = compilePattern(s.pattern); + const body = ctx.withCollectedFields(s.facetFields, () => walk(s.body)); + return t`withSelfDo(function (thisFacet) { + const _Facets = new __SYNDICATE__.Dictionary>(); + on asserted ${patternText(s.pattern)} => react { + _Facets.set([${commaJoin(sa.captureBinders.map(t=>[t.id]))}], thisFacet); + dataflow { } // TODO: horrible hack to keep the facet alive if no other endpoints + ${body} + } + on retracted ${patternText(s.pattern)} => { + const _Key = [${commaJoin(sa.captureBinders.map(t=>[t.id]))}]; + _Facets.get(_Key)?._stop(); + _Facets.delete(_Key); + } + });`; + } }); xf(ctx.parser.spawn, (s, t) => { + // TODO: parentBinders, parentInits let body = ctx.withCollectedFields(s.facetFields, () => walk(s.body)); let proc = t`function (thisFacet) {${body}}`; if (s.isDataspace) proc = t`__SYNDICATE__.inNestedDataspace(${proc})`; @@ -219,8 +263,6 @@ export function expand(tree: Items, ctx: ExpansionContext): Items { 'retracted': 'REMOVED', 'message': 'MESSAGE', })[s.triggerType]; - const destructure = sa.captureBinders.length === 0 ? '__vs' - : t`[${commaJoin(sa.captureBinders.map(i=>[i.id]))}]`; return t`addEndpoint(thisFacet => ({ assertion: __SYNDICATE__.Observe(${walk(sa.assertion)}), analysis: { @@ -228,7 +270,7 @@ export function expand(tree: Items, ctx: ExpansionContext): Items { constPaths: ${JSON.stringify(sa.constPaths)}, constVals: [${commaJoin(sa.constVals.map(walk))}], capturePaths: ${JSON.stringify(sa.capturePaths)}, - callback: thisFacet.wrap((thisFacet, __Evt, ${destructure}) => { + callback: thisFacet.wrap((thisFacet, __Evt, __vs: Array<__SYNDICATE__.Value>) => { if (__Evt === __SYNDICATE__.Skeleton.EventType.${expectedEvt}) { ${ctx.typescript ? joinItems(sa.captureBinders.map(binderTypeGuard(t)), '\n') : ''} thisFacet.scheduleScript(() => {${terminalWrap(t, s.terminal, walk(s.body))}}); @@ -251,7 +293,7 @@ ${ctx.typescript ? joinItems(sa.captureBinders.map(binderTypeGuard(t)), '\n') : xf(ctx.parser.reactStatement, (s, t) => { const body = ctx.withCollectedFields(s.facetFields, () => walk(s.body)); const fieldTypeParam = ctx.typescript - ? t`<${facetFieldObjectType(t, ctx.collectedFields)}, ${facetFieldObjectType(t, s.facetFields)}>` + ? t`<${facetFieldObjectType(t, s.facetFields)}>` : ''; return t`addChildFacet${fieldTypeParam}(function (thisFacet) {${body}});`; }); @@ -302,7 +344,7 @@ export function compile(options: CompileOptions): CompilerOutput { let tree = stripShebang(laxRead(source, { start, extraDelimiters: ':' })); const end = tree.length > 0 ? tree[tree.length - 1].end : start; - let macro = new Templates(); + const macro = new Templates(undefined, { extraDelimiters: ':' }); const ctx = new ExpansionContext(moduleType, typescript); diff --git a/packages/compiler/src/compiler/grammar.ts b/packages/compiler/src/compiler/grammar.ts index 1cb54e9..6f4a8c0 100644 --- a/packages/compiler/src/compiler/grammar.ts +++ b/packages/compiler/src/compiler/grammar.ts @@ -148,6 +148,10 @@ export interface StaticAnalysis { //--------------------------------------------------------------------------- // Parsers +function kw(text: string): Pattern { + return value(o => seq(atom(':'), bind(o, 'value', atom(text, { skipSpace: false })))); +} + export class SyndicateParser { block(acc?: Items): Pattern { return group('{', map(rest, items => (acc?.push(... items), items))); @@ -184,7 +188,7 @@ export class SyndicateParser { }; } - readonly headerExpr = this.expr(atom(':asserting'), atom(':let')); + readonly headerExpr = this.expr(kw('asserting'), kw('let')); // Principal: Facet readonly spawn: Pattern = @@ -199,10 +203,10 @@ export class SyndicateParser { option(seq(atom('dataspace'), exec(() => o.isDataspace = true))), option(seq(atom('named'), bind(o, 'name', this.headerExpr))), - repeat(alt(seq(atom(':asserting'), + repeat(alt(seq(kw('asserting'), map(this.headerExpr, e => o.initialAssertions.push(e))), map(scope((l: { b: Binder, init: Expr }) => - seq(atom(':let'), + seq(kw('let'), bind(l, 'b', this.binder), atom('='), bind(l, 'init', this.headerExpr))), @@ -227,7 +231,7 @@ export class SyndicateParser { this.facetAction(o => { o.isDynamic = true; return seq(atom('assert'), - option(map(atom(':snapshot'), _ => o.isDynamic = false)), + option(map(kw('snapshot'), _ => o.isDynamic = false)), bind(o, 'template', this.expr(seq(atom('when'), group('(', discard)))), option(seq(atom('when'), group('(', bind(o, 'test', this.expr())))), this.statementBoundary); @@ -269,7 +273,7 @@ export class SyndicateParser { alt(atomString('asserted'), atomString('retracted'), atomString('message'))), - option(map(atom(':snapshot'), _ => o.isDynamic = false)), + option(map(kw('snapshot'), _ => o.isDynamic = false)), bind(o as AssertionEventEndpointStatement, 'pattern', this.valuePattern(atom('=>'))), this.mandatoryIfNotTerminal( @@ -423,7 +427,7 @@ export class SyndicateParser { }); } }), - map(this.expr(), e => ({ type: 'PConstant', value: e })) + map(this.expr(... extraStops), e => ({ type: 'PConstant', value: e })) ); } } @@ -459,6 +463,25 @@ export function patternText(p: ValuePattern): Items { } } +export function instantiatePatternToPattern(p: ValuePattern): ValuePattern { + switch (p.type) { + case 'PDiscard': return p; + case 'PConstant': return p; + case 'PCapture': return { type: 'PConstant', value: [p.binder.id] }; + case 'PArray': + return { + type: 'PArray', + elements: p.elements.map(instantiatePatternToPattern), + }; + case 'PConstructor': + return { + type: 'PConstructor', + ctor: p.ctor, + arguments: p.arguments.map(instantiatePatternToPattern), + }; + } +} + const eDiscard: Expr = template`(__SYNDICATE__.Discard._instance)`; const eCapture = (e: Expr): Expr => template`(__SYNDICATE__.Capture(${e}))`; diff --git a/packages/compiler/src/syntax/matcher.ts b/packages/compiler/src/syntax/matcher.ts index ee3efc3..4620618 100644 --- a/packages/compiler/src/syntax/matcher.ts +++ b/packages/compiler/src/syntax/matcher.ts @@ -14,7 +14,7 @@ export type Pattern = (i: List) => PatternResult; export function match(p: Pattern, items: Items, failure: F): T | F { const r = p(new ArrayList(items)); if (r === null) return failure; - if (notAtEnd(r[1])) return failure; + if (notAtEnd(skipSpace(r[1]))) return failure; return r[0]; } diff --git a/packages/compiler/src/syntax/template.ts b/packages/compiler/src/syntax/template.ts index 772888f..d5e8816 100644 --- a/packages/compiler/src/syntax/template.ts +++ b/packages/compiler/src/syntax/template.ts @@ -1,6 +1,6 @@ import { Items } from './tokens.js'; import { Pos, startPos } from './position.js'; -import { laxRead } from './reader.js'; +import { laxRead, LaxReadOptions } from './reader.js'; import * as M from './matcher.js'; const substPat = M.scope((o: { pos: Pos }) => @@ -9,8 +9,8 @@ const substPat = M.scope((o: { pos: Pos }) => export type Substitution = Items | string; -function toItems(s: Substitution, pos: Pos): Items { - return typeof s === 'string' ? laxRead(s, { start: pos, synthetic: true }) : s; +function toItems(readOptions: LaxReadOptions, s: Substitution, pos: Pos): Items { + return typeof s === 'string' ? laxRead(s, { ... readOptions, start: pos, synthetic: true }) : s; } export type TemplateFunction = (consts: TemplateStringsArray, ... vars: Substitution[]) => Items; @@ -19,9 +19,11 @@ export class Templates { readonly sources: { [name: string]: string } = {}; readonly defaultPos: Pos; recordSources = false; + readonly readOptions: LaxReadOptions; - constructor(defaultPos: Pos = startPos(null)) { + constructor(defaultPos: Pos = startPos(null), readOptions: LaxReadOptions = {}) { this.defaultPos = defaultPos; + this.readOptions = readOptions; } template(start0: Pos | string = this.defaultPos): TemplateFunction { @@ -42,9 +44,14 @@ export class Templates { } } let i = 0; - return M.replace(laxRead(source, { start, extraDelimiters: '$', synthetic: true }), + return M.replace(laxRead(source, { ... this.readOptions, + start, + extraDelimiters: + (this.readOptions.extraDelimiters ?? '') + '$', + synthetic: true, + }), substPat, - sub => toItems(vars[i++], sub.pos)); + sub => toItems(this.readOptions, vars[i++], sub.pos)); }; } @@ -53,10 +60,13 @@ export class Templates { } } -export function joinItems(itemss: Items[], separator0: Substitution = ''): Items { +export function joinItems(itemss: Items[], + separator0: Substitution = '', + readOptions: LaxReadOptions = {}): Items +{ if (itemss.length === 0) return []; - const separator = toItems(separator0, startPos(null)); - const acc = itemss[0]; + const separator = toItems(readOptions, separator0, startPos(null)); + const acc: Items = [... itemss[0]]; for (let i = 1; i < itemss.length; i++) { acc.push(... separator, ... itemss[i]); } diff --git a/packages/core/src/runtime/dataspace.ts b/packages/core/src/runtime/dataspace.ts index 1f42ef1..782a9c6 100644 --- a/packages/core/src/runtime/dataspace.ts +++ b/packages/core/src/runtime/dataspace.ts @@ -139,10 +139,10 @@ export abstract class Dataspace { let ac = new Actor(this, name, initialAssertions, parentActor?.id); // debug('Spawn', ac && ac.toString()); this.applyPatch(ac, ac.adhocAssertions); - ac.addFacet(null, systemFacet => { + ac.addFacet<{}, {}>(null, systemFacet => { // Root facet is a dummy "system" facet that exists to hold // one-or-more "user" "root" facets. - ac.addFacet(systemFacet, bootProc); + ac.addFacet<{}, SpawnFields>(systemFacet, bootProc); // ^ The "true root", user-visible facet. initialAssertions.forEach((a) => { ac.adhocRetract(a); }); }); @@ -259,15 +259,15 @@ export class Actor { this.pendingTasks[priority].push(task); } - addFacet( + addFacet( parentFacet: Facet | null, - bootProc: Script, + bootProc: Script, checkInScript: boolean = false) { if (checkInScript && parentFacet && !parentFacet.inScript) { throw new Error("Cannot add facet outside script; are you missing a `react { ... }`?"); } - let f = new Facet(this, parentFacet); + let f = new Facet(this, parentFacet); f.invokeScript(f => f.withNonScriptContext(() => bootProc.call(f.fields, f))); this.scheduleTask(() => { if ((parentFacet && !parentFacet.isLive) || f.isInert()) { @@ -723,8 +723,8 @@ export class Facet { // delete obj[prop]; // } - addChildFacet(bootProc: Script) { - this.actor.addFacet(this, bootProc, true); + addChildFacet(bootProc: Script) { + this.actor.addFacet(this, bootProc, true); } withSelfDo(t: Script) { diff --git a/packages/ts-plugin/src/index.ts b/packages/ts-plugin/src/index.ts index 8683038..92a9c09 100644 --- a/packages/ts-plugin/src/index.ts +++ b/packages/ts-plugin/src/index.ts @@ -84,6 +84,10 @@ const boot: tslib.server.PluginModuleFactory = ({ typescript: ts }) => { get targetStart(): number { return this.target.firstItem + this.target.offset; } + + get targetEnd(): number { + return this.target.lastItem + this.target.offset; + } } function withFileName(fileName: string | undefined, @@ -96,17 +100,26 @@ const boot: tslib.server.PluginModuleFactory = ({ typescript: ts }) => { return k(new Fixup(info)); } + function withPositions(fileName: string, + positions: Array, + kNoInfo: () => T, + kNoPosition: () => T, + k: (f: Array) => T): T + { + return withFileName(fileName, kNoInfo, (fx) => { + const t = positions.map(p => fx.info.sourceToTargetMap.get(p)); + if (t.some(p => p === null)) return kNoPosition(); + return k(t.map(p => new PositionFixup(fx.info, p!))); + }); + } + function withPosition(fileName: string, position: number, kNoInfo: () => T, kNoPosition: () => T, k: (f: PositionFixup) => T): T { - return withFileName(fileName, kNoInfo, (fx) => { - const t = fx.info.sourceToTargetMap.get(position); - if (t === null) return kNoPosition(); - return k(new PositionFixup(fx.info, t)); - }); + return withPositions(fileName, [position], kNoInfo, kNoPosition, ([f]) => k(f)); } function hookHost(host0: ts.CompilerHost | undefined, @@ -487,15 +500,33 @@ const boot: tslib.server.PluginModuleFactory = ({ typescript: ts }) => { } getFormattingEditsForRange(fileName: string, start: number, end: number, options: ts.FormatCodeOptions | ts.FormatCodeSettings): ts.TextChange[] { - throw new Error('Method not implemented.'); + return withPositions( + fileName, [start, end], + () => this.inner.getFormattingEditsForRange(fileName, start, end, options), + () => [], + ([fixStart, fixEnd]) => { + const edits = this.inner.getFormattingEditsForRange(fileName, fixStart.targetStart, fixEnd.targetEnd, options); + edits.forEach(e => fixStart.span(e.span)); + return edits; + }); } getFormattingEditsForDocument(fileName: string, options: ts.FormatCodeOptions | ts.FormatCodeSettings): ts.TextChange[] { - throw new Error('Method not implemented.'); + const edits = this.inner.getFormattingEditsForDocument(fileName, options); + withFileName(fileName, () => void 0, (fixup) => edits.forEach(e => fixup.span(e.span))); + return edits; } getFormattingEditsAfterKeystroke(fileName: string, position: number, key: string, options: ts.FormatCodeOptions | ts.FormatCodeSettings): ts.TextChange[] { - throw new Error('Method not implemented.'); + return withPosition( + fileName, position, + () => this.inner.getFormattingEditsAfterKeystroke(fileName, position, key, options), + () => [], + (fixup) => { + const edits = this.inner.getFormattingEditsAfterKeystroke(fileName, fixup.targetStart, key, options); + edits.forEach(e => fixup.span(e.span)); + return edits; + }); } getDocCommentTemplateAtPosition(fileName: string, position: number): ts.TextInsertion | undefined { @@ -515,11 +546,40 @@ const boot: tslib.server.PluginModuleFactory = ({ typescript: ts }) => { } getCodeFixesAtPosition(fileName: string, start: number, end: number, errorCodes: readonly number[], formatOptions: ts.FormatCodeSettings, preferences: ts.UserPreferences): readonly ts.CodeFixAction[] { - throw new Error('Method not implemented.'); + return withPositions( + fileName, [start, end], + () => this.inner.getCodeFixesAtPosition(fileName, + start, + end, + errorCodes, + formatOptions, + preferences), + () => [], + ([fixStart, fixEnd]) => { + const fixes = this.inner.getCodeFixesAtPosition(fileName, + fixStart.targetStart, + fixEnd.targetEnd, + errorCodes, + formatOptions, + preferences); + fixes.forEach(f => + f.changes.forEach(change => + change.textChanges.forEach(c => + fixStart.span(c.span)))); + return fixes; + }); } getCombinedCodeFix(scope: ts.CombinedCodeFixScope, fixId: {}, formatOptions: ts.FormatCodeSettings, preferences: ts.UserPreferences): ts.CombinedCodeActions { - throw new Error('Method not implemented.'); + const actions = this.inner.getCombinedCodeFix(scope, fixId, formatOptions, preferences); + actions.changes.forEach(change => { + const info = getInfo(change.fileName); + if (info !== void 0) { + const fixup = new Fixup(info); + change.textChanges.forEach(c => fixup.span(c.span)); + } + }); + return actions; } applyCodeActionCommand(action: ts.InstallPackageAction, formatSettings?: ts.FormatCodeSettings): Promise; @@ -608,17 +668,22 @@ const boot: tslib.server.PluginModuleFactory = ({ typescript: ts }) => { } } + function finalSlash(s: string): string { + if (s[s.length - 1] !== '/') s = s + '/'; + return s; + } + class SyndicatePlugin implements ts.server.PluginModule { create(createInfo: ts.server.PluginCreateInfo): ts.LanguageService { const options = createInfo.project.getCompilerOptions(); if (options.rootDir !== void 0) { - syndicateRootDirs.add(options.rootDir); + syndicateRootDirs.add(finalSlash(options.rootDir)); } if (options.rootDirs !== void 0) { - options.rootDirs.forEach(d => syndicateRootDirs.add(d)); + options.rootDirs.forEach(d => syndicateRootDirs.add(finalSlash(d))); } if (options.rootDir === void 0 && options.rootDirs === void 0) { - syndicateRootDirs.add(path.resolve('.')); + syndicateRootDirs.add(finalSlash(path.resolve('.'))); } return new SyndicateLanguageService(createInfo.languageService); } diff --git a/packages/tsc/src/tsc.ts b/packages/tsc/src/tsc.ts index 5934906..d948525 100644 --- a/packages/tsc/src/tsc.ts +++ b/packages/tsc/src/tsc.ts @@ -2,12 +2,17 @@ import yargs from 'yargs/yargs'; import ts from 'typescript'; import crypto from 'crypto'; +import fs from 'fs'; +import path from 'path'; import { compile } from '@syndicate-lang/compiler'; import { SpanIndex, Token } from '@syndicate-lang/compiler/lib/syntax'; export type CommandLineArguments = { verbose: boolean; + intermediateDirectory?: string; + watch: boolean; + clear: boolean; }; interface SyndicateInfo { @@ -17,118 +22,6 @@ interface SyndicateInfo { sourceToTargetMap: SpanIndex; } -const syndicateInfo: Map = new Map(); - -function createProgram(rootNames: readonly string[] | undefined, - options: ts.CompilerOptions | undefined, - host?: ts.CompilerHost, - oldProgram?: ts.EmitAndSemanticDiagnosticsBuilderProgram, - configFileParsingDiagnostics?: readonly ts.Diagnostic[], - projectReferences?: readonly ts.ProjectReference[]) -: ts.EmitAndSemanticDiagnosticsBuilderProgram -{ - if (host === void 0) { - throw new Error("CompilerHost not present - cannot continue"); - } - - if (rootNames === void 0) { - console.warn("No Syndicate source files to compile"); - } - - const oldGetSourceFile = host.getSourceFile; - - host.getSourceFile = (fileName: string, - languageVersion: ts.ScriptTarget, - onError?: ((message: string) => void), - shouldCreateNewSourceFile?: boolean): ts.SourceFile | undefined => { - if ((rootNames?.indexOf(fileName) ?? -1) !== -1) { - try { - const inputText = host.readFile(fileName); - if (inputText === void 0) { - onError?.(`Could not read input file ${fileName}`); - return undefined; - } - const { text: expandedText, targetToSourceMap, sourceToTargetMap } = compile({ - source: inputText, - name: fileName, - typescript: true, - }); - syndicateInfo.set(fileName, { - originalSource: inputText, - languageVersion, - targetToSourceMap, - sourceToTargetMap, - }); - const sf = ts.createSourceFile(fileName, expandedText, languageVersion, true); - (sf as any).version = crypto.createHash('sha256').update(expandedText).digest('hex'); - return sf; - } catch (e) { - console.error(e); - onError?.(e.message); - return undefined; - } - } else { - return oldGetSourceFile(fileName, languageVersion, onError, shouldCreateNewSourceFile); - } - }; - - return ts.createEmitAndSemanticDiagnosticsBuilderProgram(rootNames, - options, - host, - oldProgram, - configFileParsingDiagnostics, - projectReferences); -} - -export function fixSourceMap(_ctx: ts.TransformationContext): ts.Transformer { - return sf => { - const fileName = sf.fileName; - const info = syndicateInfo.get(fileName); - if (info === void 0) throw new Error("No Syndicate info available for " + fileName); - const targetToSourceMap = info.targetToSourceMap; - const syndicateSource = ts.createSourceMapSource(fileName, info.originalSource); - - function adjustSourceMap(n: ts.Node) { - const ps = targetToSourceMap.get(n.pos); - const pe = targetToSourceMap.get(n.end); - if (ps !== null && pe !== null) { - ts.setSourceMapRange(n, { - pos: ps.firstItem.start.pos + ps.offset, - end: pe.lastItem.start.pos + pe.offset, - source: syndicateSource, - }); - } - ts.forEachChild(n, adjustSourceMap); - } - - adjustSourceMap(sf); - - return sf; - }; -} - -const syntheticSourceFiles = new Map(); -function fixupDiagnostic(d: ts.Diagnostic) { - if (d.file !== void 0 && d.start !== void 0) { - const info = syndicateInfo.get(d.file.fileName); - if (info === void 0) - return; - - if (!syntheticSourceFiles.has(d.file.fileName)) { - syntheticSourceFiles.set( - d.file.fileName, - ts.createSourceFile(d.file.fileName, - info.originalSource, - info.languageVersion, - false, - ts.ScriptKind.Unknown)); - } - d.file = syntheticSourceFiles.get(d.file.fileName); - const p = info.targetToSourceMap.get(d.start)!; - d.start = p.firstItem.start.pos + p.offset; - } -} - export function main(argv: string[]) { const options: CommandLineArguments = yargs(argv) .option('verbose', { @@ -136,16 +29,203 @@ export function main(argv: string[]) { default: false, description: "Enable verbose solution builder output", }) + .option('intermediate-directory', { + type: 'string', + description: "Save intermediate expanded Syndicate source code to this directory", + }) + .option('watch', { + alias: 'w', + type: 'boolean', + description: "Enable watch mode", + default: false, + }) + .option('clear', { + type: 'boolean', + description: "Clear screen before each build in watch mode", + default: true, + }) .argv; + if (options.watch) { + function run() { + const toWatch = new ToWatch(); + console.log((options.clear ? '\x1b[2J\x1b[H' : '\n') + (new Date()) + ': Running build'); + runBuildOnce(options, toWatch); + const watchers: Array = []; + let rebuildTriggered = false; + const cb = () => { + if (!rebuildTriggered) { + rebuildTriggered = true; + watchers.forEach(w => w.close()); + queueMicrotask(run); + } + }; + toWatch.files.forEach(f => { + const w = ts.sys.watchFile?.(f, cb); + if (w) watchers.push(w); + }); + toWatch.directories.forEach(d => { + const w = ts.sys.watchDirectory?.(d, cb, true); + if (w) watchers.push(w); + }); + console.log('\n' + (new Date()) + ': Waiting for changes to input files'); + } + run(); + } else { + ts.sys.exit(runBuildOnce(options) ? 0 : 1); + } +} + +function finalSlash(s: string): string { + if (s[s.length - 1] !== '/') s = s + '/'; + return s; +} + +const formatDiagnosticsHost: ts.FormatDiagnosticsHost = { + getCurrentDirectory: () => ts.sys.getCurrentDirectory(), + getNewLine: () => ts.sys.newLine, + getCanonicalFileName: f => f, +}; + +class ToWatch { + files: Set = new Set(); + directories: Set = new Set(); +} + +function runBuildOnce(options: CommandLineArguments, toWatch = new ToWatch()) { let problemCount = 0; let hasErrors = false; - const formatDiagnosticsHost: ts.FormatDiagnosticsHost = { - getCurrentDirectory: () => ts.sys.getCurrentDirectory(), - getNewLine: () => ts.sys.newLine, - getCanonicalFileName: f => f, - }; + const syndicateInfo: Map = new Map(); + + function createProgram(commandLineOptions: CommandLineArguments): ts.CreateProgram + { + return function (rootNames: readonly string[] | undefined, + options: ts.CompilerOptions | undefined, + host?: ts.CompilerHost, + oldProgram?: ts.EmitAndSemanticDiagnosticsBuilderProgram, + configFileParsingDiagnostics?: readonly ts.Diagnostic[], + projectReferences?: readonly ts.ProjectReference[]) + : ts.EmitAndSemanticDiagnosticsBuilderProgram + { + if (host === void 0) { + throw new Error("CompilerHost not present - cannot continue"); + } + + if (rootNames === void 0) { + console.warn("No Syndicate source files to compile"); + } + + const rootDir = finalSlash(options?.rootDir ?? path.resolve('.')); + function writeIntermediate(fileName: string, expandedText: string) { + if ('intermediateDirectory' in commandLineOptions && commandLineOptions.intermediateDirectory) { + const intermediateDirectory = commandLineOptions.intermediateDirectory; + if (fileName.startsWith(rootDir)) { + const intermediateFileName = + path.join(intermediateDirectory, fileName.substr(rootDir.length)); + fs.mkdirSync(path.dirname(intermediateFileName), { recursive: true }); + fs.writeFileSync(intermediateFileName, expandedText, 'utf-8'); + } + } + } + + const oldGetSourceFile = host.getSourceFile; + + host.getSourceFile = (fileName: string, + languageVersion: ts.ScriptTarget, + onError?: ((message: string) => void), + shouldCreateNewSourceFile?: boolean): ts.SourceFile | undefined => { + toWatch.files.add(fileName); + if ((rootNames?.indexOf(fileName) ?? -1) !== -1) { + try { + const inputText = host.readFile(fileName); + if (inputText === void 0) { + onError?.(`Could not read input file ${fileName}`); + return undefined; + } + const { text: expandedText, targetToSourceMap, sourceToTargetMap } = compile({ + source: inputText, + name: fileName, + typescript: true, + }); + writeIntermediate(fileName, expandedText); + syndicateInfo.set(fileName, { + originalSource: inputText, + languageVersion, + targetToSourceMap, + sourceToTargetMap, + }); + const sf = ts.createSourceFile(fileName, expandedText, languageVersion, true); + (sf as any).version = crypto.createHash('sha256').update(expandedText).digest('hex'); + return sf; + } catch (e) { + console.error(e); + onError?.(e.message); + return undefined; + } + } else { + return oldGetSourceFile(fileName, languageVersion, onError, shouldCreateNewSourceFile); + } + }; + + const program = ts.createEmitAndSemanticDiagnosticsBuilderProgram(rootNames, + options, + host, + oldProgram, + configFileParsingDiagnostics, + projectReferences); + return program; + }; + } + + function fixSourceMap(_ctx: ts.TransformationContext): ts.Transformer { + return sf => { + const fileName = sf.fileName; + const info = syndicateInfo.get(fileName); + if (info === void 0) throw new Error("No Syndicate info available for " + fileName); + const targetToSourceMap = info.targetToSourceMap; + const syndicateSource = ts.createSourceMapSource(fileName, info.originalSource); + + function adjustSourceMap(n: ts.Node) { + const ps = targetToSourceMap.get(n.pos); + const pe = targetToSourceMap.get(n.end); + if (ps !== null && pe !== null) { + ts.setSourceMapRange(n, { + pos: ps.firstItem.start.pos + ps.offset, + end: pe.lastItem.start.pos + pe.offset, + source: syndicateSource, + }); + } + ts.forEachChild(n, adjustSourceMap); + } + + adjustSourceMap(sf); + + return sf; + }; + } + + const syntheticSourceFiles = new Map(); + function fixupDiagnostic(d: ts.Diagnostic) { + if (d.file !== void 0 && d.start !== void 0) { + const info = syndicateInfo.get(d.file.fileName); + if (info === void 0) + return; + + if (!syntheticSourceFiles.has(d.file.fileName)) { + syntheticSourceFiles.set( + d.file.fileName, + ts.createSourceFile(d.file.fileName, + info.originalSource, + info.languageVersion, + false, + ts.ScriptKind.Unknown)); + } + d.file = syntheticSourceFiles.get(d.file.fileName); + const p = info.targetToSourceMap.get(d.start)!; + d.start = p.firstItem.start.pos + p.offset; + } + } function reportDiagnostic(d: ts.Diagnostic) { if (d.category === ts.DiagnosticCategory.Error) problemCount++; @@ -161,7 +241,7 @@ export function main(argv: string[]) { } const sbh = ts.createSolutionBuilderHost(ts.sys, - createProgram, + createProgram(options), reportDiagnostic, reportDiagnostic, reportErrorSummary); @@ -170,13 +250,26 @@ export function main(argv: string[]) { verbose: options.verbose, }); - while (true) { - const project = sb.getNextInvalidatedProject(); - if (project === void 0) break; + let project = sb.getNextInvalidatedProject(); + + // Sneakily get into secret members of the ts.SolutionBuilder and + // ts.ParsedCommandline objects to prime our set of watched + // files/directories, in case all the projects are up-to-date and + // our createProgram function is never called. + // + (((sb as any).getAllParsedConfigs?.() ?? []) as Array).forEach(c => { + const f = (c.options as any).configFilePath; + if (f) toWatch.files.add(f); + c.fileNames.forEach(f => toWatch.files.add(f)); + Object.keys(c.wildcardDirectories ?? {}).forEach(d => toWatch.directories.add(d)); + }); + + while (project !== void 0) { project.done(void 0, void 0, { before: [fixSourceMap] }); + project = sb.getNextInvalidatedProject(); } - ts.sys.exit(((problemCount > 0) || hasErrors) ? 1 : 0); + return (problemCount === 0) && !hasErrors; }