Better source tracking through syndicate/ts

packages/compiler/src/compiler/codegen.ts

@@ -2,7 +2,7 @@ import {
     isToken, isTokenType, replace, commaJoin, startPos, fixPos, joinItems,
     anonymousTemplate, laxRead,
 
-    Items, Pattern, Templates, Substitution, TokenType,
+    Items, Pattern, Templates, Substitution, TokenType, Pos,
     SourceMap, CodeWriter, TemplateFunction, Token, itemText,
 } from '../syntax/index.js';
 import {
@@ -41,9 +41,16 @@ export interface CompileOptions {
     typescript?: boolean,
 }
 
+// Essentially the same as a SourceMap, but indexed differently
+// (originally for use with the TypeScript compiler).
+export interface SourcePositionIndex {
+    sourcePositionAt(pos: number): Pos;
+}
+
 export interface CompilerOutput {
     text: string,
     map: SourceMap,
+    positionIndex: SourcePositionIndex,
 }
 
 function receiverFor(s: FacetAction): Substitution {
@@ -354,8 +361,40 @@ export function compile(options: CompileOptions): CompilerOutput {
     const cw = new CodeWriter(inputFilename);
     cw.emit(tree);
 
+    const positionMap = cw.positionMap;
+
     return {
         text: cw.text,
         map: cw.map,
+        positionIndex: {
+            sourcePositionAt(pos: number): Pos {
+                if (positionMap.length === 0) return start;
+
+                let lo = 0;
+                let hi = positionMap.length;
+
+                // console.log(`\nsearching for ${pos}`);
+                while (true) {
+                    if (lo === hi) {
+                        const e = positionMap[lo - 1] ?? [0, start];
+                        if (e[0] > pos) throw new Error();
+                        if (positionMap[lo]?.[0] <= pos) throw new Error();
+                        // console.log(`found ${JSON.stringify(e)}`);
+                        return e[1];
+                    }
+
+                    const mid = (lo + hi) >> 1;
+                    const e = positionMap[mid];
+
+                    // console.log(`${pos} lo ${lo} hi ${hi} mid ${mid} probe ${JSON.stringify([e[0], e[1].pos])}`);
+
+                    if (e[0] <= pos) {
+                        lo = mid + 1;
+                    } else {
+                        hi = mid;
+                    }
+                }
+            }
+        }
     };
 }

packages/compiler/src/syntax/codewriter.ts

@@ -51,6 +51,7 @@ export class CodeWriter {
     readonly sources: Array<string> = [];
     readonly chunks: Array<string> = [];
     readonly mappings: Array<Array<NonEmptyMapping>> = [];
+    readonly positionMap: Array<[number, Pos]> = [];
     previous: Partial<SourceNameMapping> = {};
     previousPos: Pos | null = null;
 
@@ -144,14 +145,24 @@ export class CodeWriter {
         this.mappings[this.mappings.length - 1].push(n);
     }
 
+    augmentPositionMap(p: Pos) {
+        if (this.positionMap.length > 0) {
+            const prev = this.positionMap[this.positionMap.length - 1][1];
+            if ((prev.name === p.name) && (prev.pos === p.pos)) return;
+        }
+        this.positionMap.push([this.pos.pos, { ... p }]);
+    }
+
     chunk(p: Pos, s: string, type: TokenType) {
         p = { ... p };
         this.chunks.push(s);
+        this.augmentPositionMap(p);
         if (this.mappings.length === 0) this.finishLine();
         this.addMapping(p, type);
         for (const ch of s) {
             advancePos(p, ch);
             if (advancePos(this.pos, ch)) {
+                this.augmentPositionMap(p);
                 this.finishLine();
                 this.addMapping(p, type);
             }

packages/syndicatec/bin/syndicate-maptool.js

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+try {
+  require('../lib/maptool.js').main(process.argv.slice(2));
+} catch (e) {
+  console.error(e);
+  process.exit(1);
+}

packages/syndicatec/package.json

@@ -20,6 +20,7 @@
     "yargs": "^16.2.0"
   },
   "bin": {
+    "syndicate-maptool": "./bin/syndicate-maptool.js",
     "syndicate-tsc": "./bin/syndicate-tsc.js",
     "syndicatec": "./bin/syndicatec.js"
   },

packages/syndicatec/src/maptool.ts

@@ -0,0 +1,15 @@
+import fs from 'fs';
+import { SourceMap } from '@syndicate-lang/compiler/lib/syntax/index.js';
+import { Syntax } from '@syndicate-lang/compiler';
+const { vlqDecode } = Syntax;
+
+export function main(argv: string[]) {
+    const mapFilename = argv[0];
+    console.log(mapFilename);
+    const map = JSON.parse(fs.readFileSync(mapFilename, 'utf-8')) as SourceMap;
+    console.log(map);
+
+    const entries = map.mappings.split(/;/).map(e => e.split(/,/).map(vlqDecode));
+    entries.forEach((line, lineNumber) =>
+        console.log(lineNumber + 1, line));
+}

packages/syndicatec/src/tsc.ts

@@ -1,9 +1,8 @@
-import yargs from 'yargs/yargs';
 import ts from 'typescript';
 import crypto from 'crypto';
 
 import { compile } from '@syndicate-lang/compiler';
-import { dataURL, sourceMappingComment } from './util.js';
+import { SourcePositionIndex } from '@syndicate-lang/compiler/lib/compiler/codegen';
 
 function reportDiagnostic(diagnostic: ts.Diagnostic) {
     if (diagnostic.file) {
@@ -21,6 +20,13 @@ function reportErrorSummary(n: number) {
     }
 }
 
+interface SyndicateInfo {
+    originalSource: string;
+    positionIndex: SourcePositionIndex;
+}
+
+const syndicateInfo: Map<string, SyndicateInfo> = new Map();
+
 function createProgram(rootNames: readonly string[] | undefined,
                        options: ts.CompilerOptions | undefined,
                        host?: ts.CompilerHost,
@@ -50,18 +56,15 @@ function createProgram(rootNames: readonly string[] | undefined,
                     onError?.(`Could not read input file ${fileName}`);
                     return undefined;
                 }
-                const { text: baseExpandedText, map: sourceMap } = compile({
+                const { text: expandedText, positionIndex } = compile({
                     source: inputText,
                     name: fileName,
                     typescript: true,
                 });
-                sourceMap.sourcesContent = [inputText];
-                const expandedText = baseExpandedText + sourceMappingComment(dataURL(JSON.stringify(sourceMap)));
-
-                // console.log('\n\n', fileName);
-                // expandedText.split(/\n/).forEach((line, i) => {
-                //     console.log(i + 1, line);
-                // });
+                syndicateInfo.set(fileName, {
+                    originalSource: inputText,
+                    positionIndex,
+                });
                 const sf = ts.createSourceFile(fileName, expandedText, languageVersion, true);
                 (sf as any).version = crypto.createHash('sha256').update(expandedText).digest('hex');
                 return sf;
@@ -82,6 +85,44 @@ function createProgram(rootNames: readonly string[] | undefined,
                                                              projectReferences);
 }
 
+export function fixSourceMap(ctx: ts.TransformationContext): ts.Transformer<ts.SourceFile> {
+    return sf => {
+        const fileName = sf.fileName;
+        const info = syndicateInfo.get(fileName);
+        if (info === void 0) throw new Error("No Syndicate info available for " + fileName);
+        const positionIndex = info.positionIndex;
+
+        // console.log('fixSourceMap', fileName, sf.text.length, info.originalSource.length);
+
+        const syndicateSource = ts.createSourceMapSource(fileName, info.originalSource);
+        const expandedSource = ts.createSourceMapSource(fileName + '.expanded', sf.text);
+
+        function adjustSourceMap(n: ts.Node) {
+            const ps = positionIndex.sourcePositionAt(n.pos);
+            const pe = positionIndex.sourcePositionAt(n.end);
+            if (ps.name === fileName && pe.name === fileName) {
+                ts.setSourceMapRange(n, { pos: ps.pos, end: pe.pos, source: syndicateSource });
+                // console.group(ts.SyntaxKind[n.kind], `${n.pos}-${n.end} ==> ${ps.pos}-${pe.pos}`);
+            } else if (ps.name === null && pe.name === null) {
+                ts.setSourceMapRange(n, { pos: ps.pos, end: pe.pos, source: expandedSource });
+                // console.group(ts.SyntaxKind[n.kind], n.pos, 'synthetic');
+            } else if (ps.name === null) {
+                ts.setSourceMapRange(n, { pos: pe.pos, end: pe.pos, source: expandedSource });
+                // console.group(ts.SyntaxKind[n.kind], n.pos, 'mixed end');
+            } else {
+                ts.setSourceMapRange(n, { pos: ps.pos, end: ps.pos, source: expandedSource });
+                // console.group(ts.SyntaxKind[n.kind], n.pos, 'mixed start');
+            }
+            ts.forEachChild(n, adjustSourceMap);
+            // console.groupEnd();
+        }
+
+        adjustSourceMap(sf);
+
+        return sf;
+    };
+}
+
 export function main(argv: string[]) {
     const sbh = ts.createSolutionBuilderHost(ts.sys,
                                              createProgram,
@@ -91,5 +132,12 @@ export function main(argv: string[]) {
     const sb = ts.createSolutionBuilder(sbh, ['.'], {
         verbose: true,
     });
-    ts.sys.exit(sb.build());
+    while (true) {
+        const project = sb.getNextInvalidatedProject();
+        if (project === void 0) break;
+        project.done(void 0, void 0, {
+            before: [fixSourceMap]
+        });
+    }
+    ts.sys.exit(0);
 }

watch-redo

@@ -1,2 +0,0 @@
-#!/bin/sh
-exec inotifytest redo -j$(nproc)