From 3908f2ff26542dd43c4ade5cef9a49d9448f40b5 Mon Sep 17 00:00:00 2001 From: Tony Garnock-Jones Date: Wed, 20 Jan 2021 21:56:01 +0100 Subject: [PATCH] Better source tracking through syndicate/ts --- packages/compiler/src/compiler/codegen.ts | 41 +++++++++++- packages/compiler/src/syntax/codewriter.ts | 11 +++ packages/syndicatec/bin/syndicate-maptool.js | 7 ++ packages/syndicatec/package.json | 1 + packages/syndicatec/src/maptool.ts | 15 +++++ packages/syndicatec/src/tsc.ts | 70 +++++++++++++++++--- watch-redo | 2 - 7 files changed, 133 insertions(+), 14 deletions(-) create mode 100755 packages/syndicatec/bin/syndicate-maptool.js create mode 100644 packages/syndicatec/src/maptool.ts delete mode 100755 watch-redo diff --git a/packages/compiler/src/compiler/codegen.ts b/packages/compiler/src/compiler/codegen.ts index 1c826ae..f6b60aa 100644 --- a/packages/compiler/src/compiler/codegen.ts +++ b/packages/compiler/src/compiler/codegen.ts @@ -2,7 +2,7 @@ import { isToken, isTokenType, replace, commaJoin, startPos, fixPos, joinItems, anonymousTemplate, laxRead, - Items, Pattern, Templates, Substitution, TokenType, + Items, Pattern, Templates, Substitution, TokenType, Pos, SourceMap, CodeWriter, TemplateFunction, Token, itemText, } from '../syntax/index.js'; import { @@ -41,9 +41,16 @@ export interface CompileOptions { typescript?: boolean, } +// Essentially the same as a SourceMap, but indexed differently +// (originally for use with the TypeScript compiler). +export interface SourcePositionIndex { + sourcePositionAt(pos: number): Pos; +} + export interface CompilerOutput { text: string, map: SourceMap, + positionIndex: SourcePositionIndex, } function receiverFor(s: FacetAction): Substitution { @@ -354,8 +361,40 @@ export function compile(options: CompileOptions): CompilerOutput { const cw = new CodeWriter(inputFilename); cw.emit(tree); + const positionMap = cw.positionMap; + return { text: cw.text, map: cw.map, + positionIndex: { + sourcePositionAt(pos: number): Pos { + if (positionMap.length === 0) return start; + + let lo = 0; + let hi = positionMap.length; + + // console.log(`\nsearching for ${pos}`); + while (true) { + if (lo === hi) { + const e = positionMap[lo - 1] ?? [0, start]; + if (e[0] > pos) throw new Error(); + if (positionMap[lo]?.[0] <= pos) throw new Error(); + // console.log(`found ${JSON.stringify(e)}`); + return e[1]; + } + + const mid = (lo + hi) >> 1; + const e = positionMap[mid]; + + // console.log(`${pos} lo ${lo} hi ${hi} mid ${mid} probe ${JSON.stringify([e[0], e[1].pos])}`); + + if (e[0] <= pos) { + lo = mid + 1; + } else { + hi = mid; + } + } + } + } }; } diff --git a/packages/compiler/src/syntax/codewriter.ts b/packages/compiler/src/syntax/codewriter.ts index 9fd6a2f..da5c531 100644 --- a/packages/compiler/src/syntax/codewriter.ts +++ b/packages/compiler/src/syntax/codewriter.ts @@ -51,6 +51,7 @@ export class CodeWriter { readonly sources: Array = []; readonly chunks: Array = []; readonly mappings: Array> = []; + readonly positionMap: Array<[number, Pos]> = []; previous: Partial = {}; previousPos: Pos | null = null; @@ -144,14 +145,24 @@ export class CodeWriter { this.mappings[this.mappings.length - 1].push(n); } + augmentPositionMap(p: Pos) { + if (this.positionMap.length > 0) { + const prev = this.positionMap[this.positionMap.length - 1][1]; + if ((prev.name === p.name) && (prev.pos === p.pos)) return; + } + this.positionMap.push([this.pos.pos, { ... p }]); + } + chunk(p: Pos, s: string, type: TokenType) { p = { ... p }; this.chunks.push(s); + this.augmentPositionMap(p); if (this.mappings.length === 0) this.finishLine(); this.addMapping(p, type); for (const ch of s) { advancePos(p, ch); if (advancePos(this.pos, ch)) { + this.augmentPositionMap(p); this.finishLine(); this.addMapping(p, type); } diff --git a/packages/syndicatec/bin/syndicate-maptool.js b/packages/syndicatec/bin/syndicate-maptool.js new file mode 100755 index 0000000..e5d2e50 --- /dev/null +++ b/packages/syndicatec/bin/syndicate-maptool.js @@ -0,0 +1,7 @@ +#!/usr/bin/env node +try { + require('../lib/maptool.js').main(process.argv.slice(2)); +} catch (e) { + console.error(e); + process.exit(1); +} diff --git a/packages/syndicatec/package.json b/packages/syndicatec/package.json index 92f6cce..1e6a0e3 100644 --- a/packages/syndicatec/package.json +++ b/packages/syndicatec/package.json @@ -20,6 +20,7 @@ "yargs": "^16.2.0" }, "bin": { + "syndicate-maptool": "./bin/syndicate-maptool.js", "syndicate-tsc": "./bin/syndicate-tsc.js", "syndicatec": "./bin/syndicatec.js" }, diff --git a/packages/syndicatec/src/maptool.ts b/packages/syndicatec/src/maptool.ts new file mode 100644 index 0000000..c889771 --- /dev/null +++ b/packages/syndicatec/src/maptool.ts @@ -0,0 +1,15 @@ +import fs from 'fs'; +import { SourceMap } from '@syndicate-lang/compiler/lib/syntax/index.js'; +import { Syntax } from '@syndicate-lang/compiler'; +const { vlqDecode } = Syntax; + +export function main(argv: string[]) { + const mapFilename = argv[0]; + console.log(mapFilename); + const map = JSON.parse(fs.readFileSync(mapFilename, 'utf-8')) as SourceMap; + console.log(map); + + const entries = map.mappings.split(/;/).map(e => e.split(/,/).map(vlqDecode)); + entries.forEach((line, lineNumber) => + console.log(lineNumber + 1, line)); +} diff --git a/packages/syndicatec/src/tsc.ts b/packages/syndicatec/src/tsc.ts index d3d4e31..122a297 100644 --- a/packages/syndicatec/src/tsc.ts +++ b/packages/syndicatec/src/tsc.ts @@ -1,9 +1,8 @@ -import yargs from 'yargs/yargs'; import ts from 'typescript'; import crypto from 'crypto'; import { compile } from '@syndicate-lang/compiler'; -import { dataURL, sourceMappingComment } from './util.js'; +import { SourcePositionIndex } from '@syndicate-lang/compiler/lib/compiler/codegen'; function reportDiagnostic(diagnostic: ts.Diagnostic) { if (diagnostic.file) { @@ -21,6 +20,13 @@ function reportErrorSummary(n: number) { } } +interface SyndicateInfo { + originalSource: string; + positionIndex: SourcePositionIndex; +} + +const syndicateInfo: Map = new Map(); + function createProgram(rootNames: readonly string[] | undefined, options: ts.CompilerOptions | undefined, host?: ts.CompilerHost, @@ -50,18 +56,15 @@ function createProgram(rootNames: readonly string[] | undefined, onError?.(`Could not read input file ${fileName}`); return undefined; } - const { text: baseExpandedText, map: sourceMap } = compile({ + const { text: expandedText, positionIndex } = compile({ source: inputText, name: fileName, typescript: true, }); - sourceMap.sourcesContent = [inputText]; - const expandedText = baseExpandedText + sourceMappingComment(dataURL(JSON.stringify(sourceMap))); - - // console.log('\n\n', fileName); - // expandedText.split(/\n/).forEach((line, i) => { - // console.log(i + 1, line); - // }); + syndicateInfo.set(fileName, { + originalSource: inputText, + positionIndex, + }); const sf = ts.createSourceFile(fileName, expandedText, languageVersion, true); (sf as any).version = crypto.createHash('sha256').update(expandedText).digest('hex'); return sf; @@ -82,6 +85,44 @@ function createProgram(rootNames: readonly string[] | undefined, projectReferences); } +export function fixSourceMap(ctx: ts.TransformationContext): ts.Transformer { + return sf => { + const fileName = sf.fileName; + const info = syndicateInfo.get(fileName); + if (info === void 0) throw new Error("No Syndicate info available for " + fileName); + const positionIndex = info.positionIndex; + + // console.log('fixSourceMap', fileName, sf.text.length, info.originalSource.length); + + const syndicateSource = ts.createSourceMapSource(fileName, info.originalSource); + const expandedSource = ts.createSourceMapSource(fileName + '.expanded', sf.text); + + function adjustSourceMap(n: ts.Node) { + const ps = positionIndex.sourcePositionAt(n.pos); + const pe = positionIndex.sourcePositionAt(n.end); + if (ps.name === fileName && pe.name === fileName) { + ts.setSourceMapRange(n, { pos: ps.pos, end: pe.pos, source: syndicateSource }); + // console.group(ts.SyntaxKind[n.kind], `${n.pos}-${n.end} ==> ${ps.pos}-${pe.pos}`); + } else if (ps.name === null && pe.name === null) { + ts.setSourceMapRange(n, { pos: ps.pos, end: pe.pos, source: expandedSource }); + // console.group(ts.SyntaxKind[n.kind], n.pos, 'synthetic'); + } else if (ps.name === null) { + ts.setSourceMapRange(n, { pos: pe.pos, end: pe.pos, source: expandedSource }); + // console.group(ts.SyntaxKind[n.kind], n.pos, 'mixed end'); + } else { + ts.setSourceMapRange(n, { pos: ps.pos, end: ps.pos, source: expandedSource }); + // console.group(ts.SyntaxKind[n.kind], n.pos, 'mixed start'); + } + ts.forEachChild(n, adjustSourceMap); + // console.groupEnd(); + } + + adjustSourceMap(sf); + + return sf; + }; +} + export function main(argv: string[]) { const sbh = ts.createSolutionBuilderHost(ts.sys, createProgram, @@ -91,5 +132,12 @@ export function main(argv: string[]) { const sb = ts.createSolutionBuilder(sbh, ['.'], { verbose: true, }); - ts.sys.exit(sb.build()); + while (true) { + const project = sb.getNextInvalidatedProject(); + if (project === void 0) break; + project.done(void 0, void 0, { + before: [fixSourceMap] + }); + } + ts.sys.exit(0); } diff --git a/watch-redo b/watch-redo deleted file mode 100755 index d39a939..0000000 --- a/watch-redo +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec inotifytest redo -j$(nproc)