Better source tracking through syndicate/ts

This commit is contained in:
Tony Garnock-Jones 2021-01-20 21:56:01 +01:00
parent 45e8ebef12
commit 3908f2ff26
7 changed files with 133 additions and 14 deletions

View File

@ -2,7 +2,7 @@ import {
isToken, isTokenType, replace, commaJoin, startPos, fixPos, joinItems,
anonymousTemplate, laxRead,
Items, Pattern, Templates, Substitution, TokenType,
Items, Pattern, Templates, Substitution, TokenType, Pos,
SourceMap, CodeWriter, TemplateFunction, Token, itemText,
} from '../syntax/index.js';
import {
@ -41,9 +41,16 @@ export interface CompileOptions {
typescript?: boolean,
}
// Essentially the same as a SourceMap, but indexed differently
// (originally for use with the TypeScript compiler).
export interface SourcePositionIndex {
sourcePositionAt(pos: number): Pos;
}
export interface CompilerOutput {
text: string,
map: SourceMap,
positionIndex: SourcePositionIndex,
}
function receiverFor(s: FacetAction): Substitution {
@ -354,8 +361,40 @@ export function compile(options: CompileOptions): CompilerOutput {
const cw = new CodeWriter(inputFilename);
cw.emit(tree);
const positionMap = cw.positionMap;
return {
text: cw.text,
map: cw.map,
positionIndex: {
sourcePositionAt(pos: number): Pos {
if (positionMap.length === 0) return start;
let lo = 0;
let hi = positionMap.length;
// console.log(`\nsearching for ${pos}`);
while (true) {
if (lo === hi) {
const e = positionMap[lo - 1] ?? [0, start];
if (e[0] > pos) throw new Error();
if (positionMap[lo]?.[0] <= pos) throw new Error();
// console.log(`found ${JSON.stringify(e)}`);
return e[1];
}
const mid = (lo + hi) >> 1;
const e = positionMap[mid];
// console.log(`${pos} lo ${lo} hi ${hi} mid ${mid} probe ${JSON.stringify([e[0], e[1].pos])}`);
if (e[0] <= pos) {
lo = mid + 1;
} else {
hi = mid;
}
}
}
}
};
}

View File

@ -51,6 +51,7 @@ export class CodeWriter {
readonly sources: Array<string> = [];
readonly chunks: Array<string> = [];
readonly mappings: Array<Array<NonEmptyMapping>> = [];
readonly positionMap: Array<[number, Pos]> = [];
previous: Partial<SourceNameMapping> = {};
previousPos: Pos | null = null;
@ -144,14 +145,24 @@ export class CodeWriter {
this.mappings[this.mappings.length - 1].push(n);
}
augmentPositionMap(p: Pos) {
if (this.positionMap.length > 0) {
const prev = this.positionMap[this.positionMap.length - 1][1];
if ((prev.name === p.name) && (prev.pos === p.pos)) return;
}
this.positionMap.push([this.pos.pos, { ... p }]);
}
chunk(p: Pos, s: string, type: TokenType) {
p = { ... p };
this.chunks.push(s);
this.augmentPositionMap(p);
if (this.mappings.length === 0) this.finishLine();
this.addMapping(p, type);
for (const ch of s) {
advancePos(p, ch);
if (advancePos(this.pos, ch)) {
this.augmentPositionMap(p);
this.finishLine();
this.addMapping(p, type);
}

View File

@ -0,0 +1,7 @@
#!/usr/bin/env node
try {
require('../lib/maptool.js').main(process.argv.slice(2));
} catch (e) {
console.error(e);
process.exit(1);
}

View File

@ -20,6 +20,7 @@
"yargs": "^16.2.0"
},
"bin": {
"syndicate-maptool": "./bin/syndicate-maptool.js",
"syndicate-tsc": "./bin/syndicate-tsc.js",
"syndicatec": "./bin/syndicatec.js"
},

View File

@ -0,0 +1,15 @@
import fs from 'fs';
import { SourceMap } from '@syndicate-lang/compiler/lib/syntax/index.js';
import { Syntax } from '@syndicate-lang/compiler';
const { vlqDecode } = Syntax;
export function main(argv: string[]) {
const mapFilename = argv[0];
console.log(mapFilename);
const map = JSON.parse(fs.readFileSync(mapFilename, 'utf-8')) as SourceMap;
console.log(map);
const entries = map.mappings.split(/;/).map(e => e.split(/,/).map(vlqDecode));
entries.forEach((line, lineNumber) =>
console.log(lineNumber + 1, line));
}

View File

@ -1,9 +1,8 @@
import yargs from 'yargs/yargs';
import ts from 'typescript';
import crypto from 'crypto';
import { compile } from '@syndicate-lang/compiler';
import { dataURL, sourceMappingComment } from './util.js';
import { SourcePositionIndex } from '@syndicate-lang/compiler/lib/compiler/codegen';
function reportDiagnostic(diagnostic: ts.Diagnostic) {
if (diagnostic.file) {
@ -21,6 +20,13 @@ function reportErrorSummary(n: number) {
}
}
interface SyndicateInfo {
originalSource: string;
positionIndex: SourcePositionIndex;
}
const syndicateInfo: Map<string, SyndicateInfo> = new Map();
function createProgram(rootNames: readonly string[] | undefined,
options: ts.CompilerOptions | undefined,
host?: ts.CompilerHost,
@ -50,18 +56,15 @@ function createProgram(rootNames: readonly string[] | undefined,
onError?.(`Could not read input file ${fileName}`);
return undefined;
}
const { text: baseExpandedText, map: sourceMap } = compile({
const { text: expandedText, positionIndex } = compile({
source: inputText,
name: fileName,
typescript: true,
});
sourceMap.sourcesContent = [inputText];
const expandedText = baseExpandedText + sourceMappingComment(dataURL(JSON.stringify(sourceMap)));
// console.log('\n\n', fileName);
// expandedText.split(/\n/).forEach((line, i) => {
// console.log(i + 1, line);
// });
syndicateInfo.set(fileName, {
originalSource: inputText,
positionIndex,
});
const sf = ts.createSourceFile(fileName, expandedText, languageVersion, true);
(sf as any).version = crypto.createHash('sha256').update(expandedText).digest('hex');
return sf;
@ -82,6 +85,44 @@ function createProgram(rootNames: readonly string[] | undefined,
projectReferences);
}
export function fixSourceMap(ctx: ts.TransformationContext): ts.Transformer<ts.SourceFile> {
return sf => {
const fileName = sf.fileName;
const info = syndicateInfo.get(fileName);
if (info === void 0) throw new Error("No Syndicate info available for " + fileName);
const positionIndex = info.positionIndex;
// console.log('fixSourceMap', fileName, sf.text.length, info.originalSource.length);
const syndicateSource = ts.createSourceMapSource(fileName, info.originalSource);
const expandedSource = ts.createSourceMapSource(fileName + '.expanded', sf.text);
function adjustSourceMap(n: ts.Node) {
const ps = positionIndex.sourcePositionAt(n.pos);
const pe = positionIndex.sourcePositionAt(n.end);
if (ps.name === fileName && pe.name === fileName) {
ts.setSourceMapRange(n, { pos: ps.pos, end: pe.pos, source: syndicateSource });
// console.group(ts.SyntaxKind[n.kind], `${n.pos}-${n.end} ==> ${ps.pos}-${pe.pos}`);
} else if (ps.name === null && pe.name === null) {
ts.setSourceMapRange(n, { pos: ps.pos, end: pe.pos, source: expandedSource });
// console.group(ts.SyntaxKind[n.kind], n.pos, 'synthetic');
} else if (ps.name === null) {
ts.setSourceMapRange(n, { pos: pe.pos, end: pe.pos, source: expandedSource });
// console.group(ts.SyntaxKind[n.kind], n.pos, 'mixed end');
} else {
ts.setSourceMapRange(n, { pos: ps.pos, end: ps.pos, source: expandedSource });
// console.group(ts.SyntaxKind[n.kind], n.pos, 'mixed start');
}
ts.forEachChild(n, adjustSourceMap);
// console.groupEnd();
}
adjustSourceMap(sf);
return sf;
};
}
export function main(argv: string[]) {
const sbh = ts.createSolutionBuilderHost(ts.sys,
createProgram,
@ -91,5 +132,12 @@ export function main(argv: string[]) {
const sb = ts.createSolutionBuilder(sbh, ['.'], {
verbose: true,
});
ts.sys.exit(sb.build());
while (true) {
const project = sb.getNextInvalidatedProject();
if (project === void 0) break;
project.done(void 0, void 0, {
before: [fixSourceMap]
});
}
ts.sys.exit(0);
}

View File

@ -1,2 +0,0 @@
#!/bin/sh
exec inotifytest redo -j$(nproc)