From 3b1669389c44c4d061a02256f783c16ef0a6df01 Mon Sep 17 00:00:00 2001 From: Tony Garnock-Jones Date: Mon, 13 Dec 2021 12:17:12 +0100 Subject: [PATCH] Allow cross-references to other schemas when generating code --- .../packages/schema/src/bin/cli-utils.ts | 88 ++++++++++++------- .../schema/src/bin/preserves-schema-ts.ts | 31 +++++-- .../schema/src/bin/preserves-schemac.ts | 2 +- .../packages/schema/src/compiler/context.ts | 51 ++++++----- .../schema/src/compiler/genconverter.ts | 2 +- .../javascript/packages/schema/src/meta.ts | 11 ++- 6 files changed, 118 insertions(+), 67 deletions(-) diff --git a/implementations/javascript/packages/schema/src/bin/cli-utils.ts b/implementations/javascript/packages/schema/src/bin/cli-utils.ts index ff4d652..abbc1bb 100644 --- a/implementations/javascript/packages/schema/src/bin/cli-utils.ts +++ b/implementations/javascript/packages/schema/src/bin/cli-utils.ts @@ -12,15 +12,25 @@ export interface Diagnostic { detail: Error | { message: string, pos: Position | null }; }; +export type InputFile = { + inputFilePath: string, + text: string, + baseRelPath: string, + modulePath: M.ModulePath, + schema: M.Schema, +}; + +export type XrefFile = { + xrefFilePath: string, + text: string, + modulePath: M.ModulePath, + schema: M.Schema, +}; + export type Expanded = { base: string, - inputFiles: Array<{ - inputFilePath: string, - text: string, - baseRelPath: string, - modulePath: M.ModulePath, - schema: M.Schema, - }>, + inputFiles: Array, + xrefFiles: Array, failures: Array, }; @@ -44,34 +54,52 @@ export function computeBase(paths: string[]): string { } } -export function expandInputGlob(input: string[], base0: string | undefined): Expanded { +export function expandInputGlob( + input: string[], + xrefs: string[], + base0: string | undefined, +): Expanded { const matches = input.flatMap(i => glob.sync(i)); - const base = base0 ?? computeBase(matches); const failures: Array = []; + function processInputFile(base: string, filePath: string, xref: true): XrefFile[]; + function processInputFile(base: string, filePath: string, xref: false): InputFile[]; + function processInputFile(base: string, filePath: string, xref: boolean): (InputFile | XrefFile)[] { + if (!xref && !filePath.startsWith(base)) { + throw new Error(`Input filename ${filePath} falls outside base ${base}`); + } + try { + const text = fs.readFileSync(filePath, 'utf-8'); + const baseRelPath = filePath.slice(base.length); + const modulePath = baseRelPath.split('/').map(p => p.split('.')[0]).map(Symbol.for); + const schema = readSchema(text, { + name: filePath, + readInclude(includePath: string): string { + return fs.readFileSync( + path.resolve(path.dirname(filePath), includePath), + 'utf-8'); + }, + }); + if (xref) { + return [{ xrefFilePath: filePath, text, modulePath, schema }]; + } else { + return [{ inputFilePath: filePath, text, baseRelPath, modulePath, schema }]; + } + } catch (e) { + failures.push({ type: 'error', file: filePath, detail: e as Error }); + return []; + } + } + + const base = base0 ?? computeBase(matches); + return { base, - inputFiles: matches.flatMap(inputFilePath => { - if (!inputFilePath.startsWith(base)) { - throw new Error(`Input filename ${inputFilePath} falls outside base ${base}`); - } - try { - const text = fs.readFileSync(inputFilePath, 'utf-8'); - const baseRelPath = inputFilePath.slice(base.length); - const modulePath = baseRelPath.split('/').map(p => p.split('.')[0]).map(Symbol.for); - const schema = readSchema(text, { - name: inputFilePath, - readInclude(includePath: string): string { - return fs.readFileSync( - path.resolve(path.dirname(inputFilePath), includePath), - 'utf-8'); - }, - }); - return [{ inputFilePath, text, baseRelPath, modulePath, schema }]; - } catch (e) { - failures.push({ type: 'error', file: inputFilePath, detail: e as Error }); - return []; - } + inputFiles: matches.flatMap(f => processInputFile(base, f, false)), + xrefFiles: xrefs.flatMap(i => { + const names = glob.sync(i); + const base = computeBase(names); + return names.flatMap(f => processInputFile(base, f, true)); }), failures, }; diff --git a/implementations/javascript/packages/schema/src/bin/preserves-schema-ts.ts b/implementations/javascript/packages/schema/src/bin/preserves-schema-ts.ts index d69bbf5..bfb5d74 100644 --- a/implementations/javascript/packages/schema/src/bin/preserves-schema-ts.ts +++ b/implementations/javascript/packages/schema/src/bin/preserves-schema-ts.ts @@ -5,12 +5,13 @@ import minimatch from 'minimatch'; import { Command } from 'commander'; import * as M from '../meta'; import chalk from 'chalk'; -import { Position } from '@preserves/core'; +import { is, Position } from '@preserves/core'; import chokidar from 'chokidar'; import { changeExt, Diagnostic, expandInputGlob, formatFailures } from './cli-utils'; export type CommandLineArguments = { inputs: string[]; + xrefs: string[]; base: string | undefined; output: string | undefined; stdout: boolean; @@ -22,13 +23,13 @@ export type CommandLineArguments = { export type CompilationResult = { options: CommandLineArguments, - inputFiles: Array, + inputFiles: Array, failures: Array, base: string, output: string, }; -export type InputFile = { +export type TranslatedFile = { inputFilePath: string, outputFilePath: string, schemaPath: M.ModulePath, @@ -85,10 +86,12 @@ function isAbsoluteOrExplicitlyRelative(p: string) { } export function runOnce(options: CommandLineArguments): CompilationResult { - const { base, failures, inputFiles: inputFiles0 } = - expandInputGlob(options.inputs, options.base); + const { base, failures, inputFiles: inputFiles0, xrefFiles: xrefFiles0 } = + expandInputGlob(options.inputs, options.xrefs, options.base); const output = options.output ?? base; + let xrefFiles = xrefFiles0; // filtered during construction of extensionEnv + const extensionEnv: M.Environment = options.module.map(arg => { const i = arg.indexOf('='); if (i === -1) throw new Error(`--module argument must be Namespace=path or Namespace=path:expr; got ${arg}`); @@ -96,16 +99,19 @@ export function runOnce(options: CommandLineArguments): CompilationResult { const pathAndExpr = arg.slice(i + 1); const j = pathAndExpr.lastIndexOf(':'); const path = (j === -1) ? pathAndExpr : pathAndExpr.slice(0, j); + const modulePath = ns.split('.').map(Symbol.for); const expr = (j === -1) ? null : pathAndExpr.slice(j + 1); + const e = xrefFiles.find(x => is(x.modulePath, modulePath)); + if (e) xrefFiles = xrefFiles.filter(e0 => !Object.is(e0, e)); return { - schema: null, - schemaModulePath: ns.split('.').map(Symbol.for), + schema: e ? e.schema : null, + schemaModulePath: modulePath, typescriptModulePath: path, typescriptModuleExpr: expr, }; }); - const inputFiles: Array = inputFiles0.map(i => { + const inputFiles: Array = inputFiles0.map(i => { const { inputFilePath, baseRelPath, modulePath, schema } = i; const outputFilePath = path.join(output, changeExt(baseRelPath, '.ts')); return { inputFilePath, outputFilePath, schemaPath: modulePath, schema }; @@ -120,6 +126,12 @@ export function runOnce(options: CommandLineArguments): CompilationResult { if (p === null) return []; return [{... e, typescriptModulePath: p}]; }), + ... xrefFiles.map(x => ({ + schema: x.schema, + schemaModulePath: x.modulePath, + typescriptModulePath: 'xref-typescript-modules-not-available', + typescriptModuleExpr: null, + })), ... inputFiles.map(cc => ({ schema: cc.schema, schemaModulePath: cc.schemaPath, @@ -161,6 +173,8 @@ export function main(argv: Array) { .description('Compile Preserves schema definitions to TypeScript', { input: 'Input filename or glob', }) + .option('--xref ', 'Cross-reference other textual Preserves schema definitions', + (glob, prev: string[]) => [... prev, glob], []) .option('--output ', 'Output directory for modules (default: next to sources)') .option('--stdout', 'Prints each module to stdout one after the other instead ' + 'of writing them to files in the `--output` directory') @@ -174,6 +188,7 @@ export function main(argv: Array) { .action((inputs: string[], rawOptions) => { const options: CommandLineArguments = { inputs: inputs.map(i => path.normalize(i)), + xrefs: rawOptions.xref.map((x: string) => path.normalize(x)), base: rawOptions.base, output: rawOptions.output, stdout: rawOptions.stdout, diff --git a/implementations/javascript/packages/schema/src/bin/preserves-schemac.ts b/implementations/javascript/packages/schema/src/bin/preserves-schemac.ts index 974f96a..eec9fa2 100644 --- a/implementations/javascript/packages/schema/src/bin/preserves-schemac.ts +++ b/implementations/javascript/packages/schema/src/bin/preserves-schemac.ts @@ -12,7 +12,7 @@ export type CommandLineArguments = { }; export function run(options: CommandLineArguments): void { - const { failures, inputFiles } = expandInputGlob(options.inputs, options.base); + const { failures, inputFiles } = expandInputGlob(options.inputs, [], options.base); if (!options.bundle && inputFiles.length !== 1) { failures.push({ type: 'error', file: null, detail: { diff --git a/implementations/javascript/packages/schema/src/compiler/context.ts b/implementations/javascript/packages/schema/src/compiler/context.ts index 1bdccf6..d5f77be 100644 --- a/implementations/javascript/packages/schema/src/compiler/context.ts +++ b/implementations/javascript/packages/schema/src/compiler/context.ts @@ -97,7 +97,7 @@ export class ModuleContext { return (ref) => this.lookup( ref, (_p, _t) => Type.ref(ref.name.description!, ref), - (modPath, modId, modFile, modExpr, _p, _t) => { + (modPath, modId, modFile, modExpr, _p, t) => { this.imports.add([modPath, modId, modFile, modExpr]); return Type.ref(`${modId}${modExpr}.${ref.name.description!}`, ref); }, @@ -125,31 +125,30 @@ export class ModuleContext { : R { const soughtModule = name.module.length ? name.module : (modulePath ?? this.modulePath); - for (const e of this.env) { - if (is(e.schemaModulePath, soughtModule)) { - const expr = (e.typescriptModuleExpr === null) ? '' : '.' + e.typescriptModuleExpr; - if (e.schema === null) { - // It's an artificial module, not from a schema. Assume the identifier is present. - return kOther(soughtModule, - M.modsymFor(e), - e.typescriptModulePath, - expr, - null, - null); - } else { - const p = e.schema.definitions.get(name.name); - if (p !== void 0) { - let t = () => typeForDefinition(this.resolver(soughtModule), p); - if (name.module.length) { - return kOther(soughtModule, - M.modsymFor(e), - e.typescriptModulePath, - expr, - p, - t); - } else { - return kLocal(p, t); - } + const e = M.envLookup(this.env, soughtModule); + if (e !== null) { + const expr = (e.typescriptModuleExpr === null) ? '' : '.' + e.typescriptModuleExpr; + if (e.schema === null) { + // It's an artificial module, not from a schema. Assume the identifier is present. + return kOther(soughtModule, + M.modsymFor(e), + e.typescriptModulePath, + expr, + null, + null); + } else { + const p = e.schema.definitions.get(name.name); + if (p !== void 0) { + let t = () => typeForDefinition(this.resolver(soughtModule), p); + if (name.module.length) { + return kOther(soughtModule, + M.modsymFor(e), + e.typescriptModulePath, + expr, + p, + t); + } else { + return kLocal(p, t); } } } diff --git a/implementations/javascript/packages/schema/src/compiler/genconverter.ts b/implementations/javascript/packages/schema/src/compiler/genconverter.ts index f4ff02a..11b146a 100644 --- a/implementations/javascript/packages/schema/src/compiler/genconverter.ts +++ b/implementations/javascript/packages/schema/src/compiler/genconverter.ts @@ -188,7 +188,7 @@ export function converterForSimple( (_p, _t) => [`${dest} = to${p.value.name.description!}(${src})`], (modPath, modId, modFile, modExpr, _p, _t) => { ctx.mod.imports.add([modPath, modId, modFile, modExpr]); - return [`${dest} = ${modId}${modExpr}.to${p.value.name.description!}(${src})`]; + return [`${dest} = ${modId}${modExpr}.to${p.value.name.description!}${ctx.mod.genericArgs()}(${src})`]; }); default: ((_p: never) => {})(p); diff --git a/implementations/javascript/packages/schema/src/meta.ts b/implementations/javascript/packages/schema/src/meta.ts index 9219350..acae08f 100644 --- a/implementations/javascript/packages/schema/src/meta.ts +++ b/implementations/javascript/packages/schema/src/meta.ts @@ -1,4 +1,4 @@ -import { GenericEmbedded, Value } from '@preserves/core'; +import { GenericEmbedded, is, Value } from '@preserves/core'; import * as M from './gen/schema'; import { isJsKeyword } from './compiler/jskw'; @@ -51,6 +51,15 @@ export type SchemaEnvEntry = { export type Environment = Array; +export function envLookup(env: Environment, soughtModule: M.ModulePath): SchemaEnvEntry | null { + for (const e of env) { + if (is(e.schemaModulePath, soughtModule)) { + return e; + } + } + return null; +} + export function modsymFor(e: SchemaEnvEntry): string { return '_i_' + e.schemaModulePath.map(s => s.description!).join('$'); }