Better compiler API

This commit is contained in:
Tony Garnock-Jones 2021-06-29 22:32:35 +02:00
parent dd9e190bed
commit c3bc678a46
12 changed files with 194 additions and 123 deletions

View File

@ -2,5 +2,8 @@ all:
@echo please use cargo
regenerate:
cargo run -- -o $(CURDIR)/src --prefix schema ../../../schema/schema.bin > src/metaschema.rs.tmp
mv src/metaschema.rs.tmp src/metaschema.rs
cargo run -- \
--prefix crate::gen \
--support-crate crate \
-o $(CURDIR)/src/gen \
../../../schema/schema.bin

View File

@ -1,15 +1,8 @@
use glob::glob;
use preserves::value::packed::PackedReader;
use preserves::value::Reader;
use std::convert::TryFrom;
use std::fs::File;
use std::io::Error;
use std::io::ErrorKind;
use std::path::PathBuf;
use structopt::StructOpt;
use preserves_schema::compiler::{CompilerConfig, compile};
use preserves_schema::metaschema::*;
use preserves_schema::compiler::{CompilerConfig, compile, expand_inputs};
#[derive(Clone, StructOpt, Debug)]
struct CommandLine {
@ -17,59 +10,20 @@ struct CommandLine {
output_dir: PathBuf,
#[structopt(long)]
prefix: Option<String>,
prefix: String,
#[structopt(long)]
support_crate: Option<String>,
input_glob: Vec<String>,
}
fn inputs(globs: &Vec<String>) -> Vec<PathBuf> {
let mut result = Vec::new();
for g in globs.iter() {
match glob(g) {
Ok(paths) =>
for p in paths {
match p {
Ok(s) => result.push(s),
Err(e) => println!("warning: {:?}", e),
}
}
Err(e) => println!("warning: {:?}", e),
}
}
result
}
fn main() -> Result<(), Error> {
let args = CommandLine::from_args();
let prefix = match &args.prefix {
Some(s) => s.split(".").map(str::to_string).collect(),
None => vec![],
};
let mut config = CompilerConfig::new(args.output_dir);
for i in inputs(&args.input_glob) {
let mut f = File::open(&i)?;
let mut reader = PackedReader::decode_read(&mut f);
let schema = reader.demand_next(false)?;
match Schema::try_from(&schema) {
Ok(s) => {
config.bundle.insert(prefix.clone(), s);
()
},
Err(()) => match Bundle::try_from(&schema) {
Ok(Bundle { modules }) => {
for (ModulePath(k), v) in modules.0 {
let mut name = prefix.clone();
name.extend(k);
config.bundle.insert(name, v);
}
},
Err(()) => return Err(ErrorKind::InvalidData)?,
},
}
let mut config = CompilerConfig::new(args.output_dir, args.prefix);
if let Some(c) = args.support_crate {
config.support_crate = c;
}
config.load_schemas_and_bundles(&expand_inputs(&args.input_glob)?)?;
compile(&config)
}

View File

@ -1,17 +1,20 @@
use preserves::value::{Map, IOValue};
use crate::syntax::block::Item;
use crate::syntax::block::constructors::*;
use super::CompilerConfig;
use super::types;
pub struct ModuleContext {
pub struct ModuleContext<'m> {
pub config: &'m CompilerConfig,
pub literals: Map<IOValue, String>,
pub typedefs: Vec<Item>,
pub functiondefs: Vec<Item>,
}
pub struct FunctionContext<'a> {
pub m: &'a mut ModuleContext,
pub struct FunctionContext<'a, 'm> {
pub m: &'a mut ModuleContext<'m>,
pub temp_counter: usize,
pub captures: Vec<Capture>,
}
@ -22,9 +25,10 @@ pub struct Capture {
pub source_expr: String,
}
impl ModuleContext {
pub fn new() -> Self {
impl<'m> ModuleContext<'m> {
pub fn new(config: &'m CompilerConfig) -> Self {
ModuleContext {
config: config,
literals: Map::new(),
typedefs: Vec::new(),
functiondefs: Vec::new(),
@ -44,10 +48,22 @@ impl ModuleContext {
let i = f(FunctionContext::new(self));
self.functiondefs.push(i)
}
pub fn render_ref(&self, mp: Vec<String>, n: String) -> Item {
if mp.len() == 0 {
item(n)
} else {
let mut items = Vec::new();
items.push(item(self.config.fully_qualified_module_prefix.to_owned()));
for p in mp { items.push(item(p)); }
items.push(item(n));
item(name(items))
}
}
}
impl<'a> FunctionContext<'a> {
pub fn new(m: &'a mut ModuleContext) -> Self {
impl<'a, 'm> FunctionContext<'a, 'm> {
pub fn new(m: &'a mut ModuleContext<'m>) -> Self {
FunctionContext {
m: m,
temp_counter: 0,

View File

@ -5,11 +5,25 @@ pub mod parsers;
pub mod unparsers;
pub mod codegen;
use std::path::PathBuf;
use preserves::value::Map;
use crate::gen::schema::*;
use crate::syntax::block::Formatter;
use crate::syntax::block::constructors::*;
use crate::metaschema::*;
use glob::glob;
use preserves::value::Map;
use preserves::value::PackedReader;
use preserves::value::Reader;
use preserves::value::Set;
use std::convert::TryFrom;
use std::fs::DirBuilder;
use std::fs::File;
use std::io::BufRead;
use std::io::BufReader;
use std::io::Error;
use std::io::ErrorKind;
use std::io::Write;
use std::path::PathBuf;
pub type ModulePath = Vec<String>;
@ -17,15 +31,60 @@ pub type ModulePath = Vec<String>;
pub struct CompilerConfig {
pub bundle: Map<ModulePath, Schema>,
pub output_dir: PathBuf,
pub fully_qualified_module_prefix: String,
pub support_crate: String,
}
impl CompilerConfig {
pub fn new(output_dir: PathBuf) -> Self {
pub fn new(
output_dir: PathBuf,
fully_qualified_module_prefix: String,
) -> Self {
CompilerConfig {
bundle: Map::new(),
output_dir: output_dir,
fully_qualified_module_prefix: fully_qualified_module_prefix,
support_crate: "preserves_schema".to_owned(),
}
}
pub fn load_schemas_and_bundles(&mut self, inputs: &Vec<PathBuf>) -> Result<(), Error> {
for i in inputs {
let mut f = File::open(&i)?;
let mut reader = PackedReader::decode_read(&mut f);
let blob = reader.demand_next(false)?;
if let Ok(s) = Schema::try_from(&blob) {
let prefix = i.file_stem().ok_or_else(
|| Error::new(ErrorKind::InvalidData, format!("Bad schema file stem: {:?}", i)))?
.to_str().ok_or_else(
|| Error::new(ErrorKind::InvalidData, format!("Invalid UTF-8 in schema file name: {:?}", i)))?;
self.bundle.insert(vec![prefix.to_owned()], s);
return Ok(());
}
if let Ok(Bundle { modules }) = Bundle::try_from(&blob) {
for (ModulePath(k), v) in modules.0 {
self.bundle.insert(k, v);
}
return Ok(());
}
return Err(Error::new(ErrorKind::InvalidData,
format!("Invalid schema binary blob {:?}", i)));
}
Ok(())
}
}
pub fn expand_inputs(globs: &Vec<String>) -> Result<Vec<PathBuf>, Error> {
let mut result = Vec::new();
for g in globs.iter() {
for p in glob(g).map_err(|e| Error::new(ErrorKind::InvalidData, format!("{}", e)))? {
result.push(p.map_err(glob::GlobError::into_error)?)
}
}
Ok(result)
}
pub fn compile(config: &CompilerConfig) -> Result<(), std::io::Error> {
@ -33,39 +92,71 @@ pub fn compile(config: &CompilerConfig) -> Result<(), std::io::Error> {
let mut output_path = config.output_dir.clone();
output_path.extend(k);
output_path.set_extension("rs");
let mut m = context::ModuleContext::new();
// println!("\n{:?}", &output_path);
let module_name = output_path.file_stem().unwrap().to_str().unwrap().to_owned();
DirBuilder::new().recursive(true).create(output_path.parent().unwrap())?;
let mut m = context::ModuleContext::new(config);
// TODO: embedded type
for (n, d) in &v.definitions.0 {
m.define_type(item(types::render_definition_type(n, &types::definition_type(d))));
m.define_type(item(types::render_definition_type(&m, n, &types::definition_type(d))));
parsers::gen_definition_parser(&mut m, n, d);
unparsers::gen_definition_unparser(&mut m, n, d);
}
//---------------------------------------------------------------------------
println!("#![allow(unused_parens)]");
println!();
println!("use std::convert::TryFrom;");
println!("use preserves::value::NestedValue;");
println!("use lazy_static::lazy_static;");
println!();
let mut lines: Vec<String> = Vec::new();
println!("lazy_static! {{");
lines.push("#![allow(unused_parens)]".to_owned());
lines.push("".to_owned());
lines.push("use std::convert::TryFrom;".to_owned());
lines.push("use preserves::value::NestedValue;".to_owned());
lines.push(format!("use {}::support::*;", &config.support_crate));
lines.push("".to_owned());
lines.push("lazy_static! {".to_owned());
for (value, name) in m.literals {
let bs = preserves::value::PackedWriter::encode(&value).unwrap();
println!(" pub static ref {}: preserves::value::IOValue = /* {:?} */ preserves::value::packed::from_bytes(&vec!{:?}).unwrap();",
name,
value,
bs);
lines.push(format!(" pub static ref {}: preserves::value::IOValue = /* {:?} */ preserves::value::packed::from_bytes(&vec!{:?}).unwrap();",
name,
value,
bs));
}
println!("}}\n");
lines.push("}".to_owned());
lines.push("".to_owned());
for i in m.typedefs { println!("{:?}\n", i); }
for i in m.functiondefs { println!("{:?}\n", i); }
for i in m.typedefs {
lines.push(Formatter::to_string(i));
lines.push("".to_owned());
}
for i in m.functiondefs {
lines.push(Formatter::to_string(i));
lines.push("".to_owned());
}
{
let mut f = File::create(&output_path)?;
for line in lines {
write!(f, "{}\n", line)?;
}
}
{
let mut mod_rs = output_path.clone();
mod_rs.set_file_name("mod.rs");
let mut lines = if !mod_rs.exists() {
Set::new()
} else {
BufReader::new(File::open(&mod_rs)?)
.lines().collect::<Result<Set<String>, Error>>()?
};
lines.insert(format!("pub mod {};", &module_name));
let mut f = File::create(mod_rs)?;
for line in lines {
write!(f, "{}\n", line)?;
}
}
}
Ok(())
}

View File

@ -1,13 +1,5 @@
use crate::syntax::block::Emittable;
use crate::syntax::block::constructors::*;
use convert_case::{Case, Casing};
pub fn render_ref(pieces: Vec<String>) -> impl Emittable {
let mut items = Vec::new();
for p in pieces { items.push(item(p)); }
name(items)
}
pub fn render_constructor(n: &str) -> String {
n.to_case(Case::UpperCamel)
}

View File

@ -1,5 +1,5 @@
use crate::*;
use crate::metaschema::*;
use crate::gen::schema::*;
use crate::syntax::block::Item;
use crate::syntax::block::constructors::*;
@ -85,7 +85,7 @@ fn store_wrap(is_struct: bool, ty: &TField, expr: &str) -> String {
| TField::Array(_)
| TField::Set(_)
| TField::Map(_, _) => expr.to_owned(),
TField::Ref(_) =>
TField::Ref(_, _) =>
if is_struct {
expr.to_owned()
} else {
@ -174,9 +174,7 @@ fn simple_pattern_parser(
},
SimplePattern::Ref(r) => {
let Ref { module: ModulePath(module), name } = &**r;
let mut mp = module.clone();
mp.push(name.to_owned());
let tf = name![names::render_ref(mp), "try_from"];
let tf = name![ctxt.m.render_ref(module.clone(), name.to_owned()), "try_from"];
push_let(body,
item(dest.to_owned()),
item(seq![tf, parens![src.to_owned()], "?"]));

View File

@ -1,7 +1,9 @@
use crate::*;
use crate::syntax::block::Emittable;
use crate::syntax::block::constructors::*;
use crate::metaschema::*;
use crate::gen::schema::*;
use super::context::ModuleContext;
use super::names;
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
@ -22,7 +24,7 @@ pub enum TField {
Array(Box<TField>),
Set(Box<TField>),
Map(Box<TField>, Box<TField>),
Ref(Vec<String>),
Ref(Vec<String>, String),
Base(String),
}
@ -146,51 +148,62 @@ pub fn field_type(p: &SimplePattern) -> TField {
TField::Map(Box::new(field_type(k)), Box::new(field_type(v))),
SimplePattern::Ref(r) => {
let Ref { module: ModulePath(mp), name: n } = &**r;
let mut pieces = mp.clone();
pieces.push(n.to_owned());
TField::Ref(pieces)
TField::Ref(mp.clone(), n.to_owned())
}
}
}
pub fn render_field_type(box_needed: bool, t: &TField) -> impl Emittable {
pub fn render_field_type(
ctxt: &ModuleContext,
box_needed: bool,
t: &TField,
) -> impl Emittable {
match t {
TField::Unit => seq!["()"],
TField::Array(t) => seq!["std::vec::Vec<", render_field_type(false, t), ">"],
TField::Set(t) => seq!["preserves::value::Set<", render_field_type(false, t), ">"],
TField::Array(t) => seq!["std::vec::Vec<", render_field_type(ctxt, false, t), ">"],
TField::Set(t) => seq!["preserves::value::Set<", render_field_type(ctxt, false, t), ">"],
TField::Map(k, v) => seq!["preserves::value::Map",
anglebrackets![render_field_type(false, k),
render_field_type(false, v)]],
TField::Ref(pieces) =>
anglebrackets![render_field_type(ctxt, false, k),
render_field_type(ctxt, false, v)]],
TField::Ref(mp, n) =>
if box_needed {
seq!["std::boxed::Box", anglebrackets![names::render_ref(pieces.clone())]]
seq!["std::boxed::Box", anglebrackets![ctxt.render_ref(mp.clone(), n.to_owned())]]
} else {
seq![names::render_ref(pieces.clone())]
seq![ctxt.render_ref(mp.clone(), n.to_owned())]
},
TField::Base(n) => seq![n.to_owned()],
}
}
pub fn render_recordlike_type(is_struct: bool, n: &str, d: &TSimple) -> impl Emittable {
pub fn render_recordlike_type(
ctxt: &ModuleContext,
is_struct: bool,
n: &str,
d: &TSimple,
) -> impl Emittable {
let semi = if is_struct { seq![";"] } else { seq![] };
let ppub = if is_struct { "pub " } else { "" };
seq![names::render_constructor(n), match d {
TSimple::Record(TRecord(fs)) => seq![" ", braces(
fs.iter().map(|(n, d)| item(
seq![ppub, names::render_fieldname(n), ": ", render_field_type(!is_struct, d)]
seq![ppub, names::render_fieldname(n), ": ", render_field_type(ctxt, !is_struct, d)]
)).collect())],
TSimple::Field(TField::Unit) => semi,
TSimple::Field(t) => seq![parens![seq![ppub, render_field_type(!is_struct, t)]], semi],
TSimple::Field(t) => seq![parens![seq![ppub, render_field_type(ctxt, !is_struct, t)]], semi],
}]
}
pub fn render_definition_type(n: &str, t: &TDefinition) -> impl Emittable {
pub fn render_definition_type(
ctxt: &ModuleContext,
n: &str,
t: &TDefinition,
) -> impl Emittable {
seq!["#[derive(Debug, PartialOrd, Ord, PartialEq, Eq, Clone)]\n",
match t {
TDefinition::Union(items) =>
seq!["pub enum ", names::render_constructor(n), " ", braces(
items.iter().map(|(n, d)| item(render_recordlike_type(false, n, d))).collect())],
items.iter().map(|(n, d)| item(render_recordlike_type(ctxt, false, n, d))).collect())],
TDefinition::Simple(s) =>
seq!["pub struct ", render_recordlike_type(true, n, s)],
seq!["pub struct ", render_recordlike_type(ctxt, true, n, s)],
}]
}

View File

@ -1,5 +1,5 @@
use crate::*;
use crate::metaschema::*;
use crate::gen::schema::*;
use crate::syntax::block::constructors::*;
use crate::syntax::block::{Emittable, Item};

View File

@ -0,0 +1 @@
pub mod schema;

View File

@ -2,7 +2,7 @@
use std::convert::TryFrom;
use preserves::value::NestedValue;
use lazy_static::lazy_static;
use crate::support::*;
lazy_static! {
pub static ref LIT15: preserves::value::IOValue = /* #f */ preserves::value::packed::from_bytes(&vec![128]).unwrap();

View File

@ -1,6 +1,7 @@
pub mod syntax;
pub mod compiler;
pub mod metaschema;
pub mod support;
pub mod gen;
#[cfg(test)]
mod tests {
@ -37,7 +38,7 @@ mod tests {
let mut f = std::fs::File::open("../../../schema/schema.bin")?;
let mut reader = preserves::value::PackedReader::decode_read(&mut f);
let schema = reader.demand_next(false)?;
let parsed = crate::metaschema::Schema::try_from(&schema).expect("successful parse");
let parsed = crate::gen::schema::Schema::try_from(&schema).expect("successful parse");
assert_eq!(schema, IOValue::from(&parsed));
Ok(())
}

View File

@ -0,0 +1,2 @@
pub use lazy_static::lazy_static;