//! Implementation of the Schema-to-Rust compiler. pub mod context; pub mod cycles; pub mod names; pub mod parsers; pub mod readers; pub mod types; pub mod unparsers; use crate::compiler::context::*; use crate::compiler::types::Purpose; use crate::gen::schema; use crate::gen::schema::*; use crate::gen::Language; use crate::syntax::block::constructors::*; use crate::syntax::block::{Formatter, Item}; use crate::*; use glob::glob; use preserves::value::BinarySource; use preserves::value::BytesBinarySource; use preserves::value::Map; use preserves::value::Reader; use preserves::value::Set; use std::fs::DirBuilder; use std::fs::File; use std::io; use std::io::Read; use std::io::Write; use std::path::PathBuf; pub type ModulePath = Vec; pub trait Plugin: std::fmt::Debug { fn generate_module(&self, _module_ctxt: &mut ModuleContext) {} fn generate_definition( &self, module_ctxt: &mut ModuleContext, definition_name: &str, definition: &Definition, ); } pub struct LanguageTypes { pub fallback: Option Set>>, pub definitions: Map Set>>, } impl std::fmt::Debug for LanguageTypes { fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { f.debug_struct("LanguageTypes") .field("fallback", &self.fallback.as_ref().map(|f| f("_"))) .field( "definitions", &self .definitions .iter() .map(|(k, f)| (k.clone(), f("_"))) .collect::>>(), ) .finish() } } #[derive(Debug)] pub struct ExternalModule { pub path: ModulePath, pub rust_namespace: String, pub rust_language_types: LanguageTypes, } impl ExternalModule { pub fn new(path: ModulePath, rust_namespace: &str) -> Self { ExternalModule { path, rust_namespace: rust_namespace.to_owned(), rust_language_types: LanguageTypes { fallback: None, definitions: Map::new(), }, } } pub fn set_fallback_language_types Set>( mut self, f: F, ) -> Self { self.rust_language_types.fallback = Some(Box::new(f)); self } pub fn set_definition_language_types Set>( mut self, d: &str, f: F, ) -> Self { if self .rust_language_types .definitions .insert(d.to_owned(), Box::new(f)) .is_some() { panic!( "Duplicate language types definition installed: {:?} {:?}", &self.path, d ); } self } } #[derive(Debug)] pub struct CompilerConfig { pub bundle: Map, pub output_dir: PathBuf, pub fully_qualified_module_prefix: String, pub support_crate: String, pub external_modules: Map, pub plugins: Vec>, pub rustfmt_skip: bool, } pub fn load_schema_or_bundle_with_purpose( bundle: &mut Map, i: &PathBuf, purpose: Purpose, ) -> io::Result<()> { let mut inserted = Map::::new(); load_schema_or_bundle(&mut inserted, i)?; for (k, v) in inserted.into_iter() { bundle.insert(k, (v, purpose)); } Ok(()) } pub fn load_schema_or_bundle_bin_with_purpose( bundle: &mut Map, prefix: &str, input: &[u8], purpose: Purpose, ) -> io::Result<()> { let mut inserted = Map::::new(); load_schema_or_bundle_bin(&mut inserted, prefix, input)?; for (k, v) in inserted.into_iter() { bundle.insert(k, (v, purpose)); } Ok(()) } fn bundle_prefix(i: &PathBuf) -> io::Result<&str> { i.file_stem() .ok_or_else(|| { io::Error::new( io::ErrorKind::InvalidData, format!("Bad schema file stem: {:?}", i), ) })? .to_str() .ok_or_else(|| { io::Error::new( io::ErrorKind::InvalidData, format!("Invalid UTF-8 in schema file name: {:?}", i), ) }) } pub fn load_schema_or_bundle(bundle: &mut Map, i: &PathBuf) -> io::Result<()> { let mut f = File::open(&i)?; let mut bs = vec![]; f.read_to_end(&mut bs)?; load_schema_or_bundle_bin(bundle, bundle_prefix(i)?, &bs[..]) } pub fn load_schema_or_bundle_bin( bundle: &mut Map, prefix: &str, input: &[u8], ) -> io::Result<()> { let mut src = BytesBinarySource::new(input); let mut reader = src.packed_iovalues(); let blob = reader.demand_next(false)?; let language = Language::default(); if let Ok(s) = language.parse(&blob) { bundle.insert(vec![prefix.to_owned()], s); } else if let Ok(Bundle { modules }) = language.parse(&blob) { for (ModulePath(k), v) in modules.0 { bundle.insert(k, v); } } else { return Err(io::Error::new( io::ErrorKind::InvalidData, format!("Invalid schema binary blob {:?}", prefix), )); } Ok(()) } impl CompilerConfig { pub fn new(output_dir: PathBuf, fully_qualified_module_prefix: String) -> Self { CompilerConfig { bundle: Map::new(), output_dir, fully_qualified_module_prefix, support_crate: "preserves_schema".to_owned(), external_modules: Map::new(), plugins: vec![ Box::new(types::TypePlugin), Box::new(readers::ReaderPlugin), Box::new(parsers::ParserPlugin), Box::new(unparsers::UnparserPlugin), ], rustfmt_skip: false, } } pub fn add_external_module(&mut self, m: ExternalModule) { let path = m.path.clone(); if self.external_modules.insert(path.clone(), m).is_some() { panic!("Duplicate external module installed: {:?}", path) } } pub fn load_schemas_and_bundles( &mut self, inputs: &Vec, xrefs: &Vec, ) -> io::Result<()> { for i in inputs { load_schema_or_bundle_with_purpose(&mut self.bundle, i, Purpose::Codegen)?; } for i in xrefs { load_schema_or_bundle_with_purpose(&mut self.bundle, i, Purpose::Xref)?; } Ok(()) } pub fn load_xref_bin(&mut self, prefix: &str, bundle_or_schema: &[u8]) -> io::Result<()> { load_schema_or_bundle_bin_with_purpose( &mut self.bundle, prefix, bundle_or_schema, Purpose::Xref, ) } fn build_type_cache(&self) -> Map { self.bundle .iter() .flat_map(|(modpath, s)| { let modpath = ModulePath(modpath.clone()); s.0.definitions.0.iter().map(move |(name, def)| { let ty = types::definition_type(&modpath, s.1, name, def); (ty.self_ref.clone(), ty) }) }) .collect() } fn generate_definition( &self, b: &mut BundleContext, k: &ModulePath, v: &Schema, n: &str, d: &Definition, mode: ModuleContextMode, generated: &mut Map>, ) { b.generate_module(k, v, mode, generated, |m| { for plugin in self.plugins.iter() { plugin.generate_definition(m, n, d); } }); } } pub fn expand_inputs(globs: &Vec) -> io::Result> { let mut result = Vec::new(); for g in globs.iter() { for p in glob(g).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("{}", e)))? { result.push(p.map_err(glob::GlobError::into_error)?) } } Ok(result) } fn write_if_changed(output_path: &PathBuf, contents: &[u8]) -> io::Result<()> { if output_path.exists() { if let Ok(mut f) = File::open(output_path) { let mut existing_contents = Vec::new(); f.read_to_end(&mut existing_contents)?; if existing_contents == contents { return Ok(()); } } } let mut f = File::create(output_path)?; f.write_all(contents) } impl Ref { pub fn qualify(&self, default_module_path: &schema::ModulePath) -> Ref { if self.module.0.is_empty() { Ref { module: default_module_path.clone(), name: self.name.clone(), } } else { self.clone() } } } impl Schema { pub fn has_embedded_type(&self) -> bool { self.embedded_type != EmbeddedTypeName::False } } pub fn compile(config: &CompilerConfig) -> io::Result<()> { let mut b = BundleContext::new(config); for (k, (v, module_purpose)) in config.bundle.iter() { if *module_purpose != Purpose::Codegen { continue; } let mut output_path = config.output_dir.clone(); output_path.extend(k); let module_name = output_path .file_stem() .unwrap() .to_str() .unwrap() .to_owned(); let module_name = names::render_modname(&module_name); output_path.set_file_name(format!("{}.rs", module_name)); DirBuilder::new() .recursive(true) .create(output_path.parent().unwrap())?; //--------------------------------------------------------------------------- let mut generated = Map::new(); b.generate_module(k, v, ModuleContextMode::TargetModule, &mut generated, |m| { for plugin in config.plugins.iter() { plugin.generate_module(m); } }); for (n, d) in &v.definitions.0 { use ModuleContextMode::*; config.generate_definition(&mut b, k, v, n, d, TargetToplevel, &mut generated); config.generate_definition(&mut b, k, v, n, d, TargetGeneric, &mut generated); } //--------------------------------------------------------------------------- let mut lines: Vec = Vec::new(); lines.push(Formatter::to_string(vertical( false, seq!["#![allow(unused_parens)]", "#![allow(unused_imports)]"], ))); if config.rustfmt_skip { lines.push("#![cfg_attr(rustfmt, rustfmt_skip)]".to_owned()); } lines.push(Formatter::to_string(vertical( false, seq![ "", "use std::convert::TryFrom;", format!("use {}::support as _support;", &config.support_crate), "use _support::Deserialize;", "use _support::Parse;", "use _support::Unparse;", "use _support::preserves;", "use preserves::value::Domain;", "use preserves::value::NestedValue;", "" ], ))); let mut emit_items = |items: Vec| { if !items.is_empty() { lines.push(Formatter::to_string(vertical(true, seq(items)))); lines.push("".to_owned()); } }; emit_items(generated.remove(&ModuleContextMode::TargetModule).unwrap()); emit_items( generated .remove(&ModuleContextMode::TargetToplevel) .unwrap(), ); emit_items(generated.remove(&ModuleContextMode::TargetGeneric).unwrap()); { let contents = lines.join("\n"); write_if_changed(&output_path, contents.as_bytes())?; } } { let mut mod_rs = config.output_dir.clone(); mod_rs.extend(vec!["mod.rs"]); let mut lines = Vec::new(); if config.rustfmt_skip { lines.push("#![cfg_attr(rustfmt, rustfmt_skip)]".to_owned()); lines.push("".to_owned()); } for (modpath, (_, module_purpose)) in config.bundle.iter() { if *module_purpose != Purpose::Codegen { continue; } lines.push(format!( "pub mod {};", names::render_modname(modpath.last().unwrap()) )); } lines.push("".to_owned()); lines.push(format!( "use {}::support as _support;", &config.support_crate )); lines.push("use _support::preserves;".to_owned()); lines.push("".to_owned()); lines.push("#[allow(non_snake_case)]".to_owned()); lines.push(Formatter::to_string(item(seq![ "pub struct ", b.language_struct_name(), anglebrackets!["N: preserves::value::NestedValue"], " ", vertical( false, braces( b.literals .iter() .map(|(value, name)| item(format!("pub {}: N /* {:?} */", name, value))) .collect() ) ) ]))); lines.push("".to_owned()); lines.push(Formatter::to_string(item(seq![ "impl", anglebrackets!["N: preserves::value::NestedValue"], " Default for ", b.language_struct_name(), " ", codeblock![seq![ "fn default() -> Self ", codeblock![seq![ b.language_struct_name(), " ", vertical( false, braces( b.literals .iter() .map(|(value, name)| { let bs = preserves::value::PackedWriter::encode_iovalue(&value) .unwrap(); item(format!( "{}: /* {:?} */ _support::decode_lit(&{:?}).unwrap()", name, value, bs )) }) .collect() ) ) ]] ]] ]))); lines.push("".to_owned()); { let mut b = Bundle { modules: Modules(Map::new()), }; for (modpath, (schema, purpose)) in config.bundle.iter() { if *purpose == Purpose::Codegen { b.modules .0 .insert(ModulePath(modpath.clone()), schema.clone()); } } let b_value = Language::default().unparse(&b); let b_bin = preserves::value::PackedWriter::encode_iovalue(&b_value).unwrap(); let mut hex_encoded_bundle = String::new(); let mut count = 0; for b in b_bin { if count % 16 == 0 { hex_encoded_bundle.push_str("\\\n "); } count += 1; hex_encoded_bundle.push_str(&format!("\\x{:02x}", b)); } lines.push(Formatter::to_string(item(seq![ "pub fn _bundle() -> &'static [u8] ", codeblock![seq!["b\"", hex_encoded_bundle, "\""]] ]))); } lines.push("".to_owned()); let contents = lines.join("\n"); write_if_changed(&mod_rs, contents.as_bytes())?; } Ok(()) }