syndicate-lang
/
preserves
mirror of https://gitlab.com/preserves/preserves.git
1
0
Fork 1
Code Releases Activity

New `compile_preserves_schemas!` macro

This commit is contained in:
Tony Garnock-Jones 2023-11-18 16:13:56 +01:00
parent 114875b52f
commit 284614eecb
12 changed files with 397 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
implementations/rust/Cargo.toml
View File

@ -1,8 +1,13 @@
[workspace]
exclude = [
"examples/schema-no-build",
]
members = [
"preserves",
"preserves-path",
"preserves-schema",
"preserves-schema-macros",
"preserves-tools",
]

10
implementations/rust/examples/schema-no-build/Cargo.toml Normal file
View File

@ -0,0 +1,10 @@
[package]
name = "schema-no-build"
version = "0.1.0"
edition = "2021"
[dependencies]
lazy_static = "1"
preserves = { path = "../../preserves" }
preserves-schema = { path = "../../preserves-schema" }
preserves-schema-macros = { path = "../../preserves-schema-macros" }

27
implementations/rust/examples/schema-no-build/src/main.rs Normal file
View File

@ -0,0 +1,27 @@
use preserves::value::IOValue;
use preserves_schema_macros::compile_preserves_schemas;
compile_preserves_schemas!(
crate::schemas,
load("<CARGO_MANIFEST_DIR>/../../../../path/path.bin"),
external_module(EntityRef = crate::demo_entity_ref),
);
pub mod demo_entity_ref {
use preserves::value::IOValue;
pub type Cap = IOValue;
}
preserves_schema::define_language!(language(): Language<IOValue> {
demo: crate::schemas::Language,
});
fn main() {
use crate::schemas::path::*;
use preserves::value::NestedValue;
use preserves_schema::support::Unparse;
println!("Hello, world! {:?}", (Filter::Compare {
op: Box::new(Comparison::Eq),
literal: IOValue::new(123),
}).unparse(language()));
}

6
implementations/rust/preserves-path/Cargo.toml
View File

@ -9,11 +9,11 @@ repository = "https://gitlab.com/preserves/preserves"
license = "Apache-2.0"
[build-dependencies]
preserves-schema = { path = "../preserves-schema", version = "4.991.0"}
preserves-schema = { path = "../preserves-schema", version = "4.992.0"}
[dependencies]
preserves = { path = "../preserves", version = "4.991.0"}
preserves-schema = { path = "../preserves-schema", version = "4.991.0"}
preserves = { path = "../preserves", version = "4.992.1"}
preserves-schema = { path = "../preserves-schema", version = "4.992.0"}
num = "0.4"
regex = "1.5"

4
implementations/rust/preserves-path/build.rs
View File

@ -9,10 +9,10 @@ fn main() -> Result<(), Error> {
let mut gen_dir = buildroot.clone();
gen_dir.push("src/schemas");
let mut c = CompilerConfig::new(gen_dir, "crate::schemas".to_owned());
let mut c = CompilerConfig::new("crate::schemas".to_owned());
let inputs = expand_inputs(&vec!["path.bin".to_owned()])?;
c.load_schemas_and_bundles(&inputs, &vec![])?;
compile(&c)
compile(&c, &mut CodeCollector::files(gen_dir))
}

2
implementations/rust/preserves-path/src/context.rs
View File

@ -34,7 +34,7 @@ impl Env {
Default::default()
}
pub fn load_bundle(&mut self, filename: &std::path::PathBuf) -> io::Result<()> {
pub fn load_bundle(&mut self, filename: &std::path::PathBuf) -> io::Result<bool> {
load_schema_or_bundle(&mut self.0, filename)
}

18
implementations/rust/preserves-schema-macros/Cargo.toml Normal file
View File

@ -0,0 +1,18 @@
[package]
name = "preserves-schema-macros"
version = "0.992.0"
edition = "2018"
[lib]
proc-macro = true
[dependencies]
preserves = { path = "../preserves", version = "4.992.1" }
preserves-schema = { path = "../preserves-schema", version = "4.992.0" }
proc-macro2 = { version = "1", features = ["span-locations"] }
quote = "1"
syn = { version = "2", features = ["parsing", "extra-traits", "full"] }
[package.metadata.workspaces]
independent = true

227
implementations/rust/preserves-schema-macros/src/lib.rs Normal file
View File

@ -0,0 +1,227 @@
use preserves::value::Map;
use preserves_schema::compiler::*;
use preserves_schema::compiler::types::Purpose;
use preserves_schema::gen::schema::Schema;
use proc_macro2::Span;
use quote::ToTokens;
use quote::quote;
use std::fmt::Display;
use syn::LitStr;
use syn::Token;
use syn::parenthesized;
use syn::parse::Parser;
use syn::punctuated::Punctuated;
mod kw {
use syn::custom_keyword;
custom_keyword!(load);
custom_keyword!(cross_reference);
custom_keyword!(external_module);
}
#[derive(Debug)]
enum Instruction {
Namespace(String),
Load(LitStr),
CrossReference {
namespace: String,
bundle_path: LitStr,
},
ExternalModule {
module_path: ModulePath,
namespace: String,
}
}
fn syn_path_string(p: syn::Path) -> String {
p.to_token_stream().to_string().replace(&[' ', '\t', '\n', '\r'], "")
}
fn syn_litstr_resolve(s: &syn::LitStr) -> String {
let s: String = s.value();
match s.chars().nth(0) {
Some('/') => s.into(),
Some('<') => match &s[1..].split_once('>') {
Some((envvar, remainder)) => match std::env::var(envvar) {
Ok(p) => p + "/" + remainder,
Err(_) => panic!("No such environment variable: {:?}", s),
}
None => panic!("Invalid relative path syntax: {:?}", s),
},
_ => panic!("Invalid path syntax: {:?}", s)
}
}
impl syn::parse::Parse for Instruction {
fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
let lookahead = input.lookahead1();
if lookahead.peek(kw::load) {
let _: kw::load = input.parse()?;
let content;
let _ = parenthesized!(content in input);
let bundle_path: syn::LitStr = content.parse()?;
Ok(Instruction::Load(bundle_path))
} else if lookahead.peek(kw::cross_reference) {
let _: kw::cross_reference = input.parse()?;
let content;
let _ = parenthesized!(content in input);
let namespace: syn::Path = content.parse()?;
let _: Token![=] = content.parse()?;
let bundle_path: syn::LitStr = content.parse()?;
Ok(Instruction::CrossReference {
namespace: syn_path_string(namespace),
bundle_path,
})
} else if lookahead.peek(kw::external_module) {
let _: kw::external_module = input.parse()?;
let content;
let _ = parenthesized!(content in input);
let module_path = Punctuated::<syn::Ident, syn::Token![.]>::parse_separated_nonempty(&content)?;
let _: Token![=] = content.parse()?;
let namespace: syn::Path = content.parse()?;
Ok(Instruction::ExternalModule {
module_path: module_path.into_iter().map(|p| p.to_string()).collect(),
namespace: syn_path_string(namespace),
})
} else {
let ns: syn::Path = input.parse()?;
Ok(Instruction::Namespace(syn_path_string(ns)))
}
}
}
struct ModuleTree {
own_body: String,
children: Map<String, ModuleTree>,
}
impl Default for ModuleTree {
fn default() -> Self {
ModuleTree {
own_body: String::new(),
children: Map::default(),
}
}
}
impl ModuleTree {
fn build(outputs: Map<Option<ModulePath>, String>) -> Self {
let mut mt = ModuleTree::default();
for (p, c) in outputs.into_iter() {
match p {
None => mt.own_body = c,
Some(k) => {
let mut r = &mut mt;
for e in k { r = mt.children.entry(names::render_modname(&e)).or_default(); }
r.own_body = c;
}
}
}
mt
}
}
impl Display for ModuleTree {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.own_body)?;
for (label, mt) in self.children.iter() {
write!(f, "\npub mod {} {{ ", label)?;
mt.fmt(f)?;
write!(f, "}}")?;
}
Ok(())
}
}
#[proc_macro]
pub fn compile_preserves_schemas(src: proc_macro::TokenStream) -> proc_macro::TokenStream {
let instructions = Punctuated::<Instruction, syn::Token![,]>::parse_terminated
.parse(src)
.expect("valid sequence of compile_preserves_schemas instructions");
let mut namespace = None::<String>;
let mut bundles_to_load = Vec::<LitStr>::new();
let mut bundles_to_xref = Vec::<LitStr>::new();
let mut external_modules = Vec::<ExternalModule>::new();
for i in instructions.into_iter() {
match i {
Instruction::Namespace(n) => {
if namespace.is_some() {
panic!("Only one namespace is permitted")
}
namespace = Some(n)
}
Instruction::Load(p) => bundles_to_load.push(p),
Instruction::ExternalModule { module_path, namespace } => {
external_modules.push(ExternalModule::new(module_path, &namespace));
}
Instruction::CrossReference { namespace, bundle_path } => {
let mut bundle = Map::<ModulePath, Schema>::new();
let is_schema = load_schema_or_bundle(&mut bundle, &syn_litstr_resolve(&bundle_path).into())
.expect("Invalid schema/bundle binary");
bundles_to_xref.push(bundle_path);
for (k, _v) in bundle.into_iter() {
external_modules.push(if is_schema {
ExternalModule::new(k, &namespace)
} else {
let ns = namespace.clone();
let mut pieces = vec![ns.clone()];
pieces.extend(
k.iter().map(|p| names::render_modname(&p)).collect::<Vec<_>>());
ExternalModule::new(k, &pieces.join("::"))
.set_fallback_language_types(
move |v| vec![format!("{}::Language<{}>", ns, v)].into_iter().collect())
});
}
}
}
}
let namespace = namespace.expect("Missing namespace");
let mut dependency_paths = Vec::<syn::LitStr>::new();
let mut c = CompilerConfig::new(namespace.clone());
for b in bundles_to_load.into_iter() {
dependency_paths.push(syn::LitStr::new(&syn_litstr_resolve(&b), Span::call_site()));
load_schema_or_bundle_with_purpose(&mut c.bundle, &syn_litstr_resolve(&b).into(), Purpose::Codegen)
.expect(&b.value());
}
for b in bundles_to_xref.into_iter() {
dependency_paths.push(syn::LitStr::new(&syn_litstr_resolve(&b), Span::call_site()));
load_schema_or_bundle_with_purpose(&mut c.bundle, &syn_litstr_resolve(&b).into(), Purpose::Xref)
.expect(&b.value());
}
for m in external_modules.into_iter() {
c.add_external_module(m);
}
let mut outputs = Map::<Option<ModulePath>, String>::new();
let mut collector = CodeCollector {
emit_mod_declarations: false,
collect_module: CodeModuleCollector::Custom {
collect_output: &mut |p, c| {
outputs.insert(p.cloned(), c.to_owned());
Ok(())
},
},
};
compile(&c, &mut collector).expect("Compilation failed");
let top_module_source = format!(
"pub mod {} {{ {} }}",
names::render_modname(namespace.split("::").last().unwrap()),
ModuleTree::build(outputs));
let top_module: syn::Item = syn::parse_str(&top_module_source)
.expect("Invalid generated code");
quote!{
// TODO: this is ugly, but makes the code depend on the actual schema bundle:
// See https://doc.rust-lang.org/nightly/proc_macro/tracked_path/fn.path.html
// and https://github.com/rust-lang/rust/issues/99515
#( const _: &'static [u8] = include_bytes!(#dependency_paths); )*
#top_module
}.into()
}

2
implementations/rust/preserves-schema/Cargo.toml
View File

@ -9,7 +9,7 @@ repository = "https://gitlab.com/preserves/preserves"
license = "Apache-2.0"
[dependencies]
preserves = { path = "../preserves", version = "4.991.0"}
preserves = { path = "../preserves", version = "4.992.1"}
convert_case = "0.4.0"
glob = "0.3.0"

10
implementations/rust/preserves-schema/src/bin/preserves-schema-rs.rs
View File

@ -6,7 +6,11 @@ use std::io::ErrorKind;
use std::path::PathBuf;
use structopt::StructOpt;
use preserves_schema::compiler::{compile, expand_inputs, CompilerConfig, ExternalModule};
use preserves_schema::compiler::CodeCollector;
use preserves_schema::compiler::CompilerConfig;
use preserves_schema::compiler::ExternalModule;
use preserves_schema::compiler::compile;
use preserves_schema::compiler::expand_inputs;
#[derive(Clone, StructOpt, Debug)]
struct CommandLine {
@ -33,7 +37,7 @@ struct CommandLine {
fn main() -> Result<(), Error> {
let args = CommandLine::from_args();
let mut config = CompilerConfig::new(args.output_dir, args.prefix);
let mut config = CompilerConfig::new(args.prefix);
for alias in args.module {
let (modulepath_str, target) = {
let pieces: Vec<&str> = alias.split('=').collect();
@ -56,5 +60,5 @@ fn main() -> Result<(), Error> {
&expand_inputs(&args.input_glob)?,
&expand_inputs(&args.xref)?,
)?;
compile(&config)
compile(&config, &mut CodeCollector::files(args.output_dir))
}

143
implementations/rust/preserves-schema/src/compiler/mod.rs
View File

@ -16,11 +16,11 @@
//!
//! let mut gen_dir = buildroot.clone();
//! gen_dir.push("src/schemas");
//! let mut c = CompilerConfig::new(gen_dir, "crate::schemas".to_owned());
//! let mut c = CompilerConfig::new("crate::schemas".to_owned());
//!
//! let inputs = expand_inputs(&vec![PATH_TO_PRB_FILE.to_owned()])?;
//! c.load_schemas_and_bundles(&inputs, &vec![])?;
//! compile(&c)
//! compile(&c, &mut CodeCollector::files(gen_dir))
//! }
//! ```
//!
@ -153,13 +153,31 @@ impl ExternalModule {
}
}
/// Used to collect output from the compiler.
pub enum CodeModuleCollector<'a> {
/// Default file-based code emitter.
Files {
/// Where output Rust code files will be placed.
output_dir: PathBuf,
},
Custom {
/// Used to collect the various produced source files.
/// Useful for when compiling in e.g. proc_macro context.
collect_output: &'a mut dyn FnMut(Option<&ModulePath>, &str) -> io::Result<()>,
},
}
/// Used to configure and collect output from the compiler.
pub struct CodeCollector<'a> {
pub emit_mod_declarations: bool,
pub collect_module: CodeModuleCollector<'a>,
}
/// Main entry point to the compiler.
#[derive(Debug)]
pub struct CompilerConfig {
/// All known Schema modules, indexed by [ModulePath] and annotated with a [Purpose].
pub bundle: Map<ModulePath, (Schema, Purpose)>,
/// Where output Rust code files will be placed.
pub output_dir: PathBuf,
/// Fully-qualified Rust module prefix to use for each generated module.
pub fully_qualified_module_prefix: String,
/// Rust module path to the [preserves_schema::support][crate::support] module.
@ -230,7 +248,9 @@ fn bundle_prefix(i: &PathBuf) -> io::Result<&str> {
///
/// If `i` holds a [Schema], then the file stem of `i` is used as the module name when placing
/// the schema in `bundle`.
pub fn load_schema_or_bundle(bundle: &mut Map<ModulePath, Schema>, i: &PathBuf) -> io::Result<()> {
///
/// Returns true if it was a schema, false if it was a bundle.
pub fn load_schema_or_bundle(bundle: &mut Map<ModulePath, Schema>, i: &PathBuf) -> io::Result<bool> {
let mut f = File::open(&i)?;
let mut bs = vec![];
f.read_to_end(&mut bs)?;
@ -241,11 +261,13 @@ pub fn load_schema_or_bundle(bundle: &mut Map<ModulePath, Schema>, i: &PathBuf)
///
/// If `input` corresponds to a [Schema], then `prefix` is used as its module name; otherwise,
/// it's a [Bundle], and `prefix` is ignored.
///
/// Returns true if it was a schema, false if it was a bundle.
pub fn load_schema_or_bundle_bin(
bundle: &mut Map<ModulePath, Schema>,
prefix: &str,
input: &[u8],
) -> io::Result<()> {
) -> io::Result<bool> {
let mut src = BytesBinarySource::new(input);
let mut reader = src.packed_iovalues();
let blob = reader.demand_next(false)?;
@ -253,27 +275,26 @@ pub fn load_schema_or_bundle_bin(
if let Ok(s) = language.parse(&blob) {
bundle.insert(vec![prefix.to_owned()], s);
Ok(true)
} else if let Ok(Bundle { modules }) = language.parse(&blob) {
for (ModulePath(k), v) in modules.0 {
bundle.insert(k, v);
}
Ok(false)
} else {
return Err(io::Error::new(
Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("Invalid schema binary blob {:?}", prefix),
));
))
}
Ok(())
}
impl CompilerConfig {
/// Construct a [CompilerConfig] configured to send output files to `output_dir`, and to
/// use `fully_qualified_module_prefix` as the Rust module prefix for generated code.
pub fn new(output_dir: PathBuf, fully_qualified_module_prefix: String) -> Self {
/// Construct a [CompilerConfig] configured to use `fully_qualified_module_prefix` as the
/// Rust module prefix for generated code.
pub fn new(fully_qualified_module_prefix: String) -> Self {
CompilerConfig {
bundle: Map::new(),
output_dir,
fully_qualified_module_prefix,
support_crate: "preserves_schema".to_owned(),
external_modules: Map::new(),
@ -361,18 +382,54 @@ pub fn expand_inputs(globs: &Vec<String>) -> io::Result<Vec<PathBuf>> {
Ok(result)
}
fn write_if_changed(output_path: &PathBuf, contents: &[u8]) -> io::Result<()> {
if output_path.exists() {
if let Ok(mut f) = File::open(output_path) {
let mut existing_contents = Vec::new();
f.read_to_end(&mut existing_contents)?;
if existing_contents == contents {
return Ok(());
impl<'a> CodeCollector<'a> {
/// Construct a [CodeCollector] that collects output Rust modules directly into the file
/// system tree rooted at `output_dir`.
pub fn files(output_dir: PathBuf) -> Self {
CodeCollector {
emit_mod_declarations: true,
collect_module: CodeModuleCollector::Files { output_dir },
}
}
#[doc(hidden)]
pub fn collect_output(&mut self, module: Option<&ModulePath>, contents: &str) -> io::Result<()> {
match &mut self.collect_module {
CodeModuleCollector::Files { output_dir } => {
let mut output_path = output_dir.clone();
if let Some(k) = module {
output_path.extend(k);
let module_name = output_path
.file_stem()
.unwrap()
.to_str()
.unwrap()
.to_owned();
let module_name = names::render_modname(&module_name);
output_path.set_file_name(format!("{}.rs", module_name));
} else {
output_path.push("mod.rs");
}
DirBuilder::new().recursive(true).create(output_path.parent().unwrap())?;
if output_path.exists() {
if let Ok(mut f) = File::open(&output_path) {
let mut existing_contents = String::new();
f.read_to_string(&mut existing_contents)?;
if existing_contents == contents {
return Ok(());
}
}
}
let mut f = File::create(output_path)?;
f.write_all(contents.as_bytes())
}
CodeModuleCollector::Custom { collect_output } => {
collect_output(module, contents)
}
}
}
let mut f = File::create(output_path)?;
f.write_all(contents)
}
impl Ref {
@ -395,7 +452,7 @@ impl Schema {
}
/// Main entry point: runs the compilation process.
pub fn compile(config: &CompilerConfig) -> io::Result<()> {
pub fn compile<'a>(config: &CompilerConfig, emitter: &mut CodeCollector<'a>) -> io::Result<()> {
let mut b = BundleContext::new(config);
for (k, (v, module_purpose)) in config.bundle.iter() {
@ -403,20 +460,6 @@ pub fn compile(config: &CompilerConfig) -> io::Result<()> {
continue;
}
let mut output_path = config.output_dir.clone();
output_path.extend(k);
let module_name = output_path
.file_stem()
.unwrap()
.to_str()
.unwrap()
.to_owned();
let module_name = names::render_modname(&module_name);
output_path.set_file_name(format!("{}.rs", module_name));
DirBuilder::new()
.recursive(true)
.create(output_path.parent().unwrap())?;
//---------------------------------------------------------------------------
let mut generated = Map::new();
@ -476,13 +519,11 @@ pub fn compile(config: &CompilerConfig) -> io::Result<()> {
{
let contents = lines.join("\n");
write_if_changed(&output_path, contents.as_bytes())?;
emitter.collect_output(Some(k), &contents)?;
}
}
{
let mut mod_rs = config.output_dir.clone();
mod_rs.extend(vec!["mod.rs"]);
let mut lines = Vec::new();
if config.rustfmt_skip {
@ -490,16 +531,18 @@ pub fn compile(config: &CompilerConfig) -> io::Result<()> {
lines.push("".to_owned());
}
for (modpath, (_, module_purpose)) in config.bundle.iter() {
if *module_purpose != Purpose::Codegen {
continue;
if emitter.emit_mod_declarations {
for (modpath, (_, module_purpose)) in config.bundle.iter() {
if *module_purpose != Purpose::Codegen {
continue;
}
lines.push(format!(
"pub mod {};",
names::render_modname(modpath.last().unwrap())
));
}
lines.push(format!(
"pub mod {};",
names::render_modname(modpath.last().unwrap())
));
lines.push("".to_owned());
}
lines.push("".to_owned());
lines.push(format!(
"use {}::support as _support;",
@ -586,7 +629,7 @@ pub fn compile(config: &CompilerConfig) -> io::Result<()> {
lines.push("".to_owned());
let contents = lines.join("\n");
write_if_changed(&mod_rs, contents.as_bytes())?;
emitter.collect_output(None, &contents)?;
}
Ok(())

6
implementations/rust/preserves-tools/Cargo.toml
View File

@ -9,9 +9,9 @@ repository = "https://gitlab.com/preserves/preserves"
license = "Apache-2.0"
[dependencies]
preserves = { path = "../preserves", version = "4.991.0"}
preserves-path = { path = "../preserves-path", version = "4.991.0"}
preserves-schema = { path = "../preserves-schema", version = "4.991.0"}
preserves = { path = "../preserves", version = "4.992.1"}
preserves-path = { path = "../preserves-path", version = "4.992.0"}
preserves-schema = { path = "../preserves-schema", version = "4.992.0"}
bytes = "1.0"
clap = { version = "3", features = ["derive"] }