From 405c2c0c7c9539aec954d09e59b0c8726c928ac3 Mon Sep 17 00:00:00 2001 From: Khushboo Desai Date: Tue, 26 Mar 2024 12:15:47 -0700 Subject: [PATCH 1/6] Adds changes to struct template * Add ISL type name in the Field to call read/write APIs for symbol/string or blob/clob accordingly. * Adds manifest directory in the path for the templates --- .../ion/commands/beta/generate/generator.rs | 24 +++++++++++--- .../beta/generate/templates/rust/struct.templ | 33 ++++++++++--------- src/bin/ion/commands/beta/generate/utils.rs | 1 + 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/src/bin/ion/commands/beta/generate/generator.rs b/src/bin/ion/commands/beta/generate/generator.rs index 570fb1d..519c43c 100644 --- a/src/bin/ion/commands/beta/generate/generator.rs +++ b/src/bin/ion/commands/beta/generate/generator.rs @@ -29,7 +29,11 @@ impl<'a> CodeGenerator<'a, RustLanguage> { Self { output, anonymous_type_counter: 0, - tera: Tera::new("src/bin/ion/commands/beta/generate/templates/rust/*.templ").unwrap(), + tera: Tera::new(&format!( + "{}/src/bin/ion/commands/beta/generate/templates/rust/*.templ", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(), phantom: PhantomData, } } @@ -78,7 +82,11 @@ impl<'a> CodeGenerator<'a, JavaLanguage> { Self { output, anonymous_type_counter: 0, - tera: Tera::new("src/bin/ion/commands/beta/generate/templates/java/*.templ").unwrap(), + tera: Tera::new(&format!( + "{}/src/bin/ion/commands/beta/generate/templates/java/*.templ", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(), phantom: PhantomData, } } @@ -302,6 +310,7 @@ impl<'a, L: Language> CodeGenerator<'a, L> { self.generate_struct_field( tera_fields, L::target_type_as_sequence(&type_name), + isl_type.name(), "value", )?; } @@ -315,7 +324,12 @@ impl<'a, L: Language> CodeGenerator<'a, L> { let type_name = self.type_reference_name(value.type_reference(), modules, imports)?; - self.generate_struct_field(tera_fields, type_name, name)?; + self.generate_struct_field( + tera_fields, + type_name, + value.type_reference().name(), + name, + )?; } } IslConstraintValue::Type(isl_type) => { @@ -325,7 +339,7 @@ impl<'a, L: Language> CodeGenerator<'a, L> { AbstractDataType::Value, code_gen_context, )?; - self.generate_struct_field(tera_fields, type_name, "value")?; + self.generate_struct_field(tera_fields, type_name, isl_type.name(), "value")?; } _ => {} } @@ -337,11 +351,13 @@ impl<'a, L: Language> CodeGenerator<'a, L> { &mut self, tera_fields: &mut Vec, abstract_data_type_name: String, + isl_type_name: String, field_name: &str, ) -> CodeGenResult<()> { tera_fields.push(Field { name: field_name.to_string(), value: abstract_data_type_name, + isl_type_name, }); Ok(()) } diff --git a/src/bin/ion/commands/beta/generate/templates/rust/struct.templ b/src/bin/ion/commands/beta/generate/templates/rust/struct.templ index 5290006..ba8b0a7 100644 --- a/src/bin/ion/commands/beta/generate/templates/rust/struct.templ +++ b/src/bin/ion/commands/beta/generate/templates/rust/struct.templ @@ -29,12 +29,12 @@ impl {{ target_kind_name }} { pub fn read_from(reader: &mut Reader) -> IonResult { let mut abstract_data_type = {{ target_kind_name }}::default(); {% if abstract_data_type == "Value"%} - abstract_data_type.value = {% if target_kind_name | is_built_in_type == false %} - {{ target_kind_name }}::read_from(reader)?; + abstract_data_type.value = {% if fields[0].value | is_built_in_type == false %} + {{ fields[0].value }}::read_from(reader)?; {% else %} - reader.read_{{ target_kind_name }}()?; + reader.read_{% if fields[0].isl_type_name == "symbol" %}symbol()?.text().unwrap(){% else %}{{ fields[0].value | replace(from="string", to ="str") }}()?{% endif %}{% if fields[0].value | lower == "string" %} .to_string() {% endif %}; {% endif %} - {% elif abstract_data_type is object and abstract_data_type | get(key="Structure") %} + {% elif abstract_data_type is object and abstract_data_type is containing("Structure") %} reader.step_in()?; while reader.next()? != StreamItem::Nothing { if let Some(field_name) = reader.field_name()?.text() { @@ -43,21 +43,21 @@ impl {{ target_kind_name }} { {% if field.value | is_built_in_type == false %} "{{ field.name }}" => { abstract_data_type.{{ field.name | snake }} = {{ field.value }}::read_from(reader)?; } {% else %} - "{{ field.name }}" => { abstract_data_type.{{ field.name | snake}} = reader.read_{{ field.value | lower }}()?; } + "{{ field.name }}" => { abstract_data_type.{{ field.name | snake}} = reader.read_{% if field.isl_type_name == "symbol" %}symbol()?.text().unwrap(){% else %}{{ field.value | lower | replace(from="string", to ="str") }}()?{% endif %}{% if field.value | lower== "string" %} .to_string() {% endif %}; } {% endif %} {% endfor %} _ => { {% if abstract_data_type["Structure"] %} return IonResult::decoding_error( "Can not read field name:{{ field.name }} for {{ target_kind_name }} as it doesn't exist in the given schema type definition." - ) + ); {% endif %} } } } } reader.step_out()?; - {% elif abstract_data_type is object and abstract_data_type | get(key="Sequence") %} + {% elif abstract_data_type is object and abstract_data_type is containing("Sequence") %} reader.step_in()?; abstract_data_type.value = { let mut values = vec![]; @@ -66,12 +66,14 @@ impl {{ target_kind_name }} { {% if abstract_data_type["Sequence"] | is_built_in_type == false %} values.push({{ abstract_data_type["Sequence"] }}::read_from(reader)?); {% else %} - values.push(reader.read_{{ abstract_data_type["Sequence"] | lower }}()?); + values.push(reader.read_{% if fields[0].isl_type_name == "symbol" %}symbol()?.text().unwrap(){% else %}{{ abstract_data_type["Sequence"] | lower | replace(from="string", to ="str") }}()?{% endif %}{% if abstract_data_type["Sequence"] | lower== "string" %} .to_string() {% endif %}); {% endif %} } values }; reader.step_out()?; + {% else %} + return IonResult::decoding_error("Can not resolve read API template for {{ target_kind_name }}"); {% endif %} Ok(abstract_data_type) } @@ -79,32 +81,31 @@ impl {{ target_kind_name }} { pub fn write_to(&self, writer: &mut W) -> IonResult<()> { {% if abstract_data_type == "Value" %} {% for field in fields %} - {% if field.value | is_built_in_type ==false %} + {% if field.value | is_built_in_type == false %} self.{{ field.name | snake }}.write_to(writer)?; {% else %} - writer.write_{{ field.value | lower }}(self.value)?; + writer.write_{% if field.isl_type_name == "symbol" %}symbol{% else %}{{ field.value | lower }}{% endif %}(self.value)?; {% endif %} {% endfor %} - {% elif abstract_data_type is object and abstract_data_type | get(key="Structure") %} + {% elif abstract_data_type is object and abstract_data_type is containing("Structure") %} writer.step_in(IonType::Struct)?; {% for field in fields %} writer.set_field_name("{{ field.name }}"); {% if field.value | is_built_in_type == false %} self.{{ field.name | snake }}.write_to(writer)?; {% else %} - writer.write_{{ field.value | lower }}(self.{{ field.name }})?; {# TODO: Change the following `to_owned` to only be used when writing i64,f32,f64,bool which require owned value as input #} - writer.write_{{ field.value | lower }}(self.{{ field.name | snake }}.to_owned())?; + writer.write_{% if field.isl_type_name == "symbol" %}symbol{% else %}{{ field.value | lower }}{% endif %}(self.{{ field.name | snake }}.to_owned())?; {% endif %} {% endfor %} writer.step_out()?; - {% elif abstract_data_type is object and abstract_data_type | get(key="Sequence") %} + {% elif abstract_data_type is object and abstract_data_type is containing("Sequence") %} writer.step_in(IonType::List)?; - for value in self.value { + for value in &self.value { {% if abstract_data_type["Sequence"] | is_built_in_type == false %} value.write_to(writer)?; {% else %} - writer.write_{{ abstract_data_type["Sequence"] | lower }}(value)?; + writer.write_{% if fields[0].isl_type_name == "symbol" %}symbol{% else %}{{ abstract_data_type["Sequence"] | lower }}{% endif %}(value.to_owned())?; {% endif %} } writer.step_out()?; diff --git a/src/bin/ion/commands/beta/generate/utils.rs b/src/bin/ion/commands/beta/generate/utils.rs index 98483fd..0be194e 100644 --- a/src/bin/ion/commands/beta/generate/utils.rs +++ b/src/bin/ion/commands/beta/generate/utils.rs @@ -10,6 +10,7 @@ use std::fmt::{Display, Formatter}; pub struct Field { pub(crate) name: String, pub(crate) value: String, + pub(crate) isl_type_name: String, } /// Represents an import in a generated code file. From 47f38d4bc8e251b2665208600d31d436b2a217a1 Mon Sep 17 00:00:00 2001 From: Khushboo Desai Date: Tue, 26 Mar 2024 14:26:06 -0700 Subject: [PATCH 2/6] Adds support for generating code for all schemas in authorities * Adds `CodeGenerator::generate_code_for_authorities` which generates code for each schema in given authorities * Makes `--schema` optional for code generation --- .../ion/commands/beta/generate/generator.rs | 91 ++++++++++++++++--- src/bin/ion/commands/beta/generate/mod.rs | 55 +++++++---- 2 files changed, 114 insertions(+), 32 deletions(-) diff --git a/src/bin/ion/commands/beta/generate/generator.rs b/src/bin/ion/commands/beta/generate/generator.rs index 519c43c..e493d83 100644 --- a/src/bin/ion/commands/beta/generate/generator.rs +++ b/src/bin/ion/commands/beta/generate/generator.rs @@ -7,7 +7,9 @@ use ion_schema::isl::isl_constraint::{IslConstraint, IslConstraintValue}; use ion_schema::isl::isl_type::IslType; use ion_schema::isl::isl_type_reference::IslTypeRef; use ion_schema::isl::IslSchema; +use ion_schema::system::SchemaSystem; use std::collections::HashMap; +use std::fs; use std::fs::File; use std::io::Write; use std::marker::PhantomData; @@ -26,20 +28,52 @@ pub(crate) struct CodeGenerator<'a, L: Language> { impl<'a> CodeGenerator<'a, RustLanguage> { pub fn new(output: &'a Path) -> CodeGenerator { + let tera = Tera::new(&format!( + "{}/src/bin/ion/commands/beta/generate/templates/rust/*.templ", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + Self { output, anonymous_type_counter: 0, - tera: Tera::new(&format!( - "{}/src/bin/ion/commands/beta/generate/templates/rust/*.templ", - env!("CARGO_MANIFEST_DIR") - )) - .unwrap(), + tera, phantom: PhantomData, } } + /// Generates code in Rust for all the schemas in given authorities + pub fn generate_code_for_authorities( + &mut self, + authorities: &Vec<&String>, + schema_system: &mut SchemaSystem, + ) -> CodeGenResult<()> { + for authority in authorities { + let paths = fs::read_dir(authority)?; + + for schema_file in paths { + let schema_file_path = schema_file?.path(); + let schema_id = schema_file_path.file_name().unwrap().to_str().unwrap(); + + let schema = schema_system.load_isl_schema(schema_id).unwrap(); + + self.generate(schema)?; + } + } + Ok(()) + } + /// Generates code in Rust for given Ion Schema - pub fn generate(&mut self, schema: IslSchema) -> CodeGenResult<()> { + pub fn generate_code_for_schema( + &mut self, + schema_system: &mut SchemaSystem, + schema_id: &str, + ) -> CodeGenResult<()> { + let schema = schema_system.load_isl_schema(schema_id).unwrap(); + self.generate(schema) + } + + fn generate(&mut self, schema: IslSchema) -> CodeGenResult<()> { // this will be used for Rust to create mod.rs which lists all the generated modules let mut modules = vec![]; let mut module_context = tera::Context::new(); @@ -53,6 +87,7 @@ impl<'a> CodeGenerator<'a, RustLanguage> { self.tera .register_filter("is_built_in_type", Self::is_built_in_type); + // Iterate through the ISL types, generate an abstract data type for each for isl_type in schema.types() { // unwrap here is safe because all the top-level type definition always has a name let isl_type_name = isl_type.name().clone().unwrap(); @@ -79,20 +114,52 @@ impl<'a> CodeGenerator<'a, RustLanguage> { impl<'a> CodeGenerator<'a, JavaLanguage> { pub fn new(output: &'a Path) -> CodeGenerator { + let tera = Tera::new(&format!( + "{}/src/bin/ion/commands/beta/generate/templates/java/*.templ", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap(); + Self { output, anonymous_type_counter: 0, - tera: Tera::new(&format!( - "{}/src/bin/ion/commands/beta/generate/templates/java/*.templ", - env!("CARGO_MANIFEST_DIR") - )) - .unwrap(), + tera, phantom: PhantomData, } } + /// Generates code in Java for all the schemas in given authorities + pub fn generate_code_for_authorities( + &mut self, + authorities: &Vec<&String>, + schema_system: &mut SchemaSystem, + ) -> CodeGenResult<()> { + for authority in authorities { + let paths = fs::read_dir(authority)?; + + for schema_file in paths { + let schema_file_path = schema_file?.path(); + let schema_id = schema_file_path.file_name().unwrap().to_str().unwrap(); + + let schema = schema_system.load_isl_schema(schema_id).unwrap(); + + self.generate(schema)?; + } + } + Ok(()) + } + /// Generates code in Java for given Ion Schema - pub fn generate(&mut self, schema: IslSchema) -> CodeGenResult<()> { + pub fn generate_code_for_schema( + &mut self, + schema_system: &mut SchemaSystem, + schema_id: &str, + ) -> CodeGenResult<()> { + let schema = schema_system.load_isl_schema(schema_id).unwrap(); + self.generate(schema) + } + + fn generate(&mut self, schema: IslSchema) -> CodeGenResult<()> { // this will be used for Rust to create mod.rs which lists all the generated modules let mut modules = vec![]; diff --git a/src/bin/ion/commands/beta/generate/mod.rs b/src/bin/ion/commands/beta/generate/mod.rs index 7adea3c..0b9279a 100644 --- a/src/bin/ion/commands/beta/generate/mod.rs +++ b/src/bin/ion/commands/beta/generate/mod.rs @@ -34,9 +34,8 @@ impl IonCliCommand for GenerateCommand { .arg( Arg::new("schema") .long("schema") - .required(true) .short('s') - .help("Schema file"), + .help("Schema file name or schema id"), ) .arg( Arg::new("language") @@ -75,13 +74,10 @@ impl IonCliCommand for GenerateCommand { // Extract the user provided document authorities/ directories let authorities: Vec<&String> = args.get_many("directory").unwrap().collect(); - // Extract schema file provided by user - let schema_id = args.get_one::("schema").unwrap(); - // Set up document authorities vector let mut document_authorities: Vec> = vec![]; - for authority in authorities { + for authority in &authorities { document_authorities.push(Box::new(FileSystemDocumentAuthority::new(Path::new( authority, )))) @@ -90,24 +86,43 @@ impl IonCliCommand for GenerateCommand { // Create a new schema system from given document authorities let mut schema_system = SchemaSystem::new(document_authorities); - let schema = schema_system.load_isl_schema(schema_id).unwrap(); - - // clean the target output directory if it already exists, before generating new code - if output.exists() { - fs::remove_dir_all(output).unwrap(); + // Generate directories in the output path if the path doesn't exist + if !output.exists() { + fs::create_dir_all(output).unwrap(); } - fs::create_dir_all(output).unwrap(); println!("Started generating code..."); - // generate code based on schema and programming language - match language { - "java" => CodeGenerator::::new(output).generate(schema)?, - "rust" => CodeGenerator::::new(output).generate(schema)?, - _ => bail!( - "Programming language '{}' is not yet supported. Currently supported targets: 'java', 'rust'", - language - ) + // Extract schema file provided by user + match args.get_one::("schema") { + None => { + // generate code based on schema and programming language + match language { + "java" => + CodeGenerator::::new(output) + .generate_code_for_authorities(&authorities, &mut schema_system)?, + "rust" => + CodeGenerator::::new(output) + .generate_code_for_authorities(&authorities, &mut schema_system)?, + _ => bail!( + "Programming language '{}' is not yet supported. Currently supported targets: 'java', 'rust'", + language + ) + } + } + Some(schema_id) => { + let schema = schema_system.load_isl_schema(schema_id).unwrap(); + + // generate code based on schema and programming language + match language { + "java" => CodeGenerator::::new(output).generate_code_for_schema(schema)?, + "rust" => CodeGenerator::::new(output).generate_code_for_schema(schema)?, + _ => bail!( + "Programming language '{}' is not yet supported. Currently supported targets: 'java', 'rust'", + language + ) + } + } } println!("Code generation complete successfully!"); From 9b5e242bcacc843e6db1b0dee15d2deb91ece151 Mon Sep 17 00:00:00 2001 From: Khushboo Desai Date: Tue, 26 Mar 2024 15:03:14 -0700 Subject: [PATCH 3/6] Adds `--namespace` option for Java code generation --- .../ion/commands/beta/generate/generator.rs | 2 +- src/bin/ion/commands/beta/generate/mod.rs | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/bin/ion/commands/beta/generate/generator.rs b/src/bin/ion/commands/beta/generate/generator.rs index e493d83..3d4c310 100644 --- a/src/bin/ion/commands/beta/generate/generator.rs +++ b/src/bin/ion/commands/beta/generate/generator.rs @@ -113,7 +113,7 @@ impl<'a> CodeGenerator<'a, RustLanguage> { } impl<'a> CodeGenerator<'a, JavaLanguage> { - pub fn new(output: &'a Path) -> CodeGenerator { + pub fn new(output: &'a Path, namespace: &str) -> CodeGenerator { let tera = Tera::new(&format!( "{}/src/bin/ion/commands/beta/generate/templates/java/*.templ", env!("CARGO_MANIFEST_DIR") diff --git a/src/bin/ion/commands/beta/generate/mod.rs b/src/bin/ion/commands/beta/generate/mod.rs index 0b9279a..275788e 100644 --- a/src/bin/ion/commands/beta/generate/mod.rs +++ b/src/bin/ion/commands/beta/generate/mod.rs @@ -37,6 +37,14 @@ impl IonCliCommand for GenerateCommand { .short('s') .help("Schema file name or schema id"), ) + // `--namespace` is required when Java language is specified for code generation + .arg( + Arg::new("namespace") + .long("namespace") + .short('n') + .required_if_eq("language", "java") + .help("Provide namespace for generated Java code (e.g. `org.example`)"), + ) .arg( Arg::new("language") .long("language") @@ -62,6 +70,9 @@ impl IonCliCommand for GenerateCommand { // Extract programming language for code generation let language: &str = args.get_one::("language").unwrap().as_str(); + // Extract namespace for code generation + let namespace = args.get_one::("namespace"); + // Extract output path information where the generated code will be saved // Create a module `ion_data_model` for storing all the generated code in the output directory let binding = match args.get_one::("output") { @@ -99,7 +110,7 @@ impl IonCliCommand for GenerateCommand { // generate code based on schema and programming language match language { "java" => - CodeGenerator::::new(output) + CodeGenerator::::new(output, namespace.unwrap().as_str()) .generate_code_for_authorities(&authorities, &mut schema_system)?, "rust" => CodeGenerator::::new(output) @@ -111,12 +122,10 @@ impl IonCliCommand for GenerateCommand { } } Some(schema_id) => { - let schema = schema_system.load_isl_schema(schema_id).unwrap(); - // generate code based on schema and programming language match language { - "java" => CodeGenerator::::new(output).generate_code_for_schema(schema)?, - "rust" => CodeGenerator::::new(output).generate_code_for_schema(schema)?, + "java" => CodeGenerator::::new(output, namespace.unwrap().as_str()).generate_code_for_schema(&mut schema_system, schema_id)?, + "rust" => CodeGenerator::::new(output).generate_code_for_schema(&mut schema_system, schema_id)?, _ => bail!( "Programming language '{}' is not yet supported. Currently supported targets: 'java', 'rust'", language From e172f3fc9597471f15286211fd88646c15e68ef8 Mon Sep 17 00:00:00 2001 From: Khushboo Desai Date: Tue, 26 Mar 2024 11:26:05 -0700 Subject: [PATCH 4/6] Adds changes to generate code in single module for Rust code generation * Removed Import statements from template as those were used for reference to other generated data models * Removes mod.templ as the generated code now uses single module * Removes generating `ion_data_model` module for generated code instead uses a single file `ion_generated_code` --- .../ion/commands/beta/generate/generator.rs | 104 +++++++----------- src/bin/ion/commands/beta/generate/mod.rs | 4 +- .../beta/generate/templates/rust/import.templ | 1 + .../beta/generate/templates/rust/mod.templ | 3 - .../beta/generate/templates/rust/struct.templ | 5 - src/bin/ion/commands/beta/generate/utils.rs | 11 +- 6 files changed, 45 insertions(+), 83 deletions(-) create mode 100644 src/bin/ion/commands/beta/generate/templates/rust/import.templ delete mode 100644 src/bin/ion/commands/beta/generate/templates/rust/mod.templ diff --git a/src/bin/ion/commands/beta/generate/generator.rs b/src/bin/ion/commands/beta/generate/generator.rs index 3d4c310..8b39a71 100644 --- a/src/bin/ion/commands/beta/generate/generator.rs +++ b/src/bin/ion/commands/beta/generate/generator.rs @@ -1,6 +1,6 @@ use crate::commands::beta::generate::context::{AbstractDataType, CodeGenContext}; use crate::commands::beta::generate::result::{invalid_abstract_data_type_error, CodeGenResult}; -use crate::commands::beta::generate::utils::{Field, Import, JavaLanguage, Language, RustLanguage}; +use crate::commands::beta::generate::utils::{Field, JavaLanguage, Language, RustLanguage}; use crate::commands::beta::generate::utils::{IonSchemaType, Template}; use convert_case::{Case, Casing}; use ion_schema::isl::isl_constraint::{IslConstraint, IslConstraintValue}; @@ -10,7 +10,7 @@ use ion_schema::isl::IslSchema; use ion_schema::system::SchemaSystem; use std::collections::HashMap; use std::fs; -use std::fs::File; +use std::fs::{File, OpenOptions}; use std::io::Write; use std::marker::PhantomData; use std::path::Path; @@ -21,6 +21,9 @@ pub(crate) struct CodeGenerator<'a, L: Language> { // more information: https://docs.rs/tera/latest/tera/ pub(crate) tera: Tera, output: &'a Path, + // This field is used by Java code generation to get the namespace for generated code. + // For Rust code generation, this will be set to None. + namespace: Option<&'a str>, // Represents a counter for naming anonymous type definitions pub(crate) anonymous_type_counter: usize, phantom: PhantomData, @@ -34,8 +37,18 @@ impl<'a> CodeGenerator<'a, RustLanguage> { )) .unwrap(); + // Render the imports into output file + let rendered = tera.render("import.templ", &Context::new()).unwrap(); + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(output.join("ion_generated_code.rs")) + .unwrap(); + file.write_all(rendered.as_bytes()).unwrap(); + Self { output, + namespace: None, anonymous_type_counter: 0, tera, phantom: PhantomData, @@ -74,10 +87,6 @@ impl<'a> CodeGenerator<'a, RustLanguage> { } fn generate(&mut self, schema: IslSchema) -> CodeGenResult<()> { - // this will be used for Rust to create mod.rs which lists all the generated modules - let mut modules = vec![]; - let mut module_context = tera::Context::new(); - // Register a tera filter that can be used to convert a string based on case self.tera.register_filter("upper_camel", Self::upper_camel); self.tera.register_filter("snake", Self::snake); @@ -91,29 +100,15 @@ impl<'a> CodeGenerator<'a, RustLanguage> { for isl_type in schema.types() { // unwrap here is safe because all the top-level type definition always has a name let isl_type_name = isl_type.name().clone().unwrap(); - self.generate_abstract_data_type(&isl_type_name, &mut modules, isl_type)?; + self.generate_abstract_data_type(&isl_type_name, isl_type)?; } - self.generate_modules(&mut modules, &mut module_context)?; - - Ok(()) - } - - pub fn generate_modules( - &mut self, - modules: &mut Vec, - module_context: &mut Context, - ) -> CodeGenResult<()> { - module_context.insert("modules", &modules); - let rendered = self.tera.render("mod.templ", module_context)?; - let mut file = File::create(self.output.join("mod.rs"))?; - file.write_all(rendered.as_bytes())?; Ok(()) } } impl<'a> CodeGenerator<'a, JavaLanguage> { - pub fn new(output: &'a Path, namespace: &str) -> CodeGenerator { + pub fn new(output: &'a Path, namespace: &'a str) -> CodeGenerator<'a, JavaLanguage> { let tera = Tera::new(&format!( "{}/src/bin/ion/commands/beta/generate/templates/java/*.templ", env!("CARGO_MANIFEST_DIR") @@ -122,6 +117,7 @@ impl<'a> CodeGenerator<'a, JavaLanguage> { Self { output, + namespace: Some(namespace), anonymous_type_counter: 0, tera, phantom: PhantomData, @@ -160,18 +156,16 @@ impl<'a> CodeGenerator<'a, JavaLanguage> { } fn generate(&mut self, schema: IslSchema) -> CodeGenResult<()> { - // this will be used for Rust to create mod.rs which lists all the generated modules - let mut modules = vec![]; - // Register a tera filter that can be used to convert a string based on case self.tera.register_filter("upper_camel", Self::upper_camel); self.tera.register_filter("snake", Self::snake); self.tera.register_filter("camel", Self::camel); + // Iterate through the ISL types, generate an abstract data type for each for isl_type in schema.types() { // unwrap here is safe because all the top-level type definition always has a name let isl_type_name = isl_type.name().clone().unwrap(); - self.generate_abstract_data_type(&isl_type_name, &mut modules, isl_type)?; + self.generate_abstract_data_type(&isl_type_name, isl_type)?; } Ok(()) @@ -257,12 +251,10 @@ impl<'a, L: Language> CodeGenerator<'a, L> { fn generate_abstract_data_type( &mut self, isl_type_name: &String, - modules: &mut Vec, isl_type: &IslType, ) -> CodeGenResult<()> { let mut context = Context::new(); let mut tera_fields = vec![]; - let mut imports: Vec = vec![]; let mut code_gen_context = CodeGenContext::new(); // Set the ISL type name for the generated abstract data type @@ -271,17 +263,12 @@ impl<'a, L: Language> CodeGenerator<'a, L> { let constraints = isl_type.constraints(); for constraint in constraints { self.map_constraint_to_abstract_data_type( - modules, &mut tera_fields, - &mut imports, constraint, &mut code_gen_context, )?; } - // add imports for the template - context.insert("imports", &imports); - // add fields for template // TODO: verify the `occurs` value within a field, by default the fields are optional. if let Some(abstract_data_type) = &code_gen_context.abstract_data_type { @@ -293,47 +280,41 @@ impl<'a, L: Language> CodeGenerator<'a, L> { ); } - self.render_generated_code(modules, &isl_type_name, &mut context, &mut code_gen_context) + self.render_generated_code(&isl_type_name, &mut context, &mut code_gen_context) } fn render_generated_code( &mut self, - modules: &mut Vec, - abstract_data_type_name: &str, + type_name: &str, context: &mut Context, code_gen_context: &mut CodeGenContext, ) -> CodeGenResult<()> { - modules.push(L::file_name_for_type(abstract_data_type_name)); - + // Add namespace to tera context + if let Some(namespace) = self.namespace { + context.insert("namespace", namespace); + } // Render or generate file for the template with the given context let template: &Template = &code_gen_context.abstract_data_type.as_ref().try_into()?; let rendered = self .tera .render(&format!("{}.templ", L::template_name(template)), context) .unwrap(); - let mut file = File::create(self.output.join(format!( - "{}.{}", - L::file_name_for_type(abstract_data_type_name), - L::file_extension() - )))?; + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(self.output.join(format!( + "{}.{}", + L::file_name_for_type(type_name), + L::file_extension() + )))?; file.write_all(rendered.as_bytes())?; Ok(()) } /// Provides name of the type reference that will be used for generated abstract data type - fn type_reference_name( - &mut self, - isl_type_ref: &IslTypeRef, - modules: &mut Vec, - imports: &mut Vec, - ) -> CodeGenResult { + fn type_reference_name(&mut self, isl_type_ref: &IslTypeRef) -> CodeGenResult { Ok(match isl_type_ref { IslTypeRef::Named(name, _) => { - if !L::is_built_in_type(name) { - imports.push(Import { - name: name.to_string(), - }); - } let schema_type: IonSchemaType = name.into(); L::target_type(&schema_type) } @@ -342,10 +323,8 @@ impl<'a, L: Language> CodeGenerator<'a, L> { } IslTypeRef::Anonymous(type_def, _) => { let name = self.next_anonymous_type_name(); - self.generate_abstract_data_type(&name, modules, type_def)?; - imports.push(Import { - name: name.to_string(), - }); + self.generate_abstract_data_type(&name, type_def)?; + name } }) @@ -361,15 +340,13 @@ impl<'a, L: Language> CodeGenerator<'a, L> { /// Maps the given constraint value to an abstract data type fn map_constraint_to_abstract_data_type( &mut self, - modules: &mut Vec, tera_fields: &mut Vec, - imports: &mut Vec, constraint: &IslConstraint, code_gen_context: &mut CodeGenContext, ) -> CodeGenResult<()> { match constraint.constraint() { IslConstraintValue::Element(isl_type, _) => { - let type_name = self.type_reference_name(isl_type, modules, imports)?; + let type_name = self.type_reference_name(isl_type)?; self.verify_abstract_data_type_consistency( AbstractDataType::Sequence(type_name.to_owned()), code_gen_context, @@ -388,8 +365,7 @@ impl<'a, L: Language> CodeGenerator<'a, L> { code_gen_context, )?; for (name, value) in fields.iter() { - let type_name = - self.type_reference_name(value.type_reference(), modules, imports)?; + let type_name = self.type_reference_name(value.type_reference())?; self.generate_struct_field( tera_fields, @@ -400,7 +376,7 @@ impl<'a, L: Language> CodeGenerator<'a, L> { } } IslConstraintValue::Type(isl_type) => { - let type_name = self.type_reference_name(isl_type, modules, imports)?; + let type_name = self.type_reference_name(isl_type)?; self.verify_abstract_data_type_consistency( AbstractDataType::Value, diff --git a/src/bin/ion/commands/beta/generate/mod.rs b/src/bin/ion/commands/beta/generate/mod.rs index 275788e..d258d99 100644 --- a/src/bin/ion/commands/beta/generate/mod.rs +++ b/src/bin/ion/commands/beta/generate/mod.rs @@ -76,8 +76,8 @@ impl IonCliCommand for GenerateCommand { // Extract output path information where the generated code will be saved // Create a module `ion_data_model` for storing all the generated code in the output directory let binding = match args.get_one::("output") { - Some(output_path) => PathBuf::from(output_path).join("ion_data_model"), - None => PathBuf::from("./ion_data_model"), + Some(output_path) => PathBuf::from(output_path), + None => PathBuf::from("./"), }; let output = binding.as_path(); diff --git a/src/bin/ion/commands/beta/generate/templates/rust/import.templ b/src/bin/ion/commands/beta/generate/templates/rust/import.templ new file mode 100644 index 0000000..059b9ca --- /dev/null +++ b/src/bin/ion/commands/beta/generate/templates/rust/import.templ @@ -0,0 +1 @@ +use ion_rs::{IonResult, IonReader, Reader, IonWriter, StreamItem}; diff --git a/src/bin/ion/commands/beta/generate/templates/rust/mod.templ b/src/bin/ion/commands/beta/generate/templates/rust/mod.templ deleted file mode 100644 index 0c127ee..0000000 --- a/src/bin/ion/commands/beta/generate/templates/rust/mod.templ +++ /dev/null @@ -1,3 +0,0 @@ -{% for module in modules -%} -pub mod {{ module }}; -{% endfor %} diff --git a/src/bin/ion/commands/beta/generate/templates/rust/struct.templ b/src/bin/ion/commands/beta/generate/templates/rust/struct.templ index ba8b0a7..1fc4df1 100644 --- a/src/bin/ion/commands/beta/generate/templates/rust/struct.templ +++ b/src/bin/ion/commands/beta/generate/templates/rust/struct.templ @@ -1,8 +1,3 @@ -use ion_rs::{IonResult, IonReader, Reader, IonWriter, StreamItem}; -{% for import in imports %} -use crate::ion_data_model::{{ import.name | snake }}::{{ import.name | upper_camel }}; -{% endfor %} - #[derive(Debug, Clone, Default)] pub struct {{ target_kind_name }} { {% for field in fields -%} diff --git a/src/bin/ion/commands/beta/generate/utils.rs b/src/bin/ion/commands/beta/generate/utils.rs index 0be194e..a1f1b71 100644 --- a/src/bin/ion/commands/beta/generate/utils.rs +++ b/src/bin/ion/commands/beta/generate/utils.rs @@ -13,13 +13,6 @@ pub struct Field { pub(crate) isl_type_name: String, } -/// Represents an import in a generated code file. -/// This will be used by template engine to fill import statements of a type definition. -#[derive(Serialize)] -pub struct Import { - pub(crate) name: String, -} - pub trait Language { /// Provides a file extension based on programming language fn file_extension() -> String; @@ -122,8 +115,8 @@ impl Language for RustLanguage { "rust".to_string() } - fn file_name_for_type(name: &str) -> String { - name.to_case(Case::Snake) + fn file_name_for_type(_name: &str) -> String { + "ion_generated_code".to_string() } fn target_type(ion_schema_type: &IonSchemaType) -> String { From 9e73f656f94a5ef889eb4d4dc90f6b2b6734ce1d Mon Sep 17 00:00:00 2001 From: Khushboo Desai Date: Wed, 3 Apr 2024 15:18:28 -0700 Subject: [PATCH 5/6] Adds changes for current test suite and clippy changes --- .../ion/commands/beta/generate/generator.rs | 4 +-- .../beta/generate/templates/java/class.templ | 12 ++++---- tests/cli.rs | 28 ++++++------------- 3 files changed, 17 insertions(+), 27 deletions(-) diff --git a/src/bin/ion/commands/beta/generate/generator.rs b/src/bin/ion/commands/beta/generate/generator.rs index 8b39a71..0e37ca1 100644 --- a/src/bin/ion/commands/beta/generate/generator.rs +++ b/src/bin/ion/commands/beta/generate/generator.rs @@ -10,7 +10,7 @@ use ion_schema::isl::IslSchema; use ion_schema::system::SchemaSystem; use std::collections::HashMap; use std::fs; -use std::fs::{File, OpenOptions}; +use std::fs::OpenOptions; use std::io::Write; use std::marker::PhantomData; use std::path::Path; @@ -280,7 +280,7 @@ impl<'a, L: Language> CodeGenerator<'a, L> { ); } - self.render_generated_code(&isl_type_name, &mut context, &mut code_gen_context) + self.render_generated_code(isl_type_name, &mut context, &mut code_gen_context) } fn render_generated_code( diff --git a/src/bin/ion/commands/beta/generate/templates/java/class.templ b/src/bin/ion/commands/beta/generate/templates/java/class.templ index 7435cca..79f53a6 100644 --- a/src/bin/ion/commands/beta/generate/templates/java/class.templ +++ b/src/bin/ion/commands/beta/generate/templates/java/class.templ @@ -1,19 +1,19 @@ -{% if import %} -import ion_data_model.{{ import.name | upper_camel }}; -{% endif %} +package {{ namespace }}; +import java.util.ArrayList; + public final class {{ target_kind_name }} { {% for field in fields -%} private final {{ field.value }} {{ field.name | camel }}; {% endfor %} - public {{ target_kind_name }}({% for field in fields -%}{{ field.value }} {{ field.name }}{% if not loop.last %},{% endif %}{% endfor %}) { + public {{ target_kind_name }}({% for field in fields -%}{{ field.value }} {{ field.name | camel }}{% if not loop.last %},{% endif %}{% endfor %}) { {% for field in fields -%} - this.{{ field.name }} = {{ field.name }}; + this.{{ field.name | camel }} = {{ field.name | camel }}; {% endfor %} } {% for field in fields -%}public {{ field.value }} get{% filter upper_camel %}{{ field.name }}{% endfilter %}() { - return this.{{ field.name }}; + return this.{{ field.name | camel }}; } {% endfor %} } diff --git a/tests/cli.rs b/tests/cli.rs index c401dc8..7af8696 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -220,8 +220,7 @@ fn test_write_all_values(#[case] number: i32, #[case] expected_output: &str) -> #[cfg(feature = "beta-subcommands")] #[rstest] -#[case( - "simple_struct", +#[case::simple_struct( r#" type::{ name: simple_struct, @@ -234,8 +233,7 @@ fn test_write_all_values(#[case] number: i32, #[case] expected_output: &str) -> &["id: i64", "name: String"], &["pub fn name(&self) -> &String {", "pub fn id(&self) -> &i64 {"] )] -#[case( - "value_struct", +#[case::value_struct( r#" type::{ name: value_struct, @@ -245,8 +243,7 @@ fn test_write_all_values(#[case] number: i32, #[case] expected_output: &str) -> &["value: i64"], &["pub fn value(&self) -> &i64 {"] )] -#[case( - "sequence_struct", +#[case::sequence_struct( r#" type::{ name: sequence_struct, @@ -256,8 +253,7 @@ fn test_write_all_values(#[case] number: i32, #[case] expected_output: &str) -> &["value: Vec"], &["pub fn value(&self) -> &Vec {"] )] -#[case( - "struct_with_reference_field", +#[case::struct_with_reference_field( r#" type::{ name: struct_with_reference_field, @@ -274,8 +270,7 @@ fn test_write_all_values(#[case] number: i32, #[case] expected_output: &str) -> &["reference: OtherType"], &["pub fn reference(&self) -> &OtherType {"] )] -#[case( - "struct_with_anonymous_type", +#[case::struct_with_anonymous_type( r#" type::{ name: struct_with_anonymous_type, @@ -289,7 +284,6 @@ fn test_write_all_values(#[case] number: i32, #[case] expected_output: &str) -> )] /// Calls ion-cli beta generate with different schema file. Pass the test if the return value contains the expected properties and accessors. fn test_code_generation_in_rust( - #[case] test_name: &str, #[case] test_schema: &str, #[case] expected_properties: &[&str], #[case] expected_accessors: &[&str], @@ -313,10 +307,7 @@ fn test_code_generation_in_rust( temp_dir.path().to_str().unwrap(), ]); let command_assert = cmd.assert(); - let output_file_path = temp_dir - .path() - .join("ion_data_model") - .join(format!("{}.rs", test_name)); + let output_file_path = temp_dir.path().join("ion_generated_code.rs"); command_assert.success(); let contents = fs::read_to_string(output_file_path).expect("Should have been able to read the file"); @@ -424,14 +415,13 @@ fn test_code_generation_in_java( temp_dir.path().to_str().unwrap(), "--language", "java", + "--namespace", + "org.example", "--directory", temp_dir.path().to_str().unwrap(), ]); let command_assert = cmd.assert(); - let output_file_path = temp_dir - .path() - .join("ion_data_model") - .join(format!("{}.java", test_name)); + let output_file_path = temp_dir.path().join(format!("{}.java", test_name)); command_assert.success(); let contents = fs::read_to_string(output_file_path).expect("Can not read generated code file."); for expected_property in expected_properties { From 80b6b380ff00e50c77439656383abf74dee2c827 Mon Sep 17 00:00:00 2001 From: Khushboo Desai Date: Thu, 4 Apr 2024 16:40:52 -0700 Subject: [PATCH 6/6] Adds suggested changes --- .../ion/commands/beta/generate/generator.rs | 166 +++++++----------- 1 file changed, 64 insertions(+), 102 deletions(-) diff --git a/src/bin/ion/commands/beta/generate/generator.rs b/src/bin/ion/commands/beta/generate/generator.rs index 0e37ca1..164dbab 100644 --- a/src/bin/ion/commands/beta/generate/generator.rs +++ b/src/bin/ion/commands/beta/generate/generator.rs @@ -40,8 +40,10 @@ impl<'a> CodeGenerator<'a, RustLanguage> { // Render the imports into output file let rendered = tera.render("import.templ", &Context::new()).unwrap(); let mut file = OpenOptions::new() + // In order for the file to be created, OpenOptions::write or OpenOptions::append access must be used + // reference: https://doc.rust-lang.org/std/fs/struct.OpenOptions.html#method.create + .write(true) .create(true) - .append(true) .open(output.join("ion_generated_code.rs")) .unwrap(); file.write_all(rendered.as_bytes()).unwrap(); @@ -54,57 +56,6 @@ impl<'a> CodeGenerator<'a, RustLanguage> { phantom: PhantomData, } } - - /// Generates code in Rust for all the schemas in given authorities - pub fn generate_code_for_authorities( - &mut self, - authorities: &Vec<&String>, - schema_system: &mut SchemaSystem, - ) -> CodeGenResult<()> { - for authority in authorities { - let paths = fs::read_dir(authority)?; - - for schema_file in paths { - let schema_file_path = schema_file?.path(); - let schema_id = schema_file_path.file_name().unwrap().to_str().unwrap(); - - let schema = schema_system.load_isl_schema(schema_id).unwrap(); - - self.generate(schema)?; - } - } - Ok(()) - } - - /// Generates code in Rust for given Ion Schema - pub fn generate_code_for_schema( - &mut self, - schema_system: &mut SchemaSystem, - schema_id: &str, - ) -> CodeGenResult<()> { - let schema = schema_system.load_isl_schema(schema_id).unwrap(); - self.generate(schema) - } - - fn generate(&mut self, schema: IslSchema) -> CodeGenResult<()> { - // Register a tera filter that can be used to convert a string based on case - self.tera.register_filter("upper_camel", Self::upper_camel); - self.tera.register_filter("snake", Self::snake); - self.tera.register_filter("camel", Self::camel); - - // Register a tera filter that can be used to see if a type is built in data type or not - self.tera - .register_filter("is_built_in_type", Self::is_built_in_type); - - // Iterate through the ISL types, generate an abstract data type for each - for isl_type in schema.types() { - // unwrap here is safe because all the top-level type definition always has a name - let isl_type_name = isl_type.name().clone().unwrap(); - self.generate_abstract_data_type(&isl_type_name, isl_type)?; - } - - Ok(()) - } } impl<'a> CodeGenerator<'a, JavaLanguage> { @@ -123,56 +74,9 @@ impl<'a> CodeGenerator<'a, JavaLanguage> { phantom: PhantomData, } } - - /// Generates code in Java for all the schemas in given authorities - pub fn generate_code_for_authorities( - &mut self, - authorities: &Vec<&String>, - schema_system: &mut SchemaSystem, - ) -> CodeGenResult<()> { - for authority in authorities { - let paths = fs::read_dir(authority)?; - - for schema_file in paths { - let schema_file_path = schema_file?.path(); - let schema_id = schema_file_path.file_name().unwrap().to_str().unwrap(); - - let schema = schema_system.load_isl_schema(schema_id).unwrap(); - - self.generate(schema)?; - } - } - Ok(()) - } - - /// Generates code in Java for given Ion Schema - pub fn generate_code_for_schema( - &mut self, - schema_system: &mut SchemaSystem, - schema_id: &str, - ) -> CodeGenResult<()> { - let schema = schema_system.load_isl_schema(schema_id).unwrap(); - self.generate(schema) - } - - fn generate(&mut self, schema: IslSchema) -> CodeGenResult<()> { - // Register a tera filter that can be used to convert a string based on case - self.tera.register_filter("upper_camel", Self::upper_camel); - self.tera.register_filter("snake", Self::snake); - self.tera.register_filter("camel", Self::camel); - - // Iterate through the ISL types, generate an abstract data type for each - for isl_type in schema.types() { - // unwrap here is safe because all the top-level type definition always has a name - let isl_type_name = isl_type.name().clone().unwrap(); - self.generate_abstract_data_type(&isl_type_name, isl_type)?; - } - - Ok(()) - } } -impl<'a, L: Language> CodeGenerator<'a, L> { +impl<'a, L: Language + 'static> CodeGenerator<'a, L> { /// Represents a [tera] filter that converts given tera string value to [upper camel case]. /// Returns error if the given value is not a string. /// @@ -248,6 +152,57 @@ impl<'a, L: Language> CodeGenerator<'a, L> { ))) } + /// Generates code for all the schemas in given authorities + pub fn generate_code_for_authorities( + &mut self, + authorities: &Vec<&String>, + schema_system: &mut SchemaSystem, + ) -> CodeGenResult<()> { + for authority in authorities { + let paths = fs::read_dir(authority)?; + + for schema_file in paths { + let schema_file_path = schema_file?.path(); + let schema_id = schema_file_path.file_name().unwrap().to_str().unwrap(); + + let schema = schema_system.load_isl_schema(schema_id).unwrap(); + + self.generate(schema)?; + } + } + Ok(()) + } + + /// Generates code for given Ion Schema + pub fn generate_code_for_schema( + &mut self, + schema_system: &mut SchemaSystem, + schema_id: &str, + ) -> CodeGenResult<()> { + let schema = schema_system.load_isl_schema(schema_id).unwrap(); + self.generate(schema) + } + + fn generate(&mut self, schema: IslSchema) -> CodeGenResult<()> { + // Register a tera filter that can be used to convert a string based on case + self.tera.register_filter("upper_camel", Self::upper_camel); + self.tera.register_filter("snake", Self::snake); + self.tera.register_filter("camel", Self::camel); + + // Register a tera filter that can be used to see if a type is built in data type or not + self.tera + .register_filter("is_built_in_type", Self::is_built_in_type); + + // Iterate through the ISL types, generate an abstract data type for each + for isl_type in schema.types() { + // unwrap here is safe because all the top-level type definition always has a name + let isl_type_name = isl_type.name().clone().unwrap(); + self.generate_abstract_data_type(&isl_type_name, isl_type)?; + } + + Ok(()) + } + fn generate_abstract_data_type( &mut self, isl_type_name: &String, @@ -299,9 +254,16 @@ impl<'a, L: Language> CodeGenerator<'a, L> { .tera .render(&format!("{}.templ", L::template_name(template)), context) .unwrap(); - let mut file = OpenOptions::new() + let mut file_options = OpenOptions::new(); + if L::name() == "rust" { + // since Rust code is generated into a single file, it needs append set to true. + file_options.append(true); + } + let mut file = file_options + // In order for the file to be created, OpenOptions::write or OpenOptions::append access must be used + // reference: https://doc.rust-lang.org/std/fs/struct.OpenOptions.html#method.create + .write(true) .create(true) - .append(true) .open(self.output.join(format!( "{}.{}", L::file_name_for_type(type_name),