From 52d588a056ffbae32b47429b0fc56d0a28726a6d Mon Sep 17 00:00:00 2001 From: Julius de Bruijn Date: Fri, 6 Jan 2023 14:10:14 +0000 Subject: [PATCH] Remove DML from MongoDB intro (#3557) --- .../src/sampler.rs | 19 +- .../src/sampler/field_type.rs | 42 + .../src/sampler/statistics.rs | 758 +++++++++--------- .../src/sampler/statistics/indices.rs | 318 +------- .../src/warnings.rs | 2 +- .../tests/index/mod.rs | 104 ++- .../tests/remapping_names/mod.rs | 2 +- .../datamodel-renderer/src/datamodel.rs | 14 - .../src/datamodel/composite_type.rs | 17 - .../src/datamodel/default.rs | 74 -- .../datamodel-renderer/src/datamodel/field.rs | 168 +--- .../datamodel-renderer/src/datamodel/model.rs | 210 +---- libs/mongodb-schema-describer/src/schema.rs | 7 + 13 files changed, 586 insertions(+), 1149 deletions(-) diff --git a/introspection-engine/connectors/mongodb-introspection-connector/src/sampler.rs b/introspection-engine/connectors/mongodb-introspection-connector/src/sampler.rs index d63d63b3eebf..39ff0cef751a 100644 --- a/introspection-engine/connectors/mongodb-introspection-connector/src/sampler.rs +++ b/introspection-engine/connectors/mongodb-introspection-connector/src/sampler.rs @@ -49,22 +49,19 @@ pub(super) async fn sample( } } - let data_model = statistics.into_datamodel(&mut warnings); - let is_empty = data_model.is_empty(); + let mut data_model = render::Datamodel::default(); + statistics.render(ctx.datasource(), &mut data_model, &mut warnings); - let mut rendered = render::Datamodel::default(); - rendered.push_dml(ctx.datasource(), &data_model); - let config = if ctx.render_config { - render::Configuration::from_psl(ctx.configuration()).to_string() + let psl_string = if ctx.render_config { + let config = render::Configuration::from_psl(ctx.configuration()); + format!("{config}\n{data_model}") } else { - String::new() + data_model.to_string() }; - let data_model = format!("{config}\n{rendered}"); - Ok(IntrospectionResult { - data_model: psl::reformat(&data_model, 2).unwrap(), - is_empty, + data_model: psl::reformat(&psl_string, 2).unwrap(), + is_empty: data_model.is_empty(), warnings, version: Version::NonPrisma, }) diff --git a/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/field_type.rs b/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/field_type.rs index 8a0f064ae997..1fcb5ca37f5a 100644 --- a/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/field_type.rs +++ b/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/field_type.rs @@ -62,6 +62,48 @@ impl FieldType { pub(super) fn is_array(&self) -> bool { matches!(self, Self::Array(_)) } + + pub(super) fn is_unsupported(&self) -> bool { + matches!(self, Self::Unsupported(_)) + } + + pub(super) fn is_document(&self) -> bool { + matches!(self, Self::Document(_)) + } + + pub(super) fn has_documents(&self) -> bool { + match self { + Self::Document(_) | Self::Json => true, + Self::Array(typ) => typ.is_document(), + _ => false, + } + } + + pub(super) fn prisma_type(&self) -> &str { + match self { + FieldType::String => "String", + FieldType::Double => "Float", + FieldType::BinData => "Bytes", + FieldType::ObjectId => "String", + FieldType::Bool => "Boolean", + FieldType::Date => "DateTime", + FieldType::Int32 => "Int", + FieldType::Timestamp => "DateTime", + FieldType::Int64 => "BigInt", + FieldType::Json => "Json", + FieldType::Document(ref s) => s, + FieldType::Array(ref r#type) => r#type.prisma_type(), + FieldType::Unsupported(r#type) => r#type, + } + } + + pub(super) fn native_type(&self) -> Option<&str> { + match self { + Self::ObjectId => Some("ObjectId"), + FieldType::Date => Some("Date"), + _ => None, + } + } } impl fmt::Display for FieldType { diff --git a/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/statistics.rs b/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/statistics.rs index 42fb0a85ea20..394206b8e349 100644 --- a/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/statistics.rs +++ b/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/statistics.rs @@ -2,20 +2,21 @@ mod indices; mod name; pub(crate) use name::Name; +use renderer::{ + datamodel::{IdFieldDefinition, UniqueFieldAttribute}, + value::Function, +}; use super::{field_type::FieldType, CompositeTypeDepth}; use convert_case::{Case, Casing}; +use datamodel_renderer as renderer; use introspection_connector::Warning; use mongodb::bson::{Bson, Document}; use mongodb_schema_describer::IndexWalker; use once_cell::sync::Lazy; -use psl::dml::{ - self, CompositeType, CompositeTypeField, CompositeTypeFieldType, Datamodel, DefaultValue, Field, FieldArity, Model, - PrimaryKeyDefinition, PrimaryKeyField, ScalarField, ScalarType, ValueGenerator, WithDatabaseName, -}; +use psl::datamodel_connector::constraint_names::ConstraintNames; use regex::Regex; use std::{ - borrow::Cow, cmp::Ordering, collections::{BTreeMap, HashMap, HashSet}, fmt, @@ -25,7 +26,12 @@ pub(super) const SAMPLE_SIZE: i32 = 1000; static RESERVED_NAMES: &[&str] = &["PrismaClient"]; static COMMENTED_OUT_FIELD: &str = "This field was commented out because of an invalid name. Please provide a valid one that matches [a-zA-Z][a-zA-Z0-9_]*"; -static EMPTY_TYPE_DETECTED: &str = "Nested objects had no data in the sample dataset to introspect a nested type."; + +#[derive(Default, Debug, Clone, Copy)] +struct ModelData { + document_count: usize, + has_id: bool, +} /// Statistical data from a MongoDB database for determining a Prisma data /// model. @@ -33,8 +39,10 @@ static EMPTY_TYPE_DETECTED: &str = "Nested objects had no data in the sample dat pub(super) struct Statistics<'a> { /// (container_name, field_name) -> type percentages samples: BTreeMap<(Name, String), FieldSampler>, + /// Container for composite types that are not empty + types_with_fields: HashSet, /// model_name -> document count - models: HashMap, + models: HashMap, /// model_name -> indices indices: BTreeMap>>, /// How deep we travel in nested composite types until switching to Json. None will always use @@ -43,63 +51,6 @@ pub(super) struct Statistics<'a> { } impl<'a> Statistics<'a> { - pub(super) fn new(composite_type_depth: CompositeTypeDepth) -> Self { - Self { - composite_type_depth, - ..Default::default() - } - } - - /// Track a collection as prisma model. - pub(super) fn track_model(&mut self, model: &str) { - self.models.entry(Name::Model(model.to_string())).or_insert(0); - } - - pub(super) fn track_model_fields(&mut self, model: &str, document: Document) { - self.track_document_types(Name::Model(model.to_string()), &document, self.composite_type_depth); - } - - /// Track an index for the given model. - pub(super) fn track_index(&mut self, model_name: &str, index: IndexWalker<'a>) { - let indexes = self.indices.entry(model_name.to_string()).or_default(); - indexes.push(index); - } - - /// From the given data, create a Prisma data model with best effort basis. - pub(super) fn into_datamodel(self, warnings: &mut Vec) -> Datamodel { - let mut data_model = Datamodel::new(); - let mut indices = self.indices; - let (mut models, mut types) = populate_fields(&self.models, self.samples, warnings); - - indices::add_to_models(&mut models, &mut types, &mut indices, warnings); - add_missing_ids_to_models(&mut models); - - for (_, model) in models.into_iter() { - data_model.models.push(model); - } - - for (_, mut composite_type) in types.into_iter() { - if composite_type - .fields - .iter() - .any(|f| f.database_name == Some("_id".into())) - { - if let Some(field) = composite_type - .fields - .iter_mut() - .find(|f| f.name == *"id" && f.database_name.is_none()) - { - field.name = "id_".into(); - field.database_name = Some("id".into()); - } - } - - data_model.composite_types.push(composite_type); - } - - data_model - } - /// Creates a new name for a composite type with the following rules: /// /// - if model is foo and field is bar, the type is FooBar @@ -121,18 +72,6 @@ impl<'a> Statistics<'a> { Name::CompositeType(name) } - /// Tracking the usage of types and names in a composite type. - fn track_composite_type_fields( - &mut self, - model: &str, - field: &str, - document: &Document, - depth: CompositeTypeDepth, - ) { - let name = self.composite_type_name(model, field); - self.track_document_types(name, document, depth); - } - /// If a document has a nested document, we'll introspect it as a composite /// type until a certain depth. The depth can be given by user, and if we /// reach enough nesting the following composite types are introspected as @@ -172,6 +111,285 @@ impl<'a> Statistics<'a> { (array_depth, found) } + pub(super) fn new(composite_type_depth: CompositeTypeDepth) -> Self { + Self { + composite_type_depth, + ..Default::default() + } + } + + pub(super) fn render( + &'a self, + datasource: &'a psl::Datasource, + rendered: &mut renderer::Datamodel<'a>, + warnings: &mut Vec, + ) { + let mut models: BTreeMap<&str, renderer::datamodel::Model<'_>> = self + .models + .iter() + .flat_map(|(name, doc_count)| name.as_model_name().map(|name| (name, doc_count))) + .map(|(name, doc_count)| { + let mut model = match sanitize_string(name) { + Some(sanitized) => { + let mut model = renderer::datamodel::Model::new(sanitized); + model.map(name); + model + } + None if RESERVED_NAMES.contains(&name) => { + let mut model = renderer::datamodel::Model::new(format!("Renamed{name}")); + model.map(name); + + let docs = format!("This model has been renamed to 'Renamed{name}' during introspection, because the original name '{name}' is reserved."); + model.documentation(docs); + + model + } + None => renderer::datamodel::Model::new(name), + }; + + if !doc_count.has_id { + let mut field = renderer::datamodel::Field::new("id", "String"); + + field.map("_id"); + field.native_type(&datasource.name, "ObjectId", Vec::new()); + field.default(renderer::datamodel::DefaultValue::function(Function::new("auto"))); + field.id(IdFieldDefinition::new()); + + model.push_field(field); + } + + (name, model) + }) + .collect(); + + let mut types: BTreeMap<&str, renderer::datamodel::CompositeType<'_>> = self + .models + .iter() + .flat_map(|(name, _)| name.as_type_name()) + .filter(|name| self.types_with_fields.contains(*name)) + .map(|name| (name, renderer::datamodel::CompositeType::new(name))) + .collect(); + + let mut unsupported = Vec::new(); + let mut unknown_types = Vec::new(); + let mut undecided_types = Vec::new(); + let mut fields_with_empty_names = Vec::new(); + let mut fields_with_an_empty_type = Vec::new(); + + for (model_name, indices) in self.indices.iter() { + let model = models.get_mut(model_name.as_str()).unwrap(); + + let indices = indices.iter().filter(|idx| { + !idx.is_unique() || idx.fields().len() > 1 || idx.fields().any(|f| f.name().contains('.')) + }); + + indices::render(model, model_name, indices); + } + + for ((container, field_name), sampler) in self.samples.iter() { + let doc_count = *self.models.get(container).unwrap_or(&Default::default()); + + let field_count = sampler.counter; + + let percentages = sampler.percentages(); + let most_common_type = percentages.find_most_common(); + let no_known_type = most_common_type.is_none(); + + let sanitized = sanitize_string(field_name); + + let points_to_an_empty_type = most_common_type + .as_ref() + .map(|t| t.is_document() && !types.contains_key(t.prisma_type())) + .unwrap_or(false); + + let field_type = match most_common_type { + Some(field_type) if !percentages.has_type_variety() => { + let prisma_type = field_type.prisma_type(); + + if !field_type.is_document() || types.contains_key(prisma_type) { + field_type + } else { + FieldType::Json + } + } + Some(_) if percentages.all_types_are_datetimes() => FieldType::Timestamp, + _ => FieldType::Json, + }; + + let prisma_type = field_type.prisma_type(); + + let mut field = match sanitized { + Some(sanitized) if sanitized.is_empty() => { + fields_with_empty_names.push((container.clone(), field_name.clone())); + + let mut field = renderer::datamodel::Field::new(field_name, prisma_type.to_string()); + field.map(field_name); + field.documentation(COMMENTED_OUT_FIELD); + field.commented_out(); + + field + } + Some(sanitized) => { + let mut field = renderer::datamodel::Field::new(sanitized, prisma_type.to_string()); + field.map(field_name); + + field + } + None if doc_count.has_id && field_name == "id" => { + let mut field = renderer::datamodel::Field::new("id_", prisma_type.to_string()); + field.map(field_name); + + field + } + None => renderer::datamodel::Field::new(field_name, prisma_type.to_string()), + }; + + if points_to_an_empty_type { + let docs = "Nested objects had no data in the sample dataset to introspect a nested type."; + field.documentation(docs); + fields_with_an_empty_type.push((container.clone(), field_name.clone())); + } + + if field_name == "_id" && !container.is_composite_type() { + field.id(IdFieldDefinition::default()); + + if matches!(field_type, FieldType::ObjectId) { + field.default(renderer::datamodel::DefaultValue::function(Function::new("auto"))); + } + } + + if let Some(native_type) = field_type.native_type() { + field.native_type(&datasource.name, native_type.to_string(), Vec::new()); + } + + if field_type.is_array() { + field.array(); + } else if doc_count.document_count > field_count || sampler.nullable { + field.optional(); + } + + if field_type.is_unsupported() { + field.unsupported(); + + unsupported.push(( + container.clone(), + field_name.to_string(), + field_type.prisma_type().to_string(), + )); + } + + if percentages.data.len() > 1 { + undecided_types.push((container.clone(), field_name.to_string(), field_type.to_string())); + } + + if sampler.from_index { + let docs = "Field referred in an index, but found no data to define the type."; + field.documentation(docs); + unknown_types.push((container.clone(), field_name.to_string())); + } + + if percentages.has_type_variety() { + let doc = format!("Multiple data types found: {percentages} out of {field_count} sampled entries",); + field.documentation(doc); + } + + if no_known_type { + let doc = "Could not determine type: the field only had null or empty values in the sample set."; + field.documentation(doc); + + unknown_types.push((container.clone(), field_name.to_string())); + } + + if !fields_with_an_empty_type.is_empty() { + warnings.push(crate::warnings::fields_pointing_to_an_empty_type( + &fields_with_an_empty_type, + )); + } + + match container { + Name::Model(ref model_name) => { + let unique = self.indices.get(model_name).and_then(|indices| { + indices.iter().find(|idx| { + idx.is_unique() && idx.fields().len() == 1 && idx.fields().any(|f| f.name() == field_name) + }) + }); + + if let Some(unique) = unique { + let index_field = unique.fields().next().unwrap(); + let mut attr = UniqueFieldAttribute::default(); + + let default_name = ConstraintNames::unique_index_name( + model_name, + &[index_field.name()], + psl::builtin_connectors::MONGODB, + ); + + if unique.name() != default_name { + attr.map(unique.name()); + }; + + if index_field.property.is_descending() { + attr.sort_order("Desc") + } + + field.unique(attr); + } + + let model = models.get_mut(model_name.as_str()).unwrap(); + + if field_name == "_id" { + model.insert_field_front(field); + } else { + model.push_field(field); + } + } + Name::CompositeType(ref type_name) => { + let r#type = types + .entry(type_name.as_str()) + .or_insert_with(|| renderer::datamodel::CompositeType::new(type_name)); + + r#type.push_field(field); + } + } + } + + for (_, r#type) in types { + rendered.push_composite_type(r#type); + } + + for (_, model) in models.into_iter() { + rendered.push_model(model); + } + + if !unsupported.is_empty() { + warnings.push(crate::warnings::unsupported_type(&unsupported)); + } + + if !undecided_types.is_empty() { + warnings.push(crate::warnings::undecided_field_type(&undecided_types)); + } + + if !fields_with_empty_names.is_empty() { + warnings.push(crate::warnings::fields_with_empty_names(&fields_with_empty_names)); + } + + if !unknown_types.is_empty() { + warnings.push(crate::warnings::fields_with_unknown_types(&unknown_types)); + } + } + + /// Tracking the usage of types and names in a composite type. + fn track_composite_type_fields( + &mut self, + model: &str, + field: &str, + document: &Document, + depth: CompositeTypeDepth, + ) { + let name = self.composite_type_name(model, field); + self.track_document_types(name, document, depth); + } + /// Track all fields and field types from the given document. fn track_document_types(&mut self, name: Name, document: &Document, depth: CompositeTypeDepth) { if name.is_composite_type() && depth.is_none() { @@ -179,13 +397,21 @@ impl<'a> Statistics<'a> { } let doc_count = self.models.entry(name.clone()).or_default(); - *doc_count += 1; + doc_count.document_count += 1; + doc_count.has_id = document.iter().any(|(key, _)| key == "_id"); let depth = match name { Name::CompositeType(_) => depth.level_down(), _ => depth, }; + match name { + Name::CompositeType(ref name) if !document.is_empty() => { + self.types_with_fields.insert(name.to_string()); + } + _ => (), + } + for (field, val) in document.into_iter() { let (array_layers, found_composite) = self.find_and_track_composite_types(name.as_ref(), field, val, depth); @@ -218,36 +444,102 @@ impl<'a> Statistics<'a> { } } } -} -/// A document must have a id column and the name is always `_id`. If we have no -/// data in the collection, we must assume an id field exists. -fn add_missing_ids_to_models(models: &mut BTreeMap) { - for (_, model) in models.iter_mut() { - if model.fields.iter().any(|f| f.database_name() == Some("_id")) { - continue; + /// Track an index for the given model. + pub(super) fn track_index(&mut self, model_name: &str, index: IndexWalker<'a>) { + for field in index.fields() { + if field.name().contains('.') { + let path_len = field.name().split('.').count(); + let path = field.name().split('.'); + let mut container_name = model_name.to_string(); + + for (i, field) in path.enumerate() { + let field = field.to_string(); + + let key = if i == 0 { + (Name::Model(container_name.clone()), field.clone()) + } else { + self.types_with_fields.insert(container_name.clone()); + let name = Name::CompositeType(container_name.clone()); + + if self.models.contains_key(&name) { + self.models.insert(name.clone(), Default::default()); + } + + (name, field.clone()) + }; + + let type_name = format!("{container_name}_{field}").to_case(Case::Pascal); + let type_name = sanitize_string(&type_name).unwrap_or(type_name); + container_name = type_name.clone(); + + if let Some(sampler) = self.samples.get_mut(&key) { + let has_composites = sampler.types.iter().any(|t| t.0.has_documents()); + + if i < path_len - 1 && !has_composites { + let counter = sampler.types.entry(FieldType::Document(type_name.clone())).or_default(); + *counter += 1; + } + + continue; + } + + let mut sampler = if i < path_len - 1 { + let mut sampler = FieldSampler::default(); + sampler.types.insert(FieldType::Document(type_name.clone()), 1); + + let key = Name::CompositeType(type_name); + self.models.entry(key).or_insert_with(Default::default); + + sampler + } else { + let mut sampler = FieldSampler::default(); + sampler.types.insert(FieldType::Json, 1); + + sampler + }; + + sampler.from_index = true; + sampler.nullable = true; + sampler.counter = 1; + + self.samples.insert(key, sampler); + } + } else { + let key = (Name::Model(model_name.to_string()), field.name().to_string()); + + if self.samples.contains_key(&key) { + continue; + } + + let mut sampler = FieldSampler::default(); + sampler.types.insert(FieldType::Json, 1); + sampler.from_index = true; + sampler.nullable = true; + sampler.counter = 1; + + self.samples.insert(key, sampler); + } } - let field = ScalarField { - name: String::from("id"), - field_type: dml::FieldType::from(FieldType::ObjectId), - arity: FieldArity::Required, - database_name: Some(String::from("_id")), - default_value: Some(DefaultValue::new_expression(ValueGenerator::new_auto())), - documentation: None, - is_generated: false, - is_updated_at: false, - is_commented_out: false, - is_ignored: false, - }; + let indexes = self.indices.entry(model_name.to_string()).or_default(); + indexes.push(index); + } - model.fields.insert(0, Field::ScalarField(field)); + /// Track a collection as prisma model. + pub(super) fn track_model(&mut self, model: &str) { + self.models.entry(Name::Model(model.to_string())).or_default(); + } + + pub(super) fn track_model_fields(&mut self, model: &str, document: Document) { + self.track_document_types(Name::Model(model.to_string()), &document, self.composite_type_depth); } } #[derive(Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] pub struct FieldSampler { types: BTreeMap, + from_index: bool, nullable: bool, counter: usize, } @@ -309,278 +601,6 @@ impl FieldPercentages { } } -fn new_composite_type(type_name: &str) -> CompositeType { - CompositeType { - name: type_name.to_string(), - fields: Vec::new(), - } -} - -fn new_model(model_name: &str) -> Model { - let primary_key = PrimaryKeyDefinition { - name: None, - db_name: None, - fields: vec![PrimaryKeyField { - name: "id".to_string(), - sort_order: None, - length: None, - }], - defined_on_field: true, - clustered: None, - }; - - let (name, database_name, documentation) = match sanitize_string(model_name) { - Some(sanitized) => (Cow::from(sanitized), Some(model_name.to_string()), None), - None if RESERVED_NAMES.contains(&model_name) => { - let documentation = "This model has been renamed to 'RenamedPrismaClient' during introspection, because the original name 'PrismaClient' is reserved."; - - ( - Cow::from(format!("Renamed{}", model_name)), - Some(model_name.to_string()), - Some(documentation.to_string()), - ) - } - None => (Cow::from(model_name), None, None), - }; - - Model { - name: name.to_string(), - primary_key: Some(primary_key), - fields: vec![], - database_name, - documentation, - ..Default::default() - } -} - -/// Read all samples from the data, returning models and composite types. -/// -/// ## Input -/// -/// - Samples, counting how many documents altogether there was in the model or -/// in how many documents we had data for the composite type. -/// - Fields counts from model or type and field name combination to statistics -/// of different types seen in the data. -fn populate_fields( - samples: &HashMap, - fields: BTreeMap<(Name, String), FieldSampler>, - warnings: &mut Vec, -) -> (BTreeMap, BTreeMap) { - let mut models: BTreeMap = samples - .iter() - .flat_map(|(name, _)| name.as_model_name()) - .map(|model_name| (model_name.to_string(), new_model(model_name))) - .collect(); - - let mut types: BTreeMap = samples - .iter() - .flat_map(|(name, _)| name.as_type_name()) - .map(|type_name| (type_name.to_string(), new_composite_type(type_name))) - .collect(); - - let mut unsupported = Vec::new(); - let mut unknown_types = Vec::new(); - let mut undecided_types = Vec::new(); - let mut fields_with_empty_names = Vec::new(); - - for ((container, field_name), sampler) in fields.into_iter() { - let doc_count = *samples.get(&container).unwrap_or(&0); - let field_count = sampler.counter; - - let percentages = sampler.percentages(); - let most_common_type = percentages.find_most_common(); - - let field_type = match &most_common_type { - Some(field_type) if !percentages.has_type_variety() => field_type.to_owned(), - Some(_) if percentages.all_types_are_datetimes() => FieldType::Timestamp, - _ => FieldType::Json, - }; - - if let FieldType::Unsupported(r#type) = field_type { - unsupported.push((container.clone(), field_name.to_string(), r#type)); - } - - if percentages.data.len() > 1 { - undecided_types.push((container.clone(), field_name.to_string(), field_type.to_string())); - } - - let arity = if field_type.is_array() { - FieldArity::List - } else if doc_count > field_count || sampler.nullable { - FieldArity::Optional - } else { - FieldArity::Required - }; - - let mut documentation = if percentages.has_type_variety() { - Some(format!( - "Multiple data types found: {} out of {} sampled entries", - percentages, field_count - )) - } else { - None - }; - - if most_common_type.is_none() { - static UNKNOWN_FIELD: &str = - "Could not determine type: the field only had null or empty values in the sample set."; - - match &mut documentation { - Some(docs) => { - docs.push('\n'); - docs.push_str(UNKNOWN_FIELD); - } - None => { - documentation = Some(UNKNOWN_FIELD.to_owned()); - } - } - - unknown_types.push((container.clone(), field_name.to_string())); - } - - let (name, database_name, is_commented_out) = match sanitize_string(&field_name) { - Some(sanitized) if sanitized.is_empty() => { - match documentation.as_mut() { - Some(ref mut existing) => { - existing.push('\n'); - existing.push_str(COMMENTED_OUT_FIELD); - } - None => { - documentation = Some(COMMENTED_OUT_FIELD.to_string()); - } - }; - - fields_with_empty_names.push((container.clone(), field_name.clone())); - - (field_name.clone(), Some(field_name), true) - } - Some(sanitized) => (sanitized, Some(field_name), false), - None if matches!(container, Name::Model(_)) && field_name == "id" => { - ("id_".to_string(), Some(field_name), false) - } - None => (field_name, None, false), - }; - - match container { - Name::Model(model_name) => { - let model = models.get_mut(&model_name).unwrap(); - - let mut field = ScalarField { - name, - field_type: dml::FieldType::from(field_type.clone()), - arity, - database_name, - default_value: None, - documentation, - is_generated: false, - is_updated_at: false, - is_commented_out, - is_ignored: false, - }; - - match &field.database_name { - Some(name) if name == "_id" => { - if let FieldType::ObjectId = &field_type { - field.set_default_value(DefaultValue::new_expression(ValueGenerator::new_auto())); - }; - - model.fields.insert(0, Field::ScalarField(field)); - } - _ => model.fields.push(Field::ScalarField(field)), - }; - } - Name::CompositeType(type_name) => { - let r#type = types.get_mut(&type_name).unwrap(); - - r#type.fields.push(CompositeTypeField { - name, - r#type: field_type.into(), - default_value: None, - arity, - documentation, - database_name, - is_commented_out, - }); - } - } - } - - if !unsupported.is_empty() { - warnings.push(crate::warnings::unsupported_type(&unsupported)); - } - - if !undecided_types.is_empty() { - warnings.push(crate::warnings::undecided_field_type(&undecided_types)); - } - - if !fields_with_empty_names.is_empty() { - warnings.push(crate::warnings::fields_with_empty_names(&fields_with_empty_names)); - } - - if !unknown_types.is_empty() { - warnings.push(crate::warnings::fields_with_unknown_types(&unknown_types)); - } - - filter_out_empty_types(&mut models, &mut types, warnings); - - (models, types) -} - -/// From the resulting data model, remove all types with no fields and change -/// the field types to Json. -fn filter_out_empty_types( - models: &mut BTreeMap, - types: &mut BTreeMap, - warnings: &mut Vec, -) { - let mut fields_with_an_empty_type = Vec::new(); - - // 1. remove all types that have no fields. - let empty_types: HashSet<_> = types - .iter() - .filter(|(_, r#type)| r#type.fields.is_empty()) - .map(|(name, _)| name.to_owned()) - .collect(); - - // https://github.com/rust-lang/rust/issues/70530 - types.retain(|_, r#type| !r#type.fields.is_empty()); - - // 2. change all fields in models that point to a non-existing type to Json. - for (model_name, model) in models.iter_mut() { - for field in model.fields.iter_mut().filter_map(|f| f.as_scalar_field_mut()) { - match &field.field_type { - dml::FieldType::CompositeType(ct) if empty_types.contains(ct) => { - fields_with_an_empty_type.push((Name::Model(model_name.clone()), field.name.clone())); - field.field_type = dml::FieldType::Scalar(ScalarType::Json, None); - field.documentation = Some(EMPTY_TYPE_DETECTED.to_owned()); - } - _ => (), - } - } - } - - // 3. change all fields in types that point to a non-existing type to Json. - for (type_name, r#type) in types.iter_mut() { - for field in r#type.fields.iter_mut() { - match &field.r#type { - CompositeTypeFieldType::CompositeType(name) if empty_types.contains(name) => { - fields_with_an_empty_type.push((Name::CompositeType(type_name.clone()), field.name.clone())); - field.r#type = CompositeTypeFieldType::Scalar(ScalarType::Json, None); - field.documentation = Some(EMPTY_TYPE_DETECTED.to_owned()); - } - _ => (), - } - } - } - - // 4. add warnings in the end to reduce spam - if !fields_with_an_empty_type.is_empty() { - warnings.push(crate::warnings::fields_pointing_to_an_empty_type( - &fields_with_an_empty_type, - )); - } -} - fn sanitize_string(s: &str) -> Option { static RE_START: Lazy = Lazy::new(|| Regex::new("^[^a-zA-Z]+").unwrap()); static RE: Lazy = Lazy::new(|| Regex::new("[^_a-zA-Z0-9]").unwrap()); diff --git a/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/statistics/indices.rs b/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/statistics/indices.rs index bb0fa8aae31f..f71239f2d98e 100644 --- a/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/statistics/indices.rs +++ b/introspection-engine/connectors/mongodb-introspection-connector/src/sampler/statistics/indices.rs @@ -1,290 +1,58 @@ -use super::Name; -use crate::sampler::field_type::FieldType; -use convert_case::{Case, Casing}; -use introspection_connector::Warning; -use mongodb_schema_describer::{IndexFieldProperty, IndexWalker}; -use psl::{ - datamodel_connector::constraint_names::ConstraintNames, - dml::{self, WithDatabaseName, WithName}, -}; -use std::collections::BTreeMap; - -/// Add described indices to the models. -pub(super) fn add_to_models( - models: &mut BTreeMap, - types: &mut BTreeMap, - indices: &mut BTreeMap>>, - warnings: &mut Vec, -) { - let mut fields_with_unknown_type = Vec::new(); - - for (model_name, model) in models.iter_mut() { - for index in indices.remove(model_name).into_iter().flat_map(|i| i.into_iter()) { - let defined_on_field = index.fields().len() == 1 && !index.fields().any(|f| f.name().contains('.')); - - add_missing_fields_from_index(model, index, &mut fields_with_unknown_type); - add_missing_types_from_index(types, model, index, &mut fields_with_unknown_type); - - let fields = index - .fields() - .map(|f| { - let mut path = Vec::new(); - let mut splitted_name = f.name().split('.'); - let mut next_type: Option<&dml::CompositeType> = None; - - if let Some(field_name) = splitted_name.next() { - next_type = model - .fields() - .find(|f| f.database_name() == Some(field_name) || f.name() == field_name) - .and_then(|f| f.as_scalar_field()) - .and_then(|f| match &f.field_type { - dml::FieldType::CompositeType(ref r#type) => types.get(r#type), - _ => None, - }); - - let name = super::sanitize_string(field_name).unwrap_or_else(|| field_name.to_owned()); - - path.push((name, None)); - } - - for field_name in splitted_name { - let ct_name = next_type.as_ref().map(|ct| ct.name.clone()); - let name = super::sanitize_string(field_name).unwrap_or_else(|| field_name.to_owned()); - path.push((name, ct_name)); - - next_type = next_type.as_ref().and_then(|ct| { - ct.fields - .iter() - .find(|f| f.database_name.as_deref() == Some(field_name) || f.name == field_name) - .and_then(|f| match &f.r#type { - dml::CompositeTypeFieldType::CompositeType(ref r#type) => types.get(r#type), - _ => None, - }) - }); - } - - dml::IndexField { - path, - sort_order: match f.property { - IndexFieldProperty::Text => None, - IndexFieldProperty::Ascending if index.r#type().is_fulltext() => Some(dml::SortOrder::Asc), - IndexFieldProperty::Ascending => None, - IndexFieldProperty::Descending => Some(dml::SortOrder::Desc), - }, - length: None, - operator_class: None, - } +use datamodel_renderer as renderer; +use mongodb_schema_describer::{IndexFieldProperty, IndexType, IndexWalker}; +use psl::datamodel_connector::constraint_names::ConstraintNames; +use renderer::datamodel::{IndexDefinition, IndexFieldInput, Model}; +use std::borrow::Cow; + +pub(super) fn render<'a>(model: &mut Model<'a>, model_name: &str, indices: impl Iterator>) { + for index in indices { + let fields = index.fields().map(|field| { + let name = field + .name() + .split('.') + .map(|part| { + super::sanitize_string(part) + .map(Cow::Owned) + .unwrap_or_else(|| Cow::Borrowed(part)) }) - .collect::>(); - - let tpe = match index.r#type() { - mongodb_schema_describer::IndexType::Normal => dml::IndexType::Normal, - mongodb_schema_describer::IndexType::Unique => dml::IndexType::Unique, - mongodb_schema_describer::IndexType::Fulltext => dml::IndexType::Fulltext, - }; + .collect::>() + .join("."); - // TOM WE NEED TO TALK ABOUT THIS! - let column_names = fields - .iter() - .flat_map(|f| f.path.iter().map(|p| p.0.as_str())) - .collect::>(); + let mut rendered = IndexFieldInput::new(name); - let default_name = match tpe { - dml::IndexType::Unique => { - ConstraintNames::unique_index_name(model_name, &column_names, psl::builtin_connectors::MONGODB) + match field.property { + IndexFieldProperty::Text => (), + IndexFieldProperty::Ascending if index.r#type().is_fulltext() => { + rendered.sort_order("Asc"); } - _ => { - ConstraintNames::non_unique_index_name(model_name, &column_names, psl::builtin_connectors::MONGODB) + IndexFieldProperty::Descending => { + rendered.sort_order("Desc"); } - }; - - let db_name = if index.name() == default_name { - None - } else { - Some(index.name().to_string()) - }; - - model.add_index(dml::IndexDefinition { - fields, - tpe, - defined_on_field, - db_name, - name: None, - algorithm: None, - clustered: None, - }); - } - } + IndexFieldProperty::Ascending => (), + } - if !fields_with_unknown_type.is_empty() { - warnings.push(crate::warnings::fields_with_unknown_types(&fields_with_unknown_type)); - } -} + rendered + }); -/// If an index points to a field not in the model, we'll add it as an unknown -/// field with type `Json?`. -fn add_missing_fields_from_index( - model: &mut dml::Model, - index: IndexWalker<'_>, - unknown_fields: &mut Vec<(Name, String)>, -) { - let missing_fields_in_models: Vec<_> = index - .fields() - .filter(|indf| !indf.name().contains('.')) - .filter(|indf| { - !model - .fields - .iter() - .any(|mf| mf.name() == indf.name() || mf.database_name() == Some(indf.name())) - }) - .cloned() - .collect(); + let mut rendered = match index.r#type() { + IndexType::Normal => IndexDefinition::index(fields), + IndexType::Unique => IndexDefinition::unique(fields), + IndexType::Fulltext => IndexDefinition::fulltext(fields), + }; - for field in missing_fields_in_models { - let docs = String::from("Field referred in an index, but found no data to define the type."); - unknown_fields.push((Name::Model(model.name().clone()), field.name.clone())); + let column_names = index.fields().flat_map(|f| f.name().split('.')).collect::>(); - let (name, database_name) = match super::sanitize_string(field.name()) { - Some(name) => (name, Some(field.name)), - None => (field.name, None), + let default_name = match index.r#type() { + IndexType::Unique => { + ConstraintNames::unique_index_name(model_name, &column_names, psl::builtin_connectors::MONGODB) + } + _ => ConstraintNames::non_unique_index_name(model_name, &column_names, psl::builtin_connectors::MONGODB), }; - let sf = dml::ScalarField { - name, - field_type: FieldType::Json.into(), - arity: dml::FieldArity::Optional, - database_name, - default_value: None, - documentation: Some(docs), - is_generated: false, - is_updated_at: false, - is_commented_out: false, - is_ignored: false, + if index.name() != default_name { + rendered.map(index.name()); }; - model.fields.push(dml::Field::ScalarField(sf)); - } -} - -/// If an index points to a composite field, and we have no data to describe it, -/// we create a stub type and a field of type `Json?` to mark the unknown field. -fn add_missing_types_from_index( - types: &mut BTreeMap, - model: &mut dml::Model, - index: IndexWalker<'_>, - unknown_fields: &mut Vec<(Name, String)>, -) { - let composite_fields = index.fields().filter(|indf| indf.name().contains('.')); - - for indf in composite_fields { - let path_length = indf.name().split('.').count(); - let mut splitted_name = indf.name().split('.').enumerate(); - let mut next_type = None; - - if let Some((_, field_name)) = splitted_name.next() { - let sf = model - .fields() - .find(|f| f.database_name() == Some(field_name) || f.name() == field_name) - .and_then(|f| f.as_scalar_field()); - - next_type = match sf { - Some(sf) => sf.field_type.as_composite_type().map(|tyname| tyname.to_owned()), - None => { - let docs = String::from("Field referred in an index, but found no data to define the type."); - - let (field_name, database_name) = match super::sanitize_string(field_name) { - Some(name) => (name, Some(field_name.to_string())), - None => (field_name.to_string(), None), - }; - - let type_name = format!("{}_{}", model.name, field_name).to_case(Case::Pascal); - - let ct = dml::CompositeType { - name: type_name.clone(), - fields: Vec::new(), - }; - - types.insert(type_name.clone(), ct); - - let sf = dml::ScalarField { - name: field_name, - field_type: dml::FieldType::CompositeType(type_name.clone()), - arity: dml::FieldArity::Optional, - database_name, - default_value: None, - documentation: Some(docs), - is_generated: false, - is_updated_at: false, - is_commented_out: false, - is_ignored: false, - }; - - model.fields.push(dml::Field::ScalarField(sf)); - - Some(type_name) - } - }; - } - - for (i, field_name) in splitted_name { - let type_name = match next_type.take() { - Some(name) => name, - None => continue, - }; - let ct = types.get(&type_name).unwrap(); - - let cf = ct - .fields - .iter() - .find(|f| f.database_name.as_deref() == Some(field_name) || f.name == field_name); - - next_type = match (cf, cf.and_then(|cf| cf.r#type.as_composite_type())) { - (Some(_), _) if i + 1 == path_length => None, - (Some(_), Some(type_name)) => Some(type_name.to_string()), - (None, _) | (_, None) => { - let docs = String::from("Field referred in an index, but found no data to define the type."); - - let (field_name, database_name) = match super::sanitize_string(field_name) { - Some(name) => (name, Some(field_name.to_string())), - None => (field_name.to_string(), None), - }; - - let (r#type, new_type_name) = if i + 1 < path_length { - let type_name = format!("{}_{}", type_name, field_name).to_case(Case::Pascal); - - types.insert( - type_name.clone(), - dml::CompositeType { - name: type_name.clone(), - fields: Vec::new(), - }, - ); - - ( - dml::CompositeTypeFieldType::CompositeType(type_name.clone()), - Some(type_name.clone()), - ) - } else { - unknown_fields.push((Name::CompositeType(type_name.clone()), field_name.clone())); - - (dml::CompositeTypeFieldType::Scalar(dml::ScalarType::Json, None), None) - }; - - let ct = types.get_mut(&type_name).unwrap(); - - ct.fields.push(dml::CompositeTypeField { - name: field_name, - r#type, - arity: dml::FieldArity::Optional, - database_name, - documentation: Some(docs), - default_value: None, - is_commented_out: false, - }); - - new_type_name - } - } - } + model.push_index(rendered); } } diff --git a/introspection-engine/connectors/mongodb-introspection-connector/src/warnings.rs b/introspection-engine/connectors/mongodb-introspection-connector/src/warnings.rs index 33a21aea2a70..9ce7308920d9 100644 --- a/introspection-engine/connectors/mongodb-introspection-connector/src/warnings.rs +++ b/introspection-engine/connectors/mongodb-introspection-connector/src/warnings.rs @@ -3,7 +3,7 @@ use serde_json::json; use crate::sampler::Name; -pub(crate) fn unsupported_type(affected: &[(Name, String, &str)]) -> Warning { +pub(crate) fn unsupported_type(affected: &[(Name, String, String)]) -> Warning { let affected = serde_json::Value::Array({ affected .iter() diff --git a/introspection-engine/connectors/mongodb-introspection-connector/tests/index/mod.rs b/introspection-engine/connectors/mongodb-introspection-connector/tests/index/mod.rs index 67a66cbb8ed7..f7122c86e51c 100644 --- a/introspection-engine/connectors/mongodb-introspection-connector/tests/index/mod.rs +++ b/introspection-engine/connectors/mongodb-introspection-connector/tests/index/mod.rs @@ -1353,9 +1353,9 @@ fn index_pointing_to_non_existing_field_should_add_the_field() { let expected = expect![[r#" model A { id String @id @default(auto()) @map("_id") @db.ObjectId - name String /// Field referred in an index, but found no data to define the type. age Json? + name String @@index([age], map: "age_1") } @@ -1400,9 +1400,9 @@ fn index_pointing_to_non_existing_composite_field_should_add_the_field_and_type( model Cat { id String @id @default(auto()) @map("_id") @db.ObjectId - name String /// Field referred in an index, but found no data to define the type. info CatInfo? + name String @@index([info.age], map: "info.age_1") } @@ -1412,10 +1412,16 @@ fn index_pointing_to_non_existing_composite_field_should_add_the_field_and_type( res.assert_warning("Could not determine the types for the following fields."); res.assert_warning_code(103); - res.assert_warning_affected(&json!([{ - "compositeType": "CatInfo", - "field": "age" - }])); + res.assert_warning_affected(&json!([ + { + "model": "Cat", + "field": "info" + }, + { + "compositeType": "CatInfo", + "field": "age" + } + ])); } #[test] @@ -1452,9 +1458,9 @@ fn deep_index_pointing_to_non_existing_composite_field_should_add_the_field_and_ model Cat { id String @id @default(auto()) @map("_id") @db.ObjectId - name String /// Field referred in an index, but found no data to define the type. info CatInfo? + name String @@index([info.specific.age], map: "info.specific.age_1") } @@ -1464,10 +1470,20 @@ fn deep_index_pointing_to_non_existing_composite_field_should_add_the_field_and_ res.assert_warning("Could not determine the types for the following fields."); res.assert_warning_code(103); - res.assert_warning_affected(&json!([{ - "compositeType": "CatInfoSpecific", - "field": "age" - }])); + res.assert_warning_affected(&json!([ + { + "model": "Cat", + "field": "info" + }, + { + "compositeType": "CatInfo", + "field": "specific" + }, + { + "compositeType": "CatInfoSpecific", + "field": "age" + } + ])); } #[test] @@ -1494,9 +1510,9 @@ fn index_pointing_to_mapped_non_existing_field_should_add_the_mapped_field() { let expected = expect![[r#" model A { id String @id @default(auto()) @map("_id") @db.ObjectId - name String /// Field referred in an index, but found no data to define the type. age Json? @map("_age") + name String @@index([age], map: "_age_1") } @@ -1541,9 +1557,9 @@ fn composite_index_pointing_to_mapped_non_existing_field_should_add_the_mapped_f model A { id String @id @default(auto()) @map("_id") @db.ObjectId - name String /// Field referred in an index, but found no data to define the type. info AInfo? + name String @@index([info.age], map: "info._age_1") } @@ -1553,10 +1569,16 @@ fn composite_index_pointing_to_mapped_non_existing_field_should_add_the_mapped_f res.assert_warning("Could not determine the types for the following fields."); res.assert_warning_code(103); - res.assert_warning_affected(&json!([{ - "compositeType": "AInfo", - "field": "age" - }])); + res.assert_warning_affected(&json!([ + { + "model": "A", + "field": "info" + }, + { + "compositeType": "AInfo", + "field": "_age" + } + ])); } #[test] @@ -1583,9 +1605,9 @@ fn compound_index_pointing_to_non_existing_field_should_add_the_field() { let expected = expect![[r#" model A { id String @id @default(auto()) @map("_id") @db.ObjectId - name String /// Field referred in an index, but found no data to define the type. age Json? + name String /// Field referred in an index, but found no data to define the type. play Json? @@ -1705,6 +1727,10 @@ fn deep_composite_index_with_one_existing_field_should_add_missing_stuff_only() res.assert_warning("Could not determine the types for the following fields."); res.assert_warning_code(103); res.assert_warning_affected(&json!([ + { + "compositeType": "AInfo", + "field": "special" + }, { "compositeType": "AInfoSpecial", "field": "play" @@ -1788,10 +1814,10 @@ fn deep_composite_index_should_add_missing_stuff_in_different_layers() { let expected = expect![[r#" type AInfo { - /// Field referred in an index, but found no data to define the type. - special AInfoSpecial? /// Field referred in an index, but found no data to define the type. play Json? + /// Field referred in an index, but found no data to define the type. + special AInfoSpecial? } type AInfoSpecial { @@ -1801,9 +1827,9 @@ fn deep_composite_index_should_add_missing_stuff_in_different_layers() { model A { id String @id @default(auto()) @map("_id") @db.ObjectId - name String /// Field referred in an index, but found no data to define the type. info AInfo? + name String @@index([info.special.age, info.play], map: "info.special.age_1_info.play_1") } @@ -1815,12 +1841,20 @@ fn deep_composite_index_should_add_missing_stuff_in_different_layers() { res.assert_warning_code(103); res.assert_warning_affected(&json!([ { - "compositeType": "AInfoSpecial", - "field": "age" + "model": "A", + "field": "info", }, { "compositeType": "AInfo", - "field": "play" + "field": "play", + }, + { + "compositeType": "AInfo", + "field": "special", + }, + { + "compositeType": "AInfoSpecial", + "field": "age", }, ])); } @@ -1894,9 +1928,9 @@ fn unique_index_pointing_to_non_existing_field_should_add_the_field() { let expected = expect![[r#" model A { id String @id @default(auto()) @map("_id") @db.ObjectId - name String /// Field referred in an index, but found no data to define the type. age Json? @unique(map: "age_1") + name String } "#]]; @@ -1934,9 +1968,9 @@ fn fulltext_index_pointing_to_non_existing_field_should_add_the_field() { let expected = expect![[r#" model A { id String @id @default(auto()) @map("_id") @db.ObjectId - name String /// Field referred in an index, but found no data to define the type. age Json? @unique(map: "age_1") + name String } "#]]; @@ -1972,6 +2006,18 @@ fn composite_type_index_without_corresponding_data_should_not_crash() { }); let expected = expect![[r#" + type AFoo { + /// Field referred in an index, but found no data to define the type. + bar Json? + /// Field referred in an index, but found no data to define the type. + baz AFooBaz? + } + + type AFooBaz { + /// Field referred in an index, but found no data to define the type. + quux Json? + } + model A { id String @id @default(auto()) @map("_id") @db.ObjectId /// Field referred in an index, but found no data to define the type. @@ -2003,10 +2049,10 @@ fn composite_type_index_with_non_composite_fields_in_the_middle_should_not_crash let expected = expect![[r#" type AA { - b Int + /// Nested objects had no data in the sample dataset to introspect a nested type. + /// Multiple data types found: Int: 50%, AaB: 50% out of 1 sampled entries + b Json d AaD - /// Field referred in an index, but found no data to define the type. - b AaB? } type AaB { diff --git a/introspection-engine/connectors/mongodb-introspection-connector/tests/remapping_names/mod.rs b/introspection-engine/connectors/mongodb-introspection-connector/tests/remapping_names/mod.rs index 92ae09aeb52a..d39bdb32cbc1 100644 --- a/introspection-engine/connectors/mongodb-introspection-connector/tests/remapping_names/mod.rs +++ b/introspection-engine/connectors/mongodb-introspection-connector/tests/remapping_names/mod.rs @@ -159,8 +159,8 @@ fn remapping_model_fields_with_numbers_dirty() { let expected = expect![[r#" model Outer { id String @id @default(auto()) @map("_id") @db.ObjectId - /// Multiple data types found: String: 50%, Int: 50% out of 2 sampled entries /// This field was commented out because of an invalid name. Please provide a valid one that matches [a-zA-Z][a-zA-Z0-9_]* + /// Multiple data types found: String: 50%, Int: 50% out of 2 sampled entries // 1 Json @map("1") } "#]]; diff --git a/introspection-engine/datamodel-renderer/src/datamodel.rs b/introspection-engine/datamodel-renderer/src/datamodel.rs index 01e7bc0393c8..9092da698a0d 100644 --- a/introspection-engine/datamodel-renderer/src/datamodel.rs +++ b/introspection-engine/datamodel-renderer/src/datamodel.rs @@ -19,7 +19,6 @@ pub use field_type::FieldType; pub use index::{IdDefinition, IdFieldDefinition, IndexDefinition, IndexFieldInput, IndexOps, UniqueFieldAttribute}; pub use model::{Model, Relation}; -use psl::dml; use std::fmt; /// The PSL data model declaration. @@ -69,19 +68,6 @@ impl<'a> Datamodel<'a> { self.composite_types.push(composite_type); } - /// A throwaway function to help generate a rendering from the DML structures. - /// - /// Delete when removing DML. - pub fn push_dml(&mut self, datasource: &'a psl::Datasource, dml_data_model: &dml::Datamodel) { - for dml_model in dml_data_model.models() { - self.push_model(Model::from_dml(datasource, dml_model)); - } - - for dml_ct in dml_data_model.composite_types() { - self.push_composite_type(CompositeType::from_dml(datasource, dml_ct)); - } - } - /// True if the render output would be an empty string. pub fn is_empty(&self) -> bool { self.models.is_empty() && self.enums.is_empty() && self.composite_types.is_empty() diff --git a/introspection-engine/datamodel-renderer/src/datamodel/composite_type.rs b/introspection-engine/datamodel-renderer/src/datamodel/composite_type.rs index 573a71556ac3..be48f0ccbe68 100644 --- a/introspection-engine/datamodel-renderer/src/datamodel/composite_type.rs +++ b/introspection-engine/datamodel-renderer/src/datamodel/composite_type.rs @@ -1,5 +1,4 @@ use crate::value::{Constant, Documentation}; -use psl::dml; use std::{borrow::Cow, fmt}; use super::Field; @@ -54,22 +53,6 @@ impl<'a> CompositeType<'a> { pub fn push_field(&mut self, field: Field<'a>) { self.fields.push(field); } - - /// Generate a composite type rendering from the deprecated DML structure. - /// - /// Remove when destroying the DML. - pub fn from_dml(datasource: &'a psl::Datasource, dml_ct: &dml::CompositeType) -> Self { - let mut composite_type = CompositeType::new(dml_ct.name.clone()); - let mut uniques = Default::default(); - - for dml_field in dml_ct.fields.iter() { - // TODO: remove when removing dml from mongo connector - let dml_field = dml::Field::from(dml_field.clone()); - composite_type.push_field(Field::from_dml(datasource, &dml_field, &mut uniques, None)); - } - - composite_type - } } impl<'a> fmt::Display for CompositeType<'a> { diff --git a/introspection-engine/datamodel-renderer/src/datamodel/default.rs b/introspection-engine/datamodel-renderer/src/datamodel/default.rs index 68ebd8597e4e..d0a780b1b93e 100644 --- a/introspection-engine/datamodel-renderer/src/datamodel/default.rs +++ b/introspection-engine/datamodel-renderer/src/datamodel/default.rs @@ -1,7 +1,5 @@ use std::{borrow::Cow, fmt}; -use psl::dml; - use crate::value::{Array, Constant, Function, Text, Value}; use super::attributes::FieldAttribute; @@ -110,83 +108,11 @@ impl<'a> DefaultValue<'a> { self.0.push_param(("map", Text::new(mapped_name))); } - /// Here to cope with the initial issue of needing the DML - /// structures. Remove when we don't generate DML in intro - /// anymore. - pub fn from_dml(val: &dml::DefaultValue) -> Self { - let mut dv = match &val.kind { - dml::DefaultKind::Single(dml::PrismaValue::String(val)) => Self::text(val.clone()), - dml::DefaultKind::Single(dml::PrismaValue::Boolean(val)) => Self::constant(*val), - dml::DefaultKind::Single(dml::PrismaValue::Enum(val)) => { - Self::constant(Cow::::Owned(String::clone(val))) - } - dml::DefaultKind::Single(dml::PrismaValue::Int(val)) => Self::constant(*val), - dml::DefaultKind::Single(dml::PrismaValue::Uuid(val)) => Self::constant(val.as_hyphenated().to_string()), - dml::DefaultKind::Single(dml::PrismaValue::List(ref vals)) => { - Self::array(vals.iter().map(Value::from).collect()) - } - dml::DefaultKind::Single(dml::PrismaValue::Json(val)) => Self::text(val.clone()), - dml::DefaultKind::Single(dml::PrismaValue::Xml(val)) => Self::text(val.clone()), - dml::DefaultKind::Single(dml::PrismaValue::Float(ref val)) => Self::constant(val.clone()), - dml::DefaultKind::Single(dml::PrismaValue::BigInt(val)) => Self::constant(*val), - dml::DefaultKind::Single(dml::PrismaValue::Bytes(val)) => Self::bytes(Cow::Owned(val.clone())), - dml::DefaultKind::Single(dml::PrismaValue::DateTime(val)) => Self::constant(*val), - dml::DefaultKind::Single(dml::PrismaValue::Object(_)) => unreachable!(), - dml::DefaultKind::Single(dml::PrismaValue::Null) => unreachable!(), - dml::DefaultKind::Expression(ref expr) => { - let mut fun = Function::new(expr.name().to_owned()); - fun.render_empty_parentheses(); - - for (arg_name, value) in expr.args() { - match arg_name { - Some(name) => fun.push_param((Cow::Owned(name.clone()), Value::from(value))), - None => fun.push_param(Value::from(value)), - } - } - - Self::function(fun) - } - }; - - if let Some(s) = val.db_name() { - dv.map(s.to_owned()); - } - - dv - } - fn new(inner: Function<'a>) -> Self { Self(FieldAttribute::new(inner)) } } -// TODO: remove when dml is dead. -impl From<&dml::PrismaValue> for Value<'static> { - fn from(value: &dml::PrismaValue) -> Self { - match value { - dml::PrismaValue::String(s) => Value::Text(Text(s.clone().into())), - dml::PrismaValue::Boolean(v) => Value::from(Constant::new_no_validate(v)), - dml::PrismaValue::Enum(val) => Value::from(Constant::new_no_validate(val)), - dml::PrismaValue::Int(val) => Value::from(Constant::new_no_validate(val)), - dml::PrismaValue::Uuid(val) => Value::from(Constant::new_no_validate(val.as_hyphenated())), - dml::PrismaValue::List(vals) => { - let vals = vals.iter().collect::>(); - let constant = Box::new(Array::from(vals)); - - Value::from(Constant::new_no_validate(constant)) - } - dml::PrismaValue::Json(val) => Value::Text(Text(val.clone().into())), - dml::PrismaValue::Xml(val) => Value::Text(Text(val.clone().into())), - dml::PrismaValue::Object(_) => unreachable!(), - dml::PrismaValue::Null => unreachable!(), - dml::PrismaValue::DateTime(val) => Value::from(Constant::new_no_validate(val)), - dml::PrismaValue::Float(val) => Value::from(Constant::new_no_validate(val)), - dml::PrismaValue::BigInt(val) => Value::from(Constant::new_no_validate(val)), - dml::PrismaValue::Bytes(val) => Value::from(val.clone()), - } - } -} - impl<'a> fmt::Display for DefaultValue<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) diff --git a/introspection-engine/datamodel-renderer/src/datamodel/field.rs b/introspection-engine/datamodel-renderer/src/datamodel/field.rs index 94d982a10892..34d2b95113c6 100644 --- a/introspection-engine/datamodel-renderer/src/datamodel/field.rs +++ b/introspection-engine/datamodel-renderer/src/datamodel/field.rs @@ -4,8 +4,7 @@ use crate::{ }, value::{Constant, Documentation, Function}, }; -use psl::dml; -use std::{borrow::Cow, collections::HashMap, fmt}; +use std::{borrow::Cow, fmt}; /// A field in a model block. #[derive(Debug)] @@ -224,171 +223,6 @@ impl<'a> Field<'a> { pub fn commented_out(&mut self) { self.commented_out = true; } - - /// Generate a model field rendering from the deprecated DML structure. - /// - /// Remove when destroying the DML. This API cannot really be - /// public, because we need info from the model and it doesn't - /// make much sense to call this from outside of the module. - pub(super) fn from_dml( - datasource: &'a psl::Datasource, - dml_field: &dml::Field, - uniques: &mut HashMap<&str, UniqueFieldAttribute<'static>>, - id: Option>, - ) -> Field<'a> { - match dml_field { - dml::Field::ScalarField(ref sf) => { - let (r#type, native_type): (String, _) = match sf.field_type { - dml::FieldType::Enum(ref ct) => (ct.clone(), None), - dml::FieldType::Relation(ref info) => (info.referenced_model.clone(), None), - dml::FieldType::Unsupported(ref s) => (s.clone(), None), - dml::FieldType::Scalar(ref st, ref nt) => { - (st.as_ref().to_owned(), nt.as_ref().map(|nt| (nt.name(), nt.args()))) - } - dml::FieldType::CompositeType(ref ct) => (ct.clone(), None), - }; - - let mut field = Self::new(sf.name.clone(), r#type); - - match sf.arity { - dml::FieldArity::Optional => { - field.optional(); - } - dml::FieldArity::List => { - field.array(); - } - _ => (), - }; - - if sf.field_type.is_unsupported() { - field.unsupported(); - } - - if let Some(ref docs) = sf.documentation { - field.documentation(docs.clone()); - } - - if let Some(dv) = sf.default_value() { - field.default(DefaultValue::from_dml(dv)); - } - - if let Some((name, args)) = native_type { - field.native_type(&datasource.name, name, args); - } - - if sf.is_updated_at { - field.updated_at(); - } - - if let Some(unique) = uniques.remove(sf.name.as_str()) { - field.unique(unique); - } - - if sf.is_ignored { - field.ignore(); - } - - if sf.is_commented_out { - field.commented_out(); - } - - if let Some(ref map) = sf.database_name { - field.map(map.clone()); - } - - if let Some(id) = id { - field.id(id); - } - - field - } - dml::Field::RelationField(rf) => { - let field_name = rf.name.clone(); - let referenced_model = rf.relation_info.referenced_model.clone(); - - let mut field = Self::new(field_name, referenced_model); - - match rf.arity { - dml::FieldArity::Optional => field.optional(), - dml::FieldArity::List => field.array(), - dml::FieldArity::Required => (), - } - - if let Some(ref docs) = rf.documentation { - field.documentation(docs.clone()); - } - - if rf.is_ignored { - field.ignore(); - } - - let dml_info = &rf.relation_info; - let relation_name = dml_info.name.as_str(); - - // :( - if !relation_name.is_empty() || (!dml_info.fields.is_empty() || !dml_info.references.is_empty()) { - let mut relation = Relation::new(); - - if !relation_name.is_empty() { - relation.name(relation_name.to_owned()); - } - - relation.fields(dml_info.fields.iter().map(Clone::clone).map(Cow::Owned)); - relation.references(dml_info.references.iter().map(Clone::clone).map(Cow::Owned)); - - if let Some(ref action) = dml_info.on_delete { - relation.on_delete(action.as_ref().to_owned()); - } - - if let Some(ref action) = dml_info.on_update { - relation.on_update(action.as_ref().to_owned()); - } - - if let Some(ref map) = &dml_info.fk_name { - relation.map(map.clone()); - } - - field.relation(relation); - } - - field - } - dml::Field::CompositeField(cf) => { - let name = cf.name.clone(); - let ct = cf.composite_type.clone(); - - let mut field = Self::new(name, ct); - - match cf.arity { - dml::FieldArity::Required => (), - dml::FieldArity::Optional => field.optional(), - dml::FieldArity::List => field.array(), - } - - if let Some(ref docs) = cf.documentation { - field.documentation(docs.clone()); - } - - if let Some(ref map) = cf.database_name { - field.map(map.clone()); - } - - if cf.is_commented_out { - field.commented_out(); - } - - if cf.is_ignored { - field.ignore(); - } - - if let Some(ref dv) = cf.default_value { - field.default(DefaultValue::from_dml(dv)); - } - - field - } - } - } } impl<'a> fmt::Display for Field<'a> { diff --git a/introspection-engine/datamodel-renderer/src/datamodel/model.rs b/introspection-engine/datamodel-renderer/src/datamodel/model.rs index 03f4509e0781..b81e9c9a92f2 100644 --- a/introspection-engine/datamodel-renderer/src/datamodel/model.rs +++ b/introspection-engine/datamodel-renderer/src/datamodel/model.rs @@ -1,15 +1,10 @@ mod relation; -use psl::dml; pub use relation::Relation; +use super::{attributes::BlockAttribute, field::Field, IdDefinition, IndexDefinition}; use crate::value::{Constant, Documentation, Function}; -use std::{borrow::Cow, collections::HashMap, fmt}; - -use super::{ - attributes::BlockAttribute, field::Field, IdDefinition, IdFieldDefinition, IndexDefinition, IndexFieldInput, - IndexOps, UniqueFieldAttribute, -}; +use std::{borrow::Cow, fmt}; #[derive(Debug, Clone, Copy)] enum Commented { @@ -145,11 +140,12 @@ impl<'a> Model<'a> { self.commented_out = Commented::On } - /// Push a new field to the model. + /// Push a new field to the end of the model. /// /// ```ignore /// model Foo { - /// id Int @id + /// id Int @id + /// foo String /// ^^^^^^^^^^ this /// } /// ``` @@ -157,6 +153,20 @@ impl<'a> Model<'a> { self.fields.push(field); } + /// Push a new field to the beginning of the model. + /// Extremely inefficient, prefer `push_field` if you can. + /// + /// ```ignore + /// model Foo { + /// id Int @id + /// ^^^^^^^^^^^^^^ this + /// foo String + /// } + /// ``` + pub fn insert_field_front(&mut self, field: Field<'a>) { + self.fields.insert(0, field); + } + /// Push a new index to the model. /// /// ```ignore @@ -168,188 +178,6 @@ impl<'a> Model<'a> { pub fn push_index(&mut self, index: IndexDefinition<'a>) { self.indexes.push(index); } - - /// Generate a model rendering from the deprecated DML structure. - /// - /// Remove when destroying the DML. - pub fn from_dml(datasource: &'a psl::Datasource, dml_model: &dml::Model) -> Self { - let mut model = Model::new(dml_model.name.clone()); - - if let Some(docs) = &dml_model.documentation { - model.documentation(docs.clone()); - } - - if let Some(map) = &dml_model.database_name { - model.map(map.clone()); - } - - if let Some(ref schema) = dml_model.schema { - model.schema(schema.clone()); - } - - if dml_model.is_commented_out { - model.comment_out(); - } - - if dml_model.is_ignored { - model.ignore(); - } - - match dml_model.primary_key { - Some(ref pk) if !dml_model.has_single_id_field() => { - let fields = pk.fields.iter().map(|field| IndexFieldInput { - name: Cow::Owned(field.name.clone()), - sort_order: field.sort_order.as_ref().map(|so| so.as_ref().to_owned().into()), - length: field.length, - ops: None, - }); - - let mut definition: IdDefinition<'static> = IdDefinition::new(fields); - - if let Some(ref name) = pk.name { - definition.name(name.clone()); - } - - if let Some(ref map) = &pk.db_name { - definition.map(map.clone()); - } - - if let Some(clustered) = pk.clustered { - definition.clustered(clustered); - } - - model.id(definition); - } - _ => (), - } - - // weep - let mut uniques: HashMap<&str, UniqueFieldAttribute<'static>> = dml_model - .indices - .iter() - .rev() // replicate existing behaviour on duplicate unique constraints - .filter(|ix| ix.is_unique()) - .filter(|ix| ix.defined_on_field) - .map(|ix| { - let definition = ix.fields.first().unwrap(); - let mut opts = UniqueFieldAttribute::default(); - - if let Some(clustered) = ix.clustered { - opts.clustered(clustered); - } - - if let Some(ref sort_order) = definition.sort_order { - opts.sort_order(sort_order.as_ref().to_owned()); - } - - if let Some(length) = definition.length { - opts.length(length); - } - - if let Some(ref map) = ix.db_name { - opts.map(map.clone()); - } - - (definition.from_field(), opts) - }) - .collect(); - - let primary_key = dml_model.primary_key.as_ref().filter(|pk| pk.defined_on_field); - - for dml_field in dml_model.fields.iter() { - // sob :( - let id = primary_key.and_then(|pk| { - let field = pk.fields.first().unwrap(); - - if field.name == dml_field.name() { - let mut opts = IdFieldDefinition::default(); - - if let Some(clustered) = pk.clustered { - opts.clustered(clustered); - } - - if let Some(ref sort_order) = field.sort_order { - opts.sort_order(sort_order.as_ref().to_owned()); - } - - if let Some(length) = field.length { - opts.length(length); - } - - if let Some(ref map) = pk.db_name { - opts.map(map.clone()); - } - - Some(opts) - } else { - None - } - }); - - model.push_field(Field::from_dml(datasource, dml_field, &mut uniques, id)); - } - - for dml_index in dml_model.indices.iter() { - if dml_index.defined_on_field && dml_index.is_unique() { - continue; - } - - // cry - let fields = dml_index.fields.iter().map(|f| { - let mut name = String::new(); - let mut name_path = f.path.iter().peekable(); - - while let Some((ident, _)) = name_path.next() { - name.push_str(ident); - - if name_path.peek().is_some() { - name.push('.'); - } - } - - let ops = f.operator_class.as_ref().map(|c| { - if c.is_raw() { - IndexOps::raw(c.as_ref().to_owned()) - } else { - IndexOps::managed(c.as_ref().to_owned()) - } - }); - - IndexFieldInput { - name: Cow::Owned(name), - sort_order: f.sort_order.map(|s| s.as_ref().to_string().into()), - length: f.length, - ops, - } - }); - - let mut definition = match dml_index.tpe { - dml::IndexType::Unique => IndexDefinition::unique(fields), - dml::IndexType::Normal => IndexDefinition::index(fields), - dml::IndexType::Fulltext => IndexDefinition::fulltext(fields), - }; - - if let Some(ref name) = dml_index.name { - definition.name(name.clone()); - } - - if let Some(ref map) = dml_index.db_name { - definition.map(map.clone()); - } - - if let Some(clustered) = dml_index.clustered { - definition.clustered(clustered); - } - - if let Some(ref algo) = dml_index.algorithm { - definition.index_type(algo.as_ref().to_string()); - } - - model.push_index(definition); - } - - model - } } impl<'a> fmt::Display for Model<'a> { diff --git a/libs/mongodb-schema-describer/src/schema.rs b/libs/mongodb-schema-describer/src/schema.rs index c1cdf142d58f..961f2b9329d2 100644 --- a/libs/mongodb-schema-describer/src/schema.rs +++ b/libs/mongodb-schema-describer/src/schema.rs @@ -201,6 +201,13 @@ pub enum IndexFieldProperty { Descending, } +impl IndexFieldProperty { + /// If the property is descending. + pub fn is_descending(self) -> bool { + matches!(self, Self::Descending) + } +} + impl fmt::Display for IndexFieldProperty { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self {