Skip to content

Commit

Permalink
add coerce option for text and numbers types
Browse files Browse the repository at this point in the history 
allow to coerce the field type when indexing if the type does not match
  • Loading branch information
@PSeitz
PSeitz committed Feb 24, 2023
1 parent 5f23bb7 commit 502e935
Show file tree
Hide file tree
Showing 4 changed files with 156 additions and 15 deletions.
94 changes: 84 additions & 10 deletions src/schema/field_type.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,11 +329,50 @@ impl FieldType {
Ok(DateTime::from_utc(dt_with_fixed_tz).into())
}
FieldType::Str(_) => Ok(Value::Str(field_text)),
FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) => {
Err(ValueParsingError::TypeError {
expected: "an integer",
json: JsonValue::String(field_text),
})
FieldType::U64(opt) => {
if opt.should_coerce() {
Ok(Value::U64(field_text.parse().map_err(|_| {
ValueParsingError::TypeError {
expected: "a u64 or a u64 as string",
json: JsonValue::String(field_text),
}
})?))
} else {
Err(ValueParsingError::TypeError {
expected: "a u64",
json: JsonValue::String(field_text),
})
}
}
FieldType::I64(opt) => {
if opt.should_coerce() {
Ok(Value::U64(field_text.parse().map_err(|_| {
ValueParsingError::TypeError {
expected: "a i64 or a i64 as string",
json: JsonValue::String(field_text),
}
})?))
} else {
Err(ValueParsingError::TypeError {
expected: "a i64",
json: JsonValue::String(field_text),
})
}
}
FieldType::F64(opt) => {
if opt.should_coerce() {
Ok(Value::U64(field_text.parse().map_err(|_| {
ValueParsingError::TypeError {
expected: "a f64 or a f64 as string",
json: JsonValue::String(field_text),
}
})?))
} else {
Err(ValueParsingError::TypeError {
expected: "a f64",
json: JsonValue::String(field_text),
})
}
}
FieldType::Bool(_) => Err(ValueParsingError::TypeError {
expected: "a boolean",
Expand Down Expand Up @@ -395,12 +434,20 @@ impl FieldType {
expected: "a boolean",
json: JsonValue::Number(field_val_num),
}),
FieldType::Str(_) | FieldType::Facet(_) | FieldType::Bytes(_) => {
Err(ValueParsingError::TypeError {
expected: "a string",
json: JsonValue::Number(field_val_num),
})
FieldType::Str(opt) => {
if opt.should_coerce() {
Ok(Value::Str(field_val_num.to_string()))
} else {
Err(ValueParsingError::TypeError {
expected: "a string",
json: JsonValue::Number(field_val_num),
})
}
}
FieldType::Facet(_) | FieldType::Bytes(_) => Err(ValueParsingError::TypeError {
expected: "a string",
json: JsonValue::Number(field_val_num),
}),
FieldType::JsonObject(_) => Err(ValueParsingError::TypeError {
expected: "a json object",
json: JsonValue::Number(field_val_num),
Expand Down Expand Up @@ -431,11 +478,38 @@ impl FieldType {
},
JsonValue::Bool(json_bool_val) => match self {
FieldType::Bool(_) => Ok(Value::Bool(json_bool_val)),
FieldType::Str(opt) => {
if opt.should_coerce() {
Ok(Value::Str(json_bool_val.to_string()))
} else {
Err(ValueParsingError::TypeError {
expected: "a string",
json: JsonValue::Bool(json_bool_val),
})
}
}
_ => Err(ValueParsingError::TypeError {
expected: self.value_type().name(),
json: JsonValue::Bool(json_bool_val),
}),
},
// Could also just filter them
JsonValue::Null => match self {
FieldType::Str(opt) => {
if opt.should_coerce() {
Ok(Value::Str("null".to_string()))
} else {
Err(ValueParsingError::TypeError {
expected: "a string",
json: JsonValue::Null,
})
}
}
_ => Err(ValueParsingError::TypeError {
expected: self.value_type().name(),
json: JsonValue::Null,
}),
},
_ => Err(ValueParsingError::TypeError {
expected: self.value_type().name(),
json: json.clone(),
Expand Down
1 change: 1 addition & 0 deletions src/schema/json_object_options.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ pub struct JsonObjectOptions {
/// `{"root": {"child": {"with": {"dot": "hello"}}}}`
/// and it can be search using the following query:
/// `root.child.with.dot:hello`
#[serde(default)]
expand_dots_enabled: bool,
}

Expand Down
54 changes: 50 additions & 4 deletions src/schema/numeric_options.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ pub struct NumericOptions {
fieldnorms: bool, // This attribute only has an effect if indexed is true.
fast: bool,
stored: bool,
coerce: bool,
}

/// For backward compatibility we add an intermediary to interpret the
Expand All @@ -32,6 +33,8 @@ struct NumericOptionsDeser {
#[serde(default)]
fast: bool,
stored: bool,
#[serde(default)]
coerce: bool,
}

impl From<NumericOptionsDeser> for NumericOptions {
Expand All @@ -41,6 +44,7 @@ impl From<NumericOptionsDeser> for NumericOptions {
fieldnorms: deser.fieldnorms.unwrap_or(deser.indexed),
fast: deser.fast,
stored: deser.stored,
coerce: deser.coerce,
}
}
}
Expand All @@ -66,6 +70,18 @@ impl NumericOptions {
self.fast
}

/// Returns true if values should be coerced to numbers.
pub fn should_coerce(&self) -> bool {
self.coerce
}

/// Try to coerce values if they are not a number. Defaults to false.
#[must_use]
pub fn set_coerce(mut self) -> Self {
self.coerce = true;
self
}

/// Set the field as stored.
///
/// Only the fields that are set as *stored* are
Expand Down Expand Up @@ -124,6 +140,7 @@ impl From<FastFlag> for NumericOptions {
fieldnorms: false,
stored: false,
fast: true,
coerce: false,
}
}
}
Expand All @@ -135,6 +152,7 @@ impl From<StoredFlag> for NumericOptions {
fieldnorms: false,
stored: true,
fast: false,
coerce: false,
}
}
}
Expand All @@ -146,6 +164,7 @@ impl From<IndexedFlag> for NumericOptions {
fieldnorms: true,
stored: false,
fast: false,
coerce: false,
}
}
}
Expand All @@ -160,6 +179,7 @@ impl<T: Into<NumericOptions>> BitOr<T> for NumericOptions {
fieldnorms: self.fieldnorms | other.fieldnorms,
stored: self.stored | other.stored,
fast: self.fast | other.fast,
coerce: self.coerce | other.coerce,
}
}
}
Expand Down Expand Up @@ -192,7 +212,8 @@ mod tests {
indexed: true,
fieldnorms: true,
fast: false,
stored: false
stored: false,
coerce: false,
}
);
}
Expand All @@ -210,7 +231,8 @@ mod tests {
indexed: false,
fieldnorms: false,
fast: false,
stored: false
stored: false,
coerce: false,
}
);
}
Expand All @@ -229,7 +251,8 @@ mod tests {
indexed: true,
fieldnorms: false,
fast: false,
stored: false
stored: false,
coerce: false,
}
);
}
Expand All @@ -249,7 +272,30 @@ mod tests {
indexed: false,
fieldnorms: true,
fast: false,
stored: false
stored: false,
coerce: false,
}
);
}

#[test]
fn test_int_options_deser_if_coerce_true() {
// this one is kind of useless, at least at the moment
let json = r#"{
"indexed": false,
"fieldnorms": true,
"stored": false,
"coerce": true
}"#;
let int_options: NumericOptions = serde_json::from_str(json).unwrap();
assert_eq!(
&int_options,
&NumericOptions {
indexed: false,
fieldnorms: true,
fast: false,
stored: false,
coerce: true,
}
);
}
Expand Down
22 changes: 21 additions & 1 deletion src/schema/text_options.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ pub struct TextOptions {
stored: bool,
#[serde(default)]
fast: bool,
#[serde(default)]
/// coerce values if they are not of type string
coerce: bool,
}

impl TextOptions {
Expand All @@ -35,6 +38,11 @@ impl TextOptions {
self.fast
}

/// Returns true if values should be coerced to strings (numbers, null).
pub fn should_coerce(&self) -> bool {
self.coerce
}

/// Set the field as a fast field.
///
/// Fast fields are designed for random access.
Expand All @@ -56,7 +64,14 @@ impl TextOptions {
self
}

/// Sets the field as stored
/// Coerce values if they are not of type string. Defaults to false.
#[must_use]
pub fn set_coerce(mut self) -> TextOptions {
self.coerce = true;
self
}

/// Sets the field as stored.
#[must_use]
pub fn set_stored(mut self) -> TextOptions {
self.stored = true;
Expand Down Expand Up @@ -180,6 +195,7 @@ pub const STRING: TextOptions = TextOptions {
}),
stored: false,
fast: false,
coerce: false,
};

/// The field will be tokenized and indexed.
Expand All @@ -190,6 +206,7 @@ pub const TEXT: TextOptions = TextOptions {
record: IndexRecordOption::WithFreqsAndPositions,
}),
stored: false,
coerce: false,
fast: false,
};

Expand All @@ -202,6 +219,7 @@ impl<T: Into<TextOptions>> BitOr<T> for TextOptions {
indexing: self.indexing.or(other.indexing),
stored: self.stored | other.stored,
fast: self.fast | other.fast,
coerce: self.coerce | other.coerce,
}
}
}
Expand All @@ -218,6 +236,7 @@ impl From<StoredFlag> for TextOptions {
indexing: None,
stored: true,
fast: false,
coerce: false,
}
}
}
Expand All @@ -228,6 +247,7 @@ impl From<FastFlag> for TextOptions {
indexing: None,
stored: false,
fast: true,
coerce: false,
}
}
}
Expand Down

0 comments on commit 502e935

Please sign in to comment.