Merge pull request #1382 from boraarslan/bool-fields

Add boolean fields

common/src/serialize.rs

@@ -247,6 +247,11 @@ pub mod test {
         fixed_size_test::<u64>();
     }
 
+    #[test]
+    fn test_serialize_bool() {
+        fixed_size_test::<bool>();
+    }
+
     #[test]
     fn test_serialize_string() {
         assert_eq!(serialize_test(String::from("")), 1);

src/fastfield/mod.rs

@@ -160,6 +160,37 @@ impl FastValue for f64 {
     }
 }
 
+impl FastValue for bool {
+    fn from_u64(val: u64) -> Self {
+        match val {
+            0 => false,
+            _ => true,
+        }
+    }
+
+    fn to_u64(&self) -> u64 {
+        match self {
+            false => 0,
+            true => 1,
+        }
+    }
+
+    fn fast_field_cardinality(field_type: &FieldType) -> Option<Cardinality> {
+        match *field_type {
+            FieldType::Bool(ref integer_options) => integer_options.get_fastfield_cardinality(),
+            _ => None,
+        }
+    }
+
+    fn as_u64(&self) -> u64 {
+        *self as u64
+    }
+
+    fn to_type() -> Type {
+        Type::Bool
+    }
+}
+
 impl FastValue for DateTime {
     fn from_u64(timestamp_u64: u64) -> Self {
         let unix_timestamp = i64::from_u64(timestamp_u64);
@@ -191,8 +222,9 @@ fn value_to_u64(value: &Value) -> u64 {
         Value::U64(val) => val.to_u64(),
         Value::I64(val) => val.to_u64(),
         Value::F64(val) => val.to_u64(),
+        Value::Bool(val) => val.to_u64(),
         Value::Date(val) => val.to_u64(),
-        _ => panic!("Expected a u64/i64/f64/date field, got {:?} ", value),
+        _ => panic!("Expected a u64/i64/f64/bool/date field, got {:?} ", value),
     }
 }
 
@@ -788,6 +820,118 @@ mod tests {
         }
         Ok(())
     }
+
+    #[test]
+    pub fn test_fastfield_bool() {
+        let test_fastfield = DynamicFastFieldReader::<bool>::from(vec![true, false, true, false]);
+        assert_eq!(test_fastfield.get(0), true);
+        assert_eq!(test_fastfield.get(1), false);
+        assert_eq!(test_fastfield.get(2), true);
+        assert_eq!(test_fastfield.get(3), false);
+    }
+
+    #[test]
+    pub fn test_fastfield_bool_small() -> crate::Result<()> {
+        let path = Path::new("test_bool");
+        let directory: RamDirectory = RamDirectory::create();
+
+        let mut schema_builder = Schema::builder();
+        schema_builder.add_bool_field("field_bool", FAST);
+        let schema = schema_builder.build();
+        let field = schema.get_field("field_bool").unwrap();
+
+        {
+            let write: WritePtr = directory.open_write(path).unwrap();
+            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
+            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
+            fast_field_writers.add_document(&doc!(field=>true));
+            fast_field_writers.add_document(&doc!(field=>false));
+            fast_field_writers.add_document(&doc!(field=>true));
+            fast_field_writers.add_document(&doc!(field=>false));
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new(), None)
+                .unwrap();
+            serializer.close().unwrap();
+        }
+        let file = directory.open_read(&path).unwrap();
+        assert_eq!(file.len(), 36);
+        let composite_file = CompositeFile::open(&file)?;
+        let file = composite_file.open_read(field).unwrap();
+        let fast_field_reader = DynamicFastFieldReader::<bool>::open(file)?;
+        assert_eq!(fast_field_reader.get(0), true);
+        assert_eq!(fast_field_reader.get(1), false);
+        assert_eq!(fast_field_reader.get(2), true);
+        assert_eq!(fast_field_reader.get(3), false);
+
+        Ok(())
+    }
+
+    #[test]
+    pub fn test_fastfield_bool_large() -> crate::Result<()> {
+        let path = Path::new("test_bool");
+        let directory: RamDirectory = RamDirectory::create();
+
+        let mut schema_builder = Schema::builder();
+        schema_builder.add_bool_field("field_bool", FAST);
+        let schema = schema_builder.build();
+        let field = schema.get_field("field_bool").unwrap();
+
+        {
+            let write: WritePtr = directory.open_write(path).unwrap();
+            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
+            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
+            for _ in 0..50 {
+                fast_field_writers.add_document(&doc!(field=>true));
+                fast_field_writers.add_document(&doc!(field=>false));
+            }
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new(), None)
+                .unwrap();
+            serializer.close().unwrap();
+        }
+        let file = directory.open_read(&path).unwrap();
+        assert_eq!(file.len(), 48);
+        let composite_file = CompositeFile::open(&file)?;
+        let file = composite_file.open_read(field).unwrap();
+        let fast_field_reader = DynamicFastFieldReader::<bool>::open(file)?;
+        for i in 0..25 {
+            assert_eq!(fast_field_reader.get(i * 2), true);
+            assert_eq!(fast_field_reader.get(i * 2 + 1), false);
+        }
+
+        Ok(())
+    }
+
+    #[test]
+    pub fn test_fastfield_bool_default_value() -> crate::Result<()> {
+        let path = Path::new("test_bool");
+        let directory: RamDirectory = RamDirectory::create();
+
+        let mut schema_builder = Schema::builder();
+        schema_builder.add_bool_field("field_bool", FAST);
+        let schema = schema_builder.build();
+        let field = schema.get_field("field_bool").unwrap();
+
+        {
+            let write: WritePtr = directory.open_write(path).unwrap();
+            let mut serializer = CompositeFastFieldSerializer::from_write(write).unwrap();
+            let mut fast_field_writers = FastFieldsWriter::from_schema(&schema);
+            let doc = Document::default();
+            fast_field_writers.add_document(&doc);
+            fast_field_writers
+                .serialize(&mut serializer, &HashMap::new(), None)
+                .unwrap();
+            serializer.close().unwrap();
+        }
+        let file = directory.open_read(&path).unwrap();
+        assert_eq!(file.len(), 35);
+        let composite_file = CompositeFile::open(&file)?;
+        let file = composite_file.open_read(field).unwrap();
+        let fast_field_reader = DynamicFastFieldReader::<bool>::open(file)?;
+        assert_eq!(fast_field_reader.get(0), false);
+
+        Ok(())
+    }
 }
 
 #[cfg(all(test, feature = "unstable"))]

src/fastfield/multivalued/mod.rs

@@ -226,6 +226,38 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_multivalued_bool() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let bool_field = schema_builder.add_bool_field(
+            "multifield",
+            NumericOptions::default().set_fast(Cardinality::MultiValues),
+        );
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_for_tests()?;
+        index_writer.add_document(doc!(bool_field=> true, bool_field => false))?;
+        index_writer.add_document(doc!())?;
+        index_writer.add_document(doc!(bool_field=> false))?;
+        index_writer
+            .add_document(doc!(bool_field=> true, bool_field => true, bool_field => false))?;
+        index_writer.commit()?;
+
+        let searcher = index.reader()?.searcher();
+        let segment_reader = searcher.segment_reader(0);
+        let mut vals = Vec::new();
+        let multi_value_reader = segment_reader.fast_fields().bools(bool_field).unwrap();
+        multi_value_reader.get_vals(2, &mut vals);
+        assert_eq!(&vals, &[false]);
+        multi_value_reader.get_vals(0, &mut vals);
+        assert_eq!(&vals, &[true, false]);
+        multi_value_reader.get_vals(1, &mut vals);
+        assert!(vals.is_empty());
+        multi_value_reader.get_vals(3, &mut vals);
+        assert_eq!(&vals, &[true, true, false]);
+        Ok(())
+    }
+
     fn test_multivalued_no_panic(ops: &[IndexingOp]) -> crate::Result<()> {
         let mut schema_builder = Schema::builder();
         let field = schema_builder.add_u64_field(

src/fastfield/readers.rs

@@ -21,6 +21,7 @@ pub(crate) enum FastType {
     I64,
     U64,
     F64,
+    Bool,
     Date,
 }
 
@@ -35,6 +36,9 @@ pub(crate) fn type_and_cardinality(field_type: &FieldType) -> Option<(FastType,
         FieldType::F64(options) => options
             .get_fastfield_cardinality()
             .map(|cardinality| (FastType::F64, cardinality)),
+        FieldType::Bool(options) => options
+            .get_fastfield_cardinality()
+            .map(|cardinality| (FastType::Bool, cardinality)),
         FieldType::Date(options) => options
             .get_fastfield_cardinality()
             .map(|cardinality| (FastType::Date, cardinality)),
@@ -166,6 +170,14 @@ impl FastFieldReaders {
         self.typed_fast_field_reader(field)
     }
 
+    /// Returns the `bool` fast field reader reader associated to `field`.
+    ///
+    /// If `field` is not a bool fast field, this method returns an Error.
+    pub fn bool(&self, field: Field) -> crate::Result<DynamicFastFieldReader<bool>> {
+        self.check_type(field, FastType::Bool, Cardinality::SingleValue)?;
+        self.typed_fast_field_reader(field)
+    }
+
     /// Returns a `u64s` multi-valued fast field reader reader associated to `field`.
     ///
     /// If `field` is not a u64 multi-valued fast field, this method returns an Error.
@@ -198,6 +210,14 @@ impl FastFieldReaders {
         self.typed_fast_field_multi_reader(field)
     }
 
+    /// Returns a `bools` multi-valued fast field reader reader associated to `field`.
+    ///
+    /// If `field` is not a bool multi-valued fast field, this method returns an Error.
+    pub fn bools(&self, field: Field) -> crate::Result<MultiValuedFastFieldReader<bool>> {
+        self.check_type(field, FastType::Bool, Cardinality::MultiValues)?;
+        self.typed_fast_field_multi_reader(field)
+    }
+
     /// Returns a `time::OffsetDateTime` multi-valued fast field reader reader associated to
     /// `field`.
     ///

src/fastfield/writer.rs

@@ -43,6 +43,7 @@ impl FastFieldsWriter {
                 FieldType::I64(ref int_options)
                 | FieldType::U64(ref int_options)
                 | FieldType::F64(ref int_options)
+                | FieldType::Bool(ref int_options)
                 | FieldType::Date(ref int_options) => {
                     match int_options.get_fastfield_cardinality() {
                         Some(Cardinality::SingleValue) => {
@@ -75,7 +76,7 @@ impl FastFieldsWriter {
                         bytes_value_writers.push(fast_field_writer);
                     }
                 }
-                _ => {}
+                FieldType::Str(_) | FieldType::JsonObject(_) => {}
             }
         }
         FastFieldsWriter {

src/indexer/index_writer.rs

@@ -1385,6 +1385,7 @@ mod tests {
         let mut schema_builder = schema::Schema::builder();
         let id_field = schema_builder.add_u64_field("id", FAST | INDEXED | STORED);
         let bytes_field = schema_builder.add_bytes_field("bytes", FAST | INDEXED | STORED);
+        let bool_field = schema_builder.add_bool_field("bool", FAST | INDEXED | STORED);
         let text_field = schema_builder.add_text_field(
             "text_field",
             TextOptions::default()
@@ -1403,6 +1404,12 @@ mod tests {
                 .set_fast(Cardinality::MultiValues)
                 .set_stored(),
         );
+        let multi_bools = schema_builder.add_bool_field(
+            "multi_bools",
+            NumericOptions::default()
+                .set_fast(Cardinality::MultiValues)
+                .set_stored(),
+        );
         let facet_field = schema_builder.add_facet_field("facet", FacetOptions::default());
         let schema = schema_builder.build();
         let settings = if sort_index {
@@ -1435,6 +1442,9 @@ mod tests {
                             bytes_field => id.to_le_bytes().as_slice(),
                             multi_numbers=> id,
                             multi_numbers => id,
+                            bool_field => (id % 2u64) != 0,
+                            multi_bools => (id % 2u64) != 0,
+                            multi_bools => (id % 2u64) == 0,
                             text_field => id.to_string(),
                             facet_field => facet,
                             large_text_field=> LOREM
@@ -1522,11 +1532,18 @@ mod tests {
         // multivalue fast field tests
         for segment_reader in searcher.segment_readers().iter() {
             let ff_reader = segment_reader.fast_fields().u64s(multi_numbers).unwrap();
+            let bool_ff_reader = segment_reader.fast_fields().bools(multi_bools).unwrap();
             for doc in segment_reader.doc_ids_alive() {
                 let mut vals = vec![];
                 ff_reader.get_vals(doc, &mut vals);
                 assert_eq!(vals.len(), 2);
                 assert_eq!(vals[0], vals[1]);
+
+                let mut bool_vals = vec![];
+                bool_ff_reader.get_vals(doc, &mut bool_vals);
+                assert_eq!(bool_vals.len(), 2);
+                assert_ne!(bool_vals[0], bool_vals[1]);
+
                 assert!(expected_ids_and_num_occurrences.contains_key(&vals[0]));
             }
         }
@@ -1557,6 +1574,18 @@ mod tests {
                     .as_u64()
                     .unwrap();
                 assert_eq!(id, id2);
+                let bool = store_reader
+                    .get(doc_id)
+                    .unwrap()
+                    .get_first(bool_field)
+                    .unwrap()
+                    .as_bool()
+                    .unwrap();
+                let doc = store_reader.get(doc_id).unwrap();
+                let mut bool2 = doc.get_all(multi_bools);
+                assert_eq!(bool, bool2.next().unwrap().as_bool().unwrap());
+                assert_ne!(bool, bool2.next().unwrap().as_bool().unwrap());
+                assert_eq!(None, bool2.next())
             }
         }
         // test search

src/indexer/json_term_writer.rs

@@ -123,8 +123,7 @@ fn index_json_value<'a>(
     match json_value {
         serde_json::Value::Null => {}
         serde_json::Value::Bool(val_bool) => {
-            let bool_u64 = if *val_bool { 1u64 } else { 0u64 };
-            json_term_writer.set_fast_value(bool_u64);
+            json_term_writer.set_fast_value(*val_bool);
             postings_writer.subscribe(doc, 0u32, json_term_writer.term(), ctx);
         }
         serde_json::Value::Number(number) => {
@@ -220,6 +219,9 @@ pub(crate) fn convert_to_fast_value_and_get_term(
     if let Ok(f64_val) = str::parse::<f64>(phrase) {
         return Some(set_fastvalue_and_get_term(json_term_writer, f64_val));
     }
+    if let Ok(bool_val) = str::parse::<bool>(phrase) {
+        return Some(set_fastvalue_and_get_term(json_term_writer, bool_val));
+    }
     None
 }
 // helper function to generate a Term from a json fastvalue
@@ -434,6 +436,20 @@ mod tests {
         )
     }
 
+    #[test]
+    fn test_bool_term() {
+        let field = Field::from_field_id(1);
+        let mut term = Term::new();
+        term.set_field(Type::Json, field);
+        let mut json_writer = JsonTermWriter::wrap(&mut term);
+        json_writer.push_path_segment("color");
+        json_writer.set_fast_value(true);
+        assert_eq!(
+            json_writer.term().as_slice(),
+            b"\x00\x00\x00\x01jcolor\x00o\x00\x00\x00\x00\x00\x00\x00\x01"
+        )
+    }
+
     #[test]
     fn test_push_after_set_path_segment() {
         let field = Field::from_field_id(1);