track ff dictionary memory consumption while indexing

columnar/src/columnar/writer/mod.rs

@@ -79,14 +79,18 @@ fn mutate_or_create_column<V, TMutator>(
 
 impl ColumnarWriter {
     pub fn mem_usage(&self) -> usize {
-        // TODO add dictionary builders.
         self.arena.mem_usage()
             + self.numerical_field_hash_map.mem_usage()
             + self.bool_field_hash_map.mem_usage()
             + self.bytes_field_hash_map.mem_usage()
             + self.str_field_hash_map.mem_usage()
             + self.ip_addr_field_hash_map.mem_usage()
             + self.datetime_field_hash_map.mem_usage()
+            + self
+                .dictionaries
+                .iter()
+                .map(|dict| dict.mem_usage())
+                .sum::<usize>()
     }
 
     /// Returns the list of doc ids from 0..num_docs sorted by the `sort_field`

columnar/src/dictionary.rs

@@ -32,6 +32,7 @@ pub struct OrderedId(pub u32);
 #[derive(Default)]
 pub(crate) struct DictionaryBuilder {
     dict: FnvHashMap<Vec<u8>, UnorderedId>,
+    memory_consumption: usize,
 }
 
 impl DictionaryBuilder {
@@ -43,6 +44,8 @@ impl DictionaryBuilder {
         }
         let new_id = UnorderedId(self.dict.len() as u32);
         self.dict.insert(term.to_vec(), new_id);
+        self.memory_consumption += term.len();
+        self.memory_consumption += 40; // Term Metadata + HashMap overhead
         new_id
     }
 
@@ -63,6 +66,10 @@ impl DictionaryBuilder {
         sstable_builder.finish()?;
         Ok(TermIdMapping { unordered_to_ord })
     }
+
+    pub(crate) fn mem_usage(&self) -> usize {
+        self.memory_consumption
+    }
 }
 
 #[cfg(test)]