Merge pull request #342 from pelias/addendum

allow arbitrary data to be stored alongside the rest of the document
pelias · Jan 25, 2019 · 72ec124 · 72ec124
2 parents 52997ab + 45fc319
commit 72ec124
Show file tree

Hide file tree

Showing 4 changed files with 362 additions and 4 deletions.
diff --git a/integration/dynamic_templates.js b/integration/dynamic_templates.js
@@ -18,6 +18,12 @@ module.exports.tests.dynamic_templates_phrase = function(test, common){
   test( 'document->phrase', phraseAssertion( 'myType', 'peliasPhrase', common ) );
 };
 
+// all types share the same addendum mapping
+module.exports.tests.dynamic_templates_addendum = function(test, common){
+  test( 'addendum', addendumAssertion( 'country', 'example', JSON.stringify({ example: 100 }), common ) );
+  test( 'addendum', addendumAssertion( 'myType', 'wikipedia', JSON.stringify({ slug: 'Wikipedia' }), common ) );
+};
+
 module.exports.all = function (tape, common) {
 
   function test(name, testFunction) {
@@ -96,3 +102,69 @@ function phraseAssertion( type, analyzer, common ){
     suite.run( t.end );
   };
 }
+
+function addendumAssertion( type, namespace, value, common ){
+  return function(t){
+
+    var suite = new elastictest.Suite( common.clientOpts, { schema: schema } );
+
+    // index a document including the addendum
+    suite.action( function( done ){
+      suite.client.index({
+        index: suite.props.index,
+        type: type,
+        id: '1',
+        body: { addendum: { [namespace]: value } },
+      }, done );
+    });
+
+    // check dynamically created mapping has
+    // inherited from the dynamic_template
+    suite.assert( function( done ){
+      suite.client.indices.getMapping({ index: suite.props.index, type: type }, function( err, res ){
+
+        var properties = res[suite.props.index].mappings[type].properties;
+        t.equal( properties.addendum.dynamic, 'true' );
+
+        var addendumProperties = properties.addendum.properties;
+
+        t.true([
+          'string', // elasticsearch 2.4
+          'keyword' // elasticsearch 5.6
+        ].includes( addendumProperties[namespace].type ));
+
+        t.true([
+          'no', // elasticsearch 2.4
+          false // elasticsearch 5.6
+        ].includes( addendumProperties[namespace].index ));
+
+        // elasticsearch 2.4
+        if( addendumProperties[namespace].fielddata ){
+          t.equal( addendumProperties[namespace].fielddata.format, 'disabled' );
+        }
+
+        // elasticsearch 5.6
+        if( addendumProperties[namespace].doc_values ){
+          t.equal( addendumProperties[namespace].doc_values, false );
+        }
+
+        done();
+      });
+    });
+
+    // retrieve document and check addendum was stored verbatim
+    suite.assert( function( done ){
+      suite.client.get({
+        index: suite.props.index,
+        type: type,
+        id: 1
+      }, function( err, res ){
+        t.false( err );
+        t.equal( res._source.addendum[namespace], value );
+        done();
+      });
+    });
+
+    suite.run( t.end );
+  };
+}
diff --git a/mappings/document.js b/mappings/document.js
@@ -138,7 +138,10 @@ var schema = {
     source_id: literal,
     category: literal,
     population: multiplier,
-    popularity: multiplier
+    popularity: multiplier,
+
+    // addendum (non-indexed supplimentary data)
+    addendum: hash
   },
   dynamic_templates: [{
     nameGram: {
@@ -148,7 +151,7 @@ var schema = {
         type: 'string',
         analyzer: 'peliasIndexOneEdgeGram',
         fielddata : {
-          format: "disabled"
+          format: 'disabled'
         }
       }
     },
@@ -160,7 +163,20 @@ var schema = {
         type: 'string',
         analyzer: 'peliasPhrase',
         fielddata : {
-          format: "disabled"
+          format: 'disabled'
+        }
+      }
+    }
+  },{
+    addendum: {
+      path_match: 'addendum.*',
+      match_mapping_type: 'string',
+      mapping: {
+        type: 'string',
+        index: 'no',
+        doc_values: false,
+        fielddata : {
+          format: 'disabled'
         }
       }
     }

diff --git a/test/document.js b/test/document.js
@@ -22,7 +22,7 @@ module.exports.tests.properties = function(test, common) {
 module.exports.tests.fields = function(test, common) {
   var fields = ['source', 'layer', 'name', 'phrase', 'address_parts',
     'parent', 'center_point', 'shape', 'bounding_box', 'source_id', 'category',
-    'population', 'popularity'];
+    'population', 'popularity', 'addendum'];
   test('fields specified', function(t) {
     t.deepEqual(Object.keys(schema.properties), fields);
     t.end();