Add expand_keys type support for yaml files

README.md

@@ -115,7 +115,7 @@ git2consul expects to be run on the same node as a Consul agent.  git2consul exp
     }]
   },{
     "name" : "github_data",
-    "mode" : "expand_keys",
+    "expand_keys" : true,
     "url" : "[email protected]:ryanbreen/git2consul_data.git",
     "branches" : [ "master" ],
     "hooks": [{
@@ -203,6 +203,30 @@ A few notes on how this behaves:
 
 * Any non-JSON files, including files with the extension ".json" that contain invalid JSON, are stored in your KV as if expand_keys mode was not enabled.
 
+###### YAML
+
+Similarly to JSON, git2consul can treat YAML documents in your repo as fully formed subtrees.
+
+```yaml
+---
+# file: example.yaml or example.yml
+first_level:
+  second_level:
+    third_level:
+      my_key: my_value
+```
+
+git2consul in expand_keys mode will generate the following KV:
+
+```
+/expando_keys/example.yaml/first_level/second_level/third_level/my_key
+or
+/expando_keys/example.yml/first_level/second_level/third_level/my_key
+```
+
+The value in that KV pair will be `my_value`.
+
+
 ###### .properties
 
 Similarly to JSON, git2consul can also treat [Java .properties](http://docs.oracle.com/javase/7/docs/api/java/util/Properties.html#load%28java.io.Reader%29) as a simple k/v format.

lib/consul/index.js

@@ -1,6 +1,7 @@
 var _ = require('underscore');
 var fs = require('fs');
 var path = require('path');
+var yaml = require('js-yaml');
 var utils = require('../utils.js');
 
 var logger = require('../logging.js');
@@ -104,7 +105,7 @@ var file_modified = function(branch, file, cb) {
       fs.readFile(file_path, {encoding: 'utf8'}, function (err, body) {
         /* istanbul ignore if */
         if (err) return cb('Failed to read key ' + file_path + ' due to ' + err);
-        var body = body ? body.trim() : '';
+        body ? body.trim() : '';
         try {
           var obj = JSON.parse(body);
           populate_kvs_from_object(branch, create_key_name(branch, file), obj, cb);
@@ -116,7 +117,6 @@ var file_modified = function(branch, file, cb) {
   };
 
   var handle_properties_kv_file = function(file_path, common_properties_relative_path, cb) {
-
     function extract_and_populate_properties(file_body, common_body, cb) {
       utils.load_properties(file_body, common_body, function (error, obj) {
         if (error) {
@@ -148,11 +148,26 @@ var file_modified = function(branch, file, cb) {
     });
   };
 
+  var handle_yaml_kv_file = function(file_path, cb) {
+    fs.readFile(file_path, {encoding: 'utf8'}, function (err, body) {
+      /* istanbul ignore if */
+      if (err) return cb('Failed to read key ' + file_path + ' due to ' + err);
+      body ? body.trim() : '';
+      try {
+        var obj = yaml.safeLoad(body);
+        populate_kvs_from_object(branch, create_key_name(branch, file), obj, cb);
+      } catch (e) {
+        logger.warn("Failed to parse .yaml file " + file + ".  Using body string as a KV.");
+        write_content_to_consul(create_key_name(branch, file), body, cb);
+      }
+    });
+  };
+
   var handle_as_flat_file = function(fqf, branch, file, cb) {
     fs.readFile(fqf, {encoding: 'utf8'}, function (err, body) {
       /* istanbul ignore if */
       if (err) return cb('Failed to read key ' + fqf + ' due to ' + err);
-      var body = body ? body.trim() : '';
+      body ? body.trim() : '';
       write_content_to_consul(create_key_name(branch, file), body, cb);
     });
   };
@@ -165,6 +180,11 @@ var file_modified = function(branch, file, cb) {
         if (err) return cb('Failed to delete key ' + key_name + ' due to ' + err);
         handle_json_kv_file(fqf, cb);
       });
+    } else if (file.endsWith('.yaml') || file.endsWith('.yml')) {
+      file_deleted(branch, file, function (err) {
+        if (err) return cb('Failed to delete key ' + key_name + ' due to ' + err);
+        handle_yaml_kv_file(fqf, cb);
+      });
     } else if (file.endsWith('.properties')) {
       file_deleted(branch, file, function (err) {
         if (err) return cb('Failed to delete key ' + key_name + ' due to ' + err);

package.json

@@ -22,7 +22,8 @@
     "mkdirp": "0.5.0",
     "rimraf": "2.2.8",
     "underscore": "^1.8.0",
-    "properties": "1.2.1"
+    "properties": "1.2.1",
+    "js-yaml": "^3.6.1"
   },
   "devDependencies": {
     "grunt": "^0.4.5",

test/git2consul_expand_keys_test.js

@@ -147,6 +147,119 @@ describe('Expand keys', function() {
     });
   });
 
+  /*YAML*/
+
+  it ('should handle additions of new YAML files', function(done) {
+    var sample_key = 'simple.yaml';
+    var sample_value = "---\n\nfirst_level:\n  second_level: is_all_we_need\n";
+
+    // Add the file, call branch.handleRef to sync the commit, then validate that consul contains the correct info.
+    git_utils.addFileToGitRepo(sample_key, sample_value, "Add a file.", function(err) {
+      if (err) return done(err);
+
+      branch.handleRefChange(0, function(err) {
+        if (err) return done(err);
+        // At this point, the repo should have populated consul with our sample_key
+        consul_utils.validateValue('test_repo/master/simple.yaml/first_level/second_level', 'is_all_we_need', function(err, value) {
+          if (err) return done(err);
+          done();
+        });
+      });
+    });
+  });
+
+  it ('should handle changing YAML files', function(done) {
+    var sample_key = 'changeme.yaml';
+    var sample_value = "---\n\nfirst_level:\n  is_all_we_need\n";
+
+    // Add the file, call branch.handleRef to sync the commit, then validate that consul contains the correct info.
+    git_utils.addFileToGitRepo(sample_key, sample_value, "Add a file.", function(err) {
+      if (err) return done(err);
+
+      branch.handleRefChange(0, function(err) {
+        if (err) return done(err);
+        // At this point, the repo should have populated consul with our sample_key
+        consul_utils.validateValue('test_repo/master/changeme.yaml/first_level', 'is_all_we_need', function(err, value) {
+          if (err) return done(err);
+
+          // Add the file, call branch.handleRef to sync the commit, then validate that consul contains the correct info.
+          git_utils.addFileToGitRepo(sample_key, "---\n\nfirst_level:\n  is super different\n", "Change a file.", function(err) {
+            if (err) return done(err);
+
+            branch.handleRefChange(0, function(err) {
+              if (err) return done(err);
+
+              // At this point, the repo should have populated consul with our sample_key
+              consul_utils.validateValue('test_repo/master/changeme.yaml/first_level', 'is super different', function(err, value) {
+                if (err) return done(err);
+
+                done();
+              });
+            });
+          });
+        });
+      });
+    });
+  });
+
+  it ('should handle busted YAML files', function(done) {
+    var sample_key = 'busted.yaml';
+    // from: js-yaml / test / samples-load-errors / forbidden-value.yml
+    var sample_value = "---\n\ntest: key: value\n";
+
+    // Add the file, call branch.handleRef to sync the commit, then validate that consul contains the correct info.
+    git_utils.addFileToGitRepo(sample_key, sample_value, "Add a file.", function(err) {
+      if (err) return done(err);
+
+      branch.handleRefChange(0, function(err) {
+        if (err) return done(err);
+        // At this point, the repo should have populated consul with our sample_key
+        consul_utils.validateValue('test_repo/master/busted.yaml', sample_value, function(err, value) {
+          if (err) return done(err);
+
+          // Add the file, call branch.handleRef to sync the commit, then validate that consul contains the correct info.
+          git_utils.addFileToGitRepo(sample_key, "---\n\nnot_busted: yaml\n", "Change a file.", function(err) {
+            if (err) return done(err);
+
+            branch.handleRefChange(0, function(err) {
+              if (err) return done(err);
+
+              // At this point, the repo should have populated consul with our sample_key
+              consul_utils.validateValue('test_repo/master/busted.yaml/not_busted', 'yaml', function(err, value) {
+                if (err) return done(err);
+
+                done();
+              });
+            });
+          });
+        });
+      });
+    });
+  });
+
+  it ('should handle YAML files with special characters', function(done) {
+    var sample_key = 'special.yaml';
+    var sample_value = "---\n\nfuzzy:\n  second level: ain\'t no one got time for that\n  second/level:\n    ok?: yes\n";
+
+    // Add the file, call branch.handleRef to sync the commit, then validate that consul contains the correct info.
+    git_utils.addFileToGitRepo(sample_key, sample_value, "Add a file.", function(err) {
+      if (err) return done(err);
+
+      branch.handleRefChange(0, function(err) {
+        if (err) return done(err);
+
+        // At this point, the repo should have populated consul with our sample_key
+        consul_utils.validateValue('test_repo/master/special.yaml/fuzzy/second%20level', "ain\'t no one got time for that", function(err, value) {
+          // At this point, the repo should have populated consul with our sample_key
+          consul_utils.validateValue('test_repo/master/special.yaml/fuzzy/second%2Flevel/ok%3F', 'yes', function(err, value) {
+            if (err) return done(err);
+            done();
+          });
+        });
+      });
+    });
+  });
+
   /*properties*/
 
   it ('should handle additions of new properties files', function(done) {
@@ -473,4 +586,4 @@ describe('Expand keys with invalid common properties path ', function() {
     });
   });
 
-});
+});