From abf43f63863066884fda95e67e2b29f6eea74d28 Mon Sep 17 00:00:00 2001 From: David Roetzel Date: Thu, 10 Aug 2023 14:13:05 +0200 Subject: [PATCH] Allow custom lookup functions (#165) * Upgrade dependencies. Latest security fixes for rails plus a bunch of minor updates. * Allow custom lookup functions #162 We do not (and cannot support) custom lookup functions per se, but in case they map cleanly to an existing backing, e.g. eyaml, this allows to specify this mapping, so everything should then work as expected. --------- Co-authored-by: Martin Alfke --- .rubocop_todo.yml | 8 - app/models/hiera_data/data_file.rb | 2 +- app/models/hiera_data/hierarchy.rb | 49 +-- config/hdm.yml.template | 9 + test/models/hiera_data/hierarchy_test.rb | 383 ++++++++++++----------- 5 files changed, 245 insertions(+), 206 deletions(-) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 5dcda35b..0273023d 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -292,7 +292,6 @@ Rails/FilePath: # Include: **/test/**/* Rails/RefuteMethods: Exclude: - - 'test/models/hiera_data/hierarchy_test.rb' - 'test/models/hiera_data/yaml_file_test.rb' - 'test/models/hierarchy_test.rb' - 'test/models/user_test.rb' @@ -336,7 +335,6 @@ Style/ClassAndModuleChildren: - 'test/models/hiera_data/config_test.rb' - 'test/models/hiera_data/data_file_test.rb' - 'test/models/hiera_data/git_repo_test.rb' - - 'test/models/hiera_data/hierarchy_test.rb' - 'test/models/hiera_data/interpolation_test.rb' - 'test/models/hiera_data/yaml_file_test.rb' - 'test/test_helper.rb' @@ -375,7 +373,6 @@ Style/GuardClause: - 'app/controllers/page_controller.rb' - 'app/controllers/sessions_controller.rb' - 'app/models/hiera_data/data_file.rb' - - 'app/models/hiera_data/hierarchy.rb' # Offense count: 1 # Configuration parameters: MinBranchesCount. @@ -399,7 +396,6 @@ Style/IfUnlessModifier: - 'app/controllers/sessions_controller.rb' - 'app/controllers/users_controller.rb' - 'app/models/hiera_data/config.rb' - - 'app/models/hiera_data/hierarchy.rb' # Offense count: 1 # This cop supports safe auto-correction (--auto-correct). @@ -416,7 +412,6 @@ Style/MutableConstant: Style/NumericLiteralPrefix: Exclude: - 'app/models/hiera_data/yaml_file.rb' - - 'test/models/hiera_data/hierarchy_test.rb' - 'test/models/hiera_data/yaml_file_test.rb' # Offense count: 1 @@ -431,10 +426,8 @@ Style/NumericLiterals: # Configuration parameters: PreferredDelimiters. Style/PercentLiteralDelimiters: Exclude: - - 'app/models/hiera_data/hierarchy.rb' - 'config/initializers/friendly_id.rb' - 'test/models/environment_test.rb' - - 'test/models/hiera_data/hierarchy_test.rb' - 'test/models/key_test.rb' # Offense count: 5 @@ -444,7 +437,6 @@ Style/PercentLiteralDelimiters: Style/PreferredHashMethods: Exclude: - 'app/models/hiera_data/data_file.rb' - - 'app/models/hiera_data/hierarchy.rb' - 'app/models/hiera_data/yaml_file.rb' # Offense count: 3 diff --git a/app/models/hiera_data/data_file.rb b/app/models/hiera_data/data_file.rb index a5528c61..8f341661 100644 --- a/app/models/hiera_data/data_file.rb +++ b/app/models/hiera_data/data_file.rb @@ -61,7 +61,7 @@ def create_file(type) when :yaml YamlFile.new(path: @path) else - raise HDM::Error, "unsupported data file type #{type}" + raise Hdm::Error, "unsupported data file type #{type}" end end end diff --git a/app/models/hiera_data/hierarchy.rb b/app/models/hiera_data/hierarchy.rb index 003215ad..004a8e03 100644 --- a/app/models/hiera_data/hierarchy.rb +++ b/app/models/hiera_data/hierarchy.rb @@ -1,6 +1,16 @@ class HieraData class Hierarchy - LOOKUP_FUNCTIONS = %w(lookup_key data_hash data_dig hiera3_backend).freeze + LOOKUP_FUNCTIONS = %w[lookup_key data_hash data_dig hiera3_backend].freeze + BACKENDS = { + "data_hash" => { + "json_data" => :json, + "yaml_data" => :yaml + }, + "lookup_key" => { + "eyaml_lookup_key" => :eyaml + } + }.freeze + attr_reader :raw_hash def initialize(raw_hash:, base_path:) @@ -17,17 +27,7 @@ def lookup_function end def backend - @backend ||= - case [lookup_function, raw_hash[lookup_function]] - when ["data_hash", "yaml_data"] - :yaml - when ["data_hash", "json_data"] - :json - when ["lookup_key", "eyaml_lookup_key"] - :eyaml - else - raise HDM::Error, "unknown backend #{raw_hash[lookup_function]}" - end + @backend ||= determine_backend end def datadir(facts: nil) @@ -44,15 +44,15 @@ def datadir(facts: nil) end def private_key - if backend == :eyaml - @base_path.join(raw_hash.dig("options", "pkcs7_private_key")) - end + return unless backend == :eyaml + + @base_path.join(raw_hash.dig("options", "pkcs7_private_key")) end def public_key - if backend == :eyaml - @base_path.join(raw_hash.dig("options", "pkcs7_public_key")) - end + return unless backend == :eyaml + + @base_path.join(raw_hash.dig("options", "pkcs7_public_key")) end def encryptable? @@ -62,7 +62,7 @@ def encryptable? end def uses_globs? - raw_hash.has_key?("glob") || raw_hash.has_key?("globs") + raw_hash.key?("glob") || raw_hash.key?("globs") end def paths @@ -93,5 +93,16 @@ def setup_paths base_key = uses_globs? ? "glob" : "path" Array(raw_hash[base_key] || raw_hash.fetch("#{base_key}s", [])) end + + def determine_backend + key = lookup_function + value = raw_hash[lookup_function] + backends = BACKENDS + custom_mappings = Rails.configuration.hdm[:custom_lookup_function_mapping] + backends = backends.deep_merge({ "lookup_key" => custom_mappings }) if custom_mappings.present? + backends.fetch(key).fetch(value).to_sym + rescue KeyError + raise Hdm::Error, "unknown backend #{value}" + end end end diff --git a/config/hdm.yml.template b/config/hdm.yml.template index cc95923c..0ea3cef9 100644 --- a/config/hdm.yml.template +++ b/config/hdm.yml.template @@ -79,3 +79,12 @@ production: # idp_cert_fingerprint: "aaa" # idp_cert: "cert" # use either fingerprint _or_ cert but not both +# Example for a custom lookup function, called `my_custom_function`, +# mapped to an existing backend, `eyaml` +# production: +# read_only: false +# allow_encryption: true +# puppet_db: +# server: "https://localhost:8081" +# custom_lookup_function_mapping: +# my_custom_function: eyaml diff --git a/test/models/hiera_data/hierarchy_test.rb b/test/models/hiera_data/hierarchy_test.rb index 374ce568..84ce0cd6 100644 --- a/test/models/hiera_data/hierarchy_test.rb +++ b/test/models/hiera_data/hierarchy_test.rb @@ -1,225 +1,252 @@ require 'test_helper' -class HieraData::HierarchyTest < ActiveSupport::TestCase - test "#uses_globs? returns true if `glob` key present" do - glob = "/*/singular/glob" - raw_hash = { "name" => "Singular path", "glob" => glob } - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - assert hierarchy.uses_globs? - end - - test "#uses_globs? returns true if `globs` key present" do - globs = ["/*/array/globs", "/test/**/globs"] - raw_hash = { "name" => "Singular path", "globs" => globs } - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - assert hierarchy.uses_globs? - end - - test "#paths supports the singular `path` setting" do - path = "/test/singular/path" - raw_hash = { "name" => "Singular path", "path" => path } - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - assert_equal [path], hierarchy.paths - end - - test "#paths returns all non-interpolated path names" do - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - expected_paths = [ - "nodes/%{::facts.fqdn}.yaml", - "role/%{::facts.role}-%{::facts.env}.yaml", - "role/%{::facts.role}.yaml", - "zone/%{::facts.zone}.yaml", - "common.yaml" - ] - assert_equal expected_paths, hierarchy.paths - end - - test "#paths returns an empty array if no data is available" do - hierarchy = HieraData::Hierarchy.new(raw_hash: {}, base_path: ".") - assert hierarchy.paths.empty? - end - - test "#paths supports the singular `glob` setting" do - glob = "/*/singular/glob" - raw_hash = { "name" => "Singular path", "glob" => glob } - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - assert_equal [glob], hierarchy.paths - end - - test "#paths supports the `globs` array setting" do - globs = ["/*/array/globs", "/test/**/globs"] - raw_hash = { "name" => "Singular path", "globs" => globs } - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - assert_equal globs, hierarchy.paths - end - - test "#resolved_paths uses facts to resolve paths" do - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - facts = { - "fqdn" => "testhost", - "role" => "hdm_test", - "env" => "development", - "zone" => "internal" - } - expected_resolved_paths = [ - "nodes/testhost.yaml", - "role/hdm_test-development.yaml", - "role/hdm_test.yaml", - "zone/internal.yaml", - "common.yaml" - ] - assert_equal expected_resolved_paths, hierarchy.resolved_paths(facts:) - end - - test "#resolved_paths resolves globs" do - base_path = Rails.root.join("test/fixtures/files/puppet/environments/globs") - globs = ["common/*.yaml"] - raw_hash = { "name" => "Common", "datadir" => "data", "globs" => globs } - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path:) - facts = { "fqdn" => "testhost" } - expected_resolved_paths = [ - "common/foobar.yaml", - "common/hdm.yaml" - ] - assert_equal expected_resolved_paths, hierarchy.resolved_paths(facts:) - end - - test "#name returns the existing name" do - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - assert_equal "Yaml hierarchy", hierarchy.name - end - - test "#candidate_files returns files matching paths with variables replaced by globs" do - base_path = Rails.root.join("test/fixtures/files/puppet/environments/multiple_hierarchies") - hierarchy = HieraData::Hierarchy.new( - raw_hash: raw_hash.merge("datadir" => "data"), - base_path: - ) - expected_candidate_files = [ - "nodes/4msusyei.betadots.training.yaml", - "nodes/60wxmaw5.betadots.training.yaml", - "nodes/test.host.yaml", - "role/hdm_test.yaml", - "zone/internal.yaml", - "common.yaml" - ] - assert_equal expected_candidate_files, hierarchy.candidate_files - end +class HieraData + class HierarchyTest < ActiveSupport::TestCase + test "#uses_globs? returns true if `glob` key present" do + glob = "/*/singular/glob" + raw_hash = { "name" => "Singular path", "glob" => glob } + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert hierarchy.uses_globs? + end - test "#datadir uses facts to resolve datadir" do - raw_hash = { - "name" => "dynamic datadir", - "datadir" => "%{facts.datadir}" - } - facts = { "datadir" => "data1" } - base_path = Rails.root.join("test/fixtures/files/puppet/environments/dynamic_datadir") - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path:) + test "#uses_globs? returns true if `globs` key present" do + globs = ["/*/array/globs", "/test/**/globs"] + raw_hash = { "name" => "Singular path", "globs" => globs } + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert hierarchy.uses_globs? + end - assert_equal base_path.join("data1"), hierarchy.datadir(facts:) - end + test "#paths supports the singular `path` setting" do + path = "/test/singular/path" + raw_hash = { "name" => "Singular path", "path" => path } + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert_equal [path], hierarchy.paths + end - def raw_hash - { - "name" => "Yaml hierarchy", - "paths" => [ + test "#paths returns all non-interpolated path names" do + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + expected_paths = [ "nodes/%{::facts.fqdn}.yaml", "role/%{::facts.role}-%{::facts.env}.yaml", "role/%{::facts.role}.yaml", "zone/%{::facts.zone}.yaml", "common.yaml" ] - } - end + assert_equal expected_paths, hierarchy.paths + end - class HieraData::HierarchyForYamlDataTest < ActiveSupport::TestCase - test "data_hash specified and yaml" do - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - assert_equal :yaml, hierarchy.backend + test "#paths returns an empty array if no data is available" do + hierarchy = HieraData::Hierarchy.new(raw_hash: {}, base_path: ".") + assert hierarchy.paths.empty? end - def raw_hash - { - "name" => "Yaml hierarchy", - "data_hash" => "yaml_data" - } + test "#paths supports the singular `glob` setting" do + glob = "/*/singular/glob" + raw_hash = { "name" => "Singular path", "glob" => glob } + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert_equal [glob], hierarchy.paths end - end - class HieraData::HierarchyForJSONDataTest < ActiveSupport::TestCase - test "data_hash specified and json" do + test "#paths supports the `globs` array setting" do + globs = ["/*/array/globs", "/test/**/globs"] + raw_hash = { "name" => "Singular path", "globs" => globs } hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - assert_equal :json, hierarchy.backend + assert_equal globs, hierarchy.paths end - def raw_hash - { - "name" => "JSON hierarchy", - "data_hash" => "json_data" + test "#resolved_paths uses facts to resolve paths" do + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + facts = { + "fqdn" => "testhost", + "role" => "hdm_test", + "env" => "development", + "zone" => "internal" } + expected_resolved_paths = [ + "nodes/testhost.yaml", + "role/hdm_test-development.yaml", + "role/hdm_test.yaml", + "zone/internal.yaml", + "common.yaml" + ] + assert_equal expected_resolved_paths, hierarchy.resolved_paths(facts:) end - end - class HieraData::HierarchyForEyamlDataTest < ActiveSupport::TestCase - setup do - @tmpdir = Dir.mktmpdir + test "#resolved_paths resolves globs" do + base_path = Rails.root.join("test/fixtures/files/puppet/environments/globs") + globs = ["common/*.yaml"] + raw_hash = { "name" => "Common", "datadir" => "data", "globs" => globs } + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path:) + facts = { "fqdn" => "testhost" } + expected_resolved_paths = [ + "common/foobar.yaml", + "common/hdm.yaml" + ] + assert_equal expected_resolved_paths, hierarchy.resolved_paths(facts:) end - teardown do - FileUtils.remove_entry @tmpdir + test "#name returns the existing name" do + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert_equal "Yaml hierarchy", hierarchy.name end - test "lookup function is not data_hash" do - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - assert_equal :eyaml, hierarchy.backend + test "#candidate_files returns files matching paths with variables replaced by globs" do + base_path = Rails.root.join("test/fixtures/files/puppet/environments/multiple_hierarchies") + hierarchy = HieraData::Hierarchy.new( + raw_hash: raw_hash.merge("datadir" => "data"), + base_path: + ) + expected_candidate_files = [ + "nodes/4msusyei.betadots.training.yaml", + "nodes/60wxmaw5.betadots.training.yaml", + "nodes/test.host.yaml", + "role/hdm_test.yaml", + "zone/internal.yaml", + "common.yaml" + ] + assert_equal expected_candidate_files, hierarchy.candidate_files end - test "#private_key returns path from options" do - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - assert_equal "private.key", hierarchy.private_key.to_s + test "#datadir uses facts to resolve datadir" do + raw_hash = { + "name" => "dynamic datadir", + "datadir" => "%{facts.datadir}" + } + facts = { "datadir" => "data1" } + base_path = Rails.root.join("test/fixtures/files/puppet/environments/dynamic_datadir") + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path:) + + assert_equal base_path.join("data1"), hierarchy.datadir(facts:) end - test "#public_key returns path from options" do - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") - assert_equal "public.key", hierarchy.public_key.to_s + def raw_hash + { + "name" => "Yaml hierarchy", + "paths" => [ + "nodes/%{::facts.fqdn}.yaml", + "role/%{::facts.role}-%{::facts.env}.yaml", + "role/%{::facts.role}.yaml", + "zone/%{::facts.zone}.yaml", + "common.yaml" + ] + } end - test "#encryptable? is true if all keys present and readable" do - tmpdir_path = Pathname.new(@tmpdir) - %w(private.key public.key).each { |f| FileUtils.touch(tmpdir_path.join(f)) } - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: @tmpdir) + class HierarchyForYamlDataTest < ActiveSupport::TestCase + test "data_hash specified and yaml" do + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert_equal :yaml, hierarchy.backend + end - assert hierarchy.encryptable? + def raw_hash + { + "name" => "Yaml hierarchy", + "data_hash" => "yaml_data" + } + end end - test "#encryptable? is false if a key is missing" do - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: @tmpdir) + class HierarchyForJSONDataTest < ActiveSupport::TestCase + test "data_hash specified and json" do + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert_equal :json, hierarchy.backend + end - refute hierarchy.encryptable? + def raw_hash + { + "name" => "JSON hierarchy", + "data_hash" => "json_data" + } + end end - test "#encryptable? is false if a key is not readable" do - tmpdir_path = Pathname.new(@tmpdir) - files = %w(private.key public.key).map { |f| tmpdir_path.join(f) } - files.each do |path| - FileUtils.touch(path) - File.chmod(0000, path) + class HierarchyForEyamlDataTest < ActiveSupport::TestCase + setup do + @tmpdir = Dir.mktmpdir end - hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: @tmpdir) - refute hierarchy.encryptable? + teardown do + FileUtils.remove_entry @tmpdir + end - File.chmod(0600, *files) + test "lookup function is not data_hash" do + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert_equal :eyaml, hierarchy.backend + end + + test "#private_key returns path from options" do + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert_equal "private.key", hierarchy.private_key.to_s + end + + test "#public_key returns path from options" do + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert_equal "public.key", hierarchy.public_key.to_s + end + + test "#encryptable? is true if all keys present and readable" do + tmpdir_path = Pathname.new(@tmpdir) + %w[private.key public.key].each { |f| FileUtils.touch(tmpdir_path.join(f)) } + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: @tmpdir) + + assert hierarchy.encryptable? + end + + test "#encryptable? is false if a key is missing" do + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: @tmpdir) + + assert_not hierarchy.encryptable? + end + + test "#encryptable? is false if a key is not readable" do + tmpdir_path = Pathname.new(@tmpdir) + files = %w[private.key public.key].map { |f| tmpdir_path.join(f) } + files.each do |path| + FileUtils.touch(path) + File.chmod(0o000, path) + end + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: @tmpdir) + + assert_not hierarchy.encryptable? + + File.chmod(0o600, *files) + end + + def raw_hash + { + "name" => "EYaml hierarchy", + "lookup_key" => "eyaml_lookup_key", + "options" => { + "pkcs7_private_key" => "private.key", + "pkcs7_public_key" => "public.key" + } + } + end end - def raw_hash - { - "name" => "EYaml hierarchy", - "lookup_key" => "eyaml_lookup_key", - "options" => { - "pkcs7_private_key" => "private.key", - "pkcs7_public_key" => "public.key" + class HierarchyForCustomBackend < ActiveSupport::TestCase + test "custom lookup function specified but no mapping present" do + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert_raises Hdm::Error do + hierarchy.backend + end + end + + test "custom lookup function mapped to eyaml" do + Rails.configuration.hdm[:custom_lookup_function_mapping] = { + "custom_eyaml_function" => "eyaml" } - } + hierarchy = HieraData::Hierarchy.new(raw_hash:, base_path: ".") + assert_equal :eyaml, hierarchy.backend + Rails.configuration.hdm.delete(:custom_lookup_function_mapping) + end + + def raw_hash + { + "name" => "JSON hierarchy", + "lookup_key" => "custom_eyaml_function" + } + end end end end