diff --git a/bin/genesis b/bin/genesis index efa1d2a0..7bbb91fb 100755 --- a/bin/genesis +++ b/bin/genesis @@ -312,6 +312,11 @@ define_command("manifest", { "Determines if vault values are fetched or redacted. By default, the ". "manifest will be redacted unless being output to the live console.", + 'entomb' => + "Will entomb valut secrets into the BOSH director's credhub as is done ". + "when deploying, providing a manifest that does not contain any clear ". + "text secrets", + 'prune!' => "Determines if the build metadata is pruned. Defaults to true.", @@ -384,6 +389,14 @@ define_command("deploy", { 'max-in-flight=i' => "Override the default number of maximum VMs in flight per instance group.", + + 'entomb!' => + "By default, deployments to BOSH directors will entomb vault secrets into ". + "the director's CredHub, then use CredHub variable operators to prevent ". + "secrets from being present in clear text in the manifest. This can be ". + "prevented by specifying --no-entomb as an option, or by setting ". + "'entomb_secrets: false' in your #C{~/.genesis/config} file. The option ". + "will override the configuration file setting.", ] }); # }}} @@ -450,7 +463,7 @@ define_command("info", { # genesis lookup - Find a key set in environment manifests. {{{ define_command("lookup", { summary => "Look up values from an environment's file, manifest, deployment, exodus or env state.", - usage => " lookup [--merged|--deployed|--env|--exodus|] key [default-value|--defined]\n". + usage => " lookup [--merged|--deployed|--env|--exodus|] key [default-value|--defined] [--entomb]\n". "lookup --exodus-for key [default-value|--defined]", alias => 'get', description => @@ -474,6 +487,11 @@ define_command("lookup", { "Takes an argument of the form 'env-name/deployment-type'", "env" => "Lookup environment variables used by Genesis for the given environment.", + "entomb" => + "Entomb the vault secrets in the BOSH director's credhub before ". + "rendering the manifest, so that if looking up a secret, return the ". + "reference to credhub instead. Not applicable to exodus or deployed ". + "manifests", "defined" => "Exit with 0 if key defined in specified source, 9 otherwise. No output ". "is produced, making it useful in 'if lookup ... ; then'" diff --git a/lib/Genesis/Commands/Env.pm b/lib/Genesis/Commands/Env.pm index 964df8a1..b9f602ca 100644 --- a/lib/Genesis/Commands/Env.pm +++ b/lib/Genesis/Commands/Env.pm @@ -254,24 +254,26 @@ sub manifest { ); } - print $env + output {raw => 1}, $env ->download_required_configs('blueprint', 'manifest') ->manifest( partial => get_options->{partial}, redact => get_options->{redact}, prune => get_options->{prune}, - vars_only => get_options->{'bosh-vars'} + vars_only => get_options->{'bosh-vars'}, + entomb => get_options->{entomb} ); } sub deploy { option_defaults( - redact => ! -t STDOUT + redact => ! -t STDOUT, + entomb => $Genesis::RC->get('entomb_secrets',1) ); command_usage(1) if @_ != 1; my %options = %{get_options()}; - my @invalid_create_env_opts = grep {$options{$_}} (qw/fix dry-run/); + my @invalid_create_env_opts = grep {$options{$_}} (qw/fix dry-run entomb/); $options{'disable-reactions'} = ! delete($options{reactions}); my $env = Genesis::Top->new('.')->load_env($_[0])->with_vault(); @@ -286,6 +288,8 @@ sub deploy { join(", ", @invalid_create_env_opts) ) if $env->use_create_env && @invalid_create_env_opts; + $options{entomb} = 1 unless defined($options{entomb}) || $env->use_create_env; + info "Preparing to deploy #C{%s}:\n - based on kit #c{%s}\n - using Genesis #c{%s}", $env->name, $env->kit->id, $Genesis::VERSION; if ($env->use_create_env) { info " - as a #M{create-env} deployment\n"; diff --git a/lib/Genesis/Commands/Info.pm b/lib/Genesis/Commands/Info.pm index 684e9964..94ec7d73 100644 --- a/lib/Genesis/Commands/Info.pm +++ b/lib/Genesis/Commands/Info.pm @@ -133,6 +133,13 @@ sub lookup { } my $env = $top->load_env($name); my $v; + if (get_options->{entomb}) { + bail( + "Cannot use --entombed option with --exodus, --exodus-for or --deployed", + ) if scalar( grep {$_} (@{get_options()}{qw/exodus exodus-for deployed/})); + $env->entombed_secrets_enabled(1); + } + if (get_options->{merged}) { bail( "Circular reference detected while trying to lookup merged manifest of $name" @@ -167,7 +174,7 @@ sub lookup { exit(ref($v) eq "NotFound" ? 4 : 0); } elsif (defined($v)) { $v = encode_json($v) if ref($v); - output {raw => 1}, "$v\n"; + output {raw => 1}, $v; } exit 0; } diff --git a/lib/Genesis/Env.pm b/lib/Genesis/Env.pm index 67f0043a..f34293e2 100644 --- a/lib/Genesis/Env.pm +++ b/lib/Genesis/Env.pm @@ -10,6 +10,9 @@ use Genesis::State; use Genesis::Term; use Genesis::BOSH::Director; use Genesis::BOSH::CreateEnvProxy; +use Genesis::Vault; +use Genesis::Vault::Local; +use Genesis::Vault::None; use Genesis::UI; use Genesis::IO qw/DumpYAML LoadFile/; use Genesis::Commands qw/current_command known_commands/; @@ -18,6 +21,7 @@ use JSON::PP qw/encode_json decode_json/; use POSIX qw/strftime/; use Data::Dumper; use Digest::file qw/digest_file_hex/; +use Digest::SHA qw/sha1_hex/; use Time::Seconds; ### Class Methods {{{ @@ -1062,7 +1066,7 @@ sub credhub_connection_env { $credhub_path .= "-bosh"; } $env{GENESIS_CREDHUB_EXODUS_SOURCE} = $credhub_src; - $env{GENESIS_CREDHUB_ROOT}=sprintf("%s/%s-%s", $credhub_path, $self->name, $self->type); + $env{GENESIS_CREDHUB_ROOT}=sprintf("/%s/%s-%s", $credhub_path, $self->name, $self->type); if ($credhub_src) { my $credhub_info = $self->exodus_lookup('.',undef,$credhub_src); @@ -1245,6 +1249,27 @@ sub ci_base { # }}} +# Environment Dependencies - CredHub +# credhub - get the credhub instance for the environment {{{ +sub credhub { + my $ref = $_[0]->_memoize(sub { + require Genesis::Credhub; + my ($self) = @_; + my %env = $self->credhub_connection_env; + my $credhub = Genesis::Credhub->new( + $self->deployment_name, + $env{GENESIS_CREDHUB_ROOT}, + $env{CREDHUB_SERVER}, + $env{CREDHUB_CLIENT}, + $env{CREDHUB_SECRET}, + $env{CREDHUB_CA_CERT} + ); + return $credhub; + }); + return $ref; +} +# }}} + # Environment Dependencies - BOSH and BOSH Config Files # with_bosh - ensure the BOSH director is available and authenticated {{{ sub with_bosh { @@ -1543,13 +1568,15 @@ sub manifest { my ($self, %opts) = @_; # prune by default. - my ($redact,$prune,$vars_only) = @opts{qw(redact prune vars_only)}; + my ($entomb,$redact,$prune,$vars_only) = @opts{qw(entomb redact prune vars_only)}; $prune = 1 unless defined $prune; - trace "[env $self->{name}] in manifest(): Redact %s", defined($opts{redact}) ? "'$opts{redact}'" : '#C{(undef)}'; - trace "[env $self->{name}] in manifest(): Prune: %s", defined($opts{prune}) ? "'$opts{prune}'" : '#C{(undef)}'; + trace "[env $self->{name}] in manifest(): Entomb: %s", defined($opts{entomb}) ? "'$opts{entomb}'" : '#C{(undef)}'; + trace "[env $self->{name}] in manifest(): Redact: %s", defined($opts{redact}) ? "'$opts{redact}'" : '#C{(undef)}'; + trace "[env $self->{name}] in manifest(): Prune: %s", defined($opts{prune}) ? "'$opts{prune}'" : '#C{(undef)}'; - my (undef, $path) = $self->_manifest(get_opts(\%opts, qw/no_warnings partial redact/)); + $self->entombed_secrets_enabled($entomb); + my (undef, $path) = $self->_manifest(get_opts(\%opts, qw/no_warnings partial redact msg force_msg/)); if ($vars_only) { (my $vars_path = $path) =~ s/.yml/__vars.yml/; @@ -1764,8 +1791,12 @@ sub deploy { "Preflight checks failed; deployment operation halted." ) unless $self->check(); - info "\n[#M{%s}] generating manifest...", $self->name; - $self->write_manifest("$self->{__tmp}/manifest.yml", redact => 0); + $self->write_manifest( + "$self->{__tmp}/manifest.yml", + force_msg => "generating manifest...", + entomb => $opts{entomb}, + redact => 0 + ); my ($ok, $predeploy_data,$data_fn); $self->_notify('generating BOSH vars file (#i{if applicable})...'); @@ -1806,7 +1837,7 @@ sub deploy { # Prepare the output manifest files for the repo my $manifest_file = $self->tmppath("out-manifest.yml"); my $vars_path = $self->tmppath("out-vars.yml"); - $self->write_manifest($manifest_file, redact => 1, prune => 0); + $self->write_manifest($manifest_file, entomb => $opts{entomb}, redact => 1, prune => 0); copy_or_fail($self->vars_file('redacted'), $vars_path) if ($self->vars_file('redacted')); # DEPLOY!!! @@ -2157,23 +2188,173 @@ sub remove_secrets { } } +# }}} +# entombed_secrets_enabled - allow entombing of secrets from vault to credhub {{{ +sub entombed_secrets_enabled { + my $self = shift; + $self->{__entombed_secrets_enabled} = ($_[0] ? 1 : 0) if scalar(@_); + return $self->{__entombed_secrets_enabled}; +} + +# }}} +# secrets_entombed - return the number of secrets entombed to credhub, -1 if not applicable {{{ +sub secrets_entombed { + return $_[0]->{__entombed}//-1; +} # }}} # }}} ### Private Instance Methods {{{ +# _entomb_secrets - duplicate secrets into credhub, return json to setup {{{ +sub _entomb_secrets { + my ($self) = @_; + + return $self->vault + if $self->use_create_env # Create-env can't use credhub + || $self->secrets_entombed == 0 # Entombed tried and failed + || ! $self->entombed_secrets_enabled; + + return Genesis::Vault::Local->new($self->name) if $self->secrets_entombed > 0; + + $self->with_vault(); + $self->_notify("entombing secrets into Credhub for enhanced security..."); + info ( + {pending => 1}, + "[[ >>Determining vault paths used by manifest from %s...", + $self->vault->name + ); + my $secret_paths = $self->vault_paths(); + my $secrets_count = scalar(keys %$secret_paths); + if ($secrets_count) { + info "found %d paths.", $secrets_count; + my %secret_keys = (); + + info ( + {pending => 1}, + "[[ >>Retrieving secrets from used vault paths...", + ); + for (keys %$secret_paths) { + my ($s,$k) = split ":", $_, 2; + $s =~ s#^/?#/#; # make sure the secret path starts with a / + push(@{$secret_keys{$s}}, $k) + } + my %secret_values = %{ + scalar(read_json_from( + $self->vault->query({redact_output => 1},"export", keys %secret_keys) + )) + }; + # BUG FIX for safe export on similar names + for (map {substr($_,1)} keys %secret_keys) { + $secret_values{$_} = $self->vault->get("/$_") + unless (defined($secret_values{$_})); + } + info ("#g{done!}"); + + my $local_vault = Genesis::Vault::Local->new($self->name); + my $credhub = $self->credhub(); + + #Design decision: use value-type credhub for each key, and only populate what is needed. + my $base_path = $self->secrets_base(); + my $idx = 0; + my $w = length("$secrets_count"); + my $entombment_prefix = ""; # can be set to another value to prevent conflicts if needed + info( + "[[ >>Copying Vault values to Credhub: #c{%s} => #B{%s}:", + $base_path, $credhub->base().$entombment_prefix? "/$entombment_prefix" : "/" + ); + my $previous_lines=0; + my %results = (new => 0, failed => 0, altered => 0, 'exists' => 0); + for my $secret (sort keys %secret_keys) { + my $vault_label = $secret; + $vault_label =~ s/^$base_path(.*)/csprintf("#C{$1}")/e; + my $cred_path = $secret; + $cred_path =~ s/^$base_path//; + $cred_path =~ s#^/#_/#; + for my $key (sort @{$secret_keys{$secret}}) { + my $value = $secret_values{substr($secret,1)}{$key}; + my $secret_sha = substr(sha1_hex("$cred_path--$key--".$value),0,8); + my $cred_name = "$entombment_prefix$cred_path--$key--$secret_sha"; + my $credhub_var = "(($cred_name))"; + my $existing = $credhub->get($cred_name); + my $action_color = "yi"; + my $action = "exists"; + unless ($existing && $existing eq $value) { + $credhub->set($cred_name, $value); + my $new_value = $credhub->get($cred_name); + if ($new_value ne $value) { + $action = "failed"; + $action_color = "Yr"; + } else { + $action = $existing ? "altered" : "new"; + $action_color = $existing ? "ri" : "gi"; + } + } + $local_vault->set($secret, $key, $credhub_var); + print "\r" for (1..$previous_lines); + $results{$action} += 1; + my $msg = wrap(sprintf( + "[[ [%*d/%*d] >>%s:#c{%s} #Kk{[sha1: }#Wk{%s}#Kk{]} #G{=>} #B{%s} ...#%s{%s}", + $w, ++$idx, $w, $secrets_count, "#y{$vault_label}", $key, $secret_sha, + $credhub_var, $action_color, $action + ), terminal_width); + info $msg; + $previous_lines=($existing && $existing eq $value) ? scalar(lines($msg)) : 0; + } + } + print "\r" for (1..$previous_lines); + info( + "[[ >>$idx of $secrets_count secrets processed: %s new, %s already exist, %s altered, %s failed", + @results{('new','exists','altered','failed')} + ); + + bail( + "Failed to entomb one or more secrets into Credhub. This may be due ". + "to a bug in Genesis, communication or authentication error with ". + "Credhub, or a value that Credhub can't support.\n\n". + "Please try again without the --entomb option if used, or if deploying, ". + "use the --no-entomb option, if this persists.\n\n". + "Please contact the Genesis team, or open a issue on ". + "#Bu{%s/issues/new}", + $Genesis::GITHUB + ) if ($results{failed}); + + $self->{__entombed} = $idx; + return $local_vault; + } else { + info "[[ >>No vault paths in use.\n"; + $self->{__entombed} = $secrets_count; + return $self->vault; + } +} + +# }}} # _manifest - build or return cached manifest (with/out redaction and pruning) {{{ sub _manifest { my ($self, %opts) = @_; + + my $vault_src = $self->_entomb_secrets(); + my $vault_env = $vault_src->env(); + + my $redact = $opts{redact} && (!$self->entombed_secrets_enabled); + trace "[env $self->{name}] in _manifest(): Redact %s", defined($opts{redact}) ? "'$opts{redact}'" : '#C{(undef)}'; - my $which = ($opts{partial} ? '__partial' : "").($opts{redact} ? '__redacted' : '__unredacted'); - my $path = "$self->{__tmp}/$which.yml"; + my $which = + ( + ($self->entombed_secrets_enabled && $self->secrets_entombed > 0) ? '__entombed' + : ($redact ? '__redacted' : '__unredacted') + ).($opts{partial} ? '__partial' : ""); + my $path = "$self->{__tmp}/manifest${which}.yml"; trace("[env $self->{name}] in _manifest(): looking for the '$which' cached manifest"); - if (!$self->{$which}) { + + $self->_notify($opts{force_msg}) if $opts{force_msg}; + if (!$self->{$which."__manifest"}) { trace("[env $self->{name}] in ${which}_manifest(): cache MISS; generating"); trace("[env $self->{name}] in ${which}_manifest(): cwd is ".Cwd::cwd); + $self->_notify($opts{msg}) if $opts{msg}; + my @merge_files = $self->_yaml_files($opts{partial}); trace("[env $self->{name}] in _manifest(): merging $_") for @merge_files; @@ -2181,8 +2362,8 @@ sub _manifest { my $out; my $env = { $self->get_environment_variables('manifest'), - %{$self->vault->env()}, # specify correct vault for spruce to target - REDACT => $opts{redact} ? 'yes' : '' # spruce redaction flag + %$vault_env, # specify correct vault for spruce to target + REDACT => $redact ? 'yes' : '' # spruce redaction flag }; if ($opts{partial}) { debug("running spruce merge of all files, without evaluation or cloudconfig, for parameter dereferencing"); @@ -2201,13 +2382,18 @@ sub _manifest { } popd; - debug("saving #W{%s%s} manifest to $path", $opts{partial} ? 'partial ' : '', $opts{redact} ? 'redacted' : 'unredacted'); + $out =~ s/[\r\n ]*\z/\n/ms; # Ensure output is terminated with a newline, but no blank lines + debug( + "saving #W{%s} manifest to $path [%d bytes]", + join(" ",split("__", $which)), + length($out) + ); mkfile_or_fail($path, 0400, $out); - $self->{$which} = load_yaml($out); + $self->{$which."__manifest"} = load_yaml($out); } else { trace("[env $self->{name}] in ${which}_manifest(): cache HIT!"); } - return $self->{$which}, $path; + return $self->{$which."__manifest"}, $path; } # }}} diff --git a/t/30-env.t b/t/30-env.t index 175dbee4..933b3c3c 100644 --- a/t/30-env.t +++ b/t/30-env.t @@ -879,7 +879,6 @@ deploy --max-in-flight=5 $env->{__tmp}/manifest.yml EOF - ($manifest_file, $exists, $sha1) = $env->cached_manifest_info; ok $manifest_file eq $env->path(".genesis/manifests/".$env->name.".yml"), "cached manifest path correctly determined"; ok $exists, "manifest file should exist."; @@ -1009,7 +1008,6 @@ cc: cloud-config-data collection: 1 2 3 deployment_name: standalone-thing something: valueable - EOF teardown_vault(); @@ -1623,7 +1621,7 @@ EOF GENESIS_CI_MOUNT_OVERRIDE => "true", GENESIS_CREDHUB_EXODUS_SOURCE => "root_vault/credhub", GENESIS_CREDHUB_EXODUS_SOURCE_OVERRIDE => "root_vault/credhub", # Shouldn't this be boolean? - GENESIS_CREDHUB_ROOT => "root_vault-credhub/base-extended-thing", + GENESIS_CREDHUB_ROOT => "/root_vault-credhub/base-extended-thing", GENESIS_ENV_REF => $env->name, GENESIS_ENV_KIT_OVERRIDE_FILES => re('\/(var\/folders|tmp)\/.*\/env-overrides-0.yml'), GENESIS_EXODUS_BASE => "/shhhh/exodus/base-extended/thing", @@ -1820,13 +1818,13 @@ EOF local $ENV{GENESIS_OUTPUT_COLUMNS}=120; my $fragment = <<'EOF'; -\[predeploy-reaction-fail: PRE-DEPLOY\] Running working-addon addon from kit +\[predeploy-reaction-fail/thing: PRE-DEPLOY\] Running working-addon addon from kit reactions/in-development \(dev\): This addon worked, with arguments of this that -\[predeploy-reaction-fail: PRE-DEPLOY\] Running script `bin/fail-script.sh` with -no arguments: +\[predeploy-reaction-fail/thing: PRE-DEPLOY\] Running script `bin/fail-script.sh` +with no arguments: This script failed EOF @@ -1873,48 +1871,49 @@ EOF $stderr =~ s/(Duration:|-) (\d+ minutes?, )?\d+ seconds?/$1 XXX seconds/g; eq_or_diff($stderr, <<'EOF', "deploy output should contain the correct pre-deploy output"); -[postdeploy-reaction-fail] reactions/in-development (dev) does not define a 'check' hook; BOSH configs and +[postdeploy-reaction-fail/thing] reactions/in-development (dev) does not define a 'check' hook; BOSH configs and environmental parameters checks will be skipped. -[postdeploy-reaction-fail] running secrets checks... +[postdeploy-reaction-fail/thing] running secrets checks... Parsing kit secrets descriptions ... done. - XXX seconds Retrieving all existing secrets ... done. - XXX seconds Validating 0 secrets for postdeploy-reaction-fail under path '/secret/postdeploy/reaction/fail/thing/': Completed - Duration: XXX seconds [0 validated/0 skipped/0 errors] -[postdeploy-reaction-fail] running manifest viability checks... -[postdeploy-reaction-fail] running stemcell checks... +[postdeploy-reaction-fail/thing] running manifest viability checks... + +[postdeploy-reaction-fail/thing] running stemcell checks... + +[postdeploy-reaction-fail/thing] generating manifest... -[postdeploy-reaction-fail] generating manifest... +[postdeploy-reaction-fail/thing] generating BOSH vars file (if applicable)... -[postdeploy-reaction-fail: PRE-DEPLOY] Running working-addon addon from kit reactions/in-development (dev): +[postdeploy-reaction-fail/thing: PRE-DEPLOY] Running working-addon addon from kit reactions/in-development (dev): This addon worked, with arguments of this that -[postdeploy-reaction-fail: PRE-DEPLOY] Running script `bin/pass-script.sh` with arguments of ["just a single arg with -spaces"]: +[postdeploy-reaction-fail/thing: PRE-DEPLOY] Running script `bin/pass-script.sh` with arguments of ["just a single +arg with spaces"]: This script passed Argument 1: 'just a single arg with spaces' -[postdeploy-reaction-fail] all systems ok, initiating BOSH deploy... +[postdeploy-reaction-fail/thing] all systems ok, initiating BOSH deploy... +[postdeploy-reaction-fail/thing] Deployment successful. -[postdeploy-reaction-fail] Deployment successful. - - -[postdeploy-reaction-fail: POST-DEPLOY] Running script `bin/fail-script.sh` with no arguments: +[postdeploy-reaction-fail/thing: POST-DEPLOY] Running script `bin/fail-script.sh` with no arguments: This script failed [WARNING] Environment post-deploy reaction failed! Manual intervention may be needed. -[postdeploy-reaction-fail] Preparing metadata for export... +[postdeploy-reaction-fail/thing] Preparing metadata for export... -[DONE] postdeploy-reaction-fail deployed successfully. +[DONE] postdeploy-reaction-fail/thing deployed successfully. EOF @@ -1928,33 +1927,34 @@ EOF $stderr =~ s/(Duration:|-) (\d+ minutes?, )?\d+ seconds?/$1 XXX seconds/g; eq_or_diff($stderr, <<'EOF', "deploy output should contain the correct pre-deploy output"); -[postdeploy-reaction-fail] reactions/in-development (dev) does not define a 'check' hook; BOSH configs and +[postdeploy-reaction-fail/thing] reactions/in-development (dev) does not define a 'check' hook; BOSH configs and environmental parameters checks will be skipped. -[postdeploy-reaction-fail] running secrets checks... +[postdeploy-reaction-fail/thing] running secrets checks... Parsing kit secrets descriptions ... done. - XXX seconds Retrieving all existing secrets ... done. - XXX seconds Validating 0 secrets for postdeploy-reaction-fail under path '/secret/postdeploy/reaction/fail/thing/': Completed - Duration: XXX seconds [0 validated/0 skipped/0 errors] -[postdeploy-reaction-fail] running manifest viability checks... -[postdeploy-reaction-fail] running stemcell checks... +[postdeploy-reaction-fail/thing] running manifest viability checks... -[postdeploy-reaction-fail] generating manifest... +[postdeploy-reaction-fail/thing] running stemcell checks... -[WARNING] Reactions are disabled for this deploy +[postdeploy-reaction-fail/thing] generating manifest... -[postdeploy-reaction-fail] all systems ok, initiating BOSH deploy... +[postdeploy-reaction-fail/thing] generating BOSH vars file (if applicable)... +[WARNING] Reactions are disabled for this deploy -[postdeploy-reaction-fail] Deployment successful. +[postdeploy-reaction-fail/thing] all systems ok, initiating BOSH deploy... +[postdeploy-reaction-fail/thing] Deployment successful. -[postdeploy-reaction-fail] Preparing metadata for export... +[postdeploy-reaction-fail/thing] Preparing metadata for export... -[DONE] postdeploy-reaction-fail deployed successfully. +[DONE] postdeploy-reaction-fail/thing deployed successfully. EOF diff --git a/t/31-secrets_store-vault.t b/t/31-secrets_store-vault.t deleted file mode 100644 index 2330a003..00000000 --- a/t/31-secrets_store-vault.t +++ /dev/null @@ -1,1730 +0,0 @@ -#!perl -use strict; -use warnings; -use utf8; - -use lib 'lib'; -use lib 't'; -use helper; -use Test::Exception; -use Test::Deep; -use Test::Output; -use Test::Differences; - -use Genesis::Top; -use Genesis::Env; -use_ok 'Genesis::Env::SecretsStore::Vault'; -use Genesis; - -fake_bosh; - -# To select tests, specify `prove :: test_id_1 test_id_2 ...` - -# Test ID: initialize -subtest 'initialize vault secret store' => sub { - plan skip_all => 'initialize not selected test' - if @ARGV && ! grep {$_ eq 'initialize'} @ARGV; - - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL)->link_dev_kit('t/src/creator'); - put_file $top->path("standalone.yml"), <load_env('standalone'); }; - - my $ss; - lives_ok {$ss = $env->secrets_store} "Can instantiate a secrets store for environment"; - ok ref($ss) eq "Genesis::Env::SecretsStore::Vault", "Instanciated secrets store is a Vault secrets store"; - - # Default values - ok $ss->default_mount eq '/secret/', "Vault secret store default mount is '/secret/'"; - ok $ss->mount eq '/secret/', "Vault secret store mount using the default mount"; - ok $ss->default_slug eq 'standalone/thing', "Vault secret store default slug is '/'"; - ok $ss->slug eq 'standalone/thing', "Vault secret store slug is using the default slug"; - ok $ss->base eq '/secret/standalone/thing/', "Vault secret store base is the mount + the slug + '/'"; - ok $ss->root_ca_path eq '', "Vault secret store has no root ca path"; - - # `cp /Users/dbell/.replyrc \$HOME/` unless -f $ENV{HOME}."/.replyrc"; use Pry; pry; - - teardown_vault(); -}; - -subtest 'override values for vault secret store' => sub { - plan skip_all => 'override_values not selected test' - if @ARGV && ! grep {$_ eq 'override_values'} @ARGV; - - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL)->link_dev_kit('t/src/creator'); - put_file $top->path("standalone.yml"), <load_env('standalone'); }; - - my $ss; - lives_ok {$ss = $env->secrets_store} "Can instantiate a secrets store for environment"; - ok ref($ss) eq "Genesis::Env::SecretsStore::Vault", "Instanciated secrets store is a Vault secrets store"; - - # Default values - ok $ss->default_mount eq '/secret/', "Vault secret store default mount is '/secret/'"; - ok $ss->mount eq '/shhh/', "Vault secret store mount using the formatted version of provided mount of '/shhh/'"; - ok $ss->default_slug eq 'standalone/thing', "Vault secret store default slug is '/'"; - ok $ss->slug eq 'super-cool-env/lab', "Vault secret store slug is using formatted version of provided path"; - ok $ss->base eq '/shhh/super-cool-env/lab/', "Vault secret store base is the provided mount + provided slug + '/'"; - ok $ss->root_ca_path eq '/shhh/root_ca/cert', "Vault secret store has a specified root ca path"; - - `cp /Users/dbell/.replyrc \$HOME/` unless -f $ENV{HOME}."/.replyrc"; use Pry; pry; - - teardown_vault(); -}; - -done_testing; -exit(0); - -subtest 'new() validation' => sub { - quietly { throws_ok { Genesis::Env->new() } - qr/no 'name' specified.*this is most likely a bug/is; - }; - - quietly { throws_ok { Genesis::Env->new(name => 'foo') } - qr/no 'top' specified.*this is most likely a bug/is; - }; -}; - -subtest 'name validation' => sub { - lives_ok { Genesis::Env->_validate_env_name("my-new-env"); } - "my-new-env is a good enough name"; - - quietly { throws_ok { Genesis::Env->_validate_env_name(""); } - qr/must not be empty/i; - }; - - quietly { throws_ok { Genesis::Env->_validate_env_name("my\tnew env\n"); } - qr/must not contain whitespace/i; - }; - - quietly { throws_ok { Genesis::Env->_validate_env_name("my-new-!@#%ing-env"); } - qr/can only contain lowercase letters, numbers, and hyphens/i; - }; - - quietly { throws_ok { Genesis::Env->_validate_env_name("-my-new-env"); } - qr/must start with a .*letter/i; - }; - - quietly { throws_ok { Genesis::Env->_validate_env_name("my-new-env-"); } - qr/must not end with a hyphen/i; - }; - - quietly { throws_ok { Genesis::Env->_validate_env_name("my--new--env"); } - qr/must not contain sequential hyphens/i; - }; - - for my $ok (qw( - env1 - us-east-1-prod - this-is-a-really-long-hyphenated-name-oh-god-why-would-you-do-this-to-yourself - company-us_east_1-prod - )) { - lives_ok { Genesis::Env->_validate_env_name($ok); } "$ok is a valid env name"; - } -}; - -subtest 'loading' => sub { - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL); - $top->link_dev_kit('t/src/simple'); - put_file $top->path("standalone.yml"), <load_env('standalone') } - "should be able to load the `standalone' environment."; - lives_ok { $top->load_env('standalone.yml') } - "should be able to load an environment by filename."; - teardown_vault(); -}; - -subtest 'env-to-env relation' => sub { - my $a = bless({ name => "us-west-1-preprod-a" }, 'Genesis::Env'); - my $b = bless({ name => "us-west-1-prod" }, 'Genesis::Env'); - - cmp_deeply([$a->relate($b)], [qw[ - ./us.yml - ./us-west.yml - ./us-west-1.yml - ./us-west-1-preprod.yml - ./us-west-1-preprod-a.yml - ]], "(us-west-1-preprod-a)->relate(us-west-1-prod) should return correctly"); - - cmp_deeply([$a->relate($b, ".cache")], [qw[ - .cache/us.yml - .cache/us-west.yml - .cache/us-west-1.yml - ./us-west-1-preprod.yml - ./us-west-1-preprod-a.yml - ]], "relate() should handle cache prefixes, if given"); - - cmp_deeply([$a->relate($b, ".cache", "TOP/LEVEL")], [qw[ - .cache/us.yml - .cache/us-west.yml - .cache/us-west-1.yml - TOP/LEVEL/us-west-1-preprod.yml - TOP/LEVEL/us-west-1-preprod-a.yml - ]], "relate() should handle cache and top prefixes, if both are given"); - - cmp_deeply([$a->relate("us-east-sandbox", ".cache", "TOP/LEVEL")], [qw[ - .cache/us.yml - TOP/LEVEL/us-west.yml - TOP/LEVEL/us-west-1.yml - TOP/LEVEL/us-west-1-preprod.yml - TOP/LEVEL/us-west-1-preprod-a.yml - ]], "relate() should take names for \$them, in place of actual Env objects"); - - cmp_deeply([$a->relate($a, ".cache", "TOP/LEVEL")], [qw[ - .cache/us.yml - .cache/us-west.yml - .cache/us-west-1.yml - .cache/us-west-1-preprod.yml - .cache/us-west-1-preprod-a.yml - ]], "relate()-ing an env to itself should work (if a little depraved)"); - - cmp_deeply([$a->relate(undef, ".cache", "TOP/LEVEL")], [qw[ - TOP/LEVEL/us.yml - TOP/LEVEL/us-west.yml - TOP/LEVEL/us-west-1.yml - TOP/LEVEL/us-west-1-preprod.yml - TOP/LEVEL/us-west-1-preprod-a.yml - ]], "relate()-ing to nothing (undef) should treat everything as unique"); - - cmp_deeply(scalar $a->relate($b, ".cache", "TOP/LEVEL"), { - common => [qw[ - .cache/us.yml - .cache/us-west.yml - .cache/us-west-1.yml - ]], - unique => [qw[ - TOP/LEVEL/us-west-1-preprod.yml - TOP/LEVEL/us-west-1-preprod-a.yml - ]], - }, "relate() in scalar mode passes back a hashref"); - - { - local $ENV{PREVIOUS_ENV} = 'us-west-1-sandbox'; - cmp_deeply([$a->potential_environment_files()], [qw[ - .genesis/cached/us-west-1-sandbox/us.yml - .genesis/cached/us-west-1-sandbox/us-west.yml - .genesis/cached/us-west-1-sandbox/us-west-1.yml - ./us-west-1-preprod.yml - ./us-west-1-preprod-a.yml - ]], "potential_environment_files() called with PREVIOUS_ENV should leverage the Genesis cache"); - } -}; - -subtest 'environment metadata' => sub { - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL); - quietly { $top->download_kit('bosh/0.2.0'); }; - put_file $top->path("standalone.yml"), <load_env('standalone'); }; - is($env->name, "standalone", "an environment should know its name"); - is($env->file, "standalone.yml", "an environment should know its file path"); - is($env->deployment_name, "standalone-thing", "an environment should know its deployment name"); - is($env->kit->id, "bosh/0.2.0", "an environment can ask the kit for its kit name/version"); - is($env->secrets_mount, '/secret/', "default secret mount used when none provided"); - is($env->secrets_slug, 'standalone/thing', "default secret slug generated correctly"); - is($env->secrets_base, '/secret/standalone/thing/', "default secret base path generated correctly"); - is($env->exodus_mount, '/secret/exodus/', "default exodus mount used when none provided"); - is($env->exodus_base, '/secret/exodus/standalone/thing', "correctly evaluates exodus base path"); - is($env->ci_mount, '/secret/ci/', "default ci mount used when none provided"); - is($env->ci_base, '/secret/ci/thing/standalone/', "correctly evaluates ci base path"); - - put_file $top->path("standalone-with-another.yml"), <load_env('standalone-with-another.yml');} - qr/\[ERROR\] Environment standalone-with-another.yml could not be loaded:\n\s+- kit bosh\/0.2.0 is not compatible with secrets_mount feature; check for newer kit version or remove feature.\n\s+- kit bosh\/0.2.0 is not compatible with exodus_mount feature; check for newer kit version or remove feature./ms, - "Outdated kits bail when using v2.7.0 features"; - }; - - teardown_vault(); - -}; - -subtest 'parameter lookup' => sub { - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL); - quietly { $top->download_kit('bosh/0.2.0'); }; - put_file $top->path("standalone.yml"), <load_env('enoent'); } qr/enoent.yml does not exist/; }; - quietly { throws_ok { $top->load_env('e-no-ent'); } qr/does not exist/; }; - - lives_ok { $env = $top->load_env('standalone') } - "Genesis::Env should be able to load the `standalone' environment."; - - ok($env->defines('params.state'), "standalone.yml should define params.state"); - is($env->lookup('params.state'), "awesome", "params.state in standalone.yml should be 'awesome'"); - ok($env->defines('params.false'), "params with falsey values should still be considered 'defined'"); - ok(!$env->defines('params.enoent'), "standalone.yml should not define params.enoent"); - is($env->lookup('params.enoent', 'MISSING'), 'MISSING', - "params lookup should return the default value is the param is not defined"); - is($env->lookup('params.false', 'MISSING'), undef, - "params lookup should return falsey values if they are set"); - - cmp_deeply([$env->features], [qw[vsphere second-feature]], - "features() returns the current features"); - ok($env->has_feature('vsphere'), "standalone env has the vsphere feature"); - ok($env->has_feature('second-feature'), "standalone env has the second-feature feature"); - ok(!$env->has_feature('xyzzy'), "standalone env doesn't have the xyzzy feature"); - throws_ok { $env->use_create_env() } qr/ERROR/; - throws_ok { $env->use_create_env() } - qr/\[ERROR\] This bosh environment does not use create-env \(proto\) or specify.*an alternative genesis.bosh_env/sm, - "bosh environments without specifying bosh_env require bosh create-env"; - - put_file $top->path("regular-deploy.yml"), <load_env('regular-deploy') } - "Genesis::Env should be able to load the `regular-deploy' environment."; - ok($env->has_feature('vsphere'), "regular-deploy env has the vsphere feature"); - quietly { throws_ok { $env->use_create_env() } - qr/\[ERROR\] This bosh environment specifies an alternative bosh_env, but is\n marked as a create-env \(proto\) environment./sm, - "bosh environments with bosh_env can't be a protobosh, or vice versa"; - }; - - teardown_vault(); -}; - -subtest 'manifest generation' => sub { - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - write_bosh_config 'standalone'; - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL); - $top->link_dev_kit('t/src/fancy'); - put_file $top->path('standalone.yml'), <load_env('standalone'); - cmp_deeply([$env->kit_files], [qw[ - base.yml - addons/whiskey.yml - addons/tango.yml - addons/foxtrot.yml - ]], "env gets the correct kit yaml files to merge"); - cmp_deeply([$env->potential_environment_files], [qw[ - ./standalone.yml - ]], "env formulates correct potential environment files to merge"); - cmp_deeply([$env->actual_environment_files], [qw[ - ./standalone.yml - ]], "env detects correct actual environment files to merge"); - - dies_ok { $env->manifest; } "should not be able to merge an env without a cloud-config"; - - - put_file $top->path(".cloud.yml"), <use_config($top->path(".cloud.yml"))->manifest; } - "should be able to merge an env with a cloud-config"; - - my $mfile = $top->path(".manifest.yml"); - my ($manifest, undef) = $env->_manifest(redact => 0); - $env->write_manifest($mfile, prune => 0); - ok -f $mfile, "env->write_manifest should actually write the file"; - my $mcontents; - lives_ok { $mcontents = load_yaml_file($mfile) } 'written manifest (unpruned) is valid YAML'; - cmp_deeply($mcontents, $manifest, "written manifest (unpruned) matches the raw unpruned manifest"); - cmp_deeply($mcontents, { - name => ignore, - fancy => ignore, - addons => ignore, - exodus => ignore, - genesis=> ignore, - kit => ignore, - meta => ignore, - params => ignore - }, "written manifest (unpruned) contains all the keys"); - - ok $env->manifest_lookup('addons.foxtrot'), "env manifest defines addons.foxtrot"; - is $env->manifest_lookup('addons.bravo', 'MISSING'), 'MISSING', - "env manifest doesn't define addons.bravo"; - - teardown_vault(); -}; - -subtest 'multidoc env files' => sub { - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL); - $top->link_dev_kit('t/src/fancy'); - put_file $top->path('standalone.yml'), <<'EOF'; ---- -kit: - name: dev - version: latest - features: - - whiskey - - tango - - foxtrot - -params: - env: standalone - secret: (( vault $GENESIS_SECRETS_BASE "test:secret" )) - network: (( grab networks[0].name )) - junk: (( vault "secret/passcode" )) - ---- -genesis: - env: (( grab params.env )) - -kit: - features: - - (( replace )) - - oscar ---- -params: - env: (( prune )) - -kit: - features: - - (( append )) - - kilo -EOF - - my $env = $top->load_env('standalone'); - cmp_deeply([$env->params], [{ - kit => { - features => [ "oscar", "kilo" ], - name => "dev", - version => "latest" - }, - genesis => { - env => "standalone" - }, - params => { - junk => '(( vault "secret/passcode" ))', - network => '(( grab networks.0.name ))', - secret => '(( vault $GENESIS_SECRETS_BASE "test:secret" ))', - } - }], "env contains the parameters from all document pages"); - cmp_deeply([$env->kit_files], [qw[ - base.yml - addons/oscar.yml - addons/kilo.yml - ]], "env gets the correct kit yaml files to merge"); - cmp_deeply([$env->potential_environment_files], [qw[ - ./standalone.yml - ]], "env formulates correct potential environment files to merge"); - cmp_deeply([$env->actual_environment_files], [qw[ - ./standalone.yml - ]], "env detects correct actual environment files to merge"); - - put_file $top->path('standalone.yml'), <<'EOF'; ---- -kit: - name: dev - version: latest - features: - - whiskey - - tango - - foxtrot - -params: - env: standalone - ---- -genesis: - env: (( grab params.env )) - -kit: - features: - - (( replace )) - - oscar ---- -params: - env: (( prune )) - -kit: - features: - - (( append )) - - kilo -EOF - - # Get rid of the unparsable value that would prevent manifest generation - $env = $top->load_env('standalone'); - - my $mfile = $top->path(".manifest.yml"); - my ($manifest, undef) = $env->_manifest(redact => 0); - $env->write_manifest($mfile, prune => 0); - ok -f $mfile, "env->write_manifest should actually write the file"; - my $mcontents; - lives_ok { $mcontents = load_yaml_file($mfile) } 'written manifest (unpruned) is valid YAML'; - cmp_deeply($mcontents, $manifest, "written manifest (unpruned) matches the raw unpruned manifest"); - cmp_deeply($mcontents, { - name => ignore, - fancy => ignore, - addons => ignore, - meta => ignore, - params => ignore, - exodus => ignore, - genesis=> superhashof({ - env => "standalone", - }), - kit => { - name => ignore, - version => ignore, - features => [ "oscar", "kilo" ], - }, - }, "written manifest (unpruned) contains all the keys"); - - ok $env->manifest_lookup('addons.kilo'), "env manifest defines addons.kilo"; - is $env->manifest_lookup('addons.foxtrot', 'MISSING'), 'MISSING', - "env manifest doesn't define addons.foxtrot"; - - teardown_vault(); -}; - -subtest 'manifest pruning' => sub { - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL); - $top->link_dev_kit('t/src/fancy'); - put_file $top->path(".cloud.yml"), <path('standalone.yml'), <load_env('standalone')->use_config($top->path('.cloud.yml')); - - cmp_deeply(scalar load_yaml($env->manifest(prune => 0)), { - name => ignore, - fancy => ignore, - addons => ignore, - - # Genesis stuff - meta => ignore, - pipeline => ignore, - params => ignore, - exodus => ignore, - genesis => ignore, - kit => superhashof({ name => 'dev' }), - - # cloud-config - resource_pools => ignore, - vm_types => ignore, - disk_pools => ignore, - disk_types => ignore, - networks => ignore, - azs => ignore, - vm_extensions => ignore, - compilation => ignore, - - }, "unpruned manifest should have all the top-level keys"); - - cmp_deeply(scalar load_yaml($env->manifest(prune => 1)), { - name => ignore, - fancy => ignore, - addons => ignore, - }, "pruned manifest should not have all the top-level keys"); - - my $mfile = $top->path(".manifest.yml"); - my ($manifest, undef) = $env->_manifest(redact => 0); - $env->write_manifest($mfile); - ok -f $mfile, "env->write_manifest should actually write the file"; - my $mcontents; - lives_ok { $mcontents = load_yaml_file($mfile) } 'written manifest is valid YAML'; - cmp_deeply($mcontents, subhashof($manifest), "written manifest content matches unpruned manifest for values that weren't pruned"); - cmp_deeply($mcontents, { - name => ignore, - fancy => ignore, - addons => ignore, - }, "written manifest doesn't contain the pruned keys (no cloud-config)"); - teardown_vault(); -}; - -subtest 'manifest pruning (custom bosh create-env)' => sub { - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL); - $top->link_dev_kit('t/src/custom-bosh'); - put_file $top->path(".cloud.yml"), <path('proto.yml'), <load_env('proto')->use_config($top->path('.cloud.yml')); - ok $env->use_create_env, "'proto' test env needs create-env"; - cmp_deeply(scalar load_yaml($env->manifest(prune => 0)), { - name => ignore, - fancy => ignore, - addons => ignore, - - # Genesis stuff - meta => ignore, - pipeline => ignore, - params => ignore, - exodus => ignore, - genesis => ignore, - kit => superhashof({ name => 'dev' }), - - # BOSH stuff - compilation => ignore, - - # "cloud-config" - resource_pools => ignore, - vm_types => ignore, - disk_pools => ignore, - disk_types => ignore, - networks => ignore, - azs => ignore, - vm_extensions => ignore, - - }, "unpruned proto-style manifest should have all the top-level keys"); - - cmp_deeply(scalar load_yaml($env->manifest(prune => 1)), { - name => ignore, - fancy => ignore, - addons => ignore, - - # "cloud-config" - resource_pools => ignore, - vm_types => ignore, - disk_pools => ignore, - disk_types => ignore, - networks => ignore, - azs => ignore, - vm_extensions => ignore, - }, "pruned proto-style manifest should retain 'cloud-config' keys, since create-env needs them"); - - my $mfile = $top->path(".manifest-create-env.yml"); - my ($manifest, undef) = $env->_manifest(redact => 0); - $env->write_manifest($mfile); - ok -f $mfile, "env->write_manifest should actually write the file"; - my $mcontents; - lives_ok { $mcontents = load_yaml_file($mfile) } 'written manifest for bosh-create-env is valid YAML'; - cmp_deeply($mcontents, subhashof($manifest), "written manifest for bosh-create-env content matches unpruned manifest for values that weren't pruned"); - cmp_deeply($mcontents, { - name => ignore, - fancy => ignore, - addons => ignore, - - # "cloud-config" - resource_pools => ignore, - vm_types => ignore, - disk_pools => ignore, - disk_types => ignore, - networks => ignore, - azs => ignore, - vm_extensions => ignore, - }, "written manifest for bosh-create-env doesn't contain the pruned keys (includes cloud-config)"); - - teardown_vault(); -}; - -subtest 'exodus data' => sub { - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL); - $top->link_dev_kit('t/src/fancy'); - put_file $top->path('standalone.yml'), <path(".cloud.yml"), <load_env('standalone')->use_config($top->path('.cloud.yml')); - cmp_deeply($env->exodus_data, { - version => ignore, - dated => re(qr/\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d/), - deployer => ignore, - bosh => 'standalone', - is_director => JSON::PP::false, - use_create_env => JSON::PP::false, - kit_name => 'fancy', - kit_version => '0.0.0-rc0', - kit_is_dev => JSON::PP::true, - 'addons[0]' => 'echo', - vault_base => '/secret/standalone/thing', - features => 'echo', - - 'hello.world' => 'i see you', - - # we allow multi-level arrays now - 'multilevel.arrays[0]' => 'so', - 'multilevel.arrays[1]' => 'useful', - - # we allow multi-level maps now - 'three.levels.works' => 'now', - 'three.levels.or.more.is.right' => 'on, man!', - }, "env manifest can provide exodus with flattened keys"); - - my $good_flattened = { - key => "value", - another_key => "another value", - - # flattened hash - 'this.is.a.test' => '100%', - 'this.is.a.dog' => 'woof', - 'this.is.sparta' => 300, - - # flattened array - 'matrix[0][0]' => -2, - 'matrix[0][1]' => 4, - 'matrix[1][0]' => 2, - 'matrix[1][1]' => -4, - - # flattened array of hashes - 'network[0].name' => 'default', - 'network[0].subnet' => '10.0.0.0/24', - 'network[1].name' => 'super-special', - 'network[1].subnet' => '10.0.1.0/24', - 'network[2].name' => 'secret', - 'network[2].subnet' => '10.0.2.0/24', - }; - - - cmp_deeply(Genesis::Env::_unflatten($good_flattened), { - key => "value", - another_key => "another value", - this => { - is => { - a => { - test => '100%', - dog => 'woof', - }, - sparta => 300, - } - }, - matrix => [ - [-2, 4], - [ 2,-4] - ], - network => [ - { - name => 'default', - subnet => '10.0.0.0/24', - }, { - name => 'super-special', - subnet => '10.0.1.0/24', - }, { - name => 'secret', - subnet => '10.0.2.0/24', - } - ] - }, "exodus data can be correctly unflattened"); - - teardown_vault(); -}; - -subtest 'cloud_config_and_deployment' => sub{ - local $ENV{GENESIS_BOSH_COMMAND}; - my ($director1) = fake_bosh_directors( - {alias => 'standalone'}, - ); - fake_bosh; - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - `safe set --quiet secret/code word='penguin'`; - `safe set --quiet secret/standalone/thing/admin password='drowssap'`; - - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL)->link_dev_kit('t/src/fancy'); - put_file $top->path('standalone.yml'), <set_command($ENV{GENESIS_BOSH_COMMAND}); - my $env = $top->load_env('standalone'); - quietly { lives_ok { $env->download_configs('cloud@genesis-test'); } - "download_cloud_config runs correctly"; - }; - - ok -f $env->config_file('cloud','genesis-test'), "download_cloud_config created cc file"; - eq_or_diff get_file($env->config_file('cloud','genesis-test')), <config_file('cloud'), <lookup("something","goose"), "goose", "Environment doesn't contain cloud config details"); - is($env->manifest_lookup("something","goose"), "penguin", "Manifest contains cloud config details"); - my ($manifest_file, $exists, $sha1) = $env->cached_manifest_info; - ok $manifest_file eq $env->path(".genesis/manifests/".$env->name.".yml"), "cached manifest path correctly determined"; - ok ! $exists, "manifest file doesn't exist."; - ok ! defined($sha1), "sha1 sum for manifest not computed."; - my ($stdout, $stderr) = output_from {$env->deploy(canaries => 2, "max-in-flight" => 5);}; - eq_or_diff($stdout, <{__tmp}/manifest.yml -EOF - - ($manifest_file, $exists, $sha1) = $env->cached_manifest_info; - ok $manifest_file eq $env->path(".genesis/manifests/".$env->name.".yml"), "cached manifest path correctly determined"; - ok $exists, "manifest file should exist."; - ok $sha1 =~ /[a-f0-9]{40}/, "cached manifest calculates valid SHA-1 checksum"; - ok -f $manifest_file, "deploy created cached redacted manifest file"; - - # Compare the raw exodus data - # - runs_ok('safe exists "secret/exodus/standalone/thing"', 'exodus entry created in vault'); - my ($pass, $rc, $out) = runs_ok('safe get "secret/exodus/standalone/thing" | spruce json #'); - my $exodus = load_json($out); - local %ENV = %ENV; - $ENV{USER} ||= 'unknown'; - cmp_deeply($exodus, { - dated => re(qr/\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d \+0000/), - deployer => $ENV{USER}, - kit_name => "fancy", - kit_version => "0.0.0-rc0", - kit_is_dev => 1, - features => '', - bosh => "standalone", - is_director => 0, - use_create_env => 0, - vault_base => "/secret/standalone/thing", - version => '(development)', - manifest_sha1 => $sha1, - 'hello.world' => 'i see you', - 'multilevel.arrays[0]' => 'so', - 'multilevel.arrays[1]' => 'useful', - 'three.levels.or.more.is.right' => 'on, man!', - 'three.levels.works' => 'now' - }, "exodus data was written by deployment"); - - is($env->last_deployed_lookup("something","goose"), "REDACTED", "Cached manifest contains redacted vault details"); - is($env->last_deployed_lookup("fancy.status","none"), "online", "Cached manifest contains non-redacted params"); - is($env->last_deployed_lookup("genesis.env","none"), "standalone", "Cached manifest contains pruned params"); - cmp_deeply(scalar($env->exodus_lookup("",{})), { - dated => $exodus->{dated}, - deployer => $ENV{USER}, - bosh => "standalone", - is_director => 0, - use_create_env => 0, - kit_name => "fancy", - kit_version => "0.0.0-rc0", - kit_is_dev => 1, - features => '', - vault_base => "/secret/standalone/thing", - version => '(development)', - manifest_sha1 => $sha1, - hello => { - world => 'i see you' - }, - multilevel => { - arrays => ['so','useful'] - }, - three => { - levels => { - 'or' => { more => {is => {right => 'on, man!'}}}, - 'works' => 'now' - } - } - }, "exodus data was written by deployment"); - - $director1->stop(); - teardown_vault(); -}; -subtest 'bosh variables' => sub { - local $ENV{GENESIS_BOSH_COMMAND}; - fake_bosh; - - my ($director1) = fake_bosh_directors( - {alias => 'standalone'}, - ); - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - Genesis::BOSH->set_command($ENV{GENESIS_BOSH_COMMAND}); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL)->link_dev_kit('t/src/fancy'); - put_file $top->path("standalone.yml"), <load_env('standalone'); - quietly { lives_ok { $env->download_configs('cloud'); } - "download_cloud_config runs correctly"; - }; - - put_file $env->config_file('cloud'), <vars_file(); - my ($stdout, $stderr) = output_from {eval {$env->deploy();}}; - eq_or_diff($stdout, <{__tmp}/manifest.yml -EOF - - eq_or_diff get_file($env->vars_file), < sub{ - local $ENV{GENESIS_BOSH_COMMAND}; - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - - my $name = "far-fetched"; - write_bosh_config $name; - my $top = Genesis::Top->create(workdir, 'sample', vault=>$VAULT_URL); - my $kit = $top->link_dev_kit('t/src/creator')->local_kit_version('dev'); - mkfile_or_fail $top->path("pre-existing.yml"), "I'm already here"; - - # create the environment - quietly {dies_ok {$top->create_env('', $kit)} "can't create a unnamed env"; }; - quietly {dies_ok {$top->create_env("nothing")} "can't create a env without a kit"; }; - quietly {dies_ok {$top->create_env("pre-existing", $kit)} "can't overwrite a pre-existing env"; }; - - my $env; - local $ENV{NOCOLOR} = "yes"; - local $ENV{PRY} = "1"; - my ($director1) = fake_bosh_directors( - {alias => $name}, - ); - fake_bosh; - Genesis::BOSH->set_command($ENV{GENESIS_BOSH_COMMAND}); - my $out; - lives_ok { - $out = combined_from {$env = $top->create_env($name, $kit, vault => $vault_target)} - } "successfully create an env with a dev kit"; - - $out =~ s/(Duration:|-) (\d+ minutes, )?\d+ seconds?/$1 XXX seconds/g; - eq_or_diff $out, <path($env->{file})), <check_secrets(verbose => 1), "check_secrets shows all secrets okay" - }; - $out =~ s/(Duration:|-) (\d+ minutes, )?\d+ seconds?/$1 XXX seconds/g; - - eq_or_diff $out, < /tmp/out.json); - - qx(safe rm -rf secret/far/fetched/sample/users); - qx(safe rm secret/far/fetched/sample/ssl/ca:key secret/far/fetched/sample/ssl/ca:certificate); - qx(safe rm secret/far/fetched/sample/crazy/thing:token); - - $out = combined_from { - ok !$env->check_secrets(verbose=>1), "check_secrets shows missing secrets and keys" - }; - $out =~ s/(Duration:|-) (\d+ minutes, )?\d+ seconds?/$1 XXX seconds/g; - - matches_utf8 $out, <stop(); - teardown_vault(); -}; - -subtest 'env_kit_overrides' => sub { - local $ENV{GENESIS_BOSH_COMMAND}; - fake_bosh; - - my ($director1) = fake_bosh_directors( - {alias => 'standalone'}, - ); - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - Genesis::BOSH->set_command($ENV{GENESIS_BOSH_COMMAND}); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL)->link_dev_kit('t/src/creator'); - put_file $top->path("standalone.yml"), <load_env('standalone'); - ok ! $env->use_create_env(), "env does not use create-env"; - - # check env override count and content - my @override_files = $env->kit->env_override_files(); - ok scalar(@override_files) == 1, "there is one environment kit override file"; - ok $override_files[0] =~ /\/env-overrides-0.yml$/, "override file is named correctly"; - eq_or_diff slurp($override_files[0]), <add_secrets() } - } "successfully add secrets with environment kit overrides"; - - $out =~ s/(Duration:|-) (\d+ minutes, )?\d+ seconds?/$1 XXX seconds/g; - eq_or_diff $out, <check_secrets(verbose => 1, validate => 1) } - } "successfully check secrets with environment kit overrides"; - $out =~ s/(Duration:|-) (\d+ minutes, )?\d+ seconds?/$1 XXX seconds/g; - $out =~ s/expires in (\d+) days \(([^\)]+)\)/expires in $1 days ()/g; - matches_utf8 $out, <) - [✔ ] Modulus Agreement - [✔ ] Default CA key usage: server_auth, client_auth, crl_sign, key_cert_sign - - [ 2/10] my-cert/server X509 certificate - signed by 'my-cert/ca' ... valid. - [✔ ] Not a CA Certificate - [✔ ] Signed by my-cert/ca - [✔ ] Valid: expires in 365 days () - [✔ ] Modulus Agreement - [✔ ] Subject Name 'locker' - [✔ ] Subject Alt Names: 'locker' - [✔ ] Default key usage: server_auth, client_auth - - [ 3/10] private-cert/server X509 certificate - signed by 'my-cert/ca' ... valid. - [✔ ] Not a CA Certificate - [✔ ] Signed by my-cert/ca - [✔ ] Valid: expires in 365 days () - [✔ ] Modulus Agreement - [✔ ] Subject Name 'standalone' - [✔ ] Subject Alt Names: 'standalone' - [✔ ] Default key usage: server_auth, client_auth - - [ 4/10] crazy/thing:id random password - 32 bytes, fixed ... valid. - [✔ ] 32 characters - - [ 5/10] crazy/thing:token random password - 48 bytes ... valid. - [✔ ] 48 characters - [✔ ] Only uses characters 'ABCDEF0123456789' - - [ 6/10] need-to-know:secret random password - 32 bytes ... valid. - [✔ ] 32 characters - - [ 7/10] users/admin:password random password - 64 bytes ... valid. - [✔ ] 64 characters - - [ 8/10] users/bob:password random password - 16 bytes ... valid. - [✔ ] 16 characters - - [ 9/10] work/signing_key RSA public/private keypair - 2048 bits, fixed ... valid. - [✔ ] Valid private key - [✔ ] Valid public key - [✔ ] Public/Private key agreement - [✔ ] 2048 bit - - [10/10] something/ssh SSH public/private keypair - 2048 bits, fixed ... valid. - [✔ ] Valid private key - [✔ ] Valid public key - [✔ ] Public/Private key Agreement - [✔ ] 2048 bits - -Completed - Duration: XXX seconds [10 validated/0 skipped/0 errors] - -EOF - - put_file $top->path("c.yml"), <path("c-env.yml"), <load_env('c-env'); - ok $env->use_create_env(), "env uses create-env (v2.8.0 method)"; - - # check env override count and content - @override_files = $env->kit->env_override_files(); - ok scalar(@override_files) == 2, "there is one environment kit override file"; - ok $override_files[0] =~ /\/env-overrides-0.yml$/, "first override file is named correctly"; - eq_or_diff slurp($override_files[0]), <add_secrets() } - } "successfully add secrets with environment kit overrides"; - - $out =~ s/(Duration:|-) (\d+ minutes, )?\d+ seconds?/$1 XXX seconds/g; - eq_or_diff $out, <check_secrets(verbose => 1, validate => 1) } - } "successfully check secrets with environment kit overrides"; - $out =~ s/(Duration:|-) (\d+ minutes, )?\d+ seconds?/$1 XXX seconds/g; - $out =~ s/expires in (\d+) days \(([^\)]+)\)/expires in $1 days ()/g; - matches_utf8 $out, <) - [✔ ] Modulus Agreement - [✔ ] Default CA key usage: server_auth, client_auth, crl_sign, key_cert_sign - - [ 2/16] my-cert/server X509 certificate - signed by 'my-cert/ca' ... valid. - [✔ ] Not a CA Certificate - [✔ ] Signed by my-cert/ca - [✔ ] Valid: expires in 365 days () - [✔ ] Modulus Agreement - [✔ ] Subject Name 'locker' - [✔ ] Subject Alt Names: 'locker' - [✔ ] Default key usage: server_auth, client_auth - - [ 3/16] some-ssl/ca X509 certificate - CA, self-signed ... valid. - [✔ ] CA Certificate - [✔ ] Self-Signed - [✔ ] Valid: expires in 730 days () - [✔ ] Modulus Agreement - [✔ ] Default CA key usage: server_auth, client_auth, crl_sign, key_cert_sign - - [ 4/16] some-ssl/server X509 certificate - signed by 'some-ssl/ca' ... valid. - [✔ ] Not a CA Certificate - [✔ ] Signed by some-ssl/ca - [✔ ] Valid: expires in 365 days () - [✔ ] Modulus Agreement - [✔ ] Subject Name 'proto-ssl' - [✔ ] Subject Alt Names: 'proto-ssl' - [✔ ] Default key usage: server_auth, client_auth - - [ 5/16] ssl/ca X509 certificate - CA, self-signed ... valid. - [✔ ] CA Certificate - [✔ ] Self-Signed - [✔ ] Valid: expires in 3650 days () - [✔ ] Modulus Agreement - [✔ ] Default CA key usage: server_auth, client_auth, crl_sign, key_cert_sign - - [ 6/16] ssl/server X509 certificate - signed by 'ssl/ca' ... valid. - [✔ ] Not a CA Certificate - [✔ ] Signed by ssl/ca - [✔ ] Valid: expires in 365 days () - [✔ ] Modulus Agreement - [✔ ] Subject Name 'bonus.ci' - [✔ ] Subject Alt Names: 'bonus.ci' - [✔ ] Default key usage: server_auth, client_auth - - [ 7/16] iaas:access_key user-provided - IaaS Access Key ... found. - - [ 8/16] iaas:secret_key user-provided - IaaS Secret Key ... found. - - [ 9/16] crazy/thing:id random password - 32 bytes, fixed ... valid. - [✔ ] 32 characters - - [10/16] crazy/thing:token random password - 16 bytes ... valid. - [✔ ] 16 characters - [✔ ] Only uses characters 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0987654321' - - [11/16] proto-credentials:seed random password - 64 bytes, fixed ... valid. - [✔ ] 64 characters - - [12/16] proto-credentials:token random password - 24 bytes ... valid. - [✔ ] 24 characters - [✔ ] Formatted as base64 in ':token-base64' - - [13/16] users/admin:password random password - 64 bytes ... valid. - [✔ ] 64 characters - - [14/16] users/bob:password random password - 16 bytes ... valid. - [✔ ] 16 characters - - [15/16] work/signing_key RSA public/private keypair - 2048 bits, fixed ... valid. - [✔ ] Valid private key - [✔ ] Valid public key - [✔ ] Public/Private key agreement - [✔ ] 2048 bit - - [16/16] something/ssh SSH public/private keypair - 2048 bits, fixed ... valid. - [✔ ] Valid private key - [✔ ] Valid public key - [✔ ] Public/Private key Agreement - [✔ ] 2048 bits - -Completed - Duration: XXX seconds [16 validated/0 skipped/0 errors] - -EOF - $director1->stop(); - teardown_vault(); -}; - -subtest 'load environment from env vars' => sub { - local $ENV{GENESIS_BOSH_COMMAND}; - fake_bosh; - - my ($director1) = fake_bosh_directors( - {alias => 'standalone'}, - ); - my $vault_target = vault_ok; - Genesis::Vault->clear_all(); - Genesis::BOSH->set_command($ENV{GENESIS_BOSH_COMMAND}); - my $top = Genesis::Top->create(workdir, 'thing', vault=>$VAULT_URL)->link_dev_kit('t/src/creator'); - put_file $top->path("base.yml"), <<'EOF'; # Direct YAML ---- -kit: - name: dev - version: latest - features: - - whiskey - - tango - - foxtrot - -genesis: - env: base - -params: - secret: (( vault $GENESIS_SECRETS_BASE "test:secret" )) - network: (( grab networks[0].name )) - junk: (( vault "secret/passcode" )) - -EOF - -put_file $top->path("base-extended.yml"), <<'EOF'; # Direct YAML - -kit: - features: - - (( replace )) - - alpha - - oscar - - kilo - ---- - -genesis: - env: base-extended - ci_base: (( concat "/concourse/main/" genesis.env )) - ci_mount: "/concourse" - root_ca_path: "company/root-ca" - secrets_mount: /shhhh/ - credhub_env: "root_vault/credhub" - -kit: - features: - - (( append )) - - november - ---- -genesis: - min_version: 2.8.0 - use_create_env: true - -kit: - overrides: - genesis_version_min: 2.8.0 - use_create_env: allow - certificates: - base: - private-cert: # Additional cert signed by existing CA - server: - signed_by: "my-cert/ca" - valid_for: (( defer grab certificates.base.my-cert.server.valid_for )) # Grab from kit.yml - names: [ (( grab genesis.env )) ] # Grab from env file - bonus: ~ # Deletion - credentials: - bonus: - need-to-know: - secret: random 32 #New - crazy/thing: - token: random 48 allowed-chars ABCDEF0123456789 # Update - -EOF - - my $env = $top->load_env('base-extended')->with_bosh()->with_vault(); - ok $env->use_create_env(), "env does use create-env"; - my %evs; ok %evs = $env->get_environment_variables(), "env can provide environment variables"; - - # validate expected environment variables and values - cmp_deeply(\%evs, { - GENESIS_ROOT => $env->path, - GENESIS_ENVIRONMENT => $env->name, - GENESIS_TYPE => $top->type, - GENESIS_CALL_BIN => Genesis::humanize_bin(), - GENESIS_CALL => "", - GENESIS_CI_BASE => "/concourse/main/".$env->name."/", - GENESIS_CI_MOUNT => "/concourse/", - GENESIS_CI_MOUNT_OVERRIDE => "true", - GENESIS_CREDHUB_EXODUS_SOURCE => "root_vault/credhub", - GENESIS_CREDHUB_EXODUS_SOURCE_OVERRIDE => "root_vault/credhub", # Shouldn't this be boolean? - GENESIS_CREDHUB_ROOT => "root_vault-credhub/base-extended-thing", - GENESIS_ENV_KIT_OVERRIDE_FILES => re('\/(var\/folders|tmp)\/.*\/env-overrides-0.yml'), - GENESIS_EXODUS_BASE => "/shhhh/exodus/base-extended/thing", - GENESIS_EXODUS_MOUNT => "/shhhh/exodus/", - GENESIS_EXODUS_MOUNT_OVERRIDE => "", - GENESIS_KIT_NAME => "dev", - GENESIS_KIT_VERSION => "latest", # THIS IS NOT IDEAL - GENESIS_MIN_VERSION => '2.8.0', - GENESIS_REQUESTED_FEATURES => "alpha oscar kilo november", - GENESIS_ROOT_CA_PATH => "company/root-ca", - GENESIS_SECRETS_BASE => "/shhhh/base/extended/thing/", - GENESIS_SECRETS_MOUNT => "/shhhh/", - GENESIS_SECRETS_MOUNT_OVERRIDE => "true", - GENESIS_SECRETS_PATH => "base/extended/thing", - GENESIS_SECRETS_SLUG => "base/extended/thing", - GENESIS_SECRETS_SLUG_OVERRIDE => "", - GENESIS_TARGET_VAULT => re('http://127.0.0.1:82\d\d'), - GENESIS_USE_CREATE_ENV => "1", - GENESIS_VAULT_PREFIX => "base/extended/thing", - GENESIS_VERIFY_VAULT => "1", - SAFE_TARGET => re('http://127.0.0.1:82\d\d'), - GENESIS_ENVIRONMENT_PARAMS => re('^{.*}$') - }, "environment provides the correct environment variables and values"); - - # Remove env files - unlink $top->path($_) for (@{$env->{__actual_environment_files}}); - - local %ENV = %ENV; - $ENV{$_} = $evs{$_} for (keys %evs); - dies_ok {my $env_from_evs=Genesis::Env->from_envvars($top)} "cannot load environment from env vars outside of callback"; - - $ENV{GENESIS_IS_HELPING_YOU} = 1; - $ENV{GENESIS_KIT_HOOK}="addon"; - dies_ok {my $env_from_evs=Genesis::Env->from_envvars($top)} "cannot load environment from env vars during a non-new hook"; - - $ENV{GENESIS_KIT_HOOK}="new"; - ok my $env_from_evs=Genesis::Env->from_envvars($top), "can load environment from env vars during a new hook"; - ok $env_from_evs->use_create_env, "env from env vars uses create env."; - ok $env_from_evs->{is_from_envvars}, "env from env vars indicates so."; - - my @old_properties = grep {$_ !~ /^(__actual_environment_files)$/} keys(%$env); - my @new_properties = grep {$_ !~ /^(is_from_envvars)$/} keys(%$env_from_evs); - cmp_set(\@new_properties, \@old_properties, "original and from_envvars environments have the same properties"); - - - for my $property (@old_properties) { - if ($property eq '__bosh') { - eq_or_diff ref($env->{__bosh}), ref($env_from_evs->{__bosh}), "reconstituted correct bosh director"; - } elsif ($property eq '__params') { - # Tweak some known acceptable differences - $env->{__params}{genesis}{use_create_env} = $env->{__params}{genesis}{use_create_env} ? 1 : 0; - cmp_deeply($env_from_evs->{__params},$env->{__params}, "reconstituted correct parameters"); - } elsif ($property eq '__tmp') { - eq_or_diff dirname($env->{__tmp}), dirname($env_from_evs->{__tmp}), 'reconstituted similar tmp dirs'; - } elsif ($property eq 'kit') { - eq_or_diff $env_from_evs->{path}, $env->{path}, 'reconstituted same kit'; - cmp_deeply($env_from_evs->{kit}{__metadata}, $env->{kit}{__metadata}, 'reconstituted same kit configuration'); - } elsif ($property eq '__features') { - cmp_set($env_from_evs->{__features}, $env->{__features}, 'reconstituted the same features'); - } else { - eq_or_diff $env_from_evs->{$property}, $env->{$property}, "reconstituted the correct value for $property"; - } - } - - $director1->stop(); - teardown_vault(); -}; - -done_testing;