diff --git a/command/agent/config.go b/command/agent/config.go index d730bb4c582..dea78106581 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -24,346 +24,372 @@ import ( ) // Config is the configuration for the Nomad agent. +// +// time.Duration values have two parts: +// - a string field tagged with an hcl:"foo" and json:"-" +// - a time.Duration field in the same struct and a call to duration +// in config_parse.go ParseConfigFile +// +// All config structs should have an ExtraKeysHCL field to check for +// unexpected keys type Config struct { // Region is the region this agent is in. Defaults to global. - Region string `mapstructure:"region"` + Region string `hcl:"region"` // Datacenter is the datacenter this agent is in. Defaults to dc1 - Datacenter string `mapstructure:"datacenter"` + Datacenter string `hcl:"datacenter"` // NodeName is the name we register as. Defaults to hostname. - NodeName string `mapstructure:"name"` + NodeName string `hcl:"name"` // DataDir is the directory to store our state in - DataDir string `mapstructure:"data_dir"` + DataDir string `hcl:"data_dir"` // PluginDir is the directory to lookup plugins. - PluginDir string `mapstructure:"plugin_dir"` + PluginDir string `hcl:"plugin_dir"` // LogLevel is the level of the logs to put out - LogLevel string `mapstructure:"log_level"` + LogLevel string `hcl:"log_level"` // LogJson enables log output in a JSON format - LogJson bool `mapstructure:"log_json"` + LogJson bool `hcl:"log_json"` // BindAddr is the address on which all of nomad's services will // be bound. If not specified, this defaults to 127.0.0.1. - BindAddr string `mapstructure:"bind_addr"` + BindAddr string `hcl:"bind_addr"` // EnableDebug is used to enable debugging HTTP endpoints - EnableDebug bool `mapstructure:"enable_debug"` + EnableDebug bool `hcl:"enable_debug"` // Ports is used to control the network ports we bind to. - Ports *Ports `mapstructure:"ports"` + Ports *Ports `hcl:"ports"` // Addresses is used to override the network addresses we bind to. // // Use normalizedAddrs if you need the host+port to bind to. - Addresses *Addresses `mapstructure:"addresses"` + Addresses *Addresses `hcl:"addresses"` // normalizedAddr is set to the Address+Port by normalizeAddrs() normalizedAddrs *Addresses // AdvertiseAddrs is used to control the addresses we advertise. - AdvertiseAddrs *AdvertiseAddrs `mapstructure:"advertise"` + AdvertiseAddrs *AdvertiseAddrs `hcl:"advertise"` // Client has our client related settings - Client *ClientConfig `mapstructure:"client"` + Client *ClientConfig `hcl:"client"` // Server has our server related settings - Server *ServerConfig `mapstructure:"server"` + Server *ServerConfig `hcl:"server"` // ACL has our acl related settings - ACL *ACLConfig `mapstructure:"acl"` + ACL *ACLConfig `hcl:"acl"` // Telemetry is used to configure sending telemetry - Telemetry *Telemetry `mapstructure:"telemetry"` + Telemetry *Telemetry `hcl:"telemetry"` // LeaveOnInt is used to gracefully leave on the interrupt signal - LeaveOnInt bool `mapstructure:"leave_on_interrupt"` + LeaveOnInt bool `hcl:"leave_on_interrupt"` // LeaveOnTerm is used to gracefully leave on the terminate signal - LeaveOnTerm bool `mapstructure:"leave_on_terminate"` + LeaveOnTerm bool `hcl:"leave_on_terminate"` // EnableSyslog is used to enable sending logs to syslog - EnableSyslog bool `mapstructure:"enable_syslog"` + EnableSyslog bool `hcl:"enable_syslog"` // SyslogFacility is used to control the syslog facility used. - SyslogFacility string `mapstructure:"syslog_facility"` + SyslogFacility string `hcl:"syslog_facility"` // DisableUpdateCheck is used to disable the periodic update // and security bulletin checking. - DisableUpdateCheck *bool `mapstructure:"disable_update_check"` + DisableUpdateCheck *bool `hcl:"disable_update_check"` // DisableAnonymousSignature is used to disable setting the // anonymous signature when doing the update check and looking // for security bulletins - DisableAnonymousSignature bool `mapstructure:"disable_anonymous_signature"` + DisableAnonymousSignature bool `hcl:"disable_anonymous_signature"` // Consul contains the configuration for the Consul Agent and // parameters necessary to register services, their checks, and // discover the current Nomad servers. - Consul *config.ConsulConfig `mapstructure:"consul"` + Consul *config.ConsulConfig `hcl:"consul"` // Vault contains the configuration for the Vault Agent and // parameters necessary to derive tokens. - Vault *config.VaultConfig `mapstructure:"vault"` + Vault *config.VaultConfig `hcl:"vault"` // NomadConfig is used to override the default config. // This is largely used for testing purposes. - NomadConfig *nomad.Config `mapstructure:"-" json:"-"` + NomadConfig *nomad.Config `hcl:"-" json:"-"` // ClientConfig is used to override the default config. // This is largely used for testing purposes. - ClientConfig *client.Config `mapstructure:"-" json:"-"` + ClientConfig *client.Config `hcl:"-" json:"-"` // DevMode is set by the -dev CLI flag. - DevMode bool `mapstructure:"-"` + DevMode bool `hcl:"-"` // Version information is set at compilation time Version *version.VersionInfo // List of config files that have been loaded (in order) - Files []string `mapstructure:"-"` + Files []string `hcl:"-"` // TLSConfig provides TLS related configuration for the Nomad server and // client - TLSConfig *config.TLSConfig `mapstructure:"tls"` + TLSConfig *config.TLSConfig `hcl:"tls"` // HTTPAPIResponseHeaders allows users to configure the Nomad http agent to // set arbitrary headers on API responses - HTTPAPIResponseHeaders map[string]string `mapstructure:"http_api_response_headers"` + HTTPAPIResponseHeaders map[string]string `hcl:"http_api_response_headers"` // Sentinel holds sentinel related settings - Sentinel *config.SentinelConfig `mapstructure:"sentinel"` + Sentinel *config.SentinelConfig `hcl:"sentinel"` // Autopilot contains the configuration for Autopilot behavior. - Autopilot *config.AutopilotConfig `mapstructure:"autopilot"` + Autopilot *config.AutopilotConfig `hcl:"autopilot"` // Plugins is the set of configured plugins - Plugins []*config.PluginConfig `hcl:"plugin,expand"` + Plugins []*config.PluginConfig `hcl:"plugin"` + + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } // ClientConfig is configuration specific to the client mode type ClientConfig struct { // Enabled controls if we are a client - Enabled bool `mapstructure:"enabled"` + Enabled bool `hcl:"enabled"` // StateDir is the state directory - StateDir string `mapstructure:"state_dir"` + StateDir string `hcl:"state_dir"` // AllocDir is the directory for storing allocation data - AllocDir string `mapstructure:"alloc_dir"` + AllocDir string `hcl:"alloc_dir"` // Servers is a list of known server addresses. These are as "host:port" - Servers []string `mapstructure:"servers"` + Servers []string `hcl:"servers"` // NodeClass is used to group the node by class - NodeClass string `mapstructure:"node_class"` + NodeClass string `hcl:"node_class"` // Options is used for configuration of nomad internals, // like fingerprinters and drivers. The format is: // // namespace.option = value - Options map[string]string `mapstructure:"options"` + Options map[string]string `hcl:"options"` // Metadata associated with the node - Meta map[string]string `mapstructure:"meta"` + Meta map[string]string `hcl:"meta"` // A mapping of directories on the host OS to attempt to embed inside each // task's chroot. - ChrootEnv map[string]string `mapstructure:"chroot_env"` + ChrootEnv map[string]string `hcl:"chroot_env"` // Interface to use for network fingerprinting - NetworkInterface string `mapstructure:"network_interface"` + NetworkInterface string `hcl:"network_interface"` // NetworkSpeed is used to override any detected or default network link // speed. - NetworkSpeed int `mapstructure:"network_speed"` + NetworkSpeed int `hcl:"network_speed"` // CpuCompute is used to override any detected or default total CPU compute. - CpuCompute int `mapstructure:"cpu_total_compute"` + CpuCompute int `hcl:"cpu_total_compute"` // MemoryMB is used to override any detected or default total memory. - MemoryMB int `mapstructure:"memory_total_mb"` + MemoryMB int `hcl:"memory_total_mb"` // MaxKillTimeout allows capping the user-specifiable KillTimeout. - MaxKillTimeout string `mapstructure:"max_kill_timeout"` + MaxKillTimeout string `hcl:"max_kill_timeout"` // ClientMaxPort is the upper range of the ports that the client uses for // communicating with plugin subsystems - ClientMaxPort int `mapstructure:"client_max_port"` + ClientMaxPort int `hcl:"client_max_port"` // ClientMinPort is the lower range of the ports that the client uses for // communicating with plugin subsystems - ClientMinPort int `mapstructure:"client_min_port"` + ClientMinPort int `hcl:"client_min_port"` // Reserved is used to reserve resources from being used by Nomad. This can // be used to target a certain utilization or to prevent Nomad from using a // particular set of ports. - Reserved *Resources `mapstructure:"reserved"` + Reserved *Resources `hcl:"reserved"` // GCInterval is the time interval at which the client triggers garbage // collection - GCInterval time.Duration `mapstructure:"gc_interval"` + GCInterval time.Duration + GCIntervalHCL string `hcl:"gc_interval" json:"-"` // GCParallelDestroys is the number of parallel destroys the garbage // collector will allow. - GCParallelDestroys int `mapstructure:"gc_parallel_destroys"` + GCParallelDestroys int `hcl:"gc_parallel_destroys"` // GCDiskUsageThreshold is the disk usage threshold given as a percent // beyond which the Nomad client triggers GC of terminal allocations - GCDiskUsageThreshold float64 `mapstructure:"gc_disk_usage_threshold"` + GCDiskUsageThreshold float64 `hcl:"gc_disk_usage_threshold"` // GCInodeUsageThreshold is the inode usage threshold beyond which the Nomad // client triggers GC of the terminal allocations - GCInodeUsageThreshold float64 `mapstructure:"gc_inode_usage_threshold"` + GCInodeUsageThreshold float64 `hcl:"gc_inode_usage_threshold"` // GCMaxAllocs is the maximum number of allocations a node can have // before garbage collection is triggered. - GCMaxAllocs int `mapstructure:"gc_max_allocs"` + GCMaxAllocs int `hcl:"gc_max_allocs"` // NoHostUUID disables using the host's UUID and will force generation of a // random UUID. - NoHostUUID *bool `mapstructure:"no_host_uuid"` + NoHostUUID *bool `hcl:"no_host_uuid"` // ServerJoin contains information that is used to attempt to join servers - ServerJoin *ServerJoin `mapstructure:"server_join"` + ServerJoin *ServerJoin `hcl:"server_join"` + + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } // ACLConfig is configuration specific to the ACL system type ACLConfig struct { // Enabled controls if we are enforce and manage ACLs - Enabled bool `mapstructure:"enabled"` + Enabled bool `hcl:"enabled"` // TokenTTL controls how long we cache ACL tokens. This controls // how stale they can be when we are enforcing policies. Defaults // to "30s". Reducing this impacts performance by forcing more // frequent resolution. - TokenTTL time.Duration `mapstructure:"token_ttl"` + TokenTTL time.Duration + TokenTTLHCL string `hcl:"token_ttl" json:"-"` // PolicyTTL controls how long we cache ACL policies. This controls // how stale they can be when we are enforcing policies. Defaults // to "30s". Reducing this impacts performance by forcing more // frequent resolution. - PolicyTTL time.Duration `mapstructure:"policy_ttl"` + PolicyTTL time.Duration + PolicyTTLHCL string `hcl:"policy_ttl" json:"-"` // ReplicationToken is used by servers to replicate tokens and policies // from the authoritative region. This must be a valid management token // within the authoritative region. - ReplicationToken string `mapstructure:"replication_token"` + ReplicationToken string `hcl:"replication_token"` + + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } // ServerConfig is configuration specific to the server mode type ServerConfig struct { // Enabled controls if we are a server - Enabled bool `mapstructure:"enabled"` + Enabled bool `hcl:"enabled"` // AuthoritativeRegion is used to control which region is treated as // the source of truth for global tokens and ACL policies. - AuthoritativeRegion string `mapstructure:"authoritative_region"` + AuthoritativeRegion string `hcl:"authoritative_region"` // BootstrapExpect tries to automatically bootstrap the Consul cluster, // by withholding peers until enough servers join. - BootstrapExpect int `mapstructure:"bootstrap_expect"` + BootstrapExpect int `hcl:"bootstrap_expect"` // DataDir is the directory to store our state in - DataDir string `mapstructure:"data_dir"` + DataDir string `hcl:"data_dir"` // ProtocolVersion is the protocol version to speak. This must be between // ProtocolVersionMin and ProtocolVersionMax. - ProtocolVersion int `mapstructure:"protocol_version"` + ProtocolVersion int `hcl:"protocol_version"` // RaftProtocol is the Raft protocol version to speak. This must be from [1-3]. - RaftProtocol int `mapstructure:"raft_protocol"` + RaftProtocol int `hcl:"raft_protocol"` // NumSchedulers is the number of scheduler thread that are run. // This can be as many as one per core, or zero to disable this server // from doing any scheduling work. - NumSchedulers *int `mapstructure:"num_schedulers"` + NumSchedulers *int `hcl:"num_schedulers"` // EnabledSchedulers controls the set of sub-schedulers that are // enabled for this server to handle. This will restrict the evaluations // that the workers dequeue for processing. - EnabledSchedulers []string `mapstructure:"enabled_schedulers"` + EnabledSchedulers []string `hcl:"enabled_schedulers"` // NodeGCThreshold controls how "old" a node must be to be collected by GC. // Age is not the only requirement for a node to be GCed but the threshold // can be used to filter by age. - NodeGCThreshold string `mapstructure:"node_gc_threshold"` + NodeGCThreshold string `hcl:"node_gc_threshold"` // JobGCThreshold controls how "old" a job must be to be collected by GC. // Age is not the only requirement for a Job to be GCed but the threshold // can be used to filter by age. - JobGCThreshold string `mapstructure:"job_gc_threshold"` + JobGCThreshold string `hcl:"job_gc_threshold"` // EvalGCThreshold controls how "old" an eval must be to be collected by GC. // Age is not the only requirement for a eval to be GCed but the threshold // can be used to filter by age. - EvalGCThreshold string `mapstructure:"eval_gc_threshold"` + EvalGCThreshold string `hcl:"eval_gc_threshold"` // DeploymentGCThreshold controls how "old" a deployment must be to be // collected by GC. Age is not the only requirement for a deployment to be // GCed but the threshold can be used to filter by age. - DeploymentGCThreshold string `mapstructure:"deployment_gc_threshold"` + DeploymentGCThreshold string `hcl:"deployment_gc_threshold"` // HeartbeatGrace is the grace period beyond the TTL to account for network, // processing delays and clock skew before marking a node as "down". - HeartbeatGrace time.Duration `mapstructure:"heartbeat_grace"` + HeartbeatGrace time.Duration + HeartbeatGraceHCL string `hcl:"heartbeat_grace" json:"-"` // MinHeartbeatTTL is the minimum time between heartbeats. This is used as // a floor to prevent excessive updates. - MinHeartbeatTTL time.Duration `mapstructure:"min_heartbeat_ttl"` + MinHeartbeatTTL time.Duration + MinHeartbeatTTLHCL string `hcl:"min_heartbeat_ttl" json:"-"` // MaxHeartbeatsPerSecond is the maximum target rate of heartbeats // being processed per second. This allows the TTL to be increased // to meet the target rate. - MaxHeartbeatsPerSecond float64 `mapstructure:"max_heartbeats_per_second"` + MaxHeartbeatsPerSecond float64 `hcl:"max_heartbeats_per_second"` // StartJoin is a list of addresses to attempt to join when the // agent starts. If Serf is unable to communicate with any of these // addresses, then the agent will error and exit. // Deprecated in Nomad 0.10 - StartJoin []string `mapstructure:"start_join"` + StartJoin []string `hcl:"start_join"` // RetryJoin is a list of addresses to join with retry enabled. // Deprecated in Nomad 0.10 - RetryJoin []string `mapstructure:"retry_join"` + RetryJoin []string `hcl:"retry_join"` // RetryMaxAttempts specifies the maximum number of times to retry joining a // host on startup. This is useful for cases where we know the node will be // online eventually. // Deprecated in Nomad 0.10 - RetryMaxAttempts int `mapstructure:"retry_max"` + RetryMaxAttempts int `hcl:"retry_max"` // RetryInterval specifies the amount of time to wait in between join // attempts on agent start. The minimum allowed value is 1 second and // the default is 30s. // Deprecated in Nomad 0.10 - RetryInterval time.Duration `mapstructure:"retry_interval"` + RetryInterval time.Duration + RetryIntervalHCL string `hcl:"retry_interval" json:"-"` // RejoinAfterLeave controls our interaction with the cluster after leave. // When set to false (default), a leave causes Consul to not rejoin // the cluster until an explicit join is received. If this is set to // true, we ignore the leave, and rejoin the cluster on start. - RejoinAfterLeave bool `mapstructure:"rejoin_after_leave"` + RejoinAfterLeave bool `hcl:"rejoin_after_leave"` // (Enterprise-only) NonVotingServer is whether this server will act as a // non-voting member of the cluster to help provide read scalability. - NonVotingServer bool `mapstructure:"non_voting_server"` + NonVotingServer bool `hcl:"non_voting_server"` // (Enterprise-only) RedundancyZone is the redundancy zone to use for this server. - RedundancyZone string `mapstructure:"redundancy_zone"` + RedundancyZone string `hcl:"redundancy_zone"` // (Enterprise-only) UpgradeVersion is the custom upgrade version to use when // performing upgrade migrations. - UpgradeVersion string `mapstructure:"upgrade_version"` + UpgradeVersion string `hcl:"upgrade_version"` // Encryption key to use for the Serf communication - EncryptKey string `mapstructure:"encrypt" json:"-"` + EncryptKey string `hcl:"encrypt" json:"-"` // ServerJoin contains information that is used to attempt to join servers - ServerJoin *ServerJoin `mapstructure:"server_join"` + ServerJoin *ServerJoin `hcl:"server_join"` + + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } // ServerJoin is used in both clients and servers to bootstrap connections to @@ -372,21 +398,25 @@ type ServerJoin struct { // StartJoin is a list of addresses to attempt to join when the // agent starts. If Serf is unable to communicate with any of these // addresses, then the agent will error and exit. - StartJoin []string `mapstructure:"start_join"` + StartJoin []string `hcl:"start_join"` // RetryJoin is a list of addresses to join with retry enabled, or a single // value to find multiple servers using go-discover syntax. - RetryJoin []string `mapstructure:"retry_join"` + RetryJoin []string `hcl:"retry_join"` // RetryMaxAttempts specifies the maximum number of times to retry joining a // host on startup. This is useful for cases where we know the node will be // online eventually. - RetryMaxAttempts int `mapstructure:"retry_max"` + RetryMaxAttempts int `hcl:"retry_max"` // RetryInterval specifies the amount of time to wait in between join // attempts on agent start. The minimum allowed value is 1 second and // the default is 30s. - RetryInterval time.Duration `mapstructure:"retry_interval"` + RetryInterval time.Duration + RetryIntervalHCL string `hcl:"retry_interval" json:"-"` + + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } func (s *ServerJoin) Merge(b *ServerJoin) *ServerJoin { @@ -423,38 +453,38 @@ func (s *ServerConfig) EncryptBytes() ([]byte, error) { // Telemetry is the telemetry configuration for the server type Telemetry struct { - StatsiteAddr string `mapstructure:"statsite_address"` - StatsdAddr string `mapstructure:"statsd_address"` - DataDogAddr string `mapstructure:"datadog_address"` - DataDogTags []string `mapstructure:"datadog_tags"` - PrometheusMetrics bool `mapstructure:"prometheus_metrics"` - DisableHostname bool `mapstructure:"disable_hostname"` - UseNodeName bool `mapstructure:"use_node_name"` - CollectionInterval string `mapstructure:"collection_interval"` - collectionInterval time.Duration `mapstructure:"-"` - PublishAllocationMetrics bool `mapstructure:"publish_allocation_metrics"` - PublishNodeMetrics bool `mapstructure:"publish_node_metrics"` + StatsiteAddr string `hcl:"statsite_address"` + StatsdAddr string `hcl:"statsd_address"` + DataDogAddr string `hcl:"datadog_address"` + DataDogTags []string `hcl:"datadog_tags"` + PrometheusMetrics bool `hcl:"prometheus_metrics"` + DisableHostname bool `hcl:"disable_hostname"` + UseNodeName bool `hcl:"use_node_name"` + CollectionInterval string `hcl:"collection_interval"` + collectionInterval time.Duration `hcl:"-"` + PublishAllocationMetrics bool `hcl:"publish_allocation_metrics"` + PublishNodeMetrics bool `hcl:"publish_node_metrics"` // DisableTaggedMetrics disables a new version of generating metrics which // uses tags - DisableTaggedMetrics bool `mapstructure:"disable_tagged_metrics"` + DisableTaggedMetrics bool `hcl:"disable_tagged_metrics"` // BackwardsCompatibleMetrics allows for generating metrics in a simple // key/value structure as done in older versions of Nomad - BackwardsCompatibleMetrics bool `mapstructure:"backwards_compatible_metrics"` + BackwardsCompatibleMetrics bool `hcl:"backwards_compatible_metrics"` // PrefixFilter allows for filtering out metrics from being collected - PrefixFilter []string `mapstructure:"prefix_filter"` + PrefixFilter []string `hcl:"prefix_filter"` // FilterDefault controls whether to allow metrics that have not been specified // by the filter - FilterDefault *bool `mapstructure:"filter_default"` + FilterDefault *bool `hcl:"filter_default"` // DisableDispatchedJobSummaryMetrics allows ignoring dispatched jobs when // publishing Job summary metrics. This is useful in environments that produce // high numbers of single count dispatch jobs as the metrics for each take up // a small memory overhead. - DisableDispatchedJobSummaryMetrics bool `mapstructure:"disable_dispatched_job_summary_metrics"` + DisableDispatchedJobSummaryMetrics bool `hcl:"disable_dispatched_job_summary_metrics"` // Circonus: see https://github.com/circonus-labs/circonus-gometrics // for more details on the various configuration options. @@ -472,46 +502,46 @@ type Telemetry struct { // CirconusAPIToken is a valid API Token used to create/manage check. If provided, // metric management is enabled. // Default: none - CirconusAPIToken string `mapstructure:"circonus_api_token"` + CirconusAPIToken string `hcl:"circonus_api_token"` // CirconusAPIApp is an app name associated with API token. // Default: "nomad" - CirconusAPIApp string `mapstructure:"circonus_api_app"` + CirconusAPIApp string `hcl:"circonus_api_app"` // CirconusAPIURL is the base URL to use for contacting the Circonus API. // Default: "https://api.circonus.com/v2" - CirconusAPIURL string `mapstructure:"circonus_api_url"` + CirconusAPIURL string `hcl:"circonus_api_url"` // CirconusSubmissionInterval is the interval at which metrics are submitted to Circonus. // Default: 10s - CirconusSubmissionInterval string `mapstructure:"circonus_submission_interval"` + CirconusSubmissionInterval string `hcl:"circonus_submission_interval"` // CirconusCheckSubmissionURL is the check.config.submission_url field from a // previously created HTTPTRAP check. // Default: none - CirconusCheckSubmissionURL string `mapstructure:"circonus_submission_url"` + CirconusCheckSubmissionURL string `hcl:"circonus_submission_url"` // CirconusCheckID is the check id (not check bundle id) from a previously created // HTTPTRAP check. The numeric portion of the check._cid field. // Default: none - CirconusCheckID string `mapstructure:"circonus_check_id"` + CirconusCheckID string `hcl:"circonus_check_id"` // CirconusCheckForceMetricActivation will force enabling metrics, as they are encountered, // if the metric already exists and is NOT active. If check management is enabled, the default // behavior is to add new metrics as they are encountered. If the metric already exists in the // check, it will *NOT* be activated. This setting overrides that behavior. // Default: "false" - CirconusCheckForceMetricActivation string `mapstructure:"circonus_check_force_metric_activation"` + CirconusCheckForceMetricActivation string `hcl:"circonus_check_force_metric_activation"` // CirconusCheckInstanceID serves to uniquely identify the metrics coming from this "instance". // It can be used to maintain metric continuity with transient or ephemeral instances as // they move around within an infrastructure. // Default: hostname:app - CirconusCheckInstanceID string `mapstructure:"circonus_check_instance_id"` + CirconusCheckInstanceID string `hcl:"circonus_check_instance_id"` // CirconusCheckSearchTag is a special tag which, when coupled with the instance id, helps to // narrow down the search results when neither a Submission URL or Check ID is provided. // Default: service:app (e.g. service:nomad) - CirconusCheckSearchTag string `mapstructure:"circonus_check_search_tag"` + CirconusCheckSearchTag string `hcl:"circonus_check_search_tag"` // CirconusCheckTags is a comma separated list of tags to apply to the check. Note that // the value of CirconusCheckSearchTag will always be added to the check. // Default: none - CirconusCheckTags string `mapstructure:"circonus_check_tags"` + CirconusCheckTags string `hcl:"circonus_check_tags"` // CirconusCheckDisplayName is the name for the check which will be displayed in the Circonus UI. // Default: value of CirconusCheckInstanceID - CirconusCheckDisplayName string `mapstructure:"circonus_check_display_name"` + CirconusCheckDisplayName string `hcl:"circonus_check_display_name"` // CirconusBrokerID is an explicit broker to use when creating a new check. The numeric portion // of broker._cid. If metric management is enabled and neither a Submission URL nor Check ID // is provided, an attempt will be made to search for an existing check using Instance ID and @@ -519,13 +549,16 @@ type Telemetry struct { // Default: use Select Tag if provided, otherwise, a random Enterprise Broker associated // with the specified API token or the default Circonus Broker. // Default: none - CirconusBrokerID string `mapstructure:"circonus_broker_id"` + CirconusBrokerID string `hcl:"circonus_broker_id"` // CirconusBrokerSelectTag is a special tag which will be used to select a broker when // a Broker ID is not provided. The best use of this is to as a hint for which broker // should be used based on *where* this particular instance is running. // (e.g. a specific geo location or datacenter, dc:sfo) // Default: none - CirconusBrokerSelectTag string `mapstructure:"circonus_broker_select_tag"` + CirconusBrokerSelectTag string `hcl:"circonus_broker_select_tag"` + + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } // PrefixFilters parses the PrefixFilter field and returns a list of allowed and blocked filters @@ -549,33 +582,41 @@ func (t *Telemetry) PrefixFilters() (allowed, blocked []string, err error) { // Ports encapsulates the various ports we bind to for network services. If any // are not specified then the defaults are used instead. type Ports struct { - HTTP int `mapstructure:"http"` - RPC int `mapstructure:"rpc"` - Serf int `mapstructure:"serf"` + HTTP int `hcl:"http"` + RPC int `hcl:"rpc"` + Serf int `hcl:"serf"` + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } // Addresses encapsulates all of the addresses we bind to for various // network services. Everything is optional and defaults to BindAddr. type Addresses struct { - HTTP string `mapstructure:"http"` - RPC string `mapstructure:"rpc"` - Serf string `mapstructure:"serf"` + HTTP string `hcl:"http"` + RPC string `hcl:"rpc"` + Serf string `hcl:"serf"` + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } // AdvertiseAddrs is used to control the addresses we advertise out for // different network services. All are optional and default to BindAddr and // their default Port. type AdvertiseAddrs struct { - HTTP string `mapstructure:"http"` - RPC string `mapstructure:"rpc"` - Serf string `mapstructure:"serf"` + HTTP string `hcl:"http"` + RPC string `hcl:"rpc"` + Serf string `hcl:"serf"` + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } type Resources struct { - CPU int `mapstructure:"cpu"` - MemoryMB int `mapstructure:"memory"` - DiskMB int `mapstructure:"disk"` - ReservedPorts string `mapstructure:"reserved_ports"` + CPU int `hcl:"cpu"` + MemoryMB int `hcl:"memory"` + DiskMB int `hcl:"disk"` + ReservedPorts string `hcl:"reserved_ports"` + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } // CanParseReserved returns if the reserved ports specification is parsable. diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index 52d4527365d..af9a1f5c2eb 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -6,20 +6,17 @@ import ( "io" "os" "path/filepath" + "reflect" + "strings" "time" - multierror "github.com/hashicorp/go-multierror" - version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl" - "github.com/hashicorp/hcl/hcl/ast" - "github.com/hashicorp/nomad/helper" - "github.com/hashicorp/nomad/helper/tlsutil" "github.com/hashicorp/nomad/nomad/structs/config" - "github.com/mitchellh/mapstructure" ) -// ParseConfigFile parses the given path as a config file. func ParseConfigFile(path string) (*Config, error) { + // slurp + var buf bytes.Buffer path, err := filepath.Abs(path) if err != nil { return nil, err @@ -30,1051 +27,172 @@ func ParseConfigFile(path string) (*Config, error) { return nil, err } defer f.Close() - - config, err := ParseConfig(f) - if err != nil { + if _, err := io.Copy(&buf, f); err != nil { return nil, err } - return config, nil -} - -// ParseConfig parses the config from the given io.Reader. -// -// Due to current internal limitations, the entire contents of the -// io.Reader will be copied into memory first before parsing. -func ParseConfig(r io.Reader) (*Config, error) { - // Copy the reader into an in-memory buffer first since HCL requires it. - var buf bytes.Buffer - if _, err := io.Copy(&buf, r); err != nil { - return nil, err + // parse + c := &Config{ + Client: &ClientConfig{ServerJoin: &ServerJoin{}}, + ACL: &ACLConfig{}, + Server: &ServerConfig{ServerJoin: &ServerJoin{}}, + Consul: config.DefaultConsulConfig(), + Autopilot: config.DefaultAutopilotConfig(), + Telemetry: &Telemetry{}, + Vault: config.DefaultVaultConfig(), } - // Parse the buffer - root, err := hcl.Parse(buf.String()) + err = hcl.Decode(c, buf.String()) if err != nil { - return nil, fmt.Errorf("error parsing: %s", err) - } - buf.Reset() - - // Top-level item should be a list - list, ok := root.Node.(*ast.ObjectList) - if !ok { - return nil, fmt.Errorf("error parsing: root should be an object") - } - - var config Config - if err := parseConfig(&config, list); err != nil { - return nil, fmt.Errorf("error parsing 'config': %v", err) - } - - return &config, nil -} - -func parseConfig(result *Config, list *ast.ObjectList) error { - // Check for invalid keys - valid := []string{ - "region", - "datacenter", - "name", - "data_dir", - "plugin_dir", - "log_level", - "log_json", - "bind_addr", - "enable_debug", - "ports", - "addresses", - "interfaces", - "advertise", - "client", - "server", - "telemetry", - "leave_on_interrupt", - "leave_on_terminate", - "enable_syslog", - "syslog_facility", - "disable_update_check", - "disable_anonymous_signature", - "consul", - "vault", - "tls", - "http_api_response_headers", - "acl", - "sentinel", - "autopilot", - "plugin", - } - if err := helper.CheckHCLKeys(list, valid); err != nil { - return multierror.Prefix(err, "config:") - } - - // Decode the full thing into a map[string]interface for ease - var m map[string]interface{} - if err := hcl.DecodeObject(&m, list); err != nil { - return err - } - delete(m, "ports") - delete(m, "addresses") - delete(m, "interfaces") - delete(m, "advertise") - delete(m, "client") - delete(m, "server") - delete(m, "telemetry") - delete(m, "consul") - delete(m, "vault") - delete(m, "tls") - delete(m, "http_api_response_headers") - delete(m, "acl") - delete(m, "sentinel") - delete(m, "autopilot") - delete(m, "plugin") - - // Decode the rest - if err := mapstructure.WeakDecode(m, result); err != nil { - return err - } - - // Parse ports - if o := list.Filter("ports"); len(o.Items) > 0 { - if err := parsePorts(&result.Ports, o); err != nil { - return multierror.Prefix(err, "ports ->") - } - } - - // Parse addresses - if o := list.Filter("addresses"); len(o.Items) > 0 { - if err := parseAddresses(&result.Addresses, o); err != nil { - return multierror.Prefix(err, "addresses ->") - } - } - - // Parse advertise - if o := list.Filter("advertise"); len(o.Items) > 0 { - if err := parseAdvertise(&result.AdvertiseAddrs, o); err != nil { - return multierror.Prefix(err, "advertise ->") - } - } - - // Parse client config - if o := list.Filter("client"); len(o.Items) > 0 { - if err := parseClient(&result.Client, o); err != nil { - return multierror.Prefix(err, "client ->") - } - } - - // Parse server config - if o := list.Filter("server"); len(o.Items) > 0 { - if err := parseServer(&result.Server, o); err != nil { - return multierror.Prefix(err, "server ->") - } - } - - // Parse ACL config - if o := list.Filter("acl"); len(o.Items) > 0 { - if err := parseACL(&result.ACL, o); err != nil { - return multierror.Prefix(err, "acl ->") - } - } - - // Parse telemetry config - if o := list.Filter("telemetry"); len(o.Items) > 0 { - if err := parseTelemetry(&result.Telemetry, o); err != nil { - return multierror.Prefix(err, "telemetry ->") - } - } - - // Parse the consul config - if o := list.Filter("consul"); len(o.Items) > 0 { - if err := parseConsulConfig(&result.Consul, o); err != nil { - return multierror.Prefix(err, "consul ->") - } - } - - // Parse the vault config - if o := list.Filter("vault"); len(o.Items) > 0 { - if err := parseVaultConfig(&result.Vault, o); err != nil { - return multierror.Prefix(err, "vault ->") - } - } - - // Parse the TLS config - if o := list.Filter("tls"); len(o.Items) > 0 { - if err := parseTLSConfig(&result.TLSConfig, o); err != nil { - return multierror.Prefix(err, "tls ->") - } - } - - // Parse Sentinel config - if o := list.Filter("sentinel"); len(o.Items) > 0 { - if err := parseSentinel(&result.Sentinel, o); err != nil { - return multierror.Prefix(err, "sentinel->") - } - } - - // Parse Autopilot config - if o := list.Filter("autopilot"); len(o.Items) > 0 { - if err := parseAutopilot(&result.Autopilot, o); err != nil { - return multierror.Prefix(err, "autopilot->") - } - } - - // Parse Plugin configs - if o := list.Filter("plugin"); len(o.Items) > 0 { - if err := parsePlugins(&result.Plugins, o); err != nil { - return multierror.Prefix(err, "plugin->") - } - } - - // Parse out http_api_response_headers fields. These are in HCL as a list so - // we need to iterate over them and merge them. - if headersO := list.Filter("http_api_response_headers"); len(headersO.Items) > 0 { - for _, o := range headersO.Elem().Items { - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - if err := mapstructure.WeakDecode(m, &result.HTTPAPIResponseHeaders); err != nil { - return err - } - } - } - - return nil -} - -func parsePorts(result **Ports, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'ports' block allowed") - } - - // Get our ports object - listVal := list.Items[0].Val - - // Check for invalid keys - valid := []string{ - "http", - "rpc", - "serf", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err - } - - var ports Ports - if err := mapstructure.WeakDecode(m, &ports); err != nil { - return err - } - *result = &ports - return nil -} - -func parseAddresses(result **Addresses, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'addresses' block allowed") - } - - // Get our addresses object - listVal := list.Items[0].Val - - // Check for invalid keys - valid := []string{ - "http", - "rpc", - "serf", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err - } - - var addresses Addresses - if err := mapstructure.WeakDecode(m, &addresses); err != nil { - return err - } - *result = &addresses - return nil -} - -func parseAdvertise(result **AdvertiseAddrs, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'advertise' block allowed") - } - - // Get our advertise object - listVal := list.Items[0].Val - - // Check for invalid keys - valid := []string{ - "http", - "rpc", - "serf", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err - } - - var advertise AdvertiseAddrs - if err := mapstructure.WeakDecode(m, &advertise); err != nil { - return err - } - *result = &advertise - return nil -} - -func parseClient(result **ClientConfig, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'client' block allowed") - } - - // Get our client object - obj := list.Items[0] - - // Value should be an object - var listVal *ast.ObjectList - if ot, ok := obj.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return fmt.Errorf("client value: should be an object") - } - - // Check for invalid keys - valid := []string{ - "enabled", - "state_dir", - "alloc_dir", - "servers", - "node_class", - "options", - "meta", - "chroot_env", - "network_interface", - "network_speed", - "memory_total_mb", - "cpu_total_compute", - "max_kill_timeout", - "client_max_port", - "client_min_port", - "reserved", - "stats", - "gc_interval", - "gc_disk_usage_threshold", - "gc_inode_usage_threshold", - "gc_parallel_destroys", - "gc_max_allocs", - "no_host_uuid", - "server_join", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err - } - - delete(m, "options") - delete(m, "meta") - delete(m, "chroot_env") - delete(m, "reserved") - delete(m, "stats") - delete(m, "server_join") - - var config ClientConfig - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &config, - }) - if err != nil { - return err - } - if err := dec.Decode(m); err != nil { - return err - } - - // Parse out options fields. These are in HCL as a list so we need to - // iterate over them and merge them. - if optionsO := listVal.Filter("options"); len(optionsO.Items) > 0 { - for _, o := range optionsO.Elem().Items { - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - if err := mapstructure.WeakDecode(m, &config.Options); err != nil { - return err - } - } - } - - // Parse out options meta. These are in HCL as a list so we need to - // iterate over them and merge them. - if metaO := listVal.Filter("meta"); len(metaO.Items) > 0 { - for _, o := range metaO.Elem().Items { - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - if err := mapstructure.WeakDecode(m, &config.Meta); err != nil { - return err - } - } - } - - // Parse out chroot_env fields. These are in HCL as a list so we need to - // iterate over them and merge them. - if chrootEnvO := listVal.Filter("chroot_env"); len(chrootEnvO.Items) > 0 { - for _, o := range chrootEnvO.Elem().Items { - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - if err := mapstructure.WeakDecode(m, &config.ChrootEnv); err != nil { - return err - } - } - } - - // Parse reserved config - if o := listVal.Filter("reserved"); len(o.Items) > 0 { - if err := parseReserved(&config.Reserved, o); err != nil { - return multierror.Prefix(err, "reserved ->") - } - } - - // Parse ServerJoin config - if o := listVal.Filter("server_join"); len(o.Items) > 0 { - if err := parseServerJoin(&config.ServerJoin, o); err != nil { - return multierror.Prefix(err, "server_join->") - } - } - - *result = &config - return nil -} - -func parseReserved(result **Resources, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'reserved' block allowed") - } - - // Get our reserved object - obj := list.Items[0] - - // Value should be an object - var listVal *ast.ObjectList - if ot, ok := obj.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return fmt.Errorf("client value: should be an object") - } - - // Check for invalid keys - valid := []string{ - "cpu", - "memory", - "disk", - "reserved_ports", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err - } - - var reserved Resources - if err := mapstructure.WeakDecode(m, &reserved); err != nil { - return err - } - if err := reserved.CanParseReserved(); err != nil { - return err - } - - *result = &reserved - return nil -} - -func parseServer(result **ServerConfig, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'server' block allowed") - } - - // Get our server object - obj := list.Items[0] - - // Value should be an object - var listVal *ast.ObjectList - if ot, ok := obj.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return fmt.Errorf("client value: should be an object") - } - - // Check for invalid keys - valid := []string{ - "enabled", - "bootstrap_expect", - "data_dir", - "protocol_version", - "raft_protocol", - "num_schedulers", - "enabled_schedulers", - "node_gc_threshold", - "eval_gc_threshold", - "job_gc_threshold", - "deployment_gc_threshold", - "heartbeat_grace", - "min_heartbeat_ttl", - "max_heartbeats_per_second", - "rejoin_after_leave", - "encrypt", - "authoritative_region", - "non_voting_server", - "redundancy_zone", - "upgrade_version", - - "server_join", - - // For backwards compatibility - "start_join", - "retry_join", - "retry_max", - "retry_interval", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err + return nil, err } - delete(m, "server_join") - - var config ServerConfig - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &config, + // convert strings to time.Durations + err = durations([]td{ + {"gc_interval", &c.Client.GCInterval, &c.Client.GCIntervalHCL}, + {"acl.token_ttl", &c.ACL.TokenTTL, &c.ACL.TokenTTLHCL}, + {"acl.policy_ttl", &c.ACL.PolicyTTL, &c.ACL.PolicyTTLHCL}, + {"client.server_join.retry_interval", &c.Client.ServerJoin.RetryInterval, &c.Client.ServerJoin.RetryIntervalHCL}, + {"server.heartbeat_grace", &c.Server.HeartbeatGrace, &c.Server.HeartbeatGraceHCL}, + {"server.min_heartbeat_ttl", &c.Server.MinHeartbeatTTL, &c.Server.MinHeartbeatTTLHCL}, + {"server.retry_interval", &c.Server.RetryInterval, &c.Server.RetryIntervalHCL}, + {"server.server_join.retry_interval", &c.Server.ServerJoin.RetryInterval, &c.Server.ServerJoin.RetryIntervalHCL}, + {"consul.timeout", &c.Consul.Timeout, &c.Consul.TimeoutHCL}, + {"autopilot.server_stabilization_time", &c.Autopilot.ServerStabilizationTime, &c.Autopilot.ServerStabilizationTimeHCL}, + {"autopilot.last_contact_threshold", &c.Autopilot.LastContactThreshold, &c.Autopilot.LastContactThresholdHCL}, + {"telemetry.collection_interval", &c.Telemetry.collectionInterval, &c.Telemetry.CollectionInterval}, }) if err != nil { - return err - } - if err := dec.Decode(m); err != nil { - return err - } - - if config.UpgradeVersion != "" { - if _, err := version.NewVersion(config.UpgradeVersion); err != nil { - return fmt.Errorf("error parsing upgrade_version: %v", err) - } - } - - // Parse ServerJoin config - if o := listVal.Filter("server_join"); len(o.Items) > 0 { - if err := parseServerJoin(&config.ServerJoin, o); err != nil { - return multierror.Prefix(err, "server_join->") - } - } - - *result = &config - return nil -} - -func parseServerJoin(result **ServerJoin, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'server_join' block allowed") - } - - // Get our object - listVal := list.Items[0].Val - - // Check for invalid keys - valid := []string{ - "start_join", - "retry_join", - "retry_max", - "retry_interval", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err + return nil, err } - var serverJoinInfo ServerJoin - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &serverJoinInfo, - }) + // report unexpected keys + err = extraKeys(c) if err != nil { - return err - } - if err := dec.Decode(m); err != nil { - return err + return nil, err } - *result = &serverJoinInfo - return nil + return c, nil } -func parseACL(result **ACLConfig, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'acl' block allowed") - } - - // Get our server object - obj := list.Items[0] - - // Value should be an object - var listVal *ast.ObjectList - if ot, ok := obj.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return fmt.Errorf("acl value: should be an object") - } - - // Check for invalid keys - valid := []string{ - "enabled", - "token_ttl", - "policy_ttl", - "replication_token", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err - } - - var config ACLConfig - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &config, - }) - if err != nil { - return err - } - if err := dec.Decode(m); err != nil { - return err - } - - *result = &config - return nil +// td holds args for one duration conversion +type td struct { + path string + td *time.Duration + str *string } -func parseTelemetry(result **Telemetry, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'telemetry' block allowed") - } - - // Get our telemetry object - listVal := list.Items[0].Val - - // Check for invalid keys - valid := []string{ - "statsite_address", - "statsd_address", - "disable_hostname", - "use_node_name", - "collection_interval", - "publish_allocation_metrics", - "publish_node_metrics", - "datadog_address", - "datadog_tags", - "prometheus_metrics", - "circonus_api_token", - "circonus_api_app", - "circonus_api_url", - "circonus_submission_interval", - "circonus_submission_url", - "circonus_check_id", - "circonus_check_force_metric_activation", - "circonus_check_instance_id", - "circonus_check_search_tag", - "circonus_check_display_name", - "circonus_check_tags", - "circonus_broker_id", - "circonus_broker_select_tag", - "disable_tagged_metrics", - "backwards_compatible_metrics", - "prefix_filter", - "filter_default", - "disable_dispatched_job_summary_metrics", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err - } +// durations parses the duration strings specified in the config files +// into time.Durations +func durations(xs []td) error { + for _, x := range xs { + if x.td != nil && x.str != nil && "" != *x.str { + d, err := time.ParseDuration(*x.str) + if err != nil { + return fmt.Errorf("%s can't parse time duration %s", x.path, *x.str) + } - var telemetry Telemetry - if err := mapstructure.WeakDecode(m, &telemetry); err != nil { - return err - } - if telemetry.CollectionInterval != "" { - if dur, err := time.ParseDuration(telemetry.CollectionInterval); err != nil { - return fmt.Errorf("error parsing value of %q: %v", "collection_interval", err) - } else { - telemetry.collectionInterval = dur + *x.td = d } } - *result = &telemetry - return nil -} - -func parseConsulConfig(result **config.ConsulConfig, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'consul' block allowed") - } - - // Get our Consul object - listVal := list.Items[0].Val - - // Check for invalid keys - valid := []string{ - "address", - "auth", - "auto_advertise", - "ca_file", - "cert_file", - "checks_use_advertise", - "client_auto_join", - "client_service_name", - "client_http_check_name", - "key_file", - "server_auto_join", - "server_service_name", - "server_http_check_name", - "server_serf_check_name", - "server_rpc_check_name", - "ssl", - "timeout", - "token", - "verify_ssl", - } - - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err - } - - consulConfig := config.DefaultConsulConfig() - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &consulConfig, - }) - if err != nil { - return err - } - if err := dec.Decode(m); err != nil { - return err - } - - *result = consulConfig - return nil -} -func parseTLSConfig(result **config.TLSConfig, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'tls' block allowed") - } - - // Get the TLS object - listVal := list.Items[0].Val - - valid := []string{ - "http", - "rpc", - "verify_server_hostname", - "rpc_upgrade_mode", - "ca_file", - "cert_file", - "key_file", - "verify_https_client", - "tls_cipher_suites", - "tls_min_version", - "tls_prefer_server_cipher_suites", - } - - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err - } - - var tlsConfig config.TLSConfig - if err := mapstructure.WeakDecode(m, &tlsConfig); err != nil { - return err - } - - if _, err := tlsutil.ParseCiphers(&tlsConfig); err != nil { - return err - } - - if _, err := tlsutil.ParseMinVersion(tlsConfig.TLSMinVersion); err != nil { - return err - } - - *result = &tlsConfig return nil } -func parseVaultConfig(result **config.VaultConfig, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'vault' block allowed") - } - - // Get our Vault object - listVal := list.Items[0].Val - - // Check for invalid keys - valid := []string{ - "address", - "allow_unauthenticated", - "enabled", - "task_token_ttl", - "ca_file", - "ca_path", - "cert_file", - "create_from_role", - "key_file", - "tls_server_name", - "tls_skip_verify", - "token", - "namespace", - } - - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err - } - - vaultConfig := config.DefaultVaultConfig() - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &vaultConfig, - }) - if err != nil { - return err - } - if err := dec.Decode(m); err != nil { - return err +// removeEqualFold removes the first string that EqualFold matches +func removeEqualFold(xs *[]string, search string) { + sl := *xs + for i, x := range sl { + if strings.EqualFold(x, search) { + sl = append(sl[:i], sl[i+1:]...) + if len(sl) == 0 { + *xs = nil + } else { + *xs = sl + } + return + } } - - *result = vaultConfig - return nil } -func parseSentinel(result **config.SentinelConfig, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'sentinel' block allowed") - } - - // Get our sentinel object - obj := list.Items[0] - - // Value should be an object - var listVal *ast.ObjectList - if ot, ok := obj.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return fmt.Errorf("sentinel value: should be an object") - } - - // Check for invalid keys - valid := []string{ - "import", - } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } - - var config config.SentinelConfig - if err := hcl.DecodeObject(&config, listVal); err != nil { - return err +func extraKeys(c *Config) error { + // hcl leaves behind extra keys when parsing JSON. These keys + // are kept on the top level, taken from slices or the keys of + // structs contained in slices. Clean up before looking for + // extra keys. + for range c.HTTPAPIResponseHeaders { + removeEqualFold(&c.ExtraKeysHCL, "http_api_response_headers") } - *result = &config - return nil -} - -func parseAutopilot(result **config.AutopilotConfig, list *ast.ObjectList) error { - list = list.Elem() - if len(list.Items) > 1 { - return fmt.Errorf("only one 'autopilot' block allowed") + for _, p := range c.Plugins { + removeEqualFold(&c.ExtraKeysHCL, p.Name) + removeEqualFold(&c.ExtraKeysHCL, "config") + removeEqualFold(&c.ExtraKeysHCL, "plugin") } - // Get our Autopilot object - listVal := list.Items[0].Val - - // Check for invalid keys - valid := []string{ - "cleanup_dead_servers", - "server_stabilization_time", - "last_contact_threshold", - "max_trailing_logs", - "enable_redundancy_zones", - "disable_upgrade_migration", - "enable_custom_upgrades", + for _, k := range []string{"options", "meta", "chroot_env", "servers", "server_join"} { + removeEqualFold(&c.ExtraKeysHCL, k) + removeEqualFold(&c.ExtraKeysHCL, "client") } - if err := helper.CheckHCLKeys(listVal, valid); err != nil { - return err - } + // stats is an unused key, continue to silently ignore it + removeEqualFold(&c.Client.ExtraKeysHCL, "stats") - var m map[string]interface{} - if err := hcl.DecodeObject(&m, listVal); err != nil { - return err + for _, k := range []string{"enabled_schedulers", "start_join", "retry_join", "server_join"} { + removeEqualFold(&c.ExtraKeysHCL, k) + removeEqualFold(&c.ExtraKeysHCL, "server") } - autopilotConfig := config.DefaultAutopilotConfig() - dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - WeaklyTypedInput: true, - Result: &autopilotConfig, - }) - if err != nil { - return err - } - if err := dec.Decode(m); err != nil { - return err + for _, k := range []string{"datadog_tags"} { + removeEqualFold(&c.ExtraKeysHCL, k) + removeEqualFold(&c.ExtraKeysHCL, "telemetry") } - *result = autopilotConfig - return nil + return extraKeysImpl([]string{}, reflect.ValueOf(*c)) } -func parsePlugins(result *[]*config.PluginConfig, list *ast.ObjectList) error { - listLen := len(list.Items) - plugins := make([]*config.PluginConfig, listLen) +// extraKeysImpl returns an error if any extraKeys array is not empty +func extraKeysImpl(path []string, val reflect.Value) error { + stype := val.Type() + for i := 0; i < stype.NumField(); i++ { + ftype := stype.Field(i) + fval := val.Field(i) - // Check for invalid keys - valid := []string{ - "args", - "config", - } - - for i := 0; i < listLen; i++ { - // Get the current plugin object - listVal := list.Items[i] + name := ftype.Name + prop := "" + tagSplit(ftype, "hcl", &name, &prop) - // Deal with json->hcl AST parsing incorrectness when directly nested - // items show up as additional keys. - // TODO(preetha): Add additional tests and fix other places that have the same issue - unwrapLegacyHCLObjectKeysFromJSON(listVal, 1) - if err := helper.CheckHCLKeys(listVal.Val, valid); err != nil { - return fmt.Errorf("invalid keys in plugin config %d: %v", i+1, err) + if fval.Kind() == reflect.Ptr { + fval = reflect.Indirect(fval) } - // Ensure there is a key - if len(listVal.Keys) != 1 { - return fmt.Errorf("plugin config %d doesn't incude a name key", i+1) + // struct? recurse. add the struct's key to the path + if fval.Kind() == reflect.Struct { + err := extraKeysImpl(append([]string{name}, path...), fval) + if err != nil { + return err + } } - var plugin config.PluginConfig - if err := hcl.DecodeObject(&plugin, listVal); err != nil { - return fmt.Errorf("error decoding plugin config %d: %v", i+1, err) + if "unusedKeys" == prop { + if ks, ok := fval.Interface().([]string); ok && len(ks) != 0 { + return fmt.Errorf("%s unexpected keys %s", + strings.Join(path, "."), + strings.Join(ks, ", ")) + } } - - plugins[i] = &plugin } - - *result = plugins return nil } -// unwrapLegacyHCLObjectKeysFromJSON cleans up an edge case that can occur when -// parsing JSON as input: if we're parsing JSON then directly nested -// items will show up as additional "keys". -// -// For objects that expect a fixed number of keys, this breaks the -// decoding process. This function unwraps the object into what it would've -// looked like if it came directly from HCL by specifying the number of keys -// you expect. -// -// Example: -// -// { "foo": { "baz": {} } } -// -// Will show up with Keys being: []string{"foo", "baz"} -// when we really just want the first two. This function will fix this. -func unwrapLegacyHCLObjectKeysFromJSON(item *ast.ObjectItem, depth int) { - if len(item.Keys) > depth && item.Keys[0].Token.JSON { - for len(item.Keys) > depth { - // Pop off the last key - n := len(item.Keys) - key := item.Keys[n-1] - item.Keys[n-1] = nil - item.Keys = item.Keys[:n-1] - - // Wrap our value in a list - item.Val = &ast.ObjectType{ - List: &ast.ObjectList{ - Items: []*ast.ObjectItem{ - { - Keys: []*ast.ObjectKey{key}, - Val: item.Val, - }, - }, - }, - } +// tagSplit reads the named tag from the structfield and splits its values into strings +func tagSplit(field reflect.StructField, tagName string, vars ...*string) { + tag := strings.Split(field.Tag.Get(tagName), ",") + end := len(tag) - 1 + for i, s := range vars { + if i > end { + return } + *s = tag[i] } } diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go index b58f89730f7..4c8bb0c1c4d 100644 --- a/command/agent/config_parse_test.go +++ b/command/agent/config_parse_test.go @@ -10,8 +10,361 @@ import ( "github.com/stretchr/testify/require" ) +var basicConfig = &Config{ + Region: "foobar", + Datacenter: "dc2", + NodeName: "my-web", + DataDir: "/tmp/nomad", + PluginDir: "/tmp/nomad-plugins", + LogLevel: "ERR", + LogJson: true, + BindAddr: "192.168.0.1", + EnableDebug: true, + Ports: &Ports{ + HTTP: 1234, + RPC: 2345, + Serf: 3456, + }, + Addresses: &Addresses{ + HTTP: "127.0.0.1", + RPC: "127.0.0.2", + Serf: "127.0.0.3", + }, + AdvertiseAddrs: &AdvertiseAddrs{ + RPC: "127.0.0.3", + Serf: "127.0.0.4", + }, + Client: &ClientConfig{ + Enabled: true, + StateDir: "/tmp/client-state", + AllocDir: "/tmp/alloc", + Servers: []string{"a.b.c:80", "127.0.0.1:1234"}, + NodeClass: "linux-medium-64bit", + ServerJoin: &ServerJoin{ + RetryJoin: []string{"1.1.1.1", "2.2.2.2"}, + RetryInterval: time.Duration(15) * time.Second, + RetryIntervalHCL: "15s", + RetryMaxAttempts: 3, + }, + Meta: map[string]string{ + "foo": "bar", + "baz": "zip", + }, + Options: map[string]string{ + "foo": "bar", + "baz": "zip", + }, + ChrootEnv: map[string]string{ + "/opt/myapp/etc": "/etc", + "/opt/myapp/bin": "/bin", + }, + NetworkInterface: "eth0", + NetworkSpeed: 100, + CpuCompute: 4444, + MemoryMB: 0, + MaxKillTimeout: "10s", + ClientMinPort: 1000, + ClientMaxPort: 2000, + Reserved: &Resources{ + CPU: 10, + MemoryMB: 10, + DiskMB: 10, + ReservedPorts: "1,100,10-12", + }, + GCInterval: 6 * time.Second, + GCIntervalHCL: "6s", + GCParallelDestroys: 6, + GCDiskUsageThreshold: 82, + GCInodeUsageThreshold: 91, + GCMaxAllocs: 50, + NoHostUUID: helper.BoolToPtr(false), + }, + Server: &ServerConfig{ + Enabled: true, + AuthoritativeRegion: "foobar", + BootstrapExpect: 5, + DataDir: "/tmp/data", + ProtocolVersion: 3, + RaftProtocol: 3, + NumSchedulers: helper.IntToPtr(2), + EnabledSchedulers: []string{"test"}, + NodeGCThreshold: "12h", + EvalGCThreshold: "12h", + JobGCThreshold: "12h", + DeploymentGCThreshold: "12h", + HeartbeatGrace: 30 * time.Second, + HeartbeatGraceHCL: "30s", + MinHeartbeatTTL: 33 * time.Second, + MinHeartbeatTTLHCL: "33s", + MaxHeartbeatsPerSecond: 11.0, + RetryJoin: []string{"1.1.1.1", "2.2.2.2"}, + StartJoin: []string{"1.1.1.1", "2.2.2.2"}, + RetryInterval: 15 * time.Second, + RetryIntervalHCL: "15s", + RejoinAfterLeave: true, + RetryMaxAttempts: 3, + NonVotingServer: true, + RedundancyZone: "foo", + UpgradeVersion: "0.8.0", + EncryptKey: "abc", + ServerJoin: &ServerJoin{ + RetryJoin: []string{"1.1.1.1", "2.2.2.2"}, + RetryInterval: time.Duration(15) * time.Second, + RetryIntervalHCL: "15s", + RetryMaxAttempts: 3, + }, + }, + ACL: &ACLConfig{ + Enabled: true, + TokenTTL: 60 * time.Second, + TokenTTLHCL: "60s", + PolicyTTL: 60 * time.Second, + PolicyTTLHCL: "60s", + ReplicationToken: "foobar", + }, + Telemetry: &Telemetry{ + StatsiteAddr: "127.0.0.1:1234", + StatsdAddr: "127.0.0.1:2345", + PrometheusMetrics: true, + DisableHostname: true, + UseNodeName: false, + CollectionInterval: "3s", + collectionInterval: 3 * time.Second, + PublishAllocationMetrics: true, + PublishNodeMetrics: true, + DisableTaggedMetrics: true, + BackwardsCompatibleMetrics: true, + }, + LeaveOnInt: true, + LeaveOnTerm: true, + EnableSyslog: true, + SyslogFacility: "LOCAL1", + DisableUpdateCheck: helper.BoolToPtr(true), + DisableAnonymousSignature: true, + Consul: &config.ConsulConfig{ + ServerServiceName: "nomad", + ServerHTTPCheckName: "nomad-server-http-health-check", + ServerSerfCheckName: "nomad-server-serf-health-check", + ServerRPCCheckName: "nomad-server-rpc-health-check", + ClientServiceName: "nomad-client", + ClientHTTPCheckName: "nomad-client-http-health-check", + Addr: "127.0.0.1:9500", + Token: "token1", + Auth: "username:pass", + EnableSSL: &trueValue, + VerifySSL: &trueValue, + CAFile: "/path/to/ca/file", + CertFile: "/path/to/cert/file", + KeyFile: "/path/to/key/file", + ServerAutoJoin: &trueValue, + ClientAutoJoin: &trueValue, + AutoAdvertise: &trueValue, + ChecksUseAdvertise: &trueValue, + Timeout: 5 * time.Second, + }, + Vault: &config.VaultConfig{ + Addr: "127.0.0.1:9500", + AllowUnauthenticated: &trueValue, + ConnectionRetryIntv: config.DefaultVaultConnectRetryIntv, + Enabled: &falseValue, + Role: "test_role", + TLSCaFile: "/path/to/ca/file", + TLSCaPath: "/path/to/ca", + TLSCertFile: "/path/to/cert/file", + TLSKeyFile: "/path/to/key/file", + TLSServerName: "foobar", + TLSSkipVerify: &trueValue, + TaskTokenTTL: "1s", + Token: "12345", + }, + TLSConfig: &config.TLSConfig{ + EnableHTTP: true, + EnableRPC: true, + VerifyServerHostname: true, + CAFile: "foo", + CertFile: "bar", + KeyFile: "pipe", + RPCUpgradeMode: true, + VerifyHTTPSClient: true, + TLSPreferServerCipherSuites: true, + TLSCipherSuites: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + TLSMinVersion: "tls12", + }, + HTTPAPIResponseHeaders: map[string]string{ + "Access-Control-Allow-Origin": "*", + }, + Sentinel: &config.SentinelConfig{ + Imports: []*config.SentinelImport{ + { + Name: "foo", + Path: "foo", + Args: []string{"a", "b", "c"}, + }, + { + Name: "bar", + Path: "bar", + Args: []string{"x", "y", "z"}, + }, + }, + }, + Autopilot: &config.AutopilotConfig{ + CleanupDeadServers: &trueValue, + ServerStabilizationTime: 23057 * time.Second, + ServerStabilizationTimeHCL: "23057s", + LastContactThreshold: 12705 * time.Second, + LastContactThresholdHCL: "12705s", + MaxTrailingLogs: 17849, + EnableRedundancyZones: &trueValue, + DisableUpgradeMigration: &trueValue, + EnableCustomUpgrades: &trueValue, + }, + Plugins: []*config.PluginConfig{ + { + Name: "docker", + Args: []string{"foo", "bar"}, + Config: map[string]interface{}{ + "foo": "bar", + "nested": []map[string]interface{}{ + { + "bam": 2, + }, + }, + }, + }, + { + Name: "exec", + Config: map[string]interface{}{ + "foo": true, + }, + }, + }, +} + +var pluginConfig = &Config{ + Region: "", + Datacenter: "", + NodeName: "", + DataDir: "", + PluginDir: "", + LogLevel: "", + BindAddr: "", + EnableDebug: false, + Ports: nil, + Addresses: nil, + AdvertiseAddrs: nil, + Client: &ClientConfig{ + Enabled: false, + StateDir: "", + AllocDir: "", + Servers: nil, + NodeClass: "", + Meta: nil, + Options: nil, + ChrootEnv: nil, + NetworkInterface: "", + NetworkSpeed: 0, + CpuCompute: 0, + MemoryMB: 5555, + MaxKillTimeout: "", + ClientMinPort: 0, + ClientMaxPort: 0, + Reserved: nil, + GCInterval: 0, + GCParallelDestroys: 0, + GCDiskUsageThreshold: 0, + GCInodeUsageThreshold: 0, + GCMaxAllocs: 0, + NoHostUUID: nil, + }, + Server: nil, + ACL: nil, + Telemetry: nil, + LeaveOnInt: false, + LeaveOnTerm: false, + EnableSyslog: false, + SyslogFacility: "", + DisableUpdateCheck: nil, + DisableAnonymousSignature: false, + Consul: nil, + Vault: nil, + TLSConfig: nil, + HTTPAPIResponseHeaders: nil, + Sentinel: nil, + Plugins: []*config.PluginConfig{ + { + Name: "docker", + Config: map[string]interface{}{ + "allow_privileged": true, + }, + }, + { + Name: "raw_exec", + Config: map[string]interface{}{ + "enabled": true, + }, + }, + }, +} + +var nonoptConfig = &Config{ + Region: "", + Datacenter: "", + NodeName: "", + DataDir: "", + PluginDir: "", + LogLevel: "", + BindAddr: "", + EnableDebug: false, + Ports: nil, + Addresses: nil, + AdvertiseAddrs: nil, + Client: &ClientConfig{ + Enabled: false, + StateDir: "", + AllocDir: "", + Servers: nil, + NodeClass: "", + Meta: nil, + Options: nil, + ChrootEnv: nil, + NetworkInterface: "", + NetworkSpeed: 0, + CpuCompute: 0, + MemoryMB: 5555, + MaxKillTimeout: "", + ClientMinPort: 0, + ClientMaxPort: 0, + Reserved: nil, + GCInterval: 0, + GCParallelDestroys: 0, + GCDiskUsageThreshold: 0, + GCInodeUsageThreshold: 0, + GCMaxAllocs: 0, + NoHostUUID: nil, + }, + Server: nil, + ACL: nil, + Telemetry: nil, + LeaveOnInt: false, + LeaveOnTerm: false, + EnableSyslog: false, + SyslogFacility: "", + DisableUpdateCheck: nil, + DisableAnonymousSignature: false, + Consul: nil, + Vault: nil, + TLSConfig: nil, + HTTPAPIResponseHeaders: nil, + Sentinel: nil, +} + func TestConfig_Parse(t *testing.T) { t.Parallel() + + basicConfig.addDefaults() + pluginConfig.addDefaults() + nonoptConfig.addDefaults() + cases := []struct { File string Result *Config @@ -19,637 +372,27 @@ func TestConfig_Parse(t *testing.T) { }{ { "basic.hcl", - &Config{ - Region: "foobar", - Datacenter: "dc2", - NodeName: "my-web", - DataDir: "/tmp/nomad", - PluginDir: "/tmp/nomad-plugins", - LogLevel: "ERR", - LogJson: true, - BindAddr: "192.168.0.1", - EnableDebug: true, - Ports: &Ports{ - HTTP: 1234, - RPC: 2345, - Serf: 3456, - }, - Addresses: &Addresses{ - HTTP: "127.0.0.1", - RPC: "127.0.0.2", - Serf: "127.0.0.3", - }, - AdvertiseAddrs: &AdvertiseAddrs{ - RPC: "127.0.0.3", - Serf: "127.0.0.4", - }, - Client: &ClientConfig{ - Enabled: true, - StateDir: "/tmp/client-state", - AllocDir: "/tmp/alloc", - Servers: []string{"a.b.c:80", "127.0.0.1:1234"}, - NodeClass: "linux-medium-64bit", - ServerJoin: &ServerJoin{ - RetryJoin: []string{"1.1.1.1", "2.2.2.2"}, - RetryInterval: time.Duration(15) * time.Second, - RetryMaxAttempts: 3, - }, - Meta: map[string]string{ - "foo": "bar", - "baz": "zip", - }, - Options: map[string]string{ - "foo": "bar", - "baz": "zip", - }, - ChrootEnv: map[string]string{ - "/opt/myapp/etc": "/etc", - "/opt/myapp/bin": "/bin", - }, - NetworkInterface: "eth0", - NetworkSpeed: 100, - CpuCompute: 4444, - MemoryMB: 0, - MaxKillTimeout: "10s", - ClientMinPort: 1000, - ClientMaxPort: 2000, - Reserved: &Resources{ - CPU: 10, - MemoryMB: 10, - DiskMB: 10, - ReservedPorts: "1,100,10-12", - }, - GCInterval: 6 * time.Second, - GCParallelDestroys: 6, - GCDiskUsageThreshold: 82, - GCInodeUsageThreshold: 91, - GCMaxAllocs: 50, - NoHostUUID: helper.BoolToPtr(false), - }, - Server: &ServerConfig{ - Enabled: true, - AuthoritativeRegion: "foobar", - BootstrapExpect: 5, - DataDir: "/tmp/data", - ProtocolVersion: 3, - RaftProtocol: 3, - NumSchedulers: helper.IntToPtr(2), - EnabledSchedulers: []string{"test"}, - NodeGCThreshold: "12h", - EvalGCThreshold: "12h", - JobGCThreshold: "12h", - DeploymentGCThreshold: "12h", - HeartbeatGrace: 30 * time.Second, - MinHeartbeatTTL: 33 * time.Second, - MaxHeartbeatsPerSecond: 11.0, - RetryJoin: []string{"1.1.1.1", "2.2.2.2"}, - StartJoin: []string{"1.1.1.1", "2.2.2.2"}, - RetryInterval: 15 * time.Second, - RejoinAfterLeave: true, - RetryMaxAttempts: 3, - NonVotingServer: true, - RedundancyZone: "foo", - UpgradeVersion: "0.8.0", - EncryptKey: "abc", - ServerJoin: &ServerJoin{ - RetryJoin: []string{"1.1.1.1", "2.2.2.2"}, - RetryInterval: time.Duration(15) * time.Second, - RetryMaxAttempts: 3, - }, - }, - ACL: &ACLConfig{ - Enabled: true, - TokenTTL: 60 * time.Second, - PolicyTTL: 60 * time.Second, - ReplicationToken: "foobar", - }, - Telemetry: &Telemetry{ - StatsiteAddr: "127.0.0.1:1234", - StatsdAddr: "127.0.0.1:2345", - PrometheusMetrics: true, - DisableHostname: true, - UseNodeName: false, - CollectionInterval: "3s", - collectionInterval: 3 * time.Second, - PublishAllocationMetrics: true, - PublishNodeMetrics: true, - DisableTaggedMetrics: true, - BackwardsCompatibleMetrics: true, - }, - LeaveOnInt: true, - LeaveOnTerm: true, - EnableSyslog: true, - SyslogFacility: "LOCAL1", - DisableUpdateCheck: helper.BoolToPtr(true), - DisableAnonymousSignature: true, - Consul: &config.ConsulConfig{ - ServerServiceName: "nomad", - ServerHTTPCheckName: "nomad-server-http-health-check", - ServerSerfCheckName: "nomad-server-serf-health-check", - ServerRPCCheckName: "nomad-server-rpc-health-check", - ClientServiceName: "nomad-client", - ClientHTTPCheckName: "nomad-client-http-health-check", - Addr: "127.0.0.1:9500", - Token: "token1", - Auth: "username:pass", - EnableSSL: &trueValue, - VerifySSL: &trueValue, - CAFile: "/path/to/ca/file", - CertFile: "/path/to/cert/file", - KeyFile: "/path/to/key/file", - ServerAutoJoin: &trueValue, - ClientAutoJoin: &trueValue, - AutoAdvertise: &trueValue, - ChecksUseAdvertise: &trueValue, - }, - Vault: &config.VaultConfig{ - Addr: "127.0.0.1:9500", - AllowUnauthenticated: &trueValue, - Enabled: &falseValue, - Role: "test_role", - TLSCaFile: "/path/to/ca/file", - TLSCaPath: "/path/to/ca", - TLSCertFile: "/path/to/cert/file", - TLSKeyFile: "/path/to/key/file", - TLSServerName: "foobar", - TLSSkipVerify: &trueValue, - TaskTokenTTL: "1s", - Token: "12345", - }, - TLSConfig: &config.TLSConfig{ - EnableHTTP: true, - EnableRPC: true, - VerifyServerHostname: true, - CAFile: "foo", - CertFile: "bar", - KeyFile: "pipe", - RPCUpgradeMode: true, - VerifyHTTPSClient: true, - TLSPreferServerCipherSuites: true, - TLSCipherSuites: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - TLSMinVersion: "tls12", - }, - HTTPAPIResponseHeaders: map[string]string{ - "Access-Control-Allow-Origin": "*", - }, - Sentinel: &config.SentinelConfig{ - Imports: []*config.SentinelImport{ - { - Name: "foo", - Path: "foo", - Args: []string{"a", "b", "c"}, - }, - { - Name: "bar", - Path: "bar", - Args: []string{"x", "y", "z"}, - }, - }, - }, - Autopilot: &config.AutopilotConfig{ - CleanupDeadServers: &trueValue, - ServerStabilizationTime: 23057 * time.Second, - LastContactThreshold: 12705 * time.Second, - MaxTrailingLogs: 17849, - EnableRedundancyZones: &trueValue, - DisableUpgradeMigration: &trueValue, - EnableCustomUpgrades: &trueValue, - }, - Plugins: []*config.PluginConfig{ - { - Name: "docker", - Args: []string{"foo", "bar"}, - Config: map[string]interface{}{ - "foo": "bar", - "nested": []map[string]interface{}{ - { - "bam": 2, - }, - }, - }, - }, - { - Name: "exec", - Config: map[string]interface{}{ - "foo": true, - }, - }, - }, - }, + basicConfig, false, }, { "basic.json", - &Config{ - Region: "foobar", - Datacenter: "dc2", - NodeName: "my-web", - DataDir: "/tmp/nomad", - PluginDir: "/tmp/nomad-plugins", - LogLevel: "ERR", - LogJson: true, - BindAddr: "192.168.0.1", - EnableDebug: true, - Ports: &Ports{ - HTTP: 1234, - RPC: 2345, - Serf: 3456, - }, - Addresses: &Addresses{ - HTTP: "127.0.0.1", - RPC: "127.0.0.2", - Serf: "127.0.0.3", - }, - AdvertiseAddrs: &AdvertiseAddrs{ - RPC: "127.0.0.3", - Serf: "127.0.0.4", - }, - Client: &ClientConfig{ - Enabled: true, - StateDir: "/tmp/client-state", - AllocDir: "/tmp/alloc", - Servers: []string{"a.b.c:80", "127.0.0.1:1234"}, - NodeClass: "linux-medium-64bit", - ServerJoin: &ServerJoin{ - RetryJoin: []string{"1.1.1.1", "2.2.2.2"}, - RetryInterval: time.Duration(15) * time.Second, - RetryMaxAttempts: 3, - }, - Meta: map[string]string{ - "foo": "bar", - "baz": "zip", - }, - Options: map[string]string{ - "foo": "bar", - "baz": "zip", - }, - ChrootEnv: map[string]string{ - "/opt/myapp/etc": "/etc", - "/opt/myapp/bin": "/bin", - }, - NetworkInterface: "eth0", - NetworkSpeed: 100, - CpuCompute: 4444, - MemoryMB: 0, - MaxKillTimeout: "10s", - ClientMinPort: 1000, - ClientMaxPort: 2000, - Reserved: &Resources{ - CPU: 10, - MemoryMB: 10, - DiskMB: 10, - ReservedPorts: "1,100,10-12", - }, - GCInterval: 6 * time.Second, - GCParallelDestroys: 6, - GCDiskUsageThreshold: 82, - GCInodeUsageThreshold: 91, - GCMaxAllocs: 50, - NoHostUUID: helper.BoolToPtr(false), - }, - Server: &ServerConfig{ - Enabled: true, - AuthoritativeRegion: "foobar", - BootstrapExpect: 5, - DataDir: "/tmp/data", - ProtocolVersion: 3, - RaftProtocol: 3, - NumSchedulers: helper.IntToPtr(2), - EnabledSchedulers: []string{"test"}, - NodeGCThreshold: "12h", - EvalGCThreshold: "12h", - JobGCThreshold: "12h", - DeploymentGCThreshold: "12h", - HeartbeatGrace: 30 * time.Second, - MinHeartbeatTTL: 33 * time.Second, - MaxHeartbeatsPerSecond: 11.0, - RetryJoin: []string{"1.1.1.1", "2.2.2.2"}, - StartJoin: []string{"1.1.1.1", "2.2.2.2"}, - RetryInterval: 15 * time.Second, - RejoinAfterLeave: true, - RetryMaxAttempts: 3, - NonVotingServer: true, - RedundancyZone: "foo", - UpgradeVersion: "0.8.0", - EncryptKey: "abc", - ServerJoin: &ServerJoin{ - RetryJoin: []string{"1.1.1.1", "2.2.2.2"}, - RetryInterval: time.Duration(15) * time.Second, - RetryMaxAttempts: 3, - }, - }, - ACL: &ACLConfig{ - Enabled: true, - TokenTTL: 60 * time.Second, - PolicyTTL: 60 * time.Second, - ReplicationToken: "foobar", - }, - Telemetry: &Telemetry{ - StatsiteAddr: "127.0.0.1:1234", - StatsdAddr: "127.0.0.1:2345", - PrometheusMetrics: true, - DisableHostname: true, - UseNodeName: false, - CollectionInterval: "3s", - collectionInterval: 3 * time.Second, - PublishAllocationMetrics: true, - PublishNodeMetrics: true, - DisableTaggedMetrics: true, - BackwardsCompatibleMetrics: true, - }, - LeaveOnInt: true, - LeaveOnTerm: true, - EnableSyslog: true, - SyslogFacility: "LOCAL1", - DisableUpdateCheck: helper.BoolToPtr(true), - DisableAnonymousSignature: true, - Consul: &config.ConsulConfig{ - ServerServiceName: "nomad", - ServerHTTPCheckName: "nomad-server-http-health-check", - ServerSerfCheckName: "nomad-server-serf-health-check", - ServerRPCCheckName: "nomad-server-rpc-health-check", - ClientServiceName: "nomad-client", - ClientHTTPCheckName: "nomad-client-http-health-check", - Addr: "127.0.0.1:9500", - Token: "token1", - Auth: "username:pass", - EnableSSL: &trueValue, - VerifySSL: &trueValue, - CAFile: "/path/to/ca/file", - CertFile: "/path/to/cert/file", - KeyFile: "/path/to/key/file", - ServerAutoJoin: &trueValue, - ClientAutoJoin: &trueValue, - AutoAdvertise: &trueValue, - ChecksUseAdvertise: &trueValue, - }, - Vault: &config.VaultConfig{ - Addr: "127.0.0.1:9500", - AllowUnauthenticated: &trueValue, - Enabled: &falseValue, - Role: "test_role", - TLSCaFile: "/path/to/ca/file", - TLSCaPath: "/path/to/ca", - TLSCertFile: "/path/to/cert/file", - TLSKeyFile: "/path/to/key/file", - TLSServerName: "foobar", - TLSSkipVerify: &trueValue, - TaskTokenTTL: "1s", - Token: "12345", - }, - TLSConfig: &config.TLSConfig{ - EnableHTTP: true, - EnableRPC: true, - VerifyServerHostname: true, - CAFile: "foo", - CertFile: "bar", - KeyFile: "pipe", - RPCUpgradeMode: true, - VerifyHTTPSClient: true, - TLSPreferServerCipherSuites: true, - TLSCipherSuites: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - TLSMinVersion: "tls12", - }, - HTTPAPIResponseHeaders: map[string]string{ - "Access-Control-Allow-Origin": "*", - }, - Sentinel: &config.SentinelConfig{ - Imports: []*config.SentinelImport{ - { - Name: "foo", - Path: "foo", - Args: []string{"a", "b", "c"}, - }, - { - Name: "bar", - Path: "bar", - Args: []string{"x", "y", "z"}, - }, - }, - }, - Autopilot: &config.AutopilotConfig{ - CleanupDeadServers: &trueValue, - ServerStabilizationTime: 23057 * time.Second, - LastContactThreshold: 12705 * time.Second, - MaxTrailingLogs: 17849, - EnableRedundancyZones: &trueValue, - DisableUpgradeMigration: &trueValue, - EnableCustomUpgrades: &trueValue, - }, - Plugins: []*config.PluginConfig{ - { - Name: "docker", - Args: []string{"foo", "bar"}, - Config: map[string]interface{}{ - "foo": "bar", - "nested": []map[string]interface{}{ - { - "bam": 2, - }, - }, - }, - }, - { - Name: "exec", - Config: map[string]interface{}{ - "foo": true, - }, - }, - }, - }, + basicConfig, false, }, { "plugin.hcl", - &Config{ - Region: "", - Datacenter: "", - NodeName: "", - DataDir: "", - PluginDir: "", - LogLevel: "", - BindAddr: "", - EnableDebug: false, - Ports: nil, - Addresses: nil, - AdvertiseAddrs: nil, - Client: &ClientConfig{ - Enabled: false, - StateDir: "", - AllocDir: "", - Servers: nil, - NodeClass: "", - Meta: nil, - Options: nil, - ChrootEnv: nil, - NetworkInterface: "", - NetworkSpeed: 0, - CpuCompute: 0, - MemoryMB: 5555, - MaxKillTimeout: "", - ClientMinPort: 0, - ClientMaxPort: 0, - Reserved: nil, - GCInterval: 0, - GCParallelDestroys: 0, - GCDiskUsageThreshold: 0, - GCInodeUsageThreshold: 0, - GCMaxAllocs: 0, - NoHostUUID: nil, - }, - Server: nil, - ACL: nil, - Telemetry: nil, - LeaveOnInt: false, - LeaveOnTerm: false, - EnableSyslog: false, - SyslogFacility: "", - DisableUpdateCheck: nil, - DisableAnonymousSignature: false, - Consul: nil, - Vault: nil, - TLSConfig: nil, - HTTPAPIResponseHeaders: nil, - Sentinel: nil, - Plugins: []*config.PluginConfig{ - { - Name: "docker", - Config: map[string]interface{}{ - "allow_privileged": true, - }, - }, - { - Name: "raw_exec", - Config: map[string]interface{}{ - "enabled": true, - }, - }, - }, - }, + pluginConfig, false, }, { "plugin.json", - &Config{ - Region: "", - Datacenter: "", - NodeName: "", - DataDir: "", - PluginDir: "", - LogLevel: "", - BindAddr: "", - EnableDebug: false, - Ports: nil, - Addresses: nil, - AdvertiseAddrs: nil, - Client: &ClientConfig{ - Enabled: false, - StateDir: "", - AllocDir: "", - Servers: nil, - NodeClass: "", - Meta: nil, - Options: nil, - ChrootEnv: nil, - NetworkInterface: "", - NetworkSpeed: 0, - CpuCompute: 0, - MemoryMB: 5555, - MaxKillTimeout: "", - ClientMinPort: 0, - ClientMaxPort: 0, - Reserved: nil, - GCInterval: 0, - GCParallelDestroys: 0, - GCDiskUsageThreshold: 0, - GCInodeUsageThreshold: 0, - GCMaxAllocs: 0, - NoHostUUID: nil, - }, - Server: nil, - ACL: nil, - Telemetry: nil, - LeaveOnInt: false, - LeaveOnTerm: false, - EnableSyslog: false, - SyslogFacility: "", - DisableUpdateCheck: nil, - DisableAnonymousSignature: false, - Consul: nil, - Vault: nil, - TLSConfig: nil, - HTTPAPIResponseHeaders: nil, - Sentinel: nil, - Plugins: []*config.PluginConfig{ - { - Name: "docker", - Config: map[string]interface{}{ - "allow_privileged": true, - }, - }, - { - Name: "raw_exec", - Config: map[string]interface{}{ - "enabled": true, - }, - }, - }, - }, + pluginConfig, false, }, { "non-optional.hcl", - &Config{ - Region: "", - Datacenter: "", - NodeName: "", - DataDir: "", - PluginDir: "", - LogLevel: "", - BindAddr: "", - EnableDebug: false, - Ports: nil, - Addresses: nil, - AdvertiseAddrs: nil, - Client: &ClientConfig{ - Enabled: false, - StateDir: "", - AllocDir: "", - Servers: nil, - NodeClass: "", - Meta: nil, - Options: nil, - ChrootEnv: nil, - NetworkInterface: "", - NetworkSpeed: 0, - CpuCompute: 0, - MemoryMB: 5555, - MaxKillTimeout: "", - ClientMinPort: 0, - ClientMaxPort: 0, - Reserved: nil, - GCInterval: 0, - GCParallelDestroys: 0, - GCDiskUsageThreshold: 0, - GCInodeUsageThreshold: 0, - GCMaxAllocs: 0, - NoHostUUID: nil, - }, - Server: nil, - ACL: nil, - Telemetry: nil, - LeaveOnInt: false, - LeaveOnTerm: false, - EnableSyslog: false, - SyslogFacility: "", - DisableUpdateCheck: nil, - DisableAnonymousSignature: false, - Consul: nil, - Vault: nil, - TLSConfig: nil, - HTTPAPIResponseHeaders: nil, - Sentinel: nil, - }, + nonoptConfig, false, }, } @@ -668,7 +411,7 @@ func TestConfig_Parse(t *testing.T) { } //panic(fmt.Sprintf("first: %+v \n second: %+v", actual.TLSConfig, tc.Result.TLSConfig)) - require.EqualValues(removeHelperAttributes(actual), tc.Result) + require.EqualValues(tc.Result, removeHelperAttributes(actual)) }) } } @@ -682,3 +425,174 @@ func removeHelperAttributes(c *Config) *Config { } return c } + +func (c *Config) addDefaults() { + if c.Client == nil { + c.Client = &ClientConfig{} + } + if c.Client.ServerJoin == nil { + c.Client.ServerJoin = &ServerJoin{} + } + if c.ACL == nil { + c.ACL = &ACLConfig{} + } + if c.Consul == nil { + c.Consul = config.DefaultConsulConfig() + } + if c.Autopilot == nil { + c.Autopilot = config.DefaultAutopilotConfig() + } + if c.Vault == nil { + c.Vault = config.DefaultVaultConfig() + } + if c.Telemetry == nil { + c.Telemetry = &Telemetry{} + } + if c.Server == nil { + c.Server = &ServerConfig{} + } + if c.Server.ServerJoin == nil { + c.Server.ServerJoin = &ServerJoin{} + } +} + +// Tests for a panic parsing json with an object of exactly +// length 1 described in +// https://github.com/hashicorp/nomad/issues/1290 +func TestConfig_ParsePanic(t *testing.T) { + c, err := ParseConfigFile("./testdata/obj-len-one.hcl") + if err != nil { + t.Fatalf("parse error: %s\n", err) + } + + d, err := ParseConfigFile("./testdata/obj-len-one.json") + if err != nil { + t.Fatalf("parse error: %s\n", err) + } + + require.EqualValues(t, c, d) +} + +// Top level keys left by hcl when parsing slices in the config +// structure should not be unexpected +func TestConfig_ParseSliceExtra(t *testing.T) { + c, err := ParseConfigFile("./testdata/config-slices.json") + if err != nil { + t.Fatalf("parse error: %s\n", err) + } + + opt := map[string]string{"o0": "foo", "o1": "bar"} + meta := map[string]string{"m0": "foo", "m1": "bar"} + env := map[string]string{"e0": "baz"} + srv := []string{"foo", "bar"} + + require.EqualValues(t, opt, c.Client.Options) + require.EqualValues(t, meta, c.Client.Meta) + require.EqualValues(t, env, c.Client.ChrootEnv) + require.EqualValues(t, srv, c.Client.Servers) + require.EqualValues(t, srv, c.Server.EnabledSchedulers) + require.EqualValues(t, srv, c.Server.StartJoin) + require.EqualValues(t, srv, c.Server.RetryJoin) + + // the alt format is also accepted by hcl as valid config data + c, err = ParseConfigFile("./testdata/config-slices-alt.json") + if err != nil { + t.Fatalf("parse error: %s\n", err) + } + + require.EqualValues(t, opt, c.Client.Options) + require.EqualValues(t, meta, c.Client.Meta) + require.EqualValues(t, env, c.Client.ChrootEnv) + require.EqualValues(t, srv, c.Client.Servers) + require.EqualValues(t, srv, c.Server.EnabledSchedulers) + require.EqualValues(t, srv, c.Server.StartJoin) + require.EqualValues(t, srv, c.Server.RetryJoin) + + // small files keep more extra keys than large ones + _, err = ParseConfigFile("./testdata/obj-len-one-server.json") + if err != nil { + t.Fatalf("parse error: %s\n", err) + } +} + +var sample0 = &Config{ + Region: "global", + Datacenter: "dc1", + DataDir: "/opt/data/nomad/data", + LogLevel: "INFO", + BindAddr: "0.0.0.0", + AdvertiseAddrs: &AdvertiseAddrs{ + HTTP: "host.example.com", + RPC: "host.example.com", + Serf: "host.example.com", + }, + Client: &ClientConfig{ServerJoin: &ServerJoin{}}, + Server: &ServerConfig{ + Enabled: true, + BootstrapExpect: 3, + RetryJoin: []string{"10.0.0.101", "10.0.0.102", "10.0.0.103"}, + EncryptKey: "sHck3WL6cxuhuY7Mso9BHA==", + ServerJoin: &ServerJoin{}, + }, + ACL: &ACLConfig{ + Enabled: true, + }, + Telemetry: &Telemetry{ + PrometheusMetrics: true, + DisableHostname: true, + CollectionInterval: "60s", + collectionInterval: 60 * time.Second, + PublishAllocationMetrics: true, + PublishNodeMetrics: true, + }, + LeaveOnInt: true, + LeaveOnTerm: true, + EnableSyslog: true, + SyslogFacility: "LOCAL0", + Consul: &config.ConsulConfig{ + Token: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", + ServerAutoJoin: helper.BoolToPtr(false), + ClientAutoJoin: helper.BoolToPtr(false), + // Defaults + ServerServiceName: "nomad", + ServerHTTPCheckName: "Nomad Server HTTP Check", + ServerSerfCheckName: "Nomad Server Serf Check", + ServerRPCCheckName: "Nomad Server RPC Check", + ClientServiceName: "nomad-client", + ClientHTTPCheckName: "Nomad Client HTTP Check", + AutoAdvertise: helper.BoolToPtr(true), + ChecksUseAdvertise: helper.BoolToPtr(false), + Timeout: 5 * time.Second, + EnableSSL: helper.BoolToPtr(false), + VerifySSL: helper.BoolToPtr(true), + }, + Vault: &config.VaultConfig{ + Enabled: helper.BoolToPtr(true), + Role: "nomad-cluster", + Addr: "http://host.example.com:8200", + // Defaults + AllowUnauthenticated: helper.BoolToPtr(true), + ConnectionRetryIntv: 30 * time.Second, + }, + TLSConfig: &config.TLSConfig{ + EnableHTTP: true, + EnableRPC: true, + VerifyServerHostname: true, + CAFile: "/opt/data/nomad/certs/nomad-ca.pem", + CertFile: "/opt/data/nomad/certs/server.pem", + KeyFile: "/opt/data/nomad/certs/server-key.pem", + }, + Autopilot: &config.AutopilotConfig{ + CleanupDeadServers: helper.BoolToPtr(true), + // Defaults + ServerStabilizationTime: 10 * time.Second, + LastContactThreshold: 200 * time.Millisecond, + MaxTrailingLogs: 250, + }, +} + +func TestConfig_ParseSample0(t *testing.T) { + c, err := ParseConfigFile("./testdata/sample0.json") + require.Nil(t, err) + require.EqualValues(t, sample0, c) +} diff --git a/command/agent/testdata/config-slices-alt.json b/command/agent/testdata/config-slices-alt.json new file mode 100644 index 00000000000..d259e6209d5 --- /dev/null +++ b/command/agent/testdata/config-slices-alt.json @@ -0,0 +1,67 @@ +{ + "client": [ + { + "chroot_env": [ + { + "e0": "baz" + } + ], + "meta": [ + { + "m0": "foo", + "m1": "bar" + } + ], + "options": [ + { + "o0": "foo", + "o1": "bar" + } + ], + "server_join": [ + { + "retry_join": [ + "foo", + "bar" + ], + "start_join": [ + "foo", + "bar" + ] + } + ], + "servers": [ + "foo", + "bar" + ] + } + ], + "server": [ + { + "enabled_schedulers": [ + "foo", + "bar" + ], + "retry_join": [ + "foo", + "bar" + ], + "server_join": [ + { + "retry_join": [ + "foo", + "bar" + ], + "start_join": [ + "foo", + "bar" + ] + } + ], + "start_join": [ + "foo", + "bar" + ] + } + ] +} diff --git a/command/agent/testdata/config-slices.hcl b/command/agent/testdata/config-slices.hcl new file mode 100644 index 00000000000..98be21caeb7 --- /dev/null +++ b/command/agent/testdata/config-slices.hcl @@ -0,0 +1,13 @@ +client "chroot_env" { + "e0" = "baz" +} + +client "meta" { + "m0" = "foo" + "m1" = "bar" +} + +client "options" { + "o0" = "foo" + "o1" = "bar" +} diff --git a/command/agent/testdata/config-slices.json b/command/agent/testdata/config-slices.json new file mode 100644 index 00000000000..4a0e23bfa58 --- /dev/null +++ b/command/agent/testdata/config-slices.json @@ -0,0 +1,41 @@ +{ + "client": { + "options": { + "o0": "foo", + "o1": "bar" + }, + "meta": { + "m0": "foo", + "m1": "bar" + }, + "chroot_env": { + "e0": "baz" + }, + "servers": [ + "foo", + "bar" + ], + "server_join": { + "start_join": ["foo", "bar"], + "retry_join": ["foo", "bar"] + } + }, + "server": { + "enabled_schedulers": [ + "foo", + "bar" + ], + "start_join": [ + "foo", + "bar" + ], + "retry_join": [ + "foo", + "bar" + ], + "server_join": { + "start_join": ["foo", "bar"], + "retry_join": ["foo", "bar"] + } + } +} diff --git a/command/agent/testdata/non-optional.json b/command/agent/testdata/non-optional.json new file mode 100644 index 00000000000..80d65f438b7 --- /dev/null +++ b/command/agent/testdata/non-optional.json @@ -0,0 +1,7 @@ +{ + "client": [ + { + "memory_total_mb": 5555 + } + ] +} diff --git a/command/agent/testdata/obj-len-one-server.json b/command/agent/testdata/obj-len-one-server.json new file mode 100644 index 00000000000..d1d9677b5eb --- /dev/null +++ b/command/agent/testdata/obj-len-one-server.json @@ -0,0 +1,7 @@ +{ + "server": { + "server_join": { + "start_join": ["foo", "bar"] + } + }, +} diff --git a/command/agent/testdata/obj-len-one.hcl b/command/agent/testdata/obj-len-one.hcl new file mode 100644 index 00000000000..9d154a7b943 --- /dev/null +++ b/command/agent/testdata/obj-len-one.hcl @@ -0,0 +1,5 @@ +client { + options { + driver.whitelist = "docker" + } +} diff --git a/command/agent/testdata/obj-len-one.json b/command/agent/testdata/obj-len-one.json new file mode 100644 index 00000000000..a1e46acea24 --- /dev/null +++ b/command/agent/testdata/obj-len-one.json @@ -0,0 +1,8 @@ +{ + "client": { + "options": { + "driver.whitelist": "docker" + } + }, + "server": {} +} diff --git a/command/agent/testdata/sample0.json b/command/agent/testdata/sample0.json new file mode 100644 index 00000000000..c9f9a8bd9e6 --- /dev/null +++ b/command/agent/testdata/sample0.json @@ -0,0 +1,57 @@ +{ + "autopilot": { + "cleanup_dead_servers": true + }, + "acl": { + "enabled": true + }, + "advertise": { + "http": "host.example.com", + "rpc": "host.example.com", + "serf": "host.example.com" + }, + "bind_addr": "0.0.0.0", + "consul": { + "server_auto_join": false, + "client_auto_join": false, + "token": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + }, + "data_dir": "/opt/data/nomad/data", + "datacenter": "dc1", + "enable_syslog": true, + "leave_on_interrupt": true, + "leave_on_terminate": true, + "log_level": "INFO", + "region": "global", + "server": { + "bootstrap_expect": 3, + "enabled": true, + "encrypt": "sHck3WL6cxuhuY7Mso9BHA==", + "retry_join": [ + "10.0.0.101", + "10.0.0.102", + "10.0.0.103" + ] + }, + "syslog_facility": "LOCAL0", + "telemetry": { + "collection_interval": "60s", + "disable_hostname": true, + "prometheus_metrics": true, + "publish_allocation_metrics": true, + "publish_node_metrics": true + }, + "tls": { + "ca_file": "/opt/data/nomad/certs/nomad-ca.pem", + "cert_file": "/opt/data/nomad/certs/server.pem", + "http": true, + "key_file": "/opt/data/nomad/certs/server-key.pem", + "rpc": true, + "verify_server_hostname": true + }, + "vault": { + "address": "http://host.example.com:8200", + "create_from_role": "nomad-cluster", + "enabled": true + } +} diff --git a/nomad/structs/config/autopilot.go b/nomad/structs/config/autopilot.go index ffa52bc09f7..c1bae189440 100644 --- a/nomad/structs/config/autopilot.go +++ b/nomad/structs/config/autopilot.go @@ -9,32 +9,37 @@ import ( type AutopilotConfig struct { // CleanupDeadServers controls whether to remove dead servers when a new // server is added to the Raft peers. - CleanupDeadServers *bool `mapstructure:"cleanup_dead_servers"` + CleanupDeadServers *bool `hcl:"cleanup_dead_servers"` // ServerStabilizationTime is the minimum amount of time a server must be // in a stable, healthy state before it can be added to the cluster. Only // applicable with Raft protocol version 3 or higher. - ServerStabilizationTime time.Duration `mapstructure:"server_stabilization_time"` + ServerStabilizationTime time.Duration + ServerStabilizationTimeHCL string `hcl:"server_stabilization_time" json:"-"` // LastContactThreshold is the limit on the amount of time a server can go // without leader contact before being considered unhealthy. - LastContactThreshold time.Duration `mapstructure:"last_contact_threshold"` + LastContactThreshold time.Duration + LastContactThresholdHCL string `hcl:"last_contact_threshold" json:"-"` // MaxTrailingLogs is the amount of entries in the Raft Log that a server can // be behind before being considered unhealthy. - MaxTrailingLogs int `mapstructure:"max_trailing_logs"` + MaxTrailingLogs int `hcl:"max_trailing_logs"` // (Enterprise-only) EnableRedundancyZones specifies whether to enable redundancy zones. - EnableRedundancyZones *bool `mapstructure:"enable_redundancy_zones"` + EnableRedundancyZones *bool `hcl:"enable_redundancy_zones"` // (Enterprise-only) DisableUpgradeMigration will disable Autopilot's upgrade migration // strategy of waiting until enough newer-versioned servers have been added to the // cluster before promoting them to voters. - DisableUpgradeMigration *bool `mapstructure:"disable_upgrade_migration"` + DisableUpgradeMigration *bool `hcl:"disable_upgrade_migration"` // (Enterprise-only) EnableCustomUpgrades specifies whether to enable using custom // upgrade versions when performing migrations. - EnableCustomUpgrades *bool `mapstructure:"enable_custom_upgrades"` + EnableCustomUpgrades *bool `hcl:"enable_custom_upgrades"` + + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } // DefaultAutopilotConfig() returns the canonical defaults for the Nomad diff --git a/nomad/structs/config/consul.go b/nomad/structs/config/consul.go index 9473ea814d4..9f9cd8ecf21 100644 --- a/nomad/structs/config/consul.go +++ b/nomad/structs/config/consul.go @@ -21,73 +21,77 @@ import ( type ConsulConfig struct { // ServerServiceName is the name of the service that Nomad uses to register // servers with Consul - ServerServiceName string `mapstructure:"server_service_name"` + ServerServiceName string `hcl:"server_service_name"` // ServerHTTPCheckName is the name of the health check that Nomad uses // to register the server HTTP health check with Consul - ServerHTTPCheckName string `mapstructure:"server_http_check_name"` + ServerHTTPCheckName string `hcl:"server_http_check_name"` // ServerSerfCheckName is the name of the health check that Nomad uses // to register the server Serf health check with Consul - ServerSerfCheckName string `mapstructure:"server_serf_check_name"` + ServerSerfCheckName string `hcl:"server_serf_check_name"` // ServerRPCCheckName is the name of the health check that Nomad uses // to register the server RPC health check with Consul - ServerRPCCheckName string `mapstructure:"server_rpc_check_name"` + ServerRPCCheckName string `hcl:"server_rpc_check_name"` // ClientServiceName is the name of the service that Nomad uses to register // clients with Consul - ClientServiceName string `mapstructure:"client_service_name"` + ClientServiceName string `hcl:"client_service_name"` // ClientHTTPCheckName is the name of the health check that Nomad uses // to register the client HTTP health check with Consul - ClientHTTPCheckName string `mapstructure:"client_http_check_name"` + ClientHTTPCheckName string `hcl:"client_http_check_name"` // AutoAdvertise determines if this Nomad Agent will advertise its // services via Consul. When true, Nomad Agent will register // services with Consul. - AutoAdvertise *bool `mapstructure:"auto_advertise"` + AutoAdvertise *bool `hcl:"auto_advertise"` // ChecksUseAdvertise specifies that Consul checks should use advertise // address instead of bind address - ChecksUseAdvertise *bool `mapstructure:"checks_use_advertise"` + ChecksUseAdvertise *bool `hcl:"checks_use_advertise"` // Addr is the address of the local Consul agent - Addr string `mapstructure:"address"` + Addr string `hcl:"address"` // Timeout is used by Consul HTTP Client - Timeout time.Duration `mapstructure:"timeout"` + Timeout time.Duration + TimeoutHCL string `hcl:"timeout" json:"-"` // Token is used to provide a per-request ACL token. This options overrides // the agent's default token - Token string `mapstructure:"token"` + Token string `hcl:"token"` // Auth is the information to use for http access to Consul agent - Auth string `mapstructure:"auth"` + Auth string `hcl:"auth"` // EnableSSL sets the transport scheme to talk to the Consul agent as https - EnableSSL *bool `mapstructure:"ssl"` + EnableSSL *bool `hcl:"ssl"` // VerifySSL enables or disables SSL verification when the transport scheme // for the consul api client is https - VerifySSL *bool `mapstructure:"verify_ssl"` + VerifySSL *bool `hcl:"verify_ssl"` // CAFile is the path to the ca certificate used for Consul communication - CAFile string `mapstructure:"ca_file"` + CAFile string `hcl:"ca_file"` // CertFile is the path to the certificate for Consul communication - CertFile string `mapstructure:"cert_file"` + CertFile string `hcl:"cert_file"` // KeyFile is the path to the private key for Consul communication - KeyFile string `mapstructure:"key_file"` + KeyFile string `hcl:"key_file"` // ServerAutoJoin enables Nomad servers to find peers by querying Consul and // joining them - ServerAutoJoin *bool `mapstructure:"server_auto_join"` + ServerAutoJoin *bool `hcl:"server_auto_join"` // ClientAutoJoin enables Nomad servers to find addresses of Nomad servers // and register with them - ClientAutoJoin *bool `mapstructure:"client_auto_join"` + ClientAutoJoin *bool `hcl:"client_auto_join"` + + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } // DefaultConsulConfig() returns the canonical defaults for the Nomad diff --git a/nomad/structs/config/plugins.go b/nomad/structs/config/plugins.go index 55f3a25abc6..6762be8fcf5 100644 --- a/nomad/structs/config/plugins.go +++ b/nomad/structs/config/plugins.go @@ -7,6 +7,8 @@ type PluginConfig struct { Name string `hcl:",key"` Args []string `hcl:"args"` Config map[string]interface{} `hcl:"config"` + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } func (p *PluginConfig) Merge(o *PluginConfig) *PluginConfig { diff --git a/nomad/structs/config/tls.go b/nomad/structs/config/tls.go index 875a6e20b7c..31e7e5183f2 100644 --- a/nomad/structs/config/tls.go +++ b/nomad/structs/config/tls.go @@ -14,10 +14,10 @@ import ( type TLSConfig struct { // EnableHTTP enabled TLS for http traffic to the Nomad server and clients - EnableHTTP bool `mapstructure:"http"` + EnableHTTP bool `hcl:"http"` // EnableRPC enables TLS for RPC and Raft traffic to the Nomad servers - EnableRPC bool `mapstructure:"rpc"` + EnableRPC bool `hcl:"rpc"` // VerifyServerHostname is used to enable hostname verification of servers. This // ensures that the certificate presented is valid for server..nomad @@ -25,15 +25,15 @@ type TLSConfig struct { // intercepting request traffic as well as being added as a raft peer. This should be // enabled by default with VerifyOutgoing, but for legacy reasons we cannot break // existing clients. - VerifyServerHostname bool `mapstructure:"verify_server_hostname"` + VerifyServerHostname bool `hcl:"verify_server_hostname"` // CAFile is a path to a certificate authority file. This is used with VerifyIncoming // or VerifyOutgoing to verify the TLS connection. - CAFile string `mapstructure:"ca_file"` + CAFile string `hcl:"ca_file"` // CertFile is used to provide a TLS certificate that is used for serving TLS connections. // Must be provided to serve TLS connections. - CertFile string `mapstructure:"cert_file"` + CertFile string `hcl:"cert_file"` // KeyLoader is a helper to dynamically reload TLS configuration KeyLoader *KeyLoader @@ -42,15 +42,15 @@ type TLSConfig struct { // KeyFile is used to provide a TLS key that is used for serving TLS connections. // Must be provided to serve TLS connections. - KeyFile string `mapstructure:"key_file"` + KeyFile string `hcl:"key_file"` // RPCUpgradeMode should be enabled when a cluster is being upgraded // to TLS. Allows servers to accept both plaintext and TLS connections and // should only be a temporary state. - RPCUpgradeMode bool `mapstructure:"rpc_upgrade_mode"` + RPCUpgradeMode bool `hcl:"rpc_upgrade_mode"` // Verify connections to the HTTPS API - VerifyHTTPSClient bool `mapstructure:"verify_https_client"` + VerifyHTTPSClient bool `hcl:"verify_https_client"` // Checksum is a MD5 hash of the certificate CA File, Certificate file, and // key file. @@ -58,17 +58,20 @@ type TLSConfig struct { // TLSCipherSuites are operator-defined ciphers to be used in Nomad TLS // connections - TLSCipherSuites string `mapstructure:"tls_cipher_suites"` + TLSCipherSuites string `hcl:"tls_cipher_suites"` // TLSMinVersion is used to set the minimum TLS version used for TLS // connections. Should be either "tls10", "tls11", or "tls12". - TLSMinVersion string `mapstructure:"tls_min_version"` + TLSMinVersion string `hcl:"tls_min_version"` // TLSPreferServerCipherSuites controls whether the server selects the // client's most preferred ciphersuite, or the server's most preferred // ciphersuite. If true then the server's preference, as expressed in // the order of elements in CipherSuites, is used. - TLSPreferServerCipherSuites bool `mapstructure:"tls_prefer_server_cipher_suites"` + TLSPreferServerCipherSuites bool `hcl:"tls_prefer_server_cipher_suites"` + + // ExtraKeysHCL is used by hcl to surface unexpected keys + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } type KeyLoader struct { diff --git a/nomad/structs/config/vault.go b/nomad/structs/config/vault.go index 3a0b8fab632..67dd618d26b 100644 --- a/nomad/structs/config/vault.go +++ b/nomad/structs/config/vault.go @@ -23,19 +23,19 @@ const ( type VaultConfig struct { // Enabled enables or disables Vault support. - Enabled *bool `mapstructure:"enabled"` + Enabled *bool `hcl:"enabled"` // Token is the Vault token given to Nomad such that it can // derive child tokens. Nomad will renew this token at half its lease // lifetime. - Token string `mapstructure:"token"` + Token string `hcl:"token"` // Role sets the role in which to create tokens from. The Token given to // Nomad does not have to be created from this role but must have "update" // capability on "auth/token/create/". If this value is // unset and the token is created from a role, the value is defaulted to the // role the token is from. - Role string `mapstructure:"create_from_role"` + Role string `hcl:"create_from_role"` // Namespace sets the Vault namespace used for all calls against the // Vault API. If this is unset, then Nomad does not use Vault namespaces. @@ -44,16 +44,16 @@ type VaultConfig struct { // AllowUnauthenticated allows users to submit jobs requiring Vault tokens // without providing a Vault token proving they have access to these // policies. - AllowUnauthenticated *bool `mapstructure:"allow_unauthenticated"` + AllowUnauthenticated *bool `hcl:"allow_unauthenticated"` // TaskTokenTTL is the TTL of the tokens created by Nomad Servers and used // by the client. There should be a minimum time value such that the client // does not have to renew with Vault at a very high frequency - TaskTokenTTL string `mapstructure:"task_token_ttl"` + TaskTokenTTL string `hcl:"task_token_ttl"` // Addr is the address of the local Vault agent. This should be a complete // URL such as "http://vault.example.com" - Addr string `mapstructure:"address"` + Addr string `hcl:"address"` // ConnectionRetryIntv is the interval to wait before re-attempting to // connect to Vault. @@ -61,23 +61,23 @@ type VaultConfig struct { // TLSCaFile is the path to a PEM-encoded CA cert file to use to verify the // Vault server SSL certificate. - TLSCaFile string `mapstructure:"ca_file"` + TLSCaFile string `hcl:"ca_file"` // TLSCaFile is the path to a directory of PEM-encoded CA cert files to // verify the Vault server SSL certificate. - TLSCaPath string `mapstructure:"ca_path"` + TLSCaPath string `hcl:"ca_path"` // TLSCertFile is the path to the certificate for Vault communication - TLSCertFile string `mapstructure:"cert_file"` + TLSCertFile string `hcl:"cert_file"` // TLSKeyFile is the path to the private key for Vault communication - TLSKeyFile string `mapstructure:"key_file"` + TLSKeyFile string `hcl:"key_file"` // TLSSkipVerify enables or disables SSL verification - TLSSkipVerify *bool `mapstructure:"tls_skip_verify"` + TLSSkipVerify *bool `hcl:"tls_skip_verify"` // TLSServerName, if set, is used to set the SNI host when connecting via TLS. - TLSServerName string `mapstructure:"tls_server_name"` + TLSServerName string `hcl:"tls_server_name"` } // DefaultVaultConfig() returns the canonical defaults for the Nomad diff --git a/vendor/github.com/hashicorp/hcl/appveyor.yml b/vendor/github.com/hashicorp/hcl/appveyor.yml index 3c8cdf8e977..4db0b711272 100644 --- a/vendor/github.com/hashicorp/hcl/appveyor.yml +++ b/vendor/github.com/hashicorp/hcl/appveyor.yml @@ -4,7 +4,7 @@ clone_folder: c:\gopath\src\github.com\hashicorp\hcl environment: GOPATH: c:\gopath init: - - git config --global core.autocrlf true + - git config --global core.autocrlf false install: - cmd: >- echo %Path% diff --git a/vendor/github.com/hashicorp/hcl/decoder.go b/vendor/github.com/hashicorp/hcl/decoder.go index c8a077d47f4..5e32563c09b 100644 --- a/vendor/github.com/hashicorp/hcl/decoder.go +++ b/vendor/github.com/hashicorp/hcl/decoder.go @@ -89,9 +89,9 @@ func (d *decoder) decode(name string, node ast.Node, result reflect.Value) error switch k.Kind() { case reflect.Bool: return d.decodeBool(name, node, result) - case reflect.Float64: + case reflect.Float32, reflect.Float64: return d.decodeFloat(name, node, result) - case reflect.Int: + case reflect.Int, reflect.Int32, reflect.Int64: return d.decodeInt(name, node, result) case reflect.Interface: // When we see an interface, we make our own thing @@ -117,10 +117,17 @@ func (d *decoder) decode(name string, node ast.Node, result reflect.Value) error func (d *decoder) decodeBool(name string, node ast.Node, result reflect.Value) error { switch n := node.(type) { case *ast.LiteralType: - if n.Token.Type == token.BOOL { - v, err := strconv.ParseBool(n.Token.Text) - if err != nil { - return err + switch n.Token.Type { + case token.BOOL, token.STRING, token.NUMBER: + var v bool + s := strings.ToLower(strings.Replace(n.Token.Text, "\"", "", -1)) + switch s { + case "1", "true": + v = true + case "0", "false": + v = false + default: + return fmt.Errorf("decodeBool: Unknown value for boolean: %s", n.Token.Text) } result.Set(reflect.ValueOf(v)) @@ -137,13 +144,13 @@ func (d *decoder) decodeBool(name string, node ast.Node, result reflect.Value) e func (d *decoder) decodeFloat(name string, node ast.Node, result reflect.Value) error { switch n := node.(type) { case *ast.LiteralType: - if n.Token.Type == token.FLOAT { + if n.Token.Type == token.FLOAT || n.Token.Type == token.NUMBER { v, err := strconv.ParseFloat(n.Token.Text, 64) if err != nil { return err } - result.Set(reflect.ValueOf(v)) + result.Set(reflect.ValueOf(v).Convert(result.Type())) return nil } } @@ -164,7 +171,11 @@ func (d *decoder) decodeInt(name string, node ast.Node, result reflect.Value) er return err } - result.Set(reflect.ValueOf(int(v))) + if result.Kind() == reflect.Interface { + result.Set(reflect.ValueOf(int(v))) + } else { + result.SetInt(v) + } return nil case token.STRING: v, err := strconv.ParseInt(n.Token.Value().(string), 0, 0) @@ -172,7 +183,11 @@ func (d *decoder) decodeInt(name string, node ast.Node, result reflect.Value) er return err } - result.Set(reflect.ValueOf(int(v))) + if result.Kind() == reflect.Interface { + result.Set(reflect.ValueOf(int(v))) + } else { + result.SetInt(v) + } return nil } } @@ -389,6 +404,11 @@ func (d *decoder) decodeMap(name string, node ast.Node, result reflect.Value) er } func (d *decoder) decodePtr(name string, node ast.Node, result reflect.Value) error { + // if pointer is not nil, decode into existing value + if !result.IsNil() { + return d.decode(name, node, result.Elem()) + } + // Create an element of the concrete (non pointer) type and decode // into that. Then set the value of the pointer to this type. resultType := result.Type() @@ -497,7 +517,7 @@ func expandObject(node ast.Node, result reflect.Value) ast.Node { // we need to un-flatten the ast enough to decode newNode := &ast.ObjectItem{ Keys: []*ast.ObjectKey{ - &ast.ObjectKey{ + { Token: keyToken, }, }, @@ -565,7 +585,11 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) // Compile the list of all the fields that we're going to be decoding // from all the structs. - fields := make(map[*reflect.StructField]reflect.Value) + type field struct { + field reflect.StructField + val reflect.Value + } + fields := []field{} for len(structs) > 0 { structVal := structs[0] structs = structs[1:] @@ -608,34 +632,44 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) } // Normal struct field, store it away - fields[&fieldType] = structVal.Field(i) + fields = append(fields, field{fieldType, structVal.Field(i)}) } } - usedKeys := make(map[string]struct{}) decodedFields := make([]string, 0, len(fields)) decodedFieldsVal := make([]reflect.Value, 0) unusedKeysVal := make([]reflect.Value, 0) - for fieldType, field := range fields { - if !field.IsValid() { + + // fill unusedNodeKeys with keys from the AST + // a slice because we have to do equals case fold to match Filter + unusedNodeKeys := make([]string, 0) + for _, item := range list.Items { + for _, k := range item.Keys { + unusedNodeKeys = append(unusedNodeKeys, k.Token.Value().(string)) + } + } + + for _, f := range fields { + field, fieldValue := f.field, f.val + if !fieldValue.IsValid() { // This should never happen panic("field is not valid") } // If we can't set the field, then it is unexported or something, // and we just continue onwards. - if !field.CanSet() { + if !fieldValue.CanSet() { continue } - fieldName := fieldType.Name + fieldName := field.Name - tagValue := fieldType.Tag.Get(tagName) + tagValue := field.Tag.Get(tagName) tagParts := strings.SplitN(tagValue, ",", 2) if len(tagParts) >= 2 { switch tagParts[1] { case "decodedFields": - decodedFieldsVal = append(decodedFieldsVal, field) + decodedFieldsVal = append(decodedFieldsVal, fieldValue) continue case "key": if item == nil { @@ -646,10 +680,10 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) } } - field.SetString(item.Keys[0].Token.Value().(string)) + fieldValue.SetString(item.Keys[0].Token.Value().(string)) continue case "unusedKeys": - unusedKeysVal = append(unusedKeysVal, field) + unusedKeysVal = append(unusedKeysVal, fieldValue) continue } } @@ -669,14 +703,14 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) continue } - // Track the used key - usedKeys[fieldName] = struct{}{} + // Track the used keys + unusedNodeKeys = removeCaseFold(unusedNodeKeys, fieldName) // Create the field name and decode. We range over the elements // because we actually want the value. fieldName = fmt.Sprintf("%s.%s", name, fieldName) if len(prefixMatches.Items) > 0 { - if err := d.decode(fieldName, prefixMatches, field); err != nil { + if err := d.decode(fieldName, prefixMatches, fieldValue); err != nil { return err } } @@ -686,12 +720,12 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) decodeNode = &ast.ObjectList{Items: ot.List.Items} } - if err := d.decode(fieldName, decodeNode, field); err != nil { + if err := d.decode(fieldName, decodeNode, fieldValue); err != nil { return err } } - decodedFields = append(decodedFields, fieldType.Name) + decodedFields = append(decodedFields, field.Name) } if len(decodedFieldsVal) > 0 { @@ -703,6 +737,14 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) } } + if len(unusedNodeKeys) > 0 { + // like decodedFields, populated the unusedKeys field(s) + sort.Strings(unusedNodeKeys) + for _, v := range unusedKeysVal { + v.Set(reflect.ValueOf(unusedNodeKeys)) + } + } + return nil } @@ -714,3 +756,12 @@ func findNodeType() reflect.Type { value := reflect.ValueOf(nodeContainer).FieldByName("Node") return value.Type() } + +func removeCaseFold(xs []string, y string) []string { + for i, x := range xs { + if strings.EqualFold(x, y) { + return append(xs[:i], xs[i+1:]...) + } + } + return xs +} diff --git a/vendor/github.com/hashicorp/hcl/go.mod b/vendor/github.com/hashicorp/hcl/go.mod new file mode 100644 index 00000000000..4debbbe3580 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/go.mod @@ -0,0 +1,3 @@ +module github.com/hashicorp/hcl + +require github.com/davecgh/go-spew v1.1.1 diff --git a/vendor/github.com/hashicorp/hcl/go.sum b/vendor/github.com/hashicorp/hcl/go.sum new file mode 100644 index 00000000000..b5e2922e890 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/go.sum @@ -0,0 +1,2 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/vendor/vendor.json b/vendor/vendor.json index fabf463298e..20de66fd766 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -206,7 +206,7 @@ {"path":"github.com/hashicorp/go-version","checksumSHA1":"r0pj5dMHCghpaQZ3f1BRGoKiSWw=","revision":"b5a281d3160aa11950a6182bd9a9dc2cb1e02d50","revisionTime":"2018-08-24T00:43:55Z"}, {"path":"github.com/hashicorp/golang-lru","checksumSHA1":"d9PxF1XQGLMJZRct2R8qVM/eYlE=","revision":"a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4","revisionTime":"2016-02-07T21:47:19Z"}, {"path":"github.com/hashicorp/golang-lru/simplelru","checksumSHA1":"2nOpYjx8Sn57bqlZq17yM4YJuM4=","revision":"a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4"}, - {"path":"github.com/hashicorp/hcl","checksumSHA1":"8OPDk+bKyRGJoKcS4QNw9F7dpE8=","revision":"6e968a3fcdcbab092f5307fd0d85479d5af1e4dc","revisionTime":"2016-11-01T18:00:25Z"}, + {"path":"github.com/hashicorp/hcl","checksumSHA1":"bPPNuq11pA/YSJzwxS0163WnhCo=","revision":"99e2f22d1c94b272184d97dd9d252866409100ab","revisionTime":"2019-04-30T13:52:23Z"}, {"path":"github.com/hashicorp/hcl/hcl/ast","checksumSHA1":"XQmjDva9JCGGkIecOgwtBEMCJhU=","revision":"6e968a3fcdcbab092f5307fd0d85479d5af1e4dc","revisionTime":"2016-11-01T18:00:25Z"}, {"path":"github.com/hashicorp/hcl/hcl/parser","checksumSHA1":"croNloscHsjX87X+4/cKOURf1EY=","revision":"6e968a3fcdcbab092f5307fd0d85479d5af1e4dc","revisionTime":"2016-11-01T18:00:25Z"}, {"path":"github.com/hashicorp/hcl/hcl/scanner","checksumSHA1":"lgR7PSAZ0RtvAc9OCtCnNsF/x8g=","revision":"6e968a3fcdcbab092f5307fd0d85479d5af1e4dc","revisionTime":"2016-11-01T18:00:25Z"},