From bf613bb6c3b8b526cd4db35051419328b4937a60 Mon Sep 17 00:00:00 2001 From: Chad Patel Date: Fri, 29 Sep 2023 09:50:11 -0500 Subject: [PATCH 01/24] update enhanced CI metrics and aggregations and update tests (#866) --- .../base_container_insights_config.yaml | 6 +- .../sampleConfig/complete_darwin_config.yaml | 4 +- .../sampleConfig/complete_linux_config.yaml | 4 +- .../sampleConfig/complete_windows_config.yaml | 4 +- .../sampleConfig/config_with_env.yaml | 2 +- .../emf_and_kubernetes_config.yaml | 117 +++++++++++---- .../kubernetes_on_prem_config.yaml | 121 ++++++++++++---- .../sampleConfig/log_ecs_metric_only.yaml | 6 +- .../logs_and_kubernetes_config.yaml | 117 +++++++++++---- .../sampleConfig/prometheus_config_linux.yaml | 2 +- .../prometheus_config_windows.yaml | 2 +- .../sampleConfig/trace_config_linux.yaml | 2 +- .../sampleConfig/trace_config_windows.yaml | 2 +- translator/tocwconfig/tocwconfig_test.go | 1 + .../otel/exporter/awsemf/kubernetes.go | 131 +++++++++++++++-- .../otel/exporter/awsemf/translator_test.go | 136 ++++++++++++++---- 16 files changed, 520 insertions(+), 137 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml index 2384ab5cee..7ecd23fe8f 100644 --- a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml @@ -4,7 +4,7 @@ exporters: certificate_file_path: "" emf_only: true endpoint: "https://fake_endpoint" - "imds_retries": 1 + imds_retries: 1 local_mode: false log_group_name: emf/logs/default log_retention: 0 @@ -42,7 +42,7 @@ exporters: certificate_file_path: "" endpoint: "" enhanced_container_insights: false - "imds_retries": 1 + imds_retries: 1 local_mode: false retain_initial_value_of_delta_metric: false parse_json_encoded_attr_values: [ Sources, kubernetes ] @@ -144,7 +144,7 @@ receivers: enable_control_plane_metrics: false certificate_file_path: "" endpoint: "" - "imds_retries": 1 + imds_retries: 1 prefer_full_pod_name: true leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true diff --git a/translator/tocwconfig/sampleConfig/complete_darwin_config.yaml b/translator/tocwconfig/sampleConfig/complete_darwin_config.yaml index ab62048dd0..5c6d95a9d9 100644 --- a/translator/tocwconfig/sampleConfig/complete_darwin_config.yaml +++ b/translator/tocwconfig/sampleConfig/complete_darwin_config.yaml @@ -20,7 +20,7 @@ exporters: emf_only: true certificate_file_path: "" endpoint: "https://logs-fips.us-west-2.amazonaws.com" - "imds_retries": 1 + imds_retries: 1 local_mode: false log_group_name: emf/logs/default log_retention: 0 @@ -49,7 +49,7 @@ exporters: aws_log_groups: [] certificate_file_path: "" endpoint: https://x-ray-endpoint.us-west-2.amazonaws.com - "imds_retries": 1 + imds_retries: 1 index_all_attributes: false indexed_attributes: [] local_mode: true diff --git a/translator/tocwconfig/sampleConfig/complete_linux_config.yaml b/translator/tocwconfig/sampleConfig/complete_linux_config.yaml index e549b1fb0c..d55c341d43 100644 --- a/translator/tocwconfig/sampleConfig/complete_linux_config.yaml +++ b/translator/tocwconfig/sampleConfig/complete_linux_config.yaml @@ -23,7 +23,7 @@ exporters: emf_only: true certificate_file_path: "" endpoint: "https://logs-fips.us-west-2.amazonaws.com" - "imds_retries": 1 + imds_retries: 1 local_mode: false log_group_name: emf/logs/default log_retention: 0 @@ -52,7 +52,7 @@ exporters: aws_log_groups: [] certificate_file_path: "" endpoint: https://x-ray-endpoint.us-west-2.amazonaws.com - "imds_retries": 1 + imds_retries: 1 index_all_attributes: false indexed_attributes: [] local_mode: true diff --git a/translator/tocwconfig/sampleConfig/complete_windows_config.yaml b/translator/tocwconfig/sampleConfig/complete_windows_config.yaml index 16d0b40bc7..284b6113f2 100644 --- a/translator/tocwconfig/sampleConfig/complete_windows_config.yaml +++ b/translator/tocwconfig/sampleConfig/complete_windows_config.yaml @@ -20,7 +20,7 @@ exporters: certificate_file_path: "" emf_only: true endpoint: "https://logs-fips.us-west-2.amazonaws.com" - "imds_retries": 1 + imds_retries: 1 local_mode: false log_group_name: emf/logs/default log_retention: 0 @@ -49,7 +49,7 @@ exporters: aws_log_groups: [] certificate_file_path: "" endpoint: https://x-ray-endpoint.us-west-2.amazonaws.com - "imds_retries": 1 + imds_retries: 1 index_all_attributes: false indexed_attributes: [] local_mode: true diff --git a/translator/tocwconfig/sampleConfig/config_with_env.yaml b/translator/tocwconfig/sampleConfig/config_with_env.yaml index 227fcbfac4..f5863e139b 100644 --- a/translator/tocwconfig/sampleConfig/config_with_env.yaml +++ b/translator/tocwconfig/sampleConfig/config_with_env.yaml @@ -4,7 +4,7 @@ exporters: certificate_file_path: "" emf_only: true endpoint: "" - "imds_retries": 1 + imds_retries: 1 local_mode: false log_group_name: emf/logs/default log_retention: 0 diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index efc671621b..b0f233bef0 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -4,7 +4,7 @@ exporters: emf_only: true certificate_file_path: "" endpoint: "https://fake_endpoint" - "imds_retries": 2 + imds_retries: 2 local_mode: false log_group_name: emf/logs/default log_retention: 0 @@ -36,8 +36,8 @@ exporters: eks_fargate_container_insights_enabled: false certificate_file_path: "" endpoint: "" - "imds_retries": 2 enhanced_container_insights: true + imds_retries: 2 local_mode: false log_group_name: /aws/containerinsights/{ClusterName}/performance log_retention: 0 @@ -53,11 +53,20 @@ exporters: - container_memory_utilization - container_memory_utilization_over_container_limit - container_memory_failures_total + - container_memory_limit + - container_memory_request - container_filesystem_usage + - container_filesystem_available + - container_filesystem_utilization - container_status_running - container_status_terminated - container_status_waiting - - container_status_waiting_reason_crashed + - container_status_waiting_reason_crash_loop_back_off + - container_status_waiting_reason_image_pull_error + - container_status_waiting_reason_start_error + - container_status_waiting_reason_create_container_error + - container_status_waiting_reason_create_container_config_error + - container_status_terminated_reason_oom_killed # pod metrics - dimensions: [ [ ClusterName, Namespace, PodName ], [ ClusterName ], [ ClusterName, Namespace, Service ], [ ClusterName, Namespace ], [ ClusterName, FullPodName, Namespace, PodName ] ] label_matchers: [ ] @@ -72,9 +81,7 @@ exporters: label_matchers: [ ] metric_name_selectors: - pod_interface_network_rx_dropped - - pod_interface_network_rx_errors - pod_interface_network_tx_dropped - - pod_interface_network_tx_errors - dimensions: [ [ ClusterName, Namespace, PodName ], [ ClusterName ], [ ClusterName, FullPodName, Namespace, PodName ], [ ClusterName, Namespace, Service ] ] label_matchers: [] metric_name_selectors: @@ -90,6 +97,8 @@ exporters: - pod_status_failed - pod_status_unknown - pod_status_succeeded + - pod_memory_request + - pod_memory_limit # node metrics - dimensions: [ [ ClusterName, InstanceId, NodeName ], [ ClusterName ] ] label_matchers: [ ] @@ -117,9 +126,7 @@ exporters: label_matchers: [ ] metric_name_selectors: - node_interface_network_rx_dropped - - node_interface_network_rx_errors - node_interface_network_tx_dropped - - node_interface_network_tx_errors - node_diskio_io_service_bytes_total - node_diskio_io_serviced_total # node fs metrics @@ -164,56 +171,118 @@ exporters: - dimensions: [ [ClusterName, endpoint], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: + - apiserver_storage_size_bytes + - apiserver_storage_size_bytes - etcd_db_total_size_in_bytes + - etcd_request_duration_seconds - dimensions: [ [ClusterName, resource], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - apiserver_storage_list_duration_seconds - - dimensions: [ [ClusterName, priority_level], [ ClusterName ] ] + - apiserver_longrunning_requests + - apiserver_storage_objects + - dimensions: [ [ClusterName, verb], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - - apiserver_flowcontrol_request_concurrency_limit - - dimensions: [ [ ClusterName ] ] + - apiserver_request_duration_seconds + - rest_client_request_duration_seconds + - dimensions: [ [ClusterName, code, verb], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - - apiserver_storage_objects - apiserver_request_total - apiserver_request_total_5xx - - apiserver_request_duration_seconds + - dimensions: [ [ClusterName, operation], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: - apiserver_admission_controller_admission_duration_seconds - - rest_client_request_duration_seconds - - rest_client_requests_total + - apiserver_admission_step_admission_duration_seconds - etcd_request_duration_seconds + - dimensions: [ [ClusterName, code, method], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - rest_client_requests_total + - dimensions: [ [ClusterName, request_kind], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_current_inflight_requests + - dimensions: [ [ClusterName, name], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_admission_webhook_admission_duration_seconds + - dimensions: [ [ClusterName, group], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_requested_deprecated_apis + - dimensions: [ [ClusterName, reason], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: - apiserver_flowcontrol_rejected_requests_total + - dimensions: [ [ ClusterName, priority_level ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_flowcontrol_request_concurrency_limit metric_descriptors: - - metric_name: apiserver_storage_objects + - metric_name: apiserver_admission_controller_admission_duration_seconds + unit: Seconds + overwrite: true + - metric_name: apiserver_admission_step_admission_duration_seconds + unit: Seconds + overwrite: true + - metric_name: apiserver_admission_webhook_admission_duration_seconds + unit: Seconds + overwrite: true + - metric_name: apiserver_current_inflight_requests unit: Count overwrite: true - - metric_name: apiserver_request_total + - metric_name: apiserver_current_inqueue_requests unit: Count overwrite: true - - metric_name: apiserver_request_total_5xx + - metric_name: apiserver_flowcontrol_rejected_requests_total + unit: Count + overwrite: true + - metric_name: apiserver_flowcontrol_request_concurrency_limit + unit: Count + overwrite: true + - metric_name: apiserver_longrunning_requests unit: Count overwrite: true - metric_name: apiserver_request_duration_seconds unit: Seconds overwrite: true - - metric_name: apiserver_admission_controller_admission_duration_seconds + - metric_name: apiserver_request_total + unit: Count + overwrite: true + - metric_name: apiserver_request_total_5xx + unit: Count + overwrite: true + - metric_name: apiserver_storage_objects + unit: Count + overwrite: true + - metric_name: etcd_request_duration_seconds unit: Seconds overwrite: true - - metric_name: rest_client_request_duration_seconds + - metric_name: apiserver_storage_list_duration_seconds unit: Seconds overwrite: true - - metric_name: rest_client_requests_total + - metric_name: apiserver_storage_objects unit: Count overwrite: true + - metric_name: apiserver_storage_db_total_size_in_bytes + unit: Bytes + overwrite: true + - metric_name: apiserver_storage_size_bytes + unit: Bytes + overwrite: true + - metric_name: etcd_db_total_size_in_bytes + unit: Bytes + overwrite: true - metric_name: etcd_request_duration_seconds unit: Seconds overwrite: true - - metric_name: apiserver_flowcontrol_rejected_requests_total - unit: Count + - metric_name: rest_client_request_duration_seconds + unit: Seconds overwrite: true - - metric_name: apiserver_flowcontrol_request_concurrency_limit + - metric_name: rest_client_requests_total unit: Count overwrite: true namespace: ContainerInsights @@ -270,7 +339,7 @@ receivers: enable_control_plane_metrics: true certificate_file_path: "" endpoint: "" - "imds_retries": 2 + imds_retries: 2 leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index 9d9cf38de7..69af6a05f5 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -7,8 +7,8 @@ exporters: disable_metric_extraction: true eks_fargate_container_insights_enabled: false endpoint: "" - "imds_retries": 1 enhanced_container_insights: true + imds_retries: 1 local_mode: false log_group_name: /aws/containerinsights/{ClusterName}/performance log_retention: 0 @@ -24,11 +24,20 @@ exporters: - container_memory_utilization - container_memory_utilization_over_container_limit - container_memory_failures_total + - container_memory_limit + - container_memory_request - container_filesystem_usage + - container_filesystem_available + - container_filesystem_utilization - container_status_running - container_status_terminated - container_status_waiting - - container_status_waiting_reason_crashed + - container_status_waiting_reason_crash_loop_back_off + - container_status_waiting_reason_image_pull_error + - container_status_waiting_reason_start_error + - container_status_waiting_reason_create_container_error + - container_status_waiting_reason_create_container_config_error + - container_status_terminated_reason_oom_killed # pod metrics - dimensions: [ [ ClusterName, Namespace, PodName ], [ ClusterName ], [ ClusterName, Namespace, Service ], [ ClusterName, Namespace ], [ ClusterName, FullPodName, Namespace, PodName ] ] label_matchers: [ ] @@ -43,11 +52,9 @@ exporters: label_matchers: [ ] metric_name_selectors: - pod_interface_network_rx_dropped - - pod_interface_network_rx_errors - pod_interface_network_tx_dropped - - pod_interface_network_tx_errors - dimensions: [ [ ClusterName, Namespace, PodName ], [ ClusterName ], [ ClusterName, FullPodName, Namespace, PodName ], [ ClusterName, Namespace, Service ] ] - label_matchers: [] + label_matchers: [ ] metric_name_selectors: - pod_cpu_reserved_capacity - pod_memory_reserved_capacity @@ -61,6 +68,8 @@ exporters: - pod_status_failed - pod_status_unknown - pod_status_succeeded + - pod_memory_request + - pod_memory_limit # node metrics - dimensions: [ [ ClusterName, InstanceId, NodeName ], [ ClusterName ] ] label_matchers: [ ] @@ -88,9 +97,7 @@ exporters: label_matchers: [ ] metric_name_selectors: - node_interface_network_rx_dropped - - node_interface_network_rx_errors - node_interface_network_tx_dropped - - node_interface_network_tx_errors - node_diskio_io_service_bytes_total - node_diskio_io_serviced_total # node fs metrics @@ -132,59 +139,121 @@ exporters: - cluster_failed_node_count - cluster_number_of_running_pods # control plane metrics - - dimensions: [ [ClusterName, endpoint], [ ClusterName ] ] + - dimensions: [ [ ClusterName, endpoint ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: + - apiserver_storage_size_bytes + - apiserver_storage_size_bytes - etcd_db_total_size_in_bytes - - dimensions: [ [ClusterName, resource], [ ClusterName ] ] + - etcd_request_duration_seconds + - dimensions: [ [ ClusterName, resource ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - apiserver_storage_list_duration_seconds - - dimensions: [ [ClusterName, priority_level], [ ClusterName ] ] + - apiserver_longrunning_requests + - apiserver_storage_objects + - dimensions: [ [ ClusterName, verb ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - - apiserver_flowcontrol_request_concurrency_limit - - dimensions: [ [ ClusterName ] ] + - apiserver_request_duration_seconds + - rest_client_request_duration_seconds + - dimensions: [ [ ClusterName, code, verb ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - - apiserver_storage_objects - apiserver_request_total - apiserver_request_total_5xx - - apiserver_request_duration_seconds + - dimensions: [ [ ClusterName, operation ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: - apiserver_admission_controller_admission_duration_seconds - - rest_client_request_duration_seconds - - rest_client_requests_total + - apiserver_admission_step_admission_duration_seconds - etcd_request_duration_seconds + - dimensions: [ [ ClusterName, code, method ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - rest_client_requests_total + - dimensions: [ [ ClusterName, request_kind ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_current_inflight_requests + - dimensions: [ [ ClusterName, name ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_admission_webhook_admission_duration_seconds + - dimensions: [ [ ClusterName, group ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_requested_deprecated_apis + - dimensions: [ [ ClusterName, reason ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: - apiserver_flowcontrol_rejected_requests_total + - dimensions: [ [ ClusterName, priority_level ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_flowcontrol_request_concurrency_limit metric_descriptors: - - metric_name: apiserver_storage_objects + - metric_name: apiserver_admission_controller_admission_duration_seconds + unit: Seconds + overwrite: true + - metric_name: apiserver_admission_step_admission_duration_seconds + unit: Seconds + overwrite: true + - metric_name: apiserver_admission_webhook_admission_duration_seconds + unit: Seconds + overwrite: true + - metric_name: apiserver_current_inflight_requests unit: Count overwrite: true - - metric_name: apiserver_request_total + - metric_name: apiserver_current_inqueue_requests unit: Count overwrite: true - - metric_name: apiserver_request_total_5xx + - metric_name: apiserver_flowcontrol_rejected_requests_total + unit: Count + overwrite: true + - metric_name: apiserver_flowcontrol_request_concurrency_limit + unit: Count + overwrite: true + - metric_name: apiserver_longrunning_requests unit: Count overwrite: true - metric_name: apiserver_request_duration_seconds unit: Seconds overwrite: true - - metric_name: apiserver_admission_controller_admission_duration_seconds + - metric_name: apiserver_request_total + unit: Count + overwrite: true + - metric_name: apiserver_request_total_5xx + unit: Count + overwrite: true + - metric_name: apiserver_storage_objects + unit: Count + overwrite: true + - metric_name: etcd_request_duration_seconds unit: Seconds overwrite: true - - metric_name: rest_client_request_duration_seconds + - metric_name: apiserver_storage_list_duration_seconds unit: Seconds overwrite: true - - metric_name: rest_client_requests_total + - metric_name: apiserver_storage_objects unit: Count overwrite: true + - metric_name: apiserver_storage_db_total_size_in_bytes + unit: Bytes + overwrite: true + - metric_name: apiserver_storage_size_bytes + unit: Bytes + overwrite: true + - metric_name: etcd_db_total_size_in_bytes + unit: Bytes + overwrite: true - metric_name: etcd_request_duration_seconds unit: Seconds overwrite: true - - metric_name: apiserver_flowcontrol_rejected_requests_total - unit: Count + - metric_name: rest_client_request_duration_seconds + unit: Seconds overwrite: true - - metric_name: apiserver_flowcontrol_request_concurrency_limit + - metric_name: rest_client_requests_total unit: Count overwrite: true namespace: ContainerInsights @@ -235,7 +304,7 @@ receivers: container_orchestrator: eks enable_control_plane_metrics: true endpoint: "" - "imds_retries": 1 + imds_retries: 1 leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true local_mode: true diff --git a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml index 7fc58646b2..7025cbffc6 100644 --- a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml +++ b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml @@ -4,7 +4,7 @@ exporters: emf_only: true certificate_file_path: "" endpoint: "https://fake_endpoint" - "imds_retries": 1 + imds_retries: 1 local_mode: false log_group_name: emf/logs/default log_retention: 0 @@ -36,7 +36,7 @@ exporters: eks_fargate_container_insights_enabled: false certificate_file_path: "" endpoint: "" - "imds_retries": 1 + imds_retries: 1 enhanced_container_insights: false local_mode: false log_group_name: /aws/ecs/containerinsights/{ClusterName}/performance @@ -115,7 +115,7 @@ receivers: enable_control_plane_metrics: false certificate_file_path: "" endpoint: "" - "imds_retries": 1 + imds_retries: 1 leader_lock_name: otel-container-insight-clusterleader leader_lock_using_config_map_only: false local_mode: false diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index 13421cbf89..b460f21958 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -53,11 +53,20 @@ exporters: - container_memory_utilization - container_memory_utilization_over_container_limit - container_memory_failures_total + - container_memory_limit + - container_memory_request - container_filesystem_usage + - container_filesystem_available + - container_filesystem_utilization - container_status_running - container_status_terminated - container_status_waiting - - container_status_waiting_reason_crashed + - container_status_waiting_reason_crash_loop_back_off + - container_status_waiting_reason_image_pull_error + - container_status_waiting_reason_start_error + - container_status_waiting_reason_create_container_error + - container_status_waiting_reason_create_container_config_error + - container_status_terminated_reason_oom_killed # pod metrics - dimensions: [ [ ClusterName, Namespace, PodName ], [ ClusterName ], [ ClusterName, Namespace, Service ], [ ClusterName, Namespace ], [ ClusterName, FullPodName, Namespace, PodName ] ] label_matchers: [ ] @@ -72,11 +81,9 @@ exporters: label_matchers: [ ] metric_name_selectors: - pod_interface_network_rx_dropped - - pod_interface_network_rx_errors - pod_interface_network_tx_dropped - - pod_interface_network_tx_errors - dimensions: [ [ ClusterName, Namespace, PodName ], [ ClusterName ], [ ClusterName, FullPodName, Namespace, PodName ], [ ClusterName, Namespace, Service ] ] - label_matchers: [] + label_matchers: [ ] metric_name_selectors: - pod_cpu_reserved_capacity - pod_memory_reserved_capacity @@ -90,6 +97,8 @@ exporters: - pod_status_failed - pod_status_unknown - pod_status_succeeded + - pod_memory_request + - pod_memory_limit # node metrics - dimensions: [ [ ClusterName, InstanceId, NodeName ], [ ClusterName ] ] label_matchers: [ ] @@ -117,9 +126,7 @@ exporters: label_matchers: [ ] metric_name_selectors: - node_interface_network_rx_dropped - - node_interface_network_rx_errors - node_interface_network_tx_dropped - - node_interface_network_tx_errors - node_diskio_io_service_bytes_total - node_diskio_io_serviced_total # node fs metrics @@ -161,59 +168,121 @@ exporters: - cluster_failed_node_count - cluster_number_of_running_pods # control plane metrics - - dimensions: [ [ClusterName, endpoint], [ ClusterName ] ] + - dimensions: [ [ ClusterName, endpoint ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: + - apiserver_storage_size_bytes + - apiserver_storage_size_bytes - etcd_db_total_size_in_bytes - - dimensions: [ [ClusterName, resource], [ ClusterName ] ] + - etcd_request_duration_seconds + - dimensions: [ [ ClusterName, resource ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - apiserver_storage_list_duration_seconds - - dimensions: [ [ClusterName, priority_level], [ ClusterName ] ] + - apiserver_longrunning_requests + - apiserver_storage_objects + - dimensions: [ [ ClusterName, verb ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - - apiserver_flowcontrol_request_concurrency_limit - - dimensions: [ [ ClusterName ] ] + - apiserver_request_duration_seconds + - rest_client_request_duration_seconds + - dimensions: [ [ ClusterName, code, verb ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - - apiserver_storage_objects - apiserver_request_total - apiserver_request_total_5xx - - apiserver_request_duration_seconds + - dimensions: [ [ ClusterName, operation ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: - apiserver_admission_controller_admission_duration_seconds - - rest_client_request_duration_seconds - - rest_client_requests_total + - apiserver_admission_step_admission_duration_seconds - etcd_request_duration_seconds + - dimensions: [ [ ClusterName, code, method ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - rest_client_requests_total + - dimensions: [ [ ClusterName, request_kind ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_current_inflight_requests + - dimensions: [ [ ClusterName, name ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_admission_webhook_admission_duration_seconds + - dimensions: [ [ ClusterName, group ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_requested_deprecated_apis + - dimensions: [ [ ClusterName, reason ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: - apiserver_flowcontrol_rejected_requests_total + - dimensions: [ [ ClusterName, priority_level ], [ ClusterName ] ] + label_matchers: [ ] + metric_name_selectors: + - apiserver_flowcontrol_request_concurrency_limit metric_descriptors: - - metric_name: apiserver_storage_objects + - metric_name: apiserver_admission_controller_admission_duration_seconds + unit: Seconds + overwrite: true + - metric_name: apiserver_admission_step_admission_duration_seconds + unit: Seconds + overwrite: true + - metric_name: apiserver_admission_webhook_admission_duration_seconds + unit: Seconds + overwrite: true + - metric_name: apiserver_current_inflight_requests unit: Count overwrite: true - - metric_name: apiserver_request_total + - metric_name: apiserver_current_inqueue_requests unit: Count overwrite: true - - metric_name: apiserver_request_total_5xx + - metric_name: apiserver_flowcontrol_rejected_requests_total + unit: Count + overwrite: true + - metric_name: apiserver_flowcontrol_request_concurrency_limit + unit: Count + overwrite: true + - metric_name: apiserver_longrunning_requests unit: Count overwrite: true - metric_name: apiserver_request_duration_seconds unit: Seconds overwrite: true - - metric_name: apiserver_admission_controller_admission_duration_seconds + - metric_name: apiserver_request_total + unit: Count + overwrite: true + - metric_name: apiserver_request_total_5xx + unit: Count + overwrite: true + - metric_name: apiserver_storage_objects + unit: Count + overwrite: true + - metric_name: etcd_request_duration_seconds unit: Seconds overwrite: true - - metric_name: rest_client_request_duration_seconds + - metric_name: apiserver_storage_list_duration_seconds unit: Seconds overwrite: true - - metric_name: rest_client_requests_total + - metric_name: apiserver_storage_objects unit: Count overwrite: true + - metric_name: apiserver_storage_db_total_size_in_bytes + unit: Bytes + overwrite: true + - metric_name: apiserver_storage_size_bytes + unit: Bytes + overwrite: true + - metric_name: etcd_db_total_size_in_bytes + unit: Bytes + overwrite: true - metric_name: etcd_request_duration_seconds unit: Seconds overwrite: true - - metric_name: apiserver_flowcontrol_rejected_requests_total - unit: Count + - metric_name: rest_client_request_duration_seconds + unit: Seconds overwrite: true - - metric_name: apiserver_flowcontrol_request_concurrency_limit + - metric_name: rest_client_requests_total unit: Count overwrite: true namespace: ContainerInsights diff --git a/translator/tocwconfig/sampleConfig/prometheus_config_linux.yaml b/translator/tocwconfig/sampleConfig/prometheus_config_linux.yaml index 75b0e082e3..0324968b89 100644 --- a/translator/tocwconfig/sampleConfig/prometheus_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/prometheus_config_linux.yaml @@ -9,7 +9,7 @@ exporters: eks_fargate_container_insights_enabled: false certificate_file_path: "" endpoint: "" - "imds_retries": 1 + imds_retries: 1 enhanced_container_insights: false local_mode: false log_group_name: /aws/ecs/containerinsights/TestCluster/prometheus diff --git a/translator/tocwconfig/sampleConfig/prometheus_config_windows.yaml b/translator/tocwconfig/sampleConfig/prometheus_config_windows.yaml index cf1f191a3b..ccd84ea2fb 100644 --- a/translator/tocwconfig/sampleConfig/prometheus_config_windows.yaml +++ b/translator/tocwconfig/sampleConfig/prometheus_config_windows.yaml @@ -9,7 +9,7 @@ exporters: eks_fargate_container_insights_enabled: false certificate_file_path: "" endpoint: "" - "imds_retries": 1 + imds_retries: 1 enhanced_container_insights: false local_mode: false log_group_name: /aws/ecs/containerinsights/TestCluster/prometheus diff --git a/translator/tocwconfig/sampleConfig/trace_config_linux.yaml b/translator/tocwconfig/sampleConfig/trace_config_linux.yaml index 5ad59a485b..8299964997 100644 --- a/translator/tocwconfig/sampleConfig/trace_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/trace_config_linux.yaml @@ -4,7 +4,7 @@ exporters: aws_log_groups: [] certificate_file_path: "" endpoint: "" - "imds_retries": 2 + imds_retries: 2 index_all_attributes: false indexed_attributes: [] local_mode: false diff --git a/translator/tocwconfig/sampleConfig/trace_config_windows.yaml b/translator/tocwconfig/sampleConfig/trace_config_windows.yaml index 18ed799e2f..8b6738d027 100644 --- a/translator/tocwconfig/sampleConfig/trace_config_windows.yaml +++ b/translator/tocwconfig/sampleConfig/trace_config_windows.yaml @@ -4,7 +4,7 @@ exporters: aws_log_groups: [] certificate_file_path: "" endpoint: "" - "imds_retries": 2 + imds_retries: 2 index_all_attributes: false indexed_attributes: [] local_mode: false diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index fffa845d8c..1c0e010c27 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -350,6 +350,7 @@ func verifyToYamlTranslation(t *testing.T, input interface{}, expectedYamlFilePa return pretty.Sprint(x) < pretty.Sprint(y) }) // assert.Equal(t, expected, actual) // this is useful for debugging differences between the YAML + require.True(t, cmp.Equal(expected, actual, opt), "D! YAML diff: %s", cmp.Diff(expected, actual)) } } diff --git a/translator/translate/otel/exporter/awsemf/kubernetes.go b/translator/translate/otel/exporter/awsemf/kubernetes.go index 7fcc247a16..d9ae7a7674 100644 --- a/translator/translate/otel/exporter/awsemf/kubernetes.go +++ b/translator/translate/otel/exporter/awsemf/kubernetes.go @@ -61,8 +61,11 @@ func getContainerMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.Metric Dimensions: [][]string{{"ClusterName"}, {"ContainerName", "FullPodName", "PodName", "Namespace", "ClusterName"}, {"ContainerName", "PodName", "Namespace", "ClusterName"}}, MetricNameSelectors: []string{ "container_cpu_utilization", "container_cpu_utilization_over_container_limit", - "container_memory_utilization", "container_memory_utilization_over_container_limit", "container_memory_failures_total", - "container_filesystem_usage", "container_status_running", "container_status_terminated", "container_status_waiting", "container_status_waiting_reason_crashed", + "container_memory_utilization", "container_memory_utilization_over_container_limit", "container_memory_failures_total", "container_memory_limit", "container_memory_request", + "container_filesystem_usage", "container_filesystem_available", "container_filesystem_utilization", + "container_status_running", "container_status_terminated", "container_status_waiting", "container_status_waiting_reason_crash_loop_back_off", + "container_status_waiting_reason_image_pull_error", "container_status_waiting_reason_start_error", "container_status_waiting_reason_create_container_error", + "container_status_waiting_reason_create_container_config_error", "container_status_terminated_reason_oom_killed", }, } @@ -91,7 +94,7 @@ func getPodMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDeclar podMetricDeclarations[0].Dimensions = append(podMetricDeclarations[0].Dimensions, []string{"FullPodName", "PodName", "Namespace", "ClusterName"}) selectors = append(selectors, []string{"pod_number_of_container_restarts", "pod_number_of_containers", "pod_number_of_running_containers", "pod_status_ready", "pod_status_scheduled", "pod_status_running", "pod_status_pending", "pod_status_failed", "pod_status_unknown", - "pod_status_succeeded"}...) + "pod_status_succeeded", "pod_memory_request", "pod_memory_limit"}...) } @@ -110,7 +113,7 @@ func getPodMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDeclar {"Service", "Namespace", "ClusterName"}, {"ClusterName"}, }, - MetricNameSelectors: []string{"pod_interface_network_rx_dropped", "pod_interface_network_rx_errors", "pod_interface_network_tx_dropped", "pod_interface_network_tx_errors"}, + MetricNameSelectors: []string{"pod_interface_network_rx_dropped", "pod_interface_network_tx_dropped"}, }, ) } else { @@ -147,9 +150,9 @@ func getNodeMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDecla {"ClusterName"}, }, MetricNameSelectors: []string{ - "node_interface_network_rx_dropped", "node_interface_network_rx_errors", - "node_interface_network_tx_dropped", "node_interface_network_tx_errors", - "node_diskio_io_service_bytes_total", "node_diskio_io_serviced_total"}, + "node_interface_network_rx_dropped", "node_interface_network_tx_dropped", + "node_diskio_io_service_bytes_total", "node_diskio_io_serviced_total", + }, }, } } else { @@ -266,35 +269,78 @@ func getControlPlaneMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.Met { Dimensions: [][]string{{"ClusterName", "endpoint"}, {"ClusterName"}}, MetricNameSelectors: []string{ + "apiserver_storage_size_bytes", + "apiserver_storage_size_bytes", "etcd_db_total_size_in_bytes", + "etcd_request_duration_seconds", }, }, { Dimensions: [][]string{{"ClusterName", "resource"}, {"ClusterName"}}, MetricNameSelectors: []string{ "apiserver_storage_list_duration_seconds", + "apiserver_longrunning_requests", + "apiserver_storage_objects", }, }, { - Dimensions: [][]string{{"ClusterName", "priority_level"}, {"ClusterName"}}, + Dimensions: [][]string{{"ClusterName", "verb"}, {"ClusterName"}}, MetricNameSelectors: []string{ - "apiserver_flowcontrol_request_concurrency_limit", + "apiserver_request_duration_seconds", + "rest_client_request_duration_seconds", }, }, { - Dimensions: [][]string{{"ClusterName"}}, + Dimensions: [][]string{{"ClusterName", "code", "verb"}, {"ClusterName"}}, MetricNameSelectors: []string{ - "apiserver_admission_controller_admission_duration_seconds", - "apiserver_flowcontrol_rejected_requests_total", - "apiserver_request_duration_seconds", "apiserver_request_total", "apiserver_request_total_5xx", - "apiserver_storage_objects", + }, + }, + { + Dimensions: [][]string{{"ClusterName", "operation"}, {"ClusterName"}}, + MetricNameSelectors: []string{ + "apiserver_admission_controller_admission_duration_seconds", + "apiserver_admission_step_admission_duration_seconds", "etcd_request_duration_seconds", - "rest_client_request_duration_seconds", + }, + }, + { + Dimensions: [][]string{{"ClusterName", "code", "method"}, {"ClusterName"}}, + MetricNameSelectors: []string{ "rest_client_requests_total", }, }, + { + Dimensions: [][]string{{"ClusterName", "request_kind"}, {"ClusterName"}}, + MetricNameSelectors: []string{ + "apiserver_current_inflight_requests", + }, + }, + { + Dimensions: [][]string{{"ClusterName", "name"}, {"ClusterName"}}, + MetricNameSelectors: []string{ + "apiserver_admission_webhook_admission_duration_seconds", + }, + }, + { + Dimensions: [][]string{{"ClusterName", "group"}, {"ClusterName"}}, + MetricNameSelectors: []string{ + "apiserver_requested_deprecated_apis", + }, + }, + { + Dimensions: [][]string{{"ClusterName", "reason"}, {"ClusterName"}}, + MetricNameSelectors: []string{ + "apiserver_flowcontrol_rejected_requests_total", + }, + }, + { + Dimensions: [][]string{{"ClusterName", "priority_level"}, {"ClusterName"}}, + MetricNameSelectors: []string{ + "apiserver_flowcontrol_request_concurrency_limit", + }, + }, }...) } return metricDeclarations @@ -310,6 +356,26 @@ func getControlPlaneMetricDescriptors(conf *confmap.Conf) []awsemfexporter.Metri Unit: "Seconds", Overwrite: true, }, + { + MetricName: "apiserver_admission_step_admission_duration_seconds", + Unit: "Seconds", + Overwrite: true, + }, + { + MetricName: "apiserver_admission_webhook_admission_duration_seconds", + Unit: "Seconds", + Overwrite: true, + }, + { + MetricName: "apiserver_current_inflight_requests", + Unit: "Count", + Overwrite: true, + }, + { + MetricName: "apiserver_current_inqueue_requests", + Unit: "Count", + Overwrite: true, + }, { MetricName: "apiserver_flowcontrol_rejected_requests_total", Unit: "Count", @@ -320,6 +386,11 @@ func getControlPlaneMetricDescriptors(conf *confmap.Conf) []awsemfexporter.Metri Unit: "Count", Overwrite: true, }, + { + MetricName: "apiserver_longrunning_requests", + Unit: "Count", + Overwrite: true, + }, { MetricName: "apiserver_request_duration_seconds", Unit: "Seconds", @@ -345,6 +416,36 @@ func getControlPlaneMetricDescriptors(conf *confmap.Conf) []awsemfexporter.Metri Unit: "Seconds", Overwrite: true, }, + { + MetricName: "apiserver_storage_list_duration_seconds", + Unit: "Seconds", + Overwrite: true, + }, + { + MetricName: "apiserver_storage_objects", + Unit: "Count", + Overwrite: true, + }, + { + MetricName: "apiserver_storage_db_total_size_in_bytes", + Unit: "Bytes", + Overwrite: true, + }, + { + MetricName: "apiserver_storage_size_bytes", + Unit: "Bytes", + Overwrite: true, + }, + { + MetricName: "etcd_db_total_size_in_bytes", + Unit: "Bytes", + Overwrite: true, + }, + { + MetricName: "etcd_request_duration_seconds", + Unit: "Seconds", + Overwrite: true, + }, { MetricName: "rest_client_request_duration_seconds", Unit: "Seconds", diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index b6243ffb6a..e935dcb231 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -268,8 +268,12 @@ func TestTranslator(t *testing.T) { Dimensions: [][]string{{"ClusterName"}, {"ContainerName", "FullPodName", "PodName", "Namespace", "ClusterName"}, {"ContainerName", "PodName", "Namespace", "ClusterName"}}, MetricNameSelectors: []string{ "container_cpu_utilization", "container_cpu_utilization_over_container_limit", - "container_memory_utilization", "container_memory_utilization_over_container_limit", "container_memory_failures_total", - "container_filesystem_usage", "container_status_running", "container_status_terminated", "container_status_waiting", "container_status_waiting_reason_crashed"}, + "container_memory_utilization", "container_memory_utilization_over_container_limit", "container_memory_failures_total", "container_memory_limit", "container_memory_request", + "container_filesystem_usage", "container_filesystem_available", "container_filesystem_utilization", + "container_status_running", "container_status_terminated", "container_status_waiting", "container_status_waiting_reason_crash_loop_back_off", + "container_status_waiting_reason_image_pull_error", "container_status_waiting_reason_start_error", "container_status_waiting_reason_create_container_error", + "container_status_waiting_reason_create_container_config_error", "container_status_terminated_reason_oom_killed", + }, }, { Dimensions: [][]string{{"PodName", "Namespace", "ClusterName"}, {"ClusterName"}, {"Service", "Namespace", "ClusterName"}, {"ClusterName", "Namespace"}, {"FullPodName", "PodName", "Namespace", "ClusterName"}}, @@ -277,15 +281,6 @@ func TestTranslator(t *testing.T) { "pod_network_rx_bytes", "pod_network_tx_bytes", "pod_cpu_utilization_over_pod_limit", "pod_memory_utilization_over_pod_limit"}, }, - { - Dimensions: [][]string{{"PodName", "Namespace", "ClusterName"}, {"ClusterName"}, {"FullPodName", "PodName", "Namespace", "ClusterName"}, {"Service", "Namespace", "ClusterName"}}, - MetricNameSelectors: []string{"pod_cpu_reserved_capacity", "pod_memory_reserved_capacity", "pod_number_of_container_restarts", - "pod_number_of_containers", "pod_number_of_running_containers", - "pod_status_ready", "pod_status_scheduled", - "pod_status_running", "pod_status_pending", - "pod_status_failed", "pod_status_unknown", - "pod_status_succeeded"}, - }, { Dimensions: [][]string{ {"FullPodName", "PodName", "Namespace", "ClusterName"}, @@ -293,7 +288,14 @@ func TestTranslator(t *testing.T) { {"Service", "Namespace", "ClusterName"}, {"ClusterName"}, }, - MetricNameSelectors: []string{"pod_interface_network_rx_dropped", "pod_interface_network_rx_errors", "pod_interface_network_tx_dropped", "pod_interface_network_tx_errors"}, + MetricNameSelectors: []string{"pod_interface_network_rx_dropped", "pod_interface_network_tx_dropped"}, + }, + { + Dimensions: [][]string{{"PodName", "Namespace", "ClusterName"}, {"ClusterName"}, {"FullPodName", "PodName", "Namespace", "ClusterName"}, {"Service", "Namespace", "ClusterName"}}, + MetricNameSelectors: []string{"pod_cpu_reserved_capacity", "pod_memory_reserved_capacity", "pod_number_of_container_restarts", "pod_number_of_containers", "pod_number_of_running_containers", + "pod_status_ready", "pod_status_scheduled", "pod_status_running", "pod_status_pending", "pod_status_failed", "pod_status_unknown", + "pod_status_succeeded", "pod_memory_request", "pod_memory_limit", + }, }, { Dimensions: [][]string{{"NodeName", "InstanceId", "ClusterName"}, {"ClusterName"}}, @@ -310,8 +312,7 @@ func TestTranslator(t *testing.T) { {"ClusterName"}, }, MetricNameSelectors: []string{ - "node_interface_network_rx_dropped", "node_interface_network_rx_errors", - "node_interface_network_tx_dropped", "node_interface_network_tx_errors", + "node_interface_network_rx_dropped", "node_interface_network_tx_dropped", "node_diskio_io_service_bytes_total", "node_diskio_io_serviced_total"}, }, { @@ -340,29 +341,47 @@ func TestTranslator(t *testing.T) { }, { Dimensions: [][]string{{"ClusterName", "endpoint"}, {"ClusterName"}}, - MetricNameSelectors: []string{"etcd_db_total_size_in_bytes"}, + MetricNameSelectors: []string{"apiserver_storage_size_bytes", "apiserver_storage_size_bytes", "etcd_db_total_size_in_bytes", "etcd_request_duration_seconds"}, }, { Dimensions: [][]string{{"ClusterName", "resource"}, {"ClusterName"}}, - MetricNameSelectors: []string{"apiserver_storage_list_duration_seconds"}, + MetricNameSelectors: []string{"apiserver_storage_list_duration_seconds", "apiserver_longrunning_requests", "apiserver_storage_objects"}, }, { - Dimensions: [][]string{{"ClusterName", "priority_level"}, {"ClusterName"}}, - MetricNameSelectors: []string{"apiserver_flowcontrol_request_concurrency_limit"}, + Dimensions: [][]string{{"ClusterName", "verb"}, {"ClusterName"}}, + MetricNameSelectors: []string{"apiserver_request_duration_seconds", "rest_client_request_duration_seconds"}, }, { - Dimensions: [][]string{{"ClusterName"}}, - MetricNameSelectors: []string{ - "apiserver_admission_controller_admission_duration_seconds", - "apiserver_flowcontrol_rejected_requests_total", - "apiserver_request_duration_seconds", - "apiserver_request_total", - "apiserver_request_total_5xx", - "apiserver_storage_objects", - "etcd_request_duration_seconds", - "rest_client_request_duration_seconds", - "rest_client_requests_total", - }, + Dimensions: [][]string{{"ClusterName", "code", "verb"}, {"ClusterName"}}, + MetricNameSelectors: []string{"apiserver_request_total", "apiserver_request_total_5xx"}, + }, + { + Dimensions: [][]string{{"ClusterName", "operation"}, {"ClusterName"}}, + MetricNameSelectors: []string{"apiserver_admission_controller_admission_duration_seconds", "apiserver_admission_step_admission_duration_seconds", "etcd_request_duration_seconds"}, + }, + { + Dimensions: [][]string{{"ClusterName", "code", "method"}, {"ClusterName"}}, + MetricNameSelectors: []string{"rest_client_requests_total"}, + }, + { + Dimensions: [][]string{{"ClusterName", "request_kind"}, {"ClusterName"}}, + MetricNameSelectors: []string{"apiserver_current_inflight_requests"}, + }, + { + Dimensions: [][]string{{"ClusterName", "name"}, {"ClusterName"}}, + MetricNameSelectors: []string{"apiserver_admission_webhook_admission_duration_seconds"}, + }, + { + Dimensions: [][]string{{"ClusterName", "group"}, {"ClusterName"}}, + MetricNameSelectors: []string{"apiserver_requested_deprecated_apis"}, + }, + { + Dimensions: [][]string{{"ClusterName", "reason"}, {"ClusterName"}}, + MetricNameSelectors: []string{"apiserver_flowcontrol_rejected_requests_total"}, + }, + { + Dimensions: [][]string{{"ClusterName", "priority_level"}, {"ClusterName"}}, + MetricNameSelectors: []string{"apiserver_flowcontrol_request_concurrency_limit"}, }, }, "metric_descriptors": []awsemfexporter.MetricDescriptor{ @@ -372,7 +391,22 @@ func TestTranslator(t *testing.T) { Overwrite: true, }, { - MetricName: "apiserver_flowcontrol_request_concurrency_limit", + MetricName: "apiserver_admission_step_admission_duration_seconds", + Unit: "Seconds", + Overwrite: true, + }, + { + MetricName: "apiserver_admission_webhook_admission_duration_seconds", + Unit: "Seconds", + Overwrite: true, + }, + { + MetricName: "apiserver_current_inflight_requests", + Unit: "Count", + Overwrite: true, + }, + { + MetricName: "apiserver_current_inqueue_requests", Unit: "Count", Overwrite: true, }, @@ -381,6 +415,16 @@ func TestTranslator(t *testing.T) { Unit: "Count", Overwrite: true, }, + { + MetricName: "apiserver_flowcontrol_request_concurrency_limit", + Unit: "Count", + Overwrite: true, + }, + { + MetricName: "apiserver_longrunning_requests", + Unit: "Count", + Overwrite: true, + }, { MetricName: "apiserver_request_duration_seconds", Unit: "Seconds", @@ -406,6 +450,36 @@ func TestTranslator(t *testing.T) { Unit: "Seconds", Overwrite: true, }, + { + MetricName: "apiserver_storage_list_duration_seconds", + Unit: "Seconds", + Overwrite: true, + }, + { + MetricName: "apiserver_storage_objects", + Unit: "Count", + Overwrite: true, + }, + { + MetricName: "apiserver_storage_db_total_size_in_bytes", + Unit: "Bytes", + Overwrite: true, + }, + { + MetricName: "apiserver_storage_size_bytes", + Unit: "Bytes", + Overwrite: true, + }, + { + MetricName: "etcd_db_total_size_in_bytes", + Unit: "Bytes", + Overwrite: true, + }, + { + MetricName: "etcd_request_duration_seconds", + Unit: "Seconds", + Overwrite: true, + }, { MetricName: "rest_client_request_duration_seconds", Unit: "Seconds", From 697793185639a7420d2b19ab42cb44c2b946ded8 Mon Sep 17 00:00:00 2001 From: Kaushik Surya <108111936+sky333999@users.noreply.github.com> Date: Fri, 29 Sep 2023 17:31:54 -0400 Subject: [PATCH 02/24] Pin github.com/mitchellh/mapstructure to v1.5.0 (#872) --- go.mod | 5 ++++- go.sum | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 17f2bd8250..663325d7e7 100644 --- a/go.mod +++ b/go.mod @@ -64,6 +64,9 @@ replace golang.zx2c4.com/wireguard v0.0.20200121 => golang.zx2c4.com/wireguard v // BurntSushi 0.4.1 do not decode .toml with '[]' into empty slice anymore which breaks confmigrate. replace github.com/BurntSushi/toml v0.4.1 => github.com/BurntSushi/toml v0.3.1 +// To prevent empty slices from overwriting OTel defaults such as telemetry/logs/output_paths (change in behaviour with v1.5.1) +replace github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 => github.com/mitchellh/mapstructure v1.5.0 + replace github.com/karrick/godirwalk v1.16.1 => github.com/karrick/godirwalk v1.12.0 replace github.com/docker/distribution => github.com/docker/distribution v2.8.2+incompatible @@ -104,7 +107,6 @@ require ( github.com/influxdata/wlog v0.0.0-20160411224016-7c63b0a71ef8 github.com/kardianos/service v1.2.1 // Keep this pinned to v1.2.1. v1.2.2 causes the agent to not register as a service on Windows github.com/kr/pretty v0.3.1 - github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect github.com/oklog/run v1.1.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.84.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter v0.84.0 @@ -292,6 +294,7 @@ require ( github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index 70d82c7d3a..87964eb931 100644 --- a/go.sum +++ b/go.sum @@ -876,9 +876,8 @@ github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/z github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY= -github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= From 2c2a9dd884b668742337e166c5537070470acb65 Mon Sep 17 00:00:00 2001 From: Seth L <81644108+sethAmazon@users.noreply.github.com> Date: Mon, 2 Oct 2023 16:25:20 -0400 Subject: [PATCH 03/24] Upgrade Golang Version (#884) --- .github/workflows/PR-build.yml | 4 ++-- .github/workflows/integration-test.yml | 6 +++--- .github/workflows/otel-fork-replace.yml | 2 +- .github/workflows/test-build.yml | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/PR-build.yml b/.github/workflows/PR-build.yml index 9aca3ef1a1..8d178b2a4f 100644 --- a/.github/workflows/PR-build.yml +++ b/.github/workflows/PR-build.yml @@ -47,7 +47,7 @@ jobs: if: needs.changes.outputs.lint == 'true' uses: actions/setup-go@v4 with: - go-version: ~1.20.7 + go-version: ~1.21.1 cache: false - name: Check out code @@ -102,7 +102,7 @@ jobs: if: needs.changes.outputs.build == 'true' uses: actions/setup-go@v4 with: - go-version: ~1.20.7 + go-version: ~1.21.1 cache: false - name: Check out code diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index fb2d8e2a19..ef27fe34d5 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -76,7 +76,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v4 with: - go-version: ~1.20.7 + go-version: ~1.21.1 - name: Generate matrix id: set-matrix @@ -129,7 +129,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v2 with: - go-version: ~1.19.2 + go-version: ~1.21.1 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 @@ -327,7 +327,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v4 with: - go-version: ~1.20.7 + go-version: ~1.21.1 - name: SetOutputs id: set-outputs diff --git a/.github/workflows/otel-fork-replace.yml b/.github/workflows/otel-fork-replace.yml index 15ecbcef7f..eb56e137d7 100644 --- a/.github/workflows/otel-fork-replace.yml +++ b/.github/workflows/otel-fork-replace.yml @@ -30,7 +30,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v4 with: - go-version: ~1.20.7 + go-version: ~1.21.1 cache: false - name: Update OTel fork components version id: set-matrix diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index cc9e89ded3..def7f549b8 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -63,7 +63,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v4 with: - go-version: ~1.20.7 + go-version: ~1.21.1 cache: false - name: Install rpm @@ -161,7 +161,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v4 with: - go-version: ~1.20.7 + go-version: ~1.21.1 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 @@ -226,7 +226,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v4 with: - go-version: ~1.20.7 + go-version: ~1.21.1 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 From 66f1a8cdb5d85d2778e2979c749e93e0cea81415 Mon Sep 17 00:00:00 2001 From: Chad Patel Date: Mon, 2 Oct 2023 17:35:53 -0500 Subject: [PATCH 04/24] fix apiserver_storage_db_total_size_in_bytes (#886) --- .../tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml | 2 +- .../tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml | 2 +- .../tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml | 2 +- translator/translate/otel/exporter/awsemf/kubernetes.go | 2 +- translator/translate/otel/exporter/awsemf/translator_test.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index b0f233bef0..72e0e72ccd 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -172,7 +172,7 @@ exporters: label_matchers: [ ] metric_name_selectors: - apiserver_storage_size_bytes - - apiserver_storage_size_bytes + - apiserver_storage_db_total_size_in_bytes - etcd_db_total_size_in_bytes - etcd_request_duration_seconds - dimensions: [ [ClusterName, resource], [ ClusterName ] ] diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index 69af6a05f5..6c57b41bc5 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -143,7 +143,7 @@ exporters: label_matchers: [ ] metric_name_selectors: - apiserver_storage_size_bytes - - apiserver_storage_size_bytes + - apiserver_storage_db_total_size_in_bytes - etcd_db_total_size_in_bytes - etcd_request_duration_seconds - dimensions: [ [ ClusterName, resource ], [ ClusterName ] ] diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index b460f21958..888495f51b 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -172,7 +172,7 @@ exporters: label_matchers: [ ] metric_name_selectors: - apiserver_storage_size_bytes - - apiserver_storage_size_bytes + - apiserver_storage_db_total_size_in_bytes - etcd_db_total_size_in_bytes - etcd_request_duration_seconds - dimensions: [ [ ClusterName, resource ], [ ClusterName ] ] diff --git a/translator/translate/otel/exporter/awsemf/kubernetes.go b/translator/translate/otel/exporter/awsemf/kubernetes.go index d9ae7a7674..342cd985a4 100644 --- a/translator/translate/otel/exporter/awsemf/kubernetes.go +++ b/translator/translate/otel/exporter/awsemf/kubernetes.go @@ -270,7 +270,7 @@ func getControlPlaneMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.Met Dimensions: [][]string{{"ClusterName", "endpoint"}, {"ClusterName"}}, MetricNameSelectors: []string{ "apiserver_storage_size_bytes", - "apiserver_storage_size_bytes", + "apiserver_storage_db_total_size_in_bytes", "etcd_db_total_size_in_bytes", "etcd_request_duration_seconds", }, diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index e935dcb231..3029d4b03c 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -341,7 +341,7 @@ func TestTranslator(t *testing.T) { }, { Dimensions: [][]string{{"ClusterName", "endpoint"}, {"ClusterName"}}, - MetricNameSelectors: []string{"apiserver_storage_size_bytes", "apiserver_storage_size_bytes", "etcd_db_total_size_in_bytes", "etcd_request_duration_seconds"}, + MetricNameSelectors: []string{"apiserver_storage_size_bytes", "apiserver_storage_db_total_size_in_bytes", "etcd_db_total_size_in_bytes", "etcd_request_duration_seconds"}, }, { Dimensions: [][]string{{"ClusterName", "resource"}, {"ClusterName"}}, From ed88284b1a51203a775eb91959f00e9f88adced4 Mon Sep 17 00:00:00 2001 From: Seth L <81644108+sethAmazon@users.noreply.github.com> Date: Tue, 3 Oct 2023 16:54:35 -0400 Subject: [PATCH 05/24] =?UTF-8?q?Use=20fmt.Printf=20Instead=20Of=20log.Pri?= =?UTF-8?q?ntf=20On=20Start=20Up=20Because=20Of=20A=20Bug=20Wit=E2=80=A6?= =?UTF-8?q?=20(#888)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/retryer/imdsretryer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/retryer/imdsretryer.go b/internal/retryer/imdsretryer.go index d7aaeb9dc4..d03bcba407 100644 --- a/internal/retryer/imdsretryer.go +++ b/internal/retryer/imdsretryer.go @@ -4,6 +4,7 @@ package retryer import ( + "fmt" "log" "os" "strconv" @@ -27,7 +28,7 @@ type IMDSRetryer struct { // otel component layer retries should come from aws config settings // translator layer should come from env vars see GetDefaultRetryNumber() func NewIMDSRetryer(imdsRetries int) IMDSRetryer { - log.Printf("I! imds retry client will retry %d times", imdsRetries) + fmt.Printf("I! imds retry client will retry %d times", imdsRetries) return IMDSRetryer{ DefaultRetryer: client.DefaultRetryer{ NumMaxRetries: imdsRetries, From 11de82c7fb864d76f5ec16943ff449406a926c68 Mon Sep 17 00:00:00 2001 From: Seth L <81644108+sethAmazon@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:34:38 -0400 Subject: [PATCH 06/24] Use GZIP 6 RPM Compression To Fix Install On AL2 Without Kernel Patch (#889) --- packaging/linux/amazon-cloudwatch-agent.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/linux/amazon-cloudwatch-agent.spec b/packaging/linux/amazon-cloudwatch-agent.spec index 4a897c54e5..b0e589e25e 100644 --- a/packaging/linux/amazon-cloudwatch-agent.spec +++ b/packaging/linux/amazon-cloudwatch-agent.spec @@ -10,6 +10,8 @@ Source: amazon-cloudwatch-agent.tar.gz %define _enable_debug_packages 0 %define debug_package %{nil} +%define _source_payload w6.gzdio +%define _binary_payload w6.gzdio %prep %setup -c %{name}-%{version} From 70c071bb9093bd4f63a7630abb002f8660ea8341 Mon Sep 17 00:00:00 2001 From: Seth L <81644108+sethAmazon@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:57:46 -0400 Subject: [PATCH 07/24] If IMDSV2 Fails Log fmt.Printf Not log.Printf (#890) --- internal/retryer/imdsretryer.go | 3 +-- translator/util/ec2util/ec2util.go | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/retryer/imdsretryer.go b/internal/retryer/imdsretryer.go index d03bcba407..29dec2976f 100644 --- a/internal/retryer/imdsretryer.go +++ b/internal/retryer/imdsretryer.go @@ -5,7 +5,6 @@ package retryer import ( "fmt" - "log" "os" "strconv" @@ -44,7 +43,7 @@ func (r IMDSRetryer) ShouldRetry(req *request.Request) bool { if awsError, ok := req.Error.(awserr.Error); r.DefaultRetryer.ShouldRetry(req) || (ok && awsError != nil && awsError.Code() == "EC2MetadataError") { shouldRetry = true } - log.Printf("D! should retry %t for imds error : %v", shouldRetry, req.Error) + fmt.Printf("D! should retry %t for imds error : %v", shouldRetry, req.Error) return shouldRetry } diff --git a/translator/util/ec2util/ec2util.go b/translator/util/ec2util/ec2util.go index b2af3fac74..6b3ce5e37b 100644 --- a/translator/util/ec2util/ec2util.go +++ b/translator/util/ec2util/ec2util.go @@ -5,7 +5,6 @@ package ec2util import ( "fmt" - "log" "net" "sync" "time" @@ -113,7 +112,7 @@ func (e *ec2Util) deriveEC2MetadataFromIMDS() error { if hostname, err := mdDisableFallback.GetMetadata("hostname"); err == nil { e.Hostname = hostname } else { - log.Printf("D! could not get hostname without imds v1 fallback enable thus enable fallback") + fmt.Println("D! could not get hostname without imds v1 fallback enable thus enable fallback") hostnameInner, errInner := mdEnableFallback.GetMetadata("hostname") if errInner == nil { e.Hostname = hostnameInner @@ -130,7 +129,7 @@ func (e *ec2Util) deriveEC2MetadataFromIMDS() error { e.PrivateIP = instanceIdentityDocument.PrivateIP e.InstanceID = instanceIdentityDocument.InstanceID } else { - log.Printf("D! could not get instance document without imds v1 fallback enable thus enable fallback") + fmt.Println("D! could not get instance document without imds v1 fallback enable thus enable fallback") instanceIdentityDocumentInner, errInner := mdEnableFallback.GetInstanceIdentityDocument() if errInner == nil { e.Region = instanceIdentityDocumentInner.Region From f6bc8c02a176cdd24d4d1a9a8c55a6c1fb5ba2f9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 5 Oct 2023 15:38:25 -0400 Subject: [PATCH 08/24] Filter terminated pods from node request metrics. (#882) Co-authored-by: Github Action Co-authored-by: Jeffrey Chien --- go.mod | 28 +++++++++++++++------------- go.sum | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 663325d7e7..c78a0a32cf 100644 --- a/go.mod +++ b/go.mod @@ -6,34 +6,34 @@ replace github.com/influxdata/telegraf => github.com/aws/telegraf v0.10.2-0.2022 // Replace with https://github.com/amazon-contributing/opentelemetry-collector-contrib, there are no requirements for all receivers/processors/exporters // to be all replaced since there are some changes that will always be from upstream -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231002160453-1957d95a68c7 -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231002160453-1957d95a68c7 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231002160453-1957d95a68c7 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231002160453-1957d95a68c7 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231002160453-1957d95a68c7 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231002160453-1957d95a68c7 // Replace with contrib to revert upstream change https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/20519 -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231002160453-1957d95a68c7 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231002160453-1957d95a68c7 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231002160453-1957d95a68c7 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231002160453-1957d95a68c7 -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231002160453-1957d95a68c7 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20230928170322-0df38c533713 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231002160453-1957d95a68c7 -replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230928170322-0df38c533713 +replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231002160453-1957d95a68c7 // Temporary fix, pending PR https://github.com/shirou/gopsutil/pull/957 replace github.com/shirou/gopsutil/v3 => github.com/aws/telegraf/patches/gopsutil/v3 v3.0.0-20230915153624-7629361f8380 // indirect @@ -422,3 +422,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231002160453-1957d95a68c7 diff --git a/go.sum b/go.sum index 87964eb931..9824bc2779 100644 --- a/go.sum +++ b/go.sum @@ -139,34 +139,34 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20230928170322-0df38c533713 h1:wt44oIHsyO9tNSw36V2oWZG4cTzrpqBe7JKyfXHBa5w= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20230928170322-0df38c533713/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20230928170322-0df38c533713 h1:/DboazGzxalMCGykP//7s3m1+YwUaXydlUzX8uIrXUg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20230928170322-0df38c533713/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20230928170322-0df38c533713 h1:+vsrfX+HryDhdmXbbsopiLiTQd4UKQgBEQqZ2qZ/qGw= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20230928170322-0df38c533713/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20230928170322-0df38c533713 h1:7yTgx0xsW4p1JW0aiGI1B0swLJET933/28PCwWODTr0= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20230928170322-0df38c533713/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20230928170322-0df38c533713 h1:oaZq2zS/clGw4fl59WihfY3nRRzmI5gN+7ogDYK9uB4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20230928170322-0df38c533713/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20230928170322-0df38c533713 h1:5u5narTayNWsEqMAlUYcJ+mPWY0J0J00zXTVkpFLUwU= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20230928170322-0df38c533713/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20230928170322-0df38c533713 h1:py3+pCTK1NW0GGarcMnZEgY6J3JxTgKB9BpSQ90r39k= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20230928170322-0df38c533713/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20230928170322-0df38c533713 h1:TrDLQ2BdMFt+bUp6dfzOFhG57AcY85h27UTei+jUdr0= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20230928170322-0df38c533713/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230928170322-0df38c533713 h1:N5zWIyEcfUr+t0EimIukJ0Z341YY0oew7ErHRJ1fnoc= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230928170322-0df38c533713/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231002160453-1957d95a68c7 h1:tPUjPzCJPfB8PKoU/dOybBWNqXE5F2fDOYmXJ2Aw8O4= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231002160453-1957d95a68c7/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231002160453-1957d95a68c7 h1:eF2wBE5Tf25H2KzOfKMphrp2ypt8Bpf6LNyBBF4vVVE= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231002160453-1957d95a68c7/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231002160453-1957d95a68c7 h1:OFtCEHStmq+IhXpCAmZYuJiei4buGrXz7mAdu+YVJCs= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231002160453-1957d95a68c7/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231002160453-1957d95a68c7 h1:NsqMEC9MrJDyMbbFaTB/PSz9ZueNLvB6yg37bRBKQmA= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231002160453-1957d95a68c7/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231002160453-1957d95a68c7 h1:xg/s45OU1LhcmSCxz+7qgXKHljtW7hgu4nQX8muRozw= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231002160453-1957d95a68c7/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231002160453-1957d95a68c7 h1:032FuhRi/l68/Tpx6h4DkUDwVpSIVyYhtxHzxf3JZ+I= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231002160453-1957d95a68c7/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231002160453-1957d95a68c7 h1:Vgsx20V4lcBGLYD4ayubPJ4sCnVlRGC7fn7G+zOEiXo= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231002160453-1957d95a68c7/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231002160453-1957d95a68c7 h1:riMIsVT4iVRbdlZ2oor9nTMPRGAHU+13ttT5d+21UA4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231002160453-1957d95a68c7/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231002160453-1957d95a68c7 h1:oSlkqgBdFuhGLZv5dEDlzWhW+/WUcw5UagtsG+uEvCI= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231002160453-1957d95a68c7/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 h1:2daWNVtWNvRDoCTN5GG5N+LEM9OuY3RjJ0cboU3+xmM= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713/go.mod h1:lJLumMdUeKqurOskauSjhH4J2hz8r0iNyQWDl3i5NSM= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20230928170322-0df38c533713 h1:SZf6K08K4cyoGDdkPthRGcsT1Gk6Z7o6QN01oM8Jr6A= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20230928170322-0df38c533713/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20230928170322-0df38c533713 h1:hDrhgnYst1yL2CLUmlwK1MqH4iqjLfxWZpw85X8cojE= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20230928170322-0df38c533713/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20230928170322-0df38c533713 h1:YPlfoYRq2+HxqibbpC3W+Go4WWJzVKEI5rziSHg/5Fc= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20230928170322-0df38c533713/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20230928170322-0df38c533713 h1:rAzBpcdCtVZqky+w3Nwv7tvTf7osX6WggT2/NEOHrkI= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20230928170322-0df38c533713/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231002160453-1957d95a68c7 h1:gQ29vEeVTF7wr5YP83+82uHBRkRGOhV3NeTwWDjc3dQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231002160453-1957d95a68c7/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231002160453-1957d95a68c7 h1:9hkohcGsQUL0Y80T2iMI88JsR4yFF4rq9LpXuLAATCA= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231002160453-1957d95a68c7/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231002160453-1957d95a68c7 h1:KxLOn4ISFoCZOK+ViryxmqG2eN49ZJCGmwfuvT370Aw= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231002160453-1957d95a68c7/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231002160453-1957d95a68c7 h1:yXCJd4oKpTvFVc4rAij8nfPRNxzo7yrVNDgRp6BOqFE= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231002160453-1957d95a68c7/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE= From 57616e42649485ad6064e94c5a17927e727eba8d Mon Sep 17 00:00:00 2001 From: Chad Patel Date: Thu, 5 Oct 2023 16:09:32 -0500 Subject: [PATCH 09/24] =?UTF-8?q?fix=20some=20aggregations,=20fix=20some?= =?UTF-8?q?=20double=20counted=20stuff,=20add=20in=20some=20mis=E2=80=A6?= =?UTF-8?q?=20(#891)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../emf_and_kubernetes_config.yaml | 14 ++++++------- .../kubernetes_on_prem_config.yaml | 14 ++++++------- .../logs_and_kubernetes_config.yaml | 14 ++++++------- .../otel/exporter/awsemf/kubernetes.go | 21 +++++++------------ .../otel/exporter/awsemf/translator_test.go | 17 ++++++--------- 5 files changed, 35 insertions(+), 45 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index 72e0e72ccd..a2f0bb3c7d 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -50,6 +50,8 @@ exporters: metric_name_selectors: - container_cpu_utilization - container_cpu_utilization_over_container_limit + - container_cpu_limit + - container_cpu_request - container_memory_utilization - container_memory_utilization_over_container_limit - container_memory_failures_total @@ -99,6 +101,8 @@ exporters: - pod_status_succeeded - pod_memory_request - pod_memory_limit + - pod_cpu_limit + - pod_cpu_request # node metrics - dimensions: [ [ ClusterName, InstanceId, NodeName ], [ ClusterName ] ] label_matchers: [ ] @@ -174,7 +178,6 @@ exporters: - apiserver_storage_size_bytes - apiserver_storage_db_total_size_in_bytes - etcd_db_total_size_in_bytes - - etcd_request_duration_seconds - dimensions: [ [ClusterName, resource], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: @@ -255,6 +258,9 @@ exporters: - metric_name: apiserver_request_total_5xx unit: Count overwrite: true + - metric_name: apiserver_requested_deprecated_apis + unit: Count + overwrite: true - metric_name: apiserver_storage_objects unit: Count overwrite: true @@ -264,9 +270,6 @@ exporters: - metric_name: apiserver_storage_list_duration_seconds unit: Seconds overwrite: true - - metric_name: apiserver_storage_objects - unit: Count - overwrite: true - metric_name: apiserver_storage_db_total_size_in_bytes unit: Bytes overwrite: true @@ -276,9 +279,6 @@ exporters: - metric_name: etcd_db_total_size_in_bytes unit: Bytes overwrite: true - - metric_name: etcd_request_duration_seconds - unit: Seconds - overwrite: true - metric_name: rest_client_request_duration_seconds unit: Seconds overwrite: true diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index 6c57b41bc5..9e94b187ec 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -21,6 +21,8 @@ exporters: metric_name_selectors: - container_cpu_utilization - container_cpu_utilization_over_container_limit + - container_cpu_limit + - container_cpu_request - container_memory_utilization - container_memory_utilization_over_container_limit - container_memory_failures_total @@ -70,6 +72,8 @@ exporters: - pod_status_succeeded - pod_memory_request - pod_memory_limit + - pod_cpu_limit + - pod_cpu_request # node metrics - dimensions: [ [ ClusterName, InstanceId, NodeName ], [ ClusterName ] ] label_matchers: [ ] @@ -145,7 +149,6 @@ exporters: - apiserver_storage_size_bytes - apiserver_storage_db_total_size_in_bytes - etcd_db_total_size_in_bytes - - etcd_request_duration_seconds - dimensions: [ [ ClusterName, resource ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: @@ -226,6 +229,9 @@ exporters: - metric_name: apiserver_request_total_5xx unit: Count overwrite: true + - metric_name: apiserver_requested_deprecated_apis + unit: Count + overwrite: true - metric_name: apiserver_storage_objects unit: Count overwrite: true @@ -235,9 +241,6 @@ exporters: - metric_name: apiserver_storage_list_duration_seconds unit: Seconds overwrite: true - - metric_name: apiserver_storage_objects - unit: Count - overwrite: true - metric_name: apiserver_storage_db_total_size_in_bytes unit: Bytes overwrite: true @@ -247,9 +250,6 @@ exporters: - metric_name: etcd_db_total_size_in_bytes unit: Bytes overwrite: true - - metric_name: etcd_request_duration_seconds - unit: Seconds - overwrite: true - metric_name: rest_client_request_duration_seconds unit: Seconds overwrite: true diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index 888495f51b..ef2fa0539d 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -50,6 +50,8 @@ exporters: metric_name_selectors: - container_cpu_utilization - container_cpu_utilization_over_container_limit + - container_cpu_limit + - container_cpu_request - container_memory_utilization - container_memory_utilization_over_container_limit - container_memory_failures_total @@ -99,6 +101,8 @@ exporters: - pod_status_succeeded - pod_memory_request - pod_memory_limit + - pod_cpu_limit + - pod_cpu_request # node metrics - dimensions: [ [ ClusterName, InstanceId, NodeName ], [ ClusterName ] ] label_matchers: [ ] @@ -174,7 +178,6 @@ exporters: - apiserver_storage_size_bytes - apiserver_storage_db_total_size_in_bytes - etcd_db_total_size_in_bytes - - etcd_request_duration_seconds - dimensions: [ [ ClusterName, resource ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: @@ -255,6 +258,9 @@ exporters: - metric_name: apiserver_request_total_5xx unit: Count overwrite: true + - metric_name: apiserver_requested_deprecated_apis + unit: Count + overwrite: true - metric_name: apiserver_storage_objects unit: Count overwrite: true @@ -264,9 +270,6 @@ exporters: - metric_name: apiserver_storage_list_duration_seconds unit: Seconds overwrite: true - - metric_name: apiserver_storage_objects - unit: Count - overwrite: true - metric_name: apiserver_storage_db_total_size_in_bytes unit: Bytes overwrite: true @@ -276,9 +279,6 @@ exporters: - metric_name: etcd_db_total_size_in_bytes unit: Bytes overwrite: true - - metric_name: etcd_request_duration_seconds - unit: Seconds - overwrite: true - metric_name: rest_client_request_duration_seconds unit: Seconds overwrite: true diff --git a/translator/translate/otel/exporter/awsemf/kubernetes.go b/translator/translate/otel/exporter/awsemf/kubernetes.go index 342cd985a4..1e28913f5b 100644 --- a/translator/translate/otel/exporter/awsemf/kubernetes.go +++ b/translator/translate/otel/exporter/awsemf/kubernetes.go @@ -60,7 +60,7 @@ func getContainerMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.Metric metricDeclaration := awsemfexporter.MetricDeclaration{ Dimensions: [][]string{{"ClusterName"}, {"ContainerName", "FullPodName", "PodName", "Namespace", "ClusterName"}, {"ContainerName", "PodName", "Namespace", "ClusterName"}}, MetricNameSelectors: []string{ - "container_cpu_utilization", "container_cpu_utilization_over_container_limit", + "container_cpu_utilization", "container_cpu_utilization_over_container_limit", "container_cpu_limit", "container_cpu_request", "container_memory_utilization", "container_memory_utilization_over_container_limit", "container_memory_failures_total", "container_memory_limit", "container_memory_request", "container_filesystem_usage", "container_filesystem_available", "container_filesystem_utilization", "container_status_running", "container_status_terminated", "container_status_waiting", "container_status_waiting_reason_crash_loop_back_off", @@ -94,7 +94,8 @@ func getPodMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDeclar podMetricDeclarations[0].Dimensions = append(podMetricDeclarations[0].Dimensions, []string{"FullPodName", "PodName", "Namespace", "ClusterName"}) selectors = append(selectors, []string{"pod_number_of_container_restarts", "pod_number_of_containers", "pod_number_of_running_containers", "pod_status_ready", "pod_status_scheduled", "pod_status_running", "pod_status_pending", "pod_status_failed", "pod_status_unknown", - "pod_status_succeeded", "pod_memory_request", "pod_memory_limit"}...) + "pod_status_succeeded", "pod_memory_request", "pod_memory_limit", "pod_cpu_limit", "pod_cpu_request", + }...) } @@ -272,7 +273,6 @@ func getControlPlaneMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.Met "apiserver_storage_size_bytes", "apiserver_storage_db_total_size_in_bytes", "etcd_db_total_size_in_bytes", - "etcd_request_duration_seconds", }, }, { @@ -406,6 +406,11 @@ func getControlPlaneMetricDescriptors(conf *confmap.Conf) []awsemfexporter.Metri Unit: "Count", Overwrite: true, }, + { + MetricName: "apiserver_requested_deprecated_apis", + Unit: "Count", + Overwrite: true, + }, { MetricName: "apiserver_storage_objects", Unit: "Count", @@ -421,11 +426,6 @@ func getControlPlaneMetricDescriptors(conf *confmap.Conf) []awsemfexporter.Metri Unit: "Seconds", Overwrite: true, }, - { - MetricName: "apiserver_storage_objects", - Unit: "Count", - Overwrite: true, - }, { MetricName: "apiserver_storage_db_total_size_in_bytes", Unit: "Bytes", @@ -441,11 +441,6 @@ func getControlPlaneMetricDescriptors(conf *confmap.Conf) []awsemfexporter.Metri Unit: "Bytes", Overwrite: true, }, - { - MetricName: "etcd_request_duration_seconds", - Unit: "Seconds", - Overwrite: true, - }, { MetricName: "rest_client_request_duration_seconds", Unit: "Seconds", diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index 3029d4b03c..877b77d90b 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -267,7 +267,7 @@ func TestTranslator(t *testing.T) { { Dimensions: [][]string{{"ClusterName"}, {"ContainerName", "FullPodName", "PodName", "Namespace", "ClusterName"}, {"ContainerName", "PodName", "Namespace", "ClusterName"}}, MetricNameSelectors: []string{ - "container_cpu_utilization", "container_cpu_utilization_over_container_limit", + "container_cpu_utilization", "container_cpu_utilization_over_container_limit", "container_cpu_limit", "container_cpu_request", "container_memory_utilization", "container_memory_utilization_over_container_limit", "container_memory_failures_total", "container_memory_limit", "container_memory_request", "container_filesystem_usage", "container_filesystem_available", "container_filesystem_utilization", "container_status_running", "container_status_terminated", "container_status_waiting", "container_status_waiting_reason_crash_loop_back_off", @@ -294,7 +294,7 @@ func TestTranslator(t *testing.T) { Dimensions: [][]string{{"PodName", "Namespace", "ClusterName"}, {"ClusterName"}, {"FullPodName", "PodName", "Namespace", "ClusterName"}, {"Service", "Namespace", "ClusterName"}}, MetricNameSelectors: []string{"pod_cpu_reserved_capacity", "pod_memory_reserved_capacity", "pod_number_of_container_restarts", "pod_number_of_containers", "pod_number_of_running_containers", "pod_status_ready", "pod_status_scheduled", "pod_status_running", "pod_status_pending", "pod_status_failed", "pod_status_unknown", - "pod_status_succeeded", "pod_memory_request", "pod_memory_limit", + "pod_status_succeeded", "pod_memory_request", "pod_memory_limit", "pod_cpu_limit", "pod_cpu_request", }, }, { @@ -341,7 +341,7 @@ func TestTranslator(t *testing.T) { }, { Dimensions: [][]string{{"ClusterName", "endpoint"}, {"ClusterName"}}, - MetricNameSelectors: []string{"apiserver_storage_size_bytes", "apiserver_storage_db_total_size_in_bytes", "etcd_db_total_size_in_bytes", "etcd_request_duration_seconds"}, + MetricNameSelectors: []string{"apiserver_storage_size_bytes", "apiserver_storage_db_total_size_in_bytes", "etcd_db_total_size_in_bytes"}, }, { Dimensions: [][]string{{"ClusterName", "resource"}, {"ClusterName"}}, @@ -441,13 +441,13 @@ func TestTranslator(t *testing.T) { Overwrite: true, }, { - MetricName: "apiserver_storage_objects", + MetricName: "apiserver_requested_deprecated_apis", Unit: "Count", Overwrite: true, }, { - MetricName: "etcd_request_duration_seconds", - Unit: "Seconds", + MetricName: "apiserver_storage_objects", + Unit: "Count", Overwrite: true, }, { @@ -455,11 +455,6 @@ func TestTranslator(t *testing.T) { Unit: "Seconds", Overwrite: true, }, - { - MetricName: "apiserver_storage_objects", - Unit: "Count", - Overwrite: true, - }, { MetricName: "apiserver_storage_db_total_size_in_bytes", Unit: "Bytes", From 62d92bc0d6e3b7cfabf8273102f4233870a6064c Mon Sep 17 00:00:00 2001 From: Adam <90734270+adam-mateen@users.noreply.github.com> Date: Fri, 6 Oct 2023 10:34:50 -0500 Subject: [PATCH 10/24] restrict pprof-addr to localhost (#893) --- .../amazon-cloudwatch-agent.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go b/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go index 6fb7a62013..ff6a3f0c99 100644 --- a/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go +++ b/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go @@ -51,7 +51,7 @@ const ( var fDebug = flag.Bool("debug", false, "turn on debug logging") var pprofAddr = flag.String("pprof-addr", "", - "pprof address to listen on, not activate pprof if empty") + "pprof address to listen on, disabled by default, examples: 'localhost:1234', ':4567' (restricted to localhost)") var fQuiet = flag.Bool("quiet", false, "run in quiet mode") var fTest = flag.Bool("test", false, "enable test mode: gather metrics, print them out, and exit") @@ -80,8 +80,6 @@ var fAggregatorFilters = flag.String("aggregator-filter", "", "filter the aggregators to enable, separator is :") var fProcessorFilters = flag.String("processor-filter", "", "filter the processors to enable, separator is :") -var fUsage = flag.String("usage", "", - "print usage for a plugin, ie, 'telegraf --usage mysql'") var fService = flag.String("service", "", "operate on the service (windows only)") var fServiceName = flag.String("service-name", "telegraf", "service name (windows only)") @@ -181,17 +179,17 @@ func reloadLoop( // The "config-translator" program populates that file. func loadEnvironmentVariables(path string) error { if path == "" { - return fmt.Errorf("No env config file specified") + return fmt.Errorf("no env config file specified") } bytes, err := os.ReadFile(path) if err != nil { - return fmt.Errorf("Can't read env config file %s due to: %s", path, err.Error()) + return fmt.Errorf("cannot read env config file %s due to: %s", path, err.Error()) } envVars := map[string]string{} err = json.Unmarshal(bytes, &envVars) if err != nil { - return fmt.Errorf("Can't create env config due to: %s", err.Error()) + return fmt.Errorf("cannot create env config due to: %s", err.Error()) } for key, val := range envVars { @@ -203,7 +201,7 @@ func loadEnvironmentVariables(path string) error { func getEnvConfigPath(configPath, envConfigPath string) (string, error) { if configPath == "" { - return "", fmt.Errorf("No config file specified") + return "", fmt.Errorf("no config file specified") } //load the environment variables that's saved in json env config file if envConfigPath == "" { @@ -437,6 +435,9 @@ func main() { parts := strings.Split(pprofHostPort, ":") if len(parts) == 2 && parts[0] == "" { pprofHostPort = fmt.Sprintf("localhost:%s", parts[1]) + } else if parts[0] != "localhost" { + log.Printf("W! Not starting pprof, it is restricted to localhost:nnnn") + return } pprofHostPort = "http://" + pprofHostPort + "/debug/pprof" @@ -516,6 +517,9 @@ func main() { } envVars[parts[0]] = parts[1] bytes, err = json.MarshalIndent(envVars, "", "\t") + if err != nil { + log.Fatalf("E! Failed to marshal env config: %v", err) + } if err = os.WriteFile(*fEnvConfig, bytes, 0644); err != nil { log.Fatalf("E! Failed to update env config: %v", err) } From a8302f572b28d8a29b41bf088739aedc016c7d38 Mon Sep 17 00:00:00 2001 From: Adam <90734270+adam-mateen@users.noreply.github.com> Date: Mon, 9 Oct 2023 10:28:58 -0500 Subject: [PATCH 11/24] Use default value when -otelconfig option not given (#894) --- .../amazon-cloudwatch-agent.go | 17 +++---- cmd/start-amazon-cloudwatch-agent/path.go | 38 +++++---------- .../path_windows.go | 36 ++------------- .../start-amazon-cloudwatch-agent.go | 46 ++++--------------- tool/paths/paths.go | 27 +++++++++++ tool/paths/paths_unix.go | 14 ++++++ tool/paths/paths_windows.go | 28 +++++++++++ 7 files changed, 104 insertions(+), 102 deletions(-) create mode 100644 tool/paths/paths.go diff --git a/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go b/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go index ff6a3f0c99..0cae50afe0 100644 --- a/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go +++ b/cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go @@ -42,6 +42,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/service/configprovider" "github.com/aws/amazon-cloudwatch-agent/service/defaultcomponents" "github.com/aws/amazon-cloudwatch-agent/service/registry" + "github.com/aws/amazon-cloudwatch-agent/tool/paths" ) const ( @@ -57,8 +58,8 @@ var fQuiet = flag.Bool("quiet", false, var fTest = flag.Bool("test", false, "enable test mode: gather metrics, print them out, and exit") var fTestWait = flag.Int("test-wait", 0, "wait up to this many seconds for service inputs to complete in test mode") var fSchemaTest = flag.Bool("schematest", false, "validate the toml file schema") -var fConfig = flag.String("config", "", "configuration file to load") -var fOtelConfig = flag.String("otelconfig", "", "YAML configuration file to run OTel pipeline") +var fTomlConfig = flag.String("config", "", "configuration file to load") +var fOtelConfig = flag.String("otelconfig", paths.YamlConfigPath, "YAML configuration file to run OTel pipeline") var fEnvConfig = flag.String("envconfig", "", "env configuration file to load") var fConfigDirectory = flag.String("config-directory", "", "directory containing additional *.conf files") @@ -135,7 +136,7 @@ func reloadLoop( } }(ctx) - if envConfigPath, err := getEnvConfigPath(*fConfig, *fEnvConfig); err == nil { + if envConfigPath, err := getEnvConfigPath(*fTomlConfig, *fEnvConfig); err == nil { // Reloads environment variables when file is changed go func(ctx context.Context, envConfigPath string) { var previousModTime time.Time @@ -215,7 +216,7 @@ func runAgent(ctx context.Context, inputFilters []string, outputFilters []string, ) error { - envConfigPath, err := getEnvConfigPath(*fConfig, *fEnvConfig) + envConfigPath, err := getEnvConfigPath(*fTomlConfig, *fEnvConfig) if err != nil { return err } @@ -554,8 +555,8 @@ func main() { // Handle the --service flag here to prevent any issues with tooling that // may not have an interactive session, e.g. installing from Ansible. if *fService != "" { - if *fConfig != "" { - svcConfig.Arguments = []string{"--config", *fConfig} + if *fTomlConfig != "" { + svcConfig.Arguments = []string{"--config", *fTomlConfig} } if *fConfigDirectory != "" { svcConfig.Arguments = append(svcConfig.Arguments, "--config-directory", *fConfigDirectory) @@ -606,7 +607,7 @@ func windowsRunAsService() bool { } func loadTomlConfigIntoAgent(c *config.Config) error { - err := c.LoadConfig(*fConfig) + err := c.LoadConfig(*fTomlConfig) if err != nil { return err } @@ -637,7 +638,7 @@ func validateAgentFinalConfigAndPlugins(c *config.Config) error { if *fSchemaTest { //up to this point, the given config file must be valid fmt.Println(agentinfo.FullVersion()) - fmt.Printf("The given config: %v is valid\n", *fConfig) + fmt.Printf("The given config: %v is valid\n", *fTomlConfig) os.Exit(0) } diff --git a/cmd/start-amazon-cloudwatch-agent/path.go b/cmd/start-amazon-cloudwatch-agent/path.go index 3383b06777..bf17673c52 100644 --- a/cmd/start-amazon-cloudwatch-agent/path.go +++ b/cmd/start-amazon-cloudwatch-agent/path.go @@ -24,13 +24,13 @@ func startAgent(writer io.WriteCloser) error { if os.Getenv(config.RUN_IN_CONTAINER) == config.RUN_IN_CONTAINER_TRUE { // Use exec so PID 1 changes to agent from start-agent. execArgs := []string{ - agentBinaryPath, // when using syscall.Exec, must pass binary name as args[0] - "-config", tomlConfigPath, - "-envconfig", envConfigPath, - "-otelconfig", yamlConfigPath, + paths.AgentBinaryPath, // when using syscall.Exec, must pass binary name as args[0] + "-config", paths.TomlConfigPath, + "-envconfig", paths.EnvConfigPath, + "-otelconfig", paths.YamlConfigPath, "-pidfile", paths.AgentDir + "/var/amazon-cloudwatch-agent.pid", } - if err := syscall.Exec(agentBinaryPath, execArgs, os.Environ()); err != nil { + if err := syscall.Exec(paths.AgentBinaryPath, execArgs, os.Environ()); err != nil { return fmt.Errorf("error exec as agent binary: %w", err) } // We should never reach this line but the compiler doesn't know... @@ -49,7 +49,7 @@ func startAgent(writer io.WriteCloser) error { return err } - name, err := exec.LookPath(agentBinaryPath) + name, err := exec.LookPath(paths.AgentBinaryPath) if err != nil { log.Printf("E! Failed to lookpath: %v ", err) return err @@ -62,10 +62,10 @@ func startAgent(writer io.WriteCloser) error { // linux command has pid passed while windows does not agentCmd := []string{ - agentBinaryPath, - "-config", tomlConfigPath, - "-envconfig", envConfigPath, - "-otelconfig", yamlConfigPath, + paths.AgentBinaryPath, + "-config", paths.TomlConfigPath, + "-envconfig", paths.EnvConfigPath, + "-otelconfig", paths.YamlConfigPath, "-pidfile", paths.AgentDir + "/var/amazon-cloudwatch-agent.pid", } if err = syscall.Exec(name, agentCmd, os.Environ()); err != nil { @@ -80,22 +80,8 @@ func startAgent(writer io.WriteCloser) error { func generateMergedJsonConfigMap() (map[string]interface{}, error) { ctx := context.CurrentContext() setCTXOS(ctx) - ctx.SetInputJsonFilePath(jsonConfigPath) - ctx.SetInputJsonDirPath(jsonDirPath) + ctx.SetInputJsonFilePath(paths.JsonConfigPath) + ctx.SetInputJsonDirPath(paths.JsonDirPath) ctx.SetMultiConfig("remove") return cmdutil.GenerateMergedJsonConfigMap(ctx) } - -func init() { - jsonConfigPath = paths.AgentDir + "/etc/" + JSON - jsonDirPath = paths.AgentDir + "/etc/" + paths.JsonDir - envConfigPath = paths.AgentDir + "/etc/" + ENV - tomlConfigPath = paths.AgentDir + "/etc/" + TOML - commonConfigPath = paths.AgentDir + "/etc/" + COMMON_CONFIG - yamlConfigPath = paths.AgentDir + "/etc/" + YAML - - agentLogFilePath = paths.AgentDir + "/logs/" + AGENT_LOG_FILE - - translatorBinaryPath = paths.AgentDir + "/bin/" + paths.TranslatorBinaryName - agentBinaryPath = paths.AgentDir + "/bin/" + paths.AgentBinaryName -} diff --git a/cmd/start-amazon-cloudwatch-agent/path_windows.go b/cmd/start-amazon-cloudwatch-agent/path_windows.go index 549058b5b4..19c01a2ef7 100644 --- a/cmd/start-amazon-cloudwatch-agent/path_windows.go +++ b/cmd/start-amazon-cloudwatch-agent/path_windows.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "log" - "os" "os/exec" "github.com/aws/amazon-cloudwatch-agent/tool/paths" @@ -23,40 +22,13 @@ func startAgent(writer io.WriteCloser) error { } cmd := exec.Command( - agentBinaryPath, - "-config", tomlConfigPath, - "-envconfig", envConfigPath, - "-otelconfig", yamlConfigPath, + paths.AgentBinaryPath, + "-config", paths.TomlConfigPath, + "-envconfig", paths.EnvConfigPath, + "-otelconfig", paths.YamlConfigPath, ) stdoutStderr, err := cmd.CombinedOutput() // log file is closed, so use fmt here fmt.Printf("%s \n", stdoutStderr) return err } - -func init() { - programFiles := os.Getenv("ProgramFiles") - var programData string - if _, ok := os.LookupEnv("ProgramData"); ok { - programData = os.Getenv("ProgramData") - } else { - // Windows 2003 - programData = os.Getenv("ALLUSERSPROFILE") + "\\Application Data" - } - - agentRootDir := programFiles + paths.AgentDir - agentConfigDir := programData + paths.AgentDir - - jsonConfigPath = agentConfigDir + "\\" + JSON - jsonDirPath = agentConfigDir + paths.JsonDir - envConfigPath = agentConfigDir + "\\" + ENV - tomlConfigPath = agentConfigDir + "\\" + TOML - yamlConfigPath = agentConfigDir + "\\" + YAML - - commonConfigPath = agentConfigDir + "\\" + COMMON_CONFIG - - agentLogFilePath = agentConfigDir + "\\Logs\\" + AGENT_LOG_FILE - - translatorBinaryPath = agentRootDir + "\\" + paths.TranslatorBinaryName - agentBinaryPath = agentRootDir + "\\" + paths.AgentBinaryName -} diff --git a/cmd/start-amazon-cloudwatch-agent/start-amazon-cloudwatch-agent.go b/cmd/start-amazon-cloudwatch-agent/start-amazon-cloudwatch-agent.go index d6fa6f0457..0c23787f06 100644 --- a/cmd/start-amazon-cloudwatch-agent/start-amazon-cloudwatch-agent.go +++ b/cmd/start-amazon-cloudwatch-agent/start-amazon-cloudwatch-agent.go @@ -14,47 +14,21 @@ import ( "gopkg.in/natefinch/lumberjack.v2" + "github.com/aws/amazon-cloudwatch-agent/tool/paths" "github.com/aws/amazon-cloudwatch-agent/translator/config" ) -const ( - COMMON_CONFIG = "common-config.toml" - JSON = "amazon-cloudwatch-agent.json" - TOML = "amazon-cloudwatch-agent.toml" - YAML = "amazon-cloudwatch-agent.yaml" - ENV = "env-config.json" - - AGENT_LOG_FILE = "amazon-cloudwatch-agent.log" - - //TODO this CONFIG_DIR_IN_CONTAINER should change to something indicate dir, keep it for now to avoid break testing - CONFIG_DIR_IN_CONTAINER = "/etc/cwagentconfig" -) - -var ( - jsonConfigPath string - jsonDirPath string - envConfigPath string - tomlConfigPath string - commonConfigPath string - yamlConfigPath string - - agentLogFilePath string - - translatorBinaryPath string - agentBinaryPath string -) - // We use an environment variable here because we need this condition before the translator reads agent config json file. var runInContainer = os.Getenv(config.RUN_IN_CONTAINER) func translateConfig() error { - args := []string{"--output", tomlConfigPath, "--mode", "auto"} + args := []string{"--output", paths.TomlConfigPath, "--mode", "auto"} if runInContainer == config.RUN_IN_CONTAINER_TRUE { - args = append(args, "--input-dir", CONFIG_DIR_IN_CONTAINER) + args = append(args, "--input-dir", paths.CONFIG_DIR_IN_CONTAINER) } else { - args = append(args, "--input", jsonConfigPath, "--input-dir", jsonDirPath, "--config", commonConfigPath) + args = append(args, "--input", paths.JsonConfigPath, "--input-dir", paths.JsonDirPath, "--config", paths.CommonConfigPath) } - cmd := exec.Command(translatorBinaryPath, args...) + cmd := exec.Command(paths.TranslatorBinaryPath, args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stdout err := cmd.Run() @@ -83,7 +57,7 @@ func main() { if runInContainer != config.RUN_IN_CONTAINER_TRUE { writer = &lumberjack.Logger{ - Filename: agentLogFilePath, + Filename: paths.AgentLogFilePath, MaxSize: 100, //MB MaxBackups: 5, //backup files MaxAge: 7, //days @@ -96,10 +70,10 @@ func main() { if err := translateConfig(); err != nil { log.Fatalf("E! Cannot translate JSON, ERROR is %v \n", err) } - log.Printf("I! Config has been translated into TOML %s \n", tomlConfigPath) - printFileContents(tomlConfigPath) - log.Printf("I! Config has been translated into YAML %s \n", yamlConfigPath) - printFileContents(yamlConfigPath) + log.Printf("I! Config has been translated into TOML %s \n", paths.TomlConfigPath) + printFileContents(paths.TomlConfigPath) + log.Printf("I! Config has been translated into YAML %s \n", paths.YamlConfigPath) + printFileContents(paths.YamlConfigPath) if err := startAgent(writer); err != nil { log.Printf("E! Error when starting Agent, Error is %v \n", err) diff --git a/tool/paths/paths.go b/tool/paths/paths.go new file mode 100644 index 0000000000..26543934fb --- /dev/null +++ b/tool/paths/paths.go @@ -0,0 +1,27 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package paths + +const ( + COMMON_CONFIG = "common-config.toml" + JSON = "amazon-cloudwatch-agent.json" + TOML = "amazon-cloudwatch-agent.toml" + YAML = "amazon-cloudwatch-agent.yaml" + ENV = "env-config.json" + AGENT_LOG_FILE = "amazon-cloudwatch-agent.log" + //TODO this CONFIG_DIR_IN_CONTAINER should change to something indicate dir, keep it for now to avoid break testing + CONFIG_DIR_IN_CONTAINER = "/etc/cwagentconfig" +) + +var ( + JsonConfigPath string + JsonDirPath string + EnvConfigPath string + TomlConfigPath string + CommonConfigPath string + YamlConfigPath string + AgentLogFilePath string + TranslatorBinaryPath string + AgentBinaryPath string +) diff --git a/tool/paths/paths_unix.go b/tool/paths/paths_unix.go index 111b0d622f..c59c70232a 100644 --- a/tool/paths/paths_unix.go +++ b/tool/paths/paths_unix.go @@ -6,6 +6,8 @@ package paths +import "path/filepath" + const ( AgentDir = "/opt/aws/amazon-cloudwatch-agent" BinaryDir = "bin" @@ -15,3 +17,15 @@ const ( WizardBinaryName = "amazon-cloudwatch-agent-config-wizard" AgentStartName = "amazon-cloudwatch-agent-ctl" ) + +func init() { + JsonConfigPath = filepath.Join(AgentDir, "etc", JSON) + JsonDirPath = filepath.Join(AgentDir, "etc", JsonDir) + EnvConfigPath = filepath.Join(AgentDir, "etc", ENV) + TomlConfigPath = filepath.Join(AgentDir, "etc", TOML) + CommonConfigPath = filepath.Join(AgentDir, "etc", COMMON_CONFIG) + YamlConfigPath = filepath.Join(AgentDir, "etc", YAML) + AgentLogFilePath = filepath.Join(AgentDir, "logs", AGENT_LOG_FILE) + TranslatorBinaryPath = filepath.Join(AgentDir, "bin", TranslatorBinaryName) + AgentBinaryPath = filepath.Join(AgentDir, "bin", AgentBinaryName) +} diff --git a/tool/paths/paths_windows.go b/tool/paths/paths_windows.go index ae96449be2..69c51ac889 100644 --- a/tool/paths/paths_windows.go +++ b/tool/paths/paths_windows.go @@ -6,6 +6,11 @@ package paths +import ( + "os" + "path/filepath" +) + const ( AgentDir = "\\Amazon\\AmazonCloudWatchAgent\\" JsonDir = "\\Configs" @@ -15,3 +20,26 @@ const ( WizardBinaryName = "amazon-cloudwatch-agent-config-wizard.exe" AgentStartName = "amazon-cloudwatch-agent-ctl.ps1" ) + +func init() { + programFiles := os.Getenv("ProgramFiles") + var programData string + if _, ok := os.LookupEnv("ProgramData"); ok { + programData = os.Getenv("ProgramData") + } else { + // Windows 2003 + programData = filepath.Join(os.Getenv("ALLUSERSPROFILE"), "Application Data") + } + + AgentRootDir := filepath.Join(programFiles, AgentDir) + AgentConfigDir := filepath.Join(programData, AgentDir) + JsonConfigPath = filepath.Join(AgentConfigDir, JSON) + JsonDirPath = filepath.Join(AgentConfigDir, JsonDir) + EnvConfigPath = filepath.Join(AgentConfigDir, ENV) + TomlConfigPath = filepath.Join(AgentConfigDir, TOML) + YamlConfigPath = filepath.Join(AgentConfigDir, YAML) + CommonConfigPath = filepath.Join(AgentConfigDir, COMMON_CONFIG) + AgentLogFilePath = filepath.Join(AgentConfigDir, AGENT_LOG_FILE) + TranslatorBinaryPath = filepath.Join(AgentRootDir, TranslatorBinaryName) + AgentBinaryPath = filepath.Join(AgentRootDir, AgentBinaryName) +} From 5c48d66fc60b418febdbee7570c600485b0f776e Mon Sep 17 00:00:00 2001 From: Chad Patel Date: Mon, 9 Oct 2023 12:29:43 -0500 Subject: [PATCH 12/24] add missing apiserver_current_inqueue_requests declaration (#895) --- .../tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml | 1 + .../tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml | 1 + .../tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml | 1 + translator/translate/otel/exporter/awsemf/kubernetes.go | 1 + translator/translate/otel/exporter/awsemf/translator_test.go | 2 +- 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index a2f0bb3c7d..7fe1138031 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -208,6 +208,7 @@ exporters: label_matchers: [ ] metric_name_selectors: - apiserver_current_inflight_requests + - apiserver_current_inqueue_requests - dimensions: [ [ClusterName, name], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index 9e94b187ec..3cc73c82af 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -179,6 +179,7 @@ exporters: label_matchers: [ ] metric_name_selectors: - apiserver_current_inflight_requests + - apiserver_current_inqueue_requests - dimensions: [ [ ClusterName, name ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index ef2fa0539d..5b038168ff 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -208,6 +208,7 @@ exporters: label_matchers: [ ] metric_name_selectors: - apiserver_current_inflight_requests + - apiserver_current_inqueue_requests - dimensions: [ [ ClusterName, name ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: diff --git a/translator/translate/otel/exporter/awsemf/kubernetes.go b/translator/translate/otel/exporter/awsemf/kubernetes.go index 1e28913f5b..eb083953ad 100644 --- a/translator/translate/otel/exporter/awsemf/kubernetes.go +++ b/translator/translate/otel/exporter/awsemf/kubernetes.go @@ -315,6 +315,7 @@ func getControlPlaneMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.Met Dimensions: [][]string{{"ClusterName", "request_kind"}, {"ClusterName"}}, MetricNameSelectors: []string{ "apiserver_current_inflight_requests", + "apiserver_current_inqueue_requests", }, }, { diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index 877b77d90b..72b0867bbf 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -365,7 +365,7 @@ func TestTranslator(t *testing.T) { }, { Dimensions: [][]string{{"ClusterName", "request_kind"}, {"ClusterName"}}, - MetricNameSelectors: []string{"apiserver_current_inflight_requests"}, + MetricNameSelectors: []string{"apiserver_current_inflight_requests", "apiserver_current_inqueue_requests"}, }, { Dimensions: [][]string{{"ClusterName", "name"}, {"ClusterName"}}, From d31f1fbbd8b64b6e4606bdb46e93ddce0de15f52 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 12:29:52 -0500 Subject: [PATCH 13/24] Update OTel fork components to 4b74f352a68968c240642dfb54049e9e9dd7ce1d (#892) Co-authored-by: Github Action --- go.mod | 28 ++++++++++++++-------------- go.sum | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index c78a0a32cf..494c82bbb8 100644 --- a/go.mod +++ b/go.mod @@ -6,34 +6,34 @@ replace github.com/influxdata/telegraf => github.com/aws/telegraf v0.10.2-0.2022 // Replace with https://github.com/amazon-contributing/opentelemetry-collector-contrib, there are no requirements for all receivers/processors/exporters // to be all replaced since there are some changes that will always be from upstream -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231005180140-4b74f352a689 -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231005180140-4b74f352a689 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231005180140-4b74f352a689 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231005180140-4b74f352a689 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231005180140-4b74f352a689 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231005180140-4b74f352a689 // Replace with contrib to revert upstream change https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/20519 -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231005180140-4b74f352a689 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231005180140-4b74f352a689 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231005180140-4b74f352a689 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231005180140-4b74f352a689 -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231005180140-4b74f352a689 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231002160453-1957d95a68c7 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231005180140-4b74f352a689 -replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231002160453-1957d95a68c7 +replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231005180140-4b74f352a689 // Temporary fix, pending PR https://github.com/shirou/gopsutil/pull/957 replace github.com/shirou/gopsutil/v3 => github.com/aws/telegraf/patches/gopsutil/v3 v3.0.0-20230915153624-7629361f8380 // indirect @@ -423,4 +423,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231002160453-1957d95a68c7 +replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231005180140-4b74f352a689 diff --git a/go.sum b/go.sum index 9824bc2779..7fd27813d0 100644 --- a/go.sum +++ b/go.sum @@ -139,34 +139,34 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231002160453-1957d95a68c7 h1:tPUjPzCJPfB8PKoU/dOybBWNqXE5F2fDOYmXJ2Aw8O4= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231002160453-1957d95a68c7/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231002160453-1957d95a68c7 h1:eF2wBE5Tf25H2KzOfKMphrp2ypt8Bpf6LNyBBF4vVVE= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231002160453-1957d95a68c7/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231002160453-1957d95a68c7 h1:OFtCEHStmq+IhXpCAmZYuJiei4buGrXz7mAdu+YVJCs= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231002160453-1957d95a68c7/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231002160453-1957d95a68c7 h1:NsqMEC9MrJDyMbbFaTB/PSz9ZueNLvB6yg37bRBKQmA= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231002160453-1957d95a68c7/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231002160453-1957d95a68c7 h1:xg/s45OU1LhcmSCxz+7qgXKHljtW7hgu4nQX8muRozw= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231002160453-1957d95a68c7/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231002160453-1957d95a68c7 h1:032FuhRi/l68/Tpx6h4DkUDwVpSIVyYhtxHzxf3JZ+I= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231002160453-1957d95a68c7/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231002160453-1957d95a68c7 h1:Vgsx20V4lcBGLYD4ayubPJ4sCnVlRGC7fn7G+zOEiXo= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231002160453-1957d95a68c7/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231002160453-1957d95a68c7 h1:riMIsVT4iVRbdlZ2oor9nTMPRGAHU+13ttT5d+21UA4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231002160453-1957d95a68c7/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231002160453-1957d95a68c7 h1:oSlkqgBdFuhGLZv5dEDlzWhW+/WUcw5UagtsG+uEvCI= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231002160453-1957d95a68c7/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231005180140-4b74f352a689 h1:C41hepLgo3qoZ/raJJXwg5+61r1T0wKGe7ouq2GTDCQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231005180140-4b74f352a689/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231005180140-4b74f352a689 h1:PB5QDEQZeC7Wuvmuwd6mlJbjAbU7d1jNe91Ro1sieBY= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231005180140-4b74f352a689/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231005180140-4b74f352a689 h1:Hs0LHi+VUmWUlTBrc3dp8moDSsIElpbUfWWezE4yPdg= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231005180140-4b74f352a689/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231005180140-4b74f352a689 h1:PE+2GVB1EVsRaqShTpyunefheg82SklTOeLsYIXveus= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231005180140-4b74f352a689/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231005180140-4b74f352a689 h1:H7WeHG6ERGOgPCS01RjwMo3Faja60PbozpPXDhx3OGI= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231005180140-4b74f352a689/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231005180140-4b74f352a689 h1:hpv83ENe81wZmLC93U90hT5WzbOzLCY8EhxAmvUjMuo= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231005180140-4b74f352a689/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231005180140-4b74f352a689 h1:MVjE0TuGzjWvWbphb/UQhhKz2+XtRjnhEJF20ht/D30= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231005180140-4b74f352a689/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231005180140-4b74f352a689 h1:XYgIXvOXzjDUvkw0KQ8HjGmqW2+3fLFOhwtWRZcn5YU= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231005180140-4b74f352a689/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231005180140-4b74f352a689 h1:UMqzAQZicKnFjs4M7QXXfMq1Tf/RopSD0g1kfZkgRvo= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231005180140-4b74f352a689/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 h1:2daWNVtWNvRDoCTN5GG5N+LEM9OuY3RjJ0cboU3+xmM= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713/go.mod h1:lJLumMdUeKqurOskauSjhH4J2hz8r0iNyQWDl3i5NSM= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231002160453-1957d95a68c7 h1:gQ29vEeVTF7wr5YP83+82uHBRkRGOhV3NeTwWDjc3dQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231002160453-1957d95a68c7/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231002160453-1957d95a68c7 h1:9hkohcGsQUL0Y80T2iMI88JsR4yFF4rq9LpXuLAATCA= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231002160453-1957d95a68c7/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231002160453-1957d95a68c7 h1:KxLOn4ISFoCZOK+ViryxmqG2eN49ZJCGmwfuvT370Aw= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231002160453-1957d95a68c7/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231002160453-1957d95a68c7 h1:yXCJd4oKpTvFVc4rAij8nfPRNxzo7yrVNDgRp6BOqFE= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231002160453-1957d95a68c7/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231005180140-4b74f352a689 h1:mSioY+8Y75youwy5RUzKmlR5K0SXKcsTsIPReetXyIs= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231005180140-4b74f352a689/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231005180140-4b74f352a689 h1:vMz2KlUPl8FvCit6GN+K2mYGPjN5DbgU5u9q9o0YL0s= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231005180140-4b74f352a689/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231005180140-4b74f352a689 h1:8lwu1RO1grkes/ZD4zQuNLF5FQ9jKlOQQ7vQUss4Buo= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231005180140-4b74f352a689/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231005180140-4b74f352a689 h1:PkIHZwFh7YGCEHK6Hb81VMb9J8UugN1KI3wMRoLfb2Q= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231005180140-4b74f352a689/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE= From d01e357f9f81f512500aebd59bd87381e78684b2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 10 Oct 2023 14:58:51 -0500 Subject: [PATCH 14/24] Update OTel fork components to ca00a4d960861623f3d47ada16310d961af1c2d2 (#898) Co-authored-by: Github Action --- go.mod | 28 ++++++++++++++-------------- go.sum | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index 494c82bbb8..55f21c2495 100644 --- a/go.mod +++ b/go.mod @@ -6,34 +6,34 @@ replace github.com/influxdata/telegraf => github.com/aws/telegraf v0.10.2-0.2022 // Replace with https://github.com/amazon-contributing/opentelemetry-collector-contrib, there are no requirements for all receivers/processors/exporters // to be all replaced since there are some changes that will always be from upstream -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231010194048-ca00a4d96086 -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231010194048-ca00a4d96086 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231010194048-ca00a4d96086 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231010194048-ca00a4d96086 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231010194048-ca00a4d96086 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231010194048-ca00a4d96086 // Replace with contrib to revert upstream change https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/20519 -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231010194048-ca00a4d96086 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231010194048-ca00a4d96086 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231010194048-ca00a4d96086 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231010194048-ca00a4d96086 -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231010194048-ca00a4d96086 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231005180140-4b74f352a689 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231010194048-ca00a4d96086 -replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231005180140-4b74f352a689 +replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231010194048-ca00a4d96086 // Temporary fix, pending PR https://github.com/shirou/gopsutil/pull/957 replace github.com/shirou/gopsutil/v3 => github.com/aws/telegraf/patches/gopsutil/v3 v3.0.0-20230915153624-7629361f8380 // indirect @@ -423,4 +423,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231005180140-4b74f352a689 +replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231010194048-ca00a4d96086 diff --git a/go.sum b/go.sum index 7fd27813d0..9fa90aee4e 100644 --- a/go.sum +++ b/go.sum @@ -139,34 +139,34 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231005180140-4b74f352a689 h1:C41hepLgo3qoZ/raJJXwg5+61r1T0wKGe7ouq2GTDCQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231005180140-4b74f352a689/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231005180140-4b74f352a689 h1:PB5QDEQZeC7Wuvmuwd6mlJbjAbU7d1jNe91Ro1sieBY= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231005180140-4b74f352a689/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231005180140-4b74f352a689 h1:Hs0LHi+VUmWUlTBrc3dp8moDSsIElpbUfWWezE4yPdg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231005180140-4b74f352a689/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231005180140-4b74f352a689 h1:PE+2GVB1EVsRaqShTpyunefheg82SklTOeLsYIXveus= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231005180140-4b74f352a689/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231005180140-4b74f352a689 h1:H7WeHG6ERGOgPCS01RjwMo3Faja60PbozpPXDhx3OGI= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231005180140-4b74f352a689/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231005180140-4b74f352a689 h1:hpv83ENe81wZmLC93U90hT5WzbOzLCY8EhxAmvUjMuo= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231005180140-4b74f352a689/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231005180140-4b74f352a689 h1:MVjE0TuGzjWvWbphb/UQhhKz2+XtRjnhEJF20ht/D30= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231005180140-4b74f352a689/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231005180140-4b74f352a689 h1:XYgIXvOXzjDUvkw0KQ8HjGmqW2+3fLFOhwtWRZcn5YU= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231005180140-4b74f352a689/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231005180140-4b74f352a689 h1:UMqzAQZicKnFjs4M7QXXfMq1Tf/RopSD0g1kfZkgRvo= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231005180140-4b74f352a689/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231010194048-ca00a4d96086 h1:TRJjygBgnh9fJryfwmW4mlTY5x+3YLzdbLvRH7fRKKE= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231010194048-ca00a4d96086/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231010194048-ca00a4d96086 h1:K6U7gjV7K3dphSHM58+syX2AJEDl0/L5JV+vLoMNteg= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231010194048-ca00a4d96086/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231010194048-ca00a4d96086 h1:U5DkSgPvYr3JPveOKaC2KZYYZLBMQQsU7dBOJOunbco= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231010194048-ca00a4d96086/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231010194048-ca00a4d96086 h1:DgFsom+0FLOGRmifUum7Ak5agaUkmoowMsaJV9RBuLw= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231010194048-ca00a4d96086/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231010194048-ca00a4d96086 h1:XqNAMwWdeRK3yBSNQ7qtz2m4h7V14u/uDBqFOzcIGGY= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231010194048-ca00a4d96086/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231010194048-ca00a4d96086 h1:wacwVdx2g+cgG9rwfnkRpo2livzkh/zNIlJHgS+0eCg= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231010194048-ca00a4d96086/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231010194048-ca00a4d96086 h1:YX51130ub3HUmjT/yQ9LE3jLLPPJo684DBECE4j614w= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231010194048-ca00a4d96086/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231010194048-ca00a4d96086 h1:bOmg1o07RevtPz7r6DTYA30m9BFvgk0TBqa0R+g9AAs= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231010194048-ca00a4d96086/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231010194048-ca00a4d96086 h1:o2kPeydL+WfCJs8jM6NBmvfT2hEhXjTmrodckD4ZxNA= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231010194048-ca00a4d96086/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 h1:2daWNVtWNvRDoCTN5GG5N+LEM9OuY3RjJ0cboU3+xmM= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713/go.mod h1:lJLumMdUeKqurOskauSjhH4J2hz8r0iNyQWDl3i5NSM= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231005180140-4b74f352a689 h1:mSioY+8Y75youwy5RUzKmlR5K0SXKcsTsIPReetXyIs= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231005180140-4b74f352a689/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231005180140-4b74f352a689 h1:vMz2KlUPl8FvCit6GN+K2mYGPjN5DbgU5u9q9o0YL0s= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231005180140-4b74f352a689/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231005180140-4b74f352a689 h1:8lwu1RO1grkes/ZD4zQuNLF5FQ9jKlOQQ7vQUss4Buo= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231005180140-4b74f352a689/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231005180140-4b74f352a689 h1:PkIHZwFh7YGCEHK6Hb81VMb9J8UugN1KI3wMRoLfb2Q= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231005180140-4b74f352a689/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231010194048-ca00a4d96086 h1:xdNB2JVC9WnOdhoLJVSAfa7q2T8qjDTQyJ3+Rnpr87U= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231010194048-ca00a4d96086/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231010194048-ca00a4d96086 h1:eYAppnv9Aib7y27njtMLOQeA02dgOwt/diIo4LNgIgI= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231010194048-ca00a4d96086/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231010194048-ca00a4d96086 h1:3FiCcGI5UjOzvdtM+AByUsaanTl7x+oXEwxiC70Hl0I= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231010194048-ca00a4d96086/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231010194048-ca00a4d96086 h1:nLe+YkTSldClMKWWN14TU+Ix1YBo4aAhezedGTbzjeU= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231010194048-ca00a4d96086/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE= From fc6b1c50ec76687bf025b5291944a4291bb11dbd Mon Sep 17 00:00:00 2001 From: Chad Patel Date: Tue, 10 Oct 2023 15:41:27 -0500 Subject: [PATCH 15/24] update enhanced CI metrics and tests for pod_container_status metrics (#899) --- .../emf_and_kubernetes_config.yaml | 18 ++++++++--------- .../kubernetes_on_prem_config.yaml | 20 +++++++++---------- .../logs_and_kubernetes_config.yaml | 20 +++++++++---------- .../otel/exporter/awsemf/kubernetes.go | 6 +++--- .../otel/exporter/awsemf/translator_test.go | 6 +++--- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index 7fe1138031..16804d953f 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -60,15 +60,6 @@ exporters: - container_filesystem_usage - container_filesystem_available - container_filesystem_utilization - - container_status_running - - container_status_terminated - - container_status_waiting - - container_status_waiting_reason_crash_loop_back_off - - container_status_waiting_reason_image_pull_error - - container_status_waiting_reason_start_error - - container_status_waiting_reason_create_container_error - - container_status_waiting_reason_create_container_config_error - - container_status_terminated_reason_oom_killed # pod metrics - dimensions: [ [ ClusterName, Namespace, PodName ], [ ClusterName ], [ ClusterName, Namespace, Service ], [ ClusterName, Namespace ], [ ClusterName, FullPodName, Namespace, PodName ] ] label_matchers: [ ] @@ -103,6 +94,15 @@ exporters: - pod_memory_limit - pod_cpu_limit - pod_cpu_request + - pod_container_status_running + - pod_container_status_terminated + - pod_container_status_waiting + - pod_container_status_waiting_reason_crash_loop_back_off + - pod_container_status_waiting_reason_image_pull_error + - pod_container_status_waiting_reason_start_error + - pod_container_status_waiting_reason_create_container_error + - pod_container_status_waiting_reason_create_container_config_error + - pod_container_status_terminated_reason_oom_killed # node metrics - dimensions: [ [ ClusterName, InstanceId, NodeName ], [ ClusterName ] ] label_matchers: [ ] diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index 3cc73c82af..54cdb7fe87 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -31,15 +31,6 @@ exporters: - container_filesystem_usage - container_filesystem_available - container_filesystem_utilization - - container_status_running - - container_status_terminated - - container_status_waiting - - container_status_waiting_reason_crash_loop_back_off - - container_status_waiting_reason_image_pull_error - - container_status_waiting_reason_start_error - - container_status_waiting_reason_create_container_error - - container_status_waiting_reason_create_container_config_error - - container_status_terminated_reason_oom_killed # pod metrics - dimensions: [ [ ClusterName, Namespace, PodName ], [ ClusterName ], [ ClusterName, Namespace, Service ], [ ClusterName, Namespace ], [ ClusterName, FullPodName, Namespace, PodName ] ] label_matchers: [ ] @@ -56,7 +47,7 @@ exporters: - pod_interface_network_rx_dropped - pod_interface_network_tx_dropped - dimensions: [ [ ClusterName, Namespace, PodName ], [ ClusterName ], [ ClusterName, FullPodName, Namespace, PodName ], [ ClusterName, Namespace, Service ] ] - label_matchers: [ ] + label_matchers: [] metric_name_selectors: - pod_cpu_reserved_capacity - pod_memory_reserved_capacity @@ -74,6 +65,15 @@ exporters: - pod_memory_limit - pod_cpu_limit - pod_cpu_request + - pod_container_status_running + - pod_container_status_terminated + - pod_container_status_waiting + - pod_container_status_waiting_reason_crash_loop_back_off + - pod_container_status_waiting_reason_image_pull_error + - pod_container_status_waiting_reason_start_error + - pod_container_status_waiting_reason_create_container_error + - pod_container_status_waiting_reason_create_container_config_error + - pod_container_status_terminated_reason_oom_killed # node metrics - dimensions: [ [ ClusterName, InstanceId, NodeName ], [ ClusterName ] ] label_matchers: [ ] diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index 5b038168ff..664ab2127c 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -60,15 +60,6 @@ exporters: - container_filesystem_usage - container_filesystem_available - container_filesystem_utilization - - container_status_running - - container_status_terminated - - container_status_waiting - - container_status_waiting_reason_crash_loop_back_off - - container_status_waiting_reason_image_pull_error - - container_status_waiting_reason_start_error - - container_status_waiting_reason_create_container_error - - container_status_waiting_reason_create_container_config_error - - container_status_terminated_reason_oom_killed # pod metrics - dimensions: [ [ ClusterName, Namespace, PodName ], [ ClusterName ], [ ClusterName, Namespace, Service ], [ ClusterName, Namespace ], [ ClusterName, FullPodName, Namespace, PodName ] ] label_matchers: [ ] @@ -85,7 +76,7 @@ exporters: - pod_interface_network_rx_dropped - pod_interface_network_tx_dropped - dimensions: [ [ ClusterName, Namespace, PodName ], [ ClusterName ], [ ClusterName, FullPodName, Namespace, PodName ], [ ClusterName, Namespace, Service ] ] - label_matchers: [ ] + label_matchers: [] metric_name_selectors: - pod_cpu_reserved_capacity - pod_memory_reserved_capacity @@ -103,6 +94,15 @@ exporters: - pod_memory_limit - pod_cpu_limit - pod_cpu_request + - pod_container_status_running + - pod_container_status_terminated + - pod_container_status_waiting + - pod_container_status_waiting_reason_crash_loop_back_off + - pod_container_status_waiting_reason_image_pull_error + - pod_container_status_waiting_reason_start_error + - pod_container_status_waiting_reason_create_container_error + - pod_container_status_waiting_reason_create_container_config_error + - pod_container_status_terminated_reason_oom_killed # node metrics - dimensions: [ [ ClusterName, InstanceId, NodeName ], [ ClusterName ] ] label_matchers: [ ] diff --git a/translator/translate/otel/exporter/awsemf/kubernetes.go b/translator/translate/otel/exporter/awsemf/kubernetes.go index eb083953ad..76d2f39ea5 100644 --- a/translator/translate/otel/exporter/awsemf/kubernetes.go +++ b/translator/translate/otel/exporter/awsemf/kubernetes.go @@ -63,9 +63,6 @@ func getContainerMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.Metric "container_cpu_utilization", "container_cpu_utilization_over_container_limit", "container_cpu_limit", "container_cpu_request", "container_memory_utilization", "container_memory_utilization_over_container_limit", "container_memory_failures_total", "container_memory_limit", "container_memory_request", "container_filesystem_usage", "container_filesystem_available", "container_filesystem_utilization", - "container_status_running", "container_status_terminated", "container_status_waiting", "container_status_waiting_reason_crash_loop_back_off", - "container_status_waiting_reason_image_pull_error", "container_status_waiting_reason_start_error", "container_status_waiting_reason_create_container_error", - "container_status_waiting_reason_create_container_config_error", "container_status_terminated_reason_oom_killed", }, } @@ -95,6 +92,9 @@ func getPodMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDeclar selectors = append(selectors, []string{"pod_number_of_container_restarts", "pod_number_of_containers", "pod_number_of_running_containers", "pod_status_ready", "pod_status_scheduled", "pod_status_running", "pod_status_pending", "pod_status_failed", "pod_status_unknown", "pod_status_succeeded", "pod_memory_request", "pod_memory_limit", "pod_cpu_limit", "pod_cpu_request", + "pod_container_status_running", "pod_container_status_terminated", "pod_container_status_waiting", "pod_container_status_waiting_reason_crash_loop_back_off", + "pod_container_status_waiting_reason_image_pull_error", "pod_container_status_waiting_reason_start_error", "pod_container_status_waiting_reason_create_container_error", + "pod_container_status_waiting_reason_create_container_config_error", "pod_container_status_terminated_reason_oom_killed", }...) } diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index 72b0867bbf..8af6695ee7 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -270,9 +270,6 @@ func TestTranslator(t *testing.T) { "container_cpu_utilization", "container_cpu_utilization_over_container_limit", "container_cpu_limit", "container_cpu_request", "container_memory_utilization", "container_memory_utilization_over_container_limit", "container_memory_failures_total", "container_memory_limit", "container_memory_request", "container_filesystem_usage", "container_filesystem_available", "container_filesystem_utilization", - "container_status_running", "container_status_terminated", "container_status_waiting", "container_status_waiting_reason_crash_loop_back_off", - "container_status_waiting_reason_image_pull_error", "container_status_waiting_reason_start_error", "container_status_waiting_reason_create_container_error", - "container_status_waiting_reason_create_container_config_error", "container_status_terminated_reason_oom_killed", }, }, { @@ -295,6 +292,9 @@ func TestTranslator(t *testing.T) { MetricNameSelectors: []string{"pod_cpu_reserved_capacity", "pod_memory_reserved_capacity", "pod_number_of_container_restarts", "pod_number_of_containers", "pod_number_of_running_containers", "pod_status_ready", "pod_status_scheduled", "pod_status_running", "pod_status_pending", "pod_status_failed", "pod_status_unknown", "pod_status_succeeded", "pod_memory_request", "pod_memory_limit", "pod_cpu_limit", "pod_cpu_request", + "pod_container_status_running", "pod_container_status_terminated", "pod_container_status_waiting", "pod_container_status_waiting_reason_crash_loop_back_off", + "pod_container_status_waiting_reason_image_pull_error", "pod_container_status_waiting_reason_start_error", "pod_container_status_waiting_reason_create_container_error", + "pod_container_status_waiting_reason_create_container_config_error", "pod_container_status_terminated_reason_oom_killed", }, }, { From a67ac451e380754d08930c0a71696bfa06cdc563 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 12:25:23 -0400 Subject: [PATCH 16/24] Update OTel fork components to d1a2dc44f2f34893710a29fbdb554840652309ea (#906) Co-authored-by: Github Action --- go.mod | 28 ++++++++++++++-------------- go.sum | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index 55f21c2495..a86aa781e0 100644 --- a/go.mod +++ b/go.mod @@ -6,34 +6,34 @@ replace github.com/influxdata/telegraf => github.com/aws/telegraf v0.10.2-0.2022 // Replace with https://github.com/amazon-contributing/opentelemetry-collector-contrib, there are no requirements for all receivers/processors/exporters // to be all replaced since there are some changes that will always be from upstream -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231012205229-d1a2dc44f2f3 -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231012205229-d1a2dc44f2f3 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231012205229-d1a2dc44f2f3 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231012205229-d1a2dc44f2f3 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231012205229-d1a2dc44f2f3 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231012205229-d1a2dc44f2f3 // Replace with contrib to revert upstream change https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/20519 -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231012205229-d1a2dc44f2f3 replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231012205229-d1a2dc44f2f3 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231012205229-d1a2dc44f2f3 -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231012205229-d1a2dc44f2f3 -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231012205229-d1a2dc44f2f3 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231010194048-ca00a4d96086 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231012205229-d1a2dc44f2f3 -replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231010194048-ca00a4d96086 +replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231012205229-d1a2dc44f2f3 // Temporary fix, pending PR https://github.com/shirou/gopsutil/pull/957 replace github.com/shirou/gopsutil/v3 => github.com/aws/telegraf/patches/gopsutil/v3 v3.0.0-20230915153624-7629361f8380 // indirect @@ -423,4 +423,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231010194048-ca00a4d96086 +replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231012205229-d1a2dc44f2f3 diff --git a/go.sum b/go.sum index 9fa90aee4e..930f46a297 100644 --- a/go.sum +++ b/go.sum @@ -139,34 +139,34 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231010194048-ca00a4d96086 h1:TRJjygBgnh9fJryfwmW4mlTY5x+3YLzdbLvRH7fRKKE= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231010194048-ca00a4d96086/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231010194048-ca00a4d96086 h1:K6U7gjV7K3dphSHM58+syX2AJEDl0/L5JV+vLoMNteg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231010194048-ca00a4d96086/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231010194048-ca00a4d96086 h1:U5DkSgPvYr3JPveOKaC2KZYYZLBMQQsU7dBOJOunbco= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231010194048-ca00a4d96086/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231010194048-ca00a4d96086 h1:DgFsom+0FLOGRmifUum7Ak5agaUkmoowMsaJV9RBuLw= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231010194048-ca00a4d96086/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231010194048-ca00a4d96086 h1:XqNAMwWdeRK3yBSNQ7qtz2m4h7V14u/uDBqFOzcIGGY= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231010194048-ca00a4d96086/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231010194048-ca00a4d96086 h1:wacwVdx2g+cgG9rwfnkRpo2livzkh/zNIlJHgS+0eCg= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231010194048-ca00a4d96086/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231010194048-ca00a4d96086 h1:YX51130ub3HUmjT/yQ9LE3jLLPPJo684DBECE4j614w= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231010194048-ca00a4d96086/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231010194048-ca00a4d96086 h1:bOmg1o07RevtPz7r6DTYA30m9BFvgk0TBqa0R+g9AAs= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231010194048-ca00a4d96086/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231010194048-ca00a4d96086 h1:o2kPeydL+WfCJs8jM6NBmvfT2hEhXjTmrodckD4ZxNA= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231010194048-ca00a4d96086/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231012205229-d1a2dc44f2f3 h1:2tLCZtdFZ4B7yfqhtPEWOtVoCpFE0s3zB2IHqkVAVqM= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231012205229-d1a2dc44f2f3 h1:J2tk9Eo7FBT+GbQIOd7tq0XQyxtA+O75gHKoGYH5uuE= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231012205229-d1a2dc44f2f3 h1:S1LKmk4Eczr+Ur6ReKMOkvfaYLBcJaTbRbtVmE9s9rY= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231012205229-d1a2dc44f2f3 h1:axMuDAP4C3gfDb0Ox5NGbIyoTKPOZ4Pt268hfH4jNKk= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231012205229-d1a2dc44f2f3 h1:uCQjg1aL9DjKu8BB+9VhBdEQ6PDxz3PACctDYQU0i0I= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231012205229-d1a2dc44f2f3 h1:uswozvbfuT2WbsLDd9Pk9hRrm9fsMHPTLzzbbhFEOi8= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231012205229-d1a2dc44f2f3 h1:brhehmtgpokhtJUaX/Roo/Fg+wp2JootZzj9PyJXc7I= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231012205229-d1a2dc44f2f3 h1:xlCHzoTSd8DOC0aAs1cWsGXlAtUaH0Hzh9yJYzdljgQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231012205229-d1a2dc44f2f3 h1:Igs/mTssnX3WYVpGcj72b9n1GqQdcmWBZmCgAQam8Cg= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 h1:2daWNVtWNvRDoCTN5GG5N+LEM9OuY3RjJ0cboU3+xmM= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713/go.mod h1:lJLumMdUeKqurOskauSjhH4J2hz8r0iNyQWDl3i5NSM= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231010194048-ca00a4d96086 h1:xdNB2JVC9WnOdhoLJVSAfa7q2T8qjDTQyJ3+Rnpr87U= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231010194048-ca00a4d96086/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231010194048-ca00a4d96086 h1:eYAppnv9Aib7y27njtMLOQeA02dgOwt/diIo4LNgIgI= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231010194048-ca00a4d96086/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231010194048-ca00a4d96086 h1:3FiCcGI5UjOzvdtM+AByUsaanTl7x+oXEwxiC70Hl0I= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231010194048-ca00a4d96086/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231010194048-ca00a4d96086 h1:nLe+YkTSldClMKWWN14TU+Ix1YBo4aAhezedGTbzjeU= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231010194048-ca00a4d96086/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231012205229-d1a2dc44f2f3 h1:2veJzdr7Ulzh0PVhIzk8qFlYZ2UOKlh+Th8awDT8fsw= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231012205229-d1a2dc44f2f3 h1:4Xsr3S/XBYXDOo3AihM9YUww6YxzZUJYRUpiP8LeWe0= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231012205229-d1a2dc44f2f3 h1:cvtX0kTQO/wY+33wWVv3PJS3tLCQ39x+lCfhZgbMiOY= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231012205229-d1a2dc44f2f3 h1:aNJxUF5OA5SeBosU9OxJr+L6JY8JEx2APAz+xPbb1GM= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE= From cfde177c11f0c5b1e39ed7772d1564f29b59e3e2 Mon Sep 17 00:00:00 2001 From: Seth L <81644108+sethAmazon@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:39:43 -0400 Subject: [PATCH 17/24] Skip Timestamp For Agent Log (#885) --- .../sampleConfig/no_skip_log_timestamp.conf | 37 +++ .../sampleConfig/no_skip_log_timestamp.json | 15 + .../no_skip_log_timestamp_windows.conf | 37 +++ .../no_skip_log_timestamp_windows.json | 16 + .../sampleConfig/skip_log_timestamp.conf | 35 +++ .../sampleConfig/skip_log_timestamp.json | 18 ++ .../skip_log_timestamp_default.conf | 35 +++ .../skip_log_timestamp_default.json | 15 + .../skip_log_timestamp_default_windows.conf | 35 +++ .../skip_log_timestamp_default_windows.json | 15 + .../skip_log_timestamp_windows.conf | 35 +++ .../skip_log_timestamp_windows.json | 18 ++ translator/tocwconfig/tocwconfig_test.go | 281 +++++++++++++++--- translator/translate/agent/ruleLogFile.go | 1 + .../files/collect_list/ruleTimestampFormat.go | 17 +- 15 files changed, 570 insertions(+), 40 deletions(-) create mode 100644 translator/tocwconfig/sampleConfig/no_skip_log_timestamp.conf create mode 100644 translator/tocwconfig/sampleConfig/no_skip_log_timestamp.json create mode 100644 translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.conf create mode 100644 translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.json create mode 100644 translator/tocwconfig/sampleConfig/skip_log_timestamp.conf create mode 100644 translator/tocwconfig/sampleConfig/skip_log_timestamp.json create mode 100644 translator/tocwconfig/sampleConfig/skip_log_timestamp_default.conf create mode 100644 translator/tocwconfig/sampleConfig/skip_log_timestamp_default.json create mode 100644 translator/tocwconfig/sampleConfig/skip_log_timestamp_default_windows.conf create mode 100644 translator/tocwconfig/sampleConfig/skip_log_timestamp_default_windows.json create mode 100644 translator/tocwconfig/sampleConfig/skip_log_timestamp_windows.conf create mode 100644 translator/tocwconfig/sampleConfig/skip_log_timestamp_windows.json diff --git a/translator/tocwconfig/sampleConfig/no_skip_log_timestamp.conf b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp.conf new file mode 100644 index 0000000000..5bf0ea1d61 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp.conf @@ -0,0 +1,37 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "" + interval = "60s" + logfile = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[inputs] + + [[inputs.logfile]] + destination = "cloudwatchlogs" + file_state_folder = "/opt/aws/amazon-cloudwatch-agent/logs/state" + + [[inputs.logfile.file_config]] + file_path = "/tmp/not-amazon-cloudwatch-agent.log" + from_beginning = true + log_group_name = "amazon-cloudwatch-agent.log" + pipe = false + retention_in_days = -1 + timestamp_layout = "15:04:05 06 Jan 02" + timestamp_regex = "(d{2}:d{2}:d{2} d{2} w{3} d{2})" + +[outputs] + + [[outputs.cloudwatchlogs]] + force_flush_interval = "5s" + log_stream_name = "i-UNKNOWN" + region = "us-west-2" diff --git a/translator/tocwconfig/sampleConfig/no_skip_log_timestamp.json b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp.json new file mode 100644 index 0000000000..4e04deb3de --- /dev/null +++ b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp.json @@ -0,0 +1,15 @@ +{ + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "/tmp/not-amazon-cloudwatch-agent.log", + "log_group_name": "amazon-cloudwatch-agent.log", + "timestamp_format": "%H:%M:%S %y %b %d" + } + ] + } + } + } +} \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.conf b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.conf new file mode 100644 index 0000000000..ef0961d523 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.conf @@ -0,0 +1,37 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "" + interval = "60s" + logfile = "c:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\amazon-cloudwatch-agent.log" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[inputs] + + [[inputs.logfile]] + destination = "cloudwatchlogs" + file_state_folder = "c:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\state" + + [[inputs.logfile.file_config]] + file_path = "c:\\tmp\\not-amazon-cloudwatch-agent.log" + from_beginning = true + log_group_name = "amazon-cloudwatch-agent.log" + pipe = false + retention_in_days = -1 + timestamp_layout = "15:04:05 06 Jan 02" + timestamp_regex = "(d{2}:d{2}:d{2} d{2} w{3} d{2})" + +[outputs] + + [[outputs.cloudwatchlogs]] + force_flush_interval = "5s" + log_stream_name = "i-UNKNOWN" + region = "us-west-2" diff --git a/translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.json b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.json new file mode 100644 index 0000000000..fb7bf89559 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.json @@ -0,0 +1,16 @@ +{ + + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "c:\\tmp\\not-amazon-cloudwatch-agent.log", + "log_group_name": "amazon-cloudwatch-agent.log", + "timestamp_format": "%H:%M:%S %y %b %d" + } + ] + } + } + } +} \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/skip_log_timestamp.conf b/translator/tocwconfig/sampleConfig/skip_log_timestamp.conf new file mode 100644 index 0000000000..527d132104 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/skip_log_timestamp.conf @@ -0,0 +1,35 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "" + interval = "60s" + logfile = "/opt/tmp/a.log" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[inputs] + + [[inputs.logfile]] + destination = "cloudwatchlogs" + file_state_folder = "/opt/aws/amazon-cloudwatch-agent/logs/state" + + [[inputs.logfile.file_config]] + file_path = "/opt/tmp/a.log" + from_beginning = true + log_group_name = "amazon-cloudwatch-agent.log" + pipe = false + retention_in_days = -1 + +[outputs] + + [[outputs.cloudwatchlogs]] + force_flush_interval = "5s" + log_stream_name = "i-UNKNOWN" + region = "us-west-2" diff --git a/translator/tocwconfig/sampleConfig/skip_log_timestamp.json b/translator/tocwconfig/sampleConfig/skip_log_timestamp.json new file mode 100644 index 0000000000..db134e261e --- /dev/null +++ b/translator/tocwconfig/sampleConfig/skip_log_timestamp.json @@ -0,0 +1,18 @@ +{ + "agent": { + "logfile": "/opt/tmp/a.log" + }, + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "/opt/tmp/a.log", + "log_group_name": "amazon-cloudwatch-agent.log", + "timestamp_format": "%H:%M:%S %y %b %d" + } + ] + } + } + } +} \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/skip_log_timestamp_default.conf b/translator/tocwconfig/sampleConfig/skip_log_timestamp_default.conf new file mode 100644 index 0000000000..3cb30744ec --- /dev/null +++ b/translator/tocwconfig/sampleConfig/skip_log_timestamp_default.conf @@ -0,0 +1,35 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "" + interval = "60s" + logfile = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[inputs] + + [[inputs.logfile]] + destination = "cloudwatchlogs" + file_state_folder = "/opt/aws/amazon-cloudwatch-agent/logs/state" + + [[inputs.logfile.file_config]] + file_path = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" + from_beginning = true + log_group_name = "amazon-cloudwatch-agent.log" + pipe = false + retention_in_days = -1 + +[outputs] + + [[outputs.cloudwatchlogs]] + force_flush_interval = "5s" + log_stream_name = "i-UNKNOWN" + region = "us-west-2" diff --git a/translator/tocwconfig/sampleConfig/skip_log_timestamp_default.json b/translator/tocwconfig/sampleConfig/skip_log_timestamp_default.json new file mode 100644 index 0000000000..734df33a1b --- /dev/null +++ b/translator/tocwconfig/sampleConfig/skip_log_timestamp_default.json @@ -0,0 +1,15 @@ +{ + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log", + "log_group_name": "amazon-cloudwatch-agent.log", + "timestamp_format": "%H:%M:%S %y %b %d" + } + ] + } + } + } +} \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/skip_log_timestamp_default_windows.conf b/translator/tocwconfig/sampleConfig/skip_log_timestamp_default_windows.conf new file mode 100644 index 0000000000..36c2684e47 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/skip_log_timestamp_default_windows.conf @@ -0,0 +1,35 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "" + interval = "60s" + logfile = "c:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\amazon-cloudwatch-agent.log" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[inputs] + + [[inputs.logfile]] + destination = "cloudwatchlogs" + file_state_folder = "c:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\state" + + [[inputs.logfile.file_config]] + file_path = "c:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\amazon-cloudwatch-agent.log" + from_beginning = true + log_group_name = "amazon-cloudwatch-agent.log" + pipe = false + retention_in_days = -1 + +[outputs] + + [[outputs.cloudwatchlogs]] + force_flush_interval = "5s" + log_stream_name = "i-UNKNOWN" + region = "us-west-2" diff --git a/translator/tocwconfig/sampleConfig/skip_log_timestamp_default_windows.json b/translator/tocwconfig/sampleConfig/skip_log_timestamp_default_windows.json new file mode 100644 index 0000000000..0ba8b224fb --- /dev/null +++ b/translator/tocwconfig/sampleConfig/skip_log_timestamp_default_windows.json @@ -0,0 +1,15 @@ +{ + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "c:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\amazon-cloudwatch-agent.log", + "log_group_name": "amazon-cloudwatch-agent.log", + "timestamp_format": "%H:%M:%S %y %b %d" + } + ] + } + } + } +} \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/skip_log_timestamp_windows.conf b/translator/tocwconfig/sampleConfig/skip_log_timestamp_windows.conf new file mode 100644 index 0000000000..fbd9d27b6a --- /dev/null +++ b/translator/tocwconfig/sampleConfig/skip_log_timestamp_windows.conf @@ -0,0 +1,35 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "" + interval = "60s" + logfile = "c:\\tmp\\am.log" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[inputs] + + [[inputs.logfile]] + destination = "cloudwatchlogs" + file_state_folder = "c:\\ProgramData\\Amazon\\AmazonCloudWatchAgent\\Logs\\state" + + [[inputs.logfile.file_config]] + file_path = "c:\\tmp\\am.log" + from_beginning = true + log_group_name = "amazon-cloudwatch-agent.log" + pipe = false + retention_in_days = -1 + +[outputs] + + [[outputs.cloudwatchlogs]] + force_flush_interval = "5s" + log_stream_name = "i-UNKNOWN" + region = "us-west-2" diff --git a/translator/tocwconfig/sampleConfig/skip_log_timestamp_windows.json b/translator/tocwconfig/sampleConfig/skip_log_timestamp_windows.json new file mode 100644 index 0000000000..66394e40c7 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/skip_log_timestamp_windows.json @@ -0,0 +1,18 @@ +{ + "agent": { + "logfile": "c:\\tmp\\am.log" + }, + "logs": { + "logs_collected": { + "files": { + "collect_list": [ + { + "file_path": "c:\\tmp\\am.log", + "log_group_name": "amazon-cloudwatch-agent.log", + "timestamp_format": "%H:%M:%S %y %b %d" + } + ] + } + } + } +} \ No newline at end of file diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index 1c0e010c27..bf5b0ce323 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -47,6 +47,13 @@ const ( //go:embed sampleConfig/prometheus_config.yaml var prometheusConfig string +type testCase struct { + filename string + targetPlatform string + expectedEnvVars map[string]string + appendString string +} + func TestBaseContainerInsightsConfig(t *testing.T) { resetContext(t) context.CurrentContext().SetRunInContainer(true) @@ -101,11 +108,32 @@ func TestWindowsEventOnlyConfig(t *testing.T) { } func TestStatsDConfig(t *testing.T) { - resetContext(t) - expectedEnvVars := map[string]string{} - checkTranslation(t, "statsd_config", "linux", expectedEnvVars, "_linux") - checkTranslation(t, "statsd_config", "windows", expectedEnvVars, "_windows") - checkTranslation(t, "statsd_config", "darwin", nil, "_linux") + testCases := map[string]testCase{ + "linux": { + filename: "statsd_config", + targetPlatform: "linux", + expectedEnvVars: map[string]string{}, + appendString: "_linux", + }, + "windows": { + filename: "statsd_config", + targetPlatform: "windows", + expectedEnvVars: map[string]string{}, + appendString: "_windows", + }, + "darwin": { + filename: "statsd_config", + targetPlatform: "darwin", + expectedEnvVars: nil, + appendString: "_linux", + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + resetContext(t) + checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) + }) + } } // Linux only for CollectD @@ -141,11 +169,32 @@ func TestPrometheusConfig(t *testing.T) { } func TestBasicConfig(t *testing.T) { - resetContext(t) - expectedEnvVars := map[string]string{} - checkTranslation(t, "basic_config_linux", "linux", expectedEnvVars, "") - checkTranslation(t, "basic_config_linux", "darwin", nil, "") - checkTranslation(t, "basic_config_windows", "windows", expectedEnvVars, "") + testCases := map[string]testCase{ + "linux": { + filename: "basic_config_linux", + targetPlatform: "linux", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + "darwin": { + filename: "basic_config_linux", + targetPlatform: "darwin", + expectedEnvVars: nil, + appendString: "", + }, + "windows": { + filename: "basic_config_windows", + targetPlatform: "windows", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + resetContext(t) + checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) + }) + } } func TestInvalidInputConfig(t *testing.T) { @@ -157,20 +206,62 @@ func TestInvalidInputConfig(t *testing.T) { func TestStandardConfig(t *testing.T) { // the way our config translator works is int(0) leaves an empty in the yaml // this will default to 0 on contrib side since int default is 0 for golang - resetContext(t) - t.Setenv(envconfig.IMDS_NUMBER_RETRY, "0") - expectedEnvVars := map[string]string{} - checkTranslation(t, "standard_config_linux", "linux", expectedEnvVars, "") - checkTranslation(t, "standard_config_linux", "darwin", nil, "") - checkTranslation(t, "standard_config_windows", "windows", nil, "") + testCases := map[string]testCase{ + "linux": { + filename: "standard_config_linux", + targetPlatform: "linux", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + "darwin": { + filename: "standard_config_linux", + targetPlatform: "darwin", + expectedEnvVars: nil, + appendString: "", + }, + "windows": { + filename: "standard_config_windows", + targetPlatform: "windows", + expectedEnvVars: nil, + appendString: "", + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + resetContext(t) + t.Setenv(envconfig.IMDS_NUMBER_RETRY, "0") + checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) + }) + } } func TestAdvancedConfig(t *testing.T) { - resetContext(t) - expectedEnvVars := map[string]string{} - checkTranslation(t, "advanced_config_linux", "linux", expectedEnvVars, "") - checkTranslation(t, "advanced_config_darwin", "darwin", nil, "") - checkTranslation(t, "advanced_config_windows", "windows", expectedEnvVars, "") + testCases := map[string]testCase{ + "linux": { + filename: "advanced_config_linux", + targetPlatform: "linux", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + "darwin": { + filename: "advanced_config_darwin", + targetPlatform: "darwin", + expectedEnvVars: nil, + appendString: "", + }, + "windows": { + filename: "advanced_config_windows", + targetPlatform: "windows", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + resetContext(t) + checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) + }) + } } func TestLogOnlyConfig(t *testing.T) { @@ -179,13 +270,105 @@ func TestLogOnlyConfig(t *testing.T) { checkTranslation(t, "log_only_config_windows", "windows", expectedEnvVars, "") } -func TestTraceConfig(t *testing.T) { +func TestSkipLogTimestampConfig(t *testing.T) { + testCases := map[string]testCase{ + "default_linux": { + filename: "skip_log_timestamp_default", + targetPlatform: "linux", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + "default_darwin": { + filename: "skip_log_timestamp_default", + targetPlatform: "darwin", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + "default_windows": { + filename: "skip_log_timestamp_default_windows", + targetPlatform: "windows", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + "set_linux": { + filename: "skip_log_timestamp", + targetPlatform: "linux", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + "set_darwin": { + filename: "skip_log_timestamp", + targetPlatform: "darwin", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + "set_windows": { + filename: "skip_log_timestamp_windows", + targetPlatform: "windows", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + "no_skip_linux": { + filename: "no_skip_log_timestamp", + targetPlatform: "linux", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + "no_skip_darwin": { + filename: "no_skip_log_timestamp", + targetPlatform: "darwin", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + "no_skip_windows": { + filename: "no_skip_log_timestamp_windows", + targetPlatform: "windows", + expectedEnvVars: map[string]string{}, + appendString: "", + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + resetContext(t) + checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) + }) + } +} + +func TestDoNotSkipLogDefaultTimestampConfig(t *testing.T) { resetContext(t) - readCommonConfig(t, "./sampleConfig/commonConfig/withCredentials.toml") expectedEnvVars := map[string]string{} - checkTranslation(t, "trace_config", "linux", expectedEnvVars, "_linux") - checkTranslation(t, "trace_config", "darwin", expectedEnvVars, "_linux") - checkTranslation(t, "trace_config", "windows", expectedEnvVars, "_windows") + checkTranslation(t, "log_only_config_windows", "windows", expectedEnvVars, "") +} + +func TestTraceConfig(t *testing.T) { + testCases := map[string]testCase{ + "linux": { + filename: "trace_config", + targetPlatform: "linux", + expectedEnvVars: map[string]string{}, + appendString: "_linux", + }, + "darwin": { + filename: "trace_config", + targetPlatform: "darwin", + expectedEnvVars: map[string]string{}, + appendString: "_linux", + }, + "windows": { + filename: "trace_config", + targetPlatform: "windows", + expectedEnvVars: map[string]string{}, + appendString: "_windows", + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + resetContext(t) + readCommonConfig(t, "./sampleConfig/commonConfig/withCredentials.toml") + checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) + }) + } } func TestConfigWithEnvironmentVariables(t *testing.T) { @@ -195,17 +378,43 @@ func TestConfigWithEnvironmentVariables(t *testing.T) { } func TestStandardConfigWithCommonConfig(t *testing.T) { - resetContext(t) - readCommonConfig(t, "./sampleConfig/commonConfig/withCredentialsProxySsl.toml") - expectedEnvVars := map[string]string{ - "AWS_CA_BUNDLE": "/etc/test/ca_bundle.pem", - "HTTPS_PROXY": "https://127.0.0.1:3280", - "HTTP_PROXY": "http://127.0.0.1:3280", - "NO_PROXY": "254.1.1.1", + testCases := map[string]testCase{ + "linux": { + filename: "standard_config_linux", + targetPlatform: "linux", + expectedEnvVars: map[string]string{ + "AWS_CA_BUNDLE": "/etc/test/ca_bundle.pem", + "HTTPS_PROXY": "https://127.0.0.1:3280", + "HTTP_PROXY": "http://127.0.0.1:3280", + "NO_PROXY": "254.1.1.1", + }, + appendString: "_with_common_config", + }, + "darwin": { + filename: "standard_config_linux", + targetPlatform: "darwin", + expectedEnvVars: nil, + appendString: "_with_common_config", + }, + "windows": { + filename: "standard_config_windows", + targetPlatform: "windows", + expectedEnvVars: map[string]string{ + "AWS_CA_BUNDLE": "/etc/test/ca_bundle.pem", + "HTTPS_PROXY": "https://127.0.0.1:3280", + "HTTP_PROXY": "http://127.0.0.1:3280", + "NO_PROXY": "254.1.1.1", + }, + appendString: "_with_common_config", + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + resetContext(t) + readCommonConfig(t, "./sampleConfig/commonConfig/withCredentialsProxySsl.toml") + checkTranslation(t, testCase.filename, testCase.targetPlatform, testCase.expectedEnvVars, testCase.appendString) + }) } - checkTranslation(t, "standard_config_linux", "linux", expectedEnvVars, "_with_common_config") - checkTranslation(t, "standard_config_linux", "darwin", nil, "_with_common_config") - checkTranslation(t, "standard_config_windows", "windows", expectedEnvVars, "_with_common_config") } func TestDeltaNetConfigLinux(t *testing.T) { diff --git a/translator/translate/agent/ruleLogFile.go b/translator/translate/agent/ruleLogFile.go index e2988893a7..1eff380c8a 100644 --- a/translator/translate/agent/ruleLogFile.go +++ b/translator/translate/agent/ruleLogFile.go @@ -19,6 +19,7 @@ type Logfile struct { func (l *Logfile) ApplyRule(input interface{}) (returnKey string, returnVal interface{}) { returnKey, returnVal = translator.DefaultCase("logfile", GetDefaultValue(), input) + context.CurrentContext().SetAgentLogFile(returnVal.(string)) return } diff --git a/translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat.go b/translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat.go index 65af348631..05b8ae318e 100644 --- a/translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat.go +++ b/translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/aws/amazon-cloudwatch-agent/translator" + "github.com/aws/amazon-cloudwatch-agent/translator/context" ) /* @@ -138,13 +139,17 @@ func checkAndReplace(input string, timestampFormatMap map[string]string) string type TimestampRegax struct { } +// ApplyRule add timestamp regex +// do not add timestamp check when viewing cwa logfile func (t *TimestampRegax) ApplyRule(input interface{}) (returnKey string, returnVal interface{}) { //Convert the input string into []rune and iterate the map and build the output []rune m := input.(map[string]interface{}) //If user not specify the timestamp_format, then no config entry for "timestamp_layout" in TOML if val, ok := m["timestamp_format"]; !ok { - returnKey = "" - returnVal = "" + return "", "" + } else if m["file_path"] == context.CurrentContext().GetAgentLogFile() { + fmt.Printf("timestamp_format set file_path : %s is the same as agent log file %s thus do not use timestamp_regex \n", m["file_path"], context.CurrentContext().GetAgentLogFile()) + return "", "" } else { //If user provide with the specific timestamp_format, use the one that user provide res := checkAndReplace(val.(string), TimeFormatRegexEscapeMap) @@ -170,13 +175,17 @@ func (t *TimestampRegax) ApplyRule(input interface{}) (returnKey string, returnV type TimestampLayout struct { } +// ApplyRule add timestamp layout +// do not add timestamp check when viewing cwa logfile func (t *TimestampLayout) ApplyRule(input interface{}) (returnKey string, returnVal interface{}) { //Convert the input string into []rune and iterate the map and build the output []rune m := input.(map[string]interface{}) //If user not specify the timestamp_format, then no config entry for "timestamp_layout" in TOML if val, ok := m["timestamp_format"]; !ok { - returnKey = "" - returnVal = "" + return "", "" + } else if m["file_path"] == context.CurrentContext().GetAgentLogFile() { + fmt.Printf("timestamp_format set file_path : %s is the same as agent log file %s thus do not use timestamp_layout \n", m["file_path"], context.CurrentContext().GetAgentLogFile()) + return "", "" } else { res := checkAndReplace(val.(string), TimeFormatMap) //If user provide with the specific timestamp_format, use the one that user provide From ef67caccc72b03514d64875b5a177d1c160d75f9 Mon Sep 17 00:00:00 2001 From: Bhanu Bandi Date: Mon, 16 Oct 2023 14:34:47 -0400 Subject: [PATCH 18/24] Container insights ssl (#822) --- go.mod | 2 +- go.sum | 3 ++- .../sampleConfig/base_container_insights_config.yaml | 6 +++--- .../sampleConfig/emf_and_kubernetes_config.yaml | 2 +- .../sampleConfig/kubernetes_on_prem_config.yaml | 2 +- .../tocwconfig/sampleConfig/log_ecs_metric_only.yaml | 4 ++-- .../sampleConfig/logs_and_kubernetes_config.yaml | 2 +- .../tocwconfig/sampleConfig/prometheus_config_linux.yaml | 4 ++-- .../sampleConfig/prometheus_config_windows.yaml | 4 ++-- translator/tocwconfig/tocwconfig_test.go | 5 ++++- translator/translate/otel/exporter/awsemf/translator.go | 9 ++++++++- .../otel/exporter/otel_aws_cloudwatch_logs/translator.go | 1 - 12 files changed, 27 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index a86aa781e0..db9405c94e 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,7 @@ replace github.com/go-kit/kit => github.com/go-kit/kit v0.12.1-0.20220808180842- replace github.com/openshift/api v3.9.0+incompatible => github.com/openshift/api v0.0.0-20180801171038-322a19404e37 require ( - github.com/BurntSushi/toml v0.4.1 + github.com/BurntSushi/toml v1.3.2 github.com/Jeffail/gabs v1.4.0 github.com/aws/aws-sdk-go v1.45.2 github.com/aws/aws-sdk-go-v2 v1.19.0 diff --git a/go.sum b/go.sum index 930f46a297..caaa86bf79 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,9 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= diff --git a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml index 7ecd23fe8f..94c18cc7e7 100644 --- a/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml +++ b/translator/tocwconfig/sampleConfig/base_container_insights_config.yaml @@ -1,7 +1,7 @@ connectors: {} exporters: awscloudwatchlogs/emf_logs: - certificate_file_path: "" + certificate_file_path: "/etc/test/ca_bundle.pem" emf_only: true endpoint: "https://fake_endpoint" imds_retries: 1 @@ -39,8 +39,8 @@ exporters: disable_metric_extraction: true version: "0" eks_fargate_container_insights_enabled: false - certificate_file_path: "" - endpoint: "" + certificate_file_path: "/etc/test/ca_bundle.pem" + endpoint: "https://fake_endpoint" enhanced_container_insights: false imds_retries: 1 local_mode: false diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index 16804d953f..2da8fa2114 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -35,7 +35,7 @@ exporters: disable_metric_extraction: true eks_fargate_container_insights_enabled: false certificate_file_path: "" - endpoint: "" + endpoint: "https://fake_endpoint" enhanced_container_insights: true imds_retries: 2 local_mode: false diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index 54cdb7fe87..0c98ba23de 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -6,7 +6,7 @@ exporters: dimension_rollup_option: NoDimensionRollup disable_metric_extraction: true eks_fargate_container_insights_enabled: false - endpoint: "" + endpoint: "https://fake_endpoint" enhanced_container_insights: true imds_retries: 1 local_mode: false diff --git a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml index 7025cbffc6..7d944ba2b3 100644 --- a/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml +++ b/translator/tocwconfig/sampleConfig/log_ecs_metric_only.yaml @@ -35,8 +35,8 @@ exporters: disable_metric_extraction: false eks_fargate_container_insights_enabled: false certificate_file_path: "" - endpoint: "" - imds_retries: 1 + endpoint: "https://fake_endpoint" + "imds_retries": 1 enhanced_container_insights: false local_mode: false log_group_name: /aws/ecs/containerinsights/{ClusterName}/performance diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index 664ab2127c..b9d17ebb84 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -35,7 +35,7 @@ exporters: dimension_rollup_option: NoDimensionRollup disable_metric_extraction: false eks_fargate_container_insights_enabled: false - endpoint: "" + endpoint: "https://fake_endpoint" imds_retries: 0 enhanced_container_insights: true local_mode: false diff --git a/translator/tocwconfig/sampleConfig/prometheus_config_linux.yaml b/translator/tocwconfig/sampleConfig/prometheus_config_linux.yaml index 0324968b89..d8464cc07f 100644 --- a/translator/tocwconfig/sampleConfig/prometheus_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/prometheus_config_linux.yaml @@ -8,8 +8,8 @@ exporters: retain_initial_value_of_delta_metric: false eks_fargate_container_insights_enabled: false certificate_file_path: "" - endpoint: "" - imds_retries: 1 + endpoint: "https://fake_endpoint" + "imds_retries": 1 enhanced_container_insights: false local_mode: false log_group_name: /aws/ecs/containerinsights/TestCluster/prometheus diff --git a/translator/tocwconfig/sampleConfig/prometheus_config_windows.yaml b/translator/tocwconfig/sampleConfig/prometheus_config_windows.yaml index ccd84ea2fb..639e1cd466 100644 --- a/translator/tocwconfig/sampleConfig/prometheus_config_windows.yaml +++ b/translator/tocwconfig/sampleConfig/prometheus_config_windows.yaml @@ -8,8 +8,8 @@ exporters: retain_initial_value_of_delta_metric: false eks_fargate_container_insights_enabled: false certificate_file_path: "" - endpoint: "" - imds_retries: 1 + endpoint: "https://fake_endpoint" + "imds_retries": 1 enhanced_container_insights: false local_mode: false log_group_name: /aws/ecs/containerinsights/TestCluster/prometheus diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index bf5b0ce323..816824aef8 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -59,7 +59,10 @@ func TestBaseContainerInsightsConfig(t *testing.T) { context.CurrentContext().SetRunInContainer(true) t.Setenv(config.HOST_NAME, "host_name_from_env") t.Setenv(config.HOST_IP, "127.0.0.1") - expectedEnvVars := map[string]string{} + t.Setenv(envconfig.AWS_CA_BUNDLE, "/etc/test/ca_bundle.pem") + expectedEnvVars := map[string]string{ + "AWS_CA_BUNDLE": "/etc/test/ca_bundle.pem", + } checkTranslation(t, "base_container_insights_config", "linux", expectedEnvVars, "") checkTranslation(t, "base_container_insights_config", "darwin", nil, "") } diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 0eee896963..b33881ed70 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -6,6 +6,7 @@ package awsemf import ( _ "embed" "fmt" + "os" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter" "go.opentelemetry.io/collector/component" @@ -13,6 +14,7 @@ import ( "go.opentelemetry.io/collector/exporter" "gopkg.in/yaml.v3" + "github.com/aws/amazon-cloudwatch-agent/cfg/envconfig" "github.com/aws/amazon-cloudwatch-agent/internal/retryer" "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" @@ -33,6 +35,7 @@ var ( kubernetesBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey) prometheusBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.PrometheusKey) emfProcessorBasePathKey = common.ConfigKey(prometheusBasePathKey, common.EMFProcessorKey) + endpointOverrideKey = common.ConfigKey(common.LogsKey, common.EndpointOverrideKey) ) type translator struct { @@ -79,6 +82,11 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { } } cfg.AWSSessionSettings.Region = agent.Global_Config.Region + if c.IsSet(endpointOverrideKey) { + cfg.AWSSessionSettings.Endpoint, _ = common.GetString(c, endpointOverrideKey) + } + cfg.AWSSessionSettings.CertificateFilePath = os.Getenv(envconfig.AWS_CA_BUNDLE) + cfg.AWSSessionSettings.Region = agent.Global_Config.Region if profileKey, ok := agent.Global_Config.Credentials[agent.Profile_Key]; ok { cfg.AWSSessionSettings.Profile = fmt.Sprintf("%v", profileKey) } @@ -100,7 +108,6 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { return nil, err } } - return cfg, nil } diff --git a/translator/translate/otel/exporter/otel_aws_cloudwatch_logs/translator.go b/translator/translate/otel/exporter/otel_aws_cloudwatch_logs/translator.go index ec4ee1c190..09cf02b61f 100644 --- a/translator/translate/otel/exporter/otel_aws_cloudwatch_logs/translator.go +++ b/translator/translate/otel/exporter/otel_aws_cloudwatch_logs/translator.go @@ -89,7 +89,6 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { if c.IsSet(endpointOverrideKey) { cfg.AWSSessionSettings.Endpoint, _ = common.GetString(c, endpointOverrideKey) } - cfg.AWSSessionSettings.CertificateFilePath = os.Getenv(envconfig.AWS_CA_BUNDLE) cfg.AWSSessionSettings.IMDSRetries = retryer.GetDefaultRetryNumber() return cfg, nil From 5deb8c964efd3ac8efd5a0b27f162e7244996873 Mon Sep 17 00:00:00 2001 From: Yared <45324375+ymtaye@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:51:35 -0400 Subject: [PATCH 19/24] Default timestamp_format to be compatible with zero padding and non zero padding options for month and day (#870) --- plugins/inputs/logfile/README.md | 4 +- plugins/inputs/logfile/fileconfig.go | 11 +- plugins/inputs/logfile/fileconfig_test.go | 51 ++++- plugins/inputs/logfile/logfile.go | 2 +- plugins/inputs/logfile/logfile_test.go | 2 +- plugins/outputs/cloudwatchlogs/pusher.go | 13 +- .../logs_and_kubernetes_config.conf | 2 +- .../sampleConfig/no_skip_log_timestamp.conf | 4 +- .../no_skip_log_timestamp_windows.conf | 4 +- .../files/collect_list/collect_list_test.go | 89 ++++++-- .../files/collect_list/ruleTimestampFormat.go | 31 ++- .../collect_list/ruleTimestampFormat_test.go | 199 ++++++++++++++++++ 12 files changed, 365 insertions(+), 47 deletions(-) create mode 100644 translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat_test.go diff --git a/plugins/inputs/logfile/README.md b/plugins/inputs/logfile/README.md index 3a7adcfb4b..e514ce75c4 100644 --- a/plugins/inputs/logfile/README.md +++ b/plugins/inputs/logfile/README.md @@ -45,7 +45,7 @@ The plugin expects messages in one of the log_group_name = "logfile.log" log_stream_name = "" timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$" - timestamp_layout = "02 Jan 2006 15:04:05" + timestamp_layout = ["_2 Jan 2006 15:04:05"] timezone = "UTC" multi_line_start_pattern = "{timestamp_regex}" ## Read file from beginning. @@ -63,7 +63,7 @@ The plugin expects messages in one of the log_group_name = "varlog" log_stream_name = "" timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$" - timestamp_layout = "02 Jan 2006 15:04:05" + timestamp_layout = ["_2 Jan 2006 15:04:05"] timezone = "UTC" multi_line_start_pattern = "{timestamp_regex}" ## Read file from beginning. diff --git a/plugins/inputs/logfile/fileconfig.go b/plugins/inputs/logfile/fileconfig.go index dd383ccd54..c136f9d7dc 100644 --- a/plugins/inputs/logfile/fileconfig.go +++ b/plugins/inputs/logfile/fileconfig.go @@ -43,7 +43,7 @@ type FileConfig struct { //The regex of the timestampFromLogLine presents in the log entry TimestampRegex string `toml:"timestamp_regex"` //The timestampFromLogLine layout used in GoLang to parse the timestampFromLogLine. - TimestampLayout string `toml:"timestamp_layout"` + TimestampLayout []string `toml:"timestamp_layout"` //The time zone used to parse the timestampFromLogLine in the log entry. Timezone string `toml:"timezone"` @@ -179,7 +179,14 @@ func (config *FileConfig) timestampFromLogLine(logValue string) time.Time { replacement := fmt.Sprintf(".%s", fracSecond[:3]) timestampContent = fmt.Sprintf("%s%s%s", timestampContent[:start], replacement, timestampContent[end:]) } - timestamp, err := time.ParseInLocation(config.TimestampLayout, timestampContent, config.TimezoneLoc) + var err error + var timestamp time.Time + for _, timestampLayout := range config.TimestampLayout { + timestamp, err = time.ParseInLocation(timestampLayout, timestampContent, config.TimezoneLoc) + if err == nil { + break + } + } if err != nil { log.Printf("E! Error parsing timestampFromLogLine: %s", err) return time.Time{} diff --git a/plugins/inputs/logfile/fileconfig_test.go b/plugins/inputs/logfile/fileconfig_test.go index 8f0b2b96b0..4890adb7c5 100644 --- a/plugins/inputs/logfile/fileconfig_test.go +++ b/plugins/inputs/logfile/fileconfig_test.go @@ -18,7 +18,7 @@ func TestFileConfigInit(t *testing.T) { FilePath: "/tmp/logfile.log", LogGroupName: "logfile.log", TimestampRegex: "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})", - TimestampLayout: "02 Jan 2006 15:04:05", + TimestampLayout: []string{"02 Jan 2006 15:04:05"}, Timezone: "UTC", MultiLineStartPattern: "{timestamp_regex}", } @@ -47,7 +47,7 @@ func TestFileConfigInitFailureCase(t *testing.T) { FilePath: "/tmp/logfile.log", LogGroupName: "logfile.log", TimestampRegex: "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}+)", - TimestampLayout: "02 Jan 2006 15:04:05", + TimestampLayout: []string{"02 Jan 2006 15:04:05"}, Timezone: "UTC", MultiLineStartPattern: "{timestamp_regex}", } @@ -60,7 +60,7 @@ func TestFileConfigInitFailureCase(t *testing.T) { FilePath: "/tmp/logfile.log", LogGroupName: "logfile.log", TimestampRegex: "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})", - TimestampLayout: "02 Jan 2006 15:04:05", + TimestampLayout: []string{"02 Jan 2006 15:04:05"}, Timezone: "UTC", MultiLineStartPattern: "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}+)", } @@ -82,7 +82,7 @@ func TestLogGroupName(t *testing.T) { func TestTimestampParser(t *testing.T) { timestampRegex := "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})" - timestampLayout := "02 Jan 2006 15:04:05" + timestampLayout := []string{"02 Jan 2006 15:04:05"} timezone := "UTC" timezoneLoc := time.UTC timestampRegexP, err := regexp.Compile(timestampRegex) @@ -110,7 +110,7 @@ func TestTimestampParser(t *testing.T) { func TestTimestampParserWithPadding(t *testing.T) { timestampRegex := "(\\d{1,2} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})" - timestampLayout := "1 2 15:04:05" + timestampLayout := []string{"1 2 15:04:05"} timezone := "UTC" timezoneLoc := time.UTC timestampRegexP, err := regexp.Compile(timestampRegex) @@ -133,9 +133,48 @@ func TestTimestampParserWithPadding(t *testing.T) { assert.Equal(t, 10, timestamp.Minute(), fmt.Sprintf("Timestamp does not match: %v, act: %v", "10", timestamp.Minute())) } +func TestTimestampParserDefault(t *testing.T) { + // Check when timestamp_format is "%b %d %H:%M:%S" + // %d and %-d are both treated as s{0,1}\\d{1,2} + timestampRegex := "(\\w{3} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})" + timestampLayout := []string{"test", "Jan 2 15:04:05"} + timezone := "UTC" + timezoneLoc := time.UTC + timestampRegexP, err := regexp.Compile(timestampRegex) + require.NoError(t, err, fmt.Sprintf("Failed to compile regex %s", timestampRegex)) + fileConfig := &FileConfig{ + TimestampRegex: timestampRegex, + TimestampRegexP: timestampRegexP, + TimestampLayout: timestampLayout, + Timezone: timezone, + TimezoneLoc: timezoneLoc} + + // make sure layout is compatible for "Sep 9", "Sep 9" , "Sep 09", "Sep 09" options + logEntry := fmt.Sprintf("Sep 9 02:00:43 ip-10-4-213-132 \n") + timestamp := fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, 02, timestamp.Hour()) + assert.Equal(t, 00, timestamp.Minute()) + + logEntry = fmt.Sprintf("Sep 9 02:00:43 ip-10-4-213-132 \n") + timestamp = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, 02, timestamp.Hour()) + assert.Equal(t, 00, timestamp.Minute()) + + logEntry = fmt.Sprintf("Sep 09 02:00:43 ip-10-4-213-132 \n") + timestamp = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, 02, timestamp.Hour()) + assert.Equal(t, 00, timestamp.Minute()) + + logEntry = fmt.Sprintf("Sep 09 02:00:43 ip-10-4-213-132 \n") + timestamp = fileConfig.timestampFromLogLine(logEntry) + assert.Equal(t, 02, timestamp.Hour()) + assert.Equal(t, 00, timestamp.Minute()) + +} + func TestTimestampParserWithFracSeconds(t *testing.T) { timestampRegex := "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2},(\\d{1,9}) \\w{3})" - timestampLayout := "02 Jan 2006 15:04:05,.000 MST" + timestampLayout := []string{"02 Jan 2006 15:04:05,.000 MST"} timezone := "UTC" timezoneLoc := time.UTC timestampRegexP, err := regexp.Compile(timestampRegex) diff --git a/plugins/inputs/logfile/logfile.go b/plugins/inputs/logfile/logfile.go index 3dc0578b99..1ca98da36b 100644 --- a/plugins/inputs/logfile/logfile.go +++ b/plugins/inputs/logfile/logfile.go @@ -73,7 +73,7 @@ const sampleConfig = ` log_stream_name = "" publish_multi_logs = false timestamp_regex = "^(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2}).*$" - timestamp_layout = "02 Jan 2006 15:04:05" + timestamp_layout = ["_2 Jan 2006 15:04:05"] timezone = "UTC" multi_line_start_pattern = "{timestamp_regex}" ## Read file from beginning. diff --git a/plugins/inputs/logfile/logfile_test.go b/plugins/inputs/logfile/logfile_test.go index 4efa63115a..8ce9177aec 100644 --- a/plugins/inputs/logfile/logfile_test.go +++ b/plugins/inputs/logfile/logfile_test.go @@ -504,7 +504,7 @@ append line` FilePath: tmpfile.Name(), FromBeginning: true, TimestampRegex: "(\\d{2}:\\d{2}:\\d{2} \\d{2} \\w{3} \\s{0,1}\\d{1,2})", - TimestampLayout: "15:04:05 06 Jan 2", + TimestampLayout: []string{"15:04:05 06 Jan 2"}, MultiLineStartPattern: "{timestamp_regex}", Timezone: time.UTC.String(), }} diff --git a/plugins/outputs/cloudwatchlogs/pusher.go b/plugins/outputs/cloudwatchlogs/pusher.go index 80572880bc..83b8f55282 100644 --- a/plugins/outputs/cloudwatchlogs/pusher.go +++ b/plugins/outputs/cloudwatchlogs/pusher.go @@ -20,8 +20,9 @@ import ( ) const ( - reqSizeLimit = 1024 * 1024 - reqEventsLimit = 10000 + reqSizeLimit = 1024 * 1024 + reqEventsLimit = 10000 + warnOldTimeStamp = 1 * 24 * time.Hour ) var ( @@ -51,6 +52,7 @@ type pusher struct { flushTimer *time.Timer sequenceToken *string lastValidTime int64 + lastUpdateTime time.Time needSort bool stop <-chan struct{} lastSentTime time.Time @@ -413,12 +415,19 @@ func (p *pusher) convertEvent(e logs.LogEvent) *cloudwatchlogs.InputLogEvent { // a valid timestamp and use the last valid timestamp for new entries that does // not have a timestamp. t = p.lastValidTime + if !p.lastUpdateTime.IsZero() { + // Check when timestamp has an interval of 5 days. + if time.Since(p.lastUpdateTime) > warnOldTimeStamp { + p.Log.Warnf("Unable to parse timestamp, using last valid timestamp found in the logs %v: which is at least older than 1 day for log group %v: ", p.lastValidTime, p.Group) + } + } } else { t = time.Now().UnixNano() / 1000000 } } else { t = e.Time().UnixNano() / 1000000 p.lastValidTime = t + p.lastUpdateTime = time.Now() } return &cloudwatchlogs.InputLogEvent{ Message: &message, diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.conf b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.conf index ce241e990c..62db563917 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.conf +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.conf @@ -28,7 +28,7 @@ multi_line_start_pattern = "{timestamp_regex}" pipe = false retention_in_days = -1 - timestamp_layout = "02 Jan 2006 15:04:05" + timestamp_layout = ["_2 Jan 2006 15:04:05"] timestamp_regex = "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})" timezone = "UTC" diff --git a/translator/tocwconfig/sampleConfig/no_skip_log_timestamp.conf b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp.conf index 5bf0ea1d61..18f892386e 100644 --- a/translator/tocwconfig/sampleConfig/no_skip_log_timestamp.conf +++ b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp.conf @@ -26,8 +26,8 @@ log_group_name = "amazon-cloudwatch-agent.log" pipe = false retention_in_days = -1 - timestamp_layout = "15:04:05 06 Jan 02" - timestamp_regex = "(d{2}:d{2}:d{2} d{2} w{3} d{2})" + timestamp_layout = ["15:04:05 06 Jan _2"] + timestamp_regex = "(d{2}:d{2}:d{2} d{2} w{3} s{0,1} d{1,2})" [outputs] diff --git a/translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.conf b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.conf index ef0961d523..70d57a32b2 100644 --- a/translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.conf +++ b/translator/tocwconfig/sampleConfig/no_skip_log_timestamp_windows.conf @@ -26,8 +26,8 @@ log_group_name = "amazon-cloudwatch-agent.log" pipe = false retention_in_days = -1 - timestamp_layout = "15:04:05 06 Jan 02" - timestamp_regex = "(d{2}:d{2}:d{2} d{2} w{3} d{2})" + timestamp_layout = ["15:04:05 06 Jan _2"] + timestamp_regex = "(d{2}:d{2}:d{2} d{2} w{3} s{0,1} d{1,2})" [outputs] diff --git a/translator/translate/logs/logs_collected/files/collect_list/collect_list_test.go b/translator/translate/logs/logs_collected/files/collect_list/collect_list_test.go index a2ad595f70..297e486efb 100644 --- a/translator/translate/logs/logs_collected/files/collect_list/collect_list_test.go +++ b/translator/translate/logs/logs_collected/files/collect_list/collect_list_test.go @@ -78,7 +78,7 @@ func TestTimestampFormat(t *testing.T) { "file_path": "path1", "from_beginning": true, "pipe": false, - "timestamp_layout": "15:04:05 06 Jan 2", + "timestamp_layout": []string{"15:04:05 06 Jan _2"}, "timestamp_regex": "(\\d{2}:\\d{2}:\\d{2} \\d{2} \\w{3} \\s{0,1}\\d{1,2})", "timezone": "UTC", "retention_in_days": -1, @@ -105,7 +105,7 @@ func TestTimestampFormatAll(t *testing.T) { "from_beginning": true, "pipe": false, "retention_in_days": -1, - "timestamp_layout": "15:04:05 06 Jan 2", + "timestamp_layout": []string{"15:04:05 06 Jan _2"}, "timestamp_regex": "(\\d{2}:\\d{2}:\\d{2} \\d{2} \\w{3} \\s{0,1}\\d{1,2})", }}, }, @@ -123,7 +123,7 @@ func TestTimestampFormatAll(t *testing.T) { "from_beginning": true, "pipe": false, "retention_in_days": -1, - "timestamp_layout": "1 2 15:04:05", + "timestamp_layout": []string{"1 _2 15:04:05", "01 _2 15:04:05"}, "timestamp_regex": "(\\d{1,2} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})", }}, }, @@ -141,10 +141,46 @@ func TestTimestampFormatAll(t *testing.T) { "from_beginning": true, "pipe": false, "retention_in_days": -1, - "timestamp_layout": "2 1 15:04:05", + "timestamp_layout": []string{"_2 1 15:04:05", "_2 01 15:04:05"}, "timestamp_regex": "(\\d{1,2} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})", }}, }, + { + input: `{ + "collect_list":[ + { + "file_path":"path4", + "timestamp_format": "%b %d %H:%M:%S" + } + ] + }`, + expected: []interface{}{map[string]interface{}{ + "file_path": "path4", + "from_beginning": true, + "pipe": false, + "retention_in_days": -1, + "timestamp_layout": []string{"Jan _2 15:04:05"}, + "timestamp_regex": "(\\w{3} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})", + }}, + }, + { + input: `{ + "collect_list":[ + { + "file_path":"path5", + "timestamp_format": "%b %-d %H:%M:%S" + } + ] + }`, + expected: []interface{}{map[string]interface{}{ + "file_path": "path5", + "from_beginning": true, + "pipe": false, + "retention_in_days": -1, + "timestamp_layout": []string{"Jan _2 15:04:05"}, + "timestamp_regex": "(\\w{3} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})", + }}, + }, { input: `{ "collect_list":[ @@ -159,7 +195,25 @@ func TestTimestampFormatAll(t *testing.T) { "from_beginning": true, "pipe": false, "retention_in_days": -1, - "timestamp_layout": "5 2 1 15:04:05", + "timestamp_layout": []string{"5 _2 1 15:04:05", "5 _2 01 15:04:05"}, + "timestamp_regex": "(\\d{1,2} \\s{0,1}\\d{1,2} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})", + }}, + }, + { + input: `{ + "collect_list":[ + { + "file_path":"path7", + "timestamp_format":"%-S %-d %m %H:%M:%S" + } + ] + }`, + expected: []interface{}{map[string]interface{}{ + "file_path": "path7", + "from_beginning": true, + "pipe": false, + "retention_in_days": -1, + "timestamp_layout": []string{"5 _2 01 15:04:05", "5 _2 1 15:04:05"}, "timestamp_regex": "(\\d{1,2} \\s{0,1}\\d{1,2} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})", }}, }, @@ -182,8 +236,6 @@ func applyRule1(t *testing.T, buf string) interface{} { return val } -// stdNumMonth // "1" //%-m -// stdDay // "2" //%-d // -hour:-minute:-seconds does not work for golang parser. func TestTimestampFormat_NonZeroPadding(t *testing.T) { f := new(FileConfig) @@ -201,7 +253,7 @@ func TestTimestampFormat_NonZeroPadding(t *testing.T) { assert.Fail(t, e.Error()) } _, val := f.ApplyRule(input) - expectedLayout := "3:4:5 06 1 2" + expectedLayout := []string{"3:4:5 06 1 _2", "3:4:5 06 01 _2"} expectedRegex := "(\\d{1,2}:\\d{1,2}:\\d{1,2} \\d{2} \\s{0,1}\\d{1,2} \\s{0,1}\\d{1,2})" expectVal := []interface{}{map[string]interface{}{ "file_path": "path1", @@ -223,8 +275,7 @@ func TestTimestampFormat_NonZeroPadding(t *testing.T) { match := regex.FindStringSubmatch(sampleLogEntry) assert.NotNil(t, match) assert.Equal(t, 2, len(match)) - - parsedTime, err := time.ParseInLocation(expectedLayout, match[1], time.UTC) + parsedTime, err := time.ParseInLocation(expectedLayout[0], match[1], time.UTC) assert.NoError(t, err) assert.Equal(t, time.Date(2018, 3, 8, 1, 2, 3, 0, time.UTC), parsedTime) } @@ -247,7 +298,7 @@ func TestTimestampFormat_SpecialCharacters(t *testing.T) { assert.Fail(t, e.Error()) } _, val := f.ApplyRule(input) - expectedLayout := "^.*?|[({15:04:05 06 Jan 2})]$" + expectedLayout := []string{"^.*?|[({15:04:05 06 Jan _2})]$"} expectedRegex := "(\\^\\.\\*\\?\\|\\[\\(\\{\\d{2}:\\d{2}:\\d{2} \\d{2} \\w{3} \\s{0,1}\\d{1,2}\\}\\)\\]\\$)" expectVal := []interface{}{map[string]interface{}{ "file_path": "path1", @@ -266,7 +317,7 @@ func TestTimestampFormat_SpecialCharacters(t *testing.T) { assert.NotNil(t, match) assert.Equal(t, 2, len(match)) - parsedTime, err := time.ParseInLocation(expectedLayout, match[1], time.UTC) + parsedTime, err := time.ParseInLocation(expectedLayout[0], match[1], time.UTC) assert.NoError(t, err) assert.Equal(t, time.Date(2017, 12, 27, 12, 52, 0, 0, time.UTC), parsedTime) } @@ -286,7 +337,7 @@ func TestTimestampFormat_Template(t *testing.T) { assert.Fail(t, e.Error()) } _, val := f.ApplyRule(input) - expectedLayout := "Jan 2 15:04:05" + expectedLayout := []string{"Jan _2 15:04:05"} expectedRegex := "(\\w{3} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})" expectVal := []interface{}{map[string]interface{}{ "file_path": "path1", @@ -304,14 +355,14 @@ func TestTimestampFormat_Template(t *testing.T) { assert.NotNil(t, match) assert.Equal(t, 2, len(match)) - parsedTime, err := time.ParseInLocation(expectedLayout, match[1], time.Local) + parsedTime, err := time.ParseInLocation(expectedLayout[0], match[1], time.Local) assert.NoError(t, err) assert.Equal(t, time.Date(0, 8, 9, 20, 45, 51, 0, time.Local), parsedTime) } func TestTimestampFormat_InvalidRegex(t *testing.T) { translator.ResetMessages() - r := new(TimestampRegax) + r := new(TimestampRegex) var input interface{} e := json.Unmarshal([]byte(`{ "timestamp_format":"%Y-%m-%dT%H:%M%S+00:00" @@ -347,8 +398,8 @@ func TestMultiLineStartPattern(t *testing.T) { "from_beginning": true, "pipe": false, "retention_in_days": -1, - "timestamp_layout": "15:04:05 06 Jan 02", - "timestamp_regex": "(\\d{2}:\\d{2}:\\d{2} \\d{2} \\w{3} \\d{2})", + "timestamp_layout": []string{"15:04:05 06 Jan _2"}, + "timestamp_regex": "(\\d{2}:\\d{2}:\\d{2} \\d{2} \\w{3} \\s{0,1}\\d{1,2})", "timezone": "UTC", "multi_line_start_pattern": "{timestamp_regex}", }} @@ -377,8 +428,8 @@ func TestEncoding(t *testing.T) { "from_beginning": true, "pipe": false, "retention_in_days": -1, - "timestamp_layout": "15:04:05 06 Jan 02", - "timestamp_regex": "(\\d{2}:\\d{2}:\\d{2} \\d{2} \\w{3} \\d{2})", + "timestamp_layout": []string{"15:04:05 06 Jan _2"}, + "timestamp_regex": "(\\d{2}:\\d{2}:\\d{2} \\d{2} \\w{3} \\s{0,1}\\d{1,2})", "timezone": "UTC", "encoding": "gbk", }} diff --git a/translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat.go b/translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat.go index 05b8ae318e..34690e88c9 100644 --- a/translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat.go +++ b/translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat.go @@ -62,8 +62,8 @@ var TimeFormatMap = map[string]string{ "%m": "01", "%A": "Monday", "%a": "Mon", - "%-d": "2", - "%d": "02", + "%-d": "_2", + "%d": "_2", "%H": "15", "%-I": "3", "%I": "03", @@ -83,11 +83,11 @@ var TimeFormatRexMap = map[string]string{ "%B": "\\w{7}", "%b": "\\w{3}", "%-m": "\\s{0,1}\\d{1,2}", - "%m": "\\d{2}", + "%m": "\\s{0,1}\\d{1,2}", "%A": "\\w{6,9}", "%a": "\\w{3}", "%-d": "\\s{0,1}\\d{1,2}", - "%d": "\\d{2}", + "%d": "\\s{0,1}\\d{1,2}", "%H": "\\d{2}", "%-I": "\\d{1,2}", "%I": "\\d{2}", @@ -136,12 +136,12 @@ func checkAndReplace(input string, timestampFormatMap map[string]string) string return res } -type TimestampRegax struct { +type TimestampRegex struct { } // ApplyRule add timestamp regex // do not add timestamp check when viewing cwa logfile -func (t *TimestampRegax) ApplyRule(input interface{}) (returnKey string, returnVal interface{}) { +func (t *TimestampRegex) ApplyRule(input interface{}) (returnKey string, returnVal interface{}) { //Convert the input string into []rune and iterate the map and build the output []rune m := input.(map[string]interface{}) //If user not specify the timestamp_format, then no config entry for "timestamp_layout" in TOML @@ -156,7 +156,7 @@ func (t *TimestampRegax) ApplyRule(input interface{}) (returnKey string, returnV res = checkAndReplace(res, TimeFormatRexMap) // remove the prefix, if the format startswith "%-m" or "%-d", there is an "\\s{0,1}" at the beginning. // like "timestamp_format": "%-m %-d %H:%M:%S" will be converted into following layout and regex - // timestamp_layout = "1 2 15:04:05" + // timestamp_layout = ["1 _2 15:04:05"] // timestamp_regex = "(\\s{0,1}\\d{1,2} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})" // following timestamp string " 2 1 07:10:06" matches the regex, but it can not match the layout. // After the prefix "\\s{0,1}", it can match both the regex and layout. @@ -190,7 +190,20 @@ func (t *TimestampLayout) ApplyRule(input interface{}) (returnKey string, return res := checkAndReplace(val.(string), TimeFormatMap) //If user provide with the specific timestamp_format, use the one that user provide returnKey = "timestamp_layout" - returnVal = res + timestampInput := val.(string) + // Go doesn't support _2 option for month in day as a result need to set + // timestamp_layout with 2 strings which support %m and %-m + if strings.Contains(timestampInput, "%m") { + timestampInput := strings.Replace(timestampInput, "%m", "%-m", -1) + alternativeLayout := checkAndReplace(timestampInput, TimeFormatMap) + returnVal = []string{res, alternativeLayout} + } else if strings.Contains(timestampInput, "%-m") { + timestampInput = strings.Replace(timestampInput, "%-m", "%m", -1) + alternativeLayout := checkAndReplace(timestampInput, TimeFormatMap) + returnVal = []string{res, alternativeLayout} + } else { + returnVal = []string{res} + } } return } @@ -216,7 +229,7 @@ func (t *Timezone) ApplyRule(input interface{}) (returnKey string, returnVal int } func init() { t1 := new(TimestampLayout) - t2 := new(TimestampRegax) + t2 := new(TimestampRegex) t3 := new(Timezone) r := []Rule{t1, t2, t3} RegisterRule("timestamp_format", r) diff --git a/translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat_test.go b/translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat_test.go new file mode 100644 index 0000000000..d94c54f33a --- /dev/null +++ b/translator/translate/logs/logs_collected/files/collect_list/ruleTimestampFormat_test.go @@ -0,0 +1,199 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package collect_list + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTimestampRegexRule(t *testing.T) { + regex := new(TimestampRegex) + type want struct { + key string + value interface{} + } + testCases := map[string]struct { + input map[string]interface{} + want *want + wantErr error + }{ + "WithNonZeroPaddedOptions": { + input: map[string]interface{}{ + "timestamp_format": "%-m %-d %H:%M:%S", + }, + want: &want{ + key: "timestamp_regex", + value: "(\\d{1,2} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})", + }, + }, + "WithZeroPaddedOptions": { + input: map[string]interface{}{ + "timestamp_format": "%m %d %H:%M:%S", + }, + want: &want{ + key: "timestamp_regex", + value: "(\\d{1,2} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})", + }, + }, + "WithZeroPaddedMonthWord": { + input: map[string]interface{}{ + "timestamp_format": "%b %d %H:%M:%S", + }, + want: &want{ + key: "timestamp_regex", + value: "(\\w{3} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})", + }, + }, + "WithNonZeroPaddedMonthWord": { + input: map[string]interface{}{ + "timestamp_format": "%b %-d %H:%M:%S", + }, + want: &want{ + key: "timestamp_regex", + value: "(\\w{3} \\s{0,1}\\d{1,2} \\d{2}:\\d{2}:\\d{2})", + }, + }, + "WithYearAsTwoDigits": { + input: map[string]interface{}{ + "timestamp_format": "%b %-d %y %H:%M:%S", + }, + want: &want{ + key: "timestamp_regex", + value: "(\\w{3} \\s{0,1}\\d{1,2} \\d{2} \\d{2}:\\d{2}:\\d{2})", + }, + }, + "WithYearAsFourDigits": { + input: map[string]interface{}{ + "timestamp_format": "%b %-d %Y %H:%M:%S", + }, + want: &want{ + key: "timestamp_regex", + value: "(\\w{3} \\s{0,1}\\d{1,2} \\d{4} \\d{2}:\\d{2}:\\d{2})", + }, + }, + "WithNoTimestampFormat": { + input: map[string]interface{}{ + "timestamp": "foo", + }, + want: &want{ + key: "", + value: "", + }, + }, + "WithInvalidTimestampFormat": { + input: map[string]interface{}{ + "timestamp_format": "foo", + }, + want: &want{ + key: "timestamp_regex", + value: "(foo)", + }, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + res, returnVal := regex.ApplyRule(testCase.input) + require.NotNil(t, res) + assert.Equal(t, res, testCase.want.key) + assert.Equal(t, returnVal, testCase.want.value) + }) + } +} + +func TestTimestampLayoutxRule(t *testing.T) { + layout := new(TimestampLayout) + type want struct { + key string + value interface{} + } + testCases := map[string]struct { + input map[string]interface{} + want *want + wantErr error + }{ + "WithNonZeroPaddedOptions": { + input: map[string]interface{}{ + "timestamp_format": "%-m %-d %H:%M:%S", + }, + want: &want{ + key: "timestamp_layout", + value: []string{"1 _2 15:04:05", "01 _2 15:04:05"}, + }, + }, + "WithZeroPaddedOptions": { + input: map[string]interface{}{ + "timestamp_format": "%m %d %H:%M:%S", + }, + want: &want{ + key: "timestamp_layout", + value: []string{"01 _2 15:04:05", "1 _2 15:04:05"}, + }, + }, + "WithZeroPaddedMonthWord": { + input: map[string]interface{}{ + "timestamp_format": "%b %d %H:%M:%S", + }, + want: &want{ + key: "timestamp_layout", + value: []string{"Jan _2 15:04:05"}, + }, + }, + "WithNonZeroPaddedMonthWord": { + input: map[string]interface{}{ + "timestamp_format": "%b %-d %H:%M:%S", + }, + want: &want{ + key: "timestamp_layout", + value: []string{"Jan _2 15:04:05"}, + }, + }, + "WithYearAsTwoDigits": { + input: map[string]interface{}{ + "timestamp_format": "%b %-d %y %H:%M:%S", + }, + want: &want{ + key: "timestamp_layout", + value: []string{"Jan _2 06 15:04:05"}, + }, + }, + "WithYearAsFourDigits": { + input: map[string]interface{}{ + "timestamp_format": "%b %-d %Y %H:%M:%S", + }, + want: &want{ + key: "timestamp_layout", + value: []string{"Jan _2 2006 15:04:05"}, + }, + }, + "WithNoTimestampFormat": { + input: map[string]interface{}{ + "timestamp": "foo", + }, + want: &want{ + key: "", + value: "", + }, + }, + "WithInvalidTimestampFormat": { + input: map[string]interface{}{ + "timestamp_format": "foo", + }, + want: &want{ + key: "timestamp_layout", + value: []string{"foo"}, + }, + }, + } + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + res, returnVal := layout.ApplyRule(testCase.input) + require.NotNil(t, res) + assert.Equal(t, res, testCase.want.key) + assert.Equal(t, returnVal, testCase.want.value) + }) + } +} From b1bee719ca47d334f6924343497234c158dfbe00 Mon Sep 17 00:00:00 2001 From: Adam <90734270+adam-mateen@users.noreply.github.com> Date: Thu, 19 Oct 2023 13:21:25 -0500 Subject: [PATCH 20/24] change auto_removal so it does not block waiting to finish reading previous file (#914) --- plugins/inputs/logfile/logfile_test.go | 65 ++++++++++++++---------- plugins/inputs/logfile/tail/tail.go | 5 +- plugins/inputs/logfile/tail/tail_test.go | 13 ++--- 3 files changed, 47 insertions(+), 36 deletions(-) diff --git a/plugins/inputs/logfile/logfile_test.go b/plugins/inputs/logfile/logfile_test.go index 8ce9177aec..f5a38ab48c 100644 --- a/plugins/inputs/logfile/logfile_test.go +++ b/plugins/inputs/logfile/logfile_test.go @@ -371,11 +371,12 @@ func TestLogsFileRemove(t *testing.T) { tt.Stop() } -func setupLogFileForTest(t *testing.T, file *os.File, prefix string) *LogFile { +func setupLogFileForTest(t *testing.T, monitorPath string) *LogFile { logFile := NewLogFile() logFile.Log = TestLogger{t} + t.Logf("create LogFile with FilePath = %s", monitorPath) logFile.FileConfig = []FileConfig{{ - FilePath: filepath.Join(filepath.Dir(file.Name()), prefix+"*"), + FilePath: monitorPath, FromBeginning: true, AutoRemoval: true, }} @@ -394,8 +395,12 @@ func makeTempFile(t *testing.T, prefix string) *os.File { // getLogSrc returns a LogSrc from the given LogFile, and the channel for output. // Verifies 1 and only 1 LogSrc is discovered. func getLogSrc(t *testing.T, logFile *LogFile) (*logs.LogSrc, chan logs.LogEvent) { + start := time.Now() logSources := logFile.FindLogSrc() - require.Equal(t, 1, len(logSources)) + duration := time.Since(start) + // LogFile.FindLogSrc() should not block. + require.Less(t, duration, time.Millisecond*100) + require.Equal(t, 1, len(logSources), "FindLogSrc() expected 1, got %d", len(logSources)) logSource := logSources[0] evts := make(chan logs.LogEvent) logSource.SetOutput(func(e logs.LogEvent) { @@ -407,48 +412,38 @@ func getLogSrc(t *testing.T, logFile *LogFile) (*logs.LogSrc, chan logs.LogEvent } func writeLines(t *testing.T, file *os.File, numLines int, msg string) { - t.Log("Fill temp file with sufficient lines to be read.") + t.Logf("start writing, %s", file.Name()) for i := 0; i < numLines; i++ { _, err := file.WriteString(msg + "\n") require.NoError(t, err) } + t.Logf("stop writing, %s", file.Name()) } // createWriteRead creates a temp file, writes to it, then verifies events // are received. If isParent is true, then spawn a 2nd goroutine for createWriteRead. -// Close the given channel when complete to let caller know it was successful. +// Closes "done" when complete to let caller know it was successful. func createWriteRead(t *testing.T, prefix string, logFile *LogFile, done chan bool, isParent bool) { // Let caller know when the goroutine is done. defer close(done) // done2 is only passed to child if this is the parent. done2 := make(chan bool) file := makeTempFile(t, prefix) - if isParent { - logFile = setupLogFileForTest(t, file, prefix) - defer logFile.Stop() - } logSrc, evts := getLogSrc(t, logFile) defer (*logSrc).Stop() defer close(evts) // Choose a large enough number of lines so that even high-spec hosts will not // complete receiving logEvents before the 2nd createWriteRead() goroutine begins. - const numLines int = 100000 + const numLines int = 1000000 const msg string = "this is the best log line ever written to a file" writeLines(t, file, numLines, msg) file.Close() - if !isParent { - // Child creates 2nd temp file which is NOT auto removed. - defer os.Remove(file.Name()) - } t.Log("Verify every line written to the temp file is received.") for i := 0; i < numLines; i++ { logEvent := <-evts require.Equal(t, msg, logEvent.Message()) - if i != numLines/2 { - continue - } - // Halfway through start another goroutine to create another temp file. - if isParent { + if isParent && i == numLines/2 { + // Halfway through start child goroutine to create another temp file. go createWriteRead(t, prefix, logFile, done2, false) } } @@ -457,8 +452,8 @@ func createWriteRead(t *testing.T, prefix string, logFile *LogFile, done chan bo t.Log("Verify child completed.") select { case <-done2: - t.Log("Completed before timeout (as expected)") - case <-time.After(time.Second * 5): + t.Log("Child completed before timeout (as expected)") + case <-time.After(time.Second * 10): require.Fail(t, "timeout waiting for child") } t.Log("Verify 1st temp file was auto deleted.") @@ -468,21 +463,37 @@ func createWriteRead(t *testing.T, prefix string, logFile *LogFile, done chan bo } // TestLogsFileAutoRemoval verifies when a new file matching the configured -// FilePath is discovered, the old file will be automatically deleted after -// being read to the end-of-file. +// FilePath is discovered, the old file will be automatically deleted ONLY after +// being read to the end-of-file. Also verifies the new log file is discovered +// before finishing the old file. func TestLogsFileAutoRemoval(t *testing.T) { // Override global in tailersrc.go. multilineWaitPeriod = 10 * time.Millisecond - prefix := "file_auto_removal" + prefix := "TestLogsFileAutoRemoval*" + f1 := makeTempFile(t, prefix) + f1.Close() + os.Remove(f1.Name()) + // Create the LogFile. + fileDirectoryPath := filepath.Dir(f1.Name()) + monitorPath := filepath.Join(fileDirectoryPath, prefix) + logFile := setupLogFileForTest(t, monitorPath) + defer logFile.Stop() + done := make(chan bool) - createWriteRead(t, prefix, nil, done, true) + createWriteRead(t, prefix, logFile, done, true) t.Log("Verify 1st tmp file created and discovered.") select { case <-done: - t.Log("Completed before timeout (as expected)") - case <-time.After(time.Second * 5): + t.Log("Parent completed before timeout (as expected)") + case <-time.After(time.Second * 10): require.Fail(t, "timeout waiting for 2nd temp file.") } + // Cleanup + files, _ := filepath.Glob(monitorPath) + for _, f := range files { + t.Logf("cleanup, %s", f) + os.Remove(f) + } } func TestLogsTimestampAsMultilineStarter(t *testing.T) { diff --git a/plugins/inputs/logfile/tail/tail.go b/plugins/inputs/logfile/tail/tail.go index c52418e854..826f5c6c4b 100644 --- a/plugins/inputs/logfile/tail/tail.go +++ b/plugins/inputs/logfile/tail/tail.go @@ -165,10 +165,9 @@ func (tail *Tail) Stop() error { } // StopAtEOF stops tailing as soon as the end of the file is reached. -// Blocks until tailer is dead and returns reason for death. -func (tail *Tail) StopAtEOF() error { +// Does not wait until tailer is dead. +func (tail *Tail) StopAtEOF() { tail.Kill(errStopAtEOF) - return tail.Wait() } var errStopAtEOF = errors.New("tail: stop at eof") diff --git a/plugins/inputs/logfile/tail/tail_test.go b/plugins/inputs/logfile/tail/tail_test.go index cd72b22f3d..691d8b4459 100644 --- a/plugins/inputs/logfile/tail/tail_test.go +++ b/plugins/inputs/logfile/tail/tail_test.go @@ -83,19 +83,20 @@ func TestStopAtEOF(t *testing.T) { readThreelines(t, tail) - // Since StopAtEOF() will block until the EOF is reached, run it in a goroutine. + // Since tail.Wait() will block until the EOF is reached, run it in a goroutine. done := make(chan bool) go func() { tail.StopAtEOF() + tail.Wait() close(done) }() // Verify the goroutine is blocked indefinitely. select { case <-done: - t.Fatalf("StopAtEOF() completed unexpectedly") + t.Fatalf("tail.Wait() completed unexpectedly") case <-time.After(time.Second * 1): - t.Log("timeout waiting for StopAtEOF() (as expected)") + t.Log("timeout waiting for tail.Wait() (as expected)") } assert.Equal(t, errStopAtEOF, tail.Err()) @@ -105,12 +106,12 @@ func TestStopAtEOF(t *testing.T) { <-tail.Lines } - // Verify StopAtEOF() has completed. + // Verify tail.Wait() has completed. select { case <-done: - t.Log("StopAtEOF() completed (as expected)") + t.Log("tail.Wait() completed (as expected)") case <-time.After(time.Second * 1): - t.Fatalf("StopAtEOF() has not completed") + t.Fatalf("tail.Wait() has not completed") } // Then remove the tmpfile From c6f3e33fc965c1e589cc32552be64a334fcbaed8 Mon Sep 17 00:00:00 2001 From: Mitali Salvi <44349099+mitali-salvi@users.noreply.github.com> Date: Fri, 20 Oct 2023 12:43:39 -0400 Subject: [PATCH 21/24] Removing service dimension for pod-net metrics (#916) --- .../tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml | 2 +- .../tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml | 2 +- .../tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml | 2 +- translator/translate/otel/exporter/awsemf/kubernetes.go | 2 +- translator/translate/otel/exporter/awsemf/translator_test.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml index 2da8fa2114..a1746ec979 100644 --- a/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/emf_and_kubernetes_config.yaml @@ -70,7 +70,7 @@ exporters: - pod_network_tx_bytes - pod_cpu_utilization_over_pod_limit - pod_memory_utilization_over_pod_limit - - dimensions: [ [ ClusterName, FullPodName, Namespace, PodName ], [ ClusterName, Namespace, PodName ], [ ClusterName, Namespace, Service ], [ ClusterName ] ] + - dimensions: [ [ ClusterName, FullPodName, Namespace, PodName ], [ ClusterName, Namespace, PodName ], [ ClusterName, Namespace ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - pod_interface_network_rx_dropped diff --git a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml index 0c98ba23de..64ed8e36cb 100644 --- a/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml +++ b/translator/tocwconfig/sampleConfig/kubernetes_on_prem_config.yaml @@ -41,7 +41,7 @@ exporters: - pod_network_tx_bytes - pod_cpu_utilization_over_pod_limit - pod_memory_utilization_over_pod_limit - - dimensions: [ [ ClusterName, FullPodName, Namespace, PodName ], [ ClusterName, Namespace, PodName ], [ ClusterName, Namespace, Service ], [ ClusterName ] ] + - dimensions: [ [ ClusterName, FullPodName, Namespace, PodName ], [ ClusterName, Namespace, PodName ], [ ClusterName, Namespace ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - pod_interface_network_rx_dropped diff --git a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml index b9d17ebb84..3eb5ce2d7e 100644 --- a/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/logs_and_kubernetes_config.yaml @@ -70,7 +70,7 @@ exporters: - pod_network_tx_bytes - pod_cpu_utilization_over_pod_limit - pod_memory_utilization_over_pod_limit - - dimensions: [ [ ClusterName, FullPodName, Namespace, PodName ], [ ClusterName, Namespace, PodName ], [ ClusterName, Namespace, Service ], [ ClusterName ] ] + - dimensions: [ [ ClusterName, FullPodName, Namespace, PodName ], [ ClusterName, Namespace, PodName ], [ ClusterName, Namespace ], [ ClusterName ] ] label_matchers: [ ] metric_name_selectors: - pod_interface_network_rx_dropped diff --git a/translator/translate/otel/exporter/awsemf/kubernetes.go b/translator/translate/otel/exporter/awsemf/kubernetes.go index 76d2f39ea5..564b0c2c84 100644 --- a/translator/translate/otel/exporter/awsemf/kubernetes.go +++ b/translator/translate/otel/exporter/awsemf/kubernetes.go @@ -111,7 +111,7 @@ func getPodMetricDeclarations(conf *confmap.Conf) []*awsemfexporter.MetricDeclar Dimensions: [][]string{ {"FullPodName", "PodName", "Namespace", "ClusterName"}, {"PodName", "Namespace", "ClusterName"}, - {"Service", "Namespace", "ClusterName"}, + {"Namespace", "ClusterName"}, {"ClusterName"}, }, MetricNameSelectors: []string{"pod_interface_network_rx_dropped", "pod_interface_network_tx_dropped"}, diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index 8af6695ee7..fb76195e0c 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -282,7 +282,7 @@ func TestTranslator(t *testing.T) { Dimensions: [][]string{ {"FullPodName", "PodName", "Namespace", "ClusterName"}, {"PodName", "Namespace", "ClusterName"}, - {"Service", "Namespace", "ClusterName"}, + {"Namespace", "ClusterName"}, {"ClusterName"}, }, MetricNameSelectors: []string{"pod_interface_network_rx_dropped", "pod_interface_network_tx_dropped"}, From 8a17844600e57650a785620363439a26ccaaf1b1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Oct 2023 14:23:56 -0400 Subject: [PATCH 22/24] Update OTel fork components to 8bdf732d320ad6718d88a6a42673081eb051d8a3 (#917) Co-authored-by: Github Action --- go.mod | 28 ++++++++++++++-------------- go.sum | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index db9405c94e..8f6ca28e70 100644 --- a/go.mod +++ b/go.mod @@ -6,34 +6,34 @@ replace github.com/influxdata/telegraf => github.com/aws/telegraf v0.10.2-0.2022 // Replace with https://github.com/amazon-contributing/opentelemetry-collector-contrib, there are no requirements for all receivers/processors/exporters // to be all replaced since there are some changes that will always be from upstream -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/k8s => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231020163023-8bdf732d320a // Replace with contrib to revert upstream change https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/20519 -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231020163023-8bdf732d320a replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/cwlogs => github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter => github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231020163023-8bdf732d320a -replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231020163023-8bdf732d320a -replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231020163023-8bdf732d320a // Temporary fix, pending PR https://github.com/shirou/gopsutil/pull/957 replace github.com/shirou/gopsutil/v3 => github.com/aws/telegraf/patches/gopsutil/v3 v3.0.0-20230915153624-7629361f8380 // indirect @@ -423,4 +423,4 @@ require ( sigs.k8s.io/yaml v1.3.0 // indirect ) -replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231012205229-d1a2dc44f2f3 +replace github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza => github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20231020163023-8bdf732d320a diff --git a/go.sum b/go.sum index caaa86bf79..d9b16e920a 100644 --- a/go.sum +++ b/go.sum @@ -140,34 +140,34 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1483 h1:J8HaD+Zpfi1gcel3HCKpoHHEsrcuRrZlSnx7R9SCf5I= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231012205229-d1a2dc44f2f3 h1:2tLCZtdFZ4B7yfqhtPEWOtVoCpFE0s3zB2IHqkVAVqM= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231012205229-d1a2dc44f2f3 h1:J2tk9Eo7FBT+GbQIOd7tq0XQyxtA+O75gHKoGYH5uuE= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231012205229-d1a2dc44f2f3 h1:S1LKmk4Eczr+Ur6ReKMOkvfaYLBcJaTbRbtVmE9s9rY= -github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231012205229-d1a2dc44f2f3 h1:axMuDAP4C3gfDb0Ox5NGbIyoTKPOZ4Pt268hfH4jNKk= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231012205229-d1a2dc44f2f3 h1:uCQjg1aL9DjKu8BB+9VhBdEQ6PDxz3PACctDYQU0i0I= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231012205229-d1a2dc44f2f3 h1:uswozvbfuT2WbsLDd9Pk9hRrm9fsMHPTLzzbbhFEOi8= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231012205229-d1a2dc44f2f3 h1:brhehmtgpokhtJUaX/Roo/Fg+wp2JootZzj9PyJXc7I= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231012205229-d1a2dc44f2f3 h1:xlCHzoTSd8DOC0aAs1cWsGXlAtUaH0Hzh9yJYzdljgQ= -github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231012205229-d1a2dc44f2f3 h1:Igs/mTssnX3WYVpGcj72b9n1GqQdcmWBZmCgAQam8Cg= -github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231020163023-8bdf732d320a h1:E12KX+/EG0lgJLqs6j5egsqr+H3+CB8a/UaMnTVocbE= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.0.0-20231020163023-8bdf732d320a/go.mod h1:WgmC0gq7urueR/VbZ0EHZhe3MXV6oWbaMmEWhHvagfg= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231020163023-8bdf732d320a h1:Qf01P4PgCCtViETGGhBBtnyMGPpUssyZRV0m4zWyIzA= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsemfexporter v0.0.0-20231020163023-8bdf732d320a/go.mod h1:b8pL6t9Xqk/zv0nLZsMiniuugDWiWQZRu9kh9t5SBLk= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231020163023-8bdf732d320a h1:ZrHh+eRCpcRKyQ7eyIf+OMDJ6zNj4cUCUH4HSLo7lVg= +github.com/amazon-contributing/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.0.0-20231020163023-8bdf732d320a/go.mod h1:K9h+mkX+BsA1UTuuheGJjo44KAahxaNu9jJ8/xVF6jo= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231020163023-8bdf732d320a h1:fzduYrFPdB/U4S2pr0oyo1dyLBwxtLcpnTPLw12ULjA= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/awsutil v0.0.0-20231020163023-8bdf732d320a/go.mod h1:9iAsO2SC8NIsa8/xCmC2Pj4MZPmYdvm+1/n89M74JS4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231020163023-8bdf732d320a h1:JukQdbg+7mvU0kQ92j/KocsAYZ4x3ei85kEbEdTslPI= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/containerinsight v0.0.0-20231020163023-8bdf732d320a/go.mod h1:ZwAqtlNaHJX0IUU5O40j96TDbsPA0K7o+m49AZgei7g= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231020163023-8bdf732d320a h1:g9MMmqMQXpy3HjJ64PoKtpaTSu61iv3SJNNAXQUmcdg= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/cwlogs v0.0.0-20231020163023-8bdf732d320a/go.mod h1:U0J/v82xC95JvG5QhXlrHH9OpgV8scQSGS6N7XW2y/4= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231020163023-8bdf732d320a h1:U/fex+AoERTJeFo1IjDgsjjyKshIagmxu9c67GJp0j0= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/k8s v0.0.0-20231020163023-8bdf732d320a/go.mod h1:58ZN2DUrqxJLqoXu+GZfL0RwMYiRZAAI+COKp0OmA0k= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231020163023-8bdf732d320a h1:RTebkRHNQMiZyvIR5U55deCMgEpH+Nerlhl6eeJ5dgI= +github.com/amazon-contributing/opentelemetry-collector-contrib/internal/aws/xray v0.0.0-20231020163023-8bdf732d320a/go.mod h1:8edNN/XfefbHuGLiDhFdBN1QfJfgH7wmq5ms2Gme1EA= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231020163023-8bdf732d320a h1:ad2h+5tqUTXFSdnJHGxO9UxUDpmr1r2n8Bcw/BYzfrw= +github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20231020163023-8bdf732d320a/go.mod h1:F5l/VuHtB8418NLJEsHeYz/pni6sWtOMR/SM6mgarhQ= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713 h1:2daWNVtWNvRDoCTN5GG5N+LEM9OuY3RjJ0cboU3+xmM= github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/stanza v0.0.0-20230928170322-0df38c533713/go.mod h1:lJLumMdUeKqurOskauSjhH4J2hz8r0iNyQWDl3i5NSM= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231012205229-d1a2dc44f2f3 h1:2veJzdr7Ulzh0PVhIzk8qFlYZ2UOKlh+Th8awDT8fsw= -github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231012205229-d1a2dc44f2f3 h1:4Xsr3S/XBYXDOo3AihM9YUww6YxzZUJYRUpiP8LeWe0= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231012205229-d1a2dc44f2f3 h1:cvtX0kTQO/wY+33wWVv3PJS3tLCQ39x+lCfhZgbMiOY= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231012205229-d1a2dc44f2f3 h1:aNJxUF5OA5SeBosU9OxJr+L6JY8JEx2APAz+xPbb1GM= -github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231012205229-d1a2dc44f2f3/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231020163023-8bdf732d320a h1:9xYr+Ru7GzJq9cobLqoV0DOp2XbZ+f6nuxo/wTOi4gQ= +github.com/amazon-contributing/opentelemetry-collector-contrib/pkg/translator/prometheus v0.0.0-20231020163023-8bdf732d320a/go.mod h1:9qsT0AsMflbQKz0ojK3aRU/PbyGQCDPKut3XMfAkW8k= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231020163023-8bdf732d320a h1:ILOMnnQ8bUXg0Q9591LknoPqZJWbefDe3ktQpKv41E8= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver v0.0.0-20231020163023-8bdf732d320a/go.mod h1:t/v7BcGrHUQ0/Lb/4egp0Xe8PrTceEkZVArTuRjQGBo= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231020163023-8bdf732d320a h1:Yd5943Zx/1PDXzA9QUQ+duoxEfAqLa7U9fJGBuWK+Ic= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20231020163023-8bdf732d320a/go.mod h1:akbVXOWuMWKSgqA1QKoXkm3hFt0qIvDeUr7m3ODAiS8= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231020163023-8bdf732d320a h1:nI8fWWNGJ+/zsSjdrDC6iFHlFsFp1dCbj/NRZ6s9vUc= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20231020163023-8bdf732d320a/go.mod h1:fw4J+Pn19ZgfR5ZVxWVtlvKq7+zEfXXlZV/7G9IWkko= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE= From 76214cc09c836475f626663bb2dace769edda4a0 Mon Sep 17 00:00:00 2001 From: Lisa Guo Date: Mon, 23 Oct 2023 10:09:09 -0400 Subject: [PATCH 23/24] fix unit test --- .../tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml index 648a4abaeb..cf5596a2eb 100644 --- a/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml +++ b/translator/tocwconfig/sampleConfig/apm_and_kubernetes_config.yaml @@ -127,7 +127,7 @@ exporters: dimension_rollup_option: NoDimensionRollup disable_metric_extraction: true eks_fargate_container_insights_enabled: false - endpoint: "" + endpoint: "https://fake_endpoint" enhanced_container_insights: false imds_retries: 1 local_mode: false From aebc128b0f575a065a90e34bade3530e2a5793e0 Mon Sep 17 00:00:00 2001 From: Dinakar Chappa Date: Mon, 23 Oct 2023 16:05:25 -0400 Subject: [PATCH 24/24] EMF Exporter does not forward metrics if IRSA is enabled (#915) --- plugins/inputs/logfile/logfile_test.go | 2 +- translator/translate/otel/exporter/awsemf/translator.go | 1 + translator/translate/otel/exporter/awsemf/translator_test.go | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/logfile/logfile_test.go b/plugins/inputs/logfile/logfile_test.go index f5a38ab48c..2c1a2e98dd 100644 --- a/plugins/inputs/logfile/logfile_test.go +++ b/plugins/inputs/logfile/logfile_test.go @@ -453,7 +453,7 @@ func createWriteRead(t *testing.T, prefix string, logFile *LogFile, done chan bo select { case <-done2: t.Log("Child completed before timeout (as expected)") - case <-time.After(time.Second * 10): + case <-time.After(time.Second * 20): require.Fail(t, "timeout waiting for child") } t.Log("Verify 1st temp file was auto deleted.") diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index c52a79f909..b87ca44d60 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -105,6 +105,7 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { if credentialsFileKey, ok := agent.Global_Config.Credentials[agent.CredentialsFile_Key]; ok { cfg.AWSSessionSettings.SharedCredentialsFile = []string{fmt.Sprintf("%v", credentialsFileKey)} } + cfg.AWSSessionSettings.RoleARN = agent.Global_Config.Role_arn cfg.AWSSessionSettings.IMDSRetries = retryer.GetDefaultRetryNumber() if isEcs(c) { diff --git a/translator/translate/otel/exporter/awsemf/translator_test.go b/translator/translate/otel/exporter/awsemf/translator_test.go index b2833c342d..a0d264647f 100644 --- a/translator/translate/otel/exporter/awsemf/translator_test.go +++ b/translator/translate/otel/exporter/awsemf/translator_test.go @@ -16,6 +16,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" legacytranslator "github.com/aws/amazon-cloudwatch-agent/translator" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) @@ -24,6 +25,8 @@ var nilMetricDescriptorsSlice []awsemfexporter.MetricDescriptor func TestTranslator(t *testing.T) { tt := NewTranslator() + agent.Global_Config.Region = "us-east-1" + agent.Global_Config.Role_arn = "global_arn" require.EqualValues(t, "awsemf", tt.ID().String()) testCases := map[string]struct { env map[string]string @@ -680,6 +683,8 @@ func TestTranslator(t *testing.T) { require.Equal(t, testCase.want["resource_to_telemetry_conversion"], gotCfg.ResourceToTelemetrySettings) require.ElementsMatch(t, testCase.want["metric_declarations"], gotCfg.MetricDeclarations) require.ElementsMatch(t, testCase.want["metric_descriptors"], gotCfg.MetricDescriptors) + require.Equal(t, "global_arn", gotCfg.RoleARN) + require.Equal(t, "us-east-1", gotCfg.Region) } }) }