Merge branch 'master' into rarguelloF/grpc-precursors

.circleci/config.yml

@@ -73,11 +73,17 @@ test_containers:
   - &mysql_port 3306
   - &opensearch_host opensearch
   - &container_opensearch
-    image: opensearchproject/opensearch:2.8.0
+    image: opensearchproject/opensearch:1.3.6  # Version 2.8.0 causes flaky error (https://github.com/opensearch-project/docker-images/issues/31).
     name: *opensearch_host
     environment:
       - discovery.type=single-node
       - DISABLE_SECURITY_PLUGIN=true
+      # Make sure it works on nearly full disk.
+      - cluster.routing.allocation.disk.threshold_enabled=true
+      - cluster.routing.allocation.disk.watermark.low=3gb
+      - cluster.routing.allocation.disk.watermark.high=2gb
+      - cluster.routing.allocation.disk.watermark.flood_stage=1gb
+      - cluster.routing.allocation.disk.threshold_enabled=false
   - &opensearch_port 9200
   - &container_elasticsearch
     image: elasticsearch:8.1.3

Gemfile

@@ -97,12 +97,11 @@ gem 'dogstatsd-ruby', '>= 3.3.0', '!= 5.0.0', '!= 5.0.1', '!= 5.1.0'
 gem 'opentracing', '>= 0.4.1'
 
 # Profiler optional dependencies
-# NOTE: We're excluding versions 3.7.0 and 3.7.1 for the reasons documented in #1424 and the big comment in
-#       lib/datadog/profiling.rb: it breaks for some older rubies in CI without BUNDLE_FORCE_RUBY_PLATFORM=true.
+# NOTE: We're excluding versions 3.7.0 and 3.7.1 for the reasons documented in #1424.
 #       Since most of our customers won't have BUNDLE_FORCE_RUBY_PLATFORM=true, it's not something we want to add
 #       to our CI, so we just shortcut and exclude specific versions that were affecting our CI.
 if RUBY_PLATFORM != 'java'
-  if RUBY_VERSION >= '2.5.0' # Bundler 1.x fails to find that versions >= 3.8.0 are not compatible because of binary gems
+  if RUBY_VERSION >= '2.7.0' # Bundler 1.x fails to find that versions >= 3.8.0 are not compatible because of binary gems
     gem 'google-protobuf', ['~> 3.0', '!= 3.7.0', '!= 3.7.1']
   elsif RUBY_VERSION >= '2.3.0'
     gem 'google-protobuf', ['~> 3.0', '!= 3.7.0', '!= 3.7.1', '< 3.19.2']

docker-compose.yml

@@ -464,10 +464,16 @@ services:
     ports:
       - "127.0.0.1:${TEST_MYSQL_PORT}:3306"
   opensearch:
-    image: opensearchproject/opensearch:2.8.0
+    image: opensearchproject/opensearch:1.3.6 # Version 2.8.0 causes flaky error (https://github.com/opensearch-project/docker-images/issues/31).
     environment:
       - discovery.type=single-node
       - DISABLE_SECURITY_PLUGIN=true
+      # Make sure it works on nearly full disk.
+      - cluster.routing.allocation.disk.threshold_enabled=true
+      - cluster.routing.allocation.disk.watermark.low=3gb
+      - cluster.routing.allocation.disk.watermark.high=2gb
+      - cluster.routing.allocation.disk.watermark.flood_stage=1gb
+      - cluster.routing.allocation.disk.threshold_enabled=false
     ports:
       - 9201:9200
   postgres:

docs/GettingStarted.md

@@ -122,7 +122,8 @@ To contribute, check out the [contribution guidelines][contribution docs] and [d
 
 | Type  | Documentation              | Version | Support type                         | Gem version support |
 | ----- | -------------------------- | -----   | ------------------------------------ | ------------------- |
-| MRI   | https://www.ruby-lang.org/ | 3.1     | Full                                 | Latest              |
+| MRI   | https://www.ruby-lang.org/ | 3.2     | Full                                 | Latest              |
+|       |                            | 3.1     | Full                                 | Latest              |
 |       |                            | 3.0     | Full                                 | Latest              |
 |       |                            | 2.7     | Full                                 | Latest              |
 |       |                            | 2.6     | Full                                 | Latest              |
@@ -1714,7 +1715,7 @@ end
 |  2.4          |                |  4.2.8 - 5.2   |
 |  2.5          |                |  4.2.8 - 6.1   |
 |  2.6 - 2.7    |  9.2           |  5.0 - 6.1     |
-|  3.0          |                |  6.1           |
+|  3.0 - 3.2    |                |  6.1           |
 
 ### Rake
 
@@ -2239,6 +2240,7 @@ For example, if `tracing.sampling.default_rate` is configured by [Remote Configu
 | `tracing.sampler`                                       |                                | `nil`                                                             | Advanced usage only. Sets a custom `Datadog::Tracing::Sampling::Sampler` instance. If provided, the tracer will use this sampler to determine sampling behavior. See [Application-side sampling](#application-side-sampling) for details. |
 | `tracing.sampling.default_rate`                         | `DD_TRACE_SAMPLE_RATE`         | `nil`                                                             | Sets the trace sampling rate between `0.0` (0%) and `1.0` (100%). See [Application-side sampling](#application-side-sampling) for details.                                                                                                  |
 | `tracing.sampling.rate_limit`                           | `DD_TRACE_RATE_LIMIT`          | `100` (per second)                                                | Sets a maximum number of traces per second to sample. Set a rate limit to avoid the ingestion volume overages in the case of traffic spikes.                                                                    |
+| `tracing.sampling.rules`                                | `DD_TRACE_SAMPLING_RULES`      | `nil`                                                             | Sets trace-level sampling rules, matching against the local root span. The format is a `String` with JSON, containing an Array of Objects. Each Object must have a float attribute `sample_rate` (between 0.0 and 1.0, inclusive), and optionally `name` and `service` string attributes. `name` and `service` control to which traces this sampling rule applies; if both are absent, then this rule applies to all traces. Rules are evaluted in order of declartion in the array; only the first to match is applied. If none apply, then `tracing.sampling.default_rate` is applied. |
 | `tracing.sampling.span_rules`                           | `DD_SPAN_SAMPLING_RULES`,`ENV_SPAN_SAMPLING_RULES_FILE` | `nil`                                    | Sets [Single Span Sampling](#single-span-sampling) rules. These rules allow you to keep spans even when their respective traces are dropped.                                                                                              |
 | `tracing.trace_id_128_bit_generation_enabled` | `DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED` | `false` | `true` to generate 128 bits trace ID and `false` to generate 64 bits trace ID  |
 | `tracing.report_hostname`                               | `DD_TRACE_REPORT_HOSTNAME`     | `false`                                                           | Adds hostname tag to traces.                                                                                                                                                                                                              |

integration/apps/rack/Gemfile

@@ -32,7 +32,8 @@ google_protobuf_versions = [
   '!= 3.7.0.rc.3',
   '!= 3.7.0',
   '!= 3.7.1',
-  '!= 3.8.0.rc.1'
+  '!= 3.8.0.rc.1',
+  '!= 3.24.0.rc.1',
 ]
 if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.4')
   gem 'google-protobuf', *google_protobuf_versions

integration/apps/rails-five/Gemfile

@@ -13,6 +13,7 @@ google_protobuf_versions = [
   '!= 3.8.0.rc.1',
   '!= 3.20.0.rc.1',
   '!= 3.20.0.rc.2',
+  '!= 3.24.0.rc.1',
 ]
 
 rails_version = ['~> 5.2', '>= 5.2.6']

integration/apps/rails-six/Gemfile

@@ -8,7 +8,8 @@ google_protobuf_versions = [
   '!= 3.7.0.rc.3',
   '!= 3.7.0',
   '!= 3.7.1',
-  '!= 3.8.0.rc.1'
+  '!= 3.8.0.rc.1',
+  '!= 3.24.0.rc.1',
 ]
 
 rails_version = ['~> 6.1']

integration/apps/sinatra2-classic/Gemfile

@@ -17,7 +17,8 @@ google_protobuf_versions = [
   '!= 3.7.0.rc.3',
   '!= 3.7.0',
   '!= 3.7.1',
-  '!= 3.8.0.rc.1'
+  '!= 3.8.0.rc.1',
+  '!= 3.24.0.rc.1',
 ]
 if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.4')
   gem 'google-protobuf', *google_protobuf_versions

integration/apps/sinatra2-modular/Gemfile

@@ -18,7 +18,8 @@ google_protobuf_versions = [
   '!= 3.7.0.rc.3',
   '!= 3.7.0',
   '!= 3.7.1',
-  '!= 3.8.0.rc.1'
+  '!= 3.8.0.rc.1',
+  '!= 3.24.0.rc.1',
 ]
 if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.4')
   gem 'google-protobuf', *google_protobuf_versions

lib/datadog/core/backport.rb

@@ -4,14 +4,12 @@ module Datadog
   module Core
     # This module is used to provide features from Ruby 2.5+ to older Rubies
     module BackportFrom25
-      module_function
-
       if ::String.method_defined?(:delete_prefix)
-        def string_delete_prefix(string, prefix)
+        def self.string_delete_prefix(string, prefix)
           string.delete_prefix(prefix)
         end
       else
-        def string_delete_prefix(string, prefix)
+        def self.string_delete_prefix(string, prefix)
           prefix = prefix.to_s
           if string.start_with?(prefix)
             string[prefix.length..-1] || raise('rbs-guard: String#[] is non-nil as `prefix` is guaranteed present')
@@ -21,5 +19,33 @@ def string_delete_prefix(string, prefix)
         end
       end
     end
+
+    # This module is used to provide features from Ruby 2.4+ to older Rubies
+    module BackportFrom24
+      if RUBY_VERSION < '2.4'
+        def self.dup(value)
+          case value
+          when NilClass, TrueClass, FalseClass, Numeric
+            value
+          else
+            value.dup
+          end
+        end
+      else
+        def self.dup(value)
+          value.dup
+        end
+      end
+
+      if ::Hash.method_defined?(:compact!)
+        def self.hash_compact!(hash)
+          hash.compact!
+        end
+      else
+        def self.hash_compact!(hash)
+          hash.reject! { |_key, value| value.nil? }
+        end
+      end
+    end
   end
 end

lib/datadog/core/configuration/settings.rb

@@ -540,9 +540,11 @@ def initialize(*_)
           end
 
           # Tune remote configuration polling interval.
+          # This is a private setting. Do not use outside of Datadog. It might change at any point in time.
           #
-          # @default `DD_REMOTE_CONFIGURATION_POLL_INTERVAL_SECONDS` environment variable, otherwise `5.0` seconds.
+          # @default `DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS` environment variable, otherwise `5.0` seconds.
           # @return [Float]
+          # @!visibility private
           option :poll_interval_seconds do |o|
             o.default { env_to_float(Core::Remote::Ext::ENV_POLL_INTERVAL_SECONDS, 5.0) }
           end

lib/datadog/core/remote/ext.rb

@@ -5,7 +5,7 @@ module Core
     module Remote
       module Ext
         ENV_ENABLED = 'DD_REMOTE_CONFIGURATION_ENABLED'
-        ENV_POLL_INTERVAL_SECONDS = 'DD_REMOTE_CONFIGURATION_POLL_INTERVAL_SECONDS'
+        ENV_POLL_INTERVAL_SECONDS = 'DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS'
       end
     end
   end

lib/datadog/core/telemetry/http/transport.rb

@@ -38,6 +38,7 @@ def request(request_type:, payload:)
 
           def headers(request_type:, api_version: Http::Ext::API_VERSION)
             {
+              Datadog::Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST => '1',
               Http::Ext::HEADER_CONTENT_TYPE => Http::Ext::CONTENT_TYPE_APPLICATION_JSON,
               Http::Ext::HEADER_DD_TELEMETRY_API_VERSION => api_version,
               Http::Ext::HEADER_DD_TELEMETRY_REQUEST_TYPE => request_type,

lib/datadog/core/utils/safe_dup.rb

@@ -1,40 +1,47 @@
+require_relative '../backport'
+
 module Datadog
   module Core
     module Utils
       # Helper methods for safer dup
       module SafeDup
-        if RUBY_VERSION < '2.2' # nil.dup only fails in Ruby 2.1
-          # Ensures #initialize can call nil.dup safely
-          module RefineNil
-            refine NilClass do
-              def dup
-                self
-              end
-            end
-          end
-
-          using RefineNil
-        end
-
         # String#+@ was introduced in Ruby 2.3
         if String.method_defined?(:+@) && String.method_defined?(:-@)
           def self.frozen_or_dup(v)
-            # If the string is not frozen, the +(-v) will:
-            # - first create a frozen deduplicated copy with -v
-            # - then it will dup it more efficiently with +v
-            v.frozen? ? v : +(-v)
+            # For the case of a String we use the methods +@ and -@.
+            # Those methods are only for String objects
+            # they are faster and chepaer on the memory side.
+            # Check the benchmark on
+            # https://github.com/DataDog/dd-trace-rb/pull/2704
+            if v.is_a?(String)
+              # If the string is not frozen, the +(-v) will:
+              # - first create a frozen deduplicated copy with -v
+              # - then it will dup it more efficiently with +v
+              v.frozen? ? v : +(-v)
+            else
+              v.frozen? ? v : Core::BackportFrom24.dup(v)
+            end
           end
 
           def self.frozen_dup(v)
-            -v if v
+            # For the case of a String we use the methods -@
+            # That method are only for String objects
+            # they are faster and chepaer on the memory side.
+            # Check the benchmark on
+            # https://github.com/DataDog/dd-trace-rb/pull/2704
+            if v.is_a?(String)
+              -v if v
+            else
+              v.frozen? ? v : Core::BackportFrom24.dup(v).freeze
+            end
           end
         else
           def self.frozen_or_dup(v)
-            v.frozen? ? v : v.dup
+            v.frozen? ? v : Core::BackportFrom24.dup(v)
           end
 
           def self.frozen_dup(v)
-            v.frozen? ? v : v.dup.freeze
+            v.frozen? ? v : Core::BackportFrom24.dup(v).freeze
           end
         end
       end

lib/datadog/tracing/component.rb

@@ -62,6 +62,23 @@ def build_sampler(settings)
           else
             ensure_priority_sampling(sampler, settings)
           end
+        elsif (rules = settings.tracing.sampling.rules)
+          post_sampler = Tracing::Sampling::RuleSampler.parse(
+            rules,
+            settings.tracing.sampling.rate_limit,
+            settings.tracing.sampling.default_rate
+          )
+
+          post_sampler ||= # Fallback RuleSampler in case `rules` parsing fails
+            Tracing::Sampling::RuleSampler.new(
+              rate_limit: settings.tracing.sampling.rate_limit,
+              default_sample_rate: settings.tracing.sampling.default_rate
+            )
+
+          Tracing::Sampling::PrioritySampler.new(
+            base_sampler: Tracing::Sampling::AllSampler.new,
+            post_sampler: post_sampler
+          )
         elsif settings.tracing.priority_sampling == false
           Tracing::Sampling::RuleSampler.new(
             rate_limit: settings.tracing.sampling.rate_limit,

lib/datadog/tracing/configuration/ext.rb

@@ -72,6 +72,7 @@ module NET
         module Sampling
           ENV_SAMPLE_RATE = 'DD_TRACE_SAMPLE_RATE'
           ENV_RATE_LIMIT = 'DD_TRACE_RATE_LIMIT'
+          ENV_RULES = 'DD_TRACE_SAMPLING_RULES'
 
           # @public_api
           module Span

lib/datadog/tracing/configuration/settings.rb

@@ -277,6 +277,22 @@ def self.extended(base)
                   o.default { env_to_float(Tracing::Configuration::Ext::Sampling::ENV_RATE_LIMIT, 100) }
                 end
 
+                # Trace sampling rules.
+                # These rules control whether a trace is kept or dropped by the tracer.
+                #
+                # The `rules` format is a String with a JSON array of objects:
+                # Each object must have a `sample_rate`, and the `name` and `service` fields
+                # are optional. The `sample_rate` value must be between 0.0 and 1.0 (inclusive).
+                # `name` and `service` are Strings that allow the `sample_rate` to be applied only
+                # to traces matching the `name` and `service`.
+                #
+                # @default `DD_TRACE_SAMPLING_RULES` environment variable. Otherwise `nil`.
+                # @return [String,nil]
+                # @public_api
+                option :rules do |o|
+                  o.default { ENV.fetch(Configuration::Ext::Sampling::ENV_RULES, nil) }
+                end
+
                 # Single span sampling rules.
                 # These rules allow a span to be kept when its encompassing trace is dropped.
                 #

lib/datadog/tracing/contrib/http/circuit_breaker.rb

@@ -10,7 +10,7 @@ module HTTP
         # For avoiding recursive traces.
         module CircuitBreaker
           def should_skip_tracing?(request)
-            return true if datadog_http_request?(request) || datadog_test_agent_http_request?(request)
+            return true if internal_request?(request)
 
             # we don't want a "shotgun" effect with two nested traces for one
             # logical get, and request is likely to call itself recursively
@@ -23,23 +23,9 @@ def should_skip_tracing?(request)
           # We don't want to trace our own call to the API (they use net/http)
           # TODO: We don't want this kind of soft-check on HTTP requests.
           #       Remove this when transport implements its own "skip tracing" mechanism.
-          def datadog_http_request?(request)
-            if request[Datadog::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION]
-              true
-            else
-              false
-            end
-          end
-
-          # Check if there is header present for not tracing this request. Necessary to prevent http requests
-          # used for checking if the APM Test Agent is running from being traced.
-          # TODO: Remove this when transport implements its own "skip tracing" mechanism.
-          def datadog_test_agent_http_request?(request)
-            if request['X-Datadog-Untraced-Request']
-              true
-            else
-              false
-            end
+          def internal_request?(request)
+            !!(request[Datadog::Transport::Ext::HTTP::HEADER_META_TRACER_VERSION] ||
+              request[Transport::Ext::HTTP::HEADER_DD_INTERNAL_UNTRACED_REQUEST])
           end
 
           def should_skip_distributed_tracing?(client_config)