From fa1462f9cb99465dfba16b9d23ede3e9b978e267 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Tue, 6 Aug 2024 17:16:26 -0700
Subject: [PATCH 01/41] Initial commit for OpenSearch instrumentation

---
 .github/workflows/ci_cron.yml                 |  11 +
 docker-compose.yml                            |  43 +++-
 .../agent/configuration/default_source.rb     |  24 +++
 .../agent/instrumentation/opensearch.rb       |  25 +++
 .../agent/instrumentation/opensearch/chain.rb |  21 ++
 .../opensearch/instrumentation.rb             |  66 ++++++
 .../instrumentation/opensearch/prepend.rb     |  13 ++
 .../instrumentation.thor                      |   2 +-
 .../templates/dependency_detection.tt         |   4 +-
 test/multiverse/suites/opensearch/Envfile     |  20 ++
 .../suites/opensearch/config/newrelic.yml     |  19 ++
 .../opensearch_instrumentation_test.rb        | 204 ++++++++++++++++++
 12 files changed, 438 insertions(+), 14 deletions(-)
 create mode 100644 lib/new_relic/agent/instrumentation/opensearch.rb
 create mode 100644 lib/new_relic/agent/instrumentation/opensearch/chain.rb
 create mode 100644 lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb
 create mode 100644 lib/new_relic/agent/instrumentation/opensearch/prepend.rb
 create mode 100644 test/multiverse/suites/opensearch/Envfile
 create mode 100644 test/multiverse/suites/opensearch/config/newrelic.yml
 create mode 100644 test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb

diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml
index 8a3416ac51..7e6ded35e7 100644
--- a/.github/workflows/ci_cron.yml
+++ b/.github/workflows/ci_cron.yml
@@ -149,6 +149,17 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 10
+      # opensearch:
+      #   image: opensearch:1.3.18 # latest version when instrumentation created
+      #   ports:
+      #     - "9350:9350"
+      #   environment:
+      #     - discovery.type=single-node
+      #     - plugins.security.disabled=true
+      #     - http.port=9350
+      #     - bootstrap.memory_lock=true
+      #     - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+      #   mem_limit: 1g
       memcached:
         image: memcached:latest
         ports:
diff --git a/docker-compose.yml b/docker-compose.yml
index 63f1ec1d75..3a16304195 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,14 +1,13 @@
-version: "3.9"
 services:
-  elasticsearch7:
-    image: elasticsearch:7.16.2
-    ports:
-      - "9200:9200"
-    environment:
-      - discovery.type=single-node
-      - bootstrap.memory_lock=true
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    mem_limit: 1g
+  # elasticsearch7:
+  #   image: elasticsearch:7.16.2
+  #   ports:
+  #     - "9200:9200"
+  #   environment:
+  #     - discovery.type=single-node
+  #     - bootstrap.memory_lock=true
+  #     - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+  #   mem_limit: 1g
   elasticsearch8:
     image: elasticsearch:8.13.0
     ports:
@@ -20,6 +19,25 @@ services:
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     mem_limit: 1g
+  # tests passed with
+  # docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" opensearchproject/opensearch:1.3.18
+  opensearch:
+    image: opensearchproject/opensearch:1.3.18
+    environment:
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+      - "DISABLE_INSTALL_DEMO_CONFIG=true"
+      - "DISABLE_SECURITY_PLUGIN=true"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - opensearch_data:/usr/share/opensearch/data
+    ports:
+      - 9200:9200
+      - 9600:9600
   mysql:
     image: mysql:5.7
     platform: linux/x86_64
@@ -89,8 +107,9 @@ services:
     volumes:
       - ".:/usr/src/app"
     depends_on:
-      - elasticsearch7
+      # - elasticsearch7
       - elasticsearch8
+      - opensearch
       - mysql
       - memcached
       - mongodb
@@ -106,3 +125,5 @@ volumes:
     driver: local
   postgres:
     driver: local
+  opensearch_data:
+    driver: local
diff --git a/lib/new_relic/agent/configuration/default_source.rb b/lib/new_relic/agent/configuration/default_source.rb
index c7056415b3..d1e6211cd8 100644
--- a/lib/new_relic/agent/configuration/default_source.rb
+++ b/lib/new_relic/agent/configuration/default_source.rb
@@ -1463,6 +1463,15 @@ def self.enforce_fallback(allowed_values: nil, fallback: nil)
           :allowed_from_server => false,
           :description => 'Controls auto-instrumentation of bunny at start-up. May be one of: `auto`, `prepend`, `chain`, `disabled`.'
         },
+        :'instrumentation.opensearch' => {
+          :default => 'auto',
+          :documentation_default => 'auto',
+          :public => true,
+          :type => String,
+          :dynamic_name => true,
+          :allowed_from_server => false,
+          :description => 'Controls auto-instrumentation of the opensearch-ruby library at start-up. May be one of `auto`, `prepend`, `chain`, `disabled`.'
+        },
         :'instrumentation.aws_sqs' => {
           :default => 'auto',
           :public => true,
@@ -1870,6 +1879,21 @@ def self.enforce_fallback(allowed_values: nil, fallback: nil)
           :allowed_from_server => true,
           :description => 'If `true`, the agent obfuscates Mongo queries in transaction traces.'
         },
+        # OpenSearch
+        :'opensearch.capture_queries' => {
+          :default => true,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => true,
+          :description => 'If `true`, the agent captures OpenSearch queries in transaction traces.'
+        },
+        :'opensearch.obfuscate_queries' => {
+          :default => true,
+          :public => true,
+          :type => Boolean,
+          :allowed_from_server => true,
+          :description => 'If `true`, the agent obfuscates OpenSearch queries in transaction traces.'
+        },
         # Process host
         :'process_host.display_name' => {
           :default => proc { NewRelic::Agent::Hostname.get },
diff --git a/lib/new_relic/agent/instrumentation/opensearch.rb b/lib/new_relic/agent/instrumentation/opensearch.rb
new file mode 100644
index 0000000000..42aa386df7
--- /dev/null
+++ b/lib/new_relic/agent/instrumentation/opensearch.rb
@@ -0,0 +1,25 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+require_relative 'opensearch/instrumentation'
+require_relative 'opensearch/chain'
+require_relative 'opensearch/prepend'
+
+DependencyDetection.defer do
+  named :opensearch
+
+  depends_on do
+    defined?(OpenSearch)
+  end
+
+  executes do
+    ::NewRelic::Agent.logger.info('Installing opensearch-ruby instrumentation')
+
+    if use_prepend?
+      prepend_instrument OpenSearch::Transport::Client, NewRelic::Agent::Instrumentation::OpenSearch::Prepend
+    else
+      chain_instrument NewRelic::Agent::Instrumentation::OpenSearch::Chain
+    end
+  end
+end
diff --git a/lib/new_relic/agent/instrumentation/opensearch/chain.rb b/lib/new_relic/agent/instrumentation/opensearch/chain.rb
new file mode 100644
index 0000000000..4380454247
--- /dev/null
+++ b/lib/new_relic/agent/instrumentation/opensearch/chain.rb
@@ -0,0 +1,21 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic::Agent::Instrumentation
+  module OpenSearch::Chain
+    def self.instrument!
+      ::OpenSearch::Transport::Client.class_eval do
+        include NewRelic::Agent::Instrumentation::OpenSearch
+
+        alias_method(:perform_request_without_tracing, :perform_request)
+
+        def perform_request(*args)
+          perform_request_with_tracing(*args) do
+            perform_request_without_tracing(*args)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb b/lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb
new file mode 100644
index 0000000000..20c4da2b30
--- /dev/null
+++ b/lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb
@@ -0,0 +1,66 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic::Agent::Instrumentation
+  module OpenSearch
+    PRODUCT_NAME = 'OpenSearch'
+    OPERATION = 'perform_request'
+    OPERATION_PATTERN = %r{/lib/opensearch/api/(?!.+#{OPERATION})}
+    INSTANCE_METHOD_PATTERN = /:in (?:`|')(?:.+#)?([^']+)'\z/
+    INSTRUMENTATION_NAME = NewRelic::Agent.base_name(name)
+
+    def perform_request_with_tracing(_method, _path, params = {}, body = nil, _headers = nil, _opts = {}, &_block)
+      return yield unless NewRelic::Agent::Tracer.tracing_enabled?
+
+      segment = NewRelic::Agent::Tracer.start_datastore_segment(
+        product: PRODUCT_NAME,
+        operation: nr_operation || OPERATION,
+        host: nr_hosts[:host],
+        port_path_or_id: nr_hosts[:port],
+        database_name: nr_cluster_name
+      )
+      begin
+        NewRelic::Agent::Tracer.capture_segment_error(segment) { yield }
+      ensure
+        if segment
+          segment.notice_nosql_statement(nr_reported_query(body || params))
+          segment.finish
+        end
+      end
+    end
+
+    private
+
+    # See Elasticsearch instrumentation for explanation on Ruby 3.4 changes to match instance method
+    def nr_operation
+      location = caller_locations.detect { |loc| loc.to_s.match?(OPERATION_PATTERN) }
+      return unless location && location.to_s =~ INSTANCE_METHOD_PATTERN
+
+      Regexp.last_match(1)
+    end
+
+    def nr_reported_query(query)
+      return unless NewRelic::Agent.config[:'opensearch.capture_queries']
+      return query unless NewRelic::Agent.config[:'opensearch.obfuscate_queries']
+
+      NewRelic::Agent::Datastores::NosqlObfuscator.obfuscate_statement(query)
+    end
+
+    def nr_cluster_name
+      return @nr_cluster_name if @nr_cluster_name
+      return if nr_hosts.empty?
+
+      NewRelic::Agent.disable_all_tracing do
+        @nr_cluster_name ||= perform_request('GET', '/').body['cluster_name']
+      end
+    rescue StandardError => e
+      NewRelic::Agent.logger.error('Failed to get cluster name for OpenSearch', e)
+      nil
+    end
+
+    def nr_hosts
+      @nr_hosts ||= (transport.hosts.first || NewRelic::EMPTY_HASH)
+    end
+  end
+end
diff --git a/lib/new_relic/agent/instrumentation/opensearch/prepend.rb b/lib/new_relic/agent/instrumentation/opensearch/prepend.rb
new file mode 100644
index 0000000000..993ac02bf9
--- /dev/null
+++ b/lib/new_relic/agent/instrumentation/opensearch/prepend.rb
@@ -0,0 +1,13 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+module NewRelic::Agent::Instrumentation
+  module OpenSearch::Prepend
+    include NewRelic::Agent::Instrumentation::OpenSearch
+
+    def perform_request(*args)
+      perform_request_with_tracing(*args) { super }
+    end
+  end
+end
diff --git a/lib/tasks/instrumentation_generator/instrumentation.thor b/lib/tasks/instrumentation_generator/instrumentation.thor
index 06305c1074..567f6797b3 100644
--- a/lib/tasks/instrumentation_generator/instrumentation.thor
+++ b/lib/tasks/instrumentation_generator/instrumentation.thor
@@ -103,7 +103,7 @@ class Instrumentation < Thor
     <<-CONFIG
         :'instrumentation.#{snake_name}' => {
           :default => 'auto',
-          :documentation_default => 'auto'
+          :documentation_default => 'auto',
           :public => true,
           :type => String,
           :dynamic_name => true,
diff --git a/lib/tasks/instrumentation_generator/templates/dependency_detection.tt b/lib/tasks/instrumentation_generator/templates/dependency_detection.tt
index cb8a3783a5..15849f5e14 100644
--- a/lib/tasks/instrumentation_generator/templates/dependency_detection.tt
+++ b/lib/tasks/instrumentation_generator/templates/dependency_detection.tt
@@ -12,7 +12,7 @@ DependencyDetection.defer do
   depends_on do
     # The class that needs to be defined to prepend/chain onto. This can be used
     # to determine whether the library is installed.
-    defined?(::<%= @class_name %>)
+    defined?(<%= @class_name %>)
     # Add any additional requirements to verify whether this instrumentation
     # should be installed
   end
@@ -21,7 +21,7 @@ DependencyDetection.defer do
     ::NewRelic::Agent.logger.info('Installing <%= @name.downcase %> instrumentation')
 
     if use_prepend?
-      prepend_instrument ::<%= @class_name %>, NewRelic::Agent::Instrumentation::<%= @class_name %>::Prepend
+      prepend_instrument <%= @class_name %>, NewRelic::Agent::Instrumentation::<%= @class_name %>::Prepend
     else
       chain_instrument NewRelic::Agent::Instrumentation::<%= @class_name %>::Chain
     end
diff --git a/test/multiverse/suites/opensearch/Envfile b/test/multiverse/suites/opensearch/Envfile
new file mode 100644
index 0000000000..87222e5890
--- /dev/null
+++ b/test/multiverse/suites/opensearch/Envfile
@@ -0,0 +1,20 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+instrumentation_methods :chain, :prepend
+
+OPENSEARCH_VERSIONS = [
+  ['2.1.0', 2.5],
+  ['3.4.0', 2.5]
+]
+
+def gem_list(opensearch_version = nil)
+  <<~RB
+    gem 'opensearch-ruby'#{opensearch_version}
+
+  RB
+end
+
+create_gemfiles(OPENSEARCH_VERSIONS)
+
diff --git a/test/multiverse/suites/opensearch/config/newrelic.yml b/test/multiverse/suites/opensearch/config/newrelic.yml
new file mode 100644
index 0000000000..5dd0b1c7a9
--- /dev/null
+++ b/test/multiverse/suites/opensearch/config/newrelic.yml
@@ -0,0 +1,19 @@
+---
+development:
+  error_collector:
+    enabled: true
+  apdex_t: 0.5
+  monitor_mode: true
+  license_key: bootstrap_newrelic_admin_license_key_000
+  instrumentation:
+    opensearch: <%= $instrumentation_method %>
+  app_name: test
+  log_level: debug
+  host: 127.0.0.1
+  api_host: 127.0.0.1
+  transaction_trace:
+    record_sql: obfuscated
+    enabled: true
+    stack_trace_threshold: 0.5
+    transaction_threshold: 1.0
+  capture_params: false
diff --git a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
new file mode 100644
index 0000000000..3bcbd9415d
--- /dev/null
+++ b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
@@ -0,0 +1,204 @@
+# This file is distributed under New Relic's license terms.
+# See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
+# frozen_string_literal: true
+
+class OpenSearchInstrumentationTest < Minitest::Test
+  def setup
+    @client = OpenSearch::Client.new(
+      host: "https://localhost:9200",
+      user: 'admin',
+      password: 'admin',
+      transport_options: { ssl: { verify: false } }
+    )
+
+    @client.cluster.health
+    @client.index(index: 'my-index', id: 1, body: {title: 'Test'})
+    @client.indices.refresh(index: 'my-index')
+  end
+
+  def teardown
+    @client = nil
+    @segment = nil
+  end
+
+  def search
+    txn = in_transaction do
+      @client.search(index: 'my-index', body: {query: {match: {title: 'test'}}})
+    end
+
+    @segment = txn.segments[1]
+  end
+
+  def test_datastore_segment_created
+    search
+
+    assert_equal NewRelic::Agent::Transaction::DatastoreSegment, @segment.class
+  end
+
+  def test_segment_opensearch_product
+    search
+
+    assert_equal 'OpenSearch', @segment.product
+  end
+
+  def test_segment_operation_is_search_when_search_method_called
+    search
+
+    assert_equal 'search', @segment.operation
+  end
+
+  def test_segment_operation_is_index_when_index_method_called
+    txn = in_transaction do
+      @client.index(index: 'my-index', id: 1, body: {title: 'Test'})
+    end
+
+    segment = txn.segments[1]
+
+    assert_equal 'index', segment.operation
+  end
+
+  def test_segment_operation_returns_OPERATION_when_api_not_called
+    # stubbing the constant to make sure it takes over when there's a nil value for nr_operation
+    NewRelic::Agent::Instrumentation::Elasticsearch.stub_const(:OPERATION, 'subdued-excitement') do
+      txn = in_transaction { @client.perform_request('GET', '/_search', {q: 'hi'}) }
+      segment = txn.segments[1]
+
+      assert_equal NewRelic::Agent::Instrumentation::OpenSearch::OPERATION, segment.operation
+    end
+  end
+
+  def test_segment_host
+    search
+
+    assert_equal Socket.gethostname, @segment.host
+  end
+
+  def test_segment_port_path_or_id_uses_port
+    search
+
+    assert_equal port.to_s, @segment.port_path_or_id
+  end
+
+  def test_segment_database_name
+    search
+
+    assert_equal 'docker-cluster', @segment.database_name
+  end
+
+  def test_cluster_name_doesnt_try_again_if_defined_but_nil
+    original = @client.instance_variable_get(:@transport).instance_variable_get(:@nr_cluster_name)
+    @client.instance_variable_get(:@transport).instance_variable_set(:@nr_cluster_name, nil)
+    search
+    @client.instance_variable_get(:@transport).instance_variable_set(:@nr_cluster_name, original)
+
+    assert_nil @segment.database_name
+  end
+
+  def test_nosql_statement_recorded_params_obfuscated
+    with_config(:'opensearch.obfuscate_queries' => true) do
+      txn = in_transaction do
+        # passing q: title sets the perform_request method's params argument to
+        # {q: 'title'} and leaves the body argument nil
+        @client.search(index: 'my-index', q: '?')
+      end
+      segment = txn.segments[1]
+      obfuscated_query = {q: '?'}
+
+      assert_equal obfuscated_query, segment.nosql_statement
+    end
+  end
+
+  def test_nosql_statement_recorded_params_not_obfuscated
+    with_config(:'opensearch.obfuscate_queries' => false) do
+      txn = in_transaction do
+        # passing `q: title` sets the perform_request method's params argument
+        # to {q: 'title'} and leaves the body argument nil
+        @client.search(index: 'my-index', q: 'title')
+      end
+      segment = txn.segments[1]
+      not_obfuscated_query = {q: 'title'}
+
+      assert_equal not_obfuscated_query, segment.nosql_statement
+    end
+  end
+
+
+  def test_nosql_statement_recorded_body_obfuscated
+    with_config(:'opensearch.obfuscate_queries' => true) do
+      txn = in_transaction do
+        query = {query: {match: {title: 'test'}}}
+        @client.search(index: 'my-index', body: query)
+      end
+      segment = txn.segments[1]
+      obfuscated_query = {query: {match: {title: '?'}}}
+
+      assert_equal obfuscated_query, segment.nosql_statement
+    end
+  end
+
+  def test_nosql_statement_recorded_body_not_obfuscated
+    with_config(:'opensearch.obfuscate_queries' => false) do
+      query = {query: {match: {title: 'test'}}}
+      txn = in_transaction do
+        @client.search(index: 'my-index', body: query)
+      end
+      segment = txn.segments[1]
+
+      assert_equal query, segment.nosql_statement
+    end
+  end
+
+  def test_statement_captured
+    with_config(:'opensearch.capture_queries' => true) do
+      query = {query: {match: {title: 'test'}}}
+      ob_query = {query: {match: {title: '?'}}}
+      txn = in_transaction do
+        @client.search(index: 'my-index', body: query)
+      end
+      segment = txn.segments[1]
+
+      assert_equal ob_query, segment.nosql_statement
+    end
+  end
+
+  def test_statement_not_captured
+    with_config(:'opensearch.capture_queries' => false) do
+      query = {query: {match: {title: 'test'}}}
+      txn = in_transaction do
+        @client.search(index: 'my-index', body: query)
+      end
+      segment = txn.segments[1]
+
+      assert_nil segment.nosql_statement
+    end
+  end
+
+  def test_segment_error_captured_if_raised
+    txn = nil
+    begin
+      in_transaction('opensearch') do |elastic_txn|
+        txn = elastic_txn
+        simulate_transport_error
+      end
+    rescue StandardError => e
+      # NOOP -- allowing span and transaction to notice error
+    end
+
+    expected_error_class_name = OpenSearch::Transport::Transport::Error.name
+
+    assert_segment_noticed_error txn, /opensearch$/, expected_error_class_name, /Error/i
+    assert_transaction_noticed_error txn, expected_error_class_name
+  end
+
+  private
+
+  def simulate_transport_error
+    @client.stub(:search, raise(transport_error_class.new)) do
+      @client.search(index: 'my-index', q: 'title')
+    end
+  end
+
+  def port
+    9200 # remove once we decide on a port
+  end
+end

From f11602b25bad541dfbcfa4a8c2c0c66187f1d637 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Tue, 6 Aug 2024 17:19:36 -0700
Subject: [PATCH 02/41] Draft opensearch service for CI

---
 .github/workflows/ci.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6c61b1d601..500c304699 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -136,6 +136,18 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 10
+      opensearch:
+        image: opensearchproject/opensearch:1.8.13
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+        ports:
+          - 9300:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
       memcached:
         image: memcached:latest
         ports:

From c9cd8571c97fe93f332ffac5e9900361a1ad8158 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Wed, 7 Aug 2024 12:14:06 -0700
Subject: [PATCH 03/41] Adjust test environment for opensearch

---
 .github/workflows/ci.yml                      |  8 ++---
 docker-compose.yml                            | 33 +++++++------------
 .../agent/instrumentation/opensearch.rb       |  2 +-
 .../elasticsearch_instrumentation_test.rb     |  2 +-
 test/multiverse/suites/opensearch/Envfile     |  2 +-
 .../opensearch_instrumentation_test.rb        |  7 ++--
 6 files changed, 23 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 500c304699..74200b9930 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -118,7 +118,7 @@ jobs:
         env:
           discovery.type: single-node
         ports:
-          - 9200:9200
+          - 9000:9200
         options: >-
           --health-cmd "curl http://localhost:9200/_cluster/health"
           --health-interval 10s
@@ -140,11 +140,11 @@ jobs:
         image: opensearchproject/opensearch:1.8.13
         env:
           discovery.type: single-node
-          xpack.security.enabled: false
         ports:
-          - 9300:9200
+          - 9200:9200
+          - 9600:9600
         options: >-
-          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-cmd "curl https://localhost:9200/_cluster/health"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 10
diff --git a/docker-compose.yml b/docker-compose.yml
index 3a16304195..dc873d4c71 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,13 +1,14 @@
 services:
-  # elasticsearch7:
-  #   image: elasticsearch:7.16.2
-  #   ports:
-  #     - "9200:9200"
-  #   environment:
-  #     - discovery.type=single-node
-  #     - bootstrap.memory_lock=true
-  #     - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-  #   mem_limit: 1g
+  elasticsearch7:
+    image: elasticsearch:7.16.2
+    ports:
+      - "9000:9000"
+    environment:
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - http.port=9000
+    mem_limit: 1g
   elasticsearch8:
     image: elasticsearch:8.13.0
     ports:
@@ -19,25 +20,15 @@ services:
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     mem_limit: 1g
-  # tests passed with
-  # docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" opensearchproject/opensearch:1.3.18
   opensearch:
     image: opensearchproject/opensearch:1.3.18
     environment:
       - discovery.type=single-node
-      - bootstrap.memory_lock=true
-      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
-      - "DISABLE_INSTALL_DEMO_CONFIG=true"
-      - "DISABLE_SECURITY_PLUGIN=true"
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-    volumes:
-      - opensearch_data:/usr/share/opensearch/data
     ports:
       - 9200:9200
       - 9600:9600
+      - 'OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m'
+    mem_limit: 1g
   mysql:
     image: mysql:5.7
     platform: linux/x86_64
diff --git a/lib/new_relic/agent/instrumentation/opensearch.rb b/lib/new_relic/agent/instrumentation/opensearch.rb
index 42aa386df7..0b10f1914a 100644
--- a/lib/new_relic/agent/instrumentation/opensearch.rb
+++ b/lib/new_relic/agent/instrumentation/opensearch.rb
@@ -14,7 +14,7 @@
   end
 
   executes do
-    ::NewRelic::Agent.logger.info('Installing opensearch-ruby instrumentation')
+    NewRelic::Agent.logger.info('Installing opensearch-ruby instrumentation')
 
     if use_prepend?
       prepend_instrument OpenSearch::Transport::Client, NewRelic::Agent::Instrumentation::OpenSearch::Prepend
diff --git a/test/multiverse/suites/elasticsearch/elasticsearch_instrumentation_test.rb b/test/multiverse/suites/elasticsearch/elasticsearch_instrumentation_test.rb
index 33ea640617..0b4890454e 100644
--- a/test/multiverse/suites/elasticsearch/elasticsearch_instrumentation_test.rb
+++ b/test/multiverse/suites/elasticsearch/elasticsearch_instrumentation_test.rb
@@ -222,7 +222,7 @@ def transport_error_class
 
   def port
     if ::Gem::Version.create(Elasticsearch::VERSION) < ::Gem::Version.create('8.0.0')
-      9200 # 9200 for elasticsearch 7
+      9000 # 9000 for elasticsearch 7
     else
       9250 # 9250 for elasticsearch 8
     end
diff --git a/test/multiverse/suites/opensearch/Envfile b/test/multiverse/suites/opensearch/Envfile
index 87222e5890..3030dacca2 100644
--- a/test/multiverse/suites/opensearch/Envfile
+++ b/test/multiverse/suites/opensearch/Envfile
@@ -5,7 +5,7 @@
 instrumentation_methods :chain, :prepend
 
 OPENSEARCH_VERSIONS = [
-  ['2.1.0', 2.5],
+  [nil, 2.5],
   ['3.4.0', 2.5]
 ]
 
diff --git a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
index 3bcbd9415d..e01d903011 100644
--- a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
+++ b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
@@ -4,11 +4,13 @@
 
 class OpenSearchInstrumentationTest < Minitest::Test
   def setup
+
+
     @client = OpenSearch::Client.new(
-      host: "https://localhost:9200",
+      host: "https://localhost:#{port}",
       user: 'admin',
       password: 'admin',
-      transport_options: { ssl: { verify: false } }
+      transport_options: {ssl: {verify: false}}
     )
 
     @client.cluster.health
@@ -122,7 +124,6 @@ def test_nosql_statement_recorded_params_not_obfuscated
     end
   end
 
-
   def test_nosql_statement_recorded_body_obfuscated
     with_config(:'opensearch.obfuscate_queries' => true) do
       txn = in_transaction do

From bd239ac9cca75a053353aaf0bc4e157f802ce8e6 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Wed, 7 Aug 2024 12:16:27 -0700
Subject: [PATCH 04/41] Fix spacing

---
 .../suites/opensearch/opensearch_instrumentation_test.rb        | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
index e01d903011..dad041f68a 100644
--- a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
+++ b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
@@ -4,8 +4,6 @@
 
 class OpenSearchInstrumentationTest < Minitest::Test
   def setup
-
-
     @client = OpenSearch::Client.new(
       host: "https://localhost:#{port}",
       user: 'admin',

From 8f6b7090954e63423ee8d6d0e24a4a346ef3abb1 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Wed, 7 Aug 2024 15:27:54 -0700
Subject: [PATCH 05/41] Small updates to docker-compose and ci_cron

---
 .github/workflows/ci_cron.yml | 24 ++++++++++++------------
 docker-compose.yml            |  4 +---
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml
index 7e6ded35e7..457ded437e 100644
--- a/.github/workflows/ci_cron.yml
+++ b/.github/workflows/ci_cron.yml
@@ -131,7 +131,7 @@ jobs:
         env:
           discovery.type: single-node
         ports:
-          - 9200:9200
+          - 9000:9200
         options: >-
           --health-cmd "curl http://localhost:9200/_cluster/health"
           --health-interval 10s
@@ -149,17 +149,17 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 10
-      # opensearch:
-      #   image: opensearch:1.3.18 # latest version when instrumentation created
-      #   ports:
-      #     - "9350:9350"
-      #   environment:
-      #     - discovery.type=single-node
-      #     - plugins.security.disabled=true
-      #     - http.port=9350
-      #     - bootstrap.memory_lock=true
-      #     - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
-      #   mem_limit: 1g
+      opensearch:
+        image: opensearch:1.3.18
+        env:
+          discovery.type: single-node
+        ports:
+          - 9200:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
       memcached:
         image: memcached:latest
         ports:
diff --git a/docker-compose.yml b/docker-compose.yml
index dc873d4c71..c69ce576d0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -98,7 +98,7 @@ services:
     volumes:
       - ".:/usr/src/app"
     depends_on:
-      # - elasticsearch7
+      - elasticsearch7
       - elasticsearch8
       - opensearch
       - mysql
@@ -116,5 +116,3 @@ volumes:
     driver: local
   postgres:
     driver: local
-  opensearch_data:
-    driver: local

From 7194bd065b374a7ed3b8db02e4b45bf4b9f702fd Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Wed, 7 Aug 2024 15:31:52 -0700
Subject: [PATCH 06/41] Remove unnecessary ::

---
 .../instrumentation_generator/templates/dependency_detection.tt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/tasks/instrumentation_generator/templates/dependency_detection.tt b/lib/tasks/instrumentation_generator/templates/dependency_detection.tt
index 15849f5e14..2b17a16e80 100644
--- a/lib/tasks/instrumentation_generator/templates/dependency_detection.tt
+++ b/lib/tasks/instrumentation_generator/templates/dependency_detection.tt
@@ -18,7 +18,7 @@ DependencyDetection.defer do
   end
 
   executes do
-    ::NewRelic::Agent.logger.info('Installing <%= @name.downcase %> instrumentation')
+    NewRelic::Agent.logger.info('Installing <%= @name.downcase %> instrumentation')
 
     if use_prepend?
       prepend_instrument <%= @class_name %>, NewRelic::Agent::Instrumentation::<%= @class_name %>::Prepend

From dd1f09fdbe426b7bd4e73281862f271d4137bfd7 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Wed, 7 Aug 2024 15:36:15 -0700
Subject: [PATCH 07/41] Update images for opensearch ci services

---
 .github/workflows/ci.yml      | 2 +-
 .github/workflows/ci_cron.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 74200b9930..157b5ea8f6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -137,7 +137,7 @@ jobs:
           --health-timeout 5s
           --health-retries 10
       opensearch:
-        image: opensearchproject/opensearch:1.8.13
+        image: opensearchproject/opensearch:latest
         env:
           discovery.type: single-node
         ports:
diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml
index 457ded437e..c8820df83c 100644
--- a/.github/workflows/ci_cron.yml
+++ b/.github/workflows/ci_cron.yml
@@ -150,7 +150,7 @@ jobs:
           --health-timeout 5s
           --health-retries 10
       opensearch:
-        image: opensearch:1.3.18
+        image: opensearchproject/opensearch:latest
         env:
           discovery.type: single-node
         ports:

From 7e4ba640ad3cfe0aa63b5b66d194e4d35943e48e Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Wed, 7 Aug 2024 15:41:53 -0700
Subject: [PATCH 08/41] Update opensearch version

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 157b5ea8f6..7ee3c74e91 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -137,7 +137,7 @@ jobs:
           --health-timeout 5s
           --health-retries 10
       opensearch:
-        image: opensearchproject/opensearch:latest
+        image: opensearchproject/opensearch:1.3.18
         env:
           discovery.type: single-node
         ports:

From 430d6ba6d0c388ba9f60624649cc6e4828183914 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Wed, 7 Aug 2024 15:52:48 -0700
Subject: [PATCH 09/41] disable security plugins

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7ee3c74e91..9a43817fd0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -140,6 +140,7 @@ jobs:
         image: opensearchproject/opensearch:1.3.18
         env:
           discovery.type: single-node
+          plugins.security.disabled: true
         ports:
           - 9200:9200
           - 9600:9600

From 314ea5970f6162e01ae00ed9d3b62e1819442d4e Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Wed, 7 Aug 2024 15:57:03 -0700
Subject: [PATCH 10/41] disable demo config

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9a43817fd0..87ec513c73 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -141,6 +141,7 @@ jobs:
         env:
           discovery.type: single-node
           plugins.security.disabled: true
+          DISABLE_INSTALL_DEMO_CONFIG: true
         ports:
           - 9200:9200
           - 9600:9600

From 77094d6c1f4b9ddc3313fb062e42c91a3407eaa3 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Wed, 7 Aug 2024 16:09:14 -0700
Subject: [PATCH 11/41] Remove options

---
 .github/workflows/ci.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 87ec513c73..9c7d645e4f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -145,11 +145,6 @@ jobs:
         ports:
           - 9200:9200
           - 9600:9600
-        options: >-
-          --health-cmd "curl https://localhost:9200/_cluster/health"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 10
       memcached:
         image: memcached:latest
         ports:

From 737c2b9d95ea168cc5ebf1d4b41e433d555dd3e1 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Wed, 7 Aug 2024 16:24:21 -0700
Subject: [PATCH 12/41] Simplify opensearch ci.yml service

---
 .github/workflows/ci.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9c7d645e4f..496f26a9e6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -140,8 +140,6 @@ jobs:
         image: opensearchproject/opensearch:1.3.18
         env:
           discovery.type: single-node
-          plugins.security.disabled: true
-          DISABLE_INSTALL_DEMO_CONFIG: true
         ports:
           - 9200:9200
           - 9600:9600

From d2adc1e4bf7c9019fefba3d77f3c12a888ab679f Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Wed, 7 Aug 2024 17:08:41 -0700
Subject: [PATCH 13/41] Add details for health check

---
 .github/workflows/ci.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 496f26a9e6..e3da2c7aa9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -143,6 +143,11 @@ jobs:
         ports:
           - 9200:9200
           - 9600:9600
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
       memcached:
         image: memcached:latest
         ports:

From 3e6670022c93333ab940a0c14dacfc031ea0d804 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Thu, 8 Aug 2024 07:51:46 -0700
Subject: [PATCH 14/41] Adjust opensearch ci service

---
 .github/workflows/ci.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e3da2c7aa9..928d8f4684 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -137,17 +137,16 @@ jobs:
           --health-timeout 5s
           --health-retries 10
       opensearch:
-        image: opensearchproject/opensearch:1.3.18
-        env:
-          discovery.type: single-node
+        image: opensearchproject/opensearch:2.13.0
         ports:
           - 9200:9200
-          - 9600:9600
         options: >-
-          --health-cmd "curl http://localhost:9200/_cluster/health"
+          -e "discovery.type=single-node"
+          -e "DISABLE_SECURITY_PLUGIN=true"
+          --health-cmd "curl 127.0.0.1:9200"
           --health-interval 10s
           --health-timeout 5s
-          --health-retries 10
+          --health-retries 15
       memcached:
         image: memcached:latest
         ports:

From e95edfb6a576fd5ec8ad10352f87943eb3911460 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Thu, 8 Aug 2024 08:01:15 -0700
Subject: [PATCH 15/41] Remove other services for faster feedback

---
 .github/workflows/ci.yml | 88 ++++------------------------------------
 1 file changed, 9 insertions(+), 79 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 928d8f4684..6a2eb41a04 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -113,90 +113,20 @@ jobs:
     needs: run_rubocop
     runs-on: ubuntu-22.04
     services:
-      elasticsearch7:
-        image: elasticsearch:7.16.2
-        env:
-          discovery.type: single-node
-        ports:
-          - 9000:9200
-        options: >-
-          --health-cmd "curl http://localhost:9200/_cluster/health"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 10
-      elasticsearch8:
-        image: elasticsearch:8.13.0
-        env:
-          discovery.type: single-node
-          xpack.security.enabled: false
-        ports:
-          - 9250:9200
-        options: >-
-          --health-cmd "curl http://localhost:9200/_cluster/health"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 10
       opensearch:
-        image: opensearchproject/opensearch:2.13.0
+        image: opensearchproject/opensearch:1.1.0
         ports:
           - 9200:9200
-        options: >-
-          -e "discovery.type=single-node"
-          -e "DISABLE_SECURITY_PLUGIN=true"
-          --health-cmd "curl 127.0.0.1:9200"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 15
-      memcached:
-        image: memcached:latest
-        ports:
-          - 11211:11211
-        options: >-
-          --health-cmd "timeout 5 bash -c 'cat < /dev/null > /dev/udp/127.0.0.1/11211'"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      mongodb:
-        image: ${{ contains(fromJson('["2.4.10"]'), matrix.ruby-version) && 'mongo:5.0.11' || 'mongo:latest' }}
-        ports:
-          - 27017:27017
-      mysql:
-        image: mysql:5.7
-        env:
-          MYSQL_ROOT_PASSWORD: root
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
-        ports:
-          - 3306
-      postgres:
-        image: postgres:latest
         env:
-          POSTGRES_USERNAME: postgres
-          POSTGRES_PASSWORD: password
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      rabbitmq:
-        image: rabbitmq:latest
-        ports:
-          - 5672:5672
-        options: >-
-          --health-cmd "rabbitmq-diagnostics -q check_port_connectivity"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      redis:
-        image: redis
-        ports:
-          - 6379:6379
+          discovery.type: single-node
+          DISABLE_INSTALL_DEMO_CONFIG: true
+          DISABLE_SECURITY_PLUGIN: true
         options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
+          --name=opensearch
+          --health-cmd="curl http://localhost:9200/_cluster/health"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=3
     strategy:
       fail-fast: false
       matrix:

From 4a9f08c90712be864bf74f2e35303a97bb8589ad Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Thu, 8 Aug 2024 08:05:56 -0700
Subject: [PATCH 16/41] Add some services back

---
 .github/workflows/ci.yml | 50 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6a2eb41a04..688f4207af 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -127,6 +127,56 @@ jobs:
           --health-interval=10s
           --health-timeout=5s
           --health-retries=3
+      memcached:
+        image: memcached:latest
+        ports:
+          - 11211:11211
+        options: >-
+          --health-cmd "timeout 5 bash -c 'cat < /dev/null > /dev/udp/127.0.0.1/11211'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      mongodb:
+        image: ${{ contains(fromJson('["2.4.10"]'), matrix.ruby-version) && 'mongo:5.0.11' || 'mongo:latest' }}
+        ports:
+          - 27017:27017
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: root
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+        ports:
+          - 3306
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_USERNAME: postgres
+          POSTGRES_PASSWORD: password
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      rabbitmq:
+        image: rabbitmq:latest
+        ports:
+          - 5672:5672
+        options: >-
+          --health-cmd "rabbitmq-diagnostics -q check_port_connectivity"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     strategy:
       fail-fast: false
       matrix:

From ad099e62ed9856cda41130a8de0e744dec01fb46 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Thu, 8 Aug 2024 08:18:46 -0700
Subject: [PATCH 17/41] Fix test port

---
 .../suites/opensearch/opensearch_instrumentation_test.rb        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
index dad041f68a..73d36e2e53 100644
--- a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
+++ b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
@@ -7,7 +7,7 @@ def setup
     @client = OpenSearch::Client.new(
       host: "https://localhost:#{port}",
       user: 'admin',
-      password: 'admin',
+      password: 'sn33ZeGesundheit!',
       transport_options: {ssl: {verify: false}}
     )
 

From bc556fa7fe78ad7c6296e89903d6d8261a8ba99d Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Thu, 8 Aug 2024 08:58:01 -0700
Subject: [PATCH 18/41] Remove security disabling

---
 .github/workflows/ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 688f4207af..854e4324b3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -114,16 +114,16 @@ jobs:
     runs-on: ubuntu-22.04
     services:
       opensearch:
-        image: opensearchproject/opensearch:1.1.0
+        image: opensearchproject/opensearch:1.3.18
         ports:
           - 9200:9200
         env:
           discovery.type: single-node
-          DISABLE_INSTALL_DEMO_CONFIG: true
-          DISABLE_SECURITY_PLUGIN: true
+          # DISABLE_INSTALL_DEMO_CONFIG: true
+          # DISABLE_SECURITY_PLUGIN: true
         options: >-
           --name=opensearch
-          --health-cmd="curl http://localhost:9200/_cluster/health"
+          --health-cmd="curl https://localhost:9200/_cluster/health"
           --health-interval=10s
           --health-timeout=5s
           --health-retries=3

From b496be3c02a1d2c58293d1965997e9869dca97a1 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 09:43:32 -0700
Subject: [PATCH 19/41] Add security disable configs back to opensearch

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 854e4324b3..754de2de31 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -119,8 +119,8 @@ jobs:
           - 9200:9200
         env:
           discovery.type: single-node
-          # DISABLE_INSTALL_DEMO_CONFIG: true
-          # DISABLE_SECURITY_PLUGIN: true
+          DISABLE_INSTALL_DEMO_CONFIG: true
+          DISABLE_SECURITY_PLUGIN: true
         options: >-
           --name=opensearch
           --health-cmd="curl https://localhost:9200/_cluster/health"

From 82078ae113cf481f158db45e9757dab1e57d7c9b Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 10:03:39 -0700
Subject: [PATCH 20/41] try removing demo config only

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 754de2de31..59b94bd781 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -119,11 +119,11 @@ jobs:
           - 9200:9200
         env:
           discovery.type: single-node
-          DISABLE_INSTALL_DEMO_CONFIG: true
+          # DISABLE_INSTALL_DEMO_CONFIG: true
           DISABLE_SECURITY_PLUGIN: true
         options: >-
           --name=opensearch
-          --health-cmd="curl https://localhost:9200/_cluster/health"
+          --health-cmd="curl -X GET "https://localhost:9200/_cluster/health" -ku admin:admin"
           --health-interval=10s
           --health-timeout=5s
           --health-retries=3

From b8e52ad3fd55e30ace25566e8a79cc5907d3980c Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 10:10:31 -0700
Subject: [PATCH 21/41] Revert

---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 59b94bd781..64107e3909 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -114,16 +114,16 @@ jobs:
     runs-on: ubuntu-22.04
     services:
       opensearch:
-        image: opensearchproject/opensearch:1.3.18
+        image: opensearchproject/opensearch:1.1.0
         ports:
           - 9200:9200
         env:
           discovery.type: single-node
-          # DISABLE_INSTALL_DEMO_CONFIG: true
+          DISABLE_INSTALL_DEMO_CONFIG: true
           DISABLE_SECURITY_PLUGIN: true
         options: >-
           --name=opensearch
-          --health-cmd="curl -X GET "https://localhost:9200/_cluster/health" -ku admin:admin"
+          --health-cmd="curl http://localhost:9200/_cluster/health -ku admin:admin"
           --health-interval=10s
           --health-timeout=5s
           --health-retries=3

From 2304b1ad8bd6c46efef00ecba650193b2f5867ce Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 10:20:02 -0700
Subject: [PATCH 22/41] Push the version

---
 .github/workflows/ci.yml                                        | 2 +-
 .../suites/opensearch/opensearch_instrumentation_test.rb        | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 64107e3909..60c1d89a2f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -114,7 +114,7 @@ jobs:
     runs-on: ubuntu-22.04
     services:
       opensearch:
-        image: opensearchproject/opensearch:1.1.0
+        image: opensearchproject/opensearch:1.3.18
         ports:
           - 9200:9200
         env:
diff --git a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
index 73d36e2e53..dad041f68a 100644
--- a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
+++ b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
@@ -7,7 +7,7 @@ def setup
     @client = OpenSearch::Client.new(
       host: "https://localhost:#{port}",
       user: 'admin',
-      password: 'sn33ZeGesundheit!',
+      password: 'admin',
       transport_options: {ssl: {verify: false}}
     )
 

From 27e3a0d8298a00ac8c18d24d014eb34d28084269 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 10:37:20 -0700
Subject: [PATCH 23/41] update to match docker-rails and searchkick

---
 .github/workflows/ci.yml                           | 14 +++-----------
 .../opensearch/opensearch_instrumentation_test.rb  | 13 +++++++++----
 2 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 60c1d89a2f..2df45911ed 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -114,19 +114,11 @@ jobs:
     runs-on: ubuntu-22.04
     services:
       opensearch:
-        image: opensearchproject/opensearch:1.3.18
-        ports:
-          - 9200:9200
+        image: opensearchproject/opensearch:latest
+        ports: ["9200:9200"]
         env:
           discovery.type: single-node
-          DISABLE_INSTALL_DEMO_CONFIG: true
-          DISABLE_SECURITY_PLUGIN: true
-        options: >-
-          --name=opensearch
-          --health-cmd="curl http://localhost:9200/_cluster/health -ku admin:admin"
-          --health-interval=10s
-          --health-timeout=5s
-          --health-retries=3
+          plugins.security.disabled: true
       memcached:
         image: memcached:latest
         ports:
diff --git a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
index dad041f68a..9736ba2914 100644
--- a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
+++ b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
@@ -4,11 +4,16 @@
 
 class OpenSearchInstrumentationTest < Minitest::Test
   def setup
+    # @client = OpenSearch::Client.new(
+    #   host: "https://localhost:#{port}",
+    #   user: 'admin',
+    #   password: 'admin',
+    #   transport_options: {ssl: {verify: false}}
+    # )
     @client = OpenSearch::Client.new(
-      host: "https://localhost:#{port}",
-      user: 'admin',
-      password: 'admin',
-      transport_options: {ssl: {verify: false}}
+          url: 'http://localhost:9200',
+          transport_options: {request: {timeout: 10}, headers: {content_type: "application/json"}},
+          retry_on_failure: 2
     )
 
     @client.cluster.health

From 2c9b460dae2182dc1f4f8e59b14d73a7a25f63a6 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 10:39:33 -0700
Subject: [PATCH 24/41] rubocop

---
 .../suites/opensearch/opensearch_instrumentation_test.rb    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
index 9736ba2914..8a9a2f396e 100644
--- a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
+++ b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
@@ -11,9 +11,9 @@ def setup
     #   transport_options: {ssl: {verify: false}}
     # )
     @client = OpenSearch::Client.new(
-          url: 'http://localhost:9200',
-          transport_options: {request: {timeout: 10}, headers: {content_type: "application/json"}},
-          retry_on_failure: 2
+      url: 'http://localhost:9200',
+      transport_options: {request: {timeout: 10}, headers: {content_type: 'application/json'}},
+      retry_on_failure: 2
     )
 
     @client.cluster.health

From d726f3ce4e208f1953042542abd723590ef8cb1e Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 11:52:31 -0700
Subject: [PATCH 25/41] Revert to what works locally

---
 .../opensearch/opensearch_instrumentation_test.rb  | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
index 8a9a2f396e..5708abe1b8 100644
--- a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
+++ b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
@@ -4,18 +4,12 @@
 
 class OpenSearchInstrumentationTest < Minitest::Test
   def setup
-    # @client = OpenSearch::Client.new(
-    #   host: "https://localhost:#{port}",
-    #   user: 'admin',
-    #   password: 'admin',
-    #   transport_options: {ssl: {verify: false}}
-    # )
     @client = OpenSearch::Client.new(
-      url: 'http://localhost:9200',
-      transport_options: {request: {timeout: 10}, headers: {content_type: 'application/json'}},
-      retry_on_failure: 2
+      host: "https://localhost:#{port}",
+      user: 'admin',
+      password: 'admin',
+      transport_options: {ssl: {verify: false}}
     )
-
     @client.cluster.health
     @client.index(index: 'my-index', id: 1, body: {title: 'Test'})
     @client.indices.refresh(index: 'my-index')

From 4dac92a35dff8c6cd77453fb27b56e10c5ab672c Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 11:55:29 -0700
Subject: [PATCH 26/41] Try to connect to elasticsearch

---
 .github/workflows/ci.yml | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2df45911ed..6c61b1d601 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -113,12 +113,29 @@ jobs:
     needs: run_rubocop
     runs-on: ubuntu-22.04
     services:
-      opensearch:
-        image: opensearchproject/opensearch:latest
-        ports: ["9200:9200"]
+      elasticsearch7:
+        image: elasticsearch:7.16.2
         env:
           discovery.type: single-node
-          plugins.security.disabled: true
+        ports:
+          - 9200:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      elasticsearch8:
+        image: elasticsearch:8.13.0
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+        ports:
+          - 9250:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
       memcached:
         image: memcached:latest
         ports:

From fdd258c766ad13cb01743d03d326b1ca0ebfc978 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 12:12:21 -0700
Subject: [PATCH 27/41] Downgrade to 7.10 elasticsearch

---
 .github/workflows/ci.yml                                      | 2 +-
 docker-compose.yml                                            | 4 +---
 .../elasticsearch/elasticsearch_instrumentation_test.rb       | 2 +-
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6c61b1d601..84e561f56c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -114,7 +114,7 @@ jobs:
     runs-on: ubuntu-22.04
     services:
       elasticsearch7:
-        image: elasticsearch:7.16.2
+        image: elasticsearch:7.10
         env:
           discovery.type: single-node
         ports:
diff --git a/docker-compose.yml b/docker-compose.yml
index c69ce576d0..e9f3f2c4f8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -2,12 +2,11 @@ services:
   elasticsearch7:
     image: elasticsearch:7.16.2
     ports:
-      - "9000:9000"
+      - "9200:9200"
     environment:
       - discovery.type=single-node
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-      - http.port=9000
     mem_limit: 1g
   elasticsearch8:
     image: elasticsearch:8.13.0
@@ -27,7 +26,6 @@ services:
     ports:
       - 9200:9200
       - 9600:9600
-      - 'OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m'
     mem_limit: 1g
   mysql:
     image: mysql:5.7
diff --git a/test/multiverse/suites/elasticsearch/elasticsearch_instrumentation_test.rb b/test/multiverse/suites/elasticsearch/elasticsearch_instrumentation_test.rb
index 0b4890454e..33ea640617 100644
--- a/test/multiverse/suites/elasticsearch/elasticsearch_instrumentation_test.rb
+++ b/test/multiverse/suites/elasticsearch/elasticsearch_instrumentation_test.rb
@@ -222,7 +222,7 @@ def transport_error_class
 
   def port
     if ::Gem::Version.create(Elasticsearch::VERSION) < ::Gem::Version.create('8.0.0')
-      9000 # 9000 for elasticsearch 7
+      9200 # 9200 for elasticsearch 7
     else
       9250 # 9250 for elasticsearch 8
     end

From cede740ac2f424444c33ac7d4b22c19959c428ff Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 12:16:40 -0700
Subject: [PATCH 28/41] Update elasticsearch7 image to 7.10.1

---
 .github/workflows/ci.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 84e561f56c..5e3785a2f6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -114,7 +114,7 @@ jobs:
     runs-on: ubuntu-22.04
     services:
       elasticsearch7:
-        image: elasticsearch:7.10
+        image: elasticsearch:7.10.1
         env:
           discovery.type: single-node
         ports:
@@ -197,6 +197,10 @@ jobs:
         run: 'git config --global init.defaultBranch main'
       - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag v4.1.2
 
+      #   # Runs on OpenSearch 2 by default
+      # - name: Set up OpenSearch
+      #   uses: ankane/setup-opensearch@v1
+
         # - curl is needed for Curb
         # - xslt is needed for older Nokogiris, RUBY_VERSION < 2.5
         # - sasl is needed for memcached

From ab3b26fa70727967bd96ea4c54e0abd57c221fae Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 12:24:37 -0700
Subject: [PATCH 29/41] Run tests on older opensearch

---
 test/multiverse/suites/opensearch/Envfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/multiverse/suites/opensearch/Envfile b/test/multiverse/suites/opensearch/Envfile
index 3030dacca2..e75b48d044 100644
--- a/test/multiverse/suites/opensearch/Envfile
+++ b/test/multiverse/suites/opensearch/Envfile
@@ -6,7 +6,8 @@ instrumentation_methods :chain, :prepend
 
 OPENSEARCH_VERSIONS = [
   [nil, 2.5],
-  ['3.4.0', 2.5]
+  ['3.4.0', 2.5],
+  ['2.1.0', 2.5]
 ]
 
 def gem_list(opensearch_version = nil)

From c5c94135952d774bc5853b18d5df743b0225321f Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 12:26:31 -0700
Subject: [PATCH 30/41] Run only 'rest'

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5e3785a2f6..7a4cda3450 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -189,7 +189,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        multiverse: [agent, ai, background, background_2, database, frameworks, httpclients, httpclients_2, rails, rest]
+        multiverse: [rest]
         ruby-version: [2.4.10, 3.3.4]
 
     steps:

From 3bdacb06ea1fae4981f2b507e890fcfee5ae4bfc Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 12:44:23 -0700
Subject: [PATCH 31/41] Opensearch 2 CI action step

---
 .github/workflows/ci.yml                                  | 8 +++++---
 .../suites/opensearch/opensearch_instrumentation_test.rb  | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7a4cda3450..7e3f6fb542 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -198,9 +198,11 @@ jobs:
       - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag v4.1.2
 
       #   # Runs on OpenSearch 2 by default
-      # - name: Set up OpenSearch
-      #   uses: ankane/setup-opensearch@v1
-
+      - name: Set up OpenSearch
+        uses: ankane/setup-opensearch@v1
+        with:
+          config: |
+            http.port: 9300
         # - curl is needed for Curb
         # - xslt is needed for older Nokogiris, RUBY_VERSION < 2.5
         # - sasl is needed for memcached
diff --git a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
index 5708abe1b8..b5cb41b110 100644
--- a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
+++ b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
@@ -197,6 +197,6 @@ def simulate_transport_error
   end
 
   def port
-    9200 # remove once we decide on a port
+    9300 # remove once we decide on a port
   end
 end

From d186de42911508eccbf5a440a2dd1e5cc7e53cdd Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 12:45:11 -0700
Subject: [PATCH 32/41] only opensearch

---
 .github/workflows/ci.yml                 | 4 ++--
 test/multiverse/lib/multiverse/runner.rb | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7e3f6fb542..550eca879f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -189,8 +189,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        multiverse: [rest]
-        ruby-version: [2.4.10, 3.3.4]
+        multiverse: [opensearch]
+        ruby-version: [3.3.4]
 
     steps:
       - name: Configure git
diff --git a/test/multiverse/lib/multiverse/runner.rb b/test/multiverse/lib/multiverse/runner.rb
index 1224a1c1bd..31cfda35fc 100644
--- a/test/multiverse/lib/multiverse/runner.rb
+++ b/test/multiverse/lib/multiverse/runner.rb
@@ -108,6 +108,7 @@ def execute_suites(filter, opts)
       'httpclients' => %w[async_http curb excon httpclient],
       'httpclients_2' => %w[typhoeus net_http httprb ethon httpx],
       'infinite_tracing' => %w[infinite_tracing],
+      'opensearch' => %w[opensearch]
 
       'rest' => [] # Specially handled below
     }

From e86cbcf19410a079cd2386a6c14c6c6992fa32c4 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 12:49:06 -0700
Subject: [PATCH 33/41] fix error

---
 test/multiverse/lib/multiverse/runner.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/multiverse/lib/multiverse/runner.rb b/test/multiverse/lib/multiverse/runner.rb
index 31cfda35fc..b1270f6869 100644
--- a/test/multiverse/lib/multiverse/runner.rb
+++ b/test/multiverse/lib/multiverse/runner.rb
@@ -108,7 +108,7 @@ def execute_suites(filter, opts)
       'httpclients' => %w[async_http curb excon httpclient],
       'httpclients_2' => %w[typhoeus net_http httprb ethon httpx],
       'infinite_tracing' => %w[infinite_tracing],
-      'opensearch' => %w[opensearch]
+      'opensearch' => %w[opensearch],
 
       'rest' => [] # Specially handled below
     }

From e007242669647712a0b55d9610d49da2d3f3b27b Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 12:52:18 -0700
Subject: [PATCH 34/41] Add 2.4 back

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 550eca879f..9f9caf0c86 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -190,7 +190,7 @@ jobs:
       fail-fast: false
       matrix:
         multiverse: [opensearch]
-        ruby-version: [3.3.4]
+        ruby-version: [2.4.10, 3.3.4]
 
     steps:
       - name: Configure git

From 9dcb6ad35276c8d3afae5f2d797a60bb73c66e05 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 14:45:59 -0700
Subject: [PATCH 35/41] Cleanup post-CI

Postpone the addition of OpenSearch to the CI until a later date.

The tests pass locally with a separate docker container using
instructions documented in docker-compose. Skip the suite on the CI.
---
 .github/workflows/ci.yml                       | 10 ++--------
 .github/workflows/ci_cron.yml                  | 13 +------------
 docker-compose.yml                             | 18 ++++++++++--------
 .../opensearch/instrumentation.rb              |  2 +-
 test/multiverse/lib/multiverse/runner.rb       |  1 -
 test/multiverse/suites/opensearch/Envfile      |  4 ++++
 .../opensearch_instrumentation_test.rb         | 12 +++++++-----
 7 files changed, 25 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9f9caf0c86..6c61b1d601 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -114,7 +114,7 @@ jobs:
     runs-on: ubuntu-22.04
     services:
       elasticsearch7:
-        image: elasticsearch:7.10.1
+        image: elasticsearch:7.16.2
         env:
           discovery.type: single-node
         ports:
@@ -189,7 +189,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        multiverse: [opensearch]
+        multiverse: [agent, ai, background, background_2, database, frameworks, httpclients, httpclients_2, rails, rest]
         ruby-version: [2.4.10, 3.3.4]
 
     steps:
@@ -197,12 +197,6 @@ jobs:
         run: 'git config --global init.defaultBranch main'
       - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # tag v4.1.2
 
-      #   # Runs on OpenSearch 2 by default
-      - name: Set up OpenSearch
-        uses: ankane/setup-opensearch@v1
-        with:
-          config: |
-            http.port: 9300
         # - curl is needed for Curb
         # - xslt is needed for older Nokogiris, RUBY_VERSION < 2.5
         # - sasl is needed for memcached
diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml
index c8820df83c..8a3416ac51 100644
--- a/.github/workflows/ci_cron.yml
+++ b/.github/workflows/ci_cron.yml
@@ -131,7 +131,7 @@ jobs:
         env:
           discovery.type: single-node
         ports:
-          - 9000:9200
+          - 9200:9200
         options: >-
           --health-cmd "curl http://localhost:9200/_cluster/health"
           --health-interval 10s
@@ -149,17 +149,6 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 10
-      opensearch:
-        image: opensearchproject/opensearch:latest
-        env:
-          discovery.type: single-node
-        ports:
-          - 9200:9200
-        options: >-
-          --health-cmd "curl http://localhost:9200/_cluster/health"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 10
       memcached:
         image: memcached:latest
         ports:
diff --git a/docker-compose.yml b/docker-compose.yml
index e9f3f2c4f8..11b70160df 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -19,14 +19,16 @@ services:
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     mem_limit: 1g
-  opensearch:
-    image: opensearchproject/opensearch:1.3.18
-    environment:
-      - discovery.type=single-node
-    ports:
-      - 9200:9200
-      - 9600:9600
-    mem_limit: 1g
+  # this docker-compose service doesn't always work with "docker compose run opensearch"
+  # the tests consistently work with docker run -it -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" --name opensearch-node opensearchproject/opensearch:1.3.18
+  # opensearch:
+  #   image: opensearchproject/opensearch:1.3.18
+  #   environment:
+  #     - discovery.type=single-node
+  #   ports:
+  #     - 9200:9200
+  #     - 9600:9600
+  #   mem_limit: 1g
   mysql:
     image: mysql:5.7
     platform: linux/x86_64
diff --git a/lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb b/lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb
index 20c4da2b30..bae78b2457 100644
--- a/lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb
+++ b/lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb
@@ -48,7 +48,7 @@ def nr_reported_query(query)
     end
 
     def nr_cluster_name
-      return @nr_cluster_name if @nr_cluster_name
+      return @nr_cluster_name if defined?(@nr_cluster_name)
       return if nr_hosts.empty?
 
       NewRelic::Agent.disable_all_tracing do
diff --git a/test/multiverse/lib/multiverse/runner.rb b/test/multiverse/lib/multiverse/runner.rb
index b1270f6869..1224a1c1bd 100644
--- a/test/multiverse/lib/multiverse/runner.rb
+++ b/test/multiverse/lib/multiverse/runner.rb
@@ -108,7 +108,6 @@ def execute_suites(filter, opts)
       'httpclients' => %w[async_http curb excon httpclient],
       'httpclients_2' => %w[typhoeus net_http httprb ethon httpx],
       'infinite_tracing' => %w[infinite_tracing],
-      'opensearch' => %w[opensearch],
 
       'rest' => [] # Specially handled below
     }
diff --git a/test/multiverse/suites/opensearch/Envfile b/test/multiverse/suites/opensearch/Envfile
index e75b48d044..d1ae78fa79 100644
--- a/test/multiverse/suites/opensearch/Envfile
+++ b/test/multiverse/suites/opensearch/Envfile
@@ -2,6 +2,10 @@
 # See https://github.com/newrelic/newrelic-ruby-agent/blob/main/LICENSE for complete details.
 # frozen_string_literal: true
 
+suite_condition('Skip OpenSearch on GitHub Actions, service not currently available') do
+  ENV['CI'] != true
+end
+
 instrumentation_methods :chain, :prepend
 
 OPENSEARCH_VERSIONS = [
diff --git a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
index b5cb41b110..62e07d2541 100644
--- a/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
+++ b/test/multiverse/suites/opensearch/opensearch_instrumentation_test.rb
@@ -182,14 +182,16 @@ def test_segment_error_captured_if_raised
       # NOOP -- allowing span and transaction to notice error
     end
 
-    expected_error_class_name = OpenSearch::Transport::Transport::Error.name
-
-    assert_segment_noticed_error txn, /opensearch$/, expected_error_class_name, /Error/i
-    assert_transaction_noticed_error txn, expected_error_class_name
+    assert_segment_noticed_error txn, /opensearch$/, transport_error_class.name, /Error/i
+    assert_transaction_noticed_error txn, transport_error_class.name
   end
 
   private
 
+  def transport_error_class
+    OpenSearch::Transport::Transport::Error
+  end
+
   def simulate_transport_error
     @client.stub(:search, raise(transport_error_class.new)) do
       @client.search(index: 'my-index', q: 'title')
@@ -197,6 +199,6 @@ def simulate_transport_error
   end
 
   def port
-    9300 # remove once we decide on a port
+    9200 # remove once we decide on a port
   end
 end

From 39a6758815adf07e0feb71981a370f63771f12fb Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 15:14:51 -0700
Subject: [PATCH 36/41] Make 'true' a string in suite condition

---
 test/multiverse/suites/opensearch/Envfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/multiverse/suites/opensearch/Envfile b/test/multiverse/suites/opensearch/Envfile
index d1ae78fa79..28351041cb 100644
--- a/test/multiverse/suites/opensearch/Envfile
+++ b/test/multiverse/suites/opensearch/Envfile
@@ -3,7 +3,7 @@
 # frozen_string_literal: true
 
 suite_condition('Skip OpenSearch on GitHub Actions, service not currently available') do
-  ENV['CI'] != true
+  ENV['CI'] != 'true'
 end
 
 instrumentation_methods :chain, :prepend

From 9f26446ddc5f15761f55c2a76a46cec1a08715d0 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Fri, 9 Aug 2024 15:30:36 -0700
Subject: [PATCH 37/41] Add OpenSearch changelog entry

---
 CHANGELOG.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e5dc1bf16e..7268e81719 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,11 @@
 
 ## dev
 
-Version <dev> updates framework detection, fixes Falcon dispatcher detection, and addresses a JRuby specific concurrency issue.
+Version <dev> adds experimental OpenSearch instrumentation, updates framework detection, fixes Falcon dispatcher detection, and addresses a JRuby specific concurrency issue.
+
+- **Feature: Add experimental OpenSearch instrumentation**
+
+  The agent will now automatically instrument the `opensearch-ruby` gem. We're marking this instrumentation as experimental because more work is needed to fully test it. OpenSearch instrumentation provides telemetry similar to Elasticsearch. Thank you, [@Earlopain](https://github.com/Earlopain) for reporting the issue and [@praveen-ks](https://github.com/praveen-ks) for an initial draft of the instrumentation. [Issue#2228](https://github.com/newrelic/newrelic-ruby-agent/issues/2228) [PR#2796](https://github.com/newrelic/newrelic-ruby-agent/pull/2796)
 
 - **Feature: Improve framework detection accuracy for Grape and Padrino**
 

From 0f77f60645e54392262c8cc94e50162cdab60612 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Mon, 12 Aug 2024 14:14:56 -0700
Subject: [PATCH 38/41] Add a 2.4-compatible version to the gemspec

---
 test/multiverse/suites/opensearch/Envfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/multiverse/suites/opensearch/Envfile b/test/multiverse/suites/opensearch/Envfile
index 28351041cb..7ef8dafe87 100644
--- a/test/multiverse/suites/opensearch/Envfile
+++ b/test/multiverse/suites/opensearch/Envfile
@@ -11,13 +11,12 @@ instrumentation_methods :chain, :prepend
 OPENSEARCH_VERSIONS = [
   [nil, 2.5],
   ['3.4.0', 2.5],
-  ['2.1.0', 2.5]
+  ['2.1.0', 2.4]
 ]
 
 def gem_list(opensearch_version = nil)
   <<~RB
     gem 'opensearch-ruby'#{opensearch_version}
-
   RB
 end
 

From 53b2fe49fabc4d2f10cbe735b36ccdd57d7093a2 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <87386821+kaylareopelle@users.noreply.github.com>
Date: Mon, 12 Aug 2024 14:24:48 -0700
Subject: [PATCH 39/41] Update docker-compose.yml

---
 docker-compose.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 11b70160df..4fd6bdaf0c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -20,7 +20,8 @@ services:
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     mem_limit: 1g
   # this docker-compose service doesn't always work with "docker compose run opensearch"
-  # the tests consistently work with docker run -it -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" --name opensearch-node opensearchproject/opensearch:1.3.18
+  # the tests consistently work with:
+  # docker run -it -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" --name opensearch-node opensearchproject/opensearch:1.3.18
   # opensearch:
   #   image: opensearchproject/opensearch:1.3.18
   #   environment:

From 665d621d6e243e8f46ce150ad542cf325325349f Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Mon, 12 Aug 2024 14:52:52 -0700
Subject: [PATCH 40/41] Revert changes to instrumentation generator

---
 lib/tasks/instrumentation_generator/instrumentation.thor    | 2 +-
 .../templates/dependency_detection.tt                       | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/tasks/instrumentation_generator/instrumentation.thor b/lib/tasks/instrumentation_generator/instrumentation.thor
index 567f6797b3..06305c1074 100644
--- a/lib/tasks/instrumentation_generator/instrumentation.thor
+++ b/lib/tasks/instrumentation_generator/instrumentation.thor
@@ -103,7 +103,7 @@ class Instrumentation < Thor
     <<-CONFIG
         :'instrumentation.#{snake_name}' => {
           :default => 'auto',
-          :documentation_default => 'auto',
+          :documentation_default => 'auto'
           :public => true,
           :type => String,
           :dynamic_name => true,
diff --git a/lib/tasks/instrumentation_generator/templates/dependency_detection.tt b/lib/tasks/instrumentation_generator/templates/dependency_detection.tt
index 2b17a16e80..cb8a3783a5 100644
--- a/lib/tasks/instrumentation_generator/templates/dependency_detection.tt
+++ b/lib/tasks/instrumentation_generator/templates/dependency_detection.tt
@@ -12,16 +12,16 @@ DependencyDetection.defer do
   depends_on do
     # The class that needs to be defined to prepend/chain onto. This can be used
     # to determine whether the library is installed.
-    defined?(<%= @class_name %>)
+    defined?(::<%= @class_name %>)
     # Add any additional requirements to verify whether this instrumentation
     # should be installed
   end
 
   executes do
-    NewRelic::Agent.logger.info('Installing <%= @name.downcase %> instrumentation')
+    ::NewRelic::Agent.logger.info('Installing <%= @name.downcase %> instrumentation')
 
     if use_prepend?
-      prepend_instrument <%= @class_name %>, NewRelic::Agent::Instrumentation::<%= @class_name %>::Prepend
+      prepend_instrument ::<%= @class_name %>, NewRelic::Agent::Instrumentation::<%= @class_name %>::Prepend
     else
       chain_instrument NewRelic::Agent::Instrumentation::<%= @class_name %>::Chain
     end

From 35b1090a4d7f9bf55da1dcb6ec7a5f6d897d00f3 Mon Sep 17 00:00:00 2001
From: Kayla Reopelle <kreopelle@newrelic.com>
Date: Mon, 12 Aug 2024 15:21:55 -0700
Subject: [PATCH 41/41] Update argument defaults to reduce object alloc

Use NewRelic::EMPTY_HASH for the params default
Make _opts default nil
---
 .../agent/instrumentation/opensearch/instrumentation.rb         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb b/lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb
index bae78b2457..564251d420 100644
--- a/lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb
+++ b/lib/new_relic/agent/instrumentation/opensearch/instrumentation.rb
@@ -10,7 +10,7 @@ module OpenSearch
     INSTANCE_METHOD_PATTERN = /:in (?:`|')(?:.+#)?([^']+)'\z/
     INSTRUMENTATION_NAME = NewRelic::Agent.base_name(name)
 
-    def perform_request_with_tracing(_method, _path, params = {}, body = nil, _headers = nil, _opts = {}, &_block)
+    def perform_request_with_tracing(_method, _path, params = NewRelic::EMPTY_HASH, body = nil, _headers = nil, _opts = nil, &_block)
       return yield unless NewRelic::Agent::Tracer.tracing_enabled?
 
       segment = NewRelic::Agent::Tracer.start_datastore_segment(