Add elasticsearch instrumentation (#1525)

Introduces instrumentation for the elasticsearch gem versions 7.x and 8.x and add an elasticsearch multiverse suite to the database multiverse group. The Mongo nosql obfuscator has been moved out of mongo and is now a general NoSql obfuscator used by both mongo and elasticsearch. Instrumentation generator thor task is updated with fixes that were found when using the generator to create the elasticsearch files. Elasticsearch 7 and 8 are now included in both the ci.yml and ci_cron.yml workflows to support the elasticsearch multiverse suite running on the CI. The docker compose file has also been updated to run elasticsearch 7 on port 9200 and elasticsearch 8 on port 9250 for local testing.
newrelic · Oct 14, 2022 · 169f8e7 · 169f8e7
1 parent 4586ad5
commit 169f8e7
Show file tree

Hide file tree

Showing 33 changed files with 795 additions and 221 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -101,6 +101,42 @@ jobs:
     needs: run_rubocop
     runs-on: ubuntu-22.04
     services:
+      elasticsearch7:
+        image: elasticsearch:7.16.2
+        env:
+          discovery.type: single-node
+        ports:
+          - 9200:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      elasticsearch8:
+        image: elasticsearch:8.4.2
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+        ports:
+          - 9250:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      memcached:
+        image: memcached:latest
+        ports:
+          - 11211:11211
+        options: >-
+          --health-cmd "timeout 5 bash -c 'cat < /dev/null > /dev/udp/127.0.0.1/11211'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      mongodb:
+        image: ${{ contains(fromJson('["2.2.10", "2.3.8", "2.4.10"]'), matrix.ruby-version) && 'mongo:5.0.11' || 'mongo:latest' }}
+        ports:
+          - 27017:27017
       mysql:
         image: mysql:5.7
         env:
@@ -120,19 +156,6 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-      redis:
-        image: redis
-        ports:
-          - 6379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      mongodb:
-        image: ${{ contains(fromJson('["2.2.10", "2.3.8", "2.4.10"]'), matrix.ruby-version) && 'mongo:5.0.11' || 'mongo:latest' }}
-        ports:
-          - 27017:27017
       rabbitmq:
         image: rabbitmq:latest
         ports:
@@ -142,12 +165,12 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-      memcached:
-        image: memcached:latest
+      redis:
+        image: redis
         ports:
-          - 11211:11211
+          - 6379:6379
         options: >-
-          --health-cmd "timeout 5 bash -c 'cat < /dev/null > /dev/udp/127.0.0.1/11211'"
+          --health-cmd "redis-cli ping"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
@@ -282,30 +305,53 @@ jobs:
     if: ${{ needs.check_jruby_multiverse.outputs.run_job == 'true' }}
     runs-on: ubuntu-22.04
     services:
-      mysql:
-        image: mysql:5.7
+      elasticsearch7:
+        image: elasticsearch:7.16.2
         env:
-          MYSQL_ALLOW_EMPTY_PASSWORD: yes
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+          discovery.type: single-node
         ports:
-          - "3306:3306"
-      postgres:
-        image: postgres:latest
+          - 9200:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      elasticsearch8:
+        image: elasticsearch:8.4.2
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
         ports:
-          - 5432:5432
-      redis:
-        image: redis
+          - 9250:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      memcached:
+        image: memcached:latest
         ports:
-          - 6379:6379
+          - 11211:11211
         options: >-
-          --health-cmd "redis-cli ping"
+          --health-cmd "timeout 5 bash -c 'cat < /dev/null > /dev/udp/127.0.0.1/11211'"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
       mongodb:
         image: mongo:5.0.11
         ports:
           - 27017:27017
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+        ports:
+          - "3306:3306"
+      postgres:
+        image: postgres:latest
+        ports:
+          - 5432:5432
       rabbitmq:
         image: rabbitmq:latest
         ports:
@@ -315,12 +361,12 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-      memcached:
-        image: memcached:latest
+      redis:
+        image: redis
         ports:
-          - 11211:11211
+          - 6379:6379
         options: >-
-          --health-cmd "timeout 5 bash -c 'cat < /dev/null > /dev/udp/127.0.0.1/11211'"
+          --health-cmd "redis-cli ping"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5

diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml
@@ -123,6 +123,42 @@ jobs:
     needs: run_rubocop
     runs-on: ubuntu-22.04
     services:
+      elasticsearch7:
+        image: elasticsearch:7.16.2
+        env:
+          discovery.type: single-node
+        ports:
+          - 9200:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      elasticsearch8:
+        image: elasticsearch:8.4.2
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+        ports:
+          - 9250:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      memcached:
+        image: memcached:latest
+        ports:
+          - 11211:11211
+        options: >-
+          --health-cmd "timeout 5 bash -c 'cat < /dev/null > /dev/udp/127.0.0.1/11211'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      mongodb:
+        image: ${{ contains(fromJson('["2.2.10", "2.3.8", "2.4.10"]'), matrix.ruby-version) && 'mongo:5.0.11' || 'mongo:latest' }}
+        ports:
+          - 27017:27017
       mysql:
         image: mysql:5.7
         env:
@@ -142,19 +178,6 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-      redis:
-        image: redis
-        ports:
-          - 6379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      mongodb:
-        image: ${{ contains(fromJson('["2.2.10", "2.3.8", "2.4.10"]'), matrix.ruby-version) && 'mongo:5.0.11' || 'mongo:latest' }}
-        ports:
-          - 27017:27017
       rabbitmq:
         image: rabbitmq:latest
         ports:
@@ -164,12 +187,12 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-      memcached:
-        image: memcached:latest
+      redis:
+        image: redis
         ports:
-          - 11211:11211
+          - 6379:6379
         options: >-
-          --health-cmd "timeout 5 bash -c 'cat < /dev/null > /dev/udp/127.0.0.1/11211'"
+          --health-cmd "redis-cli ping"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
@@ -271,30 +294,53 @@ jobs:
     needs: run_rubocop
     runs-on: ubuntu-22.04
     services:
-      mysql:
-        image: mysql:5.7
+      elasticsearch7:
+        image: elasticsearch:7.16.2
         env:
-          MYSQL_ALLOW_EMPTY_PASSWORD: yes
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+          discovery.type: single-node
         ports:
-          - "3306:3306"
-      postgres:
-        image: postgres:latest
+          - 9200:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      elasticsearch8:
+        image: elasticsearch:8.4.2
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
         ports:
-          - 5432:5432
-      redis:
-        image: redis
+          - 9250:9200
+        options: >-
+          --health-cmd "curl http://localhost:9200/_cluster/health"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+      memcached:
+        image: memcached:latest
         ports:
-          - 6379:6379
+          - 11211:11211
         options: >-
-          --health-cmd "redis-cli ping"
+          --health-cmd "timeout 5 bash -c 'cat < /dev/null > /dev/udp/127.0.0.1/11211'"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
       mongodb:
         image: mongo:5.0.11
         ports:
           - 27017:27017
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+        ports:
+          - "3306:3306"
+      postgres:
+        image: postgres:latest
+        ports:
+          - 5432:5432
       rabbitmq:
         image: rabbitmq:latest
         ports:
@@ -304,12 +350,12 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-      memcached:
-        image: memcached:latest
+      redis:
+        image: redis
         ports:
-          - 11211:11211
+          - 6379:6379
         options: >-
-          --health-cmd "timeout 5 bash -c 'cat < /dev/null > /dev/udp/127.0.0.1/11211'"
+          --health-cmd "redis-cli ping"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5

diff --git a/.github/workflows/scripts/rubygems-publish.rb b/.github/workflows/scripts/rubygems-publish.rb
@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 gem_name = ARGV[0]
 raise "gem name sans version must be supplied" if gem_name.to_s == ""
 
@@ -21,7 +22,7 @@
 if $?.to_i.zero?
   puts "#{gem_filename} successfully pushed to rubygems.org!"
 else
-  if result =~ /Repushing of gem versions is not allowed/
+  if result.include?('Repushing of gem versions is not allowed')
     puts "Pushing #{gem_filename} skipped because this version is already published to rubygems.org!"
     exit 0
   else

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,7 +2,17 @@
 
   ## v8.12.0
 
-  Version 8.12.0 of the agent delivers some valuable code cleanup, increases the default number of recorded Custom Events, and announces the deprecation of Ruby 2.3.
+  Version 8.12.0 of the agent delivers new Elasticsearch instrumentation, some valuable code cleanup, increases the default number of recorded Custom Events, and announces the deprecation of Ruby 2.3.
+
+  * **Support for Elasticsearch instrumentation**
+
+    This release adds support to automatically instrument the [elasticsearch](https://rubygems.org/gems/elasticsearch) gem. Versions 7.x and 8.x are supported. [PR#1525](https://github.com/newrelic/newrelic-ruby-agent/pull/1525)
+
+    | Configuration name | Default | Behavior |
+    | ----------- | ----------- |----------- |
+    | `instrumentation.elasticsearch` | auto | Controls auto-instrumentation of the elasticsearch library at start up. May be one of `auto`, `prepend`, `chain`, `disabled`. |
+    | `elasticsearch.capture_queries` | true | If `true`, the agent captures Elasticsearch queries in transaction traces. |
+    | `elasticsearch.obfuscate_queries` | true |  If `true`, the agent obfuscates Elasticsearch queries in transaction traces. |
 
   * **Cleanup: Remove orphaned code from unit tests**
 
@@ -18,6 +28,7 @@
 
     Ruby 2.3 reached end of life on March 31, 2019. The Ruby agent has deprecated support for Ruby 2.3 and will make breaking changes for this version in its next major release, v9.0.0 (release date not yet planned). All 8.x.x versions of the agent will remain compatible with Ruby 2.3.
 
+
   ## v8.11.0
 
   Version 8.11.0 of the agent updates the `newrelic deployments` command to work with API keys issued to newer accounts, fixes a memory leak in the instrumentation of Curb error handling, further preps for Ruby 3.2.0 support, and includes several community member driven cleanup and improvement efforts. Thank you to everyone involved!