x-pack/metricbeat/module/openai: Add new module

metricbeat/docs/modules/openai.asciidoc

@@ -0,0 +1,74 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+:modulename: openai
+:edit_url: https://github.com/elastic/beats/edit/main/x-pack/metricbeat/module/openai/_meta/docs.asciidoc
+
+
+[[metricbeat-module-openai]]
+[role="xpack"]
+== openai module
+
+beta[]
+
+This is the openai module.
+
+
+
+:edit_url:
+
+[float]
+=== Example configuration
+
+The openai module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: openai
+  metricsets: ["usage"]
+  enabled: false
+  period: 1h
+
+  # # Project API Keys - Multiple API keys can be specified for different projects
+  # api_keys:
+  # - key: "api_key1"
+  # - key: "api_key2"
+
+  # # API Configuration
+  # ## Base URL for the OpenAI usage API endpoint
+  # api_url: "https://api.openai.com/v1/usage"
+  # ## Custom headers to be included in API requests
+  # headers:
+  # - "k1: v1"
+  # - "k2: v2"
+  ## Rate Limiting Configuration
+  # rate_limit:
+  #   limit: 60 # requests per second
+  #   burst: 5  # burst size
+  # ## Request timeout duration
+  # timeout: 30s
+
+  # # Data Collection Configuration
+  # collection:
+  #   ## Number of days to look back when collecting usage data
+  #   lookback_days: 30
+  #   ## Whether to collect usage data in realtime. Defaults to false as how
+  #   # OpenAI usage data is collected will end up adding duplicate data to ES
+  #   # and also making it harder to do analytics. Best approach is to avoid
+  #   # realtime collection and collect only upto last day (in UTC). So, there's
+  #   # at most 24h delay.
+  #   realtime: false----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-openai-usage,usage>>
+
+include::openai/usage.asciidoc[]
+
+:edit_url!:

metricbeat/docs/modules/openai/usage.asciidoc

@@ -0,0 +1,29 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+:edit_url: https://github.com/elastic/beats/edit/main/x-pack/metricbeat/module/openai/usage/_meta/docs.asciidoc
+
+
+[[metricbeat-metricset-openai-usage]]
+[role="xpack"]
+=== openai usage metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/openai/usage/_meta/docs.asciidoc[]
+
+
+:edit_url:
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-openai,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/openai/usage/_meta/data.json[]
+----
+:edit_url!:

metricbeat/docs/modules_list.asciidoc

@@ -240,6 +240,8 @@ This file is generated! See scripts/mage/docs_collector.go
 |<<metricbeat-metricset-nats-subscriptions,subscriptions>>   
 |<<metricbeat-module-nginx,Nginx>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .1+| .1+|  |<<metricbeat-metricset-nginx-stubstatus,stubstatus>>   
+|<<metricbeat-module-openai,openai>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
+.1+| .1+|  |<<metricbeat-metricset-openai-usage,usage>> beta[]  
 |<<metricbeat-module-openmetrics,Openmetrics>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-openmetrics-collector,collector>> beta[]  
 |<<metricbeat-module-oracle,Oracle>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
@@ -381,6 +383,7 @@ include::modules/munin.asciidoc[]
 include::modules/mysql.asciidoc[]
 include::modules/nats.asciidoc[]
 include::modules/nginx.asciidoc[]
+include::modules/openai.asciidoc[]
 include::modules/openmetrics.asciidoc[]
 include::modules/oracle.asciidoc[]
 include::modules/panw.asciidoc[]

x-pack/metricbeat/metricbeat.reference.yml

@@ -1257,6 +1257,41 @@ metricbeat.modules:
   # Path to server status. Default nginx_status
   server_status_path: "nginx_status"
 
+#-------------------------------- Openai Module --------------------------------
+- module: openai
+  metricsets: ["usage"]
+  enabled: false
+  period: 1h
+
+  # # Project API Keys - Multiple API keys can be specified for different projects
+  # api_keys:
+  # - key: "api_key1"
+  # - key: "api_key2"
+
+  # # API Configuration
+  # ## Base URL for the OpenAI usage API endpoint
+  # api_url: "https://api.openai.com/v1/usage"
+  # ## Custom headers to be included in API requests
+  # headers:
+  # - "k1: v1"
+  # - "k2: v2"
+  ## Rate Limiting Configuration
+  # rate_limit:
+  #   limit: 60 # requests per second
+  #   burst: 5  # burst size
+  # ## Request timeout duration
+  # timeout: 30s
+
+  # # Data Collection Configuration
+  # collection:
+  #   ## Number of days to look back when collecting usage data
+  #   lookback_days: 30
+  #   ## Whether to collect usage data in realtime. Defaults to false as how
+  #   # OpenAI usage data is collected will end up adding duplicate data to ES
+  #   # and also making it harder to do analytics. Best approach is to avoid
+  #   # realtime collection and collect only upto last day (in UTC). So, there's
+  #   # at most 24h delay.
+  #   realtime: false
 #----------------------------- Openmetrics Module -----------------------------
 - module: openmetrics
   metricsets: ['collector']

x-pack/metricbeat/module/openai/_meta/config.yml

@@ -0,0 +1,34 @@
+- module: openai
+  metricsets: ["usage"]
+  enabled: false
+  period: 1h
+
+  # # Project API Keys - Multiple API keys can be specified for different projects
+  # api_keys:
+  # - key: "api_key1"
+  # - key: "api_key2"
+
+  # # API Configuration
+  # ## Base URL for the OpenAI usage API endpoint
+  # api_url: "https://api.openai.com/v1/usage"
+  # ## Custom headers to be included in API requests
+  # headers:
+  # - "k1: v1"
+  # - "k2: v2"
+  ## Rate Limiting Configuration
+  # rate_limit:
+  #   limit: 60 # requests per second
+  #   burst: 5  # burst size
+  # ## Request timeout duration
+  # timeout: 30s
+
+  # # Data Collection Configuration
+  # collection:
+  #   ## Number of days to look back when collecting usage data
+  #   lookback_days: 30
+  #   ## Whether to collect usage data in realtime. Defaults to false as how
+  #   # OpenAI usage data is collected will end up adding duplicate data to ES
+  #   # and also making it harder to do analytics. Best approach is to avoid
+  #   # realtime collection and collect only upto last day (in UTC). So, there's
+  #   # at most 24h delay.
+  #   realtime: false

x-pack/metricbeat/module/openai/_meta/docs.asciidoc

@@ -0,0 +1,2 @@
+This is the openai module.
+

x-pack/metricbeat/module/openai/_meta/fields.yml

@@ -0,0 +1,10 @@
+- key: openai
+  title: "openai"
+  release: beta
+  description: >
+    openai module
+  fields:
+    - name: openai
+      type: group
+      description: >
+      fields:

x-pack/metricbeat/module/openai/doc.go

@@ -0,0 +1,6 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// Package openai is a Metricbeat module that contains MetricSets.
+package openai

x-pack/metricbeat/module/openai/usage/_meta/data.json

@@ -0,0 +1,38 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "event": {
+        "dataset": "openai.usage",
+        "duration": 115000,
+        "module": "openai"
+    },
+    "metricset": {
+        "name": "usage",
+        "period": 10000
+    },
+    "openai": {
+        "usage": {
+            "data": {
+                "aggregation_timestamp": "2024-11-04T05:01:00Z",
+                "api_key_id": null,
+                "api_key_name": null,
+                "api_key_redacted": null,
+                "api_key_type": null,
+                "email": null,
+                "n_cached_context_tokens_total": 0,
+                "n_context_tokens_total": 118,
+                "n_generated_tokens_total": 35,
+                "n_requests": 1,
+                "operation": "completion-realtime",
+                "organization_id": "org-dummy",
+                "organization_name": "Personal",
+                "project_id": null,
+                "project_name": null,
+                "request_type": "",
+                "snapshot_id": "gpt-4o-realtime-preview-2024-10-01"
+            }
+        }
+    },
+    "service": {
+        "type": "openai"
+    }
+}

x-pack/metricbeat/module/openai/usage/_meta/docs.asciidoc

@@ -0,0 +1 @@
+This is the usage metricset of the module openai.