[Obs] Fix bug a but with --ephemeral-project-ids and add metadata to …

…Nginx Proxy logs (elastic#176553)

## Summary

This PR fixes a bug with the `--ephermal-project-ids` option. Instead of
using now it should use the timestamp of the first event passed to it.
This also adds `http.response.status_code` and `http.response.bytes` to
the Nginx Proxy logs.

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/message_processor/index.ts

@@ -12,6 +12,7 @@ import { badHost } from './lib/events/bad_host';
 import { weightedSample } from '../common/weighted_sample';
 
 import { Doc, GeneratorFunction, EventFunction, EventTemplate } from '../../../types';
+import { addEphemeralProjectId } from '../../../lib/add_ephemeral_project_id';
 
 let firstRun = true;
 
@@ -31,7 +32,7 @@ function getTemplate(name: string) {
   return GOOD_EVENT_TEMPLATES;
 }
 
-export const generateEvent: GeneratorFunction = (_config, schedule, _index, timestamp) => {
+export const generateEvent: GeneratorFunction = (config, schedule, _index, timestamp) => {
   let startupEvents: Doc[] = [];
   if (firstRun) {
     firstRun = false;
@@ -40,7 +41,10 @@ export const generateEvent: GeneratorFunction = (_config, schedule, _index, time
 
   const template = getTemplate(schedule.template);
   const fn = weightedSample(template) as EventFunction;
-  const events = fn(schedule, timestamp).flat();
+  const events = addEphemeralProjectId(
+    config.indexing.ephemeralProjectIds || 0,
+    fn(schedule, timestamp).flat()
+  );
 
   return [...startupEvents, ...events];
 };

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/fields/subset.yml

@@ -7,6 +7,12 @@ fields:
     fields:
       level: {}
       logger: {}
+  http:
+    fields:
+      response:
+        fields:
+          status_code: {}
+          bytes: {}
   host:
     fields:
       name: {}

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/beats/fields.ecs.yml

@@ -72,6 +72,26 @@
         It can contain what `hostname` returns on Unix systems, the fully qualified
         domain name, or a name specified by the user. The sender decides which value
         to use.'
+  - name: http
+    title: HTTP
+    group: 2
+    description: Fields related to HTTP activity. Use the `url` field set to store
+      the url of the request.
+    type: group
+    default_field: true
+    fields:
+    - name: response.bytes
+      level: extended
+      type: long
+      format: bytes
+      description: Total size in bytes of the response (body and headers).
+      example: 1437
+    - name: response.status_code
+      level: extended
+      type: long
+      format: string
+      description: HTTP response status code.
+      example: 404
   - name: log
     title: Log
     group: 2

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/csv/fields.csv

@@ -4,5 +4,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description
 8.0.0,true,base,message,match_only_text,core,,Hello World,Log message optimized for viewing in a log viewer.
 8.0.0,true,base,tags,keyword,core,array,"[""production"", ""env2""]",List of keywords used to tag each event.
 8.0.0,true,host,host.name,keyword,core,,,Name of the host.
+8.0.0,true,http,http.response.bytes,long,extended,,1437,Total size in bytes of the response (body and headers).
+8.0.0,true,http,http.response.status_code,long,extended,,404,HTTP response status code.
 8.0.0,true,log,log.level,keyword,core,,error,Log level of the log event.
 8.0.0,true,log,log.logger,keyword,core,,org.elasticsearch.bootstrap.Bootstrap,Name of the logger.

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/ecs/ecs_flat.yml

@@ -30,6 +30,28 @@ host.name:
   normalize: []
   short: Name of the host.
   type: keyword
+http.response.bytes:
+  dashed_name: http-response-bytes
+  description: Total size in bytes of the response (body and headers).
+  example: 1437
+  flat_name: http.response.bytes
+  format: bytes
+  level: extended
+  name: response.bytes
+  normalize: []
+  short: Total size in bytes of the response (body and headers).
+  type: long
+http.response.status_code:
+  dashed_name: http-response-status-code
+  description: HTTP response status code.
+  example: 404
+  flat_name: http.response.status_code
+  format: string
+  level: extended
+  name: response.status_code
+  normalize: []
+  short: HTTP response status code.
+  type: long
 labels:
   dashed_name: labels
   description: 'Custom key/value pairs.

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/ecs/ecs_nested.yml

@@ -109,6 +109,38 @@ host:
   short: Fields describing the relevant computing instance.
   title: Host
   type: group
+http:
+  description: Fields related to HTTP activity. Use the `url` field set to store the
+    url of the request.
+  fields:
+    http.response.bytes:
+      dashed_name: http-response-bytes
+      description: Total size in bytes of the response (body and headers).
+      example: 1437
+      flat_name: http.response.bytes
+      format: bytes
+      level: extended
+      name: response.bytes
+      normalize: []
+      short: Total size in bytes of the response (body and headers).
+      type: long
+    http.response.status_code:
+      dashed_name: http-response-status-code
+      description: HTTP response status code.
+      example: 404
+      flat_name: http.response.status_code
+      format: string
+      level: extended
+      name: response.status_code
+      normalize: []
+      short: HTTP response status code.
+      type: long
+  group: 2
+  name: http
+  prefix: http.
+  short: Fields describing an HTTP request.
+  title: HTTP
+  type: group
 log:
   description: 'Details about the event''s logging mechanism or logging transport.
 

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/ecs/subset/nginx_proxy/ecs_flat.yml

@@ -30,6 +30,28 @@ host.name:
   normalize: []
   short: Name of the host.
   type: keyword
+http.response.bytes:
+  dashed_name: http-response-bytes
+  description: Total size in bytes of the response (body and headers).
+  example: 1437
+  flat_name: http.response.bytes
+  format: bytes
+  level: extended
+  name: response.bytes
+  normalize: []
+  short: Total size in bytes of the response (body and headers).
+  type: long
+http.response.status_code:
+  dashed_name: http-response-status-code
+  description: HTTP response status code.
+  example: 404
+  flat_name: http.response.status_code
+  format: string
+  level: extended
+  name: response.status_code
+  normalize: []
+  short: HTTP response status code.
+  type: long
 labels:
   dashed_name: labels
   description: 'Custom key/value pairs.

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/ecs/subset/nginx_proxy/ecs_nested.yml

@@ -109,6 +109,38 @@ host:
   short: Fields describing the relevant computing instance.
   title: Host
   type: group
+http:
+  description: Fields related to HTTP activity. Use the `url` field set to store the
+    url of the request.
+  fields:
+    http.response.bytes:
+      dashed_name: http-response-bytes
+      description: Total size in bytes of the response (body and headers).
+      example: 1437
+      flat_name: http.response.bytes
+      format: bytes
+      level: extended
+      name: response.bytes
+      normalize: []
+      short: Total size in bytes of the response (body and headers).
+      type: long
+    http.response.status_code:
+      dashed_name: http-response-status-code
+      description: HTTP response status code.
+      example: 404
+      flat_name: http.response.status_code
+      format: string
+      level: extended
+      name: response.status_code
+      normalize: []
+      short: HTTP response status code.
+      type: long
+  group: 2
+  name: http
+  prefix: http.
+  short: Fields describing an HTTP request.
+  title: HTTP
+  type: group
 log:
   description: 'Details about the event''s logging mechanism or logging transport.
 

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/elasticsearch/composable/component/http.json

@@ -0,0 +1,26 @@
+{
+  "_meta": {
+    "documentation": "https://www.elastic.co/guide/en/ecs/current/ecs-http.html",
+    "ecs_version": "8.0.0"
+  },
+  "template": {
+    "mappings": {
+      "properties": {
+        "http": {
+          "properties": {
+            "response": {
+              "properties": {
+                "bytes": {
+                  "type": "long"
+                },
+                "status_code": {
+                  "type": "long"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/elasticsearch/composable/template.json

@@ -6,6 +6,7 @@
   "composed_of": [
     "ecs_8.0.0_base",
     "ecs_8.0.0_log",
+    "ecs_8.0.0_http",
     "ecs_8.0.0_host"
   ],
   "index_patterns": [

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/ecs/generated/elasticsearch/legacy/template.json

@@ -36,6 +36,20 @@
           }
         }
       },
+      "http": {
+        "properties": {
+          "response": {
+            "properties": {
+              "bytes": {
+                "type": "long"
+              },
+              "status_code": {
+                "type": "long"
+              }
+            }
+          }
+        }
+      },
       "labels": {
         "type": "object"
       },

x-pack/packages/kbn-data-forge/src/data_sources/fake_stack/nginx_proxy/lib/events/create_nginx_log.ts

@@ -37,6 +37,7 @@ export const createNginxLog = (
       )} - ${userId} ${domain} to: ${hostWithPort}: "${method} ${path} HTTP/1.1" ${statusCode} ${bytes} "${url}" "${userAgent}"`,
       log: { level: 'INFO', logger: NGINX_PROXY },
       host: { name: host },
+      http: { response: { status_code: statusCode, bytes } },
     },
   ];
 };

x-pack/packages/kbn-data-forge/src/lib/add_ephemeral_project_id.ts

@@ -6,6 +6,7 @@
  */
 
 import { random, times } from 'lodash';
+import moment from 'moment';
 import { v4 } from 'uuid';
 import { Doc } from '../types';
 
@@ -18,7 +19,7 @@ export function addEphemeralProjectId(totalProjectIds: number, events: Doc[]) {
     return events;
   }
 
-  const now = Date.now();
+  const now = (events[0] && moment(events[0]['@timestamp']).valueOf()) || Date.now();
   const workingIds = projectIds.filter(([_id, expirationDate]) => expirationDate > now);
   const numberOfIdsToCreate = totalProjectIds - workingIds.length;