ci: add docker compose file (#3)

Adds o11y_e2e, otelgw, Loki, Grafana and Prometheus to begin the ability
to test the full o11y_e2e pipeline

No actual functionality yet but makes it possible to test the syslog
reciever and other features

docker-compose.yaml

@@ -0,0 +1,91 @@
+version: "3.3"
+
+networks:
+  o11y_e2e:
+
+services:
+  o11y_e2e:
+    build:
+      dockerfile: Dockerfile
+    command:
+      - --log.level=debug
+    networks:
+      - o11y_e2e
+    ports:
+      - 9927:9227
+    restart: always
+    # TODO: needs syslog connector and config file setup to send logs to otelcol
+
+  loki:
+    image: grafana/loki:latest
+    ports:
+      - "3100:3100"
+    networks:
+      - o11y_e2e
+    restart: always
+
+  otelcol:
+  # TODO: not positive this works for loki sending but might
+    image: otel/opentelemetry-collector-contrib:latest
+    volumes:
+      - ./docker/otel-collector-config.yml:/etc/otelcol/config.yaml
+    ports:
+      - "4317:4317" # gRPC receiver port
+      - "55681:55681" # OTLP HTTP receiver port
+      - "1514:1514" # Syslog receiver port
+    networks:
+      - o11y_e2e
+    depends_on:
+      - loki
+    restart: always
+
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - ./docker/prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+    networks:
+      - o11y_e2e
+    depends_on:
+      - o11y_e2e
+    restart: always
+
+  grafana:
+      environment:
+        - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
+        - GF_AUTH_ANONYMOUS_ENABLED=true
+        - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      entrypoint:
+        - sh
+        - -euc
+        - |
+          mkdir -p /etc/grafana/provisioning/datasources
+          cat <<EOF > /etc/grafana/provisioning/datasources/ds.yaml
+          apiVersion: 1
+          datasources:
+          - name: Loki
+            type: loki
+            access: proxy
+            orgId: 1
+            url: http://loki:3100
+            basicAuth: false
+            isDefault: true
+            version: 1
+            editable: true
+          - name: prometheus
+            type: prometheus
+            access: proxy
+            orgId: 1
+            url: http://prometheus:9090
+            basicAuth: false
+            isDefault: false
+            version: 1
+            editable: true
+          EOF
+          /run.sh
+      image: grafana/grafana:latest
+      ports:
+        - "3000:3000"
+      networks:
+      - o11y_e2e

docker/otel-collector-config.yml

@@ -0,0 +1,106 @@
+exporters:
+  file/rotation:
+    format: json
+    path: /var/log/otelcol/otel.json
+    rotation:
+      localtime: false
+      max_backups: 10
+      max_days: 3
+      max_megabytes: 100
+  loki/prod:
+    endpoint: http://loki:3100/loki/api/v1/push
+    retry_on_failure:
+      initial_interval: 5s
+      max_elapsed_time: 30s
+      max_interval: 15s
+    sending_queue:
+      enabled: true
+      num_consumers: 50
+      queue_size: 5000
+      storage: file_storage
+    tls:
+      insecure: true
+extensions:
+  file_storage:
+    compaction:
+      directory: /var/lib/otelcol
+      max_transaction_size: 65536
+      on_rebound: true
+      on_start: true
+    directory: /var/lib/otelcol
+    timeout: 1s
+  health_check:
+    endpoint: :13133
+    path: /health/status
+    tls:
+      insecure: true
+  memory_ballast:
+    size_in_percentage: 50
+
+processors:
+  batch:
+    send_batch_max_size: 0
+    send_batch_size: 8192
+    timeout: 200ms
+  filter/bad_clients:
+    error_mode: ignore
+    logs:
+      log_record: []
+  memory_limiter:
+    check_interval: 5s
+    limit_percentage: 75
+    spike_limit_percentage: 10
+  resource:
+    attributes:
+    - action: insert
+      key: loki.resource.labels
+      value: instance, datacenter, filename, path, unit, environment, component, cluster, logicalcluster
+    - action: insert
+      key: loki.format
+      value: raw
+  transform/timestamps:
+    error_mode: ignore
+    log_statements:
+    - context: log
+      statements:
+      - set(time, Now()) where time > Now()
+
+receivers:
+  filelog:
+    include:
+    - /var/log/syslog
+    include_file_name: false
+    include_file_path: true
+    operators:
+    - from: attributes["log.file.path"]
+      to: resource["path"]
+      type: move
+    resource:
+      component: otelgw
+      datacenter: testdc
+      environment: production
+      instance: otelcol
+    start_at: end
+    storage: file_storage
+service:
+  extensions:
+  - health_check
+  - file_storage
+  - memory_ballast
+  pipelines:
+    logs:
+      exporters:
+      - loki/prod
+      processors:
+      - memory_limiter
+      - filter/bad_clients
+      - transform/timestamps
+      - resource
+      - batch
+      receivers:
+      - otlp
+      - filelog
+
+  telemetry:
+    metrics:
+      address: localhost:8888

docker/prometheus.yml

@@ -0,0 +1,10 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'o11y_e2e'
+    metrics_path: '/metrics'
+    scrape_interval: 15s
+    scrape_timeout: 15s
+    static_configs:
+      - targets: ["o11y_e2e:9927"]