Smart Gateway Performance Enhancement Work (#60)

* Changes for sg2 * Added new containers images for sg2 and bridge * Added socket-dir for them to communicate * Added more mode-specific code to deployment template * Removed now-unused metrics-configmap * Use best bridge args from testing * Updated CSV and container values for new SG * Updated core image name & scorecard * fixed "replaced" line in csv * Adjust template to support ceilometer metrics too * Update description in CSV (#53) Update the CSV description so that we could pass upstream Community Operator requirements should we ever publish there. * amqpDataSource should be amqp_data_source * Misplaced conditionals broke volume * Setup SG3 CI system (#59) * Use new connection configuration for metrics (#48) AMQP1Connections parameter should be used instead of nonflexible AMQP1Url. Depends-On: infrawatch/smart-gateway#83 * Bump default CSV to 2.0.0 * Fix file rename * Lock operator-courier to v2.1.7 Lock operator-courier to 2.1.7 until we can figure out what is wrong with our CSV/CRD setup or until the operator-courier issue noted in the related issue is resolved. Related: infrawatch/service-telemetry-operator#108 * Update roles/smartgateway/templates/ceilometer-metrics-configmap.yaml.j2 Co-authored-by: Chris Sibbitt <[email protected]> Co-authored-by: Martin Mágr <[email protected]> Co-authored-by: Chris Sibbitt <[email protected]> Co-authored-by: Chris Sibbitt <[email protected]> Co-authored-by: Martin Mágr <[email protected]>
infrawatch · Jul 13, 2020 · 6d8dfb7 · 6d8dfb7
1 parent 294eccd
commit 6d8dfb7
Show file tree

Hide file tree

Showing 8 changed files with 455 additions and 14 deletions.
diff --git a/.osdk-scorecard.yaml b/.osdk-scorecard.yaml
@@ -5,8 +5,8 @@ scorecard:
     - basic:
         cr-manifest:
           - "deploy/crds/smartgateway.infra.watch_v2alpha1_smartgateway.metrics_cr.yaml"
-        csv-path: "deploy/olm-catalog/smart-gateway-operator/1.0.2/smart-gateway-operator.v1.0.2.clusterserviceversion.yaml"
+        csv-path: "deploy/olm-catalog/smart-gateway-operator/2.0.0/smart-gateway-operator.v2.0.0.clusterserviceversion.yaml"
     - olm:
         cr-manifest:
           - "deploy/crds/smartgateway.infra.watch_v2alpha1_smartgateway.metrics_cr.yaml"
-        csv-path: "deploy/olm-catalog/smart-gateway-operator/1.0.2/smart-gateway-operator.v1.0.2.clusterserviceversion.yaml"
+        csv-path: "deploy/olm-catalog/smart-gateway-operator/2.0.0/smart-gateway-operator.v2.0.0.clusterserviceversion.yaml"
diff --git a/.travis.yml b/.travis.yml
@@ -7,7 +7,7 @@ language: python
 git:
   depth: 1
 install:
-- pip install operator-courier
+- pip install operator-courier==2.1.7
 - pip install ansible-lint
 script:
 - operator-courier verify --ui_validate_io deploy/olm-catalog/smart-gateway-operator

diff --git a/...log/smart-gateway-operator/2.0.0/smart-gateway-operator.v2.0.0.clusterserviceversion.yaml b/...log/smart-gateway-operator/2.0.0/smart-gateway-operator.v2.0.0.clusterserviceversion.yaml
@@ -0,0 +1,294 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: ClusterServiceVersion
+metadata:
+  annotations:
+    alm-examples: |-
+      [
+        {
+          "apiVersion": "smartgateway.infra.watch/v2alpha1",
+          "kind": "SmartGateway",
+          "metadata": {
+            "name": "cloud1-events"
+          },
+          "spec": {
+            "debug": false,
+            "prefetch": 0,
+            "resetIndex": false,
+            "serviceType": "events",
+            "size": 1
+          }
+        },
+        {
+          "apiVersion": "smartgateway.infra.watch/v2alpha1",
+          "kind": "SmartGateway",
+          "metadata": {
+            "name": "cloud1-metrics"
+          },
+          "spec": {
+            "debug": false,
+            "prefetch": 15000,
+            "serviceType": "metrics",
+            "size": 1,
+            "useTimestamp": true
+          }
+        }
+      ]
+    capabilities: Basic Install
+    categories: Monitoring
+    certified: "false"
+    containerImage: quay.io/infrawatch/smart-gateway-operator:v2.0.0
+    createdAt: "2019-11-14T14:49:00Z"
+    description: Operator for managing the Smart Gateway Custom Resources, resulting in deployments of the Smart Gateway.
+    repository: https://github.com/infrawatch/smart-gateway-operator
+    support: Red Hat (CloudOps)
+  name: smart-gateway-operator.v2.0.0
+  namespace: placeholder
+spec:
+  apiservicedefinitions: {}
+  customresourcedefinitions:
+    owned:
+    - description: Instance of the Smart Gateway
+      displayName: SmartGateway
+      kind: SmartGateway
+      name: smartgateways.smartgateway.infra.watch
+      resources:
+      - kind: Pod
+        name: ""
+        version: v1
+      - kind: Deployments
+        name: ""
+        version: v1
+      - kind: ReplicaSets
+        name: ""
+        version: v1
+      - kind: Services
+        name: ""
+        version: v1
+      - kind: ConfigMaps
+        name: ""
+        version: v1
+      - kind: SmartGateways
+        name: ""
+        version: v2alpha1
+      specDescriptors:
+      - description: Number of Smart Gateways to deploy
+        displayName: Size
+        path: size
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
+      - description: Location of the AMQP endpoint to connect the Smart Gateway to
+        displayName: AMQP URL
+        path: amqpUrl
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - description: Container image path
+        displayName: Container image path
+        path: containerImagePath
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:text
+      - description: Number of messages that we can prefetch from AMQP 1.x. By enabling
+          prefetching, the smart gateway won't need to request every message individually
+          from the AMQP bus, resulting in a round trip for every request between sender
+          and receiver. To avoid the round trip for every message, the use of prefetch
+          can be used to allow the receiver to request messages be sent in anticipation
+          of them being sent to us.
+        displayName: Prefetch
+        path: prefetch
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:number
+      - description: Use the source timestamp (time when data was collected) rather
+          than let Prometheus write when the data was scraped for that metric.
+        displayName: Use Timestamp
+        path: useTimestamp
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:booleanSwitch
+      - description: Smart Gateway Service Type
+        displayName: Service Type
+        path: serviceType
+        x-descriptors:
+        - urn:alm:descriptor:com.tectonic.ui:select:metrics
+        - urn:alm:descriptor:com.tectonic.ui:select:events
+      - description: Enable additional debugging information to console output
+        displayName: Enable debugging
+        path: debug
+        x-descriptors:
+        - urn:alm:descriptor:tectonic.ui:booleanSwitch
+      statusDescriptors:
+      - description: Conditions provided by deployment
+        displayName: Conditions
+        path: conditions
+        x-descriptors:
+        - urn:alm:descriptor:io.kubernetes.conditions
+      version: v2alpha1
+  description: |-
+    # Smart Gateway for Service Telemetry Framework
+
+    The Smart Gateway is middleware for OpenShift that takes metrics and events
+    data streams from an AMQP 1.x message bus, and provides a method to having
+    that data stream stored within Prometheus or ElasticSearch.
+
+    ## Core capabilities
+
+    The Smart Gateway provides two modes:
+
+    * metrics: provides an HTTP scrape endpoint for Prometheus
+    * events: writes events directly to an ElasticSearch endpoint
+
+    ## Operator features
+
+    * **Two Service Types** - Supports the `metrics` and `events` service types which defines the type of Smart Gateway to be deployed
+
+    * **Configuration** - Configuration of the Smart Gateway to allow for it to be connected to an AMQP 1.x bus and pointed at the appropriate storage mechanism based on service type.
+
+    ## Before getting started
+
+    1. Ensure an AMQP 1.x bus has been setup (such as AMQ Interconnect)
+    2. If the Smart Gateway is running in events mode, be sure the ElasticSearch Operator and ElasticSearch instance have been pre-deployed.
+  displayName: Smart Gateway Operator
+  icon:
+  - base64data: PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9Im5vIj8+CjxzdmcKICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50cy8xLjEvIgogICB4bWxuczpjYz0iaHR0cDovL2NyZWF0aXZlY29tbW9ucy5vcmcvbnMjIgogICB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiCiAgIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciCiAgIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIKICAgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFREL3NvZGlwb2RpLTAuZHRkIgogICB4bWxuczppbmtzY2FwZT0iaHR0cDovL3d3dy5pbmtzY2FwZS5vcmcvbmFtZXNwYWNlcy9pbmtzY2FwZSIKICAgaW5rc2NhcGU6dmVyc2lvbj0iMS4wYmV0YTIgKHVua25vd24pIgogICBoZWlnaHQ9IjEyOCIKICAgd2lkdGg9IjEyOCIKICAgc29kaXBvZGk6ZG9jbmFtZT0iSWNvbi1SZWRfSGF0LU9iamVjdHMtTGlnaHRfQnVsYi1BLUJsYWNrLVJHQi5zdmciCiAgIHZlcnNpb249IjEuMSIKICAgdmlld0JveD0iMCAwIDEyOCAxMjgiCiAgIGlkPSJJY29ucyI+CiAgPG1ldGFkYXRhCiAgICAgaWQ9Im1ldGFkYXRhMjYyIj4KICAgIDxyZGY6UkRGPgogICAgICA8Y2M6V29yawogICAgICAgICByZGY6YWJvdXQ9IiI+CiAgICAgICAgPGRjOmZvcm1hdD5pbWFnZS9zdmcreG1sPC9kYzpmb3JtYXQ+CiAgICAgICAgPGRjOnR5cGUKICAgICAgICAgICByZGY6cmVzb3VyY2U9Imh0dHA6Ly9wdXJsLm9yZy9kYy9kY21pdHlwZS9TdGlsbEltYWdlIiAvPgogICAgICAgIDxkYzp0aXRsZT5JY29uLVJlZF9IYXQtT2JqZWN0cy1MaWdodF9CdWxiLUEtQmxhY2stUkdCPC9kYzp0aXRsZT4KICAgICAgPC9jYzpXb3JrPgogICAgPC9yZGY6UkRGPgogIDwvbWV0YWRhdGE+CiAgPGRlZnMKICAgICBpZD0iZGVmczI2MCIgLz4KICA8c29kaXBvZGk6bmFtZWR2aWV3CiAgICAgaW5rc2NhcGU6Y3VycmVudC1sYXllcj0iSWNvbnMiCiAgICAgaW5rc2NhcGU6d2luZG93LW1heGltaXplZD0iMCIKICAgICBpbmtzY2FwZTp3aW5kb3cteT0iMTgiCiAgICAgaW5rc2NhcGU6d2luZG93LXg9IjM4NDAiCiAgICAgaW5rc2NhcGU6Y3k9IjQ5LjgzNjQ4OSIKICAgICBpbmtzY2FwZTpjeD0iMTAzLjgxNzM4IgogICAgIGlua3NjYXBlOnpvb209IjQuMzgwMTMzNyIKICAgICB2aWV3Ym94LXk9IjAiCiAgICAgc2NhbGUteD0iMSIKICAgICBzaG93Z3JpZD0iZmFsc2UiCiAgICAgaWQ9Im5hbWVkdmlldzI1OCIKICAgICBpbmtzY2FwZTp3aW5kb3ctaGVpZ2h0PSIxMDYyIgogICAgIGlua3NjYXBlOndpbmRvdy13aWR0aD0iMTkyMCIKICAgICBpbmtzY2FwZTpwYWdlc2hhZG93PSIyIgogICAgIGlua3NjYXBlOnBhZ2VvcGFjaXR5PSIwIgogICAgIGd1aWRldG9sZXJhbmNlPSIxMCIKICAgICBncmlkdG9sZXJhbmNlPSIxMCIKICAgICBvYmplY3R0b2xlcmFuY2U9IjEwIgogICAgIGJvcmRlcm9wYWNpdHk9IjEiCiAgICAgaW5rc2NhcGU6ZG9jdW1lbnQtcm90YXRpb249IjAiCiAgICAgYm9yZGVyY29sb3I9IiM2NjY2NjYiCiAgICAgcGFnZWNvbG9yPSIjZmZmZmZmIiAvPgogIDx0aXRsZQogICAgIGlkPSJ0aXRsZTI1MSI+SWNvbi1SZWRfSGF0LU9iamVjdHMtTGlnaHRfQnVsYi1BLUJsYWNrLVJHQjwvdGl0bGU+CiAgPHBhdGgKICAgICBzdHlsZT0ic3Ryb2tlLXdpZHRoOjUuMDIzNjciCiAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIKICAgICBpZD0icGF0aDI1MyIKICAgICBkPSJtIDg4LjMxNDkzMSw4OS4zNjA1NjcgNS4yNzQ5NjcsLTI0Ljg2NzUyNiBhIDM5LjAzNDU0MSwzOS4wMzQ1NTcgMCAxIDAgLTU5LjE3OTc5NiwwIGwgNS4yNzQ5MzgsMjUuMTE4NzI4IGEgMTAuODUxMzAyLDEwLjg1MTMwNiAwIDAgMCAxMC40OTk2NCw4LjU0MDM2OSBoIDI3LjYzMDYzMSBhIDEwLjkwMTUzOCwxMC45MDE1NDEgMCAwIDAgMTAuNDk5NjIsLTguNzkxNTcxIHogTSAzOS42ODUwMzksNTkuMjE4MDc1IEEgMzEuNDk4OTE2LDMxLjQ5ODkyNSAwIDEgMSA5NS40OTg5MDEsMzkuMTIzMDQyIDMxLjA5NzAxNSwzMS4wOTcwMjYgMCAwIDEgODguMzE0OTMxLDU5LjIxODA3NSBaIE0gODAuOTMwMDM1LDg3Ljk1MzkyMiBBIDMuMzE1Njc1MiwzLjMxNTY3NjYgMCAwIDEgNzcuODE1MzEsOTAuNTE2MDI0IEggNTAuMTg0NjggQSAzLjIxNTIwMDMsMy4yMTUyMDE4IDAgMCAxIDQ3LjA2OTk1Miw4Ny45NTM5MjIgTCA0Mi41NDg1NzgsNjYuNzAzNDgzIGggNDIuOTAyODA2IHoiIC8+CiAgPHBhdGgKICAgICBzdHlsZT0ic3Ryb2tlLXdpZHRoOjUuMDIzNjciCiAgICAgaW5rc2NhcGU6Y29ubmVjdG9yLWN1cnZhdHVyZT0iMCIKICAgICBpZD0icGF0aDI1NSIKICAgICBkPSJNIDgyLjg0Mjk5OSwxMDkuMTU0MTMgQSAzLjc2NzgxMjksMy43Njc4MTM5IDAgMCAwIDc5LjA3NTE5LDEwNS4zODYzMiBIIDQ4LjkzMjY4OSBhIDMuNzY3ODI2OSwzLjc2NzgyODIgMCAxIDAgMCw3LjUzNTY0IGggMC42MDI4NSBjIDAsMCAwLDAuMzAxNzYgMCwwLjUwMjM0IGEgMTQuNTY4ODc3LDE0LjU2ODg4MSAwIDEgMCAyOS4xMzc3NTEsMCBjIDAsMCAwLC0wLjM1MTU1IDAsLTAuNTAyMzQgaCAwLjQwMTc5MiBhIDMuODE4MDUwNSwzLjgxODA1MTUgMCAwIDAgMy43Njc4MDUsLTMuNzY3ODMgeiBtIC0xMS44MDU4MDMsNC4yNzAxOSBhIDcuMDMzMjUzLDcuMDMzMjU1OCAwIDEgMSAtMTQuMDY2NTA1LDAgMS41MDcxMjUsMS41MDcxMjU0IDAgMCAwIDAsLTAuNTAyMzQgSCA3MS4yMzgxNSBhIDEuNTA3MTI1LDEuNTA3MTI1NCAwIDAgMCAtMC4yMDA5NTgsMC42MDI4NSB6IiAvPgo8L3N2Zz4K
+    mediatype: image/svg+xml
+  install:
+    spec:
+      deployments:
+      - name: smart-gateway-operator
+        spec:
+          replicas: 1
+          selector:
+            matchLabels:
+              app: smart-gateway-operator
+          strategy: {}
+          template:
+            metadata:
+              labels:
+                app: smart-gateway-operator
+            spec:
+              containers:
+              - command:
+                - /usr/local/bin/ao-logs
+                - /tmp/ansible-operator/runner
+                - stdout
+                image: quay.io/infrawatch/smart-gateway-operator:latest
+                imagePullPolicy: Always
+                name: ansible
+                resources: {}
+                volumeMounts:
+                - mountPath: /tmp/ansible-operator/runner
+                  name: runner
+                  readOnly: true
+              - env:
+                - name: WATCH_NAMESPACE
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.annotations['olm.targetNamespaces']
+                - name: POD_NAME
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.name
+                - name: OPERATOR_NAME
+                  value: smart-gateway-operator
+                - name: ANSIBLE_GATHERING
+                  value: explicit
+                - name: SMARTGATEWAY_IMAGE
+                  value: quay.io/infrawatch/smart-gateway:v2.0.0-beta1
+                - name: CORE_SMARTGATEWAY_IMAGE
+                  value: quay.io/infrawatch/sg-core:v3.0.0
+                - name: BRIDGE_SMARTGATEWAY_IMAGE
+                  value: quay.io/infrawatch/sg-bridge:v1.0.0
+                image: quay.io/infrawatch/smart-gateway-operator:v2.0.0
+                imagePullPolicy: Always
+                name: operator
+                resources: {}
+                volumeMounts:
+                - mountPath: /tmp/ansible-operator/runner
+                  name: runner
+              serviceAccountName: smart-gateway-operator
+              volumes:
+              - emptyDir: {}
+                name: runner
+      permissions:
+      - rules:
+        - apiGroups:
+          - ""
+          resources:
+          - pods
+          - services
+          - endpoints
+          - persistentvolumeclaims
+          - events
+          - configmaps
+          - secrets
+          verbs:
+          - '*'
+        - apiGroups:
+          - ""
+          resources:
+          - namespaces
+          verbs:
+          - get
+        - apiGroups:
+          - apps
+          resources:
+          - deployments
+          - daemonsets
+          - replicasets
+          - statefulsets
+          verbs:
+          - '*'
+        - apiGroups:
+          - monitoring.coreos.com
+          resources:
+          - servicemonitors
+          verbs:
+          - get
+          - create
+          - list
+          - patch
+          - watch
+        - apiGroups:
+          - apps
+          resourceNames:
+          - smart-gateway-operator
+          resources:
+          - deployments/finalizers
+          verbs:
+          - update
+        - apiGroups:
+          - smartgateway.infra.watch
+          resources:
+          - '*'
+          verbs:
+          - '*'
+        serviceAccountName: smart-gateway-operator
+    strategy: deployment
+  installModes:
+  - supported: true
+    type: OwnNamespace
+  - supported: true
+    type: SingleNamespace
+  - supported: false
+    type: MultiNamespace
+  - supported: true
+    type: AllNamespaces
+  keywords:
+  - servicetelemetry
+  - monitoring
+  - telemetry
+  - notifications
+  - smartgateway
+  labels:
+    name: smart-gateway-operator
+  links:
+  - name: Source Code
+    url: https://github.com/infrawatch/smart-gateway-operator
+  maintainers:
+  - email: [email protected]
+    name: Red Hat CloudOps DFG
+  maturity: alpha
+  provider:
+    name: Red Hat
+  replaces: smart-gateway-operator.v1.0.2
+  selector:
+    matchLabels:
+      name: smart-gateway-operator
+  version: 2.0.0