Merge branch 'improvement/2682-deploy-loki' into q/2.6

buildchain/buildchain/image.py

@@ -184,6 +184,9 @@ def _operator_image(name: str, **kwargs: Any) -> targets.OperatorImage:
         'kube-scheduler',
         'nginx-ingress-defaultbackend-amd64',
     ],
+    constants.GRAFANA_REPOSITORY: [
+        'loki',
+    ],
     constants.INGRESS_REPOSITORY: [
         'nginx-ingress-controller',
     ],

buildchain/buildchain/iso.py

@@ -55,6 +55,7 @@
             Path('examples/new-node.yaml'),
             Path('examples/new-node_vagrant.yaml'),
             Path('examples/prometheus-sparse.yaml'),
+            Path('examples/loki-sparse.yaml'),
         ),
         destination_directory=constants.ISO_ROOT,
         task_dep=['_iso_mkdir_root']

buildchain/buildchain/salt_tree.py

@@ -269,6 +269,17 @@ def _get_parts(self) -> Iterator[str]:
         renderer=targets.Renderer.SLS,
     ),
 
+    Path('salt/metalk8s/addons/logging/deployed/init.sls'),
+    Path('salt/metalk8s/addons/logging/deployed/namespace.sls'),
+    Path('salt/metalk8s/addons/logging/loki/config/loki.yaml'),
+    Path('salt/metalk8s/addons/logging/loki/deployed/chart.sls'),
+    Path('salt/metalk8s/addons/logging/loki/deployed/init.sls'),
+    Path('salt/metalk8s/addons/logging/loki/deployed/',
+         'loki-configuration-secret.sls'),
+    Path('salt/metalk8s/addons/logging/loki/deployed/',
+         'service-configuration.sls'),
+    Path('salt/metalk8s/addons/logging/loki/deployed/storageclass.sls'),
+
     Path('salt/metalk8s/addons/prometheus-adapter/deployed/chart.sls'),
     Path('salt/metalk8s/addons/prometheus-adapter/deployed/init.sls'),
 

buildchain/buildchain/versions.py

@@ -217,6 +217,11 @@ def _version_prefix(version: str, prefix: str = 'v') -> str:
         version='latest',
         digest=None,
     ),
+    Image(
+        name='loki',
+        version='1.5.0',
+        digest='sha256:922b3f412fdd9a8fb01115b6aebf5dac162647ce1c5ee3637ce1e2cff69e097b',
+    ),
 )
 
 CONTAINER_IMAGES_MAP = {image.name: image for image in CONTAINER_IMAGES}

charts/loki.yaml

@@ -0,0 +1,250 @@
+image:
+  repository: '__image__(loki)'
+  tag: 1.5.0
+  pullPolicy: IfNotPresent
+
+  ## Optionally specify an array of imagePullSecrets.
+  ## Secrets must be manually created in the namespace.
+  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+  ##
+  # pullSecrets:
+  #   - myRegistryKeySecretName
+
+ingress:
+  enabled: false
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+  hosts:
+    - host: chart-example.local
+      paths: []
+  tls: []
+  #  - secretName: chart-example-tls
+  #    hosts:
+  #      - chart-example.local
+
+## Affinity for pod assignment
+## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+affinity: {}
+# podAntiAffinity:
+#   requiredDuringSchedulingIgnoredDuringExecution:
+#   - labelSelector:
+#       matchExpressions:
+#       - key: app
+#         operator: In
+#         values:
+#         - loki
+#     topologyKey: "kubernetes.io/hostname"
+
+## StatefulSet annotations
+annotations: {}
+
+# enable tracing for debug, need install jaeger and specify right jaeger_agent_host
+tracing:
+  jaegerAgentHost:
+
+# we disable the whole config section as we will manage it through CSC
+#config:
+#  auth_enabled: false
+#  ingester:
+#    chunk_idle_period: 3m
+#    chunk_block_size: 262144
+#    chunk_retain_period: 1m
+#    max_transfer_retries: 0
+#    lifecycler:
+#      ring:
+#        kvstore:
+#          store: inmemory
+#        replication_factor: 1
+#
+#      ## Different ring configs can be used. E.g. Consul
+#      # ring:
+#      #   store: consul
+#      #   replication_factor: 1
+#      #   consul:
+#      #     host: "consul:8500"
+#      #     prefix: ""
+#      #     http_client_timeout: "20s"
+#      #     consistent_reads: true
+#  limits_config:
+#    enforce_metric_name: false
+#    reject_old_samples: true
+#    reject_old_samples_max_age: 168h
+#  schema_config:
+#    configs:
+#    - from: 2018-04-15
+#      store: boltdb
+#      object_store: filesystem
+#      schema: v9
+#      index:
+#        prefix: index_
+#        period: 168h
+#  server:
+#    http_listen_port: 3100
+#  storage_config:
+#    boltdb:
+#      directory: /data/loki/index
+#    filesystem:
+#      directory: /data/loki/chunks
+#  chunk_store_config:
+#    max_look_back_period: 0s
+#  table_manager:
+#    retention_deletes_enabled: false
+#    retention_period: 0s
+
+## Additional Loki container arguments, e.g. log level (debug, info, warn, error)
+extraArgs: {}
+  # log.level: debug
+
+livenessProbe:
+  httpGet:
+    path: /ready
+    port: http-metrics
+  initialDelaySeconds: 45
+
+## ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/
+networkPolicy:
+  enabled: false
+
+## The app name of loki clients
+client: {}
+  # name:
+
+## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
+nodeSelector: {}
+
+## ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/
+## If you set enabled as "True", you need :
+## - create a pv which above 10Gi and has same namespace with loki
+## - keep storageClassName same with below setting
+persistence:
+  enabled: true
+  accessModes:
+  - ReadWriteOnce
+  size: 10Gi
+  annotations: {}
+  storageClassName: metalk8s-loki
+  # subPath: ""
+  # existingClaim:
+
+## Pod Labels
+podLabels: {}
+
+## Pod Annotations
+podAnnotations:
+  prometheus.io/scrape: "true"
+  prometheus.io/port: "http-metrics"
+
+podManagementPolicy: OrderedReady
+
+## Assign a PriorityClassName to pods if set
+# priorityClassName:
+
+rbac:
+  create: true
+  pspEnabled: true
+
+readinessProbe:
+  httpGet:
+    path: /ready
+    port: http-metrics
+  initialDelaySeconds: 45
+
+replicas: '__var__(loki.spec.deployment.replicas)'
+
+resources: {}
+# limits:
+#   cpu: 200m
+#   memory: 256Mi
+# requests:
+#   cpu: 100m
+#   memory: 128Mi
+
+securityContext:
+  fsGroup: 10001
+  runAsGroup: 10001
+  runAsNonRoot: true
+  runAsUser: 10001
+
+service:
+  type: ClusterIP
+  nodePort:
+  port: 3100
+  annotations: {}
+  labels: {}
+
+serviceAccount:
+  create: true
+  name:
+  annotations: {}
+
+terminationGracePeriodSeconds: 4800
+
+## Tolerations for pod assignment
+## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+tolerations:
+  - key: "node-role.kubernetes.io/bootstrap"
+    operator: "Exists"
+    effect: "NoSchedule"
+  - key: "node-role.kubernetes.io/infra"
+    operator: "Exists"
+    effect: "NoSchedule"
+
+# The values to set in the PodDisruptionBudget spec
+# If not set then a PodDisruptionBudget will not be created
+podDisruptionBudget: {}
+# minAvailable: 1
+# maxUnavailable: 1
+
+updateStrategy:
+  type: RollingUpdate
+
+serviceMonitor:
+  enabled: true
+  interval: ""
+  additionalLabels:
+    release: prometheus-operator
+  annotations: {}
+  # scrapeTimeout: 10s
+
+initContainers: []
+## Init containers to be added to the loki pod.
+# - name: my-init-container
+#   image: busybox:latest
+#   command: ['sh', '-c', 'echo hello']
+
+extraContainers: []
+## Additional containers to be added to the loki pod.
+# - name: reverse-proxy
+#   image: angelbarrera92/basic-auth-reverse-proxy:dev
+#   args:
+#     - "serve"
+#     - "--upstream=http://localhost:3100"
+#     - "--auth-config=/etc/reverse-proxy-conf/authn.yaml"
+#   ports:
+#     - name: http
+#       containerPort: 11811
+#       protocol: TCP
+#   volumeMounts:
+#     - name: reverse-proxy-auth-config
+#       mountPath: /etc/reverse-proxy-conf
+
+
+extraVolumes: []
+## Additional volumes to the loki pod.
+# - name: reverse-proxy-auth-config
+#   secret:
+#     secretName: reverse-proxy-auth-config
+
+## Extra volume mounts that will be added to the loki container
+extraVolumeMounts: []
+
+extraPorts: []
+## Additional ports to the loki services. Useful to expose extra container ports.
+# - port: 11811
+#   protocol: TCP
+#   name: http
+#   targetPort: http
+
+# Extra env variables to pass to the loki container
+env: []

charts/loki/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/loki/Chart.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+appVersion: v1.5.0
+description: 'Loki: like Prometheus, but for logs.'
+engine: gotpl
+home: https://grafana.com/loki
+icon: https://github.com/grafana/loki/raw/master/docs/logo.png
+kubeVersion: ^1.10.0-0
+maintainers:
+- email: [email protected]
+  name: Loki Maintainers
+name: loki
+sources:
+- https://github.com/grafana/loki
+version: 0.30.2

charts/loki/README.md

@@ -0,0 +1,62 @@
+# Loki Helm Chart
+
+## Prerequisites
+
+Make sure you have Helm [installed](https://helm.sh/docs/using_helm/#installing-helm) and
+[deployed](https://helm.sh/docs/using_helm/#installing-tiller) to your cluster. Then add
+Loki's chart repository to Helm:
+
+```bash
+$ helm repo add loki https://grafana.github.io/loki/charts
+```
+
+You can update the chart repository by running:
+
+```bash
+$ helm repo update
+```
+
+## Deploy Loki only
+
+```bash
+$ helm upgrade --install loki loki/loki
+```
+
+## Run Loki behind https ingress
+
+If Loki and Promtail are deployed on different clusters you can add an Ingress in front of Loki.
+By adding a certificate you create an https endpoint. For extra security enable basic authentication on the Ingress.
+
+In Promtail set the following values to communicate with https and basic auth
+
+```
+loki:
+  serviceScheme: https
+  user: user
+  password: pass
+```
+
+Sample helm template for ingress:
+```
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+annotations:
+    kubernetes.io/ingress.class: {{ .Values.ingress.class }}
+    ingress.kubernetes.io/auth-type: "basic"
+    ingress.kubernetes.io/auth-secret: {{ .Values.ingress.basic.secret }}
+name: loki
+spec:
+rules:
+- host: {{ .Values.ingress.host }}
+    http:
+    paths:
+    - backend:
+        serviceName: loki
+        servicePort: 3100
+tls:
+- secretName: {{ .Values.ingress.cert }}
+    hosts:
+    - {{ .Values.ingress.host }}
+```
+