From 5fbf4f9559e306db5ebb075ff011dfbc1c0ca41a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Wed, 10 Apr 2019 11:53:32 +0200 Subject: [PATCH 1/4] HDDS-1412. Provide example k8s deployment files as part of the release package --- hadoop-ozone/common/pom.xml | 24 ++++++ .../dev-support/bin/dist-layout-stitching | 2 + hadoop-ozone/dist/pom.xml | 77 ++++++++++++++++++- hadoop-ozone/dist/{ => src/main}/Dockerfile | 2 +- .../ozone/config.yaml} | 4 +- .../k8s/definitions/ozone/datanode-ds.yaml | 43 +++++++++++ .../ozone/definitions/prometheus.yaml} | 25 +++--- .../ozone/flekszible.yaml} | 5 +- .../definitions/ozone/freon/flekszible.yaml} | 11 +-- .../ozone/freon/freon.yaml} | 32 +++----- .../k8s/definitions/ozone/om-ss-service.yaml | 27 +++++++ .../src/main/k8s/definitions/ozone/om-ss.yaml | 38 +++++++++ .../k8s/definitions/ozone/s3g-ss-service.yaml | 27 +++++++ .../main/k8s/definitions/ozone/s3g-ss.yaml | 38 +++++++++ .../ozone/scm-ss-service.yaml} | 2 - .../ozone/scm-ss.yaml} | 43 +++-------- .../ozone/transformations/config.yaml | 26 +++++++ .../ozone/transformations/emptydir.yaml | 58 ++++++++++++++ .../main/k8s/examples/minikube/LICENSE.header | 15 ++++ .../examples/minikube/config-configmap.yaml | 34 ++++++++ .../examples/minikube/datanode-daemonset.yaml | 40 ++++++++++ .../minikube}/datanode-service.yaml | 6 +- .../minikube/datanode-statefulset.yaml | 58 ++++++++++++++ .../minikube/flekszible/flekszible.yaml | 45 +++++++++++ .../minikube/freon/freon-deployment.yaml} | 40 +++++----- .../minikube}/om-public-service.yaml | 7 +- .../minikube}/om-service.yaml | 7 +- .../minikube}/om-statefulset.yaml | 63 +++++++-------- .../minikube}/s3g-public-service.yaml | 7 +- .../minikube}/s3g-service.yaml | 7 +- .../minikube}/s3g-statefulset.yaml | 32 ++++---- .../minikube}/scm-public-service.yaml | 7 +- .../k8s/examples/minikube/scm-service.yaml | 28 +++++++ .../examples/minikube/scm-statefulset.yaml | 67 ++++++++++++++++ .../main/k8s/examples/ozone/LICENSE.header | 15 ++++ .../k8s/examples/ozone/config-configmap.yaml | 34 ++++++++ .../examples/ozone/datanode-daemonset.yaml | 56 ++++++++++++++ .../examples/ozone/flekszible/flekszible.yaml | 43 +++++++++++ .../ozone/freon/freon-deployment.yaml | 46 +++++++++++ .../ozone/om-service.yaml} | 15 ++-- .../k8s/examples/ozone/om-statefulset.yaml | 73 ++++++++++++++++++ .../main/k8s/examples/ozone/s3g-service.yaml | 28 +++++++ .../k8s/examples/ozone/s3g-statefulset.yaml | 51 ++++++++++++ .../main/k8s/examples/ozone/scm-service.yaml | 28 +++++++ .../k8s/examples/ozone/scm-statefulset.yaml | 67 ++++++++++++++++ .../src/main/k8s/prometheus/configmap.yaml | 51 ------------ .../prometheus/prometheus-public-serivce.yaml | 28 ------- .../dist/src/main/k8s/prometheus/role.yaml | 31 -------- .../src/main/k8s/prometheus/rolebindng.yaml | 27 ------- .../k8s/prometheus/scm-public-service.yaml | 29 ------- 50 files changed, 1217 insertions(+), 352 deletions(-) rename hadoop-ozone/dist/{ => src/main}/Dockerfile (93%) rename hadoop-ozone/dist/src/main/k8s/{ozone/config-configmap.yaml => definitions/ozone/config.yaml} (94%) create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/datanode-ds.yaml rename hadoop-ozone/dist/src/main/k8s/{ozone/datanode-public-service.yaml => definitions/ozone/definitions/prometheus.yaml} (78%) rename hadoop-ozone/dist/src/main/k8s/{prometheus/service-account.yaml => definitions/ozone/flekszible.yaml} (91%) rename hadoop-ozone/dist/{skaffold.yaml => src/main/k8s/definitions/ozone/freon/flekszible.yaml} (83%) rename hadoop-ozone/dist/src/main/k8s/{ozone/datanode-statefulset.yaml => definitions/ozone/freon/freon.yaml} (67%) create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss.yaml rename hadoop-ozone/dist/src/main/k8s/{ozone/scm-service.yaml => definitions/ozone/scm-ss-service.yaml} (96%) rename hadoop-ozone/dist/src/main/k8s/{ozone/scm-statefulset.yaml => definitions/ozone/scm-ss.yaml} (64%) create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/transformations/config.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/transformations/emptydir.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/LICENSE.header create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-daemonset.yaml rename hadoop-ozone/dist/src/main/k8s/{ozone => examples/minikube}/datanode-service.yaml (92%) create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/flekszible/flekszible.yaml rename hadoop-ozone/dist/src/main/k8s/{prometheus/deployment.yaml => examples/minikube/freon/freon-deployment.yaml} (61%) rename hadoop-ozone/dist/src/main/k8s/{ozone => examples/minikube}/om-public-service.yaml (93%) rename hadoop-ozone/dist/src/main/k8s/{ozone => examples/minikube}/om-service.yaml (93%) rename hadoop-ozone/dist/src/main/k8s/{ozone => examples/minikube}/om-statefulset.yaml (64%) rename hadoop-ozone/dist/src/main/k8s/{ozone => examples/minikube}/s3g-public-service.yaml (93%) rename hadoop-ozone/dist/src/main/k8s/{ozone => examples/minikube}/s3g-service.yaml (93%) rename hadoop-ozone/dist/src/main/k8s/{ozone => examples/minikube}/s3g-statefulset.yaml (75%) rename hadoop-ozone/dist/src/main/k8s/{ozone => examples/minikube}/scm-public-service.yaml (93%) create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/scm-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/scm-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/LICENSE.header create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/datanode-daemonset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/flekszible/flekszible.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/freon/freon-deployment.yaml rename hadoop-ozone/dist/src/main/k8s/{prometheus/service.yaml => examples/ozone/om-service.yaml} (90%) create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/om-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-statefulset.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/prometheus/configmap.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/prometheus/prometheus-public-serivce.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/prometheus/role.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/prometheus/rolebindng.yaml delete mode 100644 hadoop-ozone/dist/src/main/k8s/prometheus/scm-public-service.yaml diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml index efea791a754ab..6098b4d7b895a 100644 --- a/hadoop-ozone/common/pom.xml +++ b/hadoop-ozone/common/pom.xml @@ -169,4 +169,28 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> + + + k8s-dev + + + + io.fabric8 + docker-maven-plugin + 0.29.0 + + + + ${user.name}/ozone:${project.version} + + ${project.basedir} + + + + + + + + + diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index ffb469be0c441..0ce4e8fc2ec53 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -120,3 +120,5 @@ cp -r "${ROOT}/hadoop-hdds/docs/target/classes/docs" ./ run cp -p -R "${ROOT}/hadoop-ozone/dist/target/compose" . run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/smoketest" . run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/blockade" . +run cp -p -r "${ROOT}/hadoop-ozone/dist/target/k8s" kubernetes +run cp -p -r "${ROOT}/hadoop-ozone/dist/src/main/Dockerfile" . diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index b31c5b9c8d782..2c88745282ee6 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -28,6 +28,7 @@ UTF-8 true + apache/hadoop:${project.version} @@ -139,7 +140,7 @@ 3.1.0 - copy-resources + copy-compose compile copy-resources @@ -154,6 +155,22 @@ + + copy-k8s + compile + + copy-resources + + + ${basedir}/target/k8s + + + src/main/k8s + true + + + + @@ -267,4 +284,62 @@ hadoop-ozone-upgrade + + + k8s-dev + + ${user.name}/ozone:${project.version} + + + + + io.fabric8 + docker-maven-plugin + 0.29.0 + + + + build + + package + + + + + + ${docker.image} + + + ${project.build.directory}/ozone-${project.version} + + + + + + + + + + + k8s-dev-push + + + + io.fabric8 + docker-maven-plugin + 0.29.0 + + + + push + + package + + + + + + + + diff --git a/hadoop-ozone/dist/Dockerfile b/hadoop-ozone/dist/src/main/Dockerfile similarity index 93% rename from hadoop-ozone/dist/Dockerfile rename to hadoop-ozone/dist/src/main/Dockerfile index c22c3ca913cd3..9c14f498bef5e 100644 --- a/hadoop-ozone/dist/Dockerfile +++ b/hadoop-ozone/dist/src/main/Dockerfile @@ -16,6 +16,6 @@ FROM apache/hadoop-runner:latest -ADD --chown=hadoop target/ozone-0.4.0-SNAPSHOT /opt/hadoop +ADD --chown=hadoop . /opt/hadoop WORKDIR /opt/hadoop diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml similarity index 94% rename from hadoop-ozone/dist/src/main/k8s/ozone/config-configmap.yaml rename to hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml index e79a3cc905813..f98f7f5372680 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml @@ -17,9 +17,8 @@ apiVersion: v1 kind: ConfigMap metadata: name: config - labels: {} - annotations: {} data: + OZONE-SITE.XML_hdds.datanode.dir: "/data/storage" OZONE-SITE.XML_ozone.scm.datanode.id: "/data/datanode.id" OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata" OZONE-SITE.XML_ozone.scm.block.client.address: "scm-0.scm" @@ -27,7 +26,6 @@ data: OZONE-SITE.XML_ozone.scm.client.address: "scm-0.scm" OZONE-SITE.XML_ozone.scm.names: "scm-0.scm" OZONE-SITE.XML_ozone.enabled: "true" - OZONE-SITE.XML_hdds.prometheus.endpoint.enabled: "true" LOG4J.PROPERTIES_log4j.rootLogger: "INFO, stdout" LOG4J.PROPERTIES_log4j.appender.stdout: "org.apache.log4j.ConsoleAppender" LOG4J.PROPERTIES_log4j.appender.stdout.layout: "org.apache.log4j.PatternLayout" diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/datanode-ds.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/datanode-ds.yaml new file mode 100644 index 0000000000000..fbc340c714a7f --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/datanode-ds.yaml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: datanode + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: datanode + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9882" + prometheus.io/path: "/prom" + labels: + app: ozone + component: datanode + spec: + containers: + - name: datanode + image: "@docker.image@" + args: ["ozone","datanode"] + ports: + - containerPort: 9870 + name: rpc diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/datanode-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/prometheus.yaml similarity index 78% rename from hadoop-ozone/dist/src/main/k8s/ozone/datanode-public-service.yaml rename to hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/prometheus.yaml index 22154b76c97bb..581481fde94ff 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/datanode-public-service.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/prometheus.yaml @@ -13,17 +13,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: v1 -kind: Service -metadata: - name: datanode-public - labels: {} - annotations: {} -spec: - selector: - app: ozone - component: datanode - ports: - - port: 9870 - name: rpc - type: NodePort +name: ozone/prometheus +description: Enable prometheus monitoring in Ozone +--- +- type: Add + trigger: + metadata: + name: config + path: + - data + value: + OZONE-SITE.XML_hdds.prometheus.endpoint.enabled: true diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/service-account.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/flekszible.yaml similarity index 91% rename from hadoop-ozone/dist/src/main/k8s/prometheus/service-account.yaml rename to hadoop-ozone/dist/src/main/k8s/definitions/ozone/flekszible.yaml index d5ba196c747a5..2707d302d15d4 100644 --- a/hadoop-ozone/dist/src/main/k8s/prometheus/service-account.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/flekszible.yaml @@ -13,7 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus-operator +description: Apache Hadoop Ozone diff --git a/hadoop-ozone/dist/skaffold.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/freon/flekszible.yaml similarity index 83% rename from hadoop-ozone/dist/skaffold.yaml rename to hadoop-ozone/dist/src/main/k8s/definitions/ozone/freon/flekszible.yaml index cb5f375c34dee..c6e29f3d18210 100644 --- a/hadoop-ozone/dist/skaffold.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/freon/flekszible.yaml @@ -13,13 +13,4 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -apiVersion: skaffold/v1alpha5 -kind: Config -build: - artifacts: - - image: apache/ozone -deploy: - kubectl: - manifests: - - src/main/k8s/ozone/* +description: Load test tool for Apache Hadoop Ozone diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/datanode-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/freon/freon.yaml similarity index 67% rename from hadoop-ozone/dist/src/main/k8s/ozone/datanode-statefulset.yaml rename to hadoop-ozone/dist/src/main/k8s/definitions/ozone/freon/freon.yaml index f8600a0700862..40ebc98a42516 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/datanode-statefulset.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/freon/freon.yaml @@ -14,39 +14,27 @@ # See the License for the specific language governing permissions and # limitations under the License. apiVersion: apps/v1 -kind: StatefulSet +kind: Deployment metadata: - name: datanode - labels: {} - annotations: {} + name: freon + labels: + app.kubernetes.io/component: ozone spec: + replicas: 1 selector: matchLabels: app: ozone - component: datanode + component: freon template: metadata: labels: app: ozone - component: datanode + component: freon spec: containers: - - name: datanode - image: apache/ozone - args: - - ozone - - datanode - ports: - - containerPort: 9870 - name: rpc - volumeMounts: - - name: "data" - mountPath: "/data" + - name: freon + image: "@docker.image@" + args: ["ozone","freon", "rk", "--factor=THREE", "--replicationType=RATIS"] envFrom: - configMapRef: name: config - volumes: - - name: "data" - emptyDir: {} - serviceName: datanode - replicas: 3 diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss-service.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss-service.yaml new file mode 100644 index 0000000000000..a6462fe687ac1 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss-service.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: om +spec: + ports: + - port: 9874 + name: ui + clusterIP: None + selector: + app: ozone + component: om diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss.yaml new file mode 100644 index 0000000000000..f27f99bdf83ca --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: om + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: om + serviceName: om + replicas: 1 + template: + metadata: + labels: + app: ozone + component: om + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9874" + prometheus.io/path: "/prom" + spec: + initContainers: + - name: init + image: elek/ozone + args: ["ozone","om","--init"] + env: + - name: WAITFOR + value: scm-0.scm:9876 + containers: + - name: om + image: "@docker.image@" + args: ["ozone","om"] + env: + - name: WAITFOR + value: scm-0.scm:9876 + volumes: [] diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss-service.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss-service.yaml new file mode 100644 index 0000000000000..c99bbd217aef2 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss-service.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: s3g +spec: + ports: + - port: 9878 + name: rest + clusterIP: None + selector: + app: ozone + component: s3g diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss.yaml new file mode 100644 index 0000000000000..43044c90c3527 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/s3g-ss.yaml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: s3g + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: s3g + serviceName: s3g + replicas: 1 + template: + metadata: + labels: + app: ozone + component: s3g + spec: + containers: + - name: s3g + image: "@docker.image@" + args: ["ozone","s3g"] diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/scm-service.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/scm-ss-service.yaml similarity index 96% rename from hadoop-ozone/dist/src/main/k8s/ozone/scm-service.yaml rename to hadoop-ozone/dist/src/main/k8s/definitions/ozone/scm-ss-service.yaml index f311695795493..f8a05abcba928 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/scm-service.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/scm-ss-service.yaml @@ -17,8 +17,6 @@ apiVersion: v1 kind: Service metadata: name: scm - labels: {} - annotations: {} spec: ports: - port: 9876 diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/scm-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/scm-ss.yaml similarity index 64% rename from hadoop-ozone/dist/src/main/k8s/ozone/scm-statefulset.yaml rename to hadoop-ozone/dist/src/main/k8s/definitions/ozone/scm-ss.yaml index 80c59242dab56..a87111ca85d8e 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/scm-statefulset.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/scm-ss.yaml @@ -17,8 +17,8 @@ apiVersion: apps/v1 kind: StatefulSet metadata: name: scm - labels: {} - annotations: {} + labels: + app.kubernetes.io/component: ozone spec: selector: matchLabels: @@ -28,40 +28,19 @@ spec: replicas: 1 template: metadata: + labels: + app: ozone + component: scm annotations: prometheus.io/scrape: "true" prometheus.io/port: "9876" prometheus.io/path: "/prom" - labels: - app: ozone - component: scm spec: initContainers: - - name: init - image: apache/ozone - args: - - ozone - - scm - - --init - env: [] - volumeMounts: - - name: "data" - mountPath: "/data" - envFrom: - - configMapRef: - name: config + - name: init + image: "@docker.image@" + args: ["ozone","scm", "--init"] containers: - - name: scm - image: apache/ozone - args: - - ozone - - scm - volumeMounts: - - name: "data" - mountPath: "/data" - envFrom: - - configMapRef: - name: config - volumes: - - name: "data" - emptyDir: {} + - name: scm + image: elek/ozone + args: ["ozone","scm"] diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/transformations/config.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/transformations/config.yaml new file mode 100644 index 0000000000000..c8ae632cfaea2 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/transformations/config.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- type: Add + path: + - spec + - template + - spec + - ".*" + - ".*" + - envFrom + value: + - configMapRef: + name: config diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/transformations/emptydir.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/transformations/emptydir.yaml new file mode 100644 index 0000000000000..5f9b94a2c5d29 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/transformations/emptydir.yaml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +- type: Add + trigger: + metadata: + labels: + app.kubernetes.io/component: ozone + path: + - spec + - template + - spec + - containers + - "*" + - volumeMounts + value: + - name: data + mountPath: /data +- type: Add + trigger: + metadata: + labels: + app.kubernetes.io/component: ozone + path: + - spec + - template + - spec + - initContainers + - "*" + - volumeMounts + value: + - name: data + mountPath: /data +- type: Add + trigger: + metadata: + labels: + app.kubernetes.io/component: ozone + path: + - spec + - template + - spec + - volumes + value: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/LICENSE.header b/hadoop-ozone/dist/src/main/k8s/examples/minikube/LICENSE.header new file mode 100644 index 0000000000000..635f0d9e60e16 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/LICENSE.header @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml new file mode 100644 index 0000000000000..427b977daa604 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: config +data: + OZONE-SITE.XML_hdds.datanode.dir: /data/storage + OZONE-SITE.XML_ozone.scm.datanode.id: /data/datanode.id + OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata + OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm + OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm + OZONE-SITE.XML_ozone.scm.names: scm-0.scm + OZONE-SITE.XML_ozone.enabled: "true" + LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout + LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender + LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout + LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: '%d{yyyy-MM-dd + HH:mm:ss} %-5p %c{1}:%L - %m%n' diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-daemonset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-daemonset.yaml new file mode 100644 index 0000000000000..a1f8ea254360a --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-daemonset.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: datanode + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: datanode + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9882" + prometheus.io/path: /prom + labels: + app: ozone + component: datanode + spec: + containers: + - name: datanode + image: '@docker.image@' + args: + - ozone + - datanode + ports: + - containerPort: 9870 + name: rpc + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + initContainers: [] + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/datanode-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-service.yaml similarity index 92% rename from hadoop-ozone/dist/src/main/k8s/ozone/datanode-service.yaml rename to hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-service.yaml index bdc8009712b42..0e5927d1622ca 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/datanode-service.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-service.yaml @@ -13,17 +13,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + apiVersion: v1 kind: Service metadata: name: datanode - labels: {} - annotations: {} spec: clusterIP: None selector: app: ozone component: datanode - ports: - - port: 9870 - name: rpc diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-statefulset.yaml new file mode 100644 index 0000000000000..a453b9a3d6151 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-statefulset.yaml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: datanode + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: datanode + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9882" + prometheus.io/path: /prom + labels: + app: ozone + component: datanode + spec: + containers: + - name: datanode + image: '@docker.image@' + args: + - ozone + - datanode + ports: + - containerPort: 9870 + name: rpc + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + initContainers: [] + volumes: + - name: data + emptyDir: {} + serviceName: datanode + replicas: 3 diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/flekszible/flekszible.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/flekszible/flekszible.yaml new file mode 100644 index 0000000000000..81207e72e7605 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/flekszible/flekszible.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +source: + - path: ../../../definitions +import: + - path: ozone + transformations: + - type: Image + image: "@docker.image@" + - type: DaemonToStatefulSet + - type: PublishStatefulSet + - path: ozone/freon + destination: freon + transformations: + - type: Image + image: "@docker.image@" +header: |- + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/deployment.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/freon/freon-deployment.yaml similarity index 61% rename from hadoop-ozone/dist/src/main/k8s/prometheus/deployment.yaml rename to hadoop-ozone/dist/src/main/k8s/examples/minikube/freon/freon-deployment.yaml index 636840398eead..1662c4e3f2441 100644 --- a/hadoop-ozone/dist/src/main/k8s/prometheus/deployment.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/freon/freon-deployment.yaml @@ -13,34 +13,34 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -apiVersion: apps/v1beta1 + +apiVersion: apps/v1 kind: Deployment metadata: - name: prometheus + name: freon labels: - app: prometheus + app.kubernetes.io/component: ozone spec: replicas: 1 selector: matchLabels: - app: prometheus + app: ozone + component: freon template: metadata: labels: - app: prometheus + app: ozone + component: freon spec: - serviceAccountName: prometheus-operator - containers: - - name: prometheus - image: prom/prometheus - args: ["--config.file=/conf/prometheus.yaml"] - ports: - - containerPort: 9090 - volumeMounts: - - name: config - mountPath: "/conf" - readOnly: true - volumes: - - name: config - configMap: - name: prometheusconf + containers: + - name: freon + image: '@docker.image@' + args: + - ozone + - freon + - rk + - --factor=THREE + - --replicationType=RATIS + envFrom: + - configMapRef: + name: config diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/om-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/om-public-service.yaml similarity index 93% rename from hadoop-ozone/dist/src/main/k8s/ozone/om-public-service.yaml rename to hadoop-ozone/dist/src/main/k8s/examples/minikube/om-public-service.yaml index a150939c728d4..deb2c333c38f9 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/om-public-service.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/om-public-service.yaml @@ -13,16 +13,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + apiVersion: v1 kind: Service metadata: name: om-public - labels: {} - annotations: {} spec: ports: - - port: 9874 - name: ui + - port: 9874 + name: ui selector: app: ozone component: om diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/om-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/om-service.yaml similarity index 93% rename from hadoop-ozone/dist/src/main/k8s/ozone/om-service.yaml rename to hadoop-ozone/dist/src/main/k8s/examples/minikube/om-service.yaml index f363c48ead073..617277d9b8503 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/om-service.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/om-service.yaml @@ -13,16 +13,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + apiVersion: v1 kind: Service metadata: name: om - labels: {} - annotations: {} spec: ports: - - port: 9874 - name: ui + - port: 9874 + name: ui clusterIP: None selector: app: ozone diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/om-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/om-statefulset.yaml similarity index 64% rename from hadoop-ozone/dist/src/main/k8s/ozone/om-statefulset.yaml rename to hadoop-ozone/dist/src/main/k8s/examples/minikube/om-statefulset.yaml index e9eeea8bcb132..e0811e4a73e56 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/om-statefulset.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/om-statefulset.yaml @@ -13,12 +13,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + apiVersion: apps/v1 kind: StatefulSet metadata: name: om - labels: {} - annotations: {} + labels: + app.kubernetes.io/component: ozone spec: selector: matchLabels: @@ -28,39 +29,39 @@ spec: replicas: 1 template: metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9874" - prometheus.io/path: "/prom" labels: app: ozone component: om + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9874" + prometheus.io/path: /prom spec: initContainers: - - name: init - image: apache/ozone - args: - - ozone - - om - - --init - volumeMounts: - - name: "data" - mountPath: "/data" - envFrom: - - configMapRef: - name: config + - name: init + image: '@docker.image@' + args: + - ozone + - om + - --init + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data containers: - - name: om - image: apache/ozone - args: - - ozone - - om - volumeMounts: - - name: "data" - mountPath: "/data" - envFrom: - - configMapRef: - name: config + - name: om + image: '@docker.image@' + args: + - ozone + - om + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data volumes: - - name: "data" - emptyDir: {} + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/s3g-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-public-service.yaml similarity index 93% rename from hadoop-ozone/dist/src/main/k8s/ozone/s3g-public-service.yaml rename to hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-public-service.yaml index 54d10de2b5293..d2b2420576c4c 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/s3g-public-service.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-public-service.yaml @@ -13,16 +13,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + apiVersion: v1 kind: Service metadata: name: s3g-public - labels: {} - annotations: {} spec: ports: - - port: 9878 - name: rest + - port: 9878 + name: rest selector: app: ozone component: s3g diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/s3g-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-service.yaml similarity index 93% rename from hadoop-ozone/dist/src/main/k8s/ozone/s3g-service.yaml rename to hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-service.yaml index 8441e12a75a65..dd1ca83479919 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/s3g-service.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-service.yaml @@ -13,16 +13,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + apiVersion: v1 kind: Service metadata: name: s3g - labels: {} - annotations: {} spec: ports: - - port: 9878 - name: rest + - port: 9878 + name: rest clusterIP: None selector: app: ozone diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/s3g-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-statefulset.yaml similarity index 75% rename from hadoop-ozone/dist/src/main/k8s/ozone/s3g-statefulset.yaml rename to hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-statefulset.yaml index dd6fc257d526d..b608cde832ef2 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/s3g-statefulset.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/s3g-statefulset.yaml @@ -13,12 +13,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + apiVersion: apps/v1 kind: StatefulSet metadata: name: s3g - labels: {} - annotations: {} + labels: + app.kubernetes.io/component: ozone spec: selector: matchLabels: @@ -33,17 +34,18 @@ spec: component: s3g spec: containers: - - name: s3g - image: apache/ozone - args: - - ozone - - s3g - volumeMounts: - - name: "data" - mountPath: "/data" - envFrom: - - configMapRef: - name: config + - name: s3g + image: '@docker.image@' + args: + - ozone + - s3g + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + initContainers: [] volumes: - - name: "data" - emptyDir: {} + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/scm-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/scm-public-service.yaml similarity index 93% rename from hadoop-ozone/dist/src/main/k8s/ozone/scm-public-service.yaml rename to hadoop-ozone/dist/src/main/k8s/examples/minikube/scm-public-service.yaml index ddfe138d11087..e3246fc5e3568 100644 --- a/hadoop-ozone/dist/src/main/k8s/ozone/scm-public-service.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/scm-public-service.yaml @@ -13,16 +13,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + apiVersion: v1 kind: Service metadata: name: scm-public - labels: {} - annotations: {} spec: ports: - - port: 9876 - name: ui + - port: 9876 + name: ui selector: app: ozone component: scm diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/scm-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/scm-service.yaml new file mode 100644 index 0000000000000..0df15d6453159 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/scm-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: scm +spec: + ports: + - port: 9876 + name: ui + clusterIP: None + selector: + app: ozone + component: scm diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/scm-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/scm-statefulset.yaml new file mode 100644 index 0000000000000..55b56722d105b --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/minikube/scm-statefulset.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: scm + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: scm + serviceName: scm + replicas: 1 + template: + metadata: + labels: + app: ozone + component: scm + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9876" + prometheus.io/path: /prom + spec: + initContainers: + - name: init + image: '@docker.image@' + args: + - ozone + - scm + - --init + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + containers: + - name: scm + image: '@docker.image@' + args: + - ozone + - scm + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/LICENSE.header b/hadoop-ozone/dist/src/main/k8s/examples/ozone/LICENSE.header new file mode 100644 index 0000000000000..635f0d9e60e16 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/LICENSE.header @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml new file mode 100644 index 0000000000000..427b977daa604 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: config +data: + OZONE-SITE.XML_hdds.datanode.dir: /data/storage + OZONE-SITE.XML_ozone.scm.datanode.id: /data/datanode.id + OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata + OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm + OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm + OZONE-SITE.XML_ozone.scm.names: scm-0.scm + OZONE-SITE.XML_ozone.enabled: "true" + LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout + LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender + LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout + LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: '%d{yyyy-MM-dd + HH:mm:ss} %-5p %c{1}:%L - %m%n' diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/datanode-daemonset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/datanode-daemonset.yaml new file mode 100644 index 0000000000000..6e3b3b96cf09a --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/datanode-daemonset.yaml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: datanode + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: datanode + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9882" + prometheus.io/path: /prom + labels: + app: ozone + component: datanode + spec: + containers: + - name: datanode + image: '@docker.image@' + args: + - ozone + - datanode + ports: + - containerPort: 9870 + name: rpc + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + initContainers: [] + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/flekszible/flekszible.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/flekszible/flekszible.yaml new file mode 100644 index 0000000000000..5da1f20d41b55 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/flekszible/flekszible.yaml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +source: + - path: ../../../definitions +import: + - path: ozone + transformations: + - type: Image + image: "@docker.image@" + - path: ozone/freon + destination: freon + transformations: + - type: Image + image: "@docker.image@" +header: |- + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/freon/freon-deployment.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/freon/freon-deployment.yaml new file mode 100644 index 0000000000000..1662c4e3f2441 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/freon/freon-deployment.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: freon + labels: + app.kubernetes.io/component: ozone +spec: + replicas: 1 + selector: + matchLabels: + app: ozone + component: freon + template: + metadata: + labels: + app: ozone + component: freon + spec: + containers: + - name: freon + image: '@docker.image@' + args: + - ozone + - freon + - rk + - --factor=THREE + - --replicationType=RATIS + envFrom: + - configMapRef: + name: config diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/om-service.yaml similarity index 90% rename from hadoop-ozone/dist/src/main/k8s/prometheus/service.yaml rename to hadoop-ozone/dist/src/main/k8s/examples/ozone/om-service.yaml index e07aafc18696a..617277d9b8503 100644 --- a/hadoop-ozone/dist/src/main/k8s/prometheus/service.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/om-service.yaml @@ -13,13 +13,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -kind: Service + apiVersion: v1 +kind: Service metadata: - name: prometheus + name: om spec: - selector: - app: prometheus ports: - - protocol: TCP - port: 9090 + - port: 9874 + name: ui + clusterIP: None + selector: + app: ozone + component: om diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/om-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/om-statefulset.yaml new file mode 100644 index 0000000000000..884dd46241768 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/om-statefulset.yaml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: om + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: om + serviceName: om + replicas: 1 + template: + metadata: + labels: + app: ozone + component: om + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9874" + prometheus.io/path: /prom + spec: + initContainers: + - name: init + image: '@docker.image@' + args: + - ozone + - om + - --init + env: + - name: WAITFOR + value: scm-0.scm:9876 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + containers: + - name: om + image: '@docker.image@' + args: + - ozone + - om + env: + - name: WAITFOR + value: scm-0.scm:9876 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-service.yaml new file mode 100644 index 0000000000000..dd1ca83479919 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: s3g +spec: + ports: + - port: 9878 + name: rest + clusterIP: None + selector: + app: ozone + component: s3g diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml new file mode 100644 index 0000000000000..b608cde832ef2 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/s3g-statefulset.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: s3g + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: s3g + serviceName: s3g + replicas: 1 + template: + metadata: + labels: + app: ozone + component: s3g + spec: + containers: + - name: s3g + image: '@docker.image@' + args: + - ozone + - s3g + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + initContainers: [] + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-service.yaml new file mode 100644 index 0000000000000..0df15d6453159 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: scm +spec: + ports: + - port: 9876 + name: ui + clusterIP: None + selector: + app: ozone + component: scm diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-statefulset.yaml new file mode 100644 index 0000000000000..55b56722d105b --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone/scm-statefulset.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: scm + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: scm + serviceName: scm + replicas: 1 + template: + metadata: + labels: + app: ozone + component: scm + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9876" + prometheus.io/path: /prom + spec: + initContainers: + - name: init + image: '@docker.image@' + args: + - ozone + - scm + - --init + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + containers: + - name: scm + image: '@docker.image@' + args: + - ozone + - scm + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/configmap.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/configmap.yaml deleted file mode 100644 index 40b13e23d0211..0000000000000 --- a/hadoop-ozone/dist/src/main/k8s/prometheus/configmap.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheusconf -data: - prometheus.yaml: |- - global: - scrape_interval: 15s - scrape_configs: - - job_name: jmxexporter - kubernetes_sd_configs: - - role: pod - namespaces: - names: ["default"] - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - relabel_configs: - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] - action: keep - regex: true - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] - action: replace - target_label: __metrics_path__ - regex: (.+) - - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] - action: replace - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - source_labels: [__meta_kubernetes_namespace] - action: replace - target_label: kubernetes_namespace - - source_labels: [__meta_kubernetes_pod_name] - action: replace - target_label: kubernetes_pod_name diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/prometheus-public-serivce.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/prometheus-public-serivce.yaml deleted file mode 100644 index 276943dccbed9..0000000000000 --- a/hadoop-ozone/dist/src/main/k8s/prometheus/prometheus-public-serivce.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: v1 -kind: Service -metadata: - name: prometheus-public - labels: {} - annotations: {} -spec: - ports: - - port: 9090 - name: ui - selector: - app: prometheus - type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/role.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/role.yaml deleted file mode 100644 index e9412faff823a..0000000000000 --- a/hadoop-ozone/dist/src/main/k8s/prometheus/role.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: prometheus -rules: -- apiGroups: [""] - resources: - - nodes - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: - - configmaps - verbs: ["get"] diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/rolebindng.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/rolebindng.yaml deleted file mode 100644 index ffd340d28e4ed..0000000000000 --- a/hadoop-ozone/dist/src/main/k8s/prometheus/rolebindng.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: prometheus-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: prometheus -subjects: -- kind: ServiceAccount - name: prometheus-operator -# namespace: default diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/scm-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/scm-public-service.yaml deleted file mode 100644 index ddfe138d11087..0000000000000 --- a/hadoop-ozone/dist/src/main/k8s/prometheus/scm-public-service.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: v1 -kind: Service -metadata: - name: scm-public - labels: {} - annotations: {} -spec: - ports: - - port: 9876 - name: ui - selector: - app: ozone - component: scm - type: NodePort From 63407ba8825c878973ea4731d6ae96938b2118bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Wed, 17 Apr 2019 12:15:38 +0200 Subject: [PATCH 2/4] fix RAT headers --- .../src/main/k8s/definitions/ozone/om-ss.yaml | 15 +++++++ .../examples/minikube/datanode-daemonset.yaml | 40 ------------------- 2 files changed, 15 insertions(+), 40 deletions(-) delete mode 100644 hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-daemonset.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss.yaml index f27f99bdf83ca..820d5622811a9 100644 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/om-ss.yaml @@ -1,3 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-daemonset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-daemonset.yaml deleted file mode 100644 index a1f8ea254360a..0000000000000 --- a/hadoop-ozone/dist/src/main/k8s/examples/minikube/datanode-daemonset.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: datanode - labels: - app.kubernetes.io/component: ozone -spec: - selector: - matchLabels: - app: ozone - component: datanode - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9882" - prometheus.io/path: /prom - labels: - app: ozone - component: datanode - spec: - containers: - - name: datanode - image: '@docker.image@' - args: - - ozone - - datanode - ports: - - containerPort: 9870 - name: rpc - envFrom: - - configMapRef: - name: config - volumeMounts: - - name: data - mountPath: /data - initContainers: [] - volumes: - - name: data - emptyDir: {} From 6e741df4cc8bf0b5d1c36930ce14f4637730e578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Mon, 29 Apr 2019 15:47:03 +0200 Subject: [PATCH 3/4] HDDS-1478. Provide k8s resources files for prometheus and performance tests. Contributed by Elek, Marton. --- .../k8s/definitions/jaeger/flekszible.yaml | 16 ++++ .../main/k8s/definitions/jaeger/jaeger.yaml | 54 ++++++++++++++ .../ozone/definitions/profiler.yaml | 27 +++++++ .../ozone/definitions/prometheus.yaml | 2 +- .../ozone/definitions/tracing.yaml | 33 +++++++++ .../k8s/definitions/prometheus/configmap.yaml | 49 +++++++++++++ .../prometheus/definitions/enable.yaml | 26 +++++++ .../definitions/prometheus/deployment.yaml | 46 ++++++++++++ .../definitions/prometheus/flekszible.yaml | 16 ++++ .../main/k8s/definitions/prometheus/role.yaml | 33 +++++++++ .../definitions/prometheus/rolebinding.yaml | 27 +++++++ .../prometheus/service-account.yaml | 19 +++++ .../k8s/definitions/prometheus/service.yaml | 25 +++++++ .../k8s/examples/ozone-dev/LICENSE.header | 15 ++++ .../examples/ozone-dev/config-configmap.yaml | 35 +++++++++ .../ozone-dev/datanode-daemonset.yaml | 56 ++++++++++++++ .../ozone-dev/flekszible/flekszible.yaml | 36 +++++++++ .../ozone-dev/freon/freon-deployment.yaml | 46 ++++++++++++ .../examples/ozone-dev/om-public-service.yaml | 28 +++++++ .../k8s/examples/ozone-dev/om-service.yaml | 28 +++++++ .../examples/ozone-dev/om-statefulset.yaml | 73 +++++++++++++++++++ .../ozone-dev/prometheus-clusterrole.yaml | 42 +++++++++++ .../ozone-dev/prometheus-deployment.yaml | 48 ++++++++++++ ...rometheus-operator-clusterrolebinding.yaml | 28 +++++++ .../prometheus-operator-serviceaccount.yaml | 20 +++++ .../ozone-dev/prometheus-service.yaml | 26 +++++++ .../ozone-dev/prometheusconf-configmap.yaml | 50 +++++++++++++ .../ozone-dev/s3g-public-service.yaml | 28 +++++++ .../k8s/examples/ozone-dev/s3g-service.yaml | 28 +++++++ .../examples/ozone-dev/s3g-statefulset.yaml | 51 +++++++++++++ .../ozone-dev/scm-public-service.yaml | 28 +++++++ .../k8s/examples/ozone-dev/scm-service.yaml | 28 +++++++ .../examples/ozone-dev/scm-statefulset.yaml | 67 +++++++++++++++++ 33 files changed, 1133 insertions(+), 1 deletion(-) create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/jaeger/flekszible.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/jaeger/jaeger.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/profiler.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/tracing.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/prometheus/configmap.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/prometheus/definitions/enable.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/prometheus/deployment.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/prometheus/flekszible.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/prometheus/role.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/prometheus/rolebinding.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/prometheus/service-account.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/definitions/prometheus/service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/LICENSE.header create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/datanode-daemonset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/flekszible/flekszible.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/freon/freon-deployment.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-public-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-clusterrole.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-deployment.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-operator-clusterrolebinding.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-operator-serviceaccount.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheusconf-configmap.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-public-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-public-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-statefulset.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/jaeger/flekszible.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/jaeger/flekszible.yaml new file mode 100644 index 0000000000000..8fdc15566882a --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/jaeger/flekszible.yaml @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +description: Jaeger tracing server diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/jaeger/jaeger.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/jaeger/jaeger.yaml new file mode 100644 index 0000000000000..4796092657cab --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/jaeger/jaeger.yaml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: jaeger +spec: + clusterIP: None + selector: + app: jaeger + component: jaeger + ports: + - name: ui + port: 16686 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: jaeger +spec: + selector: + matchLabels: + app: jaeger + component: jaeger + replicas: 1 + serviceName: jaeger + template: + metadata: + labels: + app: jaeger + component: jaeger + spec: + containers: + - name: jaeger + image: jaegertracing/all-in-one:latest + ports: + - containerPort: 16686 + name: web + env: + - name: COLLECTOR_ZIPKIN_HTTP_PORT + value: "9411" diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/profiler.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/profiler.yaml new file mode 100644 index 0000000000000..d76931af45aad --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/profiler.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: ozone/profiler +description: Enable profiler endpoint. +--- +- type: Add + trigger: + metadata: + name: config + path: + - data + value: + OZONE-SITE.XML_hdds.profiler.endpoint.enabled: "true" + ASYNC_PROFILER_HOME: /opt/profiler \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/prometheus.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/prometheus.yaml index 581481fde94ff..604df1fe5ac15 100644 --- a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/prometheus.yaml +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/prometheus.yaml @@ -23,4 +23,4 @@ description: Enable prometheus monitoring in Ozone path: - data value: - OZONE-SITE.XML_hdds.prometheus.endpoint.enabled: true + OZONE-SITE.XML_hdds.prometheus.endpoint.enabled: "true" diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/tracing.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/tracing.yaml new file mode 100644 index 0000000000000..007b8d125176d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/ozone/definitions/tracing.yaml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: ozone/tracing +description: Enable jaeger tracing +--- +- type: Add + path: + - spec + - template + - spec + - containers + - .* + - env + value: + - name: JAEGER_SAMPLER_TYPE + value: probabilistic + - name: JAEGER_SAMPLER_PARAM + value: "0.01" + - name: JAEGER_AGENT_HOST + value: jaeger-0.jaeger diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/configmap.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/configmap.yaml new file mode 100644 index 0000000000000..f8809875cb395 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/configmap.yaml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheusconf +data: + prometheus.yaml: |- + global: + scrape_interval: 15s + scrape_configs: + - job_name: jmxexporter + kubernetes_sd_configs: + - role: pod + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/definitions/enable.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/definitions/enable.yaml new file mode 100644 index 0000000000000..6825e91da3a96 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/definitions/enable.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: ozone/prometheus +description: Enable prometheus monitoring in Ozone +--- +- type: Add + trigger: + metadata: + name: config + path: + - data + value: + OZONE-SITE.XML_hdds.prometheus.endpoint.enabled: "true" diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/deployment.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/deployment.yaml new file mode 100644 index 0000000000000..636840398eead --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/deployment.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: prometheus + labels: + app: prometheus +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + serviceAccountName: prometheus-operator + containers: + - name: prometheus + image: prom/prometheus + args: ["--config.file=/conf/prometheus.yaml"] + ports: + - containerPort: 9090 + volumeMounts: + - name: config + mountPath: "/conf" + readOnly: true + volumes: + - name: config + configMap: + name: prometheusconf diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/flekszible.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/flekszible.yaml new file mode 100644 index 0000000000000..20809e57edb73 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/flekszible.yaml @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +description: Prometheus monitoring diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/role.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/role.yaml new file mode 100644 index 0000000000000..194e9f4a2be0e --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/role.yaml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/rolebinding.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/rolebinding.yaml new file mode 100644 index 0000000000000..ef5105da4e1fa --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/rolebinding.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: prometheus-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus-operator + namespace: default diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/service-account.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/service-account.yaml new file mode 100644 index 0000000000000..d5ba196c747a5 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/service-account.yaml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-operator diff --git a/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/service.yaml b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/service.yaml new file mode 100644 index 0000000000000..e07aafc18696a --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/definitions/prometheus/service.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +kind: Service +apiVersion: v1 +metadata: + name: prometheus +spec: + selector: + app: prometheus + ports: + - protocol: TCP + port: 9090 diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/LICENSE.header b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/LICENSE.header new file mode 100644 index 0000000000000..635f0d9e60e16 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/LICENSE.header @@ -0,0 +1,15 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml new file mode 100644 index 0000000000000..d494bf530a2f0 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: config +data: + OZONE-SITE.XML_hdds.datanode.dir: /data/storage + OZONE-SITE.XML_ozone.scm.datanode.id: /data/datanode.id + OZONE-SITE.XML_ozone.metadata.dirs: /data/metadata + OZONE-SITE.XML_ozone.scm.block.client.address: scm-0.scm + OZONE-SITE.XML_ozone.om.address: om-0.om + OZONE-SITE.XML_ozone.scm.client.address: scm-0.scm + OZONE-SITE.XML_ozone.scm.names: scm-0.scm + OZONE-SITE.XML_ozone.enabled: "true" + LOG4J.PROPERTIES_log4j.rootLogger: INFO, stdout + LOG4J.PROPERTIES_log4j.appender.stdout: org.apache.log4j.ConsoleAppender + LOG4J.PROPERTIES_log4j.appender.stdout.layout: org.apache.log4j.PatternLayout + LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: '%d{yyyy-MM-dd + HH:mm:ss} %-5p %c{1}:%L - %m%n' + OZONE-SITE.XML_hdds.prometheus.endpoint.enabled: "true" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/datanode-daemonset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/datanode-daemonset.yaml new file mode 100644 index 0000000000000..6e3b3b96cf09a --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/datanode-daemonset.yaml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: datanode + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: datanode + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9882" + prometheus.io/path: /prom + labels: + app: ozone + component: datanode + spec: + containers: + - name: datanode + image: '@docker.image@' + args: + - ozone + - datanode + ports: + - containerPort: 9870 + name: rpc + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + initContainers: [] + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/flekszible/flekszible.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/flekszible/flekszible.yaml new file mode 100644 index 0000000000000..9b9f8f39d9576 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/flekszible/flekszible.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +source: + - path: ../../../definitions +import: + - path: ozone + transformations: + - type: Image + image: "@docker.image@" + - type: ozone/prometheus + - type: PublishStatefulSet + - type: ozone/tracing + - type: ozone/profiler + - path: prometheus + - path: jaeger + transformations: + - type: PublishService + - path: ozone/freon + destination: freon + transformations: + - type: Image + image: "@docker.image@" + - type: ozone/tracing \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/freon/freon-deployment.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/freon/freon-deployment.yaml new file mode 100644 index 0000000000000..1662c4e3f2441 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/freon/freon-deployment.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: freon + labels: + app.kubernetes.io/component: ozone +spec: + replicas: 1 + selector: + matchLabels: + app: ozone + component: freon + template: + metadata: + labels: + app: ozone + component: freon + spec: + containers: + - name: freon + image: '@docker.image@' + args: + - ozone + - freon + - rk + - --factor=THREE + - --replicationType=RATIS + envFrom: + - configMapRef: + name: config diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-public-service.yaml new file mode 100644 index 0000000000000..deb2c333c38f9 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: om-public +spec: + ports: + - port: 9874 + name: ui + selector: + app: ozone + component: om + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-service.yaml new file mode 100644 index 0000000000000..617277d9b8503 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: om +spec: + ports: + - port: 9874 + name: ui + clusterIP: None + selector: + app: ozone + component: om diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-statefulset.yaml new file mode 100644 index 0000000000000..884dd46241768 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-statefulset.yaml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: om + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: om + serviceName: om + replicas: 1 + template: + metadata: + labels: + app: ozone + component: om + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9874" + prometheus.io/path: /prom + spec: + initContainers: + - name: init + image: '@docker.image@' + args: + - ozone + - om + - --init + env: + - name: WAITFOR + value: scm-0.scm:9876 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + containers: + - name: om + image: '@docker.image@' + args: + - ozone + - om + env: + - name: WAITFOR + value: scm-0.scm:9876 + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-clusterrole.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-clusterrole.yaml new file mode 100644 index 0000000000000..d5df196b4a946 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-clusterrole.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: + - "" + resources: + - nodes + - services + - endpoints + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- nonResourceURLs: + - /metrics + verbs: + - get diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-deployment.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-deployment.yaml new file mode 100644 index 0000000000000..86a188af09989 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-deployment.yaml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: prometheus + labels: + app: prometheus +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + serviceAccountName: prometheus-operator + containers: + - name: prometheus + image: prom/prometheus + args: + - --config.file=/conf/prometheus.yaml + ports: + - containerPort: 9090 + volumeMounts: + - name: config + mountPath: /conf + readOnly: true + volumes: + - name: config + configMap: + name: prometheusconf diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-operator-clusterrolebinding.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-operator-clusterrolebinding.yaml new file mode 100644 index 0000000000000..b0d83185197ea --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-operator-clusterrolebinding.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: prometheus-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus-operator + namespace: default diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-operator-serviceaccount.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-operator-serviceaccount.yaml new file mode 100644 index 0000000000000..f816888b48d90 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-operator-serviceaccount.yaml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-operator diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-service.yaml new file mode 100644 index 0000000000000..312cf586d335f --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheus-service.yaml @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kind: Service +apiVersion: v1 +metadata: + name: prometheus +spec: + selector: + app: prometheus + ports: + - protocol: TCP + port: 9090 diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheusconf-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheusconf-configmap.yaml new file mode 100644 index 0000000000000..6d5b12362c502 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/prometheusconf-configmap.yaml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheusconf +data: + prometheus.yaml: |- + global: + scrape_interval: 15s + scrape_configs: + - job_name: jmxexporter + kubernetes_sd_configs: + - role: pod + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-public-service.yaml new file mode 100644 index 0000000000000..d2b2420576c4c --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: s3g-public +spec: + ports: + - port: 9878 + name: rest + selector: + app: ozone + component: s3g + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-service.yaml new file mode 100644 index 0000000000000..dd1ca83479919 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: s3g +spec: + ports: + - port: 9878 + name: rest + clusterIP: None + selector: + app: ozone + component: s3g diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml new file mode 100644 index 0000000000000..b608cde832ef2 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: s3g + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: s3g + serviceName: s3g + replicas: 1 + template: + metadata: + labels: + app: ozone + component: s3g + spec: + containers: + - name: s3g + image: '@docker.image@' + args: + - ozone + - s3g + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + initContainers: [] + volumes: + - name: data + emptyDir: {} diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-public-service.yaml new file mode 100644 index 0000000000000..e3246fc5e3568 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: scm-public +spec: + ports: + - port: 9876 + name: ui + selector: + app: ozone + component: scm + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-service.yaml new file mode 100644 index 0000000000000..0df15d6453159 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: scm +spec: + ports: + - port: 9876 + name: ui + clusterIP: None + selector: + app: ozone + component: scm diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-statefulset.yaml new file mode 100644 index 0000000000000..55b56722d105b --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-statefulset.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: scm + labels: + app.kubernetes.io/component: ozone +spec: + selector: + matchLabels: + app: ozone + component: scm + serviceName: scm + replicas: 1 + template: + metadata: + labels: + app: ozone + component: scm + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9876" + prometheus.io/path: /prom + spec: + initContainers: + - name: init + image: '@docker.image@' + args: + - ozone + - scm + - --init + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + containers: + - name: scm + image: '@docker.image@' + args: + - ozone + - scm + envFrom: + - configMapRef: + name: config + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + emptyDir: {} From e5df51b53027f2dc03822c10bb3d67e92ed3be3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Wed, 1 May 2019 14:23:17 +0200 Subject: [PATCH 4/4] regenerate k8s resources --- .../examples/ozone-dev/config-configmap.yaml | 2 + .../ozone-dev/datanode-daemonset.yaml | 7 ++++ .../ozone-dev/freon/freon-deployment.yaml | 7 ++++ .../ozone-dev/jaeger-public-service.yaml | 28 +++++++++++++ .../examples/ozone-dev/jaeger-service.yaml | 28 +++++++++++++ .../ozone-dev/jaeger-statefulset.yaml | 42 +++++++++++++++++++ .../examples/ozone-dev/om-statefulset.yaml | 6 +++ .../examples/ozone-dev/s3g-statefulset.yaml | 7 ++++ .../examples/ozone-dev/scm-statefulset.yaml | 7 ++++ 9 files changed, 134 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-public-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-service.yaml create mode 100644 hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-statefulset.yaml diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml index d494bf530a2f0..605f111f0b1ac 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml @@ -33,3 +33,5 @@ data: LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: '%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n' OZONE-SITE.XML_hdds.prometheus.endpoint.enabled: "true" + OZONE-SITE.XML_hdds.profiler.endpoint.enabled: "true" + ASYNC_PROFILER_HOME: /opt/profiler diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/datanode-daemonset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/datanode-daemonset.yaml index 6e3b3b96cf09a..0293cc25eacda 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/datanode-daemonset.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/datanode-daemonset.yaml @@ -44,6 +44,13 @@ spec: ports: - containerPort: 9870 name: rpc + env: + - name: JAEGER_SAMPLER_TYPE + value: probabilistic + - name: JAEGER_SAMPLER_PARAM + value: "0.01" + - name: JAEGER_AGENT_HOST + value: jaeger-0.jaeger envFrom: - configMapRef: name: config diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/freon/freon-deployment.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/freon/freon-deployment.yaml index 1662c4e3f2441..88c9045cb5c2d 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/freon/freon-deployment.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/freon/freon-deployment.yaml @@ -44,3 +44,10 @@ spec: envFrom: - configMapRef: name: config + env: + - name: JAEGER_SAMPLER_TYPE + value: probabilistic + - name: JAEGER_SAMPLER_PARAM + value: "0.01" + - name: JAEGER_AGENT_HOST + value: jaeger-0.jaeger diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-public-service.yaml new file mode 100644 index 0000000000000..fb06569b397af --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-public-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: jaeger-public +spec: + selector: + app: jaeger + component: jaeger + ports: + - name: ui + port: 16686 + type: NodePort diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-service.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-service.yaml new file mode 100644 index 0000000000000..6e6125ade56d2 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-service.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: jaeger +spec: + clusterIP: None + selector: + app: jaeger + component: jaeger + ports: + - name: ui + port: 16686 diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-statefulset.yaml new file mode 100644 index 0000000000000..51410148f77d4 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/jaeger-statefulset.yaml @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: jaeger +spec: + selector: + matchLabels: + app: jaeger + component: jaeger + replicas: 1 + serviceName: jaeger + template: + metadata: + labels: + app: jaeger + component: jaeger + spec: + containers: + - name: jaeger + image: jaegertracing/all-in-one:latest + ports: + - containerPort: 16686 + name: web + env: + - name: COLLECTOR_ZIPKIN_HTTP_PORT + value: "9411" diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-statefulset.yaml index 884dd46241768..927186e76c650 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-statefulset.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/om-statefulset.yaml @@ -62,6 +62,12 @@ spec: env: - name: WAITFOR value: scm-0.scm:9876 + - name: JAEGER_SAMPLER_TYPE + value: probabilistic + - name: JAEGER_SAMPLER_PARAM + value: "0.01" + - name: JAEGER_AGENT_HOST + value: jaeger-0.jaeger envFrom: - configMapRef: name: config diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml index b608cde832ef2..1033f3482071b 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/s3g-statefulset.yaml @@ -39,6 +39,13 @@ spec: args: - ozone - s3g + env: + - name: JAEGER_SAMPLER_TYPE + value: probabilistic + - name: JAEGER_SAMPLER_PARAM + value: "0.01" + - name: JAEGER_AGENT_HOST + value: jaeger-0.jaeger envFrom: - configMapRef: name: config diff --git a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-statefulset.yaml index 55b56722d105b..87c8d3878387d 100644 --- a/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-statefulset.yaml +++ b/hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/scm-statefulset.yaml @@ -56,6 +56,13 @@ spec: args: - ozone - scm + env: + - name: JAEGER_SAMPLER_TYPE + value: probabilistic + - name: JAEGER_SAMPLER_PARAM + value: "0.01" + - name: JAEGER_AGENT_HOST + value: jaeger-0.jaeger envFrom: - configMapRef: name: config