From 550596a498ee772e87214abc322a380891248504 Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Thu, 31 Oct 2019 10:44:16 -0700 Subject: [PATCH 1/6] GKE single cluster configuration archetype Demo purposes. To make it work, users will need to change some config --- README.md | 9 -- .../overlays/gke-single-cluster/README.md | 48 +++++++++++ .../admindeployment/admindeployment.yaml | 62 +++++++++++++ .../aa_namespace.yaml | 7 ++ .../admindeployment/cron.yaml | 30 +++++++ .../admindeployment/flyteadmin_config.yaml | 86 +++++++++++++++++++ .../admindeployment/kustomization.yaml | 22 +++++ .../admindeployment/service.yaml | 8 ++ .../gke-single-cluster/console/console.yaml | 15 ++++ .../console/kustomization.yaml | 8 ++ .../gke-single-cluster/console/service.yaml | 11 +++ .../datacatalog/datacatalog.yaml | 30 +++++++ .../datacatalog/datacatalog_config.yaml | 28 ++++++ .../datacatalog/kustomization.yaml | 12 +++ .../flyte/kustomization.yaml | 19 ++++ .../gke-single-cluster/propeller/config.yaml | 54 ++++++++++++ .../propeller/kustomization.yaml | 31 +++++++ .../propeller/plugins/config.yaml | 22 +++++ .../propeller/plugins/container/config.yaml | 0 .../plugins/container/propeller-patch.yaml | 17 ++++ .../propeller/plugins/qubole/config.yaml | 10 +++ .../plugins/qubole/propeller-patch.yaml | 20 +++++ .../propeller/plugins/spark/config.yaml | 16 ++++ .../plugins/spark/propeller-patch.yaml | 17 ++++ .../propeller/propeller.yaml | 18 ++++ .../redis/kustomization.yaml | 5 ++ .../gke-single-cluster/redis/storage.yaml | 11 +++ 27 files changed, 607 insertions(+), 9 deletions(-) delete mode 100644 README.md create mode 100644 kustomize/overlays/gke-single-cluster/README.md create mode 100644 kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml create mode 100644 kustomize/overlays/gke-single-cluster/admindeployment/clusterresource-templates/aa_namespace.yaml create mode 100644 kustomize/overlays/gke-single-cluster/admindeployment/cron.yaml create mode 100644 kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml create mode 100644 kustomize/overlays/gke-single-cluster/admindeployment/kustomization.yaml create mode 100644 kustomize/overlays/gke-single-cluster/admindeployment/service.yaml create mode 100644 kustomize/overlays/gke-single-cluster/console/console.yaml create mode 100644 kustomize/overlays/gke-single-cluster/console/kustomization.yaml create mode 100644 kustomize/overlays/gke-single-cluster/console/service.yaml create mode 100644 kustomize/overlays/gke-single-cluster/datacatalog/datacatalog.yaml create mode 100644 kustomize/overlays/gke-single-cluster/datacatalog/datacatalog_config.yaml create mode 100644 kustomize/overlays/gke-single-cluster/datacatalog/kustomization.yaml create mode 100644 kustomize/overlays/gke-single-cluster/flyte/kustomization.yaml create mode 100644 kustomize/overlays/gke-single-cluster/propeller/config.yaml create mode 100644 kustomize/overlays/gke-single-cluster/propeller/kustomization.yaml create mode 100644 kustomize/overlays/gke-single-cluster/propeller/plugins/config.yaml create mode 100644 kustomize/overlays/gke-single-cluster/propeller/plugins/container/config.yaml create mode 100644 kustomize/overlays/gke-single-cluster/propeller/plugins/container/propeller-patch.yaml create mode 100644 kustomize/overlays/gke-single-cluster/propeller/plugins/qubole/config.yaml create mode 100644 kustomize/overlays/gke-single-cluster/propeller/plugins/qubole/propeller-patch.yaml create mode 100644 kustomize/overlays/gke-single-cluster/propeller/plugins/spark/config.yaml create mode 100644 kustomize/overlays/gke-single-cluster/propeller/plugins/spark/propeller-patch.yaml create mode 100644 kustomize/overlays/gke-single-cluster/propeller/propeller.yaml create mode 100644 kustomize/overlays/gke-single-cluster/redis/kustomization.yaml create mode 100644 kustomize/overlays/gke-single-cluster/redis/storage.yaml diff --git a/README.md b/README.md deleted file mode 100644 index 729eee7fb5..0000000000 --- a/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Flyte - -![](https://github.com/lyft/flyte/workflows/tests/badge.svg) - -Flyte is a K8s-native, multi-tenant, distributed orchestration engine that manages highly scalable compute workflows. - -Extensibility is provided by Flyte plugins and K8s operators, including executors for Spark, Hive, AWS Batch, in addition to raw K8s compute. Flyte can orchestrate arbitrary containers at scale, providing feature fanout, automatic task parallelization, strongly-typed paramerter passing, automatic cluster scale up and scale down, cluster fail-over, and fine-grained programmatic control over retry policies, error handling, and notifications. - -See docs at https://lyft.github.io/flyte/ diff --git a/kustomize/overlays/gke-single-cluster/README.md b/kustomize/overlays/gke-single-cluster/README.md new file mode 100644 index 0000000000..f6a238b796 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/README.md @@ -0,0 +1,48 @@ +################################### +# WORK IN PROGRESS still +################################### + +SQL Database +------------ +Create a SQL database (Postgres) +https://cloud.google.com/sql/docs/postgres/create-instance + +Enable the the SQL server to be accessed from the GKE cluster that will host the FlyteAdmin service. This can be done using private networking mode and associating the shared network + +Create a database called "flyte" in this DB instance + +Configuring Flyte to access DB +------------------------------ + +In this sample we pass the username and password directly in the config file. +TODO: Example of how to use kube secrets to pass the username and password. + +Auth / IAM +---------- + +On GKE you can follow instructions listed here +https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity +to setup WorkloadIdentity and serviceAccounts. + +Important commands +kubectl create serviceaccount --namespace flytekit-development flyte-sandbox +gcloud iam service-accounts add-iam-policy-binding --role roles/iam.workloadIdentityUser --member "serviceAccount:flyte-sandbox.svc.id.goog[flytekit-development/flyte-sandbox]" flyte-sandbox@flyte-sandbox.iam.gserviceaccount.com +kubectl annotate serviceaccount --namespace flytekit-development flyte-sandbox iam.gke.io/gcp-service-account=flyte-sandbox@flyte-sandbox.iam.gserviceaccount.com + + +IAM For FLyte components +------------------------ +Create the right service accounts in GKE cluster's flyte namespace and then add the serviceaccountname to propeller and flyteadmin deployments. You may also want to add it to the various plugin +deployments. + +gcloud iam service-accounts add-iam-policy-binding --role roles/iam.workloadIdentityUser --member "serviceAccount:flyte-sandbox.svc.id.goog[flyte/flyteadmin]" flyte-sandbox@flyte-sandbox.iam.gserviceaccount.com +kubectl annotate serviceaccount --namespace flyte flyteadmin iam.gke.io/gcp-service-account=flyte-sandbox@flyte-sandbox.iam.gserviceaccount.com +gcloud iam service-accounts add-iam-policy-binding --role roles/iam.workloadIdentityUser --member "serviceAccount:flyte-sandbox.svc.id.goog[flyte/flytepropeller]" flyte-sandbox@flyte-sandbox.iam.gserviceaccount.com +kubectl annotate serviceaccount --namespace flyte flytepropeller iam.gke.io/gcp-service-account=flyte-sandbox@flyte-sandbox.iam.gserviceaccount.com + +IAM for workflows +----------------- +As a platform admin, you will need to associate service accounts with the target namespaces (project-domain) combination. Flyte allows launching workflows with serviceAccounts. Thus when the end user +requests a workflow launch or declares a workflow the right account should be associated within the right namespace. + +TODO: Future plans to automate this creation and association diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml new file mode 100644 index 0000000000..ca4add7306 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteadmin + namespace: flyte +spec: + template: + spec: + volumes: + - name: resource-templates + configMap: + name: clusterresource-template + initContainers: + - name: check-db-ready + image: postgres:10.1 + command: ['sh', '-c', + 'until pg_isready -h postgres -p 5432; + do echo waiting for database; sleep 2; done;'] + - name: run-migrations + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", + "migrate", "run"] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: seed-projects + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", + "migrate", "seed-projects", "flytesnacks", "flytetester"] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: sync-cluster-resources + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] + volumeMounts: + - name: resource-templates + mountPath: /etc/flyte/clusterresource/templates + - name: config-volume + mountPath: /etc/flyte/config + containers: + - name: flyteadmin + resources: + limits: + memory: "200Mi" + cpu: "0.1" + ephemeral-storage: "100Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: flyteadmin + namespace: flyte +spec: + ports: + - name: redoc + protocol: TCP + port: 87 + targetPort: 8087 diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/clusterresource-templates/aa_namespace.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/clusterresource-templates/aa_namespace.yaml new file mode 100644 index 0000000000..3075aa9f9e --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/admindeployment/clusterresource-templates/aa_namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ namespace }} +spec: + finalizers: + - kubernetes diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/cron.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/cron.yaml new file mode 100644 index 0000000000..5ded07db1f --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/admindeployment/cron.yaml @@ -0,0 +1,30 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: syncresources + namespace: flyte +spec: + schedule: "*/1 * * * *" + jobTemplate: + spec: + template: + spec: + serviceAccountName: flyteadmin + containers: + - name: sync-cluster-resources + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] + volumeMounts: + - name: resource-templates + mountPath: /etc/flyte/clusterresource/templates + - name: config-volume + mountPath: /etc/flyte/config + volumes: + - name: resource-templates + configMap: + name: clusterresource-template + - name: config-volume + configMap: + name: flyte-admin-config + restartPolicy: OnFailure diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml new file mode 100644 index 0000000000..6274ac4c75 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml @@ -0,0 +1,86 @@ +logger: + show-source: true + level: 5 +application: + httpPort: 8088 + grpcPort: 8089 +flyteadmin: + roleNameKey: "iam.amazonaws.com/role" + profilerPort: 10254 + metricsScope: "flyte:" + metadataStoragePrefix: + - "metadata" + - "admin" + testing: + host: http://flyteadmin +database: + # Create a database like postgres and override these values + port: 5432 + username: postgres + password: awesomesauce + # Recommended to use passwordPath and mount it using kubescrets or the like + # passwordPath: "/var/run/CREDENTIALS_DB_PASSWORD" + # host here is the ip address of the CloudSQL Db in private mode + host: 10.23.0.3 + dbname: flyte + options: sslmode=disable +storage: + type: stow + stow: + kind: google + config: + scopes: "" + project_id: flyte-sandbox + json: "" + container: "flyte-sandbox" +task_resources: + defaults: + cpu: 200m + gpu: 0 + memory: 500Mi + storage: 100Mi + limits: + cpu: 62 + gpu: 8 + memory: 256Gi + storage: 5Gi +domains: + - id: development + name: development + - id: staging + name: staging + - id: production + name: production + - id: domain + name: domain +registration: + maxWorkflowNodes: 100 +scheduler: + eventScheduler: + scheme: local + region: "us-east-1" + scheduleRole: "arn:aws:iam::173840052742:role/mbadmin-development-scheduler" + targetName: "arn:aws:sqs:us-east-1:173840052742:flyteadmin-development-scheduler" + workflowExecutor: + scheme: local + region: "us-east-1" + scheduleQueueName: "won't-work-locally" + accountId: "173840052742" +notifications: + type: local + region: "us-east-1" + publisher: + topicName: "foo" + processor: + queueName: "queue" + accountId: "bar" + emailer: + subject: "Notice: Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\"." + sender: "flyte-notifications@lyft.com" + body: > + Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\". View details at + + http://flyte.lyft.net/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. {{ error }} +cluster_resources: + templatePath: "/etc/flyte/clusterresource/templates" + refresh: 5m diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/kustomization.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/kustomization.yaml new file mode 100644 index 0000000000..fc5537643e --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/admindeployment/kustomization.yaml @@ -0,0 +1,22 @@ +bases: +- ../../../base/admindeployment + +namespace: flyte + +resources: +- cron.yaml +- service.yaml + +configMapGenerator: +# the main admin configmap +- name: flyte-admin-config + files: + - flyteadmin_config.yaml +# cluster resource templates +- name: clusterresource-template + files: +# Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". + - clusterresource-templates/aa_namespace.yaml + +patches: +- admindeployment.yaml diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/service.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/service.yaml new file mode 100644 index 0000000000..61e0e35d81 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/admindeployment/service.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Service +metadata: + name: flyteadmin + annotations: + cloud.google.com/load-balancer-type: "Internal" +spec: + type: LoadBalancer diff --git a/kustomize/overlays/gke-single-cluster/console/console.yaml b/kustomize/overlays/gke-single-cluster/console/console.yaml new file mode 100644 index 0000000000..9261654671 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/console/console.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyteconsole + namespace: flyte +spec: + template: + spec: + containers: + - name: flyteconsole + resources: + limits: + memory: "150Mi" + cpu: "0.1" + ephemeral-storage: "100Mi" diff --git a/kustomize/overlays/gke-single-cluster/console/kustomization.yaml b/kustomize/overlays/gke-single-cluster/console/kustomization.yaml new file mode 100644 index 0000000000..a6957adbe2 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/console/kustomization.yaml @@ -0,0 +1,8 @@ +bases: +- ../../../base/console + +patches: +- console.yaml + +resources: +- service.yaml diff --git a/kustomize/overlays/gke-single-cluster/console/service.yaml b/kustomize/overlays/gke-single-cluster/console/service.yaml new file mode 100644 index 0000000000..752bca4de9 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/console/service.yaml @@ -0,0 +1,11 @@ +--- +# Service +apiVersion: v1 +kind: Service +metadata: + name: flyteconsole + namespace: flyte + annotations: + cloud.google.com/load-balancer-type: "Internal" +spec: + type: LoadBalancer diff --git a/kustomize/overlays/gke-single-cluster/datacatalog/datacatalog.yaml b/kustomize/overlays/gke-single-cluster/datacatalog/datacatalog.yaml new file mode 100644 index 0000000000..40eb2cefec --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/datacatalog/datacatalog.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datacatalog + namespace: flyte +spec: + template: + spec: + initContainers: + - name: check-db-ready + image: postgres:10.1 + command: ['sh', '-c', + 'until pg_isready -h postgres -p 5432; + do echo waiting for database; sleep 2; done;'] + volumeMounts: + - name: config-volume + mountPath: /etc/datacatalog/config + containers: + - name: datacatalog + resources: + limits: + memory: "200Mi" + cpu: "0.1" + ephemeral-storage: "100Mi" +--- +apiVersion: v1 +kind: Service +metadata: + name: datacatalog + namespace: flyte diff --git a/kustomize/overlays/gke-single-cluster/datacatalog/datacatalog_config.yaml b/kustomize/overlays/gke-single-cluster/datacatalog/datacatalog_config.yaml new file mode 100644 index 0000000000..7812ba566d --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/datacatalog/datacatalog_config.yaml @@ -0,0 +1,28 @@ +logger: + show-source: true + level: 5 +datacatalog: + storage-prefix: metadata/datacatalog + metrics-scope: "datacatalog" + profiler-port: 10254 +application: + grpcPort: 8089 +storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + cache: + max_size_mbs: 10 + target_gc_percent: 100 + container: my-container + type: minio +database: + port: 5432 + username: postgres + host: postgres + dbname: datacatalog + options: sslmode=disable diff --git a/kustomize/overlays/gke-single-cluster/datacatalog/kustomization.yaml b/kustomize/overlays/gke-single-cluster/datacatalog/kustomization.yaml new file mode 100644 index 0000000000..9876c444a0 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/datacatalog/kustomization.yaml @@ -0,0 +1,12 @@ +bases: +- ../../../base/datacatalog + +namespace: flyte + +configMapGenerator: +- name: datacatalog-config + files: + - datacatalog_config.yaml + +patches: +- datacatalog.yaml diff --git a/kustomize/overlays/gke-single-cluster/flyte/kustomization.yaml b/kustomize/overlays/gke-single-cluster/flyte/kustomization.yaml new file mode 100644 index 0000000000..61f5016db3 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/flyte/kustomization.yaml @@ -0,0 +1,19 @@ +bases: +# global resources +- ../../../base/namespace +- ../../../dependencies/database +- ../../../dependencies/storage + +# user plane / control plane resources +- ../../../base/ingress +- ../../../dependencies/contour_ingress_controller +- ../admindeployment +- ../datacatalog +- ../console + +# data plane resources +- ../../../base/wf_crd +- ../../../base/operators/spark +- ../../../base/adminserviceaccount +- ../propeller +- ../redis diff --git a/kustomize/overlays/gke-single-cluster/propeller/config.yaml b/kustomize/overlays/gke-single-cluster/propeller/config.yaml new file mode 100644 index 0000000000..eab6a8a0ad --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/propeller/config.yaml @@ -0,0 +1,54 @@ +propeller: + metadata-prefix: metadata/propeller + workers: 4 + max-workflow-retries: 30 + workflow-reeval-duration: 30s + downstream-eval-duration: 30s + limit-namespace: "all" + prof-port: 10254 + metrics-prefix: flyte + enable-admin-launcher: true + leader-election: + lock-config-map: + name: propeller-leader + namespace: flyte + enabled: true + lease-duration: 15s + renew-deadline: 10s + retry-period: 2s + queue: + type: batch + batching-interval: 2s + batch-size: -1 + queue: + type: bucket + rate: 10 + capacity: 100 + sub-queue: + type: bucket + rate: 10 + capacity: 100 +logger: + show-source: true + level: 5 +storage: + type: stow + stow: + kind: google + config: + scopes: "" + project_id: flyte-sandbox + json: "" + container: "flyte-sandbox" +event: + type: admin + rate: 500 + capacity: 1000 +admin: + endpoint: flyteadmin:81 + insecure: true +# TODO may be we should disable catalog cache in the default? +catalog-cache: + endpoint: datacatalog:89 + type: datacatalog + insecure: true diff --git a/kustomize/overlays/gke-single-cluster/propeller/kustomization.yaml b/kustomize/overlays/gke-single-cluster/propeller/kustomization.yaml new file mode 100644 index 0000000000..18170c1c81 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/propeller/kustomization.yaml @@ -0,0 +1,31 @@ +bases: +- ../../../base/propeller + +namespace: flyte + +configMapGenerator: +# the main propeller configmap +- name: flyte-propeller-config + files: + - config.yaml +# the plugin-configmap +- name: flyte-plugin-config + files: + - plugins/config.yaml +# a configmap for each plugin +- name: flyte-spark-config + files: + - plugins/spark/config.yaml +- name: flyte-container-config + files: + - plugins/container/config.yaml +- name: flyte-qubole-config + files: + - plugins/qubole/config.yaml + +patches: +- propeller.yaml +# add the volumemount for each plugin configmap +- plugins/spark/propeller-patch.yaml +- plugins/container/propeller-patch.yaml +- plugins/qubole/propeller-patch.yaml diff --git a/kustomize/overlays/gke-single-cluster/propeller/plugins/config.yaml b/kustomize/overlays/gke-single-cluster/propeller/plugins/config.yaml new file mode 100644 index 0000000000..1f1db92d63 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/propeller/plugins/config.yaml @@ -0,0 +1,22 @@ +plugins: + enabled-plugins: + - container + - spark + - waitable + - hiveExecutor + - sidecar + logs: + # Log links can link to multiple options + # #1 Kubernetes dashboard + kubernetes-enabled: false + # #2 GCP stackdriver + stackdriver-enabled: true + gcp-project: flyte-sandbox + stackdriver-logresourcename: flyte + k8s: + default-annotations: + # Example annotation that will be applied to every k8s resource launched + - flyte.lyft.net/deployment: base-google-gke + # Example Environment variables that will be applied to every container executed on k8s + default-env-vars: + - FLYTE_CLOUD_PLATFORM: google diff --git a/kustomize/overlays/gke-single-cluster/propeller/plugins/container/config.yaml b/kustomize/overlays/gke-single-cluster/propeller/plugins/container/config.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kustomize/overlays/gke-single-cluster/propeller/plugins/container/propeller-patch.yaml b/kustomize/overlays/gke-single-cluster/propeller/plugins/container/propeller-patch.yaml new file mode 100644 index 0000000000..fe9e305e28 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/propeller/plugins/container/propeller-patch.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: flyte +spec: + template: + spec: + volumes: + - name: container-config-volume + configMap: + name: flyte-container-config + containers: + - name: flytepropeller + volumeMounts: + - name: container-config-volume + mountPath: /etc/flyte/config-container diff --git a/kustomize/overlays/gke-single-cluster/propeller/plugins/qubole/config.yaml b/kustomize/overlays/gke-single-cluster/propeller/plugins/qubole/config.yaml new file mode 100644 index 0000000000..f7d776659c --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/propeller/plugins/qubole/config.yaml @@ -0,0 +1,10 @@ +plugins: + qubole: + # Either create this file with your username with the real token, or set the QUBOLE_API_KEY environment variable + # See the secrets_manager.go file in the plugins repo for usage. Since the dev/test deployment of + # this has a dummy QUBOLE_API_KEY env var built in, this fake path won't break anything. + quboleTokenPath: "/Path/To/QUBOLE_CLIENT_TOKEN" + resourceManagerType: redis + redisHostPath: redis-resource-manager.flyte:6379 + redisHostKey: mypassword + quboleLimit: 10 diff --git a/kustomize/overlays/gke-single-cluster/propeller/plugins/qubole/propeller-patch.yaml b/kustomize/overlays/gke-single-cluster/propeller/plugins/qubole/propeller-patch.yaml new file mode 100644 index 0000000000..cd85986073 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/propeller/plugins/qubole/propeller-patch.yaml @@ -0,0 +1,20 @@ +# This file is only for volume mounts. The configmap itself that's being mounted is sufficiently different that +# there's no benefit to having it in this folder, since the entire thing gets overridden anyways. +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: flyte +spec: + template: + spec: + volumes: + - name: qubole-config-volume + configMap: + name: flyte-qubole-config + containers: + - name: flytepropeller + volumeMounts: + - name: qubole-config-volume + mountPath: /etc/flyte/config-qubole diff --git a/kustomize/overlays/gke-single-cluster/propeller/plugins/spark/config.yaml b/kustomize/overlays/gke-single-cluster/propeller/plugins/spark/config.yaml new file mode 100644 index 0000000000..c1ffb9209a --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/propeller/plugins/spark/config.yaml @@ -0,0 +1,16 @@ +plugins: + spark: + spark-config-default: + - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" + - spark.kubernetes.allocation.batch.size: "50" + - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" + - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.multipart.threshold: "536870912" + - spark.blacklist.enabled: "true" + - spark.blacklist.timeout: "5m" + - spark.task.maxfailures: "8" diff --git a/kustomize/overlays/gke-single-cluster/propeller/plugins/spark/propeller-patch.yaml b/kustomize/overlays/gke-single-cluster/propeller/plugins/spark/propeller-patch.yaml new file mode 100644 index 0000000000..a1c2aacd10 --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/propeller/plugins/spark/propeller-patch.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: flyte +spec: + template: + spec: + volumes: + - name: spark-config-volume + configMap: + name: flyte-spark-config + containers: + - name: flytepropeller + volumeMounts: + - name: spark-config-volume + mountPath: /etc/flyte/config-spark diff --git a/kustomize/overlays/gke-single-cluster/propeller/propeller.yaml b/kustomize/overlays/gke-single-cluster/propeller/propeller.yaml new file mode 100644 index 0000000000..498055e6fc --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/propeller/propeller.yaml @@ -0,0 +1,18 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flytepropeller + namespace: flyte +spec: + template: + spec: + containers: + - name: flytepropeller + env: + - name: QUBOLE_API_KEY + value: notarealkey + resources: + limits: + memory: "100Mi" + cpu: "0.1" + ephemeral-storage: "100Mi" diff --git a/kustomize/overlays/gke-single-cluster/redis/kustomization.yaml b/kustomize/overlays/gke-single-cluster/redis/kustomization.yaml new file mode 100644 index 0000000000..12ec57311d --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/redis/kustomization.yaml @@ -0,0 +1,5 @@ +bases: +- ../../../dependencies/redis + +patches: +- storage.yaml diff --git a/kustomize/overlays/gke-single-cluster/redis/storage.yaml b/kustomize/overlays/gke-single-cluster/redis/storage.yaml new file mode 100644 index 0000000000..0c5aed931b --- /dev/null +++ b/kustomize/overlays/gke-single-cluster/redis/storage.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: flyte +spec: + template: + spec: + volumes: + - name: redis-data + emptyDir: {} From e6a907ee4a8657e717c51b053180b7acaf8bd1a3 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Fri, 1 Nov 2019 11:29:18 -0700 Subject: [PATCH 2/6] adding a base generate script and generated yaml --- .../gke-single-cluster/flyte_generated.yaml | 1550 +++++++++++++++++ .../overlays/gke-single-cluster/README.md | 4 +- .../admindeployment/admindeployment.yaml | 2 +- .../admindeployment/flyteadmin_config.yaml | 12 +- .../admindeployment/kustomization.yaml | 2 +- .../admindeployment/service.yaml | 1 + .../console/kustomization.yaml | 2 - script/kustomize.sh | 7 +- 8 files changed, 1566 insertions(+), 14 deletions(-) create mode 100644 deployment/gke-single-cluster/flyte_generated.yaml diff --git a/deployment/gke-single-cluster/flyte_generated.yaml b/deployment/gke-single-cluster/flyte_generated.yaml new file mode 100644 index 0000000000..1d7af1ffb7 --- /dev/null +++ b/deployment/gke-single-cluster/flyte_generated.yaml @@ -0,0 +1,1550 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flyte +spec: + finalizers: + - kubernetes +status: + phase: Active +--- +apiVersion: v1 +kind: Namespace +metadata: + name: heptio-contour +--- +apiVersion: v1 +kind: Namespace +metadata: + name: sparkoperator +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: flyteworkflows.flyte.lyft.com +spec: + group: flyte.lyft.com + names: + kind: FlyteWorkflow + plural: flyteworkflows + shortNames: + - fly + singular: flyteworkflow + scope: Namespaced + version: v1alpha1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + labels: + component: ingressroute + name: ingressroutes.contour.heptio.com +spec: + additionalPrinterColumns: + - JSONPath: .spec.virtualhost.fqdn + description: Fully qualified domain name + name: FQDN + type: string + - JSONPath: .spec.virtualhost.tls.secretName + description: Secret with TLS credentials + name: TLS Secret + type: string + - JSONPath: .spec.routes[0].match + description: First routes defined + name: First route + type: string + - JSONPath: .status.currentStatus + description: The current status of the IngressRoute + name: Status + type: string + - JSONPath: .status.description + description: Description of the current status + name: Status Description + type: string + group: contour.heptio.com + names: + kind: IngressRoute + plural: ingressroutes + scope: Namespaced + validation: + openAPIV3Schema: + properties: + spec: + properties: + healthCheck: + properties: + healthyThresholdCount: + type: integer + intervalSeconds: + type: integer + path: + pattern: ^\/.*$ + type: string + timeoutSeconds: + type: integer + unhealthyThresholdCount: + type: integer + required: + - path + type: object + routes: + items: + properties: + delegate: + properties: + name: + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + namespace: + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + required: + - name + type: object + match: + pattern: ^\/.*$ + type: string + services: + items: + properties: + healthCheck: + properties: + healthyThresholdCount: + type: integer + intervalSeconds: + type: integer + path: + pattern: ^\/.*$ + type: string + timeoutSeconds: + type: integer + unhealthyThresholdCount: + type: integer + required: + - path + type: object + name: + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + type: integer + strategy: + enum: + - RoundRobin + - WeightedLeastRequest + - Random + - RingHash + - Maglev + type: string + weight: + type: integer + required: + - name + - port + type: object + type: array + required: + - match + type: array + strategy: + enum: + - RoundRobin + - LeastRequest + - Random + - RingHash + - Maglev + type: string + virtualhost: + properties: + aliases: + items: + pattern: ^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$ + type: string + type: array + fqdn: + pattern: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[a-z]{2,}$ + type: string + required: + - routes + version: v1beta1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: sparkapplications.sparkoperator.k8s.io +spec: + group: sparkoperator.k8s.io + names: + kind: SparkApplication + listKind: SparkApplicationList + plural: sparkapplications + shortNames: + - sparkapp + singular: sparkapplication + scope: Namespaced + version: v1beta1 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: datacatalog + namespace: flyte +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyteadmin + namespace: flyte +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flytepropeller + namespace: flyte +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: contour + namespace: heptio-contour +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sparkoperator + namespace: sparkoperator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyteadmin + namespace: flyte +rules: +- apiGroups: + - "" + - flyte.lyft.com + - rbac.authorization.k8s.io + resources: + - configmaps + - flyteworkflows + - namespaces + - pods + - roles + - rolebindings + - secrets + - services + - serviceaccounts + - spark-role + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flytepropeller +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - delete + - patch +- apiGroups: + - '*' + resources: + - '*' + verbs: + - get + - list + - watch + - create + - update + - delete + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - create + - delete + - update +- apiGroups: + - flyte.lyft.com + resources: + - flyteworkflows + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - post + - deletecollection +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: contour +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - contour.heptio.com + resources: + - ingressroutes + verbs: + - get + - list + - watch + - put + - post + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: sparkoperator +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - '*' +- apiGroups: + - "" + resources: + - services + - configmaps + verbs: + - create + - get + - delete +- apiGroups: + - extensions + resources: + - ingresses + verbs: + - create + - get + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - get + - update + - delete + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - update + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - update + - delete +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create + - get + - update + - delete +- apiGroups: + - sparkoperator.k8s.io + resources: + - sparkapplications + - scheduledsparkapplications + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flyteadmin-binding + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyteadmin +subjects: +- kind: ServiceAccount + name: flyteadmin + namespace: flyte +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: flytepropeller + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flytepropeller +subjects: +- kind: ServiceAccount + name: flytepropeller + namespace: flyte +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: contour +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: contour +subjects: +- kind: ServiceAccount + name: contour + namespace: heptio-contour +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: sparkoperator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sparkoperator +subjects: +- kind: ServiceAccount + name: sparkoperator + namespace: sparkoperator +--- +apiVersion: v1 +data: + aa_namespace.yaml: | + apiVersion: v1 + kind: Namespace + metadata: + name: {{ namespace }} + spec: + finalizers: + - kubernetes +kind: ConfigMap +metadata: + name: clusterresource-template-dgc9fcm2kh + namespace: flyte +--- +apiVersion: v1 +data: + datacatalog_config.yaml: | + logger: + show-source: true + level: 5 + datacatalog: + storage-prefix: metadata/datacatalog + metrics-scope: "datacatalog" + profiler-port: 10254 + application: + grpcPort: 8089 + storage: + connection: + access-key: minio + auth-type: accesskey + disable-ssl: true + endpoint: http://minio.flyte.svc.cluster.local:9000 + region: us-east-1 + secret-key: miniostorage + cache: + max_size_mbs: 10 + target_gc_percent: 100 + container: my-container + type: minio + database: + port: 5432 + username: postgres + host: postgres + dbname: datacatalog + options: sslmode=disable +kind: ConfigMap +metadata: + name: datacatalog-config-5g5bbg2k49 + namespace: flyte +--- +apiVersion: v1 +data: + flyteadmin_config.yaml: | + logger: + show-source: true + level: 5 + application: + httpPort: 8088 + grpcPort: 8089 + flyteadmin: + roleNameKey: "iam.amazonaws.com/role" + profilerPort: 10254 + metricsScope: "flyte:" + metadataStoragePrefix: + - "metadata" + - "admin" + testing: + host: http://flyteadmin + database: + # Create a database like postgres and override these values + port: 5432 + username: postgres + password: awesomesauce + # Recommended to use passwordPath and mount it using kubescrets or the like + # passwordPath: "/var/run/CREDENTIALS_DB_PASSWORD" + # host here is the private ip address of the CloudSQL Db in private mode + host: 10.23.0.3 + dbname: flyte + options: sslmode=disable + storage: + type: stow + stow: + kind: google + config: + scopes: "" + project_id: flyte-sandbox + json: "" + container: "flyte-sandbox" + task_resources: + defaults: + cpu: 200m + gpu: 0 + memory: 500Mi + storage: 100Mi + limits: + cpu: 62 + gpu: 8 + memory: 256Gi + storage: 5Gi + domains: + - id: development + name: development + - id: staging + name: staging + - id: production + name: production + - id: domain + name: domain + registration: + maxWorkflowNodes: 100 + scheduler: + eventScheduler: + scheme: local + region: "my-region" + scheduleRole: "arn:aws:iam::abc123:role/my-iam-role" + targetName: "arn:aws:sqs:my-region:abc123:my-queue" + workflowExecutor: + scheme: local + region: "my-region" + scheduleQueueName: "won't-work-locally" + accountId: "abc123" + notifications: + type: local + region: "us-east-1" + publisher: + topicName: "foo" + processor: + queueName: "queue" + accountId: "bar" + emailer: + subject: "Notice: Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\"." + sender: "flyte-notifications@lyft.com" + body: > + Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\". View details at + + http://flyte.lyft.net/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. {{ error }} + cluster_resources: + templatePath: "/etc/flyte/clusterresource/templates" + refresh: 5m +kind: ConfigMap +metadata: + name: flyte-admin-config-228b7f76mk + namespace: flyte +--- +apiVersion: v1 +data: + BASE_URL: /console + CONFIG_DIR: /etc/flyte/config +kind: ConfigMap +metadata: + name: flyte-console-config + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: "" +kind: ConfigMap +metadata: + name: flyte-container-config-5k78b9cm42 + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: | + plugins: + enabled-plugins: + - container + - spark + - waitable + - hiveExecutor + - sidecar + logs: + # Log links can link to multiple options + # #1 Kubernetes dashboard + kubernetes-enabled: false + # #2 GCP stackdriver + stackdriver-enabled: true + gcp-project: flyte-sandbox + stackdriver-logresourcename: flyte + k8s: + default-annotations: + # Example annotation that will be applied to every k8s resource launched + - flyte.lyft.net/deployment: base-google-gke + # Example Environment variables that will be applied to every container executed on k8s + default-env-vars: + - FLYTE_CLOUD_PLATFORM: google +kind: ConfigMap +metadata: + name: flyte-plugin-config-hkh64698kg + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: | + propeller: + metadata-prefix: metadata/propeller + workers: 4 + max-workflow-retries: 30 + workflow-reeval-duration: 30s + downstream-eval-duration: 30s + limit-namespace: "all" + prof-port: 10254 + metrics-prefix: flyte + enable-admin-launcher: true + leader-election: + lock-config-map: + name: propeller-leader + namespace: flyte + enabled: true + lease-duration: 15s + renew-deadline: 10s + retry-period: 2s + queue: + type: batch + batching-interval: 2s + batch-size: -1 + queue: + type: bucket + rate: 10 + capacity: 100 + sub-queue: + type: bucket + rate: 10 + capacity: 100 + logger: + show-source: true + level: 5 + storage: + type: stow + stow: + kind: google + config: + scopes: "" + project_id: flyte-sandbox + json: "" + container: "flyte-sandbox" + event: + type: admin + rate: 500 + capacity: 1000 + admin: + endpoint: flyteadmin:81 + insecure: true + # TODO may be we should disable catalog cache in the default? + catalog-cache: + endpoint: datacatalog:89 + type: datacatalog + insecure: true +kind: ConfigMap +metadata: + name: flyte-propeller-config-96k59km44m + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: | + plugins: + qubole: + # Either create this file with your username with the real token, or set the QUBOLE_API_KEY environment variable + # See the secrets_manager.go file in the plugins repo for usage. Since the dev/test deployment of + # this has a dummy QUBOLE_API_KEY env var built in, this fake path won't break anything. + quboleTokenPath: "/Path/To/QUBOLE_CLIENT_TOKEN" + resourceManagerType: redis + redisHostPath: redis-resource-manager.flyte:6379 + redisHostKey: mypassword + quboleLimit: 10 +kind: ConfigMap +metadata: + name: flyte-qubole-config-h626c9t6f7 + namespace: flyte +--- +apiVersion: v1 +data: + config.yaml: | + plugins: + spark: + spark-config-default: + - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" + - spark.kubernetes.allocation.batch.size: "50" + - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" + - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" + - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" + - spark.hadoop.fs.s3a.multipart.threshold: "536870912" + - spark.blacklist.enabled: "true" + - spark.blacklist.timeout: "5m" + - spark.task.maxfailures: "8" +kind: ConfigMap +metadata: + name: flyte-spark-config-h72bmtggc2 + namespace: flyte +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + contour.heptio.com/upstream-protocol.h2c: grpc + name: datacatalog + namespace: flyte +spec: + ports: + - name: http + port: 88 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 89 + protocol: TCP + targetPort: 8089 + selector: + app: datacatalog +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + cloud.google.com/load-balancer-type: Internal + contour.heptio.com/upstream-protocol.h2c: grpc + name: flyteadmin + namespace: flyte +spec: + ports: + - name: redoc + port: 87 + protocol: TCP + targetPort: 8087 + - name: http + port: 80 + protocol: TCP + targetPort: 8088 + - name: grpc + port: 81 + protocol: TCP + targetPort: 8089 + selector: + app: flyteadmin + type: LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + cloud.google.com/load-balancer-type: Internal + name: flyteconsole + namespace: flyte +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: flyteconsole + type: LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: flyte +spec: + externalName: minio + ports: + - port: 9000 + selector: + app: minio +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: flyte +spec: + ports: + - port: 5432 + selector: + app: postgres +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app: redis-resource-manager + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: contour + name: contour + namespace: heptio-contour +spec: + ports: + - nodePort: 30081 + port: 80 + protocol: TCP + selector: + app: contour + type: NodePort +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: datacatalog + name: datacatalog + namespace: flyte +spec: + replicas: 1 + selector: + matchLabels: + app: datacatalog + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: datacatalog + spec: + containers: + - command: + - datacatalog + - --logtostderr + - --config + - /etc/datacatalog/config/datacatalog_config.yaml + - serve + image: docker.io/lyft/datacatalog:v0.1.1 + imagePullPolicy: IfNotPresent + name: datacatalog + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: + limits: + cpu: "0.1" + ephemeral-storage: 100Mi + memory: 200Mi + volumeMounts: + - mountPath: /etc/datacatalog/config + name: config-volume + initContainers: + - command: + - sh + - -c + - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep + 2; done; + image: postgres:10.1 + name: check-db-ready + volumeMounts: + - mountPath: /etc/datacatalog/config + name: config-volume + - command: + - datacatalog + - --logtostderr + - --config + - /etc/datacatalog/config/datacatalog_config.yaml + - migrate + - run + image: docker.io/lyft/datacatalog:v0.1.1 + imagePullPolicy: IfNotPresent + name: run-migrations + volumeMounts: + - mountPath: /etc/datacatalog/config + name: config-volume + serviceAccountName: datacatalog + volumes: + - emptyDir: {} + name: shared-data + - configMap: + name: datacatalog-config-5g5bbg2k49 + name: config-volume +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: flyteadmin + name: flyteadmin + namespace: flyte +spec: + replicas: 1 + selector: + matchLabels: + app: flyteadmin + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: flyteadmin + spec: + containers: + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - serve + image: docker.io/lyft/flyteadmin:v0.1.4 + imagePullPolicy: IfNotPresent + name: flyteadmin + ports: + - containerPort: 8088 + - containerPort: 8089 + resources: + limits: + cpu: "0.1" + ephemeral-storage: 100Mi + memory: 200Mi + volumeMounts: + - mountPath: /srv/flyte + name: shared-data + - mountPath: /etc/flyte/config + name: config-volume + - command: + - sh + - -c + - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh + env: + - name: PAGE_TITLE + value: Flyte Admin OpenAPI + - name: SPEC_URL + value: /api/v1/openapi + - name: PORT + value: "8087" + image: docker.io/redocly/redoc + imagePullPolicy: IfNotPresent + name: redoc + ports: + - containerPort: 8087 + resources: + limits: + cpu: "0.1" + memory: 200Mi + initContainers: + - command: + - sh + - -c + - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep + 2; done; + image: postgres:10.1 + name: check-db-ready + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - migrate + - run + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: run-migrations + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - migrate + - seed-projects + - flytesnacks + - flytetester + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: seed-projects + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - clusterresource + - sync + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: sync-cluster-resources + volumeMounts: + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + serviceAccountName: flyteadmin + volumes: + - configMap: + name: clusterresource-template-dgc9fcm2kh + name: resource-templates + - emptyDir: {} + name: shared-data + - configMap: + name: flyte-admin-config-228b7f76mk + name: config-volume +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: flyteconsole + name: flyteconsole + namespace: flyte +spec: + replicas: 1 + selector: + matchLabels: + app: flyteconsole + template: + metadata: + labels: + app: flyteconsole + spec: + containers: + - envFrom: + - configMapRef: + name: flyte-console-config + image: docker.io/lyft/flyteconsole:v0.1.0 + name: flyteconsole + ports: + - containerPort: 8080 + resources: + limits: + cpu: "0.1" + ephemeral-storage: 100Mi + memory: 150Mi + volumeMounts: + - mountPath: /srv/flyte + name: shared-data + volumes: + - emptyDir: {} + name: shared-data +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: flytepropeller + name: flytepropeller + namespace: flyte +spec: + selector: + matchLabels: + app: flytepropeller + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: flytepropeller + app.kubernetes.io/name: flytepropeller + app.kubernetes.io/version: 0.1.9 + spec: + containers: + - args: + - --config + - /etc/flyte/config*/config.yaml + command: + - flytepropeller + env: + - name: QUBOLE_API_KEY + value: notarealkey + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: docker.io/lyft/flytepropeller:v0.1.9 + imagePullPolicy: IfNotPresent + name: flytepropeller + ports: + - containerPort: 10254 + resources: + limits: + cpu: "0.1" + ephemeral-storage: 100Mi + memory: 100Mi + volumeMounts: + - mountPath: /etc/flyte/config-qubole + name: qubole-config-volume + - mountPath: /etc/flyte/config-container + name: container-config-volume + - mountPath: /etc/flyte/config-spark + name: spark-config-volume + - mountPath: /etc/flyte/config + name: config-volume + - mountPath: /etc/flyte/config-plugin + name: plugin-config-volume + serviceAccountName: flytepropeller + volumes: + - configMap: + name: flyte-qubole-config-h626c9t6f7 + name: qubole-config-volume + - configMap: + name: flyte-container-config-5k78b9cm42 + name: container-config-volume + - configMap: + name: flyte-spark-config-h72bmtggc2 + name: spark-config-volume + - configMap: + name: flyte-propeller-config-96k59km44m + name: config-volume + - configMap: + name: flyte-plugin-config-hkh64698kg + name: plugin-config-volume +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: flyte +spec: + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - args: + - server + - /data + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: miniostorage + image: minio/minio:RELEASE.2019-06-04T01-15-58Z + name: minio + ports: + - containerPort: 9000 + name: minio + volumeMounts: + - mountPath: /data + name: minio-storage + volumes: + - emptyDir: {} + name: minio-storage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: flyte +spec: + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - image: postgres:10.1 + name: postgres + ports: + - containerPort: 5432 + name: postgres + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: postgres-storage + volumes: + - emptyDir: {} + name: postgres-storage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: contour + name: contour + namespace: heptio-contour +spec: + replicas: 1 + selector: + matchLabels: + app: contour + template: + metadata: + annotations: + prometheus.io/format: prometheus + prometheus.io/path: /stats + prometheus.io/port: "8002" + prometheus.io/scrape: "true" + labels: + app: contour + spec: + containers: + - args: + - -c + - /config/contour.yaml + - --service-cluster + - cluster0 + - --service-node + - node0 + command: + - envoy + image: docker.io/envoyproxy/envoy-alpine:v1.6.0 + name: envoy-envoyingressv1 + ports: + - containerPort: 80 + name: http + - containerPort: 8002 + name: statsd + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - mountPath: /config + name: contour-config + - args: + - serve + - --incluster + - --envoy-http-port=80 + - --debug-http-port=6069 + command: + - contour + image: gcr.io/heptio-images/contour:v0.6.1 + imagePullPolicy: Always + name: contour-unknown + ports: + - containerPort: 8000 + name: contour + dnsPolicy: ClusterFirst + initContainers: + - args: + - bootstrap + - /config/contour.yaml + - --statsd-enabled + command: + - contour + image: gcr.io/heptio-images/contour:v0.6.1 + imagePullPolicy: Always + name: envoy-initconfig + volumeMounts: + - mountPath: /config + name: contour-config + serviceAccountName: contour + terminationGracePeriodSeconds: 30 + volumes: + - emptyDir: {} + name: contour-config +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/version: v2.4.0-v1alpha1 + name: sparkoperator + namespace: sparkoperator +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/version: v2.4.0-v1alpha1 + strategy: + type: Recreate + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + initializers: + pending: [] + labels: + app.kubernetes.io/name: sparkoperator + app.kubernetes.io/version: v2.4.0-v1alpha1 + spec: + containers: + - args: + - -logtostderr + - -v=2 + - -controller-threads=20 + - -enable-metrics=true + - '-metrics-prefix=service:' + - -metrics-labels=task_name + - -metrics-labels=workflow_name + command: + - /usr/bin/spark-operator + image: gcr.io/spark-operator/spark-operator:v2.4.0-v1beta1-0.9.0 + imagePullPolicy: Always + name: sparkoperator-unknown + ports: + - containerPort: 10254 + serviceAccountName: sparkoperator +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: redis + namespace: flyte +spec: + replicas: 1 + selector: + matchLabels: + app: redis-resource-manager + serviceName: redis-resource-manager + template: + metadata: + labels: + app: redis-resource-manager + spec: + containers: + - env: + - name: REDIS_PASSWORD + value: mypassword + image: docker.io/bitnami/redis:4.0.2-r1 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: redis-resource-manager + ports: + - containerPort: 6379 + name: redis + protocol: TCP + readinessProbe: + exec: + command: + - redis-cli + - ping + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 200m + memory: 128Mi + volumeMounts: + - mountPath: /bitnami + name: redis-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - emptyDir: {} + name: redis-data +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: syncresources + namespace: flyte +spec: + jobTemplate: + spec: + template: + spec: + containers: + - command: + - flyteadmin + - --logtostderr + - --config + - /etc/flyte/config/flyteadmin_config.yaml + - clusterresource + - sync + image: docker.io/lyft/flyteadmin:v0.1.1 + imagePullPolicy: IfNotPresent + name: sync-cluster-resources + volumeMounts: + - mountPath: /etc/flyte/clusterresource/templates + name: resource-templates + - mountPath: /etc/flyte/config + name: config-volume + restartPolicy: OnFailure + serviceAccountName: flyteadmin + volumes: + - configMap: + name: clusterresource-template-dgc9fcm2kh + name: resource-templates + - configMap: + name: flyte-admin-config-228b7f76mk + name: config-volume + schedule: '*/1 * * * *' +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" + name: flytesystem + namespace: flyte +spec: + rules: + - http: + paths: + - backend: + serviceName: flyteconsole + servicePort: 80 + path: /console + - backend: + serviceName: flyteconsole + servicePort: 80 + path: /__webpack_hmr + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /api + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /healthcheck + - backend: + serviceName: flyteadmin + servicePort: 80 + path: /v1 + - backend: + serviceName: flyteadmin + servicePort: 81 + path: /flyteidl.service.AdminService + - backend: + serviceName: flyteadmin + servicePort: 87 + path: /openapi +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" + name: minio + namespace: flyte +spec: + rules: + - http: + paths: + - backend: + serviceName: minio + servicePort: 9000 + path: /minio diff --git a/kustomize/overlays/gke-single-cluster/README.md b/kustomize/overlays/gke-single-cluster/README.md index f6a238b796..d94538964e 100644 --- a/kustomize/overlays/gke-single-cluster/README.md +++ b/kustomize/overlays/gke-single-cluster/README.md @@ -7,7 +7,7 @@ SQL Database Create a SQL database (Postgres) https://cloud.google.com/sql/docs/postgres/create-instance -Enable the the SQL server to be accessed from the GKE cluster that will host the FlyteAdmin service. This can be done using private networking mode and associating the shared network +Enable the the SQL server to be accessed from the GKE cluster that will host the FlyteAdmin service. This can be done using private networking mode and associating the shared network. Create a database called "flyte" in this DB instance @@ -30,7 +30,7 @@ gcloud iam service-accounts add-iam-policy-binding --role roles/iam.workloadIden kubectl annotate serviceaccount --namespace flytekit-development flyte-sandbox iam.gke.io/gcp-service-account=flyte-sandbox@flyte-sandbox.iam.gserviceaccount.com -IAM For FLyte components +IAM For Flyte components ------------------------ Create the right service accounts in GKE cluster's flyte namespace and then add the serviceaccountname to propeller and flyteadmin deployments. You may also want to add it to the various plugin deployments. diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml index ca4add7306..b46c8047ec 100644 --- a/kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml +++ b/kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml @@ -37,7 +37,7 @@ spec: imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: - - name: resource-templates + - name: resource-templates mountPath: /etc/flyte/clusterresource/templates - name: config-volume mountPath: /etc/flyte/config diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml index 6274ac4c75..120c8feb9f 100644 --- a/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml +++ b/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml @@ -20,7 +20,7 @@ database: password: awesomesauce # Recommended to use passwordPath and mount it using kubescrets or the like # passwordPath: "/var/run/CREDENTIALS_DB_PASSWORD" - # host here is the ip address of the CloudSQL Db in private mode + # host here is the private ip address of the CloudSQL Db in private mode host: 10.23.0.3 dbname: flyte options: sslmode=disable @@ -58,14 +58,14 @@ registration: scheduler: eventScheduler: scheme: local - region: "us-east-1" - scheduleRole: "arn:aws:iam::173840052742:role/mbadmin-development-scheduler" - targetName: "arn:aws:sqs:us-east-1:173840052742:flyteadmin-development-scheduler" + region: "my-region" + scheduleRole: "arn:aws:iam::abc123:role/my-iam-role" + targetName: "arn:aws:sqs:my-region:abc123:my-queue" workflowExecutor: scheme: local - region: "us-east-1" + region: "my-region" scheduleQueueName: "won't-work-locally" - accountId: "173840052742" + accountId: "abc123" notifications: type: local region: "us-east-1" diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/kustomization.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/kustomization.yaml index fc5537643e..cfb5ade69d 100644 --- a/kustomize/overlays/gke-single-cluster/admindeployment/kustomization.yaml +++ b/kustomize/overlays/gke-single-cluster/admindeployment/kustomization.yaml @@ -5,7 +5,6 @@ namespace: flyte resources: - cron.yaml -- service.yaml configMapGenerator: # the main admin configmap @@ -20,3 +19,4 @@ configMapGenerator: patches: - admindeployment.yaml +- service.yaml diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/service.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/service.yaml index 61e0e35d81..0b35cf2f46 100644 --- a/kustomize/overlays/gke-single-cluster/admindeployment/service.yaml +++ b/kustomize/overlays/gke-single-cluster/admindeployment/service.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: Service metadata: name: flyteadmin + namespace: flyte annotations: cloud.google.com/load-balancer-type: "Internal" spec: diff --git a/kustomize/overlays/gke-single-cluster/console/kustomization.yaml b/kustomize/overlays/gke-single-cluster/console/kustomization.yaml index a6957adbe2..266afedd74 100644 --- a/kustomize/overlays/gke-single-cluster/console/kustomization.yaml +++ b/kustomize/overlays/gke-single-cluster/console/kustomization.yaml @@ -3,6 +3,4 @@ bases: patches: - console.yaml - -resources: - service.yaml diff --git a/script/kustomize.sh b/script/kustomize.sh index 1b89c6199b..9016120635 100755 --- a/script/kustomize.sh +++ b/script/kustomize.sh @@ -3,8 +3,11 @@ set -ex DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" KUSTOMIZE_IMAGE="lyft/kustomizer:v3.1.0" -# flyte test +# Flyte test docker run -v "${DIR}/../kustomize":/kustomize "$KUSTOMIZE_IMAGE" kustomize build overlays/test/flyte > "${DIR}/../deployment/test/flyte_generated.yaml" -# flyte local +# Flyte Sandbox docker run -v "${DIR}/../kustomize":/kustomize "$KUSTOMIZE_IMAGE" kustomize build overlays/sandbox/flyte > "${DIR}/../deployment/sandbox/flyte_generated.yaml" + +# canonical GCP deployment on GKE +docker run -v "${DIR}/../kustomize":/kustomize "$KUSTOMIZE_IMAGE" kustomize build overlays/gke-single-cluster/flyte > "${DIR}/../deployment/gke-single-cluster/flyte_generated.yaml" From 92bbc0a8a458d8e7bed8e5c0d5b443be9cd7ef53 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Fri, 1 Nov 2019 15:57:32 -0700 Subject: [PATCH 3/6] Remove unnecessary filler values from flyteadmin sandbox config --- deployment/sandbox/flyte_generated.yaml | 25 +++---------------- deployment/test/flyte_generated.yaml | 25 +++---------------- .../admindeployment/flyteadmin_config.yaml | 19 -------------- 3 files changed, 6 insertions(+), 63 deletions(-) diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index f46c1eb38b..c464ceb449 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -556,30 +556,11 @@ data: host: postgres dbname: postgres options: sslmode=disable - scheduler: - eventScheduler: - scheme: "local" - region: "us-east-1" - scheduleRole: "arn:aws:iam::abc123:role/my-schedule-role" - targetName: "arn:aws:sqs:my-region:abc123:my-queue" - workflowExecutor: - scheme: "local" - region: "us-east-1" - scheduleQueueName: "won't-work-locally" - accountId: "173840052742" remoteData: region: "us-east-1" scheme: "local" signedUrls: durationMinutes: 3 - notifications: - type: local - region: "us-east-1" - publisher: - topicName: "foo" - processor: - queueName: "queue" - accountId: "bar" queues: executionQueues: - parent: "FlyteNonProductionTaskCPU-staging-iad" @@ -639,7 +620,7 @@ data: refresh: 5m kind: ConfigMap metadata: - name: flyte-admin-config-4gbbgmdf6d + name: flyte-admin-config-md9bt8468h namespace: flyte --- apiVersion: v1 @@ -1107,7 +1088,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-4gbbgmdf6d + name: flyte-admin-config-md9bt8468h name: config-volume --- apiVersion: apps/v1 @@ -1509,7 +1490,7 @@ spec: name: clusterresource-template-dgc9fcm2kh name: resource-templates - configMap: - name: flyte-admin-config-4gbbgmdf6d + name: flyte-admin-config-md9bt8468h name: config-volume schedule: '*/1 * * * *' --- diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 8aad9379bf..553c2be50e 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -395,30 +395,11 @@ data: host: postgres dbname: postgres options: sslmode=disable - scheduler: - eventScheduler: - scheme: "local" - region: "us-east-1" - scheduleRole: "arn:aws:iam::abc123:role/my-schedule-role" - targetName: "arn:aws:sqs:my-region:abc123:my-queue" - workflowExecutor: - scheme: "local" - region: "us-east-1" - scheduleQueueName: "won't-work-locally" - accountId: "173840052742" remoteData: region: "us-east-1" scheme: "local" signedUrls: durationMinutes: 3 - notifications: - type: local - region: "us-east-1" - publisher: - topicName: "foo" - processor: - queueName: "queue" - accountId: "bar" queues: executionQueues: - parent: "FlyteNonProductionTaskCPU-staging-iad" @@ -478,7 +459,7 @@ data: refresh: 5m kind: ConfigMap metadata: - name: flyte-admin-config-4gbbgmdf6d + name: flyte-admin-config-md9bt8468h namespace: flyte --- apiVersion: v1 @@ -762,7 +743,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-4gbbgmdf6d + name: flyte-admin-config-md9bt8468h name: config-volume --- apiVersion: apps/v1 @@ -1059,7 +1040,7 @@ spec: name: clusterresource-template-dgc9fcm2kh name: resource-templates - configMap: - name: flyte-admin-config-4gbbgmdf6d + name: flyte-admin-config-md9bt8468h name: config-volume schedule: '*/1 * * * *' --- diff --git a/kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml b/kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml index 4baeda7fa0..c5b0054030 100644 --- a/kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml +++ b/kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml @@ -19,30 +19,11 @@ database: host: postgres dbname: postgres options: sslmode=disable -scheduler: - eventScheduler: - scheme: "local" - region: "us-east-1" - scheduleRole: "arn:aws:iam::abc123:role/my-schedule-role" - targetName: "arn:aws:sqs:my-region:abc123:my-queue" - workflowExecutor: - scheme: "local" - region: "us-east-1" - scheduleQueueName: "won't-work-locally" - accountId: "173840052742" remoteData: region: "us-east-1" scheme: "local" signedUrls: durationMinutes: 3 -notifications: - type: local - region: "us-east-1" - publisher: - topicName: "foo" - processor: - queueName: "queue" - accountId: "bar" queues: executionQueues: - parent: "FlyteNonProductionTaskCPU-staging-iad" From 81dd5980aa7cc2d8d8bae30828832d1c9d100733 Mon Sep 17 00:00:00 2001 From: Katrina Rogan Date: Fri, 1 Nov 2019 16:26:08 -0700 Subject: [PATCH 4/6] delete more --- deployment/sandbox/flyte_generated.yaml | 28 ++----------------- deployment/test/flyte_generated.yaml | 28 ++----------------- .../admindeployment/flyteadmin_config.yaml | 22 --------------- 3 files changed, 6 insertions(+), 72 deletions(-) diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index c464ceb449..728b85e173 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -561,28 +561,6 @@ data: scheme: "local" signedUrls: durationMinutes: 3 - queues: - executionQueues: - - parent: "FlyteNonProductionTaskCPU-staging-iad" - child: "FlyteNonProductionSubTaskCPU-staging-iad" - attributes: - - all - workflowConfigs: - - project: flytekit - tags: - - all - - project: "flytetester" - domain: "development" - workflowName: "app-workflows-rich-workflow-rich-workflow" - tags: - - all - - project: "flytetester" - domain: "development" - workflowName: "app-workflows-batch-workflow-batch-tasks-workflow" - tags: - - all - - tags: - - all storage: type: minio connection: @@ -620,7 +598,7 @@ data: refresh: 5m kind: ConfigMap metadata: - name: flyte-admin-config-md9bt8468h + name: flyte-admin-config-44hkc4cmdg namespace: flyte --- apiVersion: v1 @@ -1088,7 +1066,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-md9bt8468h + name: flyte-admin-config-44hkc4cmdg name: config-volume --- apiVersion: apps/v1 @@ -1490,7 +1468,7 @@ spec: name: clusterresource-template-dgc9fcm2kh name: resource-templates - configMap: - name: flyte-admin-config-md9bt8468h + name: flyte-admin-config-44hkc4cmdg name: config-volume schedule: '*/1 * * * *' --- diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 553c2be50e..754264874f 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -400,28 +400,6 @@ data: scheme: "local" signedUrls: durationMinutes: 3 - queues: - executionQueues: - - parent: "FlyteNonProductionTaskCPU-staging-iad" - child: "FlyteNonProductionSubTaskCPU-staging-iad" - attributes: - - all - workflowConfigs: - - project: flytekit - tags: - - all - - project: "flytetester" - domain: "development" - workflowName: "app-workflows-rich-workflow-rich-workflow" - tags: - - all - - project: "flytetester" - domain: "development" - workflowName: "app-workflows-batch-workflow-batch-tasks-workflow" - tags: - - all - - tags: - - all storage: type: minio connection: @@ -459,7 +437,7 @@ data: refresh: 5m kind: ConfigMap metadata: - name: flyte-admin-config-md9bt8468h + name: flyte-admin-config-44hkc4cmdg namespace: flyte --- apiVersion: v1 @@ -743,7 +721,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-md9bt8468h + name: flyte-admin-config-44hkc4cmdg name: config-volume --- apiVersion: apps/v1 @@ -1040,7 +1018,7 @@ spec: name: clusterresource-template-dgc9fcm2kh name: resource-templates - configMap: - name: flyte-admin-config-md9bt8468h + name: flyte-admin-config-44hkc4cmdg name: config-volume schedule: '*/1 * * * *' --- diff --git a/kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml b/kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml index c5b0054030..6426b56026 100644 --- a/kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml +++ b/kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml @@ -24,28 +24,6 @@ remoteData: scheme: "local" signedUrls: durationMinutes: 3 -queues: - executionQueues: - - parent: "FlyteNonProductionTaskCPU-staging-iad" - child: "FlyteNonProductionSubTaskCPU-staging-iad" - attributes: - - all -workflowConfigs: - - project: flytekit - tags: - - all - - project: "flytetester" - domain: "development" - workflowName: "app-workflows-rich-workflow-rich-workflow" - tags: - - all - - project: "flytetester" - domain: "development" - workflowName: "app-workflows-batch-workflow-batch-tasks-workflow" - tags: - - all - - tags: - - all storage: type: minio connection: From 530d44bbc5fd03d9ae0ecb7c8d37db10e81df3f6 Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Sun, 3 Nov 2019 11:19:27 -0800 Subject: [PATCH 5/6] bumping some other admin uses to the same 0.1.4 version, make kustomize --- .../gke-single-cluster/flyte_generated.yaml | 38 +++---------------- deployment/sandbox/flyte_generated.yaml | 6 +-- deployment/test/flyte_generated.yaml | 6 +-- .../admindeployment/admindeployment.yaml | 6 +-- .../admindeployment/flyteadmin_config.yaml | 26 ------------- .../admindeployment/admindeployment.yaml | 6 +-- 6 files changed, 18 insertions(+), 70 deletions(-) diff --git a/deployment/gke-single-cluster/flyte_generated.yaml b/deployment/gke-single-cluster/flyte_generated.yaml index 1d7af1ffb7..4a35267931 100644 --- a/deployment/gke-single-cluster/flyte_generated.yaml +++ b/deployment/gke-single-cluster/flyte_generated.yaml @@ -592,38 +592,12 @@ data: name: domain registration: maxWorkflowNodes: 100 - scheduler: - eventScheduler: - scheme: local - region: "my-region" - scheduleRole: "arn:aws:iam::abc123:role/my-iam-role" - targetName: "arn:aws:sqs:my-region:abc123:my-queue" - workflowExecutor: - scheme: local - region: "my-region" - scheduleQueueName: "won't-work-locally" - accountId: "abc123" - notifications: - type: local - region: "us-east-1" - publisher: - topicName: "foo" - processor: - queueName: "queue" - accountId: "bar" - emailer: - subject: "Notice: Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\"." - sender: "flyte-notifications@lyft.com" - body: > - Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\". View details at - - http://flyte.lyft.net/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. {{ error }} cluster_resources: templatePath: "/etc/flyte/clusterresource/templates" refresh: 5m kind: ConfigMap metadata: - name: flyte-admin-config-228b7f76mk + name: flyte-admin-config-k442g2d588 namespace: flyte --- apiVersion: v1 @@ -1048,7 +1022,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -1063,7 +1037,7 @@ spec: - seed-projects - flytesnacks - flytetester - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -1076,7 +1050,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -1092,7 +1066,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-228b7f76mk + name: flyte-admin-config-k442g2d588 name: config-volume --- apiVersion: apps/v1 @@ -1489,7 +1463,7 @@ spec: name: clusterresource-template-dgc9fcm2kh name: resource-templates - configMap: - name: flyte-admin-config-228b7f76mk + name: flyte-admin-config-k442g2d588 name: config-volume schedule: '*/1 * * * *' --- diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index 728b85e173..5b726a83c6 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -1022,7 +1022,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -1037,7 +1037,7 @@ spec: - seed-projects - flytesnacks - flytetester - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -1050,7 +1050,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 754264874f..be8be1897d 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -677,7 +677,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - migrate - run - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -692,7 +692,7 @@ spec: - seed-projects - flytesnacks - flytetester - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -705,7 +705,7 @@ spec: - /etc/flyte/config/flyteadmin_config.yaml - clusterresource - sync - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml index b46c8047ec..e02d5b06fe 100644 --- a/kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml +++ b/kustomize/overlays/gke-single-cluster/admindeployment/admindeployment.yaml @@ -17,7 +17,7 @@ spec: 'until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done;'] - name: run-migrations - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"] @@ -25,7 +25,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: seed-projects - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "seed-projects", "flytesnacks", "flytetester"] @@ -33,7 +33,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml index 120c8feb9f..5ff9275fae 100644 --- a/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml +++ b/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml @@ -55,32 +55,6 @@ domains: name: domain registration: maxWorkflowNodes: 100 -scheduler: - eventScheduler: - scheme: local - region: "my-region" - scheduleRole: "arn:aws:iam::abc123:role/my-iam-role" - targetName: "arn:aws:sqs:my-region:abc123:my-queue" - workflowExecutor: - scheme: local - region: "my-region" - scheduleQueueName: "won't-work-locally" - accountId: "abc123" -notifications: - type: local - region: "us-east-1" - publisher: - topicName: "foo" - processor: - queueName: "queue" - accountId: "bar" - emailer: - subject: "Notice: Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\"." - sender: "flyte-notifications@lyft.com" - body: > - Execution \"{{ name }}\" has {{ phase }} in \"{{ domain }}\". View details at - - http://flyte.lyft.net/projects/{{ project }}/domains/{{ domain }}/executions/{{ name }}. {{ error }} cluster_resources: templatePath: "/etc/flyte/clusterresource/templates" refresh: 5m diff --git a/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml b/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml index ca4add7306..5c28822aa2 100644 --- a/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml +++ b/kustomize/overlays/sandbox/admindeployment/admindeployment.yaml @@ -17,7 +17,7 @@ spec: 'until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done;'] - name: run-migrations - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "run"] @@ -25,7 +25,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: seed-projects - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "migrate", "seed-projects", "flytesnacks", "flytetester"] @@ -33,7 +33,7 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: sync-cluster-resources - image: docker.io/lyft/flyteadmin:v0.1.1 + image: docker.io/lyft/flyteadmin:v0.1.4 imagePullPolicy: IfNotPresent command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/flyteadmin_config.yaml", "clusterresource", "sync"] volumeMounts: From f9d0dd4cf5766e67fc9229db9c4403c00cdc6f3a Mon Sep 17 00:00:00 2001 From: Yee Hing Tong Date: Sun, 3 Nov 2019 11:25:30 -0800 Subject: [PATCH 6/6] remove domain domain --- deployment/gke-single-cluster/flyte_generated.yaml | 8 +++----- .../admindeployment/flyteadmin_config.yaml | 2 -- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/deployment/gke-single-cluster/flyte_generated.yaml b/deployment/gke-single-cluster/flyte_generated.yaml index 4a35267931..b20d1613d9 100644 --- a/deployment/gke-single-cluster/flyte_generated.yaml +++ b/deployment/gke-single-cluster/flyte_generated.yaml @@ -588,8 +588,6 @@ data: name: staging - id: production name: production - - id: domain - name: domain registration: maxWorkflowNodes: 100 cluster_resources: @@ -597,7 +595,7 @@ data: refresh: 5m kind: ConfigMap metadata: - name: flyte-admin-config-k442g2d588 + name: flyte-admin-config-bhkhcbfmcc namespace: flyte --- apiVersion: v1 @@ -1066,7 +1064,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-k442g2d588 + name: flyte-admin-config-bhkhcbfmcc name: config-volume --- apiVersion: apps/v1 @@ -1463,7 +1461,7 @@ spec: name: clusterresource-template-dgc9fcm2kh name: resource-templates - configMap: - name: flyte-admin-config-k442g2d588 + name: flyte-admin-config-bhkhcbfmcc name: config-volume schedule: '*/1 * * * *' --- diff --git a/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml b/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml index 5ff9275fae..1643014854 100644 --- a/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml +++ b/kustomize/overlays/gke-single-cluster/admindeployment/flyteadmin_config.yaml @@ -51,8 +51,6 @@ domains: name: staging - id: production name: production - - id: domain - name: domain registration: maxWorkflowNodes: 100 cluster_resources: