diff --git a/learning/tour-of-beam/frontend/firebase.json b/learning/tour-of-beam/frontend/firebase.json new file mode 100644 index 000000000000..66037326c171 --- /dev/null +++ b/learning/tour-of-beam/frontend/firebase.json @@ -0,0 +1,16 @@ +{ + "hosting": { + "public": "build/web", + "ignore": [ + "firebase.json", + "**/.*", + "**/node_modules/**" + ], + "rewrites": [ + { + "source": "**", + "destination": "/index.html" + } + ] + } +} diff --git a/learning/tour-of-beam/terraform/README.md b/learning/tour-of-beam/terraform/README.md new file mode 100644 index 000000000000..24b4d83e67ea --- /dev/null +++ b/learning/tour-of-beam/terraform/README.md @@ -0,0 +1,286 @@ + +# The Tour of Beam deployment on GCP +This guide provides instructions on how to deploy the Tour of Beam environment on Google Cloud Platform (GCP) and Firebase environment. +Before starting the deployment, ensure that you have the following prerequisites in place: + +## Prerequisites: + +1. [GCP project](https://cloud.google.com/resource-manager/docs/creating-managing-projects) +2. [GCP User account](https://cloud.google.com/appengine/docs/standard/access-control?tab=python) _(Note: You will find the instruction "How to create User account" for your new project)_
+ Ensure that the account has at least following privileges: + - Cloud Datastore Owner + - Create Service Accounts + - Security Admin + - Service Account User + - Service Usage Admin + - Storage Admin + - Kubernetes Engine Cluster Viewer + +3. [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for saving deployment state + +4. An OS with the following software installed: + +* [Flutter (3.7.3 >)](https://docs.flutter.dev/get-started/install) +* [Dart SDK (2.19.2)](https://dart.dev/get-dart) +* [Firebase-tools CLI](https://www.npmjs.com/package/firebase-tools) +* [Terraform](https://www.terraform.io/downloads) +* [gcloud CLI](https://cloud.google.com/sdk/docs/install-sdk) +* [Kubectl authentication plugin](https://cloud.google.com/blog/products/containers-kubernetes/kubectl-auth-changes-in-gke) + +5. Existing Beam Playground environment + +6. Apache Beam Git repository cloned locally + +# Prepare deployment configuration: + + +1. Navigate to Apache Beam cloned repository's `beam/learning/tour-of-beam/terraform` directory + +``` +cd beam/learning/tour-of-beam/terraform +``` + +2. Configure authentication for the Google Cloud Platform (GCP). _(Note: Authentication to the GCP Project required to run gcloud commands)_
+ +``` +gcloud init + +gcloud auth application-default login +``` + +3. Configure authentication in the GCP Docker registry: +``` + gcloud auth configure-docker `chosen_region`-docker.pkg.dev +``` + +4. And the authentication in GCP Google Kubernetes Engine: _(Note: Authentication to docker and GKE required to fetch GRPC router ip:port info)_
+``` +gcloud container clusters get-credentials --region `chosen_gke_zone` `gke_name` --project `project_id` +``` + +5. Create datastore indexes: +``` +gcloud datastore indexes create ../backend/internal/storage/index.yaml +``` + +# Deploy the Tour of Beam Backend Infrastructure: + +6. Initialize terraform +``` +terraform init -backend-config="bucket=`created_gcs_bucket`" +``` + +7. Run terraform apply to create Tour-Of-Beam backend infrastructure + +``` +terraform plan -var "gcloud_init_account=$(gcloud config get-value core/account)" \ +-var "environment=prod" \ +-var "region=us-west1" \ +-var "project_id=$(gcloud config get-value project)" \ +-var "datastore_namespace=playground-datastore-namespace" \ +-var "pg_router_host=$(kubectl get svc -l app=backend-router-grpc -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}')" +``` + +``` +terraform apply -var "gcloud_init_account=$(gcloud config get-value core/account)" \ +-var "environment=prod" \ +-var "region=us-west1" \ +-var "project_id=$(gcloud config get-value project)" \ +-var "datastore_namespace=playground-datastore-namespace" \ +-var "pg_router_host=$(kubectl get svc -l app=backend-router-grpc -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}')" +``` + +Where: +- **environment** - Infrastructure environment name +- **region** - GCP region for your infrastructure +- **datastore_namespace** - Beam Playground Datastore's namespace + +# Deploy the Tour of Beam Frontend Infrastructure: + +8. Update config.dart configuration file under beam/learning/tour-of-beam/frontend/lib: + + 8.1. Navigate to beam/learning/tour-of-beam/frontend/lib. + + 8.2. Update config.dart file, replacing values in ${ } with your actual values. + +Where: +- **${cloudfunctions_region}** - region where GCP Cloud Functions have been deployed +- **${project_id}** - GCP project where infrastructure being deployed +- **${environment}** - Infrastructure environment name +- **${dns_name}** - DNS record name reserved for Beam Playground environment + +``` +const _cloudFunctionsProjectRegion = '${cloudfunctions_region}'; +const _cloudFunctionsProjectId = '${project_id}'; +const cloudFunctionsBaseUrl = 'https://' + '$_cloudFunctionsProjectRegion-$_cloudFunctionsProjectId' + '.cloudfunctions.net/${environment}_'; + + +const String kAnalyticsUA = 'UA-73650088-2'; +const String kApiClientURL = +'https://router.${dns_name}'; +const String kApiJavaClientURL = +'https://java.${dns_name}'; +const String kApiGoClientURL = +'https://go.${dns_name}'; +const String kApiPythonClientURL = +'https://python.${dns_name}'; +const String kApiScioClientURL = +'https://scio.${dns_name}'; +``` + +9. Create file .firebaserc under beam/learning/tour-of-beam/frontend + + 9.1. Navigate to beam/learning/tour-of-beam/frontend. + + 9.2. Create .firebaserc file with the following content. + +Where: +- **${project_id}** - GCP project where infrastructure being deployed + +``` +{ +"projects": { +"default": "${project_id}" + } +} +``` + +10. Login into the Firebase CLI + +``` +# To use an interactive mode (forwards to a browser webpage) +firebase login +``` + +``` +# To use non-interactive mode (generates link) +firebase login --no-localhost +``` + + +11. Create Firebase Project + +``` +firebase projects:addfirebase +``` + +12. Create Firebase Web App and prepare Firebase configuration file + +``` +firebase apps:create WEB ${webapp_name} --project=$(gcloud config get-value project) +``` + +Once Firebase Web App has been created, there will be following output example: + +``` +Create your WEB app in project cloudbuild-383310: +✔ Creating your Web app + +🎉🎉🎉 Your Firebase WEB App is ready! 🎉🎉🎉 + +App information: +- App ID: WEBAPP_ID +- Display name: WEBAPP_NAME + +You can run this command to print out your new app's Google Services config: +firebase apps:sdkconfig WEB WEBAPP_ID +``` + +Copy and paste into the terminal last line to get Web App configuration. + +Output example: + +``` +✔ Downloading configuration data of your Firebase WEB app +// Copy and paste this into your JavaScript code to initialize the Firebase SDK. +// You will also need to load the Firebase SDK. +// See https://firebase.google.com/docs/web/setup for more details. + +firebase.initializeApp({ + "projectId": "cloudbuild-384304", + "appId": "1:1111111111:web:111111111111", + "storageBucket": "cloudbuild-384304.appspot.com", + "locationId": "us-west1", + "apiKey": "someApiKey", + "authDomain": "cloudbuild-384304.firebaseapp.com", + "messagingSenderId": "111111111111" +}); +``` + +Copy the lines inside the curly braces and redact them. + +You will need to: + +1) Remove "locationId" line. +2) Remove quotes (") from key of "key": "value" pair. + 3) E.g. `projectId: "cloudbuild-384304"` +4) In overall, redacted and ready to be inserted data should be as follows: + +``` + projectId: "cloudbuild-384304", + appId: "1:1111111111:web:111111111111", + storageBucket: "cloudbuild-384304.appspot.com", + apiKey: "someApiKey", + authDomain: "cloudbuild-384304.firebaseapp.com", + messagingSenderId: "111111111111" +``` + +Paste (replace) the redacted data inside the parentheses in beam/learning/tour-of-beam/frontend/lib/firebase_options.dart file. + +``` +static const FirebaseOptions web = FirebaseOptions( + + +); +``` + +13. Run flutter and firebase commands to deploy Tour of Beam frontend + +Navigate to beam/playground/frontend/playground_components and run flutter commands + +``` +# Go to beam/playground/frontend/playground_components first +flutter pub get +flutter pub run build_runner build --delete-conflicting-outputs +``` + +Navigate to beam/learning/tour-of-beam/frontend and run flutter commands + +``` +# Go to beam/learning/tour-of-beam/frontend first +flutter pub get +flutter pub run build_runner build --delete-conflicting-outputs +flutter build web --profile --dart-define=Dart2jsOptimization=O0 +firebase deploy --project=$(gcloud config get-value project) +``` + +# Validate the deployment of the Tour of Beam: + +14. Open the Tour of Beam webpage in a web browser (Hosting URL will be provided in terminal output) to ensure that deployment has been successfully completed. + +Example: +``` +✔ Deploy complete! + +Project Console: https://console.firebase.google.com/project/some-gcp-project-id/overview +Hosting URL: https://some-gcp-project-id.web.app +``` \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/api_enable/main.tf b/learning/tour-of-beam/terraform/api_enable/main.tf new file mode 100644 index 000000000000..263a6b93b627 --- /dev/null +++ b/learning/tour-of-beam/terraform/api_enable/main.tf @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# GCP API Services to be enabled +resource "google_project_service" "required_services" { + for_each = toset([ + "cloudresourcemanager", + "iam", + "cloudbuild", + "cloudfunctions", + "firebase" + ]) + service = "${each.key}.googleapis.com" + disable_on_destroy = false +} diff --git a/learning/tour-of-beam/terraform/build.gradle.kts b/learning/tour-of-beam/terraform/build.gradle.kts new file mode 100644 index 000000000000..ed4ccc7b1094 --- /dev/null +++ b/learning/tour-of-beam/terraform/build.gradle.kts @@ -0,0 +1,364 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.pswidersk.gradle.terraform.TerraformTask +import java.io.ByteArrayOutputStream +import java.util.regex.Pattern + +plugins { + id("com.pswidersk.terraform-plugin") version "1.0.0" +} + +terraformPlugin { + terraformVersion.set("1.4.2") +} + +/* init Infrastructure for migrate */ +tasks.register("terraformInit") { + // exec args can be passed by commandline, for example + args( + "init", "-migrate-state", + "-backend-config=./state.tfbackend" + ) +} + + /* refresh Infrastucture for remote state */ +tasks.register("terraformRef") { + args( + "refresh", + "-lock=false", + "-var-file=./common.tfvars" + ) + } + +tasks.register("terraformApplyBackend") { + group = "backend-deploy" + var pg_router_host = project.extensions.extraProperties["pg_router_host"] as String + args( + "apply", + "-auto-approve", + "-lock=false", + "-parallelism=3", + "-var=pg_router_host=$pg_router_host", + "-var=gcloud_init_account=$(gcloud config get-value core/account)", + "-var=project_id=$(gcloud config get-value project)", + "-var-file=./common.tfvars" + ) + + tasks.getByName("uploadLearningMaterials").mustRunAfter(this) + + } + +tasks.register("terraformDestroy") { + var pg_router_host = project.extensions.extraProperties["pg_router_host"] as String + args( + "destroy", + "-auto-approve", + "-lock=false", + "-var=pg_router_host=$pg_router_host", + "-var=gcloud_init_account=$(gcloud config get-value core/account)", + "-var=project_id=$(gcloud config get-value project)", + "-var-file=./common.tfvars" + ) +} + +tasks.register("getRouterHost") { + group = "backend-deploy" + val result = ByteArrayOutputStream() + exec { + commandLine("kubectl", "get", "svc", "-l", "app=backend-router-grpc", "-o", "jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}:{.items[0].spec.ports[0].port}'") + standardOutput = result + } + val pg_router_host = result.toString().trim().replace("'", "") + project.extensions.extraProperties["pg_router_host"] = pg_router_host +} + +tasks.register("indexcreate") { + group = "backend-deploy" + val indexpath = "../backend/internal/storage/index.yaml" + exec { + executable("gcloud") + args("datastore", "indexes", "create", indexpath) + } +} + +tasks.register("firebaseProjectCreate") { + group = "frontend-deploy" + val result = ByteArrayOutputStream() + var project_id = project.property("project_id") as String + exec { + executable("firebase") + args("projects:list") + standardOutput = result + } + val output = result.toString().trim() + if (output.contains(project_id)) { + println("Firebase is already added to project $project_id.") + } else { + exec { + executable("firebase") + args("projects:addfirebase", project_id) + }.assertNormalExitValue() + println("Firebase has been added to project $project_id.") + } +} + +tasks.register("firebaseWebAppCreate") { + group = "frontend-deploy" + val result = ByteArrayOutputStream() + var project_id = project.property("project_id") as String + var webapp_id = project.property("webapp_id") as String + exec { + executable("firebase") + args("apps:list", "--project", project_id) + standardOutput = result + } + println(result) + val output = result.toString() + if (output.contains(webapp_id)) { + println("Webapp id $webapp_id is already created on the project: $project_id.") + val regex = Regex("$webapp_id[│ ]+([\\w:]+)[│ ]+WEB[│ ]+") + val firebaseAppId = regex.find(output)?.groupValues?.get(1)?.trim() + project.extensions.extraProperties["firebaseAppId"] = firebaseAppId + } else { + val result2 = ByteArrayOutputStream() + exec { + executable("firebase") + args("apps:create", "WEB", webapp_id, "--project", project_id) + standardOutput = result2 + }.assertNormalExitValue() + val firebaseAppId = result2.toString().lines().find { it.startsWith(" - App ID:") }?.substringAfter(":")?.trim() + project.extensions.extraProperties["firebaseAppId"] = firebaseAppId + println("Firebase app ID for newly created Firebase Web App: $firebaseAppId") + } +} + +// firebase apps:sdkconfig WEB AppId +tasks.register("getSdkConfigWebApp") { + group = "frontend-deploy" + val firebaseAppId = project.extensions.extraProperties["firebaseAppId"] as String + val result = ByteArrayOutputStream() + exec { + executable("firebase") + args("apps:sdkconfig", "WEB", firebaseAppId) + standardOutput = result + } + val output = result.toString().trim() + val pattern = Pattern.compile("\\{[^{]*\"locationId\":\\s*\".*?\"[^}]*\\}", Pattern.DOTALL) + val matcher = pattern.matcher(output) + if (matcher.find()) { + val firebaseConfigData = matcher.group().replace("{", "") + .replace("}", "") + .replace("\"locationId\":\\s*\".*?\",?".toRegex(), "") + .replace("\"(\\w+)\":".toRegex(), "$1:") + .replace(":\\s*\"(.*?)\"".toRegex(), ":\"$1\"") + project.extensions.extraProperties["firebaseConfigData"] = firebaseConfigData.trim() + println("Firebase config data: $firebaseConfigData") + } +} + +tasks.register("prepareFirebaseOptionsDart") { + group = "frontend-deploy" + val firebaseConfigData = project.extensions.extraProperties["firebaseConfigData"] as String + val file = project.file("../frontend/lib/firebase_options.dart") + val content = file.readText() + val updatedContent = content.replace(Regex("""static const FirebaseOptions web = FirebaseOptions\(([^)]+)\);"""), "static const FirebaseOptions web = FirebaseOptions(${firebaseConfigData});") + file.writeText(updatedContent) +} + +tasks.register("flutterPubGetPG") { + exec { + executable("flutter") + args("pub", "get") + workingDir("../../../playground/frontend/playground_components") + } +} + +tasks.register("flutterPubRunPG") { + exec { + executable("flutter") + args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") + workingDir("../../../playground/frontend/playground_components") + } +} + +tasks.register("flutterPubGetTob") { + exec { + executable("flutter") + args("pub", "get") + workingDir("../frontend") + } +} + +tasks.register("flutterPubRunTob") { + exec { + executable("flutter") + args("pub", "run", "build_runner", "build", "--delete-conflicting-outputs") + workingDir("../frontend") + } +} + +tasks.register("flutterBuildWeb") { + exec { + executable("flutter") + args("build", "web", "--profile", "--dart-define=Dart2jsOptimization=O0") + workingDir("../frontend") + } +} + +tasks.register("firebaseDeploy") { + var project_id = project.property("project_id") as String + exec { + commandLine("firebase", "deploy", "--project", project_id) + workingDir("../frontend") + } +} + +tasks.register("prepareConfig") { + group = "frontend-deploy" + var region = project.property("region") as String + var project_id = project.property("project_id") as String + var environment = project.property("project_environment") as String + var dns_name = project.property("dns-name") as String + val configFileName = "config.dart" + val modulePath = project(":learning:tour-of-beam:frontend").projectDir.absolutePath + val file = File("$modulePath/lib/$configFileName") + + file.writeText( + """ +const _cloudFunctionsProjectRegion = '$region'; +const _cloudFunctionsProjectId = '$project_id'; +const cloudFunctionsBaseUrl = 'https://' + '$region-$project_id' + '.cloudfunctions.net/${environment}_'; + + +const String kAnalyticsUA = 'UA-73650088-2'; +const String kApiClientURL = +'https://router.${dns_name}'; +const String kApiJavaClientURL = +'https://java.${dns_name}'; +const String kApiGoClientURL = +'https://go.${dns_name}'; +const String kApiPythonClientURL = +'https://python.${dns_name}'; +const String kApiScioClientURL = +'https://scio.${dns_name}'; +""" + ) +} + +tasks.register("prepareFirebasercConfig") { + group = "frontend-deploy" + var project_id = project.property("project_id") as String + val configFileName = ".firebaserc" + val modulePath = project(":learning:tour-of-beam:frontend").projectDir.absolutePath + val file = File("$modulePath/$configFileName") + + file.writeText( + """ +{ + "projects": { + "default": "$project_id" + } +} +""" + ) +} + +tasks.register("uploadLearningMaterials") { + var project_id = project.property("project_id") as String + group = "backend-deploy" + exec { + commandLine("go", "run", "cmd/ci_cd/ci_cd.go") + environment("DATASTORE_PROJECT_ID", project_id) + environment("GOOGLE_PROJECT_ID", project_id) + environment("TOB_LEARNING_ROOT", "../learning-content/") + workingDir("../backend") + } + dependsOn("terraformApplyBackend") + mustRunAfter("terraformApplyBackend") +} + +/* Tour of Beam backend init */ +tasks.register("InitBackend") { + group = "backend-deploy" + val getRouterHost = tasks.getByName("getRouterHost") + val indexCreate = tasks.getByName("indexcreate") + val tfInit = tasks.getByName("terraformInit") + val tfApplyBackend = tasks.getByName("terraformApplyBackend") + val uploadLearningMaterials = tasks.getByName("uploadLearningMaterials") + dependsOn(getRouterHost) + dependsOn(indexCreate) + dependsOn(tfInit) + dependsOn(tfApplyBackend) + dependsOn(uploadLearningMaterials) + indexCreate.mustRunAfter(getRouterHost) + tfInit.mustRunAfter(indexCreate) + tfApplyBackend.mustRunAfter(tfInit) + uploadLearningMaterials.mustRunAfter(tfApplyBackend) + +} + +tasks.register("DestroyBackend") { + group = "backend-destroy" + val getRouterHost = tasks.getByName("getRouterHost") + val terraformDestroy = tasks.getByName("terraformDestroy") + dependsOn(getRouterHost) + dependsOn(terraformDestroy) + terraformDestroy.mustRunAfter(getRouterHost) +} + +tasks.register("InitFrontend") { + group = "frontend-deploy" + val prepareConfig = tasks.getByName("prepareConfig") + val prepareFirebasercConfig = tasks.getByName("prepareFirebasercConfig") + val firebaseProjectCreate = tasks.getByName("firebaseProjectCreate") + val firebaseWebAppCreate = tasks.getByName("firebaseWebAppCreate") + val getSdkConfigWebApp = tasks.getByName("getSdkConfigWebApp") + val prepareFirebaseOptionsDart = tasks.getByName("prepareFirebaseOptionsDart") + val flutterPubGetPG = tasks.getByName("flutterPubGetPG") + val flutterPubRunPG = tasks.getByName("flutterPubRunPG") + val flutterPubGetTob = tasks.getByName("flutterPubGetTob") + val flutterPubRunTob = tasks.getByName("flutterPubRunTob") + val flutterBuildWeb = tasks.getByName("flutterBuildWeb") + val firebaseDeploy = tasks.getByName("firebaseDeploy") + dependsOn(prepareConfig) + dependsOn(prepareFirebasercConfig) + dependsOn(firebaseProjectCreate) + dependsOn(firebaseWebAppCreate) + dependsOn(getSdkConfigWebApp) + dependsOn(prepareFirebaseOptionsDart) + dependsOn(flutterPubGetPG) + dependsOn(flutterPubRunPG) + dependsOn(flutterPubGetTob) + dependsOn(flutterPubRunTob) + dependsOn(flutterBuildWeb) + dependsOn(firebaseDeploy) + prepareFirebasercConfig.mustRunAfter(prepareConfig) + firebaseProjectCreate.mustRunAfter(prepareFirebasercConfig) + firebaseWebAppCreate.mustRunAfter(firebaseProjectCreate) + getSdkConfigWebApp.mustRunAfter(firebaseWebAppCreate) + prepareFirebaseOptionsDart.mustRunAfter(getSdkConfigWebApp) + flutterPubGetPG.mustRunAfter(prepareFirebaseOptionsDart) + flutterPubRunPG.mustRunAfter(flutterPubGetPG) + flutterPubGetTob.mustRunAfter(flutterPubRunPG) + flutterPubRunTob.mustRunAfter(flutterPubGetTob) + flutterBuildWeb.mustRunAfter(flutterPubRunTob) + firebaseDeploy.mustRunAfter(flutterBuildWeb) +} diff --git a/learning/tour-of-beam/terraform/cloud_functions/main.tf b/learning/tour-of-beam/terraform/cloud_functions/main.tf new file mode 100644 index 000000000000..e80225115dbe --- /dev/null +++ b/learning/tour-of-beam/terraform/cloud_functions/main.tf @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# GCP Cloud Functions that will serve as a part of the backend of Tour of Beam infrastructure +resource "google_cloudfunctions_function" "cloud_function" { + count = length(var.entry_point_names) + name = "${var.environment}_${var.entry_point_names[count.index]}" + runtime = "go116" + available_memory_mb = 128 + project = var.project_id + service_account_email = var.service_account_id + source_archive_bucket = var.source_archive_bucket + source_archive_object = var.source_archive_object + region = var.region + ingress_settings = "ALLOW_ALL" + # Get the source code of the cloud function as a Zip compression + trigger_http = true + # Name of the function that will be executed when the Google Cloud Function is triggered + entry_point = var.entry_point_names[count.index] + + environment_variables = { + DATASTORE_PROJECT_ID=var.project_id + GOOGLE_PROJECT_ID=var.project_id + PLAYGROUND_ROUTER_HOST=var.pg_router_host + DATASTORE_NAMESPACE=var.datastore_namespace + } + + timeouts { + create = "20m" + delete = "20m" + } + +} + +# Create IAM entry so all users can invoke the cloud functions + +# Endpoints serve content only +# Has additional firebase authentication called "Bearer token" for endpoints that update or delete user progress +resource "google_cloudfunctions_function_iam_member" "invoker" { + count = length(google_cloudfunctions_function.cloud_function) + project = var.project_id + region = var.region + cloud_function = google_cloudfunctions_function.cloud_function[count.index].name + + role = "roles/cloudfunctions.invoker" + member = "allUsers" + + depends_on = [google_cloudfunctions_function.cloud_function] +} diff --git a/learning/tour-of-beam/terraform/cloud_functions/variables.tf b/learning/tour-of-beam/terraform/cloud_functions/variables.tf new file mode 100644 index 000000000000..c7442359de62 --- /dev/null +++ b/learning/tour-of-beam/terraform/cloud_functions/variables.tf @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Taken from output of SETUP module +variable "service_account_id" { + description = "The name of Service Account to run Cloud Function" +} + +# Required for environment variables inside of cloud functions +# Generated by command (gcloud config get-value project) in Kotlin Gradle script +variable "project_id" { + description = "The GCP Project ID of function" +} + +# GCP region +variable "region" { + description = "The GCP Region where cloud functions will be created" +} + +# Source code bucket name, used for cloud functions +# Taken from output of FUNCTIONS_BUCKETS module +variable "source_archive_bucket" { + description = "The GCS bucket containing the zip archive which contains the function" +} + +# Source code objects name, used for cloud functions +# Taken from output of FUNCTIONS_BUCKETS module +variable "source_archive_object" { + description = "The source archive object (file) in archive bucket" +} + +# Constant. Will be served as cloud functions URLs/endpoints +variable "entry_point_names" { + type = list + default = ["getSdkList", "getContentTree", "getUnitContent", "getUserProgress", "postUnitComplete", "postUserCode", "postDeleteProgress"] +} + +# Existing Playground environment's router hostname:port details +# Variable assigned using "kubectl" command +variable "pg_router_host" { + description = "Hostname:port of Playground GKE cluster's router grpc workload" +} + +# To support multi environment architecture +# Env name (e.g. test, prod, dev) +variable "environment" { + description = "The name of the environment for deployment of cloud functions. Will be appended to the name of cloud functions" +} + +variable "datastore_namespace" { + description = "The name of datastore namespace" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/functions_buckets/data.tf b/learning/tour-of-beam/terraform/functions_buckets/data.tf new file mode 100644 index 000000000000..952eaa265543 --- /dev/null +++ b/learning/tour-of-beam/terraform/functions_buckets/data.tf @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Data resource to archive source code for cloud functions +data "archive_file" "source_code" { + type = "zip" + source_dir = "../backend" + output_path = "/tmp/backend.zip" +} diff --git a/learning/tour-of-beam/terraform/functions_buckets/locals.tf b/learning/tour-of-beam/terraform/functions_buckets/locals.tf new file mode 100644 index 000000000000..ec2f63cb222e --- /dev/null +++ b/learning/tour-of-beam/terraform/functions_buckets/locals.tf @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Random string generator resource. To generate source code buckets name +resource "random_string" "id" { + length = 4 + upper = false + special = false +} + +# Variable for prefix. Used in generated source code buckets name +variable "resource_name_prefix" { + type = string + description = "The resource name prefix applied to all resource naming for the application" + default = "tour-of-beam" +} + +# Local value to store generated GCS bucket name for source code (Cloud Functions) +locals { + cloudfunctions_bucket = "${var.resource_name_prefix}-cfstorage-${random_string.id.result}" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/functions_buckets/main.tf b/learning/tour-of-beam/terraform/functions_buckets/main.tf new file mode 100644 index 000000000000..6b0992833b92 --- /dev/null +++ b/learning/tour-of-beam/terraform/functions_buckets/main.tf @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# GCS bucket for source code for cloud functions +resource "google_storage_bucket" "cloud_functions_bucket" { + name = local.cloudfunctions_bucket + location = var.region + storage_class = "STANDARD" +} + +# GCS bucket object to store source code +resource "google_storage_bucket_object" "zip" { + # Use an MD5 here. If there's no changes to the source code, this won't change either. + # We can avoid unnecessary redeployments by validating the code is unchanged, and forcing + # a redeployment when it has + name = "${data.archive_file.source_code.output_md5}.zip" + bucket = google_storage_bucket.cloud_functions_bucket.name + source = data.archive_file.source_code.output_path + content_type = "application/zip" +} diff --git a/learning/tour-of-beam/terraform/functions_buckets/output.tf b/learning/tour-of-beam/terraform/functions_buckets/output.tf new file mode 100644 index 000000000000..ee858700dce3 --- /dev/null +++ b/learning/tour-of-beam/terraform/functions_buckets/output.tf @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Outputs to be used in cloud_function module +output "functions-bucket-name" { + value = google_storage_bucket.cloud_functions_bucket.name +} + +output "function-bucket-object" { + value = google_storage_bucket_object.zip.name +} + +# Output to be used as variable for google_storage_bucket resource +output "cloudfunctions-bucket-name" { + value = local.cloudfunctions_bucket +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/functions_buckets/variables.tf b/learning/tour-of-beam/terraform/functions_buckets/variables.tf new file mode 100644 index 000000000000..f1cb931f2a2b --- /dev/null +++ b/learning/tour-of-beam/terraform/functions_buckets/variables.tf @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# GCP region where GCS bucket will be created +variable "region" { + description = "The GCP region where GCS bucket will be created (For Cloud Functions source code)" +} diff --git a/learning/tour-of-beam/terraform/main.tf b/learning/tour-of-beam/terraform/main.tf new file mode 100644 index 000000000000..ba122fc7acb8 --- /dev/null +++ b/learning/tour-of-beam/terraform/main.tf @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Setup module to create service accounts, assign required IAM roles to it and deploying user account +module "setup" { + source = "./setup" + project_id = var.project_id + gcloud_init_account = var.gcloud_init_account + depends_on = [module.api_enable] +} + +# GCS buckets to create buckets, objects, archive to store source code that cloud functions will use +module "functions_buckets" { + source = "./functions_buckets" + region = var.region + depends_on = [module.setup, module.api_enable] +} + +# API services module. Enables required APIs for the infrastructure +module "api_enable" { + source = "./api_enable" +} + +# Cloud functions module. Creates cloud functions, as part of Tour of Beam backend infrastructure +module "cloud_functions" { + source = "./cloud_functions" + region = var.region + project_id = var.project_id + pg_router_host = var.pg_router_host + environment = var.environment + datastore_namespace = var.datastore_namespace + service_account_id = module.setup.service-account-email + source_archive_bucket = module.functions_buckets.functions-bucket-name + source_archive_object = module.functions_buckets.function-bucket-object + depends_on = [module.functions_buckets, module.setup, module.api_enable] +} diff --git a/learning/tour-of-beam/terraform/provider.tf b/learning/tour-of-beam/terraform/provider.tf new file mode 100644 index 000000000000..127c7d14a4a7 --- /dev/null +++ b/learning/tour-of-beam/terraform/provider.tf @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Terraform state to be saved in GCS bucket +terraform { + backend "gcs" { + } + + required_providers { + google = { + source = "hashicorp/google" + version = "4.4.0" + } + } +} + +# GCP Provider resource +provider "google" { + project = var.project_id + region = var.region +} diff --git a/learning/tour-of-beam/terraform/setup/iam.tf b/learning/tour-of-beam/terraform/setup/iam.tf new file mode 100644 index 000000000000..245872364fd6 --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/iam.tf @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Service account for GCP Cloud Functions +resource "google_service_account" "cloud_function_sa" { + account_id = local.cloudfunctions_service_account + display_name = "Service Account to run Cloud Functions" +} + +# IAM roles for Cloud Functions service account +resource "google_project_iam_member" "terraform_service_account_roles" { + for_each = toset([ + "roles/cloudfunctions.admin", "roles/storage.objectViewer", + "roles/iam.serviceAccountUser", "roles/datastore.user", + "roles/firebaseauth.viewer" + ]) + role = each.key + member = "serviceAccount:${google_service_account.cloud_function_sa.email}" + project = var.project_id +} + +# IAM roles to be granted for user account that will be running terraform scripts +resource "google_project_iam_member" "gcloud_user_required_roles" { + for_each = toset([ + "roles/cloudfunctions.admin", "roles/firebase.admin" + ]) + role = each.key + member = "user:${var.gcloud_init_account}" + project = var.project_id +} diff --git a/learning/tour-of-beam/terraform/setup/locals.tf b/learning/tour-of-beam/terraform/setup/locals.tf new file mode 100644 index 000000000000..9817b8f2821d --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/locals.tf @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Local value to store generated Cloud Functions' Service account name + +resource "random_string" "id" { + length = 4 + upper = false + special = false +} + +variable "resource_name_prefix" { + type = string + description = "The resource name prefix applied to all resource naming for the application" + default = "tour-of-beam" +} + +locals { + cloudfunctions_service_account = "${var.resource_name_prefix}-cf-sa-${random_string.id.result}" +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/output.tf b/learning/tour-of-beam/terraform/setup/output.tf new file mode 100644 index 000000000000..bd069f1ed8d8 --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/output.tf @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Output used to assign service account to cloud functions +output "service-account-email" { + value = google_service_account.cloud_function_sa.email +} \ No newline at end of file diff --git a/learning/tour-of-beam/terraform/setup/variables.tf b/learning/tour-of-beam/terraform/setup/variables.tf new file mode 100644 index 000000000000..dcb6f54808b0 --- /dev/null +++ b/learning/tour-of-beam/terraform/setup/variables.tf @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Required and not inferred from the provider argument. +# Generated by command (gcloud config get-value project) in Kotlin Gradle script +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +# This variable is generated by command (gcloud config get-value core/account) in Kotlin Gradle script +variable "gcloud_init_account" { + description = "User Account ID logged in with gcloud init command (e.g. username@domain.com)" +} diff --git a/learning/tour-of-beam/terraform/variables.tf b/learning/tour-of-beam/terraform/variables.tf new file mode 100644 index 000000000000..0a045514dd5b --- /dev/null +++ b/learning/tour-of-beam/terraform/variables.tf @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# GCP Project ID +# Provided as a result of gcloud command +variable "project_id" { + description = "The ID of the Google Cloud project within which resources are provisioned" +} + +# GCP Region where infrastructure will be deployed +variable "region" { + description = "The region of the Google Cloud project within which resources are provisioned" +} + +# User account that will be deploying Tour of Beam infrastructure +# Provided as a result of gcloud command +variable "gcloud_init_account" { + description = "User Account ID logged in with gcloud init command (e.g. username@domain.com)" +} + +# Existing Playground router hostname:port details +# Provided as a result of kubectl command +variable "pg_router_host" { + description = "Hostname:port of Playground GKE cluster's router grpc workload" +} + +# Variable for multi-environment +# To create env (e.g. prod, dev, test) +variable "environment" { + description = "The name of the environment for deployment. Will create directory where terraform config files will be stored" +} + +variable "datastore_namespace" { + description = "The name of datastore namespace" +} \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 60e064d9241d..a334f56926b3 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -82,6 +82,9 @@ include(":runners:core-construction-java") include(":runners:core-java") include(":runners:direct-java") include(":runners:extensions-java:metrics") +include(":learning") +include(":learning:tour-of-beam") +include(":learning:tour-of-beam:terraform") /* Begin Flink Runner related settings */ // Flink 1.12 include(":runners:flink:1.12")