[BEAM-13633] [Playground] Implement method to get a default example for each SDKs

playground/backend/internal/cloud_bucket/precompiled_objects.go

@@ -33,18 +33,18 @@ import (
 )
 
 const (
-	BucketName                    = "playground-precompiled-objects"
-	OutputExtension               = "output"
-	LogsExtension                 = "log"
-	GraphExtension                = "graph"
-	defaultPrecompiledObjectsInfo = "defaultPrecompiledObjects.info"
-	MetaInfoName                  = "meta.info"
-	Timeout                       = time.Minute
-	javaExtension                 = "java"
-	goExtension                   = "go"
-	pyExtension                   = "py"
-	scioExtension                 = "scala"
-	separatorsNumber              = 3
+	BucketName                   = "playground-precompiled-objects"
+	OutputExtension              = "output"
+	LogsExtension                = "log"
+	GraphExtension               = "graph"
+	defaultPrecompiledObjectInfo = "defaultPrecompiledObject.info"
+	MetaInfoName                 = "meta.info"
+	Timeout                      = time.Minute
+	javaExtension                = "java"
+	goExtension                  = "go"
+	pyExtension                  = "py"
+	scioExtension                = "scala"
+	separatorsNumber             = 3
 )
 
 type ObjectInfo struct {
@@ -69,7 +69,7 @@ type SdkToCategories map[string]CategoryToPrecompiledObjects
 // the bucket where examples are stored would be public,
 // and it has a specific structure of files, namely:
 // SDK_JAVA/
-// ----defaultPrecompiledObjects.info
+// ----defaultPrecompiledObject.info
 // ----PRECOMPILED_OBJECT_TYPE_EXAMPLE/
 // --------MinimalWordCount/
 // ----------- MinimalWordCount.java
@@ -87,7 +87,7 @@ type SdkToCategories map[string]CategoryToPrecompiledObjects
 // --------...
 // ----...
 // SDK_GO/
-// ----defaultPrecompiledObjects.info
+// ----defaultPrecompiledObject.info
 // ----PRECOMPILED_OBJECT_TYPE_EXAMPLE/
 // --------MinimalWordCount/
 // ----------- MinimalWordCount.go
@@ -100,7 +100,7 @@ type SdkToCategories map[string]CategoryToPrecompiledObjects
 // --------...
 // ----...
 //
-// defaultPrecompiledObjects.info is a file that contains path to the default example:
+// defaultPrecompiledObject.info is a file that contains path to the default example:
 // {
 //   "SDK_JAVA": "SDK_JAVA/PRECOMPILED_OBJECT_TYPE_EXAMPLE/MinimalWordCount"
 // }
@@ -273,7 +273,7 @@ func (cd *CloudStorage) GetDefaultPrecompiledObjects(ctx context.Context) (map[p
 
 // getDefaultPrecompiledObjectsPath returns path for SDK to the default precompiled object
 func (cd *CloudStorage) getDefaultPrecompiledObjectsPath(ctx context.Context, bucket *storage.BucketHandle, sdk pb.Sdk) (string, error) {
-	pathToFile := fmt.Sprintf("%s/%s", sdk.String(), defaultPrecompiledObjectsInfo)
+	pathToFile := fmt.Sprintf("%s/%s", sdk.String(), defaultPrecompiledObjectInfo)
 	rc, err := bucket.Object(pathToFile).NewReader(ctx)
 	if err != nil {
 		logger.Errorf("Object(%q).NewReader: %v", pathToFile, err.Error())

playground/infrastructure/cd_helper.py

@@ -104,44 +104,42 @@ def _save_to_cloud_storage(self, examples: List[Example]):
     self._storage_client = storage.Client()
     self._bucket = self._storage_client.bucket(Config.BUCKET_NAME)
 
-    default_examples_paths = {}
     for example in tqdm(examples):
       file_names = self._write_to_local_fs(example)
 
       if example.tag.default_example:
-        default_examples_paths[Sdk.Name(example.sdk)] = Path(
-            [*file_names].pop()).parent.__str__()
+        default_example_path = str(Path([*file_names].pop()).parent)
+        cloud_path = self._write_default_example_path_to_local_fs(
+          default_example_path)
 
-      for cloud_file_name, local_file_name in file_names.items():
         self._upload_blob(
-            source_file=local_file_name, destination_blob_name=cloud_file_name)
-
-    if len(default_examples_paths) > 0:
-      cloud_path = self._write_default_examples_paths_to_local_fs(
-          default_examples_paths)
-      self._upload_blob(
           source_file=os.path.join(Config.TEMP_FOLDER, cloud_path),
           destination_blob_name=cloud_path)
 
-  def _write_default_examples_paths_to_local_fs(self, paths: {}) -> str:
+      for cloud_file_name, local_file_name in file_names.items():
+        self._upload_blob(
+            source_file=local_file_name, destination_blob_name=cloud_file_name)
+
+  def _write_default_example_path_to_local_fs(self, path: str) -> str:
     """
-    Write paths to default examples to the file (in temp folder)
+    Write default example path to the file (in temp folder)
 
     Args:
-        paths: dict with paths
+        path: path of the default example
 
     Returns: name of the file
 
     """
-    cloud_path = os.path.join([*paths].pop(),
-                              Config.DEFAULT_PRECOMPILED_OBJECTS)
+    sdk = Path(path).parts[0]
+    cloud_path = os.path.join(sdk, Config.DEFAULT_PRECOMPILED_OBJECT)
 
-    path_to_file = os.path.join(Config.TEMP_FOLDER, [*paths].pop())
+    path_to_file = os.path.join(Config.TEMP_FOLDER, sdk)
     Path(path_to_file).mkdir(parents=True, exist_ok=True)
 
-    local_path = os.path.join(path_to_file, Config.DEFAULT_PRECOMPILED_OBJECTS)
-    content = json.dumps(paths)
-    with open(local_path, "x", encoding="utf-8") as file:
+    local_path = os.path.join(path_to_file, Config.DEFAULT_PRECOMPILED_OBJECT)
+
+    content = json.dumps({sdk: path})
+    with open(local_path, "w", encoding="utf-8") as file:
       file.write(content)
 
     return cloud_path

playground/infrastructure/config.py

@@ -43,7 +43,7 @@ class Config:
       Sdk.Name(SDK_SCIO))
   BUCKET_NAME = "playground-precompiled-objects"
   TEMP_FOLDER = "temp"
-  DEFAULT_PRECOMPILED_OBJECTS = "defaultPrecompiledObjects.info"
+  DEFAULT_PRECOMPILED_OBJECT = "defaultPrecompiledObject.info"
   SDK_TO_EXTENSION = {
       SDK_JAVA: "java", SDK_GO: "go", SDK_PYTHON: "py", SDK_SCIO: "scala"
   }

playground/infrastructure/test_cd_helper.py

@@ -14,11 +14,12 @@
 # limitations under the License.
 
 import os
+import pathlib
 import shutil
 
 import pytest
 
-from api.v1.api_pb2 import SDK_JAVA, STATUS_UNSPECIFIED, \
+from api.v1.api_pb2 import Sdk, SDK_JAVA, SDK_GO, STATUS_UNSPECIFIED, \
     PRECOMPILED_OBJECT_TYPE_UNIT_TEST
 from cd_helper import CDHelper
 from config import Config
@@ -127,3 +128,19 @@ def test__save_to_cloud_storage(mocker):
   CDHelper()._save_to_cloud_storage([example])
   write_to_os_mock.assert_called_with(example)
   upload_blob_mock.assert_called_with(source_file="", destination_blob_name="")
+
+
+def test__write_default_example_path_to_local_fs(delete_temp_folder):
+  """
+    Test writing default example link of sdk to
+    the filesystem (in temp folder)
+    Args:
+        delete_temp_folder: python fixture to clean up temp folder
+        after method execution
+  """
+  sdk = Sdk.Name(SDK_GO)
+  default_example_path = "SDK_GO\\PRECOMPILED_OBJECT_TYPE_EXAMPLE\\MinimalWordCount"
+  expected_result = str(pathlib.Path(sdk, Config.DEFAULT_PRECOMPILED_OBJECT))
+  cloud_path = CDHelper()._write_default_example_path_to_local_fs(default_example_path)
+  assert cloud_path == expected_result
+  assert os.path.exists("temp\\" + cloud_path) is True