fix: Remove unused/invalid "spark" field in session config proto

GoogleCloudDataproc · Dec 12, 2024 · d349b15 · d349b15
1 parent 5fd5d05
commit d349b15
Show file tree

Hide file tree

Showing 2 changed files with 0 additions and 20 deletions.
diff --git a/google/cloud/dataproc_spark_connect/session.py b/google/cloud/dataproc_spark_connect/session.py
@@ -185,12 +185,8 @@ def __create(self) -> "SparkSession":
                     dataproc_config, session_template
                 )
 
-                spark = self._get_spark(dataproc_config, session_template)
-
                 if not spark_connect_session:
                     dataproc_config.spark_connect_session = {}
-                if not spark:
-                    dataproc_config.spark = {}
                 os.environ["SPARK_CONNECT_MODE_ENABLED"] = "1"
                 session_request = CreateSessionRequest()
                 session_id = self.generate_dataproc_session_id()
@@ -435,14 +431,6 @@ def _get_spark_connect_session(self, dataproc_config, session_template):
                 spark_connect_session = session_template.spark_connect_session
             return spark_connect_session
 
-        def _get_spark(self, dataproc_config, session_template):
-            spark = None
-            if dataproc_config and dataproc_config.spark:
-                spark = dataproc_config.spark
-            elif session_template and session_template.spark:
-                spark = session_template.spark
-            return spark
-
         def generate_dataproc_session_id(self):
             timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
             suffix_length = 6

diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py
@@ -108,8 +108,6 @@ def test_create_spark_session_with_default_notebook_behavior(
           }
           spark_connect_session {
           }
-          spark {
-          }
         }
         session_id: "sc-20240702-103952-abcdef"
         """.replace(
@@ -203,8 +201,6 @@ def test_create_session_with_user_provided_dataproc_config(
           }
           spark_connect_session {
           }
-          spark {
-          }
         }
         session_id: "sc-20240702-103952-abcdef"
         """.replace(
@@ -314,8 +310,6 @@ def test_create_session_with_session_template(
           }
           spark_connect_session {
           }
-          spark {
-          }
           session_template: "projects/test-project/locations/test-region/sessionTemplates/test_template"
         }
         session_id: "sc-20240702-103952-abcdef"
@@ -422,8 +416,6 @@ def test_create_session_with_user_provided_dataproc_config_and_session_template(
           }
           spark_connect_session {
           }
-          spark {
-          }
           session_template: "projects/test-project/locations/test-region/sessionTemplates/test_template"
         }
         session_id: "sc-20240702-103952-abcdef"