fixes missing sparkContext on spark connect clusters (#191)

quinn/dataframe_helpers.py

@@ -20,19 +20,25 @@ def column_to_list(df: DataFrame, col_name: str) -> list[Any]:
     :return: List of values
     :rtype: List[Any]
     """
-    pyarrow_kv = ("spark.sql.execution.arrow.pyspark.enabled", "true")
-
     if "pyspark" not in sys.modules:
         raise ImportError
 
     # sparksession from df is not available in older versions of pyspark
     if sys.modules["pyspark"].__version__ < "3.3.0":
         return [row[0] for row in df.select(col_name).collect()]
 
-    spark_config = df.sparkSession.sparkContext.getConf().getAll()
+    spark_session = df.sparkSession.getActiveSession()
+    if spark_session is None:
+        return [row[0] for row in df.select(col_name).collect()]
+
+    pyarrow_enabled = (
+        spark_session.conf.get(
+            "spark.sql.execution.arrow.pyspark.enabled",
+        )
+        == "true"
+    )
 
-    pyarrow_enabled: bool = pyarrow_kv in spark_config
-    pyarrow_valid = pyarrow_enabled and sys.modules["pyarrow"] >= "0.17.0"
+    pyarrow_valid = pyarrow_enabled and sys.modules["pyarrow"].__version__ >= "0.17.0"
 
     pandas_exists = "pandas" in sys.modules
     pandas_valid = pandas_exists and sys.modules["pandas"].__version__ >= "0.24.2"