[SPARK-42821][SQL] Remove unused parameters in splitFiles methods

### What changes were proposed in this pull request? The pr aims to remove unused parameters in PartitionedFileUtil.splitFiles methods ### Why are the changes needed? Make the code more concise. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass GA. Closes #40454 from panbingkun/minor_PartitionedFileUtil. Authored-by: panbingkun <[email protected]> Signed-off-by: Sean Owen <[email protected]>
apache · Nov 8, 2023 · eabea64 · eabea64
1 parent 9d93b71
commit eabea64
Show file tree

Hide file tree

Showing 3 changed files with 1 addition and 5 deletions.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -687,7 +687,7 @@ case class FileSourceScanExec(
    * @param selectedPartitions Hive-style partition that are part of the read.
    */
   private def createReadRDD(
-      readFile: (PartitionedFile) => Iterator[InternalRow],
+      readFile: PartitionedFile => Iterator[InternalRow],
       selectedPartitions: Array[PartitionDirectory]): RDD[InternalRow] = {
     val openCostInBytes = relation.sparkSession.sessionState.conf.filesOpenCostInBytes
     val maxSplitBytes =
@@ -711,7 +711,6 @@ case class FileSourceScanExec(
           val isSplitable = relation.fileFormat.isSplitable(
               relation.sparkSession, relation.options, file.getPath)
           PartitionedFileUtil.splitFiles(
-            sparkSession = relation.sparkSession,
             file = file,
             isSplitable = isSplitable,
             maxSplitBytes = maxSplitBytes,

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/PartitionedFileUtil.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/PartitionedFileUtil.scala
@@ -20,13 +20,11 @@ package org.apache.spark.sql.execution
 import org.apache.hadoop.fs.{BlockLocation, FileStatus, LocatedFileStatus}
 
 import org.apache.spark.paths.SparkPath
-import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.datasources._
 
 object PartitionedFileUtil {
   def splitFiles(
-      sparkSession: SparkSession,
       file: FileStatusWithMetadata,
       isSplitable: Boolean,
       maxSplitBytes: Long,

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/FileScan.scala
@@ -152,7 +152,6 @@ trait FileScan extends Scan
       }
       partition.files.flatMap { file =>
         PartitionedFileUtil.splitFiles(
-          sparkSession = sparkSession,
           file = file,
           isSplitable = isSplitable(file.getPath),
           maxSplitBytes = maxSplitBytes,