com.nvidia.spark.rapids.ColumnarRdd not exposed to user for XGBoost

dist/unshimmed-common-from-spark301.txt

@@ -5,6 +5,7 @@ META-INF/maven/**
 com/nvidia/spark/ExclusiveModeGpuDiscoveryPlugin*
 com/nvidia/spark/RapidsUDF*
 com/nvidia/spark/SQLPlugin*
+com/nvidia/spark/rapids/ColumnarRdd*
 com/nvidia/spark/rapids/ExecutionPlanCaptureCallback*
 com/nvidia/spark/rapids/GpuKryoRegistrator*
 com/nvidia/spark/rapids/PlanUtils*
@@ -24,4 +25,4 @@ org/apache/spark/sql/rapids/ProxyRapidsShuffleInternalManagerBase*
 org/apache/spark/sql/rapids/VisibleShuffleManager*
 org/openucx/**
 rapids/*.py
-rapids4spark-version-info.properties
+rapids4spark-version-info.properties

sql-plugin/src/main/scala/com/nvidia/spark/rapids/ColumnarRdd.scala

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,11 +16,12 @@
 
 package com.nvidia.spark.rapids
 
+import java.lang.reflect.Method
+
 import ai.rapids.cudf.Table
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.rapids.execution.InternalColumnarRddConverter
 
 /**
  * This provides a way to get back out GPU Columnar data RDD[Table]. Each Table will have the same
@@ -39,11 +40,15 @@ import org.apache.spark.sql.rapids.execution.InternalColumnarRddConverter
  * when they no longer need it.
  */
 object ColumnarRdd {
+
+  lazy val convertMethod: Method = ShimLoader.loadColumnarRDD()
+    .getDeclaredMethod("convert", classOf[DataFrame])
+
   def apply(df: DataFrame): RDD[Table] = {
-    InternalColumnarRddConverter(df)
+    convert(df)
   }
 
   def convert(df: DataFrame): RDD[Table] = {
-    InternalColumnarRddConverter.convert(df)
+    convertMethod.invoke(null, df).asInstanceOf[RDD[Table]]
   }
 }

sql-plugin/src/main/scala/com/nvidia/spark/rapids/ShimLoader.scala

@@ -289,10 +289,14 @@ object ShimLoader extends Logging {
     shimProviderClass = classname
   }
 
-  def newInstanceOf[T](className: String): T = {
+  def loadClass(className: String): Class[_] = {
     val loader = getShimClassLoader()
     logDebug(s"Loading $className using $loader with the parent loader ${loader.getParent}")
-    instantiateClass(loader.loadClass(className)).asInstanceOf[T]
+    loader.loadClass(className)
+  }
+
+  def newInstanceOf[T](className: String): T = {
+    instantiateClass(loadClass(className)).asInstanceOf[T]
   }
 
   // avoid cached constructors
@@ -343,4 +347,8 @@ object ShimLoader extends Logging {
   def newInternalExclusiveModeGpuDiscoveryPlugin(): ResourceDiscoveryPlugin = {
     newInstanceOf("com.nvidia.spark.rapids.InternalExclusiveModeGpuDiscoveryPlugin")
   }
+
+  def loadColumnarRDD(): Class[_] = {
+    loadClass("org.apache.spark.sql.rapids.execution.InternalColumnarRddConverter")
+  }
 }