Open a subset of ~10k S3 files to reduce runtime

aws · Aug 18, 2022 · 579b0fb · 579b0fb
1 parent 35f39bb
commit 579b0fb
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_spark.ipynb b/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_spark.ipynb
@@ -663,8 +663,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# For the sake of time, open a subset of the thousands of files\n",
+    "num_files_to_open = 100 # len(s3_files)\n",
     "local_shap_values = pd.DataFrame()\n",
-    "for file in s3_files:\n",
+    "for file in s3_files[num_files_to_open:]:\n",
     "    output = sagemaker.s3.S3Downloader.read_file(file)\n",
     "    df = pd.read_csv(StringIO(output), sep=\",\")\n",
     "    local_shap_values = local_shap_values.append(df, ignore_index=True)"