From ddda6d21e806dea5b000e02a12a52d38b459a251 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 26 Apr 2022 10:51:58 -0500 Subject: [PATCH 1/3] Revise 10 minutes notebook. --- docs/cudf/source/user_guide/10min.ipynb | 485 ++++++++---------------- 1 file changed, 152 insertions(+), 333 deletions(-) diff --git a/docs/cudf/source/user_guide/10min.ipynb b/docs/cudf/source/user_guide/10min.ipynb index 9bb95406e8a..b33e2b553e6 100644 --- a/docs/cudf/source/user_guide/10min.ipynb +++ b/docs/cudf/source/user_guide/10min.ipynb @@ -2484,14 +2484,6 @@ "execution_count": 35, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/mmccarty/miniconda3/envs/cudf_dev/lib/python3.8/site-packages/cudf/core/series.py:2223: FutureWarning: Series.applymap is deprecated and will be removed in a future cuDF release. Use Series.apply instead.\n", - " warnings.warn(\n" - ] - }, { "data": { "text/plain": [ @@ -2527,7 +2519,7 @@ "def add_ten(num):\n", " return num + 10\n", "\n", - "df['a'].applymap(add_ten)" + "df['a'].apply(add_ten)" ] }, { @@ -3009,88 +3001,6 @@ "merged" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Append" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Appending values from another `Series` or array-like object." - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/mmccarty/miniconda3/envs/cudf_dev/lib/python3.8/site-packages/cudf/core/indexed_frame.py:2329: FutureWarning: append is deprecated and will be removed in a future version. Use concat instead.\n", - " warnings.warn(\n" - ] - }, - { - "data": { - "text/plain": [ - "0 1\n", - "1 2\n", - "2 3\n", - "3 \n", - "4 5\n", - "0 1\n", - "1 2\n", - "2 3\n", - "3 \n", - "4 5\n", - "dtype: int64" - ] - }, - "execution_count": 45, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "s.append(s)" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 1\n", - "1 2\n", - "2 3\n", - "3 \n", - "4 5\n", - "0 1\n", - "1 2\n", - "2 3\n", - "3 \n", - "4 5\n", - "dtype: int64" - ] - }, - "execution_count": 46, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ds2.append(ds2).compute()" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -3107,7 +3017,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 45, "metadata": {}, "outputs": [], "source": [ @@ -3126,7 +3036,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 46, "metadata": {}, "outputs": [ { @@ -3189,7 +3099,7 @@ "0 100 90 100 3" ] }, - "execution_count": 48, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -3200,7 +3110,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 47, "metadata": {}, "outputs": [ { @@ -3263,7 +3173,7 @@ "0 100 90 100 3" ] }, - "execution_count": 49, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } @@ -3281,7 +3191,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 48, "metadata": {}, "outputs": [ { @@ -3360,7 +3270,7 @@ "0 1 27 30 27" ] }, - "execution_count": 50, + "execution_count": 48, "metadata": {}, "output_type": "execute_result" } @@ -3371,7 +3281,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 49, "metadata": {}, "outputs": [ { @@ -3450,7 +3360,7 @@ "0 1 27 30 27" ] }, - "execution_count": 51, + "execution_count": 49, "metadata": {}, "output_type": "execute_result" } @@ -3468,7 +3378,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 50, "metadata": {}, "outputs": [ { @@ -3527,7 +3437,7 @@ "0 19 9.0 100" ] }, - "execution_count": 52, + "execution_count": 50, "metadata": {}, "output_type": "execute_result" } @@ -3538,7 +3448,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 51, "metadata": {}, "outputs": [ { @@ -3597,7 +3507,7 @@ "0 19 9.0 100" ] }, - "execution_count": 53, + "execution_count": 51, "metadata": {}, "output_type": "execute_result" } @@ -3622,7 +3532,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 52, "metadata": {}, "outputs": [ { @@ -3677,7 +3587,7 @@ "2 3 6" ] }, - "execution_count": 54, + "execution_count": 52, "metadata": {}, "output_type": "execute_result" } @@ -3689,7 +3599,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 53, "metadata": {}, "outputs": [ { @@ -3741,7 +3651,7 @@ "b 4 5 6" ] }, - "execution_count": 55, + "execution_count": 53, "metadata": {}, "output_type": "execute_result" } @@ -3767,7 +3677,7 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 54, "metadata": {}, "outputs": [ { @@ -3828,7 +3738,7 @@ "3 2018-11-23 0.103839" ] }, - "execution_count": 56, + "execution_count": 54, "metadata": {}, "output_type": "execute_result" } @@ -3846,7 +3756,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 55, "metadata": {}, "outputs": [ { @@ -3907,7 +3817,7 @@ "3 2018-11-23 0.103839" ] }, - "execution_count": 57, + "execution_count": 55, "metadata": {}, "output_type": "execute_result" } @@ -3934,7 +3844,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 56, "metadata": {}, "outputs": [ { @@ -4007,7 +3917,7 @@ "5 6 e" ] }, - "execution_count": 58, + "execution_count": 56, "metadata": {}, "output_type": "execute_result" } @@ -4020,7 +3930,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 57, "metadata": {}, "outputs": [ { @@ -4093,7 +4003,7 @@ "5 6 e" ] }, - "execution_count": 59, + "execution_count": 57, "metadata": {}, "output_type": "execute_result" } @@ -4112,7 +4022,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 58, "metadata": {}, "outputs": [ { @@ -4121,7 +4031,7 @@ "StringIndex(['a' 'b' 'e'], dtype='object')" ] }, - "execution_count": 60, + "execution_count": 58, "metadata": {}, "output_type": "execute_result" } @@ -4139,7 +4049,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 59, "metadata": {}, "outputs": [ { @@ -4154,7 +4064,7 @@ "dtype: uint8" ] }, - "execution_count": 61, + "execution_count": 59, "metadata": {}, "output_type": "execute_result" } @@ -4165,7 +4075,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 60, "metadata": {}, "outputs": [ { @@ -4180,7 +4090,7 @@ "dtype: uint8" ] }, - "execution_count": 62, + "execution_count": 60, "metadata": {}, "output_type": "execute_result" } @@ -4213,7 +4123,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 61, "metadata": {}, "outputs": [ { @@ -4298,7 +4208,7 @@ "4 4 15 4 1 0" ] }, - "execution_count": 63, + "execution_count": 61, "metadata": {}, "output_type": "execute_result" } @@ -4309,7 +4219,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 62, "metadata": {}, "outputs": [ { @@ -4394,7 +4304,7 @@ "4 4 15 4 1 0" ] }, - "execution_count": 64, + "execution_count": 62, "metadata": {}, "output_type": "execute_result" } @@ -4419,7 +4329,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 63, "metadata": {}, "outputs": [ { @@ -4447,7 +4357,7 @@ " [19, 0, 19, 0, 0]])" ] }, - "execution_count": 65, + "execution_count": 63, "metadata": {}, "output_type": "execute_result" } @@ -4458,7 +4368,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 64, "metadata": {}, "outputs": [ { @@ -4486,7 +4396,7 @@ " [19, 0, 19, 0, 0]])" ] }, - "execution_count": 66, + "execution_count": 64, "metadata": {}, "output_type": "execute_result" } @@ -4504,7 +4414,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 65, "metadata": {}, "outputs": [ { @@ -4514,7 +4424,7 @@ " 17, 18, 19])" ] }, - "execution_count": 67, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" } @@ -4525,7 +4435,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 66, "metadata": {}, "outputs": [ { @@ -4535,7 +4445,7 @@ " 17, 18, 19])" ] }, - "execution_count": 68, + "execution_count": 66, "metadata": {}, "output_type": "execute_result" } @@ -4560,7 +4470,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 67, "metadata": {}, "outputs": [ { @@ -4580,7 +4490,7 @@ "agg_col2: [[1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0]]" ] }, - "execution_count": 69, + "execution_count": 67, "metadata": {}, "output_type": "execute_result" } @@ -4591,7 +4501,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 68, "metadata": {}, "outputs": [ { @@ -4611,7 +4521,7 @@ "agg_col2: [[1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0]]" ] }, - "execution_count": 70, + "execution_count": 68, "metadata": {}, "output_type": "execute_result" } @@ -4644,7 +4554,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -4656,7 +4566,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 70, "metadata": {}, "outputs": [], "source": [ @@ -4672,7 +4582,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 71, "metadata": {}, "outputs": [ { @@ -4892,7 +4802,7 @@ "19 19 0 19 0 0" ] }, - "execution_count": 73, + "execution_count": 71, "metadata": {}, "output_type": "execute_result" } @@ -4904,7 +4814,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 72, "metadata": {}, "outputs": [ { @@ -5124,7 +5034,7 @@ "19 19 0 19 0 0" ] }, - "execution_count": 74, + "execution_count": 72, "metadata": {}, "output_type": "execute_result" } @@ -5143,7 +5053,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 73, "metadata": {}, "outputs": [ { @@ -5543,7 +5453,7 @@ "19 19 0 19 0 0" ] }, - "execution_count": 75, + "execution_count": 73, "metadata": {}, "output_type": "execute_result" } @@ -5569,7 +5479,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 74, "metadata": {}, "outputs": [], "source": [ @@ -5585,7 +5495,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 75, "metadata": {}, "outputs": [ { @@ -5805,7 +5715,7 @@ "19 19 0 19 0 0" ] }, - "execution_count": 77, + "execution_count": 75, "metadata": {}, "output_type": "execute_result" } @@ -5824,7 +5734,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 76, "metadata": {}, "outputs": [ { @@ -5833,7 +5743,7 @@ "(None,)" ] }, - "execution_count": 78, + "execution_count": 76, "metadata": {}, "output_type": "execute_result" } @@ -5858,32 +5768,19 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 77, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'/home/mmccarty/sandbox/rapids/cudf/python/cudf/cudf/tests/data/orc/TestOrcFile.test1.orc'" - ] - }, - "execution_count": 80, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "import os\n", "from pathlib import Path\n", - "current_dir = os.path.dirname(os.path.realpath(\"__file__\"))\n", - "cudf_root = Path(current_dir).parents[3]\n", - "file_path = os.path.join(cudf_root, \"python\", \"cudf\", \"cudf\", \"tests\", \"data\", \"orc\", \"TestOrcFile.test1.orc\")\n", - "file_path" + "cudf_root = Path(\".\").absolute().parents[3]\n", + "orc_file = Path(\"python/cudf/cudf/tests/data/orc/TestOrcFile.test1.orc\")\n", + "file_path = cudf_root / orc_file" ] }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 78, "metadata": {}, "outputs": [ { @@ -5974,7 +5871,7 @@ "1 [{'key': 'chani', 'value': {'int1': 5, 'string... " ] }, - "execution_count": 81, + "execution_count": 78, "metadata": {}, "output_type": "execute_result" } @@ -6007,17 +5904,14 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 79, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2022-04-21 10:11:07,360 - distributed.diskutils - INFO - Found stale lock file and directory '/home/mmccarty/sandbox/rapids/cudf/docs/cudf/source/user_guide/dask-worker-space/worker-ghcx5g0e', purging\n", - "2022-04-21 10:11:07,360 - distributed.diskutils - INFO - Found stale lock file and directory '/home/mmccarty/sandbox/rapids/cudf/docs/cudf/source/user_guide/dask-worker-space/worker-wh16f0h3', purging\n", - "2022-04-21 10:11:07,360 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-04-21 10:11:07,388 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n" + "2022-04-26 10:47:08,571 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n" ] }, { @@ -6027,7 +5921,7 @@ "
\n", "
\n", "

Client

\n", - "

Client-e3492c89-c17c-11ec-813e-fc3497a62adc

\n", + "

Client-2152dbb9-c578-11ec-8606-b04f13036b73

\n", " \n", "\n", " \n", @@ -6056,22 +5950,22 @@ " \n", "
\n", "

LocalCUDACluster

\n", - "

db2501e1

\n", + "

11b5c3ac

\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", @@ -6093,14 +5987,14 @@ "
\n", "
\n", "

Scheduler

\n", - "

Scheduler-6f476508-e52f-49e9-8f1f-6a8641e177bd

\n", + "

Scheduler-1f527f4a-ece1-422c-9702-87f25561cfea

\n", "
\n", " Dashboard: http://127.0.0.1:8787/status\n", " \n", - " Workers: 2\n", + " Workers: 1\n", "
\n", - " Total threads: 2\n", + " Total threads: 1\n", " \n", - " Total memory: 125.65 GiB\n", + " Total memory: 125.52 GiB\n", "
\n", " \n", " \n", " \n", " \n", " \n", @@ -6108,7 +6002,7 @@ " Dashboard:http://127.0.0.1:8787/status\n", " \n", " \n", " \n", " \n", @@ -6116,7 +6010,7 @@ " Started: Just now\n", " \n", " \n", " \n", "
\n", - " Comm: tcp://127.0.0.1:39755\n", + " Comm: tcp://127.0.0.1:43105\n", " \n", - " Workers: 2\n", + " Workers: 1\n", "
\n", - " Total threads: 2\n", + " Total threads: 1\n", "
\n", - " Total memory: 125.65 GiB\n", + " Total memory: 125.52 GiB\n", "
\n", @@ -6139,7 +6033,7 @@ " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "\n", @@ -6171,61 +6065,7 @@ " GPU: NVIDIA RTX A6000\n", " \n", " \n", - " \n", - " \n", - "\n", - " \n", - "\n", - "
\n", - " Comm: tcp://127.0.0.1:33491\n", + " Comm: tcp://127.0.0.1:45093\n", " \n", " Total threads: 1\n", @@ -6147,21 +6041,21 @@ "
\n", - " Dashboard: http://127.0.0.1:34333/status\n", + " Dashboard: http://127.0.0.1:44017/status\n", " \n", - " Memory: 62.82 GiB\n", + " Memory: 125.52 GiB\n", "
\n", - " Nanny: tcp://127.0.0.1:43093\n", + " Nanny: tcp://127.0.0.1:41841\n", "
\n", - " Local directory: /home/mmccarty/sandbox/rapids/cudf/docs/cudf/source/user_guide/dask-worker-space/worker-jsuvfju4\n", + " Local directory: /home/bdice/rapids1/cudf/docs/cudf/source/user_guide/dask-worker-space/worker-8ci7x8v3\n", "
\n", - " GPU memory: 47.51 GiB\n", - "
\n", - " \n", - "
\n", - " \n", - " \n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "

Worker: 1

\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - " \n", - " \n", - " \n", - " \n", " \n", " \n", @@ -6251,10 +6091,10 @@ "" ], "text/plain": [ - "" + "" ] }, - "execution_count": 82, + "execution_count": 79, "metadata": {}, "output_type": "execute_result" } @@ -6280,7 +6120,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 80, "metadata": {}, "outputs": [ { @@ -6356,7 +6196,7 @@ "" ] }, - "execution_count": 83, + "execution_count": 80, "metadata": {}, "output_type": "execute_result" } @@ -6372,45 +6212,38 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 81, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Thu Apr 21 10:11:07 2022 \n", - "+-----------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 495.29.05 Driver Version: 495.29.05 CUDA Version: 11.5 |\n", - "|-------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|===============================+======================+======================|\n", - "| 0 NVIDIA RTX A6000 On | 00000000:01:00.0 On | Off |\n", - "| 30% 48C P2 83W / 300W | 2970MiB / 48651MiB | 7% Default |\n", - "| | | N/A |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 1 NVIDIA RTX A6000 On | 00000000:02:00.0 Off | Off |\n", - "| 30% 36C P2 25W / 300W | 265MiB / 48685MiB | 5% Default |\n", - "| | | N/A |\n", - "+-------------------------------+----------------------+----------------------+\n", - " \n", - "+-----------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=============================================================================|\n", - "| 0 N/A N/A 2292 G /usr/lib/xorg/Xorg 871MiB |\n", - "| 0 N/A N/A 2441 G /usr/bin/gnome-shell 316MiB |\n", - "| 0 N/A N/A 1240494 G ...AAAAAAAAA= --shared-files 68MiB |\n", - "| 0 N/A N/A 1240525 G ...RendererForSitePerProcess 41MiB |\n", - "| 0 N/A N/A 1243689 C .../envs/cudf_dev/bin/python 593MiB |\n", - "| 0 N/A N/A 1245502 C .../envs/cudf_dev/bin/python 753MiB |\n", - "| 0 N/A N/A 1245751 C .../envs/cudf_dev/bin/python 257MiB |\n", - "| 1 N/A N/A 2292 G /usr/lib/xorg/Xorg 4MiB |\n", - "| 1 N/A N/A 1245748 C .../envs/cudf_dev/bin/python 257MiB |\n", - "+-----------------------------------------------------------------------------+\n" + "Tue Apr 26 10:47:09 2022 \r\n", + "+-----------------------------------------------------------------------------+\r\n", + "| NVIDIA-SMI 510.47.03 Driver Version: 510.47.03 CUDA Version: 11.6 |\r\n", + "|-------------------------------+----------------------+----------------------+\r\n", + "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\r\n", + "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\r\n", + "| | | MIG M. |\r\n", + "|===============================+======================+======================|\r\n", + "| 0 NVIDIA RTX A6000 On | 00000000:65:00.0 On | Off |\r\n", + "| 30% 43C P2 76W / 300W | 1703MiB / 49140MiB | 3% Default |\r\n", + "| | | N/A |\r\n", + "+-------------------------------+----------------------+----------------------+\r\n", + " \r\n", + "+-----------------------------------------------------------------------------+\r\n", + "| Processes: |\r\n", + "| GPU GI CI PID Type Process name GPU Memory |\r\n", + "| ID ID Usage |\r\n", + "|=============================================================================|\r\n", + "| 0 N/A N/A 1657 G 204MiB |\r\n", + "| 0 N/A N/A 1940 G 70MiB |\r\n", + "| 0 N/A N/A 37085 C 257MiB |\r\n", + "| 0 N/A N/A 42594 G 158MiB |\r\n", + "| 0 N/A N/A 47497 C 751MiB |\r\n", + "| 0 N/A N/A 47555 C 257MiB |\r\n", + "+-----------------------------------------------------------------------------+\r\n" ] } ], @@ -6427,7 +6260,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 82, "metadata": {}, "outputs": [ { @@ -6503,7 +6336,7 @@ "" ] }, - "execution_count": 85, + "execution_count": 82, "metadata": {}, "output_type": "execute_result" } @@ -6515,45 +6348,38 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 83, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Thu Apr 21 10:11:08 2022 \n", - "+-----------------------------------------------------------------------------+\n", - "| NVIDIA-SMI 495.29.05 Driver Version: 495.29.05 CUDA Version: 11.5 |\n", - "|-------------------------------+----------------------+----------------------+\n", - "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n", - "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n", - "| | | MIG M. |\n", - "|===============================+======================+======================|\n", - "| 0 NVIDIA RTX A6000 On | 00000000:01:00.0 On | Off |\n", - "| 30% 48C P2 84W / 300W | 2970MiB / 48651MiB | 3% Default |\n", - "| | | N/A |\n", - "+-------------------------------+----------------------+----------------------+\n", - "| 1 NVIDIA RTX A6000 On | 00000000:02:00.0 Off | Off |\n", - "| 30% 36C P2 37W / 300W | 265MiB / 48685MiB | 0% Default |\n", - "| | | N/A |\n", - "+-------------------------------+----------------------+----------------------+\n", - " \n", - "+-----------------------------------------------------------------------------+\n", - "| Processes: |\n", - "| GPU GI CI PID Type Process name GPU Memory |\n", - "| ID ID Usage |\n", - "|=============================================================================|\n", - "| 0 N/A N/A 2292 G /usr/lib/xorg/Xorg 871MiB |\n", - "| 0 N/A N/A 2441 G /usr/bin/gnome-shell 316MiB |\n", - "| 0 N/A N/A 1240494 G ...AAAAAAAAA= --shared-files 68MiB |\n", - "| 0 N/A N/A 1240525 G ...RendererForSitePerProcess 41MiB |\n", - "| 0 N/A N/A 1243689 C .../envs/cudf_dev/bin/python 593MiB |\n", - "| 0 N/A N/A 1245502 C .../envs/cudf_dev/bin/python 753MiB |\n", - "| 0 N/A N/A 1245751 C .../envs/cudf_dev/bin/python 257MiB |\n", - "| 1 N/A N/A 2292 G /usr/lib/xorg/Xorg 4MiB |\n", - "| 1 N/A N/A 1245748 C .../envs/cudf_dev/bin/python 257MiB |\n", - "+-----------------------------------------------------------------------------+\n" + "Tue Apr 26 10:47:09 2022 \r\n", + "+-----------------------------------------------------------------------------+\r\n", + "| NVIDIA-SMI 510.47.03 Driver Version: 510.47.03 CUDA Version: 11.6 |\r\n", + "|-------------------------------+----------------------+----------------------+\r\n", + "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\r\n", + "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\r\n", + "| | | MIG M. |\r\n", + "|===============================+======================+======================|\r\n", + "| 0 NVIDIA RTX A6000 On | 00000000:65:00.0 On | Off |\r\n", + "| 30% 43C P2 76W / 300W | 1703MiB / 49140MiB | 2% Default |\r\n", + "| | | N/A |\r\n", + "+-------------------------------+----------------------+----------------------+\r\n", + " \r\n", + "+-----------------------------------------------------------------------------+\r\n", + "| Processes: |\r\n", + "| GPU GI CI PID Type Process name GPU Memory |\r\n", + "| ID ID Usage |\r\n", + "|=============================================================================|\r\n", + "| 0 N/A N/A 1657 G 204MiB |\r\n", + "| 0 N/A N/A 1940 G 70MiB |\r\n", + "| 0 N/A N/A 37085 C 257MiB |\r\n", + "| 0 N/A N/A 42594 G 158MiB |\r\n", + "| 0 N/A N/A 47497 C 751MiB |\r\n", + "| 0 N/A N/A 47555 C 257MiB |\r\n", + "+-----------------------------------------------------------------------------+\r\n" ] } ], @@ -6580,7 +6406,7 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 84, "metadata": {}, "outputs": [], "source": [ @@ -6592,7 +6418,7 @@ "ddf1 = dask_cudf.from_cudf(df1, npartitions=100)\n", "\n", "def func(df):\n", - " time.sleep(random.randint(1, 60))\n", + " time.sleep(random.randint(1, 10))\n", " return (df + 5) * 3 - 11" ] }, @@ -6600,12 +6426,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This function will do a basic transformation of every column in the dataframe, but the time spent in the function will vary due to the `time.sleep` statement randomly adding 1-60 seconds of time. We'll run this on every partition of our dataframe using `map_partitions`, which adds the task to our task-graph, and store the result. We can then call `persist` to force execution." + "This function will do a basic transformation of every column in the dataframe, but the time spent in the function will vary due to the `time.sleep` statement randomly adding 1-10 seconds of time. We'll run this on every partition of our dataframe using `map_partitions`, which adds the task to our task-graph, and store the result. We can then call `persist` to force execution." ] }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 85, "metadata": {}, "outputs": [], "source": [ @@ -6622,16 +6448,16 @@ }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 86, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "DoneAndNotDoneFutures(done={, , , , }, not_done=set())" + "DoneAndNotDoneFutures(done={, , , , }, not_done=set())" ] }, - "execution_count": 89, + "execution_count": 86, "metadata": {}, "output_type": "execute_result" } @@ -6646,13 +6472,6 @@ "source": [ "## With `wait`, we can safely proceed on in our workflow." ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -6672,7 +6491,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.9.12" }, "toc": { "base_numbering": 1, From 59bdba5abfe9adaef86b0a354572f6cd7e14e110 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 26 Apr 2022 15:22:43 -0500 Subject: [PATCH 2/3] Remove dask client output. --- docs/cudf/source/user_guide/10min.ipynb | 189 +----------------------- 1 file changed, 2 insertions(+), 187 deletions(-) diff --git a/docs/cudf/source/user_guide/10min.ipynb b/docs/cudf/source/user_guide/10min.ipynb index b33e2b553e6..559ad6babbd 100644 --- a/docs/cudf/source/user_guide/10min.ipynb +++ b/docs/cudf/source/user_guide/10min.ipynb @@ -5911,192 +5911,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-04-26 10:47:08,571 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n" + "2022-04-26 13:04:28,153 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n" ] - }, - { - "data": { - "text/html": [ - "
\n", - "
\n", - "
\n", - "

Client

\n", - "

Client-2152dbb9-c578-11ec-8606-b04f13036b73

\n", - "
\n", - " Comm: tcp://127.0.0.1:44033\n", - " \n", - " Total threads: 1\n", - "
\n", - " Dashboard: http://127.0.0.1:45225/status\n", - " \n", - " Memory: 62.82 GiB\n", - "
\n", - " Nanny: tcp://127.0.0.1:46529\n", - "
\n", - " Local directory: /home/mmccarty/sandbox/rapids/cudf/docs/cudf/source/user_guide/dask-worker-space/worker-zlsacw8_\n", - "
\n", - " GPU: NVIDIA RTX A6000\n", - " \n", - " GPU memory: 47.54 GiB\n", + " GPU memory: 47.99 GiB\n", "
\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - "
Connection method: Cluster objectCluster type: dask_cuda.LocalCUDACluster
\n", - " Dashboard: http://127.0.0.1:8787/status\n", - "
\n", - "\n", - " \n", - "
\n", - "

Cluster Info

\n", - "
\n", - "
\n", - "
\n", - "
\n", - "

LocalCUDACluster

\n", - "

11b5c3ac

\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - "\n", - " \n", - "
\n", - " Dashboard: http://127.0.0.1:8787/status\n", - " \n", - " Workers: 1\n", - "
\n", - " Total threads: 1\n", - " \n", - " Total memory: 125.52 GiB\n", - "
Status: runningUsing processes: True
\n", - "\n", - "
\n", - " \n", - "

Scheduler Info

\n", - " \n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - "

Scheduler

\n", - "

Scheduler-1f527f4a-ece1-422c-9702-87f25561cfea

\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
\n", - " Comm: tcp://127.0.0.1:43105\n", - " \n", - " Workers: 1\n", - "
\n", - " Dashboard: http://127.0.0.1:8787/status\n", - " \n", - " Total threads: 1\n", - "
\n", - " Started: Just now\n", - " \n", - " Total memory: 125.52 GiB\n", - "
\n", - "
\n", - "
\n", - "\n", - "
\n", - " \n", - "

Workers

\n", - " \n", - "\n", - " \n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "

Worker: 0

\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - " \n", - "\n", - "
\n", - " Comm: tcp://127.0.0.1:45093\n", - " \n", - " Total threads: 1\n", - "
\n", - " Dashboard: http://127.0.0.1:44017/status\n", - " \n", - " Memory: 125.52 GiB\n", - "
\n", - " Nanny: tcp://127.0.0.1:41841\n", - "
\n", - " Local directory: /home/bdice/rapids1/cudf/docs/cudf/source/user_guide/dask-worker-space/worker-8ci7x8v3\n", - "
\n", - " GPU: NVIDIA RTX A6000\n", - " \n", - " GPU memory: 47.99 GiB\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "\n", - "
\n", - "
\n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - " \n", - "\n", - "
\n", - "
" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 79, - "metadata": {}, - "output_type": "execute_result" } ], "source": [ @@ -6106,8 +5922,7 @@ "from dask_cuda import LocalCUDACluster\n", "\n", "cluster = LocalCUDACluster()\n", - "client = Client(cluster)\n", - "client" + "client = Client(cluster)" ] }, { From 3297f3c0fdcdcec1ba42dd03682090d77787b97d Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 12 May 2022 22:43:43 -0500 Subject: [PATCH 3/3] Sleep before second nvidia-smi call. --- docs/cudf/source/user_guide/10min.ipynb | 73 +++++++++++++------------ 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/docs/cudf/source/user_guide/10min.ipynb b/docs/cudf/source/user_guide/10min.ipynb index b57e21b31b8..b9278151e64 100644 --- a/docs/cudf/source/user_guide/10min.ipynb +++ b/docs/cudf/source/user_guide/10min.ipynb @@ -5911,7 +5911,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2022-04-26 13:04:28,153 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n" + "2022-05-12 22:41:08,024 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n" ] } ], @@ -6034,31 +6034,31 @@ "name": "stdout", "output_type": "stream", "text": [ - "Tue Apr 26 10:47:09 2022 \r\n", - "+-----------------------------------------------------------------------------+\r\n", - "| NVIDIA-SMI 510.47.03 Driver Version: 510.47.03 CUDA Version: 11.6 |\r\n", - "|-------------------------------+----------------------+----------------------+\r\n", - "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\r\n", - "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\r\n", - "| | | MIG M. |\r\n", - "|===============================+======================+======================|\r\n", - "| 0 NVIDIA RTX A6000 On | 00000000:65:00.0 On | Off |\r\n", - "| 30% 43C P2 76W / 300W | 1703MiB / 49140MiB | 3% Default |\r\n", - "| | | N/A |\r\n", - "+-------------------------------+----------------------+----------------------+\r\n", - " \r\n", - "+-----------------------------------------------------------------------------+\r\n", - "| Processes: |\r\n", - "| GPU GI CI PID Type Process name GPU Memory |\r\n", - "| ID ID Usage |\r\n", - "|=============================================================================|\r\n", - "| 0 N/A N/A 1657 G 204MiB |\r\n", - "| 0 N/A N/A 1940 G 70MiB |\r\n", - "| 0 N/A N/A 37085 C 257MiB |\r\n", - "| 0 N/A N/A 42594 G 158MiB |\r\n", - "| 0 N/A N/A 47497 C 751MiB |\r\n", - "| 0 N/A N/A 47555 C 257MiB |\r\n", - "+-----------------------------------------------------------------------------+\r\n" + "Thu May 12 22:41:08 2022 \n", + "+-----------------------------------------------------------------------------+\n", + "| NVIDIA-SMI 510.47.03 Driver Version: 510.47.03 CUDA Version: 11.6 |\n", + "|-------------------------------+----------------------+----------------------+\n", + "| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n", + "| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n", + "| | | MIG M. |\n", + "|===============================+======================+======================|\n", + "| 0 NVIDIA RTX A6000 On | 00000000:65:00.0 On | Off |\n", + "| 30% 41C P2 77W / 300W | 1380MiB / 49140MiB | 2% Default |\n", + "| | | N/A |\n", + "+-------------------------------+----------------------+----------------------+\n", + " \n", + "+-----------------------------------------------------------------------------+\n", + "| Processes: |\n", + "| GPU GI CI PID Type Process name GPU Memory |\n", + "| ID ID Usage |\n", + "|=============================================================================|\n", + "| 0 N/A N/A 1674 G 159MiB |\n", + "| 0 N/A N/A 1950 G 47MiB |\n", + "| 0 N/A N/A 13521 G 132MiB |\n", + "| 0 N/A N/A 304797 G 36MiB |\n", + "| 0 N/A N/A 488366 C 743MiB |\n", + "| 0 N/A N/A 488425 C 257MiB |\n", + "+-----------------------------------------------------------------------------+\n" ] } ], @@ -6170,7 +6170,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Tue Apr 26 10:47:09 2022 \r\n", + "Thu May 12 22:41:14 2022 \r\n", "+-----------------------------------------------------------------------------+\r\n", "| NVIDIA-SMI 510.47.03 Driver Version: 510.47.03 CUDA Version: 11.6 |\r\n", "|-------------------------------+----------------------+----------------------+\r\n", @@ -6179,7 +6179,7 @@ "| | | MIG M. |\r\n", "|===============================+======================+======================|\r\n", "| 0 NVIDIA RTX A6000 On | 00000000:65:00.0 On | Off |\r\n", - "| 30% 43C P2 76W / 300W | 1703MiB / 49140MiB | 2% Default |\r\n", + "| 30% 42C P2 77W / 300W | 1942MiB / 49140MiB | 0% Default |\r\n", "| | | N/A |\r\n", "+-------------------------------+----------------------+----------------------+\r\n", " \r\n", @@ -6188,18 +6188,19 @@ "| GPU GI CI PID Type Process name GPU Memory |\r\n", "| ID ID Usage |\r\n", "|=============================================================================|\r\n", - "| 0 N/A N/A 1657 G 204MiB |\r\n", - "| 0 N/A N/A 1940 G 70MiB |\r\n", - "| 0 N/A N/A 37085 C 257MiB |\r\n", - "| 0 N/A N/A 42594 G 158MiB |\r\n", - "| 0 N/A N/A 47497 C 751MiB |\r\n", - "| 0 N/A N/A 47555 C 257MiB |\r\n", + "| 0 N/A N/A 1674 G 159MiB |\r\n", + "| 0 N/A N/A 1950 G 47MiB |\r\n", + "| 0 N/A N/A 13521 G 132MiB |\r\n", + "| 0 N/A N/A 304797 G 36MiB |\r\n", + "| 0 N/A N/A 488366 C 743MiB |\r\n", + "| 0 N/A N/A 488425 C 819MiB |\r\n", "+-----------------------------------------------------------------------------+\r\n" ] } ], "source": [ - "!nvidia-smi" + "# Sleep to ensure the persist finishes and shows in the memory usage\n", + "!sleep 5; nvidia-smi" ] }, { @@ -6269,7 +6270,7 @@ { "data": { "text/plain": [ - "DoneAndNotDoneFutures(done={, , , , }, not_done=set())" + "DoneAndNotDoneFutures(done={, , , , }, not_done=set())" ] }, "execution_count": 86,