diff --git a/tutorial/examples/notebooks/save_to_cesnet_bucket_zarr-without_find.ipynb b/tutorial/examples/notebooks/save_to_cesnet_bucket_zarr-without_find.ipynb new file mode 100644 index 0000000..2cb0c5a --- /dev/null +++ b/tutorial/examples/notebooks/save_to_cesnet_bucket_zarr-without_find.ipynb @@ -0,0 +1,825 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "several-banking", + "metadata": {}, + "source": [ + "# Save files to EOSC (CESNET)\n", + "\n", + "\n", + "- This example show you how to save your zarr file to object storage https://object-store.cloud.muni.cz" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "another-shelf", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import s3fs\n", + "import xarray as xr\n", + "import zarr" + ] + }, + { + "cell_type": "markdown", + "id": "simple-uzbekistan", + "metadata": {}, + "source": [ + "## Get a sample file" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "heard-queen", + "metadata": {}, + "outputs": [], + "source": [ + "ds = xr.tutorial.open_dataset(\"air_temperature.nc\").rename({\"air\": \"Tair\"})" + ] + }, + { + "cell_type": "markdown", + "id": "quick-style", + "metadata": {}, + "source": [ + "## Save your results to Remote object storage\n", + "- If not done, create your credentials by follwoing [this link](../EOSC_to_bucket.md)\n", + "- Verify your credentials in `$HOME/.aws/credentials` \n", + "It should look like \n", + "\n", + "```\n", + "[default]\n", + "aws_access_key_id=xxxxx\n", + "aws_secret_access_key=yyyy\n", + "aws_endpoint_url=https://object-store.cloud.muni.cz\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "id": "ecef7710-dd4b-4bb4-9d26-f56921645181", + "metadata": {}, + "source": [ + "
\n", + " It is important to save your results in 'your' bucket. [The credential created here ](../EOSC_to_bucket.md) is a common space for pangeo-eosc cloud users. You need to not to 'over write' data on other users\n", + "
\n" + ] + }, + { + "cell_type": "markdown", + "id": "9cdeab0c-ca53-4cd8-9dd8-bccb8d1f58c0", + "metadata": {}, + "source": [ + "#### Define your s3 storage parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b72337a1-e0e4-476f-9561-587198cb985c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "s3://tmp/tinaok\n" + ] + } + ], + "source": [ + "your_name='tinaok'\n", + "path='tmp/'+your_name\n", + "s3_prefix = \"s3://\"+path\n", + "print(s3_prefix)\n", + "access_key = !aws configure get aws_access_key_id\n", + "access_key = access_key[0]\n", + "secret_key = !aws configure get aws_secret_access_key\n", + "secret_key = secret_key[0]\n", + "client_kwargs={'endpoint_url': 'https://object-store.cloud.muni.cz'}" + ] + }, + { + "cell_type": "markdown", + "id": "601619f7-8103-49f5-86bb-b7401b7e0ae6", + "metadata": {}, + "source": [ + "#### Define your s3 storage and define your zarr store" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "36918d3e-488a-4099-8f60-d2a323806636", + "metadata": {}, + "outputs": [], + "source": [ + "zarr_file_name= \"FSStore_zarr\"\n", + "uri = f\"{s3_prefix}/{zarr_file_name}\"\n", + "store = zarr.storage.FSStore(uri, client_kwargs=client_kwargs,key=access_key, secret=secret_key)" + ] + }, + { + "cell_type": "markdown", + "id": "83f3fced-5ad4-4d6d-b670-d36e1f700f11", + "metadata": { + "tags": [] + }, + "source": [ + "#### Write your file down to zarr with FSStore" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "344128c6-6776-4f46-bc42-4e0db640bd06", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/srv/conda/envs/notebook/lib/python3.9/site-packages/xarray/core/dataset.py:2060: SerializationWarning: saving variable None with floating point data as an integer dtype without any _FillValue to use for NaNs\n", + " return to_zarr( # type: ignore\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 870 ms, sys: 169 ms, total: 1.04 s\n", + "Wall time: 51.3 s\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%time ds.to_zarr(store=store, mode='w', consolidated=True)" + ] + }, + { + "cell_type": "markdown", + "id": "9b8bbdd1-63dd-49d2-ad6e-deefa54a2660", + "metadata": {}, + "source": [ + "#### Verify that your 'zarr' file is at your storage" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "99cd5138-d8b3-408d-8bfc-61915798a25d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['tmp/tinaok/FSStore_zarr']" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "target = s3fs.S3FileSystem(anon=False,client_kwargs=client_kwargs)\n", + "target.ls(path)#, detail=True, refresh=True)" + ] + }, + { + "cell_type": "markdown", + "id": "52aa8b67-7525-498d-8db2-04233b17051b", + "metadata": {}, + "source": [ + "#### Open your zarr storage" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "096e617d-b8af-40b8-9229-0090c7342de6", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:  (time: 2920, lat: 25, lon: 53)\n",
+       "Coordinates:\n",
+       "  * lat      (lat) float32 75.0 72.5 70.0 67.5 65.0 ... 25.0 22.5 20.0 17.5 15.0\n",
+       "  * lon      (lon) float32 200.0 202.5 205.0 207.5 ... 322.5 325.0 327.5 330.0\n",
+       "  * time     (time) datetime64[ns] 2013-01-01 ... 2014-12-31T18:00:00\n",
+       "Data variables:\n",
+       "    Tair     (time, lat, lon) float32 dask.array<chunksize=(730, 13, 27), meta=np.ndarray>\n",
+       "Attributes:\n",
+       "    Conventions:  COARDS\n",
+       "    description:  Data is from NMC initialized reanalysis\\n(4x/day).  These a...\n",
+       "    platform:     Model\n",
+       "    references:   http://www.esrl.noaa.gov/psd/data/gridded/data.ncep.reanaly...\n",
+       "    title:        4x daily NMC reanalysis (1948)
" + ], + "text/plain": [ + "\n", + "Dimensions: (time: 2920, lat: 25, lon: 53)\n", + "Coordinates:\n", + " * lat (lat) float32 75.0 72.5 70.0 67.5 65.0 ... 25.0 22.5 20.0 17.5 15.0\n", + " * lon (lon) float32 200.0 202.5 205.0 207.5 ... 322.5 325.0 327.5 330.0\n", + " * time (time) datetime64[ns] 2013-01-01 ... 2014-12-31T18:00:00\n", + "Data variables:\n", + " Tair (time, lat, lon) float32 dask.array\n", + "Attributes:\n", + " Conventions: COARDS\n", + " description: Data is from NMC initialized reanalysis\\n(4x/day). These a...\n", + " platform: Model\n", + " references: http://www.esrl.noaa.gov/psd/data/gridded/data.ncep.reanaly...\n", + " title: 4x daily NMC reanalysis (1948)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "xr.open_zarr(store)" + ] + }, + { + "cell_type": "markdown", + "id": "3f369705-3d91-4cad-b430-8e797d3b7330", + "metadata": {}, + "source": [ + "#### Delete the zarr file you created" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "d116d941-8ba3-40d8-aa44-9c59444971a9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['tmp/tinaok/FSStore_zarr']" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "target.ls(path)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "4083647c-0dab-4aa9-a432-6805b2d4631d", + "metadata": {}, + "outputs": [], + "source": [ + "target.rm(path+'/FSStore_zarr', recursive=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ccdbc75d-ef90-4532-8142-78a231cc881e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['tmp/fepit',\n", + " 'tmp/fepitzarr-demo',\n", + " 'tmp/guillaumeeb',\n", + " 'tmp/xarray-demo-dask-s3']" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "target.ls('tmp/')" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "2030c657-4a63-454c-8f57-aa8ab76eed59", + "metadata": {}, + "outputs": [], + "source": [ + "target.rm('tmp/fepit/xesmf_regridding.ipynb')" + ] + }, + { + "cell_type": "markdown", + "id": "1b466043-2367-4ef9-8137-e978b1f5c668", + "metadata": {}, + "source": [ + "
\n", + " one users can delete another user's file.... So please be carefull especially when using recursive=True \n", + "
\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorial/examples/notebooks/save_to_cesnet_bucket_zarr.ipynb b/tutorial/examples/notebooks/save_to_cesnet_bucket_zarr.ipynb new file mode 100644 index 0000000..f4e4e66 --- /dev/null +++ b/tutorial/examples/notebooks/save_to_cesnet_bucket_zarr.ipynb @@ -0,0 +1,837 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "several-banking", + "metadata": {}, + "source": [ + "# Save files to EOSC (CESNET)\n", + "\n", + "\n", + "- This example show you how to save your zarr file to object storage https://object-store.cloud.muni.cz" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "another-shelf", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import s3fs\n", + "import xarray as xr\n", + "import zarr" + ] + }, + { + "cell_type": "markdown", + "id": "simple-uzbekistan", + "metadata": {}, + "source": [ + "## Get a sample file" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "heard-queen", + "metadata": {}, + "outputs": [], + "source": [ + "ds = xr.tutorial.open_dataset(\"air_temperature.nc\").rename({\"air\": \"Tair\"})" + ] + }, + { + "cell_type": "markdown", + "id": "quick-style", + "metadata": {}, + "source": [ + "## Save your results to Remote object storage\n", + "- If not done, create your credentials by follwoing [this link](../EOSC_to_bucket.md)\n", + "- Verify your credentials in `$HOME/.aws/credentials` \n", + "It should look like \n", + "\n", + "```\n", + "[default]\n", + "aws_access_key_id=xxxxx\n", + "aws_secret_access_key=yyyy\n", + "aws_endpoint_url=https://object-store.cloud.muni.cz\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "id": "ecef7710-dd4b-4bb4-9d26-f56921645181", + "metadata": {}, + "source": [ + "
\n", + " It is important to save your results in 'your' bucket. [The credential created here ](../EOSC_to_bucket.md) is a common space for pangeo-eosc cloud users. You need to not to 'over write' data on other users\n", + "
\n" + ] + }, + { + "cell_type": "markdown", + "id": "9cdeab0c-ca53-4cd8-9dd8-bccb8d1f58c0", + "metadata": {}, + "source": [ + "#### Define your s3 storage parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b72337a1-e0e4-476f-9561-587198cb985c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "s3://tmp/tinaok\n" + ] + } + ], + "source": [ + "your_name='tinaok'\n", + "path='tmp/'+your_name\n", + "s3_prefix = \"s3://\"+path\n", + "print(s3_prefix)\n", + "access_key = !aws configure get aws_access_key_id\n", + "access_key = access_key[0]\n", + "secret_key = !aws configure get aws_secret_access_key\n", + "secret_key = secret_key[0]\n", + "client_kwargs={'endpoint_url': 'https://object-store.cloud.muni.cz'}" + ] + }, + { + "cell_type": "markdown", + "id": "601619f7-8103-49f5-86bb-b7401b7e0ae6", + "metadata": {}, + "source": [ + "#### Define your s3 storage and define your zarr store" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "36918d3e-488a-4099-8f60-d2a323806636", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "zarr_file_name= \"FSStore_zarr\"\n", + "uri = f\"{s3_prefix}/{zarr_file_name}\"\n", + "store = zarr.storage.FSStore(uri, client_kwargs=client_kwargs,key=access_key, secret=secret_key)\n", + "store.map.fs.find(uri)" + ] + }, + { + "cell_type": "markdown", + "id": "83f3fced-5ad4-4d6d-b670-d36e1f700f11", + "metadata": { + "tags": [] + }, + "source": [ + "#### Write your file down to zarr with FSStore" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "344128c6-6776-4f46-bc42-4e0db640bd06", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/srv/conda/envs/notebook/lib/python3.9/site-packages/xarray/core/dataset.py:2060: SerializationWarning: saving variable None with floating point data as an integer dtype without any _FillValue to use for NaNs\n", + " return to_zarr( # type: ignore\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 818 ms, sys: 124 ms, total: 942 ms\n", + "Wall time: 51 s\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%time ds.to_zarr(store=store, mode='w', consolidated=True)" + ] + }, + { + "cell_type": "markdown", + "id": "9b8bbdd1-63dd-49d2-ad6e-deefa54a2660", + "metadata": {}, + "source": [ + "#### Verify that your 'zarr' file is at your storage" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "99cd5138-d8b3-408d-8bfc-61915798a25d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['tmp/tinaok/FSStore_zarr']" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "target = s3fs.S3FileSystem(anon=False,client_kwargs=client_kwargs)\n", + "target.ls(path)#, detail=True, refresh=True)" + ] + }, + { + "cell_type": "markdown", + "id": "52aa8b67-7525-498d-8db2-04233b17051b", + "metadata": {}, + "source": [ + "#### Open your zarr storage" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "096e617d-b8af-40b8-9229-0090c7342de6", + "metadata": { + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:  (time: 2920, lat: 25, lon: 53)\n",
+       "Coordinates:\n",
+       "  * lat      (lat) float32 75.0 72.5 70.0 67.5 65.0 ... 25.0 22.5 20.0 17.5 15.0\n",
+       "  * lon      (lon) float32 200.0 202.5 205.0 207.5 ... 322.5 325.0 327.5 330.0\n",
+       "  * time     (time) datetime64[ns] 2013-01-01 ... 2014-12-31T18:00:00\n",
+       "Data variables:\n",
+       "    Tair     (time, lat, lon) float32 dask.array<chunksize=(730, 13, 27), meta=np.ndarray>\n",
+       "Attributes:\n",
+       "    Conventions:  COARDS\n",
+       "    description:  Data is from NMC initialized reanalysis\\n(4x/day).  These a...\n",
+       "    platform:     Model\n",
+       "    references:   http://www.esrl.noaa.gov/psd/data/gridded/data.ncep.reanaly...\n",
+       "    title:        4x daily NMC reanalysis (1948)
" + ], + "text/plain": [ + "\n", + "Dimensions: (time: 2920, lat: 25, lon: 53)\n", + "Coordinates:\n", + " * lat (lat) float32 75.0 72.5 70.0 67.5 65.0 ... 25.0 22.5 20.0 17.5 15.0\n", + " * lon (lon) float32 200.0 202.5 205.0 207.5 ... 322.5 325.0 327.5 330.0\n", + " * time (time) datetime64[ns] 2013-01-01 ... 2014-12-31T18:00:00\n", + "Data variables:\n", + " Tair (time, lat, lon) float32 dask.array\n", + "Attributes:\n", + " Conventions: COARDS\n", + " description: Data is from NMC initialized reanalysis\\n(4x/day). These a...\n", + " platform: Model\n", + " references: http://www.esrl.noaa.gov/psd/data/gridded/data.ncep.reanaly...\n", + " title: 4x daily NMC reanalysis (1948)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "xr.open_zarr(store)" + ] + }, + { + "cell_type": "markdown", + "id": "3f369705-3d91-4cad-b430-8e797d3b7330", + "metadata": {}, + "source": [ + "#### Delete the zarr file you created" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "d116d941-8ba3-40d8-aa44-9c59444971a9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['tmp/tinaok/FSStore_zarr']" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "target.ls(path)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "4083647c-0dab-4aa9-a432-6805b2d4631d", + "metadata": {}, + "outputs": [], + "source": [ + "target.rm(path+'/FSStore_zarr', recursive=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ccdbc75d-ef90-4532-8142-78a231cc881e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['tmp/fepit',\n", + " 'tmp/fepitzarr-demo',\n", + " 'tmp/guillaumeeb',\n", + " 'tmp/xarray-demo-dask-s3']" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "target.ls('tmp/')" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "2030c657-4a63-454c-8f57-aa8ab76eed59", + "metadata": {}, + "outputs": [], + "source": [ + "target.rm('tmp/fepit/xesmf_regridding.ipynb')" + ] + }, + { + "cell_type": "markdown", + "id": "1b466043-2367-4ef9-8137-e978b1f5c668", + "metadata": {}, + "source": [ + "
\n", + " one users can delete another user's file.... So please be carefull especially when using recursive=True \n", + "
\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorial/examples/notebooks/save_to_cesnet_bucket_zarr_s3fs.ipynb b/tutorial/examples/notebooks/save_to_cesnet_bucket_zarr_s3fs.ipynb index 8d3c321..1570d9a 100644 --- a/tutorial/examples/notebooks/save_to_cesnet_bucket_zarr_s3fs.ipynb +++ b/tutorial/examples/notebooks/save_to_cesnet_bucket_zarr_s3fs.ipynb @@ -20,10 +20,9 @@ }, "outputs": [], "source": [ - "import os\n", - "import pathlib\n", "import s3fs\n", - "import xarray as xr" + "import xarray as xr\n", + "import zarr" ] }, { @@ -62,19 +61,6 @@ "```\n" ] }, - { - "cell_type": "code", - "execution_count": 3, - "id": "distributed-plaza", - "metadata": {}, - "outputs": [], - "source": [ - "target = s3fs.S3FileSystem(anon=False,\n", - " client_kwargs={\n", - " 'endpoint_url': 'https://object-store.cloud.muni.cz'\n", - " })" - ] - }, { "cell_type": "markdown", "id": "ecef7710-dd4b-4bb4-9d26-f56921645181", @@ -85,31 +71,19 @@ "\n" ] }, - { - "cell_type": "code", - "execution_count": 4, - "id": "c325dc89-131f-4b7a-a23d-5df54ba5e3be", - "metadata": {}, - "outputs": [], - "source": [ - "your_name='tinaok'" - ] - }, { "cell_type": "markdown", - "id": "0f8e176e-f3b9-4ff3-a69d-47d2b308abaa", + "id": "9cdeab0c-ca53-4cd8-9dd8-bccb8d1f58c0", "metadata": {}, "source": [ - "#### Set the bucket and place where you'll copy your data to " + "#### Define your s3 storage parameters" ] }, { "cell_type": "code", - "execution_count": 5, - "id": "eight-coach", - "metadata": { - "tags": [] - }, + "execution_count": 3, + "id": "b72337a1-e0e4-476f-9561-587198cb985c", + "metadata": {}, "outputs": [ { "name": "stdout", @@ -120,67 +94,55 @@ } ], "source": [ - "s3_prefix = \"s3://tmp/\"+your_name\n", - "print(s3_prefix)" - ] - }, - { - "cell_type": "markdown", - "id": "03aee55e-9a9e-4b0a-a3e5-9189ae15d426", - "metadata": {}, - "source": [ - "#### Give a name to your zarr file " - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "95f10fa4-5fdf-4456-a7e7-a91166aa5a19", - "metadata": {}, - "outputs": [], - "source": [ - "zarr_file_name= 'zarr-demo'" + "your_name='tinaok'\n", + "path='tmp/'+your_name\n", + "s3_prefix = \"s3://\"+path\n", + "print(s3_prefix)\n", + "access_key = !aws configure get aws_access_key_id\n", + "access_key = access_key[0]\n", + "secret_key = !aws configure get aws_secret_access_key\n", + "secret_key = secret_key[0]\n", + "client_kwargs={'endpoint_url': 'https://object-store.cloud.muni.cz'}" ] }, { "cell_type": "markdown", - "id": "9cdeab0c-ca53-4cd8-9dd8-bccb8d1f58c0", + "id": "02834a9c-228d-4031-9410-214aa108ed8e", "metadata": {}, "source": [ - "#### Define your s3 storage and define your zarr store" + "## set your zarr store (with S3Map)" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "c7783567-8c36-4875-876c-c7628faccbf1", + "execution_count": 4, + "id": "363169ee-7c47-4583-961c-9f95482d53f0", "metadata": {}, "outputs": [], "source": [ - "access_key = !aws configure get aws_access_key_id\n", - "access_key = access_key[0]\n", - "secret_key = !aws configure get aws_secret_access_key\n", - "secret_key = secret_key[0]\n", - "client_kwargs={'endpoint_url': 'https://object-store.cloud.muni.cz'}\n", + "zarr_file_name= 'S3Map_zarr'\n", "s3 = s3fs.S3FileSystem(client_kwargs=client_kwargs,key=access_key, secret=secret_key)\n", - "store_s3 = s3fs.S3Map(root=s3_prefix+zarr_file_name,\n", - " s3=s3,\n", - " check=False)" + "uri = f\"{s3_prefix}/{zarr_file_name}\"\n", + "store_s3= s3fs.S3Map(root=uri,s3=s3,check=False)" ] }, { "cell_type": "markdown", "id": "e33cb553-df83-49d7-9d27-c7d2348e3c6e", - "metadata": {}, + "metadata": { + "tags": [] + }, "source": [ - "#### Write your file down to zarr\n" + "#### Write your file down to zarr" ] }, { "cell_type": "code", - "execution_count": 8, - "id": "71317f0c-2e8f-4cb5-b635-4b75084a863b", - "metadata": {}, + "execution_count": 5, + "id": "eight-coach", + "metadata": { + "tags": [] + }, "outputs": [ { "name": "stderr", @@ -194,24 +156,23 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 1.09 s, sys: 148 ms, total: 1.24 s\n", - "Wall time: 1min 1s\n" + "CPU times: user 865 ms, sys: 154 ms, total: 1.02 s\n", + "Wall time: 52.4 s\n" ] }, { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 8, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "%%time\n", - "ds.to_zarr(store=store_s3, mode='w', consolidated=True)" + "%time ds.to_zarr(store=store_s3, mode='w', consolidated=True)" ] }, { @@ -224,28 +185,24 @@ }, { "cell_type": "code", - "execution_count": 9, - "id": "481177c4-c1fb-4421-991c-9c48b7d88cac", + "execution_count": 6, + "id": "99cd5138-d8b3-408d-8bfc-61915798a25d", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "['tmp/tinaok/2air_temperature.nc', 'tmp/tinaok/Tair_temperature.nc']" + "['tmp/tinaok/S3Map_zarr']" ] }, - "execution_count": 9, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "target = s3fs.S3FileSystem(anon=False,\n", - " client_kwargs={\n", - " 'endpoint_url': 'https://object-store.cloud.muni.cz'\n", - " })\n", - "remote_path ='tmp/'+your_name\n", - "target.ls(remote_path)" + "target = s3fs.S3FileSystem(anon=False,client_kwargs=client_kwargs)\n", + "target.ls(path)#, detail=True, refresh=True)" ] }, { @@ -258,9 +215,11 @@ }, { "cell_type": "code", - "execution_count": 10, - "id": "everyday-authorization", - "metadata": {}, + "execution_count": 7, + "id": "096e617d-b8af-40b8-9229-0090c7342de6", + "metadata": { + "tags": [] + }, "outputs": [ { "data": { @@ -630,17 +589,17 @@ " description: Data is from NMC initialized reanalysis\\n(4x/day). These a...\n", " platform: Model\n", " references: http://www.esrl.noaa.gov/psd/data/gridded/data.ncep.reanaly...\n", - " title: 4x daily NMC reanalysis (1948)