notebooks/proof-of-concept.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "eebaf61b-dc73-456c-8bd8-37206ba99052",
   "metadata": {},
   "source": [
    "# Proof of concept: Virtualizing CMIP6 netcdf files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "66af3aa4-b876-47e4-8bca-df53c450f103",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # install virtualizarr\n",
    "# !pip install git+https://github.com/jbusecke/VirtualiZarr.git@esgf-cmip-test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9b90a03-d8b6-4ae7-87f4-1463333d3f7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "from virtualizarr import open_virtual_dataset\n",
    "from virtualizarr.kerchunk import FileType"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a89ea614-48e7-45ea-967c-8cde1b04ac52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data is located on public s3 (more info: https://pangeo-data.github.io/pangeo-cmip6-cloud/overview.html#netcdf-data-overview)\n",
    "paths = [\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_185001-186012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_187101-188012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_188101-189012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_186101-187012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_189101-190012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_190101-191012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_191101-192012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_192101-193012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_193101-194012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_194101-195012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_195101-196012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_196101-197012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_197101-198012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_198101-199012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_199101-200012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_200101-201012.nc',\n",
    "    'esgf-world/CMIP6/CMIP/CCCma/CanESM5/historical/r10i1p1f1/Omon/uo/gn/v20190429/uo_Omon_CanESM5_historical_r10i1p1f1_gn_201101-201412.nc'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbc087ef-08cb-4643-b8f3-20d72f93cdb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load virtual datasets in serial\n",
    "vds_list = []\n",
    "for f in tqdm(files):\n",
    "    vds = open_virtual_dataset(f, filetype=FileType.netcdf4, indexes={})\n",
    "    vds_list.append(vds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66730a56-69b7-43c7-a24d-d91a92d208ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_vds = xr.combine_nested(vds_list, concat_dim=['time'], coords='minimal', compat='override')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25dc2ff3-aa42-4dd0-8611-a89a6a476a4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_vds.virtualize.to_kerchunk('combined_full.json', format='json')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2539bcc-6f59-4ed5-833c-86a3ba828c78",
   "metadata": {},
   "source": [
    "## Read from local json\n",
    "If you executed all steps above, you should be able to execute this cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61786379-985b-4317-934d-8ef43dcfe8db",
   "metadata": {},
   "outputs": [],
   "source": [
    "dsv_local = xr.open_dataset(\n",
    "    \"reference://\",\n",
    "    engine=\"zarr\",\n",
    "    chunks={},\n",
    "    backend_kwargs={\n",
    "        \"consolidated\": False,\n",
    "        \"storage_options\": {\n",
    "            \"fo\": \"combined_full.json\",\n",
    "            \"remote_protocol\": \"s3\",\n",
    "            \"remote_options\": {\"anon\": True},\n",
    "        },\n",
    "    },\n",
    ")\n",
    "dsv_local"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc458e9f-81aa-4e4a-8104-64cc0a756353",
   "metadata": {},
   "source": [
    "## Read from Json on public cloud storage\n",
    "I moved the resulting json to a public bucket for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09ada009-26b2-4782-871a-8b5c409c0a0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dsv_bucket = xr.open_dataset(\n",
    "    \"reference://\",\n",
    "    engine=\"zarr\",\n",
    "    chunks={'time':5, 'lev':45, 'j':291, 'i':360}, #NOTE: I changed these!\n",
    "    backend_kwargs={\n",
    "        \"consolidated\": False,\n",
    "        \"storage_options\": {\n",
    "            \"target_protocol\": \"gs\",\n",
    "            \"fo\": 'gs://cmip6/testing-virtualizarr/proof-of-concept.json',\n",
    "            \"remote_protocol\": \"s3\",\n",
    "            \"remote_options\":{'anon':True},\n",
    "        },\n",
    "    }\n",
    ")\n",
    "dsv_bucket"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}