diff --git a/README.md b/README.md index 7e0a924..d510822 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ S. Gupta et al 2022, 2024](https://github.com/CliMA/ClimaArtifacts/tree/main/soi - [ERA5 Land Forcing Data 2008](https:////github.com/CliMA/ClimaArtifacts/tree/main/era5_land_forcing_data2008) - [ERA5 Monthly Averages on Pressure Levels 1979-2024](https:////github.com/CliMA/ClimaArtifacts/tree/main/era5_monthly_averages_pressure_levels_1979_2024) - [TOPMODEL topographic index statistics](https:////github.com/CliMA/ClimaArtifacts/tree/main/topmodel) +- [ERA5 Monthly Averages on Single Levels 1979-2024](https:////github.com/CliMA/ClimaArtifacts/tree/main/era5_monthly_averages_single_level_1979_2024) # The ultimate guide to ClimaArtifacts diff --git a/era5_monthly_averages_2008/README.md b/era5_monthly_averages_2008/README.md index 155d261..e5f7e5e 100644 --- a/era5_monthly_averages_2008/README.md +++ b/era5_monthly_averages_2008/README.md @@ -1,4 +1,4 @@ -# ERA5 monthly averages +# ERA5 monthly averages 2008 This artifact processes data coming from [ERA5 monthly averaged reanalysis](https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-monthly-means?tab=download) and contains monthly averaged surface fluxes and runoff rates. @@ -41,7 +41,7 @@ There processing of the downloaded dataset results in two output datasets: Both of the output datasets contain the same variables and spatial coverage, but the variables are defined on different time dimensions. -## Spatial Coverage +### Spatial Coverage - 1 degree latitude x 1 degree longitude grid - -90N to 90N and 0E to 359E @@ -49,29 +49,29 @@ Both of the output datasets contain the same variables and spatial coverage, but The shared variables, which are all stored as Float32s and defined on the latitude, longitude, and time dimensions, are: -## `mslhf` +### `mslhf` This is the mean surface latent heat flux in units of W m**-2. No processing is done to this variable other than flipping the latitude dimension. -## `msshf` +### `msshf` This is the mean surface sensible heat flux in units of W m**-2. No processing is done to this variable other than flipping the latitude dimension. -## `mssror` +### `mssror` This is the mean sub-surface runoff rate in units of kg m**-2 s**-1. No processing is done to this variable other than flipping the latitude dimension. -## `msror` +### `msror` This is the mean surface runoff rate in units of kg m**-2 s**-1. No processing is done to this variable other than flipping the latitude dimension. -## `msuwlwrf` +### `msuwlwrf` This is the mean surface upward long-wave radiation flux in units of W m**-2. This variable is created during processing by taking the difference of mean surface downward long-wave radiation flux and mean surface net long-wave radiation flux, and then flipping the latitude dimension. -## `msuwswrf` +### `msuwswrf` This is the mean surface upward short-wave radiation flux in units of W m**-2. This variable is created during processing by taking the difference of @@ -84,7 +84,7 @@ The two output files have different temporal coverage. ### `era5_monthly_surface_fluxes_200801-200812.nc` This file contains Monthly averaged reanalysis, which is produced by averaging all daily data for each month. This results in 12 points on the -time dimension, where each point in the 15th of the month that the point represents. For example, the 6th index of `time` is 2008-06-15T00:00:00, +time dimension, where each point is the 15th of the month that the point represents. For example, the 6th index of `time` is 2008-06-15T00:00:00, which represents the whole month of June in 2008. ### `era5_monthly_surface_fluxes_hourly_200801-200812.nc` @@ -94,8 +94,8 @@ This file contains Monthly averages by hour of day, which constitutes the averag ## Citation -Hersbach, H., Bell, B., Berrisford, P., Biavati, G., Horányi, A., Muñoz Sabater, J., Nicolas, J., Peubey, C., Radu, R., Rozum, I., Schepers, D., Simmons, A., Soci, C., Dee, D., Thépaut, J-N. (2023): ERA5 monthly averaged data on single levels from 1940 to present. Copernicus Climate Change Service (C3S) Climate Data Store (CDS), DOI: 10.24381/cds.f17050d7 (Accessed on DD-MMM-YYYY) +Hersbach, H., Bell, B., Berrisford, P., Biavati, G., Horányi, A., Muñoz Sabater, J., Nicolas, J., Peubey, C., Radu, R., Rozum, I., Schepers, D., Simmons, A., Soci, C., Dee, D., Thépaut, J-N. (2023): ERA5 monthly averaged data on single levels from 1940 to present. Copernicus Climate Change Service (C3S) Climate Data Store (CDS), DOI: 10.24381/cds.f17050d7 (Accessed on 11-11-2024) ## License -See the [LICENSE](LICENSE.txt) file \ No newline at end of file +See the [LICENSE](LICENSE.txt) file diff --git a/era5_monthly_averages_2008/create_artifact.jl b/era5_monthly_averages_2008/create_artifact.jl index b7000b8..61f4192 100644 --- a/era5_monthly_averages_2008/create_artifact.jl +++ b/era5_monthly_averages_2008/create_artifact.jl @@ -30,7 +30,7 @@ if !isfile(DOWNLOAD_FILE_NAME) println("Enter your CDS Personal Access Token:") cds_PAT = readline() println("Downloading data with CDS API using PAT: $cds_PAT") - run(`python download_data.py $cds_PAT`) + run(`python download_data.py -k $cds_PAT`) end end diff --git a/era5_monthly_averages_2008/subset_of_data.jl b/era5_monthly_averages_2008/subset_of_data.jl index a9ed7bd..4607f4a 100644 --- a/era5_monthly_averages_2008/subset_of_data.jl +++ b/era5_monthly_averages_2008/subset_of_data.jl @@ -49,7 +49,6 @@ function create_new_ds_from_time_indx( Float32.(reverse(var[:, :, time_indx], dims = 2)), dimnames(var); attrib = attrib, - # attrib = copy(var.attrib), deflatelevel = 9, ) end diff --git a/era5_monthly_averages_single_level_1979_2024/LICENSE.txt b/era5_monthly_averages_single_level_1979_2024/LICENSE.txt new file mode 100644 index 0000000..ac4c789 --- /dev/null +++ b/era5_monthly_averages_single_level_1979_2024/LICENSE.txt @@ -0,0 +1,93 @@ +I. Licence to Use Copernicus Products + +1. Definitions + +1.1. ‘Licensor’ means the European Union, represented by the European Centre for Medium-Range Weather Forecasts (ECMWF). + +1.2. ‘Licensee’ means all natural or legal persons who agree to the terms of this Licence. + +1.3. ‘Licence’ means this license agreement between the Licensor and the Licensee as amended from time to time. + +1.4. ‘Copernicus Services’ means: + +1.4.1. the Copernicus Atmosphere Monitoring Service (CAMS), which is to provide information on air quality on a local, national, and European scale, and the chemical composition of the atmosphere on a global scale. + +1.4.2. the Copernicus Climate Change Service (C3S), which is to provide information to increase the knowledge base to support policies on adaptation to and mitigation of climate change + +1.5. ‘Copernicus Products’ means all products listed in the C3S or CAMS Service Product Specification or any other items available through an ECMWF Copernicus portal, except those items which are labelled/flagged as being subject to their own separate terms of use. + +1.6. ‘Intellectual Property Rights’ refers to intellectual property rights of all kinds, + +1.6.1. including: all patents; rights to inventions; copyright and related rights; moral rights; trademarks and service marks; trade names and domain names; rights in get-up; rights to goodwill or to sue for passing off or unfair competition; rights in designs; rights in computer software; database rights; rights in confidential information (including know-how and trade secrets); any other rights in the nature of intellectual property rights; + +1.6.2. in each case whether registered or unregistered and including all applications (or rights to apply) for, and renewals or extensions of, such rights and all similar or equivalent rights or forms of protection which subsist or will subsist now or in the future in any part of the world together with all rights of action in relation to the infringement of any of the above. + +1.7. ‘Copernicus Contractor’ refers to providers of Copernicus related goods and services to ECMWF, including information and data, to the Licensor and/or to the users. + +1.8. ‘Copernicus Regulations’ refers to Regulation (EU) No 377/2014 of the European Parliament and of the Council of 3 April 2014 establishing the Copernicus Programme. + +1.9. ‘ECMWF Agreement’ refers to the agreement between the European Commission and ECMWF dated 11 November 2014 on the implementation of CAMS and C3S. + +2. Introduction + +Copernicus is funded under the Copernicus Regulation and operated by ECMWF under the ECMWF Agreement. Access to all Copernicus (previously known as GMES or Global Monitoring for Environment and Security) Information and Data is regulated under Regulation (EU) No 1159/2013 of the European Parliament and of the Council of 12 July 2013 on the European Earth monitoring programme, under the ECMWF Agreement and under the European Commission’s Terms and Conditions. Access to all Copernicus information is regulated under Regulation (EU) No 1159/2013 and under the ECMWF Agreement. + +3. Terms of the Licence + +This Licence sets out the terms for use of Copernicus Products. By agreeing to these terms, the Licensee agrees to abide by all of the terms and conditions in this Licence for the use of Copernicus Products. + +4. Licence Permission + +4.1. This Licence is free of charge, worldwide, non-exclusive, royalty free and perpetual. + +4.2. Access to Copernicus Products is given for any purpose in so far as it is lawful, whereas use may include, but is not limited to: reproduction; distribution; communication to the public; adaptation, modification and combination with other data and information; or any combination of the foregoing. + +5. Attribution + +5.1. All users of Copernicus Products must provide clear and visible attribution to the Copernicus programme. The Licensee will communicate to the public the source of the Copernicus Products by crediting the Copernicus Climate Change and Atmosphere Monitoring Services: + +5.1.1. Where the Licensee communicates or distributes Copernicus Products to the public, the Licensee shall inform the recipients of the source by using the following or any similar notice: +• 'Generated using Copernicus Climate Change Service information [Year]' and/or +• 'Generated using Copernicus Atmosphere Monitoring Service information [Year]'. + +5.1.2. Where the Licensee makes or contributes to a publication or distribution containing adapted or modified Copernicus Products, the Licensee shall provide the following or any similar notice: +• 'Contains modified Copernicus Climate Change Service information [Year]'; and/or +• 'Contains modified Copernicus Atmosphere Monitoring Service information [Year]' + +5.1.3. Any such publication or distribution covered by clauses 5.1.1 and 5.1.2 shall state that neither the European Commission nor ECMWF is responsible for any use that may be made of the Copernicus information or data it contains. + +6. Intellectual Property Rights + +6.1. All Intellectual Property Rights in the Copernicus Products belong, and will continue to belong, to the European Union. + +6.2. All Intellectual Property Rights of new items created as a result of modifying or adapting the Copernicus Products through the applications and workflows accessible on the ECMWF Copernicus portals (e.g. through the CDS Toolbox) will belong to the European Union. + +6.3. All other new Intellectual Property Rights created as a result of modifying or adapting the Copernicus information will be owned by the creator. + +7. Provision of Third Party Information and Data + +This Licence only covers Copernicus Products. Access to third party products, information, and data related to Copernicus information to which the Licensee is directed or which can be directly accessed through any Copernicus portal will be subject to different licence terms. + +8. Disclaimers + +8.1. Neither the Licensor nor ECMWF warrant that Copernicus Products will be free from errors or omissions or that such errors or omissions can or will be rectified, or that the Licensee will have uninterrupted, continuous, or timely access to Copernicus Products. + +8.2. The Licensor, as well as ECMWF, exclude all warranties, conditions, terms, undertakings, obligations whether express or implied by statute including but not limited to the implied warranties of satisfactory quality and fitness for a particular purpose or otherwise to the fullest extent permitted by law. + +9. Liabilities + +Neither the Licensor nor ECMWF will accept liability for any damage, loss whether direct, indirect or consequential resulting from the Licensee’s use of the Copernicus Products. + +10. Termination of and Changes to this Licence + +The Licensor may terminate this licence if the Licensee breaches its obligations under these terms. The Licensor may revise this Licence at any time and will notify the Licensee of any revisions. + +11. Arbitration Clause and Governing Law + +In the event of a dispute arising in connection with this License, the parties shall attempt to settle their differences in an amicable manner. If any dispute cannot be so settled, it shall be settled under the Rules of Arbitration of the International Chamber of Commerce by one arbitrator appointed in accordance with the said rules sitting in London, United Kingdom. The proceedings shall be in the English language. The right of appeal by either party to regular Courts on a question of law arising in the course of any arbitral proceedings or out of an award made in any arbitral proceedings is hereby agreed to be excluded. + +It is the intention of the parties that this License shall comprehensively govern the legal relations between the parties to the Licence, without interference or contradiction by any unspecified law. However, where a matter is not specifically covered by these terms or a provision of the Licence terms is ambiguous or unclear, resolution shall be found by reference to the laws of England and Wales, including any relevant law of the European Union. + +Nothing stated in this License shall be construed as a waiver of any privileges or immunities of the Licensor or of ECMWF. + +Version 1.2 (November 2019) diff --git a/era5_monthly_averages_single_level_1979_2024/Manifest.toml b/era5_monthly_averages_single_level_1979_2024/Manifest.toml new file mode 100644 index 0000000..9c87a33 --- /dev/null +++ b/era5_monthly_averages_single_level_1979_2024/Manifest.toml @@ -0,0 +1,526 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.11.1" +manifest_format = "2.0" +project_hash = "47774df903631abd3b74628cc1eb8ab16058d1db" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.2" + +[[deps.ArtifactUtils]] +deps = ["Downloads", "Git", "HTTP", "Pkg", "ProgressLogging", "SHA", "TOML", "gh_cli_jll"] +git-tree-sha1 = "f7c3ca6c70ea6b035effe5428eb05231ba890c2d" +uuid = "8b73e784-e7d8-4ea5-973d-377fed4e3bce" +version = "0.2.4" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.11.0" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.BitFlags]] +git-tree-sha1 = "0691e34b3bb8be9307330f88d1a3c3f25466c24d" +uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" +version = "0.1.9" + +[[deps.Blosc_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Lz4_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "ef12cdd1c7fb7e1dfd6fa8fd60d4db6bc61d2f23" +uuid = "0b7ba130-8d10-5ba8-a3d6-c5182647fed9" +version = "1.21.6+0" + +[[deps.Bzip2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "8873e196c2eb87962a2048b3b8e08946535864a1" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.8+2" + +[[deps.CFTime]] +deps = ["Dates", "Printf"] +git-tree-sha1 = "5afb5c5ba2688ca43a9ad2e5a91cbb93921ccfa1" +uuid = "179af706-886a-5703-950a-314cd64e0468" +version = "0.1.3" + +[[deps.ClimaArtifactsHelper]] +deps = ["ArtifactUtils", "Downloads", "Pkg", "REPL", "SHA"] +path = "/home/treddy/.julia/dev/ClimaArtifacts/ClimaArtifactsHelper.jl" +uuid = "6ffa2572-8378-4377-82eb-ea11db28b255" +version = "0.0.1" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "bce6804e5e6044c6daab27bb533d1295e4a2e759" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.6" + +[[deps.CommonDataModel]] +deps = ["CFTime", "DataStructures", "Dates", "Preferences", "Printf", "Statistics"] +git-tree-sha1 = "98d64d5b9e5263884276656a43c45424b3a645c2" +uuid = "1fbeeb36-5f17-413c-809b-666fb144f157" +version = "0.3.7" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.16.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.1.1+0" + +[[deps.ConcurrentUtilities]] +deps = ["Serialization", "Sockets"] +git-tree-sha1 = "ea32b83ca4fefa1768dc84e504cc0a94fb1ab8d1" +uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" +version = "2.4.2" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "1d0a14036acb104d9e89698bd408f63ab58cdc82" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.20" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +version = "1.11.0" + +[[deps.DiskArrays]] +deps = ["LRUCache", "OffsetArrays"] +git-tree-sha1 = "e0e89a60637a62d13aa2107f0acd169b9b9b77e7" +uuid = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" +version = "0.4.6" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.ExceptionUnwrapping]] +deps = ["Test"] +git-tree-sha1 = "dcb08a0d93ec0b1cdc4af184b26b591e9695423a" +uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +version = "0.1.10" + +[[deps.Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "1c6317308b9dc757616f0b5cb379db10494443a7" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.6.2+0" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" +version = "1.11.0" + +[[deps.GMP_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "781609d7-10c4-51f6-84f2-b8444358ff6d" +version = "6.3.0+0" + +[[deps.Git]] +deps = ["Git_jll"] +git-tree-sha1 = "04eff47b1354d702c3a85e8ab23d539bb7d5957e" +uuid = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" +version = "1.3.1" + +[[deps.Git_jll]] +deps = ["Artifacts", "Expat_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "Libiconv_jll", "OpenSSL_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "ea372033d09e4552a04fd38361cd019f9003f4f4" +uuid = "f8c6e375-362e-5223-8a59-34ff63f689eb" +version = "2.46.2+0" + +[[deps.GnuTLS_jll]] +deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Nettle_jll", "P11Kit_jll", "Zlib_jll"] +git-tree-sha1 = "383db7d3f900f4c1f47a8a04115b053c095e48d3" +uuid = "0951126a-58fd-58f1-b5b3-b08c7c4a876d" +version = "3.8.4+0" + +[[deps.HDF5_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "LazyArtifacts", "LibCURL_jll", "Libdl", "MPICH_jll", "MPIPreferences", "MPItrampoline_jll", "MicrosoftMPI_jll", "OpenMPI_jll", "OpenSSL_jll", "TOML", "Zlib_jll", "libaec_jll"] +git-tree-sha1 = "38c8874692d48d5440d5752d6c74b0c6b0b60739" +uuid = "0234f1f7-429e-5d53-9886-15a909be8d59" +version = "1.14.2+1" + +[[deps.HTTP]] +deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] +git-tree-sha1 = "1336e07ba2eb75614c99496501a8f4b233e9fafe" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "1.10.10" + +[[deps.Hwloc_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "50aedf345a709ab75872f80a2779568dc0bb461b" +uuid = "e33a78d0-f292-5ffc-b300-72abe9b543c8" +version = "2.11.2+1" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "be3dc50a92e5a386872a493a10050136d4703f9b" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.6.1" + +[[deps.LLVMOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "78211fb6cbc872f77cad3fc0b6cf647d923f4929" +uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" +version = "18.1.7+0" + +[[deps.LRUCache]] +git-tree-sha1 = "b3cc6698599b10e652832c2f23db3cab99d51b59" +uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" +version = "1.6.1" +weakdeps = ["Serialization"] + + [deps.LRUCache.extensions] + SerializationExt = ["Serialization"] + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" +version = "1.11.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.6.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +version = "1.11.0" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.7.2+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +version = "1.11.0" + +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "61dfdba58e585066d8bce214c5a51eaa0539f269" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.17.0+1" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +version = "1.11.0" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.LoggingExtras]] +deps = ["Dates", "Logging"] +git-tree-sha1 = "f02b56007b064fbfddb4c9cd60161b6dd0f40df3" +uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" +version = "1.1.0" + +[[deps.Lz4_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "abf88ff67f4fd89839efcae2f4c39cbc4ecd0846" +uuid = "5ced341a-0733-55b8-9ab6-a4889d929147" +version = "1.10.0+1" + +[[deps.MPICH_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "7715e65c47ba3941c502bffb7f266a41a7f54423" +uuid = "7cb0a576-ebde-5e09-9194-50597f1243b4" +version = "4.2.3+0" + +[[deps.MPIPreferences]] +deps = ["Libdl", "Preferences"] +git-tree-sha1 = "c105fe467859e7f6e9a852cb15cb4301126fac07" +uuid = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267" +version = "0.1.11" + +[[deps.MPItrampoline_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML"] +git-tree-sha1 = "70e830dab5d0775183c99fc75e4c24c614ed7142" +uuid = "f1f71cc9-e9ae-5b93-9b94-4fe0e1ad3748" +version = "5.5.1+0" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] +git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.1.9" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.6+0" + +[[deps.MicrosoftMPI_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f12a29c4400ba812841c6ace3f4efbb6dbb3ba01" +uuid = "9237b28f-5490-5468-be7b-bb81f5f5e6cf" +version = "10.1.4+2" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.12.12" + +[[deps.NCDatasets]] +deps = ["CFTime", "CommonDataModel", "DataStructures", "Dates", "DiskArrays", "NetCDF_jll", "NetworkOptions", "Printf"] +git-tree-sha1 = "2c9dc92001ac06d432f363f37ff5552954d9947c" +uuid = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" +version = "0.14.6" + +[[deps.NetCDF_jll]] +deps = ["Artifacts", "Blosc_jll", "Bzip2_jll", "HDF5_jll", "JLLWrappers", "LibCURL_jll", "Libdl", "OpenMPI_jll", "XML2_jll", "Zlib_jll", "Zstd_jll", "libzip_jll"] +git-tree-sha1 = "a8af1798e4eb9ff768ce7fdefc0e957097793f15" +uuid = "7243133f-43d8-5620-bbf4-c2c921802cf3" +version = "400.902.209+0" + +[[deps.Nettle_jll]] +deps = ["Artifacts", "GMP_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "eca63e3847dad608cfa6a3329b95ef674c7160b4" +uuid = "4c82536e-c426-54e4-b420-14f461c4ed8b" +version = "3.7.2+0" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OffsetArrays]] +git-tree-sha1 = "1a27764e945a152f7ca7efa04de513d473e9542e" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.14.1" + + [deps.OffsetArrays.extensions] + OffsetArraysAdaptExt = "Adapt" + + [deps.OffsetArrays.weakdeps] + Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.27+1" + +[[deps.OpenMPI_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Hwloc_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "MPIPreferences", "TOML", "Zlib_jll"] +git-tree-sha1 = "bfce6d523861a6c562721b262c0d1aaeead2647f" +uuid = "fe0851c0-eecd-5654-98d4-656369965a5c" +version = "5.0.5+0" + +[[deps.OpenSSL]] +deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"] +git-tree-sha1 = "38cb508d080d21dc1128f7fb04f20387ed4c0af4" +uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +version = "1.4.3" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "7493f61f55a6cce7325f197443aa80d32554ba10" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.0.15+1" + +[[deps.OrderedCollections]] +git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.3" + +[[deps.P11Kit_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "2cd396108e178f3ae8dedbd8e938a18726ab2fbf" +uuid = "c2071276-7c44-58a7-b746-946036e04d0a" +version = "0.24.1+0" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.42.0+1" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.11.0" +weakdeps = ["REPL"] + + [deps.Pkg.extensions] + REPLExt = "REPL" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.3" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +version = "1.11.0" + +[[deps.ProgressLogging]] +deps = ["Logging", "SHA", "UUIDs"] +git-tree-sha1 = "80d919dee55b9c50e8d9e2da5eeafff3fe58b539" +uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" +version = "0.1.4" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "StyledStrings", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +version = "1.11.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.SimpleBufferStream]] +git-tree-sha1 = "f305871d2f381d21527c770d4788c06c097c9bc1" +uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" +version = "1.2.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +version = "1.11.0" + +[[deps.Statistics]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0" +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.11.1" + + [deps.Statistics.extensions] + SparseArraysExt = ["SparseArrays"] + + [deps.Statistics.weakdeps] + SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.11.3" + +[[deps.URIs]] +git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.5.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +version = "1.11.0" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +version = "1.11.0" + +[[deps.XML2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] +git-tree-sha1 = "6a451c6f33a176150f315726eba8b92fbfdb9ae7" +uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" +version = "2.13.4+0" + +[[deps.XZ_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "15e637a697345f6743674f1322beefbc5dcd5cfc" +uuid = "ffd25f8a-64ca-5728-b0f7-c24cf3aae800" +version = "5.6.3+0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "555d1076590a6cc2fdee2ef1469451f872d8b41b" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.6+1" + +[[deps.gh_cli_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "b54ff233a6a183ad366d29fa0e3ff9bb921692db" +uuid = "5d31d589-30fb-542f-b82d-10325e863e38" +version = "2.35.0+0" + +[[deps.libaec_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "46bf7be2917b59b761247be3f317ddf75e50e997" +uuid = "477f73a3-ac25-53e9-8cc3-50b2fa2566f0" +version = "1.1.2+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.11.0+0" + +[[deps.libzip_jll]] +deps = ["Artifacts", "Bzip2_jll", "GnuTLS_jll", "JLLWrappers", "Libdl", "XZ_jll", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "668ac0297e6bd8f4d53dfdcd3ace71f2e00f4a35" +uuid = "337d8026-41b4-5cde-a456-74a10e5b31d1" +version = "1.11.1+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.59.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" diff --git a/era5_monthly_averages_single_level_1979_2024/OutputArtifacts.toml b/era5_monthly_averages_single_level_1979_2024/OutputArtifacts.toml new file mode 100644 index 0000000..e90c60e --- /dev/null +++ b/era5_monthly_averages_single_level_1979_2024/OutputArtifacts.toml @@ -0,0 +1,17 @@ +[era5_monthly_averages_surface_single_level_1979_2024] +git-tree-sha1 = "6426ef492fdc8012c8e83427f4e45f1cda60f553" + + [[era5_monthly_averages_surface_single_level_1979_2024.download]] + sha256 = "cff72f30dac24504bcc3b8210d3b23e543ff05dc0448d7e3b1ee141654707ac3" + url = "https://caltech.box.com/shared/static/jbgtyt6oq9lxvk8il5zzck6k581q7f3k.gz" +[era5_monthly_averages_atmos_single_level_1979_2024] +git-tree-sha1 = "7fac017b817bb2e26da98bc54b6b3a36ab523a6a" + + [[era5_monthly_averages_atmos_single_level_1979_2024.download]] + sha256 = "c7e61c9714fc6e1db67fdd81f8821704b619514c2143bed621345ccf4bb724e5" + url = "https://caltech.box.com/shared/static/8mo7azrz0gukdtmo2oi0dtvdt1v3t3tv.gz" + +[era5_monthly_averages_surface_single_level_1979_2024_hourly] +git-tree-sha1 = "6f44bae3ff1d9734b0c5a1d7417ebd52ff5aca53" +[era5_monthly_averages_atmos_single_level_1979_2024_hourly] +git-tree-sha1 = "6974477ddaa472a2c1e07355887bc14650a29a12" diff --git a/era5_monthly_averages_single_level_1979_2024/Project.toml b/era5_monthly_averages_single_level_1979_2024/Project.toml new file mode 100644 index 0000000..33e6033 --- /dev/null +++ b/era5_monthly_averages_single_level_1979_2024/Project.toml @@ -0,0 +1,5 @@ +[deps] +ClimaArtifactsHelper = "6ffa2572-8378-4377-82eb-ea11db28b255" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +NCDatasets = "85f8d34a-cbdd-5861-8df4-14fed0d494ab" diff --git a/era5_monthly_averages_single_level_1979_2024/README.md b/era5_monthly_averages_single_level_1979_2024/README.md new file mode 100644 index 0000000..ce4adfd --- /dev/null +++ b/era5_monthly_averages_single_level_1979_2024/README.md @@ -0,0 +1,145 @@ +# ERA5 monthly averages 1979-2024 + +This folder creates four artifacts by processing data coming from [ERA5 monthly averaged reanalysis](https://cds.climate.copernicus.eu/datasets/reanalysis-era5-single-levels-monthly-means?tab=download) and contains +monthly averaged surface fluxes and runoff rates. + +The first artifact contains the monthly averaged reanalysis of surface variables, the second contains monthly averaged reanalysis of vertically integrated variables +the third contains monthly averaged reanalysis of surface variables by hour of day, and the fourth contains +monthly averaged reanalysis of vertically integrated variables by hour of day + +## Usage + +To recreate all four artifacts: + +1. Set up the CDI APS personal access token following the [instruction](https://cds.climate.copernicus.eu/how-to-api#install-the-cds-api-token), +or enter it in when prompted by the script. +2. Create and activate a python virtual environment +3. In the same terminal run `pip install -r requirements.txt` +4. In the same terminal run `julia --project create_artifact.jl` + +Note: The script first downloads the monthly averaged reanalysis data, and then does the same for the monthly averaged reanalysis by hour of day data. Downloading and processing the hourly averages per month takes significantly longer because it contains 24 times more data. + +## Requirements + +- Python >= 3 +- 22G of free disk space + +## Downloaded datasets + +Both of the downloaded datasets contain the following variables: + +1. `mean_surface_downward_long_wave_radiation_flux` +2. `mean_surface_downward_short_wave_radiation_flux` +3. `mean_surface_latent_heat_flux` +4. `mean_surface_net_long_wave_radiation_flux` +5. `mean_surface_net_short_wave_radiation_flux` +6. `mean_surface_sensible_heat_flux` +7. `mean_surface_runoff_rate` +8. `mean_sub_surface_runoff_rate` +9. `total_column_water` +10. `number` - Not included in output. It is introduced by default by the netCDF conversion and has a single value of 0. +11. `expver` + +## Output Datasets + +Running the script results in two output datasets: + +1. `era5_monthly_surface_fluxes_197901-202410.nc` (around 420M) +2. `era5_monthly_surface_fluxes_hourly_197901-202410.nc` (around 8.4G) + +Both of the average for each hour for each month and averages per months datasets contain the same variables and spatial coverage, but the variables are defined on different time dimensions. + +### Spatial Coverage (Identical in all four) + +- 1 degree latitude x 1 degree longitude grid +- -90N to 90N and 0E to 359E +- The latitudes are reversed during processing to be in increasing order + +## Temporal Coverage + +The output files have different temporal coverage: + +### monthly averages + +The datasets with this temporal coverage are: +- `era5_monthly_averages_atmos_single_level_197901-202410.nc` +- `era5_monthly_averages_surface_single_level_197901-202410.nc` + +These files contain Monthly averaged reanalysis from 1979 to present (October 2024 at time of creation), which is produced by averaging all daily data for each month. This results in 12*(2024-1979 + 10/12) = 550 points on the +time dimension, where each point is the 15th of the month that the point represents. For example, the 6th index of `time` is `1979-06-15T00:00:00`, +which represents the whole month of June in 1979. + +### average for each hour for each month + +The datasets with this temporal coverage are: +- `era5_monthly_averages_atmos_single_level_hourly_197901-202410.nc` +- `era5_monthly_averages_surface_single_level_hourly_197901-202410.nc` + +These files contain Monthly averages by hour of day from 1979 to present (October 2024 at time of creation), which constitutes the average over all data within the calendar month for every hour. +This results in 12 * 24 * (2024-1979 )=13200 points on the time dimension, where each point represents the average of a given hour across the month. For example, the 12th index of `time` is `1979-01-15T11:00:00`, which represents the average of each 11:00-12:00 across the month of January in 1979. + + +## Surface Variables + +These following variables are in: +- `era5_monthly_averages_surface_single_level_197901-202410.nc` +- `era5_monthly_averages_surface_single_level_hourly_197901-202410.nc` + +The following variables are stored as Float32s and are defined on the latitude, longitude, and time dimensions: + +### `mslhf` + +This is the mean surface latent heat flux in units of W m**-2. No processing is done to this variable other than flipping the latitude dimension and removing residual GRIB attributes. + +### `msshf` + +This is the mean surface sensible heat flux in units of W m**-2. No processing is done to this variable other than flipping the latitude dimension and removing residual GRIB attributes. + +### `mssror` + +This is the mean sub-surface runoff rate in units of kg m**-2 s**-1. No processing is done to this variable other than flipping the latitude dimension and removing residual GRIB attributes. + +### `msror` + +This is the mean surface runoff rate in units of kg m**-2 s**-1. No processing is done to this variable other than flipping the latitude dimension and removing residual GRIB attributes. + +### `msuwlwrf` + +This is the mean surface upward long-wave radiation flux in units of W m**-2. +This variable is created during processing by taking the difference of +mean surface downward long-wave radiation flux and mean surface net long-wave radiation flux, and then flipping the latitude dimension. +Residual GRIB attributes are also removed. + +### `msuwswrf` + +This is the mean surface upward short-wave radiation flux in units of W m**-2. +This variable is created during processing by taking the difference of +mean surface downward short-wave radiation flux and mean surface net short-wave radiation flux, and then flipping the latitude dimension. Residual GRIB attributes are also removed. + +### `expver` + +This variable is only defined on the time dimension, and a value of 5 indicates any data at that point on the time axis is from ERA5T, which is an initial release of ERA5 data. A value of 1 indicates that data for that time is no longer potentially subject to change. No preprocessing is done other than removing residual GRIB attributes. It is not included in the hourly dataset. + +## Vertically Integrated Variables + +These following variables are in: +- `era5_monthly_averages_atmos_single_level_197901-202410.nc` +- `era5_monthly_averages_atmos_single_level_hourly_197901-202410.nc` + +The following variables are stored as Float32s and are defined on the latitude, longitude, and time dimensions: + +### `tcw` + +This is the total water column in units of kg m**-2. No processing is done to this variable other than flipping the latitude dimension and removing residual GRIB attributes. + +### `expver` + +This variable is only defined on the time dimension, and a value of 5 indicates any data at that point on the time axis is from ERA5T, which is an initial release of ERA5 data. A value of 1 indicates that data for that time is no longer potentially subject to change. No preprocessing is done other than removing residual GRIB attributes. It is not included in the hourly dataset. + +## Citation + +Hersbach, H., Bell, B., Berrisford, P., Biavati, G., Horányi, A., Muñoz Sabater, J., Nicolas, J., Peubey, C., Radu, R., Rozum, I., Schepers, D., Simmons, A., Soci, C., Dee, D., Thépaut, J-N. (2023): ERA5 monthly averaged data on pressure levels from 1940 to present. Copernicus Climate Change Service (C3S) Climate Data Store (CDS), DOI: 10.24381/cds.6860a573 (Accessed on 11-11-2024) + +## License + +See the [LICENSE](LICENSE.txt) file diff --git a/era5_monthly_averages_single_level_1979_2024/create_artifact.jl b/era5_monthly_averages_single_level_1979_2024/create_artifact.jl new file mode 100644 index 0000000..160e413 --- /dev/null +++ b/era5_monthly_averages_single_level_1979_2024/create_artifact.jl @@ -0,0 +1,117 @@ +using Downloads +using ClimaArtifactsHelper +using NCDatasets +using Dates +include("process_downloads.jl") + + +# Around 30GB of storage is required to create this artifact. +# Set data_dir to the location of the data, or where you want to store it +const DATA_DIR = "" +const DOWNLOAD_FILE_NAME = + joinpath(DATA_DIR, "era5_monthly_averages_single_level_197901-202410.nc") +const DOWNLOAD_FILES_HOURLY_DIR = joinpath(DATA_DIR, "hourly_data_parts/") + +const OUTPUT_FILE_NAME_SURFACE = "era5_monthly_averages_surface_single_level_197901-202410.nc" +const OUTPUT_FILE_NAME_ATMOS = "era5_monthly_averages_atmos_single_level_197901-202410.nc" +const OUTPUT_FILE_HOURLY_NAME_SURFACE = "era5_monthly_averages_surface_single_level_hourly_197901-202410.nc" +const OUTPUT_FILE_HOURLY_NAME_ATMOS = "era5_monthly_averages_atmos_single_level_hourly_197901-202410.nc" + +const OUTPUT_DIR_SURFACE = joinpath(DATA_DIR, "surface_" * basename(@__DIR__) * "_artifact") +const OUTPUT_DIR_ATMOS = joinpath(DATA_DIR, "atmos_" * basename(@__DIR__) * "_artifact") +const OUTPUT_DIR_HOURLY_SURFACE = + joinpath(DATA_DIR, "surface_" * basename(@__DIR__) * "_hourly_artifact") +const OUTPUT_DIR_HOURLY_ATMOS = + joinpath(DATA_DIR, "atmos_" * basename(@__DIR__) * "_hourly_artifact") +cds_PAT = "" +for dir in [ + OUTPUT_DIR_SURFACE, + OUTPUT_DIR_ATMOS, + OUTPUT_DIR_HOURLY_SURFACE, + OUTPUT_DIR_HOURLY_ATMOS, +] + if isdir(dir) + @warn "$dir already exists. Content will end up in the artifact and may be overwritten." + @warn "Abort this calculation, unless you know what you are doing." + else + @info "Creating directory $dir" + mkdir(dir) + end +end +########################################################################################## +# download and process data for the monthly averages +if !isfile(DOWNLOAD_FILE_NAME) + @info "$DOWNLOAD_FILE_NAME not found, downloading it (might take a while)" + if isfile(homedir() * "/.cdsapirc") + run(`python download_monthly_data.py -t $DOWNLOAD_FILE_NAME`) + else + println("Enter your CDS Personal Access Token:") + cds_PAT = readline() + println("Downloading data with CDS API using PAT: $cds_PAT") + run(`python download_monthly_data.py -k $cds_PAT -t $DOWNLOAD_FILE_NAME`) + end +end + +@info "Processing data for the monthly averages" +input_ds = NCDataset(DOWNLOAD_FILE_NAME, "r") +output_path = joinpath(OUTPUT_DIR_SURFACE, OUTPUT_FILE_NAME_SURFACE) +create_monthly_ds_surface(input_ds, output_path) +output_path = joinpath(OUTPUT_DIR_ATMOS, OUTPUT_FILE_NAME_ATMOS) +create_monthly_ds_atmos(input_ds, output_path) +close(input_ds) +@info "Data for the monthly averages processed" +# ######################################################################################### +# download and process data for the monthly averages by hour +year_paths = + [joinpath(DOWNLOAD_FILES_HOURLY_DIR, string(year) * ".nc") for year = 1979:2024] + +if !all(isfile, year_paths) + @info "Data for 1979-2024 not found in $DOWNLOAD_FILES_HOURLY_DIR, downloading and populatinit (might take a while)" + if isfile(homedir() * "/.cdsapirc") + run(`python download_monthly_hourly_data.py -d $DOWNLOAD_FILES_HOURLY_DIR`) + else + if cds_PAT == "" + println("Enter your CDS Personal Access Token:") + cds_PAT = readline() + end + println("Downloading data with CDS API using PAT: $cds_PAT") + run( + `python download_monthly_hourly_data.py -k $cds_PAT -d $DOWNLOAD_FILES_HOURLY_DIR`, + ) + end +end + +@info "Processing data for the monthly averages by hour" +input_ds = NCDataset(year_paths[end], "r") +output_path = joinpath(DOWNLOAD_FILES_HOURLY_DIR, "current_year.nc") +fix_current_year(input_ds, output_path) +close(input_ds) +year_paths[end] = output_path + +input_ds = NCDataset(year_paths; aggdim = "time", deferopen = false) +output_path = joinpath(OUTPUT_DIR_HOURLY_SURFACE, OUTPUT_FILE_HOURLY_NAME_SURFACE) +process_hourly_data_surface(input_ds, output_path) + +output_path = joinpath(OUTPUT_DIR_HOURLY_ATMOS, OUTPUT_FILE_HOURLY_NAME_ATMOS) +process_hourly_data_atmos(input_ds, output_path) +close(input_ds) + +create_artifact_guided( + OUTPUT_DIR_SURFACE; + artifact_name = "era5_monthly_averages_surface_single_level_1979_2024", +) +create_artifact_guided( + OUTPUT_DIR_ATMOS; + artifact_name = "era5_monthly_averages_atmos_single_level_1979_2024", + append = true, +) +create_artifact_guided( + OUTPUT_DIR_HOURLY_SURFACE; + artifact_name = "era5_monthly_averages_surface_single_level_1979_2024_hourly", + append = true, +) +create_artifact_guided( + OUTPUT_DIR_HOURLY_ATMOS; + artifact_name = "era5_monthly_averages_atmos_single_level_1979_2024_hourly", + append = true, +) diff --git a/era5_monthly_averages_single_level_1979_2024/download_monthly_data.py b/era5_monthly_averages_single_level_1979_2024/download_monthly_data.py new file mode 100644 index 0000000..92cc417 --- /dev/null +++ b/era5_monthly_averages_single_level_1979_2024/download_monthly_data.py @@ -0,0 +1,64 @@ +import cdsapi +import argparse + +parser = argparse.ArgumentParser(description='Use cdsapi to download era5 mean monthly surface fluxes') +parser.add_argument('-k', '--key', help="CDS API KEY", type=str) +parser.add_argument('-t', '--target', help="download target", type=str) +args = parser.parse_args() + +URL = 'https://cds.climate.copernicus.eu/api' +dataset = "reanalysis-era5-single-levels-monthly-means" +request = { + "product_type": ["monthly_averaged_reanalysis"], + "year": [ + "1979", "1980", "1981", + "1982", "1983", "1984", + "1985", "1986", "1987", + "1988", "1989", "1990", + "1991", "1992", "1993", + "1994", "1995", "1996", + "1997", "1998", "1999", + "2000", "2001", "2002", + "2003", "2004", "2005", + "2006", "2007", "2008", + "2009", "2010", "2011", + "2012", "2013", "2014", + "2015", "2016", "2017", + "2018", "2019", "2020", + "2021", "2022", "2023", + "2024" + ], + "month": [ + "01", "02", "03", + "04", "05", "06", + "07", "08", "09", + "10", "11", "12" + ], + "time": ["00:00"], + "data_format": "netcdf", + "download_format": "unarchived", + "grid" : [1.0, 1.0], + "variable": [ + "mean_surface_downward_long_wave_radiation_flux", + "mean_surface_downward_short_wave_radiation_flux", + "mean_surface_latent_heat_flux", + "mean_surface_net_long_wave_radiation_flux", + "mean_surface_net_short_wave_radiation_flux", + "mean_surface_sensible_heat_flux", + "mean_sub_surface_runoff_rate", + "mean_surface_runoff_rate", + "total_column_water" + ] +} + +if args.key is None: + client = cdsapi.Client() +else: + client = cdsapi.Client(url=URL, key=args.key) + +if args.target is None or args.target == "": + target = "era5_surface_fluxes_monthly_197901-202412.nc" +else: + target = args.target + +client.retrieve(dataset, request, target) diff --git a/era5_monthly_averages_single_level_1979_2024/download_monthly_hourly_data.py b/era5_monthly_averages_single_level_1979_2024/download_monthly_hourly_data.py new file mode 100644 index 0000000..41aeec5 --- /dev/null +++ b/era5_monthly_averages_single_level_1979_2024/download_monthly_hourly_data.py @@ -0,0 +1,138 @@ +############################################ +# This script was inspired by CDSAPItools, which is deprecated. +# See here: https://github.com/e5k/CDSAPItools/tree/main + +# This script downloads the data year by year. It queues 19 requests at a time and waits for them to finish. +# at a time. Once a request is finished, it downloads the data and updates the status in a csv file, +# and starts a new request. The csv isused to keep track of the status of the download. +# If the download is interrupted, the script can be resumed using the csv file +############################################ + +import cdsapi +import pandas as pd +import time +import argparse +import os + +parser = argparse.ArgumentParser(description='Use cdsapi to download era5 mean monthly surface fluxes') +parser.add_argument('-k', '--key', help="CDS API KEY", type=str) +parser.add_argument('-r', '--resume', action='store_true') +parser.add_argument('-d', '--dir', help="download target dir", type=str) +args = parser.parse_args() + +dataset = "reanalysis-era5-single-levels-monthly-means" + +years = [str(year) for year in range(1979, 2025)] +if args.dir is None: + output_dir = "" +else: + output_dir = args.dir + +SLEEP_TIME = 60 +URL = 'https://cds.climate.copernicus.eu/api' +BASE_URL = "https://cds.climate.copernicus.eu/api/retrieve/v1/jobs/" + +def submit_request(year): + print("Submitting Request for data for year: ", year) + if args.key is None: + submit_client = cdsapi.Client(wait_until_complete=False, delete=False) + else: + submit_client = cdsapi.Client(wait_until_complete=False, delete=False, url=URL, key=args.key) + request = { + "product_type": ["monthly_averaged_reanalysis_by_hour_of_day"], + "variable": [ + "mean_sub_surface_runoff_rate", + "mean_surface_downward_long_wave_radiation_flux", + "mean_surface_downward_short_wave_radiation_flux", + "mean_surface_latent_heat_flux", + "mean_surface_net_long_wave_radiation_flux", + "mean_surface_net_short_wave_radiation_flux", + "mean_surface_runoff_rate", + "mean_surface_sensible_heat_flux", + "total_column_water" + ], + "year": [year], + "month": [ + "01", "02", "03", + "04", "05", "06", + "07", "08", "09", + "10", "11", "12" + ], + "time": [ + "00:00", "01:00", "02:00", + "03:00", "04:00", "05:00", + "06:00", "07:00", "08:00", + "09:00", "10:00", "11:00", + "12:00", "13:00", "14:00", + "15:00", "16:00", "17:00", + "18:00", "19:00", "20:00", + "21:00", "22:00", "23:00" + ], + "data_format": "netcdf_legacy", + "download_format": "unarchived", + "grid" : [1.0, 1.0], +} + r = submit_client.retrieve(dataset, request) + return r.reply["request_id"] + + + +# submit dummy request +if args.key is None: + client = cdsapi.Client(wait_until_complete=False, delete=False) +else: + client = cdsapi.Client(wait_until_complete=False, delete=False, url=URL, key=args.key) +r = client.retrieve('reanalysis-era5-pressure-levels', { + "variable": "temperature", + "pressure_level": "1000", + "product_type": "reanalysis", + "date": "2017-12-01/2017-12-31", + "time": "12:00", + "format": "grib" + }) +r.retry_options["maximum_tries"] = 9999 +if args.resume: + df = pd.read_csv("era5_download_status_hourly.csv", index_col=0) +else: + df = pd.DataFrame() + index = 0 + for year in years: + df = pd.concat([df, pd.DataFrame({"year": year, "status": "not started", "request_id": ""}, index=[index])]) + index += 1 + df.to_csv("era5_download_status_hourly.csv") +while True: + if all(df["status"] == "done"): + print("All files are downloaded.") + break + not_started = df[df["status"] == "not started"] + started = df[df["status"] == "started"] + if started.shape[0] < 19 and not_started.shape[0] > 0: + add_index = not_started.iloc[0].name + df.iloc[add_index, 2] = submit_request(df.iloc[add_index, 0]) + df.iloc[add_index, 1] = "started" + for index, row in started.iterrows(): + r.url = BASE_URL + row["request_id"] + r.update() + if r.reply["state"] == "completed": + + df.iloc[index, 1] = "done" + print("Downloaded data for year: ", df.iloc[index, 0], end='\r') + try: + r.download(os.path.join(output_dir, str(df.iloc[index, 0]) + ".nc")) + except: + print("Error downloading data for year: ", df.iloc[index, 0]) + elif r.reply["state"] == "failed": + df.iloc[index, 1] = "failed" + print("Failed to download data for year: ", df.iloc[index, 0]) + elif r.reply["state"] == "running": + print("Data for year: ", df.iloc[index, 0], "is running.", end='\r') + elif r.reply["state"] == "queued": + print("Data for year: ", df.iloc[index, 0], "is queued.", end='\r') + elif r.reply["state"] == "accepted": + print("Data for year: ", df.iloc[index, 0], "is accepted.", end='\r') + time.sleep(2) + time.sleep(SLEEP_TIME) + df.to_csv("era5_download_status_hourly.csv") + + + diff --git a/era5_monthly_averages_single_level_1979_2024/process_downloads.jl b/era5_monthly_averages_single_level_1979_2024/process_downloads.jl new file mode 100644 index 0000000..fb80980 --- /dev/null +++ b/era5_monthly_averages_single_level_1979_2024/process_downloads.jl @@ -0,0 +1,496 @@ +using NCDatasets +using Dates + +""" + create_monthly_ds_surface(input_ds, output_path) + +Processes the `input_ds` dataset to convert the time dimension to DateTimes, +and add the `msuwlwrf` and `msuwswrf` variables, which are the upward +long-wave and short-wave radiation fluxes, respectively. Only surface variables are kept. +""" +function create_monthly_ds_surface(input_ds, output_path) + if isfile(output_path) + rm(output_path) + @info "Removed existing file $output_path" + end + + output_ds = NCDataset(output_path, "c") + for (attrib_name, attrib_value) in input_ds.attrib + output_ds.attrib[attrib_name] = attrib_value + end + defDim(output_ds, "longitude", input_ds.dim["longitude"]) + defDim(output_ds, "latitude", input_ds.dim["latitude"]) + defDim(output_ds, "time", input_ds.dim["date"]) + ignored_attribs = + ["_FillValue", "missing_value", "add_offset", "scale_factor", "coordinates"] + for (varname, var) in input_ds + if !( + varname in [ + "latitude", + "longitude", + "date", + "msdwlwrf", + "msdwswrf", + "msnlwrf", + "msnswrf", + "expver", + "number", + "tcw", + ] + ) + attrib = copy(var.attrib) + for (key, value) in attrib + if key in ignored_attribs || + occursin("GRIB", key) || + attrib[key] == "unknown" + delete!(attrib, key) + end + end + # the _FillValue attribute is automatically added by NCDatasets + # store everything as Float32 to save space, and max compression (lossless) + defVar( + output_ds, + varname, + Float32.(reverse(var[:, :, :], dims = 2)), + (dimnames(var)[1:2]..., "time"); + attrib = attrib, + deflatelevel = 9, + ) + end + end + defVar( + output_ds, + "expver", + input_ds["expver"][:], + ("time",); + attrib = input_ds["expver"].attrib, + ) + defVar( + output_ds, + "msuwlwrf", + Float32.( + reverse( + input_ds["msdwlwrf"][:, :, :] .- input_ds["msnlwrf"][:, :, :], + dims = 2, + ) + ), + ("longitude", "latitude", "time"), + attrib = ( + "units" => "W m**-2", + "long_name" => "Mean surface upward long-wave radiation flux", + ), + deflatelevel = 9, + ) + defVar( + output_ds, + "msuwswrf", + Float32.( + reverse( + input_ds["msdwswrf"][:, :, :] .- input_ds["msnswrf"][:, :, :], + dims = 2, + ) + ), + ("longitude", "latitude", "time"), + attrib = ( + "units" => "W m**-2", + "long_name" => "Mean surface upward short-wave radiation flux", + ), + deflatelevel = 9, + ) + + defVar( + output_ds, + "latitude", + reverse(input_ds["latitude"][:]), + ("latitude",); + attrib = delete!(copy(input_ds["latitude"].attrib), "stored_direction"), + ) + + defVar( + output_ds, + "longitude", + input_ds["longitude"][:], + ("longitude",); + attrib = input_ds["longitude"].attrib, + ) + + # If data is requested as netcdf, and not netcdf_legacy, the data includes a date dimension + # instead of time, where each date is an integer in the format yyyymmdd. Here we convert it to + # a DateTime object, and set the day to the 15th of the month. + new_times = map(input_ds["date"][:]) do t + d = DateTime(string(t), "yyyymmdd") + d + (Day(15) - Day(d)) + end + + # check that there are no duplicates and that it is sorted + @assert issorted(new_times) + for i = 2:length(new_times) + @assert new_times[i] != new_times[i-1] + end + new_times_attribs = ["standard_name" => "time", "long_name" => "Time"] + + defVar(output_ds, "time", new_times, ("time",); attrib = new_times_attribs) + close(output_ds) +end + +""" + create_monthly_ds_atmos(input_ds, output_path) + +Processes the `input_ds` dataset to convert the time dimension to DateTimes, +and add the `msuwlwrf` and `msuwswrf` variables, which are the upward +long-wave and short-wave radiation fluxes, respectively. Only vertical integral variables are kept. +""" +function create_monthly_ds_atmos(input_ds, output_path) + if isfile(output_path) + rm(output_path) + @info "Removed existing file $output_path" + end + + output_ds = NCDataset(output_path, "c") + for (attrib_name, attrib_value) in input_ds.attrib + output_ds.attrib[attrib_name] = attrib_value + end + defDim(output_ds, "longitude", input_ds.dim["longitude"]) + defDim(output_ds, "latitude", input_ds.dim["latitude"]) + defDim(output_ds, "time", input_ds.dim["date"]) + ignored_attribs = + ["_FillValue", "missing_value", "add_offset", "scale_factor", "coordinates"] + var = input_ds["tcw"] + + + attrib = copy(var.attrib) + for (key, value) in attrib + if key in ignored_attribs || occursin("GRIB", key) || attrib[key] == "unknown" + delete!(attrib, key) + end + end + # the _FillValue attribute is automatically added by NCDatasets + # store everything as Float32 to save space, and max compression (lossless) + defVar( + output_ds, + "tcw", + Float32.(reverse(var[:, :, :], dims = 2)), + (dimnames(var)[1:2]..., "time"); + attrib = attrib, + deflatelevel = 9, + ) + + + defVar( + output_ds, + "expver", + input_ds["expver"][:], + ("time",); + attrib = input_ds["expver"].attrib, + ) + + defVar( + output_ds, + "latitude", + reverse(input_ds["latitude"][:]), + ("latitude",); + attrib = delete!(copy(input_ds["latitude"].attrib), "stored_direction"), + ) + + defVar( + output_ds, + "longitude", + input_ds["longitude"][:], + ("longitude",); + attrib = input_ds["longitude"].attrib, + ) + # If data is requested as netcdf, and not netcdf_legacy, the data includes a date dimension + # instead of time, where each date is an integer in the format yyyymmdd. Here we convert it to + # a DateTime object, and set the day to the 15th of the month. + new_times = map(input_ds["date"][:]) do t + d = DateTime(string(t), "yyyymmdd") + d + (Day(15) - Day(d)) + end + + # check that there are no duplicates and that it is sorted + @assert issorted(new_times) + for i = 2:length(new_times) + @assert new_times[i] != new_times[i-1] + end + new_times_attribs = ["standard_name" => "time", "long_name" => "Time"] + + defVar(output_ds, "time", new_times, ("time",); attrib = new_times_attribs) + close(output_ds) +end + +""" + process_hourly_data_surface(input_ds, output_path) + +Processes the `input_ds` dataset to add the `msuwlwrf` and `msuwswrf` variables, which are the upward +long-wave and short-wave radiation fluxes, respectively. Only surface variables are kept. +""" +function process_hourly_data_surface(input_ds, output_path) + if isfile(output_path) + rm(output_path) + @info "Removed existing file $output_path" + end + + output_ds = NCDataset(output_path, "c") + defDim(output_ds, "longitude", input_ds.dim["longitude"]) + defDim(output_ds, "latitude", input_ds.dim["latitude"]) + + # When requesting data in a netcdf, CDS converts a GRIB to netcdf. During this process, + # extra data points are added. In this case, each month has 7 extra data points, where all the data + # is missing. We remove these data points, and checked the removed data is actually missing. + ignore_mod_31 = [2, 4, 6, 8, 10, 12, 14] + time_indx = filter(i -> !(i % 31 in ignore_mod_31), 1:length(input_ds["time"][:])) + defDim(output_ds, "time", length(time_indx)) + missing_indices = filter(i -> (i % 31 in ignore_mod_31), 1:length(input_ds["time"][:])) + for index in missing_indices + if !all(input_ds["msdwswrf"][:, :, index] .=== missing) + @error "The index pattern of the invalid data is not as expected" + end + end + ignored_attribs = ["_FillValue", "missing_value", "add_offset", "scale_factor"] + deflatelevel = 9 + for (varname, var) in input_ds + if !( + varname in [ + "latitude", + "longitude", + "time", + "msdwlwrf", + "msdwswrf", + "msnlwrf", + "msnswrf", + "tcw", + ] + ) + attrib = copy(var.attrib) + for attrib_name in ignored_attribs + delete!(attrib, attrib_name) + end + defVar( + output_ds, + varname, + Float32.(reverse(var[:, :, time_indx], dims = 2)), + dimnames(var); + attrib = attrib, + deflatelevel = deflatelevel, + ) + end + end + defVar( + output_ds, + "msuwlwrf", + Float32.( + reverse( + input_ds["msdwlwrf"][:, :, time_indx] .- + input_ds["msnlwrf"][:, :, time_indx], + dims = 2, + ) + ), + ("longitude", "latitude", "time"), + attrib = ( + "units" => "W m**-2", + "long_name" => "Mean surface upward long-wave radiation flux", + ), + deflatelevel = deflatelevel, + ) + defVar( + output_ds, + "msuwswrf", + Float32.( + reverse( + input_ds["msdwswrf"][:, :, time_indx] .- + input_ds["msnswrf"][:, :, time_indx], + dims = 2, + ) + ), + ("longitude", "latitude", "time"), + attrib = ( + "units" => "W m**-2", + "long_name" => "Mean surface upward short-wave radiation flux", + ), + deflatelevel = deflatelevel, + ) + # center times on 15th of each month + new_times = map(input_ds["time"][time_indx]) do t + t + (Day(15) - Day(t)) + end + # check that there are no duplicates and that it is sorted + @assert issorted(new_times) + for i = 2:length(new_times) + @assert new_times[i] != new_times[i-1] + end + new_times_attribs = ["standard_name" => "time", "long_name" => "Time"] + + defVar(output_ds, "time", new_times, ("time",); attrib = new_times_attribs) + + defVar( + output_ds, + "latitude", + reverse(input_ds["latitude"][:]), + ("latitude",); + attrib = input_ds["latitude"].attrib, + ) + + defVar( + output_ds, + "longitude", + input_ds["longitude"][:], + ("longitude",); + attrib = input_ds["longitude"].attrib, + ) + close(output_ds) +end + +""" + process_hourly_data_atmos(input_ds, output_path) + +Processes the `input_ds` dataset to add the `msuwlwrf` and `msuwswrf` variables, which are the upward +long-wave and short-wave radiation fluxes, respectively. Only vertical integral variables are kept. +""" +function process_hourly_data_atmos(input_ds, output_path) + if isfile(output_path) + rm(output_path) + @info "Removed existing file $output_path" + end + + output_ds = NCDataset(output_path, "c") + defDim(output_ds, "longitude", input_ds.dim["longitude"]) + defDim(output_ds, "latitude", input_ds.dim["latitude"]) + + # When requesting data in a netcdf, CDS converts a GRIB to netcdf. During this process, + # extra data points are added. In this case, each month has 7 extra data points, where all the data + # is missing. We remove these data points, and checked the removed data is actually missing. + ignore_mod_31 = [1, 3, 5, 7, 9, 11, 13] + time_indx = filter(i -> !(i % 31 in ignore_mod_31), 1:length(input_ds["time"][:])) + defDim(output_ds, "time", length(time_indx)) + missing_indices = filter(i -> (i % 31 in ignore_mod_31), 1:length(input_ds["time"][:])) + for index in missing_indices + if !all(input_ds["tcw"][:, :, index] .=== missing) + @error "The index pattern of the invalid data is not as expected" + end + end + ignored_attribs = ["_FillValue", "missing_value", "add_offset", "scale_factor"] + deflatelevel = 9 + var = input_ds["tcw"] + attrib = copy(var.attrib) + for attrib_name in ignored_attribs + delete!(attrib, attrib_name) + end + defVar( + output_ds, + "tcw", + Float32.(reverse(var[:, :, time_indx], dims = 2)), + dimnames(var); + attrib = attrib, + deflatelevel = deflatelevel, + ) + + + # center times on 15th of each month + new_times = map(input_ds["time"][time_indx]) do t + t + (Day(15) - Day(t)) + end + # check that there are no duplicates and that it is sorted + @assert issorted(new_times) + for i = 2:length(new_times) + @assert new_times[i] != new_times[i-1] + end + new_times_attribs = ["standard_name" => "time", "long_name" => "Time"] + + defVar(output_ds, "time", new_times, ("time",); attrib = new_times_attribs) + + defVar( + output_ds, + "latitude", + reverse(input_ds["latitude"][:]), + ("latitude",); + attrib = input_ds["latitude"].attrib, + ) + + defVar( + output_ds, + "longitude", + input_ds["longitude"][:], + ("longitude",); + attrib = input_ds["longitude"].attrib, + ) + close(output_ds) +end + +""" + fix_current_year(input_ds, output_path) + +Processes the `input_ds` dataset to remove the `expver` dimension +""" +function fix_current_year(input_ds, output_path) + if isfile(output_path) + rm(output_path) + @info "Removed existing file $output_path" + end + ignored_attribs = + ["_FillValue", "missing_value", "add_offset", "scale_factor", "coordinates"] + output_ds = NCDataset(output_path, "c") + defDim(output_ds, "longitude", input_ds.dim["longitude"]) + defDim(output_ds, "latitude", input_ds.dim["latitude"]) + defDim(output_ds, "time", input_ds.dim["time"]) + for (varname, var) in input_ds + if !(varname in ["latitude", "longitude", "time", "expver"]) + attrib = copy(var.attrib) + for (key, value) in attrib + if key in ignored_attribs + delete!(attrib, key) + end + end + defVar( + output_ds, + varname, + remove_expver_dim(input_ds, output_ds, varname), + ("longitude", "latitude", "time"); + attrib = attrib, + ) + end + end + defVar( + output_ds, + "time", + input_ds["time"][:], + ("time",); + attrib = input_ds["time"].attrib, + ) + defVar( + output_ds, + "longitude", + input_ds["longitude"][:], + ("longitude",); + attrib = input_ds["longitude"].attrib, + ) + defVar( + output_ds, + "latitude", + input_ds["latitude"][:], + ("latitude",); + attrib = input_ds["latitude"].attrib, + ) + close(output_ds) +end + +""" + remove_expver_dim(input_ds, output_ds, varname) + +Removes the `expver` dimension from the variable `varname` in the `input_ds` +""" +function remove_expver_dim(input_ds, output_ds, varname) + input_dims = size(input_ds[varname]) + out_var = similar(input_ds[varname][:], input_dims[1], input_dims[2], input_dims[4]) + for i = 1:input_dims[4] + if input_ds[varname][1, 1, 1, i] !== missing + out_var[:, :, i] .= input_ds[varname][:, :, 1, i] + elseif input_ds[varname][1, 1, 2, i] !== missing + out_var[:, :, i] .= input_ds[varname][:, :, 2, i] + elseif all(input_ds[varname][:, :, 1, i] .!== missing) + out_var[:, :, i] .= input_ds[varname][:, :, 1, i] + else + out_var[:, :, i] .= input_ds[varname][:, :, 2, i] + end + end + return out_var +end diff --git a/era5_monthly_averages_single_level_1979_2024/requirements.txt b/era5_monthly_averages_single_level_1979_2024/requirements.txt new file mode 100644 index 0000000..8dada4d --- /dev/null +++ b/era5_monthly_averages_single_level_1979_2024/requirements.txt @@ -0,0 +1,2 @@ +cdsapi==0.7.4 +pandas==2.2.3