From 6e7f2e20b30b21a6a0450a692c70eae0f3e504f5 Mon Sep 17 00:00:00 2001 From: "Oriol Ricart Vilarrubias." <45894267+uriii3@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:11:27 +0200 Subject: [PATCH] Cf checker (#127) Co-authored-by: Renaud <38732257+renaudjester@users.noreply.github.com> Add a compliance checker to make sure that the netcdf output files are as expected. --- ARCO_smaller_area_subset_method_default.nc | Bin 23331 -> 0 bytes conda_environment_test.yaml | 1 + .../download_functions/subset_xarray.py | 6 +- example.py | 0 tests/__snapshots__/test_cf_compliance.ambr | 31 +++++++ tests/test_cf_compliance.py | 76 ++++++++++++++++++ 6 files changed, 112 insertions(+), 2 deletions(-) delete mode 100644 ARCO_smaller_area_subset_method_default.nc delete mode 100644 example.py create mode 100644 tests/__snapshots__/test_cf_compliance.ambr create mode 100644 tests/test_cf_compliance.py diff --git a/ARCO_smaller_area_subset_method_default.nc b/ARCO_smaller_area_subset_method_default.nc deleted file mode 100644 index c67d13c6d5686a4b55982d193f5d82d89f799a99..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23331 zcmeHP4UiPY6@I(R&;5WxggXHZb8;~UatjBC3IgG7ZNPAqRk1=b|J;Ygdvt1Q;JnVs*F`#;R{>e0A5?!+UD+Jh3L^ygjWRZ2jKsk9 zMAt}VGWf(_%x345qHEC3r=f*O`f1q%smAW+Cae*fJK0#g<^xrBnO(}q6J~4^T&2V{ zQF18*&T`%Jc+LHhE|dk_3e_0-WT_G<7z3+d5NhyBa=iRGnTXGhAP#R@de6TbyIUhz zyKZ_KF99gUAhGa;T3Ah4mjO^UtcvLoI6x5%^mG6;9eiR9T!RLMx$!G_lNAI>DMz+BLynrv_c!0sd`L*}Lc*Q|lJ^Y_% zAAs$|aS?SV!0T7F{#84}*H2L%9Ul#d{x5a)ovUYE9pLIOS6{j8#nsO)!*Q8}%jhCs z$)II>bh0%dC-RqkzA03)--o2Cz+qqt4I`7anQExhpt1xizl&UB%P1u&Yc}AX_7?BaxH(P>qt&~j6bht$BRh@PIZJUF1`KA@vxtSRmb~j!C zYl!1KIXmt>__L*uA`YCB=?Qpra9$@26Ne^ew&;zUZh);6fmvuWG$t&+72cu<#IrO9 zgSq*4a2v@;NO)55&TD^^!*&BD1H|?}`6^5#j&tM;J6tvSy{oU<&@EEEpQF)$b9Pi} zcJ-{Q6I~tXG7)!AI{V0FIN-E1m%+Gt{@lxU9XguEczH&x0XaL)bj*&taInLoXUFW) z_e>l)h<2#(Nfq+^dA6EACrCDcU-HE%8s4<~`~Q@xy2PO>g6u>b77rw{s%oy4GM-Z* z1m!)eYPM1pjm(0?$jmpNig4$yE$2(+&TuG~#(5$RN3Yy+c;$R4WAFkQZdM^Hw@xW{ zj7-Kg8(*A);jVM$_C~t8!tkzontTC2NGx#S=Eo1+X^KQ%`1^1Tyw1$y&tYmF(XbTHYT0Rq}@Qh*l zX;>E&#dT>i`^>nz7RXE#K$n9km7HSHq~2!;d4w-S$>eFXYc`NHt&B(~8>95T0HPO> zzdw~REpxrhPHh#{No`2lmDLALcjhbX!bxzHYT#;2n_(I03^hiSkg@b+Tu;Z3tfgpm z08WSb)B&C0NVqfH9tuZWTe`a8e5?p5GOk;C#;^b?@rDM6I6C=~88elH5=tMbe=1fW z0}?-4^3MDHOlm+p|Lm(48rVDinil&ZLFaQqJ_Q|^D4qh{Obls#hMv@VOd}D`Xg%pv zpSCiQ`nsObI%1Tts&^=3#xhz%QcvJjMnhZIBB`_y(=(Pf(2v$@#I+`p6u>2gd>~Q- z>6k2vmJn1_TStqwvbCdbK>!y*f+=nE7-=IJ!%>Ka6dJ#YvDdQtm&}{jr>D)N5san! zjdaqC4P=7GKmf-Mku{V`t~ZhvJ|QwM4>F-hZC&uf?315P#Rpe-9)N^z_j%VC39%Ac)3xf6bAQx>9tx8jEEmQRv!5iu^m3P^K zxk)j>EUOPL4lb;X8|#fksvn0kYi@76XVJ2{VBLZswceu>qk+G_`QPOLRgIUfu4=q* zM^)q9I}YptII!oM`#;~(bKh{o^V5eLwpLu*u;l9DhIHoIhVqAp8%|73JgJ@rJPmjn z@HF6Qz|+9V(tvOcoU6P}KEEFhi)GQ^G~>PuQTCuj-)r>)xf@2! zc*wxbRe#4brEbkc|FE7CedWqs!m_p^~*YqCn}b>u`X>G znP@VVCg-y9Wgp=lzPx~Yn8h?nFS!^;-x`_WwUtc^%vfsnv0hP$+e4=L0UL2ZbtJsh|L%)FA!d(`j|PgvOR5vW*@56R5peaS(^kM~S`sfQ z8i5r$wHW(P%*6B|hXKGQNH0GygayyhLNHjYp0KFVSk1gxGJ66bqPS z$$0@2W3X0!8^x|+7BNjw4>XD}b zfLz=$D=f8=bxY0OfBQ_~X=87h58Y=m0i@Dk6H{L8UE{FiD`ZCny5*QS*d1OB_r_AVs6eAAbFz247ZijR0MK_f|D=U>0(K zW#QUP2VA=oQ(SOOcsuGO!V_@qPb33pa2m1my0_jo{$-ywCutsyN(05AGu3C{>@RSd z2sjN0oE`776N%2af1M@;&W^rgY18IWwZdaQS`7$v#-~JukX4V(Ml^H`haYL`x3SOB zpfg&l(dbWa?G_8nWTAP184Y11t{cS_KveWzBe68&39O9PW#aKFt#0w6`L*-w@XtQX zbL5FdmQ7VH4CK0Pb!4F+7uhR{%p6gqI))k369%1<6lb6q$)US#9Ui{Y@hs+vKO>21 z@#s^l0eS5Fw|-v}HPJsN&&z?T?E;VCHBHDw9MY3V#OcK!Q|CkIiyD@w)+~ z(18eq=tUI{s^jeN5tlC0z2DE61~vvRzeR#!-f3rp;E)xP>#E$WOeoh?xjSiP6d2w> zGSG)gxK)7RZ6s`tG;slj-=c_+M8{JvxvK(V6tO}+6z|+Hvl_OOj7I716xdh1wNgzp zBNDyoS5F)$gJ!$s8}Hu_2PtBs#J;gxZ+=z2`qze$Nc8I8D>ql|kayni3DJNP3@g36 zdKR`(vC0ta*Z)M1IxWK0wcxZSmsz+vymI=`))UeuPiXNPSQ+YQrH4?wUxI5K1KCc$OnJB^>e0PXiAT^YfQ)XsJ!a@hpe$Ib=D67l}BY z< diff --git a/conda_environment_test.yaml b/conda_environment_test.yaml index 7075f045..42073aad 100644 --- a/conda_environment_test.yaml +++ b/conda_environment_test.yaml @@ -7,6 +7,7 @@ dependencies: - tox==4.11.4 - netcdf4==1.6.5 - syrupy==4.6.1 + - compliance-checker==5.1.1 - pip: - pytest-order==1.2.1 - freezegun==1.5.1 diff --git a/copernicusmarine/download_functions/subset_xarray.py b/copernicusmarine/download_functions/subset_xarray.py index d35ece72..6f7d63c6 100644 --- a/copernicusmarine/download_functions/subset_xarray.py +++ b/copernicusmarine/download_functions/subset_xarray.py @@ -434,10 +434,12 @@ def _adequate_dtypes_of_valid_minmax( dataset: xarray.Dataset, variable: str ) -> xarray.Dataset: dataset[variable].attrs["valid_min"] = numpy.array( - [dataset[variable].attrs["valid_min"]], dtype=dataset[variable].dtype + [dataset[variable].attrs["valid_min"]], + dtype=dataset[variable].encoding["dtype"], )[0] dataset[variable].attrs["valid_max"] = numpy.array( - [dataset[variable].attrs["valid_max"]], dtype=dataset[variable].dtype + [dataset[variable].attrs["valid_max"]], + dtype=dataset[variable].encoding["dtype"], )[0] return dataset diff --git a/example.py b/example.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/__snapshots__/test_cf_compliance.ambr b/tests/__snapshots__/test_cf_compliance.ambr new file mode 100644 index 00000000..d430df21 --- /dev/null +++ b/tests/__snapshots__/test_cf_compliance.ambr @@ -0,0 +1,31 @@ +# serializer version: 1 +# name: TestCFCompliance.test_subset_open + 'cmems_mod_nws_bgc-pft_my_7km-3D-pico_P1M-m' +# --- +# name: TestCFCompliance.test_subset_open.1 + 160 +# --- +# name: TestCFCompliance.test_subset_open.2 + 160 +# --- +# name: TestCFCompliance.test_subset_open.3 + list([ + ]) +# --- +# name: TestCFCompliance.test_subset_with_warns + 'cmems_obs-sst_med_phy-sst_nrt_diurnal-oi-0.0625deg_PT1H-m' +# --- +# name: TestCFCompliance.test_subset_with_warns.1 + 135 +# --- +# name: TestCFCompliance.test_subset_with_warns.2 + 136 +# --- +# name: TestCFCompliance.test_subset_with_warns.3 + list([ + '§2.6 Attributes', + list([ + '§2.6.1 Conventions global attribute does not contain "CF-1.6"', + ]), + ]) +# --- diff --git a/tests/test_cf_compliance.py b/tests/test_cf_compliance.py new file mode 100644 index 00000000..0b222708 --- /dev/null +++ b/tests/test_cf_compliance.py @@ -0,0 +1,76 @@ +import json + +import xarray + +from copernicusmarine import subset +from tests.test_utils import execute_in_terminal + + +class TestCFCompliance: + def test_subset_open(self, tmp_path, snapshot): + dataset_id = "cmems_mod_nws_bgc-pft_my_7km-3D-pico_P1M-m" + self.if_I_subset_a_dataset(dataset_id, tmp_path, "output_1.nc", "pico") + self.then_it_is_cf_compliant( + dataset_id, tmp_path, snapshot, "output_1" + ) + + def test_subset_with_warns(self, tmp_path, snapshot): + dataset_id = ( + "cmems_obs-sst_med_phy-sst_nrt_diurnal-oi-0.0625deg_PT1H-m" + ) + self.if_I_subset_a_dataset( + dataset_id, + tmp_path, + "output_2.nc", + "analysed_sst", + ) + self.then_it_is_cf_compliant( + dataset_id, tmp_path, snapshot, "output_2" + ) + + def if_I_subset_a_dataset( + self, dataset_id, tmp_path, output_filename, variable + ): + subset( + dataset_id=dataset_id, + variables=[variable], + output_directory=tmp_path, + output_filename=output_filename, + start_datetime="2022-01-01T00:00:00", + end_datetime="2022-01-05T00:00:00", + force_download=True, + ) + assert (tmp_path / output_filename).exists() + + def then_it_is_cf_compliant( + self, dataset_id, tmp_path, snapshot, output_filename + ): + dataset_id = dataset_id + dataset = xarray.open_dataset(f"{tmp_path}/{output_filename}.nc") + CF_convention = dataset.attrs["Conventions"][-3:] + if CF_convention < "1.6": + CF_convention = "1.6" + command = [ + "compliance-checker", + f"--test=cf:{CF_convention}", + f"{tmp_path}/{output_filename}.nc", + "-f", + "json", + "-o", + f"{tmp_path}/{output_filename}_checked.json", + ] + execute_in_terminal(command) + + f = open(f"{tmp_path}/{output_filename}_checked.json") + data = json.load(f) + + list_msgs = [] + for diccionari in data[f"cf:{CF_convention}"]["all_priorities"]: + if len(diccionari["msgs"]) > 0: + list_msgs.append(diccionari["name"]) + list_msgs.append(diccionari["msgs"]) + + assert dataset_id == snapshot + assert data[f"cf:{CF_convention}"]["scored_points"] == snapshot + assert data[f"cf:{CF_convention}"]["possible_points"] == snapshot + assert list_msgs == snapshot