Skip to content

Commit

Permalink
Update tests and test fixtures
Browse files Browse the repository at this point in the history
  • Loading branch information
David Stuebe committed Nov 25, 2024
1 parent 669a362 commit 6f9a188
Show file tree
Hide file tree
Showing 24 changed files with 29 additions and 258 deletions.
Binary file modified ...s/gfs.pgrb2.0p25/20221014/gfs.t00z.pgrb2.0p25.f000.test-limit-10.idx_grib_mapping.parquet
View file
Binary file not shown.
Binary file modified ...s/gfs.pgrb2.0p25/20221014/gfs.t06z.pgrb2.0p25.f021.test-limit-10.idx_grib_mapping.parquet
View file
Binary file not shown.
Binary file modified ...dx_fixtures/gfs.pgrb2.0p25/20231104/gfs.t00z.pgrb2.0p25.f000.test-limit-10.kindex.parquet
View file
Binary file not shown.
Binary file modified ...dx_fixtures/gfs.pgrb2.0p25/20231104/gfs.t00z.pgrb2.0p25.f021.test-limit-10.kindex.parquet
View file
Binary file not shown.
Binary file modified tests/grib_idx_fixtures/gfs.pgrb2.0p25/kerchunk_index.parquet
View file
Binary file not shown.
10 changes: 1 addition & 9 deletions ...idx_fixtures/gfs.pgrb2.0p25/reinflate/best_available/u/instant/isobaricInPa/u_chunks.json
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,4 @@
[
"u\/instant\/isobaricInPa\/u\/.zarray",
"u\/instant\/isobaricInPa\/u\/.zattrs",
"u\/instant\/isobaricInPa\/u\/0.0.0.0",
"u\/instant\/isobaricInPa\/u\/0.1.0.0",
"u\/instant\/isobaricInPa\/u\/0.2.0.0",
"u\/instant\/isobaricInPa\/u\/0.3.0.0",
"u\/instant\/isobaricInPa\/u\/0.4.0.0",
"u\/instant\/isobaricInPa\/u\/0.5.0.0",
"u\/instant\/isobaricInPa\/u\/0.6.0.0",
"u\/instant\/isobaricInPa\/u\/0.7.0.0"
"u\/instant\/isobaricInPa\/u\/.zattrs"
]
18 changes: 1 addition & 17 deletions ...s/grib_idx_fixtures/gfs.pgrb2.0p25/reinflate/horizon/u/instant/isobaricInPa/u_chunks.json
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,4 @@
[
"u\/instant\/isobaricInPa\/u\/.zarray",
"u\/instant\/isobaricInPa\/u\/.zattrs",
"u\/instant\/isobaricInPa\/u\/0.0.0.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.1.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.2.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.3.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.4.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.5.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.6.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.7.0.0",
"u\/instant\/isobaricInPa\/u\/1.1.0.0.0",
"u\/instant\/isobaricInPa\/u\/1.1.1.0.0",
"u\/instant\/isobaricInPa\/u\/1.1.2.0.0",
"u\/instant\/isobaricInPa\/u\/1.1.3.0.0",
"u\/instant\/isobaricInPa\/u\/1.1.4.0.0",
"u\/instant\/isobaricInPa\/u\/1.1.5.0.0",
"u\/instant\/isobaricInPa\/u\/1.1.6.0.0",
"u\/instant\/isobaricInPa\/u\/1.1.7.0.0"
"u\/instant\/isobaricInPa\/u\/.zattrs"
]
26 changes: 1 addition & 25 deletions .../grib_idx_fixtures/gfs.pgrb2.0p25/reinflate/run_time/u/instant/isobaricInPa/u_chunks.json
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,4 @@
[
"u\/instant\/isobaricInPa\/u\/.zarray",
"u\/instant\/isobaricInPa\/u\/.zattrs",
"u\/instant\/isobaricInPa\/u\/0.0.0.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.1.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.2.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.3.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.4.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.5.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.6.0.0",
"u\/instant\/isobaricInPa\/u\/0.0.7.0.0",
"u\/instant\/isobaricInPa\/u\/0.4.0.0.0",
"u\/instant\/isobaricInPa\/u\/0.4.1.0.0",
"u\/instant\/isobaricInPa\/u\/0.4.2.0.0",
"u\/instant\/isobaricInPa\/u\/0.4.3.0.0",
"u\/instant\/isobaricInPa\/u\/0.4.4.0.0",
"u\/instant\/isobaricInPa\/u\/0.4.5.0.0",
"u\/instant\/isobaricInPa\/u\/0.4.6.0.0",
"u\/instant\/isobaricInPa\/u\/0.4.7.0.0",
"u\/instant\/isobaricInPa\/u\/0.8.0.0.0",
"u\/instant\/isobaricInPa\/u\/0.8.1.0.0",
"u\/instant\/isobaricInPa\/u\/0.8.2.0.0",
"u\/instant\/isobaricInPa\/u\/0.8.3.0.0",
"u\/instant\/isobaricInPa\/u\/0.8.4.0.0",
"u\/instant\/isobaricInPa\/u\/0.8.5.0.0",
"u\/instant\/isobaricInPa\/u\/0.8.6.0.0",
"u\/instant\/isobaricInPa\/u\/0.8.7.0.0"
"u\/instant\/isobaricInPa\/u\/.zattrs"
]
18 changes: 1 addition & 17 deletions ...rib_idx_fixtures/gfs.pgrb2.0p25/reinflate/valid_time/u/instant/isobaricInPa/u_chunks.json
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,4 @@
[
"u\/instant\/isobaricInPa\/u\/.zarray",
"u\/instant\/isobaricInPa\/u\/.zattrs",
"u\/instant\/isobaricInPa\/u\/0.2.0.0.0",
"u\/instant\/isobaricInPa\/u\/0.2.1.0.0",
"u\/instant\/isobaricInPa\/u\/0.2.2.0.0",
"u\/instant\/isobaricInPa\/u\/0.2.3.0.0",
"u\/instant\/isobaricInPa\/u\/0.2.4.0.0",
"u\/instant\/isobaricInPa\/u\/0.2.5.0.0",
"u\/instant\/isobaricInPa\/u\/0.2.6.0.0",
"u\/instant\/isobaricInPa\/u\/0.2.7.0.0",
"u\/instant\/isobaricInPa\/u\/1.4.0.0.0",
"u\/instant\/isobaricInPa\/u\/1.4.1.0.0",
"u\/instant\/isobaricInPa\/u\/1.4.2.0.0",
"u\/instant\/isobaricInPa\/u\/1.4.3.0.0",
"u\/instant\/isobaricInPa\/u\/1.4.4.0.0",
"u\/instant\/isobaricInPa\/u\/1.4.5.0.0",
"u\/instant\/isobaricInPa\/u\/1.4.6.0.0",
"u\/instant\/isobaricInPa\/u\/1.4.7.0.0"
"u\/instant\/isobaricInPa\/u\/.zattrs"
]
Binary file modified ...es/hrrr.wrfsfcf/20221014/hrrr.t01z.wrfsfcf00.grib2.test-limit-10.idx_grib_mapping.parquet
View file
Binary file not shown.
Binary file modified ...es/hrrr.wrfsfcf/20221014/hrrr.t09z.wrfsfcf05.grib2.test-limit-10.idx_grib_mapping.parquet
View file
Binary file not shown.
Binary file modified ...idx_fixtures/hrrr.wrfsfcf/20231104/hrrr.t01z.wrfsfcf00.grib2.test-limit-10.kindex.parquet
View file
Binary file not shown.
Binary file modified ...idx_fixtures/hrrr.wrfsfcf/20231104/hrrr.t01z.wrfsfcf05.grib2.test-limit-10.kindex.parquet
View file
Binary file not shown.
Binary file modified tests/grib_idx_fixtures/hrrr.wrfsfcf/kerchunk_index.parquet
View file
Binary file not shown.
51 changes: 1 addition & 50 deletions ..._idx_fixtures/hrrr.wrfsfcf/reinflate/best_available/u/instant/isobaricInhPa/u_chunks.json
View file
Original file line number Diff line number Diff line change
@@ -1,53 +1,4 @@
[
"u\/instant\/isobaricInhPa\/u\/.zarray",
"u\/instant\/isobaricInhPa\/u\/.zattrs",
"u\/instant\/isobaricInhPa\/u\/0.0.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.0.0",
"u\/instant\/isobaricInhPa\/u\/0.3.0.0",
"u\/instant\/isobaricInhPa\/u\/0.4.0.0",
"u\/instant\/isobaricInhPa\/u\/0.5.0.0",
"u\/instant\/isobaricInhPa\/u\/0.6.0.0",
"u\/instant\/isobaricInhPa\/u\/1.0.0.0",
"u\/instant\/isobaricInhPa\/u\/1.1.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.0.0",
"u\/instant\/isobaricInhPa\/u\/1.5.0.0",
"u\/instant\/isobaricInhPa\/u\/1.6.0.0",
"u\/instant\/isobaricInhPa\/u\/2.0.0.0",
"u\/instant\/isobaricInhPa\/u\/2.1.0.0",
"u\/instant\/isobaricInhPa\/u\/2.2.0.0",
"u\/instant\/isobaricInhPa\/u\/2.3.0.0",
"u\/instant\/isobaricInhPa\/u\/2.4.0.0",
"u\/instant\/isobaricInhPa\/u\/2.5.0.0",
"u\/instant\/isobaricInhPa\/u\/2.6.0.0",
"u\/instant\/isobaricInhPa\/u\/3.0.0.0",
"u\/instant\/isobaricInhPa\/u\/3.1.0.0",
"u\/instant\/isobaricInhPa\/u\/3.2.0.0",
"u\/instant\/isobaricInhPa\/u\/3.3.0.0",
"u\/instant\/isobaricInhPa\/u\/3.4.0.0",
"u\/instant\/isobaricInhPa\/u\/3.5.0.0",
"u\/instant\/isobaricInhPa\/u\/3.6.0.0",
"u\/instant\/isobaricInhPa\/u\/4.0.0.0",
"u\/instant\/isobaricInhPa\/u\/4.1.0.0",
"u\/instant\/isobaricInhPa\/u\/4.2.0.0",
"u\/instant\/isobaricInhPa\/u\/4.3.0.0",
"u\/instant\/isobaricInhPa\/u\/4.4.0.0",
"u\/instant\/isobaricInhPa\/u\/4.5.0.0",
"u\/instant\/isobaricInhPa\/u\/4.6.0.0",
"u\/instant\/isobaricInhPa\/u\/5.0.0.0",
"u\/instant\/isobaricInhPa\/u\/5.1.0.0",
"u\/instant\/isobaricInhPa\/u\/5.2.0.0",
"u\/instant\/isobaricInhPa\/u\/5.3.0.0",
"u\/instant\/isobaricInhPa\/u\/5.4.0.0",
"u\/instant\/isobaricInhPa\/u\/5.5.0.0",
"u\/instant\/isobaricInhPa\/u\/5.6.0.0",
"u\/instant\/isobaricInhPa\/u\/6.0.0.0",
"u\/instant\/isobaricInhPa\/u\/6.1.0.0",
"u\/instant\/isobaricInhPa\/u\/6.2.0.0",
"u\/instant\/isobaricInhPa\/u\/6.3.0.0",
"u\/instant\/isobaricInhPa\/u\/6.4.0.0",
"u\/instant\/isobaricInhPa\/u\/6.5.0.0",
"u\/instant\/isobaricInhPa\/u\/6.6.0.0"
"u\/instant\/isobaricInhPa\/u\/.zattrs"
]
44 changes: 1 addition & 43 deletions tests/grib_idx_fixtures/hrrr.wrfsfcf/reinflate/horizon/u/instant/isobaricInhPa/u_chunks.json
View file
Original file line number Diff line number Diff line change
@@ -1,46 +1,4 @@
[
"u\/instant\/isobaricInhPa\/u\/.zarray",
"u\/instant\/isobaricInhPa\/u\/.zattrs",
"u\/instant\/isobaricInhPa\/u\/0.0.0.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.1.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.2.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.3.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.4.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.5.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.6.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.0.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.1.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.2.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.3.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.4.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.5.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.6.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.0.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.1.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.2.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.3.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.4.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.5.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.6.0.0",
"u\/instant\/isobaricInhPa\/u\/1.1.0.0.0",
"u\/instant\/isobaricInhPa\/u\/1.1.1.0.0",
"u\/instant\/isobaricInhPa\/u\/1.1.2.0.0",
"u\/instant\/isobaricInhPa\/u\/1.1.3.0.0",
"u\/instant\/isobaricInhPa\/u\/1.1.4.0.0",
"u\/instant\/isobaricInhPa\/u\/1.1.5.0.0",
"u\/instant\/isobaricInhPa\/u\/1.1.6.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.0.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.1.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.2.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.3.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.4.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.5.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.6.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.0.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.1.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.2.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.3.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.4.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.5.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.6.0.0"
"u\/instant\/isobaricInhPa\/u\/.zattrs"
]
44 changes: 1 addition & 43 deletions ...s/grib_idx_fixtures/hrrr.wrfsfcf/reinflate/run_time/u/instant/isobaricInhPa/u_chunks.json
View file
Original file line number Diff line number Diff line change
@@ -1,46 +1,4 @@
[
"u\/instant\/isobaricInhPa\/u\/.zarray",
"u\/instant\/isobaricInhPa\/u\/.zattrs",
"u\/instant\/isobaricInhPa\/u\/0.0.0.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.1.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.2.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.3.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.4.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.5.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.6.0.0",
"u\/instant\/isobaricInhPa\/u\/0.4.0.0.0",
"u\/instant\/isobaricInhPa\/u\/0.4.1.0.0",
"u\/instant\/isobaricInhPa\/u\/0.4.2.0.0",
"u\/instant\/isobaricInhPa\/u\/0.4.3.0.0",
"u\/instant\/isobaricInhPa\/u\/0.4.4.0.0",
"u\/instant\/isobaricInhPa\/u\/0.4.5.0.0",
"u\/instant\/isobaricInhPa\/u\/0.4.6.0.0",
"u\/instant\/isobaricInhPa\/u\/0.8.0.0.0",
"u\/instant\/isobaricInhPa\/u\/0.8.1.0.0",
"u\/instant\/isobaricInhPa\/u\/0.8.2.0.0",
"u\/instant\/isobaricInhPa\/u\/0.8.3.0.0",
"u\/instant\/isobaricInhPa\/u\/0.8.4.0.0",
"u\/instant\/isobaricInhPa\/u\/0.8.5.0.0",
"u\/instant\/isobaricInhPa\/u\/0.8.6.0.0",
"u\/instant\/isobaricInhPa\/u\/1.0.0.0.0",
"u\/instant\/isobaricInhPa\/u\/1.0.1.0.0",
"u\/instant\/isobaricInhPa\/u\/1.0.2.0.0",
"u\/instant\/isobaricInhPa\/u\/1.0.3.0.0",
"u\/instant\/isobaricInhPa\/u\/1.0.4.0.0",
"u\/instant\/isobaricInhPa\/u\/1.0.5.0.0",
"u\/instant\/isobaricInhPa\/u\/1.0.6.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.0.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.1.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.2.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.3.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.4.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.5.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.6.0.0",
"u\/instant\/isobaricInhPa\/u\/1.8.0.0.0",
"u\/instant\/isobaricInhPa\/u\/1.8.1.0.0",
"u\/instant\/isobaricInhPa\/u\/1.8.2.0.0",
"u\/instant\/isobaricInhPa\/u\/1.8.3.0.0",
"u\/instant\/isobaricInhPa\/u\/1.8.4.0.0",
"u\/instant\/isobaricInhPa\/u\/1.8.5.0.0",
"u\/instant\/isobaricInhPa\/u\/1.8.6.0.0"
"u\/instant\/isobaricInhPa\/u\/.zattrs"
]
44 changes: 1 addition & 43 deletions ...grib_idx_fixtures/hrrr.wrfsfcf/reinflate/valid_time/u/instant/isobaricInhPa/u_chunks.json
View file
Original file line number Diff line number Diff line change
@@ -1,46 +1,4 @@
[
"u\/instant\/isobaricInhPa\/u\/.zarray",
"u\/instant\/isobaricInhPa\/u\/.zattrs",
"u\/instant\/isobaricInhPa\/u\/0.0.0.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.1.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.2.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.3.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.4.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.5.0.0",
"u\/instant\/isobaricInhPa\/u\/0.0.6.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.0.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.1.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.2.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.3.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.4.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.5.0.0",
"u\/instant\/isobaricInhPa\/u\/0.1.6.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.0.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.1.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.2.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.3.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.4.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.5.0.0",
"u\/instant\/isobaricInhPa\/u\/0.2.6.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.0.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.1.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.2.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.3.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.4.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.5.0.0",
"u\/instant\/isobaricInhPa\/u\/1.2.6.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.0.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.1.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.2.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.3.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.4.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.5.0.0",
"u\/instant\/isobaricInhPa\/u\/1.3.6.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.0.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.1.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.2.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.3.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.4.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.5.0.0",
"u\/instant\/isobaricInhPa\/u\/1.4.6.0.0"
"u\/instant\/isobaricInhPa\/u\/.zattrs"
]
Binary file modified .../hrrr.wrfsubhf/20221014/hrrr.t01z.wrfsubhf00.grib2.test-limit-10.idx_grib_mapping.parquet
View file
Binary file not shown.
Binary file modified .../hrrr.wrfsubhf/20221014/hrrr.t03z.wrfsubhf09.grib2.test-limit-10.idx_grib_mapping.parquet
View file
Binary file not shown.
Binary file modified ...x_fixtures/hrrr.wrfsubhf/20231104/hrrr.t01z.wrfsubhf00.grib2.test-limit-10.kindex.parquet
View file
Binary file not shown.
Binary file modified ...x_fixtures/hrrr.wrfsubhf/20231104/hrrr.t01z.wrfsubhf09.grib2.test-limit-10.kindex.parquet
View file
Binary file not shown.
Binary file modified tests/grib_idx_fixtures/hrrr.wrfsubhf/kerchunk_index.parquet
View file
Binary file not shown.
32 changes: 21 additions & 11 deletions tests/test__grib_idx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
class DataExtractorTests(unittest.TestCase):
def test_integration(self):
# Small test file...
uri = "grib_idx_fixtures/hrrr.wrfsubhf.sample.grib2"
uri = f"{THIS_DIR}/grib_idx_fixtures/hrrr.wrfsubhf.sample.grib2"

scanned_msg_groups = scan_grib(uri)
corrected_msg_groups = [
Expand All @@ -67,14 +67,14 @@ def test_integration(self):
self.assertIsInstance(zg["vbdsf/avg/surface/vbdsf"], zarr.Array)
self.assertEqual(
zg["vbdsf/avg/surface"].attrs["coordinates"],
"surface latitude longitude time valid_time step",
"surface latitude longitude step time valid_time",
)
self.assertEqual(
zg["refc/instant/atmosphere"].attrs["coordinates"],
"atmosphere latitude longitude step time valid_time",
)
# Assert that the fill value is set correctly
self.assertIs(zg.refc.instant.atmosphere.step.fill_value, np.NaN)
self.assertIs(zg.refc.instant.atmosphere.step.fill_value, np.nan)

np.testing.assert_array_equal(
zg.refc.instant.atmosphere.time[:], np.array([1665709200])
Expand Down Expand Up @@ -223,6 +223,12 @@ def test_build_idx_grib_mapping(self):
mapper=correct_hrrr_subhf_step,
)

mapping = mapping.assign(
uri=lambda x: x.uri.str.replace(THIS_DIR, ""),
grib_uri=lambda x: x.uri.str.replace(THIS_DIR, ""),
idx_uri=lambda x: x.uri.str.replace(THIS_DIR, ""),
)

# # To update the test fixture
# write_path = os.path.join(
# THIS_DIR,
Expand All @@ -249,7 +255,6 @@ def test_build_idx_grib_mapping(self):

idxdf = parse_grib_idx(
basename=basename,
tstamp=pd.to_datetime("2023-12-31T23:59:00"),
)

# Get the runtime hour from the filename as we would in prod
Expand All @@ -265,6 +270,10 @@ def test_build_idx_grib_mapping(self):
pd.Timestamp(f"2023-11-04T{runtime_hour}"), mapping, idxdf
)

mapped_index = mapped_index.assign(
uri=lambda x: x.uri.str.replace(THIS_DIR, "")
)

# Read the expected fixture - created by test_kerchunk_indexing
kindex_test_path = os.path.join(
THIS_DIR,
Expand Down Expand Up @@ -351,6 +360,11 @@ def test_kerchunk_indexing(self):
dt, grib_tree_store, grib=True
)

# Strip the current working directory
kindex = kindex.assign(
uri=lambda x: x.uri.str.replace(THIS_DIR, "")
)

# # To update the test fixture
# write_path = os.path.join(
# THIS_DIR,
Expand Down Expand Up @@ -406,18 +420,14 @@ def _test_index_extraction(self, sample_prefix: str):

# # To update the test fixture
# write_path = os.path.join(
# TESTS_DIR, "grib_idx_fixtures", sample_prefix, "kerchunk_index.parquet"
# THIS_DIR, "grib_idx_fixtures", sample_prefix, "kerchunk_index.parquet"
# )
# k_index.to_parquet(fpath)
# k_index.to_parquet(write_path)

test_path = os.path.join(
THIS_DIR, "grib_idx_fixtures", sample_prefix, "kerchunk_index.parquet"
)
expected = pd.read_parquet(test_path)
# adjust datetime resolution
for col in ["time", "valid_time"]:
expected[col] = expected[col].dt.as_unit("ns")

pd.testing.assert_frame_equal(k_index, expected)

def test_strip_datavar_chunks(self):
Expand Down Expand Up @@ -622,7 +632,7 @@ def _reinflate_grib_store(
)

# # To update test grib_idx_fixtures
# write_path = os.path.join(TESTS_DIR, "grib_idx_fixtures", dataset, "reinflate", aggregation.value, f"{vpath}_chunks.json")
# write_path = os.path.join(THIS_DIR, "grib_idx_fixtures", dataset, "reinflate", aggregation.value, f"{vpath}_chunks.json")
# with fsspec.open(write_path, "w",) as f:
# f.write(ujson.dumps(key_set, indent=2))

Expand Down

0 comments on commit 6f9a188

Please sign in to comment.