From ead41d5a9c7102c36dfb70e79b53ab9d71cee90d Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Fri, 15 Dec 2017 11:32:10 -0500 Subject: [PATCH 01/44] 464 unit test maintenance: trash t_version_ovverride due to irrelevance: GIPS_OVERRIDE_VERSION is going away during test overhaul anyway. Unrelatedly, an assert method quit working so do it manually instead. --- gips/test/unit/t_core.py | 20 -------------------- gips/test/unit/t_modis_fetch.py | 2 +- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/gips/test/unit/t_core.py b/gips/test/unit/t_core.py index 5e38b866..a1d5066c 100644 --- a/gips/test/unit/t_core.py +++ b/gips/test/unit/t_core.py @@ -11,26 +11,6 @@ from gips.data.landsat.landsat import landsatRepository, landsatData from gips.inventory import dbinv -def t_version_override(mocker): - """Test gips.__init__.detect_version() for correct override of __version__.""" - env = mocker.patch.object(gips.os, 'environ') - # os.environ.get is called by libs as well as detect_version(); fortunately no harm seems to - # come from giving them bad results. - - # no override requested - env.get.side_effect = lambda key, default=None: default # key not found - version_a = gips.detect_version() - - # override requested - env.get.side_effect = lambda key, default=None: 'fancy-new-version' - version_b = gips.detect_version() - - env.get.assert_has_calls([ # assert two identical calls - mock.call('GIPS_OVERRIDE_VERSION', gips.version.__version__) for _ in range(2) - ]) - assert (version_a, version_b) == (gips.version.__version__, 'fancy-new-version') - - def t_repository_find_tiles_normal_case(mocker, orm): """Test Repository.find_tiles using landsatRepository as a guinea pig.""" m_list_tiles = mocker.patch('gips.data.core.dbinv.list_tiles') diff --git a/gips/test/unit/t_modis_fetch.py b/gips/test/unit/t_modis_fetch.py index 4e8b121d..8d0da6ee 100644 --- a/gips/test/unit/t_modis_fetch.py +++ b/gips/test/unit/t_modis_fetch.py @@ -69,7 +69,7 @@ def t_managed_request_returns_none(fetch_mocks, call, expected): # assertions assert expected in managed_request.call_args[0] - managed_request.assert_called_once() + assert 1 == managed_request.call_count # assert_called_once() quit working # It should skip the I/O code except for fetching the directory listing [f.assert_not_called() for f in (open, file.write)] From 53aa20695fc565e9264aa40a7d3cb0c49fbf03fe Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Fri, 15 Dec 2017 13:26:24 -0500 Subject: [PATCH 02/44] 464 have to remove directories carefully: If a/b/c and rmdir b, it breaks. Have to remove c first. --- gips/test/int/t_chirps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gips/test/int/t_chirps.py b/gips/test/int/t_chirps.py index 2952259f..47a08b02 100644 --- a/gips/test/int/t_chirps.py +++ b/gips/test/int/t_chirps.py @@ -40,7 +40,7 @@ def scene_dir_setup(): os.remove(target_product_fp) if use_fake_asset: os.remove(target_asset_fp) - [os.rmdir(d) for d in made_dirs] + [os.rmdir(d) for d in reversed(made_dirs)] # <- remove in correct order @pytest.mark.django_db def t_chirps_product_symlink(mocker, scene_dir_setup): From dfcbfe5859c231adfd64091f1f431c15dbee9a24 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Mon, 18 Dec 2017 14:58:20 -0500 Subject: [PATCH 03/44] 464 working draft non-scripttest modis t_process: * Do everything inline in def t_process; needs refactoring. * New option, --record=file, prevents the test run from actually testing; instead it just records outcomes in the specified file. * Replace depreated envoy with sh, which is apparently maintained and trendy. * new util.py code to replace scripttest, a filesystem scanner and an expectation generator that detects filetypes & hashes if appropriate --- conftest.py | 6 + dev_requirements.txt | 3 +- gips/test/sys/expected/modis.py | 379 +++++++++++++++++++++++++++----- gips/test/sys/t_modis.py | 55 ++++- gips/test/sys/util.py | 65 +++++- 5 files changed, 441 insertions(+), 67 deletions(-) diff --git a/conftest.py b/conftest.py index 8425ace8..b9670a7c 100644 --- a/conftest.py +++ b/conftest.py @@ -73,6 +73,9 @@ def pytest_addoption(parser): parser.addoption( "--expectation-format", action="store_true", help=help_str) + parser.addoption('--record', action='store', default=None, + help="Pass in a filename for expecations" + " to be written to that filename.") def pytest_configure(config): """Process user config & command-line options.""" @@ -80,6 +83,9 @@ def pytest_configure(config): level = ('warning' if raw_level is None else raw_level).upper() root_logger.setLevel(level) + # TODO complain here if --record or --record-dir are wrong: + # * --record file should not exist + # * --record-dir should exist dr = str(config.getini('data-repo')) if not dr: raise ValueError("No value specified for 'data-repo' in pytest.ini") diff --git a/dev_requirements.txt b/dev_requirements.txt index 08821eda..df93926b 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -4,4 +4,5 @@ pytest-timeout pytest-mock pytest-django scripttest -envoy +envoy # deprecated +sh diff --git a/gips/test/sys/expected/modis.py b/gips/test/sys/expected/modis.py index 58f6db2a..40b14cdc 100644 --- a/gips/test/sys/expected/modis.py +++ b/gips/test/sys/expected/modis.py @@ -28,62 +28,329 @@ t_process = { - 'updated': { - 'modis/stage': None, - 'modis/tiles/h12v04/2012336': None, - 'modis/tiles/h12v04/2012337': None, - 'modis/tiles/h12v04/2012338': None, - }, - 'created': { - # Each None is a symlink; see prism for a way to test it more meaningfully. - 'modis/tiles/h12v04/2012336/h12v04_2012336_MCD_quality.tif': None, - 'modis/tiles/h12v04/2012337/h12v04_2012337_MCD_quality.tif': None, - 'modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8td.tif': None, - 'modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8tn.tif': None, - 'modis/tiles/h12v04/2012338/h12v04_2012338_MCD_quality.tif': None, - 'modis/tiles/h12v04/2012336/h12v04_2012336_MCD_fsnow.tif': -843500181, - 'modis/tiles/h12v04/2012336/h12v04_2012336_MCD_indices.tif': -684782438, - 'modis/tiles/h12v04/2012336/h12v04_2012336_MCD_snow.tif': 388495321, - 'modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_obstime.tif': -661225282, - 'modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_temp.tif': 1278807725, - 'modis/tiles/h12v04/2012336/h12v04_2012336_MOD_clouds.tif': 161070470, - 'modis/tiles/h12v04/2012337/h12v04_2012337_MCD_fsnow.tif': 297883486, - 'modis/tiles/h12v04/2012337/h12v04_2012337_MCD_indices.tif': -1806386481, - 'modis/tiles/h12v04/2012337/h12v04_2012337_MCD_snow.tif': -748640537, - 'modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_obstime.tif': 348598487, - 'modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_temp.tif': 1853031589, - 'modis/tiles/h12v04/2012337/h12v04_2012337_MOD_clouds.tif': -832284681, - 'modis/tiles/h12v04/2012337/h12v04_2012337_MOD_ndvi8.tif': -395723296, - 'modis/tiles/h12v04/2012338/h12v04_2012338_MCD_fsnow.tif': -1930181337, - 'modis/tiles/h12v04/2012338/h12v04_2012338_MCD_indices.tif': -521042334, - 'modis/tiles/h12v04/2012338/h12v04_2012338_MCD_snow.tif': 387672365, - 'modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_obstime.tif': -1046273330, - 'modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_temp.tif': 207791084, - 'modis/tiles/h12v04/2012338/h12v04_2012338_MOD_clouds.tif': 296967275, - }, - 'ignored': [ # contain site-specific data not applicable across users/configurations - 'gips-inv-db.sqlite3', - 'modis/tiles/h12v04/2012336/MCD43A2.A2012336.h12v04.006.2016112010833.hdf.index', - 'modis/tiles/h12v04/2012336/MCD43A4.A2012336.h12v04.006.2016112010833.hdf.index', - 'modis/tiles/h12v04/2012336/MOD10A1.A2012336.h12v04.005.2012339213007.hdf.index', - 'modis/tiles/h12v04/2012336/MOD11A1.A2012336.h12v04.006.2016131164025.hdf.index', - 'modis/tiles/h12v04/2012336/MYD10A1.A2012336.h12v04.005.2012340031954.hdf.index', - 'modis/tiles/h12v04/2012336/MYD11A1.A2012336.h12v04.006.2016132100529.hdf.index', - 'modis/tiles/h12v04/2012337/MCD43A2.A2012337.h12v04.006.2016112013509.hdf.index', - 'modis/tiles/h12v04/2012337/MCD43A4.A2012337.h12v04.006.2016112013509.hdf.index', - 'modis/tiles/h12v04/2012337/MOD09Q1.A2012337.h12v04.006.2015253024347.hdf.index', - 'modis/tiles/h12v04/2012337/MOD10A1.A2012337.h12v04.005.2012340033542.hdf.index', - 'modis/tiles/h12v04/2012337/MOD11A1.A2012337.h12v04.006.2016131185851.hdf.index', - 'modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf.index', - 'modis/tiles/h12v04/2012337/MYD10A1.A2012337.h12v04.005.2012340112013.hdf.index', - 'modis/tiles/h12v04/2012337/MYD11A1.A2012337.h12v04.006.2016132101831.hdf.index', - 'modis/tiles/h12v04/2012338/MCD43A2.A2012338.h12v04.006.2016112020013.hdf.index', - 'modis/tiles/h12v04/2012338/MCD43A4.A2012338.h12v04.006.2016112020013.hdf.index', - 'modis/tiles/h12v04/2012338/MOD10A1.A2012338.h12v04.005.2012341091201.hdf.index', - 'modis/tiles/h12v04/2012338/MOD11A1.A2012338.h12v04.006.2016131192106.hdf.index', - 'modis/tiles/h12v04/2012338/MYD10A1.A2012338.h12v04.005.2012340142152.hdf.index', - 'modis/tiles/h12v04/2012338/MYD11A1.A2012338.h12v04.006.2016132112414.hdf.index', - ], + # 'landcover' [], # is annual, not available for the scene under test + # weird symlink products: + # t_process[quality] recording: + 'quality': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/data-root/modis/tiles/h12v04/2012336/MCD43A2.A2012336.h12v04.006.2016112010833.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/data-root/modis/tiles/h12v04/2012337/MCD43A2.A2012337.h12v04.006.2016112013509.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/data-root/modis/tiles/h12v04/2012338/MCD43A2.A2012338.h12v04.006.2016112020013.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo')], + + # t_process[temp8tn] recording: + 'temp8tn': + [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8tn.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/data-root/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Night_1km')], + + # t_process[temp8td] recording: + 'temp8td': + [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8td.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/data-root/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Day_1km')], + + # normal products + # t_process[satvi] recording: + 'satvi': + [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_satvi.tif', + 'hash', + 'sha256', + '94bee7cecf0f0a227fd8532223f86de10e01c3e39d1dd70b7675d91b3ca338a4'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_satvi.tif', + 'hash', + 'sha256', + '6397669a6cd62bb39813b9000d93ba50f4cabe73aebf1e6444f4e3be9f630e19'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_satvi.tif', + 'hash', + 'sha256', + '6124ee615a4f522fdf966a5479d03d87934908e5090a192192f9700020f24760')], + + # t_process[ndti] recording: + 'ndti': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndti.tif', + 'hash', + 'sha256', + '025ea7a9204ae8ad0952a92b386bf735dbbce8337e336843476d3f227cf1f1ab'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndti.tif', + 'hash', + 'sha256', + 'b6baa8c7a3996f680c4177e4157c9275dd895c8e6350afa6ef564e673bb6a5f0'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndti.tif', + 'hash', + 'sha256', + 'a1c4f1161e8a584257352e76160319c39aa4605dd565d388d0116e193773771f')], + + # t_process[ndvi] recording: + 'ndvi': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndvi.tif', + 'hash', + 'sha256', + 'efbeae3a1c5e59ccd54b6108e275f9a8f0059162ac2797f4d69b00acb80147b3'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndvi.tif', + 'hash', + 'sha256', + 'b56d5c96229a1ad4cdcbd09db10b19a216b4cde8a0ffd184ee5cf01a35ab2b46'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndvi.tif', + 'hash', + 'sha256', + 'd9ff5d2f85ac9a267a8f6b1aca34f62b8196721e212b1e092f82ce9f1e208c61')], + + # t_process[isti] recording: + 'isti': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_isti.tif', + 'hash', + 'sha256', + 'c0a38d1cd00de52eacd8abcda714bfdaa575787e27b40dc0472ffca5ab1696c0'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_isti.tif', + 'hash', + 'sha256', + 'b1759c1008e5914b955c4a9f55572baa8f7c0310677264a1057329e236d826cb'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_isti.tif', + 'hash', + 'sha256', + '677182d186dac992b52b93239af0f419536075c7cd467864fa544bcde4b89e69')], + + # t_process[ndvi8] recording: + 'ndvi8': + [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_ndvi8.tif', + 'hash', + 'sha256', + '93db24c18886fbac0b87d1404a0c80606a599cc55b02d745a09ffb051b8131ac')], + + # t_process[snow] recording: + 'snow': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_snow.tif', + 'hash', + 'sha256', + 'bea7315b736a75bcf37df86089679845ac597f3fba5d5cd6e79f5516b9657faa'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_snow.tif', + 'hash', + 'sha256', + 'de979c1a0c616d7889dd5fd45816849f9c08df54167d7f3b40301d6ac4495778'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_snow.tif', + 'hash', + 'sha256', + '5d9bbfff1522e74165cdd677f1580e9eb2236c2c5922a796a6c9ee0db24ae05a')], + + # t_process[fsnow] recording: + 'fsnow': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_fsnow.tif', + 'hash', + 'sha256', + '853a5ca95f21c4a745e41e4acc46a71a74c74e0cdd1cae9ed3dd31b2fef1ddbd'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_fsnow.tif', + 'hash', + 'sha256', + '3a5f7c95fe60e5789919cf55d7f8e652e4d668c6eb63eaa53a82249c0ea19c5a'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_fsnow.tif', + 'hash', + 'sha256', + 'd09b8a12ad17a7e715ae2e988fbf683e5fe2df46e97e9ac3ebb4444c8b1c169d')], + + # t_process[crcm] recording: + 'crcm': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crcm.tif', + 'hash', + 'sha256', + 'a45c2fb16d871a919e0c803f80e1f3b4ec831472d0466c33741fcf8e8150f6ba'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crcm.tif', + 'hash', + 'sha256', + '9aa80c0e1fa1b62b0a41aed42eeaf84632845d74f29594dfee55c22673b0a6f5'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crcm.tif', + 'hash', + 'sha256', + 'a28c7d280fb8dfe0fe1cc003e1001457abdf14acc5545b2a69b782cfbcce7b50')], + + # t_process[ndsi] recording: + 'ndsi': + [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndsi.tif', + 'hash', + 'sha256', + '1f91f738bbb28b3ebeba36affd8d0689e105e90e1547cca58497eb7c7fdf7346'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndsi.tif', + 'hash', + 'sha256', + 'a626d375fbe5094a5199fdfc2206cf94777ff574379b45811cf7a49055a9f6ec'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndsi.tif', + 'hash', + 'sha256', + 'cb2ca7461c65f66ddb13e804ae7fe8f54daaf620ade600efee24810d40d4eb34')], + + # t_process[brgt] recording: + 'brgt': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_brgt.tif', + 'hash', + 'sha256', + 'bf6bad538c7c6e703bc2f39503c2f128fa8e988a533e302c0124e085735dfaf9'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_brgt.tif', + 'hash', + 'sha256', + '5c72e8f37640ec6b4a5a33e18980d71c70482fc8f708ba25da73e074cf64729f'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_brgt.tif', + 'hash', + 'sha256', + '3460d0d35e35540b55e9eda35f253af8d85eabb84c0c44cb5f9c02d43da7e20a')], + + # t_process[msavi2] recording: + 'msavi2': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_msavi2.tif', + 'hash', + 'sha256', + 'd2c93798df10939ff8afb6cd23e18ac3281cfd844e45da47a29d0342c2b980e9'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_msavi2.tif', + 'hash', + 'sha256', + '92fe4f40d051ef3f479bde76a67bfff978bc513893af5ed62e4a48e457daabad'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_msavi2.tif', + 'hash', + 'sha256', + 'f25bd2d58d3edae72b11e73c08341479dee13af80352a4749861b6a54a81b351')], + + # t_process[bi] recording: + 'bi': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_bi.tif', + 'hash', + 'sha256', + '28c181b8d210b71759589da53295cc894c9d12c74ff1f7b0a49b88f73664282e'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_bi.tif', + 'hash', + 'sha256', + '45acfc9249e063b6b97f269338590d51e2334e9bc5df118613ad908c6f0a3f8e'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_bi.tif', + 'hash', + 'sha256', + '8526cb96d59fc9549dc8535fc8128123b9d39a63a3b7cd97ee3e611b7eeadf6f')], + + # t_process[obstime] recording: + 'obstime': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + '66e01c94910c4fdd4f6fb2ab99b0c2d0cb73618032783e3cbdf1867191981fdc'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + '117cea0497b3158b465dd7dd197c886ebb1acf5059ed83dfdd6b5d7e94bb5243'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'b04d8b07ef83e479db5650a30c84513bde7d5b05dd68187782c057ea0fd3e537')], + + # t_process[lswi] recording: + 'lswi': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_lswi.tif', + 'hash', + 'sha256', + '8de7378cdba830171c8da94424bf7807fa6c8b12523e8b34175a4b17ec770a5d'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_lswi.tif', + 'hash', + 'sha256', + '62de4bdafcc276e9c1c343f4e63f5dd06167e87f7712796f991e3a957f2b71de'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_lswi.tif', + 'hash', + 'sha256', + 'add051ef35d7ffff5eb68c48f8b8071f9cb493b5d3426867605d4c61746e3a26')], + + # t_process[vari] recording: + 'vari': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_vari.tif', + 'hash', + 'sha256', + '732d6d3d9fbe0e1b4616a3f8a91bf12d532598a0a02e8aab16c9a01c3ac56913'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_vari.tif', + 'hash', + 'sha256', + '9875717cc5e1926f11378bbf16345dbade5a2badf9f3dd06844a77f94d80ef72'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_vari.tif', + 'hash', + 'sha256', + '70fd19862ef4145c24f26a8a74cb748279b88fd973de935726ebef2294e17edd')], + + # t_process[evi] recording: + 'evi': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_evi.tif', + 'hash', + 'sha256', + '13548ae74456f08e6add657a637ca364626eafd2cc767724f57aac8ff93005bb'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_evi.tif', + 'hash', + 'sha256', + '7b926f7004cd92f2ec604fc4e2cc1489f60a335e39f4954030332a012fd33f87'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_evi.tif', + 'hash', + 'sha256', + '51db813a857f795817f529edd56d7134004566be46edab8a8dadc81286fb895f')], + + # t_process[clouds] recording: + 'clouds': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD_clouds.tif', + 'hash', + 'sha256', + '26b3e7e695f08ec2e9fb26985f9d5e392f9e8cedc331ab6721ea8e48821d93f0'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD_clouds.tif', + 'hash', + 'sha256', + 'c43eec8178d0521d20e2a5b6e9b61525007d80702c6dd705a3e52ed3a76a7b73'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_clouds.tif', + 'hash', + 'sha256', + '24a6a7050c866eed989530bec45b72004097f1e36fcc0e686652b87723a86de4')], + + # t_process[temp] recording: + 'temp': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_temp.tif', + 'hash', + 'sha256', + 'ead2622531929bba3192b8a6b609d5d920e79394ec577c5245d74d69593d827c'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '5d498bad2779f84289dfb75b14345d64d16b5bc7994b4aca6cd694f70b86064d'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '097f341fa917a0ec316c37d7907ff15b754dcd0925469be586016137ca47575f')], + + # t_process[sti] recording: + 'sti': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_sti.tif', + 'hash', + 'sha256', + '24e50766ec18826f30bbad07bc8f5a34e72f1966b325cc23f064aa5fd35dc24a'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_sti.tif', + 'hash', + 'sha256', + 'c223de9d00e9cfedc0cfc13a45163535f3c59e5587642fa2c725b00b17b7437c'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_sti.tif', + 'hash', + 'sha256', + '73c247fd2c7623e79145b6337846c97d14d1752d660b2f32245a4ccaac033350')], + + # t_process[crc] recording: + 'crc': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crc.tif', + 'hash', + 'sha256', + '07debf171aedfeba7d6c1e3220ff8e4a2a7b1033e68c72bbc121b500b6ed3c9c'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crc.tif', + 'hash', + 'sha256', + 'a7b61aabc950be7a3040b6f9572bf755522b12ad27644583e603def05ac1980b'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crc.tif', + 'hash', + 'sha256', + '942efeeee2b020123be6d7334bbb3ef5995c93d2fd0247cb79948836865c8067')], } # trailing whitespace and other junk characters are in current output diff --git a/gips/test/sys/t_modis.py b/gips/test/sys/t_modis.py index 4c1db425..76074e3b 100644 --- a/gips/test/sys/t_modis.py +++ b/gips/test/sys/t_modis.py @@ -1,9 +1,12 @@ +from __future__ import print_function + import logging from datetime import datetime import pytest import envoy +import util from .util import * logger = logging.getLogger(__name__) @@ -42,13 +45,55 @@ def t_inventory(setup_modis_data, repo_env, expected): actual = repo_env.run('gips_inventory', *STD_ARGS) assert expected == actual +from .expected import modis as expectations -def t_process(setup_modis_data, repo_env, expected): +@pytest.mark.parametrize("product", expectations.t_process.keys()) +def t_process(setup_modis_data, product): """Test gips_process on modis data.""" - # TODO add landcover test; relevant invocation: - # gips_process modis -t h12v04 -p landcover -d 2012-01-01 -v6 --fetch - actual = repo_env.run('gips_process', *STD_ARGS) - assert expected == actual + dut = util.DATA_REPO_ROOT # directory under test + record_path = pytest.config.getoption('--record') + recording_mode = record_path is not None + + if recording_mode: + initial_files = util.find_files(dut) + else: + expectation = expectations.t_process[product] + expected_filenames = [e[0] for e in expectation] + interlopers = [fn for fn in expected_filenames if os.path.exists(fn)] + if interlopers: + raise IOError('Files in the way of the test: {}'.format(interlopers)) + + # run + import sh + args = ('modis', '-s', NH_SHP_PATH, '-d', '2012-12-01,2012-12-03', + '-v', '4', '-p', product) + print("command line: `gips_process {}`".format(' '.join(args))) + outcome = sh.gips_process(*args) + + if recording_mode: + final_files = find_files(dut) + created_files = set(final_files) - set(initial_files) + + # when recording the path, do '/foo/bar/baz/file.tif' + relpath_start = os.path.dirname(dut) + rel_cf = [os.path.relpath(fp, relpath_start) for fp in created_files] + + cf_expectations = [util.generate_expectation(fn, dut) for fn in rel_cf] + print("Recording {} outcome to {}.".format(product, record_path)) + import pprint + with open(record_path, 'a') as rfo: + print('\n# {}[{}] recording:'.format(t_process.__name__, product), + file=rfo) + print("'{}':".format(product), file=rfo) + pretty_hashes = pprint.pformat(cf_expectations) + print(' ', end='', file=rfo) + print('\n '.join(pretty_hashes.split('\n')), end='', file=rfo) + print(',', file=rfo) + else: + # actual test: assemble data to compare to expectation + files = [util.generate_expectation(fn, dut) + for fn in expected_filenames] + assert outcome.exit_code == 0 and expectation == files def t_info(repo_env, expected): diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index 9cc3af7e..5f5f2d6a 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -1,9 +1,12 @@ +from __future__ import print_function + import logging, os, shutil, re import importlib +import hashlib from pprint import pformat import pytest -from scripttest import TestFileEnvironment, ProcResult, FoundFile, FoundDir +import scripttest import envoy from gips.inventory import orm # believed to be safe even though it's the code under test @@ -44,9 +47,16 @@ def extract_timestamps(files): return {k: getattr(v, 'hash', None) for k, v in files.items()} -class GipsTestFileEnv(TestFileEnvironment): +class GipsTestFileEnv(scripttest.TestFileEnvironment): """As superclass but customized for GIPS use case. + Captured values from the process under test: + standard output + standard error + Created files & checksums (for a configured directory-under-test) + Updated files & checksums (for a configured directory-under-test) + Deleted files & checksums(?) (for a configured directory-under-test) + exit status Saves ProcResult objects in self.proc_result.""" proc_result = None @@ -90,10 +100,10 @@ def remove_created(self, strict=False): created = self.proc_result.files_created # first remove all the files - fn = [n for (n, t) in created.items() if isinstance(t, FoundFile)] + fn = [n for (n, t) in created.items() if isinstance(t, scripttest.FoundFile)] [os.remove(os.path.join(DATA_REPO_ROOT, n)) for n in fn] # then remove all the directories (which should now be empty) - dn = [n for (n, t) in created.items() if isinstance(t, FoundDir)] + dn = [n for (n, t) in created.items() if isinstance(t, scripttest.FoundDir)] for n in dn: # dirs are complex because they can exist inside eachother full_n = os.path.join(DATA_REPO_ROOT, n) @@ -206,6 +216,51 @@ def rectify(driver): raise RuntimeError("failed: " + rectify_cmd, outcome.std_out, outcome.std_err, outcome) +def find_files(path): + """Finds all non-directory files under the given path. + + A lot like running find(1) with no arguments. Doesn't follow symlinks. + Doesn't report errors (see os.walk docs for details). + """ + return [os.path.join(subpath, f) + for subpath, _, filenames in os.walk(path) for f in filenames] + +def generate_expectation(filename, base_path): + """Return an expectation of the file's content, based on its file type. + + Expectation formats: + Symlink: (filename, 'symlink', prefix_string, suffix_string) + With symlinks one often can't use the whole link target + because parts differ across environments (eg /home/usr/etc). + So just use the bits before and after the base_path. + Hash: (filename, 'hash, 'sha256', ). + Used as a fallback if nothing else matches the file type. + Finally, if the file is absent: (filename, 'absent') + """ + # have to use lexists to cover for foolish abuse of symlinks by GDAL et al + if not os.path.lexists(filename): + return (filename, 'absent') + if os.path.islink(filename): + # vvvvvvvvvvvvvvvvvvvvvv--- have to rmeove this bit + # HDF4_EOS:EOS_GRID:"/home/tolson/src/gips/data-root/modis/tiles/h12v04/2012337 + # /MCD43A2.A2012337.h12v04.006.2016112013509.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo + # when recording the path, do '/foo/bar/baz/file.tif' + link_target = os.readlink(filename) + separator = os.path.dirname(base_path) + prefix, suffix = link_target.split(separator) + return (filename, 'symlink', prefix, suffix) + return (filename, 'hash', 'sha256', generate_file_hash(filename)) + +def generate_file_hash(filename, blocksize=2**20): + # stolen from SO: https://stackoverflow.com/questions/1131220/get-md5-hash-of-big-files-in-python + m = hashlib.sha256() + with open(filename, 'rb' ) as f: + while True: + buf = f.read(blocksize) + if not buf: + break + m.update(buf) + return m.hexdigest() @pytest.yield_fixture def repo_env(request): @@ -236,7 +291,7 @@ def clean_repo_env(request): _log.debug("Generating file env: {}".format(file_env)) yield file_env after = file_env._find_files() - file_env.proc_result = ProcResult(file_env, ['N/A'], '', '', '', 0, before, after) + file_env.proc_result = scripttest.ProcResult(file_env, ['N/A'], '', '', '', 0, before, after) file_env.remove_created() rectify(request.module.driver) _log.debug("Finalized file env: {}".format(file_env)) From c6bab614e79acebfb6e296b9790369fc429d106c Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 20 Dec 2017 10:58:32 -0500 Subject: [PATCH 04/44] 464 halfway through two things: * refactoring to do merra's t_process * implement conditional cleanup depending on test outcome; skipping cleanup altogether for now --- conftest.py | 4 +++ gips/test/sys/conftest.py | 33 +++++++++++++++++++++++ gips/test/sys/expected/merra.py | 48 +++++++-------------------------- gips/test/sys/t_merra.py | 16 ++++++----- gips/test/sys/t_modis.py | 9 +++---- gips/test/sys/util.py | 5 ++++ 6 files changed, 65 insertions(+), 50 deletions(-) create mode 100644 gips/test/sys/conftest.py diff --git a/conftest.py b/conftest.py index b9670a7c..bee2b91b 100644 --- a/conftest.py +++ b/conftest.py @@ -76,6 +76,10 @@ def pytest_addoption(parser): parser.addoption('--record', action='store', default=None, help="Pass in a filename for expecations" " to be written to that filename.") + # TODO cleanup may not need to be implemented at all + #parser.addoption('--cleanup-on-failure', action='store_true', + # help="Normally cleanup is skipped on failure so you can examine" + # " files; pass this option to cleanup even on failure.") def pytest_configure(config): """Process user config & command-line options.""" diff --git a/gips/test/sys/conftest.py b/gips/test/sys/conftest.py new file mode 100644 index 00000000..e6d8c9a1 --- /dev/null +++ b/gips/test/sys/conftest.py @@ -0,0 +1,33 @@ +#from __future__ import print_function + +### WHERE I LEFT THIS: +# working on optional test teardown (removal of created files) on test failure/error +# two approaches, probably going with the second one: +# PS this conftest.py only applies to system tests due to location; nifty! + +# https://stackoverflow.com/questions/28198585/pytest-how-to-take-action-on-test-failure/47908872#47908872 +""" +def pytest_exception_interact(node, call, report): + print('pytest_exception_interact') + import pdb; pdb.set_trace() + if report.failed: + print("node:", node) + print("call:", call) + print("report:", report) + # report.outcome == 'failed' + #import pdb; pdb.set_trace() +""" + +# https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures +""" +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + # execute all other hooks to obtain the report object + outcome = yield + rep = outcome.get_result() + + # set a report attribute for each phase of a call, which can + # be "setup", "call", "teardown" + + setattr(item, "rep_" + rep.when, rep) +""" diff --git a/gips/test/sys/expected/merra.py b/gips/test/sys/expected/merra.py index 30bca38c..715470de 100644 --- a/gips/test/sys/expected/merra.py +++ b/gips/test/sys/expected/merra.py @@ -40,45 +40,17 @@ """ } - t_process = { - 'updated': { - 'merra/stage': None, - 'merra/tiles/h01v01/2015135': None, - }, - 'created': { - 'merra/tiles/h01v01/2015135/h01v01_2015135_merra_patm.tif': -832862627, - 'merra/tiles/h01v01/2015135/h01v01_2015135_merra_prcp.tif': -1814551991, - 'merra/tiles/h01v01/2015135/h01v01_2015135_merra_rhum.tif': 1203124684, - 'merra/tiles/h01v01/2015135/h01v01_2015135_merra_shum.tif': -97846975, - 'merra/tiles/h01v01/2015135/h01v01_2015135_merra_srad.tif': -1745572576, - 'merra/tiles/h01v01/2015135/h01v01_2015135_merra_tave.tif': 165650774, - 'merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmax.tif': 463728356, - 'merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmin.tif': -1469140067, - 'merra/tiles/h01v01/2015135/h01v01_2015135_merra_wind.tif': 1771906367, - }, - 'ignored': [ - 'gips-inv-db.sqlite3', - ], - '_inv_stdout': """\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site NHseacoast-0 - -\x1b[1mAsset Coverage for site NHseacoast-0\x1b[0m -\x1b[1m -Tile Coverage -\x1b[4m Tile % Coverage % Tile Used\x1b[0m - h01v01 100.0% 0.0% - -\x1b[1m\x1b[4m DATE ASM FLX RAD SLV Product \x1b[0m -\x1b[1m2015 -\x1b[0m 135 100.0% 100.0% 100.0% \x1b[35mpatm\x1b[0m \x1b[35mprcp\x1b[0m \x1b[35mrhum\x1b[0m \x1b[35mshum\x1b[0m \x1b[35msrad\x1b[0m \x1b[35mtave\x1b[0m \x1b[35mtmax\x1b[0m \x1b[35mtmin\x1b[0m \x1b[35mwind\x1b[0m - - -1 files on 1 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mmerra: Modern Era Retrospective-analysis for Research and Applications\x1b[0m -""", + # test this too? 'frland': [], + 'patm': [], + 'prcp': [], + 'rhum': [], + 'shum': [], + 'srad': [], + 'tave': [], + 'tmax': [], + 'tmin': [], + 'wind': [], } t_project = { diff --git a/gips/test/sys/t_merra.py b/gips/test/sys/t_merra.py index fa8c1e05..d6d024a0 100644 --- a/gips/test/sys/t_merra.py +++ b/gips/test/sys/t_merra.py @@ -43,13 +43,17 @@ def t_inventory(setup_fixture, repo_env, expected): actual = repo_env.run('gips_inventory', *STD_ARGS) assert expected == actual +from .expected import merra as expectations -def t_process(setup_fixture, repo_env, expected): - """Test gips_process on {} data.""".format(driver) - process_actual = repo_env.run('gips_process', *STD_ARGS) - inventory_actual = envoy.run('gips_inventory ' + ' '.join(STD_ARGS)) - assert expected == process_actual - assert inventory_actual.std_out == expected._inv_stdout +@pytest.mark.parametrize("product", expectations.t_process.keys()) +def t_process(setup_fixture, record_path, product): + """Test gips_process on merra data.""" + dut = util.DATA_REPO_ROOT # directory under test + + #process_actual = repo_env.run('gips_process', *STD_ARGS) + #inventory_actual = envoy.run('gips_inventory ' + ' '.join(STD_ARGS)) + #assert expected == process_actual + #assert inventory_actual.std_out == expected._inv_stdout def t_info(repo_env, expected): diff --git a/gips/test/sys/t_modis.py b/gips/test/sys/t_modis.py index 76074e3b..55d1c0d5 100644 --- a/gips/test/sys/t_modis.py +++ b/gips/test/sys/t_modis.py @@ -48,13 +48,10 @@ def t_inventory(setup_modis_data, repo_env, expected): from .expected import modis as expectations @pytest.mark.parametrize("product", expectations.t_process.keys()) -def t_process(setup_modis_data, product): +def t_process(setup_modis_data, record_path, product): """Test gips_process on modis data.""" dut = util.DATA_REPO_ROOT # directory under test - record_path = pytest.config.getoption('--record') - recording_mode = record_path is not None - - if recording_mode: + if record_path: initial_files = util.find_files(dut) else: expectation = expectations.t_process[product] @@ -70,7 +67,7 @@ def t_process(setup_modis_data, product): print("command line: `gips_process {}`".format(' '.join(args))) outcome = sh.gips_process(*args) - if recording_mode: + if record_path: final_files = find_files(dut) created_files = set(final_files) - set(initial_files) diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index 5f5f2d6a..f47e8638 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -262,6 +262,11 @@ def generate_file_hash(filename, blocksize=2**20): m.update(buf) return m.hexdigest() +@pytest.yield_fixture +def record_path(request): + path = pytest.config.getoption('--record') + return False if path in (None, '') else path + @pytest.yield_fixture def repo_env(request): """Provide means to test files created by run & clean them up after.""" From b0abb5bb2d563728b2f2727fdbf30ea9172dab1e Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Thu, 21 Dec 2017 16:44:52 -0500 Subject: [PATCH 05/44] 464 first round of refactoring into common code --- gips/test/sys/t_modis.py | 42 ++++++----------------------------- gips/test/sys/util.py | 47 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/gips/test/sys/t_modis.py b/gips/test/sys/t_modis.py index 55d1c0d5..712a5e79 100644 --- a/gips/test/sys/t_modis.py +++ b/gips/test/sys/t_modis.py @@ -4,7 +4,8 @@ from datetime import datetime import pytest -import envoy +import envoy # deprecated +import sh import util from .util import * @@ -48,51 +49,22 @@ def t_inventory(setup_modis_data, repo_env, expected): from .expected import modis as expectations @pytest.mark.parametrize("product", expectations.t_process.keys()) -def t_process(setup_modis_data, record_path, product): +def t_process(setup_modis_data, repo_wrapper, product): """Test gips_process on modis data.""" - dut = util.DATA_REPO_ROOT # directory under test - if record_path: - initial_files = util.find_files(dut) - else: - expectation = expectations.t_process[product] - expected_filenames = [e[0] for e in expectation] - interlopers = [fn for fn in expected_filenames if os.path.exists(fn)] - if interlopers: - raise IOError('Files in the way of the test: {}'.format(interlopers)) - - # run - import sh + (dut, record_path, expectation, expected_filenames + ) = repo_wrapper + args = ('modis', '-s', NH_SHP_PATH, '-d', '2012-12-01,2012-12-03', '-v', '4', '-p', product) print("command line: `gips_process {}`".format(' '.join(args))) outcome = sh.gips_process(*args) - if record_path: - final_files = find_files(dut) - created_files = set(final_files) - set(initial_files) - - # when recording the path, do '/foo/bar/baz/file.tif' - relpath_start = os.path.dirname(dut) - rel_cf = [os.path.relpath(fp, relpath_start) for fp in created_files] - - cf_expectations = [util.generate_expectation(fn, dut) for fn in rel_cf] - print("Recording {} outcome to {}.".format(product, record_path)) - import pprint - with open(record_path, 'a') as rfo: - print('\n# {}[{}] recording:'.format(t_process.__name__, product), - file=rfo) - print("'{}':".format(product), file=rfo) - pretty_hashes = pprint.pformat(cf_expectations) - print(' ', end='', file=rfo) - print('\n '.join(pretty_hashes.split('\n')), end='', file=rfo) - print(',', file=rfo) - else: + if not record_path: # actual test: assemble data to compare to expectation files = [util.generate_expectation(fn, dut) for fn in expected_filenames] assert outcome.exit_code == 0 and expectation == files - def t_info(repo_env, expected): """Test `gips_info modis` and confirm recorded output is given.""" actual = repo_env.run('gips_info', 'modis') diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index f47e8638..125ed940 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -262,11 +262,54 @@ def generate_file_hash(filename, blocksize=2**20): m.update(buf) return m.hexdigest() -@pytest.yield_fixture -def record_path(request): +def record_path(): path = pytest.config.getoption('--record') return False if path in (None, '') else path +@pytest.yield_fixture +def repo_wrapper(request): + # DATA_REPO_ROOT is the directory under test + rp = record_path() + + expectation, expected_filenames = None, None + + # TODO use request.module to deduce expectation + from .expected import modis as expectations + product = request.getfixturevalue('product') + + if rp: + initial_files = find_files(DATA_REPO_ROOT) + else: + expectation = expectations.t_process[product] + expected_filenames = [e[0] for e in expectation] + interlopers = [fn for fn in expected_filenames if os.path.exists(fn)] + if interlopers: + raise IOError('Files in the way of the test: {}'.format(interlopers)) + + yield DATA_REPO_ROOT, rp, expectation, expected_filenames + + if rp: + final_files = find_files(DATA_REPO_ROOT) + created_files = set(final_files) - set(initial_files) + + # when recording the path, do '/foo/bar/baz/file.tif' + relpath_start = os.path.dirname(DATA_REPO_ROOT) + rel_cf = [os.path.relpath(fp, relpath_start) for fp in created_files] + + cf_expectations = [generate_expectation(fn, DATA_REPO_ROOT) for fn in rel_cf] + print("Recording {} outcome to {}.".format(product, rp)) + import pprint + with open(rp, 'a') as rfo: + import pdb; pdb.set_trace() + print('\n# {}[{}] recording:'.format(request.function.__name__, product), + file=rfo) + print("'{}':".format(product), file=rfo) + pretty_hashes = pprint.pformat(cf_expectations) + print(' ', end='', file=rfo) + print('\n '.join(pretty_hashes.split('\n')), end='', file=rfo) + print(',', file=rfo) + + @pytest.yield_fixture def repo_env(request): """Provide means to test files created by run & clean them up after.""" From 6fe6dce7df8bf32d07a0e35b76e8cb117a340afb Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Thu, 21 Dec 2017 17:43:04 -0500 Subject: [PATCH 06/44] 464 Refactor Part 2: Refactor Harder: Now ready to do merra's t_process --- gips/test/sys/t_modis.py | 19 ++++-------- gips/test/sys/util.py | 62 +++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 46 deletions(-) diff --git a/gips/test/sys/t_modis.py b/gips/test/sys/t_modis.py index 712a5e79..76c36ac6 100644 --- a/gips/test/sys/t_modis.py +++ b/gips/test/sys/t_modis.py @@ -51,19 +51,12 @@ def t_inventory(setup_modis_data, repo_env, expected): @pytest.mark.parametrize("product", expectations.t_process.keys()) def t_process(setup_modis_data, repo_wrapper, product): """Test gips_process on modis data.""" - (dut, record_path, expectation, expected_filenames - ) = repo_wrapper - - args = ('modis', '-s', NH_SHP_PATH, '-d', '2012-12-01,2012-12-03', - '-v', '4', '-p', product) - print("command line: `gips_process {}`".format(' '.join(args))) - outcome = sh.gips_process(*args) - - if not record_path: - # actual test: assemble data to compare to expectation - files = [util.generate_expectation(fn, dut) - for fn in expected_filenames] - assert outcome.exit_code == 0 and expectation == files + record_mode, expected, runner = repo_wrapper + outcome, actual = runner('gips_process', 'modis', '-s', NH_SHP_PATH, + '-d', '2012-12-01,2012-12-03', '-v', '4', '-p', + product) + if not record_mode: # don't evaluate assertions when in record-mode + assert outcome.exit_code == 0 and expected == actual def t_info(repo_env, expected): """Test `gips_info modis` and confirm recorded output is given.""" diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index 125ed940..2df309c2 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -3,11 +3,12 @@ import logging, os, shutil, re import importlib import hashlib -from pprint import pformat +import pprint import pytest import scripttest -import envoy +import envoy # deprecated +import sh from gips.inventory import orm # believed to be safe even though it's the code under test @@ -66,7 +67,7 @@ def log_findings(description, files): Logs in a format suitable for updating known good values when tests need to be updated to match code changes.""" - _log.debug("{}: {}".format(description, pformat(files))) + _log.debug("{}: {}".format(description, pprint.pformat(files))) def run(self, *args, **kwargs): """As super().run but store result & prevent premature exits.""" @@ -270,23 +271,25 @@ def record_path(): def repo_wrapper(request): # DATA_REPO_ROOT is the directory under test rp = record_path() - - expectation, expected_filenames = None, None - - # TODO use request.module to deduce expectation - from .expected import modis as expectations product = request.getfixturevalue('product') + expectations = load_expectation_module(request.module.__name__) + expected = expectations.t_process[product] + expected_filenames = [e[0] for e in expected] if rp: initial_files = find_files(DATA_REPO_ROOT) else: - expectation = expectations.t_process[product] - expected_filenames = [e[0] for e in expectation] interlopers = [fn for fn in expected_filenames if os.path.exists(fn)] if interlopers: raise IOError('Files in the way of the test: {}'.format(interlopers)) - yield DATA_REPO_ROOT, rp, expectation, expected_filenames + def wrapped_runner(cmd_string, *args): + print("command line: `{} {}`".format(cmd_string, ' '.join(args))) + outcome = sh.Command(cmd_string)(*args) + return outcome, [generate_expectation(fn, DATA_REPO_ROOT) + for fn in expected_filenames] + + yield bool(rp), expected, wrapped_runner if rp: final_files = find_files(DATA_REPO_ROOT) @@ -298,9 +301,7 @@ def repo_wrapper(request): cf_expectations = [generate_expectation(fn, DATA_REPO_ROOT) for fn in rel_cf] print("Recording {} outcome to {}.".format(product, rp)) - import pprint with open(rp, 'a') as rfo: - import pdb; pdb.set_trace() print('\n# {}[{}] recording:'.format(request.function.__name__, product), file=rfo) print("'{}':".format(product), file=rfo) @@ -351,32 +352,27 @@ def output_tfe(): gtfe = GipsTestFileEnv(OUTPUT_DIR) return gtfe - -@pytest.fixture -def expected(request): +def load_expectation_module(name): """Use introspection to find known-good values for test assertions. - For example, assume this test is in t_modis.py: - - def t_process(setup_modis_data, repo_env, expected): - '''Test gips_process on modis data.''' - actual = repo_env.run('gips_process', *STD_ARGS) - assert expected == actual - - Then expected() will end up looking in expected/modis.py for `t_process`: - - t_process = { - 'updated': {...}, - 'created': {...}, - } + Loads the model given by a part of `name`. For instance, pass in + 'foo.bar.baz.t_qux' to locate and load the module `..expected.qux`. """ - # construct expectation module name from test module name: - # e.g.: 'foo.bar.t_baz_qux' -> 'baz_qux' - mod_name = request.module.__name__.split('.')[-1].split('_', 1)[1] + _, mod_name = name.split('.')[-1].split('_', 1) relative_mod_name = '..expected.' + mod_name try: - module = importlib.import_module(relative_mod_name, __name__) + return importlib.import_module(relative_mod_name, __name__) except ImportError as ie: msg = "Eror importing expectations from {}.".format(relative_mod_name) raise ImportError(msg, ie.args) + +@pytest.fixture +def expected(request): + """Load the expectations for the running test. + + For example, assume a test named 't_process' in t_modis.py. + expected() will then load a result object with values from + expected.modis.t_process. + """ + module = load_expectation_module(request.module.__name__) return GipsProcResult(**getattr(module, request.function.func_name)) From df71c44ae24bdcabdb7df80a46866792ee07bc78 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Thu, 21 Dec 2017 17:53:41 -0500 Subject: [PATCH 07/44] 464 convert merra t_process to new framework --- gips/test/sys/expected/merra.py | 63 ++++++++++++++++++++++++++++----- gips/test/sys/t_merra.py | 16 ++++----- 2 files changed, 61 insertions(+), 18 deletions(-) diff --git a/gips/test/sys/expected/merra.py b/gips/test/sys/expected/merra.py index 715470de..eb0f2abe 100644 --- a/gips/test/sys/expected/merra.py +++ b/gips/test/sys/expected/merra.py @@ -42,15 +42,60 @@ t_process = { # test this too? 'frland': [], - 'patm': [], - 'prcp': [], - 'rhum': [], - 'shum': [], - 'srad': [], - 'tave': [], - 'tmax': [], - 'tmin': [], - 'wind': [], + # t_process[srad] recording: + 'srad': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_srad.tif', + 'hash', + 'sha256', + '882a44af70e337bf905c8695936f101792a518507a4304b23781a2f30aaabab4')], + # t_process[tave] recording: + 'tave': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tave.tif', + 'hash', + 'sha256', + '5185aebd7cda54157cad2ddbde9f6fec4871c1b85fe78d0738634ba0211f2c9b')], + # t_process[shum] recording: + 'shum': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_shum.tif', + 'hash', + 'sha256', + '558f567c1e931891553d396f81b2b90929ad48a0ae88fa95d3894eae23ab3eba')], + # t_process[rhum] recording: + 'rhum': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_rhum.tif', + 'hash', + 'sha256', + '058161eb8488f1fb3df7f5c8719833c9d68c56aa082d4605938f62c2ee0035f6')], + # t_process[tmin] recording: + 'tmin': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmin.tif', + 'hash', + 'sha256', + '35a850769a3bb8f5209574ea4835bf417d8f99024567f4af9cdee9e623d0c567')], + # t_process[tmax] recording: + 'tmax': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmax.tif', + 'hash', + 'sha256', + '5963fcf346f388fc347355153996a078f1eb5ecbbc094394b72be382ba491865')], + # t_process[prcp] recording: + 'prcp': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_prcp.tif', + 'hash', + 'sha256', + '348fd7072ef0a8625f5268e29acbf3a69a959a412327102ef6558c992e62bc9f')], + # t_process[patm] recording: + 'patm': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_patm.tif', + 'hash', + 'sha256', + '7d9deca613973cb8ffb8f15a3dfa2013e6783556b520915e89b97fe41de638e3')], + # t_process[wind] recording: + 'wind': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_wind.tif', + 'hash', + 'sha256', + 'a90c1d87fdb1c0926bd5b3004408a1f9d42ef08c38deb9d0572bc7536dbbf08a')], } t_project = { diff --git a/gips/test/sys/t_merra.py b/gips/test/sys/t_merra.py index d6d024a0..10e71c46 100644 --- a/gips/test/sys/t_merra.py +++ b/gips/test/sys/t_merra.py @@ -1,7 +1,8 @@ import logging import pytest -import envoy +import envoy # deprecated +import sh from .util import * @@ -46,15 +47,12 @@ def t_inventory(setup_fixture, repo_env, expected): from .expected import merra as expectations @pytest.mark.parametrize("product", expectations.t_process.keys()) -def t_process(setup_fixture, record_path, product): +def t_process(setup_fixture, repo_wrapper, product): """Test gips_process on merra data.""" - dut = util.DATA_REPO_ROOT # directory under test - - #process_actual = repo_env.run('gips_process', *STD_ARGS) - #inventory_actual = envoy.run('gips_inventory ' + ' '.join(STD_ARGS)) - #assert expected == process_actual - #assert inventory_actual.std_out == expected._inv_stdout - + record_mode, expected, runner = repo_wrapper + outcome, actual = runner('gips_process', *(STD_ARGS + ('-p', product))) + if not record_mode: # don't evaluate assertions when in record-mode + assert outcome.exit_code == 0 and expected == actual def t_info(repo_env, expected): """Test `gips_info {driver}` and confirm recorded output is given.""" From 8c4db0043c1e75a1b25737818379079e2247b1c2 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Fri, 22 Dec 2017 14:10:15 -0500 Subject: [PATCH 08/44] 467 full-text-diff expectation generator --- gips/test/sys/util.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index 2df309c2..65265583 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -234,13 +234,19 @@ def generate_expectation(filename, base_path): With symlinks one often can't use the whole link target because parts differ across environments (eg /home/usr/etc). So just use the bits before and after the base_path. - Hash: (filename, 'hash, 'sha256', ). + Hash: (filename, 'hash', 'sha256', ). Used as a fallback if nothing else matches the file type. + Text-full: (filename, 'text-full', list()) Finally, if the file is absent: (filename, 'absent') """ + # may want to try python-magic at some point: https://github.com/ahupp/python-magic + # but for now going by file extension seems to be sufficient. + # have to use lexists to cover for foolish abuse of symlinks by GDAL et al if not os.path.lexists(filename): return (filename, 'absent') + + # symlinks if os.path.islink(filename): # vvvvvvvvvvvvvvvvvvvvvv--- have to rmeove this bit # HDF4_EOS:EOS_GRID:"/home/tolson/src/gips/data-root/modis/tiles/h12v04/2012337 @@ -250,6 +256,24 @@ def generate_expectation(filename, base_path): separator = os.path.dirname(base_path) prefix, suffix = link_target.split(separator) return (filename, 'symlink', prefix, suffix) + + # text files + size_threshold = 80 * 25 # about one terminal-screen + if filename.endswith('.txt'): + f_size = os.stat(filename).st_size + if f_size <= size_threshold: # give up if there's too much text + with open(filename) as fo: + lines = fo.readlines() + return (filename, 'text-full', lines) + print('{} ({} bytes) exceeds max supported size for text diffs' + ' ({} bytes); defaulting to checksum'.format( + filename, f_size, size_threshold)) + + # product TIFFs + if filename.endswith('.tif'): + pass # TODO https://github.com/Applied-GeoSolutions/gips/issues/463 + + # use a hash as a fallback return (filename, 'hash', 'sha256', generate_file_hash(filename)) def generate_file_hash(filename, blocksize=2**20): From fdc106c1b0c653ac52b92dbfe447f7925e28fab0 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Fri, 22 Dec 2017 15:11:18 -0500 Subject: [PATCH 09/44] 467 generalize wrapper to observe any directory: Also port modis' t_project, and throw out the two-runs test since few/no other drivers seem to have it. --- gips/test/sys/expected/modis.py | 349 +++++++++++++++++++++++++++++--- gips/test/sys/t_modis.py | 35 +--- gips/test/sys/util.py | 32 ++- 3 files changed, 356 insertions(+), 60 deletions(-) diff --git a/gips/test/sys/expected/modis.py b/gips/test/sys/expected/modis.py index 40b14cdc..fd477886 100644 --- a/gips/test/sys/expected/modis.py +++ b/gips/test/sys/expected/modis.py @@ -378,33 +378,328 @@ landcover MCD Annual Land Cover """} -t_project = { 'created': { - '0': None, - '0/2012336_MCD_fsnow.tif': -1883071404, - '0/2012336_MCD_indices.tif': 16775451, - '0/2012336_MCD_quality.tif': -565438400, - '0/2012336_MCD_snow.tif': -1824464052, - '0/2012336_MOD-MYD_obstime.tif': 75984532, - '0/2012336_MOD-MYD_temp.tif': -1641854393, - '0/2012336_MOD_clouds.tif': -1957614367, - '0/2012337_MCD_fsnow.tif': -856980949, - '0/2012337_MCD_indices.tif': -943628281, - '0/2012337_MCD_quality.tif': -567163117, - '0/2012337_MCD_snow.tif': -1690607189, - '0/2012337_MOD-MYD_obstime.tif': 1283853420, - '0/2012337_MOD-MYD_temp.tif': 1191019971, - '0/2012337_MOD_clouds.tif': -415873821, - '0/2012337_MOD_ndvi8.tif': 486163471, - '0/2012337_MOD_temp8td.tif': 1925913017, - '0/2012337_MOD_temp8tn.tif': -1935381834, - '0/2012338_MCD_fsnow.tif': -1017381876, - '0/2012338_MCD_indices.tif': -2141895762, - '0/2012338_MCD_quality.tif': -1145887707, - '0/2012338_MCD_snow.tif': -319441628, - '0/2012338_MOD-MYD_obstime.tif': 2120166986, - '0/2012338_MOD-MYD_temp.tif': -1448563677, - '0/2012338_MOD_clouds.tif': 1789735888, -}} +t_project = { + # t_project[satvi] recording: + 'satvi': + [('output/0/2012336_MCD_satvi.tif', + 'hash', + 'sha256', + '893d968ff6359d0d34f8a587246690d1c724832505d66fbee8be8c1a43a6e3df'), + ('output/0/2012337_MCD_satvi.tif', + 'hash', + 'sha256', + '6092d7cfb609f000124b8ef107f278decaa27c3cdb69696bd094940ead97d96e'), + ('output/0/2012338_MCD_satvi.tif', + 'hash', + 'sha256', + '37dd32e0f5e761846f44615ec6e82e2cce8d6c98197f0bd721dcfb266d963b6b')], + + # t_project[ndti] recording: + 'ndti': + [('output/0/2012337_MCD_ndti.tif', + 'hash', + 'sha256', + 'db85948227815660b94d570adf0f1119d9f1001284c3de6c8b219c6455bd3c4e'), + ('output/0/2012338_MCD_ndti.tif', + 'hash', + 'sha256', + '5028c194e41d8c07076cee09631b276ef7b86c3c45b0a4f705f5e4c6b72c2812'), + ('output/0/2012336_MCD_ndti.tif', + 'hash', + 'sha256', + '157134ea0e9cbb0b2fcdb4203e17e82ae906a5779eb1f735e8c55d1f60c231e1')], + + # t_project[ndvi] recording: + 'ndvi': + [('output/0/2012337_MCD_ndvi.tif', + 'hash', + 'sha256', + '28a288720a0846348fbd6315e5d9e3801ad76eb2c873dc028e7a52d1fdcdad10'), + ('output/0/2012336_MCD_ndvi.tif', + 'hash', + 'sha256', + '78e090b1cc92304c666a8e480d11fa99985ee3d2113c4fe3be29fb85910b86b9'), + ('output/0/2012338_MCD_ndvi.tif', + 'hash', + 'sha256', + 'bbcee4ac1b49466c90264e305c20ce19a021fd177ffc35da5ed92a86555c480e')], + + # t_project[quality] recording: + 'quality': + [('output/0/2012338_MCD_quality.tif', + 'hash', + 'sha256', + '895abfc825d9da56c56202c670f94a16b28f2aaab5590db55ac6e903b6638e91'), + ('output/0/2012336_MCD_quality.tif', + 'hash', + 'sha256', + '4f40fdeb7ded507a68f404ec34a17fb6283c9b90d81ecccc7782746eaba85323'), + ('output/0/2012337_MCD_quality.tif', + 'hash', + 'sha256', + '5cdd315b2a423fbb82c19cfe859325c889a5d05cf0c3cdf71359b257524175a3')], + + # t_project[isti] recording: + 'isti': + [('output/0/2012337_MCD_isti.tif', + 'hash', + 'sha256', + '391367ee5bdd0a96435c09df85db312919121c5d9a28b5bda59a68db7b609871'), + ('output/0/2012338_MCD_isti.tif', + 'hash', + 'sha256', + 'cc082d45a2a4dd4a7e53d0970029decf24ce5033b9715930cd250b524ad87757'), + ('output/0/2012336_MCD_isti.tif', + 'hash', + 'sha256', + '16be9705c06a44f3a35e684cded2d2ceb26213510eaf0ccfbafb62ebc39bb153')], + + # t_project[ndvi8] recording: + 'ndvi8': + [('output/0/2012337_MOD_ndvi8.tif', + 'hash', + 'sha256', + '23a72df39da253ac07de5a9e98c88f40e3335f3b1b779ccf91153d1d5573a780')], + + # t_project[snow] recording: + 'snow': + [('output/0/2012336_MCD_snow.tif', + 'hash', + 'sha256', + 'b3eeca8fc933d2f43c2d96e869956447d9556737181635167744d0d420dfd813'), + ('output/0/2012338_MCD_snow.tif', + 'hash', + 'sha256', + '98a38e1a75fcea269bd75a952cde9d84af86b14ddb928e4777ff812dba8befbe'), + ('output/0/2012337_MCD_snow.tif', + 'hash', + 'sha256', + 'da26696be559bb79c399ad18942ac629a9604d857dd7fc461fd76c4577a06303')], + + # t_project[fsnow] recording: + 'fsnow': + [('output/0/2012338_MCD_fsnow.tif', + 'hash', + 'sha256', + 'f2efdf504e004caf05563f3402fdde7e771351d7d93de5970af0a54d0b5bd2c2'), + ('output/0/2012336_MCD_fsnow.tif', + 'hash', + 'sha256', + 'a9d2ea29a6d77325246e0baaf298d16cb84d6003a19faed8ba162b475703f563'), + ('output/0/2012337_MCD_fsnow.tif', + 'hash', + 'sha256', + 'e8465aa3764c60b21a943f3e078d93c0484272292f075455693ea2084971ace4')], + + # t_project[crcm] recording: + 'crcm': + [('output/0/2012337_MCD_crcm.tif', + 'hash', + 'sha256', + 'c00e7b7c6970ae23904ee52afd614a8871f3d771d26830a2aef412163d9d9228'), + ('output/0/2012338_MCD_crcm.tif', + 'hash', + 'sha256', + 'ed336a3ba4cf482936ca781ea1a5e0f04ca5ad68919c3dbe6e117edac55ec242'), + ('output/0/2012336_MCD_crcm.tif', + 'hash', + 'sha256', + 'f217a9278b195e945e2337663cf7dfe2b09faa0dc68062fbc3614a6954866eaf')], + + # t_project[ndsi] recording: + 'ndsi': + [('output/0/2012338_MCD_ndsi.tif', + 'hash', + 'sha256', + '8e26a8f6d78b9612c8cd3924936c5944f56c67ae1c63f6541d045b41a27baafe'), + ('output/0/2012337_MCD_ndsi.tif', + 'hash', + 'sha256', + '2aaea763ab1d5b54d7079e7ee2381bd987a3c72c5cbbd6f18f97aa320c186e2a'), + ('output/0/2012336_MCD_ndsi.tif', + 'hash', + 'sha256', + '83b6bbf241d080cf6860798da299038be5e6d3e20a8b544e5af51e2eefd3a5c9')], + + # t_project[brgt] recording: + 'brgt': + [('output/0/2012337_MCD_brgt.tif', + 'hash', + 'sha256', + 'fc385e6e42538a09655762d1b2557abe35af38aecef1571705cec8b971efbe4d'), + ('output/0/2012338_MCD_brgt.tif', + 'hash', + 'sha256', + 'a8f8ef66bc87c2370bd61d5eed64905ba4f832d05579687478ff5f38c65e8038'), + ('output/0/2012336_MCD_brgt.tif', + 'hash', + 'sha256', + '24f8130de39daac6e181cb2f98b48d9a14412700632d3cbd2ffbe2704283b740')], + + # t_project[msavi2] recording: + 'msavi2': + [('output/0/2012336_MCD_msavi2.tif', + 'hash', + 'sha256', + '997b562420fe223677881d577cdb2100ee653ee932b9989b95f3089c0fe38352'), + ('output/0/2012338_MCD_msavi2.tif', + 'hash', + 'sha256', + '26a7ab45723856361d108064fe0c810535de75f272a733d86f66eeb8146dee35'), + ('output/0/2012337_MCD_msavi2.tif', + 'hash', + 'sha256', + 'ee7b2006b79f65bdf11d77f678d66765abfcc024b7856ab296f855570efb9940')], + + # t_project[bi] recording: + 'bi': + [('output/0/2012336_MCD_bi.tif', + 'hash', + 'sha256', + 'e5f891a6290301508107714657274fc808fcf96cb138c652a9c22552675796a5'), + ('output/0/2012337_MCD_bi.tif', + 'hash', + 'sha256', + '58b9e3809630ba24879bb85e81bc6dc0eed548348373588d70d49de65d246582'), + ('output/0/2012338_MCD_bi.tif', + 'hash', + 'sha256', + '914ca6d15c25a22d3e0a43d0d266f72bd65d102b4d6b015a9f12b8ee3a116c13')], + + # t_project[obstime] recording: + 'obstime': + [('output/0/2012337_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'a9345d21f3d754d083d3b4d13b059f2ffe07eb42ea9a0d6f3e40ef326e4ea75f'), + ('output/0/2012336_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'fac305543c4afbf76611900579cd5f37d63e7a45814c997dce40fc46a74f13e5'), + ('output/0/2012338_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'e18456caaf9a2dd005cc6568f7aa77ac534bfa24ec364d3af28affb4ae652de9')], + + # t_project[lswi] recording: + 'lswi': + [('output/0/2012337_MCD_lswi.tif', + 'hash', + 'sha256', + '586dd3782070b983b9d383161bed82bee21c54ad312d1c993b0329965144deab'), + ('output/0/2012336_MCD_lswi.tif', + 'hash', + 'sha256', + '296c849e70d31127bcefef54ffed72832529f7ae60d50b7c639c5c6d253d30f4'), + ('output/0/2012338_MCD_lswi.tif', + 'hash', + 'sha256', + '992fa3d7b4bdc8cf95fba759a0096f9a913e84762717e93c477baae0d7d3c0ed')], + + # t_project[vari] recording: + 'vari': + [('output/0/2012336_MCD_vari.tif', + 'hash', + 'sha256', + 'be5d2273b9f0b9aca79df89b66ca0e7e1a2b8d386e6d34a6eca6eb37810b488d'), + ('output/0/2012338_MCD_vari.tif', + 'hash', + 'sha256', + '016ec5b1ee512d9306251a4c2b3630c90ea18723159a6a254c8e77c4e730abc8'), + ('output/0/2012337_MCD_vari.tif', + 'hash', + 'sha256', + 'a6c10e80436b92908637b22b40e7993a0a648e82b96993760a7b26ba43df0667')], + + # t_project[evi] recording: + 'evi': + [('output/0/2012338_MCD_evi.tif', + 'hash', + 'sha256', + '8a1beba0730ac7d113a7edb8b55035a742892bbe678cd0eaaa540e59906a6990'), + ('output/0/2012337_MCD_evi.tif', + 'hash', + 'sha256', + '824a5247b0e8e0730dec6e82b390533649aa796ef04ef4be27ac38cae9a24d04'), + ('output/0/2012336_MCD_evi.tif', + 'hash', + 'sha256', + '361798060d02c97153a9828c1f8ac6b7df396070ec243932e9ed18fd472e9669')], + + # t_project[temp8tn] recording: + 'temp8tn': + [('output/0/2012337_MOD_temp8tn.tif', + 'hash', + 'sha256', + '1600d412e2590b4f4006ac0d98ebf9c6bf83fe16967c2fd721e412d2a14040c6')], + + # t_project[clouds] recording: + 'clouds': + [('output/0/2012337_MOD_clouds.tif', + 'hash', + 'sha256', + '8b13afeec6d1a5612a3eeb51896507e2d4eeb851261881f3854d2b804e97835f'), + ('output/0/2012336_MOD_clouds.tif', + 'hash', + 'sha256', + 'cb4fab2519652bebe4830479c95425050f63d519397beb5b0c40349cc6f49372'), + ('output/0/2012338_MOD_clouds.tif', + 'hash', + 'sha256', + '54505d6d4063caa27fda74f5fdbb916c20d281f6e15deea28596c379ffa661e2')], + + # t_project[temp] recording: + 'temp': + [('output/0/2012336_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '614023db948d0b0249057ca9c458356c2222e45224a726f87a7cf512cd87745c'), + ('output/0/2012338_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '51864699b8347a30d9669ec41298665cd893a27516089cc21b4c3f03b3e65b2b'), + ('output/0/2012337_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '29373079984e8aaf55add6a1fdd7bd766eb89a1c74ebc85674da54aeb643e3ef')], + + # t_project[temp8td] recording: + 'temp8td': + [('output/0/2012337_MOD_temp8td.tif', + 'hash', + 'sha256', + 'eaa0ff94bb34cbad8d9c187b16c7cd2bd5948d1d94cf1caf30b0396ac7e8a735')], + + # t_project[sti] recording: + 'sti': + [('output/0/2012337_MCD_sti.tif', + 'hash', + 'sha256', + '9850777cae46b42fa3c8361849529e6f230dd1ce86fb6c6676aa8eb2768e71d5'), + ('output/0/2012336_MCD_sti.tif', + 'hash', + 'sha256', + 'b7fde6a04ac091946553b25b1a26de806fa7f9c23e915b415034ef811aca1d16'), + ('output/0/2012338_MCD_sti.tif', + 'hash', + 'sha256', + 'b2714cadf4e779c433415f194f366ac87877c60b383f46971843b4ab3a610fa9')], + + # t_project[crc] recording: + 'crc': + [('output/0/2012338_MCD_crc.tif', + 'hash', + 'sha256', + 'dc9a4885e694fe7117cf10df11b5589745b4eb119d5f58f9d1c14e27d0eda333'), + ('output/0/2012337_MCD_crc.tif', + 'hash', + 'sha256', + '5688d80a613c28c657237186291a88ae786037a77dad0247c0bd734da214c927'), + ('output/0/2012336_MCD_crc.tif', + 'hash', + 'sha256', + '3df0003b6153970e73c46d5e999697ce5af268744274ccd3a4a2c14e3ae932cb')], +} t_project_two_runs = t_project diff --git a/gips/test/sys/t_modis.py b/gips/test/sys/t_modis.py index 76c36ac6..6b254c2e 100644 --- a/gips/test/sys/t_modis.py +++ b/gips/test/sys/t_modis.py @@ -63,37 +63,24 @@ def t_info(repo_env, expected): actual = repo_env.run('gips_info', 'modis') assert expected == actual - -def t_project(setup_modis_data, clean_repo_env, output_tfe, expected): +@pytest.mark.parametrize("product", expectations.t_project.keys()) +def t_project(setup_modis_data, export_wrapper, product): """Test gips_project modis with warping.""" - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld') - actual = output_tfe.run('gips_project', *args) - assert expected == actual - - -def t_project_two_runs(setup_modis_data, clean_repo_env, output_tfe, expected): - """As test_project, but run twice to confirm idempotence of gips_project. - - The data repo is only cleaned up after both runs are complete; this is - intentional as changes in the data repo may influence gips_project. - """ - args = STD_ARGS + ('--res', '100', '100', - '--outdir', OUTPUT_DIR, '--notld') - - actual = output_tfe.run('gips_project', *args) - assert 'initial test_project run' and expected == actual - - actual = output_tfe.run('gips_project', *args) - expected.created = {} # no created files on second run - assert 'final test_project run' and expected == actual - + record_mode, expected, runner = export_wrapper + args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, + '--notld', '-p', product) + outcome, actual = runner('gips_project', *args) + if not record_mode: # don't evaluate assertions when in record-mode + assert outcome.exit_code == 0 and expected == actual +# TODO keep this test? +''' def t_project_no_warp(setup_modis_data, clean_repo_env, output_tfe, expected): """Test gips_project modis without warping.""" args = STD_ARGS + ('--outdir', OUTPUT_DIR, '--notld') actual = output_tfe.run('gips_project', *args) assert expected == actual - +''' def t_tiles(setup_modis_data, clean_repo_env, output_tfe, expected): """Test gips_tiles modis with warping.""" diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index 65265583..769e9dd2 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -291,17 +291,22 @@ def record_path(): path = pytest.config.getoption('--record') return False if path in (None, '') else path -@pytest.yield_fixture -def repo_wrapper(request): +def sys_test_wrapper(request, path): + """Provides for system testing by watching the specified directory. + + `request` is a pytest request object probably passed in to a fixture + that is calling this function. Yields to let some process work, then + reports on files created in the path during the wait. + """ # DATA_REPO_ROOT is the directory under test rp = record_path() product = request.getfixturevalue('product') expectations = load_expectation_module(request.module.__name__) - expected = expectations.t_process[product] + expected = getattr(expectations, request.function.__name__)[product] expected_filenames = [e[0] for e in expected] if rp: - initial_files = find_files(DATA_REPO_ROOT) + initial_files = find_files(path) else: interlopers = [fn for fn in expected_filenames if os.path.exists(fn)] if interlopers: @@ -310,20 +315,20 @@ def repo_wrapper(request): def wrapped_runner(cmd_string, *args): print("command line: `{} {}`".format(cmd_string, ' '.join(args))) outcome = sh.Command(cmd_string)(*args) - return outcome, [generate_expectation(fn, DATA_REPO_ROOT) - for fn in expected_filenames] + return outcome, [generate_expectation(fn, path) + for fn in expected_filenames] yield bool(rp), expected, wrapped_runner if rp: - final_files = find_files(DATA_REPO_ROOT) + final_files = find_files(path) created_files = set(final_files) - set(initial_files) # when recording the path, do '/foo/bar/baz/file.tif' - relpath_start = os.path.dirname(DATA_REPO_ROOT) + relpath_start = os.path.dirname(path) rel_cf = [os.path.relpath(fp, relpath_start) for fp in created_files] - cf_expectations = [generate_expectation(fn, DATA_REPO_ROOT) for fn in rel_cf] + cf_expectations = [generate_expectation(fn, path) for fn in rel_cf] print("Recording {} outcome to {}.".format(product, rp)) with open(rp, 'a') as rfo: print('\n# {}[{}] recording:'.format(request.function.__name__, product), @@ -334,6 +339,15 @@ def wrapped_runner(cmd_string, *args): print('\n '.join(pretty_hashes.split('\n')), end='', file=rfo) print(',', file=rfo) +@pytest.yield_fixture +def repo_wrapper(request): + for rv in sys_test_wrapper(request, DATA_REPO_ROOT): + yield rv + +@pytest.yield_fixture +def export_wrapper(request): + for rv in sys_test_wrapper(request, OUTPUT_DIR): + yield rv @pytest.yield_fixture def repo_env(request): From 6f643c579f051ceb9825daee8262ca0e87bb1ba3 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Fri, 22 Dec 2017 15:25:25 -0500 Subject: [PATCH 10/44] 467 port merra's t_project to no-scripttest-land --- gips/test/sys/expected/merra.py | 75 +++++++++++++++++++++++++++------ gips/test/sys/t_merra.py | 19 ++++++--- 2 files changed, 75 insertions(+), 19 deletions(-) diff --git a/gips/test/sys/expected/merra.py b/gips/test/sys/expected/merra.py index eb0f2abe..c9e54272 100644 --- a/gips/test/sys/expected/merra.py +++ b/gips/test/sys/expected/merra.py @@ -99,18 +99,69 @@ } t_project = { - 'created': { - '0': None, - '0/2015135_merra_patm.tif': 147353283, - '0/2015135_merra_prcp.tif': 1755012780, - '0/2015135_merra_rhum.tif': -157842459, - '0/2015135_merra_shum.tif': -1248597053, - '0/2015135_merra_srad.tif': 335947606, - '0/2015135_merra_tave.tif': 362688260, - '0/2015135_merra_tmax.tif': -711449232, - '0/2015135_merra_tmin.tif': 1704871784, - '0/2015135_merra_wind.tif': 1730702417, - } + + # t_project[srad] recording: + 'srad': + [('output/0/2015135_merra_srad.tif', + 'hash', + 'sha256', + 'bfc0b906f7299ff8e6fcbbe474de0fb7b877d36f2e619e2e6ac10bd84ac5b55d')], + + # t_project[tave] recording: + 'tave': + [('output/0/2015135_merra_tave.tif', + 'hash', + 'sha256', + 'c96d89ebf5d7ec90193c8a113609bfc39473d72041b6145a8cd41ee49e4f6a4d')], + + # t_project[prcp] recording: + 'prcp': + [('output/0/2015135_merra_prcp.tif', + 'hash', + 'sha256', + 'b7e68237ee3557c421856203cc22807f1ce74790ea2948f8d095cbf8a7f18a53')], + + # t_project[rhum] recording: + 'rhum': + [('output/0/2015135_merra_rhum.tif', + 'hash', + 'sha256', + '3b45ba1cd87d4781ff95b08562a00189506c76add24156f8e3719ff8872c3d3a')], + + # t_project[tmin] recording: + 'tmin': + [('output/0/2015135_merra_tmin.tif', + 'hash', + 'sha256', + '253e37e9b2c03a77a0a4fdfe5aaa0e3ff025471f64fd28d6ffb33dd8ecdd68a9')], + + # t_project[tmax] recording: + 'tmax': + [('output/0/2015135_merra_tmax.tif', + 'hash', + 'sha256', + '8371d473f0e506e6ab0fe872542b8cd0d65bc2ed2b37bbeb747c966c41e9c232')], + + # t_project[shum] recording: + 'shum': + [('output/0/2015135_merra_shum.tif', + 'hash', + 'sha256', + '208465fbf10da2e935807e00991fa6375c3c6b67d9950362a9ffe502d7690fd6')], + + # t_project[patm] recording: + 'patm': + [('output/0/2015135_merra_patm.tif', + 'hash', + 'sha256', + '5778a4d89b22ea76dc25654c8092a61bbb09634f06f66b578460a93444e513a7')], + + # t_project[wind] recording: + 'wind': + [('output/0/2015135_merra_wind.tif', + 'hash', + 'sha256', + '88029a425fbeb59149a29be793087b51ea46583bb89e62c1baca629ccfc3a36a')], } t_project_no_warp = { diff --git a/gips/test/sys/t_merra.py b/gips/test/sys/t_merra.py index 10e71c46..2c6d4259 100644 --- a/gips/test/sys/t_merra.py +++ b/gips/test/sys/t_merra.py @@ -59,19 +59,24 @@ def t_info(repo_env, expected): actual = repo_env.run('gips_info', driver) assert expected == actual +@pytest.mark.parametrize("product", expectations.t_process.keys()) # TODO <-- +def t_project(setup_fixture, export_wrapper, product): + """Test gips_project merra with warping.""" + record_mode, expected, runner = export_wrapper + args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, + '--notld', '-p', product) + outcome, actual = runner('gips_project', *args) + if not record_mode: # don't evaluate assertions when in record-mode + assert outcome.exit_code == 0 and expected == actual -def t_project(setup_fixture, clean_repo_env, output_tfe, expected): - """Test gips_project {} with warping.""".format(driver) - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld') - actual = output_tfe.run('gips_project', *args) - assert expected == actual - - +# TODO revive this test? +''' def t_project_no_warp(setup_fixture, clean_repo_env, output_tfe, expected): """Test gips_project {} without warping.""".format(driver) args = STD_ARGS + ('--outdir', OUTPUT_DIR, '--notld') actual = output_tfe.run('gips_project', *args) assert expected == actual +''' # Haven't used gips_tiles ever From d14d375d559af7fad22b4e331e1be5aad40dcd37 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Fri, 22 Dec 2017 16:06:04 -0500 Subject: [PATCH 11/44] 467 first text expectation: modis' t_stats, and: Had to move the setting of initial_files to inside the run wrapper; it was catching the preliminary files generated by gips_project. This way files produced during setup won't impact record nor playback. --- gips/test/sys/expected/modis.py | 214 ++++++++++++++++++++++++++++++-- gips/test/sys/t_modis.py | 24 ++-- gips/test/sys/util.py | 9 +- 3 files changed, 219 insertions(+), 28 deletions(-) diff --git a/gips/test/sys/expected/modis.py b/gips/test/sys/expected/modis.py index fd477886..6d64c878 100644 --- a/gips/test/sys/expected/modis.py +++ b/gips/test/sys/expected/modis.py @@ -763,18 +763,208 @@ 'h12v04/h12v04_2012338_MOD_clouds.tif': -2052728372, }} -t_stats = { 'created': { - 'clouds_stats.txt': -142855826, - 'fsnow_stats.txt': 1649245444, - 'indices_stats.txt': -896610352, - 'ndvi8_stats.txt': -493982920, - 'obstime_stats.txt': -1865008021, - 'quality_stats.txt': -914294800, - 'snow_stats.txt': 239300424, - 'temp8td_stats.txt': 1298400117, - 'temp8tn_stats.txt': -312645073, - 'temp_stats.txt': 2580004, -}} +t_stats = { + # t_stats[satvi] recording: + 'satvi': + [('output/satvi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.1199 0.2757 0.188845 0.0394886 -2.85581 221002.0 \n', + '2012-337 -0.1179 0.2955 0.189273 0.0399094 -2.7273 221002.0 \n', + '2012-338 -0.1125 0.2985 0.187186 0.0408718 -2.44579 218582.0 \n'])], + + # t_stats[ndti] recording: + 'ndti': + [('output/ndti_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -1.0048 0.8895 0.31602 0.0595381 -2.46398 221002.0 \n', + '2012-337 -0.643 0.9544 0.31893 0.0596861 -0.906354 221002.0 \n', + '2012-338 -0.5051 0.9999 0.320866 0.064446 -0.195554 218582.0 \n'])], + + # t_stats[ndvi] recording: + 'ndvi': + [('output/ndvi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.8893 0.8144 0.598198 0.130008 -4.50499 221002.0 \n', + '2012-337 -0.9126 0.8198 0.598463 0.132692 -4.61097 221002.0 \n', + '2012-338 -1.0001 0.8117 0.595837 0.134746 -4.68736 220979.0 \n'])], + + # t_stats[quality] recording: + 'quality': + [('output/quality_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0 0.0 0.0 0.0 nan 221002.0 \n', + '2012-337 0.0 0.0 0.0 0.0 nan 221002.0 \n', + '2012-338 0.0 0.0 0.0 0.0 nan 220979.0 \n'])], + + # t_stats[isti] recording: + 'isti': + [('output/isti_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0583 3.2766 0.522676 0.0837755 13.2684 220980.0 \n', + '2012-337 0.0232 3.2766 0.519636 0.0845782 12.4045 221002.0 \n', + '2012-338 -0.0001 3.0399 0.517872 0.0852503 7.69029 218582.0 \n'])], + + # t_stats[ndvi8] recording: + 'ndvi8': + [('output/ndvi8_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-337 -1.9475 3.2766 0.612297 0.162128 -3.72226 220758.0 \n'])], + + # t_stats[snow] recording: + 'snow': + [('output/snow_stats.txt', + 'text-full', + ['date Snow Cover-min Snow Cover-max Snow Cover-mean Snow Cover-sd Snow Cover-skew Snow Cover-count Fractional Snow Cover-min Fractional Snow Cover-max Fractional Snow Cover-mean Fractional Snow Cover-sd Fractional Snow Cover-skew Fractional Snow Cover-count \n', + '2012-336 0.0 100.0 66.0782 43.1646 -0.66934 2379.0 0.0 70.0 44.8928 24.6763 -0.938924 2379.0 \n', + '2012-337 0.0 0.0 0.0 0.0 nan 677.0 0.0 0.0 0.0 0.0 nan 677.0 \n', + '2012-338 0.0 100.0 7.3169 20.9113 2.95797 221002.0 0.0 38.0 0.358599 2.04345 8.82778 221002.0 \n'])], + + # t_stats[fsnow] recording: + 'fsnow': + [('output/fsnow_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0 70.0 36.4813 30.8325 -0.322887 2379.0 \n', + '2012-337 0.0 0.0 0.0 0.0 nan 677.0 \n', + '2012-338 0.0 35.0 0.0644655 1.11486 23.1911 221002.0 \n'])], + + # t_stats[crcm] recording: + 'crcm': + [('output/crcm_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.8857 1.6238 0.960025 0.21929 -3.32392 221002.0 \n', + '2012-337 -0.8639 1.5786 0.966857 0.222374 -3.25955 221002.0 \n', + '2012-338 -0.8744 1.8977 0.963618 0.23254 -2.82714 218582.0 \n'])], + + # t_stats[ndsi] recording: + 'ndsi': + [('output/ndsi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.7313 1.0045 -0.58418 0.123674 6.27293 221002.0 \n', + '2012-337 -0.731 0.9434 -0.58564 0.12352 6.14429 221002.0 \n', + '2012-338 -0.7775 0.9329 -0.581803 0.125509 5.997 218582.0 \n'])], + + # t_stats[brgt] recording: + 'brgt': + [('output/brgt_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0062 0.1361 0.0467873 0.0103586 0.699293 221002.0 \n', + '2012-337 0.0062 0.1379 0.0465373 0.0103964 0.708942 221002.0 \n', + '2012-338 0.0064 0.1351 0.0464351 0.0105081 0.660629 220979.0 \n'])], + + # t_stats[msavi2] recording: + 'msavi2': + [('output/msavi2_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.014 0.6976 0.503178 0.0608721 -2.7989 219182.0 \n', + '2012-337 -0.0001 0.6996 0.502591 0.0601847 -2.75199 219026.0 \n', + '2012-338 0.014 0.693 0.499838 0.0606011 -2.67091 219027.0 \n'])], + + # t_stats[bi] recording: + 'bi': + [('output/bi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0033 0.1717 0.0948656 0.0173088 -1.23546 221002.0 \n', + '2012-337 0.0033 0.1714 0.0944838 0.0173897 -1.24311 221002.0 \n', + '2012-338 0.0038 0.1705 0.0938487 0.0174975 -1.19669 220979.0 \n'])], + + # t_stats[obstime] recording: + 'obstime': + [('output/obstime_stats.txt', + 'text-full', + ['date Observation Time Daytime Terra-min Observation Time Daytime Terra-max Observation Time Daytime Terra-mean Observation Time Daytime Terra-sd Observation Time Daytime Terra-skew Observation Time Daytime Terra-count Observation Time Nighttime Terra-min Observation Time Nighttime Terra-max Observation Time Nighttime Terra-mean Observation Time Nighttime Terra-sd Observation Time Nighttime Terra-skew Observation Time Nighttime Terra-count Observation Time Daytime Aqua-min Observation Time Daytime Aqua-max Observation Time Daytime Aqua-mean Observation Time Daytime Aqua-sd Observation Time Daytime Aqua-skew Observation Time Daytime Aqua-count Observation Time Nighttime Aqua-min Observation Time Nighttime Aqua-max Observation Time Nighttime Aqua-mean Observation Time Nighttime Aqua-sd Observation Time Nighttime Aqua-skew Observation Time Nighttime Aqua-count \n', + '2012-336 10.8 10.8 10.8 1.90735e-07 -0.999996 716.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 1.4 1.4 1.4 2.38419e-08 inf 6685.0 \n', + '2012-337 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 \n', + '2012-338 10.6 10.6 10.6 3.8147e-07 -1.0 184032.0 255.0 0.0 nan nan nan 0.0 12.3 12.3 12.3 1.90735e-07 -0.999996 216873.0 1.2 2.8 1.61335 0.700361 1.10413 181365.0 \n'])], + + # t_stats[lswi] recording: + 'lswi': + [('output/lswi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.9413 1.004 0.0704005 0.0795676 -0.897423 221002.0 \n', + '2012-337 -0.9644 0.879 0.068998 0.0814312 -1.75006 221002.0 \n', + '2012-338 -1.0001 0.8591 0.0695461 0.0851064 -1.65515 218582.0 \n'])], + + # t_stats[vari] recording: + 'vari': + [('output/vari_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.2931 3.2766 -0.0579385 0.0896767 11.2112 221002.0 \n', + '2012-337 -0.3266 3.2766 -0.0573387 0.109661 15.3818 221002.0 \n', + '2012-338 -0.3817 3.2766 -0.0567795 0.115018 15.8625 220979.0 \n'])], + + # t_stats[evi] recording: + 'evi': + [('output/evi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.0689 0.4391 0.251258 0.0534915 -2.19097 221002.0 \n', + '2012-337 -0.0704 0.4432 0.250715 0.0535784 -2.20674 221002.0 \n', + '2012-338 -0.0727 0.4339 0.248372 0.0536924 -2.14955 220979.0 \n'])], + + # t_stats[temp8tn] recording: + 'temp8tn': + [('output/temp8tn_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-337 267.02 280.38 273.055 1.35905 0.334532 212758.0 \n'])], + + # t_stats[clouds] recording: + 'clouds': + [('output/clouds_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0 1.0 0.990964 0.094628 -10.3767 221002.0 \n', + '2012-337 0.0 1.0 0.998367 0.0403832 -24.6819 221002.0 \n', + '2012-338 0.0 1.0 0.0838499 0.277163 3.00293 221002.0 \n'])], + + # t_stats[temp] recording: + 'temp': + [('output/temp_stats.txt', + 'text-full', + ['date Temperature Daytime Terra-min Temperature Daytime Terra-max Temperature Daytime Terra-mean Temperature Daytime Terra-sd Temperature Daytime Terra-skew Temperature Daytime Terra-count Temperature Nighttime Terra-min Temperature Nighttime Terra-max Temperature Nighttime Terra-mean Temperature Nighttime Terra-sd Temperature Nighttime Terra-skew Temperature Nighttime Terra-count Temperature Daytime Aqua-min Temperature Daytime Aqua-max Temperature Daytime Aqua-mean Temperature Daytime Aqua-sd Temperature Daytime Aqua-skew Temperature Daytime Aqua-count Temperature Nighttime Aqua-min Temperature Nighttime Aqua-max Temperature Nighttime Aqua-mean Temperature Nighttime Aqua-sd Temperature Nighttime Aqua-skew Temperature Nighttime Aqua-count Temperature Best Quality-min Temperature Best Quality-max Temperature Best Quality-mean Temperature Best Quality-sd Temperature Best Quality-skew Temperature Best Quality-count \n', + '2012-336 267.44 267.9 267.67 0.140381 -0.244244 716.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 258.94 263.7 261.912 1.39539 -0.717615 6685.0 0.0 0.0 0.0 0.0 nan 221255.0 \n', + '2012-337 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 0.0 0.0 0.0 0.0 nan 221255.0 \n', + '2012-338 275.34 286.28 282.987 2.17515 -1.16826 184032.0 65535.0 0.0 nan nan nan 0.0 277.56 283.86 281.873 0.980026 -0.679654 216873.0 270.04 280.26 277.74 1.43805 -1.36599 181365.0 0.0 13.0 4.60988 3.26751 1.22493 221255.0 \n'])], + + # t_stats[temp8td] recording: + 'temp8td': + [('output/temp8td_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-337 273.32 283.76 279.933 1.98691 -1.00057 218113.0 \n'])], + + # t_stats[sti] recording: + 'sti': + [('output/sti_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.0024 3.2766 1.94182 0.235117 0.707709 221002.0 \n', + '2012-337 0.2173 3.2766 1.95394 0.239902 0.761782 221002.0 \n', + '2012-338 0.3288 3.2766 1.96524 0.263691 0.833778 218582.0 \n'])], + + # t_stats[crc] recording: + 'crc': + [('output/crc_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.8526 2.1525 1.29852 0.263609 -2.37667 221002.0 \n', + '2012-337 -0.868 2.2016 1.30555 0.268596 -2.32839 221002.0 \n', + '2012-338 -0.8795 2.8088 1.30562 0.283586 -1.87226 218582.0 \n'])], +} t_gridded_export = { 'created': { diff --git a/gips/test/sys/t_modis.py b/gips/test/sys/t_modis.py index 6b254c2e..bef0b5c6 100644 --- a/gips/test/sys/t_modis.py +++ b/gips/test/sys/t_modis.py @@ -97,22 +97,22 @@ def t_tiles_copy(setup_modis_data, clean_repo_env, output_tfe, expected): actual = output_tfe.run('gips_tiles', *args) assert expected == actual - -def t_stats(setup_modis_data, clean_repo_env, output_tfe, expected): +@pytest.mark.parametrize("product", expectations.t_stats.keys()) +def t_stats(setup_modis_data, export_wrapper, product): """Test gips_stats on projected files.""" + record_mode, expected, runner = export_wrapper + # generate data needed for stats computation - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld') - prep_run = output_tfe.run('gips_project', *args) - assert prep_run.exit_status == 0 # confirm it worked; not really in the test + args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, + '--notld', '-p', product) + outcome = sh.gips_project(*args) + assert outcome.exit_code == 0 # sanity check - # compute stats - gtfe = GipsTestFileEnv(OUTPUT_DIR, start_clear=False) - actual = gtfe.run('gips_stats', OUTPUT_DIR) + # compute & confirm stats + outcome, actual = runner('gips_stats', OUTPUT_DIR) + if not record_mode: + assert outcome.exit_code == 0 and expected == actual - # check for correct stats content - assert expected == actual - - def t_gridded_export(setup_modis_data, clean_repo_env, output_tfe, expected): """Test gips_project using rastermask spatial spec""" rastermask = os.path.join(TEST_DATA_DIR, 'site_mask.tif') diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index 769e9dd2..f6d60eda 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -298,22 +298,23 @@ def sys_test_wrapper(request, path): that is calling this function. Yields to let some process work, then reports on files created in the path during the wait. """ - # DATA_REPO_ROOT is the directory under test rp = record_path() product = request.getfixturevalue('product') expectations = load_expectation_module(request.module.__name__) expected = getattr(expectations, request.function.__name__)[product] expected_filenames = [e[0] for e in expected] - if rp: - initial_files = find_files(path) - else: + if not rp: interlopers = [fn for fn in expected_filenames if os.path.exists(fn)] if interlopers: raise IOError('Files in the way of the test: {}'.format(interlopers)) + initial_files = [] # getting around a scoping problem with python + def wrapped_runner(cmd_string, *args): print("command line: `{} {}`".format(cmd_string, ' '.join(args))) + if rp: + initial_files.extend(find_files(path)) outcome = sh.Command(cmd_string)(*args) return outcome, [generate_expectation(fn, path) for fn in expected_filenames] From 092a54e9b54bd46f31cec09b2582350b369659d1 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Fri, 22 Dec 2017 16:23:04 -0500 Subject: [PATCH 12/44] 467 port merra t_stats to non-scripttest-land --- gips/test/sys/expected/merra.py | 73 ++++++++++++++++++++++++++++----- gips/test/sys/t_merra.py | 23 ++++++----- 2 files changed, 74 insertions(+), 22 deletions(-) diff --git a/gips/test/sys/expected/merra.py b/gips/test/sys/expected/merra.py index c9e54272..a808a049 100644 --- a/gips/test/sys/expected/merra.py +++ b/gips/test/sys/expected/merra.py @@ -202,15 +202,66 @@ } t_stats = { - 'created': { - 'patm_stats.txt': 1367728250, - 'prcp_stats.txt': 1367728250, - 'rhum_stats.txt': 1367728250, - 'shum_stats.txt': 1367728250, - 'srad_stats.txt': 1367728250, - 'tave_stats.txt': 1367728250, - 'tmax_stats.txt': 1367728250, - 'tmin_stats.txt': 1367728250, - 'wind_stats.txt': 1367728250, - } + # t_stats[srad] recording: + 'srad': + [('output/srad_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[tave] recording: + 'tave': + [('output/tave_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[shum] recording: + 'shum': + [('output/shum_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[rhum] recording: + 'rhum': + [('output/rhum_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[tmin] recording: + 'tmin': + [('output/tmin_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[tmax] recording: + 'tmax': + [('output/tmax_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[prcp] recording: + 'prcp': + [('output/prcp_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[patm] recording: + 'patm': + [('output/patm_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[wind] recording: + 'wind': + [('output/wind_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], } diff --git a/gips/test/sys/t_merra.py b/gips/test/sys/t_merra.py index 2c6d4259..83ff4762 100644 --- a/gips/test/sys/t_merra.py +++ b/gips/test/sys/t_merra.py @@ -95,17 +95,18 @@ def t_tiles_copy(setup_fixture, clean_repo_env, output_tfe, expected): actual = output_tfe.run('gips_tiles', *args) assert expected == actual - -def t_stats(setup_fixture, clean_repo_env, output_tfe, expected): +@pytest.mark.parametrize("product", expectations.t_stats.keys()) +def t_stats(setup_fixture, export_wrapper, product): """Test gips_stats on projected files.""" - # generate data needed for stats computation - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld') - prep_run = output_tfe.run('gips_project', *args) - assert prep_run.exit_status == 0 # confirm it worked; not really in the test + record_mode, expected, runner = export_wrapper - # compute stats - gtfe = GipsTestFileEnv(OUTPUT_DIR, start_clear=False) - actual = gtfe.run('gips_stats', OUTPUT_DIR) + # generate data needed for stats computation + args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, + '--notld', '-p', product) + outcome = sh.gips_project(*args) + assert outcome.exit_code == 0 # sanity check - # check for correct stats content - assert expected == actual + # compute & confirm stats + outcome, actual = runner('gips_stats', OUTPUT_DIR) + if not record_mode: + assert outcome.exit_code == 0 and expected == actual From 834093ba7c8978278afb48fc251dc2d0152716fc Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Tue, 26 Dec 2017 13:23:06 -0500 Subject: [PATCH 13/44] 483 refactor t_process to handle multiple drivers --- gips/test/sys/driver_setup.py | 35 ++++ gips/test/sys/t_process.py | 362 ++++++++++++++++++++++++++++++++++ 2 files changed, 397 insertions(+) create mode 100644 gips/test/sys/driver_setup.py create mode 100644 gips/test/sys/t_process.py diff --git a/gips/test/sys/driver_setup.py b/gips/test/sys/driver_setup.py new file mode 100644 index 00000000..a9d08602 --- /dev/null +++ b/gips/test/sys/driver_setup.py @@ -0,0 +1,35 @@ +from __future__ import print_function + +import datetime + +import pytest +import sh + +import util + +STD_ARGS = { + 'modis': ('modis', '-s', util.NH_SHP_PATH, '-d', '2012-12-01,2012-12-03', '-v', '4'), + 'merra': ('merra', '-s', util.NH_SHP_PATH, '-d', '2015-135', '-v', '4'), +} + +setup_attempted = [] + +def setup_repo_data(driver): + """Use gips_inventory to ensure assets are present.""" + if driver in setup_attempted: + print(driver, "setup already attempted; skipping.") + return + setup_attempted.append(driver) + if not pytest.config.getoption('setup_repo'): + print("Skipping", driver, "repo setup per lack of option.") + return + # handle modis scheduled downtime case + if driver == 'modis' and datetime.datetime.today().date().weekday() == 2: + raise RuntimeError("It's Wednesday; modis downloads are likely to fail.") + print('Downloading', driver, 'data . . .') + args = STD_ARGS[driver] + ('--fetch',) + outcome = sh.Command('gips_inventory')(*args) + if outcome.exit_code != 0: + raise RuntimeError(driver + " data setup via `gips_inventory` failed", + outcome) + print(driver, "data download complete.") diff --git a/gips/test/sys/t_process.py b/gips/test/sys/t_process.py new file mode 100644 index 00000000..98083afc --- /dev/null +++ b/gips/test/sys/t_process.py @@ -0,0 +1,362 @@ +import pytest + +import util +from util import repo_wrapper +import driver_setup +from .expected import modis as expectations + +params = [] # (driver, product) + +params += [('modis', p) for p in ( + 'ndvi8', 'temp8td', 'temp8tn', #Terra 8-day Products + 'quality', #Nadir BRDF-Adjusted 16-day Products' + 'clouds', #Standard Products' + 'fsnow', 'obstime', 'snow', 'temp', #Terra/Aqua Daily Products' + #Index Products' + 'bi', 'brgt', 'crc', 'crcm', 'evi', 'isti', 'lswi', + 'msavi2', 'ndsi', 'ndti', 'ndvi', 'satvi', 'sti', 'vari', + #'indices', deprecated; not testing as a result + #'landcover', is annual, not available for the scene under test +)] + +#params += [('modis', p) for p in ( +#)] + +expectations = { + 'modis': { + # 'landcover' [], # is annual, not available for the scene under test + # weird symlink products: + # t_process[quality] recording: + 'quality': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/data-root/modis/tiles/h12v04/2012336/MCD43A2.A2012336.h12v04.006.2016112010833.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/data-root/modis/tiles/h12v04/2012337/MCD43A2.A2012337.h12v04.006.2016112013509.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/data-root/modis/tiles/h12v04/2012338/MCD43A2.A2012338.h12v04.006.2016112020013.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo')], + + # t_process[temp8tn] recording: + 'temp8tn': + [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8tn.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/data-root/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Night_1km')], + + # t_process[temp8td] recording: + 'temp8td': + [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8td.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/data-root/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Day_1km')], + + # normal products + # t_process[satvi] recording: + 'satvi': + [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_satvi.tif', + 'hash', + 'sha256', + '94bee7cecf0f0a227fd8532223f86de10e01c3e39d1dd70b7675d91b3ca338a4'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_satvi.tif', + 'hash', + 'sha256', + '6397669a6cd62bb39813b9000d93ba50f4cabe73aebf1e6444f4e3be9f630e19'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_satvi.tif', + 'hash', + 'sha256', + '6124ee615a4f522fdf966a5479d03d87934908e5090a192192f9700020f24760')], + + # t_process[ndti] recording: + 'ndti': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndti.tif', + 'hash', + 'sha256', + '025ea7a9204ae8ad0952a92b386bf735dbbce8337e336843476d3f227cf1f1ab'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndti.tif', + 'hash', + 'sha256', + 'b6baa8c7a3996f680c4177e4157c9275dd895c8e6350afa6ef564e673bb6a5f0'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndti.tif', + 'hash', + 'sha256', + 'a1c4f1161e8a584257352e76160319c39aa4605dd565d388d0116e193773771f')], + + # t_process[ndvi] recording: + 'ndvi': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndvi.tif', + 'hash', + 'sha256', + 'efbeae3a1c5e59ccd54b6108e275f9a8f0059162ac2797f4d69b00acb80147b3'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndvi.tif', + 'hash', + 'sha256', + 'b56d5c96229a1ad4cdcbd09db10b19a216b4cde8a0ffd184ee5cf01a35ab2b46'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndvi.tif', + 'hash', + 'sha256', + 'd9ff5d2f85ac9a267a8f6b1aca34f62b8196721e212b1e092f82ce9f1e208c61')], + + # t_process[isti] recording: + 'isti': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_isti.tif', + 'hash', + 'sha256', + 'c0a38d1cd00de52eacd8abcda714bfdaa575787e27b40dc0472ffca5ab1696c0'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_isti.tif', + 'hash', + 'sha256', + 'b1759c1008e5914b955c4a9f55572baa8f7c0310677264a1057329e236d826cb'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_isti.tif', + 'hash', + 'sha256', + '677182d186dac992b52b93239af0f419536075c7cd467864fa544bcde4b89e69')], + + # t_process[ndvi8] recording: + 'ndvi8': + [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_ndvi8.tif', + 'hash', + 'sha256', + '93db24c18886fbac0b87d1404a0c80606a599cc55b02d745a09ffb051b8131ac')], + + # t_process[snow] recording: + 'snow': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_snow.tif', + 'hash', + 'sha256', + 'bea7315b736a75bcf37df86089679845ac597f3fba5d5cd6e79f5516b9657faa'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_snow.tif', + 'hash', + 'sha256', + 'de979c1a0c616d7889dd5fd45816849f9c08df54167d7f3b40301d6ac4495778'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_snow.tif', + 'hash', + 'sha256', + '5d9bbfff1522e74165cdd677f1580e9eb2236c2c5922a796a6c9ee0db24ae05a')], + + # t_process[fsnow] recording: + 'fsnow': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_fsnow.tif', + 'hash', + 'sha256', + '853a5ca95f21c4a745e41e4acc46a71a74c74e0cdd1cae9ed3dd31b2fef1ddbd'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_fsnow.tif', + 'hash', + 'sha256', + '3a5f7c95fe60e5789919cf55d7f8e652e4d668c6eb63eaa53a82249c0ea19c5a'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_fsnow.tif', + 'hash', + 'sha256', + 'd09b8a12ad17a7e715ae2e988fbf683e5fe2df46e97e9ac3ebb4444c8b1c169d')], + + # t_process[crcm] recording: + 'crcm': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crcm.tif', + 'hash', + 'sha256', + 'a45c2fb16d871a919e0c803f80e1f3b4ec831472d0466c33741fcf8e8150f6ba'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crcm.tif', + 'hash', + 'sha256', + '9aa80c0e1fa1b62b0a41aed42eeaf84632845d74f29594dfee55c22673b0a6f5'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crcm.tif', + 'hash', + 'sha256', + 'a28c7d280fb8dfe0fe1cc003e1001457abdf14acc5545b2a69b782cfbcce7b50')], + + # t_process[ndsi] recording: + 'ndsi': + [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndsi.tif', + 'hash', + 'sha256', + '1f91f738bbb28b3ebeba36affd8d0689e105e90e1547cca58497eb7c7fdf7346'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndsi.tif', + 'hash', + 'sha256', + 'a626d375fbe5094a5199fdfc2206cf94777ff574379b45811cf7a49055a9f6ec'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndsi.tif', + 'hash', + 'sha256', + 'cb2ca7461c65f66ddb13e804ae7fe8f54daaf620ade600efee24810d40d4eb34')], + + # t_process[brgt] recording: + 'brgt': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_brgt.tif', + 'hash', + 'sha256', + 'bf6bad538c7c6e703bc2f39503c2f128fa8e988a533e302c0124e085735dfaf9'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_brgt.tif', + 'hash', + 'sha256', + '5c72e8f37640ec6b4a5a33e18980d71c70482fc8f708ba25da73e074cf64729f'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_brgt.tif', + 'hash', + 'sha256', + '3460d0d35e35540b55e9eda35f253af8d85eabb84c0c44cb5f9c02d43da7e20a')], + + # t_process[msavi2] recording: + 'msavi2': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_msavi2.tif', + 'hash', + 'sha256', + 'd2c93798df10939ff8afb6cd23e18ac3281cfd844e45da47a29d0342c2b980e9'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_msavi2.tif', + 'hash', + 'sha256', + '92fe4f40d051ef3f479bde76a67bfff978bc513893af5ed62e4a48e457daabad'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_msavi2.tif', + 'hash', + 'sha256', + 'f25bd2d58d3edae72b11e73c08341479dee13af80352a4749861b6a54a81b351')], + + # t_process[bi] recording: + 'bi': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_bi.tif', + 'hash', + 'sha256', + '28c181b8d210b71759589da53295cc894c9d12c74ff1f7b0a49b88f73664282e'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_bi.tif', + 'hash', + 'sha256', + '45acfc9249e063b6b97f269338590d51e2334e9bc5df118613ad908c6f0a3f8e'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_bi.tif', + 'hash', + 'sha256', + '8526cb96d59fc9549dc8535fc8128123b9d39a63a3b7cd97ee3e611b7eeadf6f')], + + # t_process[obstime] recording: + 'obstime': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + '66e01c94910c4fdd4f6fb2ab99b0c2d0cb73618032783e3cbdf1867191981fdc'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + '117cea0497b3158b465dd7dd197c886ebb1acf5059ed83dfdd6b5d7e94bb5243'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'b04d8b07ef83e479db5650a30c84513bde7d5b05dd68187782c057ea0fd3e537')], + + # t_process[lswi] recording: + 'lswi': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_lswi.tif', + 'hash', + 'sha256', + '8de7378cdba830171c8da94424bf7807fa6c8b12523e8b34175a4b17ec770a5d'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_lswi.tif', + 'hash', + 'sha256', + '62de4bdafcc276e9c1c343f4e63f5dd06167e87f7712796f991e3a957f2b71de'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_lswi.tif', + 'hash', + 'sha256', + 'add051ef35d7ffff5eb68c48f8b8071f9cb493b5d3426867605d4c61746e3a26')], + + # t_process[vari] recording: + 'vari': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_vari.tif', + 'hash', + 'sha256', + '732d6d3d9fbe0e1b4616a3f8a91bf12d532598a0a02e8aab16c9a01c3ac56913'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_vari.tif', + 'hash', + 'sha256', + '9875717cc5e1926f11378bbf16345dbade5a2badf9f3dd06844a77f94d80ef72'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_vari.tif', + 'hash', + 'sha256', + '70fd19862ef4145c24f26a8a74cb748279b88fd973de935726ebef2294e17edd')], + + # t_process[evi] recording: + 'evi': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_evi.tif', + 'hash', + 'sha256', + '13548ae74456f08e6add657a637ca364626eafd2cc767724f57aac8ff93005bb'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_evi.tif', + 'hash', + 'sha256', + '7b926f7004cd92f2ec604fc4e2cc1489f60a335e39f4954030332a012fd33f87'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_evi.tif', + 'hash', + 'sha256', + '51db813a857f795817f529edd56d7134004566be46edab8a8dadc81286fb895f')], + + # t_process[clouds] recording: + 'clouds': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD_clouds.tif', + 'hash', + 'sha256', + '26b3e7e695f08ec2e9fb26985f9d5e392f9e8cedc331ab6721ea8e48821d93f0'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD_clouds.tif', + 'hash', + 'sha256', + 'c43eec8178d0521d20e2a5b6e9b61525007d80702c6dd705a3e52ed3a76a7b73'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_clouds.tif', + 'hash', + 'sha256', + '24a6a7050c866eed989530bec45b72004097f1e36fcc0e686652b87723a86de4')], + + # t_process[temp] recording: + 'temp': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_temp.tif', + 'hash', + 'sha256', + 'ead2622531929bba3192b8a6b609d5d920e79394ec577c5245d74d69593d827c'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '5d498bad2779f84289dfb75b14345d64d16b5bc7994b4aca6cd694f70b86064d'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '097f341fa917a0ec316c37d7907ff15b754dcd0925469be586016137ca47575f')], + + # t_process[sti] recording: + 'sti': + [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_sti.tif', + 'hash', + 'sha256', + '24e50766ec18826f30bbad07bc8f5a34e72f1966b325cc23f064aa5fd35dc24a'), + ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_sti.tif', + 'hash', + 'sha256', + 'c223de9d00e9cfedc0cfc13a45163535f3c59e5587642fa2c725b00b17b7437c'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_sti.tif', + 'hash', + 'sha256', + '73c247fd2c7623e79145b6337846c97d14d1752d660b2f32245a4ccaac033350')], + + # t_process[crc] recording: + 'crc': + [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crc.tif', + 'hash', + 'sha256', + '07debf171aedfeba7d6c1e3220ff8e4a2a7b1033e68c72bbc121b500b6ed3c9c'), + ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crc.tif', + 'hash', + 'sha256', + 'a7b61aabc950be7a3040b6f9572bf755522b12ad27644583e603def05ac1980b'), + ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crc.tif', + 'hash', + 'sha256', + '942efeeee2b020123be6d7334bbb3ef5995c93d2fd0247cb79948836865c8067')], + }, +} + +@pytest.mark.parametrize("driver, product", params) +def t_process(repo_wrapper, driver, product): + """Test gips_process output.""" + record_mode, _, runner = repo_wrapper + driver_setup.setup_repo_data(driver) + args = ('gips_process',) + driver_setup.STD_ARGS[driver] + ('-p', product) + outcome, actual = runner(*args) + if not record_mode: # don't evaluate assertions when in record-mode + assert (outcome.exit_code == 0 + and expectations[driver][product] == actual) From 7795d1bed7836005d21b63568b4e7e2d9b1bb9f8 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Tue, 26 Dec 2017 15:10:01 -0500 Subject: [PATCH 14/44] 483 refine process test & move merra into it --- gips/test/sys/expected/merra.py | 58 ------ gips/test/sys/expected/modis.py | 327 -------------------------------- gips/test/sys/t_merra.py | 11 +- gips/test/sys/t_modis.py | 15 +- gips/test/sys/t_process.py | 86 +++++++-- gips/test/sys/util.py | 10 +- 6 files changed, 77 insertions(+), 430 deletions(-) diff --git a/gips/test/sys/expected/merra.py b/gips/test/sys/expected/merra.py index a808a049..bafe9086 100644 --- a/gips/test/sys/expected/merra.py +++ b/gips/test/sys/expected/merra.py @@ -40,64 +40,6 @@ """ } -t_process = { - # test this too? 'frland': [], - # t_process[srad] recording: - 'srad': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_srad.tif', - 'hash', - 'sha256', - '882a44af70e337bf905c8695936f101792a518507a4304b23781a2f30aaabab4')], - # t_process[tave] recording: - 'tave': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tave.tif', - 'hash', - 'sha256', - '5185aebd7cda54157cad2ddbde9f6fec4871c1b85fe78d0738634ba0211f2c9b')], - # t_process[shum] recording: - 'shum': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_shum.tif', - 'hash', - 'sha256', - '558f567c1e931891553d396f81b2b90929ad48a0ae88fa95d3894eae23ab3eba')], - # t_process[rhum] recording: - 'rhum': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_rhum.tif', - 'hash', - 'sha256', - '058161eb8488f1fb3df7f5c8719833c9d68c56aa082d4605938f62c2ee0035f6')], - # t_process[tmin] recording: - 'tmin': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmin.tif', - 'hash', - 'sha256', - '35a850769a3bb8f5209574ea4835bf417d8f99024567f4af9cdee9e623d0c567')], - # t_process[tmax] recording: - 'tmax': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmax.tif', - 'hash', - 'sha256', - '5963fcf346f388fc347355153996a078f1eb5ecbbc094394b72be382ba491865')], - # t_process[prcp] recording: - 'prcp': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_prcp.tif', - 'hash', - 'sha256', - '348fd7072ef0a8625f5268e29acbf3a69a959a412327102ef6558c992e62bc9f')], - # t_process[patm] recording: - 'patm': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_patm.tif', - 'hash', - 'sha256', - '7d9deca613973cb8ffb8f15a3dfa2013e6783556b520915e89b97fe41de638e3')], - # t_process[wind] recording: - 'wind': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_wind.tif', - 'hash', - 'sha256', - 'a90c1d87fdb1c0926bd5b3004408a1f9d42ef08c38deb9d0572bc7536dbbf08a')], -} - t_project = { # t_project[srad] recording: diff --git a/gips/test/sys/expected/modis.py b/gips/test/sys/expected/modis.py index 6d64c878..a116c629 100644 --- a/gips/test/sys/expected/modis.py +++ b/gips/test/sys/expected/modis.py @@ -26,333 +26,6 @@ """ } - -t_process = { - # 'landcover' [], # is annual, not available for the scene under test - # weird symlink products: - # t_process[quality] recording: - 'quality': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_quality.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/data-root/modis/tiles/h12v04/2012336/MCD43A2.A2012336.h12v04.006.2016112010833.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_quality.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/data-root/modis/tiles/h12v04/2012337/MCD43A2.A2012337.h12v04.006.2016112013509.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_quality.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/data-root/modis/tiles/h12v04/2012338/MCD43A2.A2012338.h12v04.006.2016112020013.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo')], - - # t_process[temp8tn] recording: - 'temp8tn': - [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8tn.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/data-root/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Night_1km')], - - # t_process[temp8td] recording: - 'temp8td': - [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8td.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/data-root/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Day_1km')], - - # normal products - # t_process[satvi] recording: - 'satvi': - [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_satvi.tif', - 'hash', - 'sha256', - '94bee7cecf0f0a227fd8532223f86de10e01c3e39d1dd70b7675d91b3ca338a4'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_satvi.tif', - 'hash', - 'sha256', - '6397669a6cd62bb39813b9000d93ba50f4cabe73aebf1e6444f4e3be9f630e19'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_satvi.tif', - 'hash', - 'sha256', - '6124ee615a4f522fdf966a5479d03d87934908e5090a192192f9700020f24760')], - - # t_process[ndti] recording: - 'ndti': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndti.tif', - 'hash', - 'sha256', - '025ea7a9204ae8ad0952a92b386bf735dbbce8337e336843476d3f227cf1f1ab'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndti.tif', - 'hash', - 'sha256', - 'b6baa8c7a3996f680c4177e4157c9275dd895c8e6350afa6ef564e673bb6a5f0'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndti.tif', - 'hash', - 'sha256', - 'a1c4f1161e8a584257352e76160319c39aa4605dd565d388d0116e193773771f')], - - # t_process[ndvi] recording: - 'ndvi': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndvi.tif', - 'hash', - 'sha256', - 'efbeae3a1c5e59ccd54b6108e275f9a8f0059162ac2797f4d69b00acb80147b3'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndvi.tif', - 'hash', - 'sha256', - 'b56d5c96229a1ad4cdcbd09db10b19a216b4cde8a0ffd184ee5cf01a35ab2b46'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndvi.tif', - 'hash', - 'sha256', - 'd9ff5d2f85ac9a267a8f6b1aca34f62b8196721e212b1e092f82ce9f1e208c61')], - - # t_process[isti] recording: - 'isti': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_isti.tif', - 'hash', - 'sha256', - 'c0a38d1cd00de52eacd8abcda714bfdaa575787e27b40dc0472ffca5ab1696c0'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_isti.tif', - 'hash', - 'sha256', - 'b1759c1008e5914b955c4a9f55572baa8f7c0310677264a1057329e236d826cb'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_isti.tif', - 'hash', - 'sha256', - '677182d186dac992b52b93239af0f419536075c7cd467864fa544bcde4b89e69')], - - # t_process[ndvi8] recording: - 'ndvi8': - [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_ndvi8.tif', - 'hash', - 'sha256', - '93db24c18886fbac0b87d1404a0c80606a599cc55b02d745a09ffb051b8131ac')], - - # t_process[snow] recording: - 'snow': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_snow.tif', - 'hash', - 'sha256', - 'bea7315b736a75bcf37df86089679845ac597f3fba5d5cd6e79f5516b9657faa'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_snow.tif', - 'hash', - 'sha256', - 'de979c1a0c616d7889dd5fd45816849f9c08df54167d7f3b40301d6ac4495778'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_snow.tif', - 'hash', - 'sha256', - '5d9bbfff1522e74165cdd677f1580e9eb2236c2c5922a796a6c9ee0db24ae05a')], - - # t_process[fsnow] recording: - 'fsnow': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_fsnow.tif', - 'hash', - 'sha256', - '853a5ca95f21c4a745e41e4acc46a71a74c74e0cdd1cae9ed3dd31b2fef1ddbd'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_fsnow.tif', - 'hash', - 'sha256', - '3a5f7c95fe60e5789919cf55d7f8e652e4d668c6eb63eaa53a82249c0ea19c5a'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_fsnow.tif', - 'hash', - 'sha256', - 'd09b8a12ad17a7e715ae2e988fbf683e5fe2df46e97e9ac3ebb4444c8b1c169d')], - - # t_process[crcm] recording: - 'crcm': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crcm.tif', - 'hash', - 'sha256', - 'a45c2fb16d871a919e0c803f80e1f3b4ec831472d0466c33741fcf8e8150f6ba'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crcm.tif', - 'hash', - 'sha256', - '9aa80c0e1fa1b62b0a41aed42eeaf84632845d74f29594dfee55c22673b0a6f5'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crcm.tif', - 'hash', - 'sha256', - 'a28c7d280fb8dfe0fe1cc003e1001457abdf14acc5545b2a69b782cfbcce7b50')], - - # t_process[ndsi] recording: - 'ndsi': - [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndsi.tif', - 'hash', - 'sha256', - '1f91f738bbb28b3ebeba36affd8d0689e105e90e1547cca58497eb7c7fdf7346'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndsi.tif', - 'hash', - 'sha256', - 'a626d375fbe5094a5199fdfc2206cf94777ff574379b45811cf7a49055a9f6ec'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndsi.tif', - 'hash', - 'sha256', - 'cb2ca7461c65f66ddb13e804ae7fe8f54daaf620ade600efee24810d40d4eb34')], - - # t_process[brgt] recording: - 'brgt': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_brgt.tif', - 'hash', - 'sha256', - 'bf6bad538c7c6e703bc2f39503c2f128fa8e988a533e302c0124e085735dfaf9'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_brgt.tif', - 'hash', - 'sha256', - '5c72e8f37640ec6b4a5a33e18980d71c70482fc8f708ba25da73e074cf64729f'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_brgt.tif', - 'hash', - 'sha256', - '3460d0d35e35540b55e9eda35f253af8d85eabb84c0c44cb5f9c02d43da7e20a')], - - # t_process[msavi2] recording: - 'msavi2': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_msavi2.tif', - 'hash', - 'sha256', - 'd2c93798df10939ff8afb6cd23e18ac3281cfd844e45da47a29d0342c2b980e9'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_msavi2.tif', - 'hash', - 'sha256', - '92fe4f40d051ef3f479bde76a67bfff978bc513893af5ed62e4a48e457daabad'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_msavi2.tif', - 'hash', - 'sha256', - 'f25bd2d58d3edae72b11e73c08341479dee13af80352a4749861b6a54a81b351')], - - # t_process[bi] recording: - 'bi': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_bi.tif', - 'hash', - 'sha256', - '28c181b8d210b71759589da53295cc894c9d12c74ff1f7b0a49b88f73664282e'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_bi.tif', - 'hash', - 'sha256', - '45acfc9249e063b6b97f269338590d51e2334e9bc5df118613ad908c6f0a3f8e'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_bi.tif', - 'hash', - 'sha256', - '8526cb96d59fc9549dc8535fc8128123b9d39a63a3b7cd97ee3e611b7eeadf6f')], - - # t_process[obstime] recording: - 'obstime': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - '66e01c94910c4fdd4f6fb2ab99b0c2d0cb73618032783e3cbdf1867191981fdc'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - '117cea0497b3158b465dd7dd197c886ebb1acf5059ed83dfdd6b5d7e94bb5243'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - 'b04d8b07ef83e479db5650a30c84513bde7d5b05dd68187782c057ea0fd3e537')], - - # t_process[lswi] recording: - 'lswi': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_lswi.tif', - 'hash', - 'sha256', - '8de7378cdba830171c8da94424bf7807fa6c8b12523e8b34175a4b17ec770a5d'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_lswi.tif', - 'hash', - 'sha256', - '62de4bdafcc276e9c1c343f4e63f5dd06167e87f7712796f991e3a957f2b71de'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_lswi.tif', - 'hash', - 'sha256', - 'add051ef35d7ffff5eb68c48f8b8071f9cb493b5d3426867605d4c61746e3a26')], - - # t_process[vari] recording: - 'vari': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_vari.tif', - 'hash', - 'sha256', - '732d6d3d9fbe0e1b4616a3f8a91bf12d532598a0a02e8aab16c9a01c3ac56913'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_vari.tif', - 'hash', - 'sha256', - '9875717cc5e1926f11378bbf16345dbade5a2badf9f3dd06844a77f94d80ef72'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_vari.tif', - 'hash', - 'sha256', - '70fd19862ef4145c24f26a8a74cb748279b88fd973de935726ebef2294e17edd')], - - # t_process[evi] recording: - 'evi': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_evi.tif', - 'hash', - 'sha256', - '13548ae74456f08e6add657a637ca364626eafd2cc767724f57aac8ff93005bb'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_evi.tif', - 'hash', - 'sha256', - '7b926f7004cd92f2ec604fc4e2cc1489f60a335e39f4954030332a012fd33f87'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_evi.tif', - 'hash', - 'sha256', - '51db813a857f795817f529edd56d7134004566be46edab8a8dadc81286fb895f')], - - # t_process[clouds] recording: - 'clouds': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD_clouds.tif', - 'hash', - 'sha256', - '26b3e7e695f08ec2e9fb26985f9d5e392f9e8cedc331ab6721ea8e48821d93f0'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD_clouds.tif', - 'hash', - 'sha256', - 'c43eec8178d0521d20e2a5b6e9b61525007d80702c6dd705a3e52ed3a76a7b73'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_clouds.tif', - 'hash', - 'sha256', - '24a6a7050c866eed989530bec45b72004097f1e36fcc0e686652b87723a86de4')], - - # t_process[temp] recording: - 'temp': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_temp.tif', - 'hash', - 'sha256', - 'ead2622531929bba3192b8a6b609d5d920e79394ec577c5245d74d69593d827c'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '5d498bad2779f84289dfb75b14345d64d16b5bc7994b4aca6cd694f70b86064d'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '097f341fa917a0ec316c37d7907ff15b754dcd0925469be586016137ca47575f')], - - # t_process[sti] recording: - 'sti': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_sti.tif', - 'hash', - 'sha256', - '24e50766ec18826f30bbad07bc8f5a34e72f1966b325cc23f064aa5fd35dc24a'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_sti.tif', - 'hash', - 'sha256', - 'c223de9d00e9cfedc0cfc13a45163535f3c59e5587642fa2c725b00b17b7437c'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_sti.tif', - 'hash', - 'sha256', - '73c247fd2c7623e79145b6337846c97d14d1752d660b2f32245a4ccaac033350')], - - # t_process[crc] recording: - 'crc': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crc.tif', - 'hash', - 'sha256', - '07debf171aedfeba7d6c1e3220ff8e4a2a7b1033e68c72bbc121b500b6ed3c9c'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crc.tif', - 'hash', - 'sha256', - 'a7b61aabc950be7a3040b6f9572bf755522b12ad27644583e603def05ac1980b'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crc.tif', - 'hash', - 'sha256', - '942efeeee2b020123be6d7334bbb3ef5995c93d2fd0247cb79948836865c8067')], -} - # trailing whitespace and other junk characters are in current output t_info = { 'stdout': u"""\x1b[1mGIPS Data Repositories (v0.8.2)\x1b[0m \x1b[1m diff --git a/gips/test/sys/t_merra.py b/gips/test/sys/t_merra.py index 83ff4762..14f059fa 100644 --- a/gips/test/sys/t_merra.py +++ b/gips/test/sys/t_merra.py @@ -44,21 +44,14 @@ def t_inventory(setup_fixture, repo_env, expected): actual = repo_env.run('gips_inventory', *STD_ARGS) assert expected == actual -from .expected import merra as expectations - -@pytest.mark.parametrize("product", expectations.t_process.keys()) -def t_process(setup_fixture, repo_wrapper, product): - """Test gips_process on merra data.""" - record_mode, expected, runner = repo_wrapper - outcome, actual = runner('gips_process', *(STD_ARGS + ('-p', product))) - if not record_mode: # don't evaluate assertions when in record-mode - assert outcome.exit_code == 0 and expected == actual def t_info(repo_env, expected): """Test `gips_info {driver}` and confirm recorded output is given.""" actual = repo_env.run('gips_info', driver) assert expected == actual +from .expected import merra as expectations + @pytest.mark.parametrize("product", expectations.t_process.keys()) # TODO <-- def t_project(setup_fixture, export_wrapper, product): """Test gips_project merra with warping.""" diff --git a/gips/test/sys/t_modis.py b/gips/test/sys/t_modis.py index bef0b5c6..0947ad23 100644 --- a/gips/test/sys/t_modis.py +++ b/gips/test/sys/t_modis.py @@ -40,29 +40,18 @@ def setup_modis_data(pytestconfig): raise RuntimeError("MODIS data setup via `gips_inventory` failed", outcome.std_out, outcome.std_err, outcome) - def t_inventory(setup_modis_data, repo_env, expected): """Test `gips_inventory modis` and confirm recorded output is given.""" actual = repo_env.run('gips_inventory', *STD_ARGS) assert expected == actual -from .expected import modis as expectations - -@pytest.mark.parametrize("product", expectations.t_process.keys()) -def t_process(setup_modis_data, repo_wrapper, product): - """Test gips_process on modis data.""" - record_mode, expected, runner = repo_wrapper - outcome, actual = runner('gips_process', 'modis', '-s', NH_SHP_PATH, - '-d', '2012-12-01,2012-12-03', '-v', '4', '-p', - product) - if not record_mode: # don't evaluate assertions when in record-mode - assert outcome.exit_code == 0 and expected == actual - def t_info(repo_env, expected): """Test `gips_info modis` and confirm recorded output is given.""" actual = repo_env.run('gips_info', 'modis') assert expected == actual +from .expected import modis as expectations + @pytest.mark.parametrize("product", expectations.t_project.keys()) def t_project(setup_modis_data, export_wrapper, product): """Test gips_project modis with warping.""" diff --git a/gips/test/sys/t_process.py b/gips/test/sys/t_process.py index 98083afc..3bddf3da 100644 --- a/gips/test/sys/t_process.py +++ b/gips/test/sys/t_process.py @@ -5,26 +5,12 @@ import driver_setup from .expected import modis as expectations -params = [] # (driver, product) +params = [] # (driver, product),... +expectations = {} # 'driver': {'product': [ (path, type, data...),...]...} -params += [('modis', p) for p in ( - 'ndvi8', 'temp8td', 'temp8tn', #Terra 8-day Products - 'quality', #Nadir BRDF-Adjusted 16-day Products' - 'clouds', #Standard Products' - 'fsnow', 'obstime', 'snow', 'temp', #Terra/Aqua Daily Products' - #Index Products' - 'bi', 'brgt', 'crc', 'crcm', 'evi', 'isti', 'lswi', - 'msavi2', 'ndsi', 'ndti', 'ndvi', 'satvi', 'sti', 'vari', - #'indices', deprecated; not testing as a result - #'landcover', is annual, not available for the scene under test -)] - -#params += [('modis', p) for p in ( -#)] - -expectations = { - 'modis': { +expectations['modis'] = { # 'landcover' [], # is annual, not available for the scene under test + # 'indices', deprecated; not testing as a result # weird symlink products: # t_process[quality] recording: 'quality': @@ -347,13 +333,73 @@ 'hash', 'sha256', '942efeeee2b020123be6d7334bbb3ef5995c93d2fd0247cb79948836865c8067')], - }, } +expectations['merra'] = { + # test this too? 'frland': [], + # t_process[srad] recording: + 'srad': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_srad.tif', + 'hash', + 'sha256', + '882a44af70e337bf905c8695936f101792a518507a4304b23781a2f30aaabab4')], + # t_process[tave] recording: + 'tave': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tave.tif', + 'hash', + 'sha256', + '5185aebd7cda54157cad2ddbde9f6fec4871c1b85fe78d0738634ba0211f2c9b')], + # t_process[shum] recording: + 'shum': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_shum.tif', + 'hash', + 'sha256', + '558f567c1e931891553d396f81b2b90929ad48a0ae88fa95d3894eae23ab3eba')], + # t_process[rhum] recording: + 'rhum': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_rhum.tif', + 'hash', + 'sha256', + '058161eb8488f1fb3df7f5c8719833c9d68c56aa082d4605938f62c2ee0035f6')], + # t_process[tmin] recording: + 'tmin': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmin.tif', + 'hash', + 'sha256', + '35a850769a3bb8f5209574ea4835bf417d8f99024567f4af9cdee9e623d0c567')], + # t_process[tmax] recording: + 'tmax': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmax.tif', + 'hash', + 'sha256', + '5963fcf346f388fc347355153996a078f1eb5ecbbc094394b72be382ba491865')], + # t_process[prcp] recording: + 'prcp': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_prcp.tif', + 'hash', + 'sha256', + '348fd7072ef0a8625f5268e29acbf3a69a959a412327102ef6558c992e62bc9f')], + # t_process[patm] recording: + 'patm': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_patm.tif', + 'hash', + 'sha256', + '7d9deca613973cb8ffb8f15a3dfa2013e6783556b520915e89b97fe41de638e3')], + # t_process[wind] recording: + 'wind': + [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_wind.tif', + 'hash', + 'sha256', + 'a90c1d87fdb1c0926bd5b3004408a1f9d42ef08c38deb9d0572bc7536dbbf08a')], +} + +params += [('modis', p) for p in expectations['modis'].keys()] +params += [('merra', p) for p in expectations['merra'].keys()] + @pytest.mark.parametrize("driver, product", params) def t_process(repo_wrapper, driver, product): """Test gips_process output.""" - record_mode, _, runner = repo_wrapper + record_mode, runner = repo_wrapper driver_setup.setup_repo_data(driver) args = ('gips_process',) + driver_setup.STD_ARGS[driver] + ('-p', product) outcome, actual = runner(*args) diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index f6d60eda..b65c793a 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -297,11 +297,15 @@ def sys_test_wrapper(request, path): `request` is a pytest request object probably passed in to a fixture that is calling this function. Yields to let some process work, then reports on files created in the path during the wait. + + Presently depends on the module of the test having an expectations name + that is a dict of dicts, and the test getting a 'driver' and 'product' + param. """ rp = record_path() product = request.getfixturevalue('product') - expectations = load_expectation_module(request.module.__name__) - expected = getattr(expectations, request.function.__name__)[product] + p = request.node.callspec.params + expected = request.module.expectations[p['driver']][p['product']] expected_filenames = [e[0] for e in expected] if not rp: @@ -319,7 +323,7 @@ def wrapped_runner(cmd_string, *args): return outcome, [generate_expectation(fn, path) for fn in expected_filenames] - yield bool(rp), expected, wrapped_runner + yield bool(rp), wrapped_runner if rp: final_files = find_files(path) From 36125dae8f14df526589840269cef08d18c94ef5 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Tue, 26 Dec 2017 15:24:00 -0500 Subject: [PATCH 15/44] 483 refactor t_project into its own file: Can accomodate multiple drivers now as t_process does. --- gips/test/sys/expected/merra.py | 66 ----- gips/test/sys/expected/modis.py | 323 ------------------------- gips/test/sys/t_merra.py | 14 +- gips/test/sys/t_modis.py | 14 +- gips/test/sys/t_project.py | 413 ++++++++++++++++++++++++++++++++ 5 files changed, 417 insertions(+), 413 deletions(-) create mode 100644 gips/test/sys/t_project.py diff --git a/gips/test/sys/expected/merra.py b/gips/test/sys/expected/merra.py index bafe9086..6d8b37d1 100644 --- a/gips/test/sys/expected/merra.py +++ b/gips/test/sys/expected/merra.py @@ -40,72 +40,6 @@ """ } -t_project = { - - # t_project[srad] recording: - 'srad': - [('output/0/2015135_merra_srad.tif', - 'hash', - 'sha256', - 'bfc0b906f7299ff8e6fcbbe474de0fb7b877d36f2e619e2e6ac10bd84ac5b55d')], - - # t_project[tave] recording: - 'tave': - [('output/0/2015135_merra_tave.tif', - 'hash', - 'sha256', - 'c96d89ebf5d7ec90193c8a113609bfc39473d72041b6145a8cd41ee49e4f6a4d')], - - # t_project[prcp] recording: - 'prcp': - [('output/0/2015135_merra_prcp.tif', - 'hash', - 'sha256', - 'b7e68237ee3557c421856203cc22807f1ce74790ea2948f8d095cbf8a7f18a53')], - - # t_project[rhum] recording: - 'rhum': - [('output/0/2015135_merra_rhum.tif', - 'hash', - 'sha256', - '3b45ba1cd87d4781ff95b08562a00189506c76add24156f8e3719ff8872c3d3a')], - - # t_project[tmin] recording: - 'tmin': - [('output/0/2015135_merra_tmin.tif', - 'hash', - 'sha256', - '253e37e9b2c03a77a0a4fdfe5aaa0e3ff025471f64fd28d6ffb33dd8ecdd68a9')], - - # t_project[tmax] recording: - 'tmax': - [('output/0/2015135_merra_tmax.tif', - 'hash', - 'sha256', - '8371d473f0e506e6ab0fe872542b8cd0d65bc2ed2b37bbeb747c966c41e9c232')], - - # t_project[shum] recording: - 'shum': - [('output/0/2015135_merra_shum.tif', - 'hash', - 'sha256', - '208465fbf10da2e935807e00991fa6375c3c6b67d9950362a9ffe502d7690fd6')], - - # t_project[patm] recording: - 'patm': - [('output/0/2015135_merra_patm.tif', - 'hash', - 'sha256', - '5778a4d89b22ea76dc25654c8092a61bbb09634f06f66b578460a93444e513a7')], - - # t_project[wind] recording: - 'wind': - [('output/0/2015135_merra_wind.tif', - 'hash', - 'sha256', - '88029a425fbeb59149a29be793087b51ea46583bb89e62c1baca629ccfc3a36a')], -} - t_project_no_warp = { 'created': { '0': None, diff --git a/gips/test/sys/expected/modis.py b/gips/test/sys/expected/modis.py index a116c629..503955f9 100644 --- a/gips/test/sys/expected/modis.py +++ b/gips/test/sys/expected/modis.py @@ -51,329 +51,6 @@ landcover MCD Annual Land Cover """} -t_project = { - # t_project[satvi] recording: - 'satvi': - [('output/0/2012336_MCD_satvi.tif', - 'hash', - 'sha256', - '893d968ff6359d0d34f8a587246690d1c724832505d66fbee8be8c1a43a6e3df'), - ('output/0/2012337_MCD_satvi.tif', - 'hash', - 'sha256', - '6092d7cfb609f000124b8ef107f278decaa27c3cdb69696bd094940ead97d96e'), - ('output/0/2012338_MCD_satvi.tif', - 'hash', - 'sha256', - '37dd32e0f5e761846f44615ec6e82e2cce8d6c98197f0bd721dcfb266d963b6b')], - - # t_project[ndti] recording: - 'ndti': - [('output/0/2012337_MCD_ndti.tif', - 'hash', - 'sha256', - 'db85948227815660b94d570adf0f1119d9f1001284c3de6c8b219c6455bd3c4e'), - ('output/0/2012338_MCD_ndti.tif', - 'hash', - 'sha256', - '5028c194e41d8c07076cee09631b276ef7b86c3c45b0a4f705f5e4c6b72c2812'), - ('output/0/2012336_MCD_ndti.tif', - 'hash', - 'sha256', - '157134ea0e9cbb0b2fcdb4203e17e82ae906a5779eb1f735e8c55d1f60c231e1')], - - # t_project[ndvi] recording: - 'ndvi': - [('output/0/2012337_MCD_ndvi.tif', - 'hash', - 'sha256', - '28a288720a0846348fbd6315e5d9e3801ad76eb2c873dc028e7a52d1fdcdad10'), - ('output/0/2012336_MCD_ndvi.tif', - 'hash', - 'sha256', - '78e090b1cc92304c666a8e480d11fa99985ee3d2113c4fe3be29fb85910b86b9'), - ('output/0/2012338_MCD_ndvi.tif', - 'hash', - 'sha256', - 'bbcee4ac1b49466c90264e305c20ce19a021fd177ffc35da5ed92a86555c480e')], - - # t_project[quality] recording: - 'quality': - [('output/0/2012338_MCD_quality.tif', - 'hash', - 'sha256', - '895abfc825d9da56c56202c670f94a16b28f2aaab5590db55ac6e903b6638e91'), - ('output/0/2012336_MCD_quality.tif', - 'hash', - 'sha256', - '4f40fdeb7ded507a68f404ec34a17fb6283c9b90d81ecccc7782746eaba85323'), - ('output/0/2012337_MCD_quality.tif', - 'hash', - 'sha256', - '5cdd315b2a423fbb82c19cfe859325c889a5d05cf0c3cdf71359b257524175a3')], - - # t_project[isti] recording: - 'isti': - [('output/0/2012337_MCD_isti.tif', - 'hash', - 'sha256', - '391367ee5bdd0a96435c09df85db312919121c5d9a28b5bda59a68db7b609871'), - ('output/0/2012338_MCD_isti.tif', - 'hash', - 'sha256', - 'cc082d45a2a4dd4a7e53d0970029decf24ce5033b9715930cd250b524ad87757'), - ('output/0/2012336_MCD_isti.tif', - 'hash', - 'sha256', - '16be9705c06a44f3a35e684cded2d2ceb26213510eaf0ccfbafb62ebc39bb153')], - - # t_project[ndvi8] recording: - 'ndvi8': - [('output/0/2012337_MOD_ndvi8.tif', - 'hash', - 'sha256', - '23a72df39da253ac07de5a9e98c88f40e3335f3b1b779ccf91153d1d5573a780')], - - # t_project[snow] recording: - 'snow': - [('output/0/2012336_MCD_snow.tif', - 'hash', - 'sha256', - 'b3eeca8fc933d2f43c2d96e869956447d9556737181635167744d0d420dfd813'), - ('output/0/2012338_MCD_snow.tif', - 'hash', - 'sha256', - '98a38e1a75fcea269bd75a952cde9d84af86b14ddb928e4777ff812dba8befbe'), - ('output/0/2012337_MCD_snow.tif', - 'hash', - 'sha256', - 'da26696be559bb79c399ad18942ac629a9604d857dd7fc461fd76c4577a06303')], - - # t_project[fsnow] recording: - 'fsnow': - [('output/0/2012338_MCD_fsnow.tif', - 'hash', - 'sha256', - 'f2efdf504e004caf05563f3402fdde7e771351d7d93de5970af0a54d0b5bd2c2'), - ('output/0/2012336_MCD_fsnow.tif', - 'hash', - 'sha256', - 'a9d2ea29a6d77325246e0baaf298d16cb84d6003a19faed8ba162b475703f563'), - ('output/0/2012337_MCD_fsnow.tif', - 'hash', - 'sha256', - 'e8465aa3764c60b21a943f3e078d93c0484272292f075455693ea2084971ace4')], - - # t_project[crcm] recording: - 'crcm': - [('output/0/2012337_MCD_crcm.tif', - 'hash', - 'sha256', - 'c00e7b7c6970ae23904ee52afd614a8871f3d771d26830a2aef412163d9d9228'), - ('output/0/2012338_MCD_crcm.tif', - 'hash', - 'sha256', - 'ed336a3ba4cf482936ca781ea1a5e0f04ca5ad68919c3dbe6e117edac55ec242'), - ('output/0/2012336_MCD_crcm.tif', - 'hash', - 'sha256', - 'f217a9278b195e945e2337663cf7dfe2b09faa0dc68062fbc3614a6954866eaf')], - - # t_project[ndsi] recording: - 'ndsi': - [('output/0/2012338_MCD_ndsi.tif', - 'hash', - 'sha256', - '8e26a8f6d78b9612c8cd3924936c5944f56c67ae1c63f6541d045b41a27baafe'), - ('output/0/2012337_MCD_ndsi.tif', - 'hash', - 'sha256', - '2aaea763ab1d5b54d7079e7ee2381bd987a3c72c5cbbd6f18f97aa320c186e2a'), - ('output/0/2012336_MCD_ndsi.tif', - 'hash', - 'sha256', - '83b6bbf241d080cf6860798da299038be5e6d3e20a8b544e5af51e2eefd3a5c9')], - - # t_project[brgt] recording: - 'brgt': - [('output/0/2012337_MCD_brgt.tif', - 'hash', - 'sha256', - 'fc385e6e42538a09655762d1b2557abe35af38aecef1571705cec8b971efbe4d'), - ('output/0/2012338_MCD_brgt.tif', - 'hash', - 'sha256', - 'a8f8ef66bc87c2370bd61d5eed64905ba4f832d05579687478ff5f38c65e8038'), - ('output/0/2012336_MCD_brgt.tif', - 'hash', - 'sha256', - '24f8130de39daac6e181cb2f98b48d9a14412700632d3cbd2ffbe2704283b740')], - - # t_project[msavi2] recording: - 'msavi2': - [('output/0/2012336_MCD_msavi2.tif', - 'hash', - 'sha256', - '997b562420fe223677881d577cdb2100ee653ee932b9989b95f3089c0fe38352'), - ('output/0/2012338_MCD_msavi2.tif', - 'hash', - 'sha256', - '26a7ab45723856361d108064fe0c810535de75f272a733d86f66eeb8146dee35'), - ('output/0/2012337_MCD_msavi2.tif', - 'hash', - 'sha256', - 'ee7b2006b79f65bdf11d77f678d66765abfcc024b7856ab296f855570efb9940')], - - # t_project[bi] recording: - 'bi': - [('output/0/2012336_MCD_bi.tif', - 'hash', - 'sha256', - 'e5f891a6290301508107714657274fc808fcf96cb138c652a9c22552675796a5'), - ('output/0/2012337_MCD_bi.tif', - 'hash', - 'sha256', - '58b9e3809630ba24879bb85e81bc6dc0eed548348373588d70d49de65d246582'), - ('output/0/2012338_MCD_bi.tif', - 'hash', - 'sha256', - '914ca6d15c25a22d3e0a43d0d266f72bd65d102b4d6b015a9f12b8ee3a116c13')], - - # t_project[obstime] recording: - 'obstime': - [('output/0/2012337_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - 'a9345d21f3d754d083d3b4d13b059f2ffe07eb42ea9a0d6f3e40ef326e4ea75f'), - ('output/0/2012336_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - 'fac305543c4afbf76611900579cd5f37d63e7a45814c997dce40fc46a74f13e5'), - ('output/0/2012338_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - 'e18456caaf9a2dd005cc6568f7aa77ac534bfa24ec364d3af28affb4ae652de9')], - - # t_project[lswi] recording: - 'lswi': - [('output/0/2012337_MCD_lswi.tif', - 'hash', - 'sha256', - '586dd3782070b983b9d383161bed82bee21c54ad312d1c993b0329965144deab'), - ('output/0/2012336_MCD_lswi.tif', - 'hash', - 'sha256', - '296c849e70d31127bcefef54ffed72832529f7ae60d50b7c639c5c6d253d30f4'), - ('output/0/2012338_MCD_lswi.tif', - 'hash', - 'sha256', - '992fa3d7b4bdc8cf95fba759a0096f9a913e84762717e93c477baae0d7d3c0ed')], - - # t_project[vari] recording: - 'vari': - [('output/0/2012336_MCD_vari.tif', - 'hash', - 'sha256', - 'be5d2273b9f0b9aca79df89b66ca0e7e1a2b8d386e6d34a6eca6eb37810b488d'), - ('output/0/2012338_MCD_vari.tif', - 'hash', - 'sha256', - '016ec5b1ee512d9306251a4c2b3630c90ea18723159a6a254c8e77c4e730abc8'), - ('output/0/2012337_MCD_vari.tif', - 'hash', - 'sha256', - 'a6c10e80436b92908637b22b40e7993a0a648e82b96993760a7b26ba43df0667')], - - # t_project[evi] recording: - 'evi': - [('output/0/2012338_MCD_evi.tif', - 'hash', - 'sha256', - '8a1beba0730ac7d113a7edb8b55035a742892bbe678cd0eaaa540e59906a6990'), - ('output/0/2012337_MCD_evi.tif', - 'hash', - 'sha256', - '824a5247b0e8e0730dec6e82b390533649aa796ef04ef4be27ac38cae9a24d04'), - ('output/0/2012336_MCD_evi.tif', - 'hash', - 'sha256', - '361798060d02c97153a9828c1f8ac6b7df396070ec243932e9ed18fd472e9669')], - - # t_project[temp8tn] recording: - 'temp8tn': - [('output/0/2012337_MOD_temp8tn.tif', - 'hash', - 'sha256', - '1600d412e2590b4f4006ac0d98ebf9c6bf83fe16967c2fd721e412d2a14040c6')], - - # t_project[clouds] recording: - 'clouds': - [('output/0/2012337_MOD_clouds.tif', - 'hash', - 'sha256', - '8b13afeec6d1a5612a3eeb51896507e2d4eeb851261881f3854d2b804e97835f'), - ('output/0/2012336_MOD_clouds.tif', - 'hash', - 'sha256', - 'cb4fab2519652bebe4830479c95425050f63d519397beb5b0c40349cc6f49372'), - ('output/0/2012338_MOD_clouds.tif', - 'hash', - 'sha256', - '54505d6d4063caa27fda74f5fdbb916c20d281f6e15deea28596c379ffa661e2')], - - # t_project[temp] recording: - 'temp': - [('output/0/2012336_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '614023db948d0b0249057ca9c458356c2222e45224a726f87a7cf512cd87745c'), - ('output/0/2012338_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '51864699b8347a30d9669ec41298665cd893a27516089cc21b4c3f03b3e65b2b'), - ('output/0/2012337_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '29373079984e8aaf55add6a1fdd7bd766eb89a1c74ebc85674da54aeb643e3ef')], - - # t_project[temp8td] recording: - 'temp8td': - [('output/0/2012337_MOD_temp8td.tif', - 'hash', - 'sha256', - 'eaa0ff94bb34cbad8d9c187b16c7cd2bd5948d1d94cf1caf30b0396ac7e8a735')], - - # t_project[sti] recording: - 'sti': - [('output/0/2012337_MCD_sti.tif', - 'hash', - 'sha256', - '9850777cae46b42fa3c8361849529e6f230dd1ce86fb6c6676aa8eb2768e71d5'), - ('output/0/2012336_MCD_sti.tif', - 'hash', - 'sha256', - 'b7fde6a04ac091946553b25b1a26de806fa7f9c23e915b415034ef811aca1d16'), - ('output/0/2012338_MCD_sti.tif', - 'hash', - 'sha256', - 'b2714cadf4e779c433415f194f366ac87877c60b383f46971843b4ab3a610fa9')], - - # t_project[crc] recording: - 'crc': - [('output/0/2012338_MCD_crc.tif', - 'hash', - 'sha256', - 'dc9a4885e694fe7117cf10df11b5589745b4eb119d5f58f9d1c14e27d0eda333'), - ('output/0/2012337_MCD_crc.tif', - 'hash', - 'sha256', - '5688d80a613c28c657237186291a88ae786037a77dad0247c0bd734da214c927'), - ('output/0/2012336_MCD_crc.tif', - 'hash', - 'sha256', - '3df0003b6153970e73c46d5e999697ce5af268744274ccd3a4a2c14e3ae932cb')], -} - t_project_two_runs = t_project t_project_no_warp = { 'created': { diff --git a/gips/test/sys/t_merra.py b/gips/test/sys/t_merra.py index 14f059fa..18f56fad 100644 --- a/gips/test/sys/t_merra.py +++ b/gips/test/sys/t_merra.py @@ -50,18 +50,6 @@ def t_info(repo_env, expected): actual = repo_env.run('gips_info', driver) assert expected == actual -from .expected import merra as expectations - -@pytest.mark.parametrize("product", expectations.t_process.keys()) # TODO <-- -def t_project(setup_fixture, export_wrapper, product): - """Test gips_project merra with warping.""" - record_mode, expected, runner = export_wrapper - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, - '--notld', '-p', product) - outcome, actual = runner('gips_project', *args) - if not record_mode: # don't evaluate assertions when in record-mode - assert outcome.exit_code == 0 and expected == actual - # TODO revive this test? ''' def t_project_no_warp(setup_fixture, clean_repo_env, output_tfe, expected): @@ -88,6 +76,8 @@ def t_tiles_copy(setup_fixture, clean_repo_env, output_tfe, expected): actual = output_tfe.run('gips_tiles', *args) assert expected == actual +from .expected import merra as expectations + @pytest.mark.parametrize("product", expectations.t_stats.keys()) def t_stats(setup_fixture, export_wrapper, product): """Test gips_stats on projected files.""" diff --git a/gips/test/sys/t_modis.py b/gips/test/sys/t_modis.py index 0947ad23..0b518045 100644 --- a/gips/test/sys/t_modis.py +++ b/gips/test/sys/t_modis.py @@ -50,18 +50,6 @@ def t_info(repo_env, expected): actual = repo_env.run('gips_info', 'modis') assert expected == actual -from .expected import modis as expectations - -@pytest.mark.parametrize("product", expectations.t_project.keys()) -def t_project(setup_modis_data, export_wrapper, product): - """Test gips_project modis with warping.""" - record_mode, expected, runner = export_wrapper - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, - '--notld', '-p', product) - outcome, actual = runner('gips_project', *args) - if not record_mode: # don't evaluate assertions when in record-mode - assert outcome.exit_code == 0 and expected == actual - # TODO keep this test? ''' def t_project_no_warp(setup_modis_data, clean_repo_env, output_tfe, expected): @@ -86,6 +74,8 @@ def t_tiles_copy(setup_modis_data, clean_repo_env, output_tfe, expected): actual = output_tfe.run('gips_tiles', *args) assert expected == actual +from .expected import modis as expectations + @pytest.mark.parametrize("product", expectations.t_stats.keys()) def t_stats(setup_modis_data, export_wrapper, product): """Test gips_stats on projected files.""" diff --git a/gips/test/sys/t_project.py b/gips/test/sys/t_project.py new file mode 100644 index 00000000..6895580e --- /dev/null +++ b/gips/test/sys/t_project.py @@ -0,0 +1,413 @@ +import pytest + +import util +from util import export_wrapper +import driver_setup +from .expected import modis as expectations + +params = [] # (driver, product),... +expectations = {} # 'driver': {'product': [ (path, type, data...),...]...} + +expectations['modis'] = { + # t_project[satvi] recording: + 'satvi': + [('output/0/2012336_MCD_satvi.tif', + 'hash', + 'sha256', + '893d968ff6359d0d34f8a587246690d1c724832505d66fbee8be8c1a43a6e3df'), + ('output/0/2012337_MCD_satvi.tif', + 'hash', + 'sha256', + '6092d7cfb609f000124b8ef107f278decaa27c3cdb69696bd094940ead97d96e'), + ('output/0/2012338_MCD_satvi.tif', + 'hash', + 'sha256', + '37dd32e0f5e761846f44615ec6e82e2cce8d6c98197f0bd721dcfb266d963b6b')], + + # t_project[ndti] recording: + 'ndti': + [('output/0/2012337_MCD_ndti.tif', + 'hash', + 'sha256', + 'db85948227815660b94d570adf0f1119d9f1001284c3de6c8b219c6455bd3c4e'), + ('output/0/2012338_MCD_ndti.tif', + 'hash', + 'sha256', + '5028c194e41d8c07076cee09631b276ef7b86c3c45b0a4f705f5e4c6b72c2812'), + ('output/0/2012336_MCD_ndti.tif', + 'hash', + 'sha256', + '157134ea0e9cbb0b2fcdb4203e17e82ae906a5779eb1f735e8c55d1f60c231e1')], + + # t_project[ndvi] recording: + 'ndvi': + [('output/0/2012337_MCD_ndvi.tif', + 'hash', + 'sha256', + '28a288720a0846348fbd6315e5d9e3801ad76eb2c873dc028e7a52d1fdcdad10'), + ('output/0/2012336_MCD_ndvi.tif', + 'hash', + 'sha256', + '78e090b1cc92304c666a8e480d11fa99985ee3d2113c4fe3be29fb85910b86b9'), + ('output/0/2012338_MCD_ndvi.tif', + 'hash', + 'sha256', + 'bbcee4ac1b49466c90264e305c20ce19a021fd177ffc35da5ed92a86555c480e')], + + # t_project[quality] recording: + 'quality': + [('output/0/2012338_MCD_quality.tif', + 'hash', + 'sha256', + '895abfc825d9da56c56202c670f94a16b28f2aaab5590db55ac6e903b6638e91'), + ('output/0/2012336_MCD_quality.tif', + 'hash', + 'sha256', + '4f40fdeb7ded507a68f404ec34a17fb6283c9b90d81ecccc7782746eaba85323'), + ('output/0/2012337_MCD_quality.tif', + 'hash', + 'sha256', + '5cdd315b2a423fbb82c19cfe859325c889a5d05cf0c3cdf71359b257524175a3')], + + # t_project[isti] recording: + 'isti': + [('output/0/2012337_MCD_isti.tif', + 'hash', + 'sha256', + '391367ee5bdd0a96435c09df85db312919121c5d9a28b5bda59a68db7b609871'), + ('output/0/2012338_MCD_isti.tif', + 'hash', + 'sha256', + 'cc082d45a2a4dd4a7e53d0970029decf24ce5033b9715930cd250b524ad87757'), + ('output/0/2012336_MCD_isti.tif', + 'hash', + 'sha256', + '16be9705c06a44f3a35e684cded2d2ceb26213510eaf0ccfbafb62ebc39bb153')], + + # t_project[ndvi8] recording: + 'ndvi8': + [('output/0/2012337_MOD_ndvi8.tif', + 'hash', + 'sha256', + '23a72df39da253ac07de5a9e98c88f40e3335f3b1b779ccf91153d1d5573a780')], + + # t_project[snow] recording: + 'snow': + [('output/0/2012336_MCD_snow.tif', + 'hash', + 'sha256', + 'b3eeca8fc933d2f43c2d96e869956447d9556737181635167744d0d420dfd813'), + ('output/0/2012338_MCD_snow.tif', + 'hash', + 'sha256', + '98a38e1a75fcea269bd75a952cde9d84af86b14ddb928e4777ff812dba8befbe'), + ('output/0/2012337_MCD_snow.tif', + 'hash', + 'sha256', + 'da26696be559bb79c399ad18942ac629a9604d857dd7fc461fd76c4577a06303')], + + # t_project[fsnow] recording: + 'fsnow': + [('output/0/2012338_MCD_fsnow.tif', + 'hash', + 'sha256', + 'f2efdf504e004caf05563f3402fdde7e771351d7d93de5970af0a54d0b5bd2c2'), + ('output/0/2012336_MCD_fsnow.tif', + 'hash', + 'sha256', + 'a9d2ea29a6d77325246e0baaf298d16cb84d6003a19faed8ba162b475703f563'), + ('output/0/2012337_MCD_fsnow.tif', + 'hash', + 'sha256', + 'e8465aa3764c60b21a943f3e078d93c0484272292f075455693ea2084971ace4')], + + # t_project[crcm] recording: + 'crcm': + [('output/0/2012337_MCD_crcm.tif', + 'hash', + 'sha256', + 'c00e7b7c6970ae23904ee52afd614a8871f3d771d26830a2aef412163d9d9228'), + ('output/0/2012338_MCD_crcm.tif', + 'hash', + 'sha256', + 'ed336a3ba4cf482936ca781ea1a5e0f04ca5ad68919c3dbe6e117edac55ec242'), + ('output/0/2012336_MCD_crcm.tif', + 'hash', + 'sha256', + 'f217a9278b195e945e2337663cf7dfe2b09faa0dc68062fbc3614a6954866eaf')], + + # t_project[ndsi] recording: + 'ndsi': + [('output/0/2012338_MCD_ndsi.tif', + 'hash', + 'sha256', + '8e26a8f6d78b9612c8cd3924936c5944f56c67ae1c63f6541d045b41a27baafe'), + ('output/0/2012337_MCD_ndsi.tif', + 'hash', + 'sha256', + '2aaea763ab1d5b54d7079e7ee2381bd987a3c72c5cbbd6f18f97aa320c186e2a'), + ('output/0/2012336_MCD_ndsi.tif', + 'hash', + 'sha256', + '83b6bbf241d080cf6860798da299038be5e6d3e20a8b544e5af51e2eefd3a5c9')], + + # t_project[brgt] recording: + 'brgt': + [('output/0/2012337_MCD_brgt.tif', + 'hash', + 'sha256', + 'fc385e6e42538a09655762d1b2557abe35af38aecef1571705cec8b971efbe4d'), + ('output/0/2012338_MCD_brgt.tif', + 'hash', + 'sha256', + 'a8f8ef66bc87c2370bd61d5eed64905ba4f832d05579687478ff5f38c65e8038'), + ('output/0/2012336_MCD_brgt.tif', + 'hash', + 'sha256', + '24f8130de39daac6e181cb2f98b48d9a14412700632d3cbd2ffbe2704283b740')], + + # t_project[msavi2] recording: + 'msavi2': + [('output/0/2012336_MCD_msavi2.tif', + 'hash', + 'sha256', + '997b562420fe223677881d577cdb2100ee653ee932b9989b95f3089c0fe38352'), + ('output/0/2012338_MCD_msavi2.tif', + 'hash', + 'sha256', + '26a7ab45723856361d108064fe0c810535de75f272a733d86f66eeb8146dee35'), + ('output/0/2012337_MCD_msavi2.tif', + 'hash', + 'sha256', + 'ee7b2006b79f65bdf11d77f678d66765abfcc024b7856ab296f855570efb9940')], + + # t_project[bi] recording: + 'bi': + [('output/0/2012336_MCD_bi.tif', + 'hash', + 'sha256', + 'e5f891a6290301508107714657274fc808fcf96cb138c652a9c22552675796a5'), + ('output/0/2012337_MCD_bi.tif', + 'hash', + 'sha256', + '58b9e3809630ba24879bb85e81bc6dc0eed548348373588d70d49de65d246582'), + ('output/0/2012338_MCD_bi.tif', + 'hash', + 'sha256', + '914ca6d15c25a22d3e0a43d0d266f72bd65d102b4d6b015a9f12b8ee3a116c13')], + + # t_project[obstime] recording: + 'obstime': + [('output/0/2012337_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'a9345d21f3d754d083d3b4d13b059f2ffe07eb42ea9a0d6f3e40ef326e4ea75f'), + ('output/0/2012336_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'fac305543c4afbf76611900579cd5f37d63e7a45814c997dce40fc46a74f13e5'), + ('output/0/2012338_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'e18456caaf9a2dd005cc6568f7aa77ac534bfa24ec364d3af28affb4ae652de9')], + + # t_project[lswi] recording: + 'lswi': + [('output/0/2012337_MCD_lswi.tif', + 'hash', + 'sha256', + '586dd3782070b983b9d383161bed82bee21c54ad312d1c993b0329965144deab'), + ('output/0/2012336_MCD_lswi.tif', + 'hash', + 'sha256', + '296c849e70d31127bcefef54ffed72832529f7ae60d50b7c639c5c6d253d30f4'), + ('output/0/2012338_MCD_lswi.tif', + 'hash', + 'sha256', + '992fa3d7b4bdc8cf95fba759a0096f9a913e84762717e93c477baae0d7d3c0ed')], + + # t_project[vari] recording: + 'vari': + [('output/0/2012336_MCD_vari.tif', + 'hash', + 'sha256', + 'be5d2273b9f0b9aca79df89b66ca0e7e1a2b8d386e6d34a6eca6eb37810b488d'), + ('output/0/2012338_MCD_vari.tif', + 'hash', + 'sha256', + '016ec5b1ee512d9306251a4c2b3630c90ea18723159a6a254c8e77c4e730abc8'), + ('output/0/2012337_MCD_vari.tif', + 'hash', + 'sha256', + 'a6c10e80436b92908637b22b40e7993a0a648e82b96993760a7b26ba43df0667')], + + # t_project[evi] recording: + 'evi': + [('output/0/2012338_MCD_evi.tif', + 'hash', + 'sha256', + '8a1beba0730ac7d113a7edb8b55035a742892bbe678cd0eaaa540e59906a6990'), + ('output/0/2012337_MCD_evi.tif', + 'hash', + 'sha256', + '824a5247b0e8e0730dec6e82b390533649aa796ef04ef4be27ac38cae9a24d04'), + ('output/0/2012336_MCD_evi.tif', + 'hash', + 'sha256', + '361798060d02c97153a9828c1f8ac6b7df396070ec243932e9ed18fd472e9669')], + + # t_project[temp8tn] recording: + 'temp8tn': + [('output/0/2012337_MOD_temp8tn.tif', + 'hash', + 'sha256', + '1600d412e2590b4f4006ac0d98ebf9c6bf83fe16967c2fd721e412d2a14040c6')], + + # t_project[clouds] recording: + 'clouds': + [('output/0/2012337_MOD_clouds.tif', + 'hash', + 'sha256', + '8b13afeec6d1a5612a3eeb51896507e2d4eeb851261881f3854d2b804e97835f'), + ('output/0/2012336_MOD_clouds.tif', + 'hash', + 'sha256', + 'cb4fab2519652bebe4830479c95425050f63d519397beb5b0c40349cc6f49372'), + ('output/0/2012338_MOD_clouds.tif', + 'hash', + 'sha256', + '54505d6d4063caa27fda74f5fdbb916c20d281f6e15deea28596c379ffa661e2')], + + # t_project[temp] recording: + 'temp': + [('output/0/2012336_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '614023db948d0b0249057ca9c458356c2222e45224a726f87a7cf512cd87745c'), + ('output/0/2012338_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '51864699b8347a30d9669ec41298665cd893a27516089cc21b4c3f03b3e65b2b'), + ('output/0/2012337_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '29373079984e8aaf55add6a1fdd7bd766eb89a1c74ebc85674da54aeb643e3ef')], + + # t_project[temp8td] recording: + 'temp8td': + [('output/0/2012337_MOD_temp8td.tif', + 'hash', + 'sha256', + 'eaa0ff94bb34cbad8d9c187b16c7cd2bd5948d1d94cf1caf30b0396ac7e8a735')], + + # t_project[sti] recording: + 'sti': + [('output/0/2012337_MCD_sti.tif', + 'hash', + 'sha256', + '9850777cae46b42fa3c8361849529e6f230dd1ce86fb6c6676aa8eb2768e71d5'), + ('output/0/2012336_MCD_sti.tif', + 'hash', + 'sha256', + 'b7fde6a04ac091946553b25b1a26de806fa7f9c23e915b415034ef811aca1d16'), + ('output/0/2012338_MCD_sti.tif', + 'hash', + 'sha256', + 'b2714cadf4e779c433415f194f366ac87877c60b383f46971843b4ab3a610fa9')], + + # t_project[crc] recording: + 'crc': + [('output/0/2012338_MCD_crc.tif', + 'hash', + 'sha256', + 'dc9a4885e694fe7117cf10df11b5589745b4eb119d5f58f9d1c14e27d0eda333'), + ('output/0/2012337_MCD_crc.tif', + 'hash', + 'sha256', + '5688d80a613c28c657237186291a88ae786037a77dad0247c0bd734da214c927'), + ('output/0/2012336_MCD_crc.tif', + 'hash', + 'sha256', + '3df0003b6153970e73c46d5e999697ce5af268744274ccd3a4a2c14e3ae932cb')], +} + +expectations['merra'] = { + # t_project[srad] recording: + 'srad': + [('output/0/2015135_merra_srad.tif', + 'hash', + 'sha256', + 'bfc0b906f7299ff8e6fcbbe474de0fb7b877d36f2e619e2e6ac10bd84ac5b55d')], + + # t_project[tave] recording: + 'tave': + [('output/0/2015135_merra_tave.tif', + 'hash', + 'sha256', + 'c96d89ebf5d7ec90193c8a113609bfc39473d72041b6145a8cd41ee49e4f6a4d')], + + # t_project[prcp] recording: + 'prcp': + [('output/0/2015135_merra_prcp.tif', + 'hash', + 'sha256', + 'b7e68237ee3557c421856203cc22807f1ce74790ea2948f8d095cbf8a7f18a53')], + + # t_project[rhum] recording: + 'rhum': + [('output/0/2015135_merra_rhum.tif', + 'hash', + 'sha256', + '3b45ba1cd87d4781ff95b08562a00189506c76add24156f8e3719ff8872c3d3a')], + + # t_project[tmin] recording: + 'tmin': + [('output/0/2015135_merra_tmin.tif', + 'hash', + 'sha256', + '253e37e9b2c03a77a0a4fdfe5aaa0e3ff025471f64fd28d6ffb33dd8ecdd68a9')], + + # t_project[tmax] recording: + 'tmax': + [('output/0/2015135_merra_tmax.tif', + 'hash', + 'sha256', + '8371d473f0e506e6ab0fe872542b8cd0d65bc2ed2b37bbeb747c966c41e9c232')], + + # t_project[shum] recording: + 'shum': + [('output/0/2015135_merra_shum.tif', + 'hash', + 'sha256', + '208465fbf10da2e935807e00991fa6375c3c6b67d9950362a9ffe502d7690fd6')], + + # t_project[patm] recording: + 'patm': + [('output/0/2015135_merra_patm.tif', + 'hash', + 'sha256', + '5778a4d89b22ea76dc25654c8092a61bbb09634f06f66b578460a93444e513a7')], + + # t_project[wind] recording: + 'wind': + [('output/0/2015135_merra_wind.tif', + 'hash', + 'sha256', + '88029a425fbeb59149a29be793087b51ea46583bb89e62c1baca629ccfc3a36a')], +} + + +params += [('modis', p) for p in expectations['modis'].keys()] +params += [('merra', p) for p in expectations['merra'].keys()] + +@pytest.mark.parametrize("driver, product", params) +def t_project(export_wrapper, driver, product): + """Test gips_project with warping.""" + record_mode, runner = export_wrapper + args = ('gips_project',) + driver_setup.STD_ARGS[driver] + ( + '--res', '100', '100', '--outdir', util.OUTPUT_DIR, '--notld', + '-p', product) + outcome, actual = runner(*args) + if not record_mode: # don't evaluate assertions when in record-mode + assert (outcome.exit_code == 0 + and expectations[driver][product] == actual) From f4fb599118b0708175a0348c3548edb702fdfb7b Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Tue, 26 Dec 2017 15:31:08 -0500 Subject: [PATCH 16/44] 483 for better readability put test at top of file --- gips/test/sys/expected/modis.py | 2 -- gips/test/sys/t_process.py | 23 +++++++++++------------ gips/test/sys/t_project.py | 24 ++++++++++++------------ 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/gips/test/sys/expected/modis.py b/gips/test/sys/expected/modis.py index 503955f9..f305b46a 100644 --- a/gips/test/sys/expected/modis.py +++ b/gips/test/sys/expected/modis.py @@ -51,8 +51,6 @@ landcover MCD Annual Land Cover """} -t_project_two_runs = t_project - t_project_no_warp = { 'created': { '0': None, # directory '0/2012336_MCD_fsnow.tif': -232655043, diff --git a/gips/test/sys/t_process.py b/gips/test/sys/t_process.py index 3bddf3da..67c8f034 100644 --- a/gips/test/sys/t_process.py +++ b/gips/test/sys/t_process.py @@ -3,11 +3,21 @@ import util from util import repo_wrapper import driver_setup -from .expected import modis as expectations params = [] # (driver, product),... expectations = {} # 'driver': {'product': [ (path, type, data...),...]...} +@pytest.mark.parametrize("driver, product", params) +def t_process(repo_wrapper, driver, product): + """Test gips_process output.""" + record_mode, runner = repo_wrapper + driver_setup.setup_repo_data(driver) + args = ('gips_process',) + driver_setup.STD_ARGS[driver] + ('-p', product) + outcome, actual = runner(*args) + if not record_mode: # don't evaluate assertions when in record-mode + assert (outcome.exit_code == 0 + and expectations[driver][product] == actual) + expectations['modis'] = { # 'landcover' [], # is annual, not available for the scene under test # 'indices', deprecated; not testing as a result @@ -395,14 +405,3 @@ params += [('modis', p) for p in expectations['modis'].keys()] params += [('merra', p) for p in expectations['merra'].keys()] - -@pytest.mark.parametrize("driver, product", params) -def t_process(repo_wrapper, driver, product): - """Test gips_process output.""" - record_mode, runner = repo_wrapper - driver_setup.setup_repo_data(driver) - args = ('gips_process',) + driver_setup.STD_ARGS[driver] + ('-p', product) - outcome, actual = runner(*args) - if not record_mode: # don't evaluate assertions when in record-mode - assert (outcome.exit_code == 0 - and expectations[driver][product] == actual) diff --git a/gips/test/sys/t_project.py b/gips/test/sys/t_project.py index 6895580e..4c4184d9 100644 --- a/gips/test/sys/t_project.py +++ b/gips/test/sys/t_project.py @@ -3,11 +3,22 @@ import util from util import export_wrapper import driver_setup -from .expected import modis as expectations params = [] # (driver, product),... expectations = {} # 'driver': {'product': [ (path, type, data...),...]...} +@pytest.mark.parametrize("driver, product", params) +def t_project(export_wrapper, driver, product): + """Test gips_project with warping.""" + record_mode, runner = export_wrapper + args = ('gips_project',) + driver_setup.STD_ARGS[driver] + ( + '--res', '100', '100', '--outdir', util.OUTPUT_DIR, '--notld', + '-p', product) + outcome, actual = runner(*args) + if not record_mode: # don't evaluate assertions when in record-mode + assert (outcome.exit_code == 0 + and expectations[driver][product] == actual) + expectations['modis'] = { # t_project[satvi] recording: 'satvi': @@ -400,14 +411,3 @@ params += [('modis', p) for p in expectations['modis'].keys()] params += [('merra', p) for p in expectations['merra'].keys()] -@pytest.mark.parametrize("driver, product", params) -def t_project(export_wrapper, driver, product): - """Test gips_project with warping.""" - record_mode, runner = export_wrapper - args = ('gips_project',) + driver_setup.STD_ARGS[driver] + ( - '--res', '100', '100', '--outdir', util.OUTPUT_DIR, '--notld', - '-p', product) - outcome, actual = runner(*args) - if not record_mode: # don't evaluate assertions when in record-mode - assert (outcome.exit_code == 0 - and expectations[driver][product] == actual) From 33707c66e8d815d602efc1522cbd55c178ab1007 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Tue, 26 Dec 2017 15:47:18 -0500 Subject: [PATCH 17/44] 483 convert t_stats to multi-driver pattern --- gips/test/sys/expected/merra.py | 65 ------- gips/test/sys/expected/modis.py | 204 ---------------------- gips/test/sys/t_stats.py | 297 ++++++++++++++++++++++++++++++++ 3 files changed, 297 insertions(+), 269 deletions(-) create mode 100644 gips/test/sys/t_stats.py diff --git a/gips/test/sys/expected/merra.py b/gips/test/sys/expected/merra.py index 6d8b37d1..2534143b 100644 --- a/gips/test/sys/expected/merra.py +++ b/gips/test/sys/expected/merra.py @@ -76,68 +76,3 @@ 'h01v01/h01v01_2015135_merra_wind.tif': 301026612, } } - -t_stats = { - # t_stats[srad] recording: - 'srad': - [('output/srad_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[tave] recording: - 'tave': - [('output/tave_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[shum] recording: - 'shum': - [('output/shum_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[rhum] recording: - 'rhum': - [('output/rhum_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[tmin] recording: - 'tmin': - [('output/tmin_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[tmax] recording: - 'tmax': - [('output/tmax_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[prcp] recording: - 'prcp': - [('output/prcp_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[patm] recording: - 'patm': - [('output/patm_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[wind] recording: - 'wind': - [('output/wind_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], -} diff --git a/gips/test/sys/expected/modis.py b/gips/test/sys/expected/modis.py index f305b46a..600ff5d5 100644 --- a/gips/test/sys/expected/modis.py +++ b/gips/test/sys/expected/modis.py @@ -111,210 +111,6 @@ 'h12v04/h12v04_2012338_MOD_clouds.tif': -2052728372, }} -t_stats = { - # t_stats[satvi] recording: - 'satvi': - [('output/satvi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.1199 0.2757 0.188845 0.0394886 -2.85581 221002.0 \n', - '2012-337 -0.1179 0.2955 0.189273 0.0399094 -2.7273 221002.0 \n', - '2012-338 -0.1125 0.2985 0.187186 0.0408718 -2.44579 218582.0 \n'])], - - # t_stats[ndti] recording: - 'ndti': - [('output/ndti_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -1.0048 0.8895 0.31602 0.0595381 -2.46398 221002.0 \n', - '2012-337 -0.643 0.9544 0.31893 0.0596861 -0.906354 221002.0 \n', - '2012-338 -0.5051 0.9999 0.320866 0.064446 -0.195554 218582.0 \n'])], - - # t_stats[ndvi] recording: - 'ndvi': - [('output/ndvi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.8893 0.8144 0.598198 0.130008 -4.50499 221002.0 \n', - '2012-337 -0.9126 0.8198 0.598463 0.132692 -4.61097 221002.0 \n', - '2012-338 -1.0001 0.8117 0.595837 0.134746 -4.68736 220979.0 \n'])], - - # t_stats[quality] recording: - 'quality': - [('output/quality_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0 0.0 0.0 0.0 nan 221002.0 \n', - '2012-337 0.0 0.0 0.0 0.0 nan 221002.0 \n', - '2012-338 0.0 0.0 0.0 0.0 nan 220979.0 \n'])], - - # t_stats[isti] recording: - 'isti': - [('output/isti_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0583 3.2766 0.522676 0.0837755 13.2684 220980.0 \n', - '2012-337 0.0232 3.2766 0.519636 0.0845782 12.4045 221002.0 \n', - '2012-338 -0.0001 3.0399 0.517872 0.0852503 7.69029 218582.0 \n'])], - - # t_stats[ndvi8] recording: - 'ndvi8': - [('output/ndvi8_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-337 -1.9475 3.2766 0.612297 0.162128 -3.72226 220758.0 \n'])], - - # t_stats[snow] recording: - 'snow': - [('output/snow_stats.txt', - 'text-full', - ['date Snow Cover-min Snow Cover-max Snow Cover-mean Snow Cover-sd Snow Cover-skew Snow Cover-count Fractional Snow Cover-min Fractional Snow Cover-max Fractional Snow Cover-mean Fractional Snow Cover-sd Fractional Snow Cover-skew Fractional Snow Cover-count \n', - '2012-336 0.0 100.0 66.0782 43.1646 -0.66934 2379.0 0.0 70.0 44.8928 24.6763 -0.938924 2379.0 \n', - '2012-337 0.0 0.0 0.0 0.0 nan 677.0 0.0 0.0 0.0 0.0 nan 677.0 \n', - '2012-338 0.0 100.0 7.3169 20.9113 2.95797 221002.0 0.0 38.0 0.358599 2.04345 8.82778 221002.0 \n'])], - - # t_stats[fsnow] recording: - 'fsnow': - [('output/fsnow_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0 70.0 36.4813 30.8325 -0.322887 2379.0 \n', - '2012-337 0.0 0.0 0.0 0.0 nan 677.0 \n', - '2012-338 0.0 35.0 0.0644655 1.11486 23.1911 221002.0 \n'])], - - # t_stats[crcm] recording: - 'crcm': - [('output/crcm_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.8857 1.6238 0.960025 0.21929 -3.32392 221002.0 \n', - '2012-337 -0.8639 1.5786 0.966857 0.222374 -3.25955 221002.0 \n', - '2012-338 -0.8744 1.8977 0.963618 0.23254 -2.82714 218582.0 \n'])], - - # t_stats[ndsi] recording: - 'ndsi': - [('output/ndsi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.7313 1.0045 -0.58418 0.123674 6.27293 221002.0 \n', - '2012-337 -0.731 0.9434 -0.58564 0.12352 6.14429 221002.0 \n', - '2012-338 -0.7775 0.9329 -0.581803 0.125509 5.997 218582.0 \n'])], - - # t_stats[brgt] recording: - 'brgt': - [('output/brgt_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0062 0.1361 0.0467873 0.0103586 0.699293 221002.0 \n', - '2012-337 0.0062 0.1379 0.0465373 0.0103964 0.708942 221002.0 \n', - '2012-338 0.0064 0.1351 0.0464351 0.0105081 0.660629 220979.0 \n'])], - - # t_stats[msavi2] recording: - 'msavi2': - [('output/msavi2_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.014 0.6976 0.503178 0.0608721 -2.7989 219182.0 \n', - '2012-337 -0.0001 0.6996 0.502591 0.0601847 -2.75199 219026.0 \n', - '2012-338 0.014 0.693 0.499838 0.0606011 -2.67091 219027.0 \n'])], - - # t_stats[bi] recording: - 'bi': - [('output/bi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0033 0.1717 0.0948656 0.0173088 -1.23546 221002.0 \n', - '2012-337 0.0033 0.1714 0.0944838 0.0173897 -1.24311 221002.0 \n', - '2012-338 0.0038 0.1705 0.0938487 0.0174975 -1.19669 220979.0 \n'])], - - # t_stats[obstime] recording: - 'obstime': - [('output/obstime_stats.txt', - 'text-full', - ['date Observation Time Daytime Terra-min Observation Time Daytime Terra-max Observation Time Daytime Terra-mean Observation Time Daytime Terra-sd Observation Time Daytime Terra-skew Observation Time Daytime Terra-count Observation Time Nighttime Terra-min Observation Time Nighttime Terra-max Observation Time Nighttime Terra-mean Observation Time Nighttime Terra-sd Observation Time Nighttime Terra-skew Observation Time Nighttime Terra-count Observation Time Daytime Aqua-min Observation Time Daytime Aqua-max Observation Time Daytime Aqua-mean Observation Time Daytime Aqua-sd Observation Time Daytime Aqua-skew Observation Time Daytime Aqua-count Observation Time Nighttime Aqua-min Observation Time Nighttime Aqua-max Observation Time Nighttime Aqua-mean Observation Time Nighttime Aqua-sd Observation Time Nighttime Aqua-skew Observation Time Nighttime Aqua-count \n', - '2012-336 10.8 10.8 10.8 1.90735e-07 -0.999996 716.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 1.4 1.4 1.4 2.38419e-08 inf 6685.0 \n', - '2012-337 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 \n', - '2012-338 10.6 10.6 10.6 3.8147e-07 -1.0 184032.0 255.0 0.0 nan nan nan 0.0 12.3 12.3 12.3 1.90735e-07 -0.999996 216873.0 1.2 2.8 1.61335 0.700361 1.10413 181365.0 \n'])], - - # t_stats[lswi] recording: - 'lswi': - [('output/lswi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.9413 1.004 0.0704005 0.0795676 -0.897423 221002.0 \n', - '2012-337 -0.9644 0.879 0.068998 0.0814312 -1.75006 221002.0 \n', - '2012-338 -1.0001 0.8591 0.0695461 0.0851064 -1.65515 218582.0 \n'])], - - # t_stats[vari] recording: - 'vari': - [('output/vari_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.2931 3.2766 -0.0579385 0.0896767 11.2112 221002.0 \n', - '2012-337 -0.3266 3.2766 -0.0573387 0.109661 15.3818 221002.0 \n', - '2012-338 -0.3817 3.2766 -0.0567795 0.115018 15.8625 220979.0 \n'])], - - # t_stats[evi] recording: - 'evi': - [('output/evi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.0689 0.4391 0.251258 0.0534915 -2.19097 221002.0 \n', - '2012-337 -0.0704 0.4432 0.250715 0.0535784 -2.20674 221002.0 \n', - '2012-338 -0.0727 0.4339 0.248372 0.0536924 -2.14955 220979.0 \n'])], - - # t_stats[temp8tn] recording: - 'temp8tn': - [('output/temp8tn_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-337 267.02 280.38 273.055 1.35905 0.334532 212758.0 \n'])], - - # t_stats[clouds] recording: - 'clouds': - [('output/clouds_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0 1.0 0.990964 0.094628 -10.3767 221002.0 \n', - '2012-337 0.0 1.0 0.998367 0.0403832 -24.6819 221002.0 \n', - '2012-338 0.0 1.0 0.0838499 0.277163 3.00293 221002.0 \n'])], - - # t_stats[temp] recording: - 'temp': - [('output/temp_stats.txt', - 'text-full', - ['date Temperature Daytime Terra-min Temperature Daytime Terra-max Temperature Daytime Terra-mean Temperature Daytime Terra-sd Temperature Daytime Terra-skew Temperature Daytime Terra-count Temperature Nighttime Terra-min Temperature Nighttime Terra-max Temperature Nighttime Terra-mean Temperature Nighttime Terra-sd Temperature Nighttime Terra-skew Temperature Nighttime Terra-count Temperature Daytime Aqua-min Temperature Daytime Aqua-max Temperature Daytime Aqua-mean Temperature Daytime Aqua-sd Temperature Daytime Aqua-skew Temperature Daytime Aqua-count Temperature Nighttime Aqua-min Temperature Nighttime Aqua-max Temperature Nighttime Aqua-mean Temperature Nighttime Aqua-sd Temperature Nighttime Aqua-skew Temperature Nighttime Aqua-count Temperature Best Quality-min Temperature Best Quality-max Temperature Best Quality-mean Temperature Best Quality-sd Temperature Best Quality-skew Temperature Best Quality-count \n', - '2012-336 267.44 267.9 267.67 0.140381 -0.244244 716.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 258.94 263.7 261.912 1.39539 -0.717615 6685.0 0.0 0.0 0.0 0.0 nan 221255.0 \n', - '2012-337 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 0.0 0.0 0.0 0.0 nan 221255.0 \n', - '2012-338 275.34 286.28 282.987 2.17515 -1.16826 184032.0 65535.0 0.0 nan nan nan 0.0 277.56 283.86 281.873 0.980026 -0.679654 216873.0 270.04 280.26 277.74 1.43805 -1.36599 181365.0 0.0 13.0 4.60988 3.26751 1.22493 221255.0 \n'])], - - # t_stats[temp8td] recording: - 'temp8td': - [('output/temp8td_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-337 273.32 283.76 279.933 1.98691 -1.00057 218113.0 \n'])], - - # t_stats[sti] recording: - 'sti': - [('output/sti_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.0024 3.2766 1.94182 0.235117 0.707709 221002.0 \n', - '2012-337 0.2173 3.2766 1.95394 0.239902 0.761782 221002.0 \n', - '2012-338 0.3288 3.2766 1.96524 0.263691 0.833778 218582.0 \n'])], - - # t_stats[crc] recording: - 'crc': - [('output/crc_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.8526 2.1525 1.29852 0.263609 -2.37667 221002.0 \n', - '2012-337 -0.868 2.2016 1.30555 0.268596 -2.32839 221002.0 \n', - '2012-338 -0.8795 2.8088 1.30562 0.283586 -1.87226 218582.0 \n'])], -} - - t_gridded_export = { 'created': { '0': None, '0/2005001_MCD_indices.tif': 1618619735, diff --git a/gips/test/sys/t_stats.py b/gips/test/sys/t_stats.py new file mode 100644 index 00000000..335245df --- /dev/null +++ b/gips/test/sys/t_stats.py @@ -0,0 +1,297 @@ +import pytest +import sh + +import util +from util import export_wrapper +import driver_setup + +params = [] # (driver, product),... +expectations = {} # 'driver': {'product': [ (path, type, data...),...]...} + +@pytest.mark.parametrize("driver, product", params) +def t_stats(export_wrapper, driver, product): + """Test gips_stats on projected files.""" + record_mode, runner = export_wrapper + + # generate data needed for stats computation + args = driver_setup.STD_ARGS[driver] + ('--res', '100', '100', '--notld', + '--outdir', util.OUTPUT_DIR, '-p', product) + outcome = sh.gips_project(*args) + assert outcome.exit_code == 0 # sanity check + + # compute & confirm stats + outcome, actual = runner('gips_stats', util.OUTPUT_DIR) + if not record_mode: + assert (outcome.exit_code == 0 + and expectations[driver][product] == actual) + +expectations['modis'] = { + # t_stats[satvi] recording: + 'satvi': + [('output/satvi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.1199 0.2757 0.188845 0.0394886 -2.85581 221002.0 \n', + '2012-337 -0.1179 0.2955 0.189273 0.0399094 -2.7273 221002.0 \n', + '2012-338 -0.1125 0.2985 0.187186 0.0408718 -2.44579 218582.0 \n'])], + + # t_stats[ndti] recording: + 'ndti': + [('output/ndti_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -1.0048 0.8895 0.31602 0.0595381 -2.46398 221002.0 \n', + '2012-337 -0.643 0.9544 0.31893 0.0596861 -0.906354 221002.0 \n', + '2012-338 -0.5051 0.9999 0.320866 0.064446 -0.195554 218582.0 \n'])], + + # t_stats[ndvi] recording: + 'ndvi': + [('output/ndvi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.8893 0.8144 0.598198 0.130008 -4.50499 221002.0 \n', + '2012-337 -0.9126 0.8198 0.598463 0.132692 -4.61097 221002.0 \n', + '2012-338 -1.0001 0.8117 0.595837 0.134746 -4.68736 220979.0 \n'])], + + # t_stats[quality] recording: + 'quality': + [('output/quality_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0 0.0 0.0 0.0 nan 221002.0 \n', + '2012-337 0.0 0.0 0.0 0.0 nan 221002.0 \n', + '2012-338 0.0 0.0 0.0 0.0 nan 220979.0 \n'])], + + # t_stats[isti] recording: + 'isti': + [('output/isti_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0583 3.2766 0.522676 0.0837755 13.2684 220980.0 \n', + '2012-337 0.0232 3.2766 0.519636 0.0845782 12.4045 221002.0 \n', + '2012-338 -0.0001 3.0399 0.517872 0.0852503 7.69029 218582.0 \n'])], + + # t_stats[ndvi8] recording: + 'ndvi8': + [('output/ndvi8_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-337 -1.9475 3.2766 0.612297 0.162128 -3.72226 220758.0 \n'])], + + # t_stats[snow] recording: + 'snow': + [('output/snow_stats.txt', + 'text-full', + ['date Snow Cover-min Snow Cover-max Snow Cover-mean Snow Cover-sd Snow Cover-skew Snow Cover-count Fractional Snow Cover-min Fractional Snow Cover-max Fractional Snow Cover-mean Fractional Snow Cover-sd Fractional Snow Cover-skew Fractional Snow Cover-count \n', + '2012-336 0.0 100.0 66.0782 43.1646 -0.66934 2379.0 0.0 70.0 44.8928 24.6763 -0.938924 2379.0 \n', + '2012-337 0.0 0.0 0.0 0.0 nan 677.0 0.0 0.0 0.0 0.0 nan 677.0 \n', + '2012-338 0.0 100.0 7.3169 20.9113 2.95797 221002.0 0.0 38.0 0.358599 2.04345 8.82778 221002.0 \n'])], + + # t_stats[fsnow] recording: + 'fsnow': + [('output/fsnow_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0 70.0 36.4813 30.8325 -0.322887 2379.0 \n', + '2012-337 0.0 0.0 0.0 0.0 nan 677.0 \n', + '2012-338 0.0 35.0 0.0644655 1.11486 23.1911 221002.0 \n'])], + + # t_stats[crcm] recording: + 'crcm': + [('output/crcm_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.8857 1.6238 0.960025 0.21929 -3.32392 221002.0 \n', + '2012-337 -0.8639 1.5786 0.966857 0.222374 -3.25955 221002.0 \n', + '2012-338 -0.8744 1.8977 0.963618 0.23254 -2.82714 218582.0 \n'])], + + # t_stats[ndsi] recording: + 'ndsi': + [('output/ndsi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.7313 1.0045 -0.58418 0.123674 6.27293 221002.0 \n', + '2012-337 -0.731 0.9434 -0.58564 0.12352 6.14429 221002.0 \n', + '2012-338 -0.7775 0.9329 -0.581803 0.125509 5.997 218582.0 \n'])], + + # t_stats[brgt] recording: + 'brgt': + [('output/brgt_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0062 0.1361 0.0467873 0.0103586 0.699293 221002.0 \n', + '2012-337 0.0062 0.1379 0.0465373 0.0103964 0.708942 221002.0 \n', + '2012-338 0.0064 0.1351 0.0464351 0.0105081 0.660629 220979.0 \n'])], + + # t_stats[msavi2] recording: + 'msavi2': + [('output/msavi2_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.014 0.6976 0.503178 0.0608721 -2.7989 219182.0 \n', + '2012-337 -0.0001 0.6996 0.502591 0.0601847 -2.75199 219026.0 \n', + '2012-338 0.014 0.693 0.499838 0.0606011 -2.67091 219027.0 \n'])], + + # t_stats[bi] recording: + 'bi': + [('output/bi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0033 0.1717 0.0948656 0.0173088 -1.23546 221002.0 \n', + '2012-337 0.0033 0.1714 0.0944838 0.0173897 -1.24311 221002.0 \n', + '2012-338 0.0038 0.1705 0.0938487 0.0174975 -1.19669 220979.0 \n'])], + + # t_stats[obstime] recording: + 'obstime': + [('output/obstime_stats.txt', + 'text-full', + ['date Observation Time Daytime Terra-min Observation Time Daytime Terra-max Observation Time Daytime Terra-mean Observation Time Daytime Terra-sd Observation Time Daytime Terra-skew Observation Time Daytime Terra-count Observation Time Nighttime Terra-min Observation Time Nighttime Terra-max Observation Time Nighttime Terra-mean Observation Time Nighttime Terra-sd Observation Time Nighttime Terra-skew Observation Time Nighttime Terra-count Observation Time Daytime Aqua-min Observation Time Daytime Aqua-max Observation Time Daytime Aqua-mean Observation Time Daytime Aqua-sd Observation Time Daytime Aqua-skew Observation Time Daytime Aqua-count Observation Time Nighttime Aqua-min Observation Time Nighttime Aqua-max Observation Time Nighttime Aqua-mean Observation Time Nighttime Aqua-sd Observation Time Nighttime Aqua-skew Observation Time Nighttime Aqua-count \n', + '2012-336 10.8 10.8 10.8 1.90735e-07 -0.999996 716.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 1.4 1.4 1.4 2.38419e-08 inf 6685.0 \n', + '2012-337 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 \n', + '2012-338 10.6 10.6 10.6 3.8147e-07 -1.0 184032.0 255.0 0.0 nan nan nan 0.0 12.3 12.3 12.3 1.90735e-07 -0.999996 216873.0 1.2 2.8 1.61335 0.700361 1.10413 181365.0 \n'])], + + # t_stats[lswi] recording: + 'lswi': + [('output/lswi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.9413 1.004 0.0704005 0.0795676 -0.897423 221002.0 \n', + '2012-337 -0.9644 0.879 0.068998 0.0814312 -1.75006 221002.0 \n', + '2012-338 -1.0001 0.8591 0.0695461 0.0851064 -1.65515 218582.0 \n'])], + + # t_stats[vari] recording: + 'vari': + [('output/vari_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.2931 3.2766 -0.0579385 0.0896767 11.2112 221002.0 \n', + '2012-337 -0.3266 3.2766 -0.0573387 0.109661 15.3818 221002.0 \n', + '2012-338 -0.3817 3.2766 -0.0567795 0.115018 15.8625 220979.0 \n'])], + + # t_stats[evi] recording: + 'evi': + [('output/evi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.0689 0.4391 0.251258 0.0534915 -2.19097 221002.0 \n', + '2012-337 -0.0704 0.4432 0.250715 0.0535784 -2.20674 221002.0 \n', + '2012-338 -0.0727 0.4339 0.248372 0.0536924 -2.14955 220979.0 \n'])], + + # t_stats[temp8tn] recording: + 'temp8tn': + [('output/temp8tn_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-337 267.02 280.38 273.055 1.35905 0.334532 212758.0 \n'])], + + # t_stats[clouds] recording: + 'clouds': + [('output/clouds_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0 1.0 0.990964 0.094628 -10.3767 221002.0 \n', + '2012-337 0.0 1.0 0.998367 0.0403832 -24.6819 221002.0 \n', + '2012-338 0.0 1.0 0.0838499 0.277163 3.00293 221002.0 \n'])], + + # t_stats[temp] recording: + 'temp': + [('output/temp_stats.txt', + 'text-full', + ['date Temperature Daytime Terra-min Temperature Daytime Terra-max Temperature Daytime Terra-mean Temperature Daytime Terra-sd Temperature Daytime Terra-skew Temperature Daytime Terra-count Temperature Nighttime Terra-min Temperature Nighttime Terra-max Temperature Nighttime Terra-mean Temperature Nighttime Terra-sd Temperature Nighttime Terra-skew Temperature Nighttime Terra-count Temperature Daytime Aqua-min Temperature Daytime Aqua-max Temperature Daytime Aqua-mean Temperature Daytime Aqua-sd Temperature Daytime Aqua-skew Temperature Daytime Aqua-count Temperature Nighttime Aqua-min Temperature Nighttime Aqua-max Temperature Nighttime Aqua-mean Temperature Nighttime Aqua-sd Temperature Nighttime Aqua-skew Temperature Nighttime Aqua-count Temperature Best Quality-min Temperature Best Quality-max Temperature Best Quality-mean Temperature Best Quality-sd Temperature Best Quality-skew Temperature Best Quality-count \n', + '2012-336 267.44 267.9 267.67 0.140381 -0.244244 716.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 258.94 263.7 261.912 1.39539 -0.717615 6685.0 0.0 0.0 0.0 0.0 nan 221255.0 \n', + '2012-337 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 0.0 0.0 0.0 0.0 nan 221255.0 \n', + '2012-338 275.34 286.28 282.987 2.17515 -1.16826 184032.0 65535.0 0.0 nan nan nan 0.0 277.56 283.86 281.873 0.980026 -0.679654 216873.0 270.04 280.26 277.74 1.43805 -1.36599 181365.0 0.0 13.0 4.60988 3.26751 1.22493 221255.0 \n'])], + + # t_stats[temp8td] recording: + 'temp8td': + [('output/temp8td_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-337 273.32 283.76 279.933 1.98691 -1.00057 218113.0 \n'])], + + # t_stats[sti] recording: + 'sti': + [('output/sti_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.0024 3.2766 1.94182 0.235117 0.707709 221002.0 \n', + '2012-337 0.2173 3.2766 1.95394 0.239902 0.761782 221002.0 \n', + '2012-338 0.3288 3.2766 1.96524 0.263691 0.833778 218582.0 \n'])], + + # t_stats[crc] recording: + 'crc': + [('output/crc_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.8526 2.1525 1.29852 0.263609 -2.37667 221002.0 \n', + '2012-337 -0.868 2.2016 1.30555 0.268596 -2.32839 221002.0 \n', + '2012-338 -0.8795 2.8088 1.30562 0.283586 -1.87226 218582.0 \n'])], +} + +expectations['merra'] = { + # t_stats[srad] recording: + 'srad': + [('output/srad_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[tave] recording: + 'tave': + [('output/tave_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[shum] recording: + 'shum': + [('output/shum_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[rhum] recording: + 'rhum': + [('output/rhum_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[tmin] recording: + 'tmin': + [('output/tmin_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[tmax] recording: + 'tmax': + [('output/tmax_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[prcp] recording: + 'prcp': + [('output/prcp_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[patm] recording: + 'patm': + [('output/patm_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[wind] recording: + 'wind': + [('output/wind_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], +} + +params += [('modis', p) for p in expectations['modis'].keys()] +params += [('merra', p) for p in expectations['merra'].keys()] From 6f338e9c77c20075551df4a1376bdbad7f4e3d82 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Tue, 26 Dec 2017 15:54:36 -0500 Subject: [PATCH 18/44] 483 remove inventory & info tests: These are believed to test nothing but stdout; programmatic tests are preferred over screen-scraping. --- gips/test/sys/expected/aod.py | 36 ---------- gips/test/sys/expected/landsat.py | 101 ---------------------------- gips/test/sys/expected/merra.py | 40 ----------- gips/test/sys/expected/modis.py | 51 -------------- gips/test/sys/expected/prism.py | 40 ----------- gips/test/sys/expected/sar.py | 66 ------------------ gips/test/sys/expected/sentinel2.py | 50 -------------- gips/test/sys/t_aod.py | 14 ---- gips/test/sys/t_landsat.py | 11 --- gips/test/sys/t_merra.py | 36 ---------- gips/test/sys/t_modis.py | 30 --------- gips/test/sys/t_prism.py | 16 ----- gips/test/sys/t_sar.py | 15 ----- gips/test/sys/t_sentinel2.py | 14 ---- 14 files changed, 520 deletions(-) diff --git a/gips/test/sys/expected/aod.py b/gips/test/sys/expected/aod.py index 72c39aa0..c6d3883f 100644 --- a/gips/test/sys/expected/aod.py +++ b/gips/test/sys/expected/aod.py @@ -1,41 +1,5 @@ """Known-good outcomes for tests, mostly stdout and created files.""" -t_info = { 'stdout': u"""\x1b[1mGIPS Data Repositories (v0.8.2)\x1b[0m -\x1b[1m -AOD Products v0.9.1\x1b[0m -\x1b[1m -Standard Products -\x1b[0m aod Aerosol Optical Depth - lta Average AOD - ltad Average daily AOD -""" } - - -t_inventory = { - 'stdout': u"""\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site NE-0 - -\x1b[1mAsset Coverage for site NE-0\x1b[0m -\x1b[1m -Tile Coverage -\x1b[4m Tile % Coverage % Tile Used\x1b[0m - h01v01 100.0% 100.0% - -\x1b[1m\x1b[4m DATE MOD08 Product \x1b[0m -\x1b[1m2017 -\x1b[0m 004 100.0% \x1b[35maod\x1b[0m - 005 100.0% \x1b[35maod\x1b[0m - 006 100.0% \x1b[35maod\x1b[0m - - -3 files on 3 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mMOD: MODIS Terra\x1b[0m -""", -} - - t_process = { 'updated': { }, diff --git a/gips/test/sys/expected/landsat.py b/gips/test/sys/expected/landsat.py index be8f3c61..2535e811 100644 --- a/gips/test/sys/expected/landsat.py +++ b/gips/test/sys/expected/landsat.py @@ -1,106 +1,5 @@ """Known-good outcomes for tests, mostly stdout and created files.""" -# trailing whitespace and other junk characters are in current output -t_info = { 'stdout': u"""\x1b[1mGIPS Data Repositories (v0.8.2)\x1b[0m -\x1b[1m -Landsat Products v1.0.0\x1b[0m - Optional qualifiers listed below each product. - Specify by appending '-option' to product (e.g., ref-toa) -\x1b[1m -Index Products -\x1b[0m bi Brightness Index - toa: use top of the atmosphere reflectance - evi Enhanced Vegetation Index - toa: use top of the atmosphere reflectance - lswi Land Surface Water Index - toa: use top of the atmosphere reflectance - msavi2 Modified Soil-Adjusted Vegetation Index (revised) - toa: use top of the atmosphere reflectance - ndsi Normalized Difference Snow Index - toa: use top of the atmosphere reflectance - ndvi Normalized Difference Vegetation Index - toa: use top of the atmosphere reflectance - ndwi Normalized Difference Water Index - toa: use top of the atmosphere reflectance - satvi Soil-Adjusted Total Vegetation Index - toa: use top of the atmosphere reflectance - vari Visible Atmospherically Resistant Index - toa: use top of the atmosphere reflectance -\x1b[1m -LC8SR Products -\x1b[0m ndvi8sr Normalized Difference Vegetation from LC8SR -\x1b[1m -ACOLITE Products -\x1b[0m acoflags 0 = water 1 = no data 2 = land - fai Floating Algae Index - oc2chl Blue-Green Ratio Chlorophyll Algorithm using bands 483 & 561 - oc3chl Blue-Green Ratio Chlorophyll Algorithm using bands 443, 483, & 561 - rhow Water-Leaving Radiance-Reflectance - spm655 Suspended Sediment Concentration 655nm - turbidity Blended Turbidity -\x1b[1m -Tillage Products -\x1b[0m crc Crop Residue Cover - toa: use top of the atmosphere reflectance - isti Inverse Standard Tillage Index - toa: use top of the atmosphere reflectance - ndti Normalized Difference Tillage Index - toa: use top of the atmosphere reflectance - sti Standard Tillage Index - toa: use top of the atmosphere reflectance -\x1b[1m -Standard Products -\x1b[0m acca Automated Cloud Cover Assessment - X: erosion kernel diameter in pixels (default: 5) - Y: dilation kernel diameter in pixels (default: 10) - Z: cloud height in meters (default: 4000) - bqa The quality band extracted into separate layers. - bqashadow LC8 QA + Shadow Smear - X: erosion kernel diameter in pixels (default: 5) - Y: dilation kernel diameter in pixels (default: 10) - Z: cloud height in meters (default: 4000) - cloudmask Cloud (and shadow) mask product based on cloud bits of the quality band - dn Raw digital numbers - fmask Fmask cloud cover - landmask Land mask from LC8SR - rad Surface-leaving radiance - toa: use top of the atmosphere reflectance - ref Surface reflectance - toa: use top of the atmosphere reflectance - tcap Tassled cap transformation - temp Brightness (apparent) temperature - volref Volumetric water reflectance - valid for water only - toa: use top of the atmosphere reflectance - wtemp Water temperature (atmospherically correct) - valid for water only -"""} - -t_inventory = { 'stdout': u"""\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site NHseacoast-0 -fname -LC08_L1GT_012030_20151218_20170224_01_T2.tar.gz -C1 asset - -\x1b[1mAsset Coverage for site NHseacoast-0\x1b[0m -\x1b[1m -Tile Coverage -\x1b[4m Tile % Coverage % Tile Used\x1b[0m - 012030 100.0% 6.7% - 013030 2.4% 0.2% - -\x1b[1m\x1b[4m DATE C1 DN SR Product \x1b[0m -\x1b[1m2015 -\x1b[0m 352 100.0% - - -1 files on 1 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mLC8: Landsat 8\x1b[0m -\x1b[31mLC8SR: Landsat 8 Surface Reflectance\x1b[0m -\x1b[32mLE7: Landsat 7\x1b[0m -\x1b[34mLT5: Landsat 5\x1b[0m -"""} - t_process = { 'compare_stderr': False, 'updated': { diff --git a/gips/test/sys/expected/merra.py b/gips/test/sys/expected/merra.py index 2534143b..76a20ef1 100644 --- a/gips/test/sys/expected/merra.py +++ b/gips/test/sys/expected/merra.py @@ -1,45 +1,5 @@ """Known-good outcomes for tests, mostly stdout and created files.""" -t_info = { 'stdout': u"""\x1b[1mGIPS Data Repositories (v0.8.2)\x1b[0m -\x1b[1m -merra Products v1.0.0\x1b[0m -\x1b[1m -Standard Products -\x1b[0m frland Fraction of land (fraction) - patm Surface atmospheric pressure (mb) - prcp Daily total precipitation (mm day-1) - rhum Relative humidity (%) - shum Relative humidity (kg kg-1) - srad Incident solar radiation (W m-2) - tave Ave daily air temperature data (K) - tmax Max daily air temperature data - tmin Min daily air temperature data - wind Daily mean wind speed (m s-1) -""" } - - -t_inventory = { - 'stdout': u"""\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site NHseacoast-0 - -\x1b[1mAsset Coverage for site NHseacoast-0\x1b[0m -\x1b[1m -Tile Coverage -\x1b[4m Tile % Coverage % Tile Used\x1b[0m - h01v01 100.0% 0.0% - -\x1b[1m\x1b[4m DATE ASM FLX RAD SLV Product \x1b[0m -\x1b[1m2015 -\x1b[0m 135 100.0% 100.0% 100.0% - - -1 files on 1 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mmerra: Modern Era Retrospective-analysis for Research and Applications\x1b[0m -""" -} - t_project_no_warp = { 'created': { '0': None, diff --git a/gips/test/sys/expected/modis.py b/gips/test/sys/expected/modis.py index 600ff5d5..735bdefd 100644 --- a/gips/test/sys/expected/modis.py +++ b/gips/test/sys/expected/modis.py @@ -1,56 +1,5 @@ """Known-good outcomes for tests, mostly stdout and created files.""" -t_inventory = { 'stdout': u"""\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site NHseacoast-0 - -\x1b[1mAsset Coverage for site NHseacoast-0\x1b[0m -\x1b[1m -Tile Coverage -\x1b[4m Tile % Coverage % Tile Used\x1b[0m - h12v04 100.0% 0.2% - -\x1b[1m\x1b[4m DATE MCD12Q1 MCD43A2 MCD43A4 MOD09Q1 MOD10A1 MOD10A2 MOD11A1 MOD11A2 MYD10A1 MYD10A2 MYD11A1 MYD11A2 Product \x1b[0m -\x1b[1m2012 -\x1b[0m 336 100.0% 100.0% 100.0% 100.0% 100.0% 100.0% - 337 100.0% 100.0% 100.0% 100.0% 100.0% 100.0% 100.0% 100.0% - 338 100.0% 100.0% 100.0% 100.0% 100.0% 100.0% - - -3 files on 3 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mMCD: Aqua/Terra Combined\x1b[0m -\x1b[31mMOD: Terra\x1b[0m -\x1b[32mMOD-MYD: Aqua/Terra together\x1b[0m -\x1b[34mMYD: Aqua\x1b[0m -""" -} - -# trailing whitespace and other junk characters are in current output -t_info = { 'stdout': u"""\x1b[1mGIPS Data Repositories (v0.8.2)\x1b[0m -\x1b[1m -Modis Products v1.0.0\x1b[0m -\x1b[1m -Terra 8-day Products -\x1b[0m ndvi8 Normalized Difference Vegetation Index: 250m - temp8td Surface temperature: 1km - temp8tn Surface temperature: 1km -\x1b[1m -Nadir BRDF-Adjusted 16-day Products -\x1b[0m indices Land indices - quality MCD Product Quality -\x1b[1m -Terra/Aqua Daily Products -\x1b[0m fsnow Fractional snow cover data - obstime MODIS Terra/Aqua overpass time - snow Snow and ice cover data - temp Surface temperature data -\x1b[1m -Standard Products -\x1b[0m clouds Cloud Mask - landcover MCD Annual Land Cover -"""} - t_project_no_warp = { 'created': { '0': None, # directory '0/2012336_MCD_fsnow.tif': -232655043, diff --git a/gips/test/sys/expected/prism.py b/gips/test/sys/expected/prism.py index 1e25274f..1f662902 100644 --- a/gips/test/sys/expected/prism.py +++ b/gips/test/sys/expected/prism.py @@ -1,45 +1,5 @@ """Known-good outcomes for tests, mostly stdout and created files.""" -t_info = { 'stdout': u"""\x1b[1mGIPS Data Repositories (v0.8.2)\x1b[0m -\x1b[1m -PRISM Products v0.1.1\x1b[0m - Optional qualifiers listed below each product. - Specify by appending '-option' to product (e.g., ref-toa) -\x1b[1m -Standard Products -\x1b[0m ppt Precipitate - pptsum Cumulative Precipitate - days: temporal window width (default: 3 days) . - tmax Daily Maximum Temperature - tmin Daily Minimum Temperature -""" } - - -t_inventory = { - 'stdout': u"""\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site NHseacoast-0 - -\x1b[1mAsset Coverage for site NHseacoast-0\x1b[0m -\x1b[1m -Tile Coverage -\x1b[4m Tile % Coverage % Tile Used\x1b[0m - CONUS 99.2% 0.0% - -\x1b[1m\x1b[4m DATE _ppt _tmax _tmin Product \x1b[0m -\x1b[1m1982 -\x1b[0m 335 99.2% 99.2% 99.2% - 336 99.2% 99.2% 99.2% - 337 99.2% 99.2% 99.2% - - -3 files on 3 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mprism: Daily Gridded Climate Data\x1b[0m -""", -} - - t_process = { 'updated': { 'prism/stage': None, diff --git a/gips/test/sys/expected/sar.py b/gips/test/sys/expected/sar.py index 6c2fd9fd..ee3b7325 100644 --- a/gips/test/sys/expected/sar.py +++ b/gips/test/sys/expected/sar.py @@ -1,71 +1,5 @@ """Known-good outcomes for tests, mostly stdout and created files.""" -t_info = { - 'stdout': u"""\x1b[1mGIPS Data Repositories (v0.8.2)\x1b[0m -\x1b[1m -SAR Products v0.9.0\x1b[0m -\x1b[1m -Standard Products -\x1b[0m date Day of year array - linci Incident angles - sign Sigma nought (radar backscatter coefficient) -""" -} - - -t_inventory = { - 'stdout': u"""\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site tiles - -\x1b[1mAsset Holdings for tile N07E099\x1b[0m -\x1b[1m\x1b[4m DATE alos1 alos2 Product \x1b[0m -\x1b[1m2015 -\x1b[0m 101 100.0% - - -1 files on 1 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mAFBD: PALSAR FineBeam Dual Polarization\x1b[0m -\x1b[31mAFBS: PALSAR FineBeam Single Polarization\x1b[0m -\x1b[32mAWB1: PALSAR WideBeam (ScanSAR Short Mode)\x1b[0m -\x1b[34mAWBD: PALSAR-2 WideBeam (ScanSAR w/Ortho Slope Correction)\x1b[0m -\x1b[33mJFBS: JERS-1 FineBeam Single Polarization\x1b[0m -Retrieving inventory for site tiles - -\x1b[1mAsset Holdings for tile N19E100\x1b[0m -\x1b[1m\x1b[4m DATE alos1 alos2 Product \x1b[0m -\x1b[1m2010 -\x1b[0m 182 100.0% - - -1 files on 1 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mAFBD: PALSAR FineBeam Dual Polarization\x1b[0m -\x1b[31mAFBS: PALSAR FineBeam Single Polarization\x1b[0m -\x1b[32mAWB1: PALSAR WideBeam (ScanSAR Short Mode)\x1b[0m -\x1b[34mAWBD: PALSAR-2 WideBeam (ScanSAR w/Ortho Slope Correction)\x1b[0m -\x1b[33mJFBS: JERS-1 FineBeam Single Polarization\x1b[0m -Retrieving inventory for site tiles - -\x1b[1mAsset Holdings for tile N00E099\x1b[0m -\x1b[1m\x1b[4m DATE alos1 alos2 Product \x1b[0m -\x1b[1m2009 -\x1b[0m 041 100.0% - - -1 files on 1 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mAFBD: PALSAR FineBeam Dual Polarization\x1b[0m -\x1b[31mAFBS: PALSAR FineBeam Single Polarization\x1b[0m -\x1b[32mAWB1: PALSAR WideBeam (ScanSAR Short Mode)\x1b[0m -\x1b[34mAWBD: PALSAR-2 WideBeam (ScanSAR w/Ortho Slope Correction)\x1b[0m -\x1b[33mJFBS: JERS-1 FineBeam Single Polarization\x1b[0m -""", -} - otro = u"""\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m Retrieving inventory for site tiles diff --git a/gips/test/sys/expected/sentinel2.py b/gips/test/sys/expected/sentinel2.py index c04d9656..e2bd53a4 100644 --- a/gips/test/sys/expected/sentinel2.py +++ b/gips/test/sys/expected/sentinel2.py @@ -1,27 +1,5 @@ """Known-good outcomes for tests, mostly stdout and created files.""" -t_inventory = { 'stdout': u"""\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site durham-0 - -\x1b[1mAsset Coverage for site durham-0\x1b[0m -\x1b[1m -Tile Coverage -\x1b[4m Tile % Coverage % Tile Used\x1b[0m - 19TCH 100.0% 0.1% - -\x1b[1m\x1b[4m DATE L1C Product \x1b[0m -\x1b[1m2017 -\x1b[0m 010 100.0% - - -1 files on 1 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mS2A: Sentinel-2, Satellite A\x1b[0m -\x1b[31mS2B: Sentinel-2, Satellite B\x1b[0m -""" -} - _en_base = ('sentinel2/tiles/19TCH/2017010' '/S2A_MSIL1C_20170110T153611_N0204_R111_T19TCH_20170110T154029.SAFE') @@ -103,34 +81,6 @@ """ } - -t_info = { 'stdout': u"""\x1b[1mGIPS Data Repositories (v0.8.2)\x1b[0m -\x1b[1m -Sentinel2 Products v0.1.0\x1b[0m -\x1b[1m -Index Products -\x1b[0m bi Brightness Index - brgt VIS and NIR reflectance, weighted by solar energy distribution. - crc Crop Residue Cover (uses BLUE) - crcm Crop Residue Cover, Modified (uses GREEN) - evi Enhanced Vegetation Index - isti Inverse Standard Tillage Index - lswi Land Surface Water Index - msavi2 Modified Soil-adjusted Vegetation Index - ndsi Normalized Difference Snow Index - ndti Normalized Difference Tillage Index - ndvi Normalized Difference Vegetation Index - satvi Soil-Adjusted Total Vegetation Index - sti Standard Tillage Index - vari Visible Atmospherically Resistant Index -\x1b[1m -Standard Products -\x1b[0m rad Surface-leaving radiance - ref Surface reflectance -""" -} - - t_project = { 'compare_stderr': False, 'created': { diff --git a/gips/test/sys/t_aod.py b/gips/test/sys/t_aod.py index ee755f48..3e1868b6 100644 --- a/gips/test/sys/t_aod.py +++ b/gips/test/sys/t_aod.py @@ -39,14 +39,6 @@ def setup_driver_data(pytestconfig): # ### SHOULD BE STANDARD BELOW HERE ##### -def t_inventory(setup_fixture, repo_env, expected): - """ - Test `gips_inventory` for {} and confirm recorded output is given. - """.format(driver) - actual = repo_env.run('gips_inventory', *STD_ARGS) - assert expected == actual - - def t_process(setup_fixture, repo_env, expected): """Test gips_process on {} data.""".format(driver) process_actual = repo_env.run('gips_process', *STD_ARGS) @@ -54,12 +46,6 @@ def t_process(setup_fixture, repo_env, expected): assert expected == process_actual and inventory_actual.std_out == expected._inv_stdout -def t_info(repo_env, expected): - """Test `gips_info {driver}` and confirm recorded output is given.""" - actual = repo_env.run('gips_info', driver) - assert expected == actual - - def t_project(setup_fixture, clean_repo_env, output_tfe, expected): """Test gips_project {} with warping.""".format(driver) args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld') diff --git a/gips/test/sys/t_landsat.py b/gips/test/sys/t_landsat.py index 60578bda..f783f4c5 100644 --- a/gips/test/sys/t_landsat.py +++ b/gips/test/sys/t_landsat.py @@ -44,12 +44,6 @@ def setup_landsat_data(pytestconfig): raise RuntimeError(err_msg) -def t_info(repo_env, expected): - """Test `gips_info modis` and confirm recorded output is given.""" - actual = repo_env.run('gips_info', 'landsat') - assert expected == actual - - def t_login(): """Test `gips_info modis` and confirm recorded output is given.""" from gips.data import landsat @@ -69,11 +63,6 @@ def t_query_service(): logger.debug(str(resp)) assert len(resp) == 1 -def t_inventory(setup_landsat_data, repo_env, expected): - """Test `gips_inventory` to confirm it notices the emplaced data file.""" - actual = repo_env.run('gips_inventory', *STD_ARGS) - assert expected == actual - @slow def t_process(setup_landsat_data, repo_env, expected): """Test gips_process on landsat data.""" diff --git a/gips/test/sys/t_merra.py b/gips/test/sys/t_merra.py index 18f56fad..f0707b83 100644 --- a/gips/test/sys/t_merra.py +++ b/gips/test/sys/t_merra.py @@ -16,7 +16,6 @@ STD_DATES = '2015-135' STD_ARGS = (driver, '-s', NH_SHP_PATH, '-d', STD_DATES, '-v', '4') - @pytest.fixture def setup_merra_data(pytestconfig): """Use gips_inventory to ensure presence of MERRA2 data in the data repo.""" @@ -31,25 +30,10 @@ def setup_merra_data(pytestconfig): raise RuntimeError("merra data setup via `gips_inventory` failed", outcome.std_out, outcome.std_err, outcome) - setup_fixture = setup_merra_data # ### SHOULD BE STANDARD BELOW HERE ##### - -def t_inventory(setup_fixture, repo_env, expected): - """ - Test `gips_inventory` for {} and confirm recorded output is given. - """.format(driver) - actual = repo_env.run('gips_inventory', *STD_ARGS) - assert expected == actual - - -def t_info(repo_env, expected): - """Test `gips_info {driver}` and confirm recorded output is given.""" - actual = repo_env.run('gips_info', driver) - assert expected == actual - # TODO revive this test? ''' def t_project_no_warp(setup_fixture, clean_repo_env, output_tfe, expected): @@ -59,7 +43,6 @@ def t_project_no_warp(setup_fixture, clean_repo_env, output_tfe, expected): assert expected == actual ''' - # Haven't used gips_tiles ever def t_tiles(setup_fixture, clean_repo_env, output_tfe, expected): """Test gips_tiles {} with warping.""".format(driver) @@ -67,7 +50,6 @@ def t_tiles(setup_fixture, clean_repo_env, output_tfe, expected): actual = output_tfe.run('gips_tiles', *args) assert expected == actual - def t_tiles_copy(setup_fixture, clean_repo_env, output_tfe, expected): """Test gips_tiles {} with copying.""".format(driver) # doesn't quite use STD_ARGS @@ -75,21 +57,3 @@ def t_tiles_copy(setup_fixture, clean_repo_env, output_tfe, expected): '--outdir', OUTPUT_DIR, '--notld') actual = output_tfe.run('gips_tiles', *args) assert expected == actual - -from .expected import merra as expectations - -@pytest.mark.parametrize("product", expectations.t_stats.keys()) -def t_stats(setup_fixture, export_wrapper, product): - """Test gips_stats on projected files.""" - record_mode, expected, runner = export_wrapper - - # generate data needed for stats computation - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, - '--notld', '-p', product) - outcome = sh.gips_project(*args) - assert outcome.exit_code == 0 # sanity check - - # compute & confirm stats - outcome, actual = runner('gips_stats', OUTPUT_DIR) - if not record_mode: - assert outcome.exit_code == 0 and expected == actual diff --git a/gips/test/sys/t_modis.py b/gips/test/sys/t_modis.py index 0b518045..29018ed4 100644 --- a/gips/test/sys/t_modis.py +++ b/gips/test/sys/t_modis.py @@ -40,16 +40,6 @@ def setup_modis_data(pytestconfig): raise RuntimeError("MODIS data setup via `gips_inventory` failed", outcome.std_out, outcome.std_err, outcome) -def t_inventory(setup_modis_data, repo_env, expected): - """Test `gips_inventory modis` and confirm recorded output is given.""" - actual = repo_env.run('gips_inventory', *STD_ARGS) - assert expected == actual - -def t_info(repo_env, expected): - """Test `gips_info modis` and confirm recorded output is given.""" - actual = repo_env.run('gips_info', 'modis') - assert expected == actual - # TODO keep this test? ''' def t_project_no_warp(setup_modis_data, clean_repo_env, output_tfe, expected): @@ -65,7 +55,6 @@ def t_tiles(setup_modis_data, clean_repo_env, output_tfe, expected): actual = output_tfe.run('gips_tiles', *args) assert expected == actual - def t_tiles_copy(setup_modis_data, clean_repo_env, output_tfe, expected): """Test gips_tiles modis with copying.""" # doesn't quite use STD_ARGS @@ -74,24 +63,6 @@ def t_tiles_copy(setup_modis_data, clean_repo_env, output_tfe, expected): actual = output_tfe.run('gips_tiles', *args) assert expected == actual -from .expected import modis as expectations - -@pytest.mark.parametrize("product", expectations.t_stats.keys()) -def t_stats(setup_modis_data, export_wrapper, product): - """Test gips_stats on projected files.""" - record_mode, expected, runner = export_wrapper - - # generate data needed for stats computation - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, - '--notld', '-p', product) - outcome = sh.gips_project(*args) - assert outcome.exit_code == 0 # sanity check - - # compute & confirm stats - outcome, actual = runner('gips_stats', OUTPUT_DIR) - if not record_mode: - assert outcome.exit_code == 0 and expected == actual - def t_gridded_export(setup_modis_data, clean_repo_env, output_tfe, expected): """Test gips_project using rastermask spatial spec""" rastermask = os.path.join(TEST_DATA_DIR, 'site_mask.tif') @@ -102,7 +73,6 @@ def t_gridded_export(setup_modis_data, clean_repo_env, output_tfe, expected): actual = output_tfe.run('gips_project', *args) assert expected == actual - def t_cubic_gridded_export(setup_modis_data, clean_repo_env, output_tfe, expected): """Test gips_project using rastermask spatial spec""" rastermask = os.path.join(TEST_DATA_DIR, 'site_mask.tif') diff --git a/gips/test/sys/t_prism.py b/gips/test/sys/t_prism.py index c1ed1d03..e0883239 100644 --- a/gips/test/sys/t_prism.py +++ b/gips/test/sys/t_prism.py @@ -35,15 +35,6 @@ def setup_prism_data(pytestconfig): # ### SHOULD BE STANDARD BELOW HERE ##### - -def t_inventory(setup_fixture, repo_env, expected): - """ - Test `gips_inventory` for {} and confirm recorded output is given. - """.format(driver) - actual = repo_env.run('gips_inventory', *STD_ARGS) - assert expected == actual - - def t_process(setup_fixture, repo_env, expected): """Test gips_process on {} data.""".format(driver) process_actual = repo_env.run('gips_process', *STD_ARGS) @@ -75,13 +66,6 @@ def t_process(setup_fixture, repo_env, expected): # assert expected == process_run2 # assert process_run1.timestamps != process_run2.timestamps - -def t_info(repo_env, expected): - """Test `gips_info {driver}` and confirm recorded output is given.""" - actual = repo_env.run('gips_info', driver) - assert expected == actual - - def t_project(setup_fixture, clean_repo_env, output_tfe, expected): """Test gips_project {} with warping.""".format(driver) args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld') diff --git a/gips/test/sys/t_sar.py b/gips/test/sys/t_sar.py index f2c13837..d8cd8734 100644 --- a/gips/test/sys/t_sar.py +++ b/gips/test/sys/t_sar.py @@ -82,24 +82,9 @@ def setup_sar_data(pytestconfig): # ### SHOULD BE STANDARD BELOW HERE ##### -def t_inventory(setup_fixture, repo_env, expected): - """ - Test `gips_inventory` for {} and confirm recorded output is given. - """.format(driver) - actual = repo_env.run('gips_inventory', *TILED_ARGS) - assert expected == actual - - def t_process(setup_fixture, repo_env, expected): """Test gips_process on {} data.""".format(driver) process_actual = repo_env.run('gips_process', *TILED_ARGS) inventory_actual = envoy.run('gips_inventory ' + ' '.join(TILED_ARGS)) assert expected == process_actual assert inventory_actual.std_out == expected._inv_stdout - - -def t_info(repo_env, expected): - """Test `gips_info {driver}` and confirm recorded output is given.""" - actual = repo_env.run('gips_info', driver) - assert expected == actual - diff --git a/gips/test/sys/t_sentinel2.py b/gips/test/sys/t_sentinel2.py index a1a604a6..6c8d43e4 100644 --- a/gips/test/sys/t_sentinel2.py +++ b/gips/test/sys/t_sentinel2.py @@ -39,13 +39,6 @@ def setup_fixture(pytestconfig): raise RuntimeError("Data setup via `gips_inventory` failed", outcome.std_out, outcome.std_err, outcome) - -def t_inventory(setup_fixture, repo_env, expected): - """Test `gips_inventory` and confirm recorded output is given.""" - actual = repo_env.run('gips_inventory', *STD_ARGS) - assert expected == actual - - @slow def t_process(setup_fixture, repo_env, expected): """Test `gips_process`, confirming products are created.""" @@ -54,13 +47,6 @@ def t_process(setup_fixture, repo_env, expected): assert (expected == process_actual and expected._inv_stdout == inventory_actual.std_out) - -def t_info(repo_env, expected): - """Test `gips_info` and confirm recorded output is given.""" - actual = repo_env.run('gips_info', driver) - assert expected == actual - - def t_project(setup_fixture, clean_repo_env, output_tfe, expected): """Test gips_project with warping.""" args = PROD_ARGS + ['--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld'] From 93baa58a7f5a147d410e7c36f529bf8feb63cc07 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Tue, 26 Dec 2017 16:52:23 -0500 Subject: [PATCH 19/44] 483 trash pointless aod process test --- gips/test/sys/driver_setup.py | 1 + gips/test/sys/expected/aod.py | 32 -------------------------------- gips/test/sys/t_aod.py | 7 ------- 3 files changed, 1 insertion(+), 39 deletions(-) diff --git a/gips/test/sys/driver_setup.py b/gips/test/sys/driver_setup.py index a9d08602..c56ac608 100644 --- a/gips/test/sys/driver_setup.py +++ b/gips/test/sys/driver_setup.py @@ -10,6 +10,7 @@ STD_ARGS = { 'modis': ('modis', '-s', util.NH_SHP_PATH, '-d', '2012-12-01,2012-12-03', '-v', '4'), 'merra': ('merra', '-s', util.NH_SHP_PATH, '-d', '2015-135', '-v', '4'), + 'aod': ('aod', '-s', util.NE_SHP_PATH, '-d', '2017-004,2017-006', '-v', '4'), } setup_attempted = [] diff --git a/gips/test/sys/expected/aod.py b/gips/test/sys/expected/aod.py index c6d3883f..befebb16 100644 --- a/gips/test/sys/expected/aod.py +++ b/gips/test/sys/expected/aod.py @@ -1,37 +1,5 @@ """Known-good outcomes for tests, mostly stdout and created files.""" -t_process = { - 'updated': { - }, - 'created': { - }, - 'ignored': [ - 'gips-inv-db.sqlite3', - ], - 'compare_stdout': False, - '_inv_stdout': """\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site NE-0 - -\x1b[1mAsset Coverage for site NE-0\x1b[0m -\x1b[1m -Tile Coverage -\x1b[4m Tile % Coverage % Tile Used\x1b[0m - h01v01 100.0% 100.0% - -\x1b[1m\x1b[4m DATE MOD08 Product \x1b[0m -\x1b[1m2017 -\x1b[0m 004 100.0% \x1b[35maod\x1b[0m - 005 100.0% \x1b[35maod\x1b[0m - 006 100.0% \x1b[35maod\x1b[0m - - -3 files on 3 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mMOD: MODIS Terra\x1b[0m -""", -} - t_project = { 'compare_stdout': False, 'compare_stderr': False, diff --git a/gips/test/sys/t_aod.py b/gips/test/sys/t_aod.py index 3e1868b6..217203ba 100644 --- a/gips/test/sys/t_aod.py +++ b/gips/test/sys/t_aod.py @@ -39,13 +39,6 @@ def setup_driver_data(pytestconfig): # ### SHOULD BE STANDARD BELOW HERE ##### -def t_process(setup_fixture, repo_env, expected): - """Test gips_process on {} data.""".format(driver) - process_actual = repo_env.run('gips_process', *STD_ARGS) - inventory_actual = envoy.run('gips_inventory ' + ' '.join(STD_ARGS)) - assert expected == process_actual and inventory_actual.std_out == expected._inv_stdout - - def t_project(setup_fixture, clean_repo_env, output_tfe, expected): """Test gips_project {} with warping.""".format(driver) args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld') From 625bd6b187e9f8e5b5ed4a99b9a166413f5fc537 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 27 Dec 2017 09:24:49 -0500 Subject: [PATCH 20/44] 483 generify the directory of interest: Tests can't rely on a directory having a specific name like 'data-root'; that can change from user to user. --- gips/test/sys/t_process.py | 154 ++++++++++++++++++------------------- gips/test/sys/util.py | 34 ++++---- 2 files changed, 94 insertions(+), 94 deletions(-) diff --git a/gips/test/sys/t_process.py b/gips/test/sys/t_process.py index 67c8f034..0db2039e 100644 --- a/gips/test/sys/t_process.py +++ b/gips/test/sys/t_process.py @@ -24,322 +24,322 @@ def t_process(repo_wrapper, driver, product): # weird symlink products: # t_process[quality] recording: 'quality': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_quality.tif', + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_quality.tif', 'symlink', 'HDF4_EOS:EOS_GRID:"', - '/data-root/modis/tiles/h12v04/2012336/MCD43A2.A2012336.h12v04.006.2016112010833.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_quality.tif', + '/modis/tiles/h12v04/2012336/MCD43A2.A2012336.h12v04.006.2016112010833.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_quality.tif', 'symlink', 'HDF4_EOS:EOS_GRID:"', - '/data-root/modis/tiles/h12v04/2012337/MCD43A2.A2012337.h12v04.006.2016112013509.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_quality.tif', + '/modis/tiles/h12v04/2012337/MCD43A2.A2012337.h12v04.006.2016112013509.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_quality.tif', 'symlink', 'HDF4_EOS:EOS_GRID:"', - '/data-root/modis/tiles/h12v04/2012338/MCD43A2.A2012338.h12v04.006.2016112020013.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo')], + '/modis/tiles/h12v04/2012338/MCD43A2.A2012338.h12v04.006.2016112020013.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo')], # t_process[temp8tn] recording: 'temp8tn': - [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8tn.tif', + [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8tn.tif', 'symlink', 'HDF4_EOS:EOS_GRID:"', - '/data-root/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Night_1km')], + '/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Night_1km')], # t_process[temp8td] recording: 'temp8td': - [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8td.tif', + [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8td.tif', 'symlink', 'HDF4_EOS:EOS_GRID:"', - '/data-root/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Day_1km')], + '/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Day_1km')], # normal products # t_process[satvi] recording: 'satvi': - [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_satvi.tif', + [('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_satvi.tif', 'hash', 'sha256', '94bee7cecf0f0a227fd8532223f86de10e01c3e39d1dd70b7675d91b3ca338a4'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_satvi.tif', + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_satvi.tif', 'hash', 'sha256', '6397669a6cd62bb39813b9000d93ba50f4cabe73aebf1e6444f4e3be9f630e19'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_satvi.tif', + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_satvi.tif', 'hash', 'sha256', '6124ee615a4f522fdf966a5479d03d87934908e5090a192192f9700020f24760')], # t_process[ndti] recording: 'ndti': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndti.tif', + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndti.tif', 'hash', 'sha256', '025ea7a9204ae8ad0952a92b386bf735dbbce8337e336843476d3f227cf1f1ab'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndti.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndti.tif', 'hash', 'sha256', 'b6baa8c7a3996f680c4177e4157c9275dd895c8e6350afa6ef564e673bb6a5f0'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndti.tif', + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndti.tif', 'hash', 'sha256', 'a1c4f1161e8a584257352e76160319c39aa4605dd565d388d0116e193773771f')], # t_process[ndvi] recording: 'ndvi': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndvi.tif', + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndvi.tif', 'hash', 'sha256', 'efbeae3a1c5e59ccd54b6108e275f9a8f0059162ac2797f4d69b00acb80147b3'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndvi.tif', + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndvi.tif', 'hash', 'sha256', 'b56d5c96229a1ad4cdcbd09db10b19a216b4cde8a0ffd184ee5cf01a35ab2b46'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndvi.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndvi.tif', 'hash', 'sha256', 'd9ff5d2f85ac9a267a8f6b1aca34f62b8196721e212b1e092f82ce9f1e208c61')], # t_process[isti] recording: 'isti': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_isti.tif', + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_isti.tif', 'hash', 'sha256', 'c0a38d1cd00de52eacd8abcda714bfdaa575787e27b40dc0472ffca5ab1696c0'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_isti.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_isti.tif', 'hash', 'sha256', 'b1759c1008e5914b955c4a9f55572baa8f7c0310677264a1057329e236d826cb'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_isti.tif', + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_isti.tif', 'hash', 'sha256', '677182d186dac992b52b93239af0f419536075c7cd467864fa544bcde4b89e69')], # t_process[ndvi8] recording: 'ndvi8': - [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_ndvi8.tif', + [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_ndvi8.tif', 'hash', 'sha256', '93db24c18886fbac0b87d1404a0c80606a599cc55b02d745a09ffb051b8131ac')], # t_process[snow] recording: 'snow': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_snow.tif', + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_snow.tif', 'hash', 'sha256', 'bea7315b736a75bcf37df86089679845ac597f3fba5d5cd6e79f5516b9657faa'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_snow.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_snow.tif', 'hash', 'sha256', 'de979c1a0c616d7889dd5fd45816849f9c08df54167d7f3b40301d6ac4495778'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_snow.tif', + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_snow.tif', 'hash', 'sha256', '5d9bbfff1522e74165cdd677f1580e9eb2236c2c5922a796a6c9ee0db24ae05a')], # t_process[fsnow] recording: 'fsnow': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_fsnow.tif', + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_fsnow.tif', 'hash', 'sha256', '853a5ca95f21c4a745e41e4acc46a71a74c74e0cdd1cae9ed3dd31b2fef1ddbd'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_fsnow.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_fsnow.tif', 'hash', 'sha256', '3a5f7c95fe60e5789919cf55d7f8e652e4d668c6eb63eaa53a82249c0ea19c5a'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_fsnow.tif', + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_fsnow.tif', 'hash', 'sha256', 'd09b8a12ad17a7e715ae2e988fbf683e5fe2df46e97e9ac3ebb4444c8b1c169d')], # t_process[crcm] recording: 'crcm': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crcm.tif', + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crcm.tif', 'hash', 'sha256', 'a45c2fb16d871a919e0c803f80e1f3b4ec831472d0466c33741fcf8e8150f6ba'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crcm.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crcm.tif', 'hash', 'sha256', '9aa80c0e1fa1b62b0a41aed42eeaf84632845d74f29594dfee55c22673b0a6f5'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crcm.tif', + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crcm.tif', 'hash', 'sha256', 'a28c7d280fb8dfe0fe1cc003e1001457abdf14acc5545b2a69b782cfbcce7b50')], # t_process[ndsi] recording: 'ndsi': - [('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndsi.tif', + [('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndsi.tif', 'hash', 'sha256', '1f91f738bbb28b3ebeba36affd8d0689e105e90e1547cca58497eb7c7fdf7346'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndsi.tif', + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndsi.tif', 'hash', 'sha256', 'a626d375fbe5094a5199fdfc2206cf94777ff574379b45811cf7a49055a9f6ec'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndsi.tif', + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndsi.tif', 'hash', 'sha256', 'cb2ca7461c65f66ddb13e804ae7fe8f54daaf620ade600efee24810d40d4eb34')], # t_process[brgt] recording: 'brgt': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_brgt.tif', + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_brgt.tif', 'hash', 'sha256', 'bf6bad538c7c6e703bc2f39503c2f128fa8e988a533e302c0124e085735dfaf9'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_brgt.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_brgt.tif', 'hash', 'sha256', '5c72e8f37640ec6b4a5a33e18980d71c70482fc8f708ba25da73e074cf64729f'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_brgt.tif', + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_brgt.tif', 'hash', 'sha256', '3460d0d35e35540b55e9eda35f253af8d85eabb84c0c44cb5f9c02d43da7e20a')], # t_process[msavi2] recording: 'msavi2': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_msavi2.tif', + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_msavi2.tif', 'hash', 'sha256', 'd2c93798df10939ff8afb6cd23e18ac3281cfd844e45da47a29d0342c2b980e9'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_msavi2.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_msavi2.tif', 'hash', 'sha256', '92fe4f40d051ef3f479bde76a67bfff978bc513893af5ed62e4a48e457daabad'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_msavi2.tif', + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_msavi2.tif', 'hash', 'sha256', 'f25bd2d58d3edae72b11e73c08341479dee13af80352a4749861b6a54a81b351')], # t_process[bi] recording: 'bi': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_bi.tif', + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_bi.tif', 'hash', 'sha256', '28c181b8d210b71759589da53295cc894c9d12c74ff1f7b0a49b88f73664282e'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_bi.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_bi.tif', 'hash', 'sha256', '45acfc9249e063b6b97f269338590d51e2334e9bc5df118613ad908c6f0a3f8e'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_bi.tif', + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_bi.tif', 'hash', 'sha256', '8526cb96d59fc9549dc8535fc8128123b9d39a63a3b7cd97ee3e611b7eeadf6f')], # t_process[obstime] recording: 'obstime': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_obstime.tif', + [('modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_obstime.tif', 'hash', 'sha256', '66e01c94910c4fdd4f6fb2ab99b0c2d0cb73618032783e3cbdf1867191981fdc'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_obstime.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_obstime.tif', 'hash', 'sha256', '117cea0497b3158b465dd7dd197c886ebb1acf5059ed83dfdd6b5d7e94bb5243'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_obstime.tif', + ('modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_obstime.tif', 'hash', 'sha256', 'b04d8b07ef83e479db5650a30c84513bde7d5b05dd68187782c057ea0fd3e537')], # t_process[lswi] recording: 'lswi': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_lswi.tif', + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_lswi.tif', 'hash', 'sha256', '8de7378cdba830171c8da94424bf7807fa6c8b12523e8b34175a4b17ec770a5d'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_lswi.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_lswi.tif', 'hash', 'sha256', '62de4bdafcc276e9c1c343f4e63f5dd06167e87f7712796f991e3a957f2b71de'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_lswi.tif', + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_lswi.tif', 'hash', 'sha256', 'add051ef35d7ffff5eb68c48f8b8071f9cb493b5d3426867605d4c61746e3a26')], # t_process[vari] recording: 'vari': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_vari.tif', + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_vari.tif', 'hash', 'sha256', '732d6d3d9fbe0e1b4616a3f8a91bf12d532598a0a02e8aab16c9a01c3ac56913'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_vari.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_vari.tif', 'hash', 'sha256', '9875717cc5e1926f11378bbf16345dbade5a2badf9f3dd06844a77f94d80ef72'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_vari.tif', + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_vari.tif', 'hash', 'sha256', '70fd19862ef4145c24f26a8a74cb748279b88fd973de935726ebef2294e17edd')], # t_process[evi] recording: 'evi': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_evi.tif', + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_evi.tif', 'hash', 'sha256', '13548ae74456f08e6add657a637ca364626eafd2cc767724f57aac8ff93005bb'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_evi.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_evi.tif', 'hash', 'sha256', '7b926f7004cd92f2ec604fc4e2cc1489f60a335e39f4954030332a012fd33f87'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_evi.tif', + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_evi.tif', 'hash', 'sha256', '51db813a857f795817f529edd56d7134004566be46edab8a8dadc81286fb895f')], # t_process[clouds] recording: 'clouds': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD_clouds.tif', + [('modis/tiles/h12v04/2012338/h12v04_2012338_MOD_clouds.tif', 'hash', 'sha256', '26b3e7e695f08ec2e9fb26985f9d5e392f9e8cedc331ab6721ea8e48821d93f0'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD_clouds.tif', + ('modis/tiles/h12v04/2012336/h12v04_2012336_MOD_clouds.tif', 'hash', 'sha256', 'c43eec8178d0521d20e2a5b6e9b61525007d80702c6dd705a3e52ed3a76a7b73'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD_clouds.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_clouds.tif', 'hash', 'sha256', '24a6a7050c866eed989530bec45b72004097f1e36fcc0e686652b87723a86de4')], # t_process[temp] recording: 'temp': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_temp.tif', + [('modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_temp.tif', 'hash', 'sha256', 'ead2622531929bba3192b8a6b609d5d920e79394ec577c5245d74d69593d827c'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_temp.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_temp.tif', 'hash', 'sha256', '5d498bad2779f84289dfb75b14345d64d16b5bc7994b4aca6cd694f70b86064d'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_temp.tif', + ('modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_temp.tif', 'hash', 'sha256', '097f341fa917a0ec316c37d7907ff15b754dcd0925469be586016137ca47575f')], # t_process[sti] recording: 'sti': - [('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_sti.tif', + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_sti.tif', 'hash', 'sha256', '24e50766ec18826f30bbad07bc8f5a34e72f1966b325cc23f064aa5fd35dc24a'), - ('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_sti.tif', + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_sti.tif', 'hash', 'sha256', 'c223de9d00e9cfedc0cfc13a45163535f3c59e5587642fa2c725b00b17b7437c'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_sti.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_sti.tif', 'hash', 'sha256', '73c247fd2c7623e79145b6337846c97d14d1752d660b2f32245a4ccaac033350')], # t_process[crc] recording: 'crc': - [('data-root/modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crc.tif', + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crc.tif', 'hash', 'sha256', '07debf171aedfeba7d6c1e3220ff8e4a2a7b1033e68c72bbc121b500b6ed3c9c'), - ('data-root/modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crc.tif', + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crc.tif', 'hash', 'sha256', 'a7b61aabc950be7a3040b6f9572bf755522b12ad27644583e603def05ac1980b'), - ('data-root/modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crc.tif', + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crc.tif', 'hash', 'sha256', '942efeeee2b020123be6d7334bbb3ef5995c93d2fd0247cb79948836865c8067')], @@ -349,55 +349,55 @@ def t_process(repo_wrapper, driver, product): # test this too? 'frland': [], # t_process[srad] recording: 'srad': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_srad.tif', + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_srad.tif', 'hash', 'sha256', '882a44af70e337bf905c8695936f101792a518507a4304b23781a2f30aaabab4')], # t_process[tave] recording: 'tave': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tave.tif', + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tave.tif', 'hash', 'sha256', '5185aebd7cda54157cad2ddbde9f6fec4871c1b85fe78d0738634ba0211f2c9b')], # t_process[shum] recording: 'shum': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_shum.tif', + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_shum.tif', 'hash', 'sha256', '558f567c1e931891553d396f81b2b90929ad48a0ae88fa95d3894eae23ab3eba')], # t_process[rhum] recording: 'rhum': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_rhum.tif', + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_rhum.tif', 'hash', 'sha256', '058161eb8488f1fb3df7f5c8719833c9d68c56aa082d4605938f62c2ee0035f6')], # t_process[tmin] recording: 'tmin': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmin.tif', + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmin.tif', 'hash', 'sha256', '35a850769a3bb8f5209574ea4835bf417d8f99024567f4af9cdee9e623d0c567')], # t_process[tmax] recording: 'tmax': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmax.tif', + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmax.tif', 'hash', 'sha256', '5963fcf346f388fc347355153996a078f1eb5ecbbc094394b72be382ba491865')], # t_process[prcp] recording: 'prcp': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_prcp.tif', + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_prcp.tif', 'hash', 'sha256', '348fd7072ef0a8625f5268e29acbf3a69a959a412327102ef6558c992e62bc9f')], # t_process[patm] recording: 'patm': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_patm.tif', + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_patm.tif', 'hash', 'sha256', '7d9deca613973cb8ffb8f15a3dfa2013e6783556b520915e89b97fe41de638e3')], # t_process[wind] recording: 'wind': - [('data-root/merra/tiles/h01v01/2015135/h01v01_2015135_merra_wind.tif', + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_wind.tif', 'hash', 'sha256', 'a90c1d87fdb1c0926bd5b3004408a1f9d42ef08c38deb9d0572bc7536dbbf08a')], diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index b65c793a..33b82393 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -241,40 +241,41 @@ def generate_expectation(filename, base_path): """ # may want to try python-magic at some point: https://github.com/ahupp/python-magic # but for now going by file extension seems to be sufficient. + full_path = os.path.join(base_path, filename) # have to use lexists to cover for foolish abuse of symlinks by GDAL et al - if not os.path.lexists(filename): + if not os.path.lexists(full_path): return (filename, 'absent') # symlinks - if os.path.islink(filename): - # vvvvvvvvvvvvvvvvvvvvvv--- have to rmeove this bit + if os.path.islink(full_path): + # have to rmeove this non-generic bit + # vvvvvvvvvvvvvvvvvvvvvvvvvvvvvv # HDF4_EOS:EOS_GRID:"/home/tolson/src/gips/data-root/modis/tiles/h12v04/2012337 # /MCD43A2.A2012337.h12v04.006.2016112013509.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo # when recording the path, do '/foo/bar/baz/file.tif' - link_target = os.readlink(filename) - separator = os.path.dirname(base_path) - prefix, suffix = link_target.split(separator) + link_target = os.readlink(full_path) + prefix, suffix = link_target.split(base_path) return (filename, 'symlink', prefix, suffix) # text files - size_threshold = 80 * 25 # about one terminal-screen if filename.endswith('.txt'): - f_size = os.stat(filename).st_size + size_threshold = 80 * 25 # about one terminal-screen + f_size = os.stat(full_path).st_size if f_size <= size_threshold: # give up if there's too much text - with open(filename) as fo: + with open(full_path) as fo: lines = fo.readlines() return (filename, 'text-full', lines) print('{} ({} bytes) exceeds max supported size for text diffs' ' ({} bytes); defaulting to checksum'.format( - filename, f_size, size_threshold)) + full_path, f_size, size_threshold)) # product TIFFs if filename.endswith('.tif'): pass # TODO https://github.com/Applied-GeoSolutions/gips/issues/463 # use a hash as a fallback - return (filename, 'hash', 'sha256', generate_file_hash(filename)) + return (filename, 'hash', 'sha256', generate_file_hash(full_path)) def generate_file_hash(filename, blocksize=2**20): # stolen from SO: https://stackoverflow.com/questions/1131220/get-md5-hash-of-big-files-in-python @@ -302,14 +303,15 @@ def sys_test_wrapper(request, path): that is a dict of dicts, and the test getting a 'driver' and 'product' param. """ - rp = record_path() + rp = record_path() # does the user want record mode? If so, save it where? product = request.getfixturevalue('product') p = request.node.callspec.params expected = request.module.expectations[p['driver']][p['product']] expected_filenames = [e[0] for e in expected] if not rp: - interlopers = [fn for fn in expected_filenames if os.path.exists(fn)] + interlopers = [fn for fn in expected_filenames + if os.path.exists(os.path.join(path, fn))] if interlopers: raise IOError('Files in the way of the test: {}'.format(interlopers)) @@ -328,10 +330,8 @@ def wrapped_runner(cmd_string, *args): if rp: final_files = find_files(path) created_files = set(final_files) - set(initial_files) - - # when recording the path, do '/foo/bar/baz/file.tif' - relpath_start = os.path.dirname(path) - rel_cf = [os.path.relpath(fp, relpath_start) for fp in created_files] + # when recording the path, don't capture the directory of interest + rel_cf = [os.path.relpath(fp, path) for fp in created_files] cf_expectations = [generate_expectation(fn, path) for fn in rel_cf] print("Recording {} outcome to {}.".format(product, rp)) From cd786c367e189ae8386d2efef4af3dc6db5917c8 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 27 Dec 2017 09:31:33 -0500 Subject: [PATCH 21/44] 483 generify t_project project dir --- gips/test/sys/t_project.py | 144 ++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/gips/test/sys/t_project.py b/gips/test/sys/t_project.py index 4c4184d9..662abfad 100644 --- a/gips/test/sys/t_project.py +++ b/gips/test/sys/t_project.py @@ -22,321 +22,321 @@ def t_project(export_wrapper, driver, product): expectations['modis'] = { # t_project[satvi] recording: 'satvi': - [('output/0/2012336_MCD_satvi.tif', + [('0/2012336_MCD_satvi.tif', 'hash', 'sha256', '893d968ff6359d0d34f8a587246690d1c724832505d66fbee8be8c1a43a6e3df'), - ('output/0/2012337_MCD_satvi.tif', + ('0/2012337_MCD_satvi.tif', 'hash', 'sha256', '6092d7cfb609f000124b8ef107f278decaa27c3cdb69696bd094940ead97d96e'), - ('output/0/2012338_MCD_satvi.tif', + ('0/2012338_MCD_satvi.tif', 'hash', 'sha256', '37dd32e0f5e761846f44615ec6e82e2cce8d6c98197f0bd721dcfb266d963b6b')], # t_project[ndti] recording: 'ndti': - [('output/0/2012337_MCD_ndti.tif', + [('0/2012337_MCD_ndti.tif', 'hash', 'sha256', 'db85948227815660b94d570adf0f1119d9f1001284c3de6c8b219c6455bd3c4e'), - ('output/0/2012338_MCD_ndti.tif', + ('0/2012338_MCD_ndti.tif', 'hash', 'sha256', '5028c194e41d8c07076cee09631b276ef7b86c3c45b0a4f705f5e4c6b72c2812'), - ('output/0/2012336_MCD_ndti.tif', + ('0/2012336_MCD_ndti.tif', 'hash', 'sha256', '157134ea0e9cbb0b2fcdb4203e17e82ae906a5779eb1f735e8c55d1f60c231e1')], # t_project[ndvi] recording: 'ndvi': - [('output/0/2012337_MCD_ndvi.tif', + [('0/2012337_MCD_ndvi.tif', 'hash', 'sha256', '28a288720a0846348fbd6315e5d9e3801ad76eb2c873dc028e7a52d1fdcdad10'), - ('output/0/2012336_MCD_ndvi.tif', + ('0/2012336_MCD_ndvi.tif', 'hash', 'sha256', '78e090b1cc92304c666a8e480d11fa99985ee3d2113c4fe3be29fb85910b86b9'), - ('output/0/2012338_MCD_ndvi.tif', + ('0/2012338_MCD_ndvi.tif', 'hash', 'sha256', 'bbcee4ac1b49466c90264e305c20ce19a021fd177ffc35da5ed92a86555c480e')], # t_project[quality] recording: 'quality': - [('output/0/2012338_MCD_quality.tif', + [('0/2012338_MCD_quality.tif', 'hash', 'sha256', '895abfc825d9da56c56202c670f94a16b28f2aaab5590db55ac6e903b6638e91'), - ('output/0/2012336_MCD_quality.tif', + ('0/2012336_MCD_quality.tif', 'hash', 'sha256', '4f40fdeb7ded507a68f404ec34a17fb6283c9b90d81ecccc7782746eaba85323'), - ('output/0/2012337_MCD_quality.tif', + ('0/2012337_MCD_quality.tif', 'hash', 'sha256', '5cdd315b2a423fbb82c19cfe859325c889a5d05cf0c3cdf71359b257524175a3')], # t_project[isti] recording: 'isti': - [('output/0/2012337_MCD_isti.tif', + [('0/2012337_MCD_isti.tif', 'hash', 'sha256', '391367ee5bdd0a96435c09df85db312919121c5d9a28b5bda59a68db7b609871'), - ('output/0/2012338_MCD_isti.tif', + ('0/2012338_MCD_isti.tif', 'hash', 'sha256', 'cc082d45a2a4dd4a7e53d0970029decf24ce5033b9715930cd250b524ad87757'), - ('output/0/2012336_MCD_isti.tif', + ('0/2012336_MCD_isti.tif', 'hash', 'sha256', '16be9705c06a44f3a35e684cded2d2ceb26213510eaf0ccfbafb62ebc39bb153')], # t_project[ndvi8] recording: 'ndvi8': - [('output/0/2012337_MOD_ndvi8.tif', + [('0/2012337_MOD_ndvi8.tif', 'hash', 'sha256', '23a72df39da253ac07de5a9e98c88f40e3335f3b1b779ccf91153d1d5573a780')], # t_project[snow] recording: 'snow': - [('output/0/2012336_MCD_snow.tif', + [('0/2012336_MCD_snow.tif', 'hash', 'sha256', 'b3eeca8fc933d2f43c2d96e869956447d9556737181635167744d0d420dfd813'), - ('output/0/2012338_MCD_snow.tif', + ('0/2012338_MCD_snow.tif', 'hash', 'sha256', '98a38e1a75fcea269bd75a952cde9d84af86b14ddb928e4777ff812dba8befbe'), - ('output/0/2012337_MCD_snow.tif', + ('0/2012337_MCD_snow.tif', 'hash', 'sha256', 'da26696be559bb79c399ad18942ac629a9604d857dd7fc461fd76c4577a06303')], # t_project[fsnow] recording: 'fsnow': - [('output/0/2012338_MCD_fsnow.tif', + [('0/2012338_MCD_fsnow.tif', 'hash', 'sha256', 'f2efdf504e004caf05563f3402fdde7e771351d7d93de5970af0a54d0b5bd2c2'), - ('output/0/2012336_MCD_fsnow.tif', + ('0/2012336_MCD_fsnow.tif', 'hash', 'sha256', 'a9d2ea29a6d77325246e0baaf298d16cb84d6003a19faed8ba162b475703f563'), - ('output/0/2012337_MCD_fsnow.tif', + ('0/2012337_MCD_fsnow.tif', 'hash', 'sha256', 'e8465aa3764c60b21a943f3e078d93c0484272292f075455693ea2084971ace4')], # t_project[crcm] recording: 'crcm': - [('output/0/2012337_MCD_crcm.tif', + [('0/2012337_MCD_crcm.tif', 'hash', 'sha256', 'c00e7b7c6970ae23904ee52afd614a8871f3d771d26830a2aef412163d9d9228'), - ('output/0/2012338_MCD_crcm.tif', + ('0/2012338_MCD_crcm.tif', 'hash', 'sha256', 'ed336a3ba4cf482936ca781ea1a5e0f04ca5ad68919c3dbe6e117edac55ec242'), - ('output/0/2012336_MCD_crcm.tif', + ('0/2012336_MCD_crcm.tif', 'hash', 'sha256', 'f217a9278b195e945e2337663cf7dfe2b09faa0dc68062fbc3614a6954866eaf')], # t_project[ndsi] recording: 'ndsi': - [('output/0/2012338_MCD_ndsi.tif', + [('0/2012338_MCD_ndsi.tif', 'hash', 'sha256', '8e26a8f6d78b9612c8cd3924936c5944f56c67ae1c63f6541d045b41a27baafe'), - ('output/0/2012337_MCD_ndsi.tif', + ('0/2012337_MCD_ndsi.tif', 'hash', 'sha256', '2aaea763ab1d5b54d7079e7ee2381bd987a3c72c5cbbd6f18f97aa320c186e2a'), - ('output/0/2012336_MCD_ndsi.tif', + ('0/2012336_MCD_ndsi.tif', 'hash', 'sha256', '83b6bbf241d080cf6860798da299038be5e6d3e20a8b544e5af51e2eefd3a5c9')], # t_project[brgt] recording: 'brgt': - [('output/0/2012337_MCD_brgt.tif', + [('0/2012337_MCD_brgt.tif', 'hash', 'sha256', 'fc385e6e42538a09655762d1b2557abe35af38aecef1571705cec8b971efbe4d'), - ('output/0/2012338_MCD_brgt.tif', + ('0/2012338_MCD_brgt.tif', 'hash', 'sha256', 'a8f8ef66bc87c2370bd61d5eed64905ba4f832d05579687478ff5f38c65e8038'), - ('output/0/2012336_MCD_brgt.tif', + ('0/2012336_MCD_brgt.tif', 'hash', 'sha256', '24f8130de39daac6e181cb2f98b48d9a14412700632d3cbd2ffbe2704283b740')], # t_project[msavi2] recording: 'msavi2': - [('output/0/2012336_MCD_msavi2.tif', + [('0/2012336_MCD_msavi2.tif', 'hash', 'sha256', '997b562420fe223677881d577cdb2100ee653ee932b9989b95f3089c0fe38352'), - ('output/0/2012338_MCD_msavi2.tif', + ('0/2012338_MCD_msavi2.tif', 'hash', 'sha256', '26a7ab45723856361d108064fe0c810535de75f272a733d86f66eeb8146dee35'), - ('output/0/2012337_MCD_msavi2.tif', + ('0/2012337_MCD_msavi2.tif', 'hash', 'sha256', 'ee7b2006b79f65bdf11d77f678d66765abfcc024b7856ab296f855570efb9940')], # t_project[bi] recording: 'bi': - [('output/0/2012336_MCD_bi.tif', + [('0/2012336_MCD_bi.tif', 'hash', 'sha256', 'e5f891a6290301508107714657274fc808fcf96cb138c652a9c22552675796a5'), - ('output/0/2012337_MCD_bi.tif', + ('0/2012337_MCD_bi.tif', 'hash', 'sha256', '58b9e3809630ba24879bb85e81bc6dc0eed548348373588d70d49de65d246582'), - ('output/0/2012338_MCD_bi.tif', + ('0/2012338_MCD_bi.tif', 'hash', 'sha256', '914ca6d15c25a22d3e0a43d0d266f72bd65d102b4d6b015a9f12b8ee3a116c13')], # t_project[obstime] recording: 'obstime': - [('output/0/2012337_MOD-MYD_obstime.tif', + [('0/2012337_MOD-MYD_obstime.tif', 'hash', 'sha256', 'a9345d21f3d754d083d3b4d13b059f2ffe07eb42ea9a0d6f3e40ef326e4ea75f'), - ('output/0/2012336_MOD-MYD_obstime.tif', + ('0/2012336_MOD-MYD_obstime.tif', 'hash', 'sha256', 'fac305543c4afbf76611900579cd5f37d63e7a45814c997dce40fc46a74f13e5'), - ('output/0/2012338_MOD-MYD_obstime.tif', + ('0/2012338_MOD-MYD_obstime.tif', 'hash', 'sha256', 'e18456caaf9a2dd005cc6568f7aa77ac534bfa24ec364d3af28affb4ae652de9')], # t_project[lswi] recording: 'lswi': - [('output/0/2012337_MCD_lswi.tif', + [('0/2012337_MCD_lswi.tif', 'hash', 'sha256', '586dd3782070b983b9d383161bed82bee21c54ad312d1c993b0329965144deab'), - ('output/0/2012336_MCD_lswi.tif', + ('0/2012336_MCD_lswi.tif', 'hash', 'sha256', '296c849e70d31127bcefef54ffed72832529f7ae60d50b7c639c5c6d253d30f4'), - ('output/0/2012338_MCD_lswi.tif', + ('0/2012338_MCD_lswi.tif', 'hash', 'sha256', '992fa3d7b4bdc8cf95fba759a0096f9a913e84762717e93c477baae0d7d3c0ed')], # t_project[vari] recording: 'vari': - [('output/0/2012336_MCD_vari.tif', + [('0/2012336_MCD_vari.tif', 'hash', 'sha256', 'be5d2273b9f0b9aca79df89b66ca0e7e1a2b8d386e6d34a6eca6eb37810b488d'), - ('output/0/2012338_MCD_vari.tif', + ('0/2012338_MCD_vari.tif', 'hash', 'sha256', '016ec5b1ee512d9306251a4c2b3630c90ea18723159a6a254c8e77c4e730abc8'), - ('output/0/2012337_MCD_vari.tif', + ('0/2012337_MCD_vari.tif', 'hash', 'sha256', 'a6c10e80436b92908637b22b40e7993a0a648e82b96993760a7b26ba43df0667')], # t_project[evi] recording: 'evi': - [('output/0/2012338_MCD_evi.tif', + [('0/2012338_MCD_evi.tif', 'hash', 'sha256', '8a1beba0730ac7d113a7edb8b55035a742892bbe678cd0eaaa540e59906a6990'), - ('output/0/2012337_MCD_evi.tif', + ('0/2012337_MCD_evi.tif', 'hash', 'sha256', '824a5247b0e8e0730dec6e82b390533649aa796ef04ef4be27ac38cae9a24d04'), - ('output/0/2012336_MCD_evi.tif', + ('0/2012336_MCD_evi.tif', 'hash', 'sha256', '361798060d02c97153a9828c1f8ac6b7df396070ec243932e9ed18fd472e9669')], # t_project[temp8tn] recording: 'temp8tn': - [('output/0/2012337_MOD_temp8tn.tif', + [('0/2012337_MOD_temp8tn.tif', 'hash', 'sha256', '1600d412e2590b4f4006ac0d98ebf9c6bf83fe16967c2fd721e412d2a14040c6')], # t_project[clouds] recording: 'clouds': - [('output/0/2012337_MOD_clouds.tif', + [('0/2012337_MOD_clouds.tif', 'hash', 'sha256', '8b13afeec6d1a5612a3eeb51896507e2d4eeb851261881f3854d2b804e97835f'), - ('output/0/2012336_MOD_clouds.tif', + ('0/2012336_MOD_clouds.tif', 'hash', 'sha256', 'cb4fab2519652bebe4830479c95425050f63d519397beb5b0c40349cc6f49372'), - ('output/0/2012338_MOD_clouds.tif', + ('0/2012338_MOD_clouds.tif', 'hash', 'sha256', '54505d6d4063caa27fda74f5fdbb916c20d281f6e15deea28596c379ffa661e2')], # t_project[temp] recording: 'temp': - [('output/0/2012336_MOD-MYD_temp.tif', + [('0/2012336_MOD-MYD_temp.tif', 'hash', 'sha256', '614023db948d0b0249057ca9c458356c2222e45224a726f87a7cf512cd87745c'), - ('output/0/2012338_MOD-MYD_temp.tif', + ('0/2012338_MOD-MYD_temp.tif', 'hash', 'sha256', '51864699b8347a30d9669ec41298665cd893a27516089cc21b4c3f03b3e65b2b'), - ('output/0/2012337_MOD-MYD_temp.tif', + ('0/2012337_MOD-MYD_temp.tif', 'hash', 'sha256', '29373079984e8aaf55add6a1fdd7bd766eb89a1c74ebc85674da54aeb643e3ef')], # t_project[temp8td] recording: 'temp8td': - [('output/0/2012337_MOD_temp8td.tif', + [('0/2012337_MOD_temp8td.tif', 'hash', 'sha256', 'eaa0ff94bb34cbad8d9c187b16c7cd2bd5948d1d94cf1caf30b0396ac7e8a735')], # t_project[sti] recording: 'sti': - [('output/0/2012337_MCD_sti.tif', + [('0/2012337_MCD_sti.tif', 'hash', 'sha256', '9850777cae46b42fa3c8361849529e6f230dd1ce86fb6c6676aa8eb2768e71d5'), - ('output/0/2012336_MCD_sti.tif', + ('0/2012336_MCD_sti.tif', 'hash', 'sha256', 'b7fde6a04ac091946553b25b1a26de806fa7f9c23e915b415034ef811aca1d16'), - ('output/0/2012338_MCD_sti.tif', + ('0/2012338_MCD_sti.tif', 'hash', 'sha256', 'b2714cadf4e779c433415f194f366ac87877c60b383f46971843b4ab3a610fa9')], # t_project[crc] recording: 'crc': - [('output/0/2012338_MCD_crc.tif', + [('0/2012338_MCD_crc.tif', 'hash', 'sha256', 'dc9a4885e694fe7117cf10df11b5589745b4eb119d5f58f9d1c14e27d0eda333'), - ('output/0/2012337_MCD_crc.tif', + ('0/2012337_MCD_crc.tif', 'hash', 'sha256', '5688d80a613c28c657237186291a88ae786037a77dad0247c0bd734da214c927'), - ('output/0/2012336_MCD_crc.tif', + ('0/2012336_MCD_crc.tif', 'hash', 'sha256', '3df0003b6153970e73c46d5e999697ce5af268744274ccd3a4a2c14e3ae932cb')], @@ -345,63 +345,63 @@ def t_project(export_wrapper, driver, product): expectations['merra'] = { # t_project[srad] recording: 'srad': - [('output/0/2015135_merra_srad.tif', + [('0/2015135_merra_srad.tif', 'hash', 'sha256', 'bfc0b906f7299ff8e6fcbbe474de0fb7b877d36f2e619e2e6ac10bd84ac5b55d')], # t_project[tave] recording: 'tave': - [('output/0/2015135_merra_tave.tif', + [('0/2015135_merra_tave.tif', 'hash', 'sha256', 'c96d89ebf5d7ec90193c8a113609bfc39473d72041b6145a8cd41ee49e4f6a4d')], # t_project[prcp] recording: 'prcp': - [('output/0/2015135_merra_prcp.tif', + [('0/2015135_merra_prcp.tif', 'hash', 'sha256', 'b7e68237ee3557c421856203cc22807f1ce74790ea2948f8d095cbf8a7f18a53')], # t_project[rhum] recording: 'rhum': - [('output/0/2015135_merra_rhum.tif', + [('0/2015135_merra_rhum.tif', 'hash', 'sha256', '3b45ba1cd87d4781ff95b08562a00189506c76add24156f8e3719ff8872c3d3a')], # t_project[tmin] recording: 'tmin': - [('output/0/2015135_merra_tmin.tif', + [('0/2015135_merra_tmin.tif', 'hash', 'sha256', '253e37e9b2c03a77a0a4fdfe5aaa0e3ff025471f64fd28d6ffb33dd8ecdd68a9')], # t_project[tmax] recording: 'tmax': - [('output/0/2015135_merra_tmax.tif', + [('0/2015135_merra_tmax.tif', 'hash', 'sha256', '8371d473f0e506e6ab0fe872542b8cd0d65bc2ed2b37bbeb747c966c41e9c232')], # t_project[shum] recording: 'shum': - [('output/0/2015135_merra_shum.tif', + [('0/2015135_merra_shum.tif', 'hash', 'sha256', '208465fbf10da2e935807e00991fa6375c3c6b67d9950362a9ffe502d7690fd6')], # t_project[patm] recording: 'patm': - [('output/0/2015135_merra_patm.tif', + [('0/2015135_merra_patm.tif', 'hash', 'sha256', '5778a4d89b22ea76dc25654c8092a61bbb09634f06f66b578460a93444e513a7')], # t_project[wind] recording: 'wind': - [('output/0/2015135_merra_wind.tif', + [('0/2015135_merra_wind.tif', 'hash', 'sha256', '88029a425fbeb59149a29be793087b51ea46583bb89e62c1baca629ccfc3a36a')], From 58afe8655c99bee8a773b87b91887ab49a43e879 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 27 Dec 2017 09:44:08 -0500 Subject: [PATCH 22/44] 483 convert to new directory-generifying code --- gips/test/sys/t_stats.py | 64 ++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/gips/test/sys/t_stats.py b/gips/test/sys/t_stats.py index 335245df..fe812976 100644 --- a/gips/test/sys/t_stats.py +++ b/gips/test/sys/t_stats.py @@ -28,7 +28,7 @@ def t_stats(export_wrapper, driver, product): expectations['modis'] = { # t_stats[satvi] recording: 'satvi': - [('output/satvi_stats.txt', + [('satvi_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 -0.1199 0.2757 0.188845 0.0394886 -2.85581 221002.0 \n', @@ -37,7 +37,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[ndti] recording: 'ndti': - [('output/ndti_stats.txt', + [('ndti_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 -1.0048 0.8895 0.31602 0.0595381 -2.46398 221002.0 \n', @@ -46,7 +46,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[ndvi] recording: 'ndvi': - [('output/ndvi_stats.txt', + [('ndvi_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 -0.8893 0.8144 0.598198 0.130008 -4.50499 221002.0 \n', @@ -55,7 +55,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[quality] recording: 'quality': - [('output/quality_stats.txt', + [('quality_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 0.0 0.0 0.0 0.0 nan 221002.0 \n', @@ -64,7 +64,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[isti] recording: 'isti': - [('output/isti_stats.txt', + [('isti_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 0.0583 3.2766 0.522676 0.0837755 13.2684 220980.0 \n', @@ -73,14 +73,14 @@ def t_stats(export_wrapper, driver, product): # t_stats[ndvi8] recording: 'ndvi8': - [('output/ndvi8_stats.txt', + [('ndvi8_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-337 -1.9475 3.2766 0.612297 0.162128 -3.72226 220758.0 \n'])], # t_stats[snow] recording: 'snow': - [('output/snow_stats.txt', + [('snow_stats.txt', 'text-full', ['date Snow Cover-min Snow Cover-max Snow Cover-mean Snow Cover-sd Snow Cover-skew Snow Cover-count Fractional Snow Cover-min Fractional Snow Cover-max Fractional Snow Cover-mean Fractional Snow Cover-sd Fractional Snow Cover-skew Fractional Snow Cover-count \n', '2012-336 0.0 100.0 66.0782 43.1646 -0.66934 2379.0 0.0 70.0 44.8928 24.6763 -0.938924 2379.0 \n', @@ -89,7 +89,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[fsnow] recording: 'fsnow': - [('output/fsnow_stats.txt', + [('fsnow_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 0.0 70.0 36.4813 30.8325 -0.322887 2379.0 \n', @@ -98,7 +98,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[crcm] recording: 'crcm': - [('output/crcm_stats.txt', + [('crcm_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 -0.8857 1.6238 0.960025 0.21929 -3.32392 221002.0 \n', @@ -107,7 +107,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[ndsi] recording: 'ndsi': - [('output/ndsi_stats.txt', + [('ndsi_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 -0.7313 1.0045 -0.58418 0.123674 6.27293 221002.0 \n', @@ -116,7 +116,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[brgt] recording: 'brgt': - [('output/brgt_stats.txt', + [('brgt_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 0.0062 0.1361 0.0467873 0.0103586 0.699293 221002.0 \n', @@ -125,7 +125,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[msavi2] recording: 'msavi2': - [('output/msavi2_stats.txt', + [('msavi2_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 0.014 0.6976 0.503178 0.0608721 -2.7989 219182.0 \n', @@ -134,7 +134,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[bi] recording: 'bi': - [('output/bi_stats.txt', + [('bi_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 0.0033 0.1717 0.0948656 0.0173088 -1.23546 221002.0 \n', @@ -143,7 +143,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[obstime] recording: 'obstime': - [('output/obstime_stats.txt', + [('obstime_stats.txt', 'text-full', ['date Observation Time Daytime Terra-min Observation Time Daytime Terra-max Observation Time Daytime Terra-mean Observation Time Daytime Terra-sd Observation Time Daytime Terra-skew Observation Time Daytime Terra-count Observation Time Nighttime Terra-min Observation Time Nighttime Terra-max Observation Time Nighttime Terra-mean Observation Time Nighttime Terra-sd Observation Time Nighttime Terra-skew Observation Time Nighttime Terra-count Observation Time Daytime Aqua-min Observation Time Daytime Aqua-max Observation Time Daytime Aqua-mean Observation Time Daytime Aqua-sd Observation Time Daytime Aqua-skew Observation Time Daytime Aqua-count Observation Time Nighttime Aqua-min Observation Time Nighttime Aqua-max Observation Time Nighttime Aqua-mean Observation Time Nighttime Aqua-sd Observation Time Nighttime Aqua-skew Observation Time Nighttime Aqua-count \n', '2012-336 10.8 10.8 10.8 1.90735e-07 -0.999996 716.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 1.4 1.4 1.4 2.38419e-08 inf 6685.0 \n', @@ -152,7 +152,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[lswi] recording: 'lswi': - [('output/lswi_stats.txt', + [('lswi_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 -0.9413 1.004 0.0704005 0.0795676 -0.897423 221002.0 \n', @@ -161,7 +161,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[vari] recording: 'vari': - [('output/vari_stats.txt', + [('vari_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 -0.2931 3.2766 -0.0579385 0.0896767 11.2112 221002.0 \n', @@ -170,7 +170,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[evi] recording: 'evi': - [('output/evi_stats.txt', + [('evi_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 -0.0689 0.4391 0.251258 0.0534915 -2.19097 221002.0 \n', @@ -179,14 +179,14 @@ def t_stats(export_wrapper, driver, product): # t_stats[temp8tn] recording: 'temp8tn': - [('output/temp8tn_stats.txt', + [('temp8tn_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-337 267.02 280.38 273.055 1.35905 0.334532 212758.0 \n'])], # t_stats[clouds] recording: 'clouds': - [('output/clouds_stats.txt', + [('clouds_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 0.0 1.0 0.990964 0.094628 -10.3767 221002.0 \n', @@ -195,7 +195,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[temp] recording: 'temp': - [('output/temp_stats.txt', + [('temp_stats.txt', 'text-full', ['date Temperature Daytime Terra-min Temperature Daytime Terra-max Temperature Daytime Terra-mean Temperature Daytime Terra-sd Temperature Daytime Terra-skew Temperature Daytime Terra-count Temperature Nighttime Terra-min Temperature Nighttime Terra-max Temperature Nighttime Terra-mean Temperature Nighttime Terra-sd Temperature Nighttime Terra-skew Temperature Nighttime Terra-count Temperature Daytime Aqua-min Temperature Daytime Aqua-max Temperature Daytime Aqua-mean Temperature Daytime Aqua-sd Temperature Daytime Aqua-skew Temperature Daytime Aqua-count Temperature Nighttime Aqua-min Temperature Nighttime Aqua-max Temperature Nighttime Aqua-mean Temperature Nighttime Aqua-sd Temperature Nighttime Aqua-skew Temperature Nighttime Aqua-count Temperature Best Quality-min Temperature Best Quality-max Temperature Best Quality-mean Temperature Best Quality-sd Temperature Best Quality-skew Temperature Best Quality-count \n', '2012-336 267.44 267.9 267.67 0.140381 -0.244244 716.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 258.94 263.7 261.912 1.39539 -0.717615 6685.0 0.0 0.0 0.0 0.0 nan 221255.0 \n', @@ -204,14 +204,14 @@ def t_stats(export_wrapper, driver, product): # t_stats[temp8td] recording: 'temp8td': - [('output/temp8td_stats.txt', + [('temp8td_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-337 273.32 283.76 279.933 1.98691 -1.00057 218113.0 \n'])], # t_stats[sti] recording: 'sti': - [('output/sti_stats.txt', + [('sti_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 -0.0024 3.2766 1.94182 0.235117 0.707709 221002.0 \n', @@ -220,7 +220,7 @@ def t_stats(export_wrapper, driver, product): # t_stats[crc] recording: 'crc': - [('output/crc_stats.txt', + [('crc_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2012-336 -0.8526 2.1525 1.29852 0.263609 -2.37667 221002.0 \n', @@ -231,63 +231,63 @@ def t_stats(export_wrapper, driver, product): expectations['merra'] = { # t_stats[srad] recording: 'srad': - [('output/srad_stats.txt', + [('srad_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], # t_stats[tave] recording: 'tave': - [('output/tave_stats.txt', + [('tave_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], # t_stats[shum] recording: 'shum': - [('output/shum_stats.txt', + [('shum_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], # t_stats[rhum] recording: 'rhum': - [('output/rhum_stats.txt', + [('rhum_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], # t_stats[tmin] recording: 'tmin': - [('output/tmin_stats.txt', + [('tmin_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], # t_stats[tmax] recording: 'tmax': - [('output/tmax_stats.txt', + [('tmax_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], # t_stats[prcp] recording: 'prcp': - [('output/prcp_stats.txt', + [('prcp_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], # t_stats[patm] recording: 'patm': - [('output/patm_stats.txt', + [('patm_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], # t_stats[wind] recording: 'wind': - [('output/wind_stats.txt', + [('wind_stats.txt', 'text-full', ['date min max mean sd skew count\n', '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], From 2e47121f37c1c565bfb03015b098cfa86e09d5b1 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 27 Dec 2017 10:50:37 -0500 Subject: [PATCH 23/44] 483 convert aod to new t_project --- gips/test/sys/expected/aod.py | 11 ----------- gips/test/sys/t_aod.py | 7 ------- gips/test/sys/t_project.py | 21 ++++++++++++++++++--- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/gips/test/sys/expected/aod.py b/gips/test/sys/expected/aod.py index befebb16..a926ddb1 100644 --- a/gips/test/sys/expected/aod.py +++ b/gips/test/sys/expected/aod.py @@ -1,16 +1,5 @@ """Known-good outcomes for tests, mostly stdout and created files.""" -t_project = { - 'compare_stdout': False, - 'compare_stderr': False, - 'created': { - '0': None, - '0/2017004_MOD_aod.tif': -766418958, - '0/2017005_MOD_aod.tif': 1283286056, - '0/2017006_MOD_aod.tif': -2004417139, - } -} - t_project_no_warp = { 'compare_stdout': False, 'created': { diff --git a/gips/test/sys/t_aod.py b/gips/test/sys/t_aod.py index 217203ba..6d971149 100644 --- a/gips/test/sys/t_aod.py +++ b/gips/test/sys/t_aod.py @@ -39,13 +39,6 @@ def setup_driver_data(pytestconfig): # ### SHOULD BE STANDARD BELOW HERE ##### -def t_project(setup_fixture, clean_repo_env, output_tfe, expected): - """Test gips_project {} with warping.""".format(driver) - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld') - actual = output_tfe.run('gips_project', *args) - assert expected == actual - - def t_project_no_warp(setup_fixture, clean_repo_env, output_tfe, expected): """Test gips_project {} without warping.""".format(driver) args = STD_ARGS + ('--outdir', OUTPUT_DIR, '--notld') diff --git a/gips/test/sys/t_project.py b/gips/test/sys/t_project.py index 662abfad..3b7d1302 100644 --- a/gips/test/sys/t_project.py +++ b/gips/test/sys/t_project.py @@ -407,7 +407,22 @@ def t_project(export_wrapper, driver, product): '88029a425fbeb59149a29be793087b51ea46583bb89e62c1baca629ccfc3a36a')], } +expectations['aod'] = { + # t_project[aod] recording: + 'aod': + [('0/2017004_MOD_aod.tif', + 'hash', + 'sha256', + '2fe93815b6d1686fd357e8685c82ff23689d99200b3aec2654277fb6db86a8c7'), + ('0/2017005_MOD_aod.tif', + 'hash', + 'sha256', + '3830875aa1603cc86fc452286d365c1dcd12af60889f0761d8df3493c7541ef7'), + ('0/2017006_MOD_aod.tif', + 'hash', + 'sha256', + 'ed9fc349c037f39eca87f2e7495024eeaf7fd0e99165ecb0f52cc7927e985977')], +} -params += [('modis', p) for p in expectations['modis'].keys()] -params += [('merra', p) for p in expectations['merra'].keys()] - +for driver, prod_expectations in expectations.items(): + params += [(driver, product) for product in prod_expectations.keys()] From bf3cbffb6103c40df5a0e3fe07cab5a741289074 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 27 Dec 2017 11:10:43 -0500 Subject: [PATCH 24/44] 483 convert aod to new t_stats --- gips/test/sys/expected/aod.py | 11 ----------- gips/test/sys/t_stats.py | 15 +++++++++++++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/gips/test/sys/expected/aod.py b/gips/test/sys/expected/aod.py index a926ddb1..5b70c8d4 100644 --- a/gips/test/sys/expected/aod.py +++ b/gips/test/sys/expected/aod.py @@ -26,14 +26,3 @@ } } -t_stats = { - 'stdout': u"""\x1b[1mGIPS Image Statistics (v0.1.0)\x1b[0m -Stats for Project directory: /tmp/test-output -Calculating statistics for 2017-01-04 -Calculating statistics for 2017-01-05 -Calculating statistics for 2017-01-06 -""", - 'created': { - 'aod_stats.txt': -769515142 - } -} diff --git a/gips/test/sys/t_stats.py b/gips/test/sys/t_stats.py index fe812976..a52bb9a0 100644 --- a/gips/test/sys/t_stats.py +++ b/gips/test/sys/t_stats.py @@ -293,5 +293,16 @@ def t_stats(export_wrapper, driver, product): '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], } -params += [('modis', p) for p in expectations['modis'].keys()] -params += [('merra', p) for p in expectations['merra'].keys()] +expectations['aod'] = { + # t_stats[aod] recording: + 'aod': + [('aod_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2017-004 45.0 107.0 68.2053 27.2408 0.683439 2.70847e+06 \n', + '2017-005 34.0 130.0 68.7497 33.5872 0.965031 4.54937e+06 \n', + '2017-006 176.0 176.0 176.0 0.0 nan 927590.0 \n'])], +} + +for driver, prod_expectations in expectations.items(): + params += [(driver, product) for product in prod_expectations.keys()] From e002402aff284386d45c03671557b97b09e910bd Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 27 Dec 2017 11:24:01 -0500 Subject: [PATCH 25/44] 483 complain if record file already exists --- conftest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/conftest.py b/conftest.py index bee2b91b..4e83ed2b 100644 --- a/conftest.py +++ b/conftest.py @@ -76,7 +76,7 @@ def pytest_addoption(parser): parser.addoption('--record', action='store', default=None, help="Pass in a filename for expecations" " to be written to that filename.") - # TODO cleanup may not need to be implemented at all + # cleanup may not need to be implemented at all #parser.addoption('--cleanup-on-failure', action='store_true', # help="Normally cleanup is skipped on failure so you can examine" # " files; pass this option to cleanup even on failure.") @@ -87,9 +87,10 @@ def pytest_configure(config): level = ('warning' if raw_level is None else raw_level).upper() root_logger.setLevel(level) - # TODO complain here if --record or --record-dir are wrong: - # * --record file should not exist - # * --record-dir should exist + record_path = config.getoption('record') + if os.path.lexists(record_path): + raise IOError("Record file already exists at {}".format(record_path)) + dr = str(config.getini('data-repo')) if not dr: raise ValueError("No value specified for 'data-repo' in pytest.ini") From aad461ede9dafa2435879f70b66af793306d2a0a Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 27 Dec 2017 12:13:17 -0500 Subject: [PATCH 26/44] 483 move expectations to separate files --- conftest.py | 2 +- gips/test/sys/expected/std_process.py | 388 ++++++++++++++++++++++++ gips/test/sys/expected/std_project.py | 406 +++++++++++++++++++++++++ gips/test/sys/expected/std_stats.py | 280 +++++++++++++++++ gips/test/sys/t_process.py | 395 +----------------------- gips/test/sys/t_project.py | 415 +------------------------- gips/test/sys/t_stats.py | 289 +----------------- gips/test/sys/util.py | 6 + 8 files changed, 1093 insertions(+), 1088 deletions(-) create mode 100644 gips/test/sys/expected/std_process.py create mode 100644 gips/test/sys/expected/std_project.py create mode 100644 gips/test/sys/expected/std_stats.py diff --git a/conftest.py b/conftest.py index 4e83ed2b..de70ec21 100644 --- a/conftest.py +++ b/conftest.py @@ -88,7 +88,7 @@ def pytest_configure(config): root_logger.setLevel(level) record_path = config.getoption('record') - if os.path.lexists(record_path): + if record_path and os.path.lexists(record_path): raise IOError("Record file already exists at {}".format(record_path)) dr = str(config.getini('data-repo')) diff --git a/gips/test/sys/expected/std_process.py b/gips/test/sys/expected/std_process.py new file mode 100644 index 00000000..ed8a7ee5 --- /dev/null +++ b/gips/test/sys/expected/std_process.py @@ -0,0 +1,388 @@ + +expectations = {} + +expectations['modis'] = { + # 'landcover' [], # is annual, not available for the scene under test + # 'indices', deprecated; not testing as a result + # weird symlink products: + # t_process[quality] recording: + 'quality': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/modis/tiles/h12v04/2012336/MCD43A2.A2012336.h12v04.006.2016112010833.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/modis/tiles/h12v04/2012337/MCD43A2.A2012337.h12v04.006.2016112013509.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/modis/tiles/h12v04/2012338/MCD43A2.A2012338.h12v04.006.2016112020013.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo')], + + # t_process[temp8tn] recording: + 'temp8tn': + [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8tn.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Night_1km')], + + # t_process[temp8td] recording: + 'temp8td': + [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8td.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Day_1km')], + + # normal products + # t_process[satvi] recording: + 'satvi': + [('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_satvi.tif', + 'hash', + 'sha256', + '94bee7cecf0f0a227fd8532223f86de10e01c3e39d1dd70b7675d91b3ca338a4'), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_satvi.tif', + 'hash', + 'sha256', + '6397669a6cd62bb39813b9000d93ba50f4cabe73aebf1e6444f4e3be9f630e19'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_satvi.tif', + 'hash', + 'sha256', + '6124ee615a4f522fdf966a5479d03d87934908e5090a192192f9700020f24760')], + + # t_process[ndti] recording: + 'ndti': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndti.tif', + 'hash', + 'sha256', + '025ea7a9204ae8ad0952a92b386bf735dbbce8337e336843476d3f227cf1f1ab'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndti.tif', + 'hash', + 'sha256', + 'b6baa8c7a3996f680c4177e4157c9275dd895c8e6350afa6ef564e673bb6a5f0'), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndti.tif', + 'hash', + 'sha256', + 'a1c4f1161e8a584257352e76160319c39aa4605dd565d388d0116e193773771f')], + + # t_process[ndvi] recording: + 'ndvi': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndvi.tif', + 'hash', + 'sha256', + 'efbeae3a1c5e59ccd54b6108e275f9a8f0059162ac2797f4d69b00acb80147b3'), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndvi.tif', + 'hash', + 'sha256', + 'b56d5c96229a1ad4cdcbd09db10b19a216b4cde8a0ffd184ee5cf01a35ab2b46'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndvi.tif', + 'hash', + 'sha256', + 'd9ff5d2f85ac9a267a8f6b1aca34f62b8196721e212b1e092f82ce9f1e208c61')], + + # t_process[isti] recording: + 'isti': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_isti.tif', + 'hash', + 'sha256', + 'c0a38d1cd00de52eacd8abcda714bfdaa575787e27b40dc0472ffca5ab1696c0'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_isti.tif', + 'hash', + 'sha256', + 'b1759c1008e5914b955c4a9f55572baa8f7c0310677264a1057329e236d826cb'), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_isti.tif', + 'hash', + 'sha256', + '677182d186dac992b52b93239af0f419536075c7cd467864fa544bcde4b89e69')], + + # t_process[ndvi8] recording: + 'ndvi8': + [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_ndvi8.tif', + 'hash', + 'sha256', + '93db24c18886fbac0b87d1404a0c80606a599cc55b02d745a09ffb051b8131ac')], + + # t_process[snow] recording: + 'snow': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_snow.tif', + 'hash', + 'sha256', + 'bea7315b736a75bcf37df86089679845ac597f3fba5d5cd6e79f5516b9657faa'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_snow.tif', + 'hash', + 'sha256', + 'de979c1a0c616d7889dd5fd45816849f9c08df54167d7f3b40301d6ac4495778'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_snow.tif', + 'hash', + 'sha256', + '5d9bbfff1522e74165cdd677f1580e9eb2236c2c5922a796a6c9ee0db24ae05a')], + + # t_process[fsnow] recording: + 'fsnow': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_fsnow.tif', + 'hash', + 'sha256', + '853a5ca95f21c4a745e41e4acc46a71a74c74e0cdd1cae9ed3dd31b2fef1ddbd'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_fsnow.tif', + 'hash', + 'sha256', + '3a5f7c95fe60e5789919cf55d7f8e652e4d668c6eb63eaa53a82249c0ea19c5a'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_fsnow.tif', + 'hash', + 'sha256', + 'd09b8a12ad17a7e715ae2e988fbf683e5fe2df46e97e9ac3ebb4444c8b1c169d')], + + # t_process[crcm] recording: + 'crcm': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crcm.tif', + 'hash', + 'sha256', + 'a45c2fb16d871a919e0c803f80e1f3b4ec831472d0466c33741fcf8e8150f6ba'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crcm.tif', + 'hash', + 'sha256', + '9aa80c0e1fa1b62b0a41aed42eeaf84632845d74f29594dfee55c22673b0a6f5'), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crcm.tif', + 'hash', + 'sha256', + 'a28c7d280fb8dfe0fe1cc003e1001457abdf14acc5545b2a69b782cfbcce7b50')], + + # t_process[ndsi] recording: + 'ndsi': + [('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndsi.tif', + 'hash', + 'sha256', + '1f91f738bbb28b3ebeba36affd8d0689e105e90e1547cca58497eb7c7fdf7346'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndsi.tif', + 'hash', + 'sha256', + 'a626d375fbe5094a5199fdfc2206cf94777ff574379b45811cf7a49055a9f6ec'), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndsi.tif', + 'hash', + 'sha256', + 'cb2ca7461c65f66ddb13e804ae7fe8f54daaf620ade600efee24810d40d4eb34')], + + # t_process[brgt] recording: + 'brgt': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_brgt.tif', + 'hash', + 'sha256', + 'bf6bad538c7c6e703bc2f39503c2f128fa8e988a533e302c0124e085735dfaf9'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_brgt.tif', + 'hash', + 'sha256', + '5c72e8f37640ec6b4a5a33e18980d71c70482fc8f708ba25da73e074cf64729f'), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_brgt.tif', + 'hash', + 'sha256', + '3460d0d35e35540b55e9eda35f253af8d85eabb84c0c44cb5f9c02d43da7e20a')], + + # t_process[msavi2] recording: + 'msavi2': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_msavi2.tif', + 'hash', + 'sha256', + 'd2c93798df10939ff8afb6cd23e18ac3281cfd844e45da47a29d0342c2b980e9'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_msavi2.tif', + 'hash', + 'sha256', + '92fe4f40d051ef3f479bde76a67bfff978bc513893af5ed62e4a48e457daabad'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_msavi2.tif', + 'hash', + 'sha256', + 'f25bd2d58d3edae72b11e73c08341479dee13af80352a4749861b6a54a81b351')], + + # t_process[bi] recording: + 'bi': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_bi.tif', + 'hash', + 'sha256', + '28c181b8d210b71759589da53295cc894c9d12c74ff1f7b0a49b88f73664282e'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_bi.tif', + 'hash', + 'sha256', + '45acfc9249e063b6b97f269338590d51e2334e9bc5df118613ad908c6f0a3f8e'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_bi.tif', + 'hash', + 'sha256', + '8526cb96d59fc9549dc8535fc8128123b9d39a63a3b7cd97ee3e611b7eeadf6f')], + + # t_process[obstime] recording: + 'obstime': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + '66e01c94910c4fdd4f6fb2ab99b0c2d0cb73618032783e3cbdf1867191981fdc'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + '117cea0497b3158b465dd7dd197c886ebb1acf5059ed83dfdd6b5d7e94bb5243'), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'b04d8b07ef83e479db5650a30c84513bde7d5b05dd68187782c057ea0fd3e537')], + + # t_process[lswi] recording: + 'lswi': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_lswi.tif', + 'hash', + 'sha256', + '8de7378cdba830171c8da94424bf7807fa6c8b12523e8b34175a4b17ec770a5d'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_lswi.tif', + 'hash', + 'sha256', + '62de4bdafcc276e9c1c343f4e63f5dd06167e87f7712796f991e3a957f2b71de'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_lswi.tif', + 'hash', + 'sha256', + 'add051ef35d7ffff5eb68c48f8b8071f9cb493b5d3426867605d4c61746e3a26')], + + # t_process[vari] recording: + 'vari': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_vari.tif', + 'hash', + 'sha256', + '732d6d3d9fbe0e1b4616a3f8a91bf12d532598a0a02e8aab16c9a01c3ac56913'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_vari.tif', + 'hash', + 'sha256', + '9875717cc5e1926f11378bbf16345dbade5a2badf9f3dd06844a77f94d80ef72'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_vari.tif', + 'hash', + 'sha256', + '70fd19862ef4145c24f26a8a74cb748279b88fd973de935726ebef2294e17edd')], + + # t_process[evi] recording: + 'evi': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_evi.tif', + 'hash', + 'sha256', + '13548ae74456f08e6add657a637ca364626eafd2cc767724f57aac8ff93005bb'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_evi.tif', + 'hash', + 'sha256', + '7b926f7004cd92f2ec604fc4e2cc1489f60a335e39f4954030332a012fd33f87'), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_evi.tif', + 'hash', + 'sha256', + '51db813a857f795817f529edd56d7134004566be46edab8a8dadc81286fb895f')], + + # t_process[clouds] recording: + 'clouds': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MOD_clouds.tif', + 'hash', + 'sha256', + '26b3e7e695f08ec2e9fb26985f9d5e392f9e8cedc331ab6721ea8e48821d93f0'), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MOD_clouds.tif', + 'hash', + 'sha256', + 'c43eec8178d0521d20e2a5b6e9b61525007d80702c6dd705a3e52ed3a76a7b73'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_clouds.tif', + 'hash', + 'sha256', + '24a6a7050c866eed989530bec45b72004097f1e36fcc0e686652b87723a86de4')], + + # t_process[temp] recording: + 'temp': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_temp.tif', + 'hash', + 'sha256', + 'ead2622531929bba3192b8a6b609d5d920e79394ec577c5245d74d69593d827c'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '5d498bad2779f84289dfb75b14345d64d16b5bc7994b4aca6cd694f70b86064d'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '097f341fa917a0ec316c37d7907ff15b754dcd0925469be586016137ca47575f')], + + # t_process[sti] recording: + 'sti': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_sti.tif', + 'hash', + 'sha256', + '24e50766ec18826f30bbad07bc8f5a34e72f1966b325cc23f064aa5fd35dc24a'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_sti.tif', + 'hash', + 'sha256', + 'c223de9d00e9cfedc0cfc13a45163535f3c59e5587642fa2c725b00b17b7437c'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_sti.tif', + 'hash', + 'sha256', + '73c247fd2c7623e79145b6337846c97d14d1752d660b2f32245a4ccaac033350')], + + # t_process[crc] recording: + 'crc': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crc.tif', + 'hash', + 'sha256', + '07debf171aedfeba7d6c1e3220ff8e4a2a7b1033e68c72bbc121b500b6ed3c9c'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crc.tif', + 'hash', + 'sha256', + 'a7b61aabc950be7a3040b6f9572bf755522b12ad27644583e603def05ac1980b'), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crc.tif', + 'hash', + 'sha256', + '942efeeee2b020123be6d7334bbb3ef5995c93d2fd0247cb79948836865c8067')], +} + +expectations['merra'] = { + # test this too? 'frland': [], + # t_process[srad] recording: + 'srad': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_srad.tif', + 'hash', + 'sha256', + '882a44af70e337bf905c8695936f101792a518507a4304b23781a2f30aaabab4')], + # t_process[tave] recording: + 'tave': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tave.tif', + 'hash', + 'sha256', + '5185aebd7cda54157cad2ddbde9f6fec4871c1b85fe78d0738634ba0211f2c9b')], + # t_process[shum] recording: + 'shum': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_shum.tif', + 'hash', + 'sha256', + '558f567c1e931891553d396f81b2b90929ad48a0ae88fa95d3894eae23ab3eba')], + # t_process[rhum] recording: + 'rhum': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_rhum.tif', + 'hash', + 'sha256', + '058161eb8488f1fb3df7f5c8719833c9d68c56aa082d4605938f62c2ee0035f6')], + # t_process[tmin] recording: + 'tmin': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmin.tif', + 'hash', + 'sha256', + '35a850769a3bb8f5209574ea4835bf417d8f99024567f4af9cdee9e623d0c567')], + # t_process[tmax] recording: + 'tmax': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmax.tif', + 'hash', + 'sha256', + '5963fcf346f388fc347355153996a078f1eb5ecbbc094394b72be382ba491865')], + # t_process[prcp] recording: + 'prcp': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_prcp.tif', + 'hash', + 'sha256', + '348fd7072ef0a8625f5268e29acbf3a69a959a412327102ef6558c992e62bc9f')], + # t_process[patm] recording: + 'patm': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_patm.tif', + 'hash', + 'sha256', + '7d9deca613973cb8ffb8f15a3dfa2013e6783556b520915e89b97fe41de638e3')], + # t_process[wind] recording: + 'wind': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_wind.tif', + 'hash', + 'sha256', + 'a90c1d87fdb1c0926bd5b3004408a1f9d42ef08c38deb9d0572bc7536dbbf08a')], +} + diff --git a/gips/test/sys/expected/std_project.py b/gips/test/sys/expected/std_project.py new file mode 100644 index 00000000..a2929942 --- /dev/null +++ b/gips/test/sys/expected/std_project.py @@ -0,0 +1,406 @@ +expectations = {} + +expectations['modis'] = { + # t_project[satvi] recording: + 'satvi': + [('0/2012336_MCD_satvi.tif', + 'hash', + 'sha256', + '893d968ff6359d0d34f8a587246690d1c724832505d66fbee8be8c1a43a6e3df'), + ('0/2012337_MCD_satvi.tif', + 'hash', + 'sha256', + '6092d7cfb609f000124b8ef107f278decaa27c3cdb69696bd094940ead97d96e'), + ('0/2012338_MCD_satvi.tif', + 'hash', + 'sha256', + '37dd32e0f5e761846f44615ec6e82e2cce8d6c98197f0bd721dcfb266d963b6b')], + + # t_project[ndti] recording: + 'ndti': + [('0/2012337_MCD_ndti.tif', + 'hash', + 'sha256', + 'db85948227815660b94d570adf0f1119d9f1001284c3de6c8b219c6455bd3c4e'), + ('0/2012338_MCD_ndti.tif', + 'hash', + 'sha256', + '5028c194e41d8c07076cee09631b276ef7b86c3c45b0a4f705f5e4c6b72c2812'), + ('0/2012336_MCD_ndti.tif', + 'hash', + 'sha256', + '157134ea0e9cbb0b2fcdb4203e17e82ae906a5779eb1f735e8c55d1f60c231e1')], + + # t_project[ndvi] recording: + 'ndvi': + [('0/2012337_MCD_ndvi.tif', + 'hash', + 'sha256', + '28a288720a0846348fbd6315e5d9e3801ad76eb2c873dc028e7a52d1fdcdad10'), + ('0/2012336_MCD_ndvi.tif', + 'hash', + 'sha256', + '78e090b1cc92304c666a8e480d11fa99985ee3d2113c4fe3be29fb85910b86b9'), + ('0/2012338_MCD_ndvi.tif', + 'hash', + 'sha256', + 'bbcee4ac1b49466c90264e305c20ce19a021fd177ffc35da5ed92a86555c480e')], + + # t_project[quality] recording: + 'quality': + [('0/2012338_MCD_quality.tif', + 'hash', + 'sha256', + '895abfc825d9da56c56202c670f94a16b28f2aaab5590db55ac6e903b6638e91'), + ('0/2012336_MCD_quality.tif', + 'hash', + 'sha256', + '4f40fdeb7ded507a68f404ec34a17fb6283c9b90d81ecccc7782746eaba85323'), + ('0/2012337_MCD_quality.tif', + 'hash', + 'sha256', + '5cdd315b2a423fbb82c19cfe859325c889a5d05cf0c3cdf71359b257524175a3')], + + # t_project[isti] recording: + 'isti': + [('0/2012337_MCD_isti.tif', + 'hash', + 'sha256', + '391367ee5bdd0a96435c09df85db312919121c5d9a28b5bda59a68db7b609871'), + ('0/2012338_MCD_isti.tif', + 'hash', + 'sha256', + 'cc082d45a2a4dd4a7e53d0970029decf24ce5033b9715930cd250b524ad87757'), + ('0/2012336_MCD_isti.tif', + 'hash', + 'sha256', + '16be9705c06a44f3a35e684cded2d2ceb26213510eaf0ccfbafb62ebc39bb153')], + + # t_project[ndvi8] recording: + 'ndvi8': + [('0/2012337_MOD_ndvi8.tif', + 'hash', + 'sha256', + '23a72df39da253ac07de5a9e98c88f40e3335f3b1b779ccf91153d1d5573a780')], + + # t_project[snow] recording: + 'snow': + [('0/2012336_MCD_snow.tif', + 'hash', + 'sha256', + 'b3eeca8fc933d2f43c2d96e869956447d9556737181635167744d0d420dfd813'), + ('0/2012338_MCD_snow.tif', + 'hash', + 'sha256', + '98a38e1a75fcea269bd75a952cde9d84af86b14ddb928e4777ff812dba8befbe'), + ('0/2012337_MCD_snow.tif', + 'hash', + 'sha256', + 'da26696be559bb79c399ad18942ac629a9604d857dd7fc461fd76c4577a06303')], + + # t_project[fsnow] recording: + 'fsnow': + [('0/2012338_MCD_fsnow.tif', + 'hash', + 'sha256', + 'f2efdf504e004caf05563f3402fdde7e771351d7d93de5970af0a54d0b5bd2c2'), + ('0/2012336_MCD_fsnow.tif', + 'hash', + 'sha256', + 'a9d2ea29a6d77325246e0baaf298d16cb84d6003a19faed8ba162b475703f563'), + ('0/2012337_MCD_fsnow.tif', + 'hash', + 'sha256', + 'e8465aa3764c60b21a943f3e078d93c0484272292f075455693ea2084971ace4')], + + # t_project[crcm] recording: + 'crcm': + [('0/2012337_MCD_crcm.tif', + 'hash', + 'sha256', + 'c00e7b7c6970ae23904ee52afd614a8871f3d771d26830a2aef412163d9d9228'), + ('0/2012338_MCD_crcm.tif', + 'hash', + 'sha256', + 'ed336a3ba4cf482936ca781ea1a5e0f04ca5ad68919c3dbe6e117edac55ec242'), + ('0/2012336_MCD_crcm.tif', + 'hash', + 'sha256', + 'f217a9278b195e945e2337663cf7dfe2b09faa0dc68062fbc3614a6954866eaf')], + + # t_project[ndsi] recording: + 'ndsi': + [('0/2012338_MCD_ndsi.tif', + 'hash', + 'sha256', + '8e26a8f6d78b9612c8cd3924936c5944f56c67ae1c63f6541d045b41a27baafe'), + ('0/2012337_MCD_ndsi.tif', + 'hash', + 'sha256', + '2aaea763ab1d5b54d7079e7ee2381bd987a3c72c5cbbd6f18f97aa320c186e2a'), + ('0/2012336_MCD_ndsi.tif', + 'hash', + 'sha256', + '83b6bbf241d080cf6860798da299038be5e6d3e20a8b544e5af51e2eefd3a5c9')], + + # t_project[brgt] recording: + 'brgt': + [('0/2012337_MCD_brgt.tif', + 'hash', + 'sha256', + 'fc385e6e42538a09655762d1b2557abe35af38aecef1571705cec8b971efbe4d'), + ('0/2012338_MCD_brgt.tif', + 'hash', + 'sha256', + 'a8f8ef66bc87c2370bd61d5eed64905ba4f832d05579687478ff5f38c65e8038'), + ('0/2012336_MCD_brgt.tif', + 'hash', + 'sha256', + '24f8130de39daac6e181cb2f98b48d9a14412700632d3cbd2ffbe2704283b740')], + + # t_project[msavi2] recording: + 'msavi2': + [('0/2012336_MCD_msavi2.tif', + 'hash', + 'sha256', + '997b562420fe223677881d577cdb2100ee653ee932b9989b95f3089c0fe38352'), + ('0/2012338_MCD_msavi2.tif', + 'hash', + 'sha256', + '26a7ab45723856361d108064fe0c810535de75f272a733d86f66eeb8146dee35'), + ('0/2012337_MCD_msavi2.tif', + 'hash', + 'sha256', + 'ee7b2006b79f65bdf11d77f678d66765abfcc024b7856ab296f855570efb9940')], + + # t_project[bi] recording: + 'bi': + [('0/2012336_MCD_bi.tif', + 'hash', + 'sha256', + 'e5f891a6290301508107714657274fc808fcf96cb138c652a9c22552675796a5'), + ('0/2012337_MCD_bi.tif', + 'hash', + 'sha256', + '58b9e3809630ba24879bb85e81bc6dc0eed548348373588d70d49de65d246582'), + ('0/2012338_MCD_bi.tif', + 'hash', + 'sha256', + '914ca6d15c25a22d3e0a43d0d266f72bd65d102b4d6b015a9f12b8ee3a116c13')], + + # t_project[obstime] recording: + 'obstime': + [('0/2012337_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'a9345d21f3d754d083d3b4d13b059f2ffe07eb42ea9a0d6f3e40ef326e4ea75f'), + ('0/2012336_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'fac305543c4afbf76611900579cd5f37d63e7a45814c997dce40fc46a74f13e5'), + ('0/2012338_MOD-MYD_obstime.tif', + 'hash', + 'sha256', + 'e18456caaf9a2dd005cc6568f7aa77ac534bfa24ec364d3af28affb4ae652de9')], + + # t_project[lswi] recording: + 'lswi': + [('0/2012337_MCD_lswi.tif', + 'hash', + 'sha256', + '586dd3782070b983b9d383161bed82bee21c54ad312d1c993b0329965144deab'), + ('0/2012336_MCD_lswi.tif', + 'hash', + 'sha256', + '296c849e70d31127bcefef54ffed72832529f7ae60d50b7c639c5c6d253d30f4'), + ('0/2012338_MCD_lswi.tif', + 'hash', + 'sha256', + '992fa3d7b4bdc8cf95fba759a0096f9a913e84762717e93c477baae0d7d3c0ed')], + + # t_project[vari] recording: + 'vari': + [('0/2012336_MCD_vari.tif', + 'hash', + 'sha256', + 'be5d2273b9f0b9aca79df89b66ca0e7e1a2b8d386e6d34a6eca6eb37810b488d'), + ('0/2012338_MCD_vari.tif', + 'hash', + 'sha256', + '016ec5b1ee512d9306251a4c2b3630c90ea18723159a6a254c8e77c4e730abc8'), + ('0/2012337_MCD_vari.tif', + 'hash', + 'sha256', + 'a6c10e80436b92908637b22b40e7993a0a648e82b96993760a7b26ba43df0667')], + + # t_project[evi] recording: + 'evi': + [('0/2012338_MCD_evi.tif', + 'hash', + 'sha256', + '8a1beba0730ac7d113a7edb8b55035a742892bbe678cd0eaaa540e59906a6990'), + ('0/2012337_MCD_evi.tif', + 'hash', + 'sha256', + '824a5247b0e8e0730dec6e82b390533649aa796ef04ef4be27ac38cae9a24d04'), + ('0/2012336_MCD_evi.tif', + 'hash', + 'sha256', + '361798060d02c97153a9828c1f8ac6b7df396070ec243932e9ed18fd472e9669')], + + # t_project[temp8tn] recording: + 'temp8tn': + [('0/2012337_MOD_temp8tn.tif', + 'hash', + 'sha256', + '1600d412e2590b4f4006ac0d98ebf9c6bf83fe16967c2fd721e412d2a14040c6')], + + # t_project[clouds] recording: + 'clouds': + [('0/2012337_MOD_clouds.tif', + 'hash', + 'sha256', + '8b13afeec6d1a5612a3eeb51896507e2d4eeb851261881f3854d2b804e97835f'), + ('0/2012336_MOD_clouds.tif', + 'hash', + 'sha256', + 'cb4fab2519652bebe4830479c95425050f63d519397beb5b0c40349cc6f49372'), + ('0/2012338_MOD_clouds.tif', + 'hash', + 'sha256', + '54505d6d4063caa27fda74f5fdbb916c20d281f6e15deea28596c379ffa661e2')], + + # t_project[temp] recording: + 'temp': + [('0/2012336_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '614023db948d0b0249057ca9c458356c2222e45224a726f87a7cf512cd87745c'), + ('0/2012338_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '51864699b8347a30d9669ec41298665cd893a27516089cc21b4c3f03b3e65b2b'), + ('0/2012337_MOD-MYD_temp.tif', + 'hash', + 'sha256', + '29373079984e8aaf55add6a1fdd7bd766eb89a1c74ebc85674da54aeb643e3ef')], + + # t_project[temp8td] recording: + 'temp8td': + [('0/2012337_MOD_temp8td.tif', + 'hash', + 'sha256', + 'eaa0ff94bb34cbad8d9c187b16c7cd2bd5948d1d94cf1caf30b0396ac7e8a735')], + + # t_project[sti] recording: + 'sti': + [('0/2012337_MCD_sti.tif', + 'hash', + 'sha256', + '9850777cae46b42fa3c8361849529e6f230dd1ce86fb6c6676aa8eb2768e71d5'), + ('0/2012336_MCD_sti.tif', + 'hash', + 'sha256', + 'b7fde6a04ac091946553b25b1a26de806fa7f9c23e915b415034ef811aca1d16'), + ('0/2012338_MCD_sti.tif', + 'hash', + 'sha256', + 'b2714cadf4e779c433415f194f366ac87877c60b383f46971843b4ab3a610fa9')], + + # t_project[crc] recording: + 'crc': + [('0/2012338_MCD_crc.tif', + 'hash', + 'sha256', + 'dc9a4885e694fe7117cf10df11b5589745b4eb119d5f58f9d1c14e27d0eda333'), + ('0/2012337_MCD_crc.tif', + 'hash', + 'sha256', + '5688d80a613c28c657237186291a88ae786037a77dad0247c0bd734da214c927'), + ('0/2012336_MCD_crc.tif', + 'hash', + 'sha256', + '3df0003b6153970e73c46d5e999697ce5af268744274ccd3a4a2c14e3ae932cb')], +} + +expectations['merra'] = { + # t_project[srad] recording: + 'srad': + [('0/2015135_merra_srad.tif', + 'hash', + 'sha256', + 'bfc0b906f7299ff8e6fcbbe474de0fb7b877d36f2e619e2e6ac10bd84ac5b55d')], + + # t_project[tave] recording: + 'tave': + [('0/2015135_merra_tave.tif', + 'hash', + 'sha256', + 'c96d89ebf5d7ec90193c8a113609bfc39473d72041b6145a8cd41ee49e4f6a4d')], + + # t_project[prcp] recording: + 'prcp': + [('0/2015135_merra_prcp.tif', + 'hash', + 'sha256', + 'b7e68237ee3557c421856203cc22807f1ce74790ea2948f8d095cbf8a7f18a53')], + + # t_project[rhum] recording: + 'rhum': + [('0/2015135_merra_rhum.tif', + 'hash', + 'sha256', + '3b45ba1cd87d4781ff95b08562a00189506c76add24156f8e3719ff8872c3d3a')], + + # t_project[tmin] recording: + 'tmin': + [('0/2015135_merra_tmin.tif', + 'hash', + 'sha256', + '253e37e9b2c03a77a0a4fdfe5aaa0e3ff025471f64fd28d6ffb33dd8ecdd68a9')], + + # t_project[tmax] recording: + 'tmax': + [('0/2015135_merra_tmax.tif', + 'hash', + 'sha256', + '8371d473f0e506e6ab0fe872542b8cd0d65bc2ed2b37bbeb747c966c41e9c232')], + + # t_project[shum] recording: + 'shum': + [('0/2015135_merra_shum.tif', + 'hash', + 'sha256', + '208465fbf10da2e935807e00991fa6375c3c6b67d9950362a9ffe502d7690fd6')], + + # t_project[patm] recording: + 'patm': + [('0/2015135_merra_patm.tif', + 'hash', + 'sha256', + '5778a4d89b22ea76dc25654c8092a61bbb09634f06f66b578460a93444e513a7')], + + # t_project[wind] recording: + 'wind': + [('0/2015135_merra_wind.tif', + 'hash', + 'sha256', + '88029a425fbeb59149a29be793087b51ea46583bb89e62c1baca629ccfc3a36a')], +} + +expectations['aod'] = { + # t_project[aod] recording: + 'aod': + [('0/2017004_MOD_aod.tif', + 'hash', + 'sha256', + '2fe93815b6d1686fd357e8685c82ff23689d99200b3aec2654277fb6db86a8c7'), + ('0/2017005_MOD_aod.tif', + 'hash', + 'sha256', + '3830875aa1603cc86fc452286d365c1dcd12af60889f0761d8df3493c7541ef7'), + ('0/2017006_MOD_aod.tif', + 'hash', + 'sha256', + 'ed9fc349c037f39eca87f2e7495024eeaf7fd0e99165ecb0f52cc7927e985977')], +} diff --git a/gips/test/sys/expected/std_stats.py b/gips/test/sys/expected/std_stats.py new file mode 100644 index 00000000..eccc30ff --- /dev/null +++ b/gips/test/sys/expected/std_stats.py @@ -0,0 +1,280 @@ +expectations = {} + +expectations['modis'] = { + # t_stats[satvi] recording: + 'satvi': + [('satvi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.1199 0.2757 0.188845 0.0394886 -2.85581 221002.0 \n', + '2012-337 -0.1179 0.2955 0.189273 0.0399094 -2.7273 221002.0 \n', + '2012-338 -0.1125 0.2985 0.187186 0.0408718 -2.44579 218582.0 \n'])], + + # t_stats[ndti] recording: + 'ndti': + [('ndti_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -1.0048 0.8895 0.31602 0.0595381 -2.46398 221002.0 \n', + '2012-337 -0.643 0.9544 0.31893 0.0596861 -0.906354 221002.0 \n', + '2012-338 -0.5051 0.9999 0.320866 0.064446 -0.195554 218582.0 \n'])], + + # t_stats[ndvi] recording: + 'ndvi': + [('ndvi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.8893 0.8144 0.598198 0.130008 -4.50499 221002.0 \n', + '2012-337 -0.9126 0.8198 0.598463 0.132692 -4.61097 221002.0 \n', + '2012-338 -1.0001 0.8117 0.595837 0.134746 -4.68736 220979.0 \n'])], + + # t_stats[quality] recording: + 'quality': + [('quality_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0 0.0 0.0 0.0 nan 221002.0 \n', + '2012-337 0.0 0.0 0.0 0.0 nan 221002.0 \n', + '2012-338 0.0 0.0 0.0 0.0 nan 220979.0 \n'])], + + # t_stats[isti] recording: + 'isti': + [('isti_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0583 3.2766 0.522676 0.0837755 13.2684 220980.0 \n', + '2012-337 0.0232 3.2766 0.519636 0.0845782 12.4045 221002.0 \n', + '2012-338 -0.0001 3.0399 0.517872 0.0852503 7.69029 218582.0 \n'])], + + # t_stats[ndvi8] recording: + 'ndvi8': + [('ndvi8_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-337 -1.9475 3.2766 0.612297 0.162128 -3.72226 220758.0 \n'])], + + # t_stats[snow] recording: + 'snow': + [('snow_stats.txt', + 'text-full', + ['date Snow Cover-min Snow Cover-max Snow Cover-mean Snow Cover-sd Snow Cover-skew Snow Cover-count Fractional Snow Cover-min Fractional Snow Cover-max Fractional Snow Cover-mean Fractional Snow Cover-sd Fractional Snow Cover-skew Fractional Snow Cover-count \n', + '2012-336 0.0 100.0 66.0782 43.1646 -0.66934 2379.0 0.0 70.0 44.8928 24.6763 -0.938924 2379.0 \n', + '2012-337 0.0 0.0 0.0 0.0 nan 677.0 0.0 0.0 0.0 0.0 nan 677.0 \n', + '2012-338 0.0 100.0 7.3169 20.9113 2.95797 221002.0 0.0 38.0 0.358599 2.04345 8.82778 221002.0 \n'])], + + # t_stats[fsnow] recording: + 'fsnow': + [('fsnow_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0 70.0 36.4813 30.8325 -0.322887 2379.0 \n', + '2012-337 0.0 0.0 0.0 0.0 nan 677.0 \n', + '2012-338 0.0 35.0 0.0644655 1.11486 23.1911 221002.0 \n'])], + + # t_stats[crcm] recording: + 'crcm': + [('crcm_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.8857 1.6238 0.960025 0.21929 -3.32392 221002.0 \n', + '2012-337 -0.8639 1.5786 0.966857 0.222374 -3.25955 221002.0 \n', + '2012-338 -0.8744 1.8977 0.963618 0.23254 -2.82714 218582.0 \n'])], + + # t_stats[ndsi] recording: + 'ndsi': + [('ndsi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.7313 1.0045 -0.58418 0.123674 6.27293 221002.0 \n', + '2012-337 -0.731 0.9434 -0.58564 0.12352 6.14429 221002.0 \n', + '2012-338 -0.7775 0.9329 -0.581803 0.125509 5.997 218582.0 \n'])], + + # t_stats[brgt] recording: + 'brgt': + [('brgt_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0062 0.1361 0.0467873 0.0103586 0.699293 221002.0 \n', + '2012-337 0.0062 0.1379 0.0465373 0.0103964 0.708942 221002.0 \n', + '2012-338 0.0064 0.1351 0.0464351 0.0105081 0.660629 220979.0 \n'])], + + # t_stats[msavi2] recording: + 'msavi2': + [('msavi2_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.014 0.6976 0.503178 0.0608721 -2.7989 219182.0 \n', + '2012-337 -0.0001 0.6996 0.502591 0.0601847 -2.75199 219026.0 \n', + '2012-338 0.014 0.693 0.499838 0.0606011 -2.67091 219027.0 \n'])], + + # t_stats[bi] recording: + 'bi': + [('bi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0033 0.1717 0.0948656 0.0173088 -1.23546 221002.0 \n', + '2012-337 0.0033 0.1714 0.0944838 0.0173897 -1.24311 221002.0 \n', + '2012-338 0.0038 0.1705 0.0938487 0.0174975 -1.19669 220979.0 \n'])], + + # t_stats[obstime] recording: + 'obstime': + [('obstime_stats.txt', + 'text-full', + ['date Observation Time Daytime Terra-min Observation Time Daytime Terra-max Observation Time Daytime Terra-mean Observation Time Daytime Terra-sd Observation Time Daytime Terra-skew Observation Time Daytime Terra-count Observation Time Nighttime Terra-min Observation Time Nighttime Terra-max Observation Time Nighttime Terra-mean Observation Time Nighttime Terra-sd Observation Time Nighttime Terra-skew Observation Time Nighttime Terra-count Observation Time Daytime Aqua-min Observation Time Daytime Aqua-max Observation Time Daytime Aqua-mean Observation Time Daytime Aqua-sd Observation Time Daytime Aqua-skew Observation Time Daytime Aqua-count Observation Time Nighttime Aqua-min Observation Time Nighttime Aqua-max Observation Time Nighttime Aqua-mean Observation Time Nighttime Aqua-sd Observation Time Nighttime Aqua-skew Observation Time Nighttime Aqua-count \n', + '2012-336 10.8 10.8 10.8 1.90735e-07 -0.999996 716.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 1.4 1.4 1.4 2.38419e-08 inf 6685.0 \n', + '2012-337 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 \n', + '2012-338 10.6 10.6 10.6 3.8147e-07 -1.0 184032.0 255.0 0.0 nan nan nan 0.0 12.3 12.3 12.3 1.90735e-07 -0.999996 216873.0 1.2 2.8 1.61335 0.700361 1.10413 181365.0 \n'])], + + # t_stats[lswi] recording: + 'lswi': + [('lswi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.9413 1.004 0.0704005 0.0795676 -0.897423 221002.0 \n', + '2012-337 -0.9644 0.879 0.068998 0.0814312 -1.75006 221002.0 \n', + '2012-338 -1.0001 0.8591 0.0695461 0.0851064 -1.65515 218582.0 \n'])], + + # t_stats[vari] recording: + 'vari': + [('vari_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.2931 3.2766 -0.0579385 0.0896767 11.2112 221002.0 \n', + '2012-337 -0.3266 3.2766 -0.0573387 0.109661 15.3818 221002.0 \n', + '2012-338 -0.3817 3.2766 -0.0567795 0.115018 15.8625 220979.0 \n'])], + + # t_stats[evi] recording: + 'evi': + [('evi_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.0689 0.4391 0.251258 0.0534915 -2.19097 221002.0 \n', + '2012-337 -0.0704 0.4432 0.250715 0.0535784 -2.20674 221002.0 \n', + '2012-338 -0.0727 0.4339 0.248372 0.0536924 -2.14955 220979.0 \n'])], + + # t_stats[temp8tn] recording: + 'temp8tn': + [('temp8tn_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-337 267.02 280.38 273.055 1.35905 0.334532 212758.0 \n'])], + + # t_stats[clouds] recording: + 'clouds': + [('clouds_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 0.0 1.0 0.990964 0.094628 -10.3767 221002.0 \n', + '2012-337 0.0 1.0 0.998367 0.0403832 -24.6819 221002.0 \n', + '2012-338 0.0 1.0 0.0838499 0.277163 3.00293 221002.0 \n'])], + + # t_stats[temp] recording: + 'temp': + [('temp_stats.txt', + 'text-full', + ['date Temperature Daytime Terra-min Temperature Daytime Terra-max Temperature Daytime Terra-mean Temperature Daytime Terra-sd Temperature Daytime Terra-skew Temperature Daytime Terra-count Temperature Nighttime Terra-min Temperature Nighttime Terra-max Temperature Nighttime Terra-mean Temperature Nighttime Terra-sd Temperature Nighttime Terra-skew Temperature Nighttime Terra-count Temperature Daytime Aqua-min Temperature Daytime Aqua-max Temperature Daytime Aqua-mean Temperature Daytime Aqua-sd Temperature Daytime Aqua-skew Temperature Daytime Aqua-count Temperature Nighttime Aqua-min Temperature Nighttime Aqua-max Temperature Nighttime Aqua-mean Temperature Nighttime Aqua-sd Temperature Nighttime Aqua-skew Temperature Nighttime Aqua-count Temperature Best Quality-min Temperature Best Quality-max Temperature Best Quality-mean Temperature Best Quality-sd Temperature Best Quality-skew Temperature Best Quality-count \n', + '2012-336 267.44 267.9 267.67 0.140381 -0.244244 716.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 258.94 263.7 261.912 1.39539 -0.717615 6685.0 0.0 0.0 0.0 0.0 nan 221255.0 \n', + '2012-337 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 0.0 0.0 0.0 0.0 nan 221255.0 \n', + '2012-338 275.34 286.28 282.987 2.17515 -1.16826 184032.0 65535.0 0.0 nan nan nan 0.0 277.56 283.86 281.873 0.980026 -0.679654 216873.0 270.04 280.26 277.74 1.43805 -1.36599 181365.0 0.0 13.0 4.60988 3.26751 1.22493 221255.0 \n'])], + + # t_stats[temp8td] recording: + 'temp8td': + [('temp8td_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-337 273.32 283.76 279.933 1.98691 -1.00057 218113.0 \n'])], + + # t_stats[sti] recording: + 'sti': + [('sti_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.0024 3.2766 1.94182 0.235117 0.707709 221002.0 \n', + '2012-337 0.2173 3.2766 1.95394 0.239902 0.761782 221002.0 \n', + '2012-338 0.3288 3.2766 1.96524 0.263691 0.833778 218582.0 \n'])], + + # t_stats[crc] recording: + 'crc': + [('crc_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2012-336 -0.8526 2.1525 1.29852 0.263609 -2.37667 221002.0 \n', + '2012-337 -0.868 2.2016 1.30555 0.268596 -2.32839 221002.0 \n', + '2012-338 -0.8795 2.8088 1.30562 0.283586 -1.87226 218582.0 \n'])], +} + +expectations['merra'] = { + # t_stats[srad] recording: + 'srad': + [('srad_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[tave] recording: + 'tave': + [('tave_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[shum] recording: + 'shum': + [('shum_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[rhum] recording: + 'rhum': + [('rhum_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[tmin] recording: + 'tmin': + [('tmin_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[tmax] recording: + 'tmax': + [('tmax_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[prcp] recording: + 'prcp': + [('prcp_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[patm] recording: + 'patm': + [('patm_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], + + # t_stats[wind] recording: + 'wind': + [('wind_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], +} + +expectations['aod'] = { + # t_stats[aod] recording: + 'aod': + [('aod_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2017-004 45.0 107.0 68.2053 27.2408 0.683439 2.70847e+06 \n', + '2017-005 34.0 130.0 68.7497 33.5872 0.965031 4.54937e+06 \n', + '2017-006 176.0 176.0 176.0 0.0 nan 927590.0 \n'])], +} diff --git a/gips/test/sys/t_process.py b/gips/test/sys/t_process.py index 0db2039e..563df2c4 100644 --- a/gips/test/sys/t_process.py +++ b/gips/test/sys/t_process.py @@ -4,10 +4,11 @@ from util import repo_wrapper import driver_setup -params = [] # (driver, product),... -expectations = {} # 'driver': {'product': [ (path, type, data...),...]...} +# 'driver': {'product': [ (path, type, data...),...]...} +from expected.std_process import expectations -@pytest.mark.parametrize("driver, product", params) +@pytest.mark.parametrize("driver, product", + util.params_from_expectations(expectations)) def t_process(repo_wrapper, driver, product): """Test gips_process output.""" record_mode, runner = repo_wrapper @@ -17,391 +18,3 @@ def t_process(repo_wrapper, driver, product): if not record_mode: # don't evaluate assertions when in record-mode assert (outcome.exit_code == 0 and expectations[driver][product] == actual) - -expectations['modis'] = { - # 'landcover' [], # is annual, not available for the scene under test - # 'indices', deprecated; not testing as a result - # weird symlink products: - # t_process[quality] recording: - 'quality': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_quality.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/modis/tiles/h12v04/2012336/MCD43A2.A2012336.h12v04.006.2016112010833.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_quality.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/modis/tiles/h12v04/2012337/MCD43A2.A2012337.h12v04.006.2016112013509.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_quality.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/modis/tiles/h12v04/2012338/MCD43A2.A2012338.h12v04.006.2016112020013.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo')], - - # t_process[temp8tn] recording: - 'temp8tn': - [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8tn.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Night_1km')], - - # t_process[temp8td] recording: - 'temp8td': - [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8td.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Day_1km')], - - # normal products - # t_process[satvi] recording: - 'satvi': - [('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_satvi.tif', - 'hash', - 'sha256', - '94bee7cecf0f0a227fd8532223f86de10e01c3e39d1dd70b7675d91b3ca338a4'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_satvi.tif', - 'hash', - 'sha256', - '6397669a6cd62bb39813b9000d93ba50f4cabe73aebf1e6444f4e3be9f630e19'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_satvi.tif', - 'hash', - 'sha256', - '6124ee615a4f522fdf966a5479d03d87934908e5090a192192f9700020f24760')], - - # t_process[ndti] recording: - 'ndti': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndti.tif', - 'hash', - 'sha256', - '025ea7a9204ae8ad0952a92b386bf735dbbce8337e336843476d3f227cf1f1ab'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndti.tif', - 'hash', - 'sha256', - 'b6baa8c7a3996f680c4177e4157c9275dd895c8e6350afa6ef564e673bb6a5f0'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndti.tif', - 'hash', - 'sha256', - 'a1c4f1161e8a584257352e76160319c39aa4605dd565d388d0116e193773771f')], - - # t_process[ndvi] recording: - 'ndvi': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndvi.tif', - 'hash', - 'sha256', - 'efbeae3a1c5e59ccd54b6108e275f9a8f0059162ac2797f4d69b00acb80147b3'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndvi.tif', - 'hash', - 'sha256', - 'b56d5c96229a1ad4cdcbd09db10b19a216b4cde8a0ffd184ee5cf01a35ab2b46'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndvi.tif', - 'hash', - 'sha256', - 'd9ff5d2f85ac9a267a8f6b1aca34f62b8196721e212b1e092f82ce9f1e208c61')], - - # t_process[isti] recording: - 'isti': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_isti.tif', - 'hash', - 'sha256', - 'c0a38d1cd00de52eacd8abcda714bfdaa575787e27b40dc0472ffca5ab1696c0'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_isti.tif', - 'hash', - 'sha256', - 'b1759c1008e5914b955c4a9f55572baa8f7c0310677264a1057329e236d826cb'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_isti.tif', - 'hash', - 'sha256', - '677182d186dac992b52b93239af0f419536075c7cd467864fa544bcde4b89e69')], - - # t_process[ndvi8] recording: - 'ndvi8': - [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_ndvi8.tif', - 'hash', - 'sha256', - '93db24c18886fbac0b87d1404a0c80606a599cc55b02d745a09ffb051b8131ac')], - - # t_process[snow] recording: - 'snow': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_snow.tif', - 'hash', - 'sha256', - 'bea7315b736a75bcf37df86089679845ac597f3fba5d5cd6e79f5516b9657faa'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_snow.tif', - 'hash', - 'sha256', - 'de979c1a0c616d7889dd5fd45816849f9c08df54167d7f3b40301d6ac4495778'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_snow.tif', - 'hash', - 'sha256', - '5d9bbfff1522e74165cdd677f1580e9eb2236c2c5922a796a6c9ee0db24ae05a')], - - # t_process[fsnow] recording: - 'fsnow': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_fsnow.tif', - 'hash', - 'sha256', - '853a5ca95f21c4a745e41e4acc46a71a74c74e0cdd1cae9ed3dd31b2fef1ddbd'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_fsnow.tif', - 'hash', - 'sha256', - '3a5f7c95fe60e5789919cf55d7f8e652e4d668c6eb63eaa53a82249c0ea19c5a'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_fsnow.tif', - 'hash', - 'sha256', - 'd09b8a12ad17a7e715ae2e988fbf683e5fe2df46e97e9ac3ebb4444c8b1c169d')], - - # t_process[crcm] recording: - 'crcm': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crcm.tif', - 'hash', - 'sha256', - 'a45c2fb16d871a919e0c803f80e1f3b4ec831472d0466c33741fcf8e8150f6ba'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crcm.tif', - 'hash', - 'sha256', - '9aa80c0e1fa1b62b0a41aed42eeaf84632845d74f29594dfee55c22673b0a6f5'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crcm.tif', - 'hash', - 'sha256', - 'a28c7d280fb8dfe0fe1cc003e1001457abdf14acc5545b2a69b782cfbcce7b50')], - - # t_process[ndsi] recording: - 'ndsi': - [('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndsi.tif', - 'hash', - 'sha256', - '1f91f738bbb28b3ebeba36affd8d0689e105e90e1547cca58497eb7c7fdf7346'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndsi.tif', - 'hash', - 'sha256', - 'a626d375fbe5094a5199fdfc2206cf94777ff574379b45811cf7a49055a9f6ec'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndsi.tif', - 'hash', - 'sha256', - 'cb2ca7461c65f66ddb13e804ae7fe8f54daaf620ade600efee24810d40d4eb34')], - - # t_process[brgt] recording: - 'brgt': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_brgt.tif', - 'hash', - 'sha256', - 'bf6bad538c7c6e703bc2f39503c2f128fa8e988a533e302c0124e085735dfaf9'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_brgt.tif', - 'hash', - 'sha256', - '5c72e8f37640ec6b4a5a33e18980d71c70482fc8f708ba25da73e074cf64729f'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_brgt.tif', - 'hash', - 'sha256', - '3460d0d35e35540b55e9eda35f253af8d85eabb84c0c44cb5f9c02d43da7e20a')], - - # t_process[msavi2] recording: - 'msavi2': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_msavi2.tif', - 'hash', - 'sha256', - 'd2c93798df10939ff8afb6cd23e18ac3281cfd844e45da47a29d0342c2b980e9'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_msavi2.tif', - 'hash', - 'sha256', - '92fe4f40d051ef3f479bde76a67bfff978bc513893af5ed62e4a48e457daabad'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_msavi2.tif', - 'hash', - 'sha256', - 'f25bd2d58d3edae72b11e73c08341479dee13af80352a4749861b6a54a81b351')], - - # t_process[bi] recording: - 'bi': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_bi.tif', - 'hash', - 'sha256', - '28c181b8d210b71759589da53295cc894c9d12c74ff1f7b0a49b88f73664282e'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_bi.tif', - 'hash', - 'sha256', - '45acfc9249e063b6b97f269338590d51e2334e9bc5df118613ad908c6f0a3f8e'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_bi.tif', - 'hash', - 'sha256', - '8526cb96d59fc9549dc8535fc8128123b9d39a63a3b7cd97ee3e611b7eeadf6f')], - - # t_process[obstime] recording: - 'obstime': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - '66e01c94910c4fdd4f6fb2ab99b0c2d0cb73618032783e3cbdf1867191981fdc'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - '117cea0497b3158b465dd7dd197c886ebb1acf5059ed83dfdd6b5d7e94bb5243'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - 'b04d8b07ef83e479db5650a30c84513bde7d5b05dd68187782c057ea0fd3e537')], - - # t_process[lswi] recording: - 'lswi': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_lswi.tif', - 'hash', - 'sha256', - '8de7378cdba830171c8da94424bf7807fa6c8b12523e8b34175a4b17ec770a5d'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_lswi.tif', - 'hash', - 'sha256', - '62de4bdafcc276e9c1c343f4e63f5dd06167e87f7712796f991e3a957f2b71de'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_lswi.tif', - 'hash', - 'sha256', - 'add051ef35d7ffff5eb68c48f8b8071f9cb493b5d3426867605d4c61746e3a26')], - - # t_process[vari] recording: - 'vari': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_vari.tif', - 'hash', - 'sha256', - '732d6d3d9fbe0e1b4616a3f8a91bf12d532598a0a02e8aab16c9a01c3ac56913'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_vari.tif', - 'hash', - 'sha256', - '9875717cc5e1926f11378bbf16345dbade5a2badf9f3dd06844a77f94d80ef72'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_vari.tif', - 'hash', - 'sha256', - '70fd19862ef4145c24f26a8a74cb748279b88fd973de935726ebef2294e17edd')], - - # t_process[evi] recording: - 'evi': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_evi.tif', - 'hash', - 'sha256', - '13548ae74456f08e6add657a637ca364626eafd2cc767724f57aac8ff93005bb'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_evi.tif', - 'hash', - 'sha256', - '7b926f7004cd92f2ec604fc4e2cc1489f60a335e39f4954030332a012fd33f87'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_evi.tif', - 'hash', - 'sha256', - '51db813a857f795817f529edd56d7134004566be46edab8a8dadc81286fb895f')], - - # t_process[clouds] recording: - 'clouds': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MOD_clouds.tif', - 'hash', - 'sha256', - '26b3e7e695f08ec2e9fb26985f9d5e392f9e8cedc331ab6721ea8e48821d93f0'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MOD_clouds.tif', - 'hash', - 'sha256', - 'c43eec8178d0521d20e2a5b6e9b61525007d80702c6dd705a3e52ed3a76a7b73'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_clouds.tif', - 'hash', - 'sha256', - '24a6a7050c866eed989530bec45b72004097f1e36fcc0e686652b87723a86de4')], - - # t_process[temp] recording: - 'temp': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_temp.tif', - 'hash', - 'sha256', - 'ead2622531929bba3192b8a6b609d5d920e79394ec577c5245d74d69593d827c'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '5d498bad2779f84289dfb75b14345d64d16b5bc7994b4aca6cd694f70b86064d'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '097f341fa917a0ec316c37d7907ff15b754dcd0925469be586016137ca47575f')], - - # t_process[sti] recording: - 'sti': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_sti.tif', - 'hash', - 'sha256', - '24e50766ec18826f30bbad07bc8f5a34e72f1966b325cc23f064aa5fd35dc24a'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_sti.tif', - 'hash', - 'sha256', - 'c223de9d00e9cfedc0cfc13a45163535f3c59e5587642fa2c725b00b17b7437c'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_sti.tif', - 'hash', - 'sha256', - '73c247fd2c7623e79145b6337846c97d14d1752d660b2f32245a4ccaac033350')], - - # t_process[crc] recording: - 'crc': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crc.tif', - 'hash', - 'sha256', - '07debf171aedfeba7d6c1e3220ff8e4a2a7b1033e68c72bbc121b500b6ed3c9c'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crc.tif', - 'hash', - 'sha256', - 'a7b61aabc950be7a3040b6f9572bf755522b12ad27644583e603def05ac1980b'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crc.tif', - 'hash', - 'sha256', - '942efeeee2b020123be6d7334bbb3ef5995c93d2fd0247cb79948836865c8067')], -} - -expectations['merra'] = { - # test this too? 'frland': [], - # t_process[srad] recording: - 'srad': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_srad.tif', - 'hash', - 'sha256', - '882a44af70e337bf905c8695936f101792a518507a4304b23781a2f30aaabab4')], - # t_process[tave] recording: - 'tave': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tave.tif', - 'hash', - 'sha256', - '5185aebd7cda54157cad2ddbde9f6fec4871c1b85fe78d0738634ba0211f2c9b')], - # t_process[shum] recording: - 'shum': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_shum.tif', - 'hash', - 'sha256', - '558f567c1e931891553d396f81b2b90929ad48a0ae88fa95d3894eae23ab3eba')], - # t_process[rhum] recording: - 'rhum': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_rhum.tif', - 'hash', - 'sha256', - '058161eb8488f1fb3df7f5c8719833c9d68c56aa082d4605938f62c2ee0035f6')], - # t_process[tmin] recording: - 'tmin': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmin.tif', - 'hash', - 'sha256', - '35a850769a3bb8f5209574ea4835bf417d8f99024567f4af9cdee9e623d0c567')], - # t_process[tmax] recording: - 'tmax': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmax.tif', - 'hash', - 'sha256', - '5963fcf346f388fc347355153996a078f1eb5ecbbc094394b72be382ba491865')], - # t_process[prcp] recording: - 'prcp': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_prcp.tif', - 'hash', - 'sha256', - '348fd7072ef0a8625f5268e29acbf3a69a959a412327102ef6558c992e62bc9f')], - # t_process[patm] recording: - 'patm': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_patm.tif', - 'hash', - 'sha256', - '7d9deca613973cb8ffb8f15a3dfa2013e6783556b520915e89b97fe41de638e3')], - # t_process[wind] recording: - 'wind': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_wind.tif', - 'hash', - 'sha256', - 'a90c1d87fdb1c0926bd5b3004408a1f9d42ef08c38deb9d0572bc7536dbbf08a')], -} - -params += [('modis', p) for p in expectations['modis'].keys()] -params += [('merra', p) for p in expectations['merra'].keys()] diff --git a/gips/test/sys/t_project.py b/gips/test/sys/t_project.py index 3b7d1302..9e89a833 100644 --- a/gips/test/sys/t_project.py +++ b/gips/test/sys/t_project.py @@ -4,10 +4,11 @@ from util import export_wrapper import driver_setup -params = [] # (driver, product),... -expectations = {} # 'driver': {'product': [ (path, type, data...),...]...} +# 'driver': {'product': [ (path, type, data...),...]...} +from expected.std_project import expectations -@pytest.mark.parametrize("driver, product", params) +@pytest.mark.parametrize("driver, product", + util.params_from_expectations(expectations)) def t_project(export_wrapper, driver, product): """Test gips_project with warping.""" record_mode, runner = export_wrapper @@ -18,411 +19,3 @@ def t_project(export_wrapper, driver, product): if not record_mode: # don't evaluate assertions when in record-mode assert (outcome.exit_code == 0 and expectations[driver][product] == actual) - -expectations['modis'] = { - # t_project[satvi] recording: - 'satvi': - [('0/2012336_MCD_satvi.tif', - 'hash', - 'sha256', - '893d968ff6359d0d34f8a587246690d1c724832505d66fbee8be8c1a43a6e3df'), - ('0/2012337_MCD_satvi.tif', - 'hash', - 'sha256', - '6092d7cfb609f000124b8ef107f278decaa27c3cdb69696bd094940ead97d96e'), - ('0/2012338_MCD_satvi.tif', - 'hash', - 'sha256', - '37dd32e0f5e761846f44615ec6e82e2cce8d6c98197f0bd721dcfb266d963b6b')], - - # t_project[ndti] recording: - 'ndti': - [('0/2012337_MCD_ndti.tif', - 'hash', - 'sha256', - 'db85948227815660b94d570adf0f1119d9f1001284c3de6c8b219c6455bd3c4e'), - ('0/2012338_MCD_ndti.tif', - 'hash', - 'sha256', - '5028c194e41d8c07076cee09631b276ef7b86c3c45b0a4f705f5e4c6b72c2812'), - ('0/2012336_MCD_ndti.tif', - 'hash', - 'sha256', - '157134ea0e9cbb0b2fcdb4203e17e82ae906a5779eb1f735e8c55d1f60c231e1')], - - # t_project[ndvi] recording: - 'ndvi': - [('0/2012337_MCD_ndvi.tif', - 'hash', - 'sha256', - '28a288720a0846348fbd6315e5d9e3801ad76eb2c873dc028e7a52d1fdcdad10'), - ('0/2012336_MCD_ndvi.tif', - 'hash', - 'sha256', - '78e090b1cc92304c666a8e480d11fa99985ee3d2113c4fe3be29fb85910b86b9'), - ('0/2012338_MCD_ndvi.tif', - 'hash', - 'sha256', - 'bbcee4ac1b49466c90264e305c20ce19a021fd177ffc35da5ed92a86555c480e')], - - # t_project[quality] recording: - 'quality': - [('0/2012338_MCD_quality.tif', - 'hash', - 'sha256', - '895abfc825d9da56c56202c670f94a16b28f2aaab5590db55ac6e903b6638e91'), - ('0/2012336_MCD_quality.tif', - 'hash', - 'sha256', - '4f40fdeb7ded507a68f404ec34a17fb6283c9b90d81ecccc7782746eaba85323'), - ('0/2012337_MCD_quality.tif', - 'hash', - 'sha256', - '5cdd315b2a423fbb82c19cfe859325c889a5d05cf0c3cdf71359b257524175a3')], - - # t_project[isti] recording: - 'isti': - [('0/2012337_MCD_isti.tif', - 'hash', - 'sha256', - '391367ee5bdd0a96435c09df85db312919121c5d9a28b5bda59a68db7b609871'), - ('0/2012338_MCD_isti.tif', - 'hash', - 'sha256', - 'cc082d45a2a4dd4a7e53d0970029decf24ce5033b9715930cd250b524ad87757'), - ('0/2012336_MCD_isti.tif', - 'hash', - 'sha256', - '16be9705c06a44f3a35e684cded2d2ceb26213510eaf0ccfbafb62ebc39bb153')], - - # t_project[ndvi8] recording: - 'ndvi8': - [('0/2012337_MOD_ndvi8.tif', - 'hash', - 'sha256', - '23a72df39da253ac07de5a9e98c88f40e3335f3b1b779ccf91153d1d5573a780')], - - # t_project[snow] recording: - 'snow': - [('0/2012336_MCD_snow.tif', - 'hash', - 'sha256', - 'b3eeca8fc933d2f43c2d96e869956447d9556737181635167744d0d420dfd813'), - ('0/2012338_MCD_snow.tif', - 'hash', - 'sha256', - '98a38e1a75fcea269bd75a952cde9d84af86b14ddb928e4777ff812dba8befbe'), - ('0/2012337_MCD_snow.tif', - 'hash', - 'sha256', - 'da26696be559bb79c399ad18942ac629a9604d857dd7fc461fd76c4577a06303')], - - # t_project[fsnow] recording: - 'fsnow': - [('0/2012338_MCD_fsnow.tif', - 'hash', - 'sha256', - 'f2efdf504e004caf05563f3402fdde7e771351d7d93de5970af0a54d0b5bd2c2'), - ('0/2012336_MCD_fsnow.tif', - 'hash', - 'sha256', - 'a9d2ea29a6d77325246e0baaf298d16cb84d6003a19faed8ba162b475703f563'), - ('0/2012337_MCD_fsnow.tif', - 'hash', - 'sha256', - 'e8465aa3764c60b21a943f3e078d93c0484272292f075455693ea2084971ace4')], - - # t_project[crcm] recording: - 'crcm': - [('0/2012337_MCD_crcm.tif', - 'hash', - 'sha256', - 'c00e7b7c6970ae23904ee52afd614a8871f3d771d26830a2aef412163d9d9228'), - ('0/2012338_MCD_crcm.tif', - 'hash', - 'sha256', - 'ed336a3ba4cf482936ca781ea1a5e0f04ca5ad68919c3dbe6e117edac55ec242'), - ('0/2012336_MCD_crcm.tif', - 'hash', - 'sha256', - 'f217a9278b195e945e2337663cf7dfe2b09faa0dc68062fbc3614a6954866eaf')], - - # t_project[ndsi] recording: - 'ndsi': - [('0/2012338_MCD_ndsi.tif', - 'hash', - 'sha256', - '8e26a8f6d78b9612c8cd3924936c5944f56c67ae1c63f6541d045b41a27baafe'), - ('0/2012337_MCD_ndsi.tif', - 'hash', - 'sha256', - '2aaea763ab1d5b54d7079e7ee2381bd987a3c72c5cbbd6f18f97aa320c186e2a'), - ('0/2012336_MCD_ndsi.tif', - 'hash', - 'sha256', - '83b6bbf241d080cf6860798da299038be5e6d3e20a8b544e5af51e2eefd3a5c9')], - - # t_project[brgt] recording: - 'brgt': - [('0/2012337_MCD_brgt.tif', - 'hash', - 'sha256', - 'fc385e6e42538a09655762d1b2557abe35af38aecef1571705cec8b971efbe4d'), - ('0/2012338_MCD_brgt.tif', - 'hash', - 'sha256', - 'a8f8ef66bc87c2370bd61d5eed64905ba4f832d05579687478ff5f38c65e8038'), - ('0/2012336_MCD_brgt.tif', - 'hash', - 'sha256', - '24f8130de39daac6e181cb2f98b48d9a14412700632d3cbd2ffbe2704283b740')], - - # t_project[msavi2] recording: - 'msavi2': - [('0/2012336_MCD_msavi2.tif', - 'hash', - 'sha256', - '997b562420fe223677881d577cdb2100ee653ee932b9989b95f3089c0fe38352'), - ('0/2012338_MCD_msavi2.tif', - 'hash', - 'sha256', - '26a7ab45723856361d108064fe0c810535de75f272a733d86f66eeb8146dee35'), - ('0/2012337_MCD_msavi2.tif', - 'hash', - 'sha256', - 'ee7b2006b79f65bdf11d77f678d66765abfcc024b7856ab296f855570efb9940')], - - # t_project[bi] recording: - 'bi': - [('0/2012336_MCD_bi.tif', - 'hash', - 'sha256', - 'e5f891a6290301508107714657274fc808fcf96cb138c652a9c22552675796a5'), - ('0/2012337_MCD_bi.tif', - 'hash', - 'sha256', - '58b9e3809630ba24879bb85e81bc6dc0eed548348373588d70d49de65d246582'), - ('0/2012338_MCD_bi.tif', - 'hash', - 'sha256', - '914ca6d15c25a22d3e0a43d0d266f72bd65d102b4d6b015a9f12b8ee3a116c13')], - - # t_project[obstime] recording: - 'obstime': - [('0/2012337_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - 'a9345d21f3d754d083d3b4d13b059f2ffe07eb42ea9a0d6f3e40ef326e4ea75f'), - ('0/2012336_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - 'fac305543c4afbf76611900579cd5f37d63e7a45814c997dce40fc46a74f13e5'), - ('0/2012338_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - 'e18456caaf9a2dd005cc6568f7aa77ac534bfa24ec364d3af28affb4ae652de9')], - - # t_project[lswi] recording: - 'lswi': - [('0/2012337_MCD_lswi.tif', - 'hash', - 'sha256', - '586dd3782070b983b9d383161bed82bee21c54ad312d1c993b0329965144deab'), - ('0/2012336_MCD_lswi.tif', - 'hash', - 'sha256', - '296c849e70d31127bcefef54ffed72832529f7ae60d50b7c639c5c6d253d30f4'), - ('0/2012338_MCD_lswi.tif', - 'hash', - 'sha256', - '992fa3d7b4bdc8cf95fba759a0096f9a913e84762717e93c477baae0d7d3c0ed')], - - # t_project[vari] recording: - 'vari': - [('0/2012336_MCD_vari.tif', - 'hash', - 'sha256', - 'be5d2273b9f0b9aca79df89b66ca0e7e1a2b8d386e6d34a6eca6eb37810b488d'), - ('0/2012338_MCD_vari.tif', - 'hash', - 'sha256', - '016ec5b1ee512d9306251a4c2b3630c90ea18723159a6a254c8e77c4e730abc8'), - ('0/2012337_MCD_vari.tif', - 'hash', - 'sha256', - 'a6c10e80436b92908637b22b40e7993a0a648e82b96993760a7b26ba43df0667')], - - # t_project[evi] recording: - 'evi': - [('0/2012338_MCD_evi.tif', - 'hash', - 'sha256', - '8a1beba0730ac7d113a7edb8b55035a742892bbe678cd0eaaa540e59906a6990'), - ('0/2012337_MCD_evi.tif', - 'hash', - 'sha256', - '824a5247b0e8e0730dec6e82b390533649aa796ef04ef4be27ac38cae9a24d04'), - ('0/2012336_MCD_evi.tif', - 'hash', - 'sha256', - '361798060d02c97153a9828c1f8ac6b7df396070ec243932e9ed18fd472e9669')], - - # t_project[temp8tn] recording: - 'temp8tn': - [('0/2012337_MOD_temp8tn.tif', - 'hash', - 'sha256', - '1600d412e2590b4f4006ac0d98ebf9c6bf83fe16967c2fd721e412d2a14040c6')], - - # t_project[clouds] recording: - 'clouds': - [('0/2012337_MOD_clouds.tif', - 'hash', - 'sha256', - '8b13afeec6d1a5612a3eeb51896507e2d4eeb851261881f3854d2b804e97835f'), - ('0/2012336_MOD_clouds.tif', - 'hash', - 'sha256', - 'cb4fab2519652bebe4830479c95425050f63d519397beb5b0c40349cc6f49372'), - ('0/2012338_MOD_clouds.tif', - 'hash', - 'sha256', - '54505d6d4063caa27fda74f5fdbb916c20d281f6e15deea28596c379ffa661e2')], - - # t_project[temp] recording: - 'temp': - [('0/2012336_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '614023db948d0b0249057ca9c458356c2222e45224a726f87a7cf512cd87745c'), - ('0/2012338_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '51864699b8347a30d9669ec41298665cd893a27516089cc21b4c3f03b3e65b2b'), - ('0/2012337_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '29373079984e8aaf55add6a1fdd7bd766eb89a1c74ebc85674da54aeb643e3ef')], - - # t_project[temp8td] recording: - 'temp8td': - [('0/2012337_MOD_temp8td.tif', - 'hash', - 'sha256', - 'eaa0ff94bb34cbad8d9c187b16c7cd2bd5948d1d94cf1caf30b0396ac7e8a735')], - - # t_project[sti] recording: - 'sti': - [('0/2012337_MCD_sti.tif', - 'hash', - 'sha256', - '9850777cae46b42fa3c8361849529e6f230dd1ce86fb6c6676aa8eb2768e71d5'), - ('0/2012336_MCD_sti.tif', - 'hash', - 'sha256', - 'b7fde6a04ac091946553b25b1a26de806fa7f9c23e915b415034ef811aca1d16'), - ('0/2012338_MCD_sti.tif', - 'hash', - 'sha256', - 'b2714cadf4e779c433415f194f366ac87877c60b383f46971843b4ab3a610fa9')], - - # t_project[crc] recording: - 'crc': - [('0/2012338_MCD_crc.tif', - 'hash', - 'sha256', - 'dc9a4885e694fe7117cf10df11b5589745b4eb119d5f58f9d1c14e27d0eda333'), - ('0/2012337_MCD_crc.tif', - 'hash', - 'sha256', - '5688d80a613c28c657237186291a88ae786037a77dad0247c0bd734da214c927'), - ('0/2012336_MCD_crc.tif', - 'hash', - 'sha256', - '3df0003b6153970e73c46d5e999697ce5af268744274ccd3a4a2c14e3ae932cb')], -} - -expectations['merra'] = { - # t_project[srad] recording: - 'srad': - [('0/2015135_merra_srad.tif', - 'hash', - 'sha256', - 'bfc0b906f7299ff8e6fcbbe474de0fb7b877d36f2e619e2e6ac10bd84ac5b55d')], - - # t_project[tave] recording: - 'tave': - [('0/2015135_merra_tave.tif', - 'hash', - 'sha256', - 'c96d89ebf5d7ec90193c8a113609bfc39473d72041b6145a8cd41ee49e4f6a4d')], - - # t_project[prcp] recording: - 'prcp': - [('0/2015135_merra_prcp.tif', - 'hash', - 'sha256', - 'b7e68237ee3557c421856203cc22807f1ce74790ea2948f8d095cbf8a7f18a53')], - - # t_project[rhum] recording: - 'rhum': - [('0/2015135_merra_rhum.tif', - 'hash', - 'sha256', - '3b45ba1cd87d4781ff95b08562a00189506c76add24156f8e3719ff8872c3d3a')], - - # t_project[tmin] recording: - 'tmin': - [('0/2015135_merra_tmin.tif', - 'hash', - 'sha256', - '253e37e9b2c03a77a0a4fdfe5aaa0e3ff025471f64fd28d6ffb33dd8ecdd68a9')], - - # t_project[tmax] recording: - 'tmax': - [('0/2015135_merra_tmax.tif', - 'hash', - 'sha256', - '8371d473f0e506e6ab0fe872542b8cd0d65bc2ed2b37bbeb747c966c41e9c232')], - - # t_project[shum] recording: - 'shum': - [('0/2015135_merra_shum.tif', - 'hash', - 'sha256', - '208465fbf10da2e935807e00991fa6375c3c6b67d9950362a9ffe502d7690fd6')], - - # t_project[patm] recording: - 'patm': - [('0/2015135_merra_patm.tif', - 'hash', - 'sha256', - '5778a4d89b22ea76dc25654c8092a61bbb09634f06f66b578460a93444e513a7')], - - # t_project[wind] recording: - 'wind': - [('0/2015135_merra_wind.tif', - 'hash', - 'sha256', - '88029a425fbeb59149a29be793087b51ea46583bb89e62c1baca629ccfc3a36a')], -} - -expectations['aod'] = { - # t_project[aod] recording: - 'aod': - [('0/2017004_MOD_aod.tif', - 'hash', - 'sha256', - '2fe93815b6d1686fd357e8685c82ff23689d99200b3aec2654277fb6db86a8c7'), - ('0/2017005_MOD_aod.tif', - 'hash', - 'sha256', - '3830875aa1603cc86fc452286d365c1dcd12af60889f0761d8df3493c7541ef7'), - ('0/2017006_MOD_aod.tif', - 'hash', - 'sha256', - 'ed9fc349c037f39eca87f2e7495024eeaf7fd0e99165ecb0f52cc7927e985977')], -} - -for driver, prod_expectations in expectations.items(): - params += [(driver, product) for product in prod_expectations.keys()] diff --git a/gips/test/sys/t_stats.py b/gips/test/sys/t_stats.py index a52bb9a0..bf624999 100644 --- a/gips/test/sys/t_stats.py +++ b/gips/test/sys/t_stats.py @@ -5,10 +5,11 @@ from util import export_wrapper import driver_setup -params = [] # (driver, product),... -expectations = {} # 'driver': {'product': [ (path, type, data...),...]...} +# 'driver': {'product': [ (path, type, data...),...]...} +from expected.std_stats import expectations -@pytest.mark.parametrize("driver, product", params) +@pytest.mark.parametrize("driver, product", + util.params_from_expectations(expectations)) def t_stats(export_wrapper, driver, product): """Test gips_stats on projected files.""" record_mode, runner = export_wrapper @@ -24,285 +25,3 @@ def t_stats(export_wrapper, driver, product): if not record_mode: assert (outcome.exit_code == 0 and expectations[driver][product] == actual) - -expectations['modis'] = { - # t_stats[satvi] recording: - 'satvi': - [('satvi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.1199 0.2757 0.188845 0.0394886 -2.85581 221002.0 \n', - '2012-337 -0.1179 0.2955 0.189273 0.0399094 -2.7273 221002.0 \n', - '2012-338 -0.1125 0.2985 0.187186 0.0408718 -2.44579 218582.0 \n'])], - - # t_stats[ndti] recording: - 'ndti': - [('ndti_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -1.0048 0.8895 0.31602 0.0595381 -2.46398 221002.0 \n', - '2012-337 -0.643 0.9544 0.31893 0.0596861 -0.906354 221002.0 \n', - '2012-338 -0.5051 0.9999 0.320866 0.064446 -0.195554 218582.0 \n'])], - - # t_stats[ndvi] recording: - 'ndvi': - [('ndvi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.8893 0.8144 0.598198 0.130008 -4.50499 221002.0 \n', - '2012-337 -0.9126 0.8198 0.598463 0.132692 -4.61097 221002.0 \n', - '2012-338 -1.0001 0.8117 0.595837 0.134746 -4.68736 220979.0 \n'])], - - # t_stats[quality] recording: - 'quality': - [('quality_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0 0.0 0.0 0.0 nan 221002.0 \n', - '2012-337 0.0 0.0 0.0 0.0 nan 221002.0 \n', - '2012-338 0.0 0.0 0.0 0.0 nan 220979.0 \n'])], - - # t_stats[isti] recording: - 'isti': - [('isti_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0583 3.2766 0.522676 0.0837755 13.2684 220980.0 \n', - '2012-337 0.0232 3.2766 0.519636 0.0845782 12.4045 221002.0 \n', - '2012-338 -0.0001 3.0399 0.517872 0.0852503 7.69029 218582.0 \n'])], - - # t_stats[ndvi8] recording: - 'ndvi8': - [('ndvi8_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-337 -1.9475 3.2766 0.612297 0.162128 -3.72226 220758.0 \n'])], - - # t_stats[snow] recording: - 'snow': - [('snow_stats.txt', - 'text-full', - ['date Snow Cover-min Snow Cover-max Snow Cover-mean Snow Cover-sd Snow Cover-skew Snow Cover-count Fractional Snow Cover-min Fractional Snow Cover-max Fractional Snow Cover-mean Fractional Snow Cover-sd Fractional Snow Cover-skew Fractional Snow Cover-count \n', - '2012-336 0.0 100.0 66.0782 43.1646 -0.66934 2379.0 0.0 70.0 44.8928 24.6763 -0.938924 2379.0 \n', - '2012-337 0.0 0.0 0.0 0.0 nan 677.0 0.0 0.0 0.0 0.0 nan 677.0 \n', - '2012-338 0.0 100.0 7.3169 20.9113 2.95797 221002.0 0.0 38.0 0.358599 2.04345 8.82778 221002.0 \n'])], - - # t_stats[fsnow] recording: - 'fsnow': - [('fsnow_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0 70.0 36.4813 30.8325 -0.322887 2379.0 \n', - '2012-337 0.0 0.0 0.0 0.0 nan 677.0 \n', - '2012-338 0.0 35.0 0.0644655 1.11486 23.1911 221002.0 \n'])], - - # t_stats[crcm] recording: - 'crcm': - [('crcm_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.8857 1.6238 0.960025 0.21929 -3.32392 221002.0 \n', - '2012-337 -0.8639 1.5786 0.966857 0.222374 -3.25955 221002.0 \n', - '2012-338 -0.8744 1.8977 0.963618 0.23254 -2.82714 218582.0 \n'])], - - # t_stats[ndsi] recording: - 'ndsi': - [('ndsi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.7313 1.0045 -0.58418 0.123674 6.27293 221002.0 \n', - '2012-337 -0.731 0.9434 -0.58564 0.12352 6.14429 221002.0 \n', - '2012-338 -0.7775 0.9329 -0.581803 0.125509 5.997 218582.0 \n'])], - - # t_stats[brgt] recording: - 'brgt': - [('brgt_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0062 0.1361 0.0467873 0.0103586 0.699293 221002.0 \n', - '2012-337 0.0062 0.1379 0.0465373 0.0103964 0.708942 221002.0 \n', - '2012-338 0.0064 0.1351 0.0464351 0.0105081 0.660629 220979.0 \n'])], - - # t_stats[msavi2] recording: - 'msavi2': - [('msavi2_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.014 0.6976 0.503178 0.0608721 -2.7989 219182.0 \n', - '2012-337 -0.0001 0.6996 0.502591 0.0601847 -2.75199 219026.0 \n', - '2012-338 0.014 0.693 0.499838 0.0606011 -2.67091 219027.0 \n'])], - - # t_stats[bi] recording: - 'bi': - [('bi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0033 0.1717 0.0948656 0.0173088 -1.23546 221002.0 \n', - '2012-337 0.0033 0.1714 0.0944838 0.0173897 -1.24311 221002.0 \n', - '2012-338 0.0038 0.1705 0.0938487 0.0174975 -1.19669 220979.0 \n'])], - - # t_stats[obstime] recording: - 'obstime': - [('obstime_stats.txt', - 'text-full', - ['date Observation Time Daytime Terra-min Observation Time Daytime Terra-max Observation Time Daytime Terra-mean Observation Time Daytime Terra-sd Observation Time Daytime Terra-skew Observation Time Daytime Terra-count Observation Time Nighttime Terra-min Observation Time Nighttime Terra-max Observation Time Nighttime Terra-mean Observation Time Nighttime Terra-sd Observation Time Nighttime Terra-skew Observation Time Nighttime Terra-count Observation Time Daytime Aqua-min Observation Time Daytime Aqua-max Observation Time Daytime Aqua-mean Observation Time Daytime Aqua-sd Observation Time Daytime Aqua-skew Observation Time Daytime Aqua-count Observation Time Nighttime Aqua-min Observation Time Nighttime Aqua-max Observation Time Nighttime Aqua-mean Observation Time Nighttime Aqua-sd Observation Time Nighttime Aqua-skew Observation Time Nighttime Aqua-count \n', - '2012-336 10.8 10.8 10.8 1.90735e-07 -0.999996 716.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 1.4 1.4 1.4 2.38419e-08 inf 6685.0 \n', - '2012-337 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 255.0 0.0 nan nan nan 0.0 \n', - '2012-338 10.6 10.6 10.6 3.8147e-07 -1.0 184032.0 255.0 0.0 nan nan nan 0.0 12.3 12.3 12.3 1.90735e-07 -0.999996 216873.0 1.2 2.8 1.61335 0.700361 1.10413 181365.0 \n'])], - - # t_stats[lswi] recording: - 'lswi': - [('lswi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.9413 1.004 0.0704005 0.0795676 -0.897423 221002.0 \n', - '2012-337 -0.9644 0.879 0.068998 0.0814312 -1.75006 221002.0 \n', - '2012-338 -1.0001 0.8591 0.0695461 0.0851064 -1.65515 218582.0 \n'])], - - # t_stats[vari] recording: - 'vari': - [('vari_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.2931 3.2766 -0.0579385 0.0896767 11.2112 221002.0 \n', - '2012-337 -0.3266 3.2766 -0.0573387 0.109661 15.3818 221002.0 \n', - '2012-338 -0.3817 3.2766 -0.0567795 0.115018 15.8625 220979.0 \n'])], - - # t_stats[evi] recording: - 'evi': - [('evi_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.0689 0.4391 0.251258 0.0534915 -2.19097 221002.0 \n', - '2012-337 -0.0704 0.4432 0.250715 0.0535784 -2.20674 221002.0 \n', - '2012-338 -0.0727 0.4339 0.248372 0.0536924 -2.14955 220979.0 \n'])], - - # t_stats[temp8tn] recording: - 'temp8tn': - [('temp8tn_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-337 267.02 280.38 273.055 1.35905 0.334532 212758.0 \n'])], - - # t_stats[clouds] recording: - 'clouds': - [('clouds_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 0.0 1.0 0.990964 0.094628 -10.3767 221002.0 \n', - '2012-337 0.0 1.0 0.998367 0.0403832 -24.6819 221002.0 \n', - '2012-338 0.0 1.0 0.0838499 0.277163 3.00293 221002.0 \n'])], - - # t_stats[temp] recording: - 'temp': - [('temp_stats.txt', - 'text-full', - ['date Temperature Daytime Terra-min Temperature Daytime Terra-max Temperature Daytime Terra-mean Temperature Daytime Terra-sd Temperature Daytime Terra-skew Temperature Daytime Terra-count Temperature Nighttime Terra-min Temperature Nighttime Terra-max Temperature Nighttime Terra-mean Temperature Nighttime Terra-sd Temperature Nighttime Terra-skew Temperature Nighttime Terra-count Temperature Daytime Aqua-min Temperature Daytime Aqua-max Temperature Daytime Aqua-mean Temperature Daytime Aqua-sd Temperature Daytime Aqua-skew Temperature Daytime Aqua-count Temperature Nighttime Aqua-min Temperature Nighttime Aqua-max Temperature Nighttime Aqua-mean Temperature Nighttime Aqua-sd Temperature Nighttime Aqua-skew Temperature Nighttime Aqua-count Temperature Best Quality-min Temperature Best Quality-max Temperature Best Quality-mean Temperature Best Quality-sd Temperature Best Quality-skew Temperature Best Quality-count \n', - '2012-336 267.44 267.9 267.67 0.140381 -0.244244 716.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 258.94 263.7 261.912 1.39539 -0.717615 6685.0 0.0 0.0 0.0 0.0 nan 221255.0 \n', - '2012-337 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 65535.0 0.0 nan nan nan 0.0 0.0 0.0 0.0 0.0 nan 221255.0 \n', - '2012-338 275.34 286.28 282.987 2.17515 -1.16826 184032.0 65535.0 0.0 nan nan nan 0.0 277.56 283.86 281.873 0.980026 -0.679654 216873.0 270.04 280.26 277.74 1.43805 -1.36599 181365.0 0.0 13.0 4.60988 3.26751 1.22493 221255.0 \n'])], - - # t_stats[temp8td] recording: - 'temp8td': - [('temp8td_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-337 273.32 283.76 279.933 1.98691 -1.00057 218113.0 \n'])], - - # t_stats[sti] recording: - 'sti': - [('sti_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.0024 3.2766 1.94182 0.235117 0.707709 221002.0 \n', - '2012-337 0.2173 3.2766 1.95394 0.239902 0.761782 221002.0 \n', - '2012-338 0.3288 3.2766 1.96524 0.263691 0.833778 218582.0 \n'])], - - # t_stats[crc] recording: - 'crc': - [('crc_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2012-336 -0.8526 2.1525 1.29852 0.263609 -2.37667 221002.0 \n', - '2012-337 -0.868 2.2016 1.30555 0.268596 -2.32839 221002.0 \n', - '2012-338 -0.8795 2.8088 1.30562 0.283586 -1.87226 218582.0 \n'])], -} - -expectations['merra'] = { - # t_stats[srad] recording: - 'srad': - [('srad_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[tave] recording: - 'tave': - [('tave_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[shum] recording: - 'shum': - [('shum_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[rhum] recording: - 'rhum': - [('rhum_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[tmin] recording: - 'tmin': - [('tmin_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[tmax] recording: - 'tmax': - [('tmax_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[prcp] recording: - 'prcp': - [('prcp_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[patm] recording: - 'patm': - [('patm_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], - - # t_stats[wind] recording: - 'wind': - [('wind_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2015-135 3.4e+38 -3.4e+38 nan nan nan 0.0 \n'])], -} - -expectations['aod'] = { - # t_stats[aod] recording: - 'aod': - [('aod_stats.txt', - 'text-full', - ['date min max mean sd skew count\n', - '2017-004 45.0 107.0 68.2053 27.2408 0.683439 2.70847e+06 \n', - '2017-005 34.0 130.0 68.7497 33.5872 0.965031 4.54937e+06 \n', - '2017-006 176.0 176.0 176.0 0.0 nan 927590.0 \n'])], -} - -for driver, prod_expectations in expectations.items(): - params += [(driver, product) for product in prod_expectations.keys()] diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index 33b82393..b8ac6fa5 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -288,6 +288,12 @@ def generate_file_hash(filename, blocksize=2**20): m.update(buf) return m.hexdigest() +def params_from_expectations(expectations): + """Generates a standard system test (driver, product) parameter list.""" + return [(driver, product) + for driver, prod_expectations in expectations.items() + for product in prod_expectations] + def record_path(): path = pytest.config.getoption('--record') return False if path in (None, '') else path From 7de944ad03d5107c5880658f1d41b50ea77d20d7 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 27 Dec 2017 13:24:18 -0500 Subject: [PATCH 27/44] 483 convert prism to new test framework --- gips/test/sys/driver_setup.py | 9 ++- gips/test/sys/expected/prism.py | 102 -------------------------- gips/test/sys/expected/std_process.py | 56 ++++++++++++++ gips/test/sys/expected/std_project.py | 54 ++++++++++++++ gips/test/sys/expected/std_stats.py | 36 +++++++++ gips/test/sys/t_prism.py | 37 ---------- 6 files changed, 152 insertions(+), 142 deletions(-) diff --git a/gips/test/sys/driver_setup.py b/gips/test/sys/driver_setup.py index c56ac608..c394e3c8 100644 --- a/gips/test/sys/driver_setup.py +++ b/gips/test/sys/driver_setup.py @@ -7,10 +7,13 @@ import util +nh_shp = util.NH_SHP_PATH + STD_ARGS = { - 'modis': ('modis', '-s', util.NH_SHP_PATH, '-d', '2012-12-01,2012-12-03', '-v', '4'), - 'merra': ('merra', '-s', util.NH_SHP_PATH, '-d', '2015-135', '-v', '4'), - 'aod': ('aod', '-s', util.NE_SHP_PATH, '-d', '2017-004,2017-006', '-v', '4'), + 'modis': ('modis', '-s', nh_shp, '-d', '2012-12-01,2012-12-03', '-v', '4'), + 'merra': ('merra', '-s', nh_shp, '-d', '2015-135', '-v', '4'), + 'aod': ('aod', '-s', nh_shp, '-d', '2017-004,2017-006', '-v', '4'), + 'prism': ('prism', '-s', nh_shp, '-d', '1982-12-01,1982-12-03', '-v4'), } setup_attempted = [] diff --git a/gips/test/sys/expected/prism.py b/gips/test/sys/expected/prism.py index 1f662902..cc48089c 100644 --- a/gips/test/sys/expected/prism.py +++ b/gips/test/sys/expected/prism.py @@ -1,91 +1,4 @@ """Known-good outcomes for tests, mostly stdout and created files.""" - -t_process = { - 'updated': { - 'prism/stage': None, - 'prism/tiles/CONUS/19821201': None, - 'prism/tiles/CONUS/19821202': None, - 'prism/tiles/CONUS/19821203': None - }, - 'created': { - 'prism/tiles/CONUS/19821201/CONUS_19821201_prism_ppt.tif': None, - 'prism/tiles/CONUS/19821201/CONUS_19821201_prism_tmax.tif': None, - 'prism/tiles/CONUS/19821201/CONUS_19821201_prism_tmin.tif': None, - 'prism/tiles/CONUS/19821202/CONUS_19821202_prism_ppt.tif': None, - 'prism/tiles/CONUS/19821202/CONUS_19821202_prism_tmax.tif': None, - 'prism/tiles/CONUS/19821202/CONUS_19821202_prism_tmin.tif': None, - 'prism/tiles/CONUS/19821203/CONUS_19821203_prism_ppt.tif': None, - 'prism/tiles/CONUS/19821203/CONUS_19821203_prism_tmax.tif': None, - 'prism/tiles/CONUS/19821203/CONUS_19821203_prism_tmin.tif': None, - 'prism/tiles/CONUS/19821203/CONUS_19821203_prism_pptsum-3.tif': 1770768528, - 'prism/tiles/CONUS/19821201/PRISM_tmax_stable_4kmD1_19821201_bil.zip.index': -74851230, - 'prism/tiles/CONUS/19821203/PRISM_ppt_stable_4kmD2_19821203_bil.zip.index': 1520480318, - 'prism/tiles/CONUS/19821203/PRISM_tmin_stable_4kmD1_19821203_bil.zip.index': -946867120, - 'prism/tiles/CONUS/19821202/PRISM_tmax_stable_4kmD1_19821202_bil.zip.index': 280913761, - 'prism/tiles/CONUS/19821202/PRISM_tmin_stable_4kmD1_19821202_bil.zip.index': 676175726, - 'prism/tiles/CONUS/19821203/PRISM_tmax_stable_4kmD1_19821203_bil.zip.index': -8647585, - 'prism/tiles/CONUS/19821202/PRISM_ppt_stable_4kmD2_19821202_bil.zip.index': -1251812608, - 'prism/tiles/CONUS/19821201/PRISM_tmin_stable_4kmD1_19821201_bil.zip.index': -1015405459, - 'prism/tiles/CONUS/19821201/PRISM_ppt_stable_4kmD2_19821201_bil.zip.index': 1582653443, - }, - '_symlinks': { - # TODO do the rest of the symlinks as time allows - 'prism/tiles/CONUS/19821201/CONUS_19821201_prism_ppt.tif': - 'prism/tiles/CONUS/19821201/PRISM_ppt_stable_4kmD2_19821201_bil.zip' - '/PRISM_ppt_stable_4kmD2_19821201_bil.bil', - }, - 'ignored': [ - 'gips-inv-db.sqlite3', - ], - '_inv_stdout': """\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site NHseacoast-0 - -\x1b[1mAsset Coverage for site NHseacoast-0\x1b[0m -\x1b[1m -Tile Coverage -\x1b[4m Tile % Coverage % Tile Used\x1b[0m - CONUS 99.2% 0.0% - -\x1b[1m\x1b[4m DATE _ppt _tmax _tmin Product \x1b[0m -\x1b[1m1982 -\x1b[0m 335 99.2% 99.2% 99.2% \x1b[35mppt\x1b[0m \x1b[35mtmax\x1b[0m \x1b[35mtmin\x1b[0m - 336 99.2% 99.2% 99.2% \x1b[35mppt\x1b[0m \x1b[35mtmax\x1b[0m \x1b[35mtmin\x1b[0m - 337 99.2% 99.2% 99.2% \x1b[35mppt\x1b[0m \x1b[35mpptsum-3\x1b[0m \x1b[35mtmax\x1b[0m \x1b[35mtmin\x1b[0m - - -3 files on 3 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mprism: Daily Gridded Climate Data\x1b[0m -""", -} - -t_project = { - 'created': { - '0': None, - '0/1982335_prism_ppt.tif': -393322682, - '0/1982335_prism_tmax.tif': 455141340, - '0/1982335_prism_tmin.tif': 847249412, - '0/1982336_prism_ppt.tif': 233388192, - '0/1982336_prism_tmax.tif': 858443300, - '0/1982336_prism_tmin.tif': -1078834087, - '0/1982337_prism_ppt.tif': 983840913, - '0/1982337_prism_pptsum.tif': 366592820, - '0/1982337_prism_tmax.tif': 1878755022, - '0/1982337_prism_tmin.tif': -100140734, -# '0/1982335_prism_ppt.tif': -1525304978, -# '0/1982335_prism_tmax.tif': 1259010415, -# '0/1982335_prism_tmin.tif': -106507302, -# '0/1982336_prism_ppt.tif': -92502584, -# '0/1982336_prism_tmax.tif': 381286493, -# '0/1982336_prism_tmin.tif': -801866142, -# '0/1982337_prism_ppt.tif': 670927499, -# '0/1982337_prism_pptsum.tif': 1195812568, -# '0/1982337_prism_tmax.tif': 1464461372, -# '0/1982337_prism_tmin.tif': -2082314142, - } -} - t_project_no_warp = { 'created': {'0': None, '0/1982335_prism_ppt.tif': -1700182526, @@ -111,11 +24,6 @@ } } -# haven't ever used tiles -# t_tiles = { -# 'created': {}, -# } - t_tiles_copy = { 'created': { 'CONUS': None, @@ -131,13 +39,3 @@ 'CONUS/CONUS_1982337_prism_pptsum.tif': 272867928, } } - -t_stats = { - 'created': { - 'ppt_stats.txt': -2108420090, - 'pptsum_stats.txt': -1221703895, - 'tmax_stats.txt': -628985934, - 'tmin_stats.txt': 691861124 - } -} - diff --git a/gips/test/sys/expected/std_process.py b/gips/test/sys/expected/std_process.py index ed8a7ee5..890f81ad 100644 --- a/gips/test/sys/expected/std_process.py +++ b/gips/test/sys/expected/std_process.py @@ -386,3 +386,59 @@ 'a90c1d87fdb1c0926bd5b3004408a1f9d42ef08c38deb9d0572bc7536dbbf08a')], } +expectations['prism'] = { + # t_process[tmin] recording: + 'tmin': + [('prism/tiles/CONUS/19821201/CONUS_19821201_prism_tmin.tif', + 'symlink', + '/vsizip/', + '/prism/tiles/CONUS/19821201/PRISM_tmin_stable_4kmD1_19821201_bil.zip/PRISM_tmin_stable_4kmD1_19821201_bil.bil'), + ('prism/tiles/CONUS/19821202/CONUS_19821202_prism_tmin.tif', + 'symlink', + '/vsizip/', + '/prism/tiles/CONUS/19821202/PRISM_tmin_stable_4kmD1_19821202_bil.zip/PRISM_tmin_stable_4kmD1_19821202_bil.bil'), + ('prism/tiles/CONUS/19821203/CONUS_19821203_prism_tmin.tif', + 'symlink', + '/vsizip/', + '/prism/tiles/CONUS/19821203/PRISM_tmin_stable_4kmD1_19821203_bil.zip/PRISM_tmin_stable_4kmD1_19821203_bil.bil')], + + # t_process[tmax] recording: + 'tmax': + [('prism/tiles/CONUS/19821203/CONUS_19821203_prism_tmax.tif', + 'symlink', + '/vsizip/', + '/prism/tiles/CONUS/19821203/PRISM_tmax_stable_4kmD1_19821203_bil.zip/PRISM_tmax_stable_4kmD1_19821203_bil.bil'), + ('prism/tiles/CONUS/19821201/CONUS_19821201_prism_tmax.tif', + 'symlink', + '/vsizip/', + '/prism/tiles/CONUS/19821201/PRISM_tmax_stable_4kmD1_19821201_bil.zip/PRISM_tmax_stable_4kmD1_19821201_bil.bil'), + ('prism/tiles/CONUS/19821202/CONUS_19821202_prism_tmax.tif', + 'symlink', + '/vsizip/', + '/prism/tiles/CONUS/19821202/PRISM_tmax_stable_4kmD1_19821202_bil.zip/PRISM_tmax_stable_4kmD1_19821202_bil.bil')], + + # IMPORTANT NOTE pptsum seems to generate ppt products as part of + # its function; as a result ppt products may exist already if pptsum + # goes first. + # t_process[ppt] recording: + 'ppt': + [('prism/tiles/CONUS/19821203/CONUS_19821203_prism_ppt.tif', + 'symlink', + '/vsizip/', + '/prism/tiles/CONUS/19821203/PRISM_ppt_stable_4kmD2_19821203_bil.zip/PRISM_ppt_stable_4kmD2_19821203_bil.bil'), + ('prism/tiles/CONUS/19821201/CONUS_19821201_prism_ppt.tif', + 'symlink', + '/vsizip/', + '/prism/tiles/CONUS/19821201/PRISM_ppt_stable_4kmD2_19821201_bil.zip/PRISM_ppt_stable_4kmD2_19821201_bil.bil'), + ('prism/tiles/CONUS/19821202/CONUS_19821202_prism_ppt.tif', + 'symlink', + '/vsizip/', + '/prism/tiles/CONUS/19821202/PRISM_ppt_stable_4kmD2_19821202_bil.zip/PRISM_ppt_stable_4kmD2_19821202_bil.bil')], + + # t_process[pptsum] recording: + 'pptsum': + [('prism/tiles/CONUS/19821203/CONUS_19821203_prism_pptsum-3.tif', + 'hash', + 'sha256', + 'd9772d10372ad86550dacae3d5645837cd9a10228db70f4e2bc7149b836495d9')], +} diff --git a/gips/test/sys/expected/std_project.py b/gips/test/sys/expected/std_project.py index a2929942..b689e763 100644 --- a/gips/test/sys/expected/std_project.py +++ b/gips/test/sys/expected/std_project.py @@ -404,3 +404,57 @@ 'sha256', 'ed9fc349c037f39eca87f2e7495024eeaf7fd0e99165ecb0f52cc7927e985977')], } + +expectations['prism'] = { + # t_project[tmax] recording: + 'tmax': + [('0/1982337_prism_tmax.tif', + 'hash', + 'sha256', + '34354af37f2f9802c3c2a6e49437b14b1a61ddf3c378d2306df21dbcd8068abe'), + ('0/1982336_prism_tmax.tif', + 'hash', + 'sha256', + 'ac083002cacf2fa456514c609e07256b08ac3fdcb8c870485145b34a113f65e0'), + ('0/1982335_prism_tmax.tif', + 'hash', + 'sha256', + '6e9b399a305ea1449d2c57752a7e013109b383de7414145017089858f1533dd3')], + + # t_project[pptsum] recording: + 'pptsum': + [('0/1982337_prism_pptsum.tif', + 'hash', + 'sha256', + '661586ac559c12a329375583f3ecee7c2837de6f276f9ee91e9b2c93b108f3ea')], + + # t_project[tmin] recording: + 'tmin': + [('0/1982337_prism_tmin.tif', + 'hash', + 'sha256', + 'e60f8950f9a1ab4a1d192aab7bf4ccf577983d591cf3f2a488de10ce44568667'), + ('0/1982336_prism_tmin.tif', + 'hash', + 'sha256', + 'a415b4b5df69bfc2ece115784f29151d6be95d8f2cc478d8c8c42e317c394129'), + ('0/1982335_prism_tmin.tif', + 'hash', + 'sha256', + 'cc22853f539a0f4049d97f3e70f1af2d6aec7254504d42b11f9f41ce12bd1a68')], + + # t_project[ppt] recording: + 'ppt': + [('0/1982336_prism_ppt.tif', + 'hash', + 'sha256', + '452953fa26769ed0c3de1c698972baeede2749d6599408a4d2b67ae9bea3ca11'), + ('0/1982337_prism_ppt.tif', + 'hash', + 'sha256', + 'f083e20b518336c4432c397d9ca35cf0d6a7da618cae230615580902fd70bca6'), + ('0/1982335_prism_ppt.tif', + 'hash', + 'sha256', + '46e0db478d2c9f9cb6ecd3d321c2ac9b69d30c6a5d2eb964ac6195d398fed661')], +} diff --git a/gips/test/sys/expected/std_stats.py b/gips/test/sys/expected/std_stats.py index eccc30ff..3d55d7ee 100644 --- a/gips/test/sys/expected/std_stats.py +++ b/gips/test/sys/expected/std_stats.py @@ -278,3 +278,39 @@ '2017-005 34.0 130.0 68.7497 33.5872 0.965031 4.54937e+06 \n', '2017-006 176.0 176.0 176.0 0.0 nan 927590.0 \n'])], } + +expectations['prism'] = { + # t_stats[tmax] recording: + 'tmax': + [('tmax_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '1982-335 6.68 10.74 9.45062 0.989373 -1.16521 221810.0 \n', + '1982-336 7.63 11.52 10.2559 0.957033 -1.03102 221810.0 \n', + '1982-337 12.97 15.32 14.5481 0.598565 -0.984863 221810.0 \n'])], + + # t_stats[pptsum] recording: + 'pptsum': + [('pptsum_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '1982-337 0.0 0.29 0.00668477 0.0376131 6.09544 221810.0 \n'])], + + # t_stats[tmin] recording: + 'tmin': + [('tmin_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '1982-335 2.57 3.83 3.04506 0.268788 0.766181 221810.0 \n', + '1982-336 -0.02 3.38 1.64335 0.757131 0.14854 221810.0 \n', + '1982-337 1.11 3.17 2.11856 0.465469 0.147961 221810.0 \n'])], + + # t_stats[ppt] recording: + 'ppt': + [('ppt_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '1982-335 0.0 0.0 0.0 0.0 nan 221810.0 \n', + '1982-336 0.0 0.0 0.0 0.0 nan 221810.0 \n', + '1982-337 0.0 0.29 0.00668477 0.0376131 6.09544 221810.0 \n'])], +} diff --git a/gips/test/sys/t_prism.py b/gips/test/sys/t_prism.py index e0883239..91afcd7e 100644 --- a/gips/test/sys/t_prism.py +++ b/gips/test/sys/t_prism.py @@ -35,27 +35,6 @@ def setup_prism_data(pytestconfig): # ### SHOULD BE STANDARD BELOW HERE ##### -def t_process(setup_fixture, repo_env, expected): - """Test gips_process on {} data.""".format(driver) - process_actual = repo_env.run('gips_process', *STD_ARGS) - inventory_actual = envoy.run('gips_inventory ' + ' '.join(STD_ARGS)) - - # TODO port this to all drivers that symlink products -- move to util, common call, yadda yadda - # Check symlinks specially: Build list of expected paths to symlinks, and their - # expected targets, then compare with actual symlinks on the filesystem. - # have to build expectations about symlinks dynamically as they depend on a user setting. - # setting in question -----vvvvvvvvvvvvvv - tgt_prefix = '/vsizip//' + DATA_REPO_ROOT.strip('/') - expected_symlinks = { - os.path.join(DATA_REPO_ROOT, partial_ln): os.path.join(tgt_prefix, partial_tgt) - for partial_ln, partial_tgt in expected._symlinks.items()} - actual_symlinks = {ln: os.readlink(ln) if os.path.islink(ln) else 'Not-a-symlink' - for ln in expected_symlinks} - - assert (expected == process_actual and inventory_actual.std_out == expected._inv_stdout - and expected_symlinks == actual_symlinks) - - # # TODO: determine why overwrite fails (see comment in prism driver) # # and then uncomment this test. # def t_process_overwrite(setup_fixture, repo_env, expected): @@ -66,28 +45,12 @@ def t_process(setup_fixture, repo_env, expected): # assert expected == process_run2 # assert process_run1.timestamps != process_run2.timestamps -def t_project(setup_fixture, clean_repo_env, output_tfe, expected): - """Test gips_project {} with warping.""".format(driver) - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld') - actual = output_tfe.run('gips_project', *args) - assert expected == actual - - def t_project_no_warp(setup_fixture, clean_repo_env, output_tfe, expected): """Test gips_project {} without warping.""".format(driver) args = STD_ARGS + ('--outdir', OUTPUT_DIR, '--notld') actual = output_tfe.run('gips_project', *args) assert expected == actual - -# # Haven't used gips_tiles ever -# def t_tiles(setup_fixture, clean_repo_env, output_tfe, expected): -# """Test gips_tiles {} with warping.""".format(driver) -# args = STD_ARGS + ('--outdir', OUTPUT_DIR, '--notld') -# actual = output_tfe.run('gips_tiles', *args) -# assert expected == actual - - def t_tiles_copy(setup_fixture, clean_repo_env, output_tfe, expected): """Test gips_tiles {} with copying.""".format(driver) # doesn't quite use STD_ARGS From 793c6d32dc9a42623d12318980a59b6c05bad9a3 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 27 Dec 2017 15:08:37 -0500 Subject: [PATCH 28/44] 483 accomodate gips_stats stupidity: So if you say `gips_stats -p tmax dir/` and tmax products from two different drivers are present in dir, gips_stats will process both and have no way to disambiguate between them. So, isolate drivers from each other to make the tests pass. --- gips/test/sys/t_project.py | 4 ++-- gips/test/sys/t_stats.py | 8 +++++--- gips/test/sys/util.py | 14 +++++++++----- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/gips/test/sys/t_project.py b/gips/test/sys/t_project.py index 9e89a833..3c273c98 100644 --- a/gips/test/sys/t_project.py +++ b/gips/test/sys/t_project.py @@ -11,9 +11,9 @@ util.params_from_expectations(expectations)) def t_project(export_wrapper, driver, product): """Test gips_project with warping.""" - record_mode, runner = export_wrapper + record_mode, runner, working_dir = export_wrapper args = ('gips_project',) + driver_setup.STD_ARGS[driver] + ( - '--res', '100', '100', '--outdir', util.OUTPUT_DIR, '--notld', + '--res', '100', '100', '--outdir', working_dir, '--notld', '-p', product) outcome, actual = runner(*args) if not record_mode: # don't evaluate assertions when in record-mode diff --git a/gips/test/sys/t_stats.py b/gips/test/sys/t_stats.py index bf624999..32e05479 100644 --- a/gips/test/sys/t_stats.py +++ b/gips/test/sys/t_stats.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import pytest import sh @@ -12,16 +14,16 @@ util.params_from_expectations(expectations)) def t_stats(export_wrapper, driver, product): """Test gips_stats on projected files.""" - record_mode, runner = export_wrapper + record_mode, runner, working_dir = export_wrapper # generate data needed for stats computation args = driver_setup.STD_ARGS[driver] + ('--res', '100', '100', '--notld', - '--outdir', util.OUTPUT_DIR, '-p', product) + '--outdir', working_dir, '-p', product) outcome = sh.gips_project(*args) assert outcome.exit_code == 0 # sanity check # compute & confirm stats - outcome, actual = runner('gips_stats', util.OUTPUT_DIR) + outcome, actual = runner('gips_stats', working_dir, '-p', product) if not record_mode: assert (outcome.exit_code == 0 and expectations[driver][product] == actual) diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index b8ac6fa5..0dd052fb 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -310,9 +310,9 @@ def sys_test_wrapper(request, path): param. """ rp = record_path() # does the user want record mode? If so, save it where? - product = request.getfixturevalue('product') - p = request.node.callspec.params - expected = request.module.expectations[p['driver']][p['product']] + driver = request.node.callspec.params['driver'] + product = request.node.callspec.params['product'] + expected = request.module.expectations[driver][product] expected_filenames = [e[0] for e in expected] if not rp: @@ -357,8 +357,12 @@ def repo_wrapper(request): @pytest.yield_fixture def export_wrapper(request): - for rv in sys_test_wrapper(request, OUTPUT_DIR): - yield rv + driver = request.node.callspec.params['driver'] + product = request.node.callspec.params['product'] + working_dir = os.path.join(OUTPUT_DIR, '{}-{}'.format(driver, product)) + os.makedirs(working_dir) # raises if leaf dir exists; this is desired + for rv in sys_test_wrapper(request, working_dir): + yield rv + (working_dir,) @pytest.yield_fixture def repo_env(request): From 066e31acb644d8d20c725efb34599f045a5253aa Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 27 Dec 2017 15:28:09 -0500 Subject: [PATCH 29/44] 483 debug aod NH_SHP_PATH vs NE_SHP_PATH: There's an hour of my life I'm never getting back. --- gips/test/sys/driver_setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gips/test/sys/driver_setup.py b/gips/test/sys/driver_setup.py index c394e3c8..a1c0781e 100644 --- a/gips/test/sys/driver_setup.py +++ b/gips/test/sys/driver_setup.py @@ -12,8 +12,9 @@ STD_ARGS = { 'modis': ('modis', '-s', nh_shp, '-d', '2012-12-01,2012-12-03', '-v', '4'), 'merra': ('merra', '-s', nh_shp, '-d', '2015-135', '-v', '4'), - 'aod': ('aod', '-s', nh_shp, '-d', '2017-004,2017-006', '-v', '4'), 'prism': ('prism', '-s', nh_shp, '-d', '1982-12-01,1982-12-03', '-v4'), + # NOT new hampshire, new england! vvv + 'aod': ('aod', '-s', util.NE_SHP_PATH, '-d', '2017-004,2017-006', '-v4'), } setup_attempted = [] From 3778839a9bf1ae5851b94299274ec36be43ff4b7 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 27 Dec 2017 15:35:31 -0500 Subject: [PATCH 30/44] 483 trash obselete test (forgot earlier) --- gips/test/sys/t_prism.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/gips/test/sys/t_prism.py b/gips/test/sys/t_prism.py index 91afcd7e..ccdb21e1 100644 --- a/gips/test/sys/t_prism.py +++ b/gips/test/sys/t_prism.py @@ -58,18 +58,3 @@ def t_tiles_copy(setup_fixture, clean_repo_env, output_tfe, expected): '--outdir', OUTPUT_DIR, '--notld') actual = output_tfe.run('gips_tiles', *args) assert expected == actual - - -def t_stats(setup_fixture, clean_repo_env, output_tfe, expected): - """Test gips_stats on projected files.""" - # generate data needed for stats computation - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld') - prep_run = output_tfe.run('gips_project', *args) - assert prep_run.exit_status == 0 # confirm it worked; not really in the test - - # compute stats - gtfe = GipsTestFileEnv(OUTPUT_DIR, start_clear=False) - actual = gtfe.run('gips_stats', OUTPUT_DIR) - - # check for correct stats content - assert expected == actual From 6a76baed10d6708db506d1cd4f5a2da0ae9f1676 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Thu, 28 Dec 2017 09:57:49 -0500 Subject: [PATCH 31/44] 483 necessary marking: * gatekeeper --sys option * per-param marking & example (sentinel2 --slow) --- gips/test/sys/driver_setup.py | 5 +++-- gips/test/sys/expected/std_process.py | 14 ++++++++++++++ gips/test/sys/t_process.py | 9 ++++++--- gips/test/sys/t_project.py | 2 ++ gips/test/sys/t_stats.py | 2 ++ gips/test/sys/util.py | 26 +++++++++++++++++++++----- 6 files changed, 48 insertions(+), 10 deletions(-) diff --git a/gips/test/sys/driver_setup.py b/gips/test/sys/driver_setup.py index a1c0781e..75bbb1e9 100644 --- a/gips/test/sys/driver_setup.py +++ b/gips/test/sys/driver_setup.py @@ -13,8 +13,9 @@ 'modis': ('modis', '-s', nh_shp, '-d', '2012-12-01,2012-12-03', '-v', '4'), 'merra': ('merra', '-s', nh_shp, '-d', '2015-135', '-v', '4'), 'prism': ('prism', '-s', nh_shp, '-d', '1982-12-01,1982-12-03', '-v4'), - # NOT new hampshire, new england! vvv - 'aod': ('aod', '-s', util.NE_SHP_PATH, '-d', '2017-004,2017-006', '-v4'), + # Here down, not NH shapefile: + 'aod': ('aod', '-s', util.NE_SHP_PATH, '-d', '2017-004,2017-006', '-v4'), + 'sentinel2': ('sentinel2', '-s', util.DURHAM_SHP_PATH, '-d2017-010', '-v4') } setup_attempted = [] diff --git a/gips/test/sys/expected/std_process.py b/gips/test/sys/expected/std_process.py index 890f81ad..0ac0dd09 100644 --- a/gips/test/sys/expected/std_process.py +++ b/gips/test/sys/expected/std_process.py @@ -1,5 +1,8 @@ +from .. import util + expectations = {} +mark_spec = {} expectations['modis'] = { # 'landcover' [], # is annual, not available for the scene under test @@ -442,3 +445,14 @@ 'sha256', 'd9772d10372ad86550dacae3d5645837cd9a10228db70f4e2bc7149b836495d9')], } + +mark_spec['sentinel2'] = util.slow + +expectations['sentinel2'] = { + 'ndvi-toa': [ + ('sentinel2/tiles/19TCH/2017010/19TCH_2017010_S2A_ndvi-toa.tif', + 'hash', + 'sha256', + '3d1decf61c3f469e342c343626d2c6e354bba48dc64da731fc774682be8752ca'), + ], +} diff --git a/gips/test/sys/t_process.py b/gips/test/sys/t_process.py index 563df2c4..6a66fa2f 100644 --- a/gips/test/sys/t_process.py +++ b/gips/test/sys/t_process.py @@ -5,10 +5,13 @@ import driver_setup # 'driver': {'product': [ (path, type, data...),...]...} -from expected.std_process import expectations +from expected.std_process import expectations, mark_spec -@pytest.mark.parametrize("driver, product", - util.params_from_expectations(expectations)) +pytestmark = util.sys # skip everything unless --sys + +params = util.params_from_expectations(expectations, mark_spec) + +@pytest.mark.parametrize("driver, product", params) def t_process(repo_wrapper, driver, product): """Test gips_process output.""" record_mode, runner = repo_wrapper diff --git a/gips/test/sys/t_project.py b/gips/test/sys/t_project.py index 3c273c98..71acd395 100644 --- a/gips/test/sys/t_project.py +++ b/gips/test/sys/t_project.py @@ -4,6 +4,8 @@ from util import export_wrapper import driver_setup +pytestmark = util.sys # skip everything unless --sys + # 'driver': {'product': [ (path, type, data...),...]...} from expected.std_project import expectations diff --git a/gips/test/sys/t_stats.py b/gips/test/sys/t_stats.py index 32e05479..daa52294 100644 --- a/gips/test/sys/t_stats.py +++ b/gips/test/sys/t_stats.py @@ -7,6 +7,8 @@ from util import export_wrapper import driver_setup +pytestmark = util.sys # skip everything unless --sys + # 'driver': {'product': [ (path, type, data...),...]...} from expected.std_stats import expectations diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index 0dd052fb..259ad1fa 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -288,11 +288,27 @@ def generate_file_hash(filename, blocksize=2**20): m.update(buf) return m.hexdigest() -def params_from_expectations(expectations): - """Generates a standard system test (driver, product) parameter list.""" - return [(driver, product) - for driver, prod_expectations in expectations.items() - for product in prod_expectations] + +def params_from_expectations(expectations, mark_spec=None): + """Generates a standard system test (driver, product) parameter list. + + Pass in a dict for mark_spec and any marks matching the given driver + or (driver, product) will be applied accordingly. These marks can be + singular or in iterables.""" + mark_spec = {} if mark_spec is None else mark_spec + params = [] + + def get_mark_spec(i): + msi = mark_spec.get(i, []) + is_mark = isinstance(msi, pytest.mark.MarkDecorator.__class__) + return [msi] if is_mark else list(msi) # handle case of multiple marks + + for driver, prod_expectations in expectations.items(): + dm = get_mark_spec(driver) # get driver-scoped marks + for product in prod_expectations: + dpm = get_mark_spec((driver, product)) # get product-scoped marks + params.append(pytest.param(driver, product, marks=(dm + dpm))) + return params def record_path(): path = pytest.config.getoption('--record') From af2bd9530ecc3f3a18f0493b3b47187e9ebc59b1 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Thu, 28 Dec 2017 11:15:40 -0500 Subject: [PATCH 32/44] 483 sentinel-2 conversion to new infrastructure --- gips/test/sys/expected/std_process.py | 72 ++++++++++++++++++++++++--- gips/test/sys/expected/std_project.py | 64 ++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 8 deletions(-) diff --git a/gips/test/sys/expected/std_process.py b/gips/test/sys/expected/std_process.py index 0ac0dd09..0e616416 100644 --- a/gips/test/sys/expected/std_process.py +++ b/gips/test/sys/expected/std_process.py @@ -1,4 +1,6 @@ +import collections + from .. import util expectations = {} @@ -448,11 +450,65 @@ mark_spec['sentinel2'] = util.slow -expectations['sentinel2'] = { - 'ndvi-toa': [ - ('sentinel2/tiles/19TCH/2017010/19TCH_2017010_S2A_ndvi-toa.tif', - 'hash', - 'sha256', - '3d1decf61c3f469e342c343626d2c6e354bba48dc64da731fc774682be8752ca'), - ], -} +expectations['sentinel2'] = collections.OrderedDict([ + # TODO acolite products; should use --acolite flag + # acoflags 0 = water 1 = no data 2 = land + # fai Floating Algae Index + # oc2chl Blue-Green Ratio Chlorophyll Algorithm using bands 483 & 561 + # oc3chl Blue-Green Ratio Chlorophyll Algorithm using bands 443, 483, & 561 + # rhow Water-Leaving Radiance-Reflectance + # spm655 Suspended Sediment Concentration 655nm + # turbidity Blended Turbidity + ('ref-toa', + [('sentinel2/tiles/19TCH/2017010/19TCH_2017010_S2A_ref-toa.tif', + 'hash', + 'sha256', + '59521f38170ff8d5d001292dc9424fe23ce0c7226f6dcc79801d7b78eb8faacd')], + ), + ('rad-toa', + [('sentinel2/tiles/19TCH/2017010/19TCH_2017010_S2A_rad-toa.tif', + 'hash', + 'sha256', + 'fa8a2cb6917d68d77931fb52c4a1dadd93501a7431f966e2641179ed570f7d3e')], + ), + ('rad', + [('sentinel2/tiles/19TCH/2017010/19TCH_2017010_S2A_rad.tif', + 'hash', + 'sha256', + '7017fdee077e01f592c80465a82a7f6b9904ed60d6fdfc941f26c2d94299438c')], + ), + ('ref', + [('sentinel2/tiles/19TCH/2017010/19TCH_2017010_S2A_ref.tif', + 'hash', + 'sha256', + '34e1f63f0680e05172dd763f78d55254c0f44c9b730f00c01646acc7b4ad452b')], + ), + ('cfmask', + [('sentinel2/tiles/19TCH/2017010/19TCH_2017010_S2A_cfmask.tif', + 'hash', + 'sha256', + 'd4438cc066e2e4cc26891635cbcaf46c00c46e240a29d5e7397344538bba7346')], + ), + + # all the index products are made by the same gippy call so given + # how slow sentinel2 is, just do a few for now; it probably + # exercises the code well enough. How to know if you're exercising + # the bands: + # https://github.com/gipit/gippy/blob/6d201870e55a7855814b3bdd3d30b05a889c24ed/GIP/GeoAlgorithms.cpp#L458 + # other indices if desired: + # 'lswi', 'bi', 'brgt', 'crc', 'isti', 'msavi2', + # 'ndti', 'satvi', 'sti', 'vari', + ('evi-toa', # nir, red, blue, TOA version + [('sentinel2/tiles/19TCH/2017010/19TCH_2017010_S2A_evi-toa.tif', + 'hash', + 'sha256', + '0d68b587a7a84c4287c5aa8621c82aa0d8be237a1b1e80f7ac4855dfdd4fe9ec')], + ), + # t_process[crcm] recording: + ('crcm', # swir1, swir2, green, surface version + [('sentinel2/tiles/19TCH/2017010/19TCH_2017010_S2A_crcm.tif', + 'hash', + 'sha256', + '5cc5d30a22fc2e24f636f1c6dc722160cb0c5eb502ed803e4ff94290ecee1109')], + ), +]) diff --git a/gips/test/sys/expected/std_project.py b/gips/test/sys/expected/std_project.py index b689e763..3ac6ec9d 100644 --- a/gips/test/sys/expected/std_project.py +++ b/gips/test/sys/expected/std_project.py @@ -1,3 +1,8 @@ +import collections + +from .. import util + +mark_spec = {} expectations = {} expectations['modis'] = { @@ -458,3 +463,62 @@ 'sha256', '46e0db478d2c9f9cb6ecd3d321c2ac9b69d30c6a5d2eb964ac6195d398fed661')], } + +# projection itself isn't terribly slow; possibly waiting on processing may +# be slow though +mark_spec['sentinel2'] = util.slow + +expectations['sentinel2'] = collections.OrderedDict([ + # TODO acolite products; should use --acolite flag + # acoflags fai oc2chl oc3chl rhow spm655 turbidity + + # t_project[ref-toa] recording, + ('ref-toa', + [('0/2017010_S2A_ref-toa.tif', + 'hash', + 'sha256', + '36effec15edef46d27f534113dc35e9fed7b44fc33706cebcac8f14ba29fdc50')], + ), + # t_project[rad-toa] recording, + ('rad-toa', + [('0/2017010_S2A_rad-toa.tif', + 'hash', + 'sha256', + '47b14e37f7ada44b978c7bbf3d6fcc51da3ccdcbdd05b01969fe1c548fb8503d')], + ), + # t_project[rad] recording, + ('rad', + [('0/2017010_S2A_rad.tif', + 'hash', + 'sha256', + '3245e591f60bc8723264a3ed960ab6c46ecfa55db881c52feda0caafb2338429')], + ), + # t_project[ref] recording, + ('ref', + [('0/2017010_S2A_ref.tif', + 'hash', + 'sha256', + 'aeb96c9c7fb857864aa71ec49d4b90798846b160f7bdfe16bc507bd2953d7bc9')], + ), + # t_project[cfmask] recording, + ('cfmask', + [('0/2017010_S2A_cfmask.tif', + 'hash', + 'sha256', + 'dfd06798ff067a6139a6bf6f38234920d643693b95e5c7f319cb1de9ff161292')], + ), + # t_project[evi-toa] recording, + ('evi-toa', # nir, red, blue, TOA version + [('0/2017010_S2A_evi-toa.tif', + 'hash', + 'sha256', + '3d4caaf7f9550c95f4e85ad87ed888f8d5bdd0e6be4d4da5a3d2868cf4f1b589')], + ), + # t_project[crcm] recording, + ('crcm', # swir1, swir2, green, surface version + [('0/2017010_S2A_crcm.tif', + 'hash', + 'sha256', + '84fc07d6d0f6df0ec3e65300d51e09cf11363cb0ab1cffe26e034ca154e3aa41')], + ), +]) From a486c575fdc6d6c77a9407a2b7a4331694f085f3 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Tue, 9 Jan 2018 12:49:13 -0500 Subject: [PATCH 33/44] 483 standardize three landsat system tests --- gips/test/sys/driver_setup.py | 1 + gips/test/sys/expected/landsat.py | 40 -------------------------- gips/test/sys/expected/std_process.py | 39 +++++++++++++++++++++++++ gips/test/sys/expected/std_project.py | 38 +++++++++++++++++++++++++ gips/test/sys/expected/std_stats.py | 41 +++++++++++++++++++++++++++ gips/test/sys/t_landsat.py | 29 ------------------- gips/test/sys/t_project.py | 4 +-- gips/test/sys/t_stats.py | 4 +-- 8 files changed, 123 insertions(+), 73 deletions(-) diff --git a/gips/test/sys/driver_setup.py b/gips/test/sys/driver_setup.py index 75bbb1e9..ffb5bdc4 100644 --- a/gips/test/sys/driver_setup.py +++ b/gips/test/sys/driver_setup.py @@ -13,6 +13,7 @@ 'modis': ('modis', '-s', nh_shp, '-d', '2012-12-01,2012-12-03', '-v', '4'), 'merra': ('merra', '-s', nh_shp, '-d', '2015-135', '-v', '4'), 'prism': ('prism', '-s', nh_shp, '-d', '1982-12-01,1982-12-03', '-v4'), + 'landsat': ('landsat', '-s', nh_shp, '-d', '2017-08-01', '-v4'), # Here down, not NH shapefile: 'aod': ('aod', '-s', util.NE_SHP_PATH, '-d', '2017-004,2017-006', '-v4'), 'sentinel2': ('sentinel2', '-s', util.DURHAM_SHP_PATH, '-d2017-010', '-v4') diff --git a/gips/test/sys/expected/landsat.py b/gips/test/sys/expected/landsat.py index 2535e811..9fed3f4d 100644 --- a/gips/test/sys/expected/landsat.py +++ b/gips/test/sys/expected/landsat.py @@ -1,25 +1,5 @@ """Known-good outcomes for tests, mostly stdout and created files.""" -t_process = { - 'compare_stderr': False, - 'updated': { - 'landsat/stage': None, - 'landsat/tiles/012030/2015352': None - }, - 'created': { - 'landsat/tiles/012030/2015352/012030_2015352_LC8_acca.tif': -531492048, - 'landsat/tiles/012030/2015352/012030_2015352_LC8_bqashadow.tif': -1819149482, - 'landsat/tiles/012030/2015352/012030_2015352_LC8_ndvi-toa.tif': 329107382, - 'landsat/tiles/012030/2015352/012030_2015352_LC8_rad-toa.tif': -1222249885, - 'landsat/tiles/012030/2015352/012030_2015352_LC8_ref-toa.tif': -871936054, - 'landsat/tiles/012030/2015352/LC08_L1GT_012030_20151218_20170224_01_T2.tar.gz.index': -394988487, - 'landsat/tiles/012030/2015352/LC08_L1GT_012030_20151218_20170224_01_T2_MTL.txt': -1453474890, - }, - 'ignored': [ - 'gips-inv-db.sqlite3', - ] -} - t_process_acolite = { 'created': { 'landsat/tiles/012030/2017213/012030_2017213_LC8_acoflags.tif': -514981863, @@ -39,18 +19,6 @@ ], } -t_project = { - 'compare_stderr': False, - 'created': { - '0': None, - '0/2015352_LC8_acca.tif': 402348046, - '0/2015352_LC8_bqashadow.tif': 923940030, - '0/2015352_LC8_ndvi-toa.tif': 728893178, - '0/2015352_LC8_rad-toa.tif': -1053542955, - '0/2015352_LC8_ref-toa.tif': -1149010214, - } -} - t_project_no_warp = { 'compare_stderr': False, 'created': { @@ -78,11 +46,3 @@ '012030/012030_2015352_LC8_ref-toa.tif': -1797834447, } } - -t_stats = { 'created': { - 'acca_stats.txt': -174967201, - 'bqashadow_stats.txt': 1868908586, - 'ndvi-toa_stats.txt': -1084861813, - 'rad-toa_stats.txt': -545320378, - 'ref-toa_stats.txt': -1132928652, -}} diff --git a/gips/test/sys/expected/std_process.py b/gips/test/sys/expected/std_process.py index 0e616416..05dae09b 100644 --- a/gips/test/sys/expected/std_process.py +++ b/gips/test/sys/expected/std_process.py @@ -512,3 +512,42 @@ '5cc5d30a22fc2e24f636f1c6dc722160cb0c5eb502ed803e4ff94290ecee1109')], ), ]) + +mark_spec['landsat'] = util.slow + +expectations['landsat'] = { + # t_process[bqashadow] recording: + 'bqashadow': + [('landsat/tiles/012030/2017213/012030_2017213_LC8_bqashadow.tif', + 'hash', + 'sha256', + '1c8c8e252afe94deb0684c56c71149d3585b997abae72a1671217aa4b74faacb')], + + # t_process[ndvi-toa] recording: + 'ndvi-toa': + [('landsat/tiles/012030/2017213/012030_2017213_LC8_ndvi-toa.tif', + 'hash', + 'sha256', + '5d2273f5eafa9fcfff1a068fba21c686b574345d1c090397a05942bb5017a715')], + + # t_process[acca] recording: + 'acca': + [('landsat/tiles/012030/2017213/012030_2017213_LC8_acca.tif', + 'hash', + 'sha256', + 'd4412cd13adad9cde0c7668fb44f728fc7a2e5bda6ddaa062fc65d5de9546765')], + + # t_process[rad-toa] recording: + 'rad-toa': + [('landsat/tiles/012030/2017213/012030_2017213_LC8_rad-toa.tif', + 'hash', + 'sha256', + '2e4923033454e9021b31def954b0e8b3c4247ed0856414995e148f672207ca46')], + + # t_process[ref-toa] recording: + 'ref-toa': + [('landsat/tiles/012030/2017213/012030_2017213_LC8_ref-toa.tif', + 'hash', + 'sha256', + 'a10ba27cd930bb6bd32c4649cd2ce7d7274e82fef043cf4a2a79c212cc4754ea')], +} diff --git a/gips/test/sys/expected/std_project.py b/gips/test/sys/expected/std_project.py index 3ac6ec9d..fd9e7632 100644 --- a/gips/test/sys/expected/std_project.py +++ b/gips/test/sys/expected/std_project.py @@ -522,3 +522,41 @@ '84fc07d6d0f6df0ec3e65300d51e09cf11363cb0ab1cffe26e034ca154e3aa41')], ), ]) + +mark_spec['landsat'] = util.slow +expectations['landsat'] = { + # t_project[bqashadow] recording: + 'bqashadow': + [('0/2017213_LC8_bqashadow.tif', + 'hash', + 'sha256', + '0f4a69505d0d37a32a086848273704ce0d0d6bf5c84d0d72758e107fda55bf36')], + + # t_project[ndvi-toa] recording: + 'ndvi-toa': + [('0/2017213_LC8_ndvi-toa.tif', + 'hash', + 'sha256', + '21f2379aaf304fb26dc942c62f05a98bc14756ee89738d28bd8e373bb3cc10ac')], + + # t_project[acca] recording: + 'acca': + [('0/2017213_LC8_acca.tif', + 'hash', + 'sha256', + '056f1d96e76ebc147b341046e6021a2f2359361c8e88fafdbe255199242b9e4e')], + + # t_project[rad-toa] recording: + 'rad-toa': + [('0/2017213_LC8_rad-toa.tif', + 'hash', + 'sha256', + 'b7ac9b05da84806de1b32de0cea35aca1dd6723fa6d5b687a3f607db95395aeb')], + + # t_project[ref-toa] recording: + 'ref-toa': + [('0/2017213_LC8_ref-toa.tif', + 'hash', + 'sha256', + '22f17fce8c02354a7180fd2739d1e9c6a5c0cd83c9991b6a1017bb2decc9736c')], +} diff --git a/gips/test/sys/expected/std_stats.py b/gips/test/sys/expected/std_stats.py index 3d55d7ee..6794dbff 100644 --- a/gips/test/sys/expected/std_stats.py +++ b/gips/test/sys/expected/std_stats.py @@ -1,4 +1,7 @@ +from .. import util + expectations = {} +mark_spec = {} expectations['modis'] = { # t_stats[satvi] recording: @@ -314,3 +317,41 @@ '1982-336 0.0 0.0 0.0 0.0 nan 221810.0 \n', '1982-337 0.0 0.29 0.00668477 0.0376131 6.09544 221810.0 \n'])], } + +mark_spec['landsat'] = util.slow +expectations['landsat'] = { + # t_stats[bqashadow] recording: + 'bqashadow': + [('bqashadow_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2017-213 1.0 1.0 1.0 0.0 nan 220848.0 \n'])], + + # t_stats[ndvi-toa] recording: + 'ndvi-toa': + [('ndvi-toa_stats.txt', + 'text-full', + ['date min max mean sd skew count\n', + '2017-213 -0.4883 0.8611 0.640444 0.223033 -2.00842 220848.0 \n'])], + + # t_stats[acca] recording: + 'acca': + [('acca_stats.txt', + 'text-full', + ['date finalmask-min finalmask-max finalmask-mean finalmask-sd finalmask-skew finalmask-count cloudmask-min cloudmask-max cloudmask-mean cloudmask-sd cloudmask-skew cloudmask-count ambclouds-min ambclouds-max ambclouds-mean ambclouds-sd ambclouds-skew ambclouds-count pass1-min pass1-max pass1-mean pass1-sd pass1-skew pass1-count \n', + '2017-213 1.0 1.0 1.0 0.0 nan 199899.0 1.0 1.0 1.0 0.0 nan 20949.0 1.0 1.0 1.0 0.0 nan 15396.0 1.0 1.0 1.0 0.0 nan 4642.0 \n'])], + + # t_stats[rad-toa] recording: + 'rad-toa': + [('rad-toa_stats.txt', + 'text-full', + ['date COASTAL-min COASTAL-max COASTAL-mean COASTAL-sd COASTAL-skew COASTAL-count BLUE-min BLUE-max BLUE-mean BLUE-sd BLUE-skew BLUE-count GREEN-min GREEN-max GREEN-mean GREEN-sd GREEN-skew GREEN-count RED-min RED-max RED-mean RED-sd RED-skew RED-count NIR-min NIR-max NIR-mean NIR-sd NIR-skew NIR-count SWIR1-min SWIR1-max SWIR1-mean SWIR1-sd SWIR1-skew SWIR1-count SWIR2-min SWIR2-max SWIR2-mean SWIR2-sd SWIR2-skew SWIR2-count CIRRUS-min CIRRUS-max CIRRUS-mean CIRRUS-sd CIRRUS-skew CIRRUS-count \n', + '2017-213 56.1 409.4 72.4368 28.7988 4.58855 220848.0 43.6 445.1 61.8659 32.3072 4.54671 220848.0 25.6 408.0 48.7223 31.1024 4.42216 220848.0 12.7 378.6 30.9724 30.2874 4.26369 220848.0 4.7 339.8 89.4949 28.9755 -0.796908 220848.0 0.0 54.9 10.4936 5.32891 1.61942 220848.0 -0.1 14.2 1.62182 1.50423 3.07054 220848.0 -0.1 1.4 0.0702492 0.0691392 3.70532 220848.0 \n'])], + + # t_stats[ref-toa] recording: + 'ref-toa': + [('ref-toa_stats.txt', + 'text-full', + ['date COASTAL-min COASTAL-max COASTAL-mean COASTAL-sd COASTAL-skew COASTAL-count BLUE-min BLUE-max BLUE-mean BLUE-sd BLUE-skew BLUE-count GREEN-min GREEN-max GREEN-mean GREEN-sd GREEN-skew GREEN-count RED-min RED-max RED-mean RED-sd RED-skew RED-count NIR-min NIR-max NIR-mean NIR-sd NIR-skew NIR-count SWIR1-min SWIR1-max SWIR1-mean SWIR1-sd SWIR1-skew SWIR1-count SWIR2-min SWIR2-max SWIR2-mean SWIR2-sd SWIR2-skew SWIR2-count CIRRUS-min CIRRUS-max CIRRUS-mean CIRRUS-sd CIRRUS-skew CIRRUS-count \n', + '2017-213 0.0802 0.5848 0.103512 0.0411366 4.58853 220848.0 0.0809 0.8259 0.114876 0.0599464 4.5467 220848.0 0.053 0.8445 0.100936 0.0643648 4.42221 220848.0 0.0231 0.6875 0.0563213 0.0549959 4.26371 220848.0 0.0142 1.0063 0.26515 0.0857833 -0.796907 220848.0 0.001 0.8389 0.161572 0.0813276 1.61928 220848.0 0.0003 0.596 0.071522 0.0625481 3.07293 220848.0 0.0002 0.0152 0.00160228 0.000582327 7.07733 220848.0 \n'])], +} diff --git a/gips/test/sys/t_landsat.py b/gips/test/sys/t_landsat.py index f783f4c5..878b10f4 100644 --- a/gips/test/sys/t_landsat.py +++ b/gips/test/sys/t_landsat.py @@ -63,13 +63,6 @@ def t_query_service(): logger.debug(str(resp)) assert len(resp) == 1 -@slow -def t_process(setup_landsat_data, repo_env, expected): - """Test gips_process on landsat data.""" - actual = repo_env.run('gips_process', *STD_PROD_ARGS) - assert expected == actual - - @slow @acolite def t_process_acolite(repo_env, expected): @@ -88,13 +81,6 @@ def t_process_acolite(repo_env, expected): assert expected == actual -def t_project(setup_landsat_data, clean_repo_env, output_tfe, expected): - """Test gips_project landsat with warping.""" - args = STD_PROD_ARGS + ('--res', '30', '30', '--outdir', OUTPUT_DIR, '--notld') - actual = output_tfe.run('gips_project', *args) - assert expected == actual - - def t_project_no_warp(setup_landsat_data, clean_repo_env, output_tfe, expected): """Test gips_project modis without warping.""" args = STD_PROD_ARGS + ('--outdir', OUTPUT_DIR, '--notld') @@ -117,18 +103,3 @@ def t_tiles_copy(setup_landsat_data, clean_repo_env, output_tfe, expected): '--outdir', OUTPUT_DIR, '--notld') + product_args actual = output_tfe.run('gips_tiles', *args) assert expected == actual - - -def t_stats(setup_landsat_data, clean_repo_env, output_tfe, expected): - """Test gips_stats on projected files.""" - # generate data needed for stats computation - args = STD_PROD_ARGS + ('--res', '30', '30', '--outdir', OUTPUT_DIR, '--notld') - prep_run = output_tfe.run('gips_project', *args) - assert prep_run.exit_status == 0 # confirm it worked; not really in the test - - # compute stats - gtfe = GipsTestFileEnv(OUTPUT_DIR, start_clear=False) - actual = gtfe.run('gips_stats', OUTPUT_DIR) - - # check for correct stats content - assert expected == actual diff --git a/gips/test/sys/t_project.py b/gips/test/sys/t_project.py index 71acd395..e20c7821 100644 --- a/gips/test/sys/t_project.py +++ b/gips/test/sys/t_project.py @@ -7,10 +7,10 @@ pytestmark = util.sys # skip everything unless --sys # 'driver': {'product': [ (path, type, data...),...]...} -from expected.std_project import expectations +from expected.std_project import expectations, mark_spec @pytest.mark.parametrize("driver, product", - util.params_from_expectations(expectations)) + util.params_from_expectations(expectations, mark_spec)) def t_project(export_wrapper, driver, product): """Test gips_project with warping.""" record_mode, runner, working_dir = export_wrapper diff --git a/gips/test/sys/t_stats.py b/gips/test/sys/t_stats.py index daa52294..72009cfc 100644 --- a/gips/test/sys/t_stats.py +++ b/gips/test/sys/t_stats.py @@ -10,10 +10,10 @@ pytestmark = util.sys # skip everything unless --sys # 'driver': {'product': [ (path, type, data...),...]...} -from expected.std_stats import expectations +from expected.std_stats import expectations, mark_spec @pytest.mark.parametrize("driver, product", - util.params_from_expectations(expectations)) + util.params_from_expectations(expectations, mark_spec)) def t_stats(export_wrapper, driver, product): """Test gips_stats on projected files.""" record_mode, runner, working_dir = export_wrapper From f6ad4869a77f47aaf54618853e95534e2b84fa91 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Tue, 9 Jan 2018 14:17:28 -0500 Subject: [PATCH 34/44] 483 sar process test pseudocode conversion --- gips/test/sys/expected/sar.py | 132 -------------------------- gips/test/sys/expected/std_process.py | 18 ++++ gips/test/sys/t_sar.py | 90 ------------------ 3 files changed, 18 insertions(+), 222 deletions(-) delete mode 100644 gips/test/sys/expected/sar.py delete mode 100644 gips/test/sys/t_sar.py diff --git a/gips/test/sys/expected/sar.py b/gips/test/sys/expected/sar.py deleted file mode 100644 index ee3b7325..00000000 --- a/gips/test/sys/expected/sar.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Known-good outcomes for tests, mostly stdout and created files.""" - -otro = u"""\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site tiles - -\x1b[1mAsset Holdings for tile N07E099\x1b[0m -\x1b[1m\x1b[4m DATE alos1 alos2 Product \x1b[0m -\x1b[1m2015 -\x1b[0m 101 100.0% - - -1 files on 1 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mAFBD: PALSAR FineBeam Dual Polarization\x1b[0m -\x1b[31mAFBS: PALSAR FineBeam Single Polarization\x1b[0m -\x1b[32mAWB1: PALSAR WideBeam (ScanSAR Short Mode)\x1b[0m -\x1b[34mAWBD: PALSAR-2 WideBeam (ScanSAR w/Ortho Slope Correction)\x1b[0m -\x1b[33mJFBS: JERS-1 FineBeam Single Polarization\x1b[0m -""" - - -t_process = { - 'updated': { - 'sar/stage': None, - 'sar/tiles/N00E099/2009041': None, - 'sar/tiles/N07E099/2015101': None, - 'sar/tiles/N19E100/2010182': None, - }, - 'created': { - 'sar/tiles/N00E099/2009041/KC_017-C25N00E099WB1ORSA1.tar.gz.index': -741988802, - 'sar/tiles/N00E099/2009041/N00E099_2009041_AWB1_date.tif': 307502521, - 'sar/tiles/N00E099/2009041/N00E099_2009041_AWB1_linci.tif': -399377879, - 'sar/tiles/N00E099/2009041/N00E099_2009041_AWB1_sign.tif': 1287316543, - 'sar/tiles/N00E099/2009041/N00E099_K25_date': 717902658, - 'sar/tiles/N00E099/2009041/N00E099_K25_date.hdr': 886355476, - 'sar/tiles/N07E099/2015101/KC_999-C019DRN07E099WBDORSA1.tar.gz.index': 541661026, - 'sar/tiles/N07E099/2015101/N07E099_2015101_AWBD_date.tif': 1151011840, - 'sar/tiles/N07E099/2015101/N07E099_2015101_AWBD_linci.tif': -2017292803, - 'sar/tiles/N07E099/2015101/N07E099_2015101_AWBD_sign.tif': 515521946, - 'sar/tiles/N07E099/2015101/N07E099_W02DC019DR_date': 1689631779, - 'sar/tiles/N07E099/2015101/N07E099_W02DC019DR_date.hdr': 1634871345, - 'sar/tiles/N19E100/2010182/KC_017-Y10N19E100FBDORSA1.tar.gz.index': 15900329, - 'sar/tiles/N19E100/2010182/N19E100_10_date': 330702546, - 'sar/tiles/N19E100/2010182/N19E100_10_date.hdr': -1922938697, - 'sar/tiles/N19E100/2010182/N19E100_2010182_AFBD_date.tif': 238211992, - 'sar/tiles/N19E100/2010182/N19E100_2010182_AFBD_linci.tif': 1373550692, - 'sar/tiles/N19E100/2010182/N19E100_2010182_AFBD_sign.tif': 832541493 - }, - '_symlinks': { - }, - 'ignored': [ - 'gips-inv-db.sqlite3', - ], - '_inv_stdout': """\x1b[1mGIPS Data Inventory (v0.8.2)\x1b[0m -Retrieving inventory for site tiles - -\x1b[1mAsset Holdings for tile N07E099\x1b[0m -\x1b[1m\x1b[4m DATE alos1 alos2 Product \x1b[0m -\x1b[1m2015 -\x1b[0m 101 100.0% \x1b[34mdate\x1b[0m \x1b[34mlinci\x1b[0m \x1b[34msign\x1b[0m - - -1 files on 1 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mAFBD: PALSAR FineBeam Dual Polarization\x1b[0m -\x1b[31mAFBS: PALSAR FineBeam Single Polarization\x1b[0m -\x1b[32mAWB1: PALSAR WideBeam (ScanSAR Short Mode)\x1b[0m -\x1b[34mAWBD: PALSAR-2 WideBeam (ScanSAR w/Ortho Slope Correction)\x1b[0m -\x1b[33mJFBS: JERS-1 FineBeam Single Polarization\x1b[0m -Retrieving inventory for site tiles - -\x1b[1mAsset Holdings for tile N19E100\x1b[0m -\x1b[1m\x1b[4m DATE alos1 alos2 Product \x1b[0m -\x1b[1m2010 -\x1b[0m 182 100.0% \x1b[35mdate\x1b[0m \x1b[35mlinci\x1b[0m \x1b[35msign\x1b[0m - - -1 files on 1 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mAFBD: PALSAR FineBeam Dual Polarization\x1b[0m -\x1b[31mAFBS: PALSAR FineBeam Single Polarization\x1b[0m -\x1b[32mAWB1: PALSAR WideBeam (ScanSAR Short Mode)\x1b[0m -\x1b[34mAWBD: PALSAR-2 WideBeam (ScanSAR w/Ortho Slope Correction)\x1b[0m -\x1b[33mJFBS: JERS-1 FineBeam Single Polarization\x1b[0m -Retrieving inventory for site tiles - -\x1b[1mAsset Holdings for tile N00E099\x1b[0m -\x1b[1m\x1b[4m DATE alos1 alos2 Product \x1b[0m -\x1b[1m2009 -\x1b[0m 041 100.0% \x1b[32mdate\x1b[0m \x1b[32mlinci\x1b[0m \x1b[32msign\x1b[0m - - -1 files on 1 dates -\x1b[1m -SENSORS\x1b[0m -\x1b[35mAFBD: PALSAR FineBeam Dual Polarization\x1b[0m -\x1b[31mAFBS: PALSAR FineBeam Single Polarization\x1b[0m -\x1b[32mAWB1: PALSAR WideBeam (ScanSAR Short Mode)\x1b[0m -\x1b[34mAWBD: PALSAR-2 WideBeam (ScanSAR w/Ortho Slope Correction)\x1b[0m -\x1b[33mJFBS: JERS-1 FineBeam Single Polarization\x1b[0m -""", -} - -t_project = { - 'created': { - } -} - -t_project_no_warp = { - 'created': {'0': None, - } -} - - -t_tiles_copy = { - 'created': { - } -} - -t_tiles_copy = { - 'created': { - } -} - -t_stats = { - 'created': { - } -} - diff --git a/gips/test/sys/expected/std_process.py b/gips/test/sys/expected/std_process.py index 05dae09b..95bb3702 100644 --- a/gips/test/sys/expected/std_process.py +++ b/gips/test/sys/expected/std_process.py @@ -551,3 +551,21 @@ 'sha256', 'a10ba27cd930bb6bd32c4649cd2ce7d7274e82fef043cf4a2a79c212cc4754ea')], } + +# TODO restore sar process test: +#expectations['sar'] = { +# 'sign': [], +# 'mask': [], # not part of the original expectation, hmm +# 'linci': [], +# 'date': [], +#} +# sar fetch currently doesn't work: +# gips_inventory sar -t N07E099 N19E100 N00E099 -d 2009,2015 -v 4 --fetch # fails +# ftp download from volga is busted too: +#ftp://anonymous:tolson@volga.ags.io/files/sar/KC_999-C019DRN07E099WBDORSA1.tar.gz +#ftp://anonymous:tolson@volga.ags.io/files/sar/KC_017-C25N00E099WB1ORSA1.tar.gz +#ftp://anonymous:tolson@volga.ags.io/files/sar/KC_017-Y10N19E100FBDORSA1.tar.gz +# original ftp code: +# 0) cd stage_dir_for_sar_driver +# 1) wget ftp_url +# 2) gips_archive sar diff --git a/gips/test/sys/t_sar.py b/gips/test/sys/t_sar.py deleted file mode 100644 index d8cd8734..00000000 --- a/gips/test/sys/t_sar.py +++ /dev/null @@ -1,90 +0,0 @@ -import logging - -import pytest -import envoy - -from .util import * - -import os - -logger = logging.getLogger(__name__) - -pytestmark = sys # skip everything unless --sys - -# changing this will require changes in expected/ -driver = 'sar' -STD_TILE = 'N07E099 N19E100 N00E099' -STD_DATES = '2009,2015' -TILED_ARGS = (driver, '-t', 'N07E099', 'N19E100', 'N00E099', '-d', STD_DATES, '-v', '4') -ASSET_URL_PREFIX = ( - 'ftp://anonymous:{USER}@volga.ags.io/files/{DRIVER}/' - .format( - USER=os.environ.get('USER', 'nobody'), - DRIVER=driver - ) -) -STORED_ASSETS = [ - ASSET_URL_PREFIX + f - for f in - [ - 'KC_999-C019DRN07E099WBDORSA1.tar.gz', - 'KC_017-C25N00E099WB1ORSA1.tar.gz', - 'KC_017-Y10N19E100FBDORSA1.tar.gz', - ] -] - - -@pytest.fixture -def setup_sar_data(pytestconfig): - """ - Use STORED_ASSETS+gips_archive or gips_inventory to ensure presence - of asset data in the data repo. - """ - if not pytestconfig.getoption('setup_repo'): - logger.debug("Skipping repo setup per lack of option.") - return - if 'STORED_ASSETS' in globals(): - cwd = os.getcwd() - stage = os.path.join(DATA_REPO_ROOT, driver, 'stage') - os.chdir(stage) - for asset in STORED_ASSETS: - cmd_str = 'wget {URL}'.format(URL=asset) - logger.info("Downloading SAR assets with " + cmd_str) - outcome = envoy.run(cmd_str) - if outcome.status_code != 0: - raise RuntimeError( - "fetchinv via STORED_ASSETS failed", - outcome.std_out, outcome.std_err, outcome - ) - cmd_str = ('gips_archive {DRIVER}'.format(DRIVER=driver)) - logger.info("Archiving SAR assets with " + cmd_str) - outcome = envoy.run(cmd_str) - logger.info("SAR data archival complete.") - if outcome.status_code != 0: - raise RuntimeError( - "sar data setup via `gips_archive` failed", - outcome.std_out, outcome.std_err, outcome - ) - else: - cmd_str = 'gips_inventory ' + ' '.join(TILED_ARGS) + ' --fetch' - logger.info("Downloading SAR assets with " + cmd_str) - outcome = envoy.run(cmd_str) - logger.info("SAR data download complete.") - if outcome.status_code != 0: - raise RuntimeError( - "sar data setup via `gips_inventory` failed", - outcome.std_out, outcome.std_err, outcome - ) - - -setup_fixture = setup_sar_data - -# ### SHOULD BE STANDARD BELOW HERE ##### - - -def t_process(setup_fixture, repo_env, expected): - """Test gips_process on {} data.""".format(driver) - process_actual = repo_env.run('gips_process', *TILED_ARGS) - inventory_actual = envoy.run('gips_inventory ' + ' '.join(TILED_ARGS)) - assert expected == process_actual - assert inventory_actual.std_out == expected._inv_stdout From e5592c3e8e55bb0a32f15175d2e653fdd8441b6c Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 10 Jan 2018 14:36:03 -0500 Subject: [PATCH 35/44] 483 lay groundwork for artifact server and: sar conversion. Also slight UX rejiggery. --- conftest.py | 12 +++++++- gips/test/sys/driver_setup.py | 56 ++++++++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/conftest.py b/conftest.py index de70ec21..cbdddf26 100644 --- a/conftest.py +++ b/conftest.py @@ -48,10 +48,18 @@ def pytest_addoption(parser): help_str = ("The directory housing the data repo for testing purposes. " "MUST match GIPS' configured REPOS setting.") parser.addini('data-repo', help=help_str) - parser.addini('output-dir', help="The directory housing output files from test runs.") + parser.addini('artifact-store-user', help="FTP artifact store username") + parser.addini('artifact-store-password', + help="FTP artifact store password") + parser.addini('artifact-store-host', help="FTP artifact store hostname") + parser.addini('artifact-store-path', + help="FTP artifact store root path; files are presumed to be" + " stored driver-wise beneath this level," + " eg path/sar/asset.tgz") + parser.addoption( "--slow", action="store_true", help="Do not skip @slow tests.") @@ -112,6 +120,8 @@ def pytest_configure(config): elif config.getoption("setup_repo"): _log.debug("--setup-repo detected; setting up data repo") setup_data_repo() + else: + print "Skipping repo setup per lack of --setup-repo." def setup_data_repo(): diff --git a/gips/test/sys/driver_setup.py b/gips/test/sys/driver_setup.py index ffb5bdc4..78ff83b9 100644 --- a/gips/test/sys/driver_setup.py +++ b/gips/test/sys/driver_setup.py @@ -1,6 +1,10 @@ from __future__ import print_function import datetime +import ftplib +import tempfile +import shutil +import os import pytest import sh @@ -19,20 +23,58 @@ 'sentinel2': ('sentinel2', '-s', util.DURHAM_SHP_PATH, '-d2017-010', '-v4') } +class CannotFetch(Exception): + pass + + +sar_asset_fns = [ + 'KC_017-C25N00E099WB1ORSA1.tar.gz', + 'KC_017-Y10N19E100FBDORSA1.tar.gz', + 'KC_999-C019DRN07E099WBDORSA1.tar.gz', +] + +def special_cases(driver): + # handle modis scheduled downtime case + if driver == 'modis' and datetime.datetime.today().date().weekday() == 2: + raise RuntimeError("It's Wednesday; modis downloads are likely to fail.") + # sar isn't fetchable; get it from the artifact server + # TODO emplace sar assets in the existing collection of system test artifacts + # and write code that works for all of these artifacts + if driver == 'sar': + username = pytest.config.getini('artifact-store-user') + password = pytest.config.getini('artifact-store-password') + host = pytest.config.getini('artifact-store-host') + path = pytest.config.getini('artifact-store-path') + url_head = 'ftp://{}:{}@{}/{}/{}/'.format( + username, password, host, path, driver) + bookmark = os.getcwd() + try: + temp_dir = tempfile.mkdtemp() + os.chdir(temp_dir) + [sh.wget(url_head + fn) for fn in sar_asset_fns] + sh.gips_archive('sar') + finally: + os.chdir(bookmark) + shutil.rmtree(temp_dir) + + # sar fetch currently doesn't work; this fails: + # gips_inventory sar -t N07E099 N19E100 N00E099 -d 2009,2015 -v 4 --fetch + raise CannotFetch("sar assets can't be fetched (reason TBD);" + " used artifact server.") + setup_attempted = [] def setup_repo_data(driver): """Use gips_inventory to ensure assets are present.""" - if driver in setup_attempted: - print(driver, "setup already attempted; skipping.") + if driver in setup_attempted or not pytest.config.getoption('setup_repo'): return setup_attempted.append(driver) - if not pytest.config.getoption('setup_repo'): - print("Skipping", driver, "repo setup per lack of option.") + try: + special_cases(driver) + except CannotFetch as cf: + print(cf) return - # handle modis scheduled downtime case - if driver == 'modis' and datetime.datetime.today().date().weekday() == 2: - raise RuntimeError("It's Wednesday; modis downloads are likely to fail.") + print('Downloading', driver, 'data . . .') args = STD_ARGS[driver] + ('--fetch',) outcome = sh.Command('gips_inventory')(*args) From ce02df66f51935802697849df41cc137dde1ad0c Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 10 Jan 2018 14:48:23 -0500 Subject: [PATCH 36/44] 483 convert sar t_process to new sys test code --- gips/test/sys/driver_setup.py | 3 +- gips/test/sys/expected/std_process.py | 78 +++++++++++++++++++++------ 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/gips/test/sys/driver_setup.py b/gips/test/sys/driver_setup.py index 78ff83b9..5e4ac257 100644 --- a/gips/test/sys/driver_setup.py +++ b/gips/test/sys/driver_setup.py @@ -20,7 +20,8 @@ 'landsat': ('landsat', '-s', nh_shp, '-d', '2017-08-01', '-v4'), # Here down, not NH shapefile: 'aod': ('aod', '-s', util.NE_SHP_PATH, '-d', '2017-004,2017-006', '-v4'), - 'sentinel2': ('sentinel2', '-s', util.DURHAM_SHP_PATH, '-d2017-010', '-v4') + 'sentinel2': ('sentinel2', '-s', util.DURHAM_SHP_PATH, '-d2017-010', '-v4'), + 'sar': ('sar', '-t', 'N07E099', 'N19E100', 'N00E099', '-d2009,2015', '-v4'), } class CannotFetch(Exception): diff --git a/gips/test/sys/expected/std_process.py b/gips/test/sys/expected/std_process.py index 95bb3702..855b4028 100644 --- a/gips/test/sys/expected/std_process.py +++ b/gips/test/sys/expected/std_process.py @@ -552,20 +552,64 @@ 'a10ba27cd930bb6bd32c4649cd2ce7d7274e82fef043cf4a2a79c212cc4754ea')], } -# TODO restore sar process test: -#expectations['sar'] = { -# 'sign': [], -# 'mask': [], # not part of the original expectation, hmm -# 'linci': [], -# 'date': [], -#} -# sar fetch currently doesn't work: -# gips_inventory sar -t N07E099 N19E100 N00E099 -d 2009,2015 -v 4 --fetch # fails -# ftp download from volga is busted too: -#ftp://anonymous:tolson@volga.ags.io/files/sar/KC_999-C019DRN07E099WBDORSA1.tar.gz -#ftp://anonymous:tolson@volga.ags.io/files/sar/KC_017-C25N00E099WB1ORSA1.tar.gz -#ftp://anonymous:tolson@volga.ags.io/files/sar/KC_017-Y10N19E100FBDORSA1.tar.gz -# original ftp code: -# 0) cd stage_dir_for_sar_driver -# 1) wget ftp_url -# 2) gips_archive sar +expectations['sar'] = { + # t_process[date] recording: + 'date': + [('sar/tiles/N07E099/2015101/N07E099_2015101_AWBD_date.tif', + 'hash', + 'sha256', + 'c29663846c6ec54f570e925e0e27bbf6fcf92824ff68c11531f266ffc073331a'), + ('sar/tiles/N19E100/2010182/N19E100_2010182_AFBD_date.tif', + 'hash', + 'sha256', + '8b6a0d270e53ca91c6cbaeaa5c9a5290ad11970e36a5b49e9b75f5e5dd272adc'), + ('sar/tiles/N00E099/2009041/N00E099_2009041_AWB1_date.tif', + 'hash', + 'sha256', + '13bdb0c9bb93a31491c8843d0c7b24ad75dfb8a33bc2d37460d09592d01a04ce')], + + # t_process[linci] recording: + 'linci': + [('sar/tiles/N07E099/2015101/N07E099_2015101_AWBD_linci.tif', + 'hash', + 'sha256', + 'ac5082457c7b7ff7c5361534a2f691469b18cf636b8c5c136694310ac6693c33'), + ('sar/tiles/N19E100/2010182/N19E100_2010182_AFBD_linci.tif', + 'hash', + 'sha256', + 'd18394c725a5f8c62b18ff8ab7fc7357a3bfb1a263e3c3dbc2eb0aed09a0f0c8'), + ('sar/tiles/N00E099/2009041/N00E099_2009041_AWB1_linci.tif', + 'hash', + 'sha256', + 'b76a9e585762f060b653d012bfeff760c5250076c174348446dc8f31d0fb4fe2')], + + # t_process[mask] recording: + 'mask': + [('sar/tiles/N00E099/2009041/N00E099_2009041_AWB1_mask.tif', + 'hash', + 'sha256', + '72311122b6b88f80dd85e76380e092bd70c33c4f26461316b4829962c1ecc1fc'), + ('sar/tiles/N07E099/2015101/N07E099_2015101_AWBD_mask.tif', + 'hash', + 'sha256', + '5e463c8174e0acb64db0d7067a6414791a795d237221af30944adaa155fec622'), + ('sar/tiles/N19E100/2010182/N19E100_2010182_AFBD_mask.tif', + 'hash', + 'sha256', + 'e3896c79e5b5ed6380bf821a2cddd65014cc35278933cb1c2c15620e27615afc')], + + # t_process[sign] recording: + 'sign': + [('sar/tiles/N07E099/2015101/N07E099_2015101_AWBD_sign.tif', + 'hash', + 'sha256', + 'a47201adaca8f0cbd9ffa64f6953d3b75a502860f48196169449db69b766aef3'), + ('sar/tiles/N00E099/2009041/N00E099_2009041_AWB1_sign.tif', + 'hash', + 'sha256', + 'f871ea0bfe899438ed7d233ed4e7f72db1d989d1af9d6c55362fac87dce74b31'), + ('sar/tiles/N19E100/2010182/N19E100_2010182_AFBD_sign.tif', + 'hash', + 'sha256', + '49e9ded5ee3a169eddaa5e50aa34b6a0c91dc1f37b345be6f2cf2aea25dfd6ce')], +} From 19eb43ba8144665dba4f58cdfcbc6ba985e96c69 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 10 Jan 2018 09:51:14 -0500 Subject: [PATCH 37/44] 463 tif comparator for expectations: It's now sometimes ambiguous which expectation type should be used, so defer to any existing expectation. For newly-recorded expectations, choose sensibly. Had to work around . . . difficulties . . . with gdalinfo -stats. It took me multiple tries to remove all the profanity from this commit. --- gips/test/sys/expected/std_process.py | 145 +++++++++++++++++++++++--- gips/test/sys/util.py | 54 ++++++++-- 2 files changed, 175 insertions(+), 24 deletions(-) diff --git a/gips/test/sys/expected/std_process.py b/gips/test/sys/expected/std_process.py index 855b4028..72d337e6 100644 --- a/gips/test/sys/expected/std_process.py +++ b/gips/test/sys/expected/std_process.py @@ -40,20 +40,6 @@ '/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Day_1km')], # normal products - # t_process[satvi] recording: - 'satvi': - [('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_satvi.tif', - 'hash', - 'sha256', - '94bee7cecf0f0a227fd8532223f86de10e01c3e39d1dd70b7675d91b3ca338a4'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_satvi.tif', - 'hash', - 'sha256', - '6397669a6cd62bb39813b9000d93ba50f4cabe73aebf1e6444f4e3be9f630e19'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_satvi.tif', - 'hash', - 'sha256', - '6124ee615a4f522fdf966a5479d03d87934908e5090a192192f9700020f24760')], # t_process[ndti] recording: 'ndti': @@ -333,6 +319,137 @@ '942efeeee2b020123be6d7334bbb3ef5995c93d2fd0247cb79948836865c8067')], } +expectations['modis']['satvi'] = [ + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_satvi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = satvi', + ' Minimum=-8110.000, Maximum=6544.000, Mean=455.555, StdDev=1827.398', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=6544', + ' STATISTICS_MEAN=455.55465105991', + ' STATISTICS_MINIMUM=-8110', + ' STATISTICS_STDDEV=1827.3983061798']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_satvi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = satvi', + ' Minimum=-8145.000, Maximum=6597.000, Mean=531.460, StdDev=1728.167', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=6597', + ' STATISTICS_MEAN=531.45965604069', + ' STATISTICS_MINIMUM=-8145', + ' STATISTICS_STDDEV=1728.166878798']), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_satvi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = satvi', + ' Minimum=-8073.000, Maximum=6491.000, Mean=480.431, StdDev=1801.390', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=6491', + ' STATISTICS_MEAN=480.43080960083', + ' STATISTICS_MINIMUM=-8073', + ' STATISTICS_STDDEV=1801.3898148172'])] + expectations['merra'] = { # test this too? 'frland': [], # t_process[srad] recording: diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index 259ad1fa..3130bef0 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -226,9 +226,12 @@ def find_files(path): return [os.path.join(subpath, f) for subpath, _, filenames in os.walk(path) for f in filenames] -def generate_expectation(filename, base_path): +def generate_expectation(filename, base_path, e_type=None): """Return an expectation of the file's content, based on its file type. + Pass in e_type to coerce the file into the given format, instead of + letting the function choose for itself. Absent files can't be coerced. + Expectation formats: Symlink: (filename, 'symlink', prefix_string, suffix_string) With symlinks one often can't use the whole link target @@ -237,6 +240,7 @@ def generate_expectation(filename, base_path): Hash: (filename, 'hash', 'sha256', ). Used as a fallback if nothing else matches the file type. Text-full: (filename, 'text-full', list()) + Raster: (filename, 'raster', 'gdalinfo-stats', list() Finally, if the file is absent: (filename, 'absent') """ # may want to try python-magic at some point: https://github.com/ahupp/python-magic @@ -247,8 +251,13 @@ def generate_expectation(filename, base_path): if not os.path.lexists(full_path): return (filename, 'absent') + if e_type is None: + e_type = ('symlink' if os.path.islink(full_path) else + 'text-full' if filename.endswith('.txt') else + 'raster' if filename.endswith('.tif') else 'hash') + # symlinks - if os.path.islink(full_path): + if e_type == 'symlink': # have to rmeove this non-generic bit # vvvvvvvvvvvvvvvvvvvvvvvvvvvvvv # HDF4_EOS:EOS_GRID:"/home/tolson/src/gips/data-root/modis/tiles/h12v04/2012337 @@ -256,26 +265,28 @@ def generate_expectation(filename, base_path): # when recording the path, do '/foo/bar/baz/file.tif' link_target = os.readlink(full_path) prefix, suffix = link_target.split(base_path) - return (filename, 'symlink', prefix, suffix) + return (filename, e_type, prefix, suffix) # text files - if filename.endswith('.txt'): + if e_type == 'text-full': size_threshold = 80 * 25 # about one terminal-screen f_size = os.stat(full_path).st_size if f_size <= size_threshold: # give up if there's too much text with open(full_path) as fo: lines = fo.readlines() - return (filename, 'text-full', lines) + return (filename, e_type, lines) print('{} ({} bytes) exceeds max supported size for text diffs' ' ({} bytes); defaulting to checksum'.format( full_path, f_size, size_threshold)) + e_type = 'hash' # product TIFFs - if filename.endswith('.tif'): - pass # TODO https://github.com/Applied-GeoSolutions/gips/issues/463 + if e_type == 'raster': + return (filename, e_type, 'gdalinfo-stats', + generate_gdalinfo_stats_expectation(full_path)) # use a hash as a fallback - return (filename, 'hash', 'sha256', generate_file_hash(full_path)) + return (filename, e_type, 'sha256', generate_file_hash(full_path)) def generate_file_hash(filename, blocksize=2**20): # stolen from SO: https://stackoverflow.com/questions/1131220/get-md5-hash-of-big-files-in-python @@ -288,6 +299,28 @@ def generate_file_hash(filename, blocksize=2**20): m.update(buf) return m.hexdigest() +def generate_gdalinfo_stats_expectation(filename): + """Use `gdalinfo -stats ` to generate an expectation object.""" + # Sooooo gdal generates these 'sidecar' files, ending in .aux.xml. + # It can be told not to do that, but can't be told to ignore them + # when they already exist. Whether or not they exist changes the + # output of gdalinfo -stats (reason unkonwn). This can't be worked + # around in any way, so we're obliged to delete them beforehand. + sidecar = filename + '.aux.xml' + os.path.exists(sidecar) and os.remove(sidecar) # RACE CONDITION LOL + os.environ['GDAL_PAM_ENABLED'] = 'NO' + # TODO look at -json + outcome = sh.gdalinfo('-stats', filename) + # The output includes unpredictable file paths, so filter them out: + lines = [] + in_files_block = False + for l in outcome.stdout.splitlines(): + # first detect whether we're in a 'Files:' block + in_files_block = (l.startswith('Files: ') + or (in_files_block and l.startswith(' '))) + if not in_files_block: + lines.append(l) + return lines def params_from_expectations(expectations, mark_spec=None): """Generates a standard system test (driver, product) parameter list. @@ -328,6 +361,7 @@ def sys_test_wrapper(request, path): rp = record_path() # does the user want record mode? If so, save it where? driver = request.node.callspec.params['driver'] product = request.node.callspec.params['product'] + # expected is an array of tuples: (filename, type, data, data, data . . .) expected = request.module.expectations[driver][product] expected_filenames = [e[0] for e in expected] @@ -344,8 +378,8 @@ def wrapped_runner(cmd_string, *args): if rp: initial_files.extend(find_files(path)) outcome = sh.Command(cmd_string)(*args) - return outcome, [generate_expectation(fn, path) - for fn in expected_filenames] + return outcome, [generate_expectation(e[0], path, e[1]) + for e in expected] yield bool(rp), wrapped_runner From 39db757c8ac552f6e6509a79bcb8b8a44b04b518 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 10 Jan 2018 17:09:22 -0500 Subject: [PATCH 38/44] 463 convert modis to gdalinfo -stats comparator --- gips/test/sys/expected/modis_process.py | 2903 +++++++++++++++++++++++ gips/test/sys/expected/std_process.py | 446 +--- 2 files changed, 2906 insertions(+), 443 deletions(-) create mode 100644 gips/test/sys/expected/modis_process.py diff --git a/gips/test/sys/expected/modis_process.py b/gips/test/sys/expected/modis_process.py new file mode 100644 index 00000000..83aa1548 --- /dev/null +++ b/gips/test/sys/expected/modis_process.py @@ -0,0 +1,2903 @@ + +expectations = { + + # t_process[satvi] recording: + 'satvi': + [('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_satvi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = satvi', + ' Minimum=-8110.000, Maximum=6544.000, Mean=455.555, StdDev=1827.398', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=6544', + ' STATISTICS_MEAN=455.55465105991', + ' STATISTICS_MINIMUM=-8110', + ' STATISTICS_STDDEV=1827.3983061798']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_satvi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = satvi', + ' Minimum=-8145.000, Maximum=6597.000, Mean=531.460, StdDev=1728.167', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=6597', + ' STATISTICS_MEAN=531.45965604069', + ' STATISTICS_MINIMUM=-8145', + ' STATISTICS_STDDEV=1728.166878798']), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_satvi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = satvi', + ' Minimum=-8073.000, Maximum=6491.000, Mean=480.431, StdDev=1801.390', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=6491', + ' STATISTICS_MEAN=480.43080960083', + ' STATISTICS_MINIMUM=-8073', + ' STATISTICS_STDDEV=1801.3898148172'])], + + # t_process[ndti] recording: + 'ndti': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndti.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = ndti', + ' Minimum=-30000.000, Maximum=32767.000, Mean=3088.770, StdDev=1732.960', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=3088.7699579987', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=1732.9601226733']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndti.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = ndti', + ' Minimum=-30000.000, Maximum=32767.000, Mean=3063.184, StdDev=1729.795', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=3063.1841212062', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=1729.7951175908']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndti.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = ndti', + ' Minimum=-30000.000, Maximum=32767.000, Mean=2981.244, StdDev=1896.016', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=2981.2441906591', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=1896.0164253292'])], + + # t_process[ndvi] recording: + 'ndvi': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndvi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = ndvi', + ' Minimum=-30000.000, Maximum=32767.000, Mean=3645.499, StdDev=3728.750', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=3645.4992438168', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=3728.7502524837']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndvi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = ndvi', + ' Minimum=-30000.000, Maximum=32767.000, Mean=3370.921, StdDev=4083.116', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=3370.921164936', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=4083.1160799659']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndvi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = ndvi', + ' Minimum=-30000.000, Maximum=32767.000, Mean=3428.079, StdDev=3862.660', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=3428.0792403057', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=3862.6596367669'])], + + # t_process[quality] recording: + 'quality': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/modis/tiles/h12v04/2012336/MCD43A2.A2012336.h12v04.006.2016112010833.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/modis/tiles/h12v04/2012337/MCD43A2.A2012337.h12v04.006.2016112013509.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_quality.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/modis/tiles/h12v04/2012338/MCD43A2.A2012338.h12v04.006.2016112020013.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo')], + + # t_process[isti] recording: + 'isti': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_isti.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = isti', + ' Minimum=-30000.000, Maximum=32767.000, Mean=5529.074, StdDev=2505.869', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=5529.073787627', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=2505.8690994676']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_isti.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = isti', + ' Minimum=-30000.000, Maximum=32767.000, Mean=5572.417, StdDev=2572.489', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=5572.4168886818', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=2572.4892195416']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_isti.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = isti', + ' Minimum=-30000.000, Maximum=32767.000, Mean=5754.650, StdDev=3082.850', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=5754.6495160657', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=3082.8503380537'])], + + # t_process[ndvi8] recording: + 'ndvi8': + [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_ndvi8.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 4800, 4800', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (231.656358263958253,-231.656358263750064)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD09Q1', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=4800x1 Type=Int16, ColorInterp=Gray', + ' Description = ndvi', + ' Minimum=-32759.000, Maximum=32767.000, Mean=2064.986, StdDev=5541.979', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=2064.9860240037', + ' STATISTICS_MINIMUM=-32759', + ' STATISTICS_STDDEV=5541.9794979241'])], + + # t_process[snow] recording: + 'snow': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_snow.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD10A1 MYD10A1', + ' FRACCLEARCOVERMISSING=0', + ' FRACCLEARCOVERSNOW=922', + ' FRACMISSINGCOVERCLEAR=11', + ' FRACMISSINGCOVERSNOW=0', + ' FRACMOSTLYCOVERCLEAR=2796', + ' FRACSNOWCOVERCLEAR=37891', + ' FRACSNOWCOVERMISSING=0', + ' GIPS Version=0.8.2', + ' NUMVALIDCOVER=1260832', + ' NUMVALIDFRAC=1260821', + ' TOTSNOWCOVER=375919', + ' TOTSNOWFRAC=237261', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=PIXEL', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Byte, ColorInterp=Gray', + ' Description = Snow Cover', + ' Minimum=0.000, Maximum=100.000, Mean=29.815, StdDev=45.669', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=100', + ' STATISTICS_MEAN=29.815193459557', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=45.669021224704', + 'Band 2 Block=2400x1 Type=Byte, ColorInterp=Undefined', + ' Description = Fractional Snow Cover', + ' Minimum=0.000, Maximum=100.000, Mean=18.818, StdDev=29.373', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=100', + ' STATISTICS_MEAN=18.817977333817', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=29.373327276096']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_snow.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD10A1 MYD10A1', + ' FRACCLEARCOVERMISSING=0', + ' FRACCLEARCOVERSNOW=3002', + ' FRACMISSINGCOVERCLEAR=487', + ' FRACMISSINGCOVERSNOW=0', + ' FRACMOSTLYCOVERCLEAR=666', + ' FRACSNOWCOVERCLEAR=11125', + ' FRACSNOWCOVERMISSING=0', + ' GIPS Version=0.8.2', + ' NUMVALIDCOVER=858463', + ' NUMVALIDFRAC=857976', + ' TOTSNOWCOVER=70234', + ' TOTSNOWFRAC=56234', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=PIXEL', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Byte, ColorInterp=Gray', + ' Description = Snow Cover', + ' Minimum=0.000, Maximum=100.000, Mean=8.181, StdDev=27.404', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=100', + ' STATISTICS_MEAN=8.1814242430949', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=27.404059620693', + 'Band 2 Block=2400x1 Type=Byte, ColorInterp=Undefined', + ' Description = Fractional Snow Cover', + ' Minimum=0.000, Maximum=100.000, Mean=6.554, StdDev=22.553', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=100', + ' STATISTICS_MEAN=6.5543500051283', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=22.553402602207']), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_snow.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD10A1 MYD10A1', + ' FRACCLEARCOVERMISSING=0', + ' FRACCLEARCOVERSNOW=6448', + ' FRACMISSINGCOVERCLEAR=1760', + ' FRACMISSINGCOVERSNOW=0', + ' FRACMOSTLYCOVERCLEAR=1363', + ' FRACSNOWCOVERCLEAR=41993', + ' FRACSNOWCOVERMISSING=0', + ' GIPS Version=0.8.2', + ' NUMVALIDCOVER=1634091', + ' NUMVALIDFRAC=1632331', + ' TOTSNOWCOVER=30508', + ' TOTSNOWFRAC=12395', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=PIXEL', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Byte, ColorInterp=Gray', + ' Description = Snow Cover', + ' Minimum=0.000, Maximum=100.000, Mean=1.867, StdDev=12.010', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=100', + ' STATISTICS_MEAN=1.8669706889029', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=12.010071248489', + 'Band 2 Block=2400x1 Type=Byte, ColorInterp=Undefined', + ' Description = Fractional Snow Cover', + ' Minimum=0.000, Maximum=100.000, Mean=0.759, StdDev=5.693', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=100', + ' STATISTICS_MEAN=0.75936436911386', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=5.6934442961109'])], + + # t_process[fsnow] recording: + 'fsnow': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_fsnow.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD10A1 MYD10A1', + ' FRACCLEARCOVERMISSING=0', + ' FRACCLEARCOVERSNOW=922', + ' FRACMISSINGCOVERCLEAR=11', + ' FRACMISSINGCOVERSNOW=0', + ' FRACMOSTLYCOVERCLEAR=0', + ' FRACSNOWCOVERCLEAR=37891', + ' FRACSNOWCOVERMISSING=0', + ' GIPS Version=0.8.2', + ' NUMVALIDCOVER=1260832', + ' NUMVALIDFRAC=1260821', + ' TOTSNOWCOVER=375919', + ' TOTSNOWFRAC=237261', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x3 Type=Byte, ColorInterp=Gray', + ' Description = Fractional Snow Cover', + ' Minimum=0.000, Maximum=100.000, Mean=18.079, StdDev=29.407', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=100', + ' STATISTICS_MEAN=18.078730552524', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=29.406815101598']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_fsnow.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD10A1 MYD10A1', + ' FRACCLEARCOVERMISSING=0', + ' FRACCLEARCOVERSNOW=3002', + ' FRACMISSINGCOVERCLEAR=487', + ' FRACMISSINGCOVERSNOW=0', + ' FRACMOSTLYCOVERCLEAR=0', + ' FRACSNOWCOVERCLEAR=11125', + ' FRACSNOWCOVERMISSING=0', + ' GIPS Version=0.8.2', + ' NUMVALIDCOVER=858463', + ' NUMVALIDFRAC=857976', + ' TOTSNOWCOVER=70234', + ' TOTSNOWFRAC=56234', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x3 Type=Byte, ColorInterp=Gray', + ' Description = Fractional Snow Cover', + ' Minimum=0.000, Maximum=100.000, Mean=6.325, StdDev=22.460', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=100', + ' STATISTICS_MEAN=6.3247816155152', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=22.460360759398']), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_fsnow.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD10A1 MYD10A1', + ' FRACCLEARCOVERMISSING=0', + ' FRACCLEARCOVERSNOW=6448', + ' FRACMISSINGCOVERCLEAR=1760', + ' FRACMISSINGCOVERSNOW=0', + ' FRACMOSTLYCOVERCLEAR=0', + ' FRACSNOWCOVERCLEAR=41993', + ' FRACSNOWCOVERMISSING=0', + ' GIPS Version=0.8.2', + ' NUMVALIDCOVER=1634091', + ' NUMVALIDFRAC=1632331', + ' TOTSNOWCOVER=30508', + ' TOTSNOWFRAC=12395', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x3 Type=Byte, ColorInterp=Gray', + ' Description = Fractional Snow Cover', + ' Minimum=0.000, Maximum=100.000, Mean=0.299, StdDev=4.325', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=100', + ' STATISTICS_MEAN=0.29908493468234', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=4.324731447589'])], + + # t_process[crcm] recording: + 'crcm': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crcm.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = crcm', + ' Minimum=-16667.000, Maximum=32767.000, Mean=2859.549, StdDev=7299.187', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=2859.5488278799', + ' STATISTICS_MINIMUM=-16667', + ' STATISTICS_STDDEV=7299.187371038']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crcm.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = crcm', + ' Minimum=-30000.000, Maximum=32767.000, Mean=2687.453, StdDev=7389.052', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=2687.4530517044', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=7389.0518213965']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crcm.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = crcm', + ' Minimum=-30000.000, Maximum=32767.000, Mean=2525.140, StdDev=7450.647', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=2525.1398757207', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=7450.6468599583'])], + + # t_process[ndsi] recording: + 'ndsi': + [('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndsi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = ndsi', + ' Minimum=-14000.000, Maximum=30000.000, Mean=-1034.395, StdDev=5424.379', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=30000', + ' STATISTICS_MEAN=-1034.3949800482', + ' STATISTICS_MINIMUM=-14000', + ' STATISTICS_STDDEV=5424.3787845071']), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndsi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = ndsi', + ' Minimum=-20000.000, Maximum=32767.000, Mean=-1183.298, StdDev=5352.399', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=-1183.2978087378', + ' STATISTICS_MINIMUM=-20000', + ' STATISTICS_STDDEV=5352.3985804984']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndsi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = ndsi', + ' Minimum=-30000.000, Maximum=30000.000, Mean=-902.639, StdDev=5506.727', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=30000', + ' STATISTICS_MEAN=-902.63861078388', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=5506.7268705181'])], + + # t_process[evi] recording: + 'evi': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_evi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = evi', + ' Minimum=-32714.000, Maximum=32767.000, Mean=2034.130, StdDev=1213.308', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=2034.129933023', + ' STATISTICS_MINIMUM=-32714', + ' STATISTICS_STDDEV=1213.3082843962']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_evi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = evi', + ' Minimum=-32648.000, Maximum=32767.000, Mean=1999.583, StdDev=1237.318', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=1999.5830646948', + ' STATISTICS_MINIMUM=-32648', + ' STATISTICS_STDDEV=1237.3180198372']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_evi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = evi', + ' Minimum=-32665.000, Maximum=32767.000, Mean=1977.732, StdDev=1231.687', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=1977.7317117551', + ' STATISTICS_MINIMUM=-32665', + ' STATISTICS_STDDEV=1231.6874352355'])], + + # t_process[msavi2] recording: + 'msavi2': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_msavi2.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = msavi2', + ' Minimum=0.000, Maximum=11225.000, Mean=4465.642, StdDev=936.916', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=11225', + ' STATISTICS_MEAN=4465.6415102532', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=936.91638748312']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_msavi2.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = msavi2', + ' Minimum=0.000, Maximum=11241.000, Mean=4446.652, StdDev=926.441', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=11241', + ' STATISTICS_MEAN=4446.6522660701', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=926.44144509475']), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_msavi2.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = msavi2', + ' Minimum=0.000, Maximum=11258.000, Mean=4472.098, StdDev=913.061', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=11258', + ' STATISTICS_MEAN=4472.0977490096', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=913.06051675922'])], + + # t_process[bi] recording: + 'bi': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_bi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = bi', + ' Minimum=-1.000, Maximum=7878.000, Mean=1195.811, StdDev=835.168', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=7878', + ' STATISTICS_MEAN=1195.8110899956', + ' STATISTICS_MINIMUM=-1', + ' STATISTICS_STDDEV=835.16845229601']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_bi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = bi', + ' Minimum=-1.000, Maximum=7875.000, Mean=1237.854, StdDev=852.914', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=7875', + ' STATISTICS_MEAN=1237.853783459', + ' STATISTICS_MINIMUM=-1', + ' STATISTICS_STDDEV=852.91417668845']), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_bi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = bi', + ' Minimum=-1.000, Maximum=7873.000, Mean=1216.722, StdDev=832.290', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=7873', + ' STATISTICS_MEAN=1216.722151189', + ' STATISTICS_MINIMUM=-1', + ' STATISTICS_STDDEV=832.29000047611'])], + + # t_process[obstime] recording: + 'obstime': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_obstime.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 1200, 1200', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.118599000386894,5559752.598833000287414)', + 'Pixel Size = (926.625433138333392,-926.625433139166944)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD11A1 MYD11A1', + ' GIPS Version=0.8.2', + ' VERSION=1', + 'Image Structure Metadata:', + ' INTERLEAVE=PIXEL', + 'Corner Coordinates:', + 'Upper Left (-6671703.119, 5559752.599) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.119, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.599, 5559752.599) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.599, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.859, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=1200x1 Type=Byte, ColorInterp=Gray', + ' Description = Observation Time Daytime Terra', + ' Minimum=99.000, Maximum=119.000, Mean=108.748, StdDev=5.824', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=119', + ' STATISTICS_MEAN=108.74766651808', + ' STATISTICS_MINIMUM=99', + ' STATISTICS_STDDEV=5.8241380829366', + 'Band 2 Block=1200x1 Type=Byte, ColorInterp=Undefined', + ' Description = Observation Time Nighttime Terra', + ' Minimum=206.000, Maximum=227.000, Mean=220.910, StdDev=5.356', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=227', + ' STATISTICS_MEAN=220.90999179166', + ' STATISTICS_MINIMUM=206', + ' STATISTICS_STDDEV=5.3560507339038', + 'Band 3 Block=1200x1 Type=Byte, ColorInterp=Undefined', + ' Description = Observation Time Daytime Aqua', + ' Minimum=120.000, Maximum=139.000, Mean=128.621, StdDev=6.059', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=139', + ' STATISTICS_MEAN=128.62144994809', + ' STATISTICS_MINIMUM=120', + ' STATISTICS_STDDEV=6.0594800008807', + 'Band 4 Block=1200x1 Type=Byte, ColorInterp=Alpha', + ' Description = Observation Time Nighttime Aqua', + ' Minimum=7.000, Maximum=30.000, Mean=23.036, StdDev=5.004', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=30', + ' STATISTICS_MEAN=23.036279683377', + ' STATISTICS_MINIMUM=7', + ' STATISTICS_STDDEV=5.0035463331778']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_obstime.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 1200, 1200', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.118599000386894,5559752.598833000287414)', + 'Pixel Size = (926.625433138333392,-926.625433139166944)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD11A1 MYD11A1', + ' GIPS Version=0.8.2', + ' VERSION=1', + 'Image Structure Metadata:', + ' INTERLEAVE=PIXEL', + 'Corner Coordinates:', + 'Upper Left (-6671703.119, 5559752.599) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.119, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.599, 5559752.599) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.599, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.859, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=1200x1 Type=Byte, ColorInterp=Gray', + ' Description = Observation Time Daytime Terra', + ' Minimum=103.000, Maximum=115.000, Mean=109.860, StdDev=2.547', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=115', + ' STATISTICS_MEAN=109.86043890866', + ' STATISTICS_MINIMUM=103', + ' STATISTICS_STDDEV=2.5470264191015', + 'Band 2 Block=1200x1 Type=Byte, ColorInterp=Undefined', + ' Description = Observation Time Nighttime Terra', + ' Minimum=206.000, Maximum=228.000, Mean=217.436, StdDev=7.626', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=228', + ' STATISTICS_MEAN=217.43558462018', + ' STATISTICS_MINIMUM=206', + ' STATISTICS_STDDEV=7.6260382004364', + 'Band 3 Block=1200x1 Type=Byte, ColorInterp=Undefined', + ' Description = Observation Time Daytime Aqua', + ' Minimum=122.000, Maximum=139.000, Mean=127.061, StdDev=3.524', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=139', + ' STATISTICS_MEAN=127.06056860321', + ' STATISTICS_MINIMUM=122', + ' STATISTICS_STDDEV=3.524392393745', + 'Band 4 Block=1200x1 Type=Byte, ColorInterp=Alpha', + ' Description = Observation Time Nighttime Aqua', + ' Minimum=11.000, Maximum=33.000, Mean=22.678, StdDev=7.648', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=33', + ' STATISTICS_MEAN=22.678463414634', + ' STATISTICS_MINIMUM=11', + ' STATISTICS_STDDEV=7.6480703499595']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_obstime.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 1200, 1200', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.118599000386894,5559752.598833000287414)', + 'Pixel Size = (926.625433138333392,-926.625433139166944)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD11A1 MYD11A1', + ' GIPS Version=0.8.2', + ' VERSION=1', + 'Image Structure Metadata:', + ' INTERLEAVE=PIXEL', + 'Corner Coordinates:', + 'Upper Left (-6671703.119, 5559752.599) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.119, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.599, 5559752.599) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.599, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.859, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=1200x1 Type=Byte, ColorInterp=Gray', + ' Description = Observation Time Daytime Terra', + ' Minimum=102.000, Maximum=112.000, Mean=105.494, StdDev=1.294', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=112', + ' STATISTICS_MEAN=105.49434299527', + ' STATISTICS_MINIMUM=102', + ' STATISTICS_STDDEV=1.2941963263825', + 'Band 2 Block=1200x1 Type=Byte, ColorInterp=Undefined', + ' Description = Observation Time Nighttime Terra', + ' Minimum=207.000, Maximum=228.000, Mean=218.812, StdDev=7.083', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=228', + ' STATISTICS_MEAN=218.81208627538', + ' STATISTICS_MINIMUM=207', + ' STATISTICS_STDDEV=7.0831187760946', + 'Band 3 Block=1200x1 Type=Byte, ColorInterp=Undefined', + ' Description = Observation Time Daytime Aqua', + ' Minimum=120.000, Maximum=137.000, Mean=122.910, StdDev=1.491', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=137', + ' STATISTICS_MEAN=122.91012838802', + ' STATISTICS_MINIMUM=120', + ' STATISTICS_STDDEV=1.4909800549212', + 'Band 4 Block=1200x1 Type=Byte, ColorInterp=Alpha', + ' Description = Observation Time Nighttime Aqua', + ' Minimum=10.000, Maximum=31.000, Mean=22.360, StdDev=5.742', + ' NoData Value=0', + ' Offset: 0, Scale:0.1', + ' Metadata:', + ' STATISTICS_MAXIMUM=31', + ' STATISTICS_MEAN=22.359684619225', + ' STATISTICS_MINIMUM=10', + ' STATISTICS_STDDEV=5.7422905503166'])], + + # t_process[lswi] recording: + 'lswi': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_lswi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = lswi', + ' Minimum=-30000.000, Maximum=32767.000, Mean=754.639, StdDev=4246.601', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=754.63877273148', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=4246.6013837712']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_lswi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = lswi', + ' Minimum=-30000.000, Maximum=32767.000, Mean=1089.544, StdDev=4148.901', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=1089.5436196297', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=4148.9014752875']), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_lswi.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = lswi', + ' Minimum=-30000.000, Maximum=32767.000, Mean=1204.967, StdDev=3983.741', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=1204.9673345758', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=3983.7411778579'])], + + # t_process[vari] recording: + 'vari': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_vari.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = vari', + ' Minimum=-32766.000, Maximum=32767.000, Mean=1040.450, StdDev=9996.309', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=1040.4500942203', + ' STATISTICS_MINIMUM=-32766', + ' STATISTICS_STDDEV=9996.3093468525']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_vari.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = vari', + ' Minimum=-32766.000, Maximum=32767.000, Mean=794.952, StdDev=9264.078', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=794.95241719667', + ' STATISTICS_MINIMUM=-32766', + ' STATISTICS_STDDEV=9264.0779079414']), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_vari.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = vari', + ' Minimum=-32766.000, Maximum=32767.000, Mean=678.058, StdDev=9126.506', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=678.05846237722', + ' STATISTICS_MINIMUM=-32766', + ' STATISTICS_STDDEV=9126.5055623721'])], + + # t_process[brgt] recording: + 'brgt': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_brgt.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = brgt', + ' Minimum=-1.000, Maximum=8007.000, Mean=855.037, StdDev=817.369', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=8007', + ' STATISTICS_MEAN=855.03682025227', + ' STATISTICS_MINIMUM=-1', + ' STATISTICS_STDDEV=817.36873337109']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_brgt.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = brgt', + ' Minimum=0.000, Maximum=8011.000, Mean=883.508, StdDev=836.901', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=8011', + ' STATISTICS_MEAN=883.5083854847', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=836.90097855261']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_brgt.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = brgt', + ' Minimum=2.000, Maximum=8016.000, Mean=842.385, StdDev=813.171', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=8016', + ' STATISTICS_MEAN=842.38453292393', + ' STATISTICS_MINIMUM=2', + ' STATISTICS_STDDEV=813.17119504428'])], + + # t_process[temp8tn] recording: + 'temp8tn': + [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8tn.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Night_1km')], + + # t_process[clouds] recording: + 'clouds': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MOD_clouds.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD10A1', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x3 Type=Byte, ColorInterp=Gray', + ' Description = Cloud Cover', + ' Minimum=0.000, Maximum=1.000, Mean=0.764, StdDev=0.425', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=1', + ' STATISTICS_MEAN=0.76357376252845', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=0.42488689283926']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MOD_clouds.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD10A1', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x3 Type=Byte, ColorInterp=Gray', + ' Description = Cloud Cover', + ' Minimum=0.000, Maximum=1.000, Mean=0.784, StdDev=0.412', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=1', + ' STATISTICS_MEAN=0.78391170259382', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=0.41157519984843']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_clouds.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD10A1', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x3 Type=Byte, ColorInterp=Gray', + ' Description = Cloud Cover', + ' Minimum=0.000, Maximum=1.000, Mean=0.863, StdDev=0.344', + ' NoData Value=127', + ' Metadata:', + ' STATISTICS_MAXIMUM=1', + ' STATISTICS_MEAN=0.86272688241637', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=0.34413545119982'])], + + # t_process[temp] recording: + 'temp': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_temp.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 1200, 1200', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.118599000386894,5559752.598833000287414)', + 'Pixel Size = (926.625433138333392,-926.625433139166944)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD11A1 MYD11A1', + ' GIPS Version=0.8.2', + ' MEANOVERPASSTIME_DAY_AQUA=2.23593', + ' MEANOVERPASSTIME_DAY_TERRA=21.8812', + ' NUMBAD_DAY_AQUA=1171621', + ' NUMBAD_DAY_TERRA=1149952', + ' NUMBEST_DAY_AQUA=52004', + ' NUMBEST_DAY_TERRA=74134', + ' NUMGOOD_DAY_AQUA=268379', + ' NUMGOOD_DAY_TERRA=290048', + ' VERSION=1.1', + 'Image Structure Metadata:', + ' INTERLEAVE=PIXEL', + 'Corner Coordinates:', + 'Upper Left (-6671703.119, 5559752.599) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.119, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.599, 5559752.599) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.599, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.859, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=1200x1 Type=UInt16, ColorInterp=Gray', + ' Description = Temperature Daytime Terra', + ' Minimum=12698.000, Maximum=14199.000, Mean=13182.023, StdDev=389.305', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=14199', + ' STATISTICS_MEAN=13182.023227235', + ' STATISTICS_MINIMUM=12698', + ' STATISTICS_STDDEV=389.30504795596', + 'Band 2 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Nighttime Terra', + ' Minimum=12194.000, Maximum=14049.000, Mean=12922.418, StdDev=309.612', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=14049', + ' STATISTICS_MEAN=12922.418403161', + ' STATISTICS_MINIMUM=12194', + ' STATISTICS_STDDEV=309.61194714359', + 'Band 3 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Daytime Aqua', + ' Minimum=12908.000, Maximum=14154.000, Mean=13515.129, StdDev=377.376', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=14154', + ' STATISTICS_MEAN=13515.128906757', + ' STATISTICS_MINIMUM=12908', + ' STATISTICS_STDDEV=377.37619731842', + 'Band 4 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Nighttime Aqua', + ' Minimum=12278.000, Maximum=14043.000, Mean=13064.797, StdDev=331.904', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=14043', + ' STATISTICS_MEAN=13064.797167439', + ' STATISTICS_MINIMUM=12278', + ' STATISTICS_STDDEV=331.90371206731', + 'Band 5 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Best Quality', + ' Minimum=0.000, Maximum=13.000, Mean=0.424, StdDev=1.577', + ' NoData Value=65535', + ' Metadata:', + ' STATISTICS_MAXIMUM=13', + ' STATISTICS_MEAN=0.42434930555556', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=1.576894548432']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_temp.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 1200, 1200', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.118599000386894,5559752.598833000287414)', + 'Pixel Size = (926.625433138333392,-926.625433139166944)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD11A1 MYD11A1', + ' GIPS Version=0.8.2', + ' MEANOVERPASSTIME_DAY_AQUA=2.23734', + ' MEANOVERPASSTIME_DAY_TERRA=21.7436', + ' NUMBAD_DAY_AQUA=1398452', + ' NUMBAD_DAY_TERRA=1385409', + ' NUMBEST_DAY_AQUA=3352', + ' NUMBEST_DAY_TERRA=1756', + ' NUMGOOD_DAY_AQUA=41548', + ' NUMGOOD_DAY_TERRA=54591', + ' VERSION=1.1', + 'Image Structure Metadata:', + ' INTERLEAVE=PIXEL', + 'Corner Coordinates:', + 'Upper Left (-6671703.119, 5559752.599) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.119, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.599, 5559752.599) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.599, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.859, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=1200x1 Type=UInt16, ColorInterp=Gray', + ' Description = Temperature Daytime Terra', + ' Minimum=12256.000, Maximum=14274.000, Mean=13427.907, StdDev=474.819', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=14274', + ' STATISTICS_MEAN=13427.90747331', + ' STATISTICS_MINIMUM=12256', + ' STATISTICS_STDDEV=474.81908783442', + 'Band 2 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Nighttime Terra', + ' Minimum=12535.000, Maximum=14114.000, Mean=13341.400, StdDev=212.633', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=14114', + ' STATISTICS_MEAN=13341.40039567', + ' STATISTICS_MINIMUM=12535', + ' STATISTICS_STDDEV=212.63257522085', + 'Band 3 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Daytime Aqua', + ' Minimum=12104.000, Maximum=14297.000, Mean=13512.830, StdDev=497.211', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=14297', + ' STATISTICS_MEAN=13512.829934075', + ' STATISTICS_MINIMUM=12104', + ' STATISTICS_STDDEV=497.21053990757', + 'Band 4 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Nighttime Aqua', + ' Minimum=12793.000, Maximum=13772.000, Mean=13231.987, StdDev=166.612', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=13772', + ' STATISTICS_MEAN=13231.98690671', + ' STATISTICS_MINIMUM=12793', + ' STATISTICS_STDDEV=166.61194311875', + 'Band 5 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Best Quality', + ' Minimum=0.000, Maximum=10.000, Mean=0.023, StdDev=0.402', + ' NoData Value=65535', + ' Metadata:', + ' STATISTICS_MAXIMUM=10', + ' STATISTICS_MEAN=0.023167361111111', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=0.40188800408023']), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_temp.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 1200, 1200', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.118599000386894,5559752.598833000287414)', + 'Pixel Size = (926.625433138333392,-926.625433139166944)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' AVAILABLE_ASSETS=MOD11A1 MYD11A1', + ' GIPS Version=0.8.2', + ' MEANOVERPASSTIME_DAY_AQUA=2.30363', + ' MEANOVERPASSTIME_DAY_TERRA=22.091', + ' NUMBAD_DAY_AQUA=1256564', + ' NUMBAD_DAY_TERRA=1305990', + ' NUMBEST_DAY_AQUA=34115', + ' NUMBEST_DAY_TERRA=11207', + ' NUMGOOD_DAY_AQUA=183436', + ' NUMGOOD_DAY_TERRA=134010', + ' VERSION=1.1', + 'Image Structure Metadata:', + ' INTERLEAVE=PIXEL', + 'Corner Coordinates:', + 'Upper Left (-6671703.119, 5559752.599) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.119, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.599, 5559752.599) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.599, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.859, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=1200x1 Type=UInt16, ColorInterp=Gray', + ' Description = Temperature Daytime Terra', + ' Minimum=13472.000, Maximum=14601.000, Mean=14165.988, StdDev=159.821', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=14601', + ' STATISTICS_MEAN=14165.988011779', + ' STATISTICS_MINIMUM=13472', + ' STATISTICS_STDDEV=159.82118558227', + 'Band 2 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Nighttime Terra', + ' Minimum=12537.000, Maximum=14111.000, Mean=13429.254, StdDev=320.629', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=14111', + ' STATISTICS_MEAN=13429.253876576', + ' STATISTICS_MINIMUM=12537', + ' STATISTICS_STDDEV=320.62907411603', + 'Band 3 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Daytime Aqua', + ' Minimum=13430.000, Maximum=14649.000, Mean=14159.739, StdDev=170.341', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=14649', + ' STATISTICS_MEAN=14159.739091361', + ' STATISTICS_MINIMUM=13430', + ' STATISTICS_STDDEV=170.34148452201', + 'Band 4 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Nighttime Aqua', + ' Minimum=12787.000, Maximum=14139.000, Mean=13558.850, StdDev=328.310', + ' NoData Value=65535', + ' Offset: 0, Scale:0.02', + ' Metadata:', + ' STATISTICS_MAXIMUM=14139', + ' STATISTICS_MEAN=13558.850345625', + ' STATISTICS_MINIMUM=12787', + ' STATISTICS_STDDEV=328.3101047801', + 'Band 5 Block=1200x1 Type=UInt16, ColorInterp=Undefined', + ' Description = Temperature Best Quality', + ' Minimum=0.000, Maximum=15.000, Mean=0.355, StdDev=1.462', + ' NoData Value=65535', + ' Metadata:', + ' STATISTICS_MAXIMUM=15', + ' STATISTICS_MEAN=0.35506597222222', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=1.462327279386'])], + + # t_process[temp8td] recording: + 'temp8td': + [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8td.tif', + 'symlink', + 'HDF4_EOS:EOS_GRID:"', + '/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Day_1km')], + + # t_process[sti] recording: + 'sti': + [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_sti.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = sti', + ' Minimum=-30000.000, Maximum=32767.000, Mean=19253.991, StdDev=5822.084', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=19253.990643803', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=5822.0838932394']), + ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_sti.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = sti', + ' Minimum=-30000.000, Maximum=32767.000, Mean=19516.385, StdDev=5506.425', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=19516.385052939', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=5506.4246903928']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_sti.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = sti', + ' Minimum=-30000.000, Maximum=32767.000, Mean=19455.885, StdDev=5525.886', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=19455.884765598', + ' STATISTICS_MINIMUM=-30000', + ' STATISTICS_STDDEV=5525.8863093302'])], + + # t_process[crc] recording: + 'crc': + [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crc.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = crc', + ' Minimum=-10500.000, Maximum=32767.000, Mean=4367.933, StdDev=8860.156', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=4367.9333322037', + ' STATISTICS_MINIMUM=-10500', + ' STATISTICS_STDDEV=8860.1557123079']), + ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crc.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = crc', + ' Minimum=-10488.000, Maximum=32767.000, Mean=4155.715, StdDev=8941.140', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=4155.7145982629', + ' STATISTICS_MINIMUM=-10488', + ' STATISTICS_STDDEV=8941.13966905']), + ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crc.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 2400, 2400', + 'Coordinate System is:', + 'PROJCS["unnamed",', + ' GEOGCS["Unknown datum based upon the custom spheroid",', + ' DATUM["Not_specified_based_on_custom_spheroid",', + ' SPHEROID["Custom spheroid",6371007.181,0]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433]],', + ' PROJECTION["Sinusoidal"],', + ' PARAMETER["longitude_of_center",0],', + ' PARAMETER["false_easting",0],', + ' PARAMETER["false_northing",0],', + ' UNIT["metre",1,', + ' AUTHORITY["EPSG","9001"]]]', + 'Origin = (-6671703.117999999783933,5559752.598333000205457)', + 'Pixel Size = (463.312716527916507,-463.312716527500129)', + 'Metadata:', + ' AREA_OR_POINT=Area', + ' GIPS Version=0.8.2', + ' VERSION=1.0', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', + 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', + 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', + 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', + 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', + 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', + ' Description = crc', + ' Minimum=-10488.000, Maximum=32767.000, Mean=3945.189, StdDev=9037.669', + ' NoData Value=-32768', + ' Unit Type: other', + ' Offset: 0, Scale:0.0001', + ' Metadata:', + ' STATISTICS_MAXIMUM=32767', + ' STATISTICS_MEAN=3945.1894356865', + ' STATISTICS_MINIMUM=-10488', + ' STATISTICS_STDDEV=9037.6685553501'])], + +} diff --git a/gips/test/sys/expected/std_process.py b/gips/test/sys/expected/std_process.py index 72d337e6..2c9a84c2 100644 --- a/gips/test/sys/expected/std_process.py +++ b/gips/test/sys/expected/std_process.py @@ -3,452 +3,12 @@ from .. import util +from . import modis_process + expectations = {} mark_spec = {} -expectations['modis'] = { - # 'landcover' [], # is annual, not available for the scene under test - # 'indices', deprecated; not testing as a result - # weird symlink products: - # t_process[quality] recording: - 'quality': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_quality.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/modis/tiles/h12v04/2012336/MCD43A2.A2012336.h12v04.006.2016112010833.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_quality.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/modis/tiles/h12v04/2012337/MCD43A2.A2012337.h12v04.006.2016112013509.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_quality.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/modis/tiles/h12v04/2012338/MCD43A2.A2012338.h12v04.006.2016112020013.hdf":MOD_Grid_BRDF:Snow_BRDF_Albedo')], - - # t_process[temp8tn] recording: - 'temp8tn': - [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8tn.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Night_1km')], - - # t_process[temp8td] recording: - 'temp8td': - [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_temp8td.tif', - 'symlink', - 'HDF4_EOS:EOS_GRID:"', - '/modis/tiles/h12v04/2012337/MOD11A2.A2012337.h12v04.006.2016137164847.hdf":MODIS_Grid_8Day_1km_LST:LST_Day_1km')], - - # normal products - - # t_process[ndti] recording: - 'ndti': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndti.tif', - 'hash', - 'sha256', - '025ea7a9204ae8ad0952a92b386bf735dbbce8337e336843476d3f227cf1f1ab'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndti.tif', - 'hash', - 'sha256', - 'b6baa8c7a3996f680c4177e4157c9275dd895c8e6350afa6ef564e673bb6a5f0'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndti.tif', - 'hash', - 'sha256', - 'a1c4f1161e8a584257352e76160319c39aa4605dd565d388d0116e193773771f')], - - # t_process[ndvi] recording: - 'ndvi': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndvi.tif', - 'hash', - 'sha256', - 'efbeae3a1c5e59ccd54b6108e275f9a8f0059162ac2797f4d69b00acb80147b3'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndvi.tif', - 'hash', - 'sha256', - 'b56d5c96229a1ad4cdcbd09db10b19a216b4cde8a0ffd184ee5cf01a35ab2b46'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndvi.tif', - 'hash', - 'sha256', - 'd9ff5d2f85ac9a267a8f6b1aca34f62b8196721e212b1e092f82ce9f1e208c61')], - - # t_process[isti] recording: - 'isti': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_isti.tif', - 'hash', - 'sha256', - 'c0a38d1cd00de52eacd8abcda714bfdaa575787e27b40dc0472ffca5ab1696c0'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_isti.tif', - 'hash', - 'sha256', - 'b1759c1008e5914b955c4a9f55572baa8f7c0310677264a1057329e236d826cb'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_isti.tif', - 'hash', - 'sha256', - '677182d186dac992b52b93239af0f419536075c7cd467864fa544bcde4b89e69')], - - # t_process[ndvi8] recording: - 'ndvi8': - [('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_ndvi8.tif', - 'hash', - 'sha256', - '93db24c18886fbac0b87d1404a0c80606a599cc55b02d745a09ffb051b8131ac')], - - # t_process[snow] recording: - 'snow': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_snow.tif', - 'hash', - 'sha256', - 'bea7315b736a75bcf37df86089679845ac597f3fba5d5cd6e79f5516b9657faa'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_snow.tif', - 'hash', - 'sha256', - 'de979c1a0c616d7889dd5fd45816849f9c08df54167d7f3b40301d6ac4495778'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_snow.tif', - 'hash', - 'sha256', - '5d9bbfff1522e74165cdd677f1580e9eb2236c2c5922a796a6c9ee0db24ae05a')], - - # t_process[fsnow] recording: - 'fsnow': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_fsnow.tif', - 'hash', - 'sha256', - '853a5ca95f21c4a745e41e4acc46a71a74c74e0cdd1cae9ed3dd31b2fef1ddbd'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_fsnow.tif', - 'hash', - 'sha256', - '3a5f7c95fe60e5789919cf55d7f8e652e4d668c6eb63eaa53a82249c0ea19c5a'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_fsnow.tif', - 'hash', - 'sha256', - 'd09b8a12ad17a7e715ae2e988fbf683e5fe2df46e97e9ac3ebb4444c8b1c169d')], - - # t_process[crcm] recording: - 'crcm': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crcm.tif', - 'hash', - 'sha256', - 'a45c2fb16d871a919e0c803f80e1f3b4ec831472d0466c33741fcf8e8150f6ba'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crcm.tif', - 'hash', - 'sha256', - '9aa80c0e1fa1b62b0a41aed42eeaf84632845d74f29594dfee55c22673b0a6f5'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crcm.tif', - 'hash', - 'sha256', - 'a28c7d280fb8dfe0fe1cc003e1001457abdf14acc5545b2a69b782cfbcce7b50')], - - # t_process[ndsi] recording: - 'ndsi': - [('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_ndsi.tif', - 'hash', - 'sha256', - '1f91f738bbb28b3ebeba36affd8d0689e105e90e1547cca58497eb7c7fdf7346'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_ndsi.tif', - 'hash', - 'sha256', - 'a626d375fbe5094a5199fdfc2206cf94777ff574379b45811cf7a49055a9f6ec'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_ndsi.tif', - 'hash', - 'sha256', - 'cb2ca7461c65f66ddb13e804ae7fe8f54daaf620ade600efee24810d40d4eb34')], - - # t_process[brgt] recording: - 'brgt': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_brgt.tif', - 'hash', - 'sha256', - 'bf6bad538c7c6e703bc2f39503c2f128fa8e988a533e302c0124e085735dfaf9'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_brgt.tif', - 'hash', - 'sha256', - '5c72e8f37640ec6b4a5a33e18980d71c70482fc8f708ba25da73e074cf64729f'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_brgt.tif', - 'hash', - 'sha256', - '3460d0d35e35540b55e9eda35f253af8d85eabb84c0c44cb5f9c02d43da7e20a')], - - # t_process[msavi2] recording: - 'msavi2': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_msavi2.tif', - 'hash', - 'sha256', - 'd2c93798df10939ff8afb6cd23e18ac3281cfd844e45da47a29d0342c2b980e9'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_msavi2.tif', - 'hash', - 'sha256', - '92fe4f40d051ef3f479bde76a67bfff978bc513893af5ed62e4a48e457daabad'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_msavi2.tif', - 'hash', - 'sha256', - 'f25bd2d58d3edae72b11e73c08341479dee13af80352a4749861b6a54a81b351')], - - # t_process[bi] recording: - 'bi': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_bi.tif', - 'hash', - 'sha256', - '28c181b8d210b71759589da53295cc894c9d12c74ff1f7b0a49b88f73664282e'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_bi.tif', - 'hash', - 'sha256', - '45acfc9249e063b6b97f269338590d51e2334e9bc5df118613ad908c6f0a3f8e'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_bi.tif', - 'hash', - 'sha256', - '8526cb96d59fc9549dc8535fc8128123b9d39a63a3b7cd97ee3e611b7eeadf6f')], - - # t_process[obstime] recording: - 'obstime': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - '66e01c94910c4fdd4f6fb2ab99b0c2d0cb73618032783e3cbdf1867191981fdc'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - '117cea0497b3158b465dd7dd197c886ebb1acf5059ed83dfdd6b5d7e94bb5243'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_obstime.tif', - 'hash', - 'sha256', - 'b04d8b07ef83e479db5650a30c84513bde7d5b05dd68187782c057ea0fd3e537')], - - # t_process[lswi] recording: - 'lswi': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_lswi.tif', - 'hash', - 'sha256', - '8de7378cdba830171c8da94424bf7807fa6c8b12523e8b34175a4b17ec770a5d'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_lswi.tif', - 'hash', - 'sha256', - '62de4bdafcc276e9c1c343f4e63f5dd06167e87f7712796f991e3a957f2b71de'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_lswi.tif', - 'hash', - 'sha256', - 'add051ef35d7ffff5eb68c48f8b8071f9cb493b5d3426867605d4c61746e3a26')], - - # t_process[vari] recording: - 'vari': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_vari.tif', - 'hash', - 'sha256', - '732d6d3d9fbe0e1b4616a3f8a91bf12d532598a0a02e8aab16c9a01c3ac56913'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_vari.tif', - 'hash', - 'sha256', - '9875717cc5e1926f11378bbf16345dbade5a2badf9f3dd06844a77f94d80ef72'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_vari.tif', - 'hash', - 'sha256', - '70fd19862ef4145c24f26a8a74cb748279b88fd973de935726ebef2294e17edd')], - - # t_process[evi] recording: - 'evi': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_evi.tif', - 'hash', - 'sha256', - '13548ae74456f08e6add657a637ca364626eafd2cc767724f57aac8ff93005bb'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_evi.tif', - 'hash', - 'sha256', - '7b926f7004cd92f2ec604fc4e2cc1489f60a335e39f4954030332a012fd33f87'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_evi.tif', - 'hash', - 'sha256', - '51db813a857f795817f529edd56d7134004566be46edab8a8dadc81286fb895f')], - - # t_process[clouds] recording: - 'clouds': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MOD_clouds.tif', - 'hash', - 'sha256', - '26b3e7e695f08ec2e9fb26985f9d5e392f9e8cedc331ab6721ea8e48821d93f0'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MOD_clouds.tif', - 'hash', - 'sha256', - 'c43eec8178d0521d20e2a5b6e9b61525007d80702c6dd705a3e52ed3a76a7b73'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD_clouds.tif', - 'hash', - 'sha256', - '24a6a7050c866eed989530bec45b72004097f1e36fcc0e686652b87723a86de4')], - - # t_process[temp] recording: - 'temp': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MOD-MYD_temp.tif', - 'hash', - 'sha256', - 'ead2622531929bba3192b8a6b609d5d920e79394ec577c5245d74d69593d827c'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '5d498bad2779f84289dfb75b14345d64d16b5bc7994b4aca6cd694f70b86064d'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MOD-MYD_temp.tif', - 'hash', - 'sha256', - '097f341fa917a0ec316c37d7907ff15b754dcd0925469be586016137ca47575f')], - - # t_process[sti] recording: - 'sti': - [('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_sti.tif', - 'hash', - 'sha256', - '24e50766ec18826f30bbad07bc8f5a34e72f1966b325cc23f064aa5fd35dc24a'), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_sti.tif', - 'hash', - 'sha256', - 'c223de9d00e9cfedc0cfc13a45163535f3c59e5587642fa2c725b00b17b7437c'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_sti.tif', - 'hash', - 'sha256', - '73c247fd2c7623e79145b6337846c97d14d1752d660b2f32245a4ccaac033350')], - - # t_process[crc] recording: - 'crc': - [('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_crc.tif', - 'hash', - 'sha256', - '07debf171aedfeba7d6c1e3220ff8e4a2a7b1033e68c72bbc121b500b6ed3c9c'), - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_crc.tif', - 'hash', - 'sha256', - 'a7b61aabc950be7a3040b6f9572bf755522b12ad27644583e603def05ac1980b'), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_crc.tif', - 'hash', - 'sha256', - '942efeeee2b020123be6d7334bbb3ef5995c93d2fd0247cb79948836865c8067')], -} - -expectations['modis']['satvi'] = [ - ('modis/tiles/h12v04/2012337/h12v04_2012337_MCD_satvi.tif', - 'raster', - 'gdalinfo-stats', - ['Driver: GTiff/GeoTIFF', - 'Size is 2400, 2400', - 'Coordinate System is:', - 'PROJCS["unnamed",', - ' GEOGCS["Unknown datum based upon the custom spheroid",', - ' DATUM["Not_specified_based_on_custom_spheroid",', - ' SPHEROID["Custom spheroid",6371007.181,0]],', - ' PRIMEM["Greenwich",0],', - ' UNIT["degree",0.0174532925199433]],', - ' PROJECTION["Sinusoidal"],', - ' PARAMETER["longitude_of_center",0],', - ' PARAMETER["false_easting",0],', - ' PARAMETER["false_northing",0],', - ' UNIT["metre",1,', - ' AUTHORITY["EPSG","9001"]]]', - 'Origin = (-6671703.117999999783933,5559752.598333000205457)', - 'Pixel Size = (463.312716527916507,-463.312716527500129)', - 'Metadata:', - ' AREA_OR_POINT=Area', - ' GIPS Version=0.8.2', - ' VERSION=1.0', - 'Image Structure Metadata:', - ' INTERLEAVE=BAND', - 'Corner Coordinates:', - 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', - 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', - 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', - 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', - 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', - 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', - ' Description = satvi', - ' Minimum=-8110.000, Maximum=6544.000, Mean=455.555, StdDev=1827.398', - ' NoData Value=-32768', - ' Unit Type: other', - ' Offset: 0, Scale:0.0001', - ' Metadata:', - ' STATISTICS_MAXIMUM=6544', - ' STATISTICS_MEAN=455.55465105991', - ' STATISTICS_MINIMUM=-8110', - ' STATISTICS_STDDEV=1827.3983061798']), - ('modis/tiles/h12v04/2012336/h12v04_2012336_MCD_satvi.tif', - 'raster', - 'gdalinfo-stats', - ['Driver: GTiff/GeoTIFF', - 'Size is 2400, 2400', - 'Coordinate System is:', - 'PROJCS["unnamed",', - ' GEOGCS["Unknown datum based upon the custom spheroid",', - ' DATUM["Not_specified_based_on_custom_spheroid",', - ' SPHEROID["Custom spheroid",6371007.181,0]],', - ' PRIMEM["Greenwich",0],', - ' UNIT["degree",0.0174532925199433]],', - ' PROJECTION["Sinusoidal"],', - ' PARAMETER["longitude_of_center",0],', - ' PARAMETER["false_easting",0],', - ' PARAMETER["false_northing",0],', - ' UNIT["metre",1,', - ' AUTHORITY["EPSG","9001"]]]', - 'Origin = (-6671703.117999999783933,5559752.598333000205457)', - 'Pixel Size = (463.312716527916507,-463.312716527500129)', - 'Metadata:', - ' AREA_OR_POINT=Area', - ' GIPS Version=0.8.2', - ' VERSION=1.0', - 'Image Structure Metadata:', - ' INTERLEAVE=BAND', - 'Corner Coordinates:', - 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', - 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', - 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', - 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', - 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', - 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', - ' Description = satvi', - ' Minimum=-8145.000, Maximum=6597.000, Mean=531.460, StdDev=1728.167', - ' NoData Value=-32768', - ' Unit Type: other', - ' Offset: 0, Scale:0.0001', - ' Metadata:', - ' STATISTICS_MAXIMUM=6597', - ' STATISTICS_MEAN=531.45965604069', - ' STATISTICS_MINIMUM=-8145', - ' STATISTICS_STDDEV=1728.166878798']), - ('modis/tiles/h12v04/2012338/h12v04_2012338_MCD_satvi.tif', - 'raster', - 'gdalinfo-stats', - ['Driver: GTiff/GeoTIFF', - 'Size is 2400, 2400', - 'Coordinate System is:', - 'PROJCS["unnamed",', - ' GEOGCS["Unknown datum based upon the custom spheroid",', - ' DATUM["Not_specified_based_on_custom_spheroid",', - ' SPHEROID["Custom spheroid",6371007.181,0]],', - ' PRIMEM["Greenwich",0],', - ' UNIT["degree",0.0174532925199433]],', - ' PROJECTION["Sinusoidal"],', - ' PARAMETER["longitude_of_center",0],', - ' PARAMETER["false_easting",0],', - ' PARAMETER["false_northing",0],', - ' UNIT["metre",1,', - ' AUTHORITY["EPSG","9001"]]]', - 'Origin = (-6671703.117999999783933,5559752.598333000205457)', - 'Pixel Size = (463.312716527916507,-463.312716527500129)', - 'Metadata:', - ' AREA_OR_POINT=Area', - ' GIPS Version=0.8.2', - ' VERSION=1.0', - 'Image Structure Metadata:', - ' INTERLEAVE=BAND', - 'Corner Coordinates:', - 'Upper Left (-6671703.118, 5559752.598) ( 93d20\'36.35"W, 50d 0\' 0.00"N)', - 'Lower Left (-6671703.118, 4447802.079) ( 78d19\'27.97"W, 40d 0\' 0.00"N)', - 'Upper Right (-5559752.598, 5559752.598) ( 77d47\'10.29"W, 50d 0\' 0.00"N)', - 'Lower Right (-5559752.598, 4447802.079) ( 65d16\'13.31"W, 40d 0\' 0.00"N)', - 'Center (-6115727.858, 5003777.339) ( 77d46\'54.29"W, 45d 0\' 0.00"N)', - 'Band 1 Block=2400x1 Type=Int16, ColorInterp=Gray', - ' Description = satvi', - ' Minimum=-8073.000, Maximum=6491.000, Mean=480.431, StdDev=1801.390', - ' NoData Value=-32768', - ' Unit Type: other', - ' Offset: 0, Scale:0.0001', - ' Metadata:', - ' STATISTICS_MAXIMUM=6491', - ' STATISTICS_MEAN=480.43080960083', - ' STATISTICS_MINIMUM=-8073', - ' STATISTICS_STDDEV=1801.3898148172'])] +expectations['modis'] = modis_process.expectations expectations['merra'] = { # test this too? 'frland': [], From f723b6159ded0327b242298bdafddb81de4a8dc4 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 10 Jan 2018 17:16:45 -0500 Subject: [PATCH 39/44] 463 convert merra to gdalinfo -stats comparator --- gips/test/sys/expected/merra_process.py | 353 ++++++++++++++++++++++++ gips/test/sys/expected/std_process.py | 60 +--- 2 files changed, 355 insertions(+), 58 deletions(-) create mode 100644 gips/test/sys/expected/merra_process.py diff --git a/gips/test/sys/expected/merra_process.py b/gips/test/sys/expected/merra_process.py new file mode 100644 index 00000000..40250aa0 --- /dev/null +++ b/gips/test/sys/expected/merra_process.py @@ -0,0 +1,353 @@ +# test this too? 'frland': +expectations = { + # t_process[srad] recording: + 'srad': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_srad.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 576, 361', + 'Coordinate System is:', + 'GEOGCS["WGS 84",', + ' DATUM["WGS_1984",', + ' SPHEROID["WGS 84",6378137,298.257223563,', + ' AUTHORITY["EPSG","7030"]],', + ' AUTHORITY["EPSG","6326"]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433],', + ' AUTHORITY["EPSG","4326"]]', + 'Origin = (-180.312500000000000,90.250000000000000)', + 'Pixel Size = (0.625000000000000,-0.500000000000000)', + 'Metadata:', + ' AREA_OR_POINT=Area', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-180.3125000, 90.2500000) (180d18\'45.00"W, 90d15\' 0.00"N)', + 'Lower Left (-180.3125000, -90.2500000) (180d18\'45.00"W, 90d15\' 0.00"S)', + 'Upper Right ( 179.6875000, 90.2500000) (179d41\'15.00"E, 90d15\' 0.00"N)', + 'Lower Right ( 179.6875000, -90.2500000) (179d41\'15.00"E, 90d15\' 0.00"S)', + 'Center ( -0.3125000, 0.0000000) ( 0d18\'45.00"W, 0d 0\' 0.01"N)', + 'Band 1 Block=576x3 Type=Float32, ColorInterp=Gray', + ' Description = srad', + ' Minimum=0.000, Maximum=393.073, Mean=158.767, StdDev=109.978', + ' NoData Value=999999986991104', + ' Unit Type: W m-2', + ' Metadata:', + ' STATISTICS_MAXIMUM=393.07275390625', + ' STATISTICS_MEAN=158.76661446295', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=109.97764298059'])], + + # t_process[tave] recording: + 'tave': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tave.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 576, 361', + 'Coordinate System is:', + 'GEOGCS["WGS 84",', + ' DATUM["WGS_1984",', + ' SPHEROID["WGS 84",6378137,298.257223563,', + ' AUTHORITY["EPSG","7030"]],', + ' AUTHORITY["EPSG","6326"]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433],', + ' AUTHORITY["EPSG","4326"]]', + 'Origin = (-180.312500000000000,90.250000000000000)', + 'Pixel Size = (0.625000000000000,-0.500000000000000)', + 'Metadata:', + ' AREA_OR_POINT=Area', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-180.3125000, 90.2500000) (180d18\'45.00"W, 90d15\' 0.00"N)', + 'Lower Left (-180.3125000, -90.2500000) (180d18\'45.00"W, 90d15\' 0.00"S)', + 'Upper Right ( 179.6875000, 90.2500000) (179d41\'15.00"E, 90d15\' 0.00"N)', + 'Lower Right ( 179.6875000, -90.2500000) (179d41\'15.00"E, 90d15\' 0.00"S)', + 'Center ( -0.3125000, 0.0000000) ( 0d18\'45.00"W, 0d 0\' 0.01"N)', + 'Band 1 Block=576x3 Type=Float32, ColorInterp=Gray', + ' Description = tave', + ' Minimum=-73.719, Maximum=38.004, Mean=5.386, StdDev=23.070', + ' NoData Value=999999986991104', + ' Unit Type: C', + ' Metadata:', + ' STATISTICS_MAXIMUM=38.004295349121', + ' STATISTICS_MEAN=5.3864481837031', + ' STATISTICS_MINIMUM=-73.719398498535', + ' STATISTICS_STDDEV=23.069743246635'])], + + # t_process[prcp] recording: + 'prcp': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_prcp.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 576, 361', + 'Coordinate System is:', + 'GEOGCS["WGS 84",', + ' DATUM["WGS_1984",', + ' SPHEROID["WGS 84",6378137,298.257223563,', + ' AUTHORITY["EPSG","7030"]],', + ' AUTHORITY["EPSG","6326"]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433],', + ' AUTHORITY["EPSG","4326"]]', + 'Origin = (-180.312500000000000,90.250000000000000)', + 'Pixel Size = (0.625000000000000,-0.500000000000000)', + 'Metadata:', + ' AREA_OR_POINT=Area', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-180.3125000, 90.2500000) (180d18\'45.00"W, 90d15\' 0.00"N)', + 'Lower Left (-180.3125000, -90.2500000) (180d18\'45.00"W, 90d15\' 0.00"S)', + 'Upper Right ( 179.6875000, 90.2500000) (179d41\'15.00"E, 90d15\' 0.00"N)', + 'Lower Right ( 179.6875000, -90.2500000) (179d41\'15.00"E, 90d15\' 0.00"S)', + 'Center ( -0.3125000, 0.0000000) ( 0d18\'45.00"W, 0d 0\' 0.01"N)', + 'Band 1 Block=576x3 Type=Float32, ColorInterp=Gray', + ' Description = prcp', + ' Minimum=0.000, Maximum=381.482, Mean=2.516, StdDev=7.518', + ' NoData Value=999999986991104', + ' Unit Type: mm d-1', + ' Metadata:', + ' STATISTICS_MAXIMUM=381.48220825195', + ' STATISTICS_MEAN=2.5160472972269', + ' STATISTICS_MINIMUM=0', + ' STATISTICS_STDDEV=7.5175707208826'])], + + # t_process[rhum] recording: + 'rhum': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_rhum.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 576, 361', + 'Coordinate System is:', + 'GEOGCS["WGS 84",', + ' DATUM["WGS_1984",', + ' SPHEROID["WGS 84",6378137,298.257223563,', + ' AUTHORITY["EPSG","7030"]],', + ' AUTHORITY["EPSG","6326"]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433],', + ' AUTHORITY["EPSG","4326"]]', + 'Origin = (-180.312500000000000,90.250000000000000)', + 'Pixel Size = (0.625000000000000,-0.500000000000000)', + 'Metadata:', + ' AREA_OR_POINT=Area', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-180.3125000, 90.2500000) (180d18\'45.00"W, 90d15\' 0.00"N)', + 'Lower Left (-180.3125000, -90.2500000) (180d18\'45.00"W, 90d15\' 0.00"S)', + 'Upper Right ( 179.6875000, 90.2500000) (179d41\'15.00"E, 90d15\' 0.00"N)', + 'Lower Right ( 179.6875000, -90.2500000) (179d41\'15.00"E, 90d15\' 0.00"S)', + 'Center ( -0.3125000, 0.0000000) ( 0d18\'45.00"W, 0d 0\' 0.01"N)', + 'Band 1 Block=576x3 Type=Float32, ColorInterp=Gray', + ' Description = rhum', + ' Minimum=4.749, Maximum=100.000, Mean=76.320, StdDev=15.480', + ' NoData Value=999999986991104', + ' Unit Type: %', + ' Metadata:', + ' STATISTICS_MAXIMUM=100', + ' STATISTICS_MEAN=76.320144796768', + ' STATISTICS_MINIMUM=4.7492995262146', + ' STATISTICS_STDDEV=15.479816083412'])], + + # t_process[tmin] recording: + 'tmin': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmin.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 576, 361', + 'Coordinate System is:', + 'GEOGCS["WGS 84",', + ' DATUM["WGS_1984",', + ' SPHEROID["WGS 84",6378137,298.257223563,', + ' AUTHORITY["EPSG","7030"]],', + ' AUTHORITY["EPSG","6326"]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433],', + ' AUTHORITY["EPSG","4326"]]', + 'Origin = (-180.312500000000000,90.250000000000000)', + 'Pixel Size = (0.625000000000000,-0.500000000000000)', + 'Metadata:', + ' AREA_OR_POINT=Area', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-180.3125000, 90.2500000) (180d18\'45.00"W, 90d15\' 0.00"N)', + 'Lower Left (-180.3125000, -90.2500000) (180d18\'45.00"W, 90d15\' 0.00"S)', + 'Upper Right ( 179.6875000, 90.2500000) (179d41\'15.00"E, 90d15\' 0.00"N)', + 'Lower Right ( 179.6875000, -90.2500000) (179d41\'15.00"E, 90d15\' 0.00"S)', + 'Center ( -0.3125000, 0.0000000) ( 0d18\'45.00"W, 0d 0\' 0.01"N)', + 'Band 1 Block=576x3 Type=Float32, ColorInterp=Gray', + ' Description = tmin', + ' Minimum=-74.883, Maximum=32.507, Mean=3.387, StdDev=23.182', + ' NoData Value=999999986991104', + ' Unit Type: C', + ' Metadata:', + ' STATISTICS_MAXIMUM=32.507019042969', + ' STATISTICS_MEAN=3.3873471134296', + ' STATISTICS_MINIMUM=-74.882934570312', + ' STATISTICS_STDDEV=23.182138404762'])], + + # t_process[tmax] recording: + 'tmax': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmax.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 576, 361', + 'Coordinate System is:', + 'GEOGCS["WGS 84",', + ' DATUM["WGS_1984",', + ' SPHEROID["WGS 84",6378137,298.257223563,', + ' AUTHORITY["EPSG","7030"]],', + ' AUTHORITY["EPSG","6326"]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433],', + ' AUTHORITY["EPSG","4326"]]', + 'Origin = (-180.312500000000000,90.250000000000000)', + 'Pixel Size = (0.625000000000000,-0.500000000000000)', + 'Metadata:', + ' AREA_OR_POINT=Area', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-180.3125000, 90.2500000) (180d18\'45.00"W, 90d15\' 0.00"N)', + 'Lower Left (-180.3125000, -90.2500000) (180d18\'45.00"W, 90d15\' 0.00"S)', + 'Upper Right ( 179.6875000, 90.2500000) (179d41\'15.00"E, 90d15\' 0.00"N)', + 'Lower Right ( 179.6875000, -90.2500000) (179d41\'15.00"E, 90d15\' 0.00"S)', + 'Center ( -0.3125000, 0.0000000) ( 0d18\'45.00"W, 0d 0\' 0.01"N)', + 'Band 1 Block=576x3 Type=Float32, ColorInterp=Gray', + ' Description = tmax', + ' Minimum=-72.602, Maximum=46.648, Mean=7.511, StdDev=23.176', + ' NoData Value=999999986991104', + ' Unit Type: C', + ' Metadata:', + ' STATISTICS_MAXIMUM=46.648315429688', + ' STATISTICS_MEAN=7.5107629172946', + ' STATISTICS_MINIMUM=-72.602355957031', + ' STATISTICS_STDDEV=23.176162684877'])], + + # t_process[shum] recording: + 'shum': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_shum.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 576, 361', + 'Coordinate System is:', + 'GEOGCS["WGS 84",', + ' DATUM["WGS_1984",', + ' SPHEROID["WGS 84",6378137,298.257223563,', + ' AUTHORITY["EPSG","7030"]],', + ' AUTHORITY["EPSG","6326"]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433],', + ' AUTHORITY["EPSG","4326"]]', + 'Origin = (-180.312500000000000,90.250000000000000)', + 'Pixel Size = (0.625000000000000,-0.500000000000000)', + 'Metadata:', + ' AREA_OR_POINT=Area', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-180.3125000, 90.2500000) (180d18\'45.00"W, 90d15\' 0.00"N)', + 'Lower Left (-180.3125000, -90.2500000) (180d18\'45.00"W, 90d15\' 0.00"S)', + 'Upper Right ( 179.6875000, 90.2500000) (179d41\'15.00"E, 90d15\' 0.00"N)', + 'Lower Right ( 179.6875000, -90.2500000) (179d41\'15.00"E, 90d15\' 0.00"S)', + 'Center ( -0.3125000, 0.0000000) ( 0d18\'45.00"W, 0d 0\' 0.01"N)', + 'Band 1 Block=576x3 Type=Float32, ColorInterp=Gray', + ' Description = shum', + ' Minimum=0.000, Maximum=0.025, Mean=0.008, StdDev=0.006', + ' NoData Value=999999986991104', + ' Unit Type: kg kg-1', + ' Metadata:', + ' STATISTICS_MAXIMUM=0.024637559428811', + ' STATISTICS_MEAN=0.0077882960847807', + ' STATISTICS_MINIMUM=1.7724577219269e-06', + ' STATISTICS_STDDEV=0.0064740740563288'])], + + # t_process[patm] recording: + 'patm': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_patm.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 576, 361', + 'Coordinate System is:', + 'GEOGCS["WGS 84",', + ' DATUM["WGS_1984",', + ' SPHEROID["WGS 84",6378137,298.257223563,', + ' AUTHORITY["EPSG","7030"]],', + ' AUTHORITY["EPSG","6326"]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433],', + ' AUTHORITY["EPSG","4326"]]', + 'Origin = (-180.312500000000000,90.250000000000000)', + 'Pixel Size = (0.625000000000000,-0.500000000000000)', + 'Metadata:', + ' AREA_OR_POINT=Area', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-180.3125000, 90.2500000) (180d18\'45.00"W, 90d15\' 0.00"N)', + 'Lower Left (-180.3125000, -90.2500000) (180d18\'45.00"W, 90d15\' 0.00"S)', + 'Upper Right ( 179.6875000, 90.2500000) (179d41\'15.00"E, 90d15\' 0.00"N)', + 'Lower Right ( 179.6875000, -90.2500000) (179d41\'15.00"E, 90d15\' 0.00"S)', + 'Center ( -0.3125000, 0.0000000) ( 0d18\'45.00"W, 0d 0\' 0.01"N)', + 'Band 1 Block=576x3 Type=Float32, ColorInterp=Gray', + ' Description = patm', + ' Minimum=519.850, Maximum=1040.560, Mean=965.135, StdDev=98.550', + ' NoData Value=999999986991104', + ' Unit Type: mb', + ' Metadata:', + ' STATISTICS_MAXIMUM=1040.5604248047', + ' STATISTICS_MEAN=965.1346876829', + ' STATISTICS_MINIMUM=519.85034179688', + ' STATISTICS_STDDEV=98.549792183355'])], + + # t_process[wind] recording: + 'wind': + [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_wind.tif', + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 576, 361', + 'Coordinate System is:', + 'GEOGCS["WGS 84",', + ' DATUM["WGS_1984",', + ' SPHEROID["WGS 84",6378137,298.257223563,', + ' AUTHORITY["EPSG","7030"]],', + ' AUTHORITY["EPSG","6326"]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433],', + ' AUTHORITY["EPSG","4326"]]', + 'Origin = (-180.312500000000000,90.250000000000000)', + 'Pixel Size = (0.625000000000000,-0.500000000000000)', + 'Metadata:', + ' AREA_OR_POINT=Area', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-180.3125000, 90.2500000) (180d18\'45.00"W, 90d15\' 0.00"N)', + 'Lower Left (-180.3125000, -90.2500000) (180d18\'45.00"W, 90d15\' 0.00"S)', + 'Upper Right ( 179.6875000, 90.2500000) (179d41\'15.00"E, 90d15\' 0.00"N)', + 'Lower Right ( 179.6875000, -90.2500000) (179d41\'15.00"E, 90d15\' 0.00"S)', + 'Center ( -0.3125000, 0.0000000) ( 0d18\'45.00"W, 0d 0\' 0.01"N)', + 'Band 1 Block=576x3 Type=Float32, ColorInterp=Gray', + ' Description = wind', + ' Minimum=0.580, Maximum=27.184, Mean=7.922, StdDev=3.816', + ' NoData Value=999999986991104', + ' Unit Type: m s-1', + ' Metadata:', + ' STATISTICS_MAXIMUM=27.184354782104', + ' STATISTICS_MEAN=7.9216990912713', + ' STATISTICS_MINIMUM=0.58011025190353', + ' STATISTICS_STDDEV=3.8163160207956'])], +} diff --git a/gips/test/sys/expected/std_process.py b/gips/test/sys/expected/std_process.py index 2c9a84c2..9c8702ca 100644 --- a/gips/test/sys/expected/std_process.py +++ b/gips/test/sys/expected/std_process.py @@ -4,69 +4,13 @@ from .. import util from . import modis_process +from . import merra_process expectations = {} mark_spec = {} expectations['modis'] = modis_process.expectations - -expectations['merra'] = { - # test this too? 'frland': [], - # t_process[srad] recording: - 'srad': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_srad.tif', - 'hash', - 'sha256', - '882a44af70e337bf905c8695936f101792a518507a4304b23781a2f30aaabab4')], - # t_process[tave] recording: - 'tave': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tave.tif', - 'hash', - 'sha256', - '5185aebd7cda54157cad2ddbde9f6fec4871c1b85fe78d0738634ba0211f2c9b')], - # t_process[shum] recording: - 'shum': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_shum.tif', - 'hash', - 'sha256', - '558f567c1e931891553d396f81b2b90929ad48a0ae88fa95d3894eae23ab3eba')], - # t_process[rhum] recording: - 'rhum': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_rhum.tif', - 'hash', - 'sha256', - '058161eb8488f1fb3df7f5c8719833c9d68c56aa082d4605938f62c2ee0035f6')], - # t_process[tmin] recording: - 'tmin': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmin.tif', - 'hash', - 'sha256', - '35a850769a3bb8f5209574ea4835bf417d8f99024567f4af9cdee9e623d0c567')], - # t_process[tmax] recording: - 'tmax': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_tmax.tif', - 'hash', - 'sha256', - '5963fcf346f388fc347355153996a078f1eb5ecbbc094394b72be382ba491865')], - # t_process[prcp] recording: - 'prcp': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_prcp.tif', - 'hash', - 'sha256', - '348fd7072ef0a8625f5268e29acbf3a69a959a412327102ef6558c992e62bc9f')], - # t_process[patm] recording: - 'patm': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_patm.tif', - 'hash', - 'sha256', - '7d9deca613973cb8ffb8f15a3dfa2013e6783556b520915e89b97fe41de638e3')], - # t_process[wind] recording: - 'wind': - [('merra/tiles/h01v01/2015135/h01v01_2015135_merra_wind.tif', - 'hash', - 'sha256', - 'a90c1d87fdb1c0926bd5b3004408a1f9d42ef08c38deb9d0572bc7536dbbf08a')], -} +expectations['merra'] = merra_process.expectations expectations['prism'] = { # t_process[tmin] recording: From 9d96c4fedf7feb2cf721b2afc28c41d4aa8e4fe5 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Wed, 10 Jan 2018 17:44:21 -0500 Subject: [PATCH 40/44] 463 convert prism to gdalinfo -stats comparator --- gips/test/sys/expected/std_process.py | 65 ++++++++++++++++++++------- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/gips/test/sys/expected/std_process.py b/gips/test/sys/expected/std_process.py index 9c8702ca..5334a01c 100644 --- a/gips/test/sys/expected/std_process.py +++ b/gips/test/sys/expected/std_process.py @@ -12,9 +12,9 @@ expectations['modis'] = modis_process.expectations expectations['merra'] = merra_process.expectations -expectations['prism'] = { +expectations['prism'] = collections.OrderedDict([ # t_process[tmin] recording: - 'tmin': + ('tmin', [('prism/tiles/CONUS/19821201/CONUS_19821201_prism_tmin.tif', 'symlink', '/vsizip/', @@ -26,28 +26,28 @@ ('prism/tiles/CONUS/19821203/CONUS_19821203_prism_tmin.tif', 'symlink', '/vsizip/', - '/prism/tiles/CONUS/19821203/PRISM_tmin_stable_4kmD1_19821203_bil.zip/PRISM_tmin_stable_4kmD1_19821203_bil.bil')], + '/prism/tiles/CONUS/19821203/PRISM_tmin_stable_4kmD1_19821203_bil.zip/PRISM_tmin_stable_4kmD1_19821203_bil.bil')]), # t_process[tmax] recording: - 'tmax': - [('prism/tiles/CONUS/19821203/CONUS_19821203_prism_tmax.tif', + ('tmax', + [('prism/tiles/CONUS/19821202/CONUS_19821202_prism_tmax.tif', 'symlink', '/vsizip/', - '/prism/tiles/CONUS/19821203/PRISM_tmax_stable_4kmD1_19821203_bil.zip/PRISM_tmax_stable_4kmD1_19821203_bil.bil'), - ('prism/tiles/CONUS/19821201/CONUS_19821201_prism_tmax.tif', + '/prism/tiles/CONUS/19821202/PRISM_tmax_stable_4kmD1_19821202_bil.zip/PRISM_tmax_stable_4kmD1_19821202_bil.bil'), + ('prism/tiles/CONUS/19821203/CONUS_19821203_prism_tmax.tif', 'symlink', '/vsizip/', - '/prism/tiles/CONUS/19821201/PRISM_tmax_stable_4kmD1_19821201_bil.zip/PRISM_tmax_stable_4kmD1_19821201_bil.bil'), - ('prism/tiles/CONUS/19821202/CONUS_19821202_prism_tmax.tif', + '/prism/tiles/CONUS/19821203/PRISM_tmax_stable_4kmD1_19821203_bil.zip/PRISM_tmax_stable_4kmD1_19821203_bil.bil'), + ('prism/tiles/CONUS/19821201/CONUS_19821201_prism_tmax.tif', 'symlink', '/vsizip/', - '/prism/tiles/CONUS/19821202/PRISM_tmax_stable_4kmD1_19821202_bil.zip/PRISM_tmax_stable_4kmD1_19821202_bil.bil')], + '/prism/tiles/CONUS/19821201/PRISM_tmax_stable_4kmD1_19821201_bil.zip/PRISM_tmax_stable_4kmD1_19821201_bil.bil')]), # IMPORTANT NOTE pptsum seems to generate ppt products as part of # its function; as a result ppt products may exist already if pptsum # goes first. # t_process[ppt] recording: - 'ppt': + ('ppt', [('prism/tiles/CONUS/19821203/CONUS_19821203_prism_ppt.tif', 'symlink', '/vsizip/', @@ -59,15 +59,46 @@ ('prism/tiles/CONUS/19821202/CONUS_19821202_prism_ppt.tif', 'symlink', '/vsizip/', - '/prism/tiles/CONUS/19821202/PRISM_ppt_stable_4kmD2_19821202_bil.zip/PRISM_ppt_stable_4kmD2_19821202_bil.bil')], + '/prism/tiles/CONUS/19821202/PRISM_ppt_stable_4kmD2_19821202_bil.zip/PRISM_ppt_stable_4kmD2_19821202_bil.bil')]), # t_process[pptsum] recording: - 'pptsum': + ('pptsum', [('prism/tiles/CONUS/19821203/CONUS_19821203_prism_pptsum-3.tif', - 'hash', - 'sha256', - 'd9772d10372ad86550dacae3d5645837cd9a10228db70f4e2bc7149b836495d9')], -} + 'raster', + 'gdalinfo-stats', + ['Driver: GTiff/GeoTIFF', + 'Size is 1405, 621', + 'Coordinate System is:', + 'GEOGCS["NAD83",', + ' DATUM["North_American_Datum_1983",', + ' SPHEROID["GRS 1980",6378137,298.2572221010042,', + ' AUTHORITY["EPSG","7019"]],', + ' AUTHORITY["EPSG","6269"]],', + ' PRIMEM["Greenwich",0],', + ' UNIT["degree",0.0174532925199433],', + ' AUTHORITY["EPSG","4269"]]', + 'Origin = (-125.020833333333329,49.937500000002032)', + 'Pixel Size = (0.041666666666670,-0.041666666666670)', + 'Metadata:', + ' AREA_OR_POINT=Area', + 'Image Structure Metadata:', + ' INTERLEAVE=BAND', + 'Corner Coordinates:', + 'Upper Left (-125.0208333, 49.9375000) (125d 1\'15.00"W, 49d56\'15.00"N)', + 'Lower Left (-125.0208333, 24.0625000) (125d 1\'15.00"W, 24d 3\'45.00"N)', + 'Upper Right ( -66.4791667, 49.9375000) ( 66d28\'45.00"W, 49d56\'15.00"N)', + 'Lower Right ( -66.4791667, 24.0625000) ( 66d28\'45.00"W, 24d 3\'45.00"N)', + 'Center ( -95.7500000, 37.0000000) ( 95d45\' 0.00"W, 37d 0\' 0.00"N)', + 'Band 1 Block=1405x1 Type=Float32, ColorInterp=Gray', + ' Description = Cumulative Precipitate(3 day window)', + ' Minimum=-29997.000, Maximum=332.600, Mean=-13428.782, StdDev=14925.792', + ' NoData Value=-9999', + ' Metadata:', + ' STATISTICS_MAXIMUM=332.60000610352', + ' STATISTICS_MEAN=-13428.782340994', + ' STATISTICS_MINIMUM=-29997', + ' STATISTICS_STDDEV=14925.791999964'])]), +]) mark_spec['sentinel2'] = util.slow From 0460844db03fc3939cba1003dd581a894bf8dc25 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Thu, 11 Jan 2018 10:19:15 -0500 Subject: [PATCH 41/44] 463 et al code review feedback --- gips/test/sys/driver_setup.py | 3 --- gips/test/sys/t_stats.py | 1 - gips/test/sys/util.py | 8 ++++++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/gips/test/sys/driver_setup.py b/gips/test/sys/driver_setup.py index 5e4ac257..dcd44426 100644 --- a/gips/test/sys/driver_setup.py +++ b/gips/test/sys/driver_setup.py @@ -79,7 +79,4 @@ def setup_repo_data(driver): print('Downloading', driver, 'data . . .') args = STD_ARGS[driver] + ('--fetch',) outcome = sh.Command('gips_inventory')(*args) - if outcome.exit_code != 0: - raise RuntimeError(driver + " data setup via `gips_inventory` failed", - outcome) print(driver, "data download complete.") diff --git a/gips/test/sys/t_stats.py b/gips/test/sys/t_stats.py index 72009cfc..212a3e6c 100644 --- a/gips/test/sys/t_stats.py +++ b/gips/test/sys/t_stats.py @@ -22,7 +22,6 @@ def t_stats(export_wrapper, driver, product): args = driver_setup.STD_ARGS[driver] + ('--res', '100', '100', '--notld', '--outdir', working_dir, '-p', product) outcome = sh.gips_project(*args) - assert outcome.exit_code == 0 # sanity check # compute & confirm stats outcome, actual = runner('gips_stats', working_dir, '-p', product) diff --git a/gips/test/sys/util.py b/gips/test/sys/util.py index 3130bef0..930bcdf3 100644 --- a/gips/test/sys/util.py +++ b/gips/test/sys/util.py @@ -402,15 +402,23 @@ def wrapped_runner(cmd_string, *args): @pytest.yield_fixture def repo_wrapper(request): + """Fixture for tests that operate mostly on the data repo.""" + # This is how you wrap a generator in python: + # https://stackoverflow.com/questions/11197186/ for rv in sys_test_wrapper(request, DATA_REPO_ROOT): yield rv @pytest.yield_fixture def export_wrapper(request): + """Fixture for tests that operate on an output directory. + + Makes a working directory that nests inside the OUTPUT_DIR.""" driver = request.node.callspec.params['driver'] product = request.node.callspec.params['product'] working_dir = os.path.join(OUTPUT_DIR, '{}-{}'.format(driver, product)) os.makedirs(working_dir) # raises if leaf dir exists; this is desired + # This is how you wrap a generator in python: + # https://stackoverflow.com/questions/11197186/ for rv in sys_test_wrapper(request, working_dir): yield rv + (working_dir,) From c2740b5e90819a1ad1e6e4d3b4602dbb2d7fa15b Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Thu, 11 Jan 2018 11:52:11 -0500 Subject: [PATCH 42/44] 463 et al update README with howto for new pattern --- gips/test/README.md | 185 ++++++++++++++++++++++++-------------------- 1 file changed, 101 insertions(+), 84 deletions(-) diff --git a/gips/test/README.md b/gips/test/README.md index 5ec33370..30687c7f 100644 --- a/gips/test/README.md +++ b/gips/test/README.md @@ -17,18 +17,25 @@ norecursedirs = .venv data-root data-repo = /home/your-user-name-here/src/gips/data-repo # a directory of your choice, used for output from, e.g., gips_project output-dir = /home/your-user-name-here/src/gips/testout + # These should remain as they are (placing them in a committed file is a TODO): python_functions = t_ python_classes = T_ python_files = t_*.py DJANGO_SETTINGS_MODULE=gips.inventory.orm.settings + +# config for artifact store +artifact-store-user = ask +artifact-store-password = for +artifact-store-host = these +artifact-store-path = values ``` Library Dependencies -------------------- GIPS automated testing uses a few libraries, mainly pytest and mock, and a library for gluing them together (pytest-mock). These are installed by -`install.sh`. Further reading: +the gips installation process. Further reading: * Pytest: http://docs.pytest.org/en/latest/index.html * Mock: http://www.voidspace.org.uk/python/mock/ @@ -39,8 +46,16 @@ Test selection -------------- Running unit tests is straightforward, as is selecting specific tests: +https://pytest.org/latest/usage.html#specifying-tests-selecting-tests + +You can also select tests based on mark (marks are tags for tests): + +https://pytest.org/latest/example/markers.html + +Examples: + ``` -py.test # only runs fast tests; all unit tests at the moment +py.test # only runs non-system tests (should be fast) py.test -k inventory # use py.test's -k to select tests conveniently: ========================= test session starts ========================= platform linux2 -- Python 2.7.11+, pytest-2.9.2, py-1.4.31, pluggy-0.3.1 @@ -60,19 +75,6 @@ gips/test/unit/t_inventory_settings.py .. ======== 4 passed, 4 skipped, 104 deselected in 0.51 seconds ========== ``` -Select specific tests per usual for pytest: - -https://pytest.org/latest/usage.html#specifying-tests-selecting-tests - -You can also select tests based on mark (marks are tags for tests): - -https://pytest.org/latest/example/markers.html - -And reading about fixtures is especially recommended as the GIPS test suite -uses them substantially: - -http://pytest.org/latest/fixture.html - **Important caveat:** If you specify the same test file multiple times on the command line (say, to specify multiple tests in that file), any module-scoped test fixtures will run once for each time the file is listed, which may not be @@ -82,6 +84,11 @@ mulitple times. Instead, avoid the problem by using `-k` to specify tests. Writing Unit Tests with Mocking =============================== +First, reading about fixtures is especially recommended as the GIPS test suite +uses them substantially: + +http://pytest.org/latest/fixture.html + Automated testing can be difficult due to the code's interaction with the outside. Some interactions may be undesirable for testing purposes, such as access to an RNG or the network. Others may be important to observe for @@ -122,19 +129,16 @@ https://pytest-django.readthedocs.io/en/latest/ Running System Tests ==================== Each test runs `gips_something` commands, usually just one, as subprocesses. -The test harness captures the command's exit status, standard output, and -standard error. In addition, the filesystem is observed before and after the -test, and any created, deleted, or updated files are noted. These observations -are then compared with known correct values to see if the test ought to pass. -Finally any cleanup actions occur, which may include removing created files to -leave the filesystem in a pristine state. - -System tests require a little setup, unfortunately: Edit `settings.py` and set -`OVERRIDE_VERSION` to the version the system tests expect. You can check this -by looking in `gips/test/sys/expected/modis.py` and checking on the version -that is expected to be output. You should also have a correct `pytest.ini` -file (see the top-level README.md). Back up any data files you want to retain -in your data repo directory; some system tests need to operate destructively on +The test harness observes the filesystem before and after the test, and any +created files are observed. These observations are then compared with known +correct values to see if the test ought to pass. + +System tests require a little setup, unfortunately: Edit `settings.py` and +set `OVERRIDE_VERSION` to the version the system tests expect. You can check +this by looking in `gips/test/sys/expected/modis.py` and checking on the +version that is expected to be output. You should also have a correct +`pytest.ini` file (see above). Back up any data files you want to retain in +your data repo directory; some system tests need to operate destructively on it. It's not normal to retain state in between test runs, but the pattern of GIPS @@ -154,6 +158,9 @@ py.test --sys --clear-repo # delete the entire repo, then fetch data files, py.test --ll debug -s -x -vv --sys --setup-repo -k prism ``` +Note that specifying the log level with `--ll` and `--log-level` is supported +for some tests, but may be deprecated for new tests. + Also some specially-marked tests are skipped unless command-line options are given; this is for performance and safety reasons. The repo-altering tests, mostly tests of `gips_inventory --fetch`, will always be skipped without the @@ -163,78 +170,88 @@ that requires `--setup-repo` or `--clear-repo` for the other tests to pass. Writing System Tests: Testing a New Driver =========================================== -Each driver's tests are split in half; here are the files for modis: +A new driver should have system tests for `gips_process`, `gips_project`, and +`gips_stats`, though in the future minimum test coverage standards may evolve +further. + +Thanks to pytest's parametrization, one test function can support many +combinations of drivers and products. See docs for details: + +https://docs.pytest.org/en/latest/parametrize.html + +In particular, the three tests live here, but shouldn't need to be modified +to establish system testing for a new driver: -* `gips/test/sys/t_modis.py`: The tests themselves -* `gips/test/sys/expected/modis.py`: A file of known good outcomes, one per - test. +* `gips/test/sys/t_process.py` +* `gips/test/sys/t_project.py` +* `gips/test/sys/t_stats.py` -As a quick demonstration, edit a new test file, `gips/test/sys/t_$DRIVER.py`, -and add a test to it: +Instead, these files should be modified: + +* `gips/test/sys/expected/*_process.py`: Known-good outcomes for `t_process` +* `gips/test/sys/expected/*_project.py`: Known-good outcomes for `t_project` +* `gips/test/sys/expected/std_stats.py`: Known-good outcomes for `t_stats` +* `gips/test/sys/driver_setup.py`: Configuration & any special code the + driver may need. + +Be aware that an older infrastructure may exist in the gips system test suite, +that exists alongside this one. They interact minimally or not at all. + +Recipe for Testing a New Driver +------------------------------- +Say we're testing the new `granitesat` driver, which has the products `snow` +and `ice`. Open the file `gips/test/sys/driver_setup.py`, and add a line to +`STD_ARGS` to tell the test suite what scene(s) to operate on: ``` -from .util import * -def t_example(repo_env, expected): - actual = repo_env.run('echo "hello world!"') - assert expected == actual +STD_ARGS = { + # . . . various drivers listed here already . . . + 'granitesat': ('granitesat', '-s', nh_shp, '-d', '2018-001,2018-005', '-v4'), +} ``` -And an output expectation to a new expectations file, -`gips/test/sys/expected/$DRIVER.py`: +Then open the file `gips/test/sys/expected/std_process.py` and add these +lines; this tells the test suite what products to test: ``` -t_example = { 'stdout': """hello world!\n""" } +expectations['granitesat'] = { + 'snow': [], # the expectation list is intentionally empty for now . . . + 'ice': [], +} ``` -Now run the test (and only this test, to save time), and you should observe it -pass: +Now record expectations for the new driver's process testing: ``` -$ py.test -k t_example -============================= test session starts ============================== -platform linux2 -- Python 2.7.11+, pytest-2.9.2, py-1.4.31, pluggy-0.3.1 -rootdir: /home/tolson/src/gips, inifile: pytest.ini -plugins: mock-1.1, cov-2.2.1, timeout-1.0.0 -collected 23 items +$ pytest -s -vv gips/test/sys/t_process.py --sys -k granitesat --record=rec.py +``` -gips/test/sys/t_$DRIVER.py . +You should observe your `snow` and `ice` tests passing; in this case it means +a recording was made successfully for each product. These recordings are +saved to `rec.py`; open it and see what files were created during the +recording. Confirm that the right files were created, and remove any entries +that aren't important to the test (such as `.index` files). Then copy the +contents of `rec.py` to `gips/test/sys/expected/std_process.py`, replacing the +content of `expectations['granitesat']`. Then re-run process testing and +observe that the tests pass: -===================== 22 tests deselected by '-kt_example' ===================== -=================== 1 passed, 22 deselected in 0.42 seconds ==================== ``` +$ # Test runs presently don't clean up after themselves, so we have to do it: +$ find your-data-repo/granitesat/ -name '*.tif' -delete # vary as needed +$ # note no --record=rec.py this time: +$ pytest -s -vv gips/test/sys/t_process.py --sys -k granitesat +``` + +The process can be repeated for `t_project` and `t_stats`. Again, the test +suite doesn't perform cleanup, so remove your `OUTPUT_DIR` between test runs. + +If your new driver has many products, follow the pattern of `modis` and place +your expectations in a separate file, and import its content into eg +`std_process.py`. -In practice, copying the patterns in `t_modis.py` for testing a new driver can -be considered adequate. Also examine any deprecated shell script in -`gips/test`, as these are organized by driver and can supply needed test cases. -For a driver's source-altering tests, edit a single pair of files regardless of -driver: - -* `sys/t_repo_src_alteration.py`: For the tests -* `sys/expected/repo_src_alteration.py`: For the expectations. - -Working with Environment Fixtures and the Expectations File ------------------------------------------------------------ -For GIPS, system tests means observing command-line processes as they execute -and comparing their output to known-good values. These outputs only take two -forms: Standard streams and altering files in the filesystem (including -creating and deleting files). Some tests are expected to modify the data repo, -while others will modify a project directory. Choose an appropriate pytest -fixture accordingly, using existing driver tests as a guide. Pytest fixtures -are available to make this easier; see `gips/test/sys/util.py` for details. - -The `expected` fixture is somewhat special: It provides a convenient way to -store bulky data needed for tests in a separate file, without needing to -explicitly access the file itself nor configure anything. See -`gips/test/sys/expected/modis.py` for the way this works in practice: The -`expected` fixture looks for an object with the same name as the currently -running test. It then loads that dictionary into a `GipsProcResult` object, -which is convenient for making assertions in these system tests. - -The known-good values stored in `expected/` are usually captured from initial -test runs. For convenience, newly developed tests may be run with -`py.test -s --log-level debug --expectation-format`; parts of the output can -be cutpasted into an `expected/` file to establish these needed known-good -values. +If your driver needs additional system tests that don't follow the standard +parttern, you can place these in their own test file, eg: +`gips/test/sys/t_granitesat.py` Caveats & Tips ============== From ad653c54bc36ef6a0ebbe1b3acc69d9a9e499b96 Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Thu, 11 Jan 2018 12:50:00 -0500 Subject: [PATCH 43/44] 463 et al forgot to remove stale code --- gips/test/sys/t_aod.py | 15 --------------- gips/test/sys/t_sentinel2.py | 15 --------------- 2 files changed, 30 deletions(-) diff --git a/gips/test/sys/t_aod.py b/gips/test/sys/t_aod.py index 6d971149..e8459a0b 100644 --- a/gips/test/sys/t_aod.py +++ b/gips/test/sys/t_aod.py @@ -61,18 +61,3 @@ def t_tiles_copy(setup_fixture, clean_repo_env, output_tfe, expected): '--outdir', OUTPUT_DIR, '--notld') actual = output_tfe.run('gips_tiles', *args) assert expected == actual - - -def t_stats(setup_fixture, clean_repo_env, output_tfe, expected): - """Test gips_stats on projected files.""" - # generate data needed for stats computation - args = STD_ARGS + ('--res', '100', '100', '--outdir', OUTPUT_DIR, '--notld') - prep_run = output_tfe.run('gips_project', *args) - assert prep_run.exit_status == 0 # confirm it worked; not really in the test - - # compute stats - gtfe = GipsTestFileEnv(OUTPUT_DIR, start_clear=False) - actual = gtfe.run('gips_stats', OUTPUT_DIR) - - # check for correct stats content - assert expected == actual diff --git a/gips/test/sys/t_sentinel2.py b/gips/test/sys/t_sentinel2.py index 6c8d43e4..bf973304 100644 --- a/gips/test/sys/t_sentinel2.py +++ b/gips/test/sys/t_sentinel2.py @@ -53,21 +53,6 @@ def t_project(setup_fixture, clean_repo_env, output_tfe, expected): actual = output_tfe.run('gips_project', *args) assert expected == actual - -def t_stats(setup_fixture, clean_repo_env, output_tfe, expected): - """Test gips_stats on projected files.""" - # generate data needed for stats computation - args = PROD_ARGS + ['--res', '30', '30', '--outdir', OUTPUT_DIR, '--notld'] - prep_run = output_tfe.run('gips_project', *args) - assert prep_run.exit_status == 0 # confirm it worked; not really in the test - - # compute stats - gtfe = GipsTestFileEnv(OUTPUT_DIR, start_clear=False) - actual = gtfe.run('gips_stats', OUTPUT_DIR) - - # check for correct stats content - assert expected == actual - @slow @acolite @pytest.mark.skip(reason="Overflows in zlib.crc32") From f475eefd9e8bb6b92d3dcde123da3f10cc5b6c0b Mon Sep 17 00:00:00 2001 From: Tom Olson Date: Fri, 12 Jan 2018 09:39:23 -0500 Subject: [PATCH 44/44] 463 correct very old error in env var name --- gips/test/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gips/test/README.md b/gips/test/README.md index 30687c7f..6edfae96 100644 --- a/gips/test/README.md +++ b/gips/test/README.md @@ -134,8 +134,8 @@ created files are observed. These observations are then compared with known correct values to see if the test ought to pass. System tests require a little setup, unfortunately: Edit `settings.py` and -set `OVERRIDE_VERSION` to the version the system tests expect. You can check -this by looking in `gips/test/sys/expected/modis.py` and checking on the +set `GIPS_OVERRIDE_VERSION` to the version the system tests expect. You can +check this by looking in `gips/test/sys/expected/modis.py` and checking on the version that is expected to be output. You should also have a correct `pytest.ini` file (see above). Back up any data files you want to retain in your data repo directory; some system tests need to operate destructively on