Skip to content

Commit

Permalink
Merge branch '463-tif-comparator' into 'dev'
Browse files Browse the repository at this point in the history 
sys test overhaul, iteration 1

Closes #463, #483, #468, #467, and #464

See merge request appliedgeosolutions/gips!485
  • Loading branch information
Ian Cooke committed Jan 15, 2018
2 parents dff5832 + f475eef commit 1245ade
Show file tree
Hide file tree
Showing 31 changed files with 5,011 additions and 1,422 deletions.
23 changes: 22 additions & 1 deletion conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,18 @@ def pytest_addoption(parser):
help_str = ("The directory housing the data repo for testing purposes. "
"MUST match GIPS' configured REPOS setting.")
parser.addini('data-repo', help=help_str)

parser.addini('output-dir',
help="The directory housing output files from test runs.")

parser.addini('artifact-store-user', help="FTP artifact store username")
parser.addini('artifact-store-password',
help="FTP artifact store password")
parser.addini('artifact-store-host', help="FTP artifact store hostname")
parser.addini('artifact-store-path',
help="FTP artifact store root path; files are presumed to be"
" stored driver-wise beneath this level,"
" eg path/sar/asset.tgz")

parser.addoption(
"--slow", action="store_true", help="Do not skip @slow tests.")

Expand All @@ -73,13 +81,24 @@ def pytest_addoption(parser):
parser.addoption(
"--expectation-format", action="store_true", help=help_str)

parser.addoption('--record', action='store', default=None,
help="Pass in a filename for expecations"
" to be written to that filename.")
# cleanup may not need to be implemented at all
#parser.addoption('--cleanup-on-failure', action='store_true',
# help="Normally cleanup is skipped on failure so you can examine"
# " files; pass this option to cleanup even on failure.")

def pytest_configure(config):
"""Process user config & command-line options."""
raw_level = config.getoption("log_level")
level = ('warning' if raw_level is None else raw_level).upper()
root_logger.setLevel(level)

record_path = config.getoption('record')
if record_path and os.path.lexists(record_path):
raise IOError("Record file already exists at {}".format(record_path))

dr = str(config.getini('data-repo'))
if not dr:
raise ValueError("No value specified for 'data-repo' in pytest.ini")
Expand All @@ -101,6 +120,8 @@ def pytest_configure(config):
elif config.getoption("setup_repo"):
_log.debug("--setup-repo detected; setting up data repo")
setup_data_repo()
else:
print "Skipping repo setup per lack of --setup-repo."


def setup_data_repo():
Expand Down
3 changes: 2 additions & 1 deletion dev_requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,5 @@ pytest-timeout
pytest-mock
pytest-django
scripttest
envoy
envoy # deprecated
sh
185 changes: 101 additions & 84 deletions gips/test/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,25 @@ norecursedirs = .venv data-root
data-repo = /home/your-user-name-here/src/gips/data-repo
# a directory of your choice, used for output from, e.g., gips_project
output-dir = /home/your-user-name-here/src/gips/testout
# These should remain as they are (placing them in a committed file is a TODO):
python_functions = t_
python_classes = T_
python_files = t_*.py
DJANGO_SETTINGS_MODULE=gips.inventory.orm.settings
# config for artifact store
artifact-store-user = ask
artifact-store-password = for
artifact-store-host = these
artifact-store-path = values
```

Library Dependencies
--------------------
GIPS automated testing uses a few libraries, mainly pytest and mock, and a
library for gluing them together (pytest-mock). These are installed by
`install.sh`. Further reading:
the gips installation process. Further reading:

* Pytest: http://docs.pytest.org/en/latest/index.html
* Mock: http://www.voidspace.org.uk/python/mock/
Expand All @@ -39,8 +46,16 @@ Test selection
--------------
Running unit tests is straightforward, as is selecting specific tests:

https://pytest.org/latest/usage.html#specifying-tests-selecting-tests

You can also select tests based on mark (marks are tags for tests):

https://pytest.org/latest/example/markers.html

Examples:

```
py.test # only runs fast tests; all unit tests at the moment
py.test # only runs non-system tests (should be fast)
py.test -k inventory # use py.test's -k to select tests conveniently:
========================= test session starts =========================
platform linux2 -- Python 2.7.11+, pytest-2.9.2, py-1.4.31, pluggy-0.3.1
Expand All @@ -60,19 +75,6 @@ gips/test/unit/t_inventory_settings.py ..
======== 4 passed, 4 skipped, 104 deselected in 0.51 seconds ==========
```

Select specific tests per usual for pytest:

https://pytest.org/latest/usage.html#specifying-tests-selecting-tests

You can also select tests based on mark (marks are tags for tests):

https://pytest.org/latest/example/markers.html

And reading about fixtures is especially recommended as the GIPS test suite
uses them substantially:

http://pytest.org/latest/fixture.html

**Important caveat:** If you specify the same test file multiple times on the
command line (say, to specify multiple tests in that file), any module-scoped
test fixtures will run once for each time the file is listed, which may not be
Expand All @@ -82,6 +84,11 @@ mulitple times. Instead, avoid the problem by using `-k` to specify tests.

Writing Unit Tests with Mocking
===============================
First, reading about fixtures is especially recommended as the GIPS test suite
uses them substantially:

http://pytest.org/latest/fixture.html

Automated testing can be difficult due to the code's interaction with the
outside. Some interactions may be undesirable for testing purposes, such as
access to an RNG or the network. Others may be important to observe for
Expand Down Expand Up @@ -122,19 +129,16 @@ https://pytest-django.readthedocs.io/en/latest/
Running System Tests
====================
Each test runs `gips_something` commands, usually just one, as subprocesses.
The test harness captures the command's exit status, standard output, and
standard error. In addition, the filesystem is observed before and after the
test, and any created, deleted, or updated files are noted. These observations
are then compared with known correct values to see if the test ought to pass.
Finally any cleanup actions occur, which may include removing created files to
leave the filesystem in a pristine state.

System tests require a little setup, unfortunately: Edit `settings.py` and set
`OVERRIDE_VERSION` to the version the system tests expect. You can check this
by looking in `gips/test/sys/expected/modis.py` and checking on the version
that is expected to be output. You should also have a correct `pytest.ini`
file (see the top-level README.md). Back up any data files you want to retain
in your data repo directory; some system tests need to operate destructively on
The test harness observes the filesystem before and after the test, and any
created files are observed. These observations are then compared with known
correct values to see if the test ought to pass.

System tests require a little setup, unfortunately: Edit `settings.py` and
set `GIPS_OVERRIDE_VERSION` to the version the system tests expect. You can
check this by looking in `gips/test/sys/expected/modis.py` and checking on the
version that is expected to be output. You should also have a correct
`pytest.ini` file (see above). Back up any data files you want to retain in
your data repo directory; some system tests need to operate destructively on
it.

It's not normal to retain state in between test runs, but the pattern of GIPS
Expand All @@ -154,6 +158,9 @@ py.test --sys --clear-repo # delete the entire repo, then fetch data files,
py.test --ll debug -s -x -vv --sys --setup-repo -k prism
```

Note that specifying the log level with `--ll` and `--log-level` is supported
for some tests, but may be deprecated for new tests.

Also some specially-marked tests are skipped unless command-line options are
given; this is for performance and safety reasons. The repo-altering tests,
mostly tests of `gips_inventory --fetch`, will always be skipped without the
Expand All @@ -163,78 +170,88 @@ that requires `--setup-repo` or `--clear-repo` for the other tests to pass.

Writing System Tests: Testing a New Driver
===========================================
Each driver's tests are split in half; here are the files for modis:
A new driver should have system tests for `gips_process`, `gips_project`, and
`gips_stats`, though in the future minimum test coverage standards may evolve
further.

Thanks to pytest's parametrization, one test function can support many
combinations of drivers and products. See docs for details:

https://docs.pytest.org/en/latest/parametrize.html

In particular, the three tests live here, but shouldn't need to be modified
to establish system testing for a new driver:

* `gips/test/sys/t_modis.py`: The tests themselves
* `gips/test/sys/expected/modis.py`: A file of known good outcomes, one per
test.
* `gips/test/sys/t_process.py`
* `gips/test/sys/t_project.py`
* `gips/test/sys/t_stats.py`

As a quick demonstration, edit a new test file, `gips/test/sys/t_$DRIVER.py`,
and add a test to it:
Instead, these files should be modified:

* `gips/test/sys/expected/*_process.py`: Known-good outcomes for `t_process`
* `gips/test/sys/expected/*_project.py`: Known-good outcomes for `t_project`
* `gips/test/sys/expected/std_stats.py`: Known-good outcomes for `t_stats`
* `gips/test/sys/driver_setup.py`: Configuration & any special code the
driver may need.

Be aware that an older infrastructure may exist in the gips system test suite,
that exists alongside this one. They interact minimally or not at all.

Recipe for Testing a New Driver
-------------------------------
Say we're testing the new `granitesat` driver, which has the products `snow`
and `ice`. Open the file `gips/test/sys/driver_setup.py`, and add a line to
`STD_ARGS` to tell the test suite what scene(s) to operate on:

```
from .util import *
def t_example(repo_env, expected):
actual = repo_env.run('echo "hello world!"')
assert expected == actual
STD_ARGS = {
# . . . various drivers listed here already . . .
'granitesat': ('granitesat', '-s', nh_shp, '-d', '2018-001,2018-005', '-v4'),
}
```

And an output expectation to a new expectations file,
`gips/test/sys/expected/$DRIVER.py`:
Then open the file `gips/test/sys/expected/std_process.py` and add these
lines; this tells the test suite what products to test:

```
t_example = { 'stdout': """hello world!\n""" }
expectations['granitesat'] = {
'snow': [], # the expectation list is intentionally empty for now . . .
'ice': [],
}
```

Now run the test (and only this test, to save time), and you should observe it
pass:
Now record expectations for the new driver's process testing:

```
$ py.test -k t_example
============================= test session starts ==============================
platform linux2 -- Python 2.7.11+, pytest-2.9.2, py-1.4.31, pluggy-0.3.1
rootdir: /home/tolson/src/gips, inifile: pytest.ini
plugins: mock-1.1, cov-2.2.1, timeout-1.0.0
collected 23 items
$ pytest -s -vv gips/test/sys/t_process.py --sys -k granitesat --record=rec.py
```

gips/test/sys/t_$DRIVER.py .
You should observe your `snow` and `ice` tests passing; in this case it means
a recording was made successfully for each product. These recordings are
saved to `rec.py`; open it and see what files were created during the
recording. Confirm that the right files were created, and remove any entries
that aren't important to the test (such as `.index` files). Then copy the
contents of `rec.py` to `gips/test/sys/expected/std_process.py`, replacing the
content of `expectations['granitesat']`. Then re-run process testing and
observe that the tests pass:

===================== 22 tests deselected by '-kt_example' =====================
=================== 1 passed, 22 deselected in 0.42 seconds ====================
```
$ # Test runs presently don't clean up after themselves, so we have to do it:
$ find your-data-repo/granitesat/ -name '*.tif' -delete # vary as needed
$ # note no --record=rec.py this time:
$ pytest -s -vv gips/test/sys/t_process.py --sys -k granitesat
```

The process can be repeated for `t_project` and `t_stats`. Again, the test
suite doesn't perform cleanup, so remove your `OUTPUT_DIR` between test runs.

If your new driver has many products, follow the pattern of `modis` and place
your expectations in a separate file, and import its content into eg
`std_process.py`.

In practice, copying the patterns in `t_modis.py` for testing a new driver can
be considered adequate. Also examine any deprecated shell script in
`gips/test`, as these are organized by driver and can supply needed test cases.
For a driver's source-altering tests, edit a single pair of files regardless of
driver:

* `sys/t_repo_src_alteration.py`: For the tests
* `sys/expected/repo_src_alteration.py`: For the expectations.

Working with Environment Fixtures and the Expectations File
-----------------------------------------------------------
For GIPS, system tests means observing command-line processes as they execute
and comparing their output to known-good values. These outputs only take two
forms: Standard streams and altering files in the filesystem (including
creating and deleting files). Some tests are expected to modify the data repo,
while others will modify a project directory. Choose an appropriate pytest
fixture accordingly, using existing driver tests as a guide. Pytest fixtures
are available to make this easier; see `gips/test/sys/util.py` for details.

The `expected` fixture is somewhat special: It provides a convenient way to
store bulky data needed for tests in a separate file, without needing to
explicitly access the file itself nor configure anything. See
`gips/test/sys/expected/modis.py` for the way this works in practice: The
`expected` fixture looks for an object with the same name as the currently
running test. It then loads that dictionary into a `GipsProcResult` object,
which is convenient for making assertions in these system tests.

The known-good values stored in `expected/` are usually captured from initial
test runs. For convenience, newly developed tests may be run with
`py.test -s --log-level debug --expectation-format`; parts of the output can
be cutpasted into an `expected/` file to establish these needed known-good
values.
If your driver needs additional system tests that don't follow the standard
parttern, you can place these in their own test file, eg:
`gips/test/sys/t_granitesat.py`

Caveats & Tips
==============
Expand Down
2 changes: 1 addition & 1 deletion gips/test/int/t_chirps.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def scene_dir_setup():
os.remove(target_product_fp)
if use_fake_asset:
os.remove(target_asset_fp)
[os.rmdir(d) for d in made_dirs]
[os.rmdir(d) for d in reversed(made_dirs)] # <- remove in correct order

@pytest.mark.django_db
def t_chirps_product_symlink(mocker, scene_dir_setup):
Expand Down
33 changes: 33 additions & 0 deletions gips/test/sys/conftest.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#from __future__ import print_function

### WHERE I LEFT THIS:
# working on optional test teardown (removal of created files) on test failure/error
# two approaches, probably going with the second one:
# PS this conftest.py only applies to system tests due to location; nifty!

# https://stackoverflow.com/questions/28198585/pytest-how-to-take-action-on-test-failure/47908872#47908872
"""
def pytest_exception_interact(node, call, report):
print('pytest_exception_interact')
import pdb; pdb.set_trace()
if report.failed:
print("node:", node)
print("call:", call)
print("report:", report)
# report.outcome == 'failed'
#import pdb; pdb.set_trace()
"""

# https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures
"""
@pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item, call):
# execute all other hooks to obtain the report object
outcome = yield
rep = outcome.get_result()
# set a report attribute for each phase of a call, which can
# be "setup", "call", "teardown"
setattr(item, "rep_" + rep.when, rep)
"""
Loading

0 comments on commit 1245ade

Please sign in to comment.