From 80b5171344875b5844d07f2df78149ac24e405ef Mon Sep 17 00:00:00 2001 From: Jacob McGill <9847006+jmcgill298@users.noreply.github.com> Date: Wed, 24 Apr 2019 13:25:49 -0400 Subject: [PATCH] TESTS: Update test framework (#402) * Test with pytest from tox instead of using Ansible * Update tests to run with python3.6 * Update netiron show monitor commands to use a single template * Remove redundant test, ``test_that_all_entries_dicts_match`` * Change to using yaml's safe_load method * Group tests together in order to load files and call TextFSM once, which creates faster test runs * Fix spacing and styling * Add failure messages to tests --- .gitignore | 2 +- .travis.yml | 12 +- .../ntc_templates}/__init__.py | 0 {ntc_templates => lib/ntc_templates}/parse.py | 0 lib/ntc_templates/templates | 1 + ntc_templates/templates | 1 - setup.py | 9 +- ...cade_netiron_show_monitor_actual.template} | 0 templates/index | 2 +- tests/__init__.py | 17 +++ .../aruba_os_show_ip_int_brief.parsed | 0 .../aruba_os_show_ip_int_brief.raw | 0 .../aruba_os_show_ipv6_int_brief.parsed | 0 .../aruba_os_show_ipv6_int_brief.raw | 0 .../brocade_netiron_show_monitor.parsed | 18 --- .../brocade_netiron_show_monitor.raw | 12 -- .../four_neighbors.parsed | 30 +++++ .../show_ip_ospf_neighbor/four_neighbors.raw | 5 + .../cisco_ios_show_spanning_tree.parsed | 0 .../cisco_ios_show_spanning_tree.raw | 0 ..._nxos_show_ip_ospf_neighbor_vrf_all.parsed | 0 ...sco_nxos_show_ip_ospf_neighbor_vrf_all.raw | 0 tests/test_index_order.py | 54 ++++----- ...red_data_against_parsed_reference_files.py | 105 +++++------------- tests/test_testcases_exists.py | 39 +++++++ .../ubiquiti_edgeswitch_show_vlan.parsed | 0 .../ubiquiti_edgeswitch_show_vlan.raw | 0 tox.ini | 17 +-- 28 files changed, 153 insertions(+), 171 deletions(-) rename {ntc_templates => lib/ntc_templates}/__init__.py (100%) rename {ntc_templates => lib/ntc_templates}/parse.py (100%) create mode 120000 lib/ntc_templates/templates delete mode 120000 ntc_templates/templates rename templates/{brocade_netiron_show_monitor.template => brocade_netiron_show_monitor_actual.template} (100%) create mode 100644 tests/__init__.py rename tests/aruba_os/{show_ip_int_brief => show_ip_interface_brief}/aruba_os_show_ip_int_brief.parsed (100%) rename tests/aruba_os/{show_ip_int_brief => show_ip_interface_brief}/aruba_os_show_ip_int_brief.raw (100%) rename tests/aruba_os/{show_ipv6_int_brief => show_ipv6_interface_brief}/aruba_os_show_ipv6_int_brief.parsed (100%) rename tests/aruba_os/{show_ipv6_int_brief => show_ipv6_interface_brief}/aruba_os_show_ipv6_int_brief.raw (100%) delete mode 100644 tests/brocade_netiron/show_monitor_config/brocade_netiron_show_monitor.parsed delete mode 100644 tests/brocade_netiron/show_monitor_config/brocade_netiron_show_monitor.raw create mode 100644 tests/cisco_ios/show_ip_ospf_neighbor/four_neighbors.parsed create mode 100644 tests/cisco_ios/show_ip_ospf_neighbor/four_neighbors.raw rename tests/cisco_ios/{show_spanning_tree => show_spanning-tree}/cisco_ios_show_spanning_tree.parsed (100%) rename tests/cisco_ios/{show_spanning_tree => show_spanning-tree}/cisco_ios_show_spanning_tree.raw (100%) rename tests/cisco_nxos/{show_ip_ospf_neighbor_vrf_all => show_ip_ospf_neighbor_vrf_vrfname}/cisco_nxos_show_ip_ospf_neighbor_vrf_all.parsed (100%) rename tests/cisco_nxos/{show_ip_ospf_neighbor_vrf_all => show_ip_ospf_neighbor_vrf_vrfname}/cisco_nxos_show_ip_ospf_neighbor_vrf_all.raw (100%) create mode 100644 tests/test_testcases_exists.py rename tests/ubiquiti_edgeswitch/{show_vlans => show_vlan}/ubiquiti_edgeswitch_show_vlan.parsed (100%) rename tests/ubiquiti_edgeswitch/{show_vlans => show_vlan}/ubiquiti_edgeswitch_show_vlan.raw (100%) diff --git a/.gitignore b/.gitignore index d16dc26a57..ede28f140f 100644 --- a/.gitignore +++ b/.gitignore @@ -93,7 +93,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +#lib/ lib64/ parts/ sdist/ diff --git a/.travis.yml b/.travis.yml index 66ebc553a8..570aa28246 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,18 +3,12 @@ language: python python: - - "2.7" + - "3.6" sudo: false install: - - pip install netmiko - - pip install textfsm - - export PYTHONPATH=$PYTHONPATH:/home/travis/build/networktocode/ntc-templates/gtextfsm-0.2.1/textfsm - - pip install ansible==1.9.2 - - pip install terminal - - git clone https://github.com/networktocode/ntc-ansible.git - + - pip install tox script: - - python test-templates.py + - tox diff --git a/ntc_templates/__init__.py b/lib/ntc_templates/__init__.py similarity index 100% rename from ntc_templates/__init__.py rename to lib/ntc_templates/__init__.py diff --git a/ntc_templates/parse.py b/lib/ntc_templates/parse.py similarity index 100% rename from ntc_templates/parse.py rename to lib/ntc_templates/parse.py diff --git a/lib/ntc_templates/templates b/lib/ntc_templates/templates new file mode 120000 index 0000000000..07531b725b --- /dev/null +++ b/lib/ntc_templates/templates @@ -0,0 +1 @@ +../../templates \ No newline at end of file diff --git a/ntc_templates/templates b/ntc_templates/templates deleted file mode 120000 index 564a409d41..0000000000 --- a/ntc_templates/templates +++ /dev/null @@ -1 +0,0 @@ -../templates \ No newline at end of file diff --git a/setup.py b/setup.py index dae9e7919f..59c2f66b19 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,9 @@ import re from codecs import open -from setuptools import setup +from setuptools import setup, find_packages version = '' -with open('ntc_templates/__init__.py', 'r') as fd: +with open('lib/ntc_templates/__init__.py', 'r') as fd: version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', fd.read(), re.MULTILINE).group(1) @@ -20,7 +20,8 @@ config = { 'name': 'ntc_templates', - 'packages': ['ntc_templates'], + 'package_dir': {'': 'lib'}, + 'packages': find_packages('lib'), 'version': version, 'package_data': {'ntc_templates': ['templates/*']}, 'description': 'Package to return structured data from the output of network devices.', @@ -35,7 +36,7 @@ 'classifiers': ['Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'Intended Audience :: System Administrators', - 'Programming Language :: Python :: 2.7'], + 'Programming Language :: Python :: 3'], 'zip_safe': False } diff --git a/templates/brocade_netiron_show_monitor.template b/templates/brocade_netiron_show_monitor_actual.template similarity index 100% rename from templates/brocade_netiron_show_monitor.template rename to templates/brocade_netiron_show_monitor_actual.template diff --git a/templates/index b/templates/index index ebfdf090e5..a2b4bbc876 100644 --- a/templates/index +++ b/templates/index @@ -83,9 +83,9 @@ brocade_netiron_show_running-config_interface.template, .*, brocade_netiron, sh[ brocade_netiron_show_lldp_neighbors_detail.template, .*, brocade_netiron, sh[[ow]] ll[[dp]] n[[eighbors]] d[[etail]] brocade_netiron_show_running-config_vlan.template, .*, brocade_netiron, sh[[ow]] ru[[nning-config]] v[[lan]] brocade_netiron_show_interfaces_brief.template, .*, brocade_netiron, sh[[ow]] in[[terfaces]] b[[rief]] +brocade_netiron_show_monitor_actual.template, .*, brocade_netiron, sh[[ow]] mon[[itor]] (?:ac|co) brocade_netiron_show_interfaces.template, .*, brocade_netiron, sh[[ow]] in[[terfaces]] brocade_netiron_show_lag_brief.template, .*, brocade_netiron, sh[[ow]] lag b[[rief]] -brocade_netiron_show_monitor.template, .*, brocade_netiron, sh[[ow]] mon[[itor]] [actual|config] brocade_netiron_show_metro.template, .*, brocade_netiron, sh[[ow]] met[[ro-ring]] brocade_netiron_show_span.template, .*, brocade_netiron, sh[[ow]] sp[[anning-tree]] brocade_netiron_show_topo.template, .*, brocade_netiron, sh[[ow]] to[[pology-group]] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..875ee2be5d --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,17 @@ +"""tests.""" + +import os +import csv + +from ntc_templates.parse import _get_template_dir + + +def load_index_data(): + """Load data from index file.""" + index_data = [] + with open('%s%sindex' % (_get_template_dir(), os.sep)) as indexfs: + data = csv.reader(indexfs) + for row in data: + if len(row) > 2 and row[0] != 'Template': + index_data.append(row) + return index_data diff --git a/tests/aruba_os/show_ip_int_brief/aruba_os_show_ip_int_brief.parsed b/tests/aruba_os/show_ip_interface_brief/aruba_os_show_ip_int_brief.parsed similarity index 100% rename from tests/aruba_os/show_ip_int_brief/aruba_os_show_ip_int_brief.parsed rename to tests/aruba_os/show_ip_interface_brief/aruba_os_show_ip_int_brief.parsed diff --git a/tests/aruba_os/show_ip_int_brief/aruba_os_show_ip_int_brief.raw b/tests/aruba_os/show_ip_interface_brief/aruba_os_show_ip_int_brief.raw similarity index 100% rename from tests/aruba_os/show_ip_int_brief/aruba_os_show_ip_int_brief.raw rename to tests/aruba_os/show_ip_interface_brief/aruba_os_show_ip_int_brief.raw diff --git a/tests/aruba_os/show_ipv6_int_brief/aruba_os_show_ipv6_int_brief.parsed b/tests/aruba_os/show_ipv6_interface_brief/aruba_os_show_ipv6_int_brief.parsed similarity index 100% rename from tests/aruba_os/show_ipv6_int_brief/aruba_os_show_ipv6_int_brief.parsed rename to tests/aruba_os/show_ipv6_interface_brief/aruba_os_show_ipv6_int_brief.parsed diff --git a/tests/aruba_os/show_ipv6_int_brief/aruba_os_show_ipv6_int_brief.raw b/tests/aruba_os/show_ipv6_interface_brief/aruba_os_show_ipv6_int_brief.raw similarity index 100% rename from tests/aruba_os/show_ipv6_int_brief/aruba_os_show_ipv6_int_brief.raw rename to tests/aruba_os/show_ipv6_interface_brief/aruba_os_show_ipv6_int_brief.raw diff --git a/tests/brocade_netiron/show_monitor_config/brocade_netiron_show_monitor.parsed b/tests/brocade_netiron/show_monitor_config/brocade_netiron_show_monitor.parsed deleted file mode 100644 index bb8ec57f4e..0000000000 --- a/tests/brocade_netiron/show_monitor_config/brocade_netiron_show_monitor.parsed +++ /dev/null @@ -1,18 +0,0 @@ ---- -parsed_sample: - -- monitoredport: '1/10' - inputmirror: '1/13' - outputmirror: '1/13' - -- monitoredport: '1/11' - inputmirror: '1/13' - outputmirror: '1/13' - -- monitoredport: '2/10' - inputmirror: '1/13' - outputmirror: '1/13' - -- monitoredport: '2/11' - inputmirror: '1/13' - outputmirror: '1/13' diff --git a/tests/brocade_netiron/show_monitor_config/brocade_netiron_show_monitor.raw b/tests/brocade_netiron/show_monitor_config/brocade_netiron_show_monitor.raw deleted file mode 100644 index 084529cea6..0000000000 --- a/tests/brocade_netiron/show_monitor_config/brocade_netiron_show_monitor.raw +++ /dev/null @@ -1,12 +0,0 @@ -Monitored Port 1/10 - Input traffic mirrored to: 1/13 - Output traffic mirrored to: 1/13 -Monitored Port 1/11 - Input traffic mirrored to: 1/13 - Output traffic mirrored to: 1/13 -Monitored Port 2/10 - Input traffic mirrored to: 1/13 - Output traffic mirrored to: 1/13 -Monitored Port 2/11 - Input traffic mirrored to: 1/13 - Output traffic mirrored to: 1/13 diff --git a/tests/cisco_ios/show_ip_ospf_neighbor/four_neighbors.parsed b/tests/cisco_ios/show_ip_ospf_neighbor/four_neighbors.parsed new file mode 100644 index 0000000000..f753b5f1cf --- /dev/null +++ b/tests/cisco_ios/show_ip_ospf_neighbor/four_neighbors.parsed @@ -0,0 +1,30 @@ +--- +parsed_sample: + +- neighbor_id: '10.190.30.2' + priority: '1' + state: 'FULL/BDR' + dead_time: '00:00:04' + address: '10.190.16.11' + interface: 'GigabitEthernet0/2.101' + +- neighbor_id: '10.190.30.3' + priority: '1' + state: 'FULL/DR' + dead_time: '00:00:04' + address: '10.190.16.12' + interface: 'GigabitEthernet0/2.101' + +- neighbor_id: '10.190.30.2' + priority: '1' + state: 'FULL/BDR' + dead_time: '00:00:04' + address: '10.190.16.3' + interface: 'GigabitEthernet0/2.100' + +- neighbor_id: '10.190.30.3' + priority: '1' + state: 'FULL/DR' + dead_time: '00:00:04' + address: '10.190.16.4' + interface: 'GigabitEthernet0/2.100' diff --git a/tests/cisco_ios/show_ip_ospf_neighbor/four_neighbors.raw b/tests/cisco_ios/show_ip_ospf_neighbor/four_neighbors.raw new file mode 100644 index 0000000000..fa8d41c7b3 --- /dev/null +++ b/tests/cisco_ios/show_ip_ospf_neighbor/four_neighbors.raw @@ -0,0 +1,5 @@ +Neighbor ID Pri State Dead Time Address Interface +10.190.30.2 1 FULL/BDR 00:00:04 10.190.16.11 GigabitEthernet0/2.101 +10.190.30.3 1 FULL/DR 00:00:04 10.190.16.12 GigabitEthernet0/2.101 +10.190.30.2 1 FULL/BDR 00:00:04 10.190.16.3 GigabitEthernet0/2.100 +10.190.30.3 1 FULL/DR 00:00:04 10.190.16.4 GigabitEthernet0/2.100 diff --git a/tests/cisco_ios/show_spanning_tree/cisco_ios_show_spanning_tree.parsed b/tests/cisco_ios/show_spanning-tree/cisco_ios_show_spanning_tree.parsed similarity index 100% rename from tests/cisco_ios/show_spanning_tree/cisco_ios_show_spanning_tree.parsed rename to tests/cisco_ios/show_spanning-tree/cisco_ios_show_spanning_tree.parsed diff --git a/tests/cisco_ios/show_spanning_tree/cisco_ios_show_spanning_tree.raw b/tests/cisco_ios/show_spanning-tree/cisco_ios_show_spanning_tree.raw similarity index 100% rename from tests/cisco_ios/show_spanning_tree/cisco_ios_show_spanning_tree.raw rename to tests/cisco_ios/show_spanning-tree/cisco_ios_show_spanning_tree.raw diff --git a/tests/cisco_nxos/show_ip_ospf_neighbor_vrf_all/cisco_nxos_show_ip_ospf_neighbor_vrf_all.parsed b/tests/cisco_nxos/show_ip_ospf_neighbor_vrf_vrfname/cisco_nxos_show_ip_ospf_neighbor_vrf_all.parsed similarity index 100% rename from tests/cisco_nxos/show_ip_ospf_neighbor_vrf_all/cisco_nxos_show_ip_ospf_neighbor_vrf_all.parsed rename to tests/cisco_nxos/show_ip_ospf_neighbor_vrf_vrfname/cisco_nxos_show_ip_ospf_neighbor_vrf_all.parsed diff --git a/tests/cisco_nxos/show_ip_ospf_neighbor_vrf_all/cisco_nxos_show_ip_ospf_neighbor_vrf_all.raw b/tests/cisco_nxos/show_ip_ospf_neighbor_vrf_vrfname/cisco_nxos_show_ip_ospf_neighbor_vrf_all.raw similarity index 100% rename from tests/cisco_nxos/show_ip_ospf_neighbor_vrf_all/cisco_nxos_show_ip_ospf_neighbor_vrf_all.raw rename to tests/cisco_nxos/show_ip_ospf_neighbor_vrf_vrfname/cisco_nxos_show_ip_ospf_neighbor_vrf_all.raw diff --git a/tests/test_index_order.py b/tests/test_index_order.py index 92b83b6425..3eef5cda63 100644 --- a/tests/test_index_order.py +++ b/tests/test_index_order.py @@ -6,19 +6,10 @@ import glob import csv +from tests import load_index_data -def load_indexdata(): - """Load data from index file.""" - index_data = [] - with open('./templates/index') as indexfs: - data = csv.reader(indexfs) - for row in data: - if len(row) > 2 and row[0] != 'Template': - index_data.append(row) - return index_data - -def check_order(current_os, prior_os,cmd_len, prior_len, os_choices, used_os, cmd, prior_cmd): +def check_order(current_os, prior_os, cmd_len, prior_len, os_choices, used_os, cmd, prior_cmd): add_os_check = [] if current_os not in used_os and used_os is not None: @@ -27,10 +18,10 @@ def check_order(current_os, prior_os,cmd_len, prior_len, os_choices, used_os, cm if used_os != sorted(used_os): msg = "OS's are not in alpabetical order, current order: '{}'".format(used_os) - return False , msg + return False, msg elif add_os_check != sorted(add_os_check): msg = "OS's are not in alpabetical order, current order: '{}'".format(add_os_check) - return False , msg + return False, msg if current_os not in os_choices: msg = "'{}' is not one of the valid options '{}'".format(current_os, used_os) @@ -38,25 +29,30 @@ def check_order(current_os, prior_os,cmd_len, prior_len, os_choices, used_os, cm if not prior_os and prior_len == 0: # Starting Point - return True , '' + return True, '' elif current_os == prior_os and cmd_len == prior_len and cmd == min(prior_cmd, cmd): - msg = "OS is the same and command same length, please set {} before {} to be in alphabetical order".format(cmd, prior_cmd) + msg = ( + "OS is the same and command same length, " + "please set {} before {} to be in alphabetical order".format(cmd, prior_cmd) + ) return False, msg elif current_os == prior_os and cmd_len <= prior_len: # OS is the same as previous, and cmd_len is smaller or equal to prior so good - return True , '' + return True, '' elif current_os != prior_os and current_os not in used_os: # prior OS has changed, do not need to check for length yet - return True , '' + return True, '' elif current_os == prior_os and cmd_len > prior_len: - msg = "Current Command len '{}' larger then previous '{}', for command '{}'".format(cmd_len, prior_len, cmd ) - return False , msg + msg = "Current Command len '{}' larger then previous '{}', for command '{}'".format( + cmd_len, prior_len, cmd + ) + return False, msg elif current_os != prior_os and current_os in used_os: msg = "'{}' OS was already used in list on command '{}'".format(current_os, cmd) - return False , msg + return False, msg else: msg = "Failed for unknown reason" - return False , msg + return False, msg def test_index_ordering(): @@ -67,7 +63,7 @@ def test_index_ordering(): 'cisco_xe', 'cisco_xr', 'dell_force10', 'enterasys', 'extreme', 'f5_ltm', 'fortinet', 'hp_comware', 'hp_procurve', 'huawei', 'juniper', 'juniper_junos', 'juniper_screenos', 'alcatel_sros', 'linux', 'ovs_linux', 'paloalto_panos', 'quanta_mesh', - 'ubiquiti_edgeswitch', 'vmware_nsxv', 'vyatta_vyos', 'vyos' + 'ubiquiti_edgeswitch', 'vmware_nsxv', 'vyatta_vyos', 'vyos', ] prior_os = "" @@ -75,26 +71,22 @@ def test_index_ordering(): prior_cmd = "" used_os = [] - index = load_indexdata() + index = load_index_data() for row in index: template = row[0].strip() os = '_'.join(template.split('_')[:2]) cmd = '_'.join(template.split('_')[2:]) cmd_len = len(cmd) - check_val, check_msg = check_order(os, prior_os, cmd_len, prior_len, os_choices, used_os, cmd, prior_cmd) + check_val, check_msg = check_order( + os, prior_os, cmd_len, prior_len, os_choices, used_os, cmd, prior_cmd + ) if not check_val: #assertFalse(check_val, msg=check_msg) print("Error on line: {}".format(row)) print("Error Message: {}".format(check_msg)) - assert check_val != False + assert check_val if os not in used_os: used_os.append(os) prior_len = cmd_len prior_cmd = cmd prior_os = os - -def main(): - test_index_ordering() - -if __name__ == "__main__": - main() diff --git a/tests/test_structured_data_against_parsed_reference_files.py b/tests/test_structured_data_against_parsed_reference_files.py index 0987b5f75e..f4708c2101 100644 --- a/tests/test_structured_data_against_parsed_reference_files.py +++ b/tests/test_structured_data_against_parsed_reference_files.py @@ -2,62 +2,24 @@ """Run tests against all the *.raw files.""" import glob + import pytest import yaml -try: - import clitable -except: - import textfsm.clitable as clitable + +from ntc_templates.parse import parse_output def return_test_files(): """Return a list of all the *.raw files to run tests against.""" platform_dirs = glob.glob('tests/*') + platforms = (glob.glob('%s/*' % platform) for platform in platform_dirs) + template_dirs = (item for sublist in platforms for item in sublist) + test_commands = (glob.glob('%s/*.raw' % template_dir) for template_dir in template_dirs) - platforms = [] - for platform in platform_dirs: - platforms.append(glob.glob('%s/*' % platform)) - - template_dirs = [item for sublist in platforms for item in sublist] - - test_commands = [] - for template_dir in template_dirs: - test_commands.append(glob.glob('%s/*.raw' % template_dir)) - - return [item for sublist in test_commands for item in sublist] - - -def clitable_to_dict(cli_table): - """Convert TextFSM cli_table object to list of dictionaries.""" - objs = [] - for row in cli_table: - temp_dict = {} - for index, element in enumerate(row): - temp_dict[cli_table.header[index].lower()] = element - objs.append(temp_dict) - - return objs - - -def get_structured_data(platform, command, rawoutput): - """Return the structured data.""" - cli_table = clitable.CliTable('index', 'templates') + return (item for sublist in test_commands for item in sublist) - attrs = dict( - Command=command, - Platform=platform - ) - cli_table.ParseCmd(rawoutput, attrs) - structured_data = clitable_to_dict(cli_table) - return structured_data - -# Populate test_collection with a list of all the .raw template files we want -# to run tests against -test_collection = return_test_files() - - -@pytest.fixture(scope='function', params=test_collection) +@pytest.fixture(scope='function', params=return_test_files()) def load_template_test(request): """Return each *.raw file to run tests on.""" return request.param @@ -71,44 +33,49 @@ def raw_template_test(raw_file): command = ' '.join(parts[2].split('_')) with open(raw_file, 'r') as data: rawoutput = data.read() - structured = get_structured_data(platform, command, rawoutput) + structured = parse_output(platform=platform, command=command, data=rawoutput) with open(parsed_file, 'r') as data: - parsed_data = yaml.load(data.read()) + parsed_data = yaml.safe_load(data.read()) return structured, parsed_data['parsed_sample'] -def test_correct_number_of_entries(load_template_test): +def test_raw_data_against_mock(load_template_test): + processed, reference = raw_template_test(load_template_test) + + correct_number_of_entries_test(processed, reference) + all_entries_have_the_same_keys_test(processed, reference) + correct_data_in_entries_test(processed, reference) + + +def correct_number_of_entries_test(processed, reference): """Test that the number of entries returned are the same as the control. This will create a test for each of the files in the test_collection variable. """ - processed, reference = raw_template_test(load_template_test) - assert len(processed) == len(reference) -def test_that_all_entries_have_the_same_keys(load_template_test): +def all_entries_have_the_same_keys_test(processed, reference): """Test that the keys of the returned data are the same as the control. This will create a test for each of the files in the test_collection variable. """ - processed, reference = raw_template_test(load_template_test) - for i in range(len(processed)): - assert sorted(processed[i].keys()) == sorted(reference[i].keys()) + proc = set(processed[i].keys()) + ref = set(reference[i].keys()) + diff = proc.symmetric_difference(ref) + assert not diff, "Key diffs: " + ", ".join(diff) -def test_correct_data_in_entries(load_template_test): +def correct_data_in_entries_test(processed, reference): """Test that the actual data in each entry is the same as the control. This will create a test for each of the files in the test_collection variable. """ - processed, reference = raw_template_test(load_template_test) - # Can be uncommented if we don't care that the parsed data isn't # in the same order as the raw data # reference = sorted(reference) @@ -116,24 +83,4 @@ def test_correct_data_in_entries(load_template_test): for i in range(len(reference)): for key in reference[i].keys(): - assert processed[i][key] == reference[i][key] - - -def test_that_all_entries_dicts_match(load_template_test): - """Test that the values of the dicts returned are the same as the control. - - This test swaps place with the processed and reference variable so it's not run - in the same order as test_correct_data_in_entries to catch dicts with extra keys - - This will create a test for each of the files in the test_collection - variable. - """ - processed, reference = raw_template_test(load_template_test) - - # Can be uncommented if we don't care that the parsed data isn't - # in the same order as the raw data - # reference = sorted(reference) - # processed = sorted(processed) - - for i in range(len(processed)): - assert processed[i] == reference[i] + assert processed[i][key] == reference[i][key], "entry #{0}, key: {1}".format(i, key) diff --git a/tests/test_testcases_exists.py b/tests/test_testcases_exists.py new file mode 100644 index 0000000000..436926f7bf --- /dev/null +++ b/tests/test_testcases_exists.py @@ -0,0 +1,39 @@ +"""Ensure that testcases exist for all templates.""" +import os +import glob +import re + +from tests import load_index_data + + +KNOWN_MISSING_TESTS = { + 'cisco_ios_show_vlan', + 'cisco_nxos_show_interface_brief', + 'cisco_nxos_show_ip_ospf_neighbor_vrf', + 'cisco_xr_show_controllers', +} + + +def test_verify_parsed_and_reference_data_exists(): + """Verify that at least one test exists for all entries in the index file. + + TODO: + Add test cases for ``KNOWN_MISSING_TESTS`` and remove related conditional. + Remove "_ssh" from ``cisco_wlc_ssh`` and rely on vendor_platform_command syntax + instead of using regex on the directories. + """ + index = sorted(load_index_data()) + for row in index: + template = row[0].strip() + template_short = template.split('.')[0] + platform = row[2].strip() + for directory in os.listdir("tests"): + if re.match(platform, directory): + platform_directory = directory + break + cut = len(platform_directory) + 1 + command = template_short[cut:] + if template_short not in KNOWN_MISSING_TESTS: + cases = 'tests/%s/%s/*.raw' % (platform_directory, command) + test_list = glob.glob(cases) + assert len(test_list) != 0, 'Could not find tests for %s' % template diff --git a/tests/ubiquiti_edgeswitch/show_vlans/ubiquiti_edgeswitch_show_vlan.parsed b/tests/ubiquiti_edgeswitch/show_vlan/ubiquiti_edgeswitch_show_vlan.parsed similarity index 100% rename from tests/ubiquiti_edgeswitch/show_vlans/ubiquiti_edgeswitch_show_vlan.parsed rename to tests/ubiquiti_edgeswitch/show_vlan/ubiquiti_edgeswitch_show_vlan.parsed diff --git a/tests/ubiquiti_edgeswitch/show_vlans/ubiquiti_edgeswitch_show_vlan.raw b/tests/ubiquiti_edgeswitch/show_vlan/ubiquiti_edgeswitch_show_vlan.raw similarity index 100% rename from tests/ubiquiti_edgeswitch/show_vlans/ubiquiti_edgeswitch_show_vlan.raw rename to tests/ubiquiti_edgeswitch/show_vlan/ubiquiti_edgeswitch_show_vlan.raw diff --git a/tox.ini b/tox.ini index 992dd8c165..5e46500fc7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,26 +1,13 @@ [tox] -envlist = py27 -skipsdist=true +envlist = py36 [testenv] -setenv = - ANSIBLE_REMOTE_TEMP = .tmp - ANSIBLE_LIBRARY = .ntc-modules - deps = pytest PyYAML - netmiko - textfsm - ansible==1.9.2 - terminal whitelist_externals = git rm -commands= - py.test - rm -rf .ntc-modules - git clone https://github.com/networktocode/ntc-ansible.git .ntc-modules - {envbindir}/python test-templates.py +commands = pytest