Use pytest for test framework and fix for "show ip ospf neighbor"

.gitignore

@@ -93,7 +93,7 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
+#lib/
 lib64/
 parts/
 sdist/

.travis.yml

@@ -8,13 +8,7 @@ python:
 sudo: false
 
 install:
-  - pip install netmiko
-  - pip install textfsm
-  - export PYTHONPATH=$PYTHONPATH:/home/travis/build/networktocode/ntc-templates/gtextfsm-0.2.1/textfsm
-  - pip install ansible==1.9.2
-  - pip install terminal
-  - git clone https://github.com/networktocode/ntc-ansible.git
-
+  - pip install tox
 
 script:
-  - python test-templates.py
+  - tox

lib/ntc_templates/templates

@@ -0,0 +1 @@
+../../templates

setup.py

@@ -1,9 +1,9 @@
 import re
 from codecs import open
-from setuptools import setup
+from setuptools import setup, find_packages
 
 version = ''
-with open('ntc_templates/__init__.py', 'r') as fd:
+with open('lib/ntc_templates/__init__.py', 'r') as fd:
     version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
                         fd.read(), re.MULTILINE).group(1)
 
@@ -20,7 +20,8 @@
 
 config = {
     'name': 'ntc_templates',
-    'packages': ['ntc_templates'],
+    'package_dir': {'': 'lib'},
+    'packages': find_packages('lib'),
     'version': version,
     'package_data': {'ntc_templates': ['templates/*']},
     'description': 'Package to return structured data from the output of network devices.',

templates/brocade_netiron_show_monitor_config.template

@@ -0,0 +1 @@
+brocade_netiron_show_monitor_actual.template

templates/index

@@ -83,9 +83,10 @@ brocade_netiron_show_running-config_interface.template, .*, brocade_netiron, sh[
 brocade_netiron_show_lldp_neighbors_detail.template, .*, brocade_netiron, sh[[ow]] ll[[dp]] n[[eighbors]] d[[etail]]
 brocade_netiron_show_running-config_vlan.template, .*, brocade_netiron, sh[[ow]] ru[[nning-config]] v[[lan]]
 brocade_netiron_show_interfaces_brief.template, .*, brocade_netiron, sh[[ow]] in[[terfaces]] b[[rief]]
+brocade_netiron_show_monitor_actual.template, .*, brocade_netiron, sh[[ow]] mon[[itor]] actual
+brocade_netiron_show_monitor_config.template, .*, brocade_netiron, sh[[ow]] mon[[itor]] config
 brocade_netiron_show_interfaces.template, .*, brocade_netiron, sh[[ow]] in[[terfaces]]
 brocade_netiron_show_lag_brief.template, .*, brocade_netiron, sh[[ow]] lag b[[rief]]
-brocade_netiron_show_monitor.template, .*, brocade_netiron, sh[[ow]] mon[[itor]] [actual|config]
 brocade_netiron_show_metro.template, .*, brocade_netiron, sh[[ow]] met[[ro-ring]]
 brocade_netiron_show_span.template, .*, brocade_netiron, sh[[ow]] sp[[anning-tree]]
 brocade_netiron_show_topo.template, .*, brocade_netiron, sh[[ow]] to[[pology-group]]

tests/__init__.py

@@ -0,0 +1 @@
+"""tests."""

tests/cisco_ios/show_ip_ospf_neighbor/four_neighbors.parsed

@@ -0,0 +1,30 @@
+---
+parsed_sample:
+
+- neighbor_id: '10.190.30.2'
+  priority: '1'
+  state: 'FULL/BDR'
+  dead_time: '00:00:04'
+  address: '10.190.16.11'
+  interface: 'GigabitEthernet0/2.101'
+
+- neighbor_id: '10.190.30.3'
+  priority: '1'
+  state: 'FULL/DR'
+  dead_time: '00:00:04'
+  address: '10.190.16.12'
+  interface: 'GigabitEthernet0/2.101'
+
+- neighbor_id: '10.190.30.2'
+  priority: '1'
+  state: 'FULL/BDR'
+  dead_time: '00:00:04'
+  address: '10.190.16.3'
+  interface: 'GigabitEthernet0/2.100'
+
+- neighbor_id: '10.190.30.3'
+  priority: '1'
+  state: 'FULL/DR'
+  dead_time: '00:00:04'
+  address: '10.190.16.4'
+  interface: 'GigabitEthernet0/2.100'

tests/cisco_ios/show_ip_ospf_neighbor/four_neighbors.raw

@@ -0,0 +1,5 @@
+Neighbor ID     Pri   State           Dead Time   Address         Interface
+10.190.30.2        1   FULL/BDR        00:00:04    10.190.16.11     GigabitEthernet0/2.101
+10.190.30.3        1   FULL/DR         00:00:04    10.190.16.12     GigabitEthernet0/2.101
+10.190.30.2        1   FULL/BDR        00:00:04    10.190.16.3      GigabitEthernet0/2.100
+10.190.30.3        1   FULL/DR         00:00:04    10.190.16.4      GigabitEthernet0/2.100

tests/ntc_template_test_helper.py

@@ -0,0 +1,19 @@
+"""Helper functions for testing."""
+import glob
+
+
+def return_test_files():
+    """Return a list of all the *.raw files to run tests against."""
+    platform_dirs = glob.glob('tests/*')
+
+    platforms = []
+    for platform in platform_dirs:
+        platforms.append(glob.glob('%s/*' % platform))
+
+    template_dirs = [item for sublist in platforms for item in sublist]
+
+    test_commands = []
+    for template_dir in template_dirs:
+        test_commands.append(glob.glob('%s/*.raw' % template_dir))
+
+    return [item for sublist in test_commands for item in sublist]

tests/test_structured_data_against_parsed_reference_files.py

@@ -1,57 +1,12 @@
 #!/usr/bin/env python
 
 """Run tests against all the *.raw files."""
-import glob
 import pytest
 import yaml
-try:
-    import clitable
-except:
-    import textfsm.clitable as clitable
+from ntc_template_test_helper import return_test_files
+from ntc_templates.parse import parse_output
 
 
-def return_test_files():
-    """Return a list of all the *.raw files to run tests against."""
-    platform_dirs = glob.glob('tests/*')
-
-    platforms = []
-    for platform in platform_dirs:
-        platforms.append(glob.glob('%s/*' % platform))
-
-    template_dirs = [item for sublist in platforms for item in sublist]
-
-    test_commands = []
-    for template_dir in template_dirs:
-        test_commands.append(glob.glob('%s/*.raw' % template_dir))
-
-    return [item for sublist in test_commands for item in sublist]
-
-
-def clitable_to_dict(cli_table):
-    """Convert TextFSM cli_table object to list of dictionaries."""
-    objs = []
-    for row in cli_table:
-        temp_dict = {}
-        for index, element in enumerate(row):
-            temp_dict[cli_table.header[index].lower()] = element
-        objs.append(temp_dict)
-
-    return objs
-
-
-def get_structured_data(platform, command, rawoutput):
-    """Return the structured data."""
-    cli_table = clitable.CliTable('index', 'templates')
-
-    attrs = dict(
-        Command=command,
-        Platform=platform
-    )
-    cli_table.ParseCmd(rawoutput, attrs)
-    structured_data = clitable_to_dict(cli_table)
-
-    return structured_data
-
 # Populate test_collection with a list of all the .raw template files we want
 # to run tests against
 test_collection = return_test_files()
@@ -71,7 +26,7 @@ def raw_template_test(raw_file):
     command = ' '.join(parts[2].split('_'))
     with open(raw_file, 'r') as data:
         rawoutput = data.read()
-    structured = get_structured_data(platform, command, rawoutput)
+    structured = parse_output(platform=platform, command=command, data=rawoutput)
     with open(parsed_file, 'r') as data:
         parsed_data = yaml.load(data.read())
 

tests/test_testcases_exists.py

@@ -0,0 +1,44 @@
+"""Ensure that testcases exist for all templates."""
+import csv
+import glob
+import os
+
+from ntc_templates.parse import _get_template_dir
+
+KNOWN_MISSING_TESTS = [
+    'cisco_ios_show_vlan',
+    'cisco_nxos_show_interface_brief',
+    'cisco_nxos_show_ip_ospf_neighbor_vrf',
+    'cisco_xr_show_controllers',
+    '.*vyos.*_os_show_interfaces',
+    '.*vyos.*_os_show_arp',
+]
+
+
+def load_indexdata():
+    """Load data from index file."""
+    index_data = []
+    with open('%s%sindex' % (_get_template_dir(), os.sep)) as indexfs:
+        data = csv.reader(indexfs)
+        for row in data:
+            if len(row) > 2 and row[0] != 'Template':
+                index_data.append(row)
+    return index_data
+
+
+def test_verify_parsed_and_reference_data_exists():
+    """Verify that at least one test exists for all entries in the index file."""
+    index = sorted(load_indexdata())
+    coverage = {}
+    for row in index:
+        template = row[0].strip()
+        platform = row[2].strip()
+        cut = len(platform) + 1
+        command = template[cut:].split('.')[0]
+        cases = 'tests/%s/%s/*.raw' % (platform, command)
+        test_list = glob.glob(cases)
+        coverage['%s_%s' % (platform, command)] = len(test_list)
+
+    for test in coverage:
+        if coverage[test] == 0 and test not in KNOWN_MISSING_TESTS:
+            assert test == 'No test cases found'

tox.ini

@@ -1,14 +1,11 @@
 [tox]
 envlist = py27
-skipsdist=true
 
 [testenv]
-setenv =
-    ANSIBLE_REMOTE_TEMP = .tmp
-    ANSIBLE_LIBRARY = .ntc-modules
 
 deps =
     pytest
+    pytest-cov
     PyYAML
     netmiko
     textfsm
@@ -20,7 +17,4 @@ whitelist_externals =
     rm
 
 commands=
-   py.test
-   rm -rf .ntc-modules
-   git clone https://github.com/networktocode/ntc-ansible.git .ntc-modules
-   {envbindir}/python test-templates.py
+   py.test --cov=ntc_templates --cov-report term-missing