Use a wrapper around yaml.safe_load instead of anymarkup so I get fil…

…e name info for parse errors

src/webapp/common.py

@@ -9,6 +9,7 @@
 from typing import Dict, List, Union, AnyStr
 
 import xmltodict
+import yaml
 
 MaybeOrderedDict = Union[None, OrderedDict]
 
@@ -199,3 +200,16 @@ def gen_id(instr: AnyStr, digits, minimum=1, hashfn=hashlib.md5) -> int:
     instr_b = instr if isinstance(instr, bytes) else instr.encode("utf-8", "surrogateescape")
     mod = (10 ** digits) - minimum
     return minimum + (int(hashfn(instr_b).hexdigest(), 16) % mod)
+
+
+def load_yaml_file(filename) -> Dict:
+    """Load a yaml file (wrapper around yaml.safe_load() because it does not
+    report the filename in which an error occurred.
+
+    """
+    try:
+        with open(filename) as stream:
+            return yaml.safe_load(stream)
+    except yaml.YAMLError as e:
+        log.error("YAML error in %s: %s", filename, e)
+        raise

src/webapp/contacts_reader.py

@@ -7,13 +7,11 @@
 import sys
 from typing import Dict
 
-import anymarkup
-
 # thanks stackoverflow
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from webapp.common import MaybeOrderedDict, to_xml, MISCUSER_SCHEMA_URL
+from webapp.common import MaybeOrderedDict, to_xml, MISCUSER_SCHEMA_URL, load_yaml_file
 
 
 log = getLogger(__name__)
@@ -116,7 +114,7 @@ def get_tree(self, authorized=False, filters=None) -> Dict:
 
 
 def get_contacts_data(infile) -> ContactsData:
-    return ContactsData(anymarkup.parse_file(infile))
+    return ContactsData(load_yaml_file(infile))
 
 
 def main(argv):

src/webapp/project_reader.py

@@ -1,10 +1,14 @@
+#!/usr/bin/env python3
 from argparse import ArgumentParser
 from collections import OrderedDict
 
 import os
+import sys
 
-import anymarkup
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+from webapp.common import load_yaml_file, to_xml
 
 
 def get_projects(indir="../projects"):
@@ -14,7 +18,7 @@ def get_projects(indir="../projects"):
     for file in os.listdir(indir):
         project = OrderedDict.fromkeys(["ID", "Name", "Description", "PIName", "Organization", "Department",
                                         "FieldOfScience", "Sponsor"])
-        project.update(anymarkup.parse_file(os.path.join(indir, file)))
+        project.update(load_yaml_file(os.path.join(indir, file)))
         projects.append(project)
 
     to_output["Projects"]["Project"] = projects
@@ -24,7 +28,7 @@ def get_projects(indir="../projects"):
 
 def get_projects_xml(indir="../projects"):
     """Returns the serialized XML as a string"""
-    return anymarkup.serialize(get_projects(indir), 'xml').decode()
+    return to_xml(get_projects(indir))
 
 
 if __name__ == "__main__":

src/webapp/rg_reader.py

@@ -16,10 +16,8 @@
 where the return value `xml` is a string.
 
 """
-import argparse
 from argparse import ArgumentParser
 
-import anymarkup
 import os
 import pprint
 import sys
@@ -29,7 +27,7 @@
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from webapp.common import ensure_list, to_xml, Filters
+from webapp.common import ensure_list, to_xml, Filters, load_yaml_file
 from webapp.contacts_reader import get_contacts_data
 from webapp.topology import CommonData, Topology
 
@@ -61,18 +59,18 @@ def get_rgsummary_rgdowntime(indir, contacts_file=None, authorized=False):
 
 def get_topology(indir="../topology", contacts_data=None):
     root = Path(indir)
-    support_centers = anymarkup.parse_file(root / "support-centers.yaml")
-    service_types = anymarkup.parse_file(root / "services.yaml")
+    support_centers = load_yaml_file(root / "support-centers.yaml")
+    service_types = load_yaml_file(root / "services.yaml")
     tables = CommonData(contacts=contacts_data, service_types=service_types, support_centers=support_centers)
     topology = Topology(tables)
 
     for facility_path in root.glob("*/FACILITY.yaml"):
         name = facility_path.parts[-2]
-        id_ = anymarkup.parse_file(facility_path)["ID"]
+        id_ = load_yaml_file(facility_path)["ID"]
         topology.add_facility(name, id_)
     for site_path in root.glob("*/*/SITE.yaml"):
         facility, name = site_path.parts[-3:-1]
-        site_info = anymarkup.parse_file(site_path)
+        site_info = load_yaml_file(site_path)
         id_ = site_info["ID"]
         topology.add_site(facility, name, id_, site_info)
     for yaml_path in root.glob("*/*/*.yaml"):
@@ -81,11 +79,11 @@ def get_topology(indir="../topology", contacts_data=None):
         if name.endswith("_downtime.yaml"): continue
 
         name = name.replace(".yaml", "")
-        rg = anymarkup.parse_file(yaml_path)
+        rg = load_yaml_file(yaml_path)
         downtime_yaml_path = yaml_path.with_name(name + "_downtime.yaml")
         downtimes = None
         if downtime_yaml_path.exists():
-            downtimes = ensure_list(anymarkup.parse_file(downtime_yaml_path))
+            downtimes = ensure_list(load_yaml_file(downtime_yaml_path))
 
         topology.add_rg(facility, site, name, rg)
         if downtimes:

src/webapp/vo_reader.py

@@ -1,27 +1,26 @@
-from argparse import ArgumentParser, FileType
+#!/usr/bin/env python3
+from argparse import ArgumentParser
 import os
 import pprint
 import sys
 
-import anymarkup
-
 # thanks stackoverflow
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from webapp.common import to_xml
+from webapp.common import load_yaml_file, to_xml
 from webapp.contacts_reader import get_contacts_data
 from webapp.vos_data import VOsData
 
 
 def get_vos_data(indir, contacts_data) -> VOsData:
-    reporting_groups_data = anymarkup.parse_file(os.path.join(indir, "REPORTING_GROUPS.yaml"))
+    reporting_groups_data = load_yaml_file(os.path.join(indir, "REPORTING_GROUPS.yaml"))
     vos_data = VOsData(contacts_data=contacts_data, reporting_groups_data=reporting_groups_data)
     for file in os.listdir(indir):
         if file == "REPORTING_GROUPS.yaml": continue
         if not file.endswith(".yaml"): continue
         name = file[:-5]
-        data = anymarkup.parse_file(os.path.join(indir, file))
+        data = load_yaml_file(os.path.join(indir, file))
         try:
             vos_data.add_vo(name, data)
         except Exception: