Check missing links for engage pages

canonicalwebteam/discourse/parsers/base_parser.py

@@ -1,6 +1,7 @@
 # Standard library
 import os
 import re
+import flask
 from urllib.parse import urlparse, urlunparse
 
 # Packages
@@ -21,6 +22,19 @@
 )
 
 
+class ParsingError(Exception):
+    flask.current_app.extensions["sentry"].captureException(Exception)
+    pass
+
+
+class MissingContentError(ParsingError):
+    def __init__(self, error):
+        super().__init__(error)
+
+        flask.current_app.extensions["sentry"].captureMessage(error)
+        pass
+
+
 class BaseParser:
     """
     Parsers used commonly by Tutorials and Engage pages
@@ -35,6 +49,7 @@ def __init__(self, api, index_topic_id, url_prefix):
         self.warnings = []
         self.url_map = {}
         self.redirect_map = {}
+        self.metadata_errors = []
 
     def parse_topic(self, topic):
         """
@@ -286,18 +301,40 @@ def _parse_metadata(self, index_soup, section_name):
                     if value.find("a"):
                         row_dict["topic_name"] = value.find("a").text
 
-                    # Beautiful soup renders URLs as anchors
-                    # Avoid that default behaviour
-                    if value.find("a") and (
-                        value.find("a")["href"] == value.find("a").text
-                    ):
-                        value.contents[0] = value.find("a").text
+                        # Only engage pages need a link
+                        if value.findAll("a", href=True):
+                            if value.find("a")["href"] == value.find("a").text:
+                                value.contents[0] = value.find("a").text
+
+                        else:
+                            error_message = f"Warning: Link not found when parsing row {index + 1} \
+                                 \"{row_dict['topic_name']}\" {titles[index]}. This row has been skipped."
+                            self.metadata_errors.append(error_message)
+                            row_dict = None
+                            MissingContentError(error_message)
+                            break
+
+                    # Missing path will cause the engage item in index to not
+                    # link to the corresponding page
+                    # Missing type will cause resource_name to be empty in
+                    # thank-you pages
+                    # This error does not need breaking, because it does not
+                    # break the page
+                    if (
+                        (titles[index] == "path") or (titles[index] == "type")
+                    ) and ((value.text == "") or (value.text is None)):
+                        error_message = f"Warning: Link not found when parsing row {index + 1} \
+                            \"{row_dict['topic_name']}\" {titles[index]}. This row has been skipped."
+                        self.metadata_errors.append(error_message)
+                        row_dict = None
+                        MissingContentError(error_message)
+                        break
 
                     row_dict[titles[index]] = "".join(
                         str(content) for content in value.contents
                     )
-
-                topics_metadata.append(row_dict)
+                if row_dict:
+                    topics_metadata.append(row_dict)
 
         return topics_metadata
 

canonicalwebteam/discourse/parsers/engage.py

@@ -36,15 +36,8 @@ def parse(self):
             raw_index_soup, self.url_prefix, self.index_topic_id, "Metadata"
         )
 
-        # Avoid markdown error to break site
-        try:
-            # Parse list of topics
-            self.metadata = self._parse_metadata(raw_index_soup, "Metadata")
-            self.takeovers = self._parse_metadata(raw_index_soup, "Takeovers")
-        except IndexError:
-            self.metadata = []
-            self.takeovers = []
-            self.warnings.append("Failed to parse metadata correctly")
+        self.metadata = self._parse_metadata(raw_index_soup, "Metadata")
+        self.takeovers = self._parse_metadata(raw_index_soup, "Takeovers")
 
         if index_topic["id"] != self.index_topic_id:
             # Get body and navigation HTML
@@ -120,7 +113,6 @@ def parse_topic(self, topic):
             ),
             "related": current_topic_related,
             "topic_path": topic_path,
-            "errors": warnings,
         }
 
     def resolve_path(self, relative_path):

setup.py

@@ -4,7 +4,7 @@
 
 setup(
     name="canonicalwebteam.discourse",
-    version="4.0.5",
+    version="4.0.6",
     author="Canonical webteam",
     author_email="[email protected]",
     url="https://github.com/canonical-webteam/canonicalwebteam.docs",