Add sort by functionality to takeovers 'cooked' content (#196)

Add new order by to sort takeovers by active="true". Data explorer query updated https://discourse.ubuntu.com/admin/plugins/explorer?id=14 and add https://discourse.ubuntu.com/admin/plugins/explorer?id=55 for tags

README.md

@@ -96,4 +96,5 @@ Similarly for takeovers, you just need to pass `page_type="takeovers"`.
 
 ## Pagination
 - `get_index` provides two additional arguments `limit` and `offset`, to provide pagination functionality. They default to 50 and 0 respectively.
+- If you want to get all engage pages, which in the case of some sites like jp.ubuntu.com there are not that many, you can pass `limit=-1`
 - Use `MaxLimitError` in the `exceptions.py` to handle excessive limit. By default, it will raise an error when it surpasses 500

canonicalwebteam/discourse/app.py

@@ -308,16 +308,29 @@ def __init__(
         self.additional_metadata_validation = additional_metadata_validation
         pass
 
-    def get_index(self, limit=50, offset=0):
+    def get_index(self, limit=50, offset=0, key=None, value=None):
         """
         Get the index topic and split it into:
         - index document content
         - URL map
         And set those as properties on this object
         """
-        list_topics = self.api.engage_pages_by_category(
-            self.category_id, limit, offset
-        )
+        if key == "tag":
+            list_topics = self.api.get_engage_pages_by_tag(
+                category_id=self.category_id,
+                limit=limit,
+                offset=offset,
+                tag=value,
+            )
+        else:
+            list_topics = self.api.get_engage_pages_by_param(
+                category_id=self.category_id,
+                limit=limit,
+                offset=offset,
+                key=key,
+                value=value,
+            )
+
         topics = []
         for topic in list_topics:
             if topic[6] not in self.exclude_topics:
@@ -327,14 +340,27 @@ def get_index(self, limit=50, offset=0):
                 except MetadataError:
                     continue
 
-        return topics
+        active_count = sum(item[9] for item in list_topics)
+        try:
+            # total_count is everything
+            # active_count is active=true
+            # current_total is the count returned after filtering
+            total_count = list_topics[0][8]
+            current_total = list_topics[0][10]
+        except IndexError:
+            total_count = 0
+            current_total = 0
+
+        # last column of list_topics is the total number of items
+        # this is appended to every item
+        return topics, total_count, active_count, current_total
 
     def get_engage_page(self, path):
         """
         Get single engage page using data-explorer
         """
         single_topic = self.api.get_engage_pages_by_param(
-            self.category_id, "path", path
+            category_id=self.category_id, key="path", value=path
         )
         try:
             single_topic[0]
@@ -348,9 +374,29 @@ def get_engage_page(self, path):
 
         return metadata
 
+    def get_engage_pages_tags(self):
+        """
+        Get all tags in all engage pages
+        for the dropdown filter
+        """
+        list_topics = self.api.get_engage_pages_by_param(
+            category_id=self.category_id, limit=-1
+        )
+        tags = set()
+        for topic in list_topics:
+            if topic[6] not in self.exclude_topics:
+                try:
+                    topics_index = self.parse_topics(topic)
+                    if "tags" in topics_index:
+                        tags = tags.union(set(topics_index["tags"].split(",")))
+                except MetadataError:
+                    continue
+
+        return tags
+
     def parse_active_takeovers(self):
         active_takeovers_topics = self.api.get_engage_pages_by_param(
-            self.category_id, "active", "true"
+            category_id=self.category_id, key="active", value="true"
         )
 
         topics = []

canonicalwebteam/discourse/models.py

@@ -1,7 +1,4 @@
-from canonicalwebteam.discourse.exceptions import (
-    DataExplorerError,
-    MaxLimitError,
-)
+from canonicalwebteam.discourse.exceptions import DataExplorerError
 
 
 class DiscourseAPI:
@@ -85,52 +82,8 @@ def get_topics_category(self, category_id, page=0):
 
         return response.json()
 
-    def engage_pages_by_category(self, category_id=50, limit=50, offset=0):
-        """
-        This endpoint returns engage pages cooked content.
-        This is possible with the Data Explorer plugin for Discourse
-        we are using it to obtain engage pages by category.
-
-        The id for the data explorer query is always 14
-
-        @params
-            - category_id [int]: 50 by default, this is set in the
-        https://discourse.ubuntu.com/admin/plugins/explorer?id=14
-            - limit [int]: 50 by default, also set in data explorer
-            - offset [int]: 0 by default (first page)
-        """
-        # Avoid abuse and spamming of pagination
-        if limit > 500:
-            raise MaxLimitError()
-
-        headers = {
-            "Accept": "application/json",
-            "Content-Type": "multipart/form-data;",
-        }
-        # See https://discourse.ubuntu.com/admin/plugins/explorer?id=14
-        data_explorer_id = 14
-
-        response = self.session.post(
-            f"{self.base_url}/admin/plugins/explorer/"
-            f"queries/{data_explorer_id}/run",
-            headers=headers,
-            data={
-                "params": (
-                    f'{{"category_id": "{category_id}", '
-                    f'"limit": "{limit}", "offset": "{offset}"}}'
-                )
-            },
-        )
-
-        result = response.json()
-        try:
-            pages = result["rows"]
-            return pages
-        except KeyError:
-            raise DataExplorerError(response["errors"][0])
-
     def get_engage_pages_by_param(
-        self, category_id, key, value, limit=50, offset=0
+        self, category_id, key=None, value=None, limit=50, offset=0
     ):
         """
         Uses data-explorer to query topics with the category
@@ -170,19 +123,83 @@ def get_engage_pages_by_param(
         # See https://discourse.ubuntu.com/admin/plugins/explorer?id=16
         data_explorer_id = 16
 
+        params = (
+            {
+                "params": (
+                    f'{{"category_id":"{category_id}", '
+                    f'"limit":"{limit}", "offset":"{offset}"}}'
+                )
+            },
+        )
+
+        if key and value:
+            params = (
+                {
+                    "params": (
+                        f'{{"category_id":"{category_id}", '
+                        f'"keyword":"{key}", "value":"{value}", '
+                        f'"limit":"{limit}", "offset":"{offset}"}}'
+                    )
+                },
+            )
+
+        if limit == -1:
+            # Get all engage pages to compile list of tags
+            # last resort if you need to get all pages, not performant
+            params = ({"params": f'{{"category_id":"{category_id}"}}'},)
+
         response = self.session.post(
             f"{self.base_url}/admin/plugins/explorer/"
             f"queries/{data_explorer_id}/run",
             headers=headers,
-            data={
+            data=params[0],
+        )
+
+        response.raise_for_status()
+        result = response.json()
+
+        if not result["success"]:
+            raise DataExplorerError(response["errors"][0])
+
+        pages = result["rows"]
+        return pages
+
+    def get_engage_pages_by_tag(self, category_id, tag, limit=50, offset=0):
+        """
+        Uses data-explorer to query engage pages
+
+        Same functionality and return values as
+        get_engage_pages_by_param, but specifically
+        for querying by tags
+
+        Args:
+        - limit [int]: 50 by default, also set in data explorer
+        - offset [int]: 0 by default (first page)
+        """
+        headers = {
+            "Accept": "application/json",
+            "Content-Type": "multipart/form-data;",
+        }
+        # See https://discourse.ubuntu.com/admin/plugins/explorer?id=16
+        data_explorer_id = 55
+
+        params = (
+            {
                 "params": (
-                    f'{{"category_id": "{category_id}", '
-                    f'"keyword": "{key}", "value": "{value}", '
-                    f'"limit": "{limit}", "offset": "{offset}"}}',
+                    f'{{"category_id":"{category_id}", '
+                    f'"tag":"{tag}", '
+                    f'"limit":"{limit}", "offset":"{offset}"}}'
                 )
             },
         )
 
+        response = self.session.post(
+            f"{self.base_url}/admin/plugins/explorer/"
+            f"queries/{data_explorer_id}/run",
+            headers=headers,
+            data=params[0],
+        )
+
         response.raise_for_status()
         result = response.json()
 

setup.py

@@ -4,7 +4,7 @@
 
 setup(
     name="canonicalwebteam.discourse",
-    version="5.7.1",
+    version="5.7.2",
     author="Canonical webteam",
     author_email="[email protected]",
     url="https://github.com/canonical/canonicalwebteam.discourse",

tests/cassettes/TestDiscourseAPI.test_index_ep_takeovers.yaml

@@ -15,7 +15,7 @@ interactions:
       User-Agent:
       - python-requests/2.32.3
     method: POST
-    uri: https://discourse.ubuntu.com/admin/plugins/explorer/queries/14/run
+    uri: https://discourse.ubuntu.com/admin/plugins/explorer/queries/16/run
   response:
     body:
       string: "{\"success\": true, \"errors\": [], \"duration\": 12.5, \"result_count\": 1, \"params\": {\"category_id\": 51, \"limit\": \"1\", \"offset\": \"0\"}, \"rows\": [[\"test metadata table\"]]}"

tests/cassettes/TestDiscourseAPI.test_pagination.yaml

@@ -15,7 +15,7 @@ interactions:
       User-Agent:
       - python-requests/2.32.3
     method: POST
-    uri: https://discourse.ubuntu.com/admin/plugins/explorer/queries/14/run
+    uri: https://discourse.ubuntu.com/admin/plugins/explorer/queries/16/run
   response:
     body:
       string: "{\"success\": true, \"errors\": [], \"duration\": 12.5, \"result_count\": 1, \"params\": {\"category_id\": 51, \"limit\": \"1\", \"offset\": \"0\"}, \"rows\": [[\"test metadata table\"]]}"

tests/test_engage_pages.py

@@ -5,7 +5,6 @@
 from vcr_unittest import VCRTestCase
 
 from canonicalwebteam.discourse import DiscourseAPI, EngagePages
-from canonicalwebteam.discourse.exceptions import MaxLimitError
 
 
 this_dir = os.path.dirname(os.path.realpath(__file__))
@@ -55,7 +54,7 @@ def test_index_ep_takeovers(self):
         Test endpoint that retrieves all takeovers/engage pages
         """
 
-        response = self.discourse_api.engage_pages_by_category()
+        response = self.discourse_api.get_engage_pages_by_param(51)
         self.assertEqual(len(response), 1)
 
     def test_individual_ep_takeovers(self):
@@ -64,7 +63,7 @@ def test_individual_ep_takeovers(self):
         """
 
         response = self.discourse_api.get_engage_pages_by_param(
-            51, "active", "true"
+            category_id=51, key="active", value="true"
         )
 
         self.assertEqual(len(response), 1)
@@ -77,20 +76,8 @@ def test_pagination(self):
         - category_id=51, should always be 51 for
         https://discourse.ubuntu.com/c/design/engage-pages/51
         """
-        response = self.discourse_api.engage_pages_by_category(
-            limit=1, offset=0
+        response = self.discourse_api.get_engage_pages_by_param(
+            category_id=51, limit=1, offset=0
         )
 
         self.assertEqual(len(response), 1)
-
-    def test_max_limit_error(self):
-        """
-        Test pagination limit abuse
-        """
-
-        self.assertRaises(
-            MaxLimitError,
-            self.discourse_api.engage_pages_by_category,
-            limit=1000,
-            offset=0,
-        )