fix: lippupiste importer

Migrate from the old csv API to use the new JSON API with minimal
changes to the old importer.

refs: LINK-2237

events/importer/lippupiste.py

@@ -1,5 +1,3 @@
-import codecs
-import csv
 import logging
 import re
 from collections import defaultdict
@@ -81,12 +79,6 @@
     "Helsingin Vanha kirkko": "tprek:43184",  # String match fails
 }
 
-# By default, only import events in the capital region
-POSTAL_CODE_RANGES = ((1, 990), (1200, 1770), (2100, 2380), (2600, 2860), (2920, 2980))
-
-# By default, only import agreed providers (lowercase required!)
-PROVIDERS_TO_IMPORT = ("helsingin kaupunginteatteri",)
-
 NAMES_TO_IGNORE_BY_PROVIDER = {
     "Helsingin kaupunginteatteri": (
         "käsiohjelma",
@@ -262,18 +254,22 @@ def setup(self):
         self.sub_event_count_by_super_event_source_id = defaultdict(lambda: 0)
 
     def _fetch_event_source_data(self, url):
-        # stream=True allows lazy iteration
-        response = requests.get(url, stream=True, timeout=self.default_timeout)
-        response_iter = response.iter_lines()
-        # CSV reader wants str instead of byte, let's decode
-        decoded_response_iter = codecs.iterdecode(response_iter, "utf-8")
-        reader = csv.DictReader(
-            decoded_response_iter, delimiter=";", quotechar='"', doublequote=True
-        )
-        return reader
+        return requests.get(
+            url,
+            auth=(
+                settings.LIPPUPISTE_EVENT_API_USERNAME,
+                settings.LIPPUPISTE_EVENT_API_PASSWORD,
+            ),
+            params={"ClientID": settings.LIPPUPISTE_EVENT_API_CLIENT_ID},
+            timeout=self.default_timeout,
+            headers={
+                "Content-Type": "application/json",
+                "CALENDARKey": settings.LIPPUPISTE_EVENT_API_CALENDAR_KEY,
+            },
+        ).json()["Events"]
 
     def _get_keywords_from_source_category(self, source_category):
-        source_category_key = source_category.lower()
+        source_category_key = source_category["Name"].lower()
         keyword_set = set()
         for keyword_id in YSO_KEYWORD_MAPS.get(source_category_key, []):
             if keyword_id in self.keyword_by_id:
@@ -282,7 +278,6 @@ def _get_keywords_from_source_category(self, source_category):
         return keyword_set
 
     def _get_keywords_from_source_categories(self, source_categories):
-        source_categories = source_categories.split("|")
         keyword_set = set()
         for category in source_categories:
             keyword_set = keyword_set.union(
@@ -480,12 +475,19 @@ def _update_event_data(self, event, source_event):
                 [lang] + self.languages_to_detect,
                 "short_description",
             )
-        event["info_url"][lang] = clean_url(source_event["EventSerieLink"])
+        event["info_url"]["fi"] = clean_url(source_event["EventSerieLinkFi"])
+        event["info_url"]["sv"] = clean_url(source_event["EventSerieLinkSv"])
+        event["info_url"]["en"] = clean_url(source_event["EventSerieLinkEn"])
+
         event["offers"] = [
             {
                 "is_free": False,
                 "description": {lang: "Tarkista hinta lippupalvelusta"},
-                "info_url": {lang: clean_url(source_event["EventLink"])},
+                "info_url": {
+                    "fi": clean_url(source_event["EventLinkFi"]),
+                    "sv": clean_url(source_event["EventLinkSv"]),
+                    "en": clean_url(source_event["EventLinkEn"]),
+                },
                 "price": None,
             },
         ]
@@ -498,7 +500,7 @@ def _update_event_data(self, event, source_event):
 
         existing_keywords = event.get("keywords", set())
         keywords_from_source = self._get_keywords_from_source_categories(
-            source_event["EventSerieCategories"]
+            source_event["Categories"]
         )
         event["keywords"] = existing_keywords.union(keywords_from_source)
 
@@ -534,7 +536,9 @@ def _import_event(self, source_event, events):
         self._update_event_data(superevent, source_event)
         superevent["origin_id"] = superevent_source_id
         superevent["super_event_type"] = Event.SuperEventType.RECURRING
-        superevent["info_url"]["fi"] = source_event["EventSerieLink"]
+        superevent["info_url"]["fi"] = source_event["EventSerieLinkFi"]
+        superevent["info_url"]["sv"] = source_event["EventSerieLinkSv"]
+        superevent["info_url"]["en"] = source_event["EventSerieLinkEn"]
 
         self.sub_event_count_by_super_event_source_id[superevent_source_id] += 1
 
@@ -636,33 +640,19 @@ def _synch_events(self, events):
         self.syncher.finish(force=self.options["force"])
 
     def import_events(self):
-        if not LIPPUPISTE_EVENT_API_URL:
+        if not settings.LIPPUPISTE_EVENT_API_URL:
             raise ImproperlyConfigured(
                 "LIPPUPISTE_EVENT_API_URL must be set in environment or config file"
             )
         logger.info("Importing Lippupiste events")
         events = recur_dict()
         event_source_data = list(
-            self._fetch_event_source_data(LIPPUPISTE_EVENT_API_URL)
+            self._fetch_event_source_data(settings.LIPPUPISTE_EVENT_API_URL)
         )
         if not event_source_data:
             raise ValidationError("Lippupiste API didn't return data, giving up")
 
         for source_event in event_source_data:
-            # check if the postal code matches
-            for range in POSTAL_CODE_RANGES:
-                if (
-                    source_event["EventZip"].isdigit()
-                    and range[0] <= int(source_event["EventZip"])
-                    and range[1] >= int(source_event["EventZip"])
-                ):
-                    break
-            else:
-                # no match, ignored
-                continue
-            # check if provider matches
-            if source_event["EventPromoterName"].lower() not in PROVIDERS_TO_IMPORT:
-                continue
             # check if we should ignore the event by name
             for provider in NAMES_TO_IGNORE_BY_PROVIDER:
                 if source_event["EventPromoterName"].lower() == provider.lower():

events/tests/importers/fixtures/lippupiste_response.json

@@ -0,0 +1,53 @@
+{
+  "EventCalendar": "EventCalendar",
+  "Events": [
+    {
+      "EventPromoterId": "999",
+      "EventPromoterName": "Helsingin Kaupunginteatteri",
+      "EventSerieId": "1231231",
+      "EventId": "11116307",
+      "EventName": "PIENET JUHLAT",
+      "EventDate": "24.04.2020",
+      "EventDayOfWeek": "Pe",
+      "EventTime": "19:00",
+      "EventDeliverable": "1",
+      "EventStatus": "2",
+      "EventVenue": "Testi Teatteri, Suuri näyttämö",
+      "EventStreet": "Testitie 123",
+      "EventZip": "1337",
+      "EventPlace": "Tampere",
+      "EventLinkFi": "https://www.lippu.fi/tickets.html?affiliate=adv&fun=evdetail&doc=evdetailb&key=1231231$11116307&language=fi",
+      "EventLinkSv": "https://www.lippu.fi/tickets.html?affiliate=adv&fun=evdetail&doc=evdetailb&key=1231231$11116307&language=sv",
+      "EventLinkEn": "https://www.lippu.fi/tickets.html?affiliate=adv&fun=evdetail&doc=evdetailb&key=1231231$11116307&language=en",
+      "LinkEventUrlFi": "",
+      "EventCapacity": "313",
+      "EventAvailable": "100",
+      "EventSold": "200",
+      "EventReserved": "13",
+      "EventSeriePictureSmall_60x60": "https://www.lippu.fi/obj/media/FI-eventim/teaser/blank.gif",
+      "EventSeriePicture_142x180": "https://www.lippu.fi/obj/media/FI-eventim/teaser/blank.gif",
+      "EventSeriePictureBig_222x222": "https://www.lippu.fi/obj/media/FI-eventim/teaser/blank.gif",
+      "EventSerieInfo": "<b>Huom!</b>Muista ottaa liput mukaan",
+      "EventSerieText": ",<b>Pienet juhlat</b> on klassikkonäytelmä \"hurmuri\" Taunosta",
+      "EventSearchText": "Testi Teatteri Pienet juhlat Tampere",
+      "EventSerieLinkFi": "https://www.lippu.fi/tickets.html?affiliate=adv&doc=erdetaila&fun=erdetail&erid=1231231&language=fi",
+      "EventSerieLinkSv": "https://www.lippu.fi/tickets.html?affiliate=adv&doc=erdetaila&fun=erdetail&erid=1231231&language=sv",
+      "EventSerieLinkEn": "https://www.lippu.fi/tickets.html?affiliate=adv&doc=erdetaila&fun=erdetail&erid=1231231&language=en",
+      "TdlEventSerieId": "234234",
+      "TdlEventId": "123123",
+      "LiveXLinkFi": "",
+      "LiveXLinkSv": "",
+      "LiveXLinkEn": "",
+      "Categories": [
+        {
+          "CategoryId": "2B",
+          "Name": "Draama"
+        },
+        {
+          "CategoryId": "XY",
+          "Name": "Peruutukset ja muutokset"
+        }
+      ]
+    }
+  ]
+}

events/tests/importers/test_lippupiste.py

@@ -0,0 +1,70 @@
+import json
+from copy import deepcopy
+
+import pytest
+from django.conf import settings
+
+from events.importer.lippupiste import LippupisteImporter
+from events.models import Event
+from events.tests.factories import DataSourceFactory, KeywordFactory
+
+
+@pytest.fixture
+def importer():
+    importer = LippupisteImporter({"force": False})
+    importer.setup()
+    return importer
+
+
+@pytest.fixture(autouse=True)
+def tprek_datasource():
+    return DataSourceFactory(id="tprek")
+
+
+@pytest.fixture
+def yso_datasource():
+    return DataSourceFactory(id="yso")
+
+
+@pytest.fixture
+def drama_keyword(yso_datasource):
+    return KeywordFactory(id="yso:p2625", data_source=yso_datasource, name="Draama")
+
+
+@pytest.fixture
+def response_with_one_event(request):
+    with open(request.path.parent / "fixtures/lippupiste_response.json", "r") as f:
+        return json.load(f)
+
+
+@pytest.fixture
+def response_with_two_events_same_super_event(response_with_one_event):
+    response = deepcopy(response_with_one_event)
+    event_2 = deepcopy(response["Events"][0])
+    event_2["EventId"] += "1"
+    response["Events"].append(event_2)
+    return response
+
+
+@pytest.mark.django_db
+def test_lippupiste_event_parse(
+    requests_mock, drama_keyword, importer, response_with_one_event
+):
+    requests_mock.get(settings.LIPPUPISTE_EVENT_API_URL, json=response_with_one_event)
+    importer.import_events()
+
+    events = Event.objects.all()
+    assert events.count() == 1
+    assert drama_keyword in events[0].keywords.all()
+
+
+@pytest.mark.django_db
+def test_lippupiste_super_event(
+    requests_mock, importer, response_with_two_events_same_super_event
+):
+    requests_mock.get(
+        settings.LIPPUPISTE_EVENT_API_URL,
+        json=response_with_two_events_same_super_event,
+    )
+    importer.import_events()
+    assert Event.objects.all().count() == 3

linkedevents/settings.py

@@ -115,6 +115,10 @@ def sentry_anonymize_user_repr(obj, hint):
     INTERNAL_IPS=(list, []),
     LANGUAGES=(list, ["fi", "sv", "en", "zh-hans", "ru", "ar"]),
     LIPPUPISTE_EVENT_API_URL=(str, None),
+    LIPPUPISTE_EVENT_API_CLIENT_ID=(str, None),
+    LIPPUPISTE_EVENT_API_CALENDAR_KEY=(str, None),
+    LIPPUPISTE_EVENT_API_PASSWORD=(str, None),
+    LIPPUPISTE_EVENT_API_USERNAME=(str, None),
     LINKED_EVENTS_UI_URL=(str, "https://linkedevents.hel.fi"),
     LINKED_REGISTRATIONS_UI_URL=(
         str,
@@ -509,6 +513,10 @@ def sentry_anonymize_user_repr(obj, hint):
 
 # Used in Lippupiste importer
 LIPPUPISTE_EVENT_API_URL = env("LIPPUPISTE_EVENT_API_URL")
+LIPPUPISTE_EVENT_API_CLIENT_ID = env("LIPPUPISTE_EVENT_API_CLIENT_ID")
+LIPPUPISTE_EVENT_API_CALENDAR_KEY = env("LIPPUPISTE_EVENT_API_CALENDAR_KEY")
+LIPPUPISTE_EVENT_API_PASSWORD = env("LIPPUPISTE_EVENT_API_PASSWORD")
+LIPPUPISTE_EVENT_API_USERNAME = env("LIPPUPISTE_EVENT_API_USERNAME")
 
 # Seat reservation duration in minutes
 SEAT_RESERVATION_DURATION = env("SEAT_RESERVATION_DURATION")

linkedevents/test_settings.py

@@ -48,3 +48,5 @@ def dummy_haystack_connection_without_warnings_for_lang(language_code):
 WEB_STORE_API_KEY = "abcd"
 WEB_STORE_API_NAMESPACE = "test"
 WEB_STORE_WEBHOOK_API_KEY = "1234"
+
+LIPPUPISTE_EVENT_API_URL = "http://lippupiste.localhost/"