diff --git a/events/importer/lippupiste.py b/events/importer/lippupiste.py
index 5ee063dd4..98f09a3b1 100644
--- a/events/importer/lippupiste.py
+++ b/events/importer/lippupiste.py
@@ -1,5 +1,3 @@
-import codecs
-import csv
import logging
import re
from collections import defaultdict
@@ -81,12 +79,6 @@
"Helsingin Vanha kirkko": "tprek:43184", # String match fails
}
-# By default, only import events in the capital region
-POSTAL_CODE_RANGES = ((1, 990), (1200, 1770), (2100, 2380), (2600, 2860), (2920, 2980))
-
-# By default, only import agreed providers (lowercase required!)
-PROVIDERS_TO_IMPORT = ("helsingin kaupunginteatteri",)
-
NAMES_TO_IGNORE_BY_PROVIDER = {
"Helsingin kaupunginteatteri": (
"käsiohjelma",
@@ -262,18 +254,22 @@ def setup(self):
self.sub_event_count_by_super_event_source_id = defaultdict(lambda: 0)
def _fetch_event_source_data(self, url):
- # stream=True allows lazy iteration
- response = requests.get(url, stream=True, timeout=self.default_timeout)
- response_iter = response.iter_lines()
- # CSV reader wants str instead of byte, let's decode
- decoded_response_iter = codecs.iterdecode(response_iter, "utf-8")
- reader = csv.DictReader(
- decoded_response_iter, delimiter=";", quotechar='"', doublequote=True
- )
- return reader
+ return requests.get(
+ url,
+ auth=(
+ settings.LIPPUPISTE_EVENT_API_USERNAME,
+ settings.LIPPUPISTE_EVENT_API_PASSWORD,
+ ),
+ params={"ClientID": settings.LIPPUPISTE_EVENT_API_CLIENT_ID},
+ timeout=self.default_timeout,
+ headers={
+ "Content-Type": "application/json",
+ "CALENDARKey": settings.LIPPUPISTE_EVENT_API_CALENDAR_KEY,
+ },
+ ).json()["Events"]
def _get_keywords_from_source_category(self, source_category):
- source_category_key = source_category.lower()
+ source_category_key = source_category["Name"].lower()
keyword_set = set()
for keyword_id in YSO_KEYWORD_MAPS.get(source_category_key, []):
if keyword_id in self.keyword_by_id:
@@ -282,7 +278,6 @@ def _get_keywords_from_source_category(self, source_category):
return keyword_set
def _get_keywords_from_source_categories(self, source_categories):
- source_categories = source_categories.split("|")
keyword_set = set()
for category in source_categories:
keyword_set = keyword_set.union(
@@ -480,12 +475,19 @@ def _update_event_data(self, event, source_event):
[lang] + self.languages_to_detect,
"short_description",
)
- event["info_url"][lang] = clean_url(source_event["EventSerieLink"])
+ event["info_url"]["fi"] = clean_url(source_event["EventSerieLinkFi"])
+ event["info_url"]["sv"] = clean_url(source_event["EventSerieLinkSv"])
+ event["info_url"]["en"] = clean_url(source_event["EventSerieLinkEn"])
+
event["offers"] = [
{
"is_free": False,
"description": {lang: "Tarkista hinta lippupalvelusta"},
- "info_url": {lang: clean_url(source_event["EventLink"])},
+ "info_url": {
+ "fi": clean_url(source_event["EventLinkFi"]),
+ "sv": clean_url(source_event["EventLinkSv"]),
+ "en": clean_url(source_event["EventLinkEn"]),
+ },
"price": None,
},
]
@@ -498,7 +500,7 @@ def _update_event_data(self, event, source_event):
existing_keywords = event.get("keywords", set())
keywords_from_source = self._get_keywords_from_source_categories(
- source_event["EventSerieCategories"]
+ source_event["Categories"]
)
event["keywords"] = existing_keywords.union(keywords_from_source)
@@ -534,7 +536,9 @@ def _import_event(self, source_event, events):
self._update_event_data(superevent, source_event)
superevent["origin_id"] = superevent_source_id
superevent["super_event_type"] = Event.SuperEventType.RECURRING
- superevent["info_url"]["fi"] = source_event["EventSerieLink"]
+ superevent["info_url"]["fi"] = source_event["EventSerieLinkFi"]
+ superevent["info_url"]["sv"] = source_event["EventSerieLinkSv"]
+ superevent["info_url"]["en"] = source_event["EventSerieLinkEn"]
self.sub_event_count_by_super_event_source_id[superevent_source_id] += 1
@@ -636,33 +640,19 @@ def _synch_events(self, events):
self.syncher.finish(force=self.options["force"])
def import_events(self):
- if not LIPPUPISTE_EVENT_API_URL:
+ if not settings.LIPPUPISTE_EVENT_API_URL:
raise ImproperlyConfigured(
"LIPPUPISTE_EVENT_API_URL must be set in environment or config file"
)
logger.info("Importing Lippupiste events")
events = recur_dict()
event_source_data = list(
- self._fetch_event_source_data(LIPPUPISTE_EVENT_API_URL)
+ self._fetch_event_source_data(settings.LIPPUPISTE_EVENT_API_URL)
)
if not event_source_data:
raise ValidationError("Lippupiste API didn't return data, giving up")
for source_event in event_source_data:
- # check if the postal code matches
- for range in POSTAL_CODE_RANGES:
- if (
- source_event["EventZip"].isdigit()
- and range[0] <= int(source_event["EventZip"])
- and range[1] >= int(source_event["EventZip"])
- ):
- break
- else:
- # no match, ignored
- continue
- # check if provider matches
- if source_event["EventPromoterName"].lower() not in PROVIDERS_TO_IMPORT:
- continue
# check if we should ignore the event by name
for provider in NAMES_TO_IGNORE_BY_PROVIDER:
if source_event["EventPromoterName"].lower() == provider.lower():
diff --git a/events/tests/importers/fixtures/lippupiste_response.json b/events/tests/importers/fixtures/lippupiste_response.json
new file mode 100644
index 000000000..3dfb3e309
--- /dev/null
+++ b/events/tests/importers/fixtures/lippupiste_response.json
@@ -0,0 +1,53 @@
+{
+ "EventCalendar": "EventCalendar",
+ "Events": [
+ {
+ "EventPromoterId": "999",
+ "EventPromoterName": "Helsingin Kaupunginteatteri",
+ "EventSerieId": "1231231",
+ "EventId": "11116307",
+ "EventName": "PIENET JUHLAT",
+ "EventDate": "24.04.2020",
+ "EventDayOfWeek": "Pe",
+ "EventTime": "19:00",
+ "EventDeliverable": "1",
+ "EventStatus": "2",
+ "EventVenue": "Testi Teatteri, Suuri näyttämö",
+ "EventStreet": "Testitie 123",
+ "EventZip": "1337",
+ "EventPlace": "Tampere",
+ "EventLinkFi": "https://www.lippu.fi/tickets.html?affiliate=adv&fun=evdetail&doc=evdetailb&key=1231231$11116307&language=fi",
+ "EventLinkSv": "https://www.lippu.fi/tickets.html?affiliate=adv&fun=evdetail&doc=evdetailb&key=1231231$11116307&language=sv",
+ "EventLinkEn": "https://www.lippu.fi/tickets.html?affiliate=adv&fun=evdetail&doc=evdetailb&key=1231231$11116307&language=en",
+ "LinkEventUrlFi": "",
+ "EventCapacity": "313",
+ "EventAvailable": "100",
+ "EventSold": "200",
+ "EventReserved": "13",
+ "EventSeriePictureSmall_60x60": "https://www.lippu.fi/obj/media/FI-eventim/teaser/blank.gif",
+ "EventSeriePicture_142x180": "https://www.lippu.fi/obj/media/FI-eventim/teaser/blank.gif",
+ "EventSeriePictureBig_222x222": "https://www.lippu.fi/obj/media/FI-eventim/teaser/blank.gif",
+ "EventSerieInfo": "Huom!Muista ottaa liput mukaan",
+ "EventSerieText": ",Pienet juhlat on klassikkonäytelmä \"hurmuri\" Taunosta",
+ "EventSearchText": "Testi Teatteri Pienet juhlat Tampere",
+ "EventSerieLinkFi": "https://www.lippu.fi/tickets.html?affiliate=adv&doc=erdetaila&fun=erdetail&erid=1231231&language=fi",
+ "EventSerieLinkSv": "https://www.lippu.fi/tickets.html?affiliate=adv&doc=erdetaila&fun=erdetail&erid=1231231&language=sv",
+ "EventSerieLinkEn": "https://www.lippu.fi/tickets.html?affiliate=adv&doc=erdetaila&fun=erdetail&erid=1231231&language=en",
+ "TdlEventSerieId": "234234",
+ "TdlEventId": "123123",
+ "LiveXLinkFi": "",
+ "LiveXLinkSv": "",
+ "LiveXLinkEn": "",
+ "Categories": [
+ {
+ "CategoryId": "2B",
+ "Name": "Draama"
+ },
+ {
+ "CategoryId": "XY",
+ "Name": "Peruutukset ja muutokset"
+ }
+ ]
+ }
+ ]
+}
diff --git a/events/tests/importers/test_lippupiste.py b/events/tests/importers/test_lippupiste.py
new file mode 100644
index 000000000..dd700a700
--- /dev/null
+++ b/events/tests/importers/test_lippupiste.py
@@ -0,0 +1,70 @@
+import json
+from copy import deepcopy
+
+import pytest
+from django.conf import settings
+
+from events.importer.lippupiste import LippupisteImporter
+from events.models import Event
+from events.tests.factories import DataSourceFactory, KeywordFactory
+
+
+@pytest.fixture
+def importer():
+ importer = LippupisteImporter({"force": False})
+ importer.setup()
+ return importer
+
+
+@pytest.fixture(autouse=True)
+def tprek_datasource():
+ return DataSourceFactory(id="tprek")
+
+
+@pytest.fixture
+def yso_datasource():
+ return DataSourceFactory(id="yso")
+
+
+@pytest.fixture
+def drama_keyword(yso_datasource):
+ return KeywordFactory(id="yso:p2625", data_source=yso_datasource, name="Draama")
+
+
+@pytest.fixture
+def response_with_one_event(request):
+ with open(request.path.parent / "fixtures/lippupiste_response.json", "r") as f:
+ return json.load(f)
+
+
+@pytest.fixture
+def response_with_two_events_same_super_event(response_with_one_event):
+ response = deepcopy(response_with_one_event)
+ event_2 = deepcopy(response["Events"][0])
+ event_2["EventId"] += "1"
+ response["Events"].append(event_2)
+ return response
+
+
+@pytest.mark.django_db
+def test_lippupiste_event_parse(
+ requests_mock, drama_keyword, importer, response_with_one_event
+):
+ requests_mock.get(settings.LIPPUPISTE_EVENT_API_URL, json=response_with_one_event)
+ importer.import_events()
+
+ events = Event.objects.all()
+ assert events.count() == 1
+ assert drama_keyword in events[0].keywords.all()
+
+
+@pytest.mark.django_db
+def test_lippupiste_super_event(
+ requests_mock, importer, response_with_two_events_same_super_event
+):
+ requests_mock.get(
+ settings.LIPPUPISTE_EVENT_API_URL,
+ json=response_with_two_events_same_super_event,
+ )
+ importer.import_events()
+ assert Event.objects.all().count() == 3
diff --git a/linkedevents/settings.py b/linkedevents/settings.py
index d4ac98264..594b1a8a8 100644
--- a/linkedevents/settings.py
+++ b/linkedevents/settings.py
@@ -115,6 +115,10 @@ def sentry_anonymize_user_repr(obj, hint):
INTERNAL_IPS=(list, []),
LANGUAGES=(list, ["fi", "sv", "en", "zh-hans", "ru", "ar"]),
LIPPUPISTE_EVENT_API_URL=(str, None),
+ LIPPUPISTE_EVENT_API_CLIENT_ID=(str, None),
+ LIPPUPISTE_EVENT_API_CALENDAR_KEY=(str, None),
+ LIPPUPISTE_EVENT_API_PASSWORD=(str, None),
+ LIPPUPISTE_EVENT_API_USERNAME=(str, None),
LINKED_EVENTS_UI_URL=(str, "https://linkedevents.hel.fi"),
LINKED_REGISTRATIONS_UI_URL=(
str,
@@ -509,6 +513,10 @@ def sentry_anonymize_user_repr(obj, hint):
# Used in Lippupiste importer
LIPPUPISTE_EVENT_API_URL = env("LIPPUPISTE_EVENT_API_URL")
+LIPPUPISTE_EVENT_API_CLIENT_ID = env("LIPPUPISTE_EVENT_API_CLIENT_ID")
+LIPPUPISTE_EVENT_API_CALENDAR_KEY = env("LIPPUPISTE_EVENT_API_CALENDAR_KEY")
+LIPPUPISTE_EVENT_API_PASSWORD = env("LIPPUPISTE_EVENT_API_PASSWORD")
+LIPPUPISTE_EVENT_API_USERNAME = env("LIPPUPISTE_EVENT_API_USERNAME")
# Seat reservation duration in minutes
SEAT_RESERVATION_DURATION = env("SEAT_RESERVATION_DURATION")
diff --git a/linkedevents/test_settings.py b/linkedevents/test_settings.py
index 38d6d5a4e..bc87d5f0d 100644
--- a/linkedevents/test_settings.py
+++ b/linkedevents/test_settings.py
@@ -48,3 +48,5 @@ def dummy_haystack_connection_without_warnings_for_lang(language_code):
WEB_STORE_API_KEY = "abcd"
WEB_STORE_API_NAMESPACE = "test"
WEB_STORE_WEBHOOK_API_KEY = "1234"
+
+LIPPUPISTE_EVENT_API_URL = "http://lippupiste.localhost/"