From 9aad538d892f1f18bd7bb0b5b897f15da20aa0e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Halil=20=C4=B0brahim=20Kasapo=C4=9Flu?= Date: Mon, 29 Apr 2024 10:48:45 +0300 Subject: [PATCH 1/6] Update docker-compose.yml --- backend/project/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/project/docker-compose.yml b/backend/project/docker-compose.yml index 869fef21..c38bede7 100644 --- a/backend/project/docker-compose.yml +++ b/backend/project/docker-compose.yml @@ -17,7 +17,7 @@ services: context: . dockerfile: Dockerfile container_name: semanticflix_backend - command: sh -c "python3 manage.py migrate --noinput && python3 manage.py collectstatic --noinput && python manage.py runserver 0.0.0.0:8000" + command: sh -c "python manage.py migrate --noinput && python manage.py collectstatic --noinput && python manage.py runserver 0.0.0.0:8000" restart: always volumes: - .:/app From 5b6cdd63a521ddba6f806b2e561877c8084fb141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Halil=20=C4=B0brahim=20Kasapo=C4=9Flu?= Date: Mon, 29 Apr 2024 10:55:17 +0300 Subject: [PATCH 2/6] Create wikidata.py --- backend/project/app/wikidata.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 backend/project/app/wikidata.py diff --git a/backend/project/app/wikidata.py b/backend/project/app/wikidata.py new file mode 100644 index 00000000..ecad1a69 --- /dev/null +++ b/backend/project/app/wikidata.py @@ -0,0 +1,30 @@ +# This helper class is used to interact with the Wikidata API +import requests +import json +from typing import List + +QUERY = """ +SELECT ?item ?itemLabel ?itemDescription ?itemAltLabel WHERE { + ?item wdt:P31 wd:Q11424. + SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } + } +""" + +class Wikidata: + def __init__(self): + self.url = "https://www.wikidata.org/w/api.php" + self.params = { + "action": "wbsearchentities", + "format": "json", + "language": "en", + } + + # Send a semantic query to the Wikidata API + def query(self, query: str) -> List[str]: + self.params["search"] = query + response = requests.get(url=self.url, params=self.params) + data = response.json() + return data["search"] + + + From c5fa952df30a57f05bcbc184221859d14a2c1a05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Halil=20=C4=B0brahim=20Kasapo=C4=9Flu?= Date: Mon, 29 Apr 2024 12:08:01 +0300 Subject: [PATCH 3/6] feat: add endpoint for wikidata queries + added a sample query file + one can send sparql queries through wikidata-query endpoint --- backend/project/app/serializers.py | 3 +++ backend/project/app/urls.py | 4 +++- backend/project/app/views.py | 30 +++++++++++++++++++++++++++++- backend/project/app/wikidata.py | 21 ++++++++++++++------- sample_queries.txt | 5 +++++ 5 files changed, 54 insertions(+), 9 deletions(-) create mode 100644 sample_queries.txt diff --git a/backend/project/app/serializers.py b/backend/project/app/serializers.py index f940c1df..62b16366 100644 --- a/backend/project/app/serializers.py +++ b/backend/project/app/serializers.py @@ -77,3 +77,6 @@ class Meta: model = Actor fields = ['name', 'surname', 'description'] + +class WikidataQuerySerializer(serializers.Serializer): + query = serializers.CharField() \ No newline at end of file diff --git a/backend/project/app/urls.py b/backend/project/app/urls.py index 338027a5..322653d6 100644 --- a/backend/project/app/urls.py +++ b/backend/project/app/urls.py @@ -3,10 +3,11 @@ from app import views from django.urls import path from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView -from .views import film_api, film_detail_api, RegisterView +from .views import film_api, film_detail_api, RegisterView, execute_query from .views import MyObtainTokenPairView from rest_framework_simplejwt.views import TokenRefreshView + urlpatterns = [ path('film/schema/', SpectacularAPIView.as_view(), name='schema'), path('film/schema/swagger-ui/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger-ui'), @@ -15,4 +16,5 @@ path('login/', MyObtainTokenPairView.as_view(), name='token_obtain_pair'), path('login/refresh/', TokenRefreshView.as_view(), name='token_refresh'), path('register/', RegisterView.as_view(), name='auth_register'), + path('wikidata-query/', execute_query, name='wikidata-query'), ] diff --git a/backend/project/app/views.py b/backend/project/app/views.py index 01661e95..078e6d8f 100644 --- a/backend/project/app/views.py +++ b/backend/project/app/views.py @@ -6,7 +6,7 @@ from django.http.response import JsonResponse from django.contrib.auth.models import User from app.models import Film, Genre, Director, Actor -from app.serializers import UserSerializer, FilmSerializer, GenreSerializer, DirectorSerializer, ActorSerializer +from app.serializers import UserSerializer, FilmSerializer, GenreSerializer, DirectorSerializer, ActorSerializer, WikidataQuerySerializer from rest_framework.decorators import api_view, permission_classes from drf_spectacular.utils import extend_schema from .serializers import MyTokenObtainPairSerializer @@ -15,6 +15,9 @@ from rest_framework.permissions import IsAuthenticated from .serializers import RegisterSerializer from rest_framework import generics +from app.wikidata import WikidataAPI +from rest_framework.response import Response +from rest_framework import status class RegisterView(generics.CreateAPIView): @@ -93,3 +96,28 @@ def film_detail_api(request, id): return JsonResponse("Film Deleted Successfully", safe=False) +# A simple endpoint for sending a semantic query to the Wikidata API +@extend_schema( + description="API endpoint for sending a semantic query to the Wikidata API.", + methods=['POST'], + request=WikidataQuerySerializer, +) +@api_view(['POST']) +def execute_query(request): + """ + Allow users to send a semantic query to the Wikidata API. + """ + if request.method == 'POST': + serializer = WikidataQuerySerializer(data=request.data) + if serializer.is_valid(): + query_text = serializer.validated_data.get('query') + + # Execute the query using the WikidataAPI class + wikidata_api = WikidataAPI() + results = wikidata_api.execute_query(query_text) + + print(results) + + return Response(results) + else: + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) diff --git a/backend/project/app/wikidata.py b/backend/project/app/wikidata.py index ecad1a69..774bb8f3 100644 --- a/backend/project/app/wikidata.py +++ b/backend/project/app/wikidata.py @@ -10,9 +10,9 @@ } """ -class Wikidata: +class WikidataAPI: def __init__(self): - self.url = "https://www.wikidata.org/w/api.php" + self.endpoint_url = "https://query.wikidata.org/sparql" self.params = { "action": "wbsearchentities", "format": "json", @@ -20,11 +20,18 @@ def __init__(self): } # Send a semantic query to the Wikidata API - def query(self, query: str) -> List[str]: - self.params["search"] = query - response = requests.get(url=self.url, params=self.params) - data = response.json() - return data["search"] + def execute_query(self, query): + try: + response = requests.get( + self.endpoint_url, + params={'query': query, 'format': 'json'} + ) + response.raise_for_status() + print(response) + return response.json() + except requests.exceptions.RequestException as e: + print("Error:", e) + return None diff --git a/sample_queries.txt b/sample_queries.txt new file mode 100644 index 00000000..1ed6b238 --- /dev/null +++ b/sample_queries.txt @@ -0,0 +1,5 @@ +======= Films played by dicaprio +{ + "query": "SELECT ?film ?filmLabel WHERE { ?film wdt:P31 wd:Q11424; wdt:P161 wd:Q38111. SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\". } }" +} +======= \ No newline at end of file From 3b5069bacff0b6e3ba506595c608d3b39233e41e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Halil=20=C4=B0brahim=20Kasapo=C4=9Flu?= Date: Mon, 29 Apr 2024 12:08:21 +0300 Subject: [PATCH 4/6] fix: moved docker compose to outer directory --- backend/project/docker-compose.yml => docker-compose.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) rename backend/project/docker-compose.yml => docker-compose.yml (78%) diff --git a/backend/project/docker-compose.yml b/docker-compose.yml similarity index 78% rename from backend/project/docker-compose.yml rename to docker-compose.yml index d923cc7f..022fb457 100644 --- a/backend/project/docker-compose.yml +++ b/docker-compose.yml @@ -17,11 +17,12 @@ services: - backend-network backend: + image: semanticflix_backend build: - context: . + context: ./backend/project dockerfile: Dockerfile container_name: semanticflix_backend - command: sh -c "python manage.py migrate --noinput && python manage.py collectstatic --noinput && python manage.py runserver 0.0.0.0:8000" + command: sh -c "python backend/project/manage.py migrate --noinput && python backend/project/manage.py collectstatic --noinput && python backend/project/manage.py runserver 0.0.0.0:8000" restart: always volumes: - .:/app From 09bddd8122e500cf0a47a658e0068e372da13f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Halil=20=C4=B0brahim=20Kasapo=C4=9Flu?= Date: Mon, 29 Apr 2024 17:29:18 +0300 Subject: [PATCH 5/6] feat: implement film search on qlever --- backend/project/app/qlever.py | 73 ++++++++++++++++++++++++++++++ backend/project/app/serializers.py | 6 ++- backend/project/app/urls.py | 9 ++-- backend/project/app/views.py | 45 +++++++++++++++++- sample_queries.txt | 51 ++++++++++++++++++++- 5 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 backend/project/app/qlever.py diff --git a/backend/project/app/qlever.py b/backend/project/app/qlever.py new file mode 100644 index 00000000..97e62829 --- /dev/null +++ b/backend/project/app/qlever.py @@ -0,0 +1,73 @@ +# This helper class is used to interact with the Wikidata API +import requests +import json +from typing import List + +class QleverAPI: + def __init__(self): + self.endpoint_url = "https://qlever.cs.uni-freiburg.de/api/wikidata" + self.params = { + "action": "wbsearchentities", + "format": "json", + "language": "en", + } + + # Send a semantic query to the Wikidata API + def execute_query(self, query): + try: + response = requests.get( + self.endpoint_url, + params={'query': query, 'format': 'json'} + ) + response.raise_for_status() + print(response) + return response.json() + except requests.exceptions.RequestException as e: + print("Error:", e) + return None + + def film_pattern_query(self, pattern, limit): + + pattern = pattern.lower() + # remove spaces from the pattern + pattern = pattern.replace(" ", "") + + SPARQL = f""" + PREFIX rdfs: + PREFIX wd: + PREFIX wdt: + SELECT DISTINCT ?film ?filmLabel ?filmId WHERE {{ + {{ + SELECT ?film ?filmLabel ?filmId (1 as ?order) WHERE {{ + ?film wdt:P31 wd:Q11424; + rdfs:label ?filmLabel. + FILTER(LANG(?filmLabel) = "en") + FILTER(STRSTARTS(REPLACE(LCASE(?filmLabel), " ", ""), "{pattern}")) + }} + }} + UNION + {{ + SELECT ?film ?filmLabel ?filmId (2 as ?order) WHERE {{ + ?film wdt:P31 wd:Q11424; + rdfs:label ?filmLabel. + FILTER(LANG(?filmLabel) = "en") + BIND(REPLACE(LCASE(?filmLabel), " ", "") AS ?formattedLabel) + FILTER(REGEX(?formattedLabel, "{pattern}", "i")) + FILTER (!STRSTARTS(REPLACE(LCASE(?filmLabel), " ", ""), "{pattern}")) + }} + }} + }} + ORDER BY ?order + LIMIT {limit} + """ + + print(SPARQL) + + results = self.execute_query(SPARQL) + + return results + + + + + diff --git a/backend/project/app/serializers.py b/backend/project/app/serializers.py index 62b16366..5f761eba 100644 --- a/backend/project/app/serializers.py +++ b/backend/project/app/serializers.py @@ -79,4 +79,8 @@ class Meta: class WikidataQuerySerializer(serializers.Serializer): - query = serializers.CharField() \ No newline at end of file + query = serializers.CharField() + +class FilmPatternWithLimitQuerySerializer(serializers.Serializer): + pattern = serializers.CharField() + limit = serializers.IntegerField() \ No newline at end of file diff --git a/backend/project/app/urls.py b/backend/project/app/urls.py index 322653d6..c24f993e 100644 --- a/backend/project/app/urls.py +++ b/backend/project/app/urls.py @@ -3,7 +3,7 @@ from app import views from django.urls import path from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView -from .views import film_api, film_detail_api, RegisterView, execute_query +from .views import film_api, film_detail_api, RegisterView, execute_query, query_film_pattern from .views import MyObtainTokenPairView from rest_framework_simplejwt.views import TokenRefreshView @@ -13,8 +13,9 @@ path('film/schema/swagger-ui/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger-ui'), path('film/', film_api, name='film-list'), path('film//', film_detail_api, name='film-detail'), - path('login/', MyObtainTokenPairView.as_view(), name='token_obtain_pair'), + path('login/', MyObtainTokenPairView.as_view(), name='token_obtain_pair'), path('login/refresh/', TokenRefreshView.as_view(), name='token_refresh'), - path('register/', RegisterView.as_view(), name='auth_register'), - path('wikidata-query/', execute_query, name='wikidata-query'), + path('register/', RegisterView.as_view(), name='auth_register'), + path('wikidata-query/', execute_query, name='wikidata-query'), + path('query-film-pattern/', query_film_pattern, name='query-film-pattern'), ] diff --git a/backend/project/app/views.py b/backend/project/app/views.py index 078e6d8f..73de3523 100644 --- a/backend/project/app/views.py +++ b/backend/project/app/views.py @@ -6,7 +6,7 @@ from django.http.response import JsonResponse from django.contrib.auth.models import User from app.models import Film, Genre, Director, Actor -from app.serializers import UserSerializer, FilmSerializer, GenreSerializer, DirectorSerializer, ActorSerializer, WikidataQuerySerializer +from app.serializers import UserSerializer, FilmSerializer, GenreSerializer, DirectorSerializer, ActorSerializer, WikidataQuerySerializer, FilmPatternWithLimitQuerySerializer from rest_framework.decorators import api_view, permission_classes from drf_spectacular.utils import extend_schema from .serializers import MyTokenObtainPairSerializer @@ -16,6 +16,7 @@ from .serializers import RegisterSerializer from rest_framework import generics from app.wikidata import WikidataAPI +from app.qlever import QleverAPI from rest_framework.response import Response from rest_framework import status @@ -121,3 +122,45 @@ def execute_query(request): return Response(results) else: return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + +# Find films with a pattern string and a limit value +@extend_schema( + description="API endpoint for finding films with a pattern string and a limit value.", + methods=['POST'], + request=FilmPatternWithLimitQuerySerializer, +) +@api_view(['POST']) +def query_film_pattern(request): + """ + Find films with a pattern string and a limit value using Qlever. + """ + if request.method == 'POST': + serializer = FilmPatternWithLimitQuerySerializer(data=request.data) + if serializer.is_valid(): + pattern = serializer.validated_data.get('pattern') + limit = serializer.validated_data.get('limit') + + # Execute the query using the Qlever class + qlever = QleverAPI() + results = qlever.film_pattern_query(pattern, limit) + + print(results) + + # change response format + # get only film ids and labels + results = results['results']['bindings'] + films = [] + for result in results: + film = { + 'id': result['film']['value'], + 'label': result['filmLabel']['value'] + } + films.append(film) + return Response(films) + + else: + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + + diff --git a/sample_queries.txt b/sample_queries.txt index 1ed6b238..30e987d7 100644 --- a/sample_queries.txt +++ b/sample_queries.txt @@ -2,4 +2,53 @@ { "query": "SELECT ?film ?filmLabel WHERE { ?film wdt:P31 wd:Q11424; wdt:P161 wd:Q38111. SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\". } }" } -======= \ No newline at end of file +======= + + +===== Film ids starting with the given string +{ + "query": "SELECT DISTINCT ?film ?filmLabel ?filmId WHERE { ?film wdt:P31 wd:Q11424; rdfs:label ?filmLabel. FILTER(STRSTARTS(?filmLabel, \"OPP\")) OPTIONAL { ?film wdt:P345 ?filmId. } SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\". } } LIMIT 10" +} + +=== Better version of it +SELECT DISTINCT ?film ?filmLabel ?filmId WHERE { + ?film wdt:P31 wd:Q11424; # Instance of film + rdfs:label ?filmLabel. # Label of the film + FILTER(LANG(?filmLabel) = "en") # Filter out non-English labels + FILTER(STRSTARTS(REPLACE(LCASE(?filmLabel), " ", ""), "opportunityk")) # Matches films with labels starting with "A" (case-insensitive and ignoring spaces) +} +LIMIT 3 +==== + + +==== +Final film listing sparql + +PREFIX rdfs: +PREFIX wd: +PREFIX wdt: +SELECT DISTINCT ?film ?filmLabel ?filmId WHERE { + { + SELECT ?film ?filmLabel ?filmId (1 as ?order) WHERE { + ?film wdt:P31 wd:Q11424; # Instance of film + rdfs:label ?filmLabel. # Label of the film + FILTER(LANG(?filmLabel) = "en") # Filter out non-English labels + FILTER(STRSTARTS(REPLACE(LCASE(?filmLabel), " ", ""), "oppen")) # Matches labels starting with "oppen" (case-insensitive and ignoring spaces) + } + } + UNION + { + SELECT ?film ?filmLabel ?filmId (2 as ?order) WHERE { + ?film wdt:P31 wd:Q11424; # Instance of film + rdfs:label ?filmLabel. # Label of the film + FILTER(LANG(?filmLabel) = "en") # Filter out non-English labels + FILTER(REGEX(?filmLabel, "oppen", "i")) # Matches labels containing "oppen" (case-insensitive) + FILTER (!STRSTARTS(REPLACE(LCASE(?filmLabel), " ", ""), "oppen")) # Ensure it's not already matched by starts with + } + } +} +ORDER BY ?order +LIMIT 3 + + +==== \ No newline at end of file From b14b1628bb770699653d481046322fd7c629d31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0REM=20NUR=20YILDIRIM?= Date: Mon, 29 Apr 2024 20:14:45 +0300 Subject: [PATCH 6/6] fix in import and settings --- backend/project/app/views.py | 3 ++- backend/project/project/settings.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/project/app/views.py b/backend/project/app/views.py index 25bc3aee..242921b0 100644 --- a/backend/project/app/views.py +++ b/backend/project/app/views.py @@ -7,7 +7,8 @@ from django.http.response import JsonResponse from django.contrib.auth.models import User from app.models import Film, Genre, Director, Actor -from app.serializers import UserSerializer, FilmSerializer, GenreSerializer, DirectorSerializer, ActorSerializer,WikidataQuerySerializer, FilmPatternWithLimitQuerySerializer +from app.serializers import * +#from app.serializers import UserSerializer, FilmSerializer, GenreSerializer, DirectorSerializer, ActorSerializer,WikidataQuerySerializer, FilmPatternWithLimitQuerySerializer, MyTokenObtainPairSerializer, LogoutSerializer from rest_framework import permissions, status , viewsets, generics from rest_framework.response import Response from rest_framework_simplejwt.tokens import RefreshToken diff --git a/backend/project/project/settings.py b/backend/project/project/settings.py index a35eba52..b02fde3a 100644 --- a/backend/project/project/settings.py +++ b/backend/project/project/settings.py @@ -44,6 +44,12 @@ 'PASSWORD': 'password', # MySQL password 'HOST': 'db', # Host where MySQL is running (in this case, Docker container) 'PORT': '3306', # Port where MySQL is running (in this case, Docker container) + + } +} +""" + +""" DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql',