diff --git a/tutorcairn/patches/k8s-deployments b/tutorcairn/patches/k8s-deployments index 0f98780..c6bf9be 100644 --- a/tutorcairn/patches/k8s-deployments +++ b/tutorcairn/patches/k8s-deployments @@ -337,11 +337,11 @@ metadata: spec: selector: matchLabels: - app.kubernetes.io/name: cairn-cairn-watchcourses + app.kubernetes.io/name: cairn-watchcourses template: metadata: labels: - app.kubernetes.io/name: cairn-cairn-watchcourses + app.kubernetes.io/name: cairn-watchcourses spec: containers: - name: cairn-watchcourses @@ -362,6 +362,8 @@ spec: subPath: auth.json securityContext: allowPrivilegeEscalation: false + ports: + - containerPort: 9282 volumes: - name: settings-lms configMap: diff --git a/tutorcairn/patches/k8s-services b/tutorcairn/patches/k8s-services index 1a076e0..60fccd0 100644 --- a/tutorcairn/patches/k8s-services +++ b/tutorcairn/patches/k8s-services @@ -49,7 +49,7 @@ kind: Service metadata: name: cairn-watchcourses spec: - type: ClusterIP + type: NodePort ports: - port: 9282 protocol: TCP diff --git a/tutorcairn/patches/k8s-volumes b/tutorcairn/patches/k8s-volumes index 0a1c22f..924d5e7 100644 --- a/tutorcairn/patches/k8s-volumes +++ b/tutorcairn/patches/k8s-volumes @@ -44,17 +44,3 @@ spec: requests: storage: 2Gi {% endif %} ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: cairn-watchcourses - labels: - app.kubernetes.io/component: volume - app.kubernetes.io/name: cairn-watchcourses -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/tutorcairn/templates/cairn/apps/openedx/scripts/server.py b/tutorcairn/templates/cairn/apps/openedx/scripts/server.py index 863c51a..49f2d16 100644 --- a/tutorcairn/templates/cairn/apps/openedx/scripts/server.py +++ b/tutorcairn/templates/cairn/apps/openedx/scripts/server.py @@ -1,13 +1,26 @@ from aiohttp import web import subprocess +import logging from opaque_keys.edx.locator import CourseLocator -async def import_course_to_clickhouse(request): +# Configure logging +logging.basicConfig(level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Get a logger instance +log = logging.getLogger(__name__) + +async def import_courses_to_clickhouse(request): data = await request.json() - if not isinstance(data, list) or 'course_id' not in data[0]: - return web.json_response({"error":"Value course_id is required."}, status=400) - + if not isinstance(data, list): + return web.json_response({"error": f"Incorrect data format. Expected list, got {data.__class__}."}, status=400) + course_ids = [] + for course in data: + course_id = course.get("course_id") + if not isinstance(course_id, str): + return web.json_response({"error": f"Incorrect course_id format. Expected str, got {course_id.__class__}."}, status=400) + # Get the list of unique course_ids unique_courses = list({course['course_id']: course for course in data}.values()) @@ -18,26 +31,17 @@ async def import_course_to_clickhouse(request): # Verify course_id is a valid course_id try: CourseLocator.from_string(course_id) - except: - continue + except Exception as e: + log.exception(f"An error occured: {str(e)}") + return web.json_response({"error": f"Incorrect arguments. Expected valid course_id, got {course_id}."}, status=400) course_ids.append(course_id) - - # If none of the course_ids are valid, return an error - if not course_ids: - return web.json_response({"error": f"Invalid course_id"}, status=400) - command = ["python", "/openedx/scripts/importcoursedata.py", "-c"] - command.extend(course_ids) - - try: - subprocess.run(command) - return web.json_response({"result": "success"}, status=200) - except Exception as e: - return web.json_response({'error': str(e)}, status=400) + subprocess.run(["python", "/openedx/scripts/importcoursedata.py", "-c", *course_ids], check=True) + return web.Response(status=204) app = web.Application() -app.router.add_post('/import_course/', import_course_to_clickhouse) +app.router.add_post('/courses/published/', import_courses_to_clickhouse) web.run_app(app, host='0.0.0.0', port=9282) diff --git a/tutorcairn/templates/cairn/apps/vector/partials/common-post.toml b/tutorcairn/templates/cairn/apps/vector/partials/common-post.toml index 8c98695..ce85ae3 100644 --- a/tutorcairn/templates/cairn/apps/vector/partials/common-post.toml +++ b/tutorcairn/templates/cairn/apps/vector/partials/common-post.toml @@ -86,8 +86,12 @@ type = "http" method = "post" encoding.codec = "json" inputs = ["course_published"] +# Batch events together to reduce the number of times +# the importcoursedata script is run +# Vector will wait 300 secs (5 mins) from the first event +# or until there are 10 events to trigger the watchcourses service batch.timeout_secs = 300 batch.max_events = 10 -uri = "http://cairn-watchcourses:9282/import_course/" +uri = "http://cairn-watchcourses:9282/courses/published/" {{ patch("cairn-vector-common-toml") }}