From 53b7bd5409b70f2ecd6135669ef94ed2c65be0e2 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Mon, 12 Feb 2024 10:07:54 -0500 Subject: [PATCH 01/10] feat: Add kafka to docker compose Nothing is configurable currently and it is not connected to the platform, but this will start all of the required services. --- .../patches/local-docker-compose-services | 67 +++++++++++++++++++ tutorevent_bus_redis/plugin.py | 2 + 2 files changed, 69 insertions(+) create mode 100644 tutorevent_bus_redis/patches/local-docker-compose-services diff --git a/tutorevent_bus_redis/patches/local-docker-compose-services b/tutorevent_bus_redis/patches/local-docker-compose-services new file mode 100644 index 0000000..9a35ccb --- /dev/null +++ b/tutorevent_bus_redis/patches/local-docker-compose-services @@ -0,0 +1,67 @@ +{% if RUN_KAFKA_EVENT_BUS %} +# needed by Kafka to keep track of nodes, topics, and messages. +zookeeper: + image: confluentinc/cp-zookeeper:6.2.1 + ports: + - "2181:2181" + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + +# Events broker +kafka: + image: confluentinc/cp-server:6.2.1 + depends_on: + - zookeeper + ports: + - "9092:9092" + - "9101:9101" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092 + KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_JMX_PORT: 9101 + KAFKA_JMX_HOSTNAME: localhost + KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:18081 + CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092 + CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 + CONFLUENT_METRICS_ENABLE: 'true' + CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' + +# browser app for monitoring local Kafka cluster. This is quite memory- and CPU-intensive, so it should only be used for local Kafka debugging +kafka-control-center: + image: confluentinc/cp-enterprise-control-center:6.2.1 + depends_on: + - kafka + - schema-registry + ports: + - "9021:9021" + environment: + CONTROL_CENTER_BOOTSTRAP_SERVERS: kafka:29092 + CONTROL_CENTER_SCHEMA_REGISTRY_URL: http://schema-registry:18081 + CONTROL_CENTER_REPLICATION_FACTOR: 1 + CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 + CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 + CONFLUENT_METRICS_TOPIC_REPLICATION: 1 + PORT: 9021 + +# storage layer for data schemas in Kafka +schema-registry: + image: confluentinc/cp-schema-registry:6.2.1 + depends_on: + - kafka + ports: + - "18081:18081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka:29092' + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:18081 +{% endif %} diff --git a/tutorevent_bus_redis/plugin.py b/tutorevent_bus_redis/plugin.py index 18b6e66..4835a0a 100644 --- a/tutorevent_bus_redis/plugin.py +++ b/tutorevent_bus_redis/plugin.py @@ -20,6 +20,8 @@ # Each new setting is a pair: (setting_name, default_value). # Prefix your setting names with 'EVENT_BUS_REDIS_'. ("EVENT_BUS_REDIS_VERSION", __version__), + ("RUN_KAFKA_EVENT_BUS", True), + ("RUN_REDIS_EVENT_BUS", True), ] ) From e5d1121ad586e4d2cdad122f70bc258ba486704e Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Mon, 12 Feb 2024 14:50:58 -0500 Subject: [PATCH 02/10] feat: Functional Kafka consumer/producer Still too many hard coded values, but it works when Kafka and the event registry actually start. --- .idea/.gitignore | 8 ++ .../patches/local-docker-compose-services | 28 +++--- .../patches/openedx-common-settings | 27 ++++++ ...penedx-dockerfile-post-python-requirements | 0 tutorevent_bus_redis/plugin.py | 95 ++++++++++++++++++- 5 files changed, 143 insertions(+), 15 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 tutorevent_bus_redis/patches/openedx-common-settings create mode 100644 tutorevent_bus_redis/patches/openedx-dockerfile-post-python-requirements diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/tutorevent_bus_redis/patches/local-docker-compose-services b/tutorevent_bus_redis/patches/local-docker-compose-services index 9a35ccb..7e12281 100644 --- a/tutorevent_bus_redis/patches/local-docker-compose-services +++ b/tutorevent_bus_redis/patches/local-docker-compose-services @@ -1,4 +1,4 @@ -{% if RUN_KAFKA_EVENT_BUS %} +{% if RUN_KAFKA_SERVER %} # needed by Kafka to keep track of nodes, topics, and messages. zookeeper: image: confluentinc/cp-zookeeper:6.2.1 @@ -36,6 +36,20 @@ kafka: CONFLUENT_METRICS_ENABLE: 'true' CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' +# storage layer for data schemas in Kafka +schema-registry: + image: confluentinc/cp-schema-registry:6.2.1 + depends_on: + - kafka + ports: + - "18081:18081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka:29092' + SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:18081 +{% endif %} + +{% if RUN_KAFKA_SERVER and RUN_KAFKA_UI %} # browser app for monitoring local Kafka cluster. This is quite memory- and CPU-intensive, so it should only be used for local Kafka debugging kafka-control-center: image: confluentinc/cp-enterprise-control-center:6.2.1 @@ -52,16 +66,4 @@ kafka-control-center: CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 CONFLUENT_METRICS_TOPIC_REPLICATION: 1 PORT: 9021 - -# storage layer for data schemas in Kafka -schema-registry: - image: confluentinc/cp-schema-registry:6.2.1 - depends_on: - - kafka - ports: - - "18081:18081" - environment: - SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka:29092' - SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:18081 {% endif %} diff --git a/tutorevent_bus_redis/patches/openedx-common-settings b/tutorevent_bus_redis/patches/openedx-common-settings new file mode 100644 index 0000000..05758a1 --- /dev/null +++ b/tutorevent_bus_redis/patches/openedx-common-settings @@ -0,0 +1,27 @@ +# Settings for producing events +SEND_CATALOG_INFO_SIGNAL = {{ EVENT_BUS_SEND_CATALOG_INFO_SIGNAL }} + + +{% if EVENT_BUS_BACKEND == 'redis' %} +# redis connection url +# https://redis.readthedocs.io/en/stable/examples/ssl_connection_examples.html#Connecting-to-a-Redis-instance-via-a-URL-string +EVENT_BUS_REDIS_CONNECTION_URL = "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/" +EVENT_BUS_TOPIC_PREFIX = "{{ EVENT_BUS_REDIS_TOPIC_PREFIX }}" +EVENT_BUS_PRODUCER = "{{ EVENT_BUS_REDIS_PRODUCER }}" +EVENT_BUS_CONSUMER = "{{ EVENT_BUS_REDIS_CONSUMER }}" +{% if EVENT_BUS_REDIS_CONSUMER_CONSECUTIVE_ERRORS_LIMIT %} +EVENT_BUS_REDIS_CONSUMER_CONSECUTIVE_ERRORS_LIMIT = int("{{ EVENT_BUS_REDIS_CONSUMER_CONSECUTIVE_ERRORS_LIMIT }}") +{% endif %} +EVENT_BUS_REDIS_CONSUMER_POLL_TIMEOUT = int("{{ EVENT_BUS_REDIS_CONSUMER_POLL_TIMEOUT }}") +EVENT_BUS_REDIS_STREAM_MAX_LEN = int("{{ EVENT_BUS_REDIS_STREAM_MAX_LEN }}") +{% endif %} + +{% if EVENT_BUS_BACKEND == 'kafka' %} +EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = "{{ EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL }}" +EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = "{{ EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS }}" +EVENT_BUS_PRODUCER = "{{ EVENT_BUS_KAFKA_PRODUCER }}" +EVENT_BUS_CONSUMER = "{{ EVENT_BUS_KAFKA_CONSUMER }}" +EVENT_BUS_TOPIC_PREFIX = "{{ EVENT_BUS_KAFKA_TOPIC_PREFIX }}" +{% endif %} + +EVENT_BUS_PRODUCER_CONFIG = {{ EVENT_BUS_PRODUCER_CONFIG }} diff --git a/tutorevent_bus_redis/patches/openedx-dockerfile-post-python-requirements b/tutorevent_bus_redis/patches/openedx-dockerfile-post-python-requirements new file mode 100644 index 0000000..e69de29 diff --git a/tutorevent_bus_redis/plugin.py b/tutorevent_bus_redis/plugin.py index 4835a0a..3dccc02 100644 --- a/tutorevent_bus_redis/plugin.py +++ b/tutorevent_bus_redis/plugin.py @@ -20,8 +20,99 @@ # Each new setting is a pair: (setting_name, default_value). # Prefix your setting names with 'EVENT_BUS_REDIS_'. ("EVENT_BUS_REDIS_VERSION", __version__), - ("RUN_KAFKA_EVENT_BUS", True), - ("RUN_REDIS_EVENT_BUS", True), + + # Possible values are "kafka", "redis", or None to disable the + # event bus + ("EVENT_BUS_BACKEND", "kafka"), + + # Settings for producing events + ("EVENT_BUS_SEND_CATALOG_INFO_SIGNAL", True), + ( + # FIXME: We should only install the one that's configured + "OPENEDX_EXTRA_PIP_REQUIREMENTS", + [ + "edx-event-bus-redis==0.3.2", + "edx-event-bus-kafka==v5.6.0", + "confluent_kafka[avro,schema-registry]", + ], + ), + + # These are a subset of known events, there are many more. + # FIXME: Update this to a saner config structure less likely to break + ("EVENT_BUS_PRODUCER_CONFIG", """{ + 'org.openedx.content_authoring.xblock.published.v1': { + 'content-authoring-xblock-lifecycle': + {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, + 'content-authoring-xblock-published': + {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, + }, + 'org.openedx.content_authoring.xblock.deleted.v1': { + 'content-authoring-xblock-lifecycle': + {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, + }, + 'org.openedx.learning.auth.session.login.completed.v1': { + 'user-login': {'event_key_field': 'user.pii.username', 'enabled': True}, + }, +} + """), + + ###################################### + # redis backend settings + # Version of https://github.com/openedx/event-bus-redis to install + # This is what follows 'pip install' so you can use official versions + # or install from forks / branches / PRs here + ("EVENT_BUS_REDIS_RELEASE", "edx-event-bus-redis=='0.3.2'"), + + # If true, this will run a separate instance of redis just for the + # event bus to prevent resource conflicts with other services + # TODO: Implement this + # ("RUN_REDIS_SERVER", True), + + # Prefix for topics sent over the event bus + ("EVENT_BUS_REDIS_TOPIC_PREFIX", "openedx"), + + # Producer class which can send events to redis streams. + ("EVENT_BUS_REDIS_PRODUCER", "edx_event_bus_redis.create_producer"), + + # Consumer class which can consume events from redis streams. + ("EVENT_BUS_REDIS_CONSUMER", "edx_event_bus_redis.RedisEventConsumer"), + + # If the consumer encounters this many consecutive errors, exit with an error. This is intended to be used in a + # context where a management system (such as Kubernetes) will relaunch the consumer automatically. + # Default is "None", which means the consumer will never relaunch. + ("EVENT_BUS_REDIS_CONSUMER_CONSECUTIVE_ERRORS_LIMIT", 0), + + # How long the consumer should wait for new entries in a stream. + # As we are running the consumer in a while True loop, changing this setting doesn't make much difference expect + # for changing number of monitoring messages while waiting for new events. + # https://redis.io/commands/xread/#blocking-for-data + ("EVENT_BUS_REDIS_CONSUMER_POLL_TIMEOUT", 60), + + # Limits stream size to approximately this number + ("EVENT_BUS_REDIS_STREAM_MAX_LEN", 10_000), + + ###################################### + # Kafka backend settings + # TODO: Move hard coded settings from local-docker-compose-services here + # Version of https://github.com/openedx/event-bus-kafka to install + # This is what follows 'pip install' so you can use official versions + # or install from forks / branches / PRs here + ("EVENT_BUS_KAFKA_RELEASE", "edx-event-bus-kafka=='v5.6.0'"), + + # This will run schema-manager, zookeeper and kafka. Set to False if you + # are using a 3rd party to host Kafka or managing it outside of Tutor. + ("RUN_KAFKA_SERVER", True), + + # This will run kafka-control-center. This consumes a lot of resources, + # you can turn it off separately from the required services. Requires + # RUN_KAFKA_SERVER to be True as well. + ("RUN_KAFKA_UI", True), + + ("EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL", "http://schema-registry:18081"), + ("EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS", "kafka:29092"), + ("EVENT_BUS_KAFKA_PRODUCER", "edx_event_bus_kafka.create_producer"), + ("EVENT_BUS_KAFKA_CONSUMER", "edx_event_bus_kafka.KafkaEventConsumer"), + ("EVENT_BUS_KAFKA_TOPIC_PREFIX", "dev"), ] ) From 80d9aca6c6657e392cee0f991578b34c946b32e1 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Mon, 12 Feb 2024 16:33:13 -0500 Subject: [PATCH 03/10] fix: Update to latest openedx-event to stop recursion --- .gitignore | 5 +++++ tutorevent_bus_redis/plugin.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f6a874f..dabb1d8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,11 @@ !.gitignore TODO __pycache__ +*.DS_Store* *.egg-info/ /build/ /dist/ +/.idea/ +venv/ +env/ +transifex_input.yaml diff --git a/tutorevent_bus_redis/plugin.py b/tutorevent_bus_redis/plugin.py index 3dccc02..77027ea 100644 --- a/tutorevent_bus_redis/plugin.py +++ b/tutorevent_bus_redis/plugin.py @@ -33,6 +33,7 @@ [ "edx-event-bus-redis==0.3.2", "edx-event-bus-kafka==v5.6.0", + "openedx-events==v9.5.1", "confluent_kafka[avro,schema-registry]", ], ), @@ -66,7 +67,7 @@ # If true, this will run a separate instance of redis just for the # event bus to prevent resource conflicts with other services # TODO: Implement this - # ("RUN_REDIS_SERVER", True), + # ("RUN_DEDICATED_REDIS_BUS_SERVER", True), # Prefix for topics sent over the event bus ("EVENT_BUS_REDIS_TOPIC_PREFIX", "openedx"), From a600a977aa0d5863a9dcc5ebdd62f4ea63d560dd Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Tue, 13 Feb 2024 08:48:34 -0500 Subject: [PATCH 04/10] chore: Remove unnecessary IDE file --- .idea/.gitignore | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .idea/.gitignore diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b8..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml From 9f2086aec1c0f996e5b950a27de40aa83f99ad3b Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Tue, 13 Feb 2024 10:11:16 -0500 Subject: [PATCH 05/10] feat: Add a default consumer for login --- .../patches/local-docker-compose-services | 20 ++++++++++ ...penedx-dockerfile-post-python-requirements | 0 tutorevent_bus_redis/plugin.py | 38 +++++++++---------- 3 files changed, 39 insertions(+), 19 deletions(-) delete mode 100644 tutorevent_bus_redis/patches/openedx-dockerfile-post-python-requirements diff --git a/tutorevent_bus_redis/patches/local-docker-compose-services b/tutorevent_bus_redis/patches/local-docker-compose-services index 7e12281..e8ec9ed 100644 --- a/tutorevent_bus_redis/patches/local-docker-compose-services +++ b/tutorevent_bus_redis/patches/local-docker-compose-services @@ -67,3 +67,23 @@ kafka-control-center: CONFLUENT_METRICS_TOPIC_REPLICATION: 1 PORT: 9021 {% endif %} + +{% if EVENT_BUS_BACKEND %} +# This is just a stub test for showing how we could run consumers +login-consumer: + image: docker.io/overhangio/openedx:17.0.1-nightly + environment: + SERVICE_VARIANT: lms + DJANGO_SETTINGS_MODULE: lms.envs.tutor.production + command: > + ./manage.py lms consume_events -t user-login -g user-activity-service {% if EVENT_BUS_BACKEND == "redis" %}--extra '{"consumer_name": "user-login-1"}'{% endif %} + restart: unless-stopped + volumes: + - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro + - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro + - ../apps/openedx/config:/openedx/config:ro + - ../../data/lms:/openedx/data + - ../../data/openedx-media:/openedx/media + depends_on: + - {% if EVENT_BUS_BACKEND == "redis" %}redis{% elif EVENT_BUS_BACKEND == "kafka" %}kafka{% endif %} +{% endif %} diff --git a/tutorevent_bus_redis/patches/openedx-dockerfile-post-python-requirements b/tutorevent_bus_redis/patches/openedx-dockerfile-post-python-requirements deleted file mode 100644 index e69de29..0000000 diff --git a/tutorevent_bus_redis/plugin.py b/tutorevent_bus_redis/plugin.py index 77027ea..a5a0fbe 100644 --- a/tutorevent_bus_redis/plugin.py +++ b/tutorevent_bus_redis/plugin.py @@ -13,6 +13,24 @@ ######################################## # CONFIGURATION ######################################## +# FIXME: Update this to a saner config structure less likely to break, and able +# to activate and deactivate individual events more easily. +PRODUCER_CONFIG = """{ + 'org.openedx.content_authoring.xblock.published.v1': { + 'content-authoring-xblock-lifecycle': + {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, + 'content-authoring-xblock-published': + {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, + }, + 'org.openedx.content_authoring.xblock.deleted.v1': { + 'content-authoring-xblock-lifecycle': + {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, + }, + 'org.openedx.learning.auth.session.login.completed.v1': { + 'user-login': {'event_key_field': 'user.pii.username', 'enabled': True}, + }, +} +""" hooks.Filters.CONFIG_DEFAULTS.add_items( [ @@ -37,25 +55,7 @@ "confluent_kafka[avro,schema-registry]", ], ), - - # These are a subset of known events, there are many more. - # FIXME: Update this to a saner config structure less likely to break - ("EVENT_BUS_PRODUCER_CONFIG", """{ - 'org.openedx.content_authoring.xblock.published.v1': { - 'content-authoring-xblock-lifecycle': - {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, - 'content-authoring-xblock-published': - {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, - }, - 'org.openedx.content_authoring.xblock.deleted.v1': { - 'content-authoring-xblock-lifecycle': - {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, - }, - 'org.openedx.learning.auth.session.login.completed.v1': { - 'user-login': {'event_key_field': 'user.pii.username', 'enabled': True}, - }, -} - """), + ("EVENT_BUS_PRODUCER_CONFIG", PRODUCER_CONFIG), ###################################### # redis backend settings From 61d75fc7c4a3679b926f56a03a5514dabcb36c68 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Tue, 13 Feb 2024 10:38:51 -0500 Subject: [PATCH 06/10] fix: Make the producer config a dict --- tutorevent_bus_redis/plugin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tutorevent_bus_redis/plugin.py b/tutorevent_bus_redis/plugin.py index a5a0fbe..19c00c1 100644 --- a/tutorevent_bus_redis/plugin.py +++ b/tutorevent_bus_redis/plugin.py @@ -15,7 +15,7 @@ ######################################## # FIXME: Update this to a saner config structure less likely to break, and able # to activate and deactivate individual events more easily. -PRODUCER_CONFIG = """{ +PRODUCER_CONFIG = { 'org.openedx.content_authoring.xblock.published.v1': { 'content-authoring-xblock-lifecycle': {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, @@ -30,7 +30,6 @@ 'user-login': {'event_key_field': 'user.pii.username', 'enabled': True}, }, } -""" hooks.Filters.CONFIG_DEFAULTS.add_items( [ From a2ba3b2fd2a85300366af8313a0de6535a77ea65 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Tue, 13 Feb 2024 10:41:48 -0500 Subject: [PATCH 07/10] fix: Make kafka images restart if the crash --- tutorevent_bus_redis/patches/local-docker-compose-services | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tutorevent_bus_redis/patches/local-docker-compose-services b/tutorevent_bus_redis/patches/local-docker-compose-services index e8ec9ed..e7d327d 100644 --- a/tutorevent_bus_redis/patches/local-docker-compose-services +++ b/tutorevent_bus_redis/patches/local-docker-compose-services @@ -2,6 +2,7 @@ # needed by Kafka to keep track of nodes, topics, and messages. zookeeper: image: confluentinc/cp-zookeeper:6.2.1 + restart: unless-stopped ports: - "2181:2181" environment: @@ -13,6 +14,7 @@ kafka: image: confluentinc/cp-server:6.2.1 depends_on: - zookeeper + restart: unless-stopped ports: - "9092:9092" - "9101:9101" @@ -41,6 +43,7 @@ schema-registry: image: confluentinc/cp-schema-registry:6.2.1 depends_on: - kafka + restart: unless-stopped ports: - "18081:18081" environment: @@ -56,6 +59,7 @@ kafka-control-center: depends_on: - kafka - schema-registry + restart: unless-stopped ports: - "9021:9021" environment: From 1073ee2041821c79c85003c60bf42244a112f0e2 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Wed, 14 Feb 2024 09:09:24 -0500 Subject: [PATCH 08/10] refactor: Put implementation independent settings together --- tutorevent_bus_redis/patches/openedx-common-settings | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tutorevent_bus_redis/patches/openedx-common-settings b/tutorevent_bus_redis/patches/openedx-common-settings index 05758a1..34fd7c0 100644 --- a/tutorevent_bus_redis/patches/openedx-common-settings +++ b/tutorevent_bus_redis/patches/openedx-common-settings @@ -1,4 +1,5 @@ -# Settings for producing events +# Backend independent settings for event production +EVENT_BUS_PRODUCER_CONFIG = {{ EVENT_BUS_PRODUCER_CONFIG }} SEND_CATALOG_INFO_SIGNAL = {{ EVENT_BUS_SEND_CATALOG_INFO_SIGNAL }} @@ -24,4 +25,3 @@ EVENT_BUS_CONSUMER = "{{ EVENT_BUS_KAFKA_CONSUMER }}" EVENT_BUS_TOPIC_PREFIX = "{{ EVENT_BUS_KAFKA_TOPIC_PREFIX }}" {% endif %} -EVENT_BUS_PRODUCER_CONFIG = {{ EVENT_BUS_PRODUCER_CONFIG }} From dd999eeb6b26457ebfdb509c215168cd14ab46da Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Wed, 14 Feb 2024 09:10:42 -0500 Subject: [PATCH 09/10] chore: Upgrade to latest event-bus-redis --- tutorevent_bus_redis/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorevent_bus_redis/plugin.py b/tutorevent_bus_redis/plugin.py index 19c00c1..a5b7663 100644 --- a/tutorevent_bus_redis/plugin.py +++ b/tutorevent_bus_redis/plugin.py @@ -48,7 +48,7 @@ # FIXME: We should only install the one that's configured "OPENEDX_EXTRA_PIP_REQUIREMENTS", [ - "edx-event-bus-redis==0.3.2", + "edx-event-bus-redis==0.3.3", "edx-event-bus-kafka==v5.6.0", "openedx-events==v9.5.1", "confluent_kafka[avro,schema-registry]", From f3a5f754bccaaf94241d7b087bee5dcf060b0a35 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Fri, 29 Mar 2024 15:33:28 -0400 Subject: [PATCH 10/10] feat: Working tracking log consumer --- .../patches/local-docker-compose-services | 23 ++++++++++++++++++- .../patches/openedx-common-settings | 17 +++++++++++++- tutorevent_bus_redis/plugin.py | 18 ++++++++------- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/tutorevent_bus_redis/patches/local-docker-compose-services b/tutorevent_bus_redis/patches/local-docker-compose-services index e7d327d..88b1f41 100644 --- a/tutorevent_bus_redis/patches/local-docker-compose-services +++ b/tutorevent_bus_redis/patches/local-docker-compose-services @@ -75,7 +75,7 @@ kafka-control-center: {% if EVENT_BUS_BACKEND %} # This is just a stub test for showing how we could run consumers login-consumer: - image: docker.io/overhangio/openedx:17.0.1-nightly + image: docker.io/overhangio/openedx:17.0.2-nightly environment: SERVICE_VARIANT: lms DJANGO_SETTINGS_MODULE: lms.envs.tutor.production @@ -90,4 +90,25 @@ login-consumer: - ../../data/openedx-media:/openedx/media depends_on: - {% if EVENT_BUS_BACKEND == "redis" %}redis{% elif EVENT_BUS_BACKEND == "kafka" %}kafka{% endif %} + +# This is just a stub test for showing how we could run consumers +tracking-consumer: + image: docker.io/overhangio/openedx:17.0.2-nightly + environment: + SERVICE_VARIANT: lms + DJANGO_SETTINGS_MODULE: lms.envs.tutor.production + command: > + ./manage.py lms consume_events -t analytics -g analytics-service {% if EVENT_BUS_BACKEND == "redis" %}--extra '{"consumer_name": "analytics-1"}'{% endif %} + restart: unless-stopped + volumes: + - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro + - ../apps/openedx/settings/cms:/openedx/edx-platform/cms/envs/tutor:ro + - ../apps/openedx/config:/openedx/config:ro + - ../../data/lms:/openedx/data + - ../../data/openedx-media:/openedx/media + {%- for mount in iter_mounts(MOUNTS, "openedx", "lms") %} + - {{ mount }} + {%- endfor %} + depends_on: + - {% if EVENT_BUS_BACKEND == "redis" %}redis{% elif EVENT_BUS_BACKEND == "kafka" %}kafka{% endif %} {% endif %} diff --git a/tutorevent_bus_redis/patches/openedx-common-settings b/tutorevent_bus_redis/patches/openedx-common-settings index 34fd7c0..087844c 100644 --- a/tutorevent_bus_redis/patches/openedx-common-settings +++ b/tutorevent_bus_redis/patches/openedx-common-settings @@ -6,7 +6,7 @@ SEND_CATALOG_INFO_SIGNAL = {{ EVENT_BUS_SEND_CATALOG_INFO_SIGNAL }} {% if EVENT_BUS_BACKEND == 'redis' %} # redis connection url # https://redis.readthedocs.io/en/stable/examples/ssl_connection_examples.html#Connecting-to-a-Redis-instance-via-a-URL-string -EVENT_BUS_REDIS_CONNECTION_URL = "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/" +EVENT_BUS_REDIS_CONNECTION_URL = "{{ EVENT_BUS_REDIS_CONNECTION_URL }}" EVENT_BUS_TOPIC_PREFIX = "{{ EVENT_BUS_REDIS_TOPIC_PREFIX }}" EVENT_BUS_PRODUCER = "{{ EVENT_BUS_REDIS_PRODUCER }}" EVENT_BUS_CONSUMER = "{{ EVENT_BUS_REDIS_CONSUMER }}" @@ -25,3 +25,18 @@ EVENT_BUS_CONSUMER = "{{ EVENT_BUS_KAFKA_CONSUMER }}" EVENT_BUS_TOPIC_PREFIX = "{{ EVENT_BUS_KAFKA_TOPIC_PREFIX }}" {% endif %} +{% if EVENT_BUS_TRACKING_LOGS %} +SEND_TRACKING_EVENT_EMITTED_SIGNAL = True + +# Update the backends to use the event bus +EVENT_TRACKING_BACKENDS["xapi"]["ENGINE"] = "eventtracking.backends.event_bus.EventBusRoutingBackend" +EVENT_TRACKING_BACKENDS["caliper"]["ENGINE"] = "eventtracking.backends.event_bus.EventBusRoutingBackend" + +# Update backend to send events in sync mode +EVENT_TRACKING_BACKENDS["xapi"]["OPTIONS"]["backends"]["xapi"]["ENGINE"] = "event_routing_backends.backends.sync_events_router.SyncEventsRouter" +EVENT_TRACKING_BACKENDS["caliper"]["OPTIONS"]["backends"]["caliper"]["ENGINE"] = "event_routing_backends.backends.sync_events_router.SyncEventsRouter" + +# Remove caliper from the tracking backends to prevent double-event-emission +EVENT_TRACKING_BACKENDS.pop("caliper") + +{% endif %} diff --git a/tutorevent_bus_redis/plugin.py b/tutorevent_bus_redis/plugin.py index a5b7663..d53bba0 100644 --- a/tutorevent_bus_redis/plugin.py +++ b/tutorevent_bus_redis/plugin.py @@ -27,7 +27,10 @@ {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, }, 'org.openedx.learning.auth.session.login.completed.v1': { - 'user-login': {'event_key_field': 'user.pii.username', 'enabled': True}, + 'user-login': {'event_key_field': 'user.pii.username', 'enabled': False}, + }, + 'org.openedx.analytics.tracking.event.emitted.v1': { + 'analytics': {'event_key_field': 'tracking_log.name', 'enabled': True} }, } @@ -40,10 +43,11 @@ # Possible values are "kafka", "redis", or None to disable the # event bus - ("EVENT_BUS_BACKEND", "kafka"), + ("EVENT_BUS_BACKEND", "redis"), # Settings for producing events ("EVENT_BUS_SEND_CATALOG_INFO_SIGNAL", True), + ("EVENT_BUS_TRACKING_LOGS", True), ( # FIXME: We should only install the one that's configured "OPENEDX_EXTRA_PIP_REQUIREMENTS", @@ -52,16 +56,13 @@ "edx-event-bus-kafka==v5.6.0", "openedx-events==v9.5.1", "confluent_kafka[avro,schema-registry]", + "git+https://github.com/openedx/platform-plugin-aspects.git@bmtcril/tracking_event_command", ], ), ("EVENT_BUS_PRODUCER_CONFIG", PRODUCER_CONFIG), ###################################### # redis backend settings - # Version of https://github.com/openedx/event-bus-redis to install - # This is what follows 'pip install' so you can use official versions - # or install from forks / branches / PRs here - ("EVENT_BUS_REDIS_RELEASE", "edx-event-bus-redis=='0.3.2'"), # If true, this will run a separate instance of redis just for the # event bus to prevent resource conflicts with other services @@ -101,18 +102,19 @@ # This will run schema-manager, zookeeper and kafka. Set to False if you # are using a 3rd party to host Kafka or managing it outside of Tutor. - ("RUN_KAFKA_SERVER", True), + ("RUN_KAFKA_SERVER", False), # This will run kafka-control-center. This consumes a lot of resources, # you can turn it off separately from the required services. Requires # RUN_KAFKA_SERVER to be True as well. - ("RUN_KAFKA_UI", True), + ("RUN_KAFKA_UI", False), ("EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL", "http://schema-registry:18081"), ("EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS", "kafka:29092"), ("EVENT_BUS_KAFKA_PRODUCER", "edx_event_bus_kafka.create_producer"), ("EVENT_BUS_KAFKA_CONSUMER", "edx_event_bus_kafka.KafkaEventConsumer"), ("EVENT_BUS_KAFKA_TOPIC_PREFIX", "dev"), + ("EVENT_BUS_REDIS_CONNECTION_URL", "redis://{% if REDIS_USERNAME and REDIS_PASSWORD %}{{ REDIS_USERNAME }}:{{""REDIS_PASSWORD }}{% endif %}@{{ REDIS_HOST }}:{{ REDIS_PORT }}/5") ] )