Skip to content

Commit

Permalink
fix: Serialize list problem_check results as JSON
Browse files Browse the repository at this point in the history
Prior to this fix, TinCan would use the repr of the list, which
made some responses effectively un-parseable downstream.
  • Loading branch information
bmtcril committed Nov 6, 2023
1 parent d841b3c commit f0b7034
Showing 9 changed files with 583 additions and 7 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
{
"name": "problem_check",
"context": {
"course_id": "course-v1:edX+DemoX+Demo_Course",
"course_user_tags": {},
"user_id": 6,
"path": "/courses/course-v1:edX+DemoX+Demo_Course/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4/handler/xmodule_handler/problem_check",
"org_id": "edX",
"enterprise_uuid": "",
"module": {
"display_name": "Multiple Choice Questions",
"usage_key": "block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4"
},
"asides": {}
},
"username": "tacotuesday",
"session": "9885dd163f65eed1fe88759c186b4ac3",
"ip": "192.168.1.1",
"agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/119.0",
"host": "local.overhang.io",
"referer": "http://local.overhang.io/xblock/block-v1:edX+DemoX+Demo_Course+type@vertical+block@54bb9b142c6c4c22afc62bcb628f0e68?show_title=0&show_bookmark_button=0&recheck_access=1&view=student_view&format=Homework",
"accept_language": "en-US,en;q=0.5",
"event": {
"state": {
"seed": 1,
"student_answers": {
"a0effb954cca4759994f1ac9e9434bf4_4_1": [
"choice_0",
"choice_1",
"choice_2"
],
"a0effb954cca4759994f1ac9e9434bf4_2_1": "blue",
"a0effb954cca4759994f1ac9e9434bf4_3_1": "choice_2"
},
"has_saved_answers": false,
"correct_map": {
"a0effb954cca4759994f1ac9e9434bf4_2_1": {
"correctness": "correct",
"npoints": null,
"msg": "",
"hint": "",
"hintmode": null,
"queuestate": null,
"answervariable": null
},
"a0effb954cca4759994f1ac9e9434bf4_3_1": {
"correctness": "correct",
"npoints": null,
"msg": "",
"hint": "",
"hintmode": null,
"queuestate": null,
"answervariable": null
},
"a0effb954cca4759994f1ac9e9434bf4_4_1": {
"correctness": "incorrect",
"npoints": null,
"msg": "",
"hint": "",
"hintmode": null,
"queuestate": null,
"answervariable": null
}
},
"input_state": {
"a0effb954cca4759994f1ac9e9434bf4_2_1": {},
"a0effb954cca4759994f1ac9e9434bf4_3_1": {},
"a0effb954cca4759994f1ac9e9434bf4_4_1": {},
"a0effb954cca4759994f1ac9e9434bf4_5_1": {}
},
"done": true
},
"problem_id": "block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4",
"answers": {
"a0effb954cca4759994f1ac9e9434bf4_4_1": [
"choice_0",
"choice_1",
"choice_2"
],
"a0effb954cca4759994f1ac9e9434bf4_5_1": [
"choice_0",
"choice_1",
"choice_2",
"choice_3",
"choice_4",
"choice_5"
],
"a0effb954cca4759994f1ac9e9434bf4_2_1": "blue",
"a0effb954cca4759994f1ac9e9434bf4_3_1": "choice_2"
},
"grade": 2,
"max_grade": 4,
"correct_map": {
"a0effb954cca4759994f1ac9e9434bf4_2_1": {
"correctness": "correct",
"npoints": null,
"msg": "",
"hint": "",
"hintmode": null,
"queuestate": null,
"answervariable": null
},
"a0effb954cca4759994f1ac9e9434bf4_3_1": {
"correctness": "correct",
"npoints": null,
"msg": "",
"hint": "",
"hintmode": null,
"queuestate": null,
"answervariable": null
},
"a0effb954cca4759994f1ac9e9434bf4_4_1": {
"correctness": "incorrect",
"npoints": null,
"msg": "",
"hint": "",
"hintmode": null,
"queuestate": null,
"answervariable": null
},
"a0effb954cca4759994f1ac9e9434bf4_5_1": {
"correctness": "incorrect",
"npoints": null,
"msg": "",
"hint": "",
"hintmode": null,
"queuestate": null,
"answervariable": null
}
},
"success": "incorrect",
"attempts": 9,
"submission": {
"a0effb954cca4759994f1ac9e9434bf4_4_1": {
"question": "",
"answer": [
"a piano",
"a tree",
"a guitar"
],
"response_type": "choiceresponse",
"input_type": "checkboxgroup",
"correct": false,
"variant": "",
"group_label": ""
},
"a0effb954cca4759994f1ac9e9434bf4_5_1": {
"question": "",
"answer": [
"Un emprunt \u00e0 l'anglais ou anglicisme",
"Un type de demande",
"Quelque chose de rapide, d'instantan\u00e9",
"Une question structur\u00e9e et pr\u00e9cise",
"Un type de poisson",
"I\"M SWP' \"SDF\""
],
"response_type": "choiceresponse",
"input_type": "checkboxgroup",
"correct": false,
"variant": "",
"group_label": ""
},
"a0effb954cca4759994f1ac9e9434bf4_2_1": {
"question": "",
"answer": "blue",
"response_type": "optionresponse",
"input_type": "optioninput",
"correct": true,
"variant": "",
"group_label": ""
},
"a0effb954cca4759994f1ac9e9434bf4_3_1": {
"question": "",
"answer": "a chair",
"response_type": "multiplechoiceresponse",
"input_type": "choicegroup",
"correct": true,
"variant": "",
"group_label": ""
}
}
},
"time": "2023-11-03T16:31:50.023274+00:00",
"event_type": "problem_check",
"event_source": "server",
"page": "x_module"
}
Original file line number Diff line number Diff line change
@@ -119,6 +119,7 @@ def test_event_transformer(self, event_filename, mocked_uuid4):
try:
self.compare_events(actual_transformed_event, expected_event)
except Exception as e: # pragma: no cover
print("Comparison failed, writing output to test_output for debugging")
with open(f"test_output/generated.{event_filename}.json", "w") as actual_transformed_event_file:
try:
actual_transformed_event_file.write(actual_transformed_event.to_json())
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""
""""
Transformers for problem interaction events.
"""
import json

from tincan import Activity, ActivityDefinition, Extensions, LanguageMap, Result

from event_routing_backends.helpers import get_problem_block_id
@@ -315,6 +317,15 @@ def get_result(self):
response = submission["answer"]
correct = submission.get("correct")
else:
# The submission key didn't exist until March 2014, prior to that
# there was usually a version of the answer in the "answers" key,
# but it is very flaky (sometimes containing xml, and without the
# appended variant identifier that we get for free in the
# submission. We don't attempt to work around those issues here due
# to how few and how old those events are and how complicated the
# parsing is. Should we ever find it necessary to make a better
# parser for them, Insights had a good effort here:
# https://github.com/openedx/edx-analytics-pipeline/blob/master/edx/analytics/tasks/insights/answer_dist.py#L260C36-L260C36
response = event_data.get('answers', None)
correct = self.get_data('success') == 'correct'

@@ -328,7 +339,17 @@ def get_result(self):
else:
scaled = 0

return Result(
# Some problems can provide a list of responses answers, but
# the Result type wants a string for "response". So we dump those
# to JSON here to provide a parsable version of the response instead
# of getting the __repr__ of the list, which is what Result will
# generate by default.
if isinstance(response, list):
cls = JSONEncodedResult
else:
cls = Result

return cls(
success=correct,
score={
'min': 0,
@@ -455,3 +476,30 @@ def get_result(self):
submission = self._get_submission() or {}
result.response = submission.get('answer')
return result


class JSONEncodedResult(Result):
"""
This is a workaround for a TinCan issue where it will coerce a value passed
in for a `response` to str. This breaks our ability to serialize list
responses into JSON, so we override it here.
"""
@property
def response(self):
"""Response for Result
:setter: Tries to JSON dump the list value.
:setter type: list
:rtype: str
"""
return self._response

Check warning on line 495 in event_routing_backends/processors/xapi/event_transformers/problem_interaction_events.py

Codecov / codecov/patch

event_routing_backends/processors/xapi/event_transformers/problem_interaction_events.py#L495

Added line #L495 was not covered by tests

@response.setter
def response(self, value):
"""
Ensures the list is serialized as JSON.
"""
if not isinstance(value, list):
raise ValueError(f"JSONEncodedResult only accepts lists, {type(value)} given.")

Check warning on line 503 in event_routing_backends/processors/xapi/event_transformers/problem_interaction_events.py

Codecov / codecov/patch

event_routing_backends/processors/xapi/event_transformers/problem_interaction_events.py#L503

Added line #L503 was not covered by tests

self._response = json.dumps(value)
Original file line number Diff line number Diff line change
@@ -39,7 +39,7 @@
"objectType": "Activity"
},
"result": {
"response": "['a correct answer', 'an incorrect answer']",
"response": "[\"a correct answer\", \"an incorrect answer\"]",
"score": {
"max": 1,
"min": 0,
Loading

0 comments on commit f0b7034

Please sign in to comment.