Persistent problem data - store directly into a new database field.

Includes changes and suggestions from Dr. Glenn Rice.
See: #1940

Fixes a minor POD error in lib/WeBWorK/HTML/AttemptsTable.pm.

lib/WeBWorK/ContentGenerator/GatewayQuiz.pm

@@ -24,6 +24,7 @@ deal with versioning sets
 =cut
 
 use Mojo::Promise;
+use Mojo::JSON qw(encode_json decode_json);
 
 use WeBWorK::PG::ImageGenerator;
 # Use the ContentGenerator formatDateTime, not the version in Utils.
@@ -315,6 +316,7 @@ async sub pre_header_initialize ($c) {
 	my $setID           = $c->stash('setID');
 	my $userID          = $c->param('user');
 	my $effectiveUserID = $c->param('effectiveUser');
+	my $isFakeSet       = 0;
 
 	# User checks
 	my $user = $db->getUser($userID);
@@ -342,6 +344,7 @@ async sub pre_header_initialize ($c) {
 	# If the set comes in as "Undefined_Set", then we're trying/editing a single problem in a set, and so create a fake
 	# set with which to work if the user has the authorization to do that.
 	if ($setID eq 'Undefined_Set') {
+		$isFakeSet = 1;
 		# Make sure these are defined
 		$requestedVersion = 1;
 		$c->{assignment_type} = 'gateway';
@@ -910,6 +913,7 @@ async sub pre_header_initialize ($c) {
 	if ($c->{submitAnswers} || (($c->{previewAnswers} || $c->param('newPage')) && $can{recordAnswers})) {
 		# If answers are being submitted, then save the problems to the database.  If this is a preview or pages change
 		# and answers can be recorded, then save the last answer for future reference.
+		# Also save the persistent data to the database even when the last answer is not saved.
 
 		# First, deal with answers being submitted for a proctored exam.  Delete the proctor keys that authorized the
 		# grading, so that it isn't possible to log in and take another proctored test without being reauthorized.
@@ -951,6 +955,28 @@ async sub pre_header_initialize ($c) {
 				my ($past_answers_string, $scores);    # Not used here
 				($past_answers_string, $encoded_last_answer_string, $scores, $answer_types_string) =
 					create_ans_str_from_responses($c, $pg_result);
+
+				# Transfer persistent problem data from the PERSISTENCE_HASH:
+				# - Get keys to update first, to avoid extra work when no updated ar
+				#   are needed. When none, we avoid the need to decode/encode JSON,
+				#   to save the pureProblem when it would not otherwise be saved.
+				# - We are assuming that there is no need to DELETE old
+				#   persistent data if the hash is empty, even if in
+				#   potential there may be some data already in the database.
+				my @persistent_data_keys = keys %{ $pg_result->{PERSISTENCE_HASH_UPDATED} };
+				if (@persistent_data_keys) {
+					my $json_data = decode_json($pureProblem->{problem_data} || '{}');
+					for my $key (@persistent_data_keys) {
+						$json_data->{$key} = $pg_result->{PERSISTENCE_HASH}{$key};
+					}
+					$pureProblem->problem_data(encode_json($json_data));
+
+					# If the pureProblem will not be saved below, we should save the
+					# persistent data here before any other changes are made to it.
+					if (($c->{submitAnswers} && !$will{recordAnswers})) {
+						$c->db->putProblemVersion($pureProblem);
+					}
+				}
 			} else {
 				my $prefix         = sprintf('Q%04d_', $problemNumbers[$i]);
 				my @fields         = sort grep {/^(?!previous).*$prefix/} (keys %{ $c->{formFields} });
@@ -1165,6 +1191,45 @@ async sub pre_header_initialize ($c) {
 			# Reset start time
 			$c->param('startTime', '');
 		}
+	} else {
+		# This 'else' case includes initial load of the first page of the
+		# quiz and checkAnswers calls, as well as when $can{recordAnswers}
+		# is false.
+
+		# Save persistent data to database even in this case, when answers
+		# would not or can not be recorded.
+		my @pureProblems = $db->getAllProblemVersions($effectiveUserID, $setID, $versionID);
+		for my $i (0 .. $#problems) {
+			# Process each problem.
+			my $pureProblem = $pureProblems[ $probOrder[$i] ];
+			my $pg_result   = $pg_results[ $probOrder[$i] ];
+
+			if (ref $pg_result) {
+				# Transfer persistent problem data from the PERSISTENCE_HASH:
+				# - Get keys to update first, to avoid extra work when no updates
+				#   are needed. When none, we avoid the need to decode/encode JSON,
+				#   or to save the pureProblem.
+				# - We are assuming that there is no need to DELETE old
+				#   persistent data if the hash is empty, even if in
+				#   potential there may be some data already in the database.
+				my @persistent_data_keys = keys %{ $pg_result->{PERSISTENCE_HASH_UPDATED} };
+				next unless (@persistent_data_keys);    # stop now if nothing to do
+				if ($isFakeSet) {
+					warn join("",
+						"This problem stores persistent data and this cannot be done in a fake set. ",
+						"Some functionality may not work properly when testing this problem in this setting.");
+					next;
+				}
+
+				my $json_data = decode_json($pureProblem->{problem_data} || '{}');
+				for my $key (@persistent_data_keys) {
+					$json_data->{$key} = $pg_result->{PERSISTENCE_HASH}{$key};
+				}
+				$pureProblem->problem_data(encode_json($json_data));
+
+				$c->db->putProblemVersion($pureProblem);
+			}
+		}
 	}
 	debug('end answer processing');
 

lib/WeBWorK/DB/Record/UserProblem.pm

@@ -54,6 +54,8 @@ BEGIN {
 		sub_status => { type => "FLOAT" },
 		# a field for flags which need to be set
 		flags => { type => "TEXT" },
+		# additional stored data for this problem, internally uses JSON:
+		problem_data => { type => "MEDIUMTEXT" },
 	);
 }
 

lib/WeBWorK/HTML/AttemptsTable.pm

@@ -46,7 +46,7 @@ answer to a WeBWorK problem.  It is used in Problem.pm, OpaqueServer.pm, standAl
 
 =head2 new
 
-	my $tbl = WeBWorK::Utils::AttemptsTable->new(
+	my $tbl = WeBWorK::HTML::AttemptsTable->new(
 		$answers,
 		answersSubmitted       => 1,
 		answerOrder            => $pg->{flags}{ANSWER_ENTRY_ORDER},

lib/WeBWorK/Utils/ProblemProcessing.pm

@@ -65,6 +65,27 @@ sub process_and_log_answer ($c) {
 	my $pureProblem = $db->getUserProblem($problem->user_id, $problem->set_id, $problem->problem_id);
 	my $answer_log  = $ce->{courseFiles}{logs}{answer_log};
 
+	# Transfer persistent problem data from the PERSISTENCE_HASH:
+	# - Get keys to update first, to avoid extra work when no updates
+	#   are needed. When none, we avoid the need to decode/encode JSON,
+	#   or to save the pureProblem.
+	# - We are assuming that there is no need to DELETE old
+	#   persistent data if the hash is empty, even if in
+	#   potential there may be some data already in the database.
+	if (defined($pureProblem)) {
+		my @persistent_data_keys = keys %{ $pg->{PERSISTENCE_HASH_UPDATED} };
+		if (@persistent_data_keys) {
+			my $json_data = decode_json($pureProblem->{problem_data} || '{}');
+			for my $key (@persistent_data_keys) {
+				$json_data->{$key} = $pg->{PERSISTENCE_HASH}{$key};
+			}
+			$pureProblem->problem_data(encode_json($json_data));
+			if (!$submitAnswers) {    # would not be saved below
+				$db->putUserProblem($pureProblem);
+			}
+		}
+	}
+
 	my ($encoded_last_answer_string, $scores2, $answer_types_string);
 	my $scoreRecordedMessage = '';
 
@@ -117,7 +138,6 @@ sub process_and_log_answer ($c) {
 			# store last answer to database for use in "sticky" answers
 			$problem->last_answer($encoded_last_answer_string);
 			$pureProblem->last_answer($encoded_last_answer_string);
-			$db->putUserProblem($pureProblem);
 
 			# store state in DB if it makes sense
 			if ($will{recordAnswers}) {
@@ -251,6 +271,8 @@ sub process_and_log_answer ($c) {
 					}
 				}
 			} else {
+				# The "sticky" answers get saved here when $will{recordAnswers} is false
+				$db->putUserProblem($pureProblem);
 				if (before($set->open_date, $c->submitTime) || after($set->due_date, $c->submitTime)) {
 					$scoreRecordedMessage =
 						$c->maketext('Your score was not recorded because this homework set is closed.');

lib/WeBWorK/Utils/Rendering.pm

@@ -120,6 +120,9 @@ sub constructPGOptions ($ce, $user, $set, $problem, $psvn, $formFields, $transla
 	$options{num_of_correct_ans}   = $problem->num_correct;
 	$options{num_of_incorrect_ans} = $problem->num_incorrect;
 
+	# Persistent problem data
+	$options{PERSISTENCE_HASH} = decode_json($problem->problem_data || '{}');
+
 	# Language
 	$options{language}            = $ce->{language};
 	$options{language_subroutine} = WeBWorK::Localize::getLoc($options{language});
@@ -260,6 +263,8 @@ sub renderPG ($c, $effectiveUser, $set, $problem, $psvn, $formFields, $translati
 				map { $_ => $pg->{pgcore}{PG_alias}{resource_list}{$_}{uri}{content} }
 					keys %{ $pg->{pgcore}{PG_alias}{resource_list} }
 			};
+			$ret->{PERSISTENCE_HASH_UPDATED} = $pg->{pgcore}{PERSISTENCE_HASH_UPDATED};
+			$ret->{PERSISTENCE_HASH}         = $pg->{pgcore}{PERSISTENCE_HASH};
 		}
 
 		# Save the problem source. This is used by Caliper::Entity. Why?