Merge remote-tracking branch 'origin/master' into asaeed/ENT-7871

CHANGELOG.rst

@@ -15,6 +15,19 @@ Change Log
 
 Unreleased
 ----------
+
+[4.7.6]
+--------
+chore: remove unnecessary logs from the integrated channels
+
+[4.7.5]
+--------
+feat: added flag to allow in progress course learner data transmission
+
+[4.7.4]
+--------
+feat: added fields for holding encrypted data in database
+
 [4.7.3]
 --------
 feat: added management command to re-encrypt enterprise customer reporting configs

enterprise/__init__.py

@@ -2,4 +2,4 @@
 Your project description goes here.
 """
 
-__version__ = "4.7.3"
+__version__ = "4.7.6"

integrated_channels/integrated_channel/transmitters/learner_data.py

@@ -302,8 +302,10 @@ def transmit(self, payload, **kwargs):  # pylint: disable=arguments-differ
                 enterprise_enrollment_id
             )
 
-            if not learner_data.course_completed:
-                # The user has not completed the course, so we shouldn't send a completion status call
+            if (not learner_data.course_completed and
+                    not getattr(self.enterprise_configuration, 'enable_incomplete_progress_transmission', False)):
+                # The user has not completed the course and enable_incomplete_progress_transmission is not set,
+                # so we shouldn't send a completion status call
                 remote_id = getattr(learner_data, kwargs.get('remote_user_id'))
                 encoded_serialized_payload = encode_data_for_logging(serialized_payload)
                 continue

integrated_channels/moodle/migrations/0028_auto_20230928_1530.py

@@ -0,0 +1,44 @@
+# Generated by Django 3.2.20 on 2023-09-28 15:30
+
+from django.db import migrations
+import fernet_fields.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('moodle', '0027_alter_historicalmoodleenterprisecustomerconfiguration_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='historicalmoodleenterprisecustomerconfiguration',
+            name='decrypted_password',
+            field=fernet_fields.fields.EncryptedCharField(blank=True, help_text="The encrypted API user's password used to obtain new tokens. It will be encrypted when stored in the database.", max_length=255, null=True, verbose_name='Encrypted Webservice Password'),
+        ),
+        migrations.AddField(
+            model_name='historicalmoodleenterprisecustomerconfiguration',
+            name='decrypted_token',
+            field=fernet_fields.fields.EncryptedCharField(blank=True, help_text="The encrypted API user's token used to obtain new tokens. It will be encrypted when stored in the database.", max_length=255, null=True, verbose_name='Encrypted Webservice Token'),
+        ),
+        migrations.AddField(
+            model_name='historicalmoodleenterprisecustomerconfiguration',
+            name='decrypted_username',
+            field=fernet_fields.fields.EncryptedCharField(blank=True, help_text="The encrypted API user's username used to obtain new tokens. It will be encrypted when stored in the database.", max_length=255, null=True, verbose_name='Encrypted Webservice Username'),
+        ),
+        migrations.AddField(
+            model_name='moodleenterprisecustomerconfiguration',
+            name='decrypted_password',
+            field=fernet_fields.fields.EncryptedCharField(blank=True, help_text="The encrypted API user's password used to obtain new tokens. It will be encrypted when stored in the database.", max_length=255, null=True, verbose_name='Encrypted Webservice Password'),
+        ),
+        migrations.AddField(
+            model_name='moodleenterprisecustomerconfiguration',
+            name='decrypted_token',
+            field=fernet_fields.fields.EncryptedCharField(blank=True, help_text="The encrypted API user's token used to obtain new tokens. It will be encrypted when stored in the database.", max_length=255, null=True, verbose_name='Encrypted Webservice Token'),
+        ),
+        migrations.AddField(
+            model_name='moodleenterprisecustomerconfiguration',
+            name='decrypted_username',
+            field=fernet_fields.fields.EncryptedCharField(blank=True, help_text="The encrypted API user's username used to obtain new tokens. It will be encrypted when stored in the database.", max_length=255, null=True, verbose_name='Encrypted Webservice Username'),
+        ),
+    ]

integrated_channels/moodle/migrations/0028_auto_20231116_1826.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.20 on 2023-11-16 18:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('moodle', '0027_alter_historicalmoodleenterprisecustomerconfiguration_options'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='historicalmoodleenterprisecustomerconfiguration',
+            name='enable_incomplete_progress_transmission',
+            field=models.BooleanField(default=False, help_text='When set to True, the configured customer will receive learner data transmissions, for incomplete courses as well'),
+        ),
+        migrations.AddField(
+            model_name='moodleenterprisecustomerconfiguration',
+            name='enable_incomplete_progress_transmission',
+            field=models.BooleanField(default=False, help_text='When set to True, the configured customer will receive learner data transmissions, for incomplete courses as well'),
+        ),
+    ]

integrated_channels/moodle/migrations/0029_auto_20231106_1233.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.20 on 2023-11-06 12:33
+
+from django.db import migrations
+
+
+def populate_decrypted_fields(apps, schema_editor):
+    """
+    Populates the encryption fields with the data previously stored in database.
+    """
+    MoodleEnterpriseCustomerConfiguration = apps.get_model('moodle', 'MoodleEnterpriseCustomerConfiguration')
+
+    for moodle_enterprise_configuration in MoodleEnterpriseCustomerConfiguration.objects.all():
+        moodle_enterprise_configuration.decrypted_username = moodle_enterprise_configuration.username
+        moodle_enterprise_configuration.decrypted_password = moodle_enterprise_configuration.password
+        moodle_enterprise_configuration.decrypted_token = moodle_enterprise_configuration.token
+        moodle_enterprise_configuration.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('moodle', '0028_auto_20230928_1530'),
+    ]
+
+    operations = [
+        migrations.RunPython(populate_decrypted_fields, reverse_code=migrations.RunPython.noop),
+    ]

integrated_channels/moodle/migrations/0030_merge_0028_auto_20231116_1826_0029_auto_20231106_1233.py

@@ -0,0 +1,14 @@
+# Generated by Django 3.2.20 on 2023-11-22 08:24
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('moodle', '0028_auto_20231116_1826'),
+        ('moodle', '0029_auto_20231106_1233'),
+    ]
+
+    operations = [
+    ]

integrated_channels/moodle/models.py

@@ -5,6 +5,7 @@
 import json
 from logging import getLogger
 
+from fernet_fields import EncryptedCharField
 from simple_history.models import HistoricalRecords
 
 from django.db import models
@@ -64,6 +65,17 @@ class MoodleEnterpriseCustomerConfiguration(EnterpriseCustomerPluginConfiguratio
         )
     )
 
+    decrypted_username = EncryptedCharField(
+        max_length=255,
+        verbose_name="Encrypted Webservice Username",
+        blank=True,
+        help_text=_(
+            "The encrypted API user's username used to obtain new tokens."
+            " It will be encrypted when stored in the database."
+        ),
+        null=True,
+    )
+
     password = models.CharField(
         max_length=255,
         blank=True,
@@ -73,6 +85,17 @@ class MoodleEnterpriseCustomerConfiguration(EnterpriseCustomerPluginConfiguratio
         )
     )
 
+    decrypted_password = EncryptedCharField(
+        max_length=255,
+        verbose_name="Encrypted Webservice Password",
+        blank=True,
+        help_text=_(
+            "The encrypted API user's password used to obtain new tokens."
+            " It will be encrypted when stored in the database."
+        ),
+        null=True,
+    )
+
     token = models.CharField(
         max_length=255,
         blank=True,
@@ -82,6 +105,17 @@ class MoodleEnterpriseCustomerConfiguration(EnterpriseCustomerPluginConfiguratio
         )
     )
 
+    decrypted_token = EncryptedCharField(
+        max_length=255,
+        verbose_name="Encrypted Webservice Token",
+        blank=True,
+        help_text=_(
+            "The encrypted API user's token used to obtain new tokens."
+            " It will be encrypted when stored in the database."
+        ),
+        null=True,
+    )
+
     transmission_chunk_size = models.IntegerField(
         default=1,
         help_text=_("The maximum number of data items to transmit to the integrated channel with each request.")
@@ -102,6 +136,12 @@ class MoodleEnterpriseCustomerConfiguration(EnterpriseCustomerPluginConfiguratio
         )
     )
 
+    enable_incomplete_progress_transmission = models.BooleanField(
+        help_text=_("When set to True, the configured customer will receive learner data transmissions, for incomplete"
+                    " courses as well"),
+        default=False,
+    )
+
     history = HistoricalRecords()
 
     class Meta:

tests/test_integrated_channels/test_integrated_channel/test_transmitters/test_learner_data.py

@@ -9,7 +9,6 @@
 import ddt
 from pytest import mark
 
-from integrated_channels.exceptions import ClientError
 from integrated_channels.integrated_channel.exporters.learner_data import LearnerExporter
 from integrated_channels.integrated_channel.tasks import transmit_single_learner_data
 from integrated_channels.integrated_channel.transmitters.learner_data import LearnerTransmitter

tests/test_integrated_channels/test_moodle/test_transmitters/test_learner_data.py

@@ -4,11 +4,15 @@
 import datetime
 import unittest
 from unittest import mock
+from unittest.mock import Mock
 
 from pytest import mark
 
+from integrated_channels.integrated_channel.exporters.learner_data import LearnerExporter, LearnerExporterUtility
+from integrated_channels.integrated_channel.transmitters.learner_data import LearnerTransmitter
 from integrated_channels.moodle.models import MoodleLearnerDataTransmissionAudit
 from integrated_channels.moodle.transmitters import learner_data
+from integrated_channels.utils import encode_data_for_logging, generate_formatted_log
 from test_utils import factories
 
 
@@ -58,6 +62,8 @@ def setUp(self):
         self.create_course_completion_mock = create_course_completion_mock.start()
         self.addCleanup(create_course_completion_mock.stop)
 
+        self.learner_transmitter = LearnerTransmitter(self.enterprise_config)
+
     def test_transmit_success(self):
         """
         Learner data transmission is successful and the payload is saved with the appropriate data.
@@ -70,3 +76,53 @@ def test_transmit_success(self):
         self.create_course_completion_mock.assert_called_with(self.payload.moodle_user_email, self.payload.serialize())
         assert self.payload.status == '200'
         assert self.payload.error_message == ''
+
+    @mock.patch("integrated_channels.integrated_channel.models.LearnerDataTransmissionAudit")
+    @mock.patch('integrated_channels.integrated_channel.transmitters.'
+                'learner_data.LOGGER', autospec=True)
+    def test_incomplete_progress_learner_data_transmission(self, mock_logger, learner_data_transmission_audit_mock):
+        """
+        Test that a customer's configuration can run in enable incomplete progress transmission mode
+        """
+        # Set boolean flag to true
+        self.enterprise_config.enable_incomplete_progress_transmission = True
+
+        self.learner_transmitter.client.create_course_completion = Mock(return_value=(200, 'success'))
+
+        LearnerExporterMock = LearnerExporter
+
+        learner_data_transmission_audit_mock.serialize = Mock(return_value='serialized data')
+        learner_data_transmission_audit_mock.user_id = 1
+        learner_data_transmission_audit_mock.enterprise_course_enrollment_id = 1
+        learner_data_transmission_audit_mock.course_completed = False
+        learner_data_transmission_audit_mock.course_id = 'course_id'
+        LearnerExporterMock.export = Mock(return_value=[learner_data_transmission_audit_mock])
+        lms_user_id = LearnerExporterUtility.lms_user_id_for_ent_course_enrollment_id(
+            learner_data_transmission_audit_mock.enterprise_course_enrollment_id
+        )
+        serialized_payload = learner_data_transmission_audit_mock.serialize(
+            enterprise_configuration=self.enterprise_config
+        )
+        encoded_serialized_payload = encode_data_for_logging(serialized_payload)
+        self.learner_transmitter.transmit(
+            LearnerExporterMock,
+            remote_user_id='user_id'
+        )
+        # with enable_incomplete_progress_transmission = True we should be able to call this method
+        assert self.learner_transmitter.client.create_course_completion.called
+
+        # Set boolean flag to false
+        self.enterprise_config.enable_incomplete_progress_transmission = False
+        self.learner_transmitter.transmit(
+            LearnerExporterMock,
+            remote_user_id='user_id'
+        )
+        mock_logger.info.assert_called_with(generate_formatted_log(
+            self.enterprise_config.channel_code(),
+            self.enterprise_config.enterprise_customer.uuid or None,
+            lms_user_id,
+            learner_data_transmission_audit_mock.course_id,
+            'Skipping in-progress enterprise enrollment record '
+            f'integrated_channel_remote_user_id={learner_data_transmission_audit_mock.user_id}, '
+            f'integrated_channel_serialized_payload_base64={encoded_serialized_payload}'
+        ))