[#12048] Data migration for course entities

src/client/java/teammates/client/scripts/sql/DataMigrationForAccountAndReadNotificationSql.java

@@ -141,16 +141,15 @@ protected void migrateEntity(teammates.storage.entity.Account oldAccount) {
 
         TypedQuery<teammates.storage.sqlentity.Account> query = HibernateUtil.createQuery(cr);
 
-
         boolean isEntityInDb = query.getResultList().size() != 0;
         HibernateUtil.commitTransaction();
 
         // In db, but somehow not set as migrated.
         if (isEntityInDb) {
             oldAccount.setMigrated(true);
-            entitiesOldAccountSavingBuffer.add(oldAccount);    
+            entitiesOldAccountSavingBuffer.add(oldAccount);
             return;
-        };
+        }
 
         teammates.storage.sqlentity.Account newAccount = new teammates.storage.sqlentity.Account(
                 oldAccount.getGoogleId(),

src/client/java/teammates/client/scripts/sql/DataMigrationForCourseSql.java

@@ -0,0 +1,57 @@
+package teammates.client.scripts.sql;
+
+import com.googlecode.objectify.cmd.Query;
+
+import teammates.storage.entity.Course;
+
+/**
+ * Data migration class for course entity.
+ */
+@SuppressWarnings("PMD")
+public class DataMigrationForCourseSql extends
+        DataMigrationEntitiesBaseScriptSql<teammates.storage.entity.Course, teammates.storage.sqlentity.Course> {
+
+    public static void main(String[] args) {
+        new DataMigrationForCourseSql().doOperationRemotely();
+    }
+
+    @Override
+    protected Query<Course> getFilterQuery() {
+        return ofy().load().type(teammates.storage.entity.Course.class);
+    }
+
+    @Override
+    protected boolean isPreview() {
+        return false;
+    }
+
+    /*
+     * Sets the migration criteria used in isMigrationNeeded.
+     */
+    @Override
+    protected void setMigrationCriteria() {
+        // No migration criteria currently needed.
+    }
+
+    @Override
+    protected boolean isMigrationNeeded(Course entity) {
+        //        HibernateUtil.beginTransaction();
+        //        teammates.storage.sqlentity.Course course = HibernateUtil.get(
+        //                teammates.storage.sqlentity.Course.class, entity.getUniqueId());
+        //        HibernateUtil.commitTransaction();
+        //        return course == null;
+        return true;
+    }
+
+    @Override
+    protected void migrateEntity(Course oldCourse) throws Exception {
+        teammates.storage.sqlentity.Course newCourse = new teammates.storage.sqlentity.Course(
+                oldCourse.getUniqueId(),
+                oldCourse.getName(),
+                oldCourse.getTimeZone(),
+                oldCourse.getInstitute());
+        // newCourse.setCreatedAt(oldCourse.getCreatedAt());
+        newCourse.setDeletedAt(oldCourse.getDeletedAt());
+        saveEntityDeferred(newCourse);
+    }
+}

src/client/java/teammates/client/scripts/sql/MarkIsMigratedForAccounts.java

@@ -1,18 +1,13 @@
 package teammates.client.scripts.sql;
 
-import java.io.IOException;
 // CHECKSTYLE.OFF:ImportOrder
 import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
-import com.google.cloud.datastore.Cursor;
-import com.google.cloud.datastore.QueryResults;
 import com.googlecode.objectify.cmd.Query;
 
 import jakarta.persistence.TypedQuery;
@@ -21,7 +16,6 @@
 import jakarta.persistence.criteria.Order;
 import jakarta.persistence.criteria.Root;
 import teammates.storage.sqlentity.Account;
-import teammates.test.FileHelper;
 import teammates.client.connector.DatastoreClient;
 import teammates.client.util.ClientProperties;
 import teammates.common.util.HibernateUtil;
@@ -32,35 +26,33 @@
  */
 @SuppressWarnings("PMD")
 public class MarkIsMigratedForAccounts extends DatastoreClient {
+    /**
+     * Batch size to fetch per page.
+     */
+    protected static final int CONST_SQL_FETCH_BASE_SIZE = 1000;
     private static final String BASE_LOG_URI = "src/client/java/teammates/client/scripts/log/";
 
     /* NOTE
      * Before running the verification, please enable hibernate.jdbc.fetch_size in HibernateUtil.java
      * for optimized batch-fetching.
     */
-
-    /**
-     * Batch size to fetch per page.
-     */
-    protected static final int CONST_SQL_FETCH_BASE_SIZE = 1000;
-
-    AtomicLong numberOfScannedKey;
-    AtomicLong numberOfAffectedEntities;
-    AtomicLong numberOfUpdatedEntities;
-
     /** Datastore entity class. */
     protected Class<teammates.storage.entity.Account> datastoreEntityClass = teammates.storage.entity.Account.class;
 
     /** SQL entity class. */
     protected Class<Account> sqlEntityClass = Account.class;
 
+    //    AtomicLong numberOfScannedKey;
+    //    AtomicLong numberOfAffectedEntities;
+    //    AtomicLong numberOfUpdatedEntities;
+
     private long entitiesVerified = 0;
     private long entitiesSetToIsMigrated = 0;
 
     public MarkIsMigratedForAccounts() {
-        numberOfScannedKey = new AtomicLong();
-        numberOfAffectedEntities = new AtomicLong();
-        numberOfUpdatedEntities = new AtomicLong();
+        //        numberOfScannedKey = new AtomicLong();
+        //        numberOfAffectedEntities = new AtomicLong();
+        //        numberOfUpdatedEntities = new AtomicLong();
 
         String connectionUrl = ClientProperties.SCRIPT_API_URL;
         String username = ClientProperties.SCRIPT_API_NAME;
@@ -184,21 +176,22 @@ protected List<Map.Entry<Account, Account>> checkAllEntitiesForFailures() {
                     failures.add(new AbstractMap.SimpleEntry<Account, Account>(sqlEntity, null));
                     continue;
                 }
-                
+
                 if (!datastoreEntity.isMigrated()) {
                     datastoreEntity.setMigrated(true);
                     setMigratedAccountBuffer.add(datastoreEntity);
                 }
             }
 
             /* Flushing the buffer */
-            if (setMigratedAccountBuffer.size() != 0 ) {
+            if (setMigratedAccountBuffer.size() != 0) {
                 long startTimeForDatastoreFlushing = System.currentTimeMillis();
                 entitiesSetToIsMigrated += setMigratedAccountBuffer.size();
                 ofy().save().entities(setMigratedAccountBuffer).now();
                 setMigratedAccountBuffer.clear();
                 long endTimeForDatastoreFlushing = System.currentTimeMillis();
-                log("Flushing for datastore " + (endTimeForDatastoreFlushing - startTimeForDatastoreFlushing) + " milliseconds");    
+                log("Flushing for datastore " + (endTimeForDatastoreFlushing - startTimeForDatastoreFlushing)
+                        + " milliseconds");
             }
         }
 
@@ -226,7 +219,7 @@ protected void runCheckAllEntities(Class<Account> sqlEntityClass,
         }
 
         long checkEndTime = System.currentTimeMillis();
-        
+
         log("Entity took " + (checkEndTime - checkStartTime) + " milliseconds to verify");
         log("Verified " + entitiesVerified + " SQL entities successfully");
         log("Number of datastore accounts set to isMigrated " + entitiesSetToIsMigrated);

src/client/java/teammates/client/scripts/sql/PatchDataMigrationForUsageStatisticsSql.java

@@ -35,7 +35,6 @@
 // CHECKSTYLE.ON:ImportOrder
 /**
  * Patch Migration Script for Usage Statistics
- * 
  * Batch select datastore usage statistics, and then batch select SQL entities
  * with the same start timestamps.
  * Compare the size of the two list, if is not equal, find the missing one in

src/client/java/teammates/client/scripts/sql/ReverseDataMigrationForAccount.java

@@ -3,7 +3,6 @@
 // CHECKSTYLE.OFF:ImportOrder
 import java.time.Instant;
 import java.time.format.DateTimeFormatter;
-import java.util.AbstractMap;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -13,20 +12,15 @@
 import jakarta.persistence.TypedQuery;
 import jakarta.persistence.criteria.CriteriaBuilder;
 import jakarta.persistence.criteria.CriteriaQuery;
-import jakarta.persistence.criteria.Order;
 import jakarta.persistence.criteria.Root;
 
 import teammates.client.connector.DatastoreClient;
 import teammates.client.util.ClientProperties;
 import teammates.common.util.HibernateUtil;
-import teammates.storage.sqlentity.AccountRequest;
 
 // CHECKSTYLE.ON:ImportOrder
 /**
  * Protected methods may be overriden.
- * 
- * @param <E> Datastore entity
- * @param <T> SQL entity
  */
 @SuppressWarnings("PMD")
 public class ReverseDataMigrationForAccount
@@ -88,7 +82,7 @@ protected List<teammates.storage.sqlentity.Account> getNewAccounts() {
         return query.getResultList();
     }
 
-    /*
+    /**
      * Reverse migrate accounts to datastore.
      */
     protected void reverseMigrateToDatastore() {
@@ -127,7 +121,7 @@ private String getLogPrefix() {
 
     /**
      * Log a line.
-     * 
+     *
      * @param logLine the line to log
      */
     protected void log(String logLine) {

src/client/java/teammates/client/scripts/sql/SeedDb.java

@@ -29,6 +29,7 @@
 import teammates.storage.api.OfyHelper;
 import teammates.storage.entity.Account;
 import teammates.storage.entity.AccountRequest;
+import teammates.storage.entity.Course;
 import teammates.storage.entity.Notification;
 import teammates.test.FileHelper;
 
@@ -37,8 +38,9 @@
  */
 @SuppressWarnings("PMD")
 public class SeedDb extends DatastoreClient {
-    private final LogicExtension logic = new LogicExtension();
 
+    private static final int MAX_ENTITY_SIZE = 10000;
+    private final LogicExtension logic = new LogicExtension();
     private Closeable closeable;
 
     /**
@@ -101,13 +103,42 @@ protected Instant getRandomInstant() {
      * Persists additional data.
      */
     protected void persistAdditionalData() {
-        int constEntitySize = 10000;
+        String[] args = {};
         // Each account will have this amount of read notifications
-        int constReadNotificationSize = 5;
-        int constNotificationSize = 1000;
-        assert constNotificationSize >= constReadNotificationSize;
+        seedNotificationAccountAndAccountRequest(5, 1000);
+        seedCourse();
 
-        String[] args = {};
+        GenerateUsageStatisticsObjects.main(args);
+    }
+
+    private void seedCourse() {
+        log("Seeding courses");
+        for (int i = 0; i < MAX_ENTITY_SIZE; i++) {
+            if (i % (MAX_ENTITY_SIZE / 5) == 0) {
+                log(String.format("Seeded %d %% of new sets of entities",
+                        (int) (100 * ((float) i / (float) MAX_ENTITY_SIZE))));
+            }
+
+            Random rand = new Random();
+
+            try {
+                String courseName = String.format("Course %s", i);
+                String courseInstitute = String.format("Institute %s", i);
+                String courseTimeZone = String.format("Time Zone %s", i);
+                Course course = new Course(UUID.randomUUID().toString(), courseName, courseTimeZone, courseInstitute,
+                        getRandomInstant(),
+                        rand.nextInt(3) > 1 ? null : getRandomInstant(), // set deletedAt randomly at 25% chance
+                        false);
+                ofy().save().entities(course).now();
+            } catch (Exception e) {
+                log(e.toString());
+            }
+        }
+    }
+
+    private void seedNotificationAccountAndAccountRequest(int constReadNotificationSize, int constNotificationSize) {
+        assert constNotificationSize >= constReadNotificationSize;
+        log("Seeding Notifications, Account and Account Request");
 
         Set<String> notificationsUuidSeen = new HashSet<String>();
         ArrayList<String> notificationUuids = new ArrayList<>();
@@ -145,11 +176,11 @@ protected void persistAdditionalData() {
             }
         }
 
-        for (int i = 0; i < constEntitySize; i++) {
+        for (int i = 0; i < MAX_ENTITY_SIZE; i++) {
 
-            if (i % (constEntitySize / 5) == 0) {
+            if (i % (MAX_ENTITY_SIZE / 5) == 0) {
                 log(String.format("Seeded %d %% of new sets of entities",
-                        (int) (100 * ((float) i / (float) constEntitySize))));
+                        (int) (100 * ((float) i / (float) MAX_ENTITY_SIZE))));
             }
 
             try {
@@ -181,8 +212,6 @@ protected void persistAdditionalData() {
                 log(e.toString());
             }
         }
-
-        GenerateUsageStatisticsObjects.main(args);
     }
 
     private void log(String logLine) {

src/client/java/teammates/client/scripts/sql/VerifyCourseAttributes.java

@@ -0,0 +1,42 @@
+package teammates.client.scripts.sql;
+
+import teammates.storage.entity.Course;
+
+/**
+ * Class for verifying account attributes.
+ */
+@SuppressWarnings("PMD")
+public class VerifyCourseAttributes
+        extends VerifyNonCourseEntityAttributesBaseScript<Course, teammates.storage.sqlentity.Course> {
+
+    public VerifyCourseAttributes() {
+        super(Course.class,
+                teammates.storage.sqlentity.Course.class);
+    }
+
+    @Override
+    protected String generateID(teammates.storage.sqlentity.Course sqlEntity) {
+        return sqlEntity.getId();
+    }
+
+    public static void main(String[] args) {
+        VerifyCourseAttributes script = new VerifyCourseAttributes();
+        script.doOperationRemotely();
+    }
+
+    // Used for sql data migration
+    @Override
+    public boolean equals(teammates.storage.sqlentity.Course sqlEntity, Course datastoreEntity) {
+        try {
+            return sqlEntity.getId().equals(datastoreEntity.getUniqueId())
+                    && sqlEntity.getName().equals(datastoreEntity.getName())
+                    && sqlEntity.getTimeZone().equals(datastoreEntity.getTimeZone())
+                    && sqlEntity.getInstitute().equals(datastoreEntity.getInstitute())
+                    // && sqlEntity.getCreatedAt().equals(datastoreEntity.getCreatedAt())
+                    && datastoreEntity.getDeletedAt() == null ? sqlEntity.getDeletedAt() == null
+                            : sqlEntity.getDeletedAt().equals(datastoreEntity.getDeletedAt());
+        } catch (IllegalArgumentException iae) {
+            return false;
+        }
+    }
+}