diff --git a/src/client/java/teammates/client/scripts/sql/DataMigrationForAccountAndReadNotificationSql.java b/src/client/java/teammates/client/scripts/sql/DataMigrationForAccountAndReadNotificationSql.java index 0d7becb01ec..24e0468fd5a 100644 --- a/src/client/java/teammates/client/scripts/sql/DataMigrationForAccountAndReadNotificationSql.java +++ b/src/client/java/teammates/client/scripts/sql/DataMigrationForAccountAndReadNotificationSql.java @@ -141,16 +141,15 @@ protected void migrateEntity(teammates.storage.entity.Account oldAccount) { TypedQuery query = HibernateUtil.createQuery(cr); - boolean isEntityInDb = query.getResultList().size() != 0; HibernateUtil.commitTransaction(); // In db, but somehow not set as migrated. if (isEntityInDb) { oldAccount.setMigrated(true); - entitiesOldAccountSavingBuffer.add(oldAccount); + entitiesOldAccountSavingBuffer.add(oldAccount); return; - }; + } teammates.storage.sqlentity.Account newAccount = new teammates.storage.sqlentity.Account( oldAccount.getGoogleId(), diff --git a/src/client/java/teammates/client/scripts/sql/DataMigrationForCourseSql.java b/src/client/java/teammates/client/scripts/sql/DataMigrationForCourseSql.java new file mode 100644 index 00000000000..dc0cee4724a --- /dev/null +++ b/src/client/java/teammates/client/scripts/sql/DataMigrationForCourseSql.java @@ -0,0 +1,57 @@ +package teammates.client.scripts.sql; + +import com.googlecode.objectify.cmd.Query; + +import teammates.storage.entity.Course; + +/** + * Data migration class for course entity. + */ +@SuppressWarnings("PMD") +public class DataMigrationForCourseSql extends + DataMigrationEntitiesBaseScriptSql { + + public static void main(String[] args) { + new DataMigrationForCourseSql().doOperationRemotely(); + } + + @Override + protected Query getFilterQuery() { + return ofy().load().type(teammates.storage.entity.Course.class); + } + + @Override + protected boolean isPreview() { + return false; + } + + /* + * Sets the migration criteria used in isMigrationNeeded. + */ + @Override + protected void setMigrationCriteria() { + // No migration criteria currently needed. + } + + @Override + protected boolean isMigrationNeeded(Course entity) { + // HibernateUtil.beginTransaction(); + // teammates.storage.sqlentity.Course course = HibernateUtil.get( + // teammates.storage.sqlentity.Course.class, entity.getUniqueId()); + // HibernateUtil.commitTransaction(); + // return course == null; + return true; + } + + @Override + protected void migrateEntity(Course oldCourse) throws Exception { + teammates.storage.sqlentity.Course newCourse = new teammates.storage.sqlentity.Course( + oldCourse.getUniqueId(), + oldCourse.getName(), + oldCourse.getTimeZone(), + oldCourse.getInstitute()); + // newCourse.setCreatedAt(oldCourse.getCreatedAt()); + newCourse.setDeletedAt(oldCourse.getDeletedAt()); + saveEntityDeferred(newCourse); + } +} diff --git a/src/client/java/teammates/client/scripts/sql/MarkIsMigratedForAccounts.java b/src/client/java/teammates/client/scripts/sql/MarkIsMigratedForAccounts.java index 75e1536b5bd..ceaeb6b6b80 100644 --- a/src/client/java/teammates/client/scripts/sql/MarkIsMigratedForAccounts.java +++ b/src/client/java/teammates/client/scripts/sql/MarkIsMigratedForAccounts.java @@ -1,18 +1,13 @@ package teammates.client.scripts.sql; -import java.io.IOException; // CHECKSTYLE.OFF:ImportOrder import java.util.AbstractMap; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; -import com.google.cloud.datastore.Cursor; -import com.google.cloud.datastore.QueryResults; import com.googlecode.objectify.cmd.Query; import jakarta.persistence.TypedQuery; @@ -21,7 +16,6 @@ import jakarta.persistence.criteria.Order; import jakarta.persistence.criteria.Root; import teammates.storage.sqlentity.Account; -import teammates.test.FileHelper; import teammates.client.connector.DatastoreClient; import teammates.client.util.ClientProperties; import teammates.common.util.HibernateUtil; @@ -32,35 +26,33 @@ */ @SuppressWarnings("PMD") public class MarkIsMigratedForAccounts extends DatastoreClient { + /** + * Batch size to fetch per page. + */ + protected static final int CONST_SQL_FETCH_BASE_SIZE = 1000; private static final String BASE_LOG_URI = "src/client/java/teammates/client/scripts/log/"; /* NOTE * Before running the verification, please enable hibernate.jdbc.fetch_size in HibernateUtil.java * for optimized batch-fetching. */ - - /** - * Batch size to fetch per page. - */ - protected static final int CONST_SQL_FETCH_BASE_SIZE = 1000; - - AtomicLong numberOfScannedKey; - AtomicLong numberOfAffectedEntities; - AtomicLong numberOfUpdatedEntities; - /** Datastore entity class. */ protected Class datastoreEntityClass = teammates.storage.entity.Account.class; /** SQL entity class. */ protected Class sqlEntityClass = Account.class; + // AtomicLong numberOfScannedKey; + // AtomicLong numberOfAffectedEntities; + // AtomicLong numberOfUpdatedEntities; + private long entitiesVerified = 0; private long entitiesSetToIsMigrated = 0; public MarkIsMigratedForAccounts() { - numberOfScannedKey = new AtomicLong(); - numberOfAffectedEntities = new AtomicLong(); - numberOfUpdatedEntities = new AtomicLong(); + // numberOfScannedKey = new AtomicLong(); + // numberOfAffectedEntities = new AtomicLong(); + // numberOfUpdatedEntities = new AtomicLong(); String connectionUrl = ClientProperties.SCRIPT_API_URL; String username = ClientProperties.SCRIPT_API_NAME; @@ -184,7 +176,7 @@ protected List> checkAllEntitiesForFailures() { failures.add(new AbstractMap.SimpleEntry(sqlEntity, null)); continue; } - + if (!datastoreEntity.isMigrated()) { datastoreEntity.setMigrated(true); setMigratedAccountBuffer.add(datastoreEntity); @@ -192,13 +184,14 @@ protected List> checkAllEntitiesForFailures() { } /* Flushing the buffer */ - if (setMigratedAccountBuffer.size() != 0 ) { + if (setMigratedAccountBuffer.size() != 0) { long startTimeForDatastoreFlushing = System.currentTimeMillis(); entitiesSetToIsMigrated += setMigratedAccountBuffer.size(); ofy().save().entities(setMigratedAccountBuffer).now(); setMigratedAccountBuffer.clear(); long endTimeForDatastoreFlushing = System.currentTimeMillis(); - log("Flushing for datastore " + (endTimeForDatastoreFlushing - startTimeForDatastoreFlushing) + " milliseconds"); + log("Flushing for datastore " + (endTimeForDatastoreFlushing - startTimeForDatastoreFlushing) + + " milliseconds"); } } @@ -226,7 +219,7 @@ protected void runCheckAllEntities(Class sqlEntityClass, } long checkEndTime = System.currentTimeMillis(); - + log("Entity took " + (checkEndTime - checkStartTime) + " milliseconds to verify"); log("Verified " + entitiesVerified + " SQL entities successfully"); log("Number of datastore accounts set to isMigrated " + entitiesSetToIsMigrated); diff --git a/src/client/java/teammates/client/scripts/sql/PatchDataMigrationForUsageStatisticsSql.java b/src/client/java/teammates/client/scripts/sql/PatchDataMigrationForUsageStatisticsSql.java index 96474589c15..b5dc8052114 100644 --- a/src/client/java/teammates/client/scripts/sql/PatchDataMigrationForUsageStatisticsSql.java +++ b/src/client/java/teammates/client/scripts/sql/PatchDataMigrationForUsageStatisticsSql.java @@ -35,7 +35,6 @@ // CHECKSTYLE.ON:ImportOrder /** * Patch Migration Script for Usage Statistics - * * Batch select datastore usage statistics, and then batch select SQL entities * with the same start timestamps. * Compare the size of the two list, if is not equal, find the missing one in diff --git a/src/client/java/teammates/client/scripts/sql/ReverseDataMigrationForAccount.java b/src/client/java/teammates/client/scripts/sql/ReverseDataMigrationForAccount.java index d3c77776f02..8a31afb4efb 100644 --- a/src/client/java/teammates/client/scripts/sql/ReverseDataMigrationForAccount.java +++ b/src/client/java/teammates/client/scripts/sql/ReverseDataMigrationForAccount.java @@ -3,7 +3,6 @@ // CHECKSTYLE.OFF:ImportOrder import java.time.Instant; import java.time.format.DateTimeFormatter; -import java.util.AbstractMap; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -13,20 +12,15 @@ import jakarta.persistence.TypedQuery; import jakarta.persistence.criteria.CriteriaBuilder; import jakarta.persistence.criteria.CriteriaQuery; -import jakarta.persistence.criteria.Order; import jakarta.persistence.criteria.Root; import teammates.client.connector.DatastoreClient; import teammates.client.util.ClientProperties; import teammates.common.util.HibernateUtil; -import teammates.storage.sqlentity.AccountRequest; // CHECKSTYLE.ON:ImportOrder /** * Protected methods may be overriden. - * - * @param Datastore entity - * @param SQL entity */ @SuppressWarnings("PMD") public class ReverseDataMigrationForAccount @@ -88,7 +82,7 @@ protected List getNewAccounts() { return query.getResultList(); } - /* + /** * Reverse migrate accounts to datastore. */ protected void reverseMigrateToDatastore() { @@ -127,7 +121,7 @@ private String getLogPrefix() { /** * Log a line. - * + * * @param logLine the line to log */ protected void log(String logLine) { diff --git a/src/client/java/teammates/client/scripts/sql/SeedDb.java b/src/client/java/teammates/client/scripts/sql/SeedDb.java index 40185c120de..483d6665914 100644 --- a/src/client/java/teammates/client/scripts/sql/SeedDb.java +++ b/src/client/java/teammates/client/scripts/sql/SeedDb.java @@ -29,6 +29,7 @@ import teammates.storage.api.OfyHelper; import teammates.storage.entity.Account; import teammates.storage.entity.AccountRequest; +import teammates.storage.entity.Course; import teammates.storage.entity.Notification; import teammates.test.FileHelper; @@ -37,8 +38,9 @@ */ @SuppressWarnings("PMD") public class SeedDb extends DatastoreClient { - private final LogicExtension logic = new LogicExtension(); + private static final int MAX_ENTITY_SIZE = 10000; + private final LogicExtension logic = new LogicExtension(); private Closeable closeable; /** @@ -101,13 +103,42 @@ protected Instant getRandomInstant() { * Persists additional data. */ protected void persistAdditionalData() { - int constEntitySize = 10000; + String[] args = {}; // Each account will have this amount of read notifications - int constReadNotificationSize = 5; - int constNotificationSize = 1000; - assert constNotificationSize >= constReadNotificationSize; + seedNotificationAccountAndAccountRequest(5, 1000); + seedCourse(); - String[] args = {}; + GenerateUsageStatisticsObjects.main(args); + } + + private void seedCourse() { + log("Seeding courses"); + for (int i = 0; i < MAX_ENTITY_SIZE; i++) { + if (i % (MAX_ENTITY_SIZE / 5) == 0) { + log(String.format("Seeded %d %% of new sets of entities", + (int) (100 * ((float) i / (float) MAX_ENTITY_SIZE)))); + } + + Random rand = new Random(); + + try { + String courseName = String.format("Course %s", i); + String courseInstitute = String.format("Institute %s", i); + String courseTimeZone = String.format("Time Zone %s", i); + Course course = new Course(UUID.randomUUID().toString(), courseName, courseTimeZone, courseInstitute, + getRandomInstant(), + rand.nextInt(3) > 1 ? null : getRandomInstant(), // set deletedAt randomly at 25% chance + false); + ofy().save().entities(course).now(); + } catch (Exception e) { + log(e.toString()); + } + } + } + + private void seedNotificationAccountAndAccountRequest(int constReadNotificationSize, int constNotificationSize) { + assert constNotificationSize >= constReadNotificationSize; + log("Seeding Notifications, Account and Account Request"); Set notificationsUuidSeen = new HashSet(); ArrayList notificationUuids = new ArrayList<>(); @@ -145,11 +176,11 @@ protected void persistAdditionalData() { } } - for (int i = 0; i < constEntitySize; i++) { + for (int i = 0; i < MAX_ENTITY_SIZE; i++) { - if (i % (constEntitySize / 5) == 0) { + if (i % (MAX_ENTITY_SIZE / 5) == 0) { log(String.format("Seeded %d %% of new sets of entities", - (int) (100 * ((float) i / (float) constEntitySize)))); + (int) (100 * ((float) i / (float) MAX_ENTITY_SIZE)))); } try { @@ -181,8 +212,6 @@ protected void persistAdditionalData() { log(e.toString()); } } - - GenerateUsageStatisticsObjects.main(args); } private void log(String logLine) { diff --git a/src/client/java/teammates/client/scripts/sql/VerifyCourseAttributes.java b/src/client/java/teammates/client/scripts/sql/VerifyCourseAttributes.java new file mode 100644 index 00000000000..121c83d67f7 --- /dev/null +++ b/src/client/java/teammates/client/scripts/sql/VerifyCourseAttributes.java @@ -0,0 +1,42 @@ +package teammates.client.scripts.sql; + +import teammates.storage.entity.Course; + +/** + * Class for verifying account attributes. + */ +@SuppressWarnings("PMD") +public class VerifyCourseAttributes + extends VerifyNonCourseEntityAttributesBaseScript { + + public VerifyCourseAttributes() { + super(Course.class, + teammates.storage.sqlentity.Course.class); + } + + @Override + protected String generateID(teammates.storage.sqlentity.Course sqlEntity) { + return sqlEntity.getId(); + } + + public static void main(String[] args) { + VerifyCourseAttributes script = new VerifyCourseAttributes(); + script.doOperationRemotely(); + } + + // Used for sql data migration + @Override + public boolean equals(teammates.storage.sqlentity.Course sqlEntity, Course datastoreEntity) { + try { + return sqlEntity.getId().equals(datastoreEntity.getUniqueId()) + && sqlEntity.getName().equals(datastoreEntity.getName()) + && sqlEntity.getTimeZone().equals(datastoreEntity.getTimeZone()) + && sqlEntity.getInstitute().equals(datastoreEntity.getInstitute()) + // && sqlEntity.getCreatedAt().equals(datastoreEntity.getCreatedAt()) + && datastoreEntity.getDeletedAt() == null ? sqlEntity.getDeletedAt() == null + : sqlEntity.getDeletedAt().equals(datastoreEntity.getDeletedAt()); + } catch (IllegalArgumentException iae) { + return false; + } + } +}