From 3e108c718f9eea7fa429bc32a3a81a9fef0a4ad2 Mon Sep 17 00:00:00 2001 From: YX Z Date: Sat, 6 Apr 2024 10:24:30 +0800 Subject: [PATCH 1/6] Add migration and verification script for feedback session --- .../DataMigrationForFeedbackSessionSql.java | 74 +++++++++++++++++++ .../teammates/client/scripts/sql/SeedDb.java | 45 ++++++++++- .../sql/VerifyFeedbackSessionAttributes.java | 55 ++++++++++++++ 3 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 src/client/java/teammates/client/scripts/sql/DataMigrationForFeedbackSessionSql.java create mode 100644 src/client/java/teammates/client/scripts/sql/VerifyFeedbackSessionAttributes.java diff --git a/src/client/java/teammates/client/scripts/sql/DataMigrationForFeedbackSessionSql.java b/src/client/java/teammates/client/scripts/sql/DataMigrationForFeedbackSessionSql.java new file mode 100644 index 00000000000..172c9b28878 --- /dev/null +++ b/src/client/java/teammates/client/scripts/sql/DataMigrationForFeedbackSessionSql.java @@ -0,0 +1,74 @@ +package teammates.client.scripts.sql; + +import java.time.Duration; + +// CHECKSTYLE.OFF:ImportOrder +import com.googlecode.objectify.cmd.Query; +import teammates.common.util.HibernateUtil; +import teammates.storage.entity.FeedbackSession; +import teammates.storage.sqlentity.Course; + +// CHECKSTYLE.ON:ImportOrder +/** + * Data migration class for feddback sessions. + */ +@SuppressWarnings("PMD") +public class DataMigrationForFeedbackSessionSql + extends DataMigrationEntitiesBaseScriptSql { + + public static void main(String[] args) { + new DataMigrationForFeedbackSessionSql().doOperationRemotely(); + } + + @Override + protected Query getFilterQuery() { + return ofy().load().type(teammates.storage.entity.FeedbackSession.class); + } + + @Override + protected boolean isPreview() { + return false; + } + + @Override + protected void setMigrationCriteria() { + // No migration criteria currently needed. + } + + @Override + protected boolean isMigrationNeeded(FeedbackSession entity) { + return true; + } + + @Override + protected void migrateEntity(FeedbackSession oldEntity) throws Exception { + HibernateUtil.beginTransaction(); + Course course = HibernateUtil.get(teammates.storage.sqlentity.Course.class, oldEntity.getCourseId()); + HibernateUtil.commitTransaction(); + + teammates.storage.sqlentity.FeedbackSession newFeedbackSession = new teammates.storage.sqlentity.FeedbackSession( + oldEntity.getFeedbackSessionName(), + course, + oldEntity.getCreatorEmail(), + oldEntity.getInstructions(), + oldEntity.getStartTime(), + oldEntity.getEndTime(), + oldEntity.getSessionVisibleFromTime(), + oldEntity.getResultsVisibleFromTime(), + Duration.ofMinutes(oldEntity.getGracePeriod()), + oldEntity.isOpeningEmailEnabled(), + oldEntity.isClosingEmailEnabled(), + oldEntity.isPublishedEmailEnabled() + ); + + newFeedbackSession.setClosedEmailSent(oldEntity.isSentClosedEmail()); + newFeedbackSession.setClosingSoonEmailSent(oldEntity.isSentClosingEmail()); + newFeedbackSession.setOpenEmailSent(oldEntity.isSentOpenEmail()); + newFeedbackSession.setOpeningSoonEmailSent(oldEntity.isSentOpeningSoonEmail()); + newFeedbackSession.setPublishedEmailSent(oldEntity.isSentPublishedEmail()); + newFeedbackSession.setDeletedAt(oldEntity.getDeletedTime()); + + saveEntityDeferred(newFeedbackSession); + } + +} diff --git a/src/client/java/teammates/client/scripts/sql/SeedDb.java b/src/client/java/teammates/client/scripts/sql/SeedDb.java index 483d6665914..60587927c80 100644 --- a/src/client/java/teammates/client/scripts/sql/SeedDb.java +++ b/src/client/java/teammates/client/scripts/sql/SeedDb.java @@ -30,6 +30,7 @@ import teammates.storage.entity.Account; import teammates.storage.entity.AccountRequest; import teammates.storage.entity.Course; +import teammates.storage.entity.FeedbackSession; import teammates.storage.entity.Notification; import teammates.test.FileHelper; @@ -40,6 +41,7 @@ public class SeedDb extends DatastoreClient { private static final int MAX_ENTITY_SIZE = 10000; + private static final int MAX_FEEDBACKSESSION_FOR_EACH_COURSE_SIZE = 3; private final LogicExtension logic = new LogicExtension(); private Closeable closeable; @@ -106,12 +108,12 @@ protected void persistAdditionalData() { String[] args = {}; // Each account will have this amount of read notifications seedNotificationAccountAndAccountRequest(5, 1000); - seedCourse(); + seedCourseAndRelatedEntites(); GenerateUsageStatisticsObjects.main(args); } - private void seedCourse() { + private void seedCourseAndRelatedEntites() { log("Seeding courses"); for (int i = 0; i < MAX_ENTITY_SIZE; i++) { if (i % (MAX_ENTITY_SIZE / 5) == 0) { @@ -120,12 +122,12 @@ private void seedCourse() { } Random rand = new Random(); - + String courseId = UUID.randomUUID().toString(); try { String courseName = String.format("Course %s", i); String courseInstitute = String.format("Institute %s", i); String courseTimeZone = String.format("Time Zone %s", i); - Course course = new Course(UUID.randomUUID().toString(), courseName, courseTimeZone, courseInstitute, + Course course = new Course(courseId, courseName, courseTimeZone, courseInstitute, getRandomInstant(), rand.nextInt(3) > 1 ? null : getRandomInstant(), // set deletedAt randomly at 25% chance false); @@ -133,6 +135,41 @@ private void seedCourse() { } catch (Exception e) { log(e.toString()); } + + // Uncomment to seed feedback sessions + // seedFeedbackSession(courseId); + } + } + + private void seedFeedbackSession(String courseId) { + Random rand = new Random(); + for (int i = 0; i < MAX_FEEDBACKSESSION_FOR_EACH_COURSE_SIZE; i++) { + try { + String feedbackSessionName = String.format("Feedback Session %s", i); + String feedbackSessionCourseId = courseId; + String feedbackSessionCreatorEmail = String.format("Creator Email %s", i); + String feedbackSessionInstructions = String.format("Instructions %s", i); + String timezone = String.format("Time Zone %s", i); + Instant feedbackSessionStartTime = getRandomInstant(); + Instant feedbackSessionEndTime = getRandomInstant(); + Instant feedbackSessionSessionVisibleFromTime = getRandomInstant(); + Instant feedbackSessionResultsVisibleFromTime = getRandomInstant(); + int feedbackSessionGracePeriod = rand.nextInt(3600); + + FeedbackSession feedbackSession = new FeedbackSession(feedbackSessionName, feedbackSessionCourseId, + feedbackSessionCreatorEmail, feedbackSessionInstructions, getRandomInstant(), + rand.nextInt(3) > 1 ? null : getRandomInstant(), // set deletedAt randomly at 25% chance + feedbackSessionStartTime, feedbackSessionEndTime, + feedbackSessionSessionVisibleFromTime, feedbackSessionResultsVisibleFromTime, + timezone, feedbackSessionGracePeriod, + rand.nextBoolean(), rand.nextBoolean(), rand.nextBoolean(), rand.nextBoolean(), + rand.nextBoolean(), rand.nextBoolean(), rand.nextBoolean(), rand.nextBoolean(), + new HashMap(), new HashMap()); + + ofy().save().entities(feedbackSession).now(); + } catch (Exception e) { + log(e.toString()); + } } } diff --git a/src/client/java/teammates/client/scripts/sql/VerifyFeedbackSessionAttributes.java b/src/client/java/teammates/client/scripts/sql/VerifyFeedbackSessionAttributes.java new file mode 100644 index 00000000000..2d442851152 --- /dev/null +++ b/src/client/java/teammates/client/scripts/sql/VerifyFeedbackSessionAttributes.java @@ -0,0 +1,55 @@ +package teammates.client.scripts.sql; + +import java.time.Duration; + +import teammates.common.util.SanitizationHelper; +import teammates.storage.entity.FeedbackSession; + +/** + * Verification of the feedback session attributes. + */ +public class VerifyFeedbackSessionAttributes + extends VerifyNonCourseEntityAttributesBaseScript { + + public VerifyFeedbackSessionAttributes() { + super(FeedbackSession.class, teammates.storage.sqlentity.FeedbackSession.class); + } + + @Override + protected String generateID(teammates.storage.sqlentity.FeedbackSession sqlEntity) { + return FeedbackSession.generateId(sqlEntity.getName(), sqlEntity.getCourse().getId()); + } + + @Override + protected boolean equals(teammates.storage.sqlentity.FeedbackSession sqlEntity, FeedbackSession datastoreEntity) { + try { + return sqlEntity.getCourse().getId().equals(datastoreEntity.getCourseId()) + && sqlEntity.getName().equals(datastoreEntity.getFeedbackSessionName()) + && sqlEntity.getCreatorEmail().equals(datastoreEntity.getCreatorEmail()) + && sqlEntity.getInstructions() + .equals(SanitizationHelper.sanitizeForRichText(datastoreEntity.getInstructions())) + && sqlEntity.getStartTime().equals(datastoreEntity.getStartTime()) + && sqlEntity.getEndTime().equals(datastoreEntity.getEndTime()) + && sqlEntity.getSessionVisibleFromTime().equals(datastoreEntity.getSessionVisibleFromTime()) + && sqlEntity.getResultsVisibleFromTime().equals(datastoreEntity.getResultsVisibleFromTime()) + && sqlEntity.getGracePeriod().equals(Duration.ofMinutes(datastoreEntity.getGracePeriod())) + && sqlEntity.isOpeningEmailEnabled() == datastoreEntity.isOpeningEmailEnabled() + && sqlEntity.isClosingEmailEnabled() == datastoreEntity.isClosingEmailEnabled() + && sqlEntity.isOpenEmailSent() == datastoreEntity.isSentOpenEmail() + && sqlEntity.isOpeningSoonEmailSent() == datastoreEntity.isSentOpeningSoonEmail() + && sqlEntity.isClosedEmailSent() == datastoreEntity.isSentClosedEmail() + && sqlEntity.isClosingSoonEmailSent() == datastoreEntity.isSentClosingEmail() + && sqlEntity.isPublishedEmailSent() == datastoreEntity.isSentPublishedEmail() + && (sqlEntity.getDeletedAt() == datastoreEntity.getDeletedTime() + || sqlEntity.getDeletedAt().equals(datastoreEntity.getDeletedTime())); + } catch (IllegalArgumentException iae) { + return false; + } + } + + public static void main(String[] args) { + VerifyFeedbackSessionAttributes script = new VerifyFeedbackSessionAttributes(); + script.doOperationRemotely(); + } + +} From 2275560e1742791e016884fcf19b1fb71eb01cbc Mon Sep 17 00:00:00 2001 From: YX Z Date: Sat, 6 Apr 2024 21:15:56 +0800 Subject: [PATCH 2/6] Optimize data migration --- .../sql/DataMigrationForFeedbackSessionSql.java | 2 +- src/main/java/teammates/common/util/HibernateUtil.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/client/java/teammates/client/scripts/sql/DataMigrationForFeedbackSessionSql.java b/src/client/java/teammates/client/scripts/sql/DataMigrationForFeedbackSessionSql.java index 172c9b28878..95c0b76a58d 100644 --- a/src/client/java/teammates/client/scripts/sql/DataMigrationForFeedbackSessionSql.java +++ b/src/client/java/teammates/client/scripts/sql/DataMigrationForFeedbackSessionSql.java @@ -43,7 +43,7 @@ protected boolean isMigrationNeeded(FeedbackSession entity) { @Override protected void migrateEntity(FeedbackSession oldEntity) throws Exception { HibernateUtil.beginTransaction(); - Course course = HibernateUtil.get(teammates.storage.sqlentity.Course.class, oldEntity.getCourseId()); + Course course = HibernateUtil.getReference(teammates.storage.sqlentity.Course.class, oldEntity.getCourseId()); HibernateUtil.commitTransaction(); teammates.storage.sqlentity.FeedbackSession newFeedbackSession = new teammates.storage.sqlentity.FeedbackSession( diff --git a/src/main/java/teammates/common/util/HibernateUtil.java b/src/main/java/teammates/common/util/HibernateUtil.java index 3410038727a..896a5acb643 100644 --- a/src/main/java/teammates/common/util/HibernateUtil.java +++ b/src/main/java/teammates/common/util/HibernateUtil.java @@ -238,6 +238,16 @@ public static T get(Class entityType, Object id) { return HibernateUtil.getCurrentSession().get(entityType, id); } + /** + * Return an instance, whose state may be lazily fetched. + * If there is no such persistent instance, EntityNotFoundException is thrown when the instance state + * is first accessed. This method can be used to increase performance, if it is known that the instance exists + * and fetching of an entity is not necessary. + */ + public static T getReference(Class entityType, Object id) { + return HibernateUtil.getCurrentSession().getReference(entityType, id); + } + /** * Return the persistent instance of the given entity class with the given natural id, * or null if there is no such persistent instance. From dba227233d13e7f4951a2764de661bf7326ead1d Mon Sep 17 00:00:00 2001 From: YX Z Date: Sat, 6 Apr 2024 21:16:29 +0800 Subject: [PATCH 3/6] Fix lint --- src/main/java/teammates/common/util/HibernateUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/teammates/common/util/HibernateUtil.java b/src/main/java/teammates/common/util/HibernateUtil.java index 896a5acb643..98ce2ef2607 100644 --- a/src/main/java/teammates/common/util/HibernateUtil.java +++ b/src/main/java/teammates/common/util/HibernateUtil.java @@ -240,7 +240,7 @@ public static T get(Class entityType, Object id) { /** * Return an instance, whose state may be lazily fetched. - * If there is no such persistent instance, EntityNotFoundException is thrown when the instance state + * If there is no such persistent instance, EntityNotFoundException is thrown when the instance state * is first accessed. This method can be used to increase performance, if it is known that the instance exists * and fetching of an entity is not necessary. */ From 64e02d897e1f3bba9636b202f3db96f0e15e6af6 Mon Sep 17 00:00:00 2001 From: YX Z Date: Sat, 6 Apr 2024 21:16:58 +0800 Subject: [PATCH 4/6] Uncomment seed feedback session function --- src/client/java/teammates/client/scripts/sql/SeedDb.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client/java/teammates/client/scripts/sql/SeedDb.java b/src/client/java/teammates/client/scripts/sql/SeedDb.java index 60587927c80..7ed88ed8c81 100644 --- a/src/client/java/teammates/client/scripts/sql/SeedDb.java +++ b/src/client/java/teammates/client/scripts/sql/SeedDb.java @@ -136,8 +136,7 @@ private void seedCourseAndRelatedEntites() { log(e.toString()); } - // Uncomment to seed feedback sessions - // seedFeedbackSession(courseId); + seedFeedbackSession(courseId); } } From 280eaed3cda6215b3cd32e6cb6cd56c82f77e422 Mon Sep 17 00:00:00 2001 From: YX Z Date: Sat, 6 Apr 2024 21:17:24 +0800 Subject: [PATCH 5/6] Add counts verification script --- .../scripts/sql/VerifyCourseEntityCounts.java | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 src/client/java/teammates/client/scripts/sql/VerifyCourseEntityCounts.java diff --git a/src/client/java/teammates/client/scripts/sql/VerifyCourseEntityCounts.java b/src/client/java/teammates/client/scripts/sql/VerifyCourseEntityCounts.java new file mode 100644 index 00000000000..f4850062749 --- /dev/null +++ b/src/client/java/teammates/client/scripts/sql/VerifyCourseEntityCounts.java @@ -0,0 +1,75 @@ +package teammates.client.scripts.sql; + +// CHECKSTYLE.OFF:ImportOrder +import java.util.HashMap; +import java.util.Map; + +import jakarta.persistence.criteria.CriteriaBuilder; +import jakarta.persistence.criteria.CriteriaQuery; +import jakarta.persistence.criteria.Root; + +import teammates.client.connector.DatastoreClient; +import teammates.client.util.ClientProperties; +import teammates.common.util.HibernateUtil; +import teammates.storage.entity.BaseEntity; +// CHECKSTYLE.ON:ImportOrder + +/** + * Verify the counts of non-course entities are correct. + */ +@SuppressWarnings("PMD") +public class VerifyCourseEntityCounts extends DatastoreClient { + private VerifyCourseEntityCounts() { + String connectionUrl = ClientProperties.SCRIPT_API_URL; + String username = ClientProperties.SCRIPT_API_NAME; + String password = ClientProperties.SCRIPT_API_PASSWORD; + + HibernateUtil.buildSessionFactory(connectionUrl, username, password); + } + + public static void main(String[] args) throws Exception { + new VerifyCourseEntityCounts().doOperationRemotely(); + } + + private void printEntityVerification(String className, int datastoreCount, long psqlCount) { + System.out.println("========================================"); + System.out.println(className); + System.out.println("Objectify count: " + datastoreCount); + System.out.println("Postgres count: " + psqlCount); + System.out.println("Correct number of rows?: " + (datastoreCount == psqlCount)); + } + + private Long countPostgresEntities(Class entity) { + HibernateUtil.beginTransaction(); + CriteriaBuilder cb = HibernateUtil.getCriteriaBuilder(); + CriteriaQuery cr = cb.createQuery(Long.class); + Root root = cr.from(entity); + + cr.select(cb.count(root)); + + Long count = HibernateUtil.createQuery(cr).getSingleResult(); + HibernateUtil.commitTransaction(); + return count; + } + + @Override + protected void doOperation() { + Map, Class> entities = + new HashMap, Class>(); + + entities.put(teammates.storage.entity.Course.class, teammates.storage.sqlentity.Course.class); + entities.put(teammates.storage.entity.FeedbackSession.class, teammates.storage.sqlentity.FeedbackSession.class); + + // Compare datastore "table" to postgres table for each entity + for (Map.Entry, Class> entry : entities + .entrySet()) { + Class objectifyClass = entry.getKey(); + Class sqlClass = entry.getValue(); + + int objectifyEntityCount = ofy().load().type(objectifyClass).count(); + Long postgresEntityCount = countPostgresEntities(sqlClass); + + printEntityVerification(objectifyClass.getSimpleName(), objectifyEntityCount, postgresEntityCount); + } + } +} From 82d06053a170518e1278d1f811db633654f9386e Mon Sep 17 00:00:00 2001 From: nicolascwy <25302138+NicolasCwy@users.noreply.github.com> Date: Sun, 7 Apr 2024 11:30:07 +0800 Subject: [PATCH 6/6] Amend getReference javadoc --- src/main/java/teammates/common/util/HibernateUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/teammates/common/util/HibernateUtil.java b/src/main/java/teammates/common/util/HibernateUtil.java index 98ce2ef2607..e35238018e5 100644 --- a/src/main/java/teammates/common/util/HibernateUtil.java +++ b/src/main/java/teammates/common/util/HibernateUtil.java @@ -240,9 +240,10 @@ public static T get(Class entityType, Object id) { /** * Return an instance, whose state may be lazily fetched. + * Id is the only field that is fetched eagerly. * If there is no such persistent instance, EntityNotFoundException is thrown when the instance state * is first accessed. This method can be used to increase performance, if it is known that the instance exists - * and fetching of an entity is not necessary. + * and fetching of other attributes of the entity is not necessary. */ public static T getReference(Class entityType, Object id) { return HibernateUtil.getCurrentSession().getReference(entityType, id);