Skip to content

Commit

Permalink
[#12048] Migrating course entity (#12980)
Browse files Browse the repository at this point in the history
* Initial commit

* Add verification script

* Finalise changes before test

* Update logs

* Add ;

* Finish course migration

* Verified Results

* Fix lint

* Fix lint

* Change isMigrationNeeded to always return true
  • Loading branch information
mingyuanc authored Apr 4, 2024
1 parent 57bc285 commit 44f3e0b
Show file tree
Hide file tree
Showing 7 changed files with 159 additions and 46 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -141,16 +141,15 @@ protected void migrateEntity(teammates.storage.entity.Account oldAccount) {

TypedQuery<teammates.storage.sqlentity.Account> query = HibernateUtil.createQuery(cr);


boolean isEntityInDb = query.getResultList().size() != 0;
HibernateUtil.commitTransaction();

// In db, but somehow not set as migrated.
if (isEntityInDb) {
oldAccount.setMigrated(true);
entitiesOldAccountSavingBuffer.add(oldAccount);
entitiesOldAccountSavingBuffer.add(oldAccount);
return;
};
}

teammates.storage.sqlentity.Account newAccount = new teammates.storage.sqlentity.Account(
oldAccount.getGoogleId(),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package teammates.client.scripts.sql;

import com.googlecode.objectify.cmd.Query;

import teammates.storage.entity.Course;

/**
* Data migration class for course entity.
*/
@SuppressWarnings("PMD")
public class DataMigrationForCourseSql extends
DataMigrationEntitiesBaseScriptSql<teammates.storage.entity.Course, teammates.storage.sqlentity.Course> {

public static void main(String[] args) {
new DataMigrationForCourseSql().doOperationRemotely();
}

@Override
protected Query<Course> getFilterQuery() {
return ofy().load().type(teammates.storage.entity.Course.class);
}

@Override
protected boolean isPreview() {
return false;
}

/*
* Sets the migration criteria used in isMigrationNeeded.
*/
@Override
protected void setMigrationCriteria() {
// No migration criteria currently needed.
}

@Override
protected boolean isMigrationNeeded(Course entity) {
// HibernateUtil.beginTransaction();
// teammates.storage.sqlentity.Course course = HibernateUtil.get(
// teammates.storage.sqlentity.Course.class, entity.getUniqueId());
// HibernateUtil.commitTransaction();
// return course == null;
return true;
}

@Override
protected void migrateEntity(Course oldCourse) throws Exception {
teammates.storage.sqlentity.Course newCourse = new teammates.storage.sqlentity.Course(
oldCourse.getUniqueId(),
oldCourse.getName(),
oldCourse.getTimeZone(),
oldCourse.getInstitute());
// newCourse.setCreatedAt(oldCourse.getCreatedAt());
newCourse.setDeletedAt(oldCourse.getDeletedAt());
saveEntityDeferred(newCourse);
}
}
Original file line number Diff line number Diff line change
@@ -1,18 +1,13 @@
package teammates.client.scripts.sql;

import java.io.IOException;
// CHECKSTYLE.OFF:ImportOrder
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;

import com.google.cloud.datastore.Cursor;
import com.google.cloud.datastore.QueryResults;
import com.googlecode.objectify.cmd.Query;

import jakarta.persistence.TypedQuery;
Expand All @@ -21,7 +16,6 @@
import jakarta.persistence.criteria.Order;
import jakarta.persistence.criteria.Root;
import teammates.storage.sqlentity.Account;
import teammates.test.FileHelper;
import teammates.client.connector.DatastoreClient;
import teammates.client.util.ClientProperties;
import teammates.common.util.HibernateUtil;
Expand All @@ -32,35 +26,33 @@
*/
@SuppressWarnings("PMD")
public class MarkIsMigratedForAccounts extends DatastoreClient {
/**
* Batch size to fetch per page.
*/
protected static final int CONST_SQL_FETCH_BASE_SIZE = 1000;
private static final String BASE_LOG_URI = "src/client/java/teammates/client/scripts/log/";

/* NOTE
* Before running the verification, please enable hibernate.jdbc.fetch_size in HibernateUtil.java
* for optimized batch-fetching.
*/

/**
* Batch size to fetch per page.
*/
protected static final int CONST_SQL_FETCH_BASE_SIZE = 1000;

AtomicLong numberOfScannedKey;
AtomicLong numberOfAffectedEntities;
AtomicLong numberOfUpdatedEntities;

/** Datastore entity class. */
protected Class<teammates.storage.entity.Account> datastoreEntityClass = teammates.storage.entity.Account.class;

/** SQL entity class. */
protected Class<Account> sqlEntityClass = Account.class;

// AtomicLong numberOfScannedKey;
// AtomicLong numberOfAffectedEntities;
// AtomicLong numberOfUpdatedEntities;

private long entitiesVerified = 0;
private long entitiesSetToIsMigrated = 0;

public MarkIsMigratedForAccounts() {
numberOfScannedKey = new AtomicLong();
numberOfAffectedEntities = new AtomicLong();
numberOfUpdatedEntities = new AtomicLong();
// numberOfScannedKey = new AtomicLong();
// numberOfAffectedEntities = new AtomicLong();
// numberOfUpdatedEntities = new AtomicLong();

String connectionUrl = ClientProperties.SCRIPT_API_URL;
String username = ClientProperties.SCRIPT_API_NAME;
Expand Down Expand Up @@ -184,21 +176,22 @@ protected List<Map.Entry<Account, Account>> checkAllEntitiesForFailures() {
failures.add(new AbstractMap.SimpleEntry<Account, Account>(sqlEntity, null));
continue;
}

if (!datastoreEntity.isMigrated()) {
datastoreEntity.setMigrated(true);
setMigratedAccountBuffer.add(datastoreEntity);
}
}

/* Flushing the buffer */
if (setMigratedAccountBuffer.size() != 0 ) {
if (setMigratedAccountBuffer.size() != 0) {
long startTimeForDatastoreFlushing = System.currentTimeMillis();
entitiesSetToIsMigrated += setMigratedAccountBuffer.size();
ofy().save().entities(setMigratedAccountBuffer).now();
setMigratedAccountBuffer.clear();
long endTimeForDatastoreFlushing = System.currentTimeMillis();
log("Flushing for datastore " + (endTimeForDatastoreFlushing - startTimeForDatastoreFlushing) + " milliseconds");
log("Flushing for datastore " + (endTimeForDatastoreFlushing - startTimeForDatastoreFlushing)
+ " milliseconds");
}
}

Expand Down Expand Up @@ -226,7 +219,7 @@ protected void runCheckAllEntities(Class<Account> sqlEntityClass,
}

long checkEndTime = System.currentTimeMillis();

log("Entity took " + (checkEndTime - checkStartTime) + " milliseconds to verify");
log("Verified " + entitiesVerified + " SQL entities successfully");
log("Number of datastore accounts set to isMigrated " + entitiesSetToIsMigrated);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
// CHECKSTYLE.ON:ImportOrder
/**
* Patch Migration Script for Usage Statistics
*
* Batch select datastore usage statistics, and then batch select SQL entities
* with the same start timestamps.
* Compare the size of the two list, if is not equal, find the missing one in
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
// CHECKSTYLE.OFF:ImportOrder
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.util.AbstractMap;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
Expand All @@ -13,20 +12,15 @@
import jakarta.persistence.TypedQuery;
import jakarta.persistence.criteria.CriteriaBuilder;
import jakarta.persistence.criteria.CriteriaQuery;
import jakarta.persistence.criteria.Order;
import jakarta.persistence.criteria.Root;

import teammates.client.connector.DatastoreClient;
import teammates.client.util.ClientProperties;
import teammates.common.util.HibernateUtil;
import teammates.storage.sqlentity.AccountRequest;

// CHECKSTYLE.ON:ImportOrder
/**
* Protected methods may be overriden.
*
* @param <E> Datastore entity
* @param <T> SQL entity
*/
@SuppressWarnings("PMD")
public class ReverseDataMigrationForAccount
Expand Down Expand Up @@ -88,7 +82,7 @@ protected List<teammates.storage.sqlentity.Account> getNewAccounts() {
return query.getResultList();
}

/*
/**
* Reverse migrate accounts to datastore.
*/
protected void reverseMigrateToDatastore() {
Expand Down Expand Up @@ -127,7 +121,7 @@ private String getLogPrefix() {

/**
* Log a line.
*
*
* @param logLine the line to log
*/
protected void log(String logLine) {
Expand Down
51 changes: 40 additions & 11 deletions src/client/java/teammates/client/scripts/sql/SeedDb.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import teammates.storage.api.OfyHelper;
import teammates.storage.entity.Account;
import teammates.storage.entity.AccountRequest;
import teammates.storage.entity.Course;
import teammates.storage.entity.Notification;
import teammates.test.FileHelper;

Expand All @@ -37,8 +38,9 @@
*/
@SuppressWarnings("PMD")
public class SeedDb extends DatastoreClient {
private final LogicExtension logic = new LogicExtension();

private static final int MAX_ENTITY_SIZE = 10000;
private final LogicExtension logic = new LogicExtension();
private Closeable closeable;

/**
Expand Down Expand Up @@ -101,13 +103,42 @@ protected Instant getRandomInstant() {
* Persists additional data.
*/
protected void persistAdditionalData() {
int constEntitySize = 10000;
String[] args = {};
// Each account will have this amount of read notifications
int constReadNotificationSize = 5;
int constNotificationSize = 1000;
assert constNotificationSize >= constReadNotificationSize;
seedNotificationAccountAndAccountRequest(5, 1000);
seedCourse();

String[] args = {};
GenerateUsageStatisticsObjects.main(args);
}

private void seedCourse() {
log("Seeding courses");
for (int i = 0; i < MAX_ENTITY_SIZE; i++) {
if (i % (MAX_ENTITY_SIZE / 5) == 0) {
log(String.format("Seeded %d %% of new sets of entities",
(int) (100 * ((float) i / (float) MAX_ENTITY_SIZE))));
}

Random rand = new Random();

try {
String courseName = String.format("Course %s", i);
String courseInstitute = String.format("Institute %s", i);
String courseTimeZone = String.format("Time Zone %s", i);
Course course = new Course(UUID.randomUUID().toString(), courseName, courseTimeZone, courseInstitute,
getRandomInstant(),
rand.nextInt(3) > 1 ? null : getRandomInstant(), // set deletedAt randomly at 25% chance
false);
ofy().save().entities(course).now();
} catch (Exception e) {
log(e.toString());
}
}
}

private void seedNotificationAccountAndAccountRequest(int constReadNotificationSize, int constNotificationSize) {
assert constNotificationSize >= constReadNotificationSize;
log("Seeding Notifications, Account and Account Request");

Set<String> notificationsUuidSeen = new HashSet<String>();
ArrayList<String> notificationUuids = new ArrayList<>();
Expand Down Expand Up @@ -145,11 +176,11 @@ protected void persistAdditionalData() {
}
}

for (int i = 0; i < constEntitySize; i++) {
for (int i = 0; i < MAX_ENTITY_SIZE; i++) {

if (i % (constEntitySize / 5) == 0) {
if (i % (MAX_ENTITY_SIZE / 5) == 0) {
log(String.format("Seeded %d %% of new sets of entities",
(int) (100 * ((float) i / (float) constEntitySize))));
(int) (100 * ((float) i / (float) MAX_ENTITY_SIZE))));
}

try {
Expand Down Expand Up @@ -181,8 +212,6 @@ protected void persistAdditionalData() {
log(e.toString());
}
}

GenerateUsageStatisticsObjects.main(args);
}

private void log(String logLine) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package teammates.client.scripts.sql;

import teammates.storage.entity.Course;

/**
* Class for verifying account attributes.
*/
@SuppressWarnings("PMD")
public class VerifyCourseAttributes
extends VerifyNonCourseEntityAttributesBaseScript<Course, teammates.storage.sqlentity.Course> {

public VerifyCourseAttributes() {
super(Course.class,
teammates.storage.sqlentity.Course.class);
}

@Override
protected String generateID(teammates.storage.sqlentity.Course sqlEntity) {
return sqlEntity.getId();
}

public static void main(String[] args) {
VerifyCourseAttributes script = new VerifyCourseAttributes();
script.doOperationRemotely();
}

// Used for sql data migration
@Override
public boolean equals(teammates.storage.sqlentity.Course sqlEntity, Course datastoreEntity) {
try {
return sqlEntity.getId().equals(datastoreEntity.getUniqueId())
&& sqlEntity.getName().equals(datastoreEntity.getName())
&& sqlEntity.getTimeZone().equals(datastoreEntity.getTimeZone())
&& sqlEntity.getInstitute().equals(datastoreEntity.getInstitute())
// && sqlEntity.getCreatedAt().equals(datastoreEntity.getCreatedAt())
&& datastoreEntity.getDeletedAt() == null ? sqlEntity.getDeletedAt() == null
: sqlEntity.getDeletedAt().equals(datastoreEntity.getDeletedAt());
} catch (IllegalArgumentException iae) {
return false;
}
}
}

0 comments on commit 44f3e0b

Please sign in to comment.