Merge pull request #5499 from onflow/fxamacker/add-diff-report-to-mig…

…rations

Add migration diff util for testing and debugging migrations

cmd/util/cmd/execution-state-extract/cmd.go

@@ -34,6 +34,8 @@ var (
 	flagValidateMigration             bool
 	flagAllowPartialStateFromPayloads bool
 	flagLogVerboseValidationError     bool
+	flagDiffMigration                 bool
+	flagLogVerboseDiff                bool
 	flagStagedContractsFile           string
 	flagInputPayloadFileName          string
 	flagOutputPayloadFileName         string
@@ -81,6 +83,12 @@ func init() {
 	Cmd.Flags().BoolVar(&flagLogVerboseValidationError, "log-verbose-validation-error", false,
 		"log entire Cadence values on validation error (atree migration)")
 
+	Cmd.Flags().BoolVar(&flagDiffMigration, "diff", false,
+		"compare Cadence values and log diff (migration)")
+
+	Cmd.Flags().BoolVar(&flagLogVerboseDiff, "log-verbose-diff", false,
+		"log entire Cadence values on diff (requires --diff flag)")
+
 	Cmd.Flags().StringVar(&flagStagedContractsFile, "staged-contracts", "",
 		"Staged contracts CSV file")
 
@@ -139,6 +147,10 @@ func run(*cobra.Command, []string) {
 		log.Fatal().Msg("--extract-payloads-by-address requires --output-payload-filename to be specified")
 	}
 
+	if flagValidateMigration && flagDiffMigration {
+		log.Fatal().Msg("Both --validate and --diff are enabled, please specify only one (or none) of these")
+	}
+
 	if len(flagBlockHash) > 0 {
 		blockID, err := flow.HexStringToIdentifier(flagBlockHash)
 		if err != nil {
@@ -241,11 +253,19 @@ func run(*cobra.Command, []string) {
 	}
 
 	if flagValidateMigration {
-		log.Warn().Msgf("atree migration validation flag is enabled and will increase duration of migration")
+		log.Warn().Msgf("--validate flag is enabled and will increase duration of migration")
 	}
 
 	if flagLogVerboseValidationError {
-		log.Warn().Msgf("atree migration has verbose validation error logging enabled which may increase size of log")
+		log.Warn().Msgf("--log-verbose-validation-error flag is enabled which may increase size of log")
+	}
+
+	if flagDiffMigration {
+		log.Warn().Msgf("--diff flag is enabled and will increase duration of migration")
+	}
+
+	if flagLogVerboseDiff {
+		log.Warn().Msgf("--log-verbose-diff flag is enabled which may increase size of log")
 	}
 
 	var inputMsg string
@@ -299,6 +319,8 @@ func run(*cobra.Command, []string) {
 			flagOutputDir,
 			flagNWorker,
 			!flagNoMigration,
+			flagDiffMigration,
+			flagLogVerboseDiff,
 			chainID,
 			evmContractChange,
 			stagedContracts,
@@ -314,6 +336,8 @@ func run(*cobra.Command, []string) {
 			flagOutputDir,
 			flagNWorker,
 			!flagNoMigration,
+			flagDiffMigration,
+			flagLogVerboseDiff,
 			chainID,
 			evmContractChange,
 			stagedContracts,

cmd/util/cmd/execution-state-extract/execution_state_extract.go

@@ -37,6 +37,8 @@ func extractExecutionState(
 	outputDir string,
 	nWorker int, // number of concurrent worker to migration payloads
 	runMigrations bool,
+	diffMigrations bool,
+	logVerboseDiff bool,
 	chainID flow.ChainID,
 	evmContractChange migrators.EVMContractChange,
 	stagedContracts []migrators.StagedContract,
@@ -97,6 +99,8 @@ func extractExecutionState(
 		dir,
 		nWorker,
 		runMigrations,
+		diffMigrations,
+		logVerboseDiff,
 		chainID,
 		evmContractChange,
 		stagedContracts,
@@ -204,6 +208,8 @@ func extractExecutionStateFromPayloads(
 	outputDir string,
 	nWorker int, // number of concurrent worker to migation payloads
 	runMigrations bool,
+	diffMigrations bool,
+	logVerboseDiff bool,
 	chainID flow.ChainID,
 	evmContractChange migrators.EVMContractChange,
 	stagedContracts []migrators.StagedContract,
@@ -224,6 +230,8 @@ func extractExecutionStateFromPayloads(
 		dir,
 		nWorker,
 		runMigrations,
+		diffMigrations,
+		logVerboseDiff,
 		chainID,
 		evmContractChange,
 		stagedContracts,
@@ -372,6 +380,8 @@ func newMigrations(
 	dir string,
 	nWorker int,
 	runMigrations bool,
+	diffMigrations bool,
+	logVerboseDiff bool,
 	chainID flow.ChainID,
 	evmContractChange migrators.EVMContractChange,
 	stagedContracts []migrators.StagedContract,
@@ -387,6 +397,8 @@ func newMigrations(
 		rwf,
 		nWorker,
 		chainID,
+		diffMigrations,
+		logVerboseDiff,
 		evmContractChange,
 		stagedContracts,
 	)

cmd/util/cmd/execution-state-extract/execution_state_extract_test.go

@@ -73,6 +73,8 @@ func TestExtractExecutionState(t *testing.T) {
 				outdir,
 				10,
 				false,
+				false,
+				false,
 				flow.Emulator,
 				// TODO:
 				migrations.EVMContractChangeNone,

cmd/util/ledger/migrations/cadence.go

@@ -166,6 +166,8 @@ func NewCadence1ValueMigrations(
 	rwf reporters.ReportWriterFactory,
 	nWorker int,
 	chainID flow.ChainID,
+	diffMigrations bool,
+	logVerboseDiff bool,
 ) (migrations []NamedMigration) {
 
 	// Populated by CadenceLinkValueMigrator,
@@ -177,17 +179,23 @@ func NewCadence1ValueMigrations(
 	for _, accountBasedMigration := range []*CadenceBaseMigrator{
 		NewCadence1ValueMigrator(
 			rwf,
+			diffMigrations,
+			logVerboseDiff,
 			errorMessageHandler,
 			NewCadence1CompositeStaticTypeConverter(chainID),
 			NewCadence1InterfaceStaticTypeConverter(chainID),
 		),
 		NewCadence1LinkValueMigrator(
 			rwf,
+			diffMigrations,
+			logVerboseDiff,
 			errorMessageHandler,
 			capabilityMapping,
 		),
 		NewCadence1CapabilityValueMigrator(
 			rwf,
+			diffMigrations,
+			logVerboseDiff,
 			errorMessageHandler,
 			capabilityMapping,
 		),
@@ -261,6 +269,8 @@ func NewCadence1Migrations(
 	rwf reporters.ReportWriterFactory,
 	nWorker int,
 	chainID flow.ChainID,
+	diffMigrations bool,
+	logVerboseDiff bool,
 	evmContractChange EVMContractChange,
 	stagedContracts []StagedContract,
 ) []NamedMigration {
@@ -277,6 +287,8 @@ func NewCadence1Migrations(
 			rwf,
 			nWorker,
 			chainID,
+			diffMigrations,
+			logVerboseDiff,
 		),
 	)
 }