From 9351cac9ceea5078dda4e36b54dea179bedc0633 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 11 Aug 2021 16:26:51 +0200 Subject: [PATCH 01/31] core/types: add legacy receipt encoding, conversion test --- core/types/legacy.go | 71 +++++++++++++++++++++++++++++++++ core/types/receipt_test.go | 80 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 core/types/legacy.go diff --git a/core/types/legacy.go b/core/types/legacy.go new file mode 100644 index 000000000000..887b3caf0974 --- /dev/null +++ b/core/types/legacy.go @@ -0,0 +1,71 @@ +package types + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" +) + +// v4StoredReceiptRLPWithLogs is the storage encoding of a receipt used in database version 4. +type v4StoredReceiptRLPWithLogs struct { + PostStateOrStatus []byte + CumulativeGasUsed uint64 + TxHash common.Hash + ContractAddress common.Address + Logs []*legacyRlpStorageLog + GasUsed uint64 +} + +// v3StoredReceiptRLP is the original storage encoding of a receipt including some unnecessary fields. +type v3StoredReceiptRLPWithLogs struct { + PostStateOrStatus []byte + CumulativeGasUsed uint64 + Bloom Bloom + TxHash common.Hash + ContractAddress common.Address + Logs []*legacyRlpStorageLog + GasUsed uint64 +} + +func encodeAsV4StoredReceiptRLPWithLogs(want *Receipt) ([]byte, error) { + stored := &v4StoredReceiptRLPWithLogs{ + PostStateOrStatus: want.statusEncoding(), + CumulativeGasUsed: want.CumulativeGasUsed, + TxHash: want.TxHash, + ContractAddress: want.ContractAddress, + Logs: make([]*legacyRlpStorageLog, len(want.Logs)), + GasUsed: want.GasUsed, + } + for i, log := range want.Logs { + stored.Logs[i] = legacyFromLog(log) + } + return rlp.EncodeToBytes(stored) +} + +func encodeAsV3StoredReceiptRLPWithLogs(want *Receipt) ([]byte, error) { + stored := &v3StoredReceiptRLPWithLogs{ + PostStateOrStatus: want.statusEncoding(), + CumulativeGasUsed: want.CumulativeGasUsed, + Bloom: want.Bloom, + TxHash: want.TxHash, + ContractAddress: want.ContractAddress, + Logs: make([]*legacyRlpStorageLog, len(want.Logs)), + GasUsed: want.GasUsed, + } + for i, log := range want.Logs { + stored.Logs[i] = legacyFromLog(log) + } + return rlp.EncodeToBytes(stored) +} + +func legacyFromLog(want *Log) *legacyRlpStorageLog { + return &legacyRlpStorageLog{ + Address: want.Address, + Topics: want.Topics, + Data: want.Data, + BlockNumber: want.BlockNumber, + TxHash: want.TxHash, + TxIndex: want.TxIndex, + BlockHash: want.BlockHash, + Index: want.Index, + } +} diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 613559a6586c..2d4e9ef0154a 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -213,6 +213,86 @@ func encodeAsV3StoredReceiptRLP(want *Receipt) ([]byte, error) { return rlp.EncodeToBytes(stored) } +func TestLegacyConversion(t *testing.T) { + tests := []struct { + name string + encode func(*Receipt) ([]byte, error) + }{ + { + "V4StoredReceiptRLP", + encodeAsV4StoredReceiptRLPWithLogs, + }, + { + "V3StoredReceiptRLP", + encodeAsV3StoredReceiptRLPWithLogs, + }, + } + + tx := NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(1), 1, big.NewInt(1), nil) + txHash := tx.Hash() + receipt := &Receipt{ + Status: ReceiptStatusFailed, + CumulativeGasUsed: 1, + Logs: []*Log{ + { + Address: common.BytesToAddress([]byte{0x11}), + Topics: []common.Hash{common.HexToHash("dead"), common.HexToHash("beef")}, + Data: []byte{0x01, 0x00, 0xff}, + TxHash: txHash, + }, + { + Address: common.BytesToAddress([]byte{0x01, 0x11}), + Topics: []common.Hash{common.HexToHash("dead"), common.HexToHash("beef")}, + Data: []byte{0x01, 0x00, 0xff}, + TxHash: txHash, + }, + }, + TxHash: txHash, + ContractAddress: common.BytesToAddress([]byte{0x01, 0x11, 0x11}), + GasUsed: 111111, + } + receipt.Bloom = CreateBloom(Receipts{receipt}) + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + enc, err := tc.encode(receipt) + if err != nil { + t.Fatalf("Error encoding receipt: %v", err) + } + + var dec ReceiptForStorage + if err := rlp.DecodeBytes(enc, &dec); err != nil { + t.Fatalf("Error decoding RLP receipt: %v\n", err) + } + + // Check whether all consensus fields are correct. + if dec.Status != receipt.Status { + t.Fatalf("Receipt status mismatch, want %v, have %v", receipt.Status, dec.Status) + } + if dec.CumulativeGasUsed != receipt.CumulativeGasUsed { + t.Fatalf("Receipt CumulativeGasUsed mismatch, want %v, have %v", receipt.CumulativeGasUsed, dec.CumulativeGasUsed) + } + if dec.Bloom != receipt.Bloom { + t.Fatalf("Bloom data mismatch, want %v, have %v", receipt.Bloom, dec.Bloom) + } + if len(dec.Logs) != len(receipt.Logs) { + t.Fatalf("Receipt log number mismatch, want %v, have %v", len(receipt.Logs), len(dec.Logs)) + } + for i := 0; i < len(dec.Logs); i++ { + if dec.Logs[i].Address != receipt.Logs[i].Address { + t.Fatalf("Receipt log %d address mismatch, want %v, have %v", i, receipt.Logs[i].Address, dec.Logs[i].Address) + } + if !reflect.DeepEqual(dec.Logs[i].Topics, receipt.Logs[i].Topics) { + t.Fatalf("Receipt log %d topics mismatch, want %v, have %v", i, receipt.Logs[i].Topics, dec.Logs[i].Topics) + } + if !bytes.Equal(dec.Logs[i].Data, receipt.Logs[i].Data) { + t.Fatalf("Receipt log %d data mismatch, want %v, have %v", i, receipt.Logs[i].Data, dec.Logs[i].Data) + } + } + }) + } +} + // Tests that receipt data can be correctly derived from the contextual infos func TestDeriveFields(t *testing.T) { // Create a few transactions to have receipts for From 6bae0aa0746dafbd3a7eefe38849ab33a261501b Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 11 Aug 2021 16:51:48 +0200 Subject: [PATCH 02/31] core/types: add convertLegacyStoredReceipt method --- core/types/legacy.go | 10 ++++++++++ core/types/receipt_test.go | 17 ++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/core/types/legacy.go b/core/types/legacy.go index 887b3caf0974..7741a7498d1e 100644 --- a/core/types/legacy.go +++ b/core/types/legacy.go @@ -5,6 +5,16 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +// convertLegacyStoredReceipt takes a legacy RLP-encoded stored receipt +// and returns a fresh RLP-encoded stored receipt. +func convertLegacyStoredReceipt(raw []byte) ([]byte, error) { + var receipt ReceiptForStorage + if err := rlp.DecodeBytes(raw, &receipt); err != nil { + return nil, err + } + return rlp.EncodeToBytes(&receipt) +} + // v4StoredReceiptRLPWithLogs is the storage encoding of a receipt used in database version 4. type v4StoredReceiptRLPWithLogs struct { PostStateOrStatus []byte diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 2d4e9ef0154a..a4366b12775b 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -253,6 +253,12 @@ func TestLegacyConversion(t *testing.T) { } receipt.Bloom = CreateBloom(Receipts{receipt}) + stored := (*ReceiptForStorage)(receipt) + wantEncoding, err := rlp.EncodeToBytes(stored) + if err != nil { + t.Fatalf("Failed to encode receipt: %v", err) + } + for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { enc, err := tc.encode(receipt) @@ -262,7 +268,7 @@ func TestLegacyConversion(t *testing.T) { var dec ReceiptForStorage if err := rlp.DecodeBytes(enc, &dec); err != nil { - t.Fatalf("Error decoding RLP receipt: %v\n", err) + t.Fatalf("Error decoding RLP receipt: %v", err) } // Check whether all consensus fields are correct. @@ -289,6 +295,15 @@ func TestLegacyConversion(t *testing.T) { t.Fatalf("Receipt log %d data mismatch, want %v, have %v", i, receipt.Logs[i].Data, dec.Logs[i].Data) } } + + // Check that converted encoding matches expected fresh encoding + got, err := convertLegacyStoredReceipt(enc) + if err != nil { + t.Fatalf("Failed to convert legacy stored receipt: %v", err) + } + if !bytes.Equal(wantEncoding, got) { + t.Fatalf("Receipt conversion mismatch, want %x, have %x", wantEncoding, got) + } }) } } From 3a3ca13ad94aff196910b4081c81f39891ee4535 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 12 Aug 2021 10:01:24 +0200 Subject: [PATCH 03/31] core/types: encode list of legacy receipts --- core/types/legacy.go | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/core/types/legacy.go b/core/types/legacy.go index 7741a7498d1e..baadce1c3bf4 100644 --- a/core/types/legacy.go +++ b/core/types/legacy.go @@ -36,6 +36,14 @@ type v3StoredReceiptRLPWithLogs struct { GasUsed uint64 } +func EncodeAsLegacyStoredReceiptsRLP(receipts []*Receipt) ([]byte, error) { + stored := make([]v3StoredReceiptRLPWithLogs, len(receipts)) + for i, r := range receipts { + stored[i] = *toV3StoredReceiptRLPWithLogs(r) + } + return rlp.EncodeToBytes(stored) +} + func encodeAsV4StoredReceiptRLPWithLogs(want *Receipt) ([]byte, error) { stored := &v4StoredReceiptRLPWithLogs{ PostStateOrStatus: want.statusEncoding(), @@ -51,16 +59,20 @@ func encodeAsV4StoredReceiptRLPWithLogs(want *Receipt) ([]byte, error) { return rlp.EncodeToBytes(stored) } -func encodeAsV3StoredReceiptRLPWithLogs(want *Receipt) ([]byte, error) { - stored := &v3StoredReceiptRLPWithLogs{ - PostStateOrStatus: want.statusEncoding(), - CumulativeGasUsed: want.CumulativeGasUsed, - Bloom: want.Bloom, - TxHash: want.TxHash, - ContractAddress: want.ContractAddress, - Logs: make([]*legacyRlpStorageLog, len(want.Logs)), - GasUsed: want.GasUsed, +func toV3StoredReceiptRLPWithLogs(from *Receipt) *v3StoredReceiptRLPWithLogs { + return &v3StoredReceiptRLPWithLogs{ + PostStateOrStatus: from.statusEncoding(), + CumulativeGasUsed: from.CumulativeGasUsed, + Bloom: from.Bloom, + TxHash: from.TxHash, + ContractAddress: from.ContractAddress, + Logs: make([]*legacyRlpStorageLog, len(from.Logs)), + GasUsed: from.GasUsed, } +} + +func encodeAsV3StoredReceiptRLPWithLogs(want *Receipt) ([]byte, error) { + stored := toV3StoredReceiptRLPWithLogs(want) for i, log := range want.Logs { stored.Logs[i] = legacyFromLog(log) } From b25078392c00618edf9ca5eea70a24e7f362329c Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 12 Aug 2021 19:31:47 +0200 Subject: [PATCH 04/31] core/types: add method checking if receipt is legacy --- core/types/legacy.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/core/types/legacy.go b/core/types/legacy.go index baadce1c3bf4..40fb55f067b8 100644 --- a/core/types/legacy.go +++ b/core/types/legacy.go @@ -1,10 +1,32 @@ package types import ( + "errors" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" ) +// IsLegacyStoredReceipts tries to parse the RLP-encoded blob +// first as an array of v3 stored receipt, then v4 stored receipt and +// returns true if successful. +func IsLegacyStoredReceipts(raw []byte) (bool, error) { + var v3 []v3StoredReceiptRLP + if err := rlp.DecodeBytes(raw, &v3); err == nil { + return true, nil + } + var v4 []v4StoredReceiptRLP + if err := rlp.DecodeBytes(raw, &v4); err == nil { + return true, nil + } + var v5 []storedReceiptRLP + // Check to see valid fresh stored receipt + if err := rlp.DecodeBytes(raw, &v5); err == nil { + return false, nil + } + return false, errors.New("Value is not a valid receipt encoding") +} + // convertLegacyStoredReceipt takes a legacy RLP-encoded stored receipt // and returns a fresh RLP-encoded stored receipt. func convertLegacyStoredReceipt(raw []byte) ([]byte, error) { From 45b73a3222fc9f0bd52d489a136aa1e033b95500 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 12 Aug 2021 19:32:38 +0200 Subject: [PATCH 05/31] cmd/geth: add boilerplate for freezer-migrate cmd --- cmd/geth/dbcmd.go | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index e1e0d77f0b2e..f6edb4b80ad0 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/console/prompt" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" @@ -69,6 +70,7 @@ Remove blockchain and state databases`, dbDumpFreezerIndex, dbImportCmd, dbExportCmd, + dbMigrateFreezer, }, } dbInspectCmd = cli.Command{ @@ -232,6 +234,21 @@ WARNING: This is a low-level operation which may cause database corruption!`, }, Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.", } + dbMigrateFreezer = cli.Command{ + Action: utils.MigrateFlags(freezerMigrate), + Name: "freezer-migrate", + Usage: "Migrate legacy parts of the freezer. (WARNING: may take a long time)", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.SyncModeFlag, + utils.MainnetFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + }, + Description: "The import command imports the specific chain data from an RLP encoded stream.", + } ) func removeDB(ctx *cli.Context) error { @@ -684,3 +701,48 @@ func exportChaindata(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true) return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop) } + +func freezerMigrate(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + db := utils.MakeChainDatabase(ctx, stack, false) + defer db.Close() + + /*path := filepath.Join(stack.ResolvePath("chaindata"), "ancient") + log.Info("Opening freezer", "location", path) + table, err := rawdb.NewFreezerTable(path, "receipts", rawdb.FreezerNoSnappy["receipts"]) + if err != nil { + log.Info("Could not open freezer table", "error", err) + return err + }*/ + /*if err := table.Close(); err != nil { + return err + }*/ + + // Check first block for legacy receipt format + numAncients, err := db.Ancients() + if err != nil { + return err + } + if numAncients < 1 { + log.Info("No blocks in freezer to migrate") + return nil + } + // TODO: check genesis or block 1? + // devnets might have an empty genesis + first, err := db.Ancient("receipts", 1) + if err != nil { + return err + } + isFirstLegacy, err := types.IsLegacyStoredReceipt(first) + if err != nil { + return err + } + if !isFirstLegacy { + log.Info("No legacy receipts to migrate") + return nil + } + log.Info("Starting migration") + return nil +} From 7474ef01329edc61aade66a4df6bdcf3210bbe60 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 17 Aug 2021 16:09:18 +0200 Subject: [PATCH 06/31] core/types: add method for converting list of legacy receipts --- core/types/legacy.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/types/legacy.go b/core/types/legacy.go index 40fb55f067b8..8476108de77c 100644 --- a/core/types/legacy.go +++ b/core/types/legacy.go @@ -27,6 +27,16 @@ func IsLegacyStoredReceipts(raw []byte) (bool, error) { return false, errors.New("Value is not a valid receipt encoding") } +// ConvertLegacyStoredReceipts takes the RLP encoding of an array of legacy +// stored receipts and returns a fresh RLP-encoded stored receipt. +func ConvertLegacyStoredReceipts(raw []byte) ([]byte, error) { + var receipts []ReceiptForStorage + if err := rlp.DecodeBytes(raw, &receipts); err != nil { + return nil, err + } + return rlp.EncodeToBytes(&receipts) +} + // convertLegacyStoredReceipt takes a legacy RLP-encoded stored receipt // and returns a fresh RLP-encoded stored receipt. func convertLegacyStoredReceipt(raw []byte) ([]byte, error) { From 39e5aeead7ff6988b58b9ff168a5aae83066420a Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 17 Aug 2021 16:10:05 +0200 Subject: [PATCH 07/31] cmd/geth: write converted receipts to tmp freezer file --- cmd/geth/dbcmd.go | 61 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index f6edb4b80ad0..114b2a88c398 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -709,17 +709,6 @@ func freezerMigrate(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, false) defer db.Close() - /*path := filepath.Join(stack.ResolvePath("chaindata"), "ancient") - log.Info("Opening freezer", "location", path) - table, err := rawdb.NewFreezerTable(path, "receipts", rawdb.FreezerNoSnappy["receipts"]) - if err != nil { - log.Info("Could not open freezer table", "error", err) - return err - }*/ - /*if err := table.Close(); err != nil { - return err - }*/ - // Check first block for legacy receipt format numAncients, err := db.Ancients() if err != nil { @@ -735,7 +724,7 @@ func freezerMigrate(ctx *cli.Context) error { if err != nil { return err } - isFirstLegacy, err := types.IsLegacyStoredReceipt(first) + isFirstLegacy, err := types.IsLegacyStoredReceipts(first) if err != nil { return err } @@ -743,6 +732,52 @@ func freezerMigrate(ctx *cli.Context) error { log.Info("No legacy receipts to migrate") return nil } - log.Info("Starting migration") + + log.Info("Starting migration", "ancients", numAncients) + start := time.Now() + // Open new freezer table for the updated receipts + path := filepath.Join(stack.ResolvePath("chaindata"), "ancient") + table, err := rawdb.NewFreezerTable(path, "ureceipts", rawdb.FreezerNoSnappy["receipts"]) + if err != nil { + log.Info("Could not open freezer table for updated receipts", "error", err) + return err + } + + // Write new freezer table files until we reach a file + // that starts with modern receipts. From that point on + // we stop and re-use the existing files. Note the last + // table file might be significantly smaller than the max size. + for i := uint64(0); i < numAncients; i++ { + blob, err := db.Ancient("receipts", i) + if err != nil { + return err + } + // Stop when first v5 receipt is spotted. + // TODO: Combine detecting legacy and converting it + // to avoid 2 decoding. + legacy, err := types.IsLegacyStoredReceipts(blob) + if err != nil { + return err + } + if !legacy { + log.Info("Spotted non-legacy receipt", "index", i) + break + } + + converted, err := types.ConvertLegacyStoredReceipts(blob) + if err != nil { + return err + } + if err := table.Append(i, converted); err != nil { + return err + } + } + + if err := table.Close(); err != nil { + return err + } + + log.Info("Migration finished", "duration", time.Since(start)) + return nil } From b1c67fd716cfaa3b0ea799ac2fa67ba13f6078ec Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 19 Aug 2021 16:19:58 +0200 Subject: [PATCH 08/31] cmd,core,ethdb: drop receipts table after copying over new receipts --- cmd/geth/dbcmd.go | 30 +++++++--- core/rawdb/database.go | 4 ++ core/rawdb/freezer.go | 15 +++++ core/rawdb/freezer_table.go | 110 ++++++++++++++++++++++++++++++++++++ core/rawdb/table.go | 4 ++ ethdb/database.go | 3 + 6 files changed, 158 insertions(+), 8 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 114b2a88c398..e03e25ab99e5 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -759,24 +759,38 @@ func freezerMigrate(ctx *cli.Context) error { if err != nil { return err } - if !legacy { - log.Info("Spotted non-legacy receipt", "index", i) - break - } - converted, err := types.ConvertLegacyStoredReceipts(blob) - if err != nil { - return err + var out []byte + if legacy { + out, err = types.ConvertLegacyStoredReceipts(blob) + if err != nil { + return err + } + } else { + out = blob } - if err := table.Append(i, converted); err != nil { + + if err := table.Append(i, out); err != nil { return err } } + log.Info("before dropping table") + // Replace old receipt files by new ones + if err := db.DropTable("receipts"); err != nil { + log.Error("Failed to drop receipts table", "error", err) + return err + } + + log.Info("Before closing table") if err := table.Close(); err != nil { return err } + log.Info("Before closing db") + if err := db.Close(); err != nil { + return err + } log.Info("Migration finished", "duration", time.Since(start)) return nil diff --git a/core/rawdb/database.go b/core/rawdb/database.go index c5af776672cf..b7288add581e 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -135,6 +135,10 @@ func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReader) error) (e return fn(db) } +func (db *nofreezedb) DropTable(_ string) error { + return errNotSupported +} + // NewDatabase creates a high level database on top of a given key-value data // store without a freezer moving immutable chain segments into cold storage. func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index e19c202adc84..5147cfb17b18 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -546,3 +546,18 @@ func (f *freezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes [] return hashes, err } + +// dropTable removes all files belonging to a table. Caution: this puts the freezer +// in an unstable position. +// TODO: should be private +func (f *freezer) DropTable(kind string) error { + table, ok := f.tables[kind] + if !ok { + return errUnknownTable + } + if err := table.drop(); err != nil { + return err + } + delete(f.tables, kind) + return nil +} diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 22405cf9b4f8..53a83637baa9 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -504,6 +504,116 @@ func (t *freezerTable) getIndices(from, count uint64) ([]*indexEntry, error) { return indices, nil } +// drop deletes the index file and all the open files. +// TODO(sina): make sure to delete non-open files +func (t *freezerTable) drop() error { + t.lock.Lock() + defer t.lock.Unlock() + + if err := t.index.Close(); err != nil { + log.Warn("failed to close index", "error", err, "index", t.index) + return err + } + log.Info("before os removing index", "name", t.index.Name()) + if err := os.Remove(t.index.Name()); err != nil { + return err + } + t.index = nil + + t.releaseFilesAfter(0, true) + t.head = nil + + return nil +} + +// Append injects a binary blob at the end of the freezer table. The item number +// is a precautionary parameter to ensure data correctness, but the table will +// reject already existing data. +// +// Note, this method will *not* flush any data to disk so be sure to explicitly +// fsync before irreversibly deleting data from the database. +func (t *freezerTable) Append(item uint64, blob []byte) error { + // Encode the blob before the lock portion + if !t.noCompression { + blob = snappy.Encode(nil, blob) + } + // Read lock prevents competition with truncate + retry, err := t.append(item, blob, false) + if err != nil { + return err + } + if retry { + // Read lock was insufficient, retry with a writelock + _, err = t.append(item, blob, true) + } + return err +} + +// append injects a binary blob at the end of the freezer table. +// Normally, inserts do not require holding the write-lock, so it should be invoked with 'wlock' set to +// false. +// However, if the data will grown the current file out of bounds, then this +// method will return 'true, nil', indicating that the caller should retry, this time +// with 'wlock' set to true. +func (t *freezerTable) append(item uint64, encodedBlob []byte, wlock bool) (bool, error) { + if wlock { + t.lock.Lock() + defer t.lock.Unlock() + } else { + t.lock.RLock() + defer t.lock.RUnlock() + } + // Ensure the table is still accessible + if t.index == nil || t.head == nil { + return false, errClosed + } + // Ensure only the next item can be written, nothing else + if atomic.LoadUint64(&t.items) != item { + return false, fmt.Errorf("appending unexpected item: want %d, have %d", t.items, item) + } + bLen := uint32(len(encodedBlob)) + if t.headBytes+bLen < bLen || + t.headBytes+bLen > t.maxFileSize { + // Writing would overflow, so we need to open a new data file. + // If we don't already hold the writelock, abort and let the caller + // invoke this method a second time. + if !wlock { + return true, nil + } + nextID := atomic.LoadUint32(&t.headId) + 1 + // We open the next file in truncated mode -- if this file already + // exists, we need to start over from scratch on it + newHead, err := t.openFile(nextID, openFreezerFileTruncated) + if err != nil { + return false, err + } + // Close old file, and reopen in RDONLY mode + t.releaseFile(t.headId) + t.openFile(t.headId, openFreezerFileForReadOnly) + + // Swap out the current head + t.head = newHead + atomic.StoreUint32(&t.headBytes, 0) + atomic.StoreUint32(&t.headId, nextID) + } + if _, err := t.head.Write(encodedBlob); err != nil { + return false, err + } + newOffset := atomic.AddUint32(&t.headBytes, bLen) + idx := indexEntry{ + filenum: atomic.LoadUint32(&t.headId), + offset: newOffset, + } + // Write indexEntry + t.index.Write(idx.marshallBinary()) + + t.writeMeter.Mark(int64(bLen + indexEntrySize)) + t.sizeGauge.Inc(int64(bLen + indexEntrySize)) + + atomic.AddUint64(&t.items, 1) + return false, nil +} + // Retrieve looks up the data offset of an item with the given number and retrieves // the raw binary blob from the data file. func (t *freezerTable) Retrieve(item uint64) ([]byte, error) { diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 91fc31b660d6..5719eb0e9256 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -101,6 +101,10 @@ func (t *table) Sync() error { return t.db.Sync() } +func (t *table) DropTable(kind string) error { + return t.db.DropTable(kind) +} + // Put inserts the given value into the database at a prefixed version of the // provided key. func (t *table) Put(key []byte, value []byte) error { diff --git a/ethdb/database.go b/ethdb/database.go index 0a5729c6c1ec..578fed3b2862 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -111,6 +111,9 @@ type AncientWriter interface { // Sync flushes all in-memory ancient store data to disk. Sync() error + + // TODO: tmp, to delete table files + DropTable(string) error } // AncientWriteOp is given to the function argument of ModifyAncients. From 82f36c983f3dc2e4fa33ae95b6f0b9892690f560 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 24 Aug 2021 16:15:40 +0200 Subject: [PATCH 09/31] cmd,core,ethdb: move logic to transform table fn --- cmd/geth/dbcmd.go | 51 +++++--------------- core/rawdb/database.go | 4 ++ core/rawdb/freezer.go | 103 +++++++++++++++++++++++++++++++++++++++++ core/rawdb/table.go | 4 ++ ethdb/database.go | 2 + 5 files changed, 124 insertions(+), 40 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index e03e25ab99e5..685926b1a203 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -733,57 +733,28 @@ func freezerMigrate(ctx *cli.Context) error { return nil } - log.Info("Starting migration", "ancients", numAncients) - start := time.Now() - // Open new freezer table for the updated receipts - path := filepath.Join(stack.ResolvePath("chaindata"), "ancient") - table, err := rawdb.NewFreezerTable(path, "ureceipts", rawdb.FreezerNoSnappy["receipts"]) - if err != nil { - log.Info("Could not open freezer table for updated receipts", "error", err) - return err - } - - // Write new freezer table files until we reach a file - // that starts with modern receipts. From that point on - // we stop and re-use the existing files. Note the last - // table file might be significantly smaller than the max size. - for i := uint64(0); i < numAncients; i++ { - blob, err := db.Ancient("receipts", i) - if err != nil { - return err - } + transformer := func(blob []byte) ([]byte, bool, error) { // Stop when first v5 receipt is spotted. // TODO: Combine detecting legacy and converting it // to avoid 2 decoding. legacy, err := types.IsLegacyStoredReceipts(blob) if err != nil { - return err + return nil, false, err } - - var out []byte - if legacy { - out, err = types.ConvertLegacyStoredReceipts(blob) - if err != nil { - return err - } - } else { - out = blob + if !legacy { + return blob, true, nil } - if err := table.Append(i, out); err != nil { - return err + out, err := types.ConvertLegacyStoredReceipts(blob) + if err != nil { + return nil, false, err } + return out, false, nil } - log.Info("before dropping table") - // Replace old receipt files by new ones - if err := db.DropTable("receipts"); err != nil { - log.Error("Failed to drop receipts table", "error", err) - return err - } - - log.Info("Before closing table") - if err := table.Close(); err != nil { + log.Info("Starting migration", "ancients", numAncients) + start := time.Now() + if err := db.TransformTable("receipts", transformer); err != nil { return err } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index b7288add581e..737a64252b35 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -139,6 +139,10 @@ func (db *nofreezedb) DropTable(_ string) error { return errNotSupported } +func (db *nofreezedb) TransformTable(_ string, _ TransformerFn) error { + return errNotSupported +} + // NewDatabase creates a high level database on top of a given key-value data // store without a freezer moving immutable chain segments into cold storage. func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 5147cfb17b18..667085dcf7a2 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -19,6 +19,7 @@ package rawdb import ( "errors" "fmt" + "io/ioutil" "math" "os" "path/filepath" @@ -561,3 +562,105 @@ func (f *freezer) DropTable(kind string) error { delete(f.tables, kind) return nil } + +type TransformerFn = func([]byte) ([]byte, bool, error) + +func (f *freezer) TransformTable(kind string, fn TransformerFn) error { + // TODO: Do we need to recover anything after error? + + table, ok := f.tables[kind] + if !ok { + return errUnknownTable + } + + numAncients, err := f.Ancients() + if err != nil { + return err + } + + ancientsPath := filepath.Dir(table.index.Name()) + migrationPath := filepath.Join(ancientsPath, "migration") + newTable, err := NewFreezerTable(migrationPath, kind, FreezerNoSnappy[kind]) + if err != nil { + return err + } + + var i uint64 + var filenum uint32 + for i = 0; i < numAncients; i++ { + blob, err := table.Retrieve(i) + if err != nil { + return err + } + out, stop, err := fn(blob) + if err != nil { + return err + } + if !stop { + newTable.Append(i, out) + } else { + // what we want is the byte offsets in the file + // for leftover items in file + + // 1. loop getBounds until filenum exceeds threshold filenum + // 2. copy verbatim to new table + // 3. need to copy rest of old index and repair the filenum in the entries + _, _, filenum, err = table.getBounds(i) + if err != nil { + return err + } + break + /*_, _, filenum, err := table.getBounds(i) + if err != nil { + return err + } + fstart, fend, err := table.getFileBounds(filenum) + if err != nil { + return err + } + log.Info("Got file bounds after stop signal", "fstart", fstart, "fend", fend)*/ + } + } + log.Info("Copying over leftover receipts", "i", i) + // Copy over left-over receipts in the file with last legacy receipt + for ; i < numAncients; i++ { + _, _, fn, err := table.getBounds(i) + if err != nil { + return err + } + if fn > filenum { + break + } + blob, err := table.Retrieve(i) + if err != nil { + return err + } + newTable.Append(i, blob) + } + log.Info("Finished copying leftovers", "i", i) + + if i < numAncients { + log.Info("Need to copy over index bits") + } + + if err := newTable.Close(); err != nil { + return err + } + + // Move new table files to ancients dir + files, err := ioutil.ReadDir(migrationPath) + if err != nil { + return err + } + for _, f := range files { + if err := os.Rename(filepath.Join(migrationPath, f.Name()), filepath.Join(ancientsPath, f.Name())); err != nil { + return err + } + } + + /*if err := table.drop(); err != nil { + return err + }*/ + log.Info("Dropped old table files") + return nil +} diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 5719eb0e9256..203aeb677605 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -105,6 +105,10 @@ func (t *table) DropTable(kind string) error { return t.db.DropTable(kind) } +func (t *table) TransformTable(kind string, fn TransformerFn) error { + return t.db.TransformTable(kind, fn) +} + // Put inserts the given value into the database at a prefixed version of the // provided key. func (t *table) Put(key []byte, value []byte) error { diff --git a/ethdb/database.go b/ethdb/database.go index 578fed3b2862..9f1ac23f96f1 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -114,6 +114,8 @@ type AncientWriter interface { // TODO: tmp, to delete table files DropTable(string) error + + TransformTable(string, func([]byte) ([]byte, bool, error)) error } // AncientWriteOp is given to the function argument of ModifyAncients. From 05dade6df6c03b4aca71e5375e898f9d6db3b4a5 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 24 Aug 2021 16:20:23 +0200 Subject: [PATCH 10/31] core/rawdb: remove getFileBounds --- core/rawdb/freezer.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 667085dcf7a2..a601799221df 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -599,30 +599,18 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { if !stop { newTable.Append(i, out) } else { - // what we want is the byte offsets in the file - // for leftover items in file - - // 1. loop getBounds until filenum exceeds threshold filenum - // 2. copy verbatim to new table - // 3. need to copy rest of old index and repair the filenum in the entries _, _, filenum, err = table.getBounds(i) if err != nil { return err } break - /*_, _, filenum, err := table.getBounds(i) - if err != nil { - return err - } - fstart, fend, err := table.getFileBounds(filenum) - if err != nil { - return err - } - log.Info("Got file bounds after stop signal", "fstart", fstart, "fend", fend)*/ } } log.Info("Copying over leftover receipts", "i", i) - // Copy over left-over receipts in the file with last legacy receipt + // Copy over left-over receipts in the file with last legacy receipt: + // 1. loop getBounds until filenum exceeds threshold filenum + // 2. copy verbatim to new table + // 3. need to copy rest of old index and repair the filenum in the entries for ; i < numAncients; i++ { _, _, fn, err := table.getBounds(i) if err != nil { From dce1336711d918549fd22392fc58eca95f5ec79b Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Fri, 27 Aug 2021 18:17:20 +0200 Subject: [PATCH 11/31] core/rawdb: copy over rest of index bits --- core/rawdb/freezer.go | 31 ++++++++++++++++++++++++++----- core/rawdb/freezer_table.go | 20 ++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index a601799221df..3e5ea1235271 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -606,17 +606,21 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { break } } - log.Info("Copying over leftover receipts", "i", i) + log.Info("Copying over leftover receipts", "i", i, "filenum", filenum) // Copy over left-over receipts in the file with last legacy receipt: // 1. loop getBounds until filenum exceeds threshold filenum // 2. copy verbatim to new table - // 3. need to copy rest of old index and repair the filenum in the entries for ; i < numAncients; i++ { - _, _, fn, err := table.getBounds(i) + /*_, _, fn, err := table.getBounds(i) + if err != nil { + return err + }*/ + idx, err := table.readEntry(i) if err != nil { return err } - if fn > filenum { + if idx.filenum > filenum { + log.Info("Reached new file with updated receipts", "fn", fn, "i", i) break } blob, err := table.Retrieve(i) @@ -627,8 +631,25 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { } log.Info("Finished copying leftovers", "i", i) + // 3. need to copy rest of old index and repair the filenum in the entries if i < numAncients { - log.Info("Need to copy over index bits") + idx, err := table.readEntry(i) + if err != nil { + return err + } + + lastFilenum := atomic.LoadUint32(&newTable.headId) + diff := int32(lastFilenum) - int32(idx.filenum) + for ; i < numAncients; i++ { + idx, err := table.readEntry(i) + if err != nil { + return err + } + // (idx.filenum + diff) is always > 0 + idx.filenum = uint32(int32(idx.filenum) + diff) + newTable.writeEntry(idx) + } + log.Info("Duplicated rest of index in new table", "i", i) } if err := newTable.Close(); err != nil { diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 53a83637baa9..3cc0742d8c37 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -850,3 +850,23 @@ func (t *freezerTable) dumpIndex(w io.Writer, start, stop int64) { } fmt.Fprintf(w, "|--------------------------|\n") } + +func (t *freezerTable) readEntry(item uint64) (indexEntry, error) { + buffer := make([]byte, indexEntrySize) + var idx indexEntry + if _, err := t.index.ReadAt(buffer, int64(item*indexEntrySize)); err != nil { + return idx, err + } + idx.unmarshalBinary(buffer) + return idx, nil +} + +// low-level, doesnt increase counters, assumes lock +func (t *freezerTable) writeEntry(idx indexEntry) error { + // Ensure the table is still accessible + if t.index == nil { + return errClosed + } + _, err := t.index.Write(idx.marshallBinary()) + return err +} From 16c55dc9a9bfd8a0f8e0395c4fc0da0606fc93c8 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 9 Nov 2021 18:02:08 +0100 Subject: [PATCH 12/31] core/rawdb: drop table append --- core/rawdb/freezer_table.go | 88 ------------------------------------- 1 file changed, 88 deletions(-) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 3cc0742d8c37..745d83516522 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -526,94 +526,6 @@ func (t *freezerTable) drop() error { return nil } -// Append injects a binary blob at the end of the freezer table. The item number -// is a precautionary parameter to ensure data correctness, but the table will -// reject already existing data. -// -// Note, this method will *not* flush any data to disk so be sure to explicitly -// fsync before irreversibly deleting data from the database. -func (t *freezerTable) Append(item uint64, blob []byte) error { - // Encode the blob before the lock portion - if !t.noCompression { - blob = snappy.Encode(nil, blob) - } - // Read lock prevents competition with truncate - retry, err := t.append(item, blob, false) - if err != nil { - return err - } - if retry { - // Read lock was insufficient, retry with a writelock - _, err = t.append(item, blob, true) - } - return err -} - -// append injects a binary blob at the end of the freezer table. -// Normally, inserts do not require holding the write-lock, so it should be invoked with 'wlock' set to -// false. -// However, if the data will grown the current file out of bounds, then this -// method will return 'true, nil', indicating that the caller should retry, this time -// with 'wlock' set to true. -func (t *freezerTable) append(item uint64, encodedBlob []byte, wlock bool) (bool, error) { - if wlock { - t.lock.Lock() - defer t.lock.Unlock() - } else { - t.lock.RLock() - defer t.lock.RUnlock() - } - // Ensure the table is still accessible - if t.index == nil || t.head == nil { - return false, errClosed - } - // Ensure only the next item can be written, nothing else - if atomic.LoadUint64(&t.items) != item { - return false, fmt.Errorf("appending unexpected item: want %d, have %d", t.items, item) - } - bLen := uint32(len(encodedBlob)) - if t.headBytes+bLen < bLen || - t.headBytes+bLen > t.maxFileSize { - // Writing would overflow, so we need to open a new data file. - // If we don't already hold the writelock, abort and let the caller - // invoke this method a second time. - if !wlock { - return true, nil - } - nextID := atomic.LoadUint32(&t.headId) + 1 - // We open the next file in truncated mode -- if this file already - // exists, we need to start over from scratch on it - newHead, err := t.openFile(nextID, openFreezerFileTruncated) - if err != nil { - return false, err - } - // Close old file, and reopen in RDONLY mode - t.releaseFile(t.headId) - t.openFile(t.headId, openFreezerFileForReadOnly) - - // Swap out the current head - t.head = newHead - atomic.StoreUint32(&t.headBytes, 0) - atomic.StoreUint32(&t.headId, nextID) - } - if _, err := t.head.Write(encodedBlob); err != nil { - return false, err - } - newOffset := atomic.AddUint32(&t.headBytes, bLen) - idx := indexEntry{ - filenum: atomic.LoadUint32(&t.headId), - offset: newOffset, - } - // Write indexEntry - t.index.Write(idx.marshallBinary()) - - t.writeMeter.Mark(int64(bLen + indexEntrySize)) - t.sizeGauge.Inc(int64(bLen + indexEntrySize)) - - atomic.AddUint64(&t.items, 1) - return false, nil -} - // Retrieve looks up the data offset of an item with the given number and retrieves // the raw binary blob from the data file. func (t *freezerTable) Retrieve(item uint64) ([]byte, error) { From 710b7d9d096998a57cf6d57397c357c57195a1a8 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 10 Nov 2021 13:26:44 +0100 Subject: [PATCH 13/31] core/rawdb: use new freezer funcs --- core/rawdb/freezer.go | 19 ++++++++++++++++--- core/rawdb/freezer_table.go | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 3e5ea1235271..f5f4292077b5 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -568,6 +568,12 @@ type TransformerFn = func([]byte) ([]byte, bool, error) func (f *freezer) TransformTable(kind string, fn TransformerFn) error { // TODO: Do we need to recover anything after error? + if f.readonly { + return errReadOnly + } + f.writeLock.Lock() + defer f.writeLock.Unlock() + table, ok := f.tables[kind] if !ok { return errUnknownTable @@ -579,11 +585,14 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { } ancientsPath := filepath.Dir(table.index.Name()) + // Set up new dir for the migrated table which we'll at the end + // move over to the ancients dir. migrationPath := filepath.Join(ancientsPath, "migration") newTable, err := NewFreezerTable(migrationPath, kind, FreezerNoSnappy[kind]) if err != nil { return err } + batch := newTable.newBatch() var i uint64 var filenum uint32 @@ -597,12 +606,16 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { return err } if !stop { - newTable.Append(i, out) + // Entry is legacy, push transformed version + batch.AppendRaw(i, out) } else { - _, _, filenum, err = table.getBounds(i) + // Reached the first up-to-date entry. + // Remember in which file the switch happens. + entry, err := table.getIndices(i, 0) if err != nil { return err } + filenum = entry[0].filenum break } } @@ -627,7 +640,7 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { if err != nil { return err } - newTable.Append(i, blob) + batch.AppendRaw(i, blob) } log.Info("Finished copying leftovers", "i", i) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 745d83516522..0cd769397ce7 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -779,6 +779,6 @@ func (t *freezerTable) writeEntry(idx indexEntry) error { if t.index == nil { return errClosed } - _, err := t.index.Write(idx.marshallBinary()) + _, err := t.index.Write(idx.append([]byte{})) return err } From 16286fa53f12ddb4b5c55f8d7ddee1e485f00b17 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 10 Nov 2021 14:24:31 +0100 Subject: [PATCH 14/31] core/rawdb: forgot batch commit --- core/rawdb/freezer.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index f5f4292077b5..56c3763bd701 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -65,6 +65,7 @@ const ( // freezerTableSize defines the maximum size of freezer data files. freezerTableSize = 2 * 1000 * 1000 * 1000 + //freezerTableSize = 2 * 1024 * 1024 ) // freezer is an memory mapped append-only database to store immutable chain data @@ -633,7 +634,7 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { return err } if idx.filenum > filenum { - log.Info("Reached new file with updated receipts", "fn", fn, "i", i) + log.Info("Reached new file with updated receipts", "fn", filenum, "i", i) break } blob, err := table.Retrieve(i) @@ -665,6 +666,10 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { log.Info("Duplicated rest of index in new table", "i", i) } + if err := batch.commit(); err != nil { + return err + } + if err := newTable.Close(); err != nil { return err } From 05812de0b82df5127cf393ea1920d51ed41e5e5a Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 15 Nov 2021 13:46:43 +0100 Subject: [PATCH 15/31] core/rawdb: comment, minor fixes --- core/rawdb/freezer.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 56c3763bd701..ff1cdcd2e036 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -596,7 +596,10 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { batch := newTable.newBatch() var i uint64 + // Number of the file in which the first up-to-date receipt appers var filenum uint32 + // Iterate through entries and transform them + // until reaching first non-legacy one. for i = 0; i < numAncients; i++ { blob, err := table.Retrieve(i) if err != nil { @@ -612,6 +615,8 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { } else { // Reached the first up-to-date entry. // Remember in which file the switch happens. + // TODO: what if it coincidentally starts in a new file? + // we won't need to copy-over that file entry, err := table.getIndices(i, 0) if err != nil { return err @@ -645,6 +650,11 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { } log.Info("Finished copying leftovers", "i", i) + if err := batch.commit(); err != nil { + return err + } + log.Info("Committed write batch", "newHeadId", newTable.headId, "headbytes", newTable.headBytes, "items", newTable.items) + // 3. need to copy rest of old index and repair the filenum in the entries if i < numAncients { idx, err := table.readEntry(i) @@ -652,8 +662,10 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { return err } - lastFilenum := atomic.LoadUint32(&newTable.headId) - diff := int32(lastFilenum) - int32(idx.filenum) + lastFilenum := newTable.headId + // idx.filenum is always >=1 + diff := int32(lastFilenum) - int32(idx.filenum-1) + log.Info("Starting duplication", "i", i, "oldFn", idx.filenum, "offset", idx.offset, "newHeadId", lastFilenum) for ; i < numAncients; i++ { idx, err := table.readEntry(i) if err != nil { @@ -666,14 +678,12 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { log.Info("Duplicated rest of index in new table", "i", i) } - if err := batch.commit(); err != nil { - return err - } - if err := newTable.Close(); err != nil { return err } + // Delete old table index & files up to cross-over point + // Move new table files to ancients dir files, err := ioutil.ReadDir(migrationPath) if err != nil { From 99c1a3282ddab670f37d26293f937745e7054df4 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 16 Nov 2021 16:41:43 +0100 Subject: [PATCH 16/31] core/rawdb: handle clean switch-over boundary --- core/rawdb/freezer.go | 57 ++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index ff1cdcd2e036..cfdb3c05b98c 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -564,6 +564,9 @@ func (f *freezer) DropTable(kind string) error { return nil } +// TransformerFn takes a freezer entry in an older format and returns +// the same in a new format. The second return argument determines +// if the entry is already of the new format. type TransformerFn = func([]byte) ([]byte, bool, error) func (f *freezer) TransformTable(kind string, fn TransformerFn) error { @@ -598,6 +601,7 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { var i uint64 // Number of the file in which the first up-to-date receipt appers var filenum uint32 + copyOver := true // Iterate through entries and transform them // until reaching first non-legacy one. for i = 0; i < numAncients; i++ { @@ -621,34 +625,42 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { if err != nil { return err } - filenum = entry[0].filenum + // First non-legacy entry coincidentally is located in a new + // file and we have a clean switch-over boundary. No need to + // copy over further elements. + if entry[0].filenum != entry[1].filenum { + copyOver = false + } + filenum = entry[1].filenum break } } log.Info("Copying over leftover receipts", "i", i, "filenum", filenum) - // Copy over left-over receipts in the file with last legacy receipt: - // 1. loop getBounds until filenum exceeds threshold filenum - // 2. copy verbatim to new table - for ; i < numAncients; i++ { - /*_, _, fn, err := table.getBounds(i) - if err != nil { - return err - }*/ - idx, err := table.readEntry(i) - if err != nil { - return err - } - if idx.filenum > filenum { - log.Info("Reached new file with updated receipts", "fn", filenum, "i", i) - break - } - blob, err := table.Retrieve(i) - if err != nil { - return err + if copyOver { + // Copy over left-over receipts in the file with last legacy receipt: + // 1. loop getBounds until filenum exceeds threshold filenum + // 2. copy verbatim to new table + for ; i < numAncients; i++ { + /*_, _, fn, err := table.getBounds(i) + if err != nil { + return err + }*/ + idx, err := table.readEntry(i) + if err != nil { + return err + } + if idx.filenum > filenum { + log.Info("Reached new file with updated receipts", "fn", filenum, "i", i) + break + } + blob, err := table.Retrieve(i) + if err != nil { + return err + } + batch.AppendRaw(i, blob) } - batch.AppendRaw(i, blob) + log.Info("Finished copying leftovers", "i", i) } - log.Info("Finished copying leftovers", "i", i) if err := batch.commit(); err != nil { return err @@ -671,6 +683,7 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { if err != nil { return err } + //log.Info("read entry", "i", i, "fn", idx.filenum, "offset", idx.offset) // (idx.filenum + diff) is always > 0 idx.filenum = uint32(int32(idx.filenum) + diff) newTable.writeEntry(idx) From 66b5285b0ef1045ca28ff2e0b4abf6617703e911 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 16 Nov 2021 16:56:58 +0100 Subject: [PATCH 17/31] core/rawdb: fix off-by-one err --- core/rawdb/freezer.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index cfdb3c05b98c..468d535aacb0 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -621,17 +621,17 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { // Remember in which file the switch happens. // TODO: what if it coincidentally starts in a new file? // we won't need to copy-over that file - entry, err := table.getIndices(i, 0) + indices, err := table.getIndices(i, 1) if err != nil { return err } // First non-legacy entry coincidentally is located in a new // file and we have a clean switch-over boundary. No need to // copy over further elements. - if entry[0].filenum != entry[1].filenum { + if indices[0].filenum != indices[1].filenum { copyOver = false } - filenum = entry[1].filenum + filenum = indices[1].filenum break } } @@ -641,15 +641,11 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { // 1. loop getBounds until filenum exceeds threshold filenum // 2. copy verbatim to new table for ; i < numAncients; i++ { - /*_, _, fn, err := table.getBounds(i) - if err != nil { - return err - }*/ - idx, err := table.readEntry(i) + indices, err := table.getIndices(i, 1) if err != nil { return err } - if idx.filenum > filenum { + if indices[1].filenum > filenum { log.Info("Reached new file with updated receipts", "fn", filenum, "i", i) break } @@ -669,23 +665,26 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { // 3. need to copy rest of old index and repair the filenum in the entries if i < numAncients { - idx, err := table.readEntry(i) + indices, err := table.getIndices(i, 1) if err != nil { return err } lastFilenum := newTable.headId // idx.filenum is always >=1 - diff := int32(lastFilenum) - int32(idx.filenum-1) - log.Info("Starting duplication", "i", i, "oldFn", idx.filenum, "offset", idx.offset, "newHeadId", lastFilenum) + diff := int32(lastFilenum) - int32(indices[1].filenum-1) + //log.Info("Starting duplication", "i", i, "oldFn", idx.filenum, "offset", idx.offset, "newHeadId", lastFilenum) for ; i < numAncients; i++ { - idx, err := table.readEntry(i) + indices, err := table.getIndices(i, 1) if err != nil { return err } + idx := indexEntry{ + // (idx.filenum + diff) is always > 0 + filenum: uint32(int32(indices[1].filenum) + diff), + offset: indices[1].offset, + } //log.Info("read entry", "i", i, "fn", idx.filenum, "offset", idx.offset) - // (idx.filenum + diff) is always > 0 - idx.filenum = uint32(int32(idx.filenum) + diff) newTable.writeEntry(idx) } log.Info("Duplicated rest of index in new table", "i", i) From 8092cdde1dd676c85de9334a3ac86c66da147021 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 16 Nov 2021 19:41:48 +0100 Subject: [PATCH 18/31] core/rawdb: add tableFilePath method --- cmd/geth/dbcmd.go | 2 ++ core/rawdb/freezer_table.go | 30 +++++++++++++----------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 685926b1a203..7d19c0e86645 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -724,6 +724,8 @@ func freezerMigrate(ctx *cli.Context) error { if err != nil { return err } + // TODO: this won't work if first blocks are empty like in mainnet + // Need to find first non-empty block and check that isFirstLegacy, err := types.IsLegacyStoredReceipts(first) if err != nil { return err diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 0cd769397ce7..4565eb1b3fc9 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -431,13 +431,8 @@ func (t *freezerTable) Close() error { func (t *freezerTable) openFile(num uint32, opener func(string) (*os.File, error)) (f *os.File, err error) { var exist bool if f, exist = t.files[num]; !exist { - var name string - if t.noCompression { - name = fmt.Sprintf("%s.%04d.rdat", t.name, num) - } else { - name = fmt.Sprintf("%s.%04d.cdat", t.name, num) - } - f, err = opener(filepath.Join(t.path, name)) + path := t.tableFilePath(num) + f, err = opener(path) if err != nil { return nil, err } @@ -446,6 +441,17 @@ func (t *freezerTable) openFile(num uint32, opener func(string) (*os.File, error return f, err } +// tableFileName returns path to a table file of a given index. +func (t *freezerTable) tableFilePath(num uint32) string { + var name string + if t.noCompression { + name = fmt.Sprintf("%s.%04d.rdat", t.name, num) + } else { + name = fmt.Sprintf("%s.%04d.cdat", t.name, num) + } + return filepath.Join(t.path, name) +} + // releaseFile closes a file, and removes it from the open file cache. // Assumes that the caller holds the write lock func (t *freezerTable) releaseFile(num uint32) { @@ -763,16 +769,6 @@ func (t *freezerTable) dumpIndex(w io.Writer, start, stop int64) { fmt.Fprintf(w, "|--------------------------|\n") } -func (t *freezerTable) readEntry(item uint64) (indexEntry, error) { - buffer := make([]byte, indexEntrySize) - var idx indexEntry - if _, err := t.index.ReadAt(buffer, int64(item*indexEntrySize)); err != nil { - return idx, err - } - idx.unmarshalBinary(buffer) - return idx, nil -} - // low-level, doesnt increase counters, assumes lock func (t *freezerTable) writeEntry(idx indexEntry) error { // Ensure the table is still accessible From 26ec3671382b68a290b4c05da6d0e601516e9a96 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 16 Nov 2021 20:39:14 +0100 Subject: [PATCH 19/31] cmd,core: rename files after switchover --- cmd/geth/dbcmd.go | 4 ++-- core/rawdb/freezer.go | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 7d19c0e86645..1baf578f6fd7 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -761,9 +761,9 @@ func freezerMigrate(ctx *cli.Context) error { } log.Info("Before closing db") - if err := db.Close(); err != nil { + /*if err := db.Close(); err != nil { return err - } + }*/ log.Info("Migration finished", "duration", time.Since(start)) return nil diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 468d535aacb0..e1468940ba72 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -663,6 +663,9 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { } log.Info("Committed write batch", "newHeadId", newTable.headId, "headbytes", newTable.headBytes, "items", newTable.items) + var diff int32 + toRename := make(map[uint32]struct{}) + // 3. need to copy rest of old index and repair the filenum in the entries if i < numAncients { indices, err := table.getIndices(i, 1) @@ -672,7 +675,7 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { lastFilenum := newTable.headId // idx.filenum is always >=1 - diff := int32(lastFilenum) - int32(indices[1].filenum-1) + diff = int32(lastFilenum) - int32(indices[1].filenum-1) //log.Info("Starting duplication", "i", i, "oldFn", idx.filenum, "offset", idx.offset, "newHeadId", lastFilenum) for ; i < numAncients; i++ { indices, err := table.getIndices(i, 1) @@ -686,15 +689,25 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { } //log.Info("read entry", "i", i, "fn", idx.filenum, "offset", idx.offset) newTable.writeEntry(idx) + toRename[indices[1].filenum] = struct{}{} } log.Info("Duplicated rest of index in new table", "i", i) } + // TODO: close table here or in cmd? + if err := table.Close(); err != nil { + return err + } if err := newTable.Close(); err != nil { return err } - // Delete old table index & files up to cross-over point + // Rename table files after the switchover point + for k := range toRename { + if err := os.Rename(table.tableFilePath(k), newTable.tableFilePath(uint32(int32(k)+diff))); err != nil { + return err + } + } // Move new table files to ancients dir files, err := ioutil.ReadDir(migrationPath) @@ -702,6 +715,7 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { return err } for _, f := range files { + // This will replace the index + table files up to and including the switchover file if err := os.Rename(filepath.Join(migrationPath, f.Name()), filepath.Join(ancientsPath, f.Name())); err != nil { return err } From fd000bc5b95566927bebcf2539ed3d8ac86a4fde Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 17 Nov 2021 13:30:16 +0100 Subject: [PATCH 20/31] core/rawdb: fix file juggling --- core/rawdb/freezer.go | 36 +++++++++++++++++++----------------- core/rawdb/freezer_table.go | 36 ++++++++++++++---------------------- 2 files changed, 33 insertions(+), 39 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index e1468940ba72..6f1587b42a5b 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -676,7 +676,6 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { lastFilenum := newTable.headId // idx.filenum is always >=1 diff = int32(lastFilenum) - int32(indices[1].filenum-1) - //log.Info("Starting duplication", "i", i, "oldFn", idx.filenum, "offset", idx.offset, "newHeadId", lastFilenum) for ; i < numAncients; i++ { indices, err := table.getIndices(i, 1) if err != nil { @@ -687,43 +686,46 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { filenum: uint32(int32(indices[1].filenum) + diff), offset: indices[1].offset, } - //log.Info("read entry", "i", i, "fn", idx.filenum, "offset", idx.offset) newTable.writeEntry(idx) toRename[indices[1].filenum] = struct{}{} } log.Info("Duplicated rest of index in new table", "i", i) } - // TODO: close table here or in cmd? - if err := table.Close(); err != nil { - return err - } - if err := newTable.Close(); err != nil { - return err - } - - // Rename table files after the switchover point + // Warning: file juggling to follow. + // First release open table files because we need to move some of them. + table.releaseFilesAfter(0, false) + // Move table files after the switchover point to migration dir. for k := range toRename { if err := os.Rename(table.tableFilePath(k), newTable.tableFilePath(uint32(int32(k)+diff))); err != nil { return err } } - // Move new table files to ancients dir + // Now we can delete all the rest. + if err := table.deleteFiles(); err != nil { + return err + } + log.Info("Deleted old table files") + + // Move migrated files to ancients dir. + if err := newTable.Close(); err != nil { + return err + } files, err := ioutil.ReadDir(migrationPath) if err != nil { return err } for _, f := range files { - // This will replace the index + table files up to and including the switchover file + // This will replace the index + table files up to and including the switchover file. if err := os.Rename(filepath.Join(migrationPath, f.Name()), filepath.Join(ancientsPath, f.Name())); err != nil { return err } } - - /*if err := table.drop(); err != nil { + // Delete by now empty dir. + if err := os.Remove(migrationPath); err != nil { return err - }*/ - log.Info("Dropped old table files") + } + return nil } diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 4565eb1b3fc9..b0043b7ea56e 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -474,6 +474,20 @@ func (t *freezerTable) releaseFilesAfter(num uint32, remove bool) { } } +// deleteFiles deletes table files based on tail and head +// indices, and assumes these files are not open. +func (t *freezerTable) deleteFiles() error { + t.lock.Lock() + defer t.lock.Unlock() + for i := t.tailId; i < t.headId; i++ { + path := t.tableFilePath(i) + if err := os.Remove(path); err != nil { + return err + } + } + return nil +} + // getIndices returns the index entries for the given from-item, covering 'count' items. // N.B: The actual number of returned indices for N items will always be N+1 (unless an // error is returned). @@ -510,28 +524,6 @@ func (t *freezerTable) getIndices(from, count uint64) ([]*indexEntry, error) { return indices, nil } -// drop deletes the index file and all the open files. -// TODO(sina): make sure to delete non-open files -func (t *freezerTable) drop() error { - t.lock.Lock() - defer t.lock.Unlock() - - if err := t.index.Close(); err != nil { - log.Warn("failed to close index", "error", err, "index", t.index) - return err - } - log.Info("before os removing index", "name", t.index.Name()) - if err := os.Remove(t.index.Name()); err != nil { - return err - } - t.index = nil - - t.releaseFilesAfter(0, true) - t.head = nil - - return nil -} - // Retrieve looks up the data offset of an item with the given number and retrieves // the raw binary blob from the data file. func (t *freezerTable) Retrieve(item uint64) ([]byte, error) { From e4028b6a53b75b799a66695d96011b375c866e22 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 17 Nov 2021 13:39:06 +0100 Subject: [PATCH 21/31] core/rawdb: remove DropTable method --- core/rawdb/database.go | 4 ---- core/rawdb/freezer.go | 15 --------------- core/rawdb/table.go | 4 ---- ethdb/database.go | 3 --- 4 files changed, 26 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 737a64252b35..7fc95abc9b70 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -135,10 +135,6 @@ func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReader) error) (e return fn(db) } -func (db *nofreezedb) DropTable(_ string) error { - return errNotSupported -} - func (db *nofreezedb) TransformTable(_ string, _ TransformerFn) error { return errNotSupported } diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 6f1587b42a5b..500d16d05b6b 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -549,21 +549,6 @@ func (f *freezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes [] return hashes, err } -// dropTable removes all files belonging to a table. Caution: this puts the freezer -// in an unstable position. -// TODO: should be private -func (f *freezer) DropTable(kind string) error { - table, ok := f.tables[kind] - if !ok { - return errUnknownTable - } - if err := table.drop(); err != nil { - return err - } - delete(f.tables, kind) - return nil -} - // TransformerFn takes a freezer entry in an older format and returns // the same in a new format. The second return argument determines // if the entry is already of the new format. diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 203aeb677605..9b7fbcbbf060 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -101,10 +101,6 @@ func (t *table) Sync() error { return t.db.Sync() } -func (t *table) DropTable(kind string) error { - return t.db.DropTable(kind) -} - func (t *table) TransformTable(kind string, fn TransformerFn) error { return t.db.TransformTable(kind, fn) } diff --git a/ethdb/database.go b/ethdb/database.go index 9f1ac23f96f1..d306bf4b477b 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -112,9 +112,6 @@ type AncientWriter interface { // Sync flushes all in-memory ancient store data to disk. Sync() error - // TODO: tmp, to delete table files - DropTable(string) error - TransformTable(string, func([]byte) ([]byte, bool, error)) error } From e3b1daa953611fba44f129b35e0400ada0ca464c Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 17 Nov 2021 13:39:59 +0100 Subject: [PATCH 22/31] ethdb: minor --- ethdb/database.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ethdb/database.go b/ethdb/database.go index d306bf4b477b..7c4850b7cd7f 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -112,6 +112,7 @@ type AncientWriter interface { // Sync flushes all in-memory ancient store data to disk. Sync() error + // Processes and migrates entries in a table to a new format according to a transformer function. TransformTable(string, func([]byte) ([]byte, bool, error)) error } From 0d22403e53eab10fd49f03d30644dcaf93bd0eac Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 17 Nov 2021 19:35:54 +0100 Subject: [PATCH 23/31] core/rawdb: improve readability --- core/rawdb/freezer.go | 45 +++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 500d16d05b6b..88455fa5ebc8 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -554,9 +554,9 @@ func (f *freezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes [] // if the entry is already of the new format. type TransformerFn = func([]byte) ([]byte, bool, error) +// TransformTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. func (f *freezer) TransformTable(kind string, fn TransformerFn) error { - // TODO: Do we need to recover anything after error? - if f.readonly { return errReadOnly } @@ -567,15 +567,9 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { if !ok { return errUnknownTable } - - numAncients, err := f.Ancients() - if err != nil { - return err - } - ancientsPath := filepath.Dir(table.index.Name()) - // Set up new dir for the migrated table which we'll at the end - // move over to the ancients dir. + // Set up new dir for the migrated table, the content of which + // we'll at the end move over to the ancients dir. migrationPath := filepath.Join(ancientsPath, "migration") newTable, err := NewFreezerTable(migrationPath, kind, FreezerNoSnappy[kind]) if err != nil { @@ -583,6 +577,10 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { } batch := newTable.newBatch() + numAncients, err := f.Ancients() + if err != nil { + return err + } var i uint64 // Number of the file in which the first up-to-date receipt appers var filenum uint32 @@ -590,6 +588,9 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { // Iterate through entries and transform them // until reaching first non-legacy one. for i = 0; i < numAncients; i++ { + if i%500000 == 0 { + log.Info("Processing legacy elements", "number", i) + } blob, err := table.Retrieve(i) if err != nil { return err @@ -604,8 +605,6 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { } else { // Reached the first up-to-date entry. // Remember in which file the switch happens. - // TODO: what if it coincidentally starts in a new file? - // we won't need to copy-over that file indices, err := table.getIndices(i, 1) if err != nil { return err @@ -617,10 +616,10 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { copyOver = false } filenum = indices[1].filenum + log.Info("Found first non-legacy element", "number", i, "filenum", filenum, "copyOver", copyOver) break } } - log.Info("Copying over leftover receipts", "i", i, "filenum", filenum) if copyOver { // Copy over left-over receipts in the file with last legacy receipt: // 1. loop getBounds until filenum exceeds threshold filenum @@ -631,7 +630,7 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { return err } if indices[1].filenum > filenum { - log.Info("Reached new file with updated receipts", "fn", filenum, "i", i) + log.Info("Copied over rest of switch-over file", "number", i, "nextFile", indices[1].filenum) break } blob, err := table.Retrieve(i) @@ -640,15 +639,13 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { } batch.AppendRaw(i, blob) } - log.Info("Finished copying leftovers", "i", i) } if err := batch.commit(); err != nil { return err } - log.Info("Committed write batch", "newHeadId", newTable.headId, "headbytes", newTable.headBytes, "items", newTable.items) - var diff int32 + var fileCountDiff int32 toRename := make(map[uint32]struct{}) // 3. need to copy rest of old index and repair the filenum in the entries @@ -658,31 +655,30 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { return err } - lastFilenum := newTable.headId - // idx.filenum is always >=1 - diff = int32(lastFilenum) - int32(indices[1].filenum-1) + log.Info("Duplicating rest of index", "fromFile", indices[1].filenum, "toFile", newTable.headId+1) + fileCountDiff = int32(newTable.headId+1) - int32(indices[1].filenum) for ; i < numAncients; i++ { indices, err := table.getIndices(i, 1) if err != nil { return err } idx := indexEntry{ - // (idx.filenum + diff) is always > 0 - filenum: uint32(int32(indices[1].filenum) + diff), + // (idx.filenum + fileCountDiff) is always > 0 + filenum: uint32(int32(indices[1].filenum) + fileCountDiff), offset: indices[1].offset, } newTable.writeEntry(idx) toRename[indices[1].filenum] = struct{}{} } - log.Info("Duplicated rest of index in new table", "i", i) } + log.Info("Replacing table files") // Warning: file juggling to follow. // First release open table files because we need to move some of them. table.releaseFilesAfter(0, false) // Move table files after the switchover point to migration dir. for k := range toRename { - if err := os.Rename(table.tableFilePath(k), newTable.tableFilePath(uint32(int32(k)+diff))); err != nil { + if err := os.Rename(table.tableFilePath(k), newTable.tableFilePath(uint32(int32(k)+fileCountDiff))); err != nil { return err } } @@ -691,7 +687,6 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { if err := table.deleteFiles(); err != nil { return err } - log.Info("Deleted old table files") // Move migrated files to ancients dir. if err := newTable.Close(); err != nil { From 84668876269b806883a04c049174fd44d6930fb0 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 17 Nov 2021 20:37:33 +0100 Subject: [PATCH 24/31] core/rawdb: fix initial legacy check --- cmd/geth/dbcmd.go | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 1baf578f6fd7..b74c10814a7d 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -718,22 +718,36 @@ func freezerMigrate(ctx *cli.Context) error { log.Info("No blocks in freezer to migrate") return nil } - // TODO: check genesis or block 1? - // devnets might have an empty genesis - first, err := db.Ancient("receipts", 1) + + // Find first block with non-empty receipt + i := uint64(0) + emptyRLPList := []byte{192} + for ; i < numAncients; i++ { + r, err := db.Ancient("receipts", i) + if err != nil { + return err + } + if len(r) == 0 { + continue + } + if !bytes.Equal(r, emptyRLPList) { + break + } + } + // Is first non-empty receipt legacy? + first, err := db.Ancient("receipts", i) if err != nil { return err } - // TODO: this won't work if first blocks are empty like in mainnet - // Need to find first non-empty block and check that isFirstLegacy, err := types.IsLegacyStoredReceipts(first) if err != nil { return err } if !isFirstLegacy { - log.Info("No legacy receipts to migrate") + log.Info("No legacy receipts to migrate", "number", i) return nil } + log.Info("First legacy receipt", "number", i) transformer := func(blob []byte) ([]byte, bool, error) { // Stop when first v5 receipt is spotted. @@ -760,10 +774,9 @@ func freezerMigrate(ctx *cli.Context) error { return err } - log.Info("Before closing db") - /*if err := db.Close(); err != nil { + if err := db.Close(); err != nil { return err - }*/ + } log.Info("Migration finished", "duration", time.Since(start)) return nil From eb306b65ec155b77d1a256b480aa649dc1df01b9 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 17 Nov 2021 21:51:23 +0100 Subject: [PATCH 25/31] Add start param to migrateTable --- cmd/geth/dbcmd.go | 13 +++++++------ core/rawdb/database.go | 2 +- core/rawdb/freezer.go | 11 +++++++---- core/rawdb/table.go | 4 ++-- ethdb/database.go | 4 ++-- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index b74c10814a7d..b1273ac358a3 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -720,9 +720,9 @@ func freezerMigrate(ctx *cli.Context) error { } // Find first block with non-empty receipt - i := uint64(0) + firstIdx := uint64(0) emptyRLPList := []byte{192} - for ; i < numAncients; i++ { + for i := uint64(0); i < numAncients; i++ { r, err := db.Ancient("receipts", i) if err != nil { return err @@ -731,11 +731,12 @@ func freezerMigrate(ctx *cli.Context) error { continue } if !bytes.Equal(r, emptyRLPList) { + firstIdx = i break } } // Is first non-empty receipt legacy? - first, err := db.Ancient("receipts", i) + first, err := db.Ancient("receipts", firstIdx) if err != nil { return err } @@ -744,10 +745,10 @@ func freezerMigrate(ctx *cli.Context) error { return err } if !isFirstLegacy { - log.Info("No legacy receipts to migrate", "number", i) + log.Info("No legacy receipts to migrate", "number", firstIdx) return nil } - log.Info("First legacy receipt", "number", i) + log.Info("First legacy receipt", "number", firstIdx) transformer := func(blob []byte) ([]byte, bool, error) { // Stop when first v5 receipt is spotted. @@ -770,7 +771,7 @@ func freezerMigrate(ctx *cli.Context) error { log.Info("Starting migration", "ancients", numAncients) start := time.Now() - if err := db.TransformTable("receipts", transformer); err != nil { + if err := db.MigrateTable("receipts", firstIdx, transformer); err != nil { return err } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 7fc95abc9b70..627ab2b6159f 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -135,7 +135,7 @@ func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReader) error) (e return fn(db) } -func (db *nofreezedb) TransformTable(_ string, _ TransformerFn) error { +func (db *nofreezedb) MigrateTable(_ string, _ uint64, _ TransformerFn) error { return errNotSupported } diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 88455fa5ebc8..628f1488d9f1 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -554,9 +554,9 @@ func (f *freezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes [] // if the entry is already of the new format. type TransformerFn = func([]byte) ([]byte, bool, error) -// TransformTable processes the entries in a given table in sequence +// MigrateTable processes the entries in a given table in sequence // converting them to a new format if they're of an old format. -func (f *freezer) TransformTable(kind string, fn TransformerFn) error { +func (f *freezer) MigrateTable(kind string, start uint64, fn TransformerFn) error { if f.readonly { return errReadOnly } @@ -581,13 +581,16 @@ func (f *freezer) TransformTable(kind string, fn TransformerFn) error { if err != nil { return err } - var i uint64 + if start >= numAncients { + return errors.New("not enough elements in freezer") + } + i := start // Number of the file in which the first up-to-date receipt appers var filenum uint32 copyOver := true // Iterate through entries and transform them // until reaching first non-legacy one. - for i = 0; i < numAncients; i++ { + for ; i < numAncients; i++ { if i%500000 == 0 { log.Info("Processing legacy elements", "number", i) } diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 9b7fbcbbf060..1381b2560542 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -101,8 +101,8 @@ func (t *table) Sync() error { return t.db.Sync() } -func (t *table) TransformTable(kind string, fn TransformerFn) error { - return t.db.TransformTable(kind, fn) +func (t *table) MigrateTable(kind string, start uint64, fn TransformerFn) error { + return t.db.MigrateTable(kind, start, fn) } // Put inserts the given value into the database at a prefixed version of the diff --git a/ethdb/database.go b/ethdb/database.go index 7c4850b7cd7f..f0d913aee037 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -112,8 +112,8 @@ type AncientWriter interface { // Sync flushes all in-memory ancient store data to disk. Sync() error - // Processes and migrates entries in a table to a new format according to a transformer function. - TransformTable(string, func([]byte) ([]byte, bool, error)) error + // MigrateTable Processes and migrates entries in a table to a new format. + MigrateTable(string, uint64, func([]byte) ([]byte, bool, error)) error } // AncientWriteOp is given to the function argument of ModifyAncients. From 76426e35a9c969a2844924feb2ca46b73e7d7a77 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 18 Nov 2021 09:48:17 +0100 Subject: [PATCH 26/31] check batch append err --- core/rawdb/freezer.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 628f1488d9f1..5bd08d601a7f 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -604,7 +604,9 @@ func (f *freezer) MigrateTable(kind string, start uint64, fn TransformerFn) erro } if !stop { // Entry is legacy, push transformed version - batch.AppendRaw(i, out) + if err := batch.AppendRaw(i, out); err != nil { + return err + } } else { // Reached the first up-to-date entry. // Remember in which file the switch happens. @@ -640,7 +642,9 @@ func (f *freezer) MigrateTable(kind string, start uint64, fn TransformerFn) erro if err != nil { return err } - batch.AppendRaw(i, blob) + if err := batch.AppendRaw(i, blob); err != nil { + return err + } } } From 1279bb3ca6414e316add8dd027ed99a23cd8218e Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 18 Nov 2021 09:54:43 +0100 Subject: [PATCH 27/31] drop start param from migrateTable --- cmd/geth/dbcmd.go | 2 +- core/rawdb/database.go | 2 +- core/rawdb/freezer.go | 7 ++----- core/rawdb/table.go | 4 ++-- ethdb/database.go | 2 +- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index b1273ac358a3..8d4f61d913a5 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -771,7 +771,7 @@ func freezerMigrate(ctx *cli.Context) error { log.Info("Starting migration", "ancients", numAncients) start := time.Now() - if err := db.MigrateTable("receipts", firstIdx, transformer); err != nil { + if err := db.MigrateTable("receipts", transformer); err != nil { return err } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 627ab2b6159f..05097d6e862b 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -135,7 +135,7 @@ func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReader) error) (e return fn(db) } -func (db *nofreezedb) MigrateTable(_ string, _ uint64, _ TransformerFn) error { +func (db *nofreezedb) MigrateTable(_ string, _ TransformerFn) error { return errNotSupported } diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 5bd08d601a7f..e1e50f6ad9fe 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -556,7 +556,7 @@ type TransformerFn = func([]byte) ([]byte, bool, error) // MigrateTable processes the entries in a given table in sequence // converting them to a new format if they're of an old format. -func (f *freezer) MigrateTable(kind string, start uint64, fn TransformerFn) error { +func (f *freezer) MigrateTable(kind string, fn TransformerFn) error { if f.readonly { return errReadOnly } @@ -581,10 +581,7 @@ func (f *freezer) MigrateTable(kind string, start uint64, fn TransformerFn) erro if err != nil { return err } - if start >= numAncients { - return errors.New("not enough elements in freezer") - } - i := start + i := uint64(0) // Number of the file in which the first up-to-date receipt appers var filenum uint32 copyOver := true diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 1381b2560542..c46b0bfcd07a 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -101,8 +101,8 @@ func (t *table) Sync() error { return t.db.Sync() } -func (t *table) MigrateTable(kind string, start uint64, fn TransformerFn) error { - return t.db.MigrateTable(kind, start, fn) +func (t *table) MigrateTable(kind string, fn TransformerFn) error { + return t.db.MigrateTable(kind, fn) } // Put inserts the given value into the database at a prefixed version of the diff --git a/ethdb/database.go b/ethdb/database.go index f0d913aee037..832e6f60491c 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -113,7 +113,7 @@ type AncientWriter interface { Sync() error // MigrateTable Processes and migrates entries in a table to a new format. - MigrateTable(string, uint64, func([]byte) ([]byte, bool, error)) error + MigrateTable(string, func([]byte) ([]byte, bool, error)) error } // AncientWriteOp is given to the function argument of ModifyAncients. From 7338e21ba21bbcf7e9fd4a558ecfb400d46aa210 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 18 Nov 2021 09:55:05 +0100 Subject: [PATCH 28/31] add license to legacy file --- core/types/legacy.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/core/types/legacy.go b/core/types/legacy.go index 8476108de77c..46e4fc891e84 100644 --- a/core/types/legacy.go +++ b/core/types/legacy.go @@ -1,3 +1,19 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package types import ( From 63fb341dd26e6ca7623be947d5def1a0a76cd222 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 18 Nov 2021 10:00:27 +0100 Subject: [PATCH 29/31] rm helper funcs and their tests --- core/types/legacy.go | 88 ----------------------------------- core/types/receipt_test.go | 95 -------------------------------------- 2 files changed, 183 deletions(-) diff --git a/core/types/legacy.go b/core/types/legacy.go index 46e4fc891e84..1796c85d9a0b 100644 --- a/core/types/legacy.go +++ b/core/types/legacy.go @@ -19,7 +19,6 @@ package types import ( "errors" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" ) @@ -52,90 +51,3 @@ func ConvertLegacyStoredReceipts(raw []byte) ([]byte, error) { } return rlp.EncodeToBytes(&receipts) } - -// convertLegacyStoredReceipt takes a legacy RLP-encoded stored receipt -// and returns a fresh RLP-encoded stored receipt. -func convertLegacyStoredReceipt(raw []byte) ([]byte, error) { - var receipt ReceiptForStorage - if err := rlp.DecodeBytes(raw, &receipt); err != nil { - return nil, err - } - return rlp.EncodeToBytes(&receipt) -} - -// v4StoredReceiptRLPWithLogs is the storage encoding of a receipt used in database version 4. -type v4StoredReceiptRLPWithLogs struct { - PostStateOrStatus []byte - CumulativeGasUsed uint64 - TxHash common.Hash - ContractAddress common.Address - Logs []*legacyRlpStorageLog - GasUsed uint64 -} - -// v3StoredReceiptRLP is the original storage encoding of a receipt including some unnecessary fields. -type v3StoredReceiptRLPWithLogs struct { - PostStateOrStatus []byte - CumulativeGasUsed uint64 - Bloom Bloom - TxHash common.Hash - ContractAddress common.Address - Logs []*legacyRlpStorageLog - GasUsed uint64 -} - -func EncodeAsLegacyStoredReceiptsRLP(receipts []*Receipt) ([]byte, error) { - stored := make([]v3StoredReceiptRLPWithLogs, len(receipts)) - for i, r := range receipts { - stored[i] = *toV3StoredReceiptRLPWithLogs(r) - } - return rlp.EncodeToBytes(stored) -} - -func encodeAsV4StoredReceiptRLPWithLogs(want *Receipt) ([]byte, error) { - stored := &v4StoredReceiptRLPWithLogs{ - PostStateOrStatus: want.statusEncoding(), - CumulativeGasUsed: want.CumulativeGasUsed, - TxHash: want.TxHash, - ContractAddress: want.ContractAddress, - Logs: make([]*legacyRlpStorageLog, len(want.Logs)), - GasUsed: want.GasUsed, - } - for i, log := range want.Logs { - stored.Logs[i] = legacyFromLog(log) - } - return rlp.EncodeToBytes(stored) -} - -func toV3StoredReceiptRLPWithLogs(from *Receipt) *v3StoredReceiptRLPWithLogs { - return &v3StoredReceiptRLPWithLogs{ - PostStateOrStatus: from.statusEncoding(), - CumulativeGasUsed: from.CumulativeGasUsed, - Bloom: from.Bloom, - TxHash: from.TxHash, - ContractAddress: from.ContractAddress, - Logs: make([]*legacyRlpStorageLog, len(from.Logs)), - GasUsed: from.GasUsed, - } -} - -func encodeAsV3StoredReceiptRLPWithLogs(want *Receipt) ([]byte, error) { - stored := toV3StoredReceiptRLPWithLogs(want) - for i, log := range want.Logs { - stored.Logs[i] = legacyFromLog(log) - } - return rlp.EncodeToBytes(stored) -} - -func legacyFromLog(want *Log) *legacyRlpStorageLog { - return &legacyRlpStorageLog{ - Address: want.Address, - Topics: want.Topics, - Data: want.Data, - BlockNumber: want.BlockNumber, - TxHash: want.TxHash, - TxIndex: want.TxIndex, - BlockHash: want.BlockHash, - Index: want.Index, - } -} diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index a4366b12775b..613559a6586c 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -213,101 +213,6 @@ func encodeAsV3StoredReceiptRLP(want *Receipt) ([]byte, error) { return rlp.EncodeToBytes(stored) } -func TestLegacyConversion(t *testing.T) { - tests := []struct { - name string - encode func(*Receipt) ([]byte, error) - }{ - { - "V4StoredReceiptRLP", - encodeAsV4StoredReceiptRLPWithLogs, - }, - { - "V3StoredReceiptRLP", - encodeAsV3StoredReceiptRLPWithLogs, - }, - } - - tx := NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(1), 1, big.NewInt(1), nil) - txHash := tx.Hash() - receipt := &Receipt{ - Status: ReceiptStatusFailed, - CumulativeGasUsed: 1, - Logs: []*Log{ - { - Address: common.BytesToAddress([]byte{0x11}), - Topics: []common.Hash{common.HexToHash("dead"), common.HexToHash("beef")}, - Data: []byte{0x01, 0x00, 0xff}, - TxHash: txHash, - }, - { - Address: common.BytesToAddress([]byte{0x01, 0x11}), - Topics: []common.Hash{common.HexToHash("dead"), common.HexToHash("beef")}, - Data: []byte{0x01, 0x00, 0xff}, - TxHash: txHash, - }, - }, - TxHash: txHash, - ContractAddress: common.BytesToAddress([]byte{0x01, 0x11, 0x11}), - GasUsed: 111111, - } - receipt.Bloom = CreateBloom(Receipts{receipt}) - - stored := (*ReceiptForStorage)(receipt) - wantEncoding, err := rlp.EncodeToBytes(stored) - if err != nil { - t.Fatalf("Failed to encode receipt: %v", err) - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - enc, err := tc.encode(receipt) - if err != nil { - t.Fatalf("Error encoding receipt: %v", err) - } - - var dec ReceiptForStorage - if err := rlp.DecodeBytes(enc, &dec); err != nil { - t.Fatalf("Error decoding RLP receipt: %v", err) - } - - // Check whether all consensus fields are correct. - if dec.Status != receipt.Status { - t.Fatalf("Receipt status mismatch, want %v, have %v", receipt.Status, dec.Status) - } - if dec.CumulativeGasUsed != receipt.CumulativeGasUsed { - t.Fatalf("Receipt CumulativeGasUsed mismatch, want %v, have %v", receipt.CumulativeGasUsed, dec.CumulativeGasUsed) - } - if dec.Bloom != receipt.Bloom { - t.Fatalf("Bloom data mismatch, want %v, have %v", receipt.Bloom, dec.Bloom) - } - if len(dec.Logs) != len(receipt.Logs) { - t.Fatalf("Receipt log number mismatch, want %v, have %v", len(receipt.Logs), len(dec.Logs)) - } - for i := 0; i < len(dec.Logs); i++ { - if dec.Logs[i].Address != receipt.Logs[i].Address { - t.Fatalf("Receipt log %d address mismatch, want %v, have %v", i, receipt.Logs[i].Address, dec.Logs[i].Address) - } - if !reflect.DeepEqual(dec.Logs[i].Topics, receipt.Logs[i].Topics) { - t.Fatalf("Receipt log %d topics mismatch, want %v, have %v", i, receipt.Logs[i].Topics, dec.Logs[i].Topics) - } - if !bytes.Equal(dec.Logs[i].Data, receipt.Logs[i].Data) { - t.Fatalf("Receipt log %d data mismatch, want %v, have %v", i, receipt.Logs[i].Data, dec.Logs[i].Data) - } - } - - // Check that converted encoding matches expected fresh encoding - got, err := convertLegacyStoredReceipt(enc) - if err != nil { - t.Fatalf("Failed to convert legacy stored receipt: %v", err) - } - if !bytes.Equal(wantEncoding, got) { - t.Fatalf("Receipt conversion mismatch, want %x, have %x", wantEncoding, got) - } - }) - } -} - // Tests that receipt data can be correctly derived from the contextual infos func TestDeriveFields(t *testing.T) { // Create a few transactions to have receipts for From fa28df56a34043d7770fccc2970aef0325d4c3e6 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 30 Nov 2021 18:10:08 +0100 Subject: [PATCH 30/31] freezer: rm unneeded line Co-authored-by: Martin Holst Swende --- core/rawdb/freezer.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index e1e50f6ad9fe..e4d3444d36b8 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -65,7 +65,6 @@ const ( // freezerTableSize defines the maximum size of freezer data files. freezerTableSize = 2 * 1000 * 1000 * 1000 - //freezerTableSize = 2 * 1024 * 1024 ) // freezer is an memory mapped append-only database to store immutable chain data From 6bef4813fcf133ad6686d86a8563624d68444050 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 30 Nov 2021 18:29:20 +0100 Subject: [PATCH 31/31] core, ethdb: add comments --- core/rawdb/freezer.go | 3 ++- core/rawdb/table.go | 2 ++ ethdb/database.go | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index e1e50f6ad9fe..38eed0f7f98c 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -551,7 +551,8 @@ func (f *freezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes [] // TransformerFn takes a freezer entry in an older format and returns // the same in a new format. The second return argument determines -// if the entry is already of the new format. +// if the entry is not in the legacy format. Note that empty items can +// be considered both legacy and non-legacy. type TransformerFn = func([]byte) ([]byte, bool, error) // MigrateTable processes the entries in a given table in sequence diff --git a/core/rawdb/table.go b/core/rawdb/table.go index c46b0bfcd07a..2c8452afb329 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -101,6 +101,8 @@ func (t *table) Sync() error { return t.db.Sync() } +// MigrateTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. func (t *table) MigrateTable(kind string, fn TransformerFn) error { return t.db.MigrateTable(kind, fn) } diff --git a/ethdb/database.go b/ethdb/database.go index 832e6f60491c..153bb4b1157b 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -113,6 +113,10 @@ type AncientWriter interface { Sync() error // MigrateTable Processes and migrates entries in a table to a new format. + // The function takes a freezer entry in an older format and returns + // the same in a new format. The second return argument determines + // if the entry is not in the legacy format. Note that empty items can + // be considered both legacy and non-legacy. MigrateTable(string, func([]byte) ([]byte, bool, error)) error }