Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use custom codec for validator metadata #1510

Merged
merged 81 commits into from
Oct 23, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
81 commits
Select commit Hold shift + click to select a range
e2e96fc
detached validator metadata codec
abi87 May 16, 2023
7b371af
nits and fixes
abi87 May 16, 2023
c17a619
Merge branch 'dev' into validator_metadata_codec
abi87 May 17, 2023
5a7eeaa
nit
abi87 May 17, 2023
9b3b10e
Merge branch 'dev' into validator_metadata_codec
abi87 May 18, 2023
f08f9e5
Merge branch 'dev' into validator_metadata_codec
abi87 May 18, 2023
302e6ab
Merge branch 'dev' into validator_metadata_codec
abi87 May 19, 2023
7c98dca
Merge branch 'dev' into validator_metadata_codec
abi87 May 20, 2023
c5ec0db
Merge branch 'dev' into validator_metadata_codec
abi87 May 23, 2023
d45a56f
Merge branch 'dev' into validator_metadata_codec
abi87 May 24, 2023
e7989fc
Merge branch 'dev' into validator_metadata_codec
abi87 May 25, 2023
f3c3106
Merge branch 'dev' into validator_metadata_codec
abi87 May 28, 2023
8f450c8
Merge branch 'dev' into validator_metadata_codec
abi87 May 31, 2023
400af13
Merge branch 'dev' into validator_metadata_codec
abi87 May 31, 2023
c1c05f2
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 1, 2023
005ad9b
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 6, 2023
cad02b9
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 8, 2023
d6bc37d
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 9, 2023
940089b
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 11, 2023
73a8d63
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 13, 2023
f473bcd
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 14, 2023
91a867d
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 15, 2023
7f234c2
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 15, 2023
9fcff0f
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 16, 2023
46d8042
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 19, 2023
f04fbcc
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 19, 2023
1a477c6
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 28, 2023
32f846f
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 28, 2023
1e3858d
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 29, 2023
7389334
Merge branch 'dev' into validator_metadata_codec
abi87 Jun 30, 2023
6aa2c75
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 7, 2023
616d69a
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 10, 2023
195d32c
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 11, 2023
11cf52d
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 13, 2023
4cf68cd
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 14, 2023
d85f475
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 19, 2023
64e4973
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 20, 2023
f9d9c8a
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 21, 2023
f8b959d
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 22, 2023
d518ac9
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 24, 2023
2b424cb
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 25, 2023
ef58ed1
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 26, 2023
73a880a
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 27, 2023
ea6abe8
Merge branch 'dev' into validator_metadata_codec
abi87 Jul 28, 2023
65dad4e
Merge branch 'dev' into validator_metadata_codec
abi87 Aug 9, 2023
2494246
Merge branch 'dev' into validator_metadata_codec
abi87 Aug 14, 2023
68cde28
Merge branch 'dev' into validator_metadata_codec
abi87 Aug 16, 2023
2cfb6cc
nit
abi87 Aug 16, 2023
e049f41
nit
abi87 Aug 16, 2023
dc241a9
Merge branch 'dev' into validator_metadata_codec
abi87 Aug 22, 2023
e8c8425
Merge branch 'dev' into validator_metadata_codec
abi87 Aug 24, 2023
e0f5e03
Merge branch 'dev' into validator_metadata_codec
abi87 Aug 25, 2023
68fc5a0
Merge branch 'dev' into validator_metadata_codec
abi87 Aug 29, 2023
42cb462
Merge branch 'dev' into validator_metadata_codec
abi87 Aug 30, 2023
60c4bbf
Merge branch 'dev' into validator_metadata_codec
abi87 Aug 31, 2023
d7698e6
Merge branch 'dev' into validator_metadata_codec
abi87 Sep 2, 2023
f09d0b8
Merge branch 'dev' into validator_metadata_codec
abi87 Sep 6, 2023
cbe1b4e
Merge branch 'dev' into validator_metadata_codec
abi87 Sep 12, 2023
51119a4
Merge branch 'dev' into validator_metadata_codec
abi87 Sep 14, 2023
ea9ca44
Merge branch 'dev' into validator_metadata_codec
abi87 Sep 18, 2023
47301bc
Merge branch 'dev' into validator_metadata_codec
abi87 Sep 20, 2023
ab086d3
Merge branch 'dev' into validator_metadata_codec
abi87 Sep 23, 2023
276c1d2
Merge branch 'dev' into validator_metadata_codec
abi87 Sep 27, 2023
3ce6889
Merge branch 'dev' into validator_metadata_codec
abi87 Sep 28, 2023
ff63392
introduced delegator metadata
abi87 Sep 28, 2023
25db2fd
Merge branch 'dev' into validator_metadata_codec
abi87 Sep 29, 2023
d6ef189
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 4, 2023
54a6eef
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 6, 2023
9bde97b
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 10, 2023
7d57392
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 11, 2023
dad193c
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 12, 2023
bb1ff16
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 15, 2023
28cb766
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 16, 2023
9311560
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 16, 2023
c01adef
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 17, 2023
b086883
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 18, 2023
e82e829
improved delegator metadata writing
abi87 Oct 18, 2023
94fa5d5
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 19, 2023
23e4b34
Merge branch 'dev' into validator_metadata_codec
abi87 Oct 20, 2023
a6ae416
nit
abi87 Oct 20, 2023
884a347
Merge branch 'dev' into validator_metadata_codec
StephenButtolph Oct 23, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/keystore/codec.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (

const (
maxPackerSize = 1 * units.GiB // max size, in bytes, of something being marshalled by Marshal()
maxSliceLength = 256 * 1024
maxSliceLength = linearcodec.DefaultMaxSliceLength
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just removed some code duplication, once linearcodec.DefaultMaxSliceLength is exported


codecVersion = 0
)
Expand Down
4 changes: 2 additions & 2 deletions codec/linearcodec/codec.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (

const (
// default max length of a slice being marshalled by Marshal(). Should be <= math.MaxUint32.
defaultMaxSliceLength = 256 * 1024
DefaultMaxSliceLength = 256 * 1024
)

var (
Expand Down Expand Up @@ -56,7 +56,7 @@ func New(tagNames []string, maxSliceLen uint32) Codec {

// NewDefault is a convenience constructor; it returns a new codec with reasonable default values
func NewDefault() Codec {
return New([]string{reflectcodec.DefaultTagName}, defaultMaxSliceLength)
return New([]string{reflectcodec.DefaultTagName}, DefaultMaxSliceLength)
}

// NewCustomMaxLength is a convenience constructor; it returns a new codec with custom max length and default tags
Expand Down
2 changes: 1 addition & 1 deletion codec/linearcodec/codec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ func TestVectors(t *testing.T) {

func TestMultipleTags(t *testing.T) {
for _, test := range codec.MultipleTagsTests {
c := New([]string{"tag1", "tag2"}, defaultMaxSliceLength)
c := New([]string{"tag1", "tag2"}, DefaultMaxSliceLength)
test(c, t)
}
}
Expand Down
28 changes: 28 additions & 0 deletions vms/platformvm/state/metadata_codec.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package state

import (
"math"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/linearcodec"
)

const (
v0tag = "v0"
v0 = uint16(0)
)

var metadataCodec codec.Manager

func init() {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we initialize the metadata code similarly to what we do with txs.Codec

c := linearcodec.New([]string{v0tag}, math.MaxInt32)
metadataCodec = codec.NewManager(math.MaxInt32)

err := metadataCodec.RegisterCodec(v0, c)
if err != nil {
panic(err)
}
}
25 changes: 25 additions & 0 deletions vms/platformvm/state/metadata_delegator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package state

import (
"github.com/ava-labs/avalanchego/database"
"github.com/ava-labs/avalanchego/ids"
)

type delegatorMetadata struct {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is currently redundant really, but when dropping stakers we'll extend it to store delegator StartTime.
Introducing the scaffolding here to reduce diffs in main PR

PotentialReward uint64

txID ids.ID
}

func parseDelegatorMetadata(bytes []byte, metadata *delegatorMetadata) error {
var err error
metadata.PotentialReward, err = database.ParseUInt64(bytes)
return err
}

func writeDelegatorMetadata(db database.KeyValueWriter, metadata *delegatorMetadata) error {
return database.PutUInt64(db, metadata.txID[:], metadata.PotentialReward)
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ import (
"github.com/ava-labs/avalanchego/utils/constants"
"github.com/ava-labs/avalanchego/utils/set"
"github.com/ava-labs/avalanchego/utils/wrappers"
"github.com/ava-labs/avalanchego/vms/platformvm/genesis"
"github.com/ava-labs/avalanchego/vms/platformvm/txs"
)

// preDelegateeRewardSize is the size of codec marshalling
Expand All @@ -24,16 +22,16 @@ const preDelegateeRewardSize = wrappers.ShortLen + 3*wrappers.LongLen
var _ validatorState = (*metadata)(nil)

type preDelegateeRewardMetadata struct {
UpDuration time.Duration `serialize:"true"`
LastUpdated uint64 `serialize:"true"` // Unix time in seconds
PotentialReward uint64 `serialize:"true"`
UpDuration time.Duration `v0:"true"`
LastUpdated uint64 `v0:"true"` // Unix time in seconds
PotentialReward uint64 `v0:"true"`
}

type validatorMetadata struct {
UpDuration time.Duration `serialize:"true"`
LastUpdated uint64 `serialize:"true"` // Unix time in seconds
PotentialReward uint64 `serialize:"true"`
PotentialDelegateeReward uint64 `serialize:"true"`
UpDuration time.Duration `v0:"true"`
LastUpdated uint64 `v0:"true"` // Unix time in seconds
PotentialReward uint64 `v0:"true"`
PotentialDelegateeReward uint64 `v0:"true"`

txID ids.ID
lastUpdated time.Time
Expand All @@ -60,7 +58,7 @@ func parseValidatorMetadata(bytes []byte, metadata *validatorMetadata) error {
// potential reward and uptime was stored but potential delegatee reward
// was not
tmp := preDelegateeRewardMetadata{}
if _, err := txs.Codec.Unmarshal(bytes, &tmp); err != nil {
if _, err := metadataCodec.Unmarshal(bytes, &tmp); err != nil {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the main change in the PR: I need to marshal/unmarshal validator metadata with their own codec metadataCodec, to be free to change version without impacting txs.Codec

return err
}

Expand All @@ -69,7 +67,7 @@ func parseValidatorMetadata(bytes []byte, metadata *validatorMetadata) error {
metadata.PotentialReward = tmp.PotentialReward
default:
// everything was stored
if _, err := txs.Codec.Unmarshal(bytes, metadata); err != nil {
if _, err := metadataCodec.Unmarshal(bytes, metadata); err != nil {
return err
}
}
Expand Down Expand Up @@ -238,7 +236,7 @@ func (m *metadata) WriteValidatorMetadata(
metadata := m.metadata[vdrID][subnetID]
metadata.LastUpdated = uint64(metadata.lastUpdated.Unix())

metadataBytes, err := genesis.Codec.Marshal(txs.Version, metadata)
metadataBytes, err := metadataCodec.Marshal(v0, metadata)
if err != nil {
return err
}
Expand Down
16 changes: 11 additions & 5 deletions vms/platformvm/state/state.go
Original file line number Diff line number Diff line change
Expand Up @@ -1568,8 +1568,10 @@ func (s *state) loadCurrentValidators() error {
return err
}

potentialRewardBytes := delegatorIt.Value()
potentialReward, err := database.ParseUInt64(potentialRewardBytes)
metadata := &delegatorMetadata{
txID: txID,
}
err = parseDelegatorMetadata(delegatorIt.Value(), metadata)
if err != nil {
return err
}
Expand All @@ -1579,7 +1581,7 @@ func (s *state) loadCurrentValidators() error {
return fmt.Errorf("expected tx type txs.Staker but got %T", tx.Unsigned)
}

staker, err := NewCurrentStaker(txID, stakerTx, potentialReward)
staker, err := NewCurrentStaker(txID, stakerTx, metadata.PotentialReward)
if err != nil {
return err
}
Expand Down Expand Up @@ -2032,7 +2034,7 @@ func (s *state) writeCurrentStakers(updateValidators bool, height uint64) error
PotentialDelegateeReward: 0,
}

metadataBytes, err := block.GenesisCodec.Marshal(block.Version, metadata)
metadataBytes, err := metadataCodec.Marshal(v0, metadata)
if err != nil {
return fmt.Errorf("failed to serialize current validator: %w", err)
}
Expand Down Expand Up @@ -2175,7 +2177,11 @@ func writeCurrentDelegatorDiff(
return fmt.Errorf("failed to increase node weight diff: %w", err)
}

if err := database.PutUInt64(currentDelegatorList, staker.TxID[:], staker.PotentialReward); err != nil {
metadata := &delegatorMetadata{
txID: staker.TxID,
PotentialReward: staker.PotentialReward,
}
if err := writeDelegatorMetadata(currentDelegatorList, metadata); err != nil {
Comment on lines +2180 to +2184
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just encapsulating delegator metadata writing into delegatorMetadata object

return fmt.Errorf("failed to write current delegator to list: %w", err)
}
}
Expand Down