From a4cd57eff3cccdba88219eb06fa3d556ad8c02ed Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 27 Nov 2024 12:27:50 +0100 Subject: [PATCH] feat(f3): tool for explicit power table generation (#12723) * feat(f3): tool for explicit power table generation Signed-off-by: Jakub Sztandera * Fix indent, ignore # lines and empty lines Signed-off-by: Jakub Sztandera * Fix good list with no power and allow to specify tipset Signed-off-by: Jakub Sztandera * appease formatter Signed-off-by: Jakub Sztandera * I'm on my knees linter Signed-off-by: Jakub Sztandera * Update cmd/lotus-shed/f3.go Co-authored-by: Rod Vagg * Update cmd/lotus-shed/f3.go Co-authored-by: Rod Vagg --------- Signed-off-by: Jakub Sztandera Co-authored-by: Rod Vagg --- cmd/lotus-shed/f3.go | 175 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/cmd/lotus-shed/f3.go b/cmd/lotus-shed/f3.go index dfe49ed3459..e1169a798f9 100644 --- a/cmd/lotus-shed/f3.go +++ b/cmd/lotus-shed/f3.go @@ -1,14 +1,25 @@ package main import ( + "bufio" "context" + "encoding/json" "fmt" + "math/rand" + "os" + "sort" + "strconv" + "time" "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" "github.com/urfave/cli/v2" "golang.org/x/xerrors" + "github.com/filecoin-project/go-f3/gpbft" + + lcli "github.com/filecoin-project/lotus/cli" + cliutil "github.com/filecoin-project/lotus/cli/util" "github.com/filecoin-project/lotus/node/repo" ) @@ -17,6 +28,170 @@ var f3Cmd = &cli.Command{ Description: "f3 related commands", Subcommands: []*cli.Command{ f3ClearStateCmd, + f3GenExplicitPower, + }, +} + +func loadF3IDList(path string) ([]gpbft.ActorID, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open file: %w", err) + } + defer file.Close() //nolint:errcheck + + var ids []gpbft.ActorID + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if line == "" || line[0] == '#' { + continue + } + id, err := strconv.ParseUint(line, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse ID: %w", err) + } + + ids = append(ids, gpbft.ActorID(id)) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading file: %w", err) + } + + return ids, nil +} + +var f3GenExplicitPower = &cli.Command{ + Name: "gen-explicit-power", + Description: "generates an explicit power table", + + Flags: []cli.Flag{ + &cli.PathFlag{ + Name: "good-list", + Usage: "new line delimited file with known good IDs to be included", + }, + &cli.PathFlag{ + Name: "bad-list", + Usage: "new line delimited file with known bad IDs to be excluded", + }, + &cli.IntFlag{ + Name: "n", + Usage: "generate N entries, exclusive with ratio", + }, + &cli.Float64Flag{ + Name: "ratio", + Usage: "generate given ratio of full power table, exclusive with N", + }, + &cli.Int64Flag{ + Name: "seed", + Usage: "seed for randomization, -1 will use current nano time", + Value: -1, + }, + &cli.Uint64Flag{ + Name: "iteration", + Usage: "the iteration of randomization, random entries will be exclusive across iterations", + Value: 0, + }, + &cli.StringFlag{ + Name: "tipset", + Usage: "specify tipset to call method on (pass comma separated array of cids) or @epoch", + }, + }, + + Action: func(cctx *cli.Context) error { + ctx := cliutil.ReqContext(cctx) + api, closer, err := cliutil.GetFullNodeAPIV1(cctx) + if err != nil { + return fmt.Errorf("getting api: %w", err) + } + defer closer() + + ts, err := lcli.LoadTipSet(ctx, cctx, api) + if err != nil { + return fmt.Errorf("getting chain head: %w", err) + } + if cctx.IsSet("N") && cctx.IsSet("ratio") { + return fmt.Errorf("N and ratio options are exclusive") + } + + allPowerEntries, err := api.F3GetECPowerTable(ctx, ts.Key()) + if err != nil { + return fmt.Errorf("getting power entries: %w", err) + } + + powerMap := map[gpbft.ActorID]gpbft.PowerEntry{} + for _, pe := range allPowerEntries { + powerMap[pe.ID] = pe + } + var goodList []gpbft.ActorID + if goodPath := cctx.Path("good-list"); goodPath != "" { + goodList, err = loadF3IDList(goodPath) + if err != nil { + return fmt.Errorf("loading good list: %w", err) + } + } + + var badList []gpbft.ActorID + if badPath := cctx.Path("bad-list"); badPath != "" { + badList, err = loadF3IDList(badPath) + if err != nil { + return fmt.Errorf("loading bad list: %w", err) + } + } + total := len(powerMap) + for _, id := range badList { + delete(powerMap, id) + } + + var result gpbft.PowerEntries + add := func(id gpbft.ActorID) { + result = append(result, powerMap[id]) + delete(powerMap, id) + } + + for _, id := range goodList { + if _, ok := powerMap[id]; ok { + add(id) + } + } + + seed := cctx.Int64("seed") + if seed == -1 { + seed = time.Now().UnixNano() + } + rng := rand.New(rand.NewSource(seed)) + + endSize := cctx.Int("N") + if cctx.IsSet("ratio") { + endSize = int(float64(total) * cctx.Float64("ratio")) + } + if toAdd := endSize - len(result); toAdd > 0 { + var powerList gpbft.PowerEntries + for _, pe := range powerMap { + powerList = append(powerList, pe) + } + rng.Shuffle(len(powerList), powerList.Swap) + + iteration := cctx.Int("iteration") + startIdx := min(toAdd*iteration, len(powerList)) + endIdx := min(toAdd*(iteration+1), len(powerList)) + result = append(result, powerList[startIdx:endIdx]...) + } + + if len(result) > endSize { + result = result[:endSize] + } + sort.Sort(result) + res, err := json.MarshalIndent(result, " ", " ") + if err != nil { + return fmt.Errorf("marshalling to json: %w", err) + } + _, err = cctx.App.Writer.Write(res) + if err != nil { + return fmt.Errorf("writing result: %w", err) + } + + return nil }, }