feat(f3): tool for explicit power table generation (#12723)

* feat(f3): tool for explicit power table generation Signed-off-by: Jakub Sztandera <[email protected]> * Fix indent, ignore # lines and empty lines Signed-off-by: Jakub Sztandera <[email protected]> * Fix good list with no power and allow to specify tipset Signed-off-by: Jakub Sztandera <[email protected]> * appease formatter Signed-off-by: Jakub Sztandera <[email protected]> * I'm on my knees linter Signed-off-by: Jakub Sztandera <[email protected]> * Update cmd/lotus-shed/f3.go Co-authored-by: Rod Vagg <[email protected]> * Update cmd/lotus-shed/f3.go Co-authored-by: Rod Vagg <[email protected]> --------- Signed-off-by: Jakub Sztandera <[email protected]> Co-authored-by: Rod Vagg <[email protected]>
filecoin-project · Nov 27, 2024 · a4cd57e · a4cd57e
1 parent 70996d0
commit a4cd57e
Showing 1 changed file with 175 additions and 0 deletions.
diff --git a/cmd/lotus-shed/f3.go b/cmd/lotus-shed/f3.go
@@ -1,14 +1,25 @@
 package main
 
 import (
+	"bufio"
 	"context"
+	"encoding/json"
 	"fmt"
+	"math/rand"
+	"os"
+	"sort"
+	"strconv"
+	"time"
 
 	"github.com/ipfs/go-datastore"
 	dsq "github.com/ipfs/go-datastore/query"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
+	"github.com/filecoin-project/go-f3/gpbft"
+
+	lcli "github.com/filecoin-project/lotus/cli"
+	cliutil "github.com/filecoin-project/lotus/cli/util"
 	"github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -17,6 +28,170 @@ var f3Cmd = &cli.Command{
 	Description: "f3 related commands",
 	Subcommands: []*cli.Command{
 		f3ClearStateCmd,
+		f3GenExplicitPower,
+	},
+}
+
+func loadF3IDList(path string) ([]gpbft.ActorID, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %w", err)
+	}
+	defer file.Close() //nolint:errcheck
+
+	var ids []gpbft.ActorID
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" || line[0] == '#' {
+			continue
+		}
+		id, err := strconv.ParseUint(line, 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse ID: %w", err)
+		}
+
+		ids = append(ids, gpbft.ActorID(id))
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("error reading file: %w", err)
+	}
+
+	return ids, nil
+}
+
+var f3GenExplicitPower = &cli.Command{
+	Name:        "gen-explicit-power",
+	Description: "generates an explicit power table",
+
+	Flags: []cli.Flag{
+		&cli.PathFlag{
+			Name:  "good-list",
+			Usage: "new line delimited file with known good IDs to be included",
+		},
+		&cli.PathFlag{
+			Name:  "bad-list",
+			Usage: "new line delimited file with known bad IDs to be excluded",
+		},
+		&cli.IntFlag{
+			Name:  "n",
+			Usage: "generate N entries, exclusive with ratio",
+		},
+		&cli.Float64Flag{
+			Name:  "ratio",
+			Usage: "generate given ratio of full power table, exclusive with N",
+		},
+		&cli.Int64Flag{
+			Name:  "seed",
+			Usage: "seed for randomization, -1 will use current nano time",
+			Value: -1,
+		},
+		&cli.Uint64Flag{
+			Name:  "iteration",
+			Usage: "the iteration of randomization, random entries will be exclusive across iterations",
+			Value: 0,
+		},
+		&cli.StringFlag{
+			Name:  "tipset",
+			Usage: "specify tipset to call method on (pass comma separated array of cids) or @epoch",
+		},
+	},
+
+	Action: func(cctx *cli.Context) error {
+		ctx := cliutil.ReqContext(cctx)
+		api, closer, err := cliutil.GetFullNodeAPIV1(cctx)
+		if err != nil {
+			return fmt.Errorf("getting api: %w", err)
+		}
+		defer closer()
+
+		ts, err := lcli.LoadTipSet(ctx, cctx, api)
+		if err != nil {
+			return fmt.Errorf("getting chain head: %w", err)
+		}
+		if cctx.IsSet("N") && cctx.IsSet("ratio") {
+			return fmt.Errorf("N and ratio options are exclusive")
+		}
+
+		allPowerEntries, err := api.F3GetECPowerTable(ctx, ts.Key())
+		if err != nil {
+			return fmt.Errorf("getting power entries: %w", err)
+		}
+
+		powerMap := map[gpbft.ActorID]gpbft.PowerEntry{}
+		for _, pe := range allPowerEntries {
+			powerMap[pe.ID] = pe
+		}
+		var goodList []gpbft.ActorID
+		if goodPath := cctx.Path("good-list"); goodPath != "" {
+			goodList, err = loadF3IDList(goodPath)
+			if err != nil {
+				return fmt.Errorf("loading good list: %w", err)
+			}
+		}
+
+		var badList []gpbft.ActorID
+		if badPath := cctx.Path("bad-list"); badPath != "" {
+			badList, err = loadF3IDList(badPath)
+			if err != nil {
+				return fmt.Errorf("loading bad list: %w", err)
+			}
+		}
+		total := len(powerMap)
+		for _, id := range badList {
+			delete(powerMap, id)
+		}
+
+		var result gpbft.PowerEntries
+		add := func(id gpbft.ActorID) {
+			result = append(result, powerMap[id])
+			delete(powerMap, id)
+		}
+
+		for _, id := range goodList {
+			if _, ok := powerMap[id]; ok {
+				add(id)
+			}
+		}
+
+		seed := cctx.Int64("seed")
+		if seed == -1 {
+			seed = time.Now().UnixNano()
+		}
+		rng := rand.New(rand.NewSource(seed))
+
+		endSize := cctx.Int("N")
+		if cctx.IsSet("ratio") {
+			endSize = int(float64(total) * cctx.Float64("ratio"))
+		}
+		if toAdd := endSize - len(result); toAdd > 0 {
+			var powerList gpbft.PowerEntries
+			for _, pe := range powerMap {
+				powerList = append(powerList, pe)
+			}
+			rng.Shuffle(len(powerList), powerList.Swap)
+
+			iteration := cctx.Int("iteration")
+			startIdx := min(toAdd*iteration, len(powerList))
+			endIdx := min(toAdd*(iteration+1), len(powerList))
+			result = append(result, powerList[startIdx:endIdx]...)
+		}
+
+		if len(result) > endSize {
+			result = result[:endSize]
+		}
+		sort.Sort(result)
+		res, err := json.MarshalIndent(result, "  ", "  ")
+		if err != nil {
+			return fmt.Errorf("marshalling to json: %w", err)
+		}
+		_, err = cctx.App.Writer.Write(res)
+		if err != nil {
+			return fmt.Errorf("writing result: %w", err)
+		}
+
+		return nil
 	},
 }