Feat/cron chillout (#1398)

Cron defer and recycle logic
* Migration
* Add invariants for new state
* Clamp BR at epsilon and small review
* Remove race detection from CI to stop OOMing box

Co-authored-by: ZenGround0 <[email protected]>

.circleci/config.yml

@@ -76,7 +76,6 @@ jobs:
             make test-coverage
             mkdir -p /tmp/artifacts
             mv coverage.out /tmp/artifacts/coverage.out
-            make test-migration
       - codecov/upload:
           file: /tmp/artifacts/coverage.out
       - store_artifacts:

Makefile

@@ -8,11 +8,11 @@ build:
 
 test:
 	$(GO_BIN) test ./...
-	$(GO_BIN) test -race ./actors/migration/nv10/test
+	$(GO_BIN) test -race ./actors/migration/nv12/test
 .PHONY: test
 
 test-migration:
-	$(GO_BIN) test -race ./actors/migration/nv10/test
+	$(GO_BIN) test -race ./actors/migration/nv12/test
 .PHONY: test-migration
 
 test-coverage:

actors/builtin/codes.go

@@ -35,17 +35,17 @@ func init() {
 	builtinActors = make(map[cid.Cid]*actorInfo)
 
 	for id, info := range map[*cid.Cid]*actorInfo{ //nolint:nomaprange
-		&SystemActorCodeID:           {name: "fil/3/system"},
-		&InitActorCodeID:             {name: "fil/3/init"},
-		&CronActorCodeID:             {name: "fil/3/cron"},
-		&StoragePowerActorCodeID:     {name: "fil/3/storagepower"},
-		&StorageMinerActorCodeID:     {name: "fil/3/storageminer"},
-		&StorageMarketActorCodeID:    {name: "fil/3/storagemarket"},
-		&PaymentChannelActorCodeID:   {name: "fil/3/paymentchannel"},
-		&RewardActorCodeID:           {name: "fil/3/reward"},
-		&VerifiedRegistryActorCodeID: {name: "fil/3/verifiedregistry"},
-		&AccountActorCodeID:          {name: "fil/3/account", signer: true},
-		&MultisigActorCodeID:         {name: "fil/3/multisig", signer: true},
+		&SystemActorCodeID:           {name: "fil/4/system"},
+		&InitActorCodeID:             {name: "fil/4/init"},
+		&CronActorCodeID:             {name: "fil/4/cron"},
+		&StoragePowerActorCodeID:     {name: "fil/4/storagepower"},
+		&StorageMinerActorCodeID:     {name: "fil/4/storageminer"},
+		&StorageMarketActorCodeID:    {name: "fil/4/storagemarket"},
+		&PaymentChannelActorCodeID:   {name: "fil/4/paymentchannel"},
+		&RewardActorCodeID:           {name: "fil/4/reward"},
+		&VerifiedRegistryActorCodeID: {name: "fil/4/verifiedregistry"},
+		&AccountActorCodeID:          {name: "fil/4/account", signer: true},
+		&MultisigActorCodeID:         {name: "fil/4/multisig", signer: true},
 	} {
 		c, err := builder.Sum([]byte(info.name))
 		if err != nil {

actors/builtin/miner/deadlines.go

@@ -94,3 +94,13 @@ func deadlineAvailableForCompaction(provingPeriodStart abi.ChainEpoch, dlIdx uin
 	return deadlineIsMutable(provingPeriodStart, dlIdx, currentEpoch) &&
 		!deadlineAvailableForOptimisticPoStDispute(provingPeriodStart, dlIdx, currentEpoch)
 }
+
+// Determine current period start and deadline index directly from current epoch and
+// the offset implied by the proving period. This works correctly even for the state
+// of a miner actor without an active deadline cron
+func NewDeadlineInfoFromOffsetAndEpoch(periodStartSeed abi.ChainEpoch, currEpoch abi.ChainEpoch) *dline.Info {
+	q := NewQuantSpec(WPoStProvingPeriod, periodStartSeed)
+	currentPeriodStart := q.QuantizeDown(currEpoch)
+	currentDeadlineIdx := uint64((currEpoch-currentPeriodStart)/WPoStChallengeWindow) % WPoStPeriodDeadlines
+	return NewDeadlineInfo(currentPeriodStart, currentDeadlineIdx, currEpoch)
+}

actors/builtin/miner/deadlines_test.go

@@ -19,3 +19,68 @@ func TestProvingPeriodDeadlines(t *testing.T) {
 		assert.Equal(t, d.NextNotElapsed().Last(), quant.QuantizeUp(curr))
 	})
 }
+
+func TestDeadlineInfoFromOffsetAndEpoch(t *testing.T) {
+
+	// All proving periods equivalent mod WPoStProving period should give equivalent
+	// dlines for a given epoch. Only the offset property should matter
+	t.Run("Offset and epoch invariant checking", func(t *testing.T) {
+		pp := abi.ChainEpoch(1972)
+		ppThree := abi.ChainEpoch(1972 + 2880*3)
+		ppMillion := abi.ChainEpoch(1972 + 2880*10e6)
+
+		epochs := []abi.ChainEpoch{4, 2000, 400000, 5000000}
+		for _, epoch := range epochs {
+			dlineA := miner.NewDeadlineInfoFromOffsetAndEpoch(pp, epoch)
+			dlineB := miner.NewDeadlineInfoFromOffsetAndEpoch(ppThree, epoch)
+			dlineC := miner.NewDeadlineInfoFromOffsetAndEpoch(ppMillion, epoch)
+
+			assert.Equal(t, *dlineA, *dlineB)
+			assert.Equal(t, *dlineB, *dlineC)
+		}
+	})
+	t.Run("sanity checks", func(t *testing.T) {
+		offset := abi.ChainEpoch(7)
+		start := abi.ChainEpoch(2880*103) + offset
+		// epoch 2880*103 + offset we are in deadline 0, pp start = 2880*103 + offset
+		dline := miner.NewDeadlineInfoFromOffsetAndEpoch(offset, start)
+		assert.Equal(t, uint64(0), dline.Index)
+		assert.Equal(t, start, dline.PeriodStart)
+
+		// epoch 2880*103 + offset + WPoStChallengeWindow - 1 we are in deadline 0
+		dline = miner.NewDeadlineInfoFromOffsetAndEpoch(offset, start+miner.WPoStChallengeWindow-1)
+		assert.Equal(t, uint64(0), dline.Index)
+		assert.Equal(t, start, dline.PeriodStart)
+
+		// epoch 2880*103 + offset + WPoStChallengeWindow we are in deadline 1
+		dline = miner.NewDeadlineInfoFromOffsetAndEpoch(offset, start+miner.WPoStChallengeWindow)
+		assert.Equal(t, uint64(1), dline.Index)
+		assert.Equal(t, start, dline.PeriodStart)
+
+		// epoch 2880*103 + offset + 40*WPoStChallengeWindow we are in deadline 40
+		dline = miner.NewDeadlineInfoFromOffsetAndEpoch(offset, start+40*miner.WPoStChallengeWindow)
+		assert.Equal(t, uint64(40), dline.Index)
+		assert.Equal(t, start, dline.PeriodStart)
+
+		// epoch 2880*103 + offset + 40*WPoStChallengeWindow - 1 we are in deadline 39
+		dline = miner.NewDeadlineInfoFromOffsetAndEpoch(offset, start+40*miner.WPoStChallengeWindow-1)
+		assert.Equal(t, uint64(39), dline.Index)
+		assert.Equal(t, start, dline.PeriodStart)
+
+		// epoch 2880*103 + offset + 40*WPoStChallengeWindow + 1 we are in deadline 40
+		dline = miner.NewDeadlineInfoFromOffsetAndEpoch(offset, start+40*miner.WPoStChallengeWindow+1)
+		assert.Equal(t, uint64(40), dline.Index)
+		assert.Equal(t, start, dline.PeriodStart)
+
+		// epoch 2880*103 + offset + WPoStPeriodDeadlines*WPoStChallengeWindow -1 we are in deadline WPoStPeriodDeadlines - 1
+		dline = miner.NewDeadlineInfoFromOffsetAndEpoch(offset, start+abi.ChainEpoch(miner.WPoStPeriodDeadlines)*miner.WPoStChallengeWindow-abi.ChainEpoch(1))
+		assert.Equal(t, uint64(miner.WPoStPeriodDeadlines-1), dline.Index)
+		assert.Equal(t, start, dline.PeriodStart)
+
+		// epoch 2880*103 + offset + WPoStPeriodDeadlines*WPoStChallengeWindow + 1 we are in deadline 0, pp start = 2880*104 + offset
+		dline = miner.NewDeadlineInfoFromOffsetAndEpoch(offset, start+abi.ChainEpoch(miner.WPoStPeriodDeadlines)*miner.WPoStChallengeWindow)
+		assert.Equal(t, uint64(0), dline.Index)
+		assert.Equal(t, start+miner.WPoStProvingPeriod, dline.PeriodStart)
+
+	})
+}

actors/builtin/miner/miner_actor.go

@@ -126,11 +126,6 @@ func (a Actor) Constructor(rt Runtime, params *ConstructorParams) *abi.EmptyValu
 	builtin.RequireNoErr(rt, err, exitcode.ErrIllegalState, "failed to construct state")
 	rt.StateCreate(state)
 
-	// Register first cron callback for epoch before the next deadline starts.
-	deadlineClose := periodStart + WPoStChallengeWindow*abi.ChainEpoch(1+deadlineIndex)
-	enrollCronEvent(rt, deadlineClose-1, &CronEventPayload{
-		EventType: CronEventProvingDeadline,
-	})
 	return nil
 }
 
@@ -508,7 +503,8 @@ func (a Actor) DisputeWindowedPoSt(rt Runtime, params *DisputeWindowedPoStParams
 	powerDelta := NewPowerPairZero()
 	var st State
 	rt.StateTransaction(&st, func() {
-		if !deadlineAvailableForOptimisticPoStDispute(st.ProvingPeriodStart, params.Deadline, currEpoch) {
+		dlInfo := st.DeadlineInfo(currEpoch)
+		if !deadlineAvailableForOptimisticPoStDispute(dlInfo.PeriodStart, params.Deadline, currEpoch) {
 			rt.Abortf(exitcode.ErrForbidden, "can only dispute window posts during the dispute window (%d epochs after the challenge window closes)", WPoStDisputeWindow)
 		}
 
@@ -519,8 +515,8 @@ func (a Actor) DisputeWindowedPoSt(rt Runtime, params *DisputeWindowedPoStParams
 		// Check proof
 		{
 			// Find the proving period start for the deadline in question.
-			ppStart := st.ProvingPeriodStart
-			if st.CurrentDeadline < params.Deadline {
+			ppStart := dlInfo.PeriodStart
+			if dlInfo.Index < params.Deadline {
 				ppStart -= WPoStProvingPeriod
 			}
 			targetDeadline := NewDeadlineInfo(ppStart, params.Deadline, currEpoch)
@@ -711,6 +707,7 @@ func (a Actor) PreCommitSector(rt Runtime, params *PreCommitSectorParams) *abi.E
 	var err error
 	newlyVested := big.Zero()
 	feeToBurn := abi.NewTokenAmount(0)
+	var needsCron bool
 	rt.StateTransaction(&st, func() {
 		// available balance already accounts for fee debt so it is correct to call
 		// this before RepayDebts. We would have to
@@ -791,12 +788,21 @@ func (a Actor) PreCommitSector(rt Runtime, params *PreCommitSectorParams) *abi.E
 
 		err = st.AddPreCommitExpiry(store, expiryBound, params.SectorNumber)
 		builtin.RequireNoErr(rt, err, exitcode.ErrIllegalState, "failed to add pre-commit expiry to queue")
-	})
 
+		// activate miner cron
+		needsCron = !st.DeadlineCronActive
+		st.DeadlineCronActive = true
+	})
 	burnFunds(rt, feeToBurn)
 	rt.StateReadonly(&st)
 	err = st.CheckBalanceInvariants(rt.CurrentBalance())
 	builtin.RequireNoErr(rt, err, ErrBalanceInvariantBroken, "balance invariants broken")
+	if needsCron {
+		newDlInfo := st.DeadlineInfo(rt.CurrEpoch())
+		enrollCronEvent(rt, newDlInfo.Last(), &CronEventPayload{
+			EventType: CronEventProvingDeadline,
+		})
+	}
 
 	notifyPledgeChanged(rt, newlyVested.Neg())
 
@@ -1331,7 +1337,7 @@ func (a Actor) TerminateSectors(rt Runtime, params *TerminateSectorsParams) *Ter
 		err = toProcess.ForEach(func(dlIdx uint64, partitionSectors PartitionSectorMap) error {
 			// If the deadline the current or next deadline to prove, don't allow terminating sectors.
 			// We assume that deadlines are immutable when being proven.
-			if !deadlineIsMutable(st.ProvingPeriodStart, dlIdx, currEpoch) {
+			if !deadlineIsMutable(st.CurrentProvingPeriodStart(currEpoch), dlIdx, currEpoch) {
 				rt.Abortf(exitcode.ErrIllegalArgument, "cannot terminate sectors in immutable deadline %d", dlIdx)
 			}
 
@@ -1427,8 +1433,9 @@ func (a Actor) DeclareFaults(rt Runtime, params *DeclareFaultsParams) *abi.Empty
 		sectors, err := LoadSectors(store, st.Sectors)
 		builtin.RequireNoErr(rt, err, exitcode.ErrIllegalState, "failed to load sectors array")
 
+		currEpoch := rt.CurrEpoch()
 		err = toProcess.ForEach(func(dlIdx uint64, pm PartitionSectorMap) error {
-			targetDeadline, err := declarationDeadlineInfo(st.ProvingPeriodStart, dlIdx, rt.CurrEpoch())
+			targetDeadline, err := declarationDeadlineInfo(st.CurrentProvingPeriodStart(currEpoch), dlIdx, currEpoch)
 			builtin.RequireNoErr(rt, err, exitcode.ErrIllegalArgument, "invalid fault declaration deadline %d", dlIdx)
 
 			err = validateFRDeclarationDeadline(targetDeadline)
@@ -1516,8 +1523,9 @@ func (a Actor) DeclareFaultsRecovered(rt Runtime, params *DeclareFaultsRecovered
 		sectors, err := LoadSectors(store, st.Sectors)
 		builtin.RequireNoErr(rt, err, exitcode.ErrIllegalState, "failed to load sectors array")
 
+		currEpoch := rt.CurrEpoch()
 		err = toProcess.ForEach(func(dlIdx uint64, pm PartitionSectorMap) error {
-			targetDeadline, err := declarationDeadlineInfo(st.ProvingPeriodStart, dlIdx, rt.CurrEpoch())
+			targetDeadline, err := declarationDeadlineInfo(st.CurrentProvingPeriodStart(currEpoch), dlIdx, currEpoch)
 			builtin.RequireNoErr(rt, err, exitcode.ErrIllegalArgument, "invalid recovery declaration deadline %d", dlIdx)
 			err = validateFRDeclarationDeadline(targetDeadline)
 			builtin.RequireNoErr(rt, err, exitcode.ErrIllegalArgument, "failed recovery declaration at deadline %d", dlIdx)
@@ -1577,7 +1585,7 @@ func (a Actor) CompactPartitions(rt Runtime, params *CompactPartitionsParams) *a
 		info := getMinerInfo(rt, &st)
 		rt.ValidateImmediateCallerIs(append(info.ControlAddresses, info.Owner, info.Worker)...)
 
-		if !deadlineAvailableForCompaction(st.ProvingPeriodStart, params.Deadline, rt.CurrEpoch()) {
+		if !deadlineAvailableForCompaction(st.CurrentProvingPeriodStart(rt.CurrEpoch()), params.Deadline, rt.CurrEpoch()) {
 			rt.Abortf(exitcode.ErrForbidden,
 				"cannot compact deadline %d during its challenge window, or the prior challenge window, or before %d epochs have passed since its last challenge window ended", params.Deadline, WPoStDisputeWindow)
 		}
@@ -2025,6 +2033,7 @@ func handleProvingDeadline(rt Runtime) {
 	penaltyTotal := abi.NewTokenAmount(0)
 	pledgeDeltaTotal := abi.NewTokenAmount(0)
 
+	var continueCron bool
 	var st State
 	rt.StateTransaction(&st, func() {
 		{
@@ -2077,6 +2086,11 @@ func handleProvingDeadline(rt Runtime) {
 			penaltyTotal = big.Add(penaltyFromVesting, penaltyFromBalance)
 			pledgeDeltaTotal = big.Sub(pledgeDeltaTotal, penaltyFromVesting)
 		}
+
+		continueCron = st.ContinueDeadlineCron()
+		if !continueCron {
+			st.DeadlineCronActive = false
+		}
 	})
 
 	// Remove power for new faults, and burn penalties.
@@ -2085,10 +2099,14 @@ func handleProvingDeadline(rt Runtime) {
 	notifyPledgeChanged(rt, pledgeDeltaTotal)
 
 	// Schedule cron callback for next deadline's last epoch.
-	newDlInfo := st.DeadlineInfo(currEpoch)
-	enrollCronEvent(rt, newDlInfo.Last(), &CronEventPayload{
-		EventType: CronEventProvingDeadline,
-	})
+	if continueCron {
+		newDlInfo := st.DeadlineInfo(currEpoch + 1)
+		enrollCronEvent(rt, newDlInfo.Last(), &CronEventPayload{
+			EventType: CronEventProvingDeadline,
+		})
+	} else {
+		rt.Log(rtt.INFO, "miner %s going inactive, deadline cron discontinued", rt.Receiver())
+	}
 
 	// Record whether or not we _have_ early terminations now.
 	hasEarlyTerminations := havePendingEarlyTerminations(rt, &st)