Prover: beta v1.2 integration changes (#692)

* bump go-corset

* fix compile errors

* constraints: bump to v0.1.0-rc1 for beta-v1.2

* bump to latest go-corset

* constraints: bump to beta-v1.2 / v0.1.0-rc2

* bump go-corset

* bump zkevm bin

* use next power of two value for non-power of two size columns (e.g., MMIO)

* remove a check for the power of two size

* bump corset to 9.7.18

* bump zkevm.bin

* bump corset to v9.7.18

* update zkevm.bin

* added interleaved to the compilediop columns

* adjusted size for corset columns

* Prover/Codehash Non Power of Two Column Size (#618)

* Revert "adjusted size for corset columns"

This reverts commit b1a7319.

* fixed bug and added panic message for a non power of two size column

* removing panic

* reinsteaded the panic

---------

Co-authored-by: gusiri <[email protected]>

* adjusted size for corset columns

* constraints: bump to beta v1.2/v0.1.0-rc3

* update constraints version to rc3

* bump to latest go-corset

* apply hotfix for BLOCKDATA

* move NextPowerOfTwo unit test to utils

* add logs for adjusted columns with non-power of two size

* turn off trace version check

* fix golangcli-lint

* Prover/fix public input timestamps from new blockdata (#644)

* updated timestamp fetcher and arithmetization mock data for unit testing.

* fix(codehash): uses 0x0 for the codehash of non-existing accounts instead of the default EOA codehash

* fix(mimccodehash): unimport the rom codehash for initialization code

* fixup(execDataHash): revert the exec-data hash check

* timestamp byte change

* fix(execdatahash): adds the real blockhash in the execdata hash instead of 0x0

* fixup previous commit

* fixup(build): removes imports

* Revert "fixup(execDataHash): revert the exec-data hash check"

This reverts commit eb8d984.

* fix(consistency): adds a smaller size to the consistency module

* feat(mimc): alex -- mimc simplification -- start

* optimize factorExpression

* feat(exec): uses the ProveCheck in the execution proof

* Revert "feat(mimc): alex -- mimc simplification -- start"

This reverts commit 184771b.

* fix (public-input): changed the hashing method to match compression

* perf(mem): adds a detector for constant regular column.

* fixup(mem): support the edge-case for smartvectors of size 1

* fix(codehash): support the case where the ROM is empty

* feat(csv): adds a feature to rename columns when using fmtcsv

* fixup(codehash): supports the case where no codehash are available

* test(codehash): adds test for empty rom or statesummary

* fix(ss-connect): skip the integration connector test

---------

Co-authored-by: gusiri <[email protected]>
Co-authored-by: Soleimani193 <[email protected]>
Co-authored-by: Arijit Dutta <[email protected]>
Co-authored-by: Bogdan Ursu <[email protected]>

prover/backend/execution/constraints-versions.txt

@@ -1,2 +1 @@
-v0.8.0-rc7
-v0.8.0-rc8
+beta-v1.2/v0.1.0-rc3

prover/backend/execution/craft.go

@@ -36,13 +36,13 @@ func CraftProverOutput(
 ) Response {
 
 	// Split the embedded file contents into a string slice
-	constraintsVersions := strings.Split(strings.TrimSpace(constraintsVersionsStr), "\n")
+	// constraintsVersions := strings.Split(strings.TrimSpace(constraintsVersionsStr), "\n")
 
 	// Check the arithmetization version used to generate the trace is contained in the prover request
 	// and fail fast if the constraint version is not supported
-	if err := checkArithmetizationVersion(req.ConflatedExecutionTracesFile, req.TracesEngineVersion, constraintsVersions); err != nil {
-		panic(err.Error())
-	}
+	// if err := checkArithmetizationVersion(req.ConflatedExecutionTracesFile, req.TracesEngineVersion, constraintsVersions); err != nil {
+	// 	panic(err.Error())
+	// }
 
 	var (
 		l2BridgeAddress = cfg.Layer2.MsgSvcContract
@@ -238,6 +238,7 @@ func NewWitness(cfg *config.Config, req *Request, rsp *Response) *Witness {
 			TxHashes:        txHashes,
 			L2BridgeAddress: cfg.Layer2.MsgSvcContract,
 			ChainID:         cfg.Layer2.ChainID,
+			BlockHashList:   getBlockHashList(rsp),
 		},
 		FuncInp: rsp.FuncInput(),
 	}
@@ -286,3 +287,11 @@ func validateAndExtractVersion(traceFileName string) (string, error) {
 	}
 	return "", fmt.Errorf("conflated trace file: %s not in the appropriate format or version not found", traceFileName)
 }
+
+func getBlockHashList(rsp *Response) []types.FullBytes32 {
+	res := []types.FullBytes32{}
+	for i := range rsp.BlocksData {
+		res = append(res, rsp.BlocksData[i].BlockHash)
+	}
+	return res
+}

prover/circuits/execution/circuit.go

@@ -100,17 +100,12 @@ func MakeProof(
 ) string {
 
 	assignment := assign(limits, comp, wproof, funcInputs)
-	witness, err := frontend.NewWitness(&assignment, ecc.BLS12_377.ScalarField())
-	if err != nil {
-		panic(err)
-	}
 
-	proof, err := plonk.Prove(
-		setup.Circuit,
-		setup.ProvingKey,
-		witness,
-		emPlonk.GetNativeProverOptions(ecc.BW6_761.ScalarField(), setup.Circuit.Field()),
+	proof, err := circuits.ProveCheck(
+		&setup,
+		&assignment,
 	)
+
 	if err != nil {
 		panic(err)
 	}

prover/go.mod

@@ -10,10 +10,11 @@ require (
 	github.com/consensys/compress v0.2.5
 	github.com/consensys/gnark v0.11.1-0.20250107100237-2cb190338a01
 	github.com/consensys/gnark-crypto v0.14.1-0.20241217134352-810063550bd4
-	github.com/consensys/go-corset v0.0.0-20241125005324-5cb0c289c021
+	github.com/consensys/go-corset v0.0.0-20250129030528-04c6bba7ccfd
 	github.com/crate-crypto/go-kzg-4844 v1.1.0
 	github.com/dlclark/regexp2 v1.11.2
 	github.com/fxamacker/cbor/v2 v2.7.0
+	github.com/go-playground/assert/v2 v2.2.0
 	github.com/go-playground/validator/v10 v10.22.0
 	github.com/iancoleman/strcase v0.3.0
 	github.com/icza/bitio v1.1.0

prover/go.sum

@@ -100,8 +100,8 @@ github.com/consensys/gnark v0.11.1-0.20250107100237-2cb190338a01 h1:YCHI04nMKFC6
 github.com/consensys/gnark v0.11.1-0.20250107100237-2cb190338a01/go.mod h1:8YNyW/+XsYiLRzROLaj/PSktYO4VAdv6YW1b1P3UsZk=
 github.com/consensys/gnark-crypto v0.14.1-0.20241217134352-810063550bd4 h1:Kp6egjRqKZf4469dfAWqFe6gi3MRs4VvNHmTfEjUlS8=
 github.com/consensys/gnark-crypto v0.14.1-0.20241217134352-810063550bd4/go.mod h1:GMPeN3dUSslNBYJsK3WTjIGd3l0ccfMbcEh/d5knFrc=
-github.com/consensys/go-corset v0.0.0-20241125005324-5cb0c289c021 h1:zAPMHjY72pXmjuyb/niQ816pd+B9RAmZoL/W/f5uJSU=
-github.com/consensys/go-corset v0.0.0-20241125005324-5cb0c289c021/go.mod h1:J64guTfpmfXl4Yk2D7lsWdYg0ilP+N8JWPudP7+sZpA=
+github.com/consensys/go-corset v0.0.0-20250129030528-04c6bba7ccfd h1:GQFLOm7l3hKtngBCuxw7f1NZajizIfWkK1gkUet+kFE=
+github.com/consensys/go-corset v0.0.0-20250129030528-04c6bba7ccfd/go.mod h1:J64guTfpmfXl4Yk2D7lsWdYg0ilP+N8JWPudP7+sZpA=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=

prover/maths/common/smartvectors/smartvectors.go

@@ -233,3 +233,50 @@ func WindowExt(v SmartVector) []fext.Element {
 		panic(fmt.Sprintf("unexpected type %T", v))
 	}
 }
+
+// TryReduceSize detects if the input smart-vector can be reduced to a constant
+// smart-vector. It will only apply over the following types: [Regular].
+func TryReduceSize(v SmartVector) SmartVector {
+
+	switch w := v.(type) {
+	case *Constant, *Rotated, *Pooled, *PaddedCircularWindow:
+		return w
+	case *Regular:
+
+		// to detect if a regular vector can be reduced to a constant, we need to
+		// check if all the values are equals. That's an expensive, so we instead
+		// by comparing values that would be likely to be unequal if it was not a
+		// constant. Also, we need to rule out the case where len(*w) because it
+		// is irrelevant to reducing the size.
+		if len(*w) <= 1 {
+			return w
+		}
+
+		if (*w)[0] != (*w)[1] {
+			return w
+		}
+
+		if (*w)[0] != (*w)[len(*w)-1] {
+			return w
+		}
+
+		if (*w)[0] != (*w)[len(*w)/2] {
+			return w
+		}
+
+		// This is expensive check where we check all the values in the vector
+		// to see if they are all equal. This is not the most efficient way to
+		// detect if a vector is a constant but the only reliable one.
+		for i := range *w {
+			if (*w)[i] != (*w)[0] {
+				return w
+			}
+		}
+
+		return NewConstant((*w)[0], len(*w))
+
+	default:
+		panic(fmt.Sprintf("unexpected type %T", v))
+	}
+
+}

prover/protocol/compiler/globalcs/factoring.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"reflect"
+	"sync"
 
 	"github.com/consensys/linea-monorepo/prover/protocol/accessors"
 	"github.com/consensys/linea-monorepo/prover/protocol/serialization"
@@ -13,12 +14,20 @@ import (
 	"github.com/consensys/linea-monorepo/prover/utils"
 )
 
-// factorExpressionList applies [factorExpression] over a list of expression
+// factorExpressionList applies [factorExpression] over a list of expressions
 func factorExpressionList(comp *wizard.CompiledIOP, exprList []*symbolic.Expression) []*symbolic.Expression {
 	res := make([]*symbolic.Expression, len(exprList))
+	var wg sync.WaitGroup
+
 	for i, expr := range exprList {
-		res[i] = factorExpression(comp, expr)
+		wg.Add(1)
+		go func(i int, expr *symbolic.Expression) {
+			defer wg.Done()
+			res[i] = factorExpression(comp, expr)
+		}(i, expr)
 	}
+
+	wg.Wait()
 	return res
 }
 

prover/protocol/wizard/compiled.go

@@ -166,7 +166,10 @@ func (c *CompiledIOP) InsertCommit(round int, name ifaces.ColID, size int) iface
 //   - if the size of the column is not a power of 2
 //   - if a column using the same name has already been registered
 func (c *CompiledIOP) InsertColumn(round int, name ifaces.ColID, size int, status column.Status) ifaces.Column {
-
+	// Panic if the size is not a power of 2
+	if !utils.IsPowerOfTwo(size) {
+		utils.Panic("Registering column %v with a non power of two size = %v", name, size)
+	}
 	// @alex: this has actually caught a few typos. When wrongly setting an
 	// incorrect but very large size here, it will generate a disproportionate
 	// wizard

prover/protocol/wizard/prover.go

@@ -388,6 +388,10 @@ func (run *ProverRuntime) AssignColumn(name ifaces.ColID, witness ifaces.ColAssi
 		utils.Panic("Witness with non-power of two sizes, should have been caught earlier")
 	}
 
+	// This reduction is a trade-off between runtime and memory. It
+	// costs CPU but can save a significant amount of memory.
+	witness = smartvectors.TryReduceSize(witness)
+
 	// Adds it to the assignments
 	run.Columns.InsertNew(handle.GetColID(), witness)
 }

prover/symbolic/expression.go

@@ -3,6 +3,7 @@ package symbolic
 import (
 	"fmt"
 	"reflect"
+	"sync"
 
 	"github.com/consensys/gnark/frontend"
 	"github.com/consensys/linea-monorepo/prover/maths/common/mempool"
@@ -244,9 +245,17 @@ func (e *Expression) ReconstructBottomUp(
 	// LinComb or Product or PolyEval. This is an intermediate expression.
 	case LinComb, Product, PolyEval:
 		children := make([]*Expression, len(e.Children))
+		var wg sync.WaitGroup
+		wg.Add(len(e.Children))
+
 		for i, c := range e.Children {
-			children[i] = c.ReconstructBottomUp(constructor)
+			go func(i int, c *Expression) {
+				defer wg.Done()
+				children[i] = c.ReconstructBottomUp(constructor)
+			}(i, c)
 		}
+
+		wg.Wait()
 		return constructor(e, children)
 	}
 

prover/symbolic/simplify/cost_stat.go

@@ -2,6 +2,7 @@ package simplify
 
 import (
 	"math/bits"
+	"sync"
 
 	sym "github.com/consensys/linea-monorepo/prover/symbolic"
 )
@@ -21,10 +22,21 @@ func (s *costStats) add(cost costStats) {
 // Returns the cost stats of a boarded expression
 func evaluateCostStat(expr *sym.Expression) (s costStats) {
 	board := expr.Board()
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+
 	for i := 1; i < len(board.Nodes); i++ {
-		s_ := evaluateNodeCosts(board.Nodes[i]...)
-		s.add(s_)
+		wg.Add(1)
+		go func(nodes []sym.Node) {
+			defer wg.Done()
+			s_ := evaluateNodeCosts(nodes...)
+			mu.Lock()
+			s.add(s_)
+			mu.Unlock()
+		}(board.Nodes[i])
 	}
+	wg.Wait()
+
 	return s
 }
 

prover/symbolic/simplify/factor.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"math"
 	"sort"
+	"sync"
 
 	"github.com/consensys/linea-monorepo/prover/maths/field"
 	sym "github.com/consensys/linea-monorepo/prover/symbolic"
@@ -17,7 +18,8 @@ import (
 func factorizeExpression(expr *sym.Expression, iteration int) *sym.Expression {
 	res := expr
 	initEsh := expr.ESHash
-	alreadyWalked := map[field.Element]*sym.Expression{}
+	alreadyWalked := sync.Map{}
+	factorMemo := sync.Map{}
 
 	logrus.Infof("factoring expression : init stats %v", evaluateCostStat(expr))
 
@@ -26,10 +28,9 @@ func factorizeExpression(expr *sym.Expression, iteration int) *sym.Expression {
 		scoreInit := evaluateCostStat(res)
 
 		res = res.ReconstructBottomUp(func(lincomb *sym.Expression, newChildren []*sym.Expression) *sym.Expression {
-
 			// Time save, we reuse the results we got for that particular node.
-			if ret, ok := alreadyWalked[lincomb.ESHash]; ok {
-				return ret
+			if ret, ok := alreadyWalked.Load(lincomb.ESHash); ok {
+				return ret.(*sym.Expression)
 			}
 
 			// Incorporate the new children inside of the expression to account
@@ -55,31 +56,29 @@ func factorizeExpression(expr *sym.Expression, iteration int) *sym.Expression {
 
 				group := findGdChildrenGroup(new)
 
-				logrus.Tracef("found children group: %v\n", group)
-
 				if len(group) < 1 {
-					if k > 0 {
-						logrus.Tracef("finished factoring : %v opportunities", k)
-					}
 					return new
 				}
 
-				logrus.Tracef(
-					"factoring an expression with a set of %v siblings",
-					len(group),
-				)
+				// Memoize the factorLinCompFromGroup result
+				cacheKey := fmt.Sprintf("%v-%v", new.ESHash, group)
 
-				new = factorLinCompFromGroup(new, group)
+				if cachedResult, ok := factorMemo.Load(cacheKey); ok {
+					new = cachedResult.(*sym.Expression)
+
+				} else {
+					new = factorLinCompFromGroup(new, group)
+					factorMemo.Store(cacheKey, new)
+				}
 
 				if len(new.Children) >= prevSize {
-					logrus.Tracef("factorization did not help. stopping")
 					return new
 				}
 
 				prevSize = len(new.Children)
 			}
 
-			logrus.Tracef("finished factoring slow node")
+			alreadyWalked.Store(new.ESHash, new)
 			return new
 		})
 

prover/symbolic/simplify/rmpolyeval.go

@@ -17,20 +17,27 @@ func removePolyEval(e *sym.Expression) *sym.Expression {
 		x := newChildren[0]
 		cs := newChildren[1:]
 
+		if len(cs) == 0 {
+			return oldExpr // Handle edge case where there are no coefficients
+		}
+
 		acc := cs[0]
-		xPowi := x
+
+		// Precompute powers of x
+		powersOfX := make([]*sym.Expression, len(cs))
+		powersOfX[0] = x
+		for i := 1; i < len(cs); i++ {
+			// We don't use the default constructor because it will collapse the
+			// intermediate terms into a single term. The intermediates are useful because
+			// they tell the evaluator to reuse the intermediate terms instead of
+			// computing x^i for every term.
+			powersOfX[i] = sym.NewProduct([]*sym.Expression{powersOfX[i-1], x}, []int{1, 1})
+		}
 
 		for i := 1; i < len(cs); i++ {
 			// Here we want to use the default constructor to ensure that we
 			// will have a merged sum at the end.
-			acc = sym.Add(acc, sym.Mul(xPowi, cs[i]))
-			if i+1 < len(cs) {
-				// We don't use the default construct because it will collapse the
-				// xPowi into a single term. The intermediate are useful because
-				// it tells the evaluator to reuse the intermediate terms instead of
-				// computing x^i for every term.
-				xPowi = sym.NewProduct([]*sym.Expression{xPowi, x}, []int{1, 1})
-			}
+			acc = sym.Add(acc, sym.Mul(powersOfX[i-1], cs[i]))
 		}
 
 		if oldExpr.ESHash != acc.ESHash {