Merge pull request #1074 from k1LoW/refactor-store

Refactor store ( Start adding `secrets:` sections )

book.go

@@ -26,6 +26,7 @@ type book struct {
 	needs                map[string]string
 	runners              map[string]any
 	vars                 map[string]any
+	secrets              []string
 	rawSteps             []map[string]any
 	hostRules            hostRules
 	hostRulesFromOpts    hostRules

dbg.go

@@ -189,7 +189,7 @@ L:
 				_, _ = fmt.Fprintf(os.Stderr, "args required")
 				continue
 			}
-			store := s.parent.store.toMapWithOutFuncs()
+			store := s.parent.store.toMapForDbg()
 			store[storeRootKeyIncluded] = s.parent.included
 			store[storeRootKeyPrevious] = s.parent.store.latest()
 			e, err := Eval(cmd[1], store)
@@ -240,7 +240,7 @@ L:
 				}
 				table.Render()
 			case "variables", "v":
-				store := s.parent.store.toMapWithOutFuncs()
+				store := s.parent.store.toMapForDbg()
 				store[storeRootKeyIncluded] = s.parent.included
 				store[storeRootKeyPrevious] = s.parent.store.latest()
 				keys := lo.Keys(store)

dump_test.go

@@ -78,8 +78,9 @@ func TestDumpRunnerRun(t *testing.T) {
 				stepMap: map[string]map[string]any{
 					"stepkey": {"key": "value"},
 				},
-				vars:   map[string]any{},
-				useMap: true,
+				vars:        map[string]any{},
+				useMap:      true,
+				stepMapKeys: []string{"stepkey", "stepnext"},
 			},
 			"steps",
 			false,
@@ -114,8 +115,9 @@ func TestDumpRunnerRun(t *testing.T) {
 				stepMap: map[string]map[string]any{
 					"0": {"key": "value"},
 				},
-				vars:   map[string]any{},
-				useMap: true,
+				vars:        map[string]any{},
+				useMap:      true,
+				stepMapKeys: []string{"0", "1"},
 			},
 			"steps['0']",
 			false,
@@ -234,8 +236,9 @@ func TestDumpRunnerRunWithOut(t *testing.T) {
 				stepMap: map[string]map[string]any{
 					"stepkey": {"key": "value"},
 				},
-				vars:   map[string]any{},
-				useMap: true,
+				vars:        map[string]any{},
+				useMap:      true,
+				stepMapKeys: []string{"stepkey", "stepnext"},
 			},
 			"steps",
 			false,
@@ -270,8 +273,9 @@ func TestDumpRunnerRunWithOut(t *testing.T) {
 				stepMap: map[string]map[string]any{
 					"0": {"key": "value"},
 				},
-				vars:   map[string]any{},
-				useMap: true,
+				vars:        map[string]any{},
+				useMap:      true,
+				stepMapKeys: []string{"0", "1"},
 			},
 			"steps['0']",
 			false,

go.mod

@@ -45,6 +45,7 @@ require (
 	github.com/k1LoW/grpcstub v0.24.1
 	github.com/k1LoW/grpcurlreq v0.2.1
 	github.com/k1LoW/httpstub v0.17.0
+	github.com/k1LoW/maskedio v0.3.0
 	github.com/k1LoW/protoresolv v0.1.1
 	github.com/k1LoW/repin v0.3.4
 	github.com/k1LoW/sshc/v4 v4.2.0

go.sum

@@ -1028,6 +1028,8 @@ github.com/k1LoW/grpcurlreq v0.2.1 h1:vV5XPcudceX5+DvnEvyDY11E/MkJ01RmYoHO3/8aNj
 github.com/k1LoW/grpcurlreq v0.2.1/go.mod h1:fYHtOo6/5ans9ub1CDRIQQfvo4NyTGwPud+FH/Rnx90=
 github.com/k1LoW/httpstub v0.17.0 h1:VOeyfSriYbHQL1wGk9sfqa8TDpP6Zti/MPT3xcYDtF0=
 github.com/k1LoW/httpstub v0.17.0/go.mod h1:r9Cu3m/G+ILM8jjPebA/P4dXbIH8s8jnPVrhFETAFYA=
+github.com/k1LoW/maskedio v0.3.0 h1:MTtPgMC47v+VhX2i8c/XLMtAlH7E5RFC+oSjh8f22cA=
+github.com/k1LoW/maskedio v0.3.0/go.mod h1:ugf3OzV7imakceGtmkOY13oJsDynL2YVXRcn9VUZEiE=
 github.com/k1LoW/protoresolv v0.1.1 h1:3Q5Sl4bOGyXy9etMZJraF60XqPQBqodFMfGoqBEBwYE=
 github.com/k1LoW/protoresolv v0.1.1/go.mod h1:eVklYuDOPjSNzmbHZiNmXwmXX4OHqscY0MduCGQbSK0=
 github.com/k1LoW/repin v0.3.4 h1:xcNuBBc/ISHUNBzjXNTCux4OYZND5ZMiyz4SrRtpDhg=

http.go

@@ -554,7 +554,7 @@ func (rnr *httpRunner) run(ctx context.Context, r *httpRequest, s *step) error {
 		}
 
 		d[httpStoreCookieKey] = keyMap
-		o.recordToCookie(cookies)
+		o.recordCookie(cookies)
 	} else {
 		d[httpStoreCookieKey] = map[string]http.Cookie{}
 	}

include.go

@@ -188,7 +188,7 @@ func (rnr *includeRunner) run(ctx context.Context, oo *operator, s *step) error
 		}
 		rnr.runResults = append(rnr.runResults, ooo.runResult)
 	}
-	o.record(oo.store.toNormalizedMap())
+	o.record(oo.store.toMapForIncludeRunner())
 	return nil
 }
 

kv_test.go

@@ -78,7 +78,7 @@ func TestRunNWithKV(t *testing.T) {
 	go func() {
 		time.Sleep(50 * time.Millisecond)
 		ops.SetKV("email", "[email protected]")
-		ops.SetKV("map", map[string]interface{}{
+		ops.SetKV("map", map[string]any{
 			"str": "hello",
 			"int": 123,
 		})

operator.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/k1LoW/concgroup"
 	"github.com/k1LoW/donegroup"
+	"github.com/k1LoW/maskedio"
 	"github.com/k1LoW/runn/exprtrace"
 	"github.com/k1LoW/runn/internal/deprecation"
 	"github.com/k1LoW/stopw"
@@ -83,6 +84,7 @@ type operator struct {
 	runResult       *RunResult
 	dbg             *dbg
 	hasRunnerRunner bool
+	maskRule        *maskedio.Rule
 
 	mu sync.Mutex
 }
@@ -372,51 +374,25 @@ func (op *operator) recordNotRun(i int) {
 		return
 	}
 	v := map[string]any{}
-	if op.useMap {
-		op.recordAsMapped(v)
-		return
-	}
-	op.recordAsListed(v)
+	op.store.record(v)
 }
 
 func (op *operator) record(v map[string]any) {
 	if v == nil {
 		v = map[string]any{}
 	}
-	if op.useMap {
-		op.recordAsMapped(v)
-		return
-	}
-	op.recordAsListed(v)
+	op.store.record(v)
 }
 
-func (op *operator) recordAsListed(v map[string]any) {
-	if op.store.loopIndex != nil && *op.store.loopIndex > 0 {
-		// delete values of prevous loop
-		op.store.steps = op.store.steps[:op.store.length()-1]
-	}
-	op.store.recordAsListed(v)
-}
-
-func (op *operator) recordAsMapped(v map[string]any) {
-	if op.store.loopIndex != nil && *op.store.loopIndex > 0 {
-		// delete values of prevous loop
-		op.store.removeLatestAsMapped()
-	}
-	// Get next key
-	k := op.steps[op.store.length()].key
-	op.store.recordAsMapped(k, v)
-}
-
-func (op *operator) recordToLatest(key string, value any) error {
+func (op *operator) recordResultToLatest(v result) error {
 	r := op.Result()
 	r.StepResults = op.StepResults()
 	op.capturers.captureResultByStep(op.trails(), r)
-	return op.store.recordToLatest(key, value)
+	return op.store.recordToLatestStep(storeStepKeyOutcome, v)
 }
 
-func (op *operator) recordToCookie(cookies []*http.Cookie) {
-	op.store.recordToCookie(cookies)
+func (op *operator) recordCookie(cookies []*http.Cookie) {
+	op.store.recordCookie(cookies)
 }
 
 func (op *operator) generateTrail() Trail {
@@ -454,7 +430,7 @@ func New(opts ...Option) (*operator, error) {
 	if err != nil {
 		return nil, err
 	}
-	st := newStore(bk)
+	st := newStore(bk.vars, bk.funcs, bk.secrets, bk.useMap, bk.stepKeys)
 	op := &operator{
 		id:             id,
 		httpRunners:    map[string]*httpRunner{},
@@ -491,6 +467,7 @@ func New(opts ...Option) (*operator, error) {
 		capturers:      bk.capturers,
 		runResult:      newRunResult(bk.desc, bk.labels, bk.path, bk.included, st),
 		dbg:            newDBG(bk.attach),
+		maskRule:       st.maskRule(),
 	}
 
 	if op.debug {
@@ -957,9 +934,6 @@ func (op *operator) run(ctx context.Context) error {
 		select {
 		case <-ctx.Done():
 		case v := <-op.nm.Chan(n.path):
-			if op.store.needsVars == nil {
-				op.store.needsVars = map[string]any{}
-			}
 			if len(v.bindVars) > 0 {
 				op.store.needsVars[k] = v.bindVars
 			} else {
@@ -1205,7 +1179,7 @@ func (op *operator) runInternal(ctx context.Context) (rerr error) {
 		if failed && !force {
 			s.setResult(errStepSkiped)
 			op.recordNotRun(i)
-			if err := op.recordToLatest(storeStepKeyOutcome, resultSkipped); err != nil {
+			if err := op.recordResultToLatest(resultSkipped); err != nil {
 				return err
 			}
 			continue
@@ -1215,18 +1189,18 @@ func (op *operator) runInternal(ctx context.Context) (rerr error) {
 		switch {
 		case errors.Is(errStepSkiped, err):
 			op.recordNotRun(i)
-			if err := op.recordToLatest(storeStepKeyOutcome, resultSkipped); err != nil {
+			if err := op.recordResultToLatest(resultSkipped); err != nil {
 				return err
 			}
 		case err != nil:
 			op.recordNotRun(i)
-			if err := op.recordToLatest(storeStepKeyOutcome, resultFailure); err != nil {
+			if err := op.recordResultToLatest(resultFailure); err != nil {
 				return err
 			}
 			rerr = errors.Join(rerr, err)
 			failed = true
 		default:
-			if err := op.recordToLatest(storeStepKeyOutcome, resultSuccess); err != nil {
+			if err := op.recordResultToLatest(resultSuccess); err != nil {
 				return err
 			}
 		}
@@ -1310,7 +1284,7 @@ func (op *operator) skip() error {
 	for i, s := range op.steps {
 		s.setResult(errStepSkiped)
 		op.recordNotRun(i)
-		if err := op.recordToLatest(storeStepKeyOutcome, resultSkipped); err != nil {
+		if err := op.recordResultToLatest(resultSkipped); err != nil {
 			return err
 		}
 	}

operator_test.go

@@ -970,7 +970,7 @@ func TestShard(t *testing.T) {
 				cmp.AllowUnexported(allow...),
 				cmpopts.IgnoreUnexported(ignore...),
 				cmpopts.IgnoreFields(stopw.Span{}, "ID"),
-				cmpopts.IgnoreFields(operator{}, "id", "concurrency", "mu", "dbg", "needs", "nm"),
+				cmpopts.IgnoreFields(operator{}, "id", "concurrency", "mu", "dbg", "needs", "nm", "maskRule"),
 				cmpopts.IgnoreFields(cdpRunner{}, "ctx", "cancel", "opts", "mu", "operatorID"),
 				cmpopts.IgnoreFields(sshRunner{}, "client", "sess", "stdin", "stdout", "stderr", "operatorID"),
 				cmpopts.IgnoreFields(grpcRunner{}, "mu", "operatorID"),

runbook.go

@@ -44,6 +44,7 @@ type runbook struct {
 	Needs       map[string]string `yaml:"needs,omitempty"`
 	Runners     map[string]any    `yaml:"runners,omitempty"`
 	Vars        map[string]any    `yaml:"vars,omitempty"`
+	Secrets     []string          `yaml:"secrets,omitempty"`
 	Steps       []yaml.MapSlice   `yaml:"steps"`
 	HostRules   yaml.MapSlice     `yaml:"hostRules,omitempty"`
 	Debug       bool              `yaml:"debug,omitempty"`
@@ -65,6 +66,7 @@ type runbookMapped struct {
 	Needs       map[string]string `yaml:"needs,omitempty"`
 	Runners     map[string]any    `yaml:"runners,omitempty"`
 	Vars        map[string]any    `yaml:"vars,omitempty"`
+	Secrets     []string          `yaml:"secrets,omitempty"`
 	Steps       yaml.MapSlice     `yaml:"steps,omitempty"`
 	HostRules   yaml.MapSlice     `yaml:"hostRules,omitempty"`
 	Debug       bool              `yaml:"debug,omitempty"`
@@ -165,6 +167,7 @@ func parseRunbookMapped(b []byte, rb *runbook) error {
 	rb.Needs = m.Needs
 	rb.Runners = m.Runners
 	rb.Vars = m.Vars
+	rb.Secrets = m.Secrets
 	rb.HostRules = m.HostRules
 	rb.Debug = m.Debug
 	rb.Interval = m.Interval
@@ -232,6 +235,7 @@ func (rb *runbook) MarshalYAML() (any, error) {
 	m.Needs = rb.Needs
 	m.Runners = rb.Runners
 	m.Vars = rb.Vars
+	m.Secrets = rb.Secrets
 	m.HostRules = rb.HostRules
 	m.Debug = rb.Debug
 	m.Interval = rb.Interval
@@ -441,6 +445,7 @@ func (rb *runbook) toBook() (*book, error) {
 	if !ok {
 		return nil, fmt.Errorf("failed to normalize vars: %v", rb.Vars)
 	}
+	bk.secrets = rb.Secrets
 	for _, s := range rb.Steps {
 		v, ok := normalize(s).(map[string]any)
 		if !ok {