Merge branch 'main' into performance-play

bob/bob.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"runtime"
 
+	nixbuilder "github.com/benchkram/bob/bob/nix-builder"
 	"github.com/benchkram/bob/pkg/auth"
 	"github.com/benchkram/bob/pkg/dockermobyutil"
 	"github.com/benchkram/bob/pkg/envutil"
@@ -61,7 +62,7 @@ type B struct {
 	enablePull bool
 
 	// nix builds dependencies for tasks
-	nix *NixBuilder
+	nix *nixbuilder.NB
 
 	// authStore is used to store authentication credentials for remote store
 	authStore *auth.Store
@@ -128,6 +129,12 @@ func BobWithBaseStoreDir(baseStoreDir string, opts ...Option) (*B, error) {
 	}
 	bob.authStore = authStore
 
+	nixBuilder, err := NixBuilder(baseStoreDir)
+	if err != nil {
+		return nil, err
+	}
+	bob.nix = nixBuilder
+
 	for _, opt := range opts {
 		if opt == nil {
 			continue
@@ -170,7 +177,7 @@ func Bob(opts ...Option) (*B, error) {
 	}
 
 	if bob.nix == nil {
-		nix, err := DefaultNix(bob.envStore)
+		nix, err := DefaultNixBuilder()
 		if err != nil {
 			return nil, err
 		}
@@ -192,7 +199,7 @@ func (b *B) Dir() string {
 	return b.dir
 }
 
-func (b *B) Nix() *NixBuilder {
+func (b *B) Nix() *nixbuilder.NB {
 	return b.nix
 }
 

bob/bob_stores.go → bob/bob_defaults.go

@@ -7,8 +7,10 @@ import (
 	"github.com/benchkram/errz"
 
 	"github.com/benchkram/bob/bob/global"
+	nixbuilder "github.com/benchkram/bob/bob/nix-builder"
 	"github.com/benchkram/bob/pkg/auth"
 	"github.com/benchkram/bob/pkg/buildinfostore"
+	"github.com/benchkram/bob/pkg/nix"
 	"github.com/benchkram/bob/pkg/store"
 	"github.com/benchkram/bob/pkg/store/filestore"
 )
@@ -46,10 +48,10 @@ func DefaultBuildinfoStore() (s buildinfostore.Store, err error) {
 	return BuildinfoStore(home)
 }
 
-func BuildinfoStore(dir string) (s buildinfostore.Store, err error) {
+func BuildinfoStore(baseDir string) (s buildinfostore.Store, err error) {
 	defer errz.Recover(&err)
 
-	storeDir := filepath.Join(dir, global.BobCacheBuildinfoDir)
+	storeDir := filepath.Join(baseDir, global.BobCacheBuildinfoDir)
 	err = os.MkdirAll(storeDir, 0775)
 	errz.Fatal(err)
 
@@ -66,10 +68,10 @@ func (b *B) Localstore() store.Store {
 	return b.local
 }
 
-func AuthStore(dir string) (s *auth.Store, err error) {
+func AuthStore(baseDir string) (s *auth.Store, err error) {
 	defer errz.Recover(&err)
 
-	storeDir := filepath.Join(dir, global.BobAuthStoreDir)
+	storeDir := filepath.Join(baseDir, global.BobAuthStoreDir)
 	err = os.MkdirAll(storeDir, 0775)
 	errz.Fatal(err)
 
@@ -84,3 +86,37 @@ func DefaultAuthStore() (s *auth.Store, err error) {
 
 	return AuthStore(home)
 }
+
+// NixBuilder initialises a new nix builder object with the cache setup
+// in the given location.
+//
+// It's save to use the same base dir as for BuildinfoStore(),
+// Filestore() and AuthStore().
+func NixBuilder(baseDir string) (_ *nixbuilder.NB, err error) {
+
+	cacheDir := filepath.Join(baseDir, global.BobCacheNixFileName)
+
+	err = os.MkdirAll(filepath.Dir(cacheDir), 0775)
+	errz.Fatal(err)
+
+	nixCache, err := nix.NewCacheStore(nix.WithPath(cacheDir))
+	errz.Fatal(err)
+
+	shellCache := nix.NewShellCache(filepath.Join(baseDir, global.BobCacheNixShellCacheDir))
+
+	nb := nixbuilder.New(
+		nixbuilder.WithCache(nixCache),
+		nixbuilder.WithShellCache(shellCache),
+	)
+
+	return nb, nil
+}
+
+func DefaultNixBuilder() (_ *nixbuilder.NB, err error) {
+	defer errz.Recover(&err)
+
+	home, err := os.UserHomeDir()
+	errz.Fatal(err)
+
+	return NixBuilder(home)
+}

bob/build.go

@@ -37,8 +37,6 @@ func (b *B) Build(ctx context.Context, taskName string) (err error) {
 		playbook.WithLocalStore(b.local),
 		playbook.WithPushEnabled(b.enablePush),
 		playbook.WithPullEnabled(b.enablePull),
-		playbook.WitNixCache(b.Nix().cache),
-		playbook.WitNixShellCache(b.Nix().shellCache),
 	)
 	errz.Fatal(err)
 

bob/builder.go

@@ -4,22 +4,23 @@ import (
 	"context"
 
 	"github.com/benchkram/bob/bob/bobfile"
+	nixbuilder "github.com/benchkram/bob/bob/nix-builder"
 	"github.com/benchkram/bob/pkg/ctl"
 )
 
 var _ ctl.Builder = (*builder)(nil)
 
-type BuildFunc func(_ context.Context, runname string, aggregate *bobfile.Bobfile, nix *NixBuilder) error
+type BuildFunc func(_ context.Context, runname string, aggregate *bobfile.Bobfile, nix *nixbuilder.NB) error
 
 // builder holds all dependencies to build a build task
 type builder struct {
 	task      string
 	aggregate *bobfile.Bobfile
 	f         BuildFunc
-	nix       *NixBuilder
+	nix       *nixbuilder.NB
 }
 
-func NewBuilder(task string, aggregate *bobfile.Bobfile, f BuildFunc, nix *NixBuilder) ctl.Builder {
+func NewBuilder(task string, aggregate *bobfile.Bobfile, f BuildFunc, nix *nixbuilder.NB) ctl.Builder {
 	builder := &builder{
 		task:      task,
 		aggregate: aggregate,

bob/nix_builder.go → bob/nix-builder/nix_builder.go

@@ -1,4 +1,4 @@
-package bob
+package nixbuilder
 
 import (
 	"fmt"
@@ -11,9 +11,9 @@ import (
 	"github.com/benchkram/bob/pkg/usererror"
 )
 
-// NixBuilder acts as a wrapper for github.com/benchkram/bob/pkg/nix package
+// NB acts as a wrapper for github.com/benchkram/bob/pkg/nix package
 // and is used for building tasks dependencies
-type NixBuilder struct {
+type NB struct {
 	// cache allows caching the dependency to store path
 	cache *nix.Cache
 
@@ -25,29 +25,29 @@ type NixBuilder struct {
 	envStore envutil.Store
 }
 
-type NixOption func(n *NixBuilder)
+type NixOption func(n *NB)
 
 func WithCache(cache *nix.Cache) NixOption {
-	return func(n *NixBuilder) {
+	return func(n *NB) {
 		n.cache = cache
 	}
 }
 
 func WithShellCache(cache *nix.ShellCache) NixOption {
-	return func(n *NixBuilder) {
+	return func(n *NB) {
 		n.shellCache = cache
 	}
 }
 
 func WithEnvironmentStore(store envutil.Store) NixOption {
-	return func(n *NixBuilder) {
+	return func(n *NB) {
 		n.envStore = store
 	}
 }
 
-// NewNixBuilder instantiates a new Nix builder instance
-func NewNixBuilder(opts ...NixOption) *NixBuilder {
-	n := &NixBuilder{
+// NewNB instantiates a new Nix builder instance
+func New(opts ...NixOption) *NB {
+	n := &NB{
 		envStore: envutil.NewStore(),
 	}
 
@@ -62,7 +62,7 @@ func NewNixBuilder(opts ...NixOption) *NixBuilder {
 }
 
 // BuildNixDependenciesInPipeline collects and builds nix-dependencies for a pipeline starting at taskName.
-func (n *NixBuilder) BuildNixDependenciesInPipeline(ag *bobfile.Bobfile, taskName string) (err error) {
+func (n *NB) BuildNixDependenciesInPipeline(ag *bobfile.Bobfile, taskName string) (err error) {
 	defer errz.Recover(&err)
 
 	if !nix.IsInstalled() {
@@ -77,7 +77,7 @@ func (n *NixBuilder) BuildNixDependenciesInPipeline(ag *bobfile.Bobfile, taskNam
 
 // BuildNixDependencies builds nix dependencies and prepares the affected tasks
 // by setting the store paths on each task in the given aggregate.
-func (n *NixBuilder) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, runTasksInPipeline []string) (err error) {
+func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, runTasksInPipeline []string) (err error) {
 	defer errz.Recover(&err)
 
 	if !nix.IsInstalled() {
@@ -146,11 +146,11 @@ func (n *NixBuilder) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipel
 }
 
 // BuildDependencies builds the list of all nix deps
-func (n *NixBuilder) BuildDependencies(deps []nix.Dependency) error {
+func (n *NB) BuildDependencies(deps []nix.Dependency) error {
 	return nix.BuildDependencies(deps, n.cache)
 }
 
 // BuildEnvironment builds the environment with all nix deps
-func (n *NixBuilder) BuildEnvironment(deps []nix.Dependency, nixpkgs string) (_ []string, err error) {
+func (n *NB) BuildEnvironment(deps []nix.Dependency, nixpkgs string) (_ []string, err error) {
 	return nix.BuildEnvironment(deps, nixpkgs, n.cache, n.shellCache)
 }

bob/options.go

@@ -1,6 +1,7 @@
 package bob
 
 import (
+	nixbuilder "github.com/benchkram/bob/bob/nix-builder"
 	"github.com/benchkram/bob/pkg/auth"
 	"github.com/benchkram/bob/pkg/buildinfostore"
 	"github.com/benchkram/bob/pkg/envutil"
@@ -77,7 +78,7 @@ func WithInsecure(allow bool) Option {
 	}
 }
 
-func WithNixBuilder(nix *NixBuilder) Option {
+func WithNixBuilder(nix *nixbuilder.NB) Option {
 	return func(b *B) {
 		b.nix = nix
 	}

bob/playbook/build_internal.go

@@ -108,7 +108,7 @@ func (p *Playbook) build(ctx context.Context, task *bobtask.Task) (err error) {
 	err = task.Clean()
 	errz.Fatal(err)
 
-	err = task.Run(ctx, p.namePad, p.nixCache, p.nixShellCache)
+	err = task.Run(ctx, p.namePad)
 	if err != nil {
 		taskSuccessFul = false
 		taskErr = err

bob/playbook/options.go

@@ -1,7 +1,6 @@
 package playbook
 
 import (
-	"github.com/benchkram/bob/pkg/nix"
 	"github.com/benchkram/bob/pkg/store"
 )
 
@@ -48,15 +47,3 @@ func WithLocalStore(s store.Store) Option {
 		p.localStore = s
 	}
 }
-
-func WitNixCache(c *nix.Cache) Option {
-	return func(p *Playbook) {
-		p.nixCache = c
-	}
-}
-
-func WitNixShellCache(c *nix.ShellCache) Option {
-	return func(p *Playbook) {
-		p.nixShellCache = c
-	}
-}

bob/playbook/playbook.go

@@ -12,7 +12,6 @@ import (
 	"github.com/benchkram/bob/bobtask/buildinfo"
 	"github.com/benchkram/bob/bobtask/hash"
 	"github.com/benchkram/bob/pkg/boberror"
-	"github.com/benchkram/bob/pkg/nix"
 	"github.com/benchkram/bob/pkg/store"
 	"github.com/benchkram/bob/pkg/usererror"
 	"github.com/benchkram/errz"
@@ -79,12 +78,6 @@ type Playbook struct {
 	// enablePull allows pulling artifacts from remote store
 	enablePull bool
 
-	// nixCache caches the Nix store paths
-	nixCache *nix.Cache
-
-	// nixShellCache caches the nix-shell --command='env' command output
-	nixShellCache *nix.ShellCache
-
 	// oncePrepareOptimizedAccess is used to initalize the optimized
 	// slice to access tasks.
 	oncePrepareOptimizedAccess sync.Once

bob/run.go

@@ -7,6 +7,7 @@ import (
 	"github.com/benchkram/errz"
 
 	"github.com/benchkram/bob/bob/bobfile"
+	nixbuilder "github.com/benchkram/bob/bob/nix-builder"
 	"github.com/benchkram/bob/pkg/boberror"
 	"github.com/benchkram/bob/pkg/ctl"
 	"github.com/benchkram/bob/pkg/sliceutil"
@@ -140,7 +141,7 @@ func executeBuildTasksInPipeline(
 	ctx context.Context,
 	runTaskName string,
 	aggregate *bobfile.Bobfile,
-	nix *NixBuilder,
+	nix *nixbuilder.NB,
 ) (err error) {
 	defer errz.Recover(&err)
 

bobtask/run.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/benchkram/bob/pkg/boblog"
 	"github.com/benchkram/bob/pkg/envutil"
-	"github.com/benchkram/bob/pkg/nix"
 	"github.com/benchkram/bob/pkg/usererror"
 	"github.com/logrusorgru/aurora"
 	"mvdan.cc/sh/expand"
@@ -19,15 +18,9 @@ import (
 	"github.com/benchkram/errz"
 )
 
-func (t *Task) Run(ctx context.Context, namePad int, nixCache *nix.Cache, shellCache *nix.ShellCache) (err error) {
+func (t *Task) Run(ctx context.Context, namePad int) (err error) {
 	defer errz.Recover(&err)
 
-	// if len(t.Env()) == 0 {
-	// 	nixShellEnv, err := nix.BuildEnvironment(t.dependencies, t.nixpkgs, nixCache, shellCache)
-	// 	errz.Fatal(err)
-	// 	t.SetEnv(envutil.Merge(nixShellEnv, t.env))
-	// }
-
 	nixEnv, ok := t.envStore[t.envID]
 	if !ok {
 		return fmt.Errorf("missing nix environment in envStore")