diff --git a/solver/llbsolver/ops/file.go b/solver/llbsolver/ops/file.go new file mode 100644 index 0000000000000..0579cac2aa7d2 --- /dev/null +++ b/solver/llbsolver/ops/file.go @@ -0,0 +1,258 @@ +package ops + +import ( + "context" + "fmt" + "sync" + + "github.com/moby/buildkit/solver/llbsolver/ops/fileoptypes" + "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/flightcontrol" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +func NewFileOpSolver(b fileoptypes.Backend, r fileoptypes.RefManager) *FileOpSolver { + return &FileOpSolver{ + b: b, + r: r, + outs: map[int]int{}, + ins: map[int]input{}, + } +} + +type FileOpSolver struct { + b fileoptypes.Backend + r fileoptypes.RefManager + + mu sync.Mutex + outs map[int]int + ins map[int]input + g flightcontrol.Group +} + +type input struct { + requiresCommit bool + mount fileoptypes.Mount + ref fileoptypes.Ref +} + +func (s *FileOpSolver) Solve(ctx context.Context, inputs []fileoptypes.Ref, actions []*pb.FileAction) ([]fileoptypes.Ref, error) { + for i, a := range actions { + if int(a.Input) < -1 || int(a.Input) >= len(inputs)+len(actions) { + return nil, errors.Errorf("invalid input index %d, %d provided", a.Input, len(inputs)) + } + if int(a.SecondaryInput) < -1 || int(a.SecondaryInput) >= len(inputs)+len(actions) { + return nil, errors.Errorf("invalid secondary input index %d, %d provided", a.Input, len(inputs)) + } + + inp, ok := s.ins[int(a.Input)] + if ok { + inp.requiresCommit = true + } + s.ins[int(a.Input)] = inp + + inp, ok = s.ins[int(a.SecondaryInput)] + if ok { + inp.requiresCommit = true + } + s.ins[int(a.SecondaryInput)] = inp + + if a.Output != -1 { + if _, ok := s.outs[int(a.Output)]; ok { + return nil, errors.Errorf("duplicate output %d", a.Output) + } + idx := len(inputs) + i + s.outs[int(a.Output)] = idx + s.ins[idx] = input{requiresCommit: true} + } + } + + if len(s.outs) == 0 { + return nil, errors.Errorf("no outputs specified") + } + + for i := 0; i < len(s.outs); i++ { + if _, ok := s.outs[i]; !ok { + return nil, errors.Errorf("missing output index %d", i) + } + } + + outs := make([]fileoptypes.Ref, len(s.outs)) + + eg, ctx := errgroup.WithContext(ctx) + for i, idx := range s.outs { + func(i, idx int) { + eg.Go(func() error { + if err := s.validate(idx, inputs, actions, nil); err != nil { + return err + } + inp, err := s.getInput(ctx, idx, inputs, actions) + if err != nil { + return err + } + outs[i] = inp.ref + return nil + }) + }(i, idx) + } + + if err := eg.Wait(); err != nil { + return nil, err + } + + return outs, nil +} + +func (s *FileOpSolver) validate(idx int, inputs []fileoptypes.Ref, actions []*pb.FileAction, loaded []int) error { + for _, check := range loaded { + if idx == check { + return errors.Errorf("loop from index %d", idx) + } + } + if idx < len(inputs) { + return nil + } + loaded = append(loaded, idx) + action := actions[idx-len(inputs)] + for _, inp := range []int{int(action.Input), int(action.SecondaryInput)} { + if err := s.validate(inp, inputs, actions, loaded); err != nil { + return err + } + } + return nil +} + +func (s *FileOpSolver) getInput(ctx context.Context, idx int, inputs []fileoptypes.Ref, actions []*pb.FileAction) (input, error) { + inp, err := s.g.Do(ctx, fmt.Sprintf("inp-%d", idx), func(ctx context.Context) (interface{}, error) { + s.mu.Lock() + inp := s.ins[idx] + s.mu.Unlock() + if inp.mount != nil || inp.ref != nil { + return inp, nil + } + + if idx < len(inputs) { + inp.ref = inputs[idx] + s.mu.Lock() + s.ins[idx] = inp + s.mu.Unlock() + return inp, nil + } + + var inpMount, inpMountSecondary fileoptypes.Mount + action := actions[idx-len(inputs)] + + loadInput := func(ctx context.Context) func() error { + return func() error { + inp, err := s.getInput(ctx, int(action.Input), inputs, actions) + if err != nil { + return err + } + if inp.ref != nil { + m, err := s.r.Prepare(ctx, inp.ref, false) + if err != nil { + return err + } + inpMount = m + return nil + } + inpMount = inp.mount + return nil + } + } + + loadSecondaryInput := func(ctx context.Context) func() error { + return func() error { + inp, err := s.getInput(ctx, int(action.SecondaryInput), inputs, actions) + if err != nil { + return err + } + if inp.ref != nil { + m, err := s.r.Prepare(ctx, inp.ref, true) + if err != nil { + return err + } + inpMountSecondary = m + return nil + } + inpMountSecondary = inp.mount + return nil + } + } + + if action.Input != -1 && action.SecondaryInput != -1 { + eg, ctx := errgroup.WithContext(ctx) + eg.Go(loadInput(ctx)) + eg.Go(loadSecondaryInput(ctx)) + if err := eg.Wait(); err != nil { + return nil, err + } + } else { + if action.Input != -1 { + if err := loadInput(ctx)(); err != nil { + return nil, err + } + } + if action.SecondaryInput != -1 { + if err := loadSecondaryInput(ctx)(); err != nil { + return nil, err + } + } + } + + if inpMount == nil { + m, err := s.r.Prepare(ctx, nil, false) + if err != nil { + return nil, err + } + inpMount = m + } + + switch a := action.Action.(type) { + case *pb.FileAction_Mkdir: + if err := s.b.Mkdir(ctx, inpMount, *a.Mkdir); err != nil { + return nil, err + } + case *pb.FileAction_Mkfile: + if err := s.b.Mkfile(ctx, inpMount, *a.Mkfile); err != nil { + return nil, err + } + case *pb.FileAction_Rm: + if err := s.b.Rm(ctx, inpMount, *a.Rm); err != nil { + return nil, err + } + case *pb.FileAction_Copy: + if inpMountSecondary == nil { + m, err := s.r.Prepare(ctx, nil, true) + if err != nil { + return nil, err + } + inpMountSecondary = m + } + if err := s.b.Copy(ctx, inpMountSecondary, inpMount, *a.Copy); err != nil { + return nil, err + } + default: + return nil, errors.Errorf("invalid action type %T", action.Action) + } + + if inp.requiresCommit { + ref, err := s.r.Commit(ctx, inpMount) + if err != nil { + return nil, err + } + inp.ref = ref + } else { + inp.mount = inpMount + } + s.mu.Lock() + s.ins[idx] = inp + s.mu.Unlock() + return inp, nil + }) + if err != nil { + return input{}, err + } + return inp.(input), err +} diff --git a/solver/llbsolver/ops/file_test.go b/solver/llbsolver/ops/file_test.go new file mode 100644 index 0000000000000..5859499aee3d7 --- /dev/null +++ b/solver/llbsolver/ops/file_test.go @@ -0,0 +1,468 @@ +package ops + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/moby/buildkit/solver/llbsolver/ops/fileoptypes" + "github.com/moby/buildkit/solver/pb" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestMkdirMkfile(t *testing.T) { + fo := &pb.FileOp{ + Actions: []*pb.FileAction{ + { + Input: 0, + SecondaryInput: -1, + Output: -1, + Action: &pb.FileAction_Mkdir{ + Mkdir: &pb.FileActionMkDir{ + Path: "/foo/bar", + MakeParents: true, + Mode: 0700, + }, + }, + }, + { + Input: 1, + SecondaryInput: -1, + Output: 0, + Action: &pb.FileAction_Mkfile{ + Mkfile: &pb.FileActionMkFile{ + Path: "/foo/bar/baz", + Mode: 0700, + }, + }, + }, + }, + } + + s := newTestFileSolver() + inp := newTestRef("ref1") + outs, err := s.Solve(context.TODO(), []fileoptypes.Ref{inp}, fo.Actions) + require.NoError(t, err) + require.Equal(t, len(outs), 1) + + o := outs[0].(*testFileRef) + require.Equal(t, "mount-ref1-mkdir-mkfile-commit", o.id) + require.Equal(t, 2, len(o.mount.chain)) + require.Equal(t, fo.Actions[0].Action.(*pb.FileAction_Mkdir).Mkdir, o.mount.chain[0].mkdir) + require.Equal(t, fo.Actions[1].Action.(*pb.FileAction_Mkfile).Mkfile, o.mount.chain[1].mkfile) +} + +func TestInvalidNoOutput(t *testing.T) { + fo := &pb.FileOp{ + Actions: []*pb.FileAction{ + { + Input: 0, + SecondaryInput: -1, + Output: -1, + Action: &pb.FileAction_Mkdir{ + Mkdir: &pb.FileActionMkDir{ + Path: "/foo/bar", + MakeParents: true, + Mode: 0700, + }, + }, + }, + }, + } + + s := newTestFileSolver() + _, err := s.Solve(context.TODO(), []fileoptypes.Ref{}, fo.Actions) + require.Error(t, err) + require.Contains(t, err.Error(), "no outputs specified") +} + +func TestInvalidDuplicateOutput(t *testing.T) { + fo := &pb.FileOp{ + Actions: []*pb.FileAction{ + { + Input: 0, + SecondaryInput: -1, + Output: 0, + Action: &pb.FileAction_Mkdir{ + Mkdir: &pb.FileActionMkDir{ + Path: "/foo/bar", + MakeParents: true, + Mode: 0700, + }, + }, + }, + { + Input: 1, + SecondaryInput: -1, + Output: 0, + Action: &pb.FileAction_Mkfile{ + Mkfile: &pb.FileActionMkFile{ + Path: "/foo/bar/baz", + Mode: 0700, + }, + }, + }, + }, + } + + s := newTestFileSolver() + _, err := s.Solve(context.TODO(), []fileoptypes.Ref{}, fo.Actions) + require.Error(t, err) + require.Contains(t, err.Error(), "duplicate output") +} + +func TestActionInvalidIndex(t *testing.T) { + fo := &pb.FileOp{ + Actions: []*pb.FileAction{ + { + Input: 0, + SecondaryInput: -1, + Output: 0, + Action: &pb.FileAction_Mkdir{ + Mkdir: &pb.FileActionMkDir{ + Path: "/foo/bar", + MakeParents: true, + Mode: 0700, + }, + }, + }, + }, + } + + s := newTestFileSolver() + _, err := s.Solve(context.TODO(), []fileoptypes.Ref{}, fo.Actions) + require.Error(t, err) + require.Contains(t, err.Error(), "loop from index") +} + +func TestActionLoop(t *testing.T) { + fo := &pb.FileOp{ + Actions: []*pb.FileAction{ + { + Input: 1, + SecondaryInput: -1, + Output: -1, + Action: &pb.FileAction_Mkdir{ + Mkdir: &pb.FileActionMkDir{ + Path: "/foo/bar", + MakeParents: true, + Mode: 0700, + }, + }, + }, + { + Input: 0, + SecondaryInput: -1, + Output: 0, + Action: &pb.FileAction_Mkfile{ + Mkfile: &pb.FileActionMkFile{ + Path: "/foo/bar/baz", + Mode: 0700, + }, + }, + }, + }, + } + + s := newTestFileSolver() + _, err := s.Solve(context.TODO(), []fileoptypes.Ref{}, fo.Actions) + require.Error(t, err) + require.Contains(t, err.Error(), "loop from index") +} + +func TestMultiOutput(t *testing.T) { + fo := &pb.FileOp{ + Actions: []*pb.FileAction{ + { + Input: 0, + SecondaryInput: -1, + Output: 0, + Action: &pb.FileAction_Mkdir{ + Mkdir: &pb.FileActionMkDir{ + Path: "/foo/bar", + MakeParents: true, + Mode: 0700, + }, + }, + }, + { + Input: 1, + SecondaryInput: -1, + Output: 1, + Action: &pb.FileAction_Mkfile{ + Mkfile: &pb.FileActionMkFile{ + Path: "/foo/bar/baz", + Mode: 0700, + }, + }, + }, + }, + } + + s := newTestFileSolver() + inp := newTestRef("ref1") + outs, err := s.Solve(context.TODO(), []fileoptypes.Ref{inp}, fo.Actions) + require.NoError(t, err) + require.Equal(t, len(outs), 2) + + o := outs[0].(*testFileRef) + require.Equal(t, "mount-ref1-mkdir-commit", o.id) + require.Equal(t, 1, len(o.mount.chain)) + require.Equal(t, fo.Actions[0].Action.(*pb.FileAction_Mkdir).Mkdir, o.mount.chain[0].mkdir) + + o = outs[1].(*testFileRef) + require.Equal(t, "mount-ref1-mkdir-mkfile-commit", o.id) + require.Equal(t, 2, len(o.mount.chain)) + require.Equal(t, fo.Actions[0].Action.(*pb.FileAction_Mkdir).Mkdir, o.mount.chain[0].mkdir) + require.Equal(t, fo.Actions[1].Action.(*pb.FileAction_Mkfile).Mkfile, o.mount.chain[1].mkfile) +} + +func TestFileFromScratch(t *testing.T) { + fo := &pb.FileOp{ + Actions: []*pb.FileAction{ + { + Input: -1, + SecondaryInput: -1, + Output: -1, + Action: &pb.FileAction_Mkdir{ + Mkdir: &pb.FileActionMkDir{ + Path: "/foo/bar", + MakeParents: true, + Mode: 0700, + }, + }, + }, + { + Input: 0, + SecondaryInput: -1, + Output: 0, + Action: &pb.FileAction_Mkfile{ + Mkfile: &pb.FileActionMkFile{ + Path: "/foo/bar/baz", + Mode: 0700, + }, + }, + }, + }, + } + + s := newTestFileSolver() + outs, err := s.Solve(context.TODO(), []fileoptypes.Ref{}, fo.Actions) + require.NoError(t, err) + require.Equal(t, len(outs), 1) + + o := outs[0].(*testFileRef) + + require.Equal(t, "scratch-mkdir-mkfile-commit", o.id) + require.Equal(t, 2, len(o.mount.chain)) + require.Equal(t, fo.Actions[0].Action.(*pb.FileAction_Mkdir).Mkdir, o.mount.chain[0].mkdir) + require.Equal(t, fo.Actions[1].Action.(*pb.FileAction_Mkfile).Mkfile, o.mount.chain[1].mkfile) +} + +func TestFileCopyInputRm(t *testing.T) { + fo := &pb.FileOp{ + Actions: []*pb.FileAction{ + { + Input: 0, + SecondaryInput: -1, + Output: -1, + Action: &pb.FileAction_Mkdir{ + Mkdir: &pb.FileActionMkDir{ + Path: "/foo/bar", + MakeParents: true, + Mode: 0700, + }, + }, + }, + { + Input: 1, + SecondaryInput: 2, + Output: -1, + Action: &pb.FileAction_Copy{ + Copy: &pb.FileActionCopy{ + Src: "/src", + Dest: "/dest", + }, + }, + }, + { + Input: 3, + SecondaryInput: -1, + Output: 0, + Action: &pb.FileAction_Rm{ + Rm: &pb.FileActionRm{ + Path: "/foo/bar/baz", + }, + }, + }, + }, + } + + s := newTestFileSolver() + inp0 := newTestRef("srcref") + inp1 := newTestRef("destref") + outs, err := s.Solve(context.TODO(), []fileoptypes.Ref{inp0, inp1}, fo.Actions) + require.NoError(t, err) + require.Equal(t, len(outs), 1) + + o := outs[0].(*testFileRef) + require.Equal(t, "mount-destref-copy(mount-srcref-mkdir)-rm-commit", o.id) + require.Equal(t, 2, len(o.mount.chain)) + require.Equal(t, fo.Actions[0].Action.(*pb.FileAction_Mkdir).Mkdir, o.mount.chain[0].copySrc[0].mkdir) + require.Equal(t, fo.Actions[1].Action.(*pb.FileAction_Copy).Copy, o.mount.chain[0].copy) + require.Equal(t, fo.Actions[2].Action.(*pb.FileAction_Rm).Rm, o.mount.chain[1].rm) +} + +func TestFileParallelActions(t *testing.T) { + // two mkdirs from scratch copied over each other. mkdirs should happen in parallel + fo := &pb.FileOp{ + Actions: []*pb.FileAction{ + { + Input: 0, + SecondaryInput: -1, + Output: -1, + Action: &pb.FileAction_Mkdir{ + Mkdir: &pb.FileActionMkDir{ + Path: "/foo", + }, + }, + }, + { + Input: 0, + SecondaryInput: -1, + Output: -1, + Action: &pb.FileAction_Mkdir{ + Mkdir: &pb.FileActionMkDir{ + Path: "/bar", + }, + }, + }, + { + Input: 2, + SecondaryInput: 1, + Output: 0, + Action: &pb.FileAction_Copy{ + Copy: &pb.FileActionCopy{ + Src: "/src", + Dest: "/dest", + }, + }, + }, + }, + } + + s := newTestFileSolver() + inp := newTestRef("inpref") + + ch := make(chan struct{}) + var sem int64 + inp.mount.callback = func() { + if atomic.AddInt64(&sem, 1) == 2 { + close(ch) + } + <-ch + } + + outs, err := s.Solve(context.TODO(), []fileoptypes.Ref{inp}, fo.Actions) + require.NoError(t, err) + require.Equal(t, len(outs), 1) + + require.Equal(t, int64(2), sem) +} + +func newTestFileSolver() *FileOpSolver { + return NewFileOpSolver(&testFileBackend{}, &testFileRefBackend{}) +} + +type testFileRef struct { + id string + mount testMount + released bool +} + +func (r *testFileRef) Release(context.Context) error { + if r.released { + return errors.Errorf("ref already released") + } + r.released = true + return nil +} + +func newTestRef(id string) *testFileRef { + return &testFileRef{mount: testMount{id: "mount-" + id}, id: id} +} + +type testMount struct { + id string + released bool + chain []mod + callback func() +} + +type mod struct { + mkdir *pb.FileActionMkDir + rm *pb.FileActionRm + mkfile *pb.FileActionMkFile + copy *pb.FileActionCopy + copySrc []mod +} + +func (m *testMount) Release(context.Context) error { + if m.released { + return errors.Errorf("already released") + } + m.released = true + return nil +} + +func (m *testMount) IsFileOpMount() {} + +type testFileBackend struct { +} + +func (b *testFileBackend) Mkdir(_ context.Context, m fileoptypes.Mount, a pb.FileActionMkDir) error { + mm := m.(*testMount) + if mm.callback != nil { + mm.callback() + } + mm.id += "-mkdir" + mm.chain = append(mm.chain, mod{mkdir: &a}) + return nil +} + +func (b *testFileBackend) Mkfile(_ context.Context, m fileoptypes.Mount, a pb.FileActionMkFile) error { + mm := m.(*testMount) + mm.id += "-mkfile" + mm.chain = append(mm.chain, mod{mkfile: &a}) + return nil +} +func (b *testFileBackend) Rm(_ context.Context, m fileoptypes.Mount, a pb.FileActionRm) error { + mm := m.(*testMount) + mm.id += "-rm" + mm.chain = append(mm.chain, mod{rm: &a}) + return nil +} +func (b *testFileBackend) Copy(_ context.Context, m1 fileoptypes.Mount, m fileoptypes.Mount, a pb.FileActionCopy) error { + mm := m.(*testMount) + mm1 := m1.(*testMount) + mm.id += "-copy(" + mm1.id + ")" + mm.chain = append(mm.chain, mod{copy: &a, copySrc: mm1.chain}) + return nil +} + +type testFileRefBackend struct { +} + +func (b *testFileRefBackend) Prepare(ctx context.Context, ref fileoptypes.Ref, readonly bool) (fileoptypes.Mount, error) { + if ref == nil { + return &testMount{id: "scratch"}, nil + } + m := ref.(*testFileRef).mount + m.chain = append([]mod{}, m.chain...) + return &m, nil +} +func (b *testFileRefBackend) Commit(ctx context.Context, mount fileoptypes.Mount) (fileoptypes.Ref, error) { + m := *mount.(*testMount) + return &testFileRef{mount: m, id: m.id + "-commit"}, nil +} diff --git a/solver/llbsolver/ops/fileoptypes/types.go b/solver/llbsolver/ops/fileoptypes/types.go new file mode 100644 index 0000000000000..870395200580c --- /dev/null +++ b/solver/llbsolver/ops/fileoptypes/types.go @@ -0,0 +1,28 @@ +package fileoptypes + +import ( + "context" + + "github.com/moby/buildkit/solver/pb" +) + +type Ref interface { + Release(context.Context) error +} + +type Mount interface { + Release(context.Context) error + IsFileOpMount() +} + +type Backend interface { + Mkdir(context.Context, Mount, pb.FileActionMkDir) error + Mkfile(context.Context, Mount, pb.FileActionMkFile) error + Rm(context.Context, Mount, pb.FileActionRm) error + Copy(context.Context, Mount, Mount, pb.FileActionCopy) error +} + +type RefManager interface { + Prepare(ctx context.Context, ref Ref, readonly bool) (Mount, error) + Commit(ctx context.Context, mount Mount) (Ref, error) +}