parser: reference FS instead of files contents map

internal/commands/run.go

@@ -149,7 +149,7 @@ func run(cmd *cobra.Command, args []string) error {
 	var result *parser.Result
 	if experimentalParser {
 		p := parser.New(parser.WithEnvironment(userSpecifiedEnvironment))
-		result, err = p.Parse(mergedYAML)
+		result, err = p.Parse(cmd.Context(), mergedYAML)
 		if err != nil {
 			return err
 		}

internal/evaluator/evaluator.go

@@ -75,10 +75,10 @@ func evaluateConfig(ctx context.Context, request *api.EvaluateConfigRequest) (*a
 		yamlConfigs = append(yamlConfigs, request.YamlConfig)
 	}
 
+	fs := fsFromEnvironment(request.Environment)
+
 	// Run Starlark script and register generated YAML configuration (if any)
 	if request.StarlarkConfig != "" {
-		fs := fsFromEnvironment(request.Environment)
-
 		lrk := larker.New(
 			larker.WithFileSystem(fs),
 			larker.WithEnvironment(request.Environment),
@@ -95,10 +95,10 @@ func evaluateConfig(ctx context.Context, request *api.EvaluateConfigRequest) (*a
 	// Parse combined YAML
 	p := parser.New(
 		parser.WithEnvironment(request.Environment),
-		parser.WithFilesContents(request.FilesContents),
+		parser.WithFileSystem(fs),
 	)
 
-	result, err := p.Parse(strings.Join(yamlConfigs, "\n"))
+	result, err := p.Parse(ctx, strings.Join(yamlConfigs, "\n"))
 	if err != nil {
 		return nil, status.Error(codes.InvalidArgument, err.Error())
 	}

pkg/larker/fs/memory/memory.go

@@ -0,0 +1,65 @@
+package memory
+
+import (
+	"context"
+	"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
+	"github.com/go-git/go-billy/v5"
+	"github.com/go-git/go-billy/v5/memfs"
+	"github.com/go-git/go-billy/v5/util"
+	"io/ioutil"
+)
+
+type Memory struct {
+	fs billy.Filesystem
+}
+
+func New(fileContents map[string][]byte) (*Memory, error) {
+	memory := &Memory{
+		fs: memfs.New(),
+	}
+
+	for path, contents := range fileContents {
+		if err := util.WriteFile(memory.fs, path, contents, 0600); err != nil {
+			return nil, err
+		}
+	}
+
+	return memory, nil
+}
+
+func (memory *Memory) Stat(ctx context.Context, path string) (*fs.FileInfo, error) {
+	fileInfo, err := memory.fs.Stat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return &fs.FileInfo{IsDir: fileInfo.IsDir()}, nil
+}
+
+func (memory *Memory) Get(ctx context.Context, path string) ([]byte, error) {
+	file, err := memory.fs.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	fileBytes, err := ioutil.ReadAll(file)
+	if err != nil {
+		return nil, err
+	}
+
+	return fileBytes, nil
+}
+
+func (memory *Memory) ReadDir(ctx context.Context, path string) ([]string, error) {
+	fileInfos, err := memory.fs.ReadDir(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var result []string
+	for _, fileInfo := range fileInfos {
+		result = append(result, fileInfo.Name())
+	}
+
+	return result, nil
+}

pkg/parser/options.go

@@ -1,6 +1,7 @@
 package parser
 
 import (
+	"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
 	"google.golang.org/protobuf/reflect/protoreflect"
 )
 
@@ -12,9 +13,9 @@ func WithEnvironment(environment map[string]string) Option {
 	}
 }
 
-func WithFilesContents(filesContents map[string]string) Option {
+func WithFileSystem(fs fs.FileSystem) Option {
 	return func(parser *Parser) {
-		parser.filesContents = filesContents
+		parser.fs = fs
 	}
 }
 

pkg/parser/parser.go

@@ -1,12 +1,14 @@
 package parser
 
 import (
+	"context"
 	"crypto/md5" // nolint:gosec // backwards compatibility
 	"errors"
 	"fmt"
 	"github.com/cirruslabs/cirrus-ci-agent/api"
 	"github.com/cirruslabs/cirrus-cli/internal/executor/environment"
-	"github.com/cirruslabs/cirrus-cli/internal/executor/instance"
+	"github.com/cirruslabs/cirrus-cli/pkg/larker/fs"
+	"github.com/cirruslabs/cirrus-cli/pkg/larker/fs/dummy"
 	"github.com/cirruslabs/cirrus-cli/pkg/parser/modifier/matrix"
 	"github.com/cirruslabs/cirrus-cli/pkg/parser/nameable"
 	"github.com/cirruslabs/cirrus-cli/pkg/parser/node"
@@ -18,7 +20,7 @@ import (
 	"github.com/lestrrat-go/jsschema"
 	"google.golang.org/protobuf/reflect/protoreflect"
 	"io/ioutil"
-	"path/filepath"
+	"os"
 	"regexp"
 	"strconv"
 )
@@ -32,8 +34,10 @@ type Parser struct {
 	// Environment to take into account when expanding variables.
 	environment map[string]string
 
-	// Paths and contents of the files that might influence the parser.
-	filesContents map[string]string
+	// Filesystem to reference when calculating file hashes.
+	//
+	// For example, Dockerfile contents are hashed to avoid duplicate builds.
+	fs fs.FileSystem
 
 	parsers             map[nameable.Nameable]parseable.Parseable
 	numbering           int64
@@ -47,8 +51,8 @@ type Result struct {
 
 func New(opts ...Option) *Parser {
 	parser := &Parser{
-		environment:   make(map[string]string),
-		filesContents: make(map[string]string),
+		environment: make(map[string]string),
+		fs:          dummy.New(),
 	}
 
 	// Apply options
@@ -117,7 +121,7 @@ func (p *Parser) parseTasks(tree *node.Node) ([]task.ParseableTaskLike, error) {
 	return tasks, nil
 }
 
-func (p *Parser) Parse(config string) (*Result, error) {
+func (p *Parser) Parse(ctx context.Context, config string) (*Result, error) {
 	var parsed yaml.MapSlice
 
 	// Unmarshal YAML
@@ -170,7 +174,7 @@ func (p *Parser) Parse(config string) (*Result, error) {
 	}
 
 	// Create service tasks
-	serviceTasks, err := p.createServiceTasks(protoTasks)
+	serviceTasks, err := p.createServiceTasks(ctx, protoTasks)
 	if err != nil {
 		return &Result{
 			Errors: []string{err.Error()},
@@ -196,59 +200,27 @@ func (p *Parser) Parse(config string) (*Result, error) {
 	}, nil
 }
 
-func (p *Parser) ParseFromFile(path string) (*Result, error) {
+func (p *Parser) ParseFromFile(ctx context.Context, path string) (*Result, error) {
 	config, err := ioutil.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
 
-	result, err := p.Parse(string(config))
-	if err != nil || len(result.Errors) != 0 {
-		return result, err
-	}
-
-	// Get the contents of files that might influence the parser results
-	//
-	// For example, when using Dockerfile as CI environment feature[1], the unique hash of the container
-	// image is calculated from the file specified in the "dockerfile" field.
-	//
-	// [1]: https://cirrus-ci.org/guide/docker-builder-vm/#dockerfile-as-a-ci-environment
-	filesContents := make(map[string]string)
-	for _, task := range result.Tasks {
-		inst, err := instance.NewFromProto(task.Instance, []*api.Command{})
-		if err != nil {
-			continue
-		}
-		prebuilt, ok := inst.(*instance.PrebuiltInstance)
-		if !ok {
-			continue
-		}
-		contents, err := ioutil.ReadFile(filepath.Join(filepath.Dir(path), prebuilt.Dockerfile))
-		if err != nil {
-			return nil, fmt.Errorf("%w: %v", ErrFilesContents, err)
-		}
-		filesContents[prebuilt.Dockerfile] = string(contents)
-	}
-
-	// Short-circuit if we've found no special files
-	if len(filesContents) == 0 {
-		return result, nil
-	}
-
-	// Parse again with the file contents supplied
-	p.filesContents = filesContents
-	return p.Parse(string(config))
+	return p.Parse(ctx, string(config))
 }
 
-func (p *Parser) ContentHash(filePath string) string {
+func (p *Parser) fileHash(ctx context.Context, path string) (string, error) {
 	// Note that this will be empty if we don't know anything about the file,
 	// so we'll return MD5(""), but that's OK since the purpose is caching
-	fileContents := p.filesContents[filePath]
+	fileBytes, err := p.fs.Get(ctx, path)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return "", err
+	}
 
 	// nolint:gosec // backwards compatibility
-	digest := md5.Sum([]byte(fileContents))
+	digest := md5.Sum(fileBytes)
 
-	return fmt.Sprintf("%x", digest)
+	return fmt.Sprintf("%x", digest), nil
 }
 
 func (p *Parser) NextTaskID() int64 {
@@ -304,7 +276,7 @@ func resolveDependencies(tasks []task.ParseableTaskLike) error {
 	return nil
 }
 
-func (p *Parser) createServiceTasks(protoTasks []*api.Task) ([]*api.Task, error) {
+func (p *Parser) createServiceTasks(ctx context.Context, protoTasks []*api.Task) ([]*api.Task, error) {
 	var serviceTasks []*api.Task
 
 	for _, protoTask := range protoTasks {
@@ -327,7 +299,10 @@ func (p *Parser) createServiceTasks(protoTasks []*api.Task) ([]*api.Task, error)
 			continue
 		}
 
-		dockerfileHash := p.ContentHash(taskContainer.DockerfilePath)
+		dockerfileHash, err := p.fileHash(ctx, taskContainer.DockerfilePath)
+		if err != nil {
+			return nil, err
+		}
 
 		prebuiltInstance := &api.PrebuiltImageInstance{
 			Repository:     fmt.Sprintf("cirrus-ci-community/%s", dockerfileHash),

pkg/parser/parser_test.go

@@ -1,11 +1,13 @@
 package parser_test
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"github.com/cirruslabs/cirrus-ci-agent/api"
 	"github.com/cirruslabs/cirrus-cli/internal/testutil"
+	"github.com/cirruslabs/cirrus-cli/pkg/larker/fs/memory"
 	"github.com/cirruslabs/cirrus-cli/pkg/rpcparser"
 	"github.com/goccy/go-yaml"
 	"github.com/stretchr/testify/require"
@@ -40,7 +42,7 @@ func TestValidConfigs(t *testing.T) {
 		file := validCase
 		t.Run(file, func(t *testing.T) {
 			p := parser.New()
-			result, err := p.ParseFromFile(absolutize(file + ".yml"))
+			result, err := p.ParseFromFile(context.Background(), absolutize(file+".yml"))
 
 			require.Nil(t, err)
 			require.Empty(t, result.Errors)
@@ -62,7 +64,7 @@ func TestAdditionalInstances(t *testing.T) {
 	p := parser.New(parser.WithAdditionalInstances(map[string]protoreflect.MessageDescriptor{
 		"proto_container": containerInstanceReflect.Descriptor(),
 	}))
-	result, err := p.ParseFromFile(absolutize("proto-instance.yml"))
+	result, err := p.ParseFromFile(context.Background(), absolutize("proto-instance.yml"))
 
 	require.Nil(t, err)
 	require.Empty(t, result.Errors)
@@ -83,7 +85,7 @@ func TestInvalidConfigs(t *testing.T) {
 		file := invalidCase
 		t.Run(file, func(t *testing.T) {
 			p := parser.New()
-			result, err := p.ParseFromFile(absolutize(file))
+			result, err := p.ParseFromFile(context.Background(), absolutize(file))
 
 			require.Nil(t, err)
 			assert.NotEmpty(t, result.Errors)
@@ -136,12 +138,24 @@ func viaRPCRunSingle(t *testing.T, cloudDir string, yamlConfigName string) {
 		t.Fatal(err)
 	}
 
+	// Craft virtual in-memory filesystem with test-specific files
+	fileContents := make(map[string][]byte)
+
+	for key, value := range viaRPCLoadMap(t, fcPath) {
+		fileContents[key] = []byte(value)
+	}
+
+	fs, err := memory.New(fileContents)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// Obtain the actual result by parsing YAML configuration using the local parser
 	localParser := parser.New(
 		parser.WithEnvironment(viaRPCLoadMap(t, envPath)),
-		parser.WithFilesContents(viaRPCLoadMap(t, fcPath)),
+		parser.WithFileSystem(fs),
 	)
-	localResult, err := localParser.Parse(string(yamlBytes))
+	localResult, err := localParser.Parse(context.Background(), string(yamlBytes))
 	if err != nil {
 		t.Fatal(err)
 	}