From ae434652bd631b1f597509fdecca31f3714b88af Mon Sep 17 00:00:00 2001 From: Milas Bowman Date: Fri, 18 Aug 2023 09:56:57 -0400 Subject: [PATCH] load: include details about included files on *Project (#444) Add a map to `Project` that has Compose YAML filename as key and any `IncludeConfig`s loaded from it. This is populated as the project is recursively loaded. For example: ``` proj/ compose.yaml a/ compose.yaml b/ compose.yaml ``` If `project/compose.yaml` has: ``` include: ['./a/compose.yaml'] ``` And `project/a/compose.yaml` has: ``` include: ['../b/compose.yaml'] ``` The final result after load is conceptually: ``` { "proj/compose.yaml": ["proj/a/compose.yaml"], "proj/a/compose.yaml": ["proj/b/compose.yaml"], } ``` (Note: in reality, it's a list of `IncludeConfig`, which has multiple fields. Example is simplified.) Relative path resolution is based on overall loader configuration, but note that disabling it does not work properly for `include` currently due to other issues. This makes it possible for the caller to understand a bit more about the loaded project resources. We're really overdue for a bit of an overhaul/refactor of the loader - at that point, I think it might be better to have a `Loader` object type that can track stuff like this on the instance because it's weirdly both part of the project and NOT part of the project at the moment (similar to `Profiles`, project name, etc). Signed-off-by: Milas Bowman --- loader/include.go | 18 ++++++++++++------ loader/loader.go | 11 ++++++++++- loader/loader_test.go | 9 +++++++++ loader/paths.go | 30 ++++++++++++++++++++++++++++++ types/project.go | 26 ++++++++++++++++---------- 5 files changed, 77 insertions(+), 17 deletions(-) diff --git a/loader/include.go b/loader/include.go index 026dcf9a..7a736aae 100644 --- a/loader/include.go +++ b/loader/include.go @@ -45,14 +45,17 @@ var transformIncludeConfig TransformerFunc = func(data interface{}) (interface{} } } -func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, error) { +func loadInclude(ctx context.Context, filename string, configDetails types.ConfigDetails, model *types.Config, options *Options, loaded []string) (*types.Config, map[string][]types.IncludeConfig, error) { + included := make(map[string][]types.IncludeConfig) for _, r := range model.Include { + included[filename] = append(included[filename], r) + for i, p := range r.Path { for _, loader := range options.ResourceLoaders { if loader.Accept(p) { path, err := loader.Load(ctx, p) if err != nil { - return nil, err + return nil, nil, err } p = path break @@ -72,7 +75,7 @@ func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model * env, err := dotenv.GetEnvFromFile(configDetails.Environment, r.ProjectDirectory, r.EnvFile) if err != nil { - return nil, err + return nil, nil, err } config := types.ConfigDetails{ @@ -87,16 +90,19 @@ func loadInclude(ctx context.Context, configDetails types.ConfigDetails, model * } imported, err := load(ctx, config, loadOptions, loaded) if err != nil { - return nil, err + return nil, nil, err + } + for k, v := range imported.IncludeReferences { + included[k] = append(included[k], v...) } err = importResources(model, imported, r.Path) if err != nil { - return nil, err + return nil, nil, err } } model.Include = nil - return model, nil + return model, included, nil } // importResources import into model all resources defined by imported, and report error on conflict diff --git a/loader/loader.go b/loader/loader.go index b6f8fe90..6b80ab23 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -250,6 +250,7 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options, } loaded = append(loaded, mainFile) + includeRefs := make(map[string][]types.IncludeConfig) for i, file := range configDetails.ConfigFiles { var postProcessor PostProcessor configDict := file.Config @@ -285,10 +286,14 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options, } if !opts.SkipInclude { - cfg, err = loadInclude(ctx, configDetails, cfg, opts, loaded) + var included map[string][]types.IncludeConfig + cfg, included, err = loadInclude(ctx, file.Filename, configDetails, cfg, opts, loaded) if err != nil { return nil, err } + for k, v := range included { + includeRefs[k] = append(includeRefs[k], v...) + } } if i == 0 { @@ -321,6 +326,10 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options, Extensions: model.Extensions, } + if len(includeRefs) != 0 { + project.IncludeReferences = includeRefs + } + if !opts.SkipNormalization { err := Normalize(project) if err != nil { diff --git a/loader/loader_test.go b/loader/loader_test.go index 317b89b0..130c7787 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -2545,6 +2545,15 @@ services: }, }, }) + assert.DeepEqual(t, p.IncludeReferences, map[string][]types.IncludeConfig{ + filepath.Join(workingDir, "filename0.yml"): { + { + Path: []string{filepath.Join(workingDir, "testdata", "subdir", "compose-test-extends-imported.yaml")}, + ProjectDirectory: workingDir, + EnvFile: []string{filepath.Join(workingDir, "testdata", "subdir", "extra.env")}, + }, + }, + }) assert.NilError(t, err) } diff --git a/loader/paths.go b/loader/paths.go index 45bbc545..61e79f01 100644 --- a/loader/paths.go +++ b/loader/paths.go @@ -64,6 +64,25 @@ func ResolveRelativePaths(project *types.Project) error { project.Volumes[name] = config } } + + // don't coerce a nil map to an empty map + if project.IncludeReferences != nil { + absIncludes := make(map[string][]types.IncludeConfig, len(project.IncludeReferences)) + for filename, config := range project.IncludeReferences { + filename = absPath(project.WorkingDir, filename) + absConfigs := make([]types.IncludeConfig, len(config)) + for i, c := range config { + absConfigs[i] = types.IncludeConfig{ + Path: resolvePaths(project.WorkingDir, c.Path), + ProjectDirectory: absPath(project.WorkingDir, c.ProjectDirectory), + EnvFile: resolvePaths(project.WorkingDir, c.EnvFile), + } + } + absIncludes[filename] = absConfigs + } + project.IncludeReferences = absIncludes + } + return nil } @@ -133,3 +152,14 @@ func isRemoteContext(maybeURL string) bool { } return false } + +func resolvePaths(basePath string, in types.StringList) types.StringList { + if in == nil { + return nil + } + ret := make(types.StringList, len(in)) + for i := range in { + ret[i] = absPath(basePath, in[i]) + } + return ret +} diff --git a/types/project.go b/types/project.go index 59b6b2b9..2a82ccfb 100644 --- a/types/project.go +++ b/types/project.go @@ -36,16 +36,22 @@ import ( // Project is the result of loading a set of compose files type Project struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - WorkingDir string `yaml:"-" json:"-"` - Services Services `yaml:"services" json:"services"` - Networks Networks `yaml:"networks,omitempty" json:"networks,omitempty"` - Volumes Volumes `yaml:"volumes,omitempty" json:"volumes,omitempty"` - Secrets Secrets `yaml:"secrets,omitempty" json:"secrets,omitempty"` - Configs Configs `yaml:"configs,omitempty" json:"configs,omitempty"` - Extensions Extensions `yaml:"#extensions,inline" json:"-"` // https://github.com/golang/go/issues/6213 - ComposeFiles []string `yaml:"-" json:"-"` - Environment Mapping `yaml:"-" json:"-"` + Name string `yaml:"name,omitempty" json:"name,omitempty"` + WorkingDir string `yaml:"-" json:"-"` + Services Services `yaml:"services" json:"services"` + Networks Networks `yaml:"networks,omitempty" json:"networks,omitempty"` + Volumes Volumes `yaml:"volumes,omitempty" json:"volumes,omitempty"` + Secrets Secrets `yaml:"secrets,omitempty" json:"secrets,omitempty"` + Configs Configs `yaml:"configs,omitempty" json:"configs,omitempty"` + Extensions Extensions `yaml:"#extensions,inline" json:"-"` // https://github.com/golang/go/issues/6213 + + // IncludeReferences is keyed by Compose YAML filename and contains config for + // other Compose YAML files it directly triggered a load of via `include`. + // + // Note: this is + IncludeReferences map[string][]IncludeConfig `yaml:"-" json:"-"` + ComposeFiles []string `yaml:"-" json:"-"` + Environment Mapping `yaml:"-" json:"-"` // DisabledServices track services which have been disable as profile is not active DisabledServices Services `yaml:"-" json:"-"`