diff --git a/pkg/julia/manifest/parse.go b/pkg/julia/manifest/parse.go index cd30be75..2c0f4aae 100644 --- a/pkg/julia/manifest/parse.go +++ b/pkg/julia/manifest/parse.go @@ -12,15 +12,28 @@ import ( "golang.org/x/xerrors" ) -type Manifest struct { - JuliaVersion string `toml:"julia_version"` - ManifestFormat string `toml:"manifest_format"` - Dependencies map[string][]Dependency `toml:"deps"` // e.g. [[deps.Foo]] +type PrimitiveManifest struct { + JuliaVersion string `toml:"julia_version"` + ManifestFormat string `toml:"manifest_format"` + Dependencies map[string][]PrimitiveDependency `toml:"deps"` // e.g. [[deps.Foo]] } -type Dependency struct { - Dependencies []string `toml:"deps"` // by name. e.g. deps = ["Foo"] - UUID string `toml:"uuid"` - Version string `toml:"version"` // not specified for stdlib packages, which are of the Julia version + +type DecodedManifest struct { + JuliaVersion string + ManifestFormat string + Dependencies map[string][]DecodedDependency +} + +type PrimitiveDependency struct { + Dependencies toml.Primitive `toml:"deps"` // by name. e.g. deps = ["Foo"] or [deps.Foo.deps] + UUID string `toml:"uuid"` + Version string `toml:"version"` // not specified for stdlib packages, which are of the Julia version +} + +type DecodedDependency struct { + Dependencies map[string]*string + UUID string + Version string } type Parser struct{} @@ -30,26 +43,32 @@ func NewParser() types.Parser { } func (p *Parser) Parse(r dio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) { - var oldDeps map[string][]Dependency - var man Manifest + var oldDeps map[string][]PrimitiveDependency + var primMan PrimitiveManifest + var manMetadata toml.MetaData decoder := toml.NewDecoder(r) // Try to read the old Manifest format. If that fails, try the new format. if _, err := decoder.Decode(&oldDeps); err != nil { r.Seek(0, io.SeekStart) - if _, err := decoder.Decode(&man); err != nil { + if manMetadata, err = decoder.Decode(&primMan); err != nil { return nil, nil, xerrors.Errorf("decode error: %w", err) } } // We can't know the Julia version on an old manifest. // All newer manifests include a manifest version and a julia version. - if man.ManifestFormat == "" { - man = Manifest{ + if primMan.ManifestFormat == "" { + primMan = PrimitiveManifest{ JuliaVersion: "unknown", Dependencies: oldDeps, } } + man, err := decodeManifest(&primMan, &manMetadata) + if err != nil { + return nil, nil, xerrors.Errorf("unable to decode manifest dependencies: %w", err) + } + if _, err := r.Seek(0, io.SeekStart); err != nil { return nil, nil, xerrors.Errorf("seek error: %w", err) } @@ -61,27 +80,27 @@ func (p *Parser) Parse(r dio.ReadSeekerAt) ([]types.Library, []types.Dependency, var libs []types.Library var deps []types.Dependency for name, manifestDeps := range man.Dependencies { - // The TOML schema forces man.Dependencies to be a list, but Julia manifests always specify exactly one - // dependency using this format. Check that this invariant holds. - if len(manifestDeps) > 1 { - return nil, nil, xerrors.Errorf("multiple entries for dep: %s", name) - } - manifestDep := manifestDeps[0] - version := depVersion(manifestDep, man.JuliaVersion) - pkgID := utils.PackageID(manifestDep.UUID, version) - lib := types.Library{ - ID: pkgID, - Name: name, - Version: version, - } - if pos, ok := lineNumIdx[manifestDep.UUID]; ok { - lib.Locations = []types.Location{{StartLine: pos.start, EndLine: pos.end}} - } + for _, manifestDep := range manifestDeps { + version := depVersion(&manifestDep, man.JuliaVersion) + pkgID := utils.PackageID(manifestDep.UUID, version) + lib := types.Library{ + ID: pkgID, + Name: name, + Version: version, + } + if pos, ok := lineNumIdx[manifestDep.UUID]; ok { + lib.Locations = []types.Location{{StartLine: pos.start, EndLine: pos.end}} + } + + libs = append(libs, lib) - libs = append(libs, lib) - dep := parseDependencies(pkgID, manifestDep.Dependencies, man.Dependencies, man.JuliaVersion) - if dep != nil { - deps = append(deps, *dep) + dep, err := parseDependencies(pkgID, manifestDep.Dependencies, man.Dependencies, man.JuliaVersion) + if err != nil { + return nil, nil, xerrors.Errorf("failed to parse dependencies: %w", err) + } + if dep != nil { + deps = append(deps, *dep) + } } } sort.Sort(types.Libraries(libs)) @@ -89,31 +108,111 @@ func (p *Parser) Parse(r dio.ReadSeekerAt) ([]types.Library, []types.Dependency, return libs, deps, nil } -// Returns the matching dependencies in `allDeps` of the given `depNames`. If there are no matching dependencies, returns `nil`. -func parseDependencies(pkgId string, depNames []string, allDeps map[string][]Dependency, juliaVersion string) *types.Dependency { +// Returns the matching dependencies in `allDeps` of the given `deps`. If there are no matching dependencies, returns `nil`. +func parseDependencies(pkgId string, deps map[string]*string, allDeps map[string][]DecodedDependency, juliaVersion string) (*types.Dependency, error) { var dependOn []string - for _, depName := range depNames { - dep := allDeps[depName][0] + for depName, depUUID := range deps { + dep, err := lookupDep(depName, depUUID, allDeps) + if err != nil { + return nil, err + } version := depVersion(dep, juliaVersion) dependOn = append(dependOn, utils.PackageID(dep.UUID, version)) } + if len(dependOn) > 0 { sort.Strings(dependOn) return &types.Dependency{ ID: pkgId, DependsOn: dependOn, - } + }, nil } else { - return nil + return nil, nil + } +} + +// Returns the matching dependency in `allDeps` given the dep with the `name` and `uuid`. +// The `uuid` may be `nil` if the given dep is the only one with its name in the manifest. +// Otherwise, if there are multiple deps with the same name, the `uuid` must be specified. +func lookupDep(name string, uuid *string, allDeps map[string][]DecodedDependency) (*DecodedDependency, error) { + if uuid == nil { + // No UUID was set in the manifest, which means there is only one dep with this name + return &allDeps[name][0], nil + } else { + for _, candidateDep := range allDeps[name] { + if candidateDep.UUID == *uuid { + return &candidateDep, nil + } + } + return nil, xerrors.Errorf("failed to find dep with name %s and UUID %s", name, *uuid) } } // Returns the effective version of the `dep`. // stdlib packages do not have a version in the manifest because they are packaged with julia itself -func depVersion(dep Dependency, juliaVersion string) string { +func depVersion(dep *DecodedDependency, juliaVersion string) string { if len(dep.Version) == 0 { return juliaVersion } return dep.Version } + +// Decodes a primitive manifest using the metadata from parse time. +func decodeManifest(man *PrimitiveManifest, metadata *toml.MetaData) (*DecodedManifest, error) { + // Copy over already decoded fields + decodedManifest := DecodedManifest{ + JuliaVersion: man.JuliaVersion, + ManifestFormat: man.ManifestFormat, + Dependencies: make(map[string][]DecodedDependency), + } + + // Decode each dependency into the new manifest + for depName, primDeps := range man.Dependencies { + decodedDeps := []DecodedDependency{} + for _, primDep := range primDeps { + decodedDep, err := decodeDependency(&primDep, metadata) + if err != nil { + return nil, err + } + decodedDeps = append(decodedDeps, *decodedDep) + } + decodedManifest.Dependencies[depName] = decodedDeps + } + + return &decodedManifest, nil +} + +// Decodes a primitive dependency using the metadata from parse time. +func decodeDependency(dep *PrimitiveDependency, metadata *toml.MetaData) (*DecodedDependency, error) { + // Try to decode as []string first where the manifest looks like deps = ["A", "B"] + var possibleDeps []string + err := metadata.PrimitiveDecode(dep.Dependencies, &possibleDeps) + if err == nil { + finalDeps := make(map[string]*string) + for _, depName := range possibleDeps { + finalDeps[depName] = nil + } + return &DecodedDependency{ + Dependencies: finalDeps, + UUID: dep.UUID, + Version: dep.Version, + }, nil + } + + // The other possibility is a map where the manifest looks like + // [deps.A.deps] + // B = "..." + var possibleDepsMap map[string]*string + err = metadata.PrimitiveDecode(dep.Dependencies, &possibleDepsMap) + if err == nil { + return &DecodedDependency{ + Dependencies: possibleDepsMap, + UUID: dep.UUID, + Version: dep.Version, + }, nil + } + + // We don't know what the shape of the data is -- i.e. an invalid manifest + return nil, err +} diff --git a/pkg/julia/manifest/parse_test.go b/pkg/julia/manifest/parse_test.go index a98f37e0..60785d91 100644 --- a/pkg/julia/manifest/parse_test.go +++ b/pkg/julia/manifest/parse_test.go @@ -48,6 +48,12 @@ func TestParse(t *testing.T) { want: juliaV1_9DepExtLibs, wantDeps: nil, }, + { + name: "shadowed dep v1.9", + file: "testdata/shadowed_dep_v1.9/Manifest.toml", + want: juliaV1_9ShadowedDepLibs, + wantDeps: juliaV1_9ShadowedDepDeps, + }, } for _, tt := range tests { diff --git a/pkg/julia/manifest/parse_testcase.go b/pkg/julia/manifest/parse_testcase.go index 4780f987..37a77df4 100644 --- a/pkg/julia/manifest/parse_testcase.go +++ b/pkg/julia/manifest/parse_testcase.go @@ -64,4 +64,14 @@ var ( juliaV1_9DepExtLibs = []types.Library{ {ID: "621f4979-c628-5d54-868e-fcf4e3e8185c@1.3.1", Name: "AbstractFFTs", Version: "1.3.1", Locations: []types.Location{{StartLine: 7, EndLine: 10}}}, } + + juliaV1_9ShadowedDepLibs = []types.Library{ + {ID: "ead4f63c-334e-11e9-00e6-e7f0a5f21b60@1.9.0", Name: "A", Version: "1.9.0", Locations: []types.Location{{StartLine: 7, EndLine: 8}}}, + {ID: "f41f7b98-334e-11e9-1257-49272045fb24@1.9.0", Name: "B", Version: "1.9.0", Locations: []types.Location{{StartLine: 13, EndLine: 14}}}, + {ID: "edca9bc6-334e-11e9-3554-9595dbb4349c@1.9.0", Name: "B", Version: "1.9.0", Locations: []types.Location{{StartLine: 15, EndLine: 16}}}, + } + + juliaV1_9ShadowedDepDeps = []types.Dependency{ + {ID: "ead4f63c-334e-11e9-00e6-e7f0a5f21b60@1.9.0", DependsOn: []string{"f41f7b98-334e-11e9-1257-49272045fb24@1.9.0"}}, + } ) diff --git a/pkg/julia/manifest/testdata/shadowed_dep_v1.9/Manifest.toml b/pkg/julia/manifest/testdata/shadowed_dep_v1.9/Manifest.toml new file mode 100644 index 00000000..dd4ea00b --- /dev/null +++ b/pkg/julia/manifest/testdata/shadowed_dep_v1.9/Manifest.toml @@ -0,0 +1,16 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.9.0" +manifest_format = "2.0" +project_hash = "f0a796fb78285c02ad123fec6e14c8bac09a2ccc" + +[[deps.A]] +uuid = "ead4f63c-334e-11e9-00e6-e7f0a5f21b60" + + [deps.A.deps] + B = "f41f7b98-334e-11e9-1257-49272045fb24" + +[[deps.B]] +uuid = "f41f7b98-334e-11e9-1257-49272045fb24" +[[deps.B]] +uuid = "edca9bc6-334e-11e9-3554-9595dbb4349c"