Support the new Dev field to parse Julia's extras deps as dev deps

pkg/fanal/analyzer/language/julia/pkg/pkg.go

@@ -42,6 +42,7 @@ type juliaAnalyzer struct {
 
 type Project struct {
 	Dependencies map[string]string `toml:"deps"`
+	Extras       map[string]string `toml:"extras"`
 }
 
 func newJuliaAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
@@ -67,8 +68,8 @@ func (a juliaAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysi
 		}
 
 		// Parse Project.toml alongside Manifest.toml to identify the direct dependencies. This mutates `app`.
-		if err = a.removeExtraDependencies(input.FS, filepath.Dir(path), app); err != nil {
-			log.Logger.Warnf("Unable to parse %q to identify direct dependencies: %s", filepath.Join(filepath.Dir(path), types.JuliaProject), err)
+		if err = analyzeDependencies(input.FS, filepath.Dir(path), app); err != nil {
+			log.Logger.Warnf("Unable to parse %q to analyze dependencies: %s", filepath.Join(filepath.Dir(path), types.JuliaProject), err)
 		}
 
 		sort.Sort(app.Libraries)
@@ -101,44 +102,29 @@ func (a juliaAnalyzer) parseJuliaManifest(path string, r io.Reader) (*types.Appl
 	return language.Parse(types.Julia, path, r, a.lockParser)
 }
 
-// Removes extra dependencies (e.g. test dependencies) in the Project.toml file.
-// This is not strictly necessary, but given that test dependencies are in flux right now, this is future-proofing.
-// https://pkgdocs.julialang.org/v1/creating-packages/#target-based-test-specific-dependencies
-func (a juliaAnalyzer) removeExtraDependencies(fsys fs.FS, dir string, app *types.Application) error {
+func analyzeDependencies(fsys fs.FS, dir string, app *types.Application) error {
+	deps, devDeps, err := parseDependencies(fsys, dir)
+	if err != nil {
+		return err
+	}
+
+	pkgs := walkDependencies(deps, app.Libraries, false)
+	devPkgs := walkDependencies(devDeps, app.Libraries, true)
+	pkgs = lo.Assign(devPkgs, pkgs)
+	app.Libraries = maps.Values(pkgs)
+	return nil
+}
+
+func parseDependencies(fsys fs.FS, dir string) (map[string]string, map[string]string, error) {
 	projectPath := filepath.Join(dir, types.JuliaProject)
 	project, err := parseJuliaProject(fsys, projectPath)
 	if errors.Is(err, fs.ErrNotExist) {
 		log.Logger.Debugf("Julia: %s not found", projectPath)
-		return nil
+		return nil, nil, nil
 	} else if err != nil {
-		return xerrors.Errorf("unable to parse %s: %w", projectPath, err)
-	}
-
-	// Project.toml file can contain same libraries with different versions.
-	// Save versions separately for version comparison by comparator
-	pkgIDs := lo.SliceToMap(app.Libraries, func(pkg types.Package) (string, types.Package) {
-		return pkg.ID, pkg
-	})
-
-	// Identify direct dependencies
-	visited := map[string]types.Package{}
-	for _, uuid := range project.Dependencies {
-		// uuid is a direct dep since it's in the Project's dependencies section. Search through Libraries to mark the matching one as a direct dep.
-		if pkg, ok := pkgIDs[uuid]; ok {
-			// Mark as a direct dependency
-			pkg.Indirect = false
-			visited[pkg.ID] = pkg
-		}
+		return nil, nil, xerrors.Errorf("unable to parse %s: %w", projectPath, err)
 	}
-
-	// Identify indirect dependencies
-	for _, pkg := range visited {
-		walkIndirectDependencies(pkg, pkgIDs, visited)
-	}
-
-	// Include only the packages we visited so that we don't include any deps from the [extras] section
-	app.Libraries = maps.Values(visited)
-	return nil
+	return project.Dependencies, project.Extras, nil
 }
 
 // Parses Project.toml
@@ -156,8 +142,35 @@ func parseJuliaProject(fsys fs.FS, path string) (Project, error) {
 	return proj, nil
 }
 
+// Marks the given direct dependencies as direct, then marks those packages' dependencies as indirect.
+// Marks all encountered packages' Dev flag according to `dev`.
+// Modifies the packages in `allPackages`.
+func walkDependencies(directDeps map[string]string, allPackages types.Packages, dev bool) map[string]types.Package {
+	pkgsByID := lo.SliceToMap(allPackages, func(pkg types.Package) (string, types.Package) {
+		return pkg.ID, pkg
+	})
+
+	// Identify direct dependencies
+	// Everything in `directDeps` is assumed to be direct
+	visited := map[string]types.Package{}
+	for _, uuid := range directDeps {
+		if pkg, ok := pkgsByID[uuid]; ok {
+			pkg.Indirect = false
+			pkg.Dev = dev
+			visited[pkg.ID] = pkg
+		}
+	}
+
+	// Identify indirect dependencies
+	for _, pkg := range visited {
+		walkIndirectDependencies(pkg, pkgsByID, visited, dev)
+	}
+
+	return visited
+}
+
 // Marks all indirect dependencies as indirect. Starts from `rootPkg`. Visited deps are added to `visited`.
-func walkIndirectDependencies(rootPkg types.Package, allPkgIDs map[string]types.Package, visited map[string]types.Package) {
+func walkIndirectDependencies(rootPkg types.Package, allPkgIDs map[string]types.Package, visited map[string]types.Package, dev bool) {
 	for _, pkgID := range rootPkg.DependsOn {
 		if _, ok := visited[pkgID]; ok {
 			continue
@@ -169,7 +182,8 @@ func walkIndirectDependencies(rootPkg types.Package, allPkgIDs map[string]types.
 		}
 
 		dep.Indirect = true
+		dep.Dev = dev
 		visited[dep.ID] = dep
-		walkIndirectDependencies(dep, allPkgIDs, visited)
+		walkIndirectDependencies(dep, allPkgIDs, visited, dev)
 	}
 }

pkg/fanal/analyzer/language/julia/pkg/pkg_test.go

@@ -34,6 +34,23 @@ func Test_juliaAnalyzer_Analyze(t *testing.T) {
 								Locations: []types.Location{{StartLine: 7, EndLine: 9}},
 								DependsOn: []string{"de0858da-6303-5e67-8744-51eddeeeb8d7"},
 							},
+							{
+								ID:        "d9a60922-03b4-4a1b-81be-b8d05b827236",
+								Name:      "DevDep",
+								Version:   "1.0.0",
+								Indirect:  false,
+								Dev:       true,
+								Locations: []types.Location{{StartLine: 65, EndLine: 68}},
+								DependsOn: []string{"b637660b-5035-4894-8335-b3805a4b50d8"},
+							},
+							{
+								ID:        "b637660b-5035-4894-8335-b3805a4b50d8",
+								Name:      "IndirectDevDep",
+								Version:   "2.0.0",
+								Indirect:  true,
+								Dev:       true,
+								Locations: []types.Location{{StartLine: 70, EndLine: 72}},
+							},
 							{
 								ID:        "682c06a0-de6a-54ab-a142-c8b1cf79cde6",
 								Name:      "JSON",

pkg/fanal/analyzer/language/julia/pkg/testdata/happy/Manifest.toml

@@ -61,3 +61,12 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [[deps.Unicode]]
 uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
+
+[[deps.DevDep]]
+deps = ["IndirectDevDep"]
+uuid = "d9a60922-03b4-4a1b-81be-b8d05b827236"
+version = "1.0.0"
+
+[[deps.IndirectDevDep]]
+uuid = "b637660b-5035-4894-8335-b3805a4b50d8"
+version = "2.0.0"

pkg/fanal/analyzer/language/julia/pkg/testdata/happy/Project.toml

@@ -11,7 +11,7 @@ JSON = "0.21"
 julia = "1.8"
 
 [extras]
-FastHistograms = "06971c4e-2824-4b19-bcf3-55442efb9bc7"
+DevDep = "d9a60922-03b4-4a1b-81be-b8d05b827236"
 
 [targets]
-test = ["FastHistograms"]
+test = ["DevDep"]