Add additional exclusions and inclusions (#215)

* Add additional exclusions and inclusions * Default exclude venv3 folder * Change test for new bz2 format
debricked · Mar 21, 2024 · 18e3bf9 · 18e3bf9
1 parent c6d673a
commit 18e3bf9
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 12 deletions.
diff --git a/internal/file/exclusion.go b/internal/file/exclusion.go
@@ -36,7 +36,7 @@ var EXCLUDED_DIRS_FINGERPRINT = []string{
 	"__pycache__", "_yardoc", "eggs",
 	"wheels", "htmlcov", "__pypackages__", ".git"}
 
-var EXCLUDED_DIRS_FINGERPRINT_RAW = []string{"**/*.egg-info/**", "**/*venv/**"}
+var EXCLUDED_DIRS_FINGERPRINT_RAW = []string{"**/*.egg-info/**", "**/*venv/**", "**/*venv3/**"}
 
 func DefaultExclusionsFingerprint() []string {
 	output := []string{}

diff --git a/internal/file/exclusion_test.go b/internal/file/exclusion_test.go
@@ -79,6 +79,7 @@ func TestDefaultExclusionsFingerprint(t *testing.T) {
 		filepath.Join("**", ".git", "**"),
 		"**/*.egg-info/**",
 		"**/*venv/**",
+		"**/*venv3/**",
 	}
 
 	exclusions := DefaultExclusionsFingerprint()

diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go
@@ -34,16 +34,27 @@ var EXCLUDED_EXT = []string{
 	".toml", ".ttf", ".txt", ".utf-8", ".vim", ".wav", ".whl", ".woff", ".woff2", ".xht",
 	".xhtml", ".xls", ".xlsx", ".xpm", ".xsd", ".xul", ".yaml", ".yml", ".wfp",
 	".editorconfig", ".dotcover", ".pid", ".lcov", ".egg", ".manifest", ".cache", ".coverage", ".cover",
-	".gem", ".lst", ".pickle", ".pdb", ".gml", ".pot", ".plt",
+	".gem", ".lst", ".pickle", ".pdb", ".gml", ".pot", ".plt", "", ".pyi",
 }
 
-var EXCLUDED_FILE_ENDINGS = []string{"-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice",
-	"readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3"}
+var EXCLUDED_FILE_ENDINGS = []string{
+	"-doc", "changelog", "config", "copying", "license", "authors", "news", "licenses", "notice",
+	"readme", "swiftdoc", "texidoc", "todo", "version", "ignore", "manifest", "sqlite", "sqlite3",
+	"nycrc", "targ", "eslintrc", "prettierrc",
+}
 
 var EXCLUDED_FILES = []string{
 	"gradlew", "gradlew.bat", "mvnw", "mvnw.cmd", "gradle-wrapper.jar", "maven-wrapper.jar",
 	"thumbs.db", "babel.config.js", "license.txt", "license.md", "copying.lib", "makefile",
-	"[content_types].xml",
+	"[content_types].xml", "py.typed", "LICENSE.APACHE2", "LICENSE.MIT",
+}
+
+var EXCLUDED_DIRS = []string{
+	".idea",
+}
+
+var INCLUDED_FILES = []string{
+	"package.json",
 }
 
 var ZIP_FILE_ENDINGS = []string{".jar", ".nupkg", ".war", ".zip", ".ear"}
@@ -65,9 +76,21 @@ const (
 
 func isExcludedFile(path string) bool {
 
-	return isExcludedByExtension(path) ||
+	return (isExcludedByExtension(path) ||
 		isExcludedByFilename(path) ||
-		isExcludedByEnding(path)
+		isExcludedByEnding(path) ||
+		isInExcludedDir(path)) && !isIncludedFile(path)
+}
+
+func isIncludedFile(path string) bool {
+	filename := filepath.Base(path)
+	for _, file := range INCLUDED_FILES {
+		if filename == file {
+			return true
+		}
+	}
+
+	return false
 }
 
 func isExcludedByExtension(path string) bool {
@@ -104,6 +127,16 @@ func isExcludedByEnding(path string) bool {
 	return false
 }
 
+func isInExcludedDir(path string) bool {
+	for _, dirname := range EXCLUDED_DIRS {
+		if strings.Contains(path, dirname) {
+			return true
+		}
+	}
+
+	return false
+}
+
 type IFingerprint interface {
 	FingerprintFiles(rootPath string, exclusions []string, fingerprintCompressedContent bool) (Fingerprints, error)
 }

diff --git a/internal/fingerprint/fingerprint_test.go b/internal/fingerprint/fingerprint_test.go
@@ -14,14 +14,14 @@ import (
 func TestIsExcludedFile(t *testing.T) {
 
 	// Test excluded file extensions
-	excludedExts := []string{".doc", ".pdf", ".txt"}
+	excludedExts := []string{".doc", ".pdf", ".txt", ""}
 	for _, ext := range excludedExts {
 		filename := "file" + ext
 		assert.True(t, isExcludedFile(filename), "Expected %q to be excluded", filename)
 	}
 
 	// Test excluded files
-	excludedFiles := []string{"LICENSE", "README.md", "Makefile", "mvnw", "[content_types].xml"}
+	excludedFiles := []string{"LICENSE", "README.md", "Makefile", "mvnw", "[content_types].xml", "Stockholm", "hello.json"}
 	for _, filename := range excludedFiles {
 		assert.True(t, isExcludedFile(filename), "Expected %q to be excluded", filename)
 		filepath := "foo/bar/" + filename
@@ -36,6 +36,20 @@ func TestIsExcludedFile(t *testing.T) {
 		assert.True(t, isExcludedFile(filename), "Expected %q to be excluded", filename)
 	}
 
+	// Test excluded dirnames
+	filesInExcludedDir := []string{"package/.idea/test.txt"}
+	for _, filename := range filesInExcludedDir {
+		assert.True(t, isExcludedFile(filename), "Expected %q to be excluded", filename)
+	}
+
+	// Test included files
+	includedFiles := []string{"package.json"}
+	for _, filename := range includedFiles {
+		assert.False(t, isExcludedFile(filename), "Expected %q to not be excluded", filename)
+		filepath := "foo/bar/" + filename
+		assert.False(t, isExcludedFile(filepath), "Expected %q to not be excluded", filepath)
+	}
+
 	// Test non-excluded files
 	assert.False(t, isExcludedFile("file.py"), "Expected file.txt to not be excluded")
 	assert.False(t, isExcludedFile("file.go"), "Expected .go to not be excluded")
@@ -436,7 +450,7 @@ func TestInMemFingerprintingCompressedContent(t *testing.T) {
 		{
 			name:        "Jar",
 			path:        "testdata/archive/jar",
-			expected:    196,
+			expected:    195,
 			suffix:      "log4j-api-2.18.0.jar",
 			shouldUnzip: true,
 		},
@@ -450,7 +464,7 @@ func TestInMemFingerprintingCompressedContent(t *testing.T) {
 		{
 			name:        "TGz",
 			path:        "testdata/archive/tgz",
-			expected:    1050,
+			expected:    1051,
 			suffix:      "lodash.tgz",
 			shouldUnzip: true,
 		},
@@ -464,7 +478,7 @@ func TestInMemFingerprintingCompressedContent(t *testing.T) {
 		{
 			name:        "BZip2",
 			path:        "testdata/archive/bz2",
-			expected:    10,
+			expected:    7,
 			suffix:      "stuf-0.1.tar.bz2",
 			shouldUnzip: true,
 		},