Rename bandwidth to capacity, logging tweaks, default config updates

configs/go/fileshot/fileshot.yaml

@@ -10,6 +10,7 @@ throughput:
   chunk: 98304
   delay: 1s
 log:
+  verbose: False
   path: 'C:\temp\fileshot.log'
   maxsize: 1      # Log file size limit in Megabytes
 	maxbackups: 0   # Log file backup count limit
@@ -20,18 +21,21 @@ files:
     # path: asc
     # size: asc
   patterns:
-    - '/glob/to/your/files/*'
-  minsize: 0                  # Limit minimum file size in bytes
+    positive:
+      - '/glob/to/your/files/*'
+    # negative
+    #   - '/*/badfiles/*'
+  minsize: 1                  # Limit minimum file size in bytes
   maxsize: 999999999          # Limit max file size in bytes
-  limitbandwidth: 999999999   # Limit max file bandwidth in bytes
+  limitcapacity: 999999999    # Limit max file bandwidth in bytes
   limitpattern: 999           # Limit each file path to a max number of files
   limittotal: 99999           # Limit all collection to a max number of total files
   modified: 99999             # Last modified in hours, e.g. 168 Hours = Last Modified < 7 Days Ago
   mimetypes:                  # Exclude all files with specific mimetypes e.g. application/octet-stream
-    negative:
-      - 'application/mimetype'
     # positive:
     #   - 'application/mimetype'
+    negative:
+      - 'application/mimetype'
   excludehashes: # Enabling hash exclusions will cause matching files to be hashed and cause a performance penalty
     # path: fileshot-exclusions.md5      # Path to a file containing a list of hashes
     # list:

src/go/cmd/strelka-fileshot/main.go

@@ -431,11 +431,13 @@ func getFilePaths(conf structs.FileShot, verbose *bool, hashes []string, sortMet
 
 	paths := make([]string, 0)
 	var matchRichFiles []matchRich
+	patternLimitReached := false
+	capacityLimitReached := false
 
 	// Set Total Limiter for Max Files to be consumed by host
 	totalCount := 0
-	// Set Total Limiter for Max Bandwidth to be consumed per run
-	var bandwidthCount int64
+	// Set Total Limiter for Max Capacity to be consumed per run
+	var capacityCount int64
 
 	// var patternCount map[string]int
 	patternCount := make(map[string]int)
@@ -448,7 +450,7 @@ func getFilePaths(conf structs.FileShot, verbose *bool, hashes []string, sortMet
 	for _, p := range conf.Files.Patterns.Positive {
 
 		if *verbose {
-			log.Printf("Collecting files from: %s.", p)
+			log.Printf("Collecting files from: %s", p)
 		}
 
 		// Expand the pattern to a list of matching file paths
@@ -481,7 +483,7 @@ func getFilePaths(conf structs.FileShot, verbose *bool, hashes []string, sortMet
 
 				if result {
 					if *verbose {
-						log.Printf("[IGNORING] Negative pattern matched: %s %s.", negativePattern, filePath)
+						log.Printf("[IGNORING] Negative pattern matched: %s %s", negativePattern, filePath)
 					}
 					continue OUTERMATCH
 				}
@@ -526,14 +528,19 @@ func getFilePaths(conf structs.FileShot, verbose *bool, hashes []string, sortMet
 		var checkType bool
 
 		// If total files collected exceeds amount allowed per collection, finish.
-		if conf.Files.LimitTotal > 0 && totalCount > conf.Files.LimitTotal {
+		if conf.Files.LimitTotal > 0 && totalCount >= conf.Files.LimitTotal {
 			log.Printf("[LIMIT REACHED] Total file collection limit of %d reached.", conf.Files.LimitTotal)
-			continue
+			break
 		}
 
 		// If current path exceeds amount allowed per collection path, move onto next path.
-		if conf.Files.LimitPattern > 0 && patternCount[f.pattern] > conf.Files.LimitPattern {
-			log.Printf("[LIMIT REACHED] Total pattern collection limit of %d reached for %s.", conf.Files.LimitPattern, f.pattern)
+		// log.Println(f.pattern, patternCount[f.pattern])
+		if conf.Files.LimitPattern > 0 && patternCount[f.pattern] >= conf.Files.LimitPattern {
+			// Only show the pattern limit warning once per pattern
+			if !patternLimitReached {
+				log.Printf("[LIMIT REACHED] Total pattern collection limit of %d reached for %s.", conf.Files.LimitPattern, f.pattern)
+				patternLimitReached = true
+			}
 			continue
 		}
 
@@ -557,15 +564,20 @@ func getFilePaths(conf structs.FileShot, verbose *bool, hashes []string, sortMet
 		if !(conf.Files.Minsize < 0) && conf.Files.Maxsize > 0 {
 			if !checkSize {
 				if *verbose {
-					log.Printf("[IGNORING] File size (%d) is not within configured Minsize (%d) and Maxsize (%d): %s.", fileSize, int64(conf.Files.Minsize), int64(conf.Files.Maxsize), f.filePath)
+					log.Printf("[IGNORING] File size (%d) is not within configured Minsize (%d) and Maxsize (%d): %s", fileSize, int64(conf.Files.Minsize), int64(conf.Files.Maxsize), f.filePath)
 				}
 				continue
 			}
 		}
 
-		// If current path exceeds amount of bandwidth allowed, move onto next path.
-		if conf.Files.LimitBandwidth > 0 && (bandwidthCount+f.fileSize) > conf.Files.LimitBandwidth {
-			log.Printf("[LIMIT REACHED] File would exceed total bandwidth limit of %d reached on %s.", conf.Files.LimitBandwidth, f.filePath)
+		// If current path exceeds amount of capacity allowed, move onto next path.
+		if conf.Files.LimitCapacity > 0 && (capacityCount+f.fileSize) > conf.Files.LimitCapacity {
+			if !capacityLimitReached {
+				capacityLimitReached = true
+			}
+			if *verbose {
+				log.Printf("[LIMIT REACHED] File would exceed total capacity limit of %d on %s", conf.Files.LimitCapacity, f.filePath)
+			}
 			continue
 		}
 
@@ -605,13 +617,19 @@ func getFilePaths(conf structs.FileShot, verbose *bool, hashes []string, sortMet
 
 		// Iterate Limiter Counts
 		totalCount += 1
-		patternCount[f.pattern] =+ 1
-		bandwidthCount += fileSize
+		patternCount[f.pattern] += 1
+		capacityCount += fileSize
 
 		paths = append(paths, f.filePath)
 
 		log.Printf("File staged for submission: %s %d %s %s %s", f.filePath, fileSize, fileType, fileHash, modTime)
 	}
 
+	if capacityLimitReached {
+		log.Printf("[LIMIT REACHED] File collection was limited by total capacity limit of %d. Use verbose logging mode to learn more.", conf.Files.LimitCapacity)
+	}
+
+	log.Printf("Collected %d total files.", totalCount)
+
 	return paths
 }

src/go/pkg/structs/structs.go

@@ -62,7 +62,7 @@ type ConfFiles struct {
 	Maxsize        int64    // optional
 	LimitPattern   int      // optional
 	LimitTotal     int      // optional
-	LimitBandwidth int64    // optional
+	LimitCapacity  int64    // optional
 	Modified       int      // optional
 	Delete         bool     // optional
 	Gatekeeper     bool     // required