[feat] - tmp file diffs

pkg/gitparse/gitparse.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
-	"github.com/go-logr/logr"
 	"io"
 	"os"
 	"os/exec"
@@ -13,8 +12,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-logr/logr"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/writers/buffered_file_writer"
 )
 
 const (
@@ -42,10 +44,45 @@ type Commit struct {
 type Diff struct {
 	PathB     string
 	LineStart int
-	Content   bytes.Buffer
-	IsBinary  bool
+	// Used to keep small diffs in memory and larger diffs in a file.
+	contentWriter *bufferedfilewriter.BufferedFileWriter
+	IsBinary      bool
+}
+
+type diffOption func(*Diff)
+
+// withPathB sets the PathB option.
+func withPathB(pathB string) diffOption { return func(d *Diff) { d.PathB = pathB } }
+
+// NewDiff creates a new Diff with a threshold.
+func NewDiff(opts ...diffOption) *Diff {
+	const defaultThreshold = 20 * 1024 * 1024 // 20MB
+	d := new(Diff)
+	for _, opt := range opts {
+		opt(d)
+	}
+	d.contentWriter = bufferedfilewriter.New(bufferedfilewriter.WithThreshold(defaultThreshold))
+
+	return d
+}
+
+// Len returns the length of the storage.
+func (d *Diff) Len() int { return d.contentWriter.Len() }
+
+// ReadCloser returns a ReadCloser for the contentWriter.
+func (d *Diff) ReadCloser() (io.ReadCloser, error) { return d.contentWriter.ReadCloser() }
+
+// write delegates to the contentWriter.
+func (d *Diff) write(ctx context.Context, p []byte) error {
+	_, err := d.contentWriter.Write(ctx, p)
+	return err
 }
 
+// finalize ensures proper closure of resources associated with the Diff.
+// handle the final flush in the finalize method, in case there's data remaining in the buffer.
+// This method should be called to release resources, especially when writing to a file.
+func (d *Diff) finalize() error { return d.contentWriter.Close() }
+
 // Parser sets values used in GitParse.
 type Parser struct {
 	maxDiffSize   int
@@ -152,14 +189,13 @@ func (c1 *Commit) Equal(c2 *Commit) bool {
 			return false
 		case d1.LineStart != d2.LineStart:
 			return false
-		case d1.Content.String() != d2.Content.String():
+		case d1.contentWriter.String() != d2.contentWriter.String():
 			return false
 		case d1.IsBinary != d2.IsBinary:
 			return false
 		}
 	}
 	return true
-
 }
 
 // RepoPath parses the output of the `git log` command for the `source` path.
@@ -254,11 +290,11 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 	outReader := bufio.NewReader(stdOut)
 	var (
 		currentCommit *Commit
-		currentDiff   Diff
 
 		totalLogSize int
 	)
 	var latestState = Initial
+	currentDiff := NewDiff()
 
 	defer common.RecoverWithExit(ctx)
 	defer close(commitChan)
@@ -277,20 +313,21 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			latestState = CommitLine
 
 			// If there is a currentDiff, add it to currentCommit.
-			if currentDiff.Content.Len() > 0 || currentDiff.IsBinary {
-				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
-				currentCommit.Size += currentDiff.Content.Len()
+			if currentDiff.Len() > 0 || currentDiff.IsBinary {
+				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+				currentCommit.Size += currentDiff.Len()
 			}
 			// If there is a currentCommit, send it to the channel.
 			if currentCommit != nil {
+				if err := currentDiff.finalize(); err != nil {
+					ctx.Logger().Error(err, "failed to finalize diff")
+				}
 				commitChan <- *currentCommit
 				totalLogSize += currentCommit.Size
 			}
 			// Create a new currentDiff and currentCommit
-			currentDiff = Diff{}
-			currentCommit = &Commit{
-				Message: strings.Builder{},
-			}
+			currentDiff = NewDiff()
+			currentCommit = &Commit{Message: strings.Builder{}}
 			// Check that the commit line contains a hash and set it.
 			if len(line) >= 47 {
 				currentCommit.Hash = string(line[7:47])
@@ -326,12 +363,15 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			if currentCommit == nil {
 				currentCommit = &Commit{}
 			}
-			if currentDiff.Content.Len() > 0 || currentDiff.IsBinary {
-				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
+			if currentDiff.Len() > 0 || currentDiff.IsBinary {
+				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+				if err := currentDiff.finalize(); err != nil {
+					ctx.Logger().Error(err, "failed to finalize diff")
+				}
 				// If the currentDiff is over 1GB, drop it into the channel so it isn't held in memory waiting for more commits.
 				totalSize := 0
 				for _, diff := range currentCommit.Diffs {
-					totalSize += diff.Content.Len()
+					totalSize += diff.Len()
 				}
 				if totalSize > c.maxCommitSize {
 					oldCommit := currentCommit
@@ -348,7 +388,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 					currentCommit.Message.WriteString(oldCommit.Message.String())
 				}
 			}
-			currentDiff = Diff{}
+			currentDiff = NewDiff()
 		case isModeLine(isStaged, latestState, line):
 			latestState = ModeLine
 			// NoOp
@@ -375,12 +415,10 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 		case isHunkLineNumberLine(isStaged, latestState, line):
 			latestState = HunkLineNumberLine
 
-			if currentDiff.Content.Len() > 0 || currentDiff.IsBinary {
-				currentCommit.Diffs = append(currentCommit.Diffs, currentDiff)
-			}
-			currentDiff = Diff{
-				PathB: currentDiff.PathB,
+			if currentDiff.Len() > 0 || currentDiff.IsBinary {
+				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 			}
+			currentDiff = NewDiff(withPathB(currentDiff.PathB))
 
 			words := bytes.Split(line, []byte(" "))
 			if len(words) >= 3 {
@@ -395,24 +433,21 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 				latestState = HunkContentLine
 			}
 			// TODO: Why do we care about this? It creates empty lines in the diff. If there are no plusLines, it's just newlines.
-			currentDiff.Content.Write([]byte("\n"))
+			if err := currentDiff.write(ctx, []byte("\n")); err != nil {
+				ctx.Logger().Error(err, "failed to write to diff")
+			}
 		case isHunkPlusLine(isStaged, latestState, line):
 			if latestState != HunkContentLine {
 				latestState = HunkContentLine
 			}
 
-			currentDiff.Content.Write(line[1:])
-		case isHunkMinusLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
+			if err := currentDiff.write(ctx, line[1:]); err != nil {
+				ctx.Logger().Error(err, "failed to write to diff")
 			}
 			// NoOp. We only care about additions.
-		case isHunkNewlineWarningLine(isStaged, latestState, line):
-			if latestState != HunkContentLine {
-				latestState = HunkContentLine
-			}
-			// NoOp
-		case isHunkEmptyLine(isStaged, latestState, line):
+		case isHunkMinusLine(isStaged, latestState, line),
+			isHunkNewlineWarningLine(isStaged, latestState, line),
+			isHunkEmptyLine(isStaged, latestState, line):
 			if latestState != HunkContentLine {
 				latestState = HunkContentLine
 			}
@@ -439,14 +474,14 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			latestState = ParseFailure
 		}
 
-		if currentDiff.Content.Len() > c.maxDiffSize {
+		if currentDiff.Len() > c.maxDiffSize {
 			ctx.Logger().V(2).Info(fmt.Sprintf(
 				"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
 			))
 			break
 		}
 	}
-	cleanupParse(currentCommit, &currentDiff, commitChan, &totalLogSize)
+	cleanupParse(currentCommit, currentDiff, commitChan, &totalLogSize)
 
 	ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize)
 }
@@ -718,7 +753,7 @@ func isCommitSeparatorLine(isStaged bool, latestState ParseState, line []byte) b
 
 func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) {
 	// Ignore empty or binary diffs (this condition may be redundant).
-	if currentDiff != nil && (currentDiff.Content.Len() > 0 || currentDiff.IsBinary) {
+	if currentDiff != nil && (currentDiff.Len() > 0 || currentDiff.IsBinary) {
 		currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 	}
 	if currentCommit != nil {