custom: increase default chunk size

pkg/handlers/apk_test.go

@@ -22,7 +22,7 @@ func TestAPKHandler(t *testing.T) {
 	}{
 		"apk_with_3_leaked_keys": {
 			archiveURL:     "https://github.com/joeleonjr/leakyAPK/raw/refs/heads/main/aws_leak.apk",
-			expectedChunks: 942,
+			expectedChunks: 178,
 			// Note: the secret count is 4 instead of 3 b/c we're not actually running the secret detection engine,
 			// we're just looking for a string match. There is one extra string match in the APK (but only 3 detected secrets).
 			expectedSecrets: 4,

pkg/handlers/ar_test.go

@@ -26,7 +26,7 @@ func TestHandleARFile(t *testing.T) {
 	dataOrErrChan := handler.HandleFile(context.AddLogger(ctx), rdr)
 	assert.NoError(t, err)
 
-	wantChunkCount := 102
+	wantChunkCount := 12
 	count := 0
 	for range dataOrErrChan {
 		count++

pkg/handlers/archive_test.go

@@ -64,7 +64,7 @@ func TestArchiveHandler(t *testing.T) {
 		},
 		"gzip-large": {
 			"https://raw.githubusercontent.com/bill-rich/bad-secrets/master/FifteenMB.gz",
-			1543,
+			155,
 			"AKIAYVP4CIPPH5TNP3SW",
 			false,
 		},

pkg/handlers/default_test.go

@@ -26,7 +26,7 @@ func TestHandleNonArchiveFile(t *testing.T) {
 	dataOrErrChan := handler.HandleFile(context.AddLogger(ctx), rdr)
 	assert.NoError(t, err)
 
-	wantChunkCount := 6
+	wantChunkCount := 1
 	count := 0
 	for range dataOrErrChan {
 		count++

pkg/handlers/handlers_test.go

@@ -67,7 +67,7 @@ func TestHandleHTTPJson(t *testing.T) {
 		assert.NoError(t, err)
 	}()
 
-	wantCount := 513
+	wantCount := 52
 	count := 0
 	for range chunkCh {
 		count++
@@ -91,7 +91,7 @@ func TestHandleHTTPJsonZip(t *testing.T) {
 		assert.NoError(t, err)
 	}()
 
-	wantCount := 513
+	wantCount := 52
 	count := 0
 	for range chunkCh {
 		count++
@@ -240,7 +240,7 @@ func TestExtractTarContent(t *testing.T) {
 		assert.NoError(t, err)
 	}()
 
-	wantCount := 4
+	wantCount := 1
 	count := 0
 	for range chunkCh {
 		count++
@@ -271,7 +271,7 @@ func TestNestedDirArchive(t *testing.T) {
 }
 
 func TestHandleFileRPM(t *testing.T) {
-	wantChunkCount := 179
+	wantChunkCount := 19
 	reporter := sources.ChanReporter{Ch: make(chan *sources.Chunk, wantChunkCount)}
 
 	file, err := os.Open("testdata/test.rpm")
@@ -283,7 +283,7 @@ func TestHandleFileRPM(t *testing.T) {
 }
 
 func TestHandleFileAR(t *testing.T) {
-	wantChunkCount := 102
+	wantChunkCount := 12
 	reporter := sources.ChanReporter{Ch: make(chan *sources.Chunk, wantChunkCount)}
 
 	file, err := os.Open("testdata/test.deb")
@@ -320,7 +320,7 @@ func BenchmarkHandleAR(b *testing.B) {
 }
 
 func TestHandleFileNonArchive(t *testing.T) {
-	wantChunkCount := 6
+	wantChunkCount := 1
 	reporter := sources.ChanReporter{Ch: make(chan *sources.Chunk, wantChunkCount)}
 
 	file, err := os.Open("testdata/nonarchive.txt")
@@ -345,7 +345,7 @@ func TestExtractTarContentWithEmptyFile(t *testing.T) {
 		assert.NoError(t, err)
 	}()
 
-	wantCount := 4
+	wantCount := 1
 	count := 0
 	for range chunkCh {
 		count++
@@ -417,7 +417,7 @@ func TestHandleLargeHTTPJson(t *testing.T) {
 		assert.NoError(t, err)
 	}()
 
-	wantCount := 5121
+	wantCount := 513
 	count := 0
 	for range chunkCh {
 		count++
@@ -497,7 +497,7 @@ func TestHandleGitCatFile(t *testing.T) {
 			fileName:       "largefile.bin",
 			fileSize:       50 * 1024 * 1024, // 50 MB
 			supportedType:  true,
-			expectedChunks: 5120,
+			expectedChunks: 512,
 		},
 		{
 			name:           "UnsupportedType",

pkg/handlers/rpm_test.go

@@ -26,7 +26,7 @@ func TestHandleRPMFile(t *testing.T) {
 	dataOrErrChan := handler.HandleFile(context.AddLogger(ctx), rdr)
 	assert.NoError(t, err)
 
-	wantChunkCount := 179
+	wantChunkCount := 19
 	count := 0
 	for range dataOrErrChan {
 		count++

pkg/sources/chunker.go

@@ -11,9 +11,9 @@ import (
 
 const (
 	// ChunkSize is the maximum size of a chunk.
-	ChunkSize = 10 * 1024
+	ChunkSize = 100 * 1024
 	// PeekSize is the size of the peek into the previous chunk.
-	PeekSize = 3 * 1024
+	PeekSize = 10 * 1024
 	// TotalChunkSize is the total size of a chunk with peek data.
 	TotalChunkSize = ChunkSize + PeekSize
 )