zstd: Allow to ignore checksum checking

There might be scenarios for which checksum validation is not necessary.
klauspost · Apr 27, 2022 · 90dccaa · 90dccaa
1 parent 595e86d
commit 90dccaa
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 8 deletions.
diff --git a/zstd/decoder.go b/zstd/decoder.go
@@ -451,7 +451,7 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) {
 		got := d.current.crc.Sum64()
 		var tmp [4]byte
 		binary.LittleEndian.PutUint32(tmp[:], uint32(got))
-		if !bytes.Equal(tmp[:], next.d.checkCRC) && !ignoreCRC {
+		if !d.o.ignoreChecksum && !bytes.Equal(tmp[:], next.d.checkCRC) && !ignoreCRC {
 			if debugDecoder {
 				println("CRC Check Failed:", tmp[:], " (got) !=", next.d.checkCRC, "(on stream)")
 			}

diff --git a/zstd/decoder_options.go b/zstd/decoder_options.go
@@ -19,6 +19,7 @@ type decoderOptions struct {
 	maxDecodedSize uint64
 	maxWindowSize  uint64
 	dicts          []dict
+	ignoreChecksum bool
 }
 
 func (o *decoderOptions) setDefault() {
@@ -112,3 +113,11 @@ func WithDecoderMaxWindow(size uint64) DOption {
 		return nil
 	}
 }
+
+// IgnoreChecksum allows to forcibly ignore checksum checking.
+func IgnoreChecksum(b bool) DOption {
+	return func(o *decoderOptions) error {
+		o.ignoreChecksum = b
+		return nil
+	}
+}
diff --git a/zstd/decoder_test.go b/zstd/decoder_test.go
@@ -1737,6 +1737,57 @@ func TestResetNil(t *testing.T) {
 	}
 }
 
+func TestIgnoreChecksum(t *testing.T) {
+	// zstd file containing text "compress\n" and has a xxhash checksum
+	zstdBlob := []byte{0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x09, 0x49, 0x00, 0x00, 'C', 'o', 'm', 'p', 'r', 'e', 's', 's', '\n', 0x79, 0x6e, 0xe0, 0xd2}
+
+	// replace letter 'c' with 'C', so decoding should fail.
+	zstdBlob[9] = 'C'
+
+	{
+		// Check if the file is indeed incorrect
+		dec, err := NewReader(nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer dec.Close()
+
+		dec.Reset(bytes.NewBuffer(zstdBlob))
+
+		_, err = ioutil.ReadAll(dec)
+		if err == nil {
+			t.Fatal("Expected decoding error")
+		}
+
+		if !errors.Is(err, ErrCRCMismatch) {
+			t.Fatalf("Expected checksum error, got '%s'", err)
+		}
+	}
+
+	{
+		// Ignore CRC error and decompress the content
+		dec, err := NewReader(nil, IgnoreChecksum(true))
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer dec.Close()
+
+		dec.Reset(bytes.NewBuffer(zstdBlob))
+
+		res, err := ioutil.ReadAll(dec)
+		if err != nil {
+			t.Fatalf("Unexpected error: '%s'", err)
+		}
+
+		want := []byte{'C', 'o', 'm', 'p', 'r', 'e', 's', 's', '\n'}
+		if !bytes.Equal(res, want) {
+			t.Logf("want: %s", want)
+			t.Logf("got:  %s", res)
+			t.Fatalf("Wrong output")
+		}
+	}
+}
+
 func timeout(after time.Duration) (cancel func()) {
 	if isRaceTest {
 		return func() {}

diff --git a/zstd/framedec.go b/zstd/framedec.go
@@ -290,13 +290,6 @@ func (d *frameDec) checkCRC() error {
 	if !d.HasCheckSum {
 		return nil
 	}
-	var tmp [4]byte
-	got := d.crc.Sum64()
-	// Flip to match file order.
-	tmp[0] = byte(got >> 0)
-	tmp[1] = byte(got >> 8)
-	tmp[2] = byte(got >> 16)
-	tmp[3] = byte(got >> 24)
 
 	// We can overwrite upper tmp now
 	want, err := d.rawInput.readSmall(4)
@@ -305,6 +298,18 @@ func (d *frameDec) checkCRC() error {
 		return err
 	}
 
+	if d.o.ignoreChecksum {
+		return nil
+	}
+
+	var tmp [4]byte
+	got := d.crc.Sum64()
+	// Flip to match file order.
+	tmp[0] = byte(got >> 0)
+	tmp[1] = byte(got >> 8)
+	tmp[2] = byte(got >> 16)
+	tmp[3] = byte(got >> 24)
+
 	if !bytes.Equal(tmp[:], want) && !ignoreCRC {
 		if debugDecoder {
 			println("CRC Check Failed:", tmp[:], "!=", want)