Use external LibLZ4, detected using standard pkg-config (ME-768)

.circleci/config.yml

@@ -0,0 +1,66 @@
+# Check https://circleci.com/docs/2.0/language-go/ for more details
+version: 2
+jobs:
+  test-lz4-stretch:
+    docker:
+      - image: circleci/golang:1.13-stretch
+
+    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
+    steps:
+      - run:
+          name: Install liblz4-dev package
+          command: |
+            sudo apt-get update -qy
+            sudo apt-get install -y --no-install-recommends liblz4-dev
+      - checkout
+      - run: go test -v ./...
+
+  test-lz4-source:
+    docker:
+      - image: circleci/golang:1.13-stretch
+
+    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
+
+    steps:
+      - run:
+          name: Build LZ4 from source
+          environment:
+            LZ4_VERSION: 1.9.2
+            LZ4_TARBALL_SHA256SUM: 658ba6191fa44c92280d4aa2c271b0f4fbc0e34d249578dd05e50e76d0e5efcc
+            _builddir: /tmp/build
+            _tmptgz: /tmp/liblz4.tgz
+          command: |
+            curl -L https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz -o ${_tmptgz}
+            echo "${LZ4_TARBALL_SHA256SUM} ${_tmptgz}" | sha256sum -c -
+            mkdir ${_builddir}
+            tar xzf /tmp/liblz4.tgz -C ${_builddir} --strip-components=1
+            make -C ${_builddir}
+            sudo make -C ${_builddir} install
+            rm -rf ${_tmptgz} ${_builddir}
+      - checkout
+      - run: go test -v ./...
+
+  lint:
+    docker:
+      - image: circleci/golang:1.13-stretch
+    working_directory: /go/src/github.com/{{ORG_NAME}}/{{REPO_NAME}}
+    steps:
+      - checkout
+      - run:
+          name: Check for files that need go fmt
+          command: |
+            needs_gofmt=$(gofmt -l -e .)
+            if test "${needs_gofmt}"
+            then
+              echo "The following files are not properly formatted. Run "go fmt" and commit those changes."
+              echo ${needs_gofmt}
+              exit 1
+            fi
+
+workflows:
+  version: 2
+  tests:
+    jobs:
+      - test-lz4-stretch
+      - test-lz4-source
+      - lint

CHANGELOG.md

@@ -1,5 +1,14 @@
 # golz4 CHANGELOG
 
+## v1.0.0
+
+While this release **does not break API compatibility**, it changes the way the library is built.
+Starting with this version, the C source code for `liblz4` **is not included in the Go package anymore**.
+
+**The `liblz4` needs to be provided externally**, using a package manager or a manual, from source installation, for example.
+
+Detection of `liblz4` now relies on `pkg-config` to add the correct `CFLAGS` and `LDFLAGS`.
+
 ## v0.0.131
 
 * Initial release, using lz4 source code version r131

README.md

@@ -1,14 +1,14 @@
 golz4
 =====
 
-[![Godoc](http://img.shields.io/badge/godoc-reference-blue.svg?style=flat)](https://godoc.org/github.com/DataDog/golz4) [![license](http://img.shields.io/badge/license-BSD-red.svg?style=flat)](https://raw.githubusercontent.com/DataDog/golz4/master/LICENSE)
+[![Godoc](http://img.shields.io/badge/godoc-reference-blue.svg?style=flat)](https://godoc.org/github.com/DataDog/golz4) [![license](http://img.shields.io/badge/license-BSD-red.svg?style=flat)](https://raw.githubusercontent.com/DataDog/golz4/master/LICENSE) [![CircleCI](https://circleci.com/gh/DataDog/golz4.svg?style=svg)](https://circleci.com/gh/DataDog/golz4)
 
 Golang interface to LZ4 compression.
 
 Forked from `github.com/cloudflare/golz4` but with significant differences:
 
 * input/output arg order has been swapped to follow Go convention, ie `Compress(in, out)` -> `Compress(out, in)`
-* lz4 131 used which fixes [several segfaults](https://github.com/cloudflare/golz4/pull/7)
+* builds against the liblz4 that it detects using pkgconfig
 
 Benchmark 
 ```
@@ -17,3 +17,24 @@ BenchmarkCompressUncompress-8   	20000000	        62.4 ns/op	 688.60 MB/s
 BenchmarkStreamCompress-8       	   50000	     32842 ns/op	2003.41 MB/s	  278537 B/op	       4 allocs/op
 BenchmarkStreamUncompress-8     	  500000	      2867 ns/op	22855.34 MB/s	      52 B/op	       2 allocs/op
 ```
+
+Building
+--------
+
+Building `golz4` requires that [lz4](https://github.com/lz4/lz4) library be available.
+
+On Debian or Ubuntu, this is as easy as running
+
+```
+$ sudo apt-get install liblz4-dev
+```
+
+On MacOS
+
+```
+$ brew install lz4
+```
+
+If the library version provided for your OS is too old and does not include a `liblz4.pc` pkg-config file, the [upstream documentation](https://github.com/lz4/lz4#installation) describes how to build and install from source.
+
+_NOTE_: if `lz4` is not installed in standard directories, setting `PKG_CONFIG_PATH` environment variable with the directory containing the `liblz4.pc` file will help.

header_test.go

@@ -25,8 +25,13 @@ func TestCompressHdrRatio(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if want := corpusSize + 4; want != outSize {
-		t.Fatalf("Compressed output length != expected: %d != %d", outSize, want)
+	outputNoHdr := make([]byte, CompressBoundHdr(input))
+	outNoHdrSize, err := Compress(outputNoHdr, input)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if outSize != outNoHdrSize+4 {
+		t.Fatalf("Compressed output length != expected: %d != %d", outSize, outNoHdrSize+4)
 	}
 }
 

lz4.go

@@ -1,8 +1,8 @@
 package lz4
 
-// #cgo CFLAGS: -O3
-// #include "src/lz4.h"
-// #include "src/lz4.c"
+// #cgo pkg-config: liblz4
+// #include <lz4.h>
+// #include <stdlib.h>
 import "C"
 
 import (

lz4_hc.go

@@ -1,8 +1,7 @@
 package lz4
 
-// #cgo CFLAGS: -O3
-// #include "src/lz4hc.h"
-// #include "src/lz4hc.c"
+// #cgo pkg-config: liblz4
+// #include <lz4hc.h>
 import "C"
 
 import (

lz4_hc_test.go

@@ -23,8 +23,12 @@ func TestCompressionHCRatio(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if want := 4317; want != outSize {
-		t.Fatalf("HC Compressed output length != expected: %d != %d", want, outSize)
+	// Should be at most 85% of the input size
+	maxSize := 85 * len(input) / 100
+
+	if outSize > maxSize {
+		t.Fatalf("HC Compressed output length should be at most 85%% of input size: input=%d, compressed=%d, maxExpected=%d",
+			len(input), outSize, maxSize)
 	}
 }
 
@@ -34,40 +38,22 @@ func TestCompressionHCLevels(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cases := []struct {
-		Level   int
-		Outsize int
-	}{
-		{0, 4317},
-		{1, 4415},
-		{2, 4359},
-		{3, 4339},
-		{4, 4321},
-		{5, 4317},
-		{6, 4317},
-		{7, 4317},
-		{8, 4317},
-		{9, 4317},
-		{10, 4317},
-		{11, 4317},
-		{12, 4317},
-		{13, 4317},
-		{14, 4317},
-		{15, 4317},
-		{16, 4317},
-	}
-
-	for _, tt := range cases {
+	// Should be at most 85% of the input size
+	previousCompressedSize := 85 * len(input) / 100
+
+	// NOTE: lvl == 0 means auto, 1 worst, 16 best
+	for lvl := 1; lvl <= 16; lvl++ {
 		output := make([]byte, CompressBound(input))
-		outSize, err := CompressHCLevel(output, input, tt.Level)
+		outSize, err := CompressHCLevel(output, input, lvl)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		if want := tt.Outsize; want != outSize {
-			t.Errorf("HC level %d length != expected: %d != %d",
-				tt.Level, want, outSize)
+		if outSize > previousCompressedSize {
+			t.Errorf("HC level %d should lead to a better or equal compression than HC level %d (previousSize=%d, currentSize=%d)",
+				lvl, lvl-1, previousCompressedSize, outSize)
 		}
+		previousCompressedSize = outSize
 	}
 }
 

lz4_test.go

@@ -18,8 +18,6 @@ import (
 	"testing/quick"
 )
 
-const corpusSize = 4746
-
 var plaintext0 = []byte("jkoedasdcnegzb.,ewqegmovobspjikodecedegds[]")
 
 func failOnError(t *testing.T, msg string, err error) {
@@ -40,8 +38,9 @@ func TestCompressionRatio(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if want := corpusSize; want != outSize {
-		t.Fatalf("Compressed output length != expected: %d != %d", outSize, want)
+	maxCompressedSize := 100 * len(input) / 85
+	if outSize >= maxCompressedSize {
+		t.Fatalf("Compressed output length %d should be smaller than %d", outSize, maxCompressedSize)
 	}
 }