Skip to content

Commit

Permalink
Upgrade upstream zstd from v1.5.0 to v1.5.1
Browse files Browse the repository at this point in the history
  • Loading branch information
valyala committed Jan 12, 2022
1 parent 85e7e5e commit 8ab2e9b
Show file tree
Hide file tree
Showing 161 changed files with 14,974 additions and 5,922 deletions.
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ GOARCH ?= $(shell go env GOARCH)
GOOS_GOARCH := $(GOOS)_$(GOARCH)
GOOS_GOARCH_NATIVE := $(shell go env GOHOSTOS)_$(shell go env GOHOSTARCH)
LIBZSTD_NAME := libzstd_$(GOOS_GOARCH).a
ZSTD_VERSION ?= v1.5.0
MUSL_BUILDER_IMAGE=golang:1.17.1-alpine
ZSTD_VERSION ?= v1.5.1
MUSL_BUILDER_IMAGE=golang:1.17.6-alpine
BUILDER_IMAGE := local/builder_musl:2.0.0-$(shell echo $(MUSL_BUILDER_IMAGE) | tr : _)

.PHONY: libzstd.a
Expand Down
Binary file modified libzstd_linux_amd64.a
Binary file not shown.
Binary file modified libzstd_linux_arm.a
Binary file not shown.
Binary file modified libzstd_linux_arm64.a
Binary file not shown.
Binary file modified libzstd_linux_musl_amd64.a
Binary file not shown.
8 changes: 4 additions & 4 deletions zdict.h
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ extern "C" {
*
* Zstd can use dictionaries to improve compression ratio of small data.
* Traditionally small files don't compress well because there is very little
* repetion in a single sample, since it is small. But, if you are compressing
* repetition in a single sample, since it is small. But, if you are compressing
* many similar files, like a bunch of JSON records that share the same
* structure, you can train a dictionary on ahead of time on some samples of
* these files. Then, zstd can use the dictionary to find repetitions that are
Expand Down Expand Up @@ -132,7 +132,7 @@ extern "C" {
*
* # Benchmark levels 1-3 without a dictionary
* zstd -b1e3 -r /path/to/my/files
* # Benchmark levels 1-3 with a dictioanry
* # Benchmark levels 1-3 with a dictionary
* zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary
*
* When should I retrain a dictionary?
Expand Down Expand Up @@ -237,7 +237,6 @@ typedef struct {
* is presumed that the most profitable content is at the end of the dictionary,
* since that is the cheapest to reference.
*
* `dictContentSize` must be >= ZDICT_CONTENTSIZE_MIN bytes.
* `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN).
*
* @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`),
Expand Down Expand Up @@ -272,8 +271,9 @@ ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode);
* Use them only in association with static linking.
* ==================================================================================== */

#define ZDICT_CONTENTSIZE_MIN 128
#define ZDICT_DICTSIZE_MIN 256
/* Deprecated: Remove in v1.6.0 */
#define ZDICT_CONTENTSIZE_MIN 128

/*! ZDICT_cover_params_t:
* k and d are the only required parameters.
Expand Down
315 changes: 179 additions & 136 deletions zstd.h

Large diffs are not rendered by default.

45 changes: 13 additions & 32 deletions zstd/.circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,21 @@ jobs:
# preinstalled to reduce installation time.
docker:
- image: fbopensource/zstd-circleci-primary:0.0.1
# TODO: Re-enable aarch64 build:
# make aarch64build && make clean
steps:
- checkout
- run:
name: Test
command: |
./tests/test-license.py
cc -v; CFLAGS="-O0 -Werror -pedantic" make all && make clean
make c99build ; make clean
make c11build ; make clean
make aarch64build ; make clean
cc -v
CFLAGS="-O0 -Werror -pedantic" make allmost; make clean
make c99build; make clean
make c11build; make clean
make -j regressiontest; make clean
make shortest ; make clean
make cxxtest ; make clean
make shortest; make clean
make cxxtest; make clean
# the second half of the jobs are in this test
short-tests-1:
docker:
Expand Down Expand Up @@ -84,31 +86,10 @@ workflows:
commit:
jobs:
# Run the tests in parallel
- short-tests-0:
filters:
tags:
only: /.*/
- short-tests-1:
filters:
tags:
only: /.*/
# Create a branch called regression and set it to dev to force a
# regression test run
- regression-test:
filters:
branches:
only:
- regression
# Only run on release tags.
- publish-github-release:
requires:
- short-tests-0
- short-tests-1
filters:
branches:
ignore: /.*/
tags:
only: /^v\d+\.\d+\.\d+$/
- short-tests-0
- short-tests-1
- regression-test

nightly:
triggers:
- schedule:
Expand All @@ -120,7 +101,7 @@ workflows:
- dev
- master
jobs:
# Run daily long regression tests
# Run daily regression tests
- regression-test


Expand Down
240 changes: 240 additions & 0 deletions zstd/.github/workflows/dev-long-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,240 @@
name: dev-long-tests
# Tests longer than 10mn

concurrency:
group: long-${{ github.ref }}
cancel-in-progress: true

on:
pull_request:
branches: [ dev, release, actionsTest ]

jobs:
make-all:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: make all
run: make all

# lasts ~24mn
make-test:
runs-on: ubuntu-latest
env:
DEVNULLRIGHTS: 1
READFROMBLOCKDEVICE: 1
steps:
- uses: actions/checkout@v2
- name: make test
run: make test

# lasts ~26mn
make-test-osx:
runs-on: macos-latest
steps:
- uses: actions/checkout@v2
- name: OS-X test
run: make test # make -c lib all doesn't work because of the fact that it's not a tty

no-intrinsics-fuzztest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: no intrinsics fuzztest
run: MOREFLAGS="-DZSTD_NO_INTRINSICS" make -C tests fuzztest

tsan-zstreamtest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: thread sanitizer zstreamtest
run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream

# lasts ~15mn
tsan-fuzztest:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: thread sanitizer fuzztest
run: CC=clang make tsan-fuzztest

# lasts ~23mn
gcc-8-asan-ubsan-testzstd:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: gcc-8 + ASan + UBSan + Test Zstd
run: |
sudo apt-get -qqq update
make gcc8install
CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
gcc-asan-ubsan-testzstd-32bit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: ASan + UBSan + Test Zstd, 32bit mode
run: |
make libc6install
make -j uasan-test-zstd32 V=1
# Note : external libraries must be turned off when using MSAN tests,
# because they are not msan-instrumented,
# so any data coming from these libraries is always considered "uninitialized"

gcc-8-asan-ubsan-fuzz:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: gcc-8 + ASan + UBSan + Fuzz Test
run: |
make gcc8install
CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
gcc-asan-ubsan-fuzz32:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: ASan + UBSan + Fuzz Test 32bit
run: |
make libc6install
CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
asan-ubsan-regression:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: ASan + UBSan + Regression Test
run: make -j uasanregressiontest

msan-regression:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: MSan + Regression Test
run: make -j msanregressiontest

clang-msan-fuzz:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: clang + MSan + Fuzz Test
run: |
sudo apt-get -qqq update
sudo apt-get install clang
CC=clang FUZZER_FLAGS="--long-tests" make clean msan-fuzztest
# lasts ~24mn
clang-msan-testzstd:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: clang + MSan + Test Zstd
run: |
sudo apt-get update
sudo apt-get install clang
CC=clang make msan-test-zstd HAVE_ZLIB=0 HAVE_LZ4=0 HAVE_LZMA=0 V=1
armfuzz:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Qemu ARM emulation + Fuzz Test
run: |
sudo apt-get -qqq update
make arminstall
make armfuzz
valgrind-fuzz-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: valgrind + fuzz test stack mode # ~ 7mn
shell: 'script -q -e -c "bash {0}"'
run: |
make valgrindinstall
make -C tests valgrindTest
make clean
make -C tests test-fuzzer-stackmode
mingw-long-test:
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
include: [
{ compiler: clang, platform: x64, action: build, script: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion -Wno-unused-command-line-argument -Wno-implicit-int-float-conversion' make -j allzstd V=1"},
{ compiler: gcc, platform: x64, action: test, script: ""},
]
steps:
- uses: actions/checkout@v2
- name: Mingw long test
run: |
$env:PATH_ORIGINAL = $env:PATH
$env:PATH_MINGW32 = "C:\msys64\mingw32\bin"
$env:PATH_MINGW64 = "C:\msys64\mingw64\bin"
COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw32\bin\make.exe
COPY C:\msys64\usr\bin\make.exe C:\msys64\mingw64\bin\make.exe
IF ("${{matrix.platform}}" -eq "x64")
{
$env:PATH = $env:PATH_MINGW64 + ";" + $env:PATH_ORIGINAL
}
ELSEIF ("${{matrix.platform}}" -eq "x86")
{
$env:PATH = $env:PATH_MINGW32 + ";" + $env:PATH_ORIGINAL
}
IF ("${{matrix.action}}" -eq "build")
{
make -v
sh -c "${{matrix.compiler}} -v"
ECHO "Building zlib to static link"
$env:CC = "${{matrix.compiler}}"
sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib"
sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a"
ECHO "Building zstd"
$env:CPPFLAGS = "-I../../zlib"
$env:LDFLAGS = "../../zlib/libz.a"
sh -c "${{matrix.script}}"
}
ELSEIF ("${{matrix.action}}" -eq "test")
{
ECHO "Testing ${{matrix.compiler}} ${{matrix.platform}}"
$env:CC = "gcc"
$env:CXX = "g++"
MKDIR build\cmake\build
CD build\cmake\build
$env:FUZZERTEST = "-T2mn"
$env:ZSTREAM_TESTTIME = "-T2mn"
cmake -G "Visual Studio 14 2015 Win64" ..
cd ..\..\..
make clean
}
# lasts ~20mn
oss-fuzz:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
sanitizer: [address, undefined, memory]
steps:
- name: Build Fuzzers (${{ matrix.sanitizer }})
id: build
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
with:
oss-fuzz-project-name: 'zstd'
dry-run: false
sanitizer: ${{ matrix.sanitizer }}
- name: Run Fuzzers (${{ matrix.sanitizer }})
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
with:
oss-fuzz-project-name: 'zstd'
fuzz-seconds: 600
dry-run: false
sanitizer: ${{ matrix.sanitizer }}
- name: Upload Crash
uses: actions/upload-artifact@v1
if: failure() && steps.build.outcome == 'success'
with:
name: ${{ matrix.sanitizer }}-artifacts
path: ./out/artifacts
Loading

0 comments on commit 8ab2e9b

Please sign in to comment.