Import unmodified ZStandard v1.4.4

This imports the unmodified ZStandard source to contrib/ which will be used by ZFS. This code shall not be modified in any way to keep it easily updatable. Only the required files from lib/ are imported. contrib/zstd is excluded from codecov calculation as dependencies don't need full codecov. Signed-off-by: Sebastian Gottschall <[email protected]>
openzfs · May 1, 2020 · 60fe768 · 60fe768
1 parent ac00df4
commit 60fe768
Show file tree

Hide file tree

Showing 84 changed files with 7,488 additions and 2,666 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -104,7 +104,8 @@ commitcheck:
 	fi
 
 cstyle:
-	@find ${top_srcdir} -name build -prune -o -type f -name '*.[hc]' \
+	@find ${top_srcdir} \( -name build -o -path ./contrib/zstd \) -prune \
+		-o -name '*.[hc]' \
 		! -name 'zfs_config.*' ! -name '*.mod.c' \
 		! -name 'opt_global.h' ! -name '*_if*.h' \
 		-exec ${top_srcdir}/scripts/cstyle.pl -cpP {} \+

diff --git a/config/ax_code_coverage.m4 b/config/ax_code_coverage.m4
@@ -222,7 +222,7 @@ CODE_COVERAGE_GENHTML_OPTIONS ?= $(CODE_COVERAGE_GENHTML_OPTIONS_DEFAULT)
 
 # Add any folders you want to ignore here
 # Ignore tmp and tests themselves
-CODE_COVERAGE_IGNORE_PATTERN ?= "/tmp/*" "*/tests/*"  
+CODE_COVERAGE_IGNORE_PATTERN ?= "/tmp/*" "*/tests/*" "*/contrib/zstd/*"
 
 GITIGNOREFILES ?=
 GITIGNOREFILES += $(CODE_COVERAGE_OUTPUT_FILE) $(CODE_COVERAGE_OUTPUT_DIRECTORY)

diff --git a/configure.ac b/configure.ac
@@ -98,6 +98,10 @@ AC_CONFIG_FILES([
 	contrib/pyzfs/Makefile
 	contrib/pyzfs/setup.py
 	contrib/zcp/Makefile
+	contrib/zstd/Makefile
+	contrib/zstd/common/Makefile
+	contrib/zstd/compress/Makefile
+	contrib/zstd/decompress/Makefile
 	etc/Makefile
 	etc/default/Makefile
 	etc/init.d/Makefile
@@ -163,6 +167,7 @@ AC_CONFIG_FILES([
 	lib/libzfs/libzfs_core.pc
 	lib/libzfs_core/Makefile
 	lib/libzpool/Makefile
+	lib/libzstd/Makefile
 	lib/libzutil/Makefile
 	man/Makefile
 	man/man1/Makefile

diff --git a/contrib/Makefile.am b/contrib/Makefile.am
@@ -1,5 +1,5 @@
-SUBDIRS = bash_completion.d pyzfs zcp
+SUBDIRS = bash_completion.d pyzfs zcp zstd
 if BUILD_LINUX
 SUBDIRS += bpftrace dracut initramfs
 endif
-DIST_SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp
+DIST_SUBDIRS = bash_completion.d bpftrace dracut initramfs pyzfs zcp zstd
diff --git a/contrib/zstd/.gitignore b/contrib/zstd/.gitignore
@@ -0,0 +1 @@
+*.o.cmd
diff --git a/contrib/zstd/Makefile.am b/contrib/zstd/Makefile.am
@@ -0,0 +1,2 @@
+SUBDIRS = common compress decompress
+EXTRA_DIST = common compress decompress zstd.h
diff --git a/contrib/zstd/README.md b/contrib/zstd/README.md
@@ -0,0 +1,24 @@
+# ZSTD Contrib Library Manual
+
+## Introduction
+
+This `contrib` contains the ZSTD library used in ZFS. It is heavily cut-down by
+dropping any unneeded files but otherwise is intentionally unmodified. Please do
+not alter these files in any way, besides upgrading to a newer ZSTD release.
+
+## Updating ZSTD
+
+To update ZSTD the following steps need to be taken:
+
+1. Grab the latest release of [ZSTD](https://github.com/facebook/zstd/releases).
+2. Replace (not merge) `common`, `compress`, `decompress` and `zstd.h` with the
+   new versions from `lib/`.
+3. Make sure any newly required files and/or folders are also included.
+   1. Add an empty file `Makefile.am` inside any new folder.
+   2. Add them to `AC_CONFIG_FILES` in `configure.ac` accordingly.
+   3. Make sure new files/folders are listed in
+      - `contrib/zstd/Makefile.am`
+      - `lib/libzstd/Makefile.am`
+      - `module/zstd/Makefile.in`
+      - this README
+4. Update the version `ZFS_MODULE_VERSION("x.y.z")` in `module/zstd/zstd.c`.
diff --git a/contrib/zstd/THIRDPARTYLICENSE b/contrib/zstd/THIRDPARTYLICENSE
@@ -0,0 +1,30 @@
+BSD License
+
+For Zstandard software
+
+Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+ * Neither the name Facebook nor the names of its contributors may be used to
+   endorse or promote products derived from this software without specific
+   prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/contrib/zstd/THIRDPARTYLICENSE.gplv2 b/contrib/zstd/THIRDPARTYLICENSE.gplv2
diff --git a/contrib/zstd/common/Makefile.am b/contrib/zstd/common/Makefile.am
diff --git a/include/sys/zstd/bitstream.h → contrib/zstd/common/bitstream.h b/include/sys/zstd/bitstream.h → contrib/zstd/common/bitstream.h
@@ -1,4 +1,3 @@
-/* BEGIN CSTYLED */
 /* ******************************************************************
    bitstream
    Part of FSE library
@@ -48,14 +47,19 @@ extern "C" {
 /*-****************************************
 *  Dependencies
 ******************************************/
-#include <sys/zstd/mem.h>            /* unaligned access routines */
-#include <sys/zstd/debug.h>          /* assert(), DEBUGLOG(), RAWLOG() */
-#include <sys/zstd/error_private.h>  /* error codes and messages */
+#include "mem.h"            /* unaligned access routines */
+#include "debug.h"          /* assert(), DEBUGLOG(), RAWLOG() */
+#include "error_private.h"  /* error codes and messages */
 
 
 /*=========================================
 *  Target specific
 =========================================*/
+#if defined(__BMI__) && defined(__GNUC__)
+#  include <immintrin.h>   /* support for bextr (experimental) */
+#elif defined(__ICCARM__)
+#  include <intrinsics.h>
+#endif
 
 #define STREAM_ACCUMULATOR_MIN_32  25
 #define STREAM_ACCUMULATOR_MIN_64  57
@@ -155,7 +159,27 @@ MEM_STATIC unsigned BIT_highbit32 (U32 val)
 {
     assert(val != 0);
     {
+#   if defined(_MSC_VER)   /* Visual */
+        unsigned long r=0;
+        _BitScanReverse ( &r, val );
+        return (unsigned) r;
+#   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* Use GCC Intrinsic */
         return __builtin_clz (val) ^ 31;
+#   elif defined(__ICCARM__)    /* IAR Intrinsic */
+        return 31 - __CLZ(val);
+#   else   /* Software version */
+        static const unsigned DeBruijnClz[32] = { 0,  9,  1, 10, 13, 21,  2, 29,
+                                                 11, 14, 16, 18, 22, 25,  3, 30,
+                                                  8, 12, 20, 28, 15, 17, 24,  7,
+                                                 19, 27, 23,  6, 26,  5,  4, 31 };
+        U32 v = val;
+        v |= v >> 1;
+        v |= v >> 2;
+        v |= v >> 4;
+        v |= v >> 8;
+        v |= v >> 16;
+        return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
+#   endif
     }
 }
 
@@ -434,4 +458,3 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
 #endif
 
 #endif /* BITSTREAM_H_MODULE */
-/* END CSTYLED */
diff --git a/include/sys/zstd/compiler.h → contrib/zstd/common/compiler.h b/include/sys/zstd/compiler.h → contrib/zstd/common/compiler.h
@@ -1,4 +1,3 @@
-/* BEGIN CSTYLED */
 /*
  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
  * All rights reserved.
@@ -16,19 +15,27 @@
 *  Compiler specifics
 *********************************************************/
 /* force inlining */
+
+#if !defined(ZSTD_NO_INLINE)
+#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
 #  define INLINE_KEYWORD inline
-#  define FORCE_INLINE_ATTR __always_inline
+#else
+#  define INLINE_KEYWORD
+#endif
 
-/* vectorization
- * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */
-#if !defined(__clang__) && defined(__GNUC__)
-#  if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5)
-#    define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
-#  else
-#    define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")")
-#  endif
+#if defined(__GNUC__) || defined(__ICCARM__)
+#  define FORCE_INLINE_ATTR __attribute__((always_inline))
+#elif defined(_MSC_VER)
+#  define FORCE_INLINE_ATTR __forceinline
 #else
-#  define DONT_VECTORIZE
+#  define FORCE_INLINE_ATTR
+#endif
+
+#else
+
+#define INLINE_KEYWORD
+#define FORCE_INLINE_ATTR
+
 #endif
 
 /**
@@ -61,28 +68,22 @@
 #  define UNUSED_ATTR
 #endif
 
-#ifdef _KERNEL
 /* force no inlining */
-#  ifdef __GNUC__
-#    define FORCE_NOINLINE static noinline
-#  else
-#    define FORCE_NOINLINE static
-#  endif
+#ifdef _MSC_VER
+#  define FORCE_NOINLINE static __declspec(noinline)
 #else
-#  ifdef __GNUC__
-#    define FORCE_NOINLINE static __attribute__((noinline))
+#  if defined(__GNUC__) || defined(__ICCARM__)
+#    define FORCE_NOINLINE static __attribute__((__noinline__))
 #  else
 #    define FORCE_NOINLINE static
 #  endif
-
-
 #endif
 
 /* target attribute */
 #ifndef __has_attribute
   #define __has_attribute(x) 0  /* Compatibility with non-clang compilers. */
 #endif
-#if defined(__GNUC__)
+#if defined(__GNUC__) || defined(__ICCARM__)
 #  define TARGET_ATTRIBUTE(target) __attribute__((__target__(target)))
 #else
 #  define TARGET_ATTRIBUTE(target)
@@ -102,13 +103,18 @@
   #  define DYNAMIC_BMI2 0
   #endif
 #endif
-/* prefetch 
+
+/* prefetch
  * can be disabled, by declaring NO_PREFETCH build macro */
 #if defined(NO_PREFETCH)
 #  define PREFETCH_L1(ptr)  (void)(ptr)  /* disabled */
 #  define PREFETCH_L2(ptr)  (void)(ptr)  /* disabled */
 #else
-#  if defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
+#  if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86))  /* _mm_prefetch() is not defined outside of x86/x64 */
+#    include <mmintrin.h>   /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
+#    define PREFETCH_L1(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
+#    define PREFETCH_L2(ptr)  _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
+#  elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
 #    define PREFETCH_L1(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
 #    define PREFETCH_L2(ptr)  __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
 #  else
@@ -117,11 +123,7 @@
 #  endif
 #endif  /* NO_PREFETCH */
 
-#if defined(__x86_64__) || defined(__i386__)
 #define CACHELINE_SIZE 64
-#else
-#define CACHELINE_SIZE 32
-#endif
 
 #define PREFETCH_AREA(p, s)  {            \
     const char* const _ptr = (const char*)(p);  \
@@ -132,5 +134,26 @@
     }                                     \
 }
 
+/* vectorization
+ * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax */
+#if !defined(__clang__) && defined(__GNUC__)
+#  if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5)
+#    define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize")))
+#  else
+#    define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")")
+#  endif
+#else
+#  define DONT_VECTORIZE
+#endif
+
+/* disable warnings */
+#ifdef _MSC_VER    /* Visual Studio */
+#  include <intrin.h>                    /* For Visual 2005 */
+#  pragma warning(disable : 4100)        /* disable: C4100: unreferenced formal parameter */
+#  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
+#  pragma warning(disable : 4204)        /* disable: C4204: non-constant aggregate initializer */
+#  pragma warning(disable : 4214)        /* disable: C4214: non-int bitfields */
+#  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
+#endif
+
 #endif /* ZSTD_COMPILER_H */
-/* END CSTYLED */
diff --git a/include/sys/zstd/cpu.h → contrib/zstd/common/cpu.h b/include/sys/zstd/cpu.h → contrib/zstd/common/cpu.h
@@ -1,4 +1,3 @@
-/* BEGIN CSTYLED */
 /*
  * Copyright (c) 2018-present, Facebook, Inc.
  * All rights reserved.
@@ -17,7 +16,13 @@
  * https://github.com/facebook/folly/blob/master/folly/CpuId.h
  */
 
-#include <sys/zstd/mem.h>
+#include <string.h>
+
+#include "mem.h"
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#endif
 
 typedef struct {
     U32 f1c;
@@ -31,7 +36,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
     U32 f1d = 0;
     U32 f7b = 0;
     U32 f7c = 0;
-#ifdef _MSC_VER
+#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
     int reg[4];
     __cpuid((int*)reg, 0);
     {
@@ -67,8 +72,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
           "cpuid\n\t"
           "popl %%ebx\n\t"
           : "=a"(f1a), "=c"(f1c), "=d"(f1d)
-          : "a"(1)
-          :);
+          : "a"(1));
     }
     if (n >= 7) {
       __asm__(
@@ -209,4 +213,3 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
 #undef X
 
 #endif /* ZSTD_COMMON_CPU_H */
-/* END CSTYLED */
diff --git a/contrib/zstd/common/debug.c b/contrib/zstd/common/debug.c
@@ -0,0 +1,44 @@
+/* ******************************************************************
+   debug
+   Part of FSE library
+   Copyright (C) 2013-present, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
+****************************************************************** */
+
+
+/*
+ * This module only hosts one global variable
+ * which can be used to dynamically influence the verbosity of traces,
+ * such as DEBUGLOG and RAWLOG
+ */
+
+#include "debug.h"
+
+int g_debuglevel = DEBUGLEVEL;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		SUBDIRS = common compress decompress
		EXTRA_DIST = common compress decompress zstd.h