Support for the LZ4 algorithm

cmake/modules/FindLZ4.cmake

@@ -0,0 +1,41 @@
+# Find the LZ4 includes and library.
+#
+# This module defines
+# LZ4_INCLUDE_DIR, where to locate LZ4 header files
+# LZ4_LIBRARIES, the libraries to link against to use LZ4
+# LZ4_FOUND.  If false, you cannot build anything that requires LZ4.
+
+if(LZ4_CONFIG_EXECUTABLE)
+  set(LZ4_FIND_QUIETLY 1)
+endif()
+set(LZ4_FOUND 0)
+
+find_path(LZ4_INCLUDE_DIR lz4.h
+  $ENV{LZ4_DIR}/include
+  /usr/local/include
+  /opt/lzma/include
+  DOC "Specify the directory containing lz4.h"
+)
+
+find_library(LZ4_LIBRARY NAMES lz4 PATHS
+  $ENV{LZ4_DIR}/lib
+  /usr/local/lz4/lib
+  /usr/local/lib
+  /usr/lib/lz4
+  /usr/local/lib/lz4
+  /usr/lz4/lib /usr/lib
+  /usr/lz4 /usr/local/lz4
+  /opt/lz4 /opt/lz4/lib
+  DOC "Specify the lz4 library here."
+)
+
+if(LZ4_INCLUDE_DIR AND LZ4_LIBRARY)
+  set(LZ4_FOUND 1)
+  if(NOT LZ4_FIND_QUIETLY)
+     message(STATUS "Found LZ4 includes at ${LZ4_INCLUDE_DIR}")
+     message(STATUS "Found LZ4 library at ${LZ4_LIBRARY}")
+  endif()
+endif()
+
+set(LZ4_LIBRARIES ${LZ4_LIBRARY})
+mark_as_advanced(LZ4_FOUND LZ4_LIBRARY LZ4_INCLUDE_DIR)

cmake/modules/FindLZMA.cmake

@@ -2,8 +2,8 @@
 #
 # This module defines
 # LZMA_INCLUDE_DIR, where to locate LZMA header files
-# LZMA_LIBRARIES, the libraries to link against to use Pythia6
-# LZMA_FOUND.  If false, you cannot build anything that requires Pythia6.
+# LZMA_LIBRARIES, the libraries to link against to use LZMA
+# LZMA_FOUND.  If false, you cannot build anything that requires LZMA.
 
 if(LZMA_CONFIG_EXECUTABLE)
   set(LZMA_FIND_QUIETLY 1)

cmake/modules/RootBuildOptions.cmake

@@ -101,6 +101,7 @@ ROOT_BUILD_OPTION(builtin_glew ON "Built included libGLEW, or use system libGLEW
 ROOT_BUILD_OPTION(builtin_pcre OFF "Built included libpcre, or use system libpcre")
 ROOT_BUILD_OPTION(builtin_zlib OFF "Built included libz, or use system libz")
 ROOT_BUILD_OPTION(builtin_lzma OFF "Built included liblzma, or use system liblzma")
+ROOT_BUILD_OPTION(builtin_lz4 OFF "Built included liblz4, or use system liblz4")
 ROOT_BUILD_OPTION(builtin_davix OFF "Built the Davix library internally (downloading tarfile from the Web)")
 ROOT_BUILD_OPTION(builtin_gsl OFF "Built the GSL library internally (downloading tarfile from the Web)")
 ROOT_BUILD_OPTION(builtin_cfitsio OFF "Built the FITSIO library internally (downloading tarfile from the Web)")

cmake/modules/RootConfiguration.cmake

@@ -435,6 +435,11 @@ if(lzma)
 else()
   set(haslzmacompression undef)
 endif()
+if(lz4)
+  set(haslz4compression define)
+else()
+  set(haslz4compression undef)
+endif()
 if(cocoa)
   set(hascocoa define)
 else()

cmake/modules/SearchInstalledSoftware.cmake

@@ -91,7 +91,7 @@ if(builtin_lzma)
      URL_MD5  65693dc257802b6778c28ed53ecca678
      PREFIX LZMA
      INSTALL_DIR ${CMAKE_BINARY_DIR}
-      CONFIGURE_COMMAND "" BUILD_COMMAND ""
+     CONFIGURE_COMMAND "" BUILD_COMMAND ""
      INSTALL_COMMAND cmake -E copy lib/liblzma.dll <INSTALL_DIR>/bin
      BUILD_IN_SOURCE 1)
     install(FILES ${CMAKE_BINARY_DIR}/LZMA/src/LZMA/lib/liblzma.dll DESTINATION ${CMAKE_INSTALL_BINDIR})
@@ -116,6 +116,38 @@ if(builtin_lzma)
 endif()
 
 
+#---Check for LZ4--------------------------------------------------------------------
+if(NOT builtin_lz4)
+  message(STATUS "Looking for LZ4")
+  find_package(LZ4)
+  if(LZ4_FOUND)
+  else()
+    message(STATUS "LZ4 not found. Switching on builtin_lz4 option")
+    set(builtin_lz4 ON CACHE BOOL "" FORCE)
+  endif()
+endif()
+if(builtin_lz4)
+  set(lz4_version r127)
+  message(STATUS "Building LZ4 version ${lz4_version} included in ROOT itself")
+  if(CMAKE_CXX_COMPILER_ID STREQUAL Clang)
+    set(LZ4_CFLAGS "-Wno-format-nonliteral")
+  elseif( CMAKE_CXX_COMPILER_ID STREQUAL Intel)
+    set(LZ4_CFLAGS "-wd188 -wd181 -wd1292 -wd10006 -wd10156 -wd2259 -wd981 -wd128 -wd3179")
+  endif()
+  ExternalProject_Add(
+    LZ4
+    URL ${CMAKE_SOURCE_DIR}/core/lz4/src/lz4-${lz4_version}.tar.gz
+    URL_MD5 5ec580285a5c9bf1e5db3df0fce980b1
+    INSTALL_DIR ${CMAKE_BINARY_DIR}
+    CONFIGURE_COMMAND cd <SOURCE_DIR>/cmake_unofficial && cmake . -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> -DBUILD_LIBS=1 -DCMAKE_C_FLAGS=-O3
+    BUILD_COMMAND cd cmake_unofficial && make
+    INSTALL_COMMAND cd cmake_unofficial && make install
+    BUILD_IN_SOURCE 1)
+  set(LZ4_LIBRARIES ${CMAKE_BINARY_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lz4${CMAKE_STATIC_LIBRARY_SUFFIX})
+  set(LZ4_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include)
+endif()
+
+
 #---Check for X11 which is mandatory lib on Unix--------------------------------------
 if(x11)
   message(STATUS "Looking for X11")

core/CMakeLists.txt

@@ -14,6 +14,7 @@ ROOT_USE_PACKAGE(core/winnt)
 ROOT_USE_PACKAGE(core/macosx)
 ROOT_USE_PACKAGE(core/zip)
 ROOT_USE_PACKAGE(core/lzma)
+ROOT_USE_PACKAGE(core/lz4)
 
 if(builtin_pcre)
   add_subdirectory(pcre)
@@ -46,13 +47,15 @@ if(cocoa)
 endif()
 add_subdirectory(zip)
 add_subdirectory(lzma)
+add_subdirectory(lz4)
 add_subdirectory(base)
 
 set(objectlibs $<TARGET_OBJECTS:Base>
                $<TARGET_OBJECTS:Clib>
                $<TARGET_OBJECTS:Cont>
-               $<TARGET_OBJECTS:Lzma>
                $<TARGET_OBJECTS:Zip>
+               $<TARGET_OBJECTS:Lzma>
+               $<TARGET_OBJECTS:Lz4>
                $<TARGET_OBJECTS:MetaUtils>
                $<TARGET_OBJECTS:Meta>
                $<TARGET_OBJECTS:TextInput>
@@ -111,7 +114,7 @@ add_subdirectory(utils)
 ROOT_LINKER_LIBRARY(Core
                     $<TARGET_OBJECTS:BaseTROOT>
                     ${objectlibs}
-                    LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${ZLIB_LIBRARY}
+                    LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${LZ4_LIBRARIES} ${ZLIB_LIBRARY}
                               ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${corelinklibs} )
 
 if(cling)
@@ -123,5 +126,8 @@ endif()
 if(builtin_lzma)
   ROOT_ADD_BUILTIN_DEPENDENCIES(Core LZMA)
 endif()
+if(builtin_lz4)
+  ROOT_ADD_BUILTIN_DEPENDENCIES(Core LZ4)
+endif()
 
 #----------------------------------------------------------------------------------------

core/lz4/CMakeLists.txt

@@ -0,0 +1,22 @@
+############################################################################
+# CMakeLists.txt file for building ROOT core/lz4 package
+############################################################################
+
+
+#---The builtin LZ4 library is built using the CMake ExternalProject standard module
+#   in cmake/modules/SearchInstalledSoftare.cmake
+
+#---Declare ZipLZ4 sources as part of libCore-------------------------------
+set(headers ${CMAKE_CURRENT_SOURCE_DIR}/inc/ZipLZ4.h)
+set(sources ${CMAKE_CURRENT_SOURCE_DIR}/src/ZipLZ4.c)
+
+
+include_directories(${LZ4_INCLUDE_DIR})
+ROOT_OBJECT_LIBRARY(Lz4 ${sources})
+
+if(builtin_lz4)
+  add_dependencies(Lz4 LZ4)
+endif()
+
+ROOT_INSTALL_HEADERS()
+install(FILES ${LZ4_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})

core/lz4/inc/ZipLZ4.h

@@ -0,0 +1,13 @@
+// Author: Brian Bockelman March 2015
+
+/*************************************************************************
+ * Copyright (C) 1995-2015, Rene Brun and Fons Rademakers.               *
+ * All rights reserved.                                                  *
+ *                                                                       *
+ * For the licensing terms see $ROOTSYS/LICENSE.                         *
+ * For the list of contributors see $ROOTSYS/README/CREDITS.             *
+ *************************************************************************/
+
+void R__zipLZ4(int cxlevel, int *srcsize, char *src, int *tgtsize, char *tgt, int *irep);
+
+void R__unzipLZ4(int *srcsize, unsigned char *src, int *tgtsize, unsigned char *tgt, int *irep);

core/lz4/src/ZipLZ4.c

@@ -0,0 +1,97 @@
+// Original Author: Brian Bockelman
+
+/*************************************************************************
+ * Copyright (C) 1995-2015, Rene Brun and Fons Rademakers.               *
+ * All rights reserved.                                                  *
+ *                                                                       *
+ * For the licensing terms see $ROOTSYS/LICENSE.                         *
+ * For the list of contributors see $ROOTSYS/README/CREDITS.             *
+ *************************************************************************/
+
+#include "ZipLZ4.h"
+#include "lz4.h"
+#include "lz4hc.h"
+#include <stdio.h>
+#include <stdint.h>
+
+#if (__GNUC__ >= 3) || defined(__INTEL_COMPILER)
+#if !defined(R__unlikely)
+  #define R__unlikely(expr) __builtin_expect(!!(expr), 0)
+#endif
+#if !defined(R__likely)
+  #define R__likely(expr) __builtin_expect(!!(expr), 1)
+#endif
+#else
+  #define R__unlikely(expr) expr
+  #define R__likely(expr) expr
+#endif
+
+static const int kHeaderSize = 9;
+
+void R__zipLZ4(int cxlevel, int *srcsize, char *src, int *tgtsize, char *tgt, int *irep)
+{
+   int LZ4_version = LZ4_versionNumber();
+   uint64_t out_size; /* compressed size */
+   uint64_t in_size = (unsigned) (*srcsize);
+
+   *irep = 0;
+
+   if (*tgtsize <= 0) {
+      return;
+   }
+
+   if (*srcsize > 0xffffff || *srcsize < 0) {
+      return;
+   }
+
+   int returnStatus;
+   unsigned char cxByte = cxlevel >= 4;
+   if (cxByte) {
+      returnStatus = LZ4_compressHC_limitedOutput(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize);
+   } else {
+      returnStatus = LZ4_compress_limitedOutput(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize);
+   }
+
+   if (R__unlikely(returnStatus == 0)) { /* LZ4 compression failed */
+      return;
+   }
+
+   tgt[0] = 'L';
+   tgt[1] = '4';
+   tgt[2] = (LZ4_version / (100*100));
+
+   out_size  = returnStatus;             /* compressed size */
+
+   tgt[3] = (char)(out_size & 0xff);
+   tgt[4] = (char)((out_size >> 8) & 0xff);
+   tgt[5] = (char)((out_size >> 16) & 0xff);
+
+   tgt[6] = (char)(in_size & 0xff);         /* decompressed size */
+   tgt[7] = (char)((in_size >> 8) & 0xff);
+   tgt[8] = (char)((in_size >> 16) & 0xff);
+
+   *irep = (int)returnStatus + kHeaderSize;
+}
+
+void R__unzipLZ4(int *srcsize, unsigned char *src, int *tgtsize, unsigned char *tgt, int *irep)
+{
+   int LZ4_version = LZ4_versionNumber()/(100*100);
+   *irep = 0;
+   if (R__unlikely(src[0] != 'L' || src[1] != '4')) {
+      fprintf(stderr, "R__unzipLZ4: algorithm run against buffer with incorrect header (got %d%d; expected %d%d).\n", src[0], src[1], 'L', '4');
+      return;
+   }
+   if (R__unlikely(src[2] != LZ4_version)) {
+      fprintf(stderr, "R__unzipLZ4: This version of LZ4 is incompatible with the on-disk version (got %d; expected %d).\n", src[2], LZ4_version);
+      return;
+   }
+
+   int returnStatus = LZ4_decompress_safe((char *)(&src[kHeaderSize]), (char *)(tgt), *srcsize-kHeaderSize, *tgtsize);
+   if (R__unlikely(returnStatus < 0)) {
+      fprintf(stderr, "R__unzipLZ4: error in decompression around byte %d out of maximum %d.\n", -returnStatus, *tgtsize);
+      return;
+   }
+
+   *irep = returnStatus;
+}
+

core/utils/CMakeLists.txt

@@ -24,13 +24,14 @@ ROOT_EXECUTABLE(rootcling src/LinkdefReader.cxx src/DictSelectionReader.cxx
                           $<TARGET_OBJECTS:Base>
                           $<TARGET_OBJECTS:Cont>
                           $<TARGET_OBJECTS:Lzma>
+                          $<TARGET_OBJECTS:Lz4>
                           $<TARGET_OBJECTS:Zip>
                           $<TARGET_OBJECTS:Meta>
                           $<TARGET_OBJECTS:TextInput>
                           ${macosx_objects}
                           ${unix_objects}
                           ${winnt_objects}
-                          LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${ZLIB_LIBRARY}
+                          LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${LZ4_LIBRARIES} ${ZLIB_LIBRARY}
                                     ${CLING_LIBRARIES} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT}
                                     ${CMAKE_TINFO_LIBS} ${corelinklibs})
 

core/zip/inc/Compression.h

@@ -26,6 +26,9 @@ namespace ROOT {
    // and memory when compressing.  LZMA memory usage is particularly
    // high for compression levels 8 and 9.
    //
+   // Finally, the LZ4 package results in worse compression ratios
+   // than ZLIB but achieves much faster decompression rates.
+   //
    // The current algorithms support level 1 to 9. The higher
    // the level the greater the compression and more CPU time
    // and memory resources used during compression. Level 0
@@ -34,6 +37,7 @@ namespace ROOT {
                                 kZLIB,
                                 kLZMA,
                                 kOldCompressionAlgo,
+                                kLZ4,
                                 // if adding new algorithm types,
                                 // keep this enum value last
                                 kUndefinedCompressionAlgorithm

core/zip/src/Bits.h

@@ -15,6 +15,7 @@
 #include "zlib.h"
 #include "RConfigure.h"
 #include "ZipLZMA.h"
+#include "ZipLZ4.h"
 
 #include <stdio.h>
 #include <assert.h>
@@ -287,11 +288,16 @@ struct bits_internal_state {
    and when R__zipMultipleAlgorithm is called with its last argument set to 0.
    R__ZipMode = 1 : ZLIB compression algorithm is used (default)
    R__ZipMode = 2 : LZMA compression algorithm is used
+   R__ZipMode = 4 : LZ4  compression algorithm is used
    R__ZipMode = 0 or 3 : a very old compression algorithm is used
    (the very old algorithm is supported for backward compatibility)
    The LZMA algorithm requires the external XZ package be installed when linking
    is done. LZMA typically has significantly higher compression factors, but takes
    more CPU time and memory resources while compressing.
+
+  The LZ4 algorithm requires the external LZ4 package to be installed when linking
+  is done.  LZ4 typically has the worst compression ratios, but much faster decompression
+  speeds - sometimes by an order of magnitude.
 */
 int R__ZipMode = 1;
 
@@ -583,6 +589,9 @@ void R__zipMultipleAlgorithm(int cxlevel, int *srcsize, char *src, int *tgtsize,
   if (compressionAlgorithm == 2) {
     R__zipLZMA(cxlevel, srcsize, src, tgtsize, tgt, irep);
     return;
+  } else if (compressionAlgorithm == 4) {
+    R__zipLZ4(cxlevel, srcsize, src, tgtsize, tgt, irep);
+    return;
   }
 
   // The very old algorithm for backward compatibility