Merge remote-tracking branch 'refs/remotes/origin/dev'

inikep · Nov 20, 2016 · c2c4743 · c2c4743
2 parents f187eda + 71ce310
commit c2c4743
Show file tree

Hide file tree

Showing 148 changed files with 14,657 additions and 38,513 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -2,17 +2,70 @@ language: c
 matrix:
   fast_finish: true
   include:
+    # OS X Mavericks
+    - os: osx
+      env: CC_COMPILER=clang CXX_COMPILER=clang++
+
+
     # Container-based Ubuntu 12.04 LTS Server Edition 64 bit (doesn't support 32-bit includes)
     - os: linux
       sudo: false
       env: CC_COMPILER=gcc CXX_COMPILER=g++
+
     - os: linux
       sudo: false
       env: CC_COMPILER=clang CXX_COMPILER=clang++
+
+
     # Ubuntu 14.04 LTS Server Edition 64 bit
+    - dist: trusty
+      sudo: required
+      addons:
+        apt:
+          packages:
+            - qemu-system-arm
+            - qemu-user-static
+            - gcc-aarch64-linux-gnu
+            - g++-aarch64-linux-gnu
+      env: QEMU_INTERPRETER=qemu-aarch64-static CC_COMPILER=aarch64-linux-gnu-gcc CXX_COMPILER=aarch64-linux-gnu-g++
+
+    - dist: trusty
+      sudo: required
+      addons:
+        apt:
+          packages:
+            - qemu-system-arm
+            - qemu-user-static
+            - gcc-arm-linux-gnueabi
+            - g++-arm-linux-gnueabi
+      env: QEMU_INTERPRETER=qemu-arm-static CC_COMPILER=arm-linux-gnueabi-gcc CXX_COMPILER=arm-linux-gnueabi-g++
+
+    - dist: trusty
+      sudo: required
+      addons:
+        apt:
+          packages:
+            - qemu-system-ppc
+            - qemu-user-static
+            - gcc-powerpc-linux-gnu
+            - g++-powerpc-linux-gnu
+      env: QEMU_INTERPRETER=qemu-ppc64-static CC_COMPILER=powerpc-linux-gnu-gcc CXX_COMPILER=powerpc-linux-gnu-g++ MORE_OPTIONS="MOREFLAGS=-m64 DONT_BUILD_YAPPY=1 DONT_BUILD_DENSITY=1 DONT_BUILD_ZLING=1"
+
+    - dist: trusty
+      sudo: required
+      addons:
+        apt:
+          packages:
+            - qemu-system-ppc
+            - qemu-user-static
+            - gcc-powerpc-linux-gnu
+            - g++-powerpc-linux-gnu
+      env: QEMU_INTERPRETER=qemu-ppc-static CC_COMPILER=powerpc-linux-gnu-gcc CXX_COMPILER=powerpc-linux-gnu-g++ MORE_OPTIONS="DONT_BUILD_YAPPY=1 DONT_BUILD_DENSITY=1 DONT_BUILD_ZLING=1"
+
     - dist: trusty
       sudo: required
       env: CC_COMPILER=gcc CXX_COMPILER=g++
+
     - dist: trusty
       sudo: required
       addons:
@@ -21,6 +74,7 @@ matrix:
             - libc6-dev-i386
             - g++-multilib
       env: CC_COMPILER=gcc CXX_COMPILER=g++ MORE_OPTIONS="BUILD_ARCH=32-bit"
+
     - dist: trusty
       sudo: required
       addons:
@@ -30,6 +84,7 @@ matrix:
           packages:
             - g++-4.9
       env: CC_COMPILER=gcc-4.9 CXX_COMPILER=g++-4.9
+
     - dist: trusty
       sudo: required
       addons:
@@ -39,6 +94,7 @@ matrix:
           packages:
             - g++-5
       env: CC_COMPILER=gcc-5 CXX_COMPILER=g++-5
+
     - dist: trusty
       sudo: required
       addons:
@@ -50,6 +106,7 @@ matrix:
             - g++-5-multilib
             - gcc-multilib
       env: CC_COMPILER=gcc-5 CXX_COMPILER=g++-5 MORE_OPTIONS="BUILD_ARCH=32-bit"
+
     - dist: trusty
       sudo: required
       addons:
@@ -59,6 +116,7 @@ matrix:
           packages:
             - g++-6
       env: CC_COMPILER=gcc-6 CXX_COMPILER=g++-6
+
     - dist: trusty
       sudo: required
       addons:
@@ -70,6 +128,7 @@ matrix:
             - g++-6-multilib
             - gcc-multilib
       env: CC_COMPILER=gcc-6 CXX_COMPILER=g++-6 MORE_OPTIONS="BUILD_ARCH=32-bit"
+
     - dist: trusty
       sudo: required
       addons:
@@ -80,6 +139,7 @@ matrix:
           packages:
             - clang-3.5
       env: CC_COMPILER=clang-3.5 CXX_COMPILER=clang++-3.5
+
     - dist: trusty
       sudo: required
       addons:
@@ -90,6 +150,7 @@ matrix:
           packages:
             - clang-3.6
       env: CC_COMPILER=clang-3.6 CXX_COMPILER=clang++-3.6
+
     - dist: trusty
       sudo: required
       addons:
@@ -100,14 +161,11 @@ matrix:
           packages:
             - clang-3.8
       env: CC_COMPILER=clang-3.8 CXX_COMPILER=clang++-3.8
-    # OS X Mavericks
-    - os: osx
-      compiler: gcc
-      env: CC_COMPILER=clang CXX_COMPILER=clang++
 
 script:
   - $CC_COMPILER -v
   - $CXX_COMPILER -v
   - make CC=$CC_COMPILER CXX=$CXX_COMPILER $MORE_OPTIONS
-  - ./lzbench
-  - ./lzbench -eall ./lzbench
+  - file ./lzbench
+  - $QEMU_INTERPRETER ./lzbench
+  - $QEMU_INTERPRETER ./lzbench -eall ./lzbench
diff --git a/Makefile b/Makefile
@@ -1,7 +1,8 @@
 #BUILD_ARCH = 32-bit
 
-# LZSSE requires gcc with support of __SSE4_1__
-ifeq ($(shell echo|$(CC) -dM -E - -march=native|grep -c SSE4_1), 0)
+ifeq ($(BUILD_ARCH),32-bit)
+	CODE_FLAGS += -m32
+	LDFLAGS += -m32
 	DONT_BUILD_LZSSE ?= 1
 endif
 
@@ -14,13 +15,9 @@ ifeq (1,$(filter 1,$(shell [ "$(COMPILER)" = "gcc" ] && expr $(GCC_VERSION) \< 4
     DONT_BUILD_GLZA ?= 1
 endif
 
-# if BUILD_ARCH is 32-bit
-ifeq ($(BUILD_ARCH),32-bit)
-	CODE_FLAGS += -m32
-	LDFLAGS += -m32
+# LZSSE requires gcc with support of __SSE4_1__
+ifeq ($(shell echo|$(CC) -dM -E - -march=native|grep -c SSE4_1), 0)
 	DONT_BUILD_LZSSE ?= 1
-else
-	DEFINES	+= -D__x86_64__
 endif
 
 
@@ -31,18 +28,26 @@ ifneq (,$(filter Windows%,$(OS)))
 	endif
 	LDFLAGS += -lshell32 -lole32 -loleaut32 -static
 else
-    # MacOS doesn't support -lrt -static
-    ifeq ($(shell uname -s),Darwin)
-        DONT_BUILD_LZHAM ?= 1
-    else
-        LDFLAGS	+= -lrt -static
-    endif
-    LDFLAGS	+= -lpthread
+	ifeq ($(shell uname -p),powerpc)
+		# density and yappy don't work with big-endian PowerPC
+		DONT_BUILD_DENSITY ?= 1
+		DONT_BUILD_YAPPY ?= 1
+		DONT_BUILD_ZLING ?= 1
+	endif
+
+	# MacOS doesn't support -lrt -static
+	ifeq ($(shell uname -s),Darwin)
+		DONT_BUILD_LZHAM ?= 1
+		DONT_BUILD_CSC ?= 1
+	else
+		LDFLAGS	+= -lrt -static
+	endif
+	LDFLAGS	+= -lpthread
 endif
 
 
-DEFINES     += -I. -Izstd/lib -Izstd/lib/common -Ixpack/common
-DEFINES     += -DHAVE_CONFIG_H -DXXH_NAMESPACE=ZSTD_
+DEFINES     += -I. -Izstd/lib -Izstd/lib/common -Ixpack/common -Ilibcsc
+DEFINES     += -DHAVE_CONFIG_H
 CODE_FLAGS  += -Wno-unknown-pragmas -Wno-sign-compare -Wno-conversion
 OPT_FLAGS   ?= -fomit-frame-pointer -fstrict-aliasing -ffast-math
 
@@ -57,32 +62,9 @@ endif
 
 CFLAGS = $(MOREFLAGS) $(CODE_FLAGS) $(OPT_FLAGS_O3) $(DEFINES)
 CFLAGS_O2 = $(MOREFLAGS) $(CODE_FLAGS) $(OPT_FLAGS_O2) $(DEFINES)
+LDFLAGS += $(MOREFLAGS)
 
 
-
-ifeq "$(DONT_BUILD_LZSSE)" "1"
-    DEFINES += -DBENCH_REMOVE_LZSSE
-else
-    LZSSE_FILES = lzsse/lzsse2/lzsse2.o lzsse/lzsse4/lzsse4.o lzsse/lzsse8/lzsse8.o
-endif
-
-ifeq "$(DONT_BUILD_LZHAM)" "1"
-    DEFINES += -DBENCH_REMOVE_LZHAM
-else
-    LZHAM_FILES = lzham/lzham_assert.o lzham/lzham_checksum.o lzham/lzham_huffman_codes.o lzham/lzham_lzbase.o
-    LZHAM_FILES += lzham/lzham_lzcomp.o lzham/lzham_lzcomp_internal.o lzham/lzham_lzdecomp.o lzham/lzham_lzdecompbase.o
-    LZHAM_FILES += lzham/lzham_match_accel.o lzham/lzham_mem.o lzham/lzham_platform.o lzham/lzham_lzcomp_state.o
-    LZHAM_FILES += lzham/lzham_prefix_coding.o lzham/lzham_symbol_codec.o lzham/lzham_timer.o lzham/lzham_vector.o lzham/lzham_lib.o
-endif
-
-ifeq "$(DONT_BUILD_GLZA)" "1"
-    DEFINES += -DBENCH_REMOVE_GLZA
-else
-    GLZA_FILES = glza/GLZAcomp.o glza/GLZAformat.o glza/GLZAcompress.o glza/GLZAencode.o glza/GLZAdecode.o glza/GLZAmodel.o
-endif
-
-ZLING_FILES = libzling/libzling.o libzling/libzling_huffman.o libzling/libzling_lz.o libzling/libzling_utils.o
-
 LZO_FILES = lzo/lzo1.o lzo/lzo1a.o lzo/lzo1a_99.o lzo/lzo1b_1.o lzo/lzo1b_2.o lzo/lzo1b_3.o lzo/lzo1b_4.o lzo/lzo1b_5.o
 LZO_FILES += lzo/lzo1b_6.o lzo/lzo1b_7.o lzo/lzo1b_8.o lzo/lzo1b_9.o lzo/lzo1b_99.o lzo/lzo1b_9x.o lzo/lzo1b_cc.o
 LZO_FILES += lzo/lzo1b_d1.o lzo/lzo1b_d2.o lzo/lzo1b_rr.o lzo/lzo1b_xx.o lzo/lzo1c_1.o lzo/lzo1c_2.o lzo/lzo1c_3.o
@@ -108,35 +90,25 @@ LZRW_FILES = lzrw/lzrw1-a.o lzrw/lzrw1.o lzrw/lzrw2.o lzrw/lzrw3.o lzrw/lzrw3-a.
 
 LZMA_FILES = lzma/LzFind.o lzma/LzmaDec.o lzma/LzmaEnc.o
 
-LZ4_FILES = lz5/lz5.o lz5/lz5hc.o lz4/lz4.o lz4/lz4hc.o 
+LZ4_FILES = lz5/lz5_compress.o lz5/lz5_decompress.o lz4/lz4.o lz4/lz4hc.o 
 
 LZF_FILES = lzf/lzf_c_ultra.o lzf/lzf_c_very.o lzf/lzf_d.o
 
 LZFSE_FILES = lzfse/lzfse_decode.o lzfse/lzfse_decode_base.o lzfse/lzfse_encode.o lzfse/lzfse_encode_base.o lzfse/lzfse_fse.o lzfse/lzvn_decode.o lzfse/lzvn_decode_base.o lzfse/lzvn_encode_base.o 
 
 QUICKLZ_FILES = quicklz/quicklz151b7.o quicklz/quicklz1.o quicklz/quicklz2.o quicklz/quicklz3.o
 
-DENSITY_FILES = density/block_decode.o density/block_encode.o density/block_footer.o density/block_header.o density/block_mode_marker.o
-DENSITY_FILES += density/buffer.o density/globals.o density/kernel_chameleon_decode.o density/kernel_chameleon_dictionary.o
-DENSITY_FILES += density/kernel_chameleon_encode.o density/kernel_cheetah_decode.o density/kernel_cheetah_dictionary.o
-DENSITY_FILES += density/kernel_cheetah_encode.o density/kernel_lion_decode.o density/kernel_lion_dictionary.o
-DENSITY_FILES += density/kernel_lion_encode.o density/kernel_lion_form_model.o density/main_decode.o density/main_encode.o
-DENSITY_FILES += density/main_footer.o density/main_header.o density/memory_location.o density/memory_teleport.o density/stream.o
-DENSITY_FILES += density/spookyhash/spookyhash.o density/spookyhash/context.o
-
 SNAPPY_FILES = snappy/snappy-sinksource.o snappy/snappy-stubs-internal.o snappy/snappy.o 
 
-CSC_FILES = libcsc/csc_analyzer.o libcsc/csc_coder.o libcsc/csc_dec.o libcsc/csc_enc.o libcsc/csc_encoder_main.o
-CSC_FILES += libcsc/csc_filters.o libcsc/csc_lz.o libcsc/csc_memio.o libcsc/csc_mf.o libcsc/csc_model.o libcsc/csc_profiler.o 
-
 BROTLI_FILES = brotli/common/dictionary.o brotli/dec/bit_reader.o brotli/dec/decode.o brotli/dec/huffman.o brotli/dec/state.o
 BROTLI_FILES += brotli/enc/backward_references.o brotli/enc/block_splitter.o brotli/enc/brotli_bit_stream.o brotli/enc/encode.o
 BROTLI_FILES += brotli/enc/encode_parallel.o brotli/enc/entropy_encode.o brotli/enc/histogram.o brotli/enc/literal_cost.o brotli/enc/memory.o
 BROTLI_FILES += brotli/enc/metablock.o brotli/enc/static_dict.o brotli/enc/streams.o brotli/enc/utf8_util.o brotli/enc/compress_fragment.o brotli/enc/compress_fragment_two_pass.o
 BROTLI_FILES += brotli/enc/streams.o brotli/enc/cluster.o brotli/enc/bit_cost.o 
 
-ZSTD_FILES = zstd/lib/decompress/huf_decompress.o zstd/lib/decompress/zstd_decompress.o zstd/lib/common/fse_decompress.o zstd/lib/common/entropy_common.o zstd/lib/common/zstd_common.o zstd/lib/common/xxhash.o
-ZSTD_FILES += zstd/lib/compress/fse_compress.o zstd/lib/compress/huf_compress.o zstd/lib/compress/zstd_compress.o 
+ZSTD_FILES = zstd/lib/decompress/zstd_decompress.o zstd/lib/decompress/huf_decompress.o zstd/lib/common/zstd_common.o zstd/lib/common/fse_decompress.o 
+ZSTD_FILES += zstd/lib/common/xxhash.o zstd/lib/common/error_private.o zstd/lib/common/entropy_common.o 
+ZSTD_FILES += zstd/lib/compress/zstd_compress.o zstd/lib/compress/fse_compress.o zstd/lib/compress/huf_compress.o 
 
 BRIEFLZ_FILES = brieflz/brieflz.o brieflz/depacks.o 
 
@@ -153,10 +125,72 @@ LIBDEFLATE_FILES = libdeflate/adler32.o libdeflate/aligned_malloc.o libdeflate/c
 LIBDEFLATE_FILES += libdeflate/deflate_decompress.o libdeflate/gzip_compress.o libdeflate/gzip_decompress.o
 LIBDEFLATE_FILES += libdeflate/x86_cpu_features.o libdeflate/zlib_compress.o libdeflate/zlib_decompress.o
 
-MISC_FILES = crush/crush.o shrinker/shrinker.o yappy/yappy.o fastlz/fastlz.o tornado/tor_test.o pithy/pithy.o lzjb/lzjb2010.o wflz/wfLZ.o
+MISC_FILES = crush/crush.o shrinker/shrinker.o fastlz/fastlz.o pithy/pithy.o lzjb/lzjb2010.o wflz/wfLZ.o
 MISC_FILES += lzlib/lzlib.o blosclz/blosclz.o slz/slz.o
 
 
+ifeq "$(DONT_BUILD_CSC)" "1"
+    DEFINES += -DBENCH_REMOVE_CSC
+else
+	CSC_FILES = libcsc/csc_analyzer.o libcsc/csc_coder.o libcsc/csc_dec.o libcsc/csc_enc.o libcsc/csc_encoder_main.o
+	CSC_FILES += libcsc/csc_filters.o libcsc/csc_lz.o libcsc/csc_memio.o libcsc/csc_mf.o libcsc/csc_model.o libcsc/csc_profiler.o libcsc/csc_default_alloc.o 
+endif
+
+ifeq "$(DONT_BUILD_DENSITY)" "1"
+    DEFINES += -DBENCH_REMOVE_DENSITY
+else
+	DENSITY_FILES = density/block_decode.o density/block_encode.o density/block_footer.o density/block_header.o density/block_mode_marker.o
+	DENSITY_FILES += density/buffer.o density/globals.o density/kernel_chameleon_decode.o density/kernel_chameleon_dictionary.o
+	DENSITY_FILES += density/kernel_chameleon_encode.o density/kernel_cheetah_decode.o density/kernel_cheetah_dictionary.o
+	DENSITY_FILES += density/kernel_cheetah_encode.o density/kernel_lion_decode.o density/kernel_lion_dictionary.o
+	DENSITY_FILES += density/kernel_lion_encode.o density/kernel_lion_form_model.o density/main_decode.o density/main_encode.o
+	DENSITY_FILES += density/main_footer.o density/main_header.o density/memory_location.o density/memory_teleport.o density/stream.o
+	DENSITY_FILES += density/spookyhash/spookyhash.o density/spookyhash/context.o
+endif
+
+ifeq "$(DONT_BUILD_GLZA)" "1"
+    DEFINES += -DBENCH_REMOVE_GLZA
+else
+    GLZA_FILES = glza/GLZAcomp.o glza/GLZAformat.o glza/GLZAcompress.o glza/GLZAencode.o glza/GLZAdecode.o glza/GLZAmodel.o
+endif
+
+ifeq "$(DONT_BUILD_LZHAM)" "1"
+    DEFINES += -DBENCH_REMOVE_LZHAM
+else
+    LZHAM_FILES = lzham/lzham_assert.o lzham/lzham_checksum.o lzham/lzham_huffman_codes.o lzham/lzham_lzbase.o
+    LZHAM_FILES += lzham/lzham_lzcomp.o lzham/lzham_lzcomp_internal.o lzham/lzham_lzdecomp.o lzham/lzham_lzdecompbase.o
+    LZHAM_FILES += lzham/lzham_match_accel.o lzham/lzham_mem.o lzham/lzham_platform.o lzham/lzham_lzcomp_state.o
+    LZHAM_FILES += lzham/lzham_prefix_coding.o lzham/lzham_symbol_codec.o lzham/lzham_timer.o lzham/lzham_vector.o lzham/lzham_lib.o
+endif
+
+ifeq "$(DONT_BUILD_LZSSE)" "1"
+    DEFINES += -DBENCH_REMOVE_LZSSE
+else
+    LZSSE_FILES = lzsse/lzsse2/lzsse2.o lzsse/lzsse4/lzsse4.o lzsse/lzsse8/lzsse8.o
+endif
+
+ifeq "$(DONT_BUILD_TORNADO)" "1"
+    DEFINES += "-DBENCH_REMOVE_TORNADO"
+    LZMA_FILES += lzma/Alloc.o
+else
+    MISC_FILES += tornado/tor_test.o
+endif
+
+ifeq "$(DONT_BUILD_YAPPY)" "1"
+    DEFINES += -DBENCH_REMOVE_YAPPY
+else
+    MISC_FILES += yappy/yappy.o 
+endif
+
+ifeq "$(DONT_BUILD_ZLING)" "1"
+    DEFINES += -DBENCH_REMOVE_ZLING
+else
+	ZLING_FILES = libzling/libzling.o libzling/libzling_huffman.o libzling/libzling_lz.o libzling/libzling_utils.o
+endif
+
+
+
+
 all: lzbench
 
 # FIX for SEGFAULT on GCC 4.9+
@@ -187,7 +221,7 @@ lzsse/lzsse8/lzsse8.o: lzsse/lzsse8/lzsse8.cpp
 
 _lzbench/lzbench.o: _lzbench/lzbench.cpp _lzbench/lzbench.h
 
-lzbench: $(GLZA_FILES) $(ZSTD_FILES) $(LZSSE_FILES) $(LZFSE_FILES) $(XPACK_FILES) $(GIPFELI_FILES) $(XZ_FILES) $(LIBLZG_FILES) $(BRIEFLZ_FILES) $(LZF_FILES) $(LZRW_FILES) $(BROTLI_FILES) $(CSC_FILES) $(LZMA_FILES) $(DENSITY_FILES) $(ZLING_FILES) $(QUICKLZ_FILES) $(SNAPPY_FILES) $(ZLIB_FILES) $(LZHAM_FILES) $(LZO_FILES) $(UCL_FILES) $(LZMAT_FILES) $(LZ4_FILES) $(LIBDEFLATE_FILES) $(MISC_FILES) _lzbench/lzbench.o _lzbench/compressors.o
+lzbench: $(ZSTD_FILES) $(GLZA_FILES) $(LZSSE_FILES) $(LZFSE_FILES) $(XPACK_FILES) $(GIPFELI_FILES) $(XZ_FILES) $(LIBLZG_FILES) $(BRIEFLZ_FILES) $(LZF_FILES) $(LZRW_FILES) $(BROTLI_FILES) $(CSC_FILES) $(LZMA_FILES) $(DENSITY_FILES) $(ZLING_FILES) $(QUICKLZ_FILES) $(SNAPPY_FILES) $(ZLIB_FILES) $(LZHAM_FILES) $(LZO_FILES) $(UCL_FILES) $(LZMAT_FILES) $(LZ4_FILES) $(LIBDEFLATE_FILES) $(MISC_FILES) _lzbench/lzbench.o _lzbench/compressors.o
 	$(CXX) $^ -o $@ $(LDFLAGS)
 	echo Linked GCC_VERSION=$(GCC_VERSION) CLANG_VERSION=$(CLANG_VERSION) COMPILER=$(COMPILER)
 

diff --git a/NEWS b/NEWS
@@ -1,3 +1,14 @@
+v1.5
+- added the "-r" option : operate recursively on directories
+- added the "-j" option : join files in memory but compress them independently (for many small files)
+- lz5 updated to v2.0 RC2
+- lz4 updated to v1.7.3
+- zstd updated to v1.1.1
+- libdeflate updated to v0.6
+- csc updated to 2016-10-13
+- improved PowerPC support
+- fixed -m option (thanks to Julian Kunkel)
+
 v1.4
 - zstd updated to v1.0.0
 - added libdeflate 2016-08-29 (thanks to Jørgen Ibsen)