feat: add ricepp compression and simple FITS categorizer

Author: mhx

.docker/Dockerfile

@@ -44,6 +44,7 @@ RUN apt install -y \
         liblz4-dev \
         liblzma-dev \
         libmagic-dev \
+        librange-v3-dev \
         libssl-dev \
         libunwind-dev \
         libdwarf-dev \

.docker/build-linux.sh

@@ -115,8 +115,8 @@ if [[ "-$BUILD_TYPE-" == *-coverage-* ]]; then
   rm -rf /tmp-runner/coverage
   mkdir -p /tmp-runner/coverage
   llvm-profdata-17 merge -sparse profile/* -o dwarfs.profdata
-  for binary in mkdwarfs dwarfs dwarfsck dwarfsextract *_test; do
-    llvm-cov-17 show -instr-profile=dwarfs.profdata $binary >/tmp-runner/coverage/$binary.txt
+  for binary in mkdwarfs dwarfs dwarfsck dwarfsextract *_test ricepp/ricepp_test; do
+    llvm-cov-17 show -instr-profile=dwarfs.profdata $binary >/tmp-runner/coverage/$(basename $binary).txt
   done
 fi
 

CMakeLists.txt

@@ -33,6 +33,7 @@ option(WITH_FUZZ "build with fuzzing binaries" OFF)
 option(WITH_MAN_OPTION "build with --man option" ON)
 option(ENABLE_PERFMON "enable performance monitor in all tools" ON)
 option(ENABLE_FLAC "build with FLAC support" ON)
+option(ENABLE_RICEPP "build with RICEPP compression support" ON)
 if(WIN32)
   set(PREFER_SYSTEM_LIBFMT ON)
   set(PREFER_SYSTEM_ZSTD ON)
@@ -218,6 +219,7 @@ else()
 endif()
 
 find_package(Boost 1.67 REQUIRED COMPONENTS chrono iostreams program_options)
+find_package(range-v3 CONFIG REQUIRED)
 
 if(STATIC_BUILD_DO_NOT_USE)
   set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
@@ -397,6 +399,10 @@ if(DWARFS_OPTIMIZE)
   string(REPLACE "-O3" "-O${DWARFS_OPTIMIZE}" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
 endif()
 
+if(ENABLE_RICEPP)
+  add_subdirectory(ricepp)
+endif()
+
 list(APPEND LIBDWARFS_SRC
   src/dwarfs/block_cache.cpp
   src/dwarfs/block_compressor.cpp
@@ -507,10 +513,15 @@ list(
   APPEND
   LIBDWARFS_CATEGORIZER_SRC
   # src/dwarfs/categorizer/binary_categorizer.cpp
+  src/dwarfs/categorizer/fits_categorizer.cpp
   src/dwarfs/categorizer/incompressible_categorizer.cpp
   src/dwarfs/categorizer/pcmaudio_categorizer.cpp
 )
 
+if(ENABLE_RICEPP)
+  list(APPEND LIBDWARFS_COMPRESSION_SRC src/dwarfs/compression/ricepp.cpp)
+endif()
+
 # if(LIBMAGIC_FOUND)
 #   list(APPEND LIBDWARFS_CATEGORIZER_SRC src/dwarfs/categorizer/libmagic_categorizer.cpp)
 # endif()
@@ -540,8 +551,11 @@ target_compile_definitions(
   PRJ_COMPILER_ID="${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}"
 )
 
-target_link_libraries(dwarfs_categorizer folly)
+target_link_libraries(dwarfs_categorizer folly range-v3::range-v3)
 target_link_libraries(dwarfs_compression folly compression_thrift)
+if(ENABLE_RICEPP)
+  target_link_libraries(dwarfs_compression ricepp)
+endif()
 target_link_libraries(dwarfs_tool dwarfs)
 
 if(STATIC_BUILD_DO_NOT_USE)
@@ -660,6 +674,7 @@ if(WITH_TESTS)
     error_test
     file_access_test
     filesystem_test
+    fits_categorizer_test
     fragment_category_test
     incompressible_categorizer_test
     integral_value_parser_test
@@ -683,6 +698,10 @@ if(WITH_TESTS)
     list(APPEND DWARFS_TESTS flac_compressor_test)
   endif()
 
+  if(ENABLE_RICEPP)
+    list(APPEND DWARFS_TESTS ricepp_compressor_test)
+  endif()
+
   foreach (test ${DWARFS_TESTS})
     add_executable(${test} test/${test}.cpp)
     target_link_libraries(
@@ -823,6 +842,7 @@ foreach(tgt dwarfs dwarfs_compression dwarfs_categorizer
             $<$<BOOL:${LIBLZMA_FOUND}>:DWARFS_HAVE_LIBLZMA>
             $<$<AND:$<BOOL:${LIBBROTLIDEC_FOUND}>,$<BOOL:${LIBBROTLIENC_FOUND}>>:DWARFS_HAVE_LIBBROTLI>
             $<$<BOOL:${FLAC_FOUND}>:DWARFS_HAVE_FLAC>
+            $<$<BOOL:${ENABLE_RICEPP}>:DWARFS_HAVE_RICEPP>
   )
 
   if(DWARFS_USE_EXCEPTION_TRACER)
@@ -1068,6 +1088,8 @@ add_custom_target(
 )
 
 file(GLOB_RECURSE ALL_SOURCES LIST_DIRECTORIES false
+        ${CMAKE_CURRENT_SOURCE_DIR}/ricepp/*.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/ricepp/*.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/include/*.h
         ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/test/*.h

include/dwarfs/compression.h

@@ -31,7 +31,8 @@
   DWARFS_COMPRESSION_TYPE(LZ4,    3) SEPARATOR                           \
   DWARFS_COMPRESSION_TYPE(LZ4HC,  4) SEPARATOR                           \
   DWARFS_COMPRESSION_TYPE(BROTLI, 5) SEPARATOR                           \
-  DWARFS_COMPRESSION_TYPE(FLAC,   6)
+  DWARFS_COMPRESSION_TYPE(FLAC,   6) SEPARATOR                           \
+  DWARFS_COMPRESSION_TYPE(RICEPP, 7)
 // clang-format on
 
 namespace dwarfs {

ricepp/.clang-format

@@ -0,0 +1,18 @@
+---
+AlwaysBreakTemplateDeclarations: 'true'
+AccessModifierOffset: -1
+BreakConstructorInitializersBeforeComma: 'true'
+ColumnLimit: '80'
+ConstructorInitializerAllOnOneLineOrOnePerLine: 'false'
+ConstructorInitializerIndentWidth: '4'
+Cpp11BracedListStyle: 'true'
+IndentWidth: '2'
+Language: Cpp
+MaxEmptyLinesToKeep: '1'
+NamespaceIndentation: None
+PenaltyReturnTypeOnItsOwnLine: '0'
+PointerAlignment: Left
+Standard: Cpp11
+UseTab: Never
+
+...

ricepp/.gitignore

@@ -0,0 +1,13 @@
+/@*
+/tmp/
+/build*/
+*.log
+/.gdb_history
+
+*~
+.*.swp
+
+# VS build environment
+.vs/
+out/
+CMakeSettings.json

ricepp/CMakeLists.txt

@@ -0,0 +1,80 @@
+#
+# Copyright (c) Marcus Holland-Moritz
+#
+# This file is part of ricepp.
+#
+# ricepp is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# ricepp is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# ricepp.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+cmake_minimum_required(VERSION 3.25.0)
+
+project(ricepp)
+
+include(FetchContent)
+
+if(NOT TARGET range-v3)
+  FetchContent_Declare(
+    range-v3
+    GIT_REPOSITORY https://github.com/ericniebler/range-v3
+    GIT_TAG 0.12.0
+  )
+  FetchContent_MakeAvailable(range-v3)
+endif()
+
+add_library(ricepp ricepp.cpp)
+
+target_include_directories(ricepp PUBLIC include)
+target_link_libraries(ricepp PUBLIC range-v3)
+target_compile_features(ricepp PUBLIC cxx_std_20)
+
+# # TODO: remove/rework
+# add_executable(ricepp_demo ricepp_demo.cpp)
+# target_link_libraries(ricepp_demo PRIVATE ricepp fmt)
+
+if(WITH_TESTS)
+  if(NOT TARGET gtest)
+    FetchContent_Declare(
+      googletest
+      GIT_REPOSITORY https://github.com/google/googletest.git
+      GIT_TAG v1.14.0
+    )
+    FetchContent_MakeAvailable(googletest)
+  endif()
+
+  enable_testing()
+  include(GoogleTest)
+
+  add_executable(ricepp_test
+    test/bitstream_test.cpp
+    test/byteorder_test.cpp
+    test/codec_test.cpp
+  )
+
+  target_link_libraries(ricepp_test PRIVATE ricepp gtest gmock gtest_main)
+
+  if(ENABLE_COVERAGE)
+    if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+      foreach(target ricepp ricepp_test)
+        target_compile_options(
+          ${target} PRIVATE -fprofile-instr-generate -fcoverage-mapping
+                            -fno-omit-frame-pointer)
+        target_link_options(${target} PRIVATE -fprofile-instr-generate
+                                              -fcoverage-mapping)
+      endforeach()
+    endif()
+  endif()
+
+  if(NOT CMAKE_CROSSCOMPILING)
+    gtest_discover_tests(ricepp_test DISCOVERY_TIMEOUT 120)
+  endif()
+endif()