Experimental AVIF Sample Transform bit depth ext

Allow encoding and decoding images of more than 12 bits, the limit of
the AV1 format. This commit experiments 16-bit AVIF images, made of
one primary lossless 8-bit image and one hidden lossy/lossless bit
depth extension 8-bit image. The two are combined thanks to a Sample
Transform derived image item. Alpha is supported.

Warning: This feature is experimental and not covered by the current
         AVIF specification.

.github/workflows/ci-unix-static.yml

@@ -109,6 +109,7 @@ jobs:
         -DAVIF_ENABLE_EXPERIMENTAL_YCGCO_R=ON
         -DAVIF_ENABLE_EXPERIMENTAL_GAIN_MAP=ON
         -DAVIF_ENABLE_EXPERIMENTAL_AVIR=ON
+        -DAVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM=ON
         -DAVIF_ENABLE_WERROR=ON
     - name: Build libavif (ninja)
       working-directory: ./build

.github/workflows/ci-windows.yml

@@ -125,7 +125,9 @@ jobs:
         -DAVIF_BUILD_TESTS=ON -DAVIF_ENABLE_GTEST=ON -DAVIF_LOCAL_GTEST=ON
         -DAVIF_ENABLE_EXPERIMENTAL_YCGCO_R=ON
         -DAVIF_ENABLE_EXPERIMENTAL_GAIN_MAP=ON
-        -DAVIF_ENABLE_EXPERIMENTAL_AVIR=ON -DAVIF_ENABLE_WERROR=ON
+        -DAVIF_ENABLE_EXPERIMENTAL_AVIR=ON
+        -DAVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM=ON
+        -DAVIF_ENABLE_WERROR=ON
     - name: Build libavif (ninja)
       working-directory: ./build
       run: ninja

CHANGELOG.md

@@ -51,6 +51,8 @@ The changes are relative to the previous release, unless the baseline is specifi
 * Require libyuv by default (but it can still be disabled with
   -DAVIF_LIBYUV=OFF).
 * Add avifdec --icc flag to override the output color profile.
+* Add experimental API for reading and writing 16-bit AVIF files behind the
+  compilation flag AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM.
 
 ### Changed since 1.0.0
 * Update aom.cmd: v3.8.1

CMakeLists.txt

@@ -42,6 +42,7 @@ option(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP
        "Enable experimental gain map code (for HDR images that look good both on HDR and SDR displays)" OFF
 )
 option(AVIF_ENABLE_EXPERIMENTAL_AVIR "Enable experimental reduced header" OFF)
+option(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM "Enable experimental sample transform code" OFF)
 
 set(AVIF_PKG_CONFIG_EXTRA_LIBS_PRIVATE "")
 set(AVIF_PKG_CONFIG_EXTRA_REQUIRES_PRIVATE "")
@@ -328,6 +329,10 @@ if(AVIF_ENABLE_EXPERIMENTAL_AVIR)
     add_compile_definitions(AVIF_ENABLE_EXPERIMENTAL_AVIR)
 endif()
 
+if(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    add_compile_definitions(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+endif()
+
 set(AVIF_SRCS
     src/alpha.c
     src/avif.c
@@ -351,6 +356,9 @@ set(AVIF_SRCS
 if(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP)
     list(APPEND AVIF_SRCS src/gainmap.c)
 endif()
+if(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    list(APPEND AVIF_SRCS src/sampletransform.c)
+endif()
 
 if(AVIF_ENABLE_COMPLIANCE_WARDEN)
     if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/ext/ComplianceWarden")

include/avif/avif.h

@@ -192,6 +192,10 @@ typedef enum AVIF_NODISCARD avifResult
     AVIF_RESULT_DECODE_GAIN_MAP_FAILED = 31,
     AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE = 32,
 #endif
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    AVIF_RESULT_ENCODE_SAMPLE_TRANSFORM_FAILED = 33,
+    AVIF_RESULT_DECODE_SAMPLE_TRANSFORM_FAILED = 34,
+#endif
 
     // Kept for backward compatibility; please use the symbols above instead.
     AVIF_RESULT_NO_AV1_ITEMS_FOUND = AVIF_RESULT_MISSING_IMAGE_ITEM
@@ -706,6 +710,39 @@ AVIF_NODISCARD AVIF_API avifBool avifGainMapMetadataFractionsToDouble(avifGainMa
 
 #endif // AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP
 
+// ---------------------------------------------------------------------------
+
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+// Sample Transforms are a HIGHLY EXPERIMENTAL FEATURE. The format might still
+// change and images containing a sample transform item encoded with the current
+// version of libavif might not decode with a future version of libavif.
+// Use are your own risk.
+// This is based on a proposal from the Alliance for Open Media.
+
+typedef enum avifSampleTransformRecipe
+{
+    AVIF_SAMPLE_TRANSFORM_NONE,
+    // Encode the 8 most significant bits of each input image sample losslessly
+    // into one base image. The remaining least 8 significant bits are encoded
+    // in a separate hidden image item. The two are combined at decoding into
+    // one image with the same bit depth as the original image.
+    // It is backward compatible in the sense that only the base image may be
+    // decoded (ignoring the hidden image item), leading to a valid image but
+    // with precision loss (16-bit samples truncated to the 8 most significant
+    // bits).
+    AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_8B_8B,
+    // Encode the 12 most significant bits of each input image sample losslessly
+    // into one base image. The remaining least 4 significant bits are encoded
+    // in a separate hidden image item. The two are combined at decoding into
+    // one image with the same bit depth as the original image.
+    // It is backward compatible in the sense that only the base image may be
+    // decoded (ignoring the hidden image item), leading to a valid image but
+    // with precision loss (16-bit samples truncated to the 12 most significant
+    // bits).
+    AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_EXTENSION_12B_4B
+} avifSampleTransformRecipe;
+#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM
+
 // ---------------------------------------------------------------------------
 // avifImage
 
@@ -1458,6 +1495,11 @@ typedef struct avifEncoder
 #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP)
     int qualityGainMap; // changeable encoder setting
 #endif
+
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    // Perform extra steps at encoding and decoding to extend AV1 features as bundled additional image items.
+    avifSampleTransformRecipe sampleTransformRecipe;
+#endif
 } avifEncoder;
 
 // avifEncoderCreate() returns NULL if a memory allocation failed.

include/avif/internal.h

@@ -156,6 +156,79 @@ void avifImageCopyNoAlloc(avifImage * dstImage, const avifImage * srcImage);
 // Ignores the gainMap field (which exists only if AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP is defined).
 void avifImageCopySamples(avifImage * dstImage, const avifImage * srcImage, avifPlanesFlags planes);
 
+// ---------------------------------------------------------------------------
+
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+// Mapping used in the coding of Sample Transform metadata.
+typedef enum avifSampleTransformBitDepth
+{
+    AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_8 = 0,  // Signed 8-bit.
+    AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_16 = 1, // Signed 16-bit.
+    AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_32 = 2, // Signed 32-bit.
+    AVIF_SAMPLE_TRANSFORM_BIT_DEPTH_64 = 3  // Signed 64-bit.
+} avifSampleTransformBitDepth;
+
+// Meaning of an operand or operator in Sample Transform metadata.
+typedef enum avifSampleTransformTokenType
+{
+    // Operands.
+    AVIF_SAMPLE_TRANSFORM_CONSTANT = 0,
+    AVIF_SAMPLE_TRANSFORM_INPUT_IMAGE_ITEM_INDEX = 1,
+
+    // Operators. L is the left operand and R is the right operand.
+    AVIF_SAMPLE_TRANSFORM_SUM = 2,        // S = L + R
+    AVIF_SAMPLE_TRANSFORM_DIFFERENCE = 3, // S = L - R
+    AVIF_SAMPLE_TRANSFORM_PRODUCT = 4,    // S = L * R
+    AVIF_SAMPLE_TRANSFORM_DIVIDE = 5,     // S = R==0 ? L : floor(L / R)
+    AVIF_SAMPLE_TRANSFORM_AND = 6,        // S = L & R
+    AVIF_SAMPLE_TRANSFORM_OR = 7,         // S = L | R
+    AVIF_SAMPLE_TRANSFORM_XOR = 8,        // S = L ^ R
+    AVIF_SAMPLE_TRANSFORM_NOR = 9,        // S = ~(L | R)
+    AVIF_SAMPLE_TRANSFORM_MSB = 10,       // S = L<=0 ? R : floor(log2(L))
+    AVIF_SAMPLE_TRANSFORM_POW = 11,       // S = pow(L, abs(R))
+    AVIF_SAMPLE_TRANSFORM_MIN = 12,       // S = L<=R ? L : R
+    AVIF_SAMPLE_TRANSFORM_MAX = 13,       // S = L<=R ? R : L
+    AVIF_SAMPLE_TRANSFORM_RESERVED
+} avifSampleTransformTokenType;
+
+typedef struct avifSampleTransformToken
+{
+    uint8_t value;               // avifSampleTransformTokenType
+    int32_t constant;            // If value is AVIF_SAMPLE_TRANSFORM_CONSTANT.
+    uint8_t inputImageItemIndex; // If value is AVIF_SAMPLE_TRANSFORM_INPUT_IMAGE_ITEM_INDEX. 1-based.
+} avifSampleTransformToken;
+
+AVIF_ARRAY_DECLARE(avifSampleTransformExpression, avifSampleTransformToken, tokens);
+avifBool avifSampleTransformExpressionIsValid(const avifSampleTransformExpression * expression, uint32_t numInputImageItems);
+avifBool avifSampleTransformExpressionIsEquivalentTo(const avifSampleTransformExpression * a, const avifSampleTransformExpression * b);
+
+avifResult avifSampleTransformRecipeToExpression(avifSampleTransformRecipe recipe, avifSampleTransformExpression * expression);
+avifResult avifSampleTransformExpressionToRecipe(const avifSampleTransformExpression * expression, avifSampleTransformRecipe * recipe);
+
+// Applies the expression to the samples of the inputImageItems in the selected planes and stores
+// the results in dstImage. dstImage can be part of the inputImageItems.
+// dstImage and inputImageItems must be allocated and have the same planes and dimensions.
+avifResult avifImageApplyExpression(avifImage * dstImage,
+                                    avifSampleTransformBitDepth bitDepth,
+                                    const avifSampleTransformExpression * expression,
+                                    uint8_t numInputImageItems,
+                                    const avifImage * inputImageItems[],
+                                    avifPlanesFlags planes);
+
+// Same as avifImageApplyExpression(). Convenience function.
+avifResult avifImageApplyOperations(avifImage * dstImage,
+                                    avifSampleTransformBitDepth bitDepth,
+                                    uint32_t numTokens,
+                                    const avifSampleTransformToken tokens[],
+                                    uint8_t numInputImageItems,
+                                    const avifImage * inputImageItems[],
+                                    avifPlanesFlags planes);
+
+#endif // AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM
+
+// ---------------------------------------------------------------------------
+// Alpha
+
 typedef struct avifAlphaParams
 {
     uint32_t width;
@@ -317,10 +390,31 @@ typedef enum avifItemCategory
     AVIF_ITEM_ALPHA,
 #if defined(AVIF_ENABLE_EXPERIMENTAL_GAIN_MAP)
     AVIF_ITEM_GAIN_MAP,
+#endif
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    AVIF_ITEM_SAMPLE_TRANSFORM, // Sample Transform derived image item 'sato'.
+    // Extra input image items for AVIF_ITEM_SAMPLE_TRANSFORM. "Extra" because AVIF_ITEM_COLOR could be one too.
+    AVIF_ITEM_SAMPLE_TRANSFORM_INPUT_0_COLOR,
+    AVIF_ITEM_SAMPLE_TRANSFORM_INPUT_1_COLOR,
+    AVIF_ITEM_SAMPLE_TRANSFORM_INPUT_0_ALPHA,
+    AVIF_ITEM_SAMPLE_TRANSFORM_INPUT_1_ALPHA,
 #endif
     AVIF_ITEM_CATEGORY_COUNT
 } avifItemCategory;
 
+avifBool avifIsAlpha(avifItemCategory itemCategory);
+
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+#define AVIF_SAMPLE_TRANSFORM_MAX_NUM_EXTRA_INPUT_IMAGE_ITEMS \
+    (AVIF_ITEM_SAMPLE_TRANSFORM_INPUT_0_ALPHA - AVIF_ITEM_SAMPLE_TRANSFORM_INPUT_0_COLOR)
+#define AVIF_SAMPLE_TRANSFORM_MAX_NUM_INPUT_IMAGE_ITEMS \
+    (1 /* for AVIF_ITEM_COLOR */ + AVIF_SAMPLE_TRANSFORM_MAX_NUM_EXTRA_INPUT_IMAGE_ITEMS)
+
+#define AVIF_SAMPLE_TRANSFORM_MIN_CATEGORY AVIF_ITEM_SAMPLE_TRANSFORM_INPUT_0_COLOR
+#define AVIF_SAMPLE_TRANSFORM_MAX_CATEGORY \
+    (AVIF_ITEM_SAMPLE_TRANSFORM_INPUT_0_ALPHA + AVIF_SAMPLE_TRANSFORM_MAX_NUM_EXTRA_INPUT_IMAGE_ITEMS - 1)
+#endif
+
 // ---------------------------------------------------------------------------
 
 #if defined(AVIF_ENABLE_EXPERIMENTAL_AVIR)

src/avif.c

@@ -108,6 +108,10 @@ const char * avifResultToString(avifResult result)
         case AVIF_RESULT_ENCODE_GAIN_MAP_FAILED:        return "Encoding of gain map planes failed";
         case AVIF_RESULT_DECODE_GAIN_MAP_FAILED:        return "Decoding of gain map planes failed";
         case AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE:     return "Invalid tone mapped image item";
+#endif
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+        case AVIF_RESULT_ENCODE_SAMPLE_TRANSFORM_FAILED: return "Encoding of sample transformed image failed";
+        case AVIF_RESULT_DECODE_SAMPLE_TRANSFORM_FAILED: return "Decoding of sample transformed image failed";
 #endif
         case AVIF_RESULT_UNKNOWN_ERROR:
         default:
@@ -868,6 +872,22 @@ avifBool avifCleanApertureBoxConvertCropRect(avifCleanApertureBox * clap,
 
 // ---------------------------------------------------------------------------
 
+avifBool avifIsAlpha(avifItemCategory itemCategory)
+{
+    if (itemCategory == AVIF_ITEM_ALPHA) {
+        return AVIF_TRUE;
+    }
+#if defined(AVIF_ENABLE_EXPERIMENTAL_SAMPLE_TRANSFORM)
+    if (itemCategory >= AVIF_ITEM_SAMPLE_TRANSFORM_INPUT_0_ALPHA &&
+        itemCategory < AVIF_ITEM_SAMPLE_TRANSFORM_INPUT_0_ALPHA + AVIF_SAMPLE_TRANSFORM_MAX_NUM_EXTRA_INPUT_IMAGE_ITEMS) {
+        return AVIF_TRUE;
+    }
+#endif
+    return AVIF_FALSE;
+}
+
+// ---------------------------------------------------------------------------
+
 avifBool avifAreGridDimensionsValid(avifPixelFormat yuvFormat, uint32_t imageW, uint32_t imageH, uint32_t tileW, uint32_t tileH, avifDiagnostics * diag)
 {
     // ISO/IEC 23000-22:2019, Section 7.3.11.4.2: