feat: replace vector_byte_buffer with malloc_byte_buffer

The latter doesn't always explicitly initialize the allocated memory,
which can help improve performance.

Author: mhx

cmake/libdwarfs.cmake

@@ -34,6 +34,7 @@ add_library(
   src/history.cpp
   src/library_dependencies.cpp
   src/logger.cpp
+  src/malloc_byte_buffer.cpp
   src/mapped_byte_buffer.cpp
   src/mmap.cpp
   src/option_map.cpp
@@ -44,15 +45,14 @@ add_library(
   src/thread_pool.cpp
   src/util.cpp
   src/varint.cpp
-  src/vector_byte_buffer.cpp
-  src/vector_byte_buffer_factory.cpp
   src/xattr.cpp
 
   src/internal/features.cpp
   src/internal/file_status_conv.cpp
   src/internal/fs_section.cpp
   src/internal/fs_section_checker.cpp
   src/internal/glob_to_regex.cpp
+  src/internal/malloc_buffer.cpp
   src/internal/metadata_utils.cpp
   src/internal/string_table.cpp
   src/internal/unicode_case_folding.cpp

include/dwarfs/byte_buffer.h

@@ -49,6 +49,12 @@ compare_spans(std::span<uint8_t const> lhs, std::span<uint8_t const> rhs);
 
 } // namespace detail
 
+namespace internal {
+
+class malloc_buffer;
+
+}
+
 class byte_buffer_interface {
  public:
   virtual ~byte_buffer_interface() = default;
@@ -72,9 +78,7 @@ class mutable_byte_buffer_interface : public byte_buffer_interface {
   // that would reallocate the buffer will throw.
   virtual void freeze_location() = 0;
 
-  // TODO: See if we can do without this. This will *only* be implemented
-  //       in the vector_byte_buffer, other implementations will throw.
-  virtual std::vector<uint8_t>& raw_vector() = 0;
+  virtual internal::malloc_buffer& raw_buffer() = 0;
 };
 
 class shared_byte_buffer {
@@ -164,7 +168,7 @@ class mutable_byte_buffer {
 
   void freeze_location() { bb_->freeze_location(); }
 
-  std::vector<uint8_t>& raw_vector() { return bb_->raw_vector(); }
+  internal::malloc_buffer& raw_buffer() { return bb_->raw_buffer(); }
 
   void swap(mutable_byte_buffer& other) noexcept { std::swap(bb_, other.bb_); }
 

include/dwarfs/internal/malloc_buffer.h

@@ -0,0 +1,93 @@
+/* vim:set ts=2 sw=2 sts=2 et: */
+/**
+ * \author     Marcus Holland-Moritz ([email protected])
+ * \copyright  Copyright (c) Marcus Holland-Moritz
+ *
+ * This file is part of dwarfs.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the “Software”), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <cstring>
+#include <span>
+
+namespace dwarfs::internal {
+
+class malloc_buffer {
+ public:
+  using value_type = uint8_t;
+
+  malloc_buffer() = default;
+  malloc_buffer(size_t size);
+  malloc_buffer(void const* data, size_t size);
+  malloc_buffer(std::span<value_type const> data);
+
+  ~malloc_buffer();
+
+  malloc_buffer(malloc_buffer&&) = default;
+  malloc_buffer& operator=(malloc_buffer&&) = default;
+
+  bool empty() const { return size_ == 0; }
+
+  size_t size() const { return size_; }
+  size_t capacity() const { return capacity_; }
+
+  value_type const* data() const { return data_; }
+  value_type* data() { return data_; }
+
+  void append(void const* data, size_t size) {
+    reserve(size_ + size);
+    copy(data_ + size_, data, size);
+    size_ += size;
+  }
+
+  void clear() { size_ = 0; }
+
+  void resize(size_t new_size) {
+    reserve(new_size);
+    size_ = new_size;
+  }
+
+  void reserve(size_t new_capacity) {
+    if (new_capacity > capacity_) {
+      grow(new_capacity);
+    }
+  }
+
+  void shrink_to_fit();
+
+ private:
+  void grow(size_t new_size);
+  void resize_buffer(size_t new_size);
+  static void copy(void* dest, void const* src, size_t size) {
+    // TODO: try std::copy or even something custom
+    std::memcpy(dest, src, size);
+  }
+
+  value_type* data_{nullptr};
+  size_t size_{0};
+  size_t capacity_{0};
+};
+
+} // namespace dwarfs::internal

include/dwarfs/vector_byte_buffer.h include/dwarfs/malloc_byte_buffer.hinclude/dwarfs/vector_byte_buffer.h renamed to include/dwarfs/malloc_byte_buffer.h

@@ -34,14 +34,19 @@
 
 namespace dwarfs {
 
-class vector_byte_buffer {
+namespace internal {
+class malloc_buffer;
+}
+
+class malloc_byte_buffer {
  public:
   static mutable_byte_buffer create();
   static mutable_byte_buffer create(size_t size);
+  static mutable_byte_buffer create_zeroed(size_t size);
   static mutable_byte_buffer create_reserve(size_t size);
   static mutable_byte_buffer create(std::string_view data);
   static mutable_byte_buffer create(std::span<uint8_t const> data);
-  static mutable_byte_buffer create(std::vector<uint8_t>&& data);
+  static mutable_byte_buffer create(internal::malloc_buffer&& data);
 };
 
 } // namespace dwarfs

…lude/dwarfs/vector_byte_buffer_factory.h …lude/dwarfs/malloc_byte_buffer_factory.hinclude/dwarfs/vector_byte_buffer_factory.h renamed to include/dwarfs/malloc_byte_buffer_factory.h include/dwarfs/vector_byte_buffer_factory.h renamed to include/dwarfs/malloc_byte_buffer_factory.h

@@ -32,7 +32,7 @@
 
 namespace dwarfs {
 
-class vector_byte_buffer_factory {
+class malloc_byte_buffer_factory {
  public:
   static byte_buffer_factory create();
 };

src/block_decompressor.cpp

@@ -29,7 +29,7 @@
 #include <dwarfs/block_decompressor.h>
 #include <dwarfs/decompressor_registry.h>
 #include <dwarfs/fstypes.h>
-#include <dwarfs/vector_byte_buffer.h>
+#include <dwarfs/malloc_byte_buffer.h>
 
 namespace dwarfs {
 
@@ -42,7 +42,7 @@ shared_byte_buffer
 block_decompressor::decompress(compression_type type,
                                std::span<uint8_t const> data) {
   block_decompressor bd(type, data);
-  auto target = vector_byte_buffer::create_reserve(bd.uncompressed_size());
+  auto target = malloc_byte_buffer::create_reserve(bd.uncompressed_size());
   bd.start_decompression(target);
   bd.decompress_frame(bd.uncompressed_size());
   return target.share();

src/compression/brotli.cpp

@@ -37,9 +37,9 @@
 #include <dwarfs/decompressor_registry.h>
 #include <dwarfs/error.h>
 #include <dwarfs/fstypes.h>
+#include <dwarfs/malloc_byte_buffer.h>
 #include <dwarfs/option_map.h>
 #include <dwarfs/varint.h>
-#include <dwarfs/vector_byte_buffer.h>
 
 #include "base.h"
 
@@ -61,7 +61,7 @@ class brotli_block_compressor final : public block_compressor::impl {
 
   shared_byte_buffer compress(shared_byte_buffer const& data,
                               std::string const* /*metadata*/) const override {
-    auto compressed = vector_byte_buffer::create(); // TODO: make configurable
+    auto compressed = malloc_byte_buffer::create(); // TODO: make configurable
     compressed.resize(varint::max_size +
                       ::BrotliEncoderMaxCompressedSize(data.size()));
     size_t size_size = varint::encode(data.size(), compressed.data());

src/compression/flac.cpp

@@ -42,10 +42,10 @@
 #include <dwarfs/compressor_registry.h>
 #include <dwarfs/decompressor_registry.h>
 #include <dwarfs/error.h>
+#include <dwarfs/malloc_byte_buffer.h>
 #include <dwarfs/option_map.h>
 #include <dwarfs/pcm_sample_transformer.h>
 #include <dwarfs/varint.h>
-#include <dwarfs/vector_byte_buffer.h>
 
 #include <dwarfs/gen-cpp2/compression_types.h>
 
@@ -281,7 +281,7 @@ class flac_block_compressor final : public block_compressor::impl {
       pcm_pad = pcm_sample_padding::Msb;
     }
 
-    auto compressed = vector_byte_buffer::create(); // TODO: make configurable
+    auto compressed = malloc_byte_buffer::create(); // TODO: make configurable
 
     {
       using namespace ::apache::thrift;

src/compression/lz4.cpp

@@ -36,8 +36,8 @@
 #include <dwarfs/decompressor_registry.h>
 #include <dwarfs/error.h>
 #include <dwarfs/fstypes.h>
+#include <dwarfs/malloc_byte_buffer.h>
 #include <dwarfs/option_map.h>
-#include <dwarfs/vector_byte_buffer.h>
 
 #include "base.h"
 
@@ -82,7 +82,7 @@ class lz4_block_compressor final : public block_compressor::impl {
 
   shared_byte_buffer compress(shared_byte_buffer const& data,
                               std::string const* /*metadata*/) const override {
-    auto compressed = vector_byte_buffer::create(); // TODO: make configurable
+    auto compressed = malloc_byte_buffer::create(); // TODO: make configurable
     compressed.resize(sizeof(uint32_t) +
                       LZ4_compressBound(to<int>(data.size())));
     // TODO: this should have been a varint; also, if we ever support

src/compression/lzma.cpp

@@ -42,10 +42,10 @@
 #include <dwarfs/decompressor_registry.h>
 #include <dwarfs/error.h>
 #include <dwarfs/fstypes.h>
+#include <dwarfs/malloc_byte_buffer.h>
 #include <dwarfs/option_map.h>
 #include <dwarfs/sorted_array_map.h>
 #include <dwarfs/types.h>
-#include <dwarfs/vector_byte_buffer.h>
 
 #include "base.h"
 
@@ -221,7 +221,7 @@ lzma_block_compressor::compress(shared_byte_buffer const& data,
 
   lzma_action action = LZMA_FINISH;
 
-  auto compressed = vector_byte_buffer::create(); // TODO: make configurable
+  auto compressed = malloc_byte_buffer::create(); // TODO: make configurable
   compressed.resize(data.size() - 1);
 
   s.next_in = data.data();