Skip to content

Commit

Permalink
no unique F14Chunk empty-tag-vector instance
Browse files Browse the repository at this point in the history 
Summary:
With `F14Table`-based containers like `F14FastMap`, we have observed the impossibility of guaranteeing a single, process-wide, globally-unique instance of the chunk empty-tag-vector.

Instead, we pick a unique invalid representation of the first pointer-sized word of the chunk and compare against it. This comes to a single extra indirection, plus the instructions for comparing against a constant which were already present anyway.

Reviewed By: akrieger

Differential Revision: D62062572

fbshipit-source-id: 8f84f85276f1abcf2fdf70385e578e6b9f3895af
  • Loading branch information
@yfeldblum @facebook-github-bot
yfeldblum authored and facebook-github-bot committed Sep 17, 2024
1 parent 46366b4 commit 366de7f
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 49 deletions.
4 changes: 0 additions & 4 deletions folly/container/detail/F14Table.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,6 @@ namespace detail {
// everywhere or disabled everywhere.
void F14LinkCheck<getF14IntrinsicsMode()>::check() noexcept {}

#if FOLLY_F14_VECTOR_INTRINSICS_AVAILABLE
EmptyTagVectorType kEmptyTagVector = {};
#endif

//// Debug and ASAN stuff

bool tlsPendingSafeInserts(std::ptrdiff_t delta) {
Expand Down
108 changes: 63 additions & 45 deletions folly/container/detail/F14Table.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
#include <folly/Bits.h>
#include <folly/ConstexprMath.h>
#include <folly/Likely.h>
#include <folly/Memory.h>
#include <folly/Portability.h>
#include <folly/ScopeGuard.h>
#include <folly/Traits.h>
Expand Down Expand Up @@ -474,11 +475,21 @@ using TagVector = __uint128_t;
constexpr std::size_t kRequiredVectorAlignment =
constexpr_max(std::size_t{16}, alignof(max_align_t));

using EmptyTagVectorType = std::aligned_storage_t<
sizeof(TagVector) + kRequiredVectorAlignment,
alignof(max_align_t)>;
struct alignas(kRequiredVectorAlignment) F14EmptyTagVector {
std::array<uint8_t, 15> bytes_{};
uint8_t marker_{255}; // coincides with outboundOverflowCount_
};

FOLLY_EXPORT extern EmptyTagVectorType kEmptyTagVector;
FOLLY_EXPORT inline F14EmptyTagVector& getF14EmptyTagVector() noexcept {
static constexpr F14EmptyTagVector instance;
auto const raw = reinterpret_cast<uintptr_t>(&instance);
FOLLY_SAFE_DCHECK(
(raw % kRequiredVectorAlignment) == 0,
raw,
" not aligned to ",
kRequiredVectorAlignment);
return const_cast<F14EmptyTagVector&>(instance);
}

template <typename ItemType>
struct alignas(kRequiredVectorAlignment) F14Chunk {
Expand Down Expand Up @@ -506,6 +517,9 @@ struct alignas(kRequiredVectorAlignment) F14Chunk {

static constexpr MaskType kFullMask = FullMask<kCapacity>::value;

static constexpr std::uint8_t kOutboundOverflowMax = 254;
static constexpr std::uint8_t kOutboundOverflowEmpty = 255;

// Non-empty tags have their top bit set. tags_ array might be bigger
// than kCapacity to keep alignment of first item.
std::array<uint8_t, 14> tags_;
Expand All @@ -518,26 +532,19 @@ struct alignas(kRequiredVectorAlignment) F14Chunk {

// The number of values that would have been placed into this chunk if
// there had been space, including values that also overflowed previous
// full chunks. This value saturates; once it becomes 255 it no longer
// full chunks. This value saturates; once it becomes 254 it no longer
// increases nor decreases.
uint8_t outboundOverflowCount_;

std::array<aligned_storage_for_t<Item>, kAllocatedCapacity> rawItems_;

static F14Chunk* emptyInstance() {
auto raw = reinterpret_cast<char*>(&kEmptyTagVector);
if constexpr (kRequiredVectorAlignment > alignof(max_align_t)) {
auto delta = kRequiredVectorAlignment -
(reinterpret_cast<uintptr_t>(raw) % kRequiredVectorAlignment);
raw += delta;
}
auto rv = reinterpret_cast<F14Chunk*>(raw);
FOLLY_SAFE_DCHECK(
(reinterpret_cast<uintptr_t>(rv) % kRequiredVectorAlignment) == 0,
reinterpret_cast<uintptr_t>(rv),
" not aligned to ",
kRequiredVectorAlignment);
return rv;
FOLLY_EXPORT static F14Chunk* getSomeEmptyInstance() noexcept {
return reinterpret_cast<F14Chunk*>(&getF14EmptyTagVector());
}

static bool isEmptyInstance(F14Chunk const* const chunk) noexcept {
auto const empty = reinterpret_cast<F14EmptyTagVector const*>(chunk);
return empty->marker_ == kOutboundOverflowEmpty;
}

void clear() {
Expand All @@ -550,6 +557,7 @@ struct alignas(kRequiredVectorAlignment) F14Chunk {
}

void copyOverflowInfoFrom(F14Chunk const& rhs) {
FOLLY_SAFE_DCHECK(!isEmptyInstance(&rhs));
FOLLY_SAFE_DCHECK(hostedOverflowCount() == 0, "");
control_ += static_cast<uint8_t>(rhs.control_ & 0xf0);
outboundOverflowCount_ = rhs.outboundOverflowCount_;
Expand All @@ -563,21 +571,22 @@ struct alignas(kRequiredVectorAlignment) F14Chunk {

void adjustHostedOverflowCount(uint8_t op) { control_ += op; }

bool eof() const { return capacityScale() != 0; }
bool eof() const { return capacityScale(this) != 0; }

std::size_t capacityScale() const {
static std::size_t capacityScale(F14Chunk const* const chunk) {
auto const empty = reinterpret_cast<F14EmptyTagVector const*>(chunk);
if constexpr (kCapacityScaleBits == 4) {
return control_ & 0xf;
return empty->bytes_[offsetof(F14Chunk, control_)] & 0xf;
} else {
uint16_t v;
std::memcpy(&v, &tags_[12], 2);
std::memcpy(&v, &empty->bytes_[12], 2);
return v;
}
}

void setCapacityScale(std::size_t scale) {
FOLLY_SAFE_DCHECK(
this != emptyInstance() && scale > 0 &&
!isEmptyInstance(this) && scale > 0 &&
scale < (std::size_t{1} << kCapacityScaleBits),
"");
if constexpr (kCapacityScaleBits == 4) {
Expand All @@ -596,22 +605,22 @@ struct alignas(kRequiredVectorAlignment) F14Chunk {
unsigned outboundOverflowCount() const { return outboundOverflowCount_; }

void incrOutboundOverflowCount() {
if (outboundOverflowCount_ != 255) {
if (outboundOverflowCount_ != kOutboundOverflowMax) {
++outboundOverflowCount_;
}
}

void decrOutboundOverflowCount() {
if (outboundOverflowCount_ != 255) {
FOLLY_SAFE_DCHECK(outboundOverflowCount_ != 0);
if (outboundOverflowCount_ != kOutboundOverflowMax) {
--outboundOverflowCount_;
}
}

std::size_t tag(std::size_t index) const { return tags_[index]; }

void setTag(std::size_t index, std::size_t tag) {
FOLLY_SAFE_DCHECK(
this != emptyInstance() && tag >= 0x80 && tag <= 0xff, "");
FOLLY_SAFE_DCHECK(!isEmptyInstance(this) && tag >= 0x80 && tag <= 0xff, "");
FOLLY_SAFE_CHECK(tags_[index] == 0, "");
tags_[index] = static_cast<uint8_t>(tag);
}
Expand Down Expand Up @@ -1235,7 +1244,7 @@ class F14Table : public Policy {
private:
//////// begin fields

ChunkPtr chunks_{Chunk::emptyInstance()};
ChunkPtr chunks_{Chunk::getSomeEmptyInstance()};
SizeAndChunkShiftAndPackedBegin<ItemIter, kEnableItemIteration>
sizeAndChunkShiftAndPackedBegin_;

Expand Down Expand Up @@ -1460,7 +1469,8 @@ class F14Table : public Policy {

std::size_t itemCount() const noexcept {
if (chunkShift() == 0) {
return computeCapacity(1, chunks_->capacityScale());
return computeCapacity(
1, Chunk::capacityScale(access::to_address(chunks_)));
} else {
return chunkCount() * Chunk::kCapacity;
}
Expand All @@ -1487,7 +1497,8 @@ class F14Table : public Policy {
}

std::size_t bucket_count() const noexcept {
return computeCapacity(chunkCount(), chunks_->capacityScale());
return computeCapacity(
chunkCount(), Chunk::capacityScale(access::to_address(chunks_)));
}

std::size_t max_bucket_count() const noexcept { return max_size(); }
Expand Down Expand Up @@ -1780,7 +1791,7 @@ class F14Table : public Policy {
!this->destroyItemOnClear() && itemCount() == src.itemCount()) {
FOLLY_SAFE_DCHECK(chunkShift() == src.chunkShift(), "");

auto scale = chunks_->capacityScale();
auto scale = Chunk::capacityScale(access::to_address(chunks_));

// most happy path
auto n = chunkAllocSize(chunkCount(), scale);
Expand Down Expand Up @@ -1947,7 +1958,9 @@ class F14Table : public Policy {

// Use the source's capacity, unless it is oversized.
auto upperLimit = computeChunkCountAndScale(src.size(), false, false);
auto ccas = std::make_pair(src.chunkCount(), src.chunks_->capacityScale());
auto ccas = std::make_pair(
src.chunkCount(),
Chunk::capacityScale(access::to_address(src.chunks_)));
FOLLY_SAFE_DCHECK(
ccas.first >= upperLimit.first,
"rounded chunk count can't be bigger than actual");
Expand All @@ -1971,7 +1984,7 @@ class F14Table : public Policy {

void maybeRehash(std::size_t desiredCapacity, bool attemptExact) {
auto origChunkCount = chunkCount();
auto origCapacityScale = chunks_->capacityScale();
auto origCapacityScale = Chunk::capacityScale(access::to_address(chunks_));
auto origCapacity = computeCapacity(origChunkCount, origCapacityScale);

std::size_t newChunkCount;
Expand Down Expand Up @@ -2052,7 +2065,8 @@ class F14Table : public Policy {
void initialReserve(std::size_t desiredCapacity) {
FOLLY_SAFE_DCHECK(size() == 0, "");
FOLLY_SAFE_DCHECK(chunkShift() == 0, "");
FOLLY_SAFE_DCHECK(chunks_ == Chunk::emptyInstance(), "");
FOLLY_SAFE_DCHECK(!!chunks_);
FOLLY_SAFE_DCHECK(Chunk::isEmptyInstance(access::to_address(chunks_)), "");
if (desiredCapacity == 0) {
return;
}
Expand Down Expand Up @@ -2219,7 +2233,7 @@ class F14Table : public Policy {

void debugModeSpuriousRehash() {
auto cc = chunkCount();
auto ss = chunks_->capacityScale();
auto ss = Chunk::capacityScale(access::to_address(chunks_));
rehashImpl(size(), cc, ss, cc, ss);
}

Expand Down Expand Up @@ -2270,7 +2284,8 @@ class F14Table : public Policy {
// We want to support the pattern
// map.reserve(map.size() + 2); auto& r1 = map[k1]; auto& r2 = map[k2];
debugModeOnReserve(capacity);
if (chunks_ == Chunk::emptyInstance()) {
FOLLY_SAFE_DCHECK(!!chunks_);
if (Chunk::isEmptyInstance(access::to_address(chunks_))) {
initialReserve(capacity);
} else {
reserveImpl(capacity);
Expand All @@ -2282,7 +2297,7 @@ class F14Table : public Policy {

auto needed = size() + incoming;
auto chunkCount_ = chunkCount();
auto scale = chunks_->capacityScale();
auto scale = Chunk::capacityScale(access::to_address(chunks_));
auto existing = computeCapacity(chunkCount_, scale);
if (needed - 1 >= existing) {
reserveForInsertImpl(needed - 1, chunkCount_, scale, existing);
Expand Down Expand Up @@ -2354,7 +2369,8 @@ class F14Table : public Policy {
private:
template <bool Reset>
void clearImpl() noexcept {
if (chunks_ == Chunk::emptyInstance()) {
FOLLY_SAFE_DCHECK(!!chunks_);
if (Chunk::isEmptyInstance(access::to_address(chunks_))) {
FOLLY_SAFE_DCHECK(empty() && bucket_count() == 0, "");
return;
}
Expand Down Expand Up @@ -2390,7 +2406,7 @@ class F14Table : public Policy {
// It's okay to do this in a separate loop because we only do it
// when the chunk count is small. That avoids a branch when we
// are promoting a clear to a reset for a large table.
auto scale = chunks_[0].capacityScale();
auto scale = Chunk::capacityScale(access::to_address(chunks_));
for (std::size_t ci = 0; ci < chunkCount(); ++ci) {
chunks_[ci].clear();
}
Expand All @@ -2405,10 +2421,10 @@ class F14Table : public Policy {
if (willReset) {
BytePtr rawAllocation = std::pointer_traits<BytePtr>::pointer_to(
*static_cast<uint8_t*>(static_cast<void*>(&*chunks_)));
std::size_t rawSize =
chunkAllocSize(chunkCount(), chunks_->capacityScale());
std::size_t rawSize = chunkAllocSize(
chunkCount(), Chunk::capacityScale(access::to_address(chunks_)));

chunks_ = Chunk::emptyInstance();
chunks_ = Chunk::getSomeEmptyInstance();
sizeAndChunkShiftAndPackedBegin_.setChunkCount(1);

this->afterReset(origSize, origCapacity, rawAllocation, rawSize);
Expand Down Expand Up @@ -2491,7 +2507,7 @@ class F14Table : public Policy {
// be called with a zero allocationCount.
template <typename V>
void visitAllocationClasses(V&& visitor) const {
auto scale = chunks_->capacityScale();
auto scale = Chunk::capacityScale(access::to_address(chunks_));
this->visitPolicyAllocationClasses(
scale == 0 ? 0 : chunkAllocSize(chunkCount(), scale),
size(),
Expand Down Expand Up @@ -2566,7 +2582,9 @@ class F14Table : public Policy {
}

FOLLY_SAFE_DCHECK(
(chunks_ == Chunk::emptyInstance()) == (bucket_count() == 0), "");
(Chunk::isEmptyInstance(access::to_address(chunks_))) ==
(bucket_count() == 0),
"");

std::size_t n1 = 0;
std::size_t n2 = 0;
Expand Down

0 comments on commit 366de7f

Please sign in to comment.