diff --git a/absl/cleanup/cleanup.h b/absl/cleanup/cleanup.h index 61b53d5..960ccd0 100644 --- a/absl/cleanup/cleanup.h +++ b/absl/cleanup/cleanup.h
@@ -86,25 +86,25 @@ "Callbacks that return values are not supported."); public: - Cleanup(Callback callback) // NOLINT - : storage_(std::move(callback), /* is_callback_engaged = */ true) {} + Cleanup(Callback callback) : storage_(std::move(callback)) {} // NOLINT Cleanup(Cleanup&& other) = default; void Cancel() && { ABSL_HARDENING_ASSERT(storage_.IsCallbackEngaged()); - storage_.DisengageCallback(); + storage_.DestroyCallback(); } void Invoke() && { ABSL_HARDENING_ASSERT(storage_.IsCallbackEngaged()); - storage_.DisengageCallback(); storage_.InvokeCallback(); + storage_.DestroyCallback(); } ~Cleanup() { if (storage_.IsCallbackEngaged()) { storage_.InvokeCallback(); + storage_.DestroyCallback(); } }
diff --git a/absl/cleanup/cleanup_test.cc b/absl/cleanup/cleanup_test.cc index 792595d..46b8858 100644 --- a/absl/cleanup/cleanup_test.cc +++ b/absl/cleanup/cleanup_test.cc
@@ -264,4 +264,48 @@ EXPECT_FALSE(called); // Destructor shouldn't invoke the callback } +int DestructionCount = 0; + +struct DestructionCounter { + void operator()() {} + + ~DestructionCounter() { ++DestructionCount; } +}; + +TYPED_TEST(CleanupTest, DestructorDestroys) { + { + auto cleanup = + absl::MakeCleanup(TypeParam::AsCallback(DestructionCounter())); + DestructionCount = 0; + } + + EXPECT_EQ(DestructionCount, 1); // Engaged cleanup destroys +} + +TYPED_TEST(CleanupTest, CancelDestroys) { + { + auto cleanup = + absl::MakeCleanup(TypeParam::AsCallback(DestructionCounter())); + DestructionCount = 0; + + std::move(cleanup).Cancel(); + EXPECT_EQ(DestructionCount, 1); // Cancel destroys + } + + EXPECT_EQ(DestructionCount, 1); // Canceled cleanup does not double destroy +} + +TYPED_TEST(CleanupTest, InvokeDestroys) { + { + auto cleanup = + absl::MakeCleanup(TypeParam::AsCallback(DestructionCounter())); + DestructionCount = 0; + + std::move(cleanup).Invoke(); + EXPECT_EQ(DestructionCount, 1); // Invoke destroys + } + + EXPECT_EQ(DestructionCount, 1); // Invoked cleanup does not double destroy +} + } // namespace
diff --git a/absl/cleanup/internal/cleanup.h b/absl/cleanup/internal/cleanup.h index b4c4073..2783fcb 100644 --- a/absl/cleanup/internal/cleanup.h +++ b/absl/cleanup/internal/cleanup.h
@@ -15,10 +15,12 @@ #ifndef ABSL_CLEANUP_INTERNAL_CLEANUP_H_ #define ABSL_CLEANUP_INTERNAL_CLEANUP_H_ +#include <new> #include <type_traits> #include <utility> #include "absl/base/internal/invoke.h" +#include "absl/base/macros.h" #include "absl/base/thread_annotations.h" #include "absl/utility/utility.h" @@ -45,14 +47,22 @@ public: Storage() = delete; - Storage(Callback callback, bool is_callback_engaged) - : callback_(std::move(callback)), - is_callback_engaged_(is_callback_engaged) {} + explicit Storage(Callback callback) { + // Placement-new into a character buffer is used for eager destruction when + // the cleanup is invoked or cancelled. To ensure this optimizes well, the + // behavior is implemented locally instead of using an absl::optional. + ::new (GetCallbackBuffer()) Callback(std::move(callback)); + is_callback_engaged_ = true; + } - Storage(Storage&& other) - : callback_(std::move(other.callback_)), - is_callback_engaged_( - absl::exchange(other.is_callback_engaged_, false)) {} + Storage(Storage&& other) { + ABSL_HARDENING_ASSERT(other.IsCallbackEngaged()); + + ::new (GetCallbackBuffer()) Callback(std::move(other.GetCallback())); + is_callback_engaged_ = true; + + other.DestroyCallback(); + } Storage(const Storage& other) = delete; @@ -60,17 +70,26 @@ Storage& operator=(const Storage& other) = delete; + void* GetCallbackBuffer() { return static_cast<void*>(+callback_buffer_); } + + Callback& GetCallback() { + return *reinterpret_cast<Callback*>(GetCallbackBuffer()); + } + bool IsCallbackEngaged() const { return is_callback_engaged_; } - void DisengageCallback() { is_callback_engaged_ = false; } + void DestroyCallback() { + is_callback_engaged_ = false; + GetCallback().~Callback(); + } void InvokeCallback() ABSL_NO_THREAD_SAFETY_ANALYSIS { - std::move(callback_)(); + std::move(GetCallback())(); } private: - Callback callback_; bool is_callback_engaged_; + alignas(Callback) char callback_buffer_[sizeof(Callback)]; }; } // namespace cleanup_internal
diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index a5dee6a..e219956 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc
@@ -23,6 +23,12 @@ ABSL_NAMESPACE_BEGIN namespace container_internal { +ABSL_CONST_INIT ABSL_DLL alignas(16) const ctrl_t kEmptyGroup[16] = { + ctrl_t::kSentinel, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty}; + constexpr size_t Group::kWidth; // Returns "random" seed.
diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index d778326..bafafd4 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h
@@ -291,13 +291,9 @@ // A single block of empty control bytes for tables without any slots allocated. // This enables removing a branch in the hot path of find(). +ABSL_DLL extern const ctrl_t kEmptyGroup[16]; inline ctrl_t* EmptyGroup() { - alignas(16) static constexpr ctrl_t empty_group[] = { - ctrl_t::kSentinel, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, - ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, - ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, - ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty}; - return const_cast<ctrl_t*>(empty_group); + return const_cast<ctrl_t*>(kEmptyGroup); } // Mixes a randomly generated per-process seed with `hash` and `ctrl` to
diff --git a/absl/flags/internal/sequence_lock.h b/absl/flags/internal/sequence_lock.h index 807b2a7..36318ab 100644 --- a/absl/flags/internal/sequence_lock.h +++ b/absl/flags/internal/sequence_lock.h
@@ -49,7 +49,7 @@ // The memory reads and writes protected by this lock must use the provided // `TryRead()` and `Write()` functions. These functions behave similarly to // `memcpy()`, with one oddity: the protected data must be an array of -// `std::atomic<int64>`. This is to comply with the C++ standard, which +// `std::atomic<uint64>`. This is to comply with the C++ standard, which // considers data races on non-atomic objects to be undefined behavior. See "Can // Seqlocks Get Along With Programming Language Memory Models?"[1] by Hans J. // Boehm for more details.
diff --git a/absl/flags/internal/usage.cc b/absl/flags/internal/usage.cc index 949709e..a883567 100644 --- a/absl/flags/internal/usage.cc +++ b/absl/flags/internal/usage.cc
@@ -17,6 +17,7 @@ #include <stdint.h> +#include <algorithm> #include <functional> #include <map> #include <ostream> @@ -255,9 +256,6 @@ matching_flags; flags_internal::ForEachFlag([&](absl::CommandLineFlag& flag) { - // Ignore retired flags. - if (flag.IsRetired()) return; - // If the flag has been stripped, pretend that it doesn't exist. if (flag.Help() == flags_internal::kStrippedFlagHelp) return; @@ -275,6 +273,14 @@ absl::string_view file_separator; // controls blank lines between files for (auto& package : matching_flags) { if (format == HelpFormat::kHumanReadable) { + // Hide packages with only retired flags + bool all_package_flags_are_retired = true; + for (const auto& flags_in_file : package.second) { + for (const auto* flag : flags_in_file.second) { + all_package_flags_are_retired &= flag->IsRetired(); + } + } + if (all_package_flags_are_retired) continue; out << package_separator; package_separator = "\n\n"; } @@ -334,8 +340,11 @@ // Produces the help message describing specific flag. void FlagHelp(std::ostream& out, const CommandLineFlag& flag, HelpFormat format) { - if (format == HelpFormat::kHumanReadable) + if (format == HelpFormat::kHumanReadable) { + // Ignore retired flags + if (flag.IsRetired()) return; flags_internal::FlagHelpHumanReadable(flag, out); + } } // --------------------------------------------------------------------
diff --git a/absl/flags/internal/usage_test.cc b/absl/flags/internal/usage_test.cc index 044d71c..5055640 100644 --- a/absl/flags/internal/usage_test.cc +++ b/absl/flags/internal/usage_test.cc
@@ -61,6 +61,9 @@ "Even more long long long long long long long long long long long long " "help message."); +ABSL_RETIRED_FLAG(int64_t, usage_reporting_test_flag_07, 1, + "usage_reporting_test_flag_07 help message"); + namespace { namespace flags = absl::flags_internal;
diff --git a/absl/numeric/int128.cc b/absl/numeric/int128.cc index 5160df7..17d8874 100644 --- a/absl/numeric/int128.cc +++ b/absl/numeric/int128.cc
@@ -138,28 +138,21 @@ uint128::uint128(double v) : uint128(MakeUint128FromFloat(v)) {} uint128::uint128(long double v) : uint128(MakeUint128FromFloat(v)) {} +#if !defined(ABSL_HAVE_INTRINSIC_INT128) uint128 operator/(uint128 lhs, uint128 rhs) { -#if defined(ABSL_HAVE_INTRINSIC_INT128) - return static_cast<unsigned __int128>(lhs) / - static_cast<unsigned __int128>(rhs); -#else // ABSL_HAVE_INTRINSIC_INT128 uint128 quotient = 0; uint128 remainder = 0; DivModImpl(lhs, rhs, "ient, &remainder); return quotient; -#endif // ABSL_HAVE_INTRINSIC_INT128 } + uint128 operator%(uint128 lhs, uint128 rhs) { -#if defined(ABSL_HAVE_INTRINSIC_INT128) - return static_cast<unsigned __int128>(lhs) % - static_cast<unsigned __int128>(rhs); -#else // ABSL_HAVE_INTRINSIC_INT128 uint128 quotient = 0; uint128 remainder = 0; DivModImpl(lhs, rhs, "ient, &remainder); return remainder; -#endif // ABSL_HAVE_INTRINSIC_INT128 } +#endif // !defined(ABSL_HAVE_INTRINSIC_INT128) namespace {
diff --git a/absl/numeric/int128.h b/absl/numeric/int128.h index 198aa19..a9cbe48 100644 --- a/absl/numeric/int128.h +++ b/absl/numeric/int128.h
@@ -18,6 +18,10 @@ // ----------------------------------------------------------------------------- // // This header file defines 128-bit integer types, `uint128` and `int128`. +// +// TODO(absl-team): This module is inconsistent as many inline `uint128` methods +// are defined in this file, while many inline `int128` methods are defined in +// the `int128_*_intrinsic.inc` files. #ifndef ABSL_NUMERIC_INT128_H_ #define ABSL_NUMERIC_INT128_H_ @@ -783,8 +787,13 @@ // Comparison operators. inline bool operator==(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) == + static_cast<unsigned __int128>(rhs); +#else return (Uint128Low64(lhs) == Uint128Low64(rhs) && Uint128High64(lhs) == Uint128High64(rhs)); +#endif } inline bool operator!=(uint128 lhs, uint128 rhs) { @@ -819,52 +828,76 @@ } inline uint128 operator-(uint128 val) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return -static_cast<unsigned __int128>(val); +#else uint64_t hi = ~Uint128High64(val); uint64_t lo = ~Uint128Low64(val) + 1; if (lo == 0) ++hi; // carry return MakeUint128(hi, lo); +#endif } constexpr inline bool operator!(uint128 val) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return !static_cast<unsigned __int128>(val); +#else return !Uint128High64(val) && !Uint128Low64(val); +#endif } // Logical operators. constexpr inline uint128 operator~(uint128 val) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return ~static_cast<unsigned __int128>(val); +#else return MakeUint128(~Uint128High64(val), ~Uint128Low64(val)); +#endif } constexpr inline uint128 operator|(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) | + static_cast<unsigned __int128>(rhs); +#else return MakeUint128(Uint128High64(lhs) | Uint128High64(rhs), - Uint128Low64(lhs) | Uint128Low64(rhs)); + Uint128Low64(lhs) | Uint128Low64(rhs)); +#endif } constexpr inline uint128 operator&(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) & + static_cast<unsigned __int128>(rhs); +#else return MakeUint128(Uint128High64(lhs) & Uint128High64(rhs), - Uint128Low64(lhs) & Uint128Low64(rhs)); + Uint128Low64(lhs) & Uint128Low64(rhs)); +#endif } constexpr inline uint128 operator^(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) ^ + static_cast<unsigned __int128>(rhs); +#else return MakeUint128(Uint128High64(lhs) ^ Uint128High64(rhs), - Uint128Low64(lhs) ^ Uint128Low64(rhs)); + Uint128Low64(lhs) ^ Uint128Low64(rhs)); +#endif } inline uint128& uint128::operator|=(uint128 other) { - hi_ |= other.hi_; - lo_ |= other.lo_; + *this = *this | other; return *this; } inline uint128& uint128::operator&=(uint128 other) { - hi_ &= other.hi_; - lo_ &= other.lo_; + *this = *this & other; return *this; } inline uint128& uint128::operator^=(uint128 other) { - hi_ ^= other.hi_; - lo_ ^= other.lo_; + *this = *this ^ other; return *this; } @@ -907,21 +940,31 @@ } inline uint128 operator+(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) + + static_cast<unsigned __int128>(rhs); +#else uint128 result = MakeUint128(Uint128High64(lhs) + Uint128High64(rhs), Uint128Low64(lhs) + Uint128Low64(rhs)); if (Uint128Low64(result) < Uint128Low64(lhs)) { // check for carry return MakeUint128(Uint128High64(result) + 1, Uint128Low64(result)); } return result; +#endif } inline uint128 operator-(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) - + static_cast<unsigned __int128>(rhs); +#else uint128 result = MakeUint128(Uint128High64(lhs) - Uint128High64(rhs), Uint128Low64(lhs) - Uint128Low64(rhs)); if (Uint128Low64(lhs) < Uint128Low64(rhs)) { // check for carry return MakeUint128(Uint128High64(result) - 1, Uint128Low64(result)); } return result; +#endif } inline uint128 operator*(uint128 lhs, uint128 rhs) { @@ -951,6 +994,18 @@ #endif // ABSL_HAVE_INTRINSIC128 } +#if defined(ABSL_HAVE_INTRINSIC_INT128) +inline uint128 operator/(uint128 lhs, uint128 rhs) { + return static_cast<unsigned __int128>(lhs) / + static_cast<unsigned __int128>(rhs); +} + +inline uint128 operator%(uint128 lhs, uint128 rhs) { + return static_cast<unsigned __int128>(lhs) % + static_cast<unsigned __int128>(rhs); +} +#endif + // Increment/decrement operators. inline uint128 uint128::operator++(int) {
diff --git a/absl/random/internal/generate_real.h b/absl/random/internal/generate_real.h index 4f62873..d5fbb44 100644 --- a/absl/random/internal/generate_real.h +++ b/absl/random/internal/generate_real.h
@@ -127,10 +127,8 @@ // Construct the 32-bit or 64-bit IEEE 754 floating-point value from // the individual fields: sign, exp, mantissa(bits). - uint_type val = - (std::is_same<SignedTag, GeneratePositiveTag>::value ? 0u : sign) | - (static_cast<uint_type>(exp) << kExp) | - (static_cast<uint_type>(bits) & kMask); + uint_type val = sign | (static_cast<uint_type>(exp) << kExp) | + (static_cast<uint_type>(bits) & kMask); // bit_cast to the output-type real_type result;
diff --git a/absl/strings/BUILD.bazel b/absl/strings/BUILD.bazel index 9720b68..f9735b4 100644 --- a/absl/strings/BUILD.bazel +++ b/absl/strings/BUILD.bazel
@@ -318,6 +318,7 @@ ":strings", "//absl/base:config", "//absl/base:raw_logging_internal", + "//absl/cleanup", "@com_google_googletest//:gtest_main", ], )
diff --git a/absl/strings/CMakeLists.txt b/absl/strings/CMakeLists.txt index 88f076a..8ad5f9c 100644 --- a/absl/strings/CMakeLists.txt +++ b/absl/strings/CMakeLists.txt
@@ -952,6 +952,7 @@ ${ABSL_TEST_COPTS} DEPS absl::base + absl::cleanup absl::config absl::cord_internal absl::cord_rep_test_util
diff --git a/absl/strings/cord.cc b/absl/strings/cord.cc index f5aa6e4..e9d72fa 100644 --- a/absl/strings/cord.cc +++ b/absl/strings/cord.cc
@@ -36,8 +36,8 @@ #include "absl/container/inlined_vector.h" #include "absl/strings/escaping.h" #include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" #include "absl/strings/internal/cord_rep_flat.h" -#include "absl/strings/internal/cord_rep_ring.h" #include "absl/strings/internal/cordz_statistics.h" #include "absl/strings/internal/cordz_update_scope.h" #include "absl/strings/internal/cordz_update_tracker.h" @@ -51,22 +51,16 @@ ABSL_NAMESPACE_BEGIN using ::absl::cord_internal::CordRep; +using ::absl::cord_internal::CordRepBtree; using ::absl::cord_internal::CordRepConcat; using ::absl::cord_internal::CordRepExternal; using ::absl::cord_internal::CordRepFlat; -using ::absl::cord_internal::CordRepRing; using ::absl::cord_internal::CordRepSubstring; using ::absl::cord_internal::CordzUpdateTracker; using ::absl::cord_internal::InlineData; using ::absl::cord_internal::kMaxFlatLength; using ::absl::cord_internal::kMinFlatLength; -using ::absl::cord_internal::CONCAT; -using ::absl::cord_internal::EXTERNAL; -using ::absl::cord_internal::FLAT; -using ::absl::cord_internal::RING; -using ::absl::cord_internal::SUBSTRING; - using ::absl::cord_internal::kInlinedVectorSize; using ::absl::cord_internal::kMaxBytesToCopy; @@ -100,13 +94,13 @@ static const int kMinLengthSize = ABSL_ARRAYSIZE(min_length); -static inline bool cord_ring_enabled() { - return cord_internal::cord_ring_buffer_enabled.load( +static inline bool btree_enabled() { + return cord_internal::cord_btree_enabled.load( std::memory_order_relaxed); } static inline bool IsRootBalanced(CordRep* node) { - if (node->tag != CONCAT) { + if (!node->IsConcat()) { return true; } else if (node->concat()->depth() <= 15) { return true; @@ -143,7 +137,7 @@ // Return the depth of a node static int Depth(const CordRep* rep) { - if (rep->tag == CONCAT) { + if (rep->IsConcat()) { return rep->concat()->depth(); } else { return 0; @@ -176,7 +170,7 @@ } CordRepConcat* rep = new CordRepConcat(); - rep->tag = CONCAT; + rep->tag = cord_internal::CONCAT; SetConcatChildren(rep, left, right); return rep; @@ -218,27 +212,25 @@ return flat; } -// Creates a new flat or ringbuffer out of the specified array. +// Creates a new flat or Btree out of the specified array. // The returned node has a refcount of 1. -static CordRep* RingNewTree(const char* data, size_t length, - size_t alloc_hint) { +static CordRep* NewBtree(const char* data, size_t length, size_t alloc_hint) { if (length <= kMaxFlatLength) { return CreateFlat(data, length, alloc_hint); } CordRepFlat* flat = CreateFlat(data, kMaxFlatLength, 0); data += kMaxFlatLength; length -= kMaxFlatLength; - size_t extra = (length - 1) / kMaxFlatLength + 1; - auto* root = CordRepRing::Create(flat, extra); - return CordRepRing::Append(root, {data, length}, alloc_hint); + auto* root = CordRepBtree::Create(flat); + return CordRepBtree::Append(root, {data, length}, alloc_hint); } // Create a new tree out of the specified array. // The returned node has a refcount of 1. static CordRep* NewTree(const char* data, size_t length, size_t alloc_hint) { if (length == 0) return nullptr; - if (cord_ring_enabled()) { - return RingNewTree(data, length, alloc_hint); + if (btree_enabled()) { + return NewBtree(data, length, alloc_hint); } absl::FixedArray<CordRep*> reps((length - 1) / kMaxFlatLength + 1); size_t n = 0; @@ -275,7 +267,7 @@ CordRepSubstring* rep = new CordRepSubstring(); assert((offset + length) <= child->length); rep->length = length; - rep->tag = SUBSTRING; + rep->tag = cord_internal::SUBSTRING; rep->start = offset; rep->child = child; return VerifyTree(rep); @@ -346,10 +338,10 @@ reduce_size(n); } -// Returns `rep` converted into a CordRepRing. -// Directly returns `rep` if `rep` is already a CordRepRing. -static CordRepRing* ForceRing(CordRep* rep, size_t extra) { - return (rep->tag == RING) ? rep->ring() : CordRepRing::Create(rep, extra); +// Returns `rep` converted into a CordRepBtree. +// Directly returns `rep` if `rep` is already a CordRepBtree. +static CordRepBtree* ForceBtree(CordRep* rep) { + return rep->IsBtree() ? rep->btree() : CordRepBtree::Create(rep); } void Cord::InlineRep::AppendTreeToInlined(CordRep* tree, @@ -357,8 +349,8 @@ assert(!is_tree()); if (!data_.is_empty()) { CordRepFlat* flat = MakeFlatWithExtraCapacity(0); - if (cord_ring_enabled()) { - tree = CordRepRing::Append(CordRepRing::Create(flat, 1), tree); + if (btree_enabled()) { + tree = CordRepBtree::Append(CordRepBtree::Create(flat), tree); } else { tree = Concat(flat, tree); } @@ -369,8 +361,8 @@ void Cord::InlineRep::AppendTreeToTree(CordRep* tree, MethodIdentifier method) { assert(is_tree()); const CordzUpdateScope scope(data_.cordz_info(), method); - if (cord_ring_enabled()) { - tree = CordRepRing::Append(ForceRing(data_.as_tree(), 1), tree); + if (btree_enabled()) { + tree = CordRepBtree::Append(ForceBtree(data_.as_tree()), tree); } else { tree = Concat(data_.as_tree(), tree); } @@ -391,8 +383,8 @@ assert(!is_tree()); if (!data_.is_empty()) { CordRepFlat* flat = MakeFlatWithExtraCapacity(0); - if (cord_ring_enabled()) { - tree = CordRepRing::Prepend(CordRepRing::Create(flat, 1), tree); + if (btree_enabled()) { + tree = CordRepBtree::Prepend(CordRepBtree::Create(flat), tree); } else { tree = Concat(tree, flat); } @@ -404,8 +396,8 @@ MethodIdentifier method) { assert(is_tree()); const CordzUpdateScope scope(data_.cordz_info(), method); - if (cord_ring_enabled()) { - tree = CordRepRing::Prepend(ForceRing(data_.as_tree(), 1), tree); + if (btree_enabled()) { + tree = CordRepBtree::Prepend(ForceBtree(data_.as_tree()), tree); } else { tree = Concat(tree, data_.as_tree()); } @@ -427,8 +419,8 @@ // written to region and the actual size increase will be written to size. static inline bool PrepareAppendRegion(CordRep* root, char** region, size_t* size, size_t max_length) { - if (root->tag == RING && root->refcount.IsOne()) { - Span<char> span = root->ring()->GetAppendBuffer(max_length); + if (root->IsBtree() && root->refcount.IsOne()) { + Span<char> span = root->btree()->GetAppendBuffer(max_length); if (!span.empty()) { *region = span.data(); *size = span.size(); @@ -438,11 +430,11 @@ // Search down the right-hand path for a non-full FLAT node. CordRep* dst = root; - while (dst->tag == CONCAT && dst->refcount.IsOne()) { + while (dst->IsConcat() && dst->refcount.IsOne()) { dst = dst->concat()->right; } - if (dst->tag < FLAT || !dst->refcount.IsOne()) { + if (!dst->IsFlat() || !dst->refcount.IsOne()) { *region = nullptr; *size = 0; return false; @@ -500,22 +492,48 @@ *region = new_node->Data(); *size = new_node->length; - if (cord_ring_enabled()) { - rep = CordRepRing::Append(ForceRing(rep, 1), new_node); + if (btree_enabled()) { + rep = CordRepBtree::Append(ForceBtree(rep), new_node); } else { rep = Concat(rep, new_node); } CommitTree(root, rep, scope, method); } +// Computes the memory side of the provided edge which must be a valid data edge +// for a btrtee, i.e., a FLAT, EXTERNAL or SUBSTRING of a FLAT or EXTERNAL node. +static bool RepMemoryUsageDataEdge(const CordRep* rep, + size_t* total_mem_usage) { + size_t maybe_sub_size = 0; + if (ABSL_PREDICT_FALSE(rep->IsSubstring())) { + maybe_sub_size = sizeof(cord_internal::CordRepSubstring); + rep = rep->substring()->child; + } + if (rep->IsFlat()) { + *total_mem_usage += maybe_sub_size + rep->flat()->AllocatedSize(); + return true; + } + if (rep->IsExternal()) { + // We don't know anything about the embedded / bound data, but we can safely + // assume it is 'at least' a word / pointer to data. In the future we may + // choose to use the 'data' byte as a tag to identify the types of some + // well-known externals, such as a std::string instance. + *total_mem_usage += maybe_sub_size + + sizeof(cord_internal::CordRepExternalImpl<intptr_t>) + + rep->length; + return true; + } + return false; +} + // If the rep is a leaf, this will increment the value at total_mem_usage and // will return true. static bool RepMemoryUsageLeaf(const CordRep* rep, size_t* total_mem_usage) { - if (rep->tag >= FLAT) { + if (rep->IsFlat()) { *total_mem_usage += rep->flat()->AllocatedSize(); return true; } - if (rep->tag == EXTERNAL) { + if (rep->IsExternal()) { // We don't know anything about the embedded / bound data, but we can safely // assume it is 'at least' a word / pointer to data. In the future we may // choose to use the 'data' byte as a tag to identify the types of some @@ -630,7 +648,7 @@ } if (tree != nullptr) { CordzUpdateScope scope(contents_.cordz_info(), method); - if (tree->tag >= FLAT && tree->flat()->Capacity() >= length && + if (tree->IsFlat() && tree->flat()->Capacity() >= length && tree->refcount.IsOne()) { // Copy in place if the existing FLAT node is reusable. memmove(tree->flat()->Data(), data, length); @@ -690,9 +708,11 @@ return; } - if (cord_ring_enabled()) { - rep = ForceRing(rep, (src.size() - 1) / kMaxFlatLength + 1); - rep = CordRepRing::Append(rep->ring(), src); + if (btree_enabled()) { + // TODO(b/192061034): keep legacy 10% growth rate: consider other rates. + rep = ForceBtree(rep); + const size_t alloc_hint = (std::min)(kMaxFlatLength, rep->length / 10); + rep = CordRepBtree::Append(rep->btree(), src, alloc_hint); } else { // Use new block(s) for any remaining bytes that were not handled above. // Alloc extra memory only if the right child of the root of the new tree @@ -747,7 +767,7 @@ contents_.AppendArray({src.contents_.data(), src_size}, method); return; } - if (src_tree->tag >= FLAT) { + if (src_tree->IsFlat()) { // src tree just has one flat node. contents_.AppendArray({src_tree->flat()->Data(), src_size}, method); return; @@ -769,9 +789,13 @@ contents_.AppendTree(rep, CordzUpdateTracker::kAppendCord); } -void Cord::Append(const Cord& src) { AppendImpl(src); } +void Cord::Append(const Cord& src) { + AppendImpl(src); +} -void Cord::Append(Cord&& src) { AppendImpl(std::move(src)); } +void Cord::Append(Cord&& src) { + AppendImpl(std::move(src)); +} template <typename T, Cord::EnableIfString<T>> void Cord::Append(T&& src) { @@ -833,7 +857,7 @@ if (n == 0) return CordRep::Ref(node); absl::InlinedVector<CordRep*, kInlinedVectorSize> rhs_stack; - while (node->tag == CONCAT) { + while (node->IsConcat()) { assert(n <= node->length); if (n < node->concat()->left->length) { // Push right to stack, descend left. @@ -852,7 +876,7 @@ } else { size_t start = n; size_t len = node->length - n; - if (node->tag == SUBSTRING) { + if (node->IsSubstring()) { // Consider in-place update of node, similar to in RemoveSuffixFrom(). start += node->substring()->start; node = node->substring()->child; @@ -875,7 +899,7 @@ absl::InlinedVector<CordRep*, kInlinedVectorSize> lhs_stack; bool inplace_ok = node->refcount.IsOne(); - while (node->tag == CONCAT) { + while (node->IsConcat()) { assert(n <= node->length); if (n < node->concat()->right->length) { // Push left to stack, descend right. @@ -892,7 +916,7 @@ if (n == 0) { CordRep::Ref(node); - } else if (inplace_ok && node->tag != EXTERNAL) { + } else if (inplace_ok && !node->IsExternal()) { // Consider making a new buffer if the current node capacity is much // larger than the new length. CordRep::Ref(node); @@ -900,7 +924,7 @@ } else { size_t start = 0; size_t len = node->length - n; - if (node->tag == SUBSTRING) { + if (node->IsSubstring()) { start = node->substring()->start; node = node->substring()->child; } @@ -923,8 +947,10 @@ } else { auto constexpr method = CordzUpdateTracker::kRemovePrefix; CordzUpdateScope scope(contents_.cordz_info(), method); - if (tree->tag == RING) { - tree = CordRepRing::RemovePrefix(tree->ring(), n); + if (tree->IsBtree()) { + CordRep* old = tree; + tree = tree->btree()->SubTree(n, tree->length - n); + CordRep::Unref(old); } else { CordRep* newrep = RemovePrefixFrom(tree, n); CordRep::Unref(tree); @@ -944,8 +970,10 @@ } else { auto constexpr method = CordzUpdateTracker::kRemoveSuffix; CordzUpdateScope scope(contents_.cordz_info(), method); - if (tree->tag == RING) { - tree = CordRepRing::RemoveSuffix(tree->ring(), n); + if (tree->IsBtree()) { + CordRep* old = tree; + tree = tree->btree()->SubTree(0, tree->length - n); + CordRep::Unref(old); } else { CordRep* newrep = RemoveSuffixFrom(tree, n); CordRep::Unref(tree); @@ -984,8 +1012,8 @@ results.push_back(Concat(left, right)); } else if (pos == 0 && n == node->length) { results.push_back(CordRep::Ref(node)); - } else if (node->tag != CONCAT) { - if (node->tag == SUBSTRING) { + } else if (!node->IsConcat()) { + if (node->IsSubstring()) { pos += node->substring()->start; node = node->substring()->child; } @@ -1037,9 +1065,8 @@ return sub_cord; } - if (tree->tag == RING) { - CordRepRing* ring = CordRep::Ref(tree)->ring(); - tree = CordRepRing::SubRing(ring, pos, new_size); + if (tree->IsBtree()) { + tree = tree->btree()->SubTree(pos, new_size); } else { tree = NewSubRange(tree, pos, new_size); } @@ -1063,7 +1090,7 @@ CordRep* node = pending.back(); pending.pop_back(); CheckNode(node); - if (ABSL_PREDICT_FALSE(node->tag != CONCAT)) { + if (ABSL_PREDICT_FALSE(!node->IsConcat())) { AddNode(node); continue; } @@ -1157,7 +1184,7 @@ static void CheckNode(CordRep* node) { ABSL_INTERNAL_CHECK(node->length != 0u, ""); - if (node->tag == CONCAT) { + if (node->IsConcat()) { ABSL_INTERNAL_CHECK(node->concat()->left != nullptr, ""); ABSL_INTERNAL_CHECK(node->concat()->right != nullptr, ""); ABSL_INTERNAL_CHECK(node->length == (node->concat()->left->length + @@ -1177,7 +1204,7 @@ static CordRep* Rebalance(CordRep* node) { VerifyTree(node); - assert(node->tag == CONCAT); + assert(node->IsConcat()); if (node->length == 0) { return nullptr; @@ -1227,28 +1254,33 @@ } // namespace -// Helper routine. Locates the first flat chunk of the Cord without -// initializing the iterator. +// Helper routine. Locates the first flat or external chunk of the Cord without +// initializing the iterator, and returns a string_view referencing the data. inline absl::string_view Cord::InlineRep::FindFlatStartPiece() const { if (!is_tree()) { return absl::string_view(data_.as_chars(), data_.inline_size()); } CordRep* node = tree(); - if (node->tag >= FLAT) { + if (node->IsFlat()) { return absl::string_view(node->flat()->Data(), node->length); } - if (node->tag == EXTERNAL) { + if (node->IsExternal()) { return absl::string_view(node->external()->base, node->length); } - if (node->tag == RING) { - return node->ring()->entry_data(node->ring()->head()); + if (node->IsBtree()) { + CordRepBtree* tree = node->btree(); + int height = tree->height(); + while (--height >= 0) { + tree = tree->Edge(CordRepBtree::kFront)->btree(); + } + return tree->Data(tree->begin()); } // Walk down the left branches until we hit a non-CONCAT node. - while (node->tag == CONCAT) { + while (node->IsConcat()) { node = node->concat()->left; } @@ -1257,16 +1289,16 @@ size_t length = node->length; assert(length != 0); - if (node->tag == SUBSTRING) { + if (node->IsSubstring()) { offset = node->substring()->start; node = node->substring()->child; } - if (node->tag >= FLAT) { + if (node->IsFlat()) { return absl::string_view(node->flat()->Data() + offset, length); } - assert((node->tag == EXTERNAL) && "Expect FLAT or EXTERNAL node here"); + assert(node->IsExternal() && "Expect FLAT or EXTERNAL node here"); return absl::string_view(node->external()->base + offset, length); } @@ -1460,7 +1492,7 @@ // Walk down the left branches until we hit a non-CONCAT node. Save the // right children to the stack for subsequent traversal. - while (node->tag == CONCAT) { + while (node->IsConcat()) { stack_of_right_children.push_back(node->concat()->right); node = node->concat()->left; } @@ -1468,15 +1500,15 @@ // Get the child node if we encounter a SUBSTRING. size_t offset = 0; size_t length = node->length; - if (node->tag == SUBSTRING) { + if (node->IsSubstring()) { offset = node->substring()->start; node = node->substring()->child; } - assert(node->tag == EXTERNAL || node->tag >= FLAT); + assert(node->IsExternal() || node->IsFlat()); assert(length != 0); const char* data = - node->tag == EXTERNAL ? node->external()->base : node->flat()->Data(); + node->IsExternal() ? node->external()->base : node->flat()->Data(); current_chunk_ = absl::string_view(data + offset, length); current_leaf_ = node; return *this; @@ -1506,22 +1538,21 @@ return subcord; } - if (ring_reader_) { + if (btree_reader_) { size_t chunk_size = current_chunk_.size(); if (n <= chunk_size && n <= kMaxBytesToCopy) { subcord = Cord(current_chunk_.substr(0, n), method); + if (n < chunk_size) { + current_chunk_.remove_prefix(n); + } else { + current_chunk_ = btree_reader_.Next(); + } } else { - auto* ring = CordRep::Ref(ring_reader_.ring())->ring(); - size_t offset = ring_reader_.length() - bytes_remaining_; - CordRep* rep = CordRepRing::SubRing(ring, offset, n); + CordRep* rep; + current_chunk_ = btree_reader_.Read(n, chunk_size, rep); subcord.contents_.EmplaceTree(rep, method); } - if (n < chunk_size) { - bytes_remaining_ -= n; - current_chunk_.remove_prefix(n); - } else { - AdvanceBytesRing(n); - } + bytes_remaining_ -= n; return subcord; } @@ -1530,8 +1561,8 @@ // Range to read is a proper subrange of the current chunk. assert(current_leaf_ != nullptr); CordRep* subnode = CordRep::Ref(current_leaf_); - const char* data = subnode->tag == EXTERNAL ? subnode->external()->base - : subnode->flat()->Data(); + const char* data = subnode->IsExternal() ? subnode->external()->base + : subnode->flat()->Data(); subnode = NewSubstring(subnode, current_chunk_.data() - data, n); subcord.contents_.EmplaceTree(VerifyTree(subnode), method); RemoveChunkPrefix(n); @@ -1543,8 +1574,8 @@ assert(current_leaf_ != nullptr); CordRep* subnode = CordRep::Ref(current_leaf_); if (current_chunk_.size() < subnode->length) { - const char* data = subnode->tag == EXTERNAL ? subnode->external()->base - : subnode->flat()->Data(); + const char* data = subnode->IsExternal() ? subnode->external()->base + : subnode->flat()->Data(); subnode = NewSubstring(subnode, current_chunk_.data() - data, current_chunk_.size()); } @@ -1582,7 +1613,7 @@ // Walk down the appropriate branches until we hit a non-CONCAT node. Save the // right children to the stack for subsequent traversal. - while (node->tag == CONCAT) { + while (node->IsConcat()) { if (node->concat()->left->length > n) { // Push right, descend left. stack_of_right_children.push_back(node->concat()->right); @@ -1599,20 +1630,20 @@ // Get the child node if we encounter a SUBSTRING. size_t offset = 0; size_t length = node->length; - if (node->tag == SUBSTRING) { + if (node->IsSubstring()) { offset = node->substring()->start; node = node->substring()->child; } // Range to read ends with a proper (possibly empty) subrange of the current // chunk. - assert(node->tag == EXTERNAL || node->tag >= FLAT); + assert(node->IsExternal() || node->IsFlat()); assert(length > n); if (n > 0) { subnode = Concat(subnode, NewSubstring(CordRep::Ref(node), offset, n)); } const char* data = - node->tag == EXTERNAL ? node->external()->base : node->flat()->Data(); + node->IsExternal() ? node->external()->base : node->flat()->Data(); current_chunk_ = absl::string_view(data + offset + n, length - n); current_leaf_ = node; bytes_remaining_ -= n; @@ -1655,7 +1686,7 @@ // Walk down the appropriate branches until we hit a non-CONCAT node. Save the // right children to the stack for subsequent traversal. - while (node->tag == CONCAT) { + while (node->IsConcat()) { if (node->concat()->left->length > n) { // Push right, descend left. stack_of_right_children.push_back(node->concat()->right); @@ -1671,15 +1702,15 @@ // Get the child node if we encounter a SUBSTRING. size_t offset = 0; size_t length = node->length; - if (node->tag == SUBSTRING) { + if (node->IsSubstring()) { offset = node->substring()->start; node = node->substring()->child; } - assert(node->tag == EXTERNAL || node->tag >= FLAT); + assert(node->IsExternal() || node->IsFlat()); assert(length > n); const char* data = - node->tag == EXTERNAL ? node->external()->base : node->flat()->Data(); + node->IsExternal() ? node->external()->base : node->flat()->Data(); current_chunk_ = absl::string_view(data + offset + n, length - n); current_leaf_ = node; bytes_remaining_ -= n; @@ -1695,15 +1726,15 @@ while (true) { assert(rep != nullptr); assert(offset < rep->length); - if (rep->tag >= FLAT) { + if (rep->IsFlat()) { // Get the "i"th character directly from the flat array. return rep->flat()->Data()[offset]; - } else if (rep->tag == RING) { - return rep->ring()->GetCharacter(offset); - } else if (rep->tag == EXTERNAL) { + } else if (rep->IsBtree()) { + return rep->btree()->GetCharacter(offset); + } else if (rep->IsExternal()) { // Get the "i"th character from the external array. return rep->external()->base[offset]; - } else if (rep->tag == CONCAT) { + } else if (rep->IsConcat()) { // Recursively branch to the side of the concatenation that the "i"th // character is on. size_t left_length = rep->concat()->left->length; @@ -1715,7 +1746,7 @@ } } else { // This must be a substring a node, so bypass it to get to the child. - assert(rep->tag == SUBSTRING); + assert(rep->IsSubstring()); offset += rep->substring()->start; rep = rep->substring()->child; } @@ -1752,27 +1783,27 @@ /* static */ bool Cord::GetFlatAux(CordRep* rep, absl::string_view* fragment) { assert(rep != nullptr); - if (rep->tag >= FLAT) { + if (rep->IsFlat()) { *fragment = absl::string_view(rep->flat()->Data(), rep->length); return true; - } else if (rep->tag == EXTERNAL) { + } else if (rep->IsExternal()) { *fragment = absl::string_view(rep->external()->base, rep->length); return true; - } else if (rep->tag == RING) { - return rep->ring()->IsFlat(fragment); - } else if (rep->tag == SUBSTRING) { + } else if (rep->IsBtree()) { + return rep->btree()->IsFlat(fragment); + } else if (rep->IsSubstring()) { CordRep* child = rep->substring()->child; - if (child->tag >= FLAT) { + if (child->IsFlat()) { *fragment = absl::string_view( child->flat()->Data() + rep->substring()->start, rep->length); return true; - } else if (child->tag == EXTERNAL) { + } else if (child->IsExternal()) { *fragment = absl::string_view( child->external()->base + rep->substring()->start, rep->length); return true; - } else if (child->tag == RING) { - return child->ring()->IsFlat(rep->substring()->start, rep->length, - fragment); + } else if (child->IsBtree()) { + return child->btree()->IsFlat(rep->substring()->start, rep->length, + fragment); } } return false; @@ -1781,7 +1812,7 @@ /* static */ void Cord::ForEachChunkAux( absl::cord_internal::CordRep* rep, absl::FunctionRef<void(absl::string_view)> callback) { - if (rep->tag == RING) { + if (rep->IsBtree()) { ChunkIterator it(rep), end; while (it != end) { callback(*it); @@ -1797,7 +1828,7 @@ absl::cord_internal::CordRep* stack[stack_max]; absl::cord_internal::CordRep* current_node = rep; while (true) { - if (current_node->tag == CONCAT) { + if (current_node->IsConcat()) { if (stack_pos == stack_max) { // There's no more room on our stack array to add another right branch, // and the idea is to avoid allocations, so call this function @@ -1844,37 +1875,29 @@ *os << "]"; *os << " " << (IsRootBalanced(rep) ? 'b' : 'u'); *os << " " << std::setw(indent) << ""; - if (rep->tag == CONCAT) { + if (rep->IsConcat()) { *os << "CONCAT depth=" << Depth(rep) << "\n"; indent += kIndentStep; indents.push_back(indent); stack.push_back(rep->concat()->right); rep = rep->concat()->left; - } else if (rep->tag == SUBSTRING) { + } else if (rep->IsSubstring()) { *os << "SUBSTRING @ " << rep->substring()->start << "\n"; indent += kIndentStep; rep = rep->substring()->child; } else { // Leaf or ring - if (rep->tag == EXTERNAL) { + if (rep->IsExternal()) { *os << "EXTERNAL ["; if (include_data) *os << absl::CEscape(std::string(rep->external()->base, rep->length)); *os << "]\n"; - } else if (rep->tag >= FLAT) { + } else if (rep->IsFlat()) { *os << "FLAT cap=" << rep->flat()->Capacity() << " ["; if (include_data) *os << absl::CEscape(std::string(rep->flat()->Data(), rep->length)); *os << "]\n"; } else { - assert(rep->tag == RING); - auto* ring = rep->ring(); - *os << "RING, entries = " << ring->entries() << "\n"; - CordRepRing::index_type head = ring->head(); - do { - DumpNode(ring->entry_child(head), include_data, os, - indent + kIndentStep); - head = ring->advance(head); - } while (head != ring->tail()); + CordRepBtree::Dump(rep, /*label=*/ "", include_data, *os); } if (stack.empty()) break; rep = stack.back(); @@ -1906,7 +1929,7 @@ ABSL_INTERNAL_CHECK(node->length != 0, ReportError(root, node)); } - if (node->tag == CONCAT) { + if (node->IsConcat()) { ABSL_INTERNAL_CHECK(node->concat()->left != nullptr, ReportError(root, node)); ABSL_INTERNAL_CHECK(node->concat()->right != nullptr, @@ -1918,13 +1941,13 @@ worklist.push_back(node->concat()->right); worklist.push_back(node->concat()->left); } - } else if (node->tag >= FLAT) { + } else if (node->IsFlat()) { ABSL_INTERNAL_CHECK(node->length <= node->flat()->Capacity(), ReportError(root, node)); - } else if (node->tag == EXTERNAL) { + } else if (node->IsExternal()) { ABSL_INTERNAL_CHECK(node->external()->base != nullptr, ReportError(root, node)); - } else if (node->tag == SUBSTRING) { + } else if (node->IsSubstring()) { ABSL_INTERNAL_CHECK( node->substring()->start < node->substring()->child->length, ReportError(root, node)); @@ -1953,7 +1976,7 @@ while (true) { const CordRep* next_node = nullptr; - if (cur_node->tag == CONCAT) { + if (cur_node->IsConcat()) { total_mem_usage += sizeof(CordRepConcat); const CordRep* left = cur_node->concat()->left; if (!RepMemoryUsageLeaf(left, &total_mem_usage)) { @@ -1967,18 +1990,21 @@ } next_node = right; } - } else if (cur_node->tag == RING) { - total_mem_usage += CordRepRing::AllocSize(cur_node->ring()->capacity()); - const CordRepRing* ring = cur_node->ring(); - CordRepRing::index_type pos = ring->head(), tail = ring->tail(); - do { - CordRep* node = ring->entry_child(pos); - assert(node->tag >= FLAT || node->tag == EXTERNAL); - RepMemoryUsageLeaf(node, &total_mem_usage); - } while ((pos = ring->advance(pos)) != tail); + } else if (cur_node->IsBtree()) { + total_mem_usage += sizeof(CordRepBtree); + const CordRepBtree* node = cur_node->btree(); + if (node->height() == 0) { + for (const CordRep* edge : node->Edges()) { + RepMemoryUsageDataEdge(edge, &total_mem_usage); + } + } else { + for (const CordRep* edge : node->Edges()) { + tree_stack.push_back(edge); + } + } } else { // Since cur_node is not a leaf or a concat node it must be a substring. - assert(cur_node->tag == SUBSTRING); + assert(cur_node->IsSubstring()); total_mem_usage += sizeof(CordRepSubstring); next_node = cur_node->substring()->child; if (RepMemoryUsageLeaf(next_node, &total_mem_usage)) {
diff --git a/absl/strings/cord.h b/absl/strings/cord.h index e758f1c..ac1832f 100644 --- a/absl/strings/cord.h +++ b/absl/strings/cord.h
@@ -79,8 +79,9 @@ #include "absl/functional/function_ref.h" #include "absl/meta/type_traits.h" #include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_btree_reader.h" #include "absl/strings/internal/cord_rep_ring.h" -#include "absl/strings/internal/cord_rep_ring_reader.h" #include "absl/strings/internal/cordz_functions.h" #include "absl/strings/internal/cordz_info.h" #include "absl/strings/internal/cordz_statistics.h" @@ -370,8 +371,8 @@ private: using CordRep = absl::cord_internal::CordRep; - using CordRepRing = absl::cord_internal::CordRepRing; - using CordRepRingReader = absl::cord_internal::CordRepRingReader; + using CordRepBtree = absl::cord_internal::CordRepBtree; + using CordRepBtreeReader = absl::cord_internal::CordRepBtreeReader; // Stack of right children of concat nodes that we have to visit. // Keep this at the end of the structure to avoid cache-thrashing. @@ -397,9 +398,9 @@ // Stack specific operator++ ChunkIterator& AdvanceStack(); - // Ring buffer specific operator++ - ChunkIterator& AdvanceRing(); - void AdvanceBytesRing(size_t n); + // Btree specific operator++ + ChunkIterator& AdvanceBtree(); + void AdvanceBytesBtree(size_t n); // Iterates `n` bytes, where `n` is expected to be greater than or equal to // `current_chunk_.size()`. @@ -415,8 +416,8 @@ // The number of bytes left in the `Cord` over which we are iterating. size_t bytes_remaining_ = 0; - // Cord reader for ring buffers. Empty if not traversing a ring buffer. - CordRepRingReader ring_reader_; + // Cord reader for cord btrees. Empty if not traversing a btree. + CordRepBtreeReader btree_reader_; // See 'Stack' alias definition. Stack stack_of_right_children_; @@ -1247,8 +1248,8 @@ } inline void Cord::ChunkIterator::InitTree(cord_internal::CordRep* tree) { - if (tree->tag == cord_internal::RING) { - current_chunk_ = ring_reader_.Reset(tree->ring()); + if (tree->tag == cord_internal::BTREE) { + current_chunk_ = btree_reader_.Init(tree->btree()); return; } @@ -1271,20 +1272,20 @@ } } -inline Cord::ChunkIterator& Cord::ChunkIterator::AdvanceRing() { - current_chunk_ = ring_reader_.Next(); +inline Cord::ChunkIterator& Cord::ChunkIterator::AdvanceBtree() { + current_chunk_ = btree_reader_.Next(); return *this; } -inline void Cord::ChunkIterator::AdvanceBytesRing(size_t n) { +inline void Cord::ChunkIterator::AdvanceBytesBtree(size_t n) { assert(n >= current_chunk_.size()); bytes_remaining_ -= n; if (bytes_remaining_) { if (n == current_chunk_.size()) { - current_chunk_ = ring_reader_.Next(); + current_chunk_ = btree_reader_.Next(); } else { - size_t offset = ring_reader_.length() - bytes_remaining_; - current_chunk_ = ring_reader_.Seek(offset); + size_t offset = btree_reader_.length() - bytes_remaining_; + current_chunk_ = btree_reader_.Seek(offset); } } else { current_chunk_ = {}; @@ -1297,7 +1298,7 @@ assert(bytes_remaining_ >= current_chunk_.size()); bytes_remaining_ -= current_chunk_.size(); if (bytes_remaining_ > 0) { - return ring_reader_ ? AdvanceRing() : AdvanceStack(); + return btree_reader_ ? AdvanceBtree() : AdvanceStack(); } else { current_chunk_ = {}; } @@ -1339,7 +1340,7 @@ if (ABSL_PREDICT_TRUE(n < current_chunk_.size())) { RemoveChunkPrefix(n); } else if (n != 0) { - ring_reader_ ? AdvanceBytesRing(n) : AdvanceBytesSlowPath(n); + btree_reader_ ? AdvanceBytesBtree(n) : AdvanceBytesSlowPath(n); } }
diff --git a/absl/strings/cord_test.cc b/absl/strings/cord_test.cc index 597378c..d029633 100644 --- a/absl/strings/cord_test.cc +++ b/absl/strings/cord_test.cc
@@ -206,7 +206,32 @@ ABSL_NAMESPACE_END } // namespace absl -TEST(Cord, AllFlatSizes) { +// The CordTest fixture runs all tests with and without Cord Btree enabled. +class CordTest : public testing::TestWithParam<bool> { + public: + CordTest() : was_btree_(absl::cord_internal::cord_btree_enabled.load()) { + absl::cord_internal::cord_btree_enabled.store(UseBtree()); + } + ~CordTest() override { + absl::cord_internal::cord_btree_enabled.store(was_btree_); + } + + // Returns true if test is running with btree enabled. + bool UseBtree() const { return GetParam(); } + + // Returns human readable string representation of the test parameter. + static std::string ToString(testing::TestParamInfo<bool> param) { + return param.param ? "Btree" : "Concat"; + } + + private: + const bool was_btree_; +}; + +INSTANTIATE_TEST_SUITE_P(WithParam, CordTest, testing::Bool(), + CordTest::ToString); + +TEST_P(CordTest, AllFlatSizes) { using absl::strings_internal::CordTestAccess; for (size_t s = 0; s < CordTestAccess::MaxFlatLength(); s++) { @@ -224,7 +249,7 @@ // We create a Cord at least 128GB in size using the fact that Cords can // internally reference-count; thus the Cord is enormous without actually // consuming very much memory. -TEST(GigabyteCord, FromExternal) { +TEST_P(CordTest, GigabyteCordFromExternal) { const size_t one_gig = 1024U * 1024U * 1024U; size_t max_size = 2 * one_gig; if (sizeof(max_size) > 4) max_size = 128 * one_gig; @@ -273,7 +298,7 @@ extern bool my_unique_true_boolean; bool my_unique_true_boolean = true; -TEST(Cord, Assignment) { +TEST_P(CordTest, Assignment) { absl::Cord x(absl::string_view("hi there")); absl::Cord y(x); ASSERT_EQ(std::string(x), "hi there"); @@ -327,7 +352,7 @@ } } -TEST(Cord, StartsEndsWith) { +TEST_P(CordTest, StartsEndsWith) { absl::Cord x(absl::string_view("abcde")); absl::Cord empty(""); @@ -360,13 +385,13 @@ ASSERT_TRUE(!empty.EndsWith("xyz")); } -TEST(Cord, Subcord) { +TEST_P(CordTest, Subcord) { RandomEngine rng(GTEST_FLAG_GET(random_seed)); const std::string s = RandomLowercaseString(&rng, 1024); absl::Cord a; AppendWithFragments(s, &rng, &a); - ASSERT_EQ(s.size(), a.size()); + ASSERT_EQ(s, std::string(a)); // Check subcords of a, from a variety of interesting points. std::set<size_t> positions; @@ -421,7 +446,7 @@ EXPECT_TRUE(sa.empty()); } -TEST(Cord, Swap) { +TEST_P(CordTest, Swap) { absl::string_view a("Dexter"); absl::string_view b("Mandark"); absl::Cord x(a); @@ -453,7 +478,7 @@ } } -TEST(Cord, CopyToString) { +TEST_P(CordTest, CopyToString) { VerifyCopyToString(absl::Cord()); VerifyCopyToString(absl::Cord("small cord")); VerifyCopyToString( @@ -461,45 +486,45 @@ "copying ", "to ", "a ", "string."})); } -TEST(TryFlat, Empty) { +TEST_P(CordTest, TryFlatEmpty) { absl::Cord c; EXPECT_EQ(c.TryFlat(), ""); } -TEST(TryFlat, Flat) { +TEST_P(CordTest, TryFlatFlat) { absl::Cord c("hello"); EXPECT_EQ(c.TryFlat(), "hello"); } -TEST(TryFlat, SubstrInlined) { +TEST_P(CordTest, TryFlatSubstrInlined) { absl::Cord c("hello"); c.RemovePrefix(1); EXPECT_EQ(c.TryFlat(), "ello"); } -TEST(TryFlat, SubstrFlat) { +TEST_P(CordTest, TryFlatSubstrFlat) { absl::Cord c("longer than 15 bytes"); absl::Cord sub = absl::CordTestPeer::MakeSubstring(c, 1, c.size() - 1); EXPECT_EQ(sub.TryFlat(), "onger than 15 bytes"); } -TEST(TryFlat, Concat) { +TEST_P(CordTest, TryFlatConcat) { absl::Cord c = absl::MakeFragmentedCord({"hel", "lo"}); EXPECT_EQ(c.TryFlat(), absl::nullopt); } -TEST(TryFlat, External) { +TEST_P(CordTest, TryFlatExternal) { absl::Cord c = absl::MakeCordFromExternal("hell", [](absl::string_view) {}); EXPECT_EQ(c.TryFlat(), "hell"); } -TEST(TryFlat, SubstrExternal) { +TEST_P(CordTest, TryFlatSubstrExternal) { absl::Cord c = absl::MakeCordFromExternal("hell", [](absl::string_view) {}); absl::Cord sub = absl::CordTestPeer::MakeSubstring(c, 1, c.size() - 1); EXPECT_EQ(sub.TryFlat(), "ell"); } -TEST(TryFlat, SubstrConcat) { +TEST_P(CordTest, TryFlatSubstrConcat) { absl::Cord c = absl::MakeFragmentedCord({"hello", " world"}); absl::Cord sub = absl::CordTestPeer::MakeSubstring(c, 1, c.size() - 1); EXPECT_EQ(sub.TryFlat(), absl::nullopt); @@ -507,7 +532,7 @@ EXPECT_EQ(c.TryFlat(), absl::nullopt); } -TEST(TryFlat, CommonlyAssumedInvariants) { +TEST_P(CordTest, TryFlatCommonlyAssumedInvariants) { // The behavior tested below is not part of the API contract of Cord, but it's // something we intend to be true in our current implementation. This test // exists to detect and prevent accidental breakage of the implementation. @@ -563,7 +588,7 @@ EXPECT_TRUE(IsFlat(c)); } -TEST(Cord, Flatten) { +TEST_P(CordTest, Flatten) { VerifyFlatten(absl::Cord()); VerifyFlatten(absl::Cord("small cord")); VerifyFlatten(absl::Cord("larger than small buffer optimization")); @@ -617,7 +642,7 @@ }; } // namespace -TEST(Cord, MultipleLengths) { +TEST_P(CordTest, MultipleLengths) { TestData d; for (size_t i = 0; i < d.size(); i++) { std::string a = d.data(i); @@ -693,7 +718,7 @@ namespace { -TEST(Cord, RemoveSuffixWithExternalOrSubstring) { +TEST_P(CordTest, RemoveSuffixWithExternalOrSubstring) { absl::Cord cord = absl::MakeCordFromExternal( "foo bar baz", [](absl::string_view s) { DoNothing(s, nullptr); }); @@ -708,7 +733,7 @@ EXPECT_EQ("foo", std::string(cord)); } -TEST(Cord, RemoveSuffixMakesZeroLengthNode) { +TEST_P(CordTest, RemoveSuffixMakesZeroLengthNode) { absl::Cord c; c.Append(absl::Cord(std::string(100, 'x'))); absl::Cord other_ref = c; // Prevent inplace appends @@ -735,7 +760,7 @@ } // Establish that ZedBlock does what we think it does. -TEST(CordSpliceTest, ZedBlock) { +TEST_P(CordTest, CordSpliceTestZedBlock) { absl::Cord blob = CordWithZedBlock(10); EXPECT_EQ(10, blob.size()); std::string s; @@ -743,7 +768,7 @@ EXPECT_EQ("zzzzzzzzzz", s); } -TEST(CordSpliceTest, ZedBlock0) { +TEST_P(CordTest, CordSpliceTestZedBlock0) { absl::Cord blob = CordWithZedBlock(0); EXPECT_EQ(0, blob.size()); std::string s; @@ -751,7 +776,7 @@ EXPECT_EQ("", s); } -TEST(CordSpliceTest, ZedBlockSuffix1) { +TEST_P(CordTest, CordSpliceTestZedBlockSuffix1) { absl::Cord blob = CordWithZedBlock(10); EXPECT_EQ(10, blob.size()); absl::Cord suffix(blob); @@ -763,7 +788,7 @@ } // Remove all of a prefix block -TEST(CordSpliceTest, ZedBlockSuffix0) { +TEST_P(CordTest, CordSpliceTestZedBlockSuffix0) { absl::Cord blob = CordWithZedBlock(10); EXPECT_EQ(10, blob.size()); absl::Cord suffix(blob); @@ -795,7 +820,7 @@ } // Taking an empty suffix of a block breaks appending. -TEST(CordSpliceTest, RemoveEntireBlock1) { +TEST_P(CordTest, CordSpliceTestRemoveEntireBlock1) { absl::Cord zero = CordWithZedBlock(10); absl::Cord suffix(zero); suffix.RemovePrefix(10); @@ -803,7 +828,7 @@ result.Append(suffix); } -TEST(CordSpliceTest, RemoveEntireBlock2) { +TEST_P(CordTest, CordSpliceTestRemoveEntireBlock2) { absl::Cord zero = CordWithZedBlock(10); absl::Cord prefix(zero); prefix.RemoveSuffix(10); @@ -813,7 +838,7 @@ result.Append(suffix); } -TEST(CordSpliceTest, RemoveEntireBlock3) { +TEST_P(CordTest, CordSpliceTestRemoveEntireBlock3) { absl::Cord blob = CordWithZedBlock(10); absl::Cord block = BigCord(10, 'b'); blob = SpliceCord(blob, 0, block); @@ -844,7 +869,7 @@ << "LHS=" << rhs_string << "; RHS=" << lhs_string; } -TEST(Cord, Compare) { +TEST_P(CordTest, Compare) { absl::Cord subcord("aaaaaBBBBBcccccDDDDD"); subcord = subcord.Subcord(3, 10); @@ -907,7 +932,7 @@ } } -TEST(Cord, CompareAfterAssign) { +TEST_P(CordTest, CompareAfterAssign) { absl::Cord a("aaaaaa1111111"); absl::Cord b("aaaaaa2222222"); a = "cccccc"; @@ -936,7 +961,7 @@ EXPECT_EQ(expected, sign(c.Compare(d))) << c << ", " << d; } -TEST(Compare, ComparisonIsUnsigned) { +TEST_P(CordTest, CompareComparisonIsUnsigned) { RandomEngine rng(GTEST_FLAG_GET(random_seed)); std::uniform_int_distribution<uint32_t> uniform_uint8(0, 255); char x = static_cast<char>(uniform_uint8(rng)); @@ -945,7 +970,7 @@ absl::Cord(std::string(GetUniformRandomUpTo(&rng, 100), x ^ 0x80)), &rng); } -TEST(Compare, RandomComparisons) { +TEST_P(CordTest, CompareRandomComparisons) { const int kIters = 5000; RandomEngine rng(GTEST_FLAG_GET(random_seed)); @@ -1003,43 +1028,43 @@ EXPECT_FALSE(b <= a); } -TEST(ComparisonOperators, Cord_Cord) { +TEST_P(CordTest, ComparisonOperators_Cord_Cord) { CompareOperators<absl::Cord, absl::Cord>(); } -TEST(ComparisonOperators, Cord_StringPiece) { +TEST_P(CordTest, ComparisonOperators_Cord_StringPiece) { CompareOperators<absl::Cord, absl::string_view>(); } -TEST(ComparisonOperators, StringPiece_Cord) { +TEST_P(CordTest, ComparisonOperators_StringPiece_Cord) { CompareOperators<absl::string_view, absl::Cord>(); } -TEST(ComparisonOperators, Cord_string) { +TEST_P(CordTest, ComparisonOperators_Cord_string) { CompareOperators<absl::Cord, std::string>(); } -TEST(ComparisonOperators, string_Cord) { +TEST_P(CordTest, ComparisonOperators_string_Cord) { CompareOperators<std::string, absl::Cord>(); } -TEST(ComparisonOperators, stdstring_Cord) { +TEST_P(CordTest, ComparisonOperators_stdstring_Cord) { CompareOperators<std::string, absl::Cord>(); } -TEST(ComparisonOperators, Cord_stdstring) { +TEST_P(CordTest, ComparisonOperators_Cord_stdstring) { CompareOperators<absl::Cord, std::string>(); } -TEST(ComparisonOperators, charstar_Cord) { +TEST_P(CordTest, ComparisonOperators_charstar_Cord) { CompareOperators<const char*, absl::Cord>(); } -TEST(ComparisonOperators, Cord_charstar) { +TEST_P(CordTest, ComparisonOperators_Cord_charstar) { CompareOperators<absl::Cord, const char*>(); } -TEST(ConstructFromExternal, ReleaserInvoked) { +TEST_P(CordTest, ConstructFromExternalReleaserInvoked) { // Empty external memory means the releaser should be called immediately. { bool invoked = false; @@ -1081,7 +1106,7 @@ } } -TEST(ConstructFromExternal, CompareContents) { +TEST_P(CordTest, ConstructFromExternalCompareContents) { RandomEngine rng(GTEST_FLAG_GET(random_seed)); for (int length = 1; length <= 2048; length *= 2) { @@ -1097,7 +1122,7 @@ } } -TEST(ConstructFromExternal, LargeReleaser) { +TEST_P(CordTest, ConstructFromExternalLargeReleaser) { RandomEngine rng(GTEST_FLAG_GET(random_seed)); constexpr size_t kLength = 256; std::string data = RandomLowercaseString(&rng, kLength); @@ -1112,7 +1137,7 @@ EXPECT_TRUE(invoked); } -TEST(ConstructFromExternal, FunctionPointerReleaser) { +TEST_P(CordTest, ConstructFromExternalFunctionPointerReleaser) { static absl::string_view data("hello world"); static bool invoked; auto* releaser = @@ -1129,7 +1154,7 @@ EXPECT_TRUE(invoked); } -TEST(ConstructFromExternal, MoveOnlyReleaser) { +TEST_P(CordTest, ConstructFromExternalMoveOnlyReleaser) { struct Releaser { explicit Releaser(bool* invoked) : invoked(invoked) {} Releaser(Releaser&& other) noexcept : invoked(other.invoked) {} @@ -1143,20 +1168,20 @@ EXPECT_TRUE(invoked); } -TEST(ConstructFromExternal, NoArgLambda) { +TEST_P(CordTest, ConstructFromExternalNoArgLambda) { bool invoked = false; (void)absl::MakeCordFromExternal("dummy", [&invoked]() { invoked = true; }); EXPECT_TRUE(invoked); } -TEST(ConstructFromExternal, StringViewArgLambda) { +TEST_P(CordTest, ConstructFromExternalStringViewArgLambda) { bool invoked = false; (void)absl::MakeCordFromExternal( "dummy", [&invoked](absl::string_view) { invoked = true; }); EXPECT_TRUE(invoked); } -TEST(ConstructFromExternal, NonTrivialReleaserDestructor) { +TEST_P(CordTest, ConstructFromExternalNonTrivialReleaserDestructor) { struct Releaser { explicit Releaser(bool* destroyed) : destroyed(destroyed) {} ~Releaser() { *destroyed = true; } @@ -1171,7 +1196,7 @@ EXPECT_TRUE(destroyed); } -TEST(ConstructFromExternal, ReferenceQualifierOverloads) { +TEST_P(CordTest, ConstructFromExternalReferenceQualifierOverloads) { struct Releaser { void operator()(absl::string_view) & { *lvalue_invoked = true; } void operator()(absl::string_view) && { *rvalue_invoked = true; } @@ -1199,7 +1224,7 @@ EXPECT_TRUE(rvalue_invoked); } -TEST(ExternalMemory, BasicUsage) { +TEST_P(CordTest, ExternalMemoryBasicUsage) { static const char* strings[] = {"", "hello", "there"}; for (const char* str : strings) { absl::Cord dst("(prefix)"); @@ -1210,7 +1235,7 @@ } } -TEST(ExternalMemory, RemovePrefixSuffix) { +TEST_P(CordTest, ExternalMemoryRemovePrefixSuffix) { // Exhaustively try all sub-strings. absl::Cord cord = MakeComposite(); std::string s = std::string(cord); @@ -1225,7 +1250,7 @@ } } -TEST(ExternalMemory, Get) { +TEST_P(CordTest, ExternalMemoryGet) { absl::Cord cord("hello"); AddExternalMemory(" world!", &cord); AddExternalMemory(" how are ", &cord); @@ -1244,16 +1269,16 @@ // Additionally we have some whiteboxed expectations based on our knowledge of // the layout and size of empty and inlined cords, and flat nodes. -TEST(CordMemoryUsage, Empty) { +TEST_P(CordTest, CordMemoryUsageEmpty) { EXPECT_EQ(sizeof(absl::Cord), absl::Cord().EstimatedMemoryUsage()); } -TEST(CordMemoryUsage, Embedded) { +TEST_P(CordTest, CordMemoryUsageEmbedded) { absl::Cord a("hello"); EXPECT_EQ(a.EstimatedMemoryUsage(), sizeof(absl::Cord)); } -TEST(CordMemoryUsage, EmbeddedAppend) { +TEST_P(CordTest, CordMemoryUsageEmbeddedAppend) { absl::Cord a("a"); absl::Cord b("bcd"); EXPECT_EQ(b.EstimatedMemoryUsage(), sizeof(absl::Cord)); @@ -1261,7 +1286,7 @@ EXPECT_EQ(a.EstimatedMemoryUsage(), sizeof(absl::Cord)); } -TEST(CordMemoryUsage, ExternalMemory) { +TEST_P(CordTest, CordMemoryUsageExternalMemory) { static const int kLength = 1000; absl::Cord cord; AddExternalMemory(std::string(kLength, 'x'), &cord); @@ -1269,14 +1294,14 @@ EXPECT_LE(cord.EstimatedMemoryUsage(), kLength * 1.5); } -TEST(CordMemoryUsage, Flat) { +TEST_P(CordTest, CordMemoryUsageFlat) { static const int kLength = 125; absl::Cord a(std::string(kLength, 'a')); EXPECT_GT(a.EstimatedMemoryUsage(), kLength); EXPECT_LE(a.EstimatedMemoryUsage(), kLength * 1.5); } -TEST(CordMemoryUsage, AppendFlat) { +TEST_P(CordTest, CordMemoryUsageAppendFlat) { using absl::strings_internal::CordTestAccess; absl::Cord a(std::string(CordTestAccess::MaxFlatLength(), 'a')); size_t length = a.EstimatedMemoryUsage(); @@ -1286,9 +1311,32 @@ EXPECT_LE(delta, CordTestAccess::MaxFlatLength() * 1.5); } +TEST_P(CordTest, CordMemoryUsageAppendExternal) { + static const int kLength = 1000; + using absl::strings_internal::CordTestAccess; + absl::Cord a(std::string(CordTestAccess::MaxFlatLength(), 'a')); + size_t length = a.EstimatedMemoryUsage(); + AddExternalMemory(std::string(kLength, 'b'), &a); + size_t delta = a.EstimatedMemoryUsage() - length; + EXPECT_GT(delta, kLength); + EXPECT_LE(delta, kLength * 1.5); +} + +TEST_P(CordTest, CordMemoryUsageSubString) { + static const int kLength = 2000; + using absl::strings_internal::CordTestAccess; + absl::Cord a(std::string(kLength, 'a')); + size_t length = a.EstimatedMemoryUsage(); + AddExternalMemory(std::string(kLength, 'b'), &a); + absl::Cord b = a.Subcord(0, kLength + kLength / 2); + size_t delta = b.EstimatedMemoryUsage() - length; + EXPECT_GT(delta, kLength); + EXPECT_LE(delta, kLength * 1.5); +} + // Regtest for a change that had to be rolled back because it expanded out // of the InlineRep too soon, which was observable through MemoryUsage(). -TEST(CordMemoryUsage, InlineRep) { +TEST_P(CordTest, CordMemoryUsageInlineRep) { constexpr size_t kMaxInline = 15; // Cord::InlineRep::N const std::string small_string(kMaxInline, 'x'); absl::Cord c1(small_string); @@ -1302,7 +1350,7 @@ } // namespace // Regtest for 7510292 (fix a bug introduced by 7465150) -TEST(Cord, Concat_Append) { +TEST_P(CordTest, Concat_Append) { // Create a rep of type CONCAT absl::Cord s1("foobarbarbarbarbar"); s1.Append("abcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefg"); @@ -1317,7 +1365,7 @@ EXPECT_EQ(s2.size(), size + 1); } -TEST(Cord, DiabolicalGrowth) { +TEST_P(CordTest, DiabolicalGrowth) { // This test exercises a diabolical Append(<one char>) on a cord, making the // cord shared before each Append call resulting in a terribly fragmented // resulting cord. @@ -1337,7 +1385,7 @@ cord.EstimatedMemoryUsage()); } -TEST(MakeFragmentedCord, MakeFragmentedCordFromInitializerList) { +TEST_P(CordTest, MakeFragmentedCordFromInitializerList) { absl::Cord fragmented = absl::MakeFragmentedCord({"A ", "fragmented ", "Cord"}); @@ -1357,7 +1405,7 @@ ASSERT_TRUE(++chunk_it == fragmented.chunk_end()); } -TEST(MakeFragmentedCord, MakeFragmentedCordFromVector) { +TEST_P(CordTest, MakeFragmentedCordFromVector) { std::vector<absl::string_view> chunks = {"A ", "fragmented ", "Cord"}; absl::Cord fragmented = absl::MakeFragmentedCord(chunks); @@ -1377,7 +1425,7 @@ ASSERT_TRUE(++chunk_it == fragmented.chunk_end()); } -TEST(CordChunkIterator, Traits) { +TEST_P(CordTest, CordChunkIteratorTraits) { static_assert(std::is_copy_constructible<absl::Cord::ChunkIterator>::value, ""); static_assert(std::is_copy_assignable<absl::Cord::ChunkIterator>::value, ""); @@ -1458,7 +1506,7 @@ EXPECT_TRUE(post_iter == cord.chunk_end()); // NOLINT } -TEST(CordChunkIterator, Operations) { +TEST_P(CordTest, CordChunkIteratorOperations) { absl::Cord empty_cord; VerifyChunkIterator(empty_cord, 0); @@ -1628,7 +1676,7 @@ VerifyCharIterator(subcords); } -TEST(Cord, StreamingOutput) { +TEST_P(CordTest, StreamingOutput) { absl::Cord c = absl::MakeFragmentedCord({"A ", "small ", "fragmented ", "Cord", "."}); std::stringstream output; @@ -1636,7 +1684,7 @@ EXPECT_EQ("A small fragmented Cord.", output.str()); } -TEST(Cord, ForEachChunk) { +TEST_P(CordTest, ForEachChunk) { for (int num_elements : {1, 10, 200}) { SCOPED_TRACE(num_elements); std::vector<std::string> cord_chunks; @@ -1654,7 +1702,7 @@ } } -TEST(Cord, SmallBufferAssignFromOwnData) { +TEST_P(CordTest, SmallBufferAssignFromOwnData) { constexpr size_t kMaxInline = 15; std::string contents = "small buff cord"; EXPECT_EQ(contents.size(), kMaxInline); @@ -1669,7 +1717,7 @@ } } -TEST(Cord, Format) { +TEST_P(CordTest, Format) { absl::Cord c; absl::Format(&c, "There were %04d little %s.", 3, "pigs"); EXPECT_EQ(c, "There were 0003 little pigs."); @@ -1770,7 +1818,7 @@ }; -TEST(Cord, ConstinitConstructor) { +TEST_P(CordTest, ConstinitConstructor) { TestConstinitConstructor( absl::strings_internal::MakeStringConstant(ShortView{})); TestConstinitConstructor(
diff --git a/absl/strings/internal/cord_internal.cc b/absl/strings/internal/cord_internal.cc index f79cb62..1767e6f 100644 --- a/absl/strings/internal/cord_internal.cc +++ b/absl/strings/internal/cord_internal.cc
@@ -31,6 +31,7 @@ kCordEnableRingBufferDefault); ABSL_CONST_INIT std::atomic<bool> shallow_subcords_enabled( kCordShallowSubcordsDefault); +ABSL_CONST_INIT std::atomic<bool> cord_btree_exhaustive_validation(false); void CordRep::Destroy(CordRep* rep) { assert(rep != nullptr);
diff --git a/absl/strings/internal/cord_internal.h b/absl/strings/internal/cord_internal.h index 371a7f9..b7f3f4c 100644 --- a/absl/strings/internal/cord_internal.h +++ b/absl/strings/internal/cord_internal.h
@@ -46,6 +46,12 @@ extern std::atomic<bool> cord_ring_buffer_enabled; extern std::atomic<bool> shallow_subcords_enabled; +// `cord_btree_exhaustive_validation` can be set to force exhaustive validation +// in debug assertions, and code that calls `IsValid()` explicitly. By default, +// assertions should be relatively cheap and AssertValid() can easily lead to +// O(n^2) complexity as recursive / full tree validation is O(n). +extern std::atomic<bool> cord_btree_exhaustive_validation; + inline void enable_cord_btree(bool enable) { cord_btree_enabled.store(enable, std::memory_order_relaxed); } @@ -210,6 +216,14 @@ // padding space from the base class (clang and gcc do, MSVC does not, etc) uint8_t storage[3]; + // Returns true if this instance's tag matches the requested type. + constexpr bool IsRing() const { return tag == RING; } + constexpr bool IsConcat() const { return tag == CONCAT; } + constexpr bool IsSubstring() const { return tag == SUBSTRING; } + constexpr bool IsExternal() const { return tag == EXTERNAL; } + constexpr bool IsFlat() const { return tag >= FLAT; } + constexpr bool IsBtree() const { return tag == BTREE; } + inline CordRepRing* ring(); inline const CordRepRing* ring() const; inline CordRepConcat* concat(); @@ -220,7 +234,6 @@ inline const CordRepExternal* external() const; inline CordRepFlat* flat(); inline const CordRepFlat* flat() const; - inline CordRepBtree* btree(); inline const CordRepBtree* btree() const; @@ -271,7 +284,7 @@ ExternalReleaserInvoker releaser_invoker; // Deletes (releases) the external rep. - // Requires rep != nullptr and rep->tag == EXTERNAL + // Requires rep != nullptr and rep->IsExternal() static void Delete(CordRep* rep); }; @@ -314,7 +327,7 @@ }; inline void CordRepExternal::Delete(CordRep* rep) { - assert(rep != nullptr && rep->tag == EXTERNAL); + assert(rep != nullptr && rep->IsExternal()); auto* rep_external = static_cast<CordRepExternal*>(rep); assert(rep_external->releaser_invoker != nullptr); rep_external->releaser_invoker(rep_external); @@ -525,32 +538,32 @@ static_assert(sizeof(InlineData) == kMaxInline + 1, ""); inline CordRepConcat* CordRep::concat() { - assert(tag == CONCAT); + assert(IsConcat()); return static_cast<CordRepConcat*>(this); } inline const CordRepConcat* CordRep::concat() const { - assert(tag == CONCAT); + assert(IsConcat()); return static_cast<const CordRepConcat*>(this); } inline CordRepSubstring* CordRep::substring() { - assert(tag == SUBSTRING); + assert(IsSubstring()); return static_cast<CordRepSubstring*>(this); } inline const CordRepSubstring* CordRep::substring() const { - assert(tag == SUBSTRING); + assert(IsSubstring()); return static_cast<const CordRepSubstring*>(this); } inline CordRepExternal* CordRep::external() { - assert(tag == EXTERNAL); + assert(IsExternal()); return static_cast<CordRepExternal*>(this); } inline const CordRepExternal* CordRep::external() const { - assert(tag == EXTERNAL); + assert(IsExternal()); return static_cast<const CordRepExternal*>(this); }
diff --git a/absl/strings/internal/cord_rep_btree.cc b/absl/strings/internal/cord_rep_btree.cc index 6978cfd..8fe589f 100644 --- a/absl/strings/internal/cord_rep_btree.cc +++ b/absl/strings/internal/cord_rep_btree.cc
@@ -32,6 +32,8 @@ ABSL_NAMESPACE_BEGIN namespace cord_internal { +constexpr size_t CordRepBtree::kMaxCapacity; // NOLINT: needed for c++ < c++17 + namespace { using NodeStack = CordRepBtree * [CordRepBtree::kMaxDepth]; @@ -42,6 +44,10 @@ constexpr auto kFront = CordRepBtree::kFront; constexpr auto kBack = CordRepBtree::kBack; +inline bool exhaustive_validation() { + return cord_btree_exhaustive_validation.load(std::memory_order_relaxed); +} + // Implementation of the various 'Dump' functions. // Prints the entire tree structure or 'rep'. External callers should // not specify 'depth' and leave it to its default (0) value. @@ -73,7 +79,7 @@ // indented by two spaces per recursive depth. stream << std::string(depth * 2, ' ') << sharing << " (" << sptr << ") "; - if (rep->tag == BTREE) { + if (rep->IsBtree()) { const CordRepBtree* node = rep->btree(); std::string label = node->height() ? absl::StrCat("Node(", node->height(), ")") : "Leaf"; @@ -357,7 +363,7 @@ Delete(tree); } -bool CordRepBtree::IsValid(const CordRepBtree* tree) { +bool CordRepBtree::IsValid(const CordRepBtree* tree, bool shallow) { #define NODE_CHECK_VALID(x) \ if (!(x)) { \ ABSL_RAW_LOG(ERROR, "CordRepBtree::CheckValid() FAILED: %s", #x); \ @@ -372,7 +378,7 @@ } NODE_CHECK_VALID(tree != nullptr); - NODE_CHECK_EQ(tree->tag, BTREE); + NODE_CHECK_VALID(tree->IsBtree()); NODE_CHECK_VALID(tree->height() <= kMaxHeight); NODE_CHECK_VALID(tree->begin() < tree->capacity()); NODE_CHECK_VALID(tree->end() <= tree->capacity()); @@ -381,7 +387,7 @@ for (CordRep* edge : tree->Edges()) { NODE_CHECK_VALID(edge != nullptr); if (tree->height() > 0) { - NODE_CHECK_VALID(edge->tag == BTREE); + NODE_CHECK_VALID(edge->IsBtree()); NODE_CHECK_VALID(edge->btree()->height() == tree->height() - 1); } else { NODE_CHECK_VALID(IsDataEdge(edge)); @@ -389,9 +395,9 @@ child_length += edge->length; } NODE_CHECK_EQ(child_length, tree->length); - if (tree->height() > 0) { + if ((!shallow || exhaustive_validation()) && tree->height() > 0) { for (CordRep* edge : tree->Edges()) { - if (!IsValid(edge->btree())) return false; + if (!IsValid(edge->btree(), shallow)) return false; } } return true; @@ -402,16 +408,17 @@ #ifndef NDEBUG -CordRepBtree* CordRepBtree::AssertValid(CordRepBtree* tree) { - if (!IsValid(tree)) { +CordRepBtree* CordRepBtree::AssertValid(CordRepBtree* tree, bool shallow) { + if (!IsValid(tree, shallow)) { Dump(tree, "CordRepBtree validation failed:", false, std::cout); ABSL_RAW_LOG(FATAL, "CordRepBtree::CheckValid() FAILED"); } return tree; } -const CordRepBtree* CordRepBtree::AssertValid(const CordRepBtree* tree) { - if (!IsValid(tree)) { +const CordRepBtree* CordRepBtree::AssertValid(const CordRepBtree* tree, + bool shallow) { + if (!IsValid(tree, shallow)) { Dump(tree, "CordRepBtree validation failed:", false, std::cout); ABSL_RAW_LOG(FATAL, "CordRepBtree::CheckValid() FAILED"); } @@ -882,7 +889,7 @@ } CordRepBtree* CordRepBtree::CreateSlow(CordRep* rep) { - if (rep->tag == BTREE) return rep->btree(); + if (rep->IsBtree()) return rep->btree(); CordRepBtree* node = nullptr; auto consume = [&node](CordRep* r, size_t offset, size_t length) { @@ -898,7 +905,7 @@ } CordRepBtree* CordRepBtree::AppendSlow(CordRepBtree* tree, CordRep* rep) { - if (ABSL_PREDICT_TRUE(rep->tag == BTREE)) { + if (ABSL_PREDICT_TRUE(rep->IsBtree())) { return MergeTrees(tree, rep->btree()); } auto consume = [&tree](CordRep* r, size_t offset, size_t length) { @@ -910,7 +917,7 @@ } CordRepBtree* CordRepBtree::PrependSlow(CordRepBtree* tree, CordRep* rep) { - if (ABSL_PREDICT_TRUE(rep->tag == BTREE)) { + if (ABSL_PREDICT_TRUE(rep->IsBtree())) { return MergeTrees(rep->btree(), tree); } auto consume = [&tree](CordRep* r, size_t offset, size_t length) {
diff --git a/absl/strings/internal/cord_rep_btree.h b/absl/strings/internal/cord_rep_btree.h index 7d85473..8f000ca 100644 --- a/absl/strings/internal/cord_rep_btree.h +++ b/absl/strings/internal/cord_rep_btree.h
@@ -153,7 +153,7 @@ }; // Creates a btree from the given input. Adopts a ref of `rep`. - // If the input `rep` is itself a btree, i.e., `tag == BTREE`, then this + // If the input `rep` is itself a btree, i.e., `IsBtree()`, then this // function immediately returns `rep->btree()`. If the input is a valid data // edge (see IsDataEdge()), then a new leaf node is returned containing `rep` // as the sole data edge. Else, the input is assumed to be a (legacy) concat @@ -266,10 +266,28 @@ // holding a FLAT or EXTERNAL child rep. static bool IsDataEdge(const CordRep* rep); - // Diagnostics - static bool IsValid(const CordRepBtree* tree); - static CordRepBtree* AssertValid(CordRepBtree* tree); - static const CordRepBtree* AssertValid(const CordRepBtree* tree); + // Diagnostics: returns true if `tree` is valid and internally consistent. + // If `shallow` is false, then the provided top level node and all child nodes + // below it are recursively checked. If `shallow` is true, only the provided + // node in `tree` and the cumulative length, type and height of the direct + // child nodes of `tree` are checked. The value of `shallow` is ignored if the + // internal `cord_btree_exhaustive_validation` diagnostics variable is true, + // in which case the performed validations works as if `shallow` were false. + // This function is intended for debugging and testing purposes only. + static bool IsValid(const CordRepBtree* tree, bool shallow = false); + + // Diagnostics: asserts that the provided tree is valid. + // `AssertValid()` performs a shallow validation by default. `shallow` can be + // set to false in which case an exhaustive validation is performed. This + // function is implemented in terms of calling `IsValid()` and asserting the + // return value to be true. See `IsValid()` for more information. + // This function is intended for debugging and testing purposes only. + static CordRepBtree* AssertValid(CordRepBtree* tree, bool shallow = true); + static const CordRepBtree* AssertValid(const CordRepBtree* tree, + bool shallow = true); + + // Diagnostics: dump the contents of this tree to `stream`. + // This function is intended for debugging and testing purposes only. static void Dump(const CordRep* rep, std::ostream& stream); static void Dump(const CordRep* rep, absl::string_view label, std::ostream& stream); @@ -496,12 +514,12 @@ }; inline CordRepBtree* CordRep::btree() { - assert(tag == BTREE); + assert(IsBtree()); return static_cast<CordRepBtree*>(this); } inline const CordRepBtree* CordRep::btree() const { - assert(tag == BTREE); + assert(IsBtree()); return static_cast<const CordRepBtree*>(this); } @@ -571,7 +589,7 @@ inline CordRepBtree* CordRepBtree::New(CordRep* rep) { CordRepBtree* tree = new CordRepBtree; - int height = rep->tag == BTREE ? rep->btree()->height() + 1 : 0; + int height = rep->IsBtree() ? rep->btree()->height() + 1 : 0; tree->length = rep->length; tree->InitInstance(height, /*begin=*/0, /*end=*/1); tree->edges_[0] = rep; @@ -834,11 +852,13 @@ #ifdef NDEBUG -inline CordRepBtree* CordRepBtree::AssertValid(CordRepBtree* tree) { +inline CordRepBtree* CordRepBtree::AssertValid(CordRepBtree* tree, + bool /* shallow */) { return tree; } -inline const CordRepBtree* CordRepBtree::AssertValid(const CordRepBtree* tree) { +inline const CordRepBtree* CordRepBtree::AssertValid(const CordRepBtree* tree, + bool /* shallow */) { return tree; }
diff --git a/absl/strings/internal/cord_rep_btree_test.cc b/absl/strings/internal/cord_rep_btree_test.cc index 7a0b781..073a7d4 100644 --- a/absl/strings/internal/cord_rep_btree_test.cc +++ b/absl/strings/internal/cord_rep_btree_test.cc
@@ -24,6 +24,7 @@ #include "gtest/gtest.h" #include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" +#include "absl/cleanup/cleanup.h" #include "absl/strings/internal/cord_internal.h" #include "absl/strings/internal/cord_rep_test_util.h" #include "absl/strings/str_cat.h" @@ -1346,6 +1347,48 @@ CordRep::Unref(tree); } +TEST(CordRepBtreeTest, CheckAssertValidShallowVsDeep) { + // Restore exhaustive validation on any exit. + const bool exhaustive_validation = cord_btree_exhaustive_validation.load(); + auto cleanup = absl::MakeCleanup([exhaustive_validation] { + cord_btree_exhaustive_validation.store(exhaustive_validation); + }); + + // Create a tree of at least 2 levels, and mess with the original flat, which + // should go undetected in shallow mode as the flat is too far away, but + // should be detected in forced non-shallow mode. + CordRep* flat = MakeFlat("abc"); + CordRepBtree* tree = CordRepBtree::Create(flat); + constexpr size_t max_cap = CordRepBtree::kMaxCapacity; + const size_t n = max_cap * max_cap * 2; + for (size_t i = 0; i < n; ++i) { + tree = CordRepBtree::Append(tree, MakeFlat("Hello world")); + } + flat->length = 100; + + cord_btree_exhaustive_validation.store(false); + EXPECT_FALSE(CordRepBtree::IsValid(tree)); + EXPECT_TRUE(CordRepBtree::IsValid(tree, true)); + EXPECT_FALSE(CordRepBtree::IsValid(tree, false)); + CordRepBtree::AssertValid(tree); + CordRepBtree::AssertValid(tree, true); +#if defined(GTEST_HAS_DEATH_TEST) + EXPECT_DEBUG_DEATH(CordRepBtree::AssertValid(tree, false), ".*"); +#endif + + cord_btree_exhaustive_validation.store(true); + EXPECT_FALSE(CordRepBtree::IsValid(tree)); + EXPECT_FALSE(CordRepBtree::IsValid(tree, true)); + EXPECT_FALSE(CordRepBtree::IsValid(tree, false)); +#if defined(GTEST_HAS_DEATH_TEST) + EXPECT_DEBUG_DEATH(CordRepBtree::AssertValid(tree), ".*"); + EXPECT_DEBUG_DEATH(CordRepBtree::AssertValid(tree, true), ".*"); +#endif + + flat->length = 3; + CordRep::Unref(tree); +} + } // namespace } // namespace cord_internal ABSL_NAMESPACE_END
diff --git a/absl/strings/internal/cord_rep_ring.cc b/absl/strings/internal/cord_rep_ring.cc index 20a6fc2..db1f63f 100644 --- a/absl/strings/internal/cord_rep_ring.cc +++ b/absl/strings/internal/cord_rep_ring.cc
@@ -40,7 +40,7 @@ enum class Direction { kForward, kReversed }; inline bool IsFlatOrExternal(CordRep* rep) { - return rep->tag >= FLAT || rep->tag == EXTERNAL; + return rep->IsFlat() || rep->IsExternal(); } // Verifies that n + extra <= kMaxCapacity: throws std::length_error otherwise. @@ -229,7 +229,7 @@ } void CordRepRing::Delete(CordRepRing* rep) { - assert(rep != nullptr && rep->tag == RING); + assert(rep != nullptr && rep->IsRing()); #if defined(__cpp_sized_deallocation) size_t size = AllocSize(rep->capacity_); rep->~CordRepRing(); @@ -360,7 +360,7 @@ if (IsFlatOrExternal(child)) { return CreateFromLeaf(child, 0, length, extra); } - if (child->tag == RING) { + if (child->IsRing()) { return Mutable(child->ring(), extra); } return CreateSlow(child, extra); @@ -433,7 +433,7 @@ CordRepRing* CordRepRing::AppendSlow(CordRepRing* rep, CordRep* child) { Consume(child, [&rep](CordRep* child_arg, size_t offset, size_t len) { - if (child_arg->tag == RING) { + if (child_arg->IsRing()) { rep = AddRing<AddMode::kAppend>(rep, child_arg->ring(), offset, len); } else { rep = AppendLeaf(rep, child_arg, offset, len); @@ -460,7 +460,7 @@ if (IsFlatOrExternal(child)) { return AppendLeaf(rep, child, 0, length); } - if (child->tag == RING) { + if (child->IsRing()) { return AddRing<AddMode::kAppend>(rep, child->ring(), 0, length); } return AppendSlow(rep, child); @@ -496,7 +496,7 @@ if (IsFlatOrExternal(child)) { return PrependLeaf(rep, child, 0, length); } - if (child->tag == RING) { + if (child->IsRing()) { return AddRing<AddMode::kPrepend>(rep, child->ring(), 0, length); } return PrependSlow(rep, child);
diff --git a/absl/strings/internal/cord_rep_ring.h b/absl/strings/internal/cord_rep_ring.h index 2082a56..44db849 100644 --- a/absl/strings/internal/cord_rep_ring.h +++ b/absl/strings/internal/cord_rep_ring.h
@@ -570,12 +570,12 @@ // Now that CordRepRing is defined, we can define CordRep's helper casts: inline CordRepRing* CordRep::ring() { - assert(tag == RING); + assert(IsRing()); return static_cast<CordRepRing*>(this); } inline const CordRepRing* CordRep::ring() const { - assert(tag == RING); + assert(IsRing()); return static_cast<const CordRepRing*>(this); }
diff --git a/absl/strings/internal/cordz_info.cc b/absl/strings/internal/cordz_info.cc index a3a0b9c..5c18bbc 100644 --- a/absl/strings/internal/cordz_info.cc +++ b/absl/strings/internal/cordz_info.cc
@@ -19,6 +19,7 @@ #include "absl/container/inlined_vector.h" #include "absl/debugging/stacktrace.h" #include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" #include "absl/strings/internal/cord_rep_ring.h" #include "absl/strings/internal/cordz_handle.h" #include "absl/strings/internal/cordz_statistics.h" @@ -83,19 +84,23 @@ // Process all top level linear nodes (substrings and flats). repref = CountLinearReps(repref, memory_usage_); - // We should have have either a concat or ring node node if not null. if (repref.rep != nullptr) { - assert(repref.rep->tag == RING || repref.rep->tag == CONCAT); if (repref.rep->tag == RING) { AnalyzeRing(repref); + } else if (repref.rep->tag == BTREE) { + AnalyzeBtree(repref); } else if (repref.rep->tag == CONCAT) { AnalyzeConcat(repref); + } else { + // We should have either a concat, btree, or ring node if not null. + assert(false); } } // Adds values to output statistics_.estimated_memory_usage += memory_usage_.total; - statistics_.estimated_fair_share_memory_usage += memory_usage_.fair_share; + statistics_.estimated_fair_share_memory_usage += + static_cast<size_t>(memory_usage_.fair_share); } private: @@ -117,13 +122,13 @@ // Memory usage values struct MemoryUsage { size_t total = 0; - size_t fair_share = 0; + double fair_share = 0.0; // Adds 'size` memory usage to this class, with a cumulative (recursive) // reference count of `refcount` void Add(size_t size, size_t refcount) { total += size; - fair_share += size / refcount; + fair_share += static_cast<double>(size) / refcount; } }; @@ -215,28 +220,32 @@ } } - // Counts the provided ring buffer child into `child_usage`. - void CountRingChild(const CordRep* child, MemoryUsage& child_usage) { - RepRef rep{child, static_cast<size_t>(child->refcount.Get())}; - rep = CountLinearReps(rep, child_usage); - assert(rep.rep == nullptr); - } - - // Analyzes the provided ring. As ring buffers can have many child nodes, the - // effect of rounding errors can become non trivial, so we compute the totals - // first at the ring level, and then divide the fair share of the total - // including children fair share totals. + // Analyzes the provided ring. void AnalyzeRing(RepRef rep) { statistics_.node_count++; statistics_.node_counts.ring++; - MemoryUsage ring_usage; const CordRepRing* ring = rep.rep->ring(); - ring_usage.Add(CordRepRing::AllocSize(ring->capacity()), 1); + memory_usage_.Add(CordRepRing::AllocSize(ring->capacity()), rep.refcount); ring->ForEach([&](CordRepRing::index_type pos) { - CountRingChild(ring->entry_child(pos), ring_usage); + CountLinearReps(rep.Child(ring->entry_child(pos)), memory_usage_); }); - memory_usage_.total += ring_usage.total; - memory_usage_.fair_share += ring_usage.fair_share / rep.refcount; + } + + // Analyzes the provided btree. + void AnalyzeBtree(RepRef rep) { + statistics_.node_count++; + statistics_.node_counts.btree++; + memory_usage_.Add(sizeof(CordRepBtree), rep.refcount); + const CordRepBtree* tree = rep.rep->btree(); + if (tree->height() > 0) { + for (CordRep* edge : tree->Edges()) { + AnalyzeBtree(rep.Child(edge)); + } + } else { + for (CordRep* edge : tree->Edges()) { + CountLinearReps(rep.Child(edge), memory_usage_); + } + } } CordzStatistics& statistics_;
diff --git a/absl/strings/internal/cordz_info_statistics_test.cc b/absl/strings/internal/cordz_info_statistics_test.cc index 9f2842d..7430d28 100644 --- a/absl/strings/internal/cordz_info_statistics_test.cc +++ b/absl/strings/internal/cordz_info_statistics_test.cc
@@ -21,6 +21,7 @@ #include "absl/base/config.h" #include "absl/strings/cord.h" #include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" #include "absl/strings/internal/cord_rep_flat.h" #include "absl/strings/internal/cord_rep_ring.h" #include "absl/strings/internal/cordz_info.h" @@ -42,6 +43,8 @@ namespace { +using ::testing::Ge; + // Creates a flat of the specified allocated size CordRepFlat* Flat(size_t size) { // Round up to a tag size, as we are going to poke an exact tag size back into @@ -134,8 +137,8 @@ } // Computes fair share memory used in a naive 'we dare to recurse' way. -size_t FairShare(CordRep* rep, size_t ref = 1) { - size_t self = 0, children = 0; +double FairShareImpl(CordRep* rep, size_t ref) { + double self = 0.0, children = 0.0; ref *= rep->refcount.Get(); if (rep->tag >= FLAT) { self = SizeOf(rep->flat()); @@ -143,22 +146,32 @@ self = SizeOf(rep->external()); } else if (rep->tag == SUBSTRING) { self = SizeOf(rep->substring()); - children = FairShare(rep->substring()->child, ref); + children = FairShareImpl(rep->substring()->child, ref); + } else if (rep->tag == BTREE) { + self = SizeOf(rep->btree()); + for (CordRep*edge : rep->btree()->Edges()) { + children += FairShareImpl(edge, ref); + } } else if (rep->tag == RING) { self = SizeOf(rep->ring()); rep->ring()->ForEach([&](CordRepRing::index_type i) { - self += FairShare(rep->ring()->entry_child(i)); + self += FairShareImpl(rep->ring()->entry_child(i), 1); }); } else if (rep->tag == CONCAT) { self = SizeOf(rep->concat()); - children = FairShare(rep->concat()->left, ref) + - FairShare(rep->concat()->right, ref); + children = FairShareImpl(rep->concat()->left, ref) + + FairShareImpl(rep->concat()->right, ref); } else { assert(false); } return self / ref + children; } +// Returns the fair share memory size from `ShareFhareImpl()` as a size_t. +size_t FairShare(CordRep* rep, size_t ref = 1) { + return static_cast<size_t>(FairShareImpl(rep, ref)); +} + // Samples the cord and returns CordzInfo::GetStatistics() CordzStatistics SampleCord(CordRep* rep) { InlineData cord(rep); @@ -191,6 +204,7 @@ STATS_MATCHER_EXPECT_EQ(node_counts.concat); STATS_MATCHER_EXPECT_EQ(node_counts.substring); STATS_MATCHER_EXPECT_EQ(node_counts.ring); + STATS_MATCHER_EXPECT_EQ(node_counts.btree); STATS_MATCHER_EXPECT_EQ(estimated_memory_usage); STATS_MATCHER_EXPECT_EQ(estimated_fair_share_memory_usage); @@ -424,6 +438,103 @@ EXPECT_THAT(SampleCord(substring), EqStatistics(expected)); } +TEST(CordzInfoStatisticsTest, BtreeLeaf) { + ASSERT_THAT(CordRepBtree::kMaxCapacity, Ge(3)); + RefHelper ref; + auto* flat1 = Flat(2000); + auto* flat2 = Flat(200); + auto* substr = Substring(flat2); + auto* external = External(3000); + + CordRepBtree* tree = CordRepBtree::Create(flat1); + tree = CordRepBtree::Append(tree, substr); + tree = CordRepBtree::Append(tree, external); + size_t flat3_count = CordRepBtree::kMaxCapacity - 3; + size_t flat3_size = 0; + for (size_t i = 0; i < flat3_count; ++i) { + auto* flat3 = Flat(70); + flat3_size += SizeOf(flat3); + tree = CordRepBtree::Append(tree, flat3); + } + ref.NeedsUnref(tree); + + CordzStatistics expected; + expected.size = tree->length; + expected.estimated_memory_usage = SizeOf(tree) + SizeOf(flat1) + + SizeOf(flat2) + SizeOf(substr) + + flat3_size + SizeOf(external); + expected.estimated_fair_share_memory_usage = expected.estimated_memory_usage; + expected.node_count = 1 + 3 + 1 + flat3_count; + expected.node_counts.flat = 2 + flat3_count; + expected.node_counts.flat_128 = flat3_count; + expected.node_counts.flat_256 = 1; + expected.node_counts.external = 1; + expected.node_counts.substring = 1; + expected.node_counts.btree = 1; + + EXPECT_THAT(SampleCord(tree), EqStatistics(expected)); +} + +TEST(CordzInfoStatisticsTest, BtreeNodeShared) { + RefHelper ref; + static constexpr int leaf_count = 3; + const size_t flat3_count = CordRepBtree::kMaxCapacity - 3; + ASSERT_THAT(flat3_count, Ge(0)); + + CordRepBtree* tree = nullptr; + size_t mem_size = 0; + for (int i = 0; i < leaf_count; ++i) { + auto* flat1 = ref.Ref(Flat(2000), 9); + mem_size += SizeOf(flat1); + if (i == 0) { + tree = CordRepBtree::Create(flat1); + } else { + tree = CordRepBtree::Append(tree, flat1); + } + + auto* flat2 = Flat(200); + auto* substr = Substring(flat2); + mem_size += SizeOf(flat2) + SizeOf(substr); + tree = CordRepBtree::Append(tree, substr); + + auto* external = External(30); + mem_size += SizeOf(external); + tree = CordRepBtree::Append(tree, external); + + for (size_t i = 0; i < flat3_count; ++i) { + auto* flat3 = Flat(70); + mem_size += SizeOf(flat3); + tree = CordRepBtree::Append(tree, flat3); + } + + if (i == 0) { + mem_size += SizeOf(tree); + } else { + mem_size += SizeOf(tree->Edges().back()->btree()); + } + } + ref.NeedsUnref(tree); + + // Ref count: 2 for top (add 1), 5 for leaf 0 (add 4). + ref.Ref(tree, 1); + ref.Ref(tree->Edges().front(), 4); + + CordzStatistics expected; + expected.size = tree->length; + expected.estimated_memory_usage = SizeOf(tree) + mem_size; + expected.estimated_fair_share_memory_usage = FairShare(tree); + + expected.node_count = 1 + leaf_count * (1 + 3 + 1 + flat3_count); + expected.node_counts.flat = leaf_count * (2 + flat3_count); + expected.node_counts.flat_128 = leaf_count * flat3_count; + expected.node_counts.flat_256 = leaf_count; + expected.node_counts.external = leaf_count; + expected.node_counts.substring = leaf_count; + expected.node_counts.btree = 1 + leaf_count; + + EXPECT_THAT(SampleCord(tree), EqStatistics(expected)); +} + TEST(CordzInfoStatisticsTest, ThreadSafety) { Notification stop; static constexpr int kNumThreads = 8; @@ -471,9 +582,15 @@ CordRep::Unref(cord.as_tree()); cord.set_inline_size(0); } else { - // 50/50 Ring or Flat coin toss + // Coin toss to 25% ring, 25% btree, and 50% flat. CordRep* rep = Flat(256); - rep = (coin_toss(gen) != 0) ? CordRepRing::Create(rep) : rep; + if (coin_toss(gen) != 0) { + if (coin_toss(gen) != 0) { + rep = CordRepRing::Create(rep); + } else { + rep = CordRepBtree::Create(rep); + } + } cord.make_tree(rep); // 50/50 sample
diff --git a/absl/strings/internal/cordz_statistics.h b/absl/strings/internal/cordz_statistics.h index e03c651..da4c7db 100644 --- a/absl/strings/internal/cordz_statistics.h +++ b/absl/strings/internal/cordz_statistics.h
@@ -40,6 +40,7 @@ size_t substring = 0; // #substring reps size_t concat = 0; // #concat reps size_t ring = 0; // #ring buffer reps + size_t btree = 0; // #btree reps }; // The size of the cord in bytes. This matches the result of Cord::size(). @@ -61,6 +62,8 @@ // The total number of nodes referenced by this cord. // For ring buffer Cords, this includes the 'ring buffer' node. + // For btree Cords, this includes all 'CordRepBtree' tree nodes as well as all + // the substring, flat and external nodes referenced by the tree. // A value of 0 implies the property has not been recorded. int64_t node_count = 0;
diff --git a/absl/strings/string_view.h b/absl/strings/string_view.h index ec6c431..ea76052 100644 --- a/absl/strings/string_view.h +++ b/absl/strings/string_view.h
@@ -198,9 +198,9 @@ // Implicit constructor of a `string_view` from NUL-terminated `str`. When // accepting possibly null strings, use `absl::NullSafeStringView(str)` // instead (see below). + // The length check is skipped since it is unnecessary and causes code bloat. constexpr string_view(const char* str) // NOLINT(runtime/explicit) - : ptr_(str), - length_(str ? CheckLengthInternal(StrlenInternal(str)) : 0) {} + : ptr_(str), length_(str ? StrlenInternal(str) : 0) {} // Implicit constructor of a `string_view` from a `const char*` and length. constexpr string_view(const char* data, size_type len)