diff --git a/absl/base/internal/low_level_alloc.cc b/absl/base/internal/low_level_alloc.cc index a5bd71d..3887a92 100644 --- a/absl/base/internal/low_level_alloc.cc +++ b/absl/base/internal/low_level_alloc.cc
@@ -47,6 +47,12 @@ #ifdef __linux__ #include <sys/prctl.h> +#ifndef PR_SET_VMA +#define PR_SET_VMA 0x53564d41 +#endif +#ifndef PR_SET_VMA_ANON_NAME +#define PR_SET_VMA_ANON_NAME 0 +#endif #endif #include <string.h>
diff --git a/absl/base/macros.h b/absl/base/macros.h index d42ecf1..392b7b8 100644 --- a/absl/base/macros.h +++ b/absl/base/macros.h
@@ -56,6 +56,24 @@ ABSL_NAMESPACE_END } // namespace absl +// ABSL_INTERNAL_UNEVALUATED() +// +// Expands into a no-op expression that contains the given expression. Used to +// avoid unused-variable warnings in configurations that don't need to evaluate +// the given expression (e.g., NDEBUG). +#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L +// We use `decltype` here to avoid generating unnecessary code that the +// optimizer then has to optimize away. +// This not only improves compilation performance by reducing codegen bloat +// and optimization work, but also guarantees fast run-time performance without +// having to rely on the optimizer. +#define ABSL_INTERNAL_UNEVALUATED(expr) (decltype((void)(expr))()) +#else +// Pre-C++20, lambdas can't be inside unevaluated operands, so we're forced to +// rely on the optimizer. +#define ABSL_INTERNAL_UNEVALUATED(expr) (false ? (void)(expr) : void()) +#endif + // ABSL_BAD_CALL_IF() // // Used on a function overload to trap bad calls: any call that matches the @@ -93,18 +111,7 @@ // This macro is inspired by // https://akrzemi1.wordpress.com/2017/05/18/asserts-in-constexpr-functions/ #if defined(NDEBUG) -#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L -// We use `decltype` here to avoid generating unnecessary code that the -// optimizer then has to optimize away. -// This not only improves compilation performance by reducing codegen bloat -// and optimization work, but also guarantees fast run-time performance without -// having to rely on the optimizer. -#define ABSL_ASSERT(expr) (decltype((expr) ? void() : void())()) -#else -// Pre-C++20, lambdas can't be inside unevaluated operands, so we're forced to -// rely on the optimizer. -#define ABSL_ASSERT(expr) (false ? ((expr) ? void() : void()) : void()) -#endif +#define ABSL_ASSERT(expr) ABSL_INTERNAL_UNEVALUATED((expr) ? void() : void()) #else #define ABSL_ASSERT(expr) \ (ABSL_PREDICT_TRUE((expr)) ? static_cast<void>(0) \
diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index be10bb5..65081ee 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel
@@ -133,6 +133,7 @@ "//absl/base:base_internal", "//absl/base:config", "//absl/base:core_headers", + "//absl/base:hardening", "//absl/memory", "//absl/meta:type_traits", "//absl/types:span", @@ -148,6 +149,7 @@ ":inlined_vector_internal", "//absl/algorithm", "//absl/base:core_headers", + "//absl/base:hardening", "//absl/base:iterator_traits_internal", "//absl/base:throw_delegate", "//absl/hash:weakly_mixed_integer",
diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 589a3a4..983d187 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt
@@ -181,6 +181,7 @@ absl::compressed_tuple absl::config absl::core_headers + absl::hardening absl::memory absl::span absl::type_traits @@ -197,6 +198,7 @@ DEPS absl::algorithm absl::core_headers + absl::hardening absl::inlined_vector_internal absl::throw_delegate absl::memory
diff --git a/absl/container/inlined_vector.h b/absl/container/inlined_vector.h index c1b5785..020e37f 100644 --- a/absl/container/inlined_vector.h +++ b/absl/container/inlined_vector.h
@@ -47,6 +47,7 @@ #include "absl/algorithm/algorithm.h" #include "absl/base/attributes.h" +#include "absl/base/internal/hardening.h" #include "absl/base/internal/iterator_traits.h" #include "absl/base/macros.h" #include "absl/base/optimization.h" @@ -363,14 +364,14 @@ // // Returns a `reference` to the `i`th element of the inlined vector. reference operator[](size_type i) ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(i < size()); + absl::base_internal::HardeningAssertLT(i, size()); return data()[i]; } // Overload of `InlinedVector::operator[](...)` that returns a // `const_reference` to the `i`th element of the inlined vector. const_reference operator[](size_type i) const ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(i < size()); + absl::base_internal::HardeningAssertLT(i, size()); return data()[i]; } @@ -404,14 +405,14 @@ // // Returns a `reference` to the first element of the inlined vector. reference front() ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(!empty()); + absl::base_internal::HardeningAssertNonEmpty(*this); return data()[0]; } // Overload of `InlinedVector::front()` that returns a `const_reference` to // the first element of the inlined vector. const_reference front() const ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(!empty()); + absl::base_internal::HardeningAssertNonEmpty(*this); return data()[0]; } @@ -419,14 +420,14 @@ // // Returns a `reference` to the last element of the inlined vector. reference back() ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(!empty()); + absl::base_internal::HardeningAssertNonEmpty(*this); return data()[size() - 1]; } // Overload of `InlinedVector::back()` that returns a `const_reference` to the // last element of the inlined vector. const_reference back() const ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(!empty()); + absl::base_internal::HardeningAssertNonEmpty(*this); return data()[size() - 1]; } @@ -600,7 +601,7 @@ // NOTE: If `n` is smaller than `size()`, extra elements are destroyed. If `n` // is larger than `size()`, new elements are value-initialized. void resize(size_type n) { - ABSL_HARDENING_ASSERT(n <= max_size()); + absl::base_internal::HardeningAssertLE(n, max_size()); storage_.Resize(DefaultValueAdapter<A>(), n); } @@ -610,7 +611,7 @@ // NOTE: if `n` is smaller than `size()`, extra elements are destroyed. If `n` // is larger than `size()`, new elements are copied-constructed from `v`. void resize(size_type n, const_reference v) { - ABSL_HARDENING_ASSERT(n <= max_size()); + absl::base_internal::HardeningAssertLE(n, max_size()); storage_.Resize(CopyValueAdapter<A>(std::addressof(v)), n); } @@ -635,8 +636,8 @@ // the newly inserted elements. iterator insert(const_iterator pos, size_type n, const_reference v) ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(pos >= begin()); - ABSL_HARDENING_ASSERT(pos <= end()); + absl::base_internal::HardeningAssertGE(pos, cbegin()); + absl::base_internal::HardeningAssertLE(pos, cend()); if (ABSL_PREDICT_TRUE(n != 0)) { value_type dealias = v; @@ -676,8 +677,8 @@ EnableIfAtLeastForwardIterator<ForwardIterator> = 0> iterator insert(const_iterator pos, ForwardIterator first, ForwardIterator last) ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(pos >= begin()); - ABSL_HARDENING_ASSERT(pos <= end()); + absl::base_internal::HardeningAssertGE(pos, cbegin()); + absl::base_internal::HardeningAssertLE(pos, cend()); if (ABSL_PREDICT_TRUE(first != last)) { return storage_.Insert( @@ -697,8 +698,8 @@ DisableIfAtLeastForwardIterator<InputIterator> = 0> iterator insert(const_iterator pos, InputIterator first, InputIterator last) ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(pos >= begin()); - ABSL_HARDENING_ASSERT(pos <= end()); + absl::base_internal::HardeningAssertGE(pos, cbegin()); + absl::base_internal::HardeningAssertLE(pos, cend()); size_type index = static_cast<size_type>(std::distance(cbegin(), pos)); for (size_type i = index; first != last; ++i, static_cast<void>(++first)) { @@ -715,8 +716,8 @@ template <typename... Args> iterator emplace(const_iterator pos, Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(pos >= begin()); - ABSL_HARDENING_ASSERT(pos <= end()); + absl::base_internal::HardeningAssertGE(pos, cbegin()); + absl::base_internal::HardeningAssertLE(pos, cend()); value_type dealias(std::forward<Args>(args)...); // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102329#c2 @@ -761,7 +762,7 @@ // // Destroys the element at `back()`, reducing the size by `1`. void pop_back() noexcept { - ABSL_HARDENING_ASSERT(!empty()); + absl::base_internal::HardeningAssertNonEmpty(*this); AllocatorTraits<A>::destroy(storage_.GetAllocator(), data() + (size() - 1)); storage_.SubtractSize(1); @@ -774,8 +775,8 @@ // // NOTE: may return `end()`, which is not dereferenceable. iterator erase(const_iterator pos) ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(pos >= begin()); - ABSL_HARDENING_ASSERT(pos < end()); + absl::base_internal::HardeningAssertGE(pos, cbegin()); + absl::base_internal::HardeningAssertLT(pos, cend()); // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102329#c2 // It appears that GCC thinks that since `pos` is a const pointer and may @@ -800,9 +801,9 @@ // NOTE: may return `end()`, which is not dereferenceable. iterator erase(const_iterator from, const_iterator to) ABSL_ATTRIBUTE_LIFETIME_BOUND { - ABSL_HARDENING_ASSERT(from >= begin()); - ABSL_HARDENING_ASSERT(from <= to); - ABSL_HARDENING_ASSERT(to <= end()); + absl::base_internal::HardeningAssertGE(from, cbegin()); + absl::base_internal::HardeningAssertLE(from, to); + absl::base_internal::HardeningAssertLE(to, cend()); if (ABSL_PREDICT_TRUE(from != to)) { return storage_.Erase(from, to); @@ -875,7 +876,7 @@ // // REQUIRES: other.storage_.GetIsAllocated() void DestroyExistingAndAdopt(InlinedVector&& other) { - ABSL_HARDENING_ASSERT(other.storage_.GetIsAllocated()); + absl::base_internal::HardeningAssert(other.storage_.GetIsAllocated()); inlined_vector_internal::DestroyAdapter<A>::DestroyElements( storage_.GetAllocator(), data(), size());
diff --git a/absl/container/internal/inlined_vector.h b/absl/container/internal/inlined_vector.h index 42878ac..6bda9d5 100644 --- a/absl/container/internal/inlined_vector.h +++ b/absl/container/internal/inlined_vector.h
@@ -27,6 +27,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/internal/hardening.h" #include "absl/base/macros.h" #include "absl/container/internal/compressed_tuple.h" #include "absl/memory/memory.h" @@ -476,7 +477,7 @@ } void SubtractSize(SizeType<A> count) { - ABSL_HARDENING_ASSERT(count <= GetSize()); + absl::base_internal::HardeningAssertLE(count, GetSize()); GetSizeAndIsAllocated() -= count << static_cast<SizeType<A>>(1); } @@ -500,18 +501,17 @@ // { using V = ValueType<A>; - ABSL_HARDENING_ASSERT( - other_storage.GetIsAllocated() || - (std::is_same<A, std::allocator<V>>::value && - ( - // First case above - absl::is_trivially_relocatable<V>::value || - // Second case above - (std::is_trivially_move_assignable<V>::value && - std::is_trivially_destructible<V>::value) || - // Third case above - (std::is_trivially_copy_constructible<V>::value || - std::is_trivially_copy_assignable<V>::value)))); + ABSL_ASSERT(other_storage.GetIsAllocated() || + (std::is_same<A, std::allocator<V>>::value && + ( + // First case above + absl::is_trivially_relocatable<V>::value || + // Second case above + (std::is_trivially_move_assignable<V>::value && + std::is_trivially_destructible<V>::value) || + // Third case above + (std::is_trivially_copy_constructible<V>::value || + std::is_trivially_copy_assignable<V>::value)))); } GetSizeAndIsAllocated() = other_storage.GetSizeAndIsAllocated(); @@ -575,7 +575,7 @@ template <typename T, size_t N, typename A> void Storage<T, N, A>::InitFrom(const Storage& other) { const SizeType<A> n = other.GetSize(); - ABSL_HARDENING_ASSERT(n > 0); // Empty sources handled handled in caller. + ABSL_ASSERT(n > 0); // Empty sources handled in caller. ConstPointer<A> src; Pointer<A> dst; if (!other.GetIsAllocated()) { @@ -613,8 +613,8 @@ auto Storage<T, N, A>::Initialize(ValueAdapter values, SizeType<A> new_size) -> void { // Only callable from constructors! - ABSL_HARDENING_ASSERT(!GetIsAllocated()); - ABSL_HARDENING_ASSERT(GetSize() == 0); + ABSL_ASSERT(!GetIsAllocated()); + ABSL_ASSERT(GetSize() == 0); Pointer<A> construct_data; if (new_size > GetInlinedCapacity()) { @@ -930,7 +930,7 @@ template <typename T, size_t N, typename A> auto Storage<T, N, A>::ShrinkToFit() -> void { // May only be called on allocated instances! - ABSL_HARDENING_ASSERT(GetIsAllocated()); + ABSL_ASSERT(GetIsAllocated()); StorageView<A> storage_view{GetAllocatedData(), GetSize(), GetAllocatedCapacity()}; @@ -979,7 +979,7 @@ template <typename T, size_t N, typename A> auto Storage<T, N, A>::Swap(Storage* other_storage_ptr) -> void { using std::swap; - ABSL_HARDENING_ASSERT(this != other_storage_ptr); + ABSL_ASSERT(this != other_storage_ptr); if (GetIsAllocated() && other_storage_ptr->GetIsAllocated()) { swap(data_.allocated, other_storage_ptr->data_.allocated);
diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index d7379ef..fa6b982 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc
@@ -135,8 +135,8 @@ // Must be defined out-of-line to avoid MSVC error C2482 on some platforms, // which is caused by non-constexpr initialization. -uint16_t HashtableInlineData::NextSeed() { - static_assert(PerTableSeed::kBitCount == 16); +uint16_t NextHashTableSeed() { + static_assert(PerTableSeed::kBitCount <= 16); thread_local uint16_t seed = static_cast<uint16_t>(reinterpret_cast<uintptr_t>(&seed)); seed += uint16_t{0xad53};
diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 71f16df..7cf92a4 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h
@@ -493,6 +493,14 @@ return (~size_t{}) >> leading_zeros; } +// The mode we store capacity in the table. +enum HashtableCapacityStorageMode { + // Capacity stored as size_t as a full number. + kCapacityByValue, + // Capacity stored as uint8_t as log2, i.e. capacity = 2^capacity_ - 1. + kCapacityByLog, +}; + // The number of slots in the backing array. This is always 2^N-1 for an // integer N. // NOTE: this class exists to simplify experiments with different ways to store @@ -502,44 +510,67 @@ // 2^N-1), and (b) storing 2^N as the most significant bit of size_ and storing // size in the low bits. Both of these experiments were regressions, presumably // because we need capacity to do find operations. -class HashtableCapacity { +template <HashtableCapacityStorageMode StorageMode> +class HashtableCapacityImpl { + using IntType = + std::conditional_t<StorageMode == kCapacityByValue, size_t, uint8_t>; + public: - static constexpr HashtableCapacity CreateDestroyed() { - return HashtableCapacity(kDestroyed); + static constexpr HashtableCapacityImpl CreateDestroyed() { + return HashtableCapacityImpl(kDestroyed); } - static constexpr HashtableCapacity CreateReentrance() { - return HashtableCapacity(kReentrance); + static constexpr HashtableCapacityImpl CreateReentrance() { + return HashtableCapacityImpl(kReentrance); } - static constexpr HashtableCapacity CreateMovedFrom() { - return HashtableCapacity(kMovedFrom); + static constexpr HashtableCapacityImpl CreateMovedFrom() { + return HashtableCapacityImpl(kMovedFrom); } - static constexpr HashtableCapacity CreateSelfMovedFrom() { - return HashtableCapacity(kSelfMovedFrom); + static constexpr HashtableCapacityImpl CreateSelfMovedFrom() { + return HashtableCapacityImpl(kSelfMovedFrom); } - explicit HashtableCapacity(uninitialized_tag_t) {} - explicit constexpr HashtableCapacity(size_t capacity) : capacity_(capacity) { + explicit HashtableCapacityImpl(uninitialized_tag_t) {} + explicit constexpr HashtableCapacityImpl(size_t capacity) + : capacity_data_(static_cast<IntType>( + StorageMode == kCapacityByValue ? capacity + : TrailingZeros(capacity + 1))) { ABSL_SWISSTABLE_ASSERT(capacity == 0 || IsValidCapacity(capacity)); } - constexpr bool IsValid() const { return capacity_ <= kAboveMaxValidCapacity; } + // Creates capacity from the value that was returned by `ToRawData()`. + // This is needed to use bitfield for capacity. + // At least on Windows combination uint8_t and uint64_t bitfield in one struct + // is not optimized by compiler. + static HashtableCapacityImpl FromRawData(uint64_t capacity) { + auto cap = HashtableCapacityImpl(uninitialized_tag_t{}); + cap.capacity_data_ = static_cast<IntType>(capacity); + return cap; + } + IntType ToRawData() const { return capacity_data_; } - constexpr bool IsDestroyed() const { return capacity_ == kDestroyed; } - constexpr bool IsReentrance() const { return capacity_ == kReentrance; } + constexpr bool IsValid() const { + return capacity_data_ <= kAboveMaxValidCapacity; + } + + constexpr bool IsDestroyed() const { return capacity_data_ == kDestroyed; } + constexpr bool IsReentrance() const { return capacity_data_ == kReentrance; } // Returns true if the table is moved-from including self moved-from. - constexpr bool IsMovedFrom() const { return capacity_ >= kMovedFrom; } - constexpr bool IsSelfMovedFrom() const { return capacity_ == kSelfMovedFrom; } + constexpr bool IsMovedFrom() const { return capacity_data_ >= kMovedFrom; } + constexpr bool IsSelfMovedFrom() const { + return capacity_data_ == kSelfMovedFrom; + } constexpr size_t capacity() const { ABSL_SWISSTABLE_ASSERT(IsValid()); - return capacity_; + return StorageMode == kCapacityByValue ? capacity_data_ + : (size_t{1} << capacity_data_) - 1; } private: // We use these sentinel capacity values in debug mode to indicate different // classes of bugs. - enum InvalidCapacity : size_t { - kAboveMaxValidCapacity = ~size_t{} - 100, + enum InvalidCapacity : IntType { + kAboveMaxValidCapacity = (std::numeric_limits<IntType>::max)() - 100, kReentrance, kDestroyed, @@ -548,60 +579,85 @@ kSelfMovedFrom, }; - explicit constexpr HashtableCapacity(InvalidCapacity capacity) - : capacity_(capacity) { - ABSL_SWISSTABLE_ASSERT(capacity_ > kAboveMaxValidCapacity); + explicit constexpr HashtableCapacityImpl(InvalidCapacity capacity) + : capacity_data_(capacity) { + ABSL_SWISSTABLE_ASSERT(capacity_data_ > kAboveMaxValidCapacity); } - size_t capacity_; + // Capacity is stored as a value or as a log2 depending on `StorageMode`. + IntType capacity_data_; }; +template <HashtableCapacityStorageMode StorageMode> +class HashtableInlineDataImpl; + +// Returns next per-table seed. +uint16_t NextHashTableSeed(); + // Per table hash salt. This gets mixed into H1 to randomize iteration order // per-table. // The seed is needed to ensure non-determinism of iteration order. -class PerTableSeed { +template <typename StorageType> +class PerTableSeedImpl { public: + using IntType = StorageType; + // The number of bits in the seed. // It is big enough to ensure non-determinism of iteration order. // We store the seed inside a uint64_t together with size and other metadata. - // Using 16 bits allows us to save one `and` instruction in H1 (we use + // Using 8 or 16 bits allows us to save one `and` instruction in H1 (we use // zero-extended move instead of mov+and). When absl::Hash is inlined, it can // also have lower latency knowing that the high bits of the seed are zero. - static constexpr size_t kBitCount = 16; + static constexpr size_t kBitCount = sizeof(IntType) * 8; + + // We need to use a constant seed when the table is sampled so that sampled + // hashes use the same seed and can e.g. identify stuck bits accurately. + static constexpr IntType kSampledSeed = static_cast<IntType>(~IntType{0}); // Returns the seed for the table. size_t seed() const { return seed_; } private: - friend class HashtableInlineData; - explicit PerTableSeed(uint16_t seed) : seed_(seed) {} + template <HashtableCapacityStorageMode StorageMode> + friend class HashtableInlineDataImpl; - // The most significant bit of the seed is always 1 when there is a non-zero - // seed. This way, when sign-extended the seed has non-zero high bits. - const uint16_t seed_; + explicit PerTableSeedImpl(uint64_t seed) + : seed_(static_cast<IntType>(seed)) {} + + const IntType seed_; }; // Capacity, size and also has additionally // 1) one bit that stores whether we have infoz. // 2) PerTableSeed::kBitCount bits for the seed. (For SOO tables, the lowest // bit of the seed is repurposed to track if sampling has been tried). -class HashtableInlineData { +template <HashtableCapacityStorageMode StorageMode> +class HashtableInlineDataImpl { public: - static constexpr size_t kSizeBitCount = 64 - PerTableSeed::kBitCount - 1; + using PerTableSeed = PerTableSeedImpl< + std::conditional_t<StorageMode == kCapacityByValue, uint16_t, uint8_t>>; + using HashtableCapacity = HashtableCapacityImpl<StorageMode>; + static constexpr size_t kSizeBitCount = + StorageMode == kCapacityByValue + ? 64 - PerTableSeed::kBitCount - 1 + : 64 - PerTableSeed::kBitCount - sizeof(HashtableCapacity) * 8 - 1; - explicit HashtableInlineData(uninitialized_tag_t) - : capacity_(uninitialized_tag_t{}) {} - explicit HashtableInlineData(HashtableCapacity capacity, no_seed_empty_tag_t) - : capacity_(capacity), data_(0) {} - HashtableInlineData(HashtableCapacity capacity, full_soo_tag_t, - bool has_tried_sampling) - : capacity_(capacity), + explicit HashtableInlineDataImpl(uninitialized_tag_t) {} + explicit HashtableInlineDataImpl(HashtableCapacity capacity, + no_seed_empty_tag_t) + : capacity_internal_(capacity.ToRawData()), data_(0) {} + HashtableInlineDataImpl(HashtableCapacity capacity, full_soo_tag_t, + bool has_tried_sampling) + : capacity_internal_(capacity.ToRawData()), data_(kSizeOneNoMetadata | (has_tried_sampling ? kSooHasTriedSamplingMask : 0)) {} - size_t capacity() const { return capacity_.capacity(); } - HashtableCapacity maybe_invalid_capacity() const { return capacity_; } - void set_capacity(HashtableCapacity c) { capacity_ = c; } + HashtableCapacity maybe_invalid_capacity() const { + return HashtableCapacity::FromRawData(capacity_internal_); + } + size_t capacity() const { return maybe_invalid_capacity().capacity(); } + + void set_capacity(HashtableCapacity c) { capacity_internal_ = c.ToRawData(); } void set_capacity(size_t c) { set_capacity(HashtableCapacity(c)); } // Returns actual size of the table. @@ -623,23 +679,23 @@ void set_soo_has_tried_sampling() { data_ |= kSooHasTriedSamplingMask; } // Sets the size, but keeps all the metadata bits. - void set_size_keep_metadata(size_t size) { + void set_size(size_t size) { data_ = (data_ & kMetadataMask) | (static_cast<uint64_t>(size) << kSizeShift); } - PerTableSeed seed() const { - return PerTableSeed(static_cast<size_t>(data_) & kSeedMask); - } + PerTableSeed seed() const { return PerTableSeed(data_ & kSeedMask); } - void generate_new_seed() { set_seed(NextSeed()); } + void generate_new_seed() { + set_seed(static_cast<typename PerTableSeed::IntType>(NextHashTableSeed())); + } // We need to use a constant seed when the table is sampled so that sampled // hashes use the same seed and can e.g. identify stuck bits accurately. - void set_sampled_seed() { set_seed((std::numeric_limits<uint16_t>::max)()); } + void set_sampled_seed() { set_seed(PerTableSeed::kSampledSeed); } bool is_sampled_seed() const { - return (data_ & kSeedMask) == (std::numeric_limits<uint16_t>::max)(); + return seed().seed() == PerTableSeed::kSampledSeed; } // Returns true if the table has infoz. @@ -652,16 +708,16 @@ void set_no_seed_for_testing() { data_ &= ~kSeedMask; } - // Returns next per-table seed. - static uint16_t NextSeed(); - private: - void set_seed(uint16_t seed) { data_ = (data_ & ~kSeedMask) | seed; } - // Bit layout of `data_`: - // [63 ... 17] (47 bits) : size - // [16] (1 bit) : has_infoz - // [15 ... 0] (16 bits) : seed - static constexpr size_t kSizeShift = 64 - kSizeBitCount; + // Bit layout of `data_` from MSB to LSB: + // (47 bits) : size + // (1 bit) : has_infoz + // (16 or 8 bits) : seed + // We don't split these components of `data_` into separate bit field elements + // because we get worse generated code that way. + static constexpr size_t kDataBitCount = + PerTableSeed::kBitCount + 1 + kSizeBitCount; + static constexpr size_t kSizeShift = kDataBitCount - kSizeBitCount; static constexpr uint64_t kSizeOneNoMetadata = uint64_t{1} << kSizeShift; static constexpr uint64_t kMetadataMask = kSizeOneNoMetadata - 1; static constexpr uint64_t kSeedMask = @@ -671,11 +727,30 @@ // For SOO tables, the seed is unused, and bit 0 is repurposed to track // whether the table has already queried should_sample_soo(). static constexpr uint64_t kSooHasTriedSamplingMask = 1; - HashtableCapacity capacity_; - // Stores the size and metadata bits. See above for layout. - uint64_t data_; + + void set_seed(typename PerTableSeed::IntType seed) { + data_ = (data_ & ~kSeedMask) | seed; + } + + uint64_t capacity_internal_ : sizeof(HashtableCapacity) * 8; + uint64_t data_ : kDataBitCount; }; +static_assert( + sizeof(HashtableInlineDataImpl<kCapacityByValue>::HashtableCapacity) == + sizeof(size_t)); +// NOTE: some platforms have this size to be equal to 12 for two reasons: +// 1) alignof(uint64_t) == 4. +// 2) sizeof(size_t) == sizeof(HashtableCapacityImpl<kCapacityByValue>) == 4. +static_assert(sizeof(HashtableInlineDataImpl<kCapacityByValue>) <= 16); +static_assert( + sizeof(HashtableInlineDataImpl<kCapacityByLog>::HashtableCapacity) == 1); +static_assert(sizeof(HashtableInlineDataImpl<kCapacityByLog>) == 8); + +using HashtableInlineData = HashtableInlineDataImpl<kCapacityByValue>; +using PerTableSeed = HashtableInlineData::PerTableSeed; +using HashtableCapacity = HashtableInlineData::HashtableCapacity; + // H1 is just the low bits of the hash. inline size_t H1(size_t hash) { return hash; } @@ -1111,14 +1186,14 @@ // The number of filled slots. size_t size() const { return inline_data_.size(); } // Sets the size to zero, but keeps hashinfoz bit and seed. - void set_size_to_zero() { inline_data_.set_size_keep_metadata(0); } + void set_size_to_zero() { inline_data_.set_size(0); } void set_empty_soo() { AssertInSooMode(); - inline_data_.set_size_keep_metadata(0); + inline_data_.set_size(0); } void set_full_soo() { AssertInSooMode(); - inline_data_.set_size_keep_metadata(1); + inline_data_.set_size(1); } void increment_size() { ABSL_SWISSTABLE_ASSERT(size() < capacity());
diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 59717ef..8c7199e 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc
@@ -352,6 +352,186 @@ EXPECT_THAT(offsets, ElementsAre(0, 16, 48, 96, 32, 112, 80, 64)); } +template <typename T> +class HashtableDataTest : public ::testing::Test {}; + +using StorageModes = ::testing::Types< + std::integral_constant<HashtableCapacityStorageMode, + HashtableCapacityStorageMode::kCapacityByValue>, + std::integral_constant<HashtableCapacityStorageMode, + HashtableCapacityStorageMode::kCapacityByLog>>; + +TYPED_TEST_SUITE(HashtableDataTest, StorageModes); + +TYPED_TEST(HashtableDataTest, HashtableCapacity) { + constexpr HashtableCapacityStorageMode kMode = TypeParam::value; + using Capacity = HashtableCapacityImpl<kMode>; + + Capacity cap0(0); + EXPECT_TRUE(cap0.IsValid()); + EXPECT_EQ(cap0.capacity(), 0); + EXPECT_FALSE(cap0.IsDestroyed()); + EXPECT_FALSE(cap0.IsReentrance()); + EXPECT_FALSE(cap0.IsMovedFrom()); + EXPECT_FALSE(cap0.IsSelfMovedFrom()); + + for (size_t i = 0, cap = 0; i < 20; ++i, cap = NextCapacity(cap)) { + Capacity capacity(cap); + ASSERT_TRUE(capacity.IsValid()); + ASSERT_EQ(capacity.capacity(), cap); + } + + auto destroyed = Capacity::CreateDestroyed(); + EXPECT_FALSE(destroyed.IsValid()); + EXPECT_TRUE(destroyed.IsDestroyed()); + + auto reentrance = Capacity::CreateReentrance(); + EXPECT_FALSE(reentrance.IsValid()); + EXPECT_TRUE(reentrance.IsReentrance()); + + auto moved_from = Capacity::CreateMovedFrom(); + EXPECT_FALSE(moved_from.IsValid()); + EXPECT_TRUE(moved_from.IsMovedFrom()); + EXPECT_FALSE(moved_from.IsSelfMovedFrom()); + + auto self_moved_from = Capacity::CreateSelfMovedFrom(); + EXPECT_FALSE(self_moved_from.IsValid()); + EXPECT_TRUE(self_moved_from.IsSelfMovedFrom()); + EXPECT_TRUE(self_moved_from.IsMovedFrom()); +} + +TYPED_TEST(HashtableDataTest, RawData) { + constexpr HashtableCapacityStorageMode kMode = TypeParam::value; + using Capacity = HashtableCapacityImpl<kMode>; + + for (size_t i = 0, cap = 0; i < 20; ++i, cap = NextCapacity(cap)) { + Capacity orig_capacity(cap); + Capacity capacity = Capacity::FromRawData(orig_capacity.ToRawData()); + ASSERT_TRUE(capacity.IsValid()); + ASSERT_EQ(capacity.capacity(), cap); + } + auto orig_reentrance = Capacity::CreateReentrance(); + Capacity reentrance = Capacity::FromRawData(orig_reentrance.ToRawData()); + EXPECT_TRUE(reentrance.IsReentrance()); +} + +TYPED_TEST(HashtableDataTest, HashtableInlineDataCapacity) { + constexpr HashtableCapacityStorageMode kMode = TypeParam::value; + using InlineData = HashtableInlineDataImpl<kMode>; + using Capacity = HashtableCapacityImpl<kMode>; + + InlineData data(Capacity(0), no_seed_empty_tag_t{}); + EXPECT_EQ(data.capacity(), 0); + EXPECT_EQ(data.size(), 0); + EXPECT_TRUE(data.empty()); + + for (size_t i = 0, cap = 0; i < 20; ++i, cap = NextCapacity(cap)) { + data.set_capacity(cap); + ASSERT_EQ(data.capacity(), cap); + } + + // Test overload from `Capacity` object. + for (size_t i = 0, cap = 0; i < 20; ++i, cap = NextCapacity(cap)) { + data.set_capacity(Capacity(cap)); + ASSERT_EQ(data.capacity(), cap); + } + + auto reentrance = Capacity::CreateReentrance(); + data.set_capacity(reentrance); + EXPECT_TRUE(data.maybe_invalid_capacity().IsReentrance()); +} + +TYPED_TEST(HashtableDataTest, HashtableInlineDataSize) { + constexpr HashtableCapacityStorageMode kMode = TypeParam::value; + using InlineData = HashtableInlineDataImpl<kMode>; + using Capacity = HashtableCapacityImpl<kMode>; + + InlineData data(Capacity(0), no_seed_empty_tag_t{}); + EXPECT_EQ(data.size(), 0); + data.increment_size(); + EXPECT_EQ(data.size(), 1); + EXPECT_FALSE(data.empty()); + + data.increment_size(5); + EXPECT_EQ(data.size(), 6); + + data.decrement_size(); + EXPECT_EQ(data.size(), 5); + + constexpr size_t kHugeIncrement = + (size_t(1) << (sizeof(size_t) == 4 ? 31 : 42)); + data.increment_size(kHugeIncrement); + EXPECT_EQ(data.size(), kHugeIncrement + 5); + + EXPECT_FALSE(data.has_infoz()); + data.set_has_infoz(); + data.set_size(10); + EXPECT_EQ(data.size(), 10); + EXPECT_TRUE(data.has_infoz()); +} + +TYPED_TEST(HashtableDataTest, HashtableInlineDataMetadata) { + constexpr HashtableCapacityStorageMode kMode = TypeParam::value; + using InlineData = HashtableInlineDataImpl<kMode>; + using Capacity = HashtableCapacityImpl<kMode>; + + InlineData data(Capacity(0), no_seed_empty_tag_t{}); + + // SOO sampling (test this first before seed messes with bit 0) + EXPECT_FALSE(data.soo_has_tried_sampling()); + data.set_soo_has_tried_sampling(); + EXPECT_TRUE(data.soo_has_tried_sampling()); + + data.set_no_seed_for_testing(); + EXPECT_EQ(data.seed().seed(), 0); + + data.set_sampled_seed(); + EXPECT_TRUE(data.is_sampled_seed()); + + // Infoz + EXPECT_FALSE(data.has_infoz()); + data.set_has_infoz(); + EXPECT_TRUE(data.has_infoz()); +} + +TYPED_TEST(HashtableDataTest, HashtableInlineDataFullSooConstructor) { + constexpr HashtableCapacityStorageMode kMode = TypeParam::value; + using InlineData = HashtableInlineDataImpl<kMode>; + using Capacity = HashtableCapacityImpl<kMode>; + + { + InlineData data_soo(Capacity(1), full_soo_tag_t{}, + /*has_tried_sampling=*/true); + EXPECT_EQ(data_soo.capacity(), 1); + EXPECT_EQ(data_soo.size(), 1); + EXPECT_TRUE(data_soo.soo_has_tried_sampling()); + } + + { + InlineData data_soo(Capacity(1), full_soo_tag_t{}, + /*has_tried_sampling=*/false); + EXPECT_EQ(data_soo.capacity(), 1); + EXPECT_EQ(data_soo.size(), 1); + EXPECT_FALSE(data_soo.soo_has_tried_sampling()); + } +} + +TYPED_TEST(HashtableDataTest, GenerateNewSeedDoesntChangeSize) { + constexpr HashtableCapacityStorageMode kMode = TypeParam::value; + using InlineData = HashtableInlineDataImpl<kMode>; + using Capacity = HashtableCapacityImpl<kMode>; + size_t size = 1; + do { + InlineData inline_data(Capacity(15), no_seed_empty_tag_t{}); + inline_data.increment_size(size); + EXPECT_EQ(inline_data.size(), size); + inline_data.generate_new_seed(); + EXPECT_EQ(inline_data.size(), size); + size = size * 2 + 1; + } while (size < std::min(MaxStorableSize(), + MaxSizeAtMaxValidCapacity(/*slot_size=*/1))); +} + TEST(Batch, DropDeletes) { constexpr size_t kCapacity = 63; constexpr size_t kGroupWidth = container_internal::Group::kWidth; @@ -2602,7 +2782,7 @@ // in seed. void GenerateIrrelevantSeeds(int cnt) { for (int i = cnt % 17; i > 0; --i) { - HashtableInlineData::NextSeed(); + NextHashTableSeed(); } } @@ -2640,8 +2820,8 @@ for (bool do_reserve : {false, true}) { for (size_t size : {2u, 3u, 6u, 7u, 12u, 15u, 20u, 50u}) { for (size_t rehash_size : { - size_t{0}, // Force rehash is guaranteed. - size * 10 // Rehash to the larger capacity is guaranteed. + size_t{0}, // Force rehash is guaranteed. + size * 10 // Rehash to the larger capacity is guaranteed. }) { SCOPED_TRACE(absl::StrCat("size: ", size, " rehash_size: ", rehash_size, " do_reserve: ", do_reserve)); @@ -2803,8 +2983,7 @@ using RawHashSamplerTestTypes = ::testing::Types< // 32 bits to make sure that table is Soo for 32 bits platform as well. // 64 bits table is not SOO due to alignment. - SooInt32Table, - NonSooIntTable>; + SooInt32Table, NonSooIntTable>; TYPED_TEST_SUITE(RawHashSamplerTest, RawHashSamplerTestTypes); TYPED_TEST(RawHashSamplerTest, Sample) { @@ -3080,9 +3259,7 @@ t_orig.insert(1); std::vector<const HashtablezInfo*> infos = - SampleSooMutation([&t_orig](SooInt32Table& t) { - t = t_orig; - }); + SampleSooMutation([&t_orig](SooInt32Table& t) { t = t_orig; }); for (const HashtablezInfo* info : infos) { ASSERT_EQ(info->inline_element_size, @@ -4125,19 +4302,6 @@ } } -TEST(HashtableSize, GenerateNewSeedDoesntChangeSize) { - size_t size = 1; - do { - HashtableInlineData hs(HashtableCapacity(15), no_seed_empty_tag_t{}); - hs.increment_size(size); - EXPECT_EQ(hs.size(), size); - hs.generate_new_seed(); - EXPECT_EQ(hs.size(), size); - size = size * 2 + 1; - } while (size < std::min(MaxStorableSize(), - MaxSizeAtMaxValidCapacity(/*slot_size=*/1))); -} - TEST(Table, MaxValidSize) { IntTable t; EXPECT_EQ(
diff --git a/absl/crc/internal/crc32_x86_arm_combined_simd.h b/absl/crc/internal/crc32_x86_arm_combined_simd.h index 5a9b61a..07f5a69 100644 --- a/absl/crc/internal/crc32_x86_arm_combined_simd.h +++ b/absl/crc/internal/crc32_x86_arm_combined_simd.h
@@ -15,6 +15,7 @@ #ifndef ABSL_CRC_INTERNAL_CRC32_X86_ARM_COMBINED_SIMD_H_ #define ABSL_CRC_INTERNAL_CRC32_X86_ARM_COMBINED_SIMD_H_ +#include <array> #include <cstdint> #include "absl/base/config.h" @@ -65,6 +66,13 @@ using V128 = __m128i; #endif +#if defined(__AVX__) +using V256 = __m256i; +#else +// Placeholder for V256 when AVX is not available. +using V256 = std::array<uint64_t, 4>; +#endif + // Starting with the initial value in |crc|, accumulates a CRC32 value for // unsigned integers of different sizes. uint32_t CRC32_u8(uint32_t crc, uint8_t v); @@ -119,6 +127,17 @@ // Add packed 64-bit integers in |l| and |r|. V128 V128_Add64(const V128 l, const V128 r); +#if defined(__AVX__) +inline V256 V256_LoadU(const V256* src); +inline V256 V256_Broadcast128(const V128* src); +#else +template <typename T = V256> +T V256_LoadU(const T* src); + +template <typename T = V256> +T V256_Broadcast128(const V128* src); +#endif + #endif #if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) @@ -271,6 +290,28 @@ #endif +#if defined(__AVX__) && defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) +inline V256 V256_LoadU(const V256* src) { return _mm256_loadu_si256(src); } + +inline V256 V256_Broadcast128(const V128* src) { + return _mm256_castps_si256( + _mm256_broadcast_ps(reinterpret_cast<const __m128*>(src))); +} +#elif defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) || \ + defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) +template <typename T> +inline T V256_LoadU(const T* src) { + (void)src; + return T{}; +} + +template <typename T> +inline T V256_Broadcast128(const V128* src) { + (void)src; + return T{}; +} +#endif + } // namespace crc_internal ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index d639aec..b81490f 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -357,6 +357,72 @@ crc[2] = crc2; } +#if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && defined(__AVX__) + // This is only used if we have vector version of PCLMULQDQ. + // We don't have it on arm, and it isn't supported by default + // compiler targets on x86. If we want to use it, we need to either use + // new compiler flags for the whole function and compile it twice + // with new and default flags or use inline asm. + // The code below is the same as FinalizePclmulStream, but with + // PCLMUL and XOR operating on 2 values in a vector at the same time. + ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t + FinalizeVpclmulStream(V256* partialCRC) const { + uint64_t crc = 0; + uint64_t low64, high64; + __asm__( + // reduce 2 256-bit vectors into s single 256 vector + "vbroadcasti128 %[k256], %%ymm0 \n" + "vpclmulqdq $0x00, %%ymm0, %[crc0], %%ymm1 \n" + "vpclmulqdq $0x11, %%ymm0, %[crc0], %%ymm2 \n" + "vpxor %%ymm2, %%ymm1, %%ymm1 \n" + "vpxor %[crc1], %%ymm1, %%ymm1 \n" + // reduce upper and lower parts of 256-bit vector + "vextracti128 $1, %%ymm1, %%xmm2 \n" + "vpclmulqdq $0x00, %[k128], %%xmm1, %%xmm3 \n" + "vpclmulqdq $0x11, %[k128], %%xmm1, %%xmm1 \n" + "vpxor %%xmm1, %%xmm3, %%xmm3 \n" + "vpxor %%xmm2, %%xmm3, %%xmm3 \n" + // Move 2 parts of 128-bit vector into scalar register + // and reduce using sacalr crc instruction + "vmovq %%xmm3, %[low] \n" + "vpextrq $1, %%xmm3, %[high] \n" + "crc32q %[low], %[crc_out] \n" + "crc32q %[high], %[crc_out] \n" + : [crc_out] "+r"(crc), [low] "=&r"(low64), [high] "=&r"(high64) + : [k256] "m"(*(const __m128i*)kFoldAcross256Bits), + [crc0] "x"(partialCRC[0]), [crc1] "x"(partialCRC[1]), + [k128] "m"(*(const __m128i*)kFoldAcross128Bits) + : "ymm0", "ymm1", "ymm2", "ymm3"); + return crc; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesVpclmul( + const uint8_t* p, V256* vpartialCRC, V256 loopMultiplicands) const { + __asm__ volatile( + "vpclmulqdq $0x11, %3, %0, %%ymm0 \n" + "vpclmulqdq $0x11, %3, %1, %%ymm1 \n" + "vpclmulqdq $0x00, %3, %0, %0 \n" + "vpclmulqdq $0x00, %3, %1, %1 \n" + "vpxor %%ymm0, %0, %0 \n" + "vpxor %%ymm1, %1, %1 \n" + "vpxor (%2), %0, %0 \n" + "vpxor 32(%2), %1, %1 \n" + : "+x"(vpartialCRC[0]), "+x"(vpartialCRC[1]) + : "r"(p), "x"(loopMultiplicands) + : "ymm0", "ymm1"); + } +#else + template <typename T = V256> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesVpclmul( + const uint8_t* p, T* vpartialCRC, T loopMultiplicands) const { + static_assert(sizeof(T) == 0, "Vector PCLMUL not supported"); + } + ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t + FinalizeVpclmulStream(V256* partialCRC) const { + return 0; + } +#endif // defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && defined(__AVX__) + // Constants generated by './scripts/gen-crc-consts.py x86_pclmul // crc32_lsb_0x82f63b78' from the Linux kernel. alignas(16) static constexpr uint64_t kFoldAcross512Bits[2] = { @@ -386,7 +452,7 @@ }; template <size_t num_crc_streams, size_t num_pclmul_streams, - CutoffStrategy strategy> + size_t num_vpclmul_streams, CutoffStrategy strategy> class CRC32AcceleratedX86ARMCombinedMultipleStreams : public CRC32AcceleratedX86ARMCombinedMultipleStreamsBase { ABSL_ATTRIBUTE_HOT @@ -396,6 +462,9 @@ "Invalid number of crc streams"); static_assert(num_pclmul_streams >= 0 && num_pclmul_streams <= kMaxStreams, "Invalid number of pclmul streams"); + static_assert( + num_vpclmul_streams >= 0 && num_vpclmul_streams <= kMaxStreams, + "Invalid number of vpclmul streams"); const uint8_t* p = static_cast<const uint8_t*>(bytes); const uint8_t* e = p + length; uint32_t l = *crc; @@ -474,17 +543,23 @@ } size_t bs = static_cast<size_t>(e - p) / - (num_crc_streams + num_pclmul_streams) / 64; + (num_crc_streams + num_pclmul_streams + num_vpclmul_streams) / + 64; + const uint8_t* stream_start = p; const uint8_t* crc_streams[kMaxStreams]; - const uint8_t* pclmul_streams[kMaxStreams]; - // We are guaranteed to have at least one crc stream. - crc_streams[0] = p; - for (size_t i = 1; i < num_crc_streams; i++) { - crc_streams[i] = crc_streams[i - 1] + bs * 64; + for (size_t i = 0; i < num_crc_streams; i++) { + crc_streams[i] = stream_start; + stream_start += bs * 64; } - pclmul_streams[0] = crc_streams[num_crc_streams - 1] + bs * 64; - for (size_t i = 1; i < num_pclmul_streams; i++) { - pclmul_streams[i] = pclmul_streams[i - 1] + bs * 64; + const uint8_t* pclmul_streams[kMaxStreams]; + for (size_t i = 0; i < num_pclmul_streams; i++) { + pclmul_streams[i] = stream_start; + stream_start += bs * 64; + } + const uint8_t* vpclmul_streams[kMaxStreams]; + for (size_t i = 0; i < num_vpclmul_streams; i++) { + vpclmul_streams[i] = stream_start; + stream_start += bs * 64; } // Per stream crc sums. @@ -520,6 +595,18 @@ pclmul_streams[i] += 16 * 4; } + V256 vpartialCRC[kMaxStreams][2]; + V256 loopMultiplicands{}; + loopMultiplicands = + V256_Broadcast128(reinterpret_cast<const V128*>(kFoldAcross512Bits)); + for (size_t i = 0; i < num_vpclmul_streams; i++) { + vpartialCRC[i][0] = V256_LoadU( + reinterpret_cast<const V256*>(vpclmul_streams[i] + 32 * 0)); + vpartialCRC[i][1] = V256_LoadU( + reinterpret_cast<const V256*>(vpclmul_streams[i] + 32 * 1)); + vpclmul_streams[i] += 16 * 4; + } + for (size_t i = 1; i < bs; i++) { // Prefetch data for next iterations. for (size_t j = 0; j < num_crc_streams; j++) { @@ -530,6 +617,10 @@ PrefetchToLocalCache(reinterpret_cast<const char*>(pclmul_streams[j] + kPrefetchHorizon)); } + for (size_t j = 0; j < num_vpclmul_streams; j++) { + PrefetchToLocalCache(reinterpret_cast<const char*>( + vpclmul_streams[j] + kPrefetchHorizon)); + } // We process each stream in 64 byte blocks. This can be written as // for (int i = 0; i < num_pclmul_streams; i++) { @@ -568,6 +659,12 @@ Process64BytesPclmul(pclmul_streams[2], partialCRC[2]); pclmul_streams[2] += 16 * 4; } + + if constexpr (num_vpclmul_streams > 0) { + Process64BytesVpclmul(vpclmul_streams[0], vpartialCRC[0], + loopMultiplicands); + vpclmul_streams[0] += 16 * 4; + } } // PCLMULQDQ based streams require special final step; @@ -576,6 +673,13 @@ l64_pclmul[i] = FinalizePclmulStream(partialCRC[i]); } + uint64_t l64_vpclmul[kMaxStreams] = {0}; + if constexpr (num_vpclmul_streams > 0) { + for (size_t i = 0; i < num_vpclmul_streams; i++) { + l64_vpclmul[i] = FinalizeVpclmulStream(vpartialCRC[i]); + } + } + // Combine all streams into single result. static_assert(64 % (1 << kNumDroppedBits) == 0); uint32_t magic = ComputeZeroConstant(bs * 64); @@ -588,9 +692,15 @@ l64 = MultiplyWithExtraX33(static_cast<uint32_t>(l64), magic); l64 ^= l64_pclmul[i]; } + for (size_t i = 0; i < num_vpclmul_streams; i++) { + l64 = MultiplyWithExtraX33(static_cast<uint32_t>(l64), magic); + l64 ^= l64_vpclmul[i]; + } // Update p. - if (num_pclmul_streams > 0) { + if constexpr (num_vpclmul_streams > 0) { + p = vpclmul_streams[num_vpclmul_streams - 1]; + } else if constexpr (num_pclmul_streams > 0) { p = pclmul_streams[num_pclmul_streams - 1]; } else { p = crc_streams[num_crc_streams - 1]; @@ -618,6 +728,10 @@ ABSL_INTERNAL_STEP1(l, p); } + *crc = l; + } +}; + #undef ABSL_INTERNAL_STEP8BY3 #undef ABSL_INTERNAL_STEP8BY2 #undef ABSL_INTERNAL_STEP8 @@ -625,10 +739,6 @@ #undef ABSL_INTERNAL_STEP2 #undef ABSL_INTERNAL_STEP1 - *crc = l; - } -}; - } // namespace // Intel processors with SSE4.2 have an instruction for one particular @@ -639,11 +749,20 @@ case CpuType::kIntelHaswell: case CpuType::kAmdRome: case CpuType::kAmdNaples: + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 3, 1, 0, CutoffStrategy::Fold3>(); case CpuType::kAmdMilan: case CpuType::kAmdGenoa: case CpuType::kAmdTurin: +#if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && defined(__AVX__) + // We don't have vector pclmul on arm, but this still needs to + // compile. return new CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 1, CutoffStrategy::Fold3>(); + 3, 0, 1, CutoffStrategy::Fold3>(); +#else + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 3, 1, 0, CutoffStrategy::Fold3>(); +#endif // PCLMULQDQ is fast, use combined PCLMULQDQ + CRC implementation. case CpuType::kIntelCascadelakeXeon: case CpuType::kIntelSkylakeXeon: @@ -654,32 +773,32 @@ case CpuType::kIntelEmeraldrapids: case CpuType::kIntelGraniterapidsap: return new CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 2, CutoffStrategy::Fold3>(); + 3, 2, 0, CutoffStrategy::Fold3>(); // PCLMULQDQ is slow, don't use it. case CpuType::kIntelIvybridge: case CpuType::kIntelSandybridge: case CpuType::kIntelWestmere: return new CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 0, CutoffStrategy::Fold3>(); + 3, 0, 0, CutoffStrategy::Fold3>(); case CpuType::kArmNeoverseN1: case CpuType::kArmNeoverseN2: case CpuType::kArmNeoverseV1: case CpuType::kArmNeoverseN3: return new CRC32AcceleratedX86ARMCombinedMultipleStreams< - 1, 1, CutoffStrategy::Unroll64CRC>(); + 1, 1, 0, CutoffStrategy::Unroll64CRC>(); case CpuType::kAmpereSiryn: return new CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 2, CutoffStrategy::Fold3>(); + 3, 2, 0, CutoffStrategy::Fold3>(); case CpuType::kArmNeoverseV2: return new CRC32AcceleratedX86ARMCombinedMultipleStreams< - 1, 2, CutoffStrategy::Unroll64CRC>(); + 1, 2, 0, CutoffStrategy::Unroll64CRC>(); #if defined(__aarch64__) default: // Not all ARM processors support the needed instructions, so check here // before trying to use an accelerated implementation. if (SupportsArmCRC32PMULL()) { return new CRC32AcceleratedX86ARMCombinedMultipleStreams< - 1, 1, CutoffStrategy::Unroll64CRC>(); + 1, 1, 0, CutoffStrategy::Unroll64CRC>(); } else { return nullptr; } @@ -687,71 +806,13 @@ default: // Something else, play it safe and assume slow PCLMULQDQ. return new CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 0, CutoffStrategy::Fold3>(); + 3, 0, 0, CutoffStrategy::Fold3>(); #endif } } -std::vector<std::unique_ptr<CRCImpl>> NewCRC32AcceleratedX86ARMCombinedAll() { - auto ret = std::vector<std::unique_ptr<CRCImpl>>(); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 1, 0, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 1, 1, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 1, 2, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 1, 3, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 2, 0, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 2, 1, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 2, 2, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 2, 3, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 0, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 1, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 2, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 3, CutoffStrategy::Fold3>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 1, 0, CutoffStrategy::Unroll64CRC>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 1, 1, CutoffStrategy::Unroll64CRC>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 1, 2, CutoffStrategy::Unroll64CRC>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 1, 3, CutoffStrategy::Unroll64CRC>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 2, 0, CutoffStrategy::Unroll64CRC>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 2, 1, CutoffStrategy::Unroll64CRC>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 2, 2, CutoffStrategy::Unroll64CRC>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 2, 3, CutoffStrategy::Unroll64CRC>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 0, CutoffStrategy::Unroll64CRC>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 1, CutoffStrategy::Unroll64CRC>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 2, CutoffStrategy::Unroll64CRC>>()); - ret.push_back(std::make_unique<CRC32AcceleratedX86ARMCombinedMultipleStreams< - 3, 3, CutoffStrategy::Unroll64CRC>>()); - - return ret; -} - #else // !ABSL_INTERNAL_CAN_USE_SIMD_CRC32C -std::vector<std::unique_ptr<CRCImpl>> NewCRC32AcceleratedX86ARMCombinedAll() { - return std::vector<std::unique_ptr<CRCImpl>>(); -} - // no hardware acceleration available CRCImpl* TryNewCRC32AcceleratedX86ARMCombined() { return nullptr; }
diff --git a/absl/debugging/internal/stacktrace_aarch64-inl.inc b/absl/debugging/internal/stacktrace_aarch64-inl.inc index bbdce77..03f2294 100644 --- a/absl/debugging/internal/stacktrace_aarch64-inl.inc +++ b/absl/debugging/internal/stacktrace_aarch64-inl.inc
@@ -212,6 +212,17 @@ stack_info.sig_stack_low = stack_info.stack_low; stack_info.sig_stack_high = kUnknownStackEnd; +#if defined(__linux__) + // Similarly for the signal stack, retrieve the information into sig_stack and + // then pass it down per call. Calling sigaltstack with nullptr as the first + // argument doesn't change the signal stack. + stack_t sig_stack; + if (sigaltstack(nullptr, &sig_stack) == 0) { + stack_info.sig_stack_low = reinterpret_cast<uintptr_t>(sig_stack.ss_sp); + stack_info.sig_stack_high = stack_info.sig_stack_low + sig_stack.ss_size; + } +#endif + // The frame pointer points to low address of a frame. The first 64-bit // word of a frame points to the next frame up the call chain, which normally // is just after the high address of the current frame. The second word of
diff --git a/absl/log/absl_vlog_is_on.h b/absl/log/absl_vlog_is_on.h index e4ec86d..1a6792c 100644 --- a/absl/log/absl_vlog_is_on.h +++ b/absl/log/absl_vlog_is_on.h
@@ -63,8 +63,6 @@ #include "absl/log/internal/vlog_config.h" // IWYU pragma: export #include "absl/strings/string_view.h" -// IWYU pragma: private, include "absl/log/log.h" - // This is expanded at the callsite to allow the compiler to optimize // always-false cases out of the build. // An ABSL_MAX_VLOG_VERBOSITY of 2 means that VLOG(3) and above should never
diff --git a/absl/log/internal/proto.cc b/absl/log/internal/proto.cc index 821be2b..8e7bda9 100644 --- a/absl/log/internal/proto.cc +++ b/absl/log/internal/proto.cc
@@ -12,6 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. +// All of the data that passes through this code is trusted because it flows +// through a closed loop within the absl::LogMessage object. It is not a robust +// protocol buffer encoder or decoder. +// +// Encoding: When `LOG(INFO) << "foo"` is called, the library uses the Encode* +// functions to build a protocol buffer in a private, fixed-size internal buffer +// (`LogMessageData::encoded_buf`). +// +// Decoding: During the same logging call, `LogMessage::Flush()` calls +// `FinalizeEncodingAndFormat()`, which uses `ProtoField::DecodeFrom` to parse +// that same internal buffer to generate the human-readable string for +// text-based log sinks. + #include "absl/log/internal/proto.h" #include <algorithm>
diff --git a/absl/log/vlog_is_on.h b/absl/log/vlog_is_on.h index c33fcc1..67c6fb3 100644 --- a/absl/log/vlog_is_on.h +++ b/absl/log/vlog_is_on.h
@@ -60,8 +60,6 @@ #include "absl/log/absl_vlog_is_on.h" // IWYU pragma: export -// IWYU pragma: private, include "absl/log/log.h" - // Each VLOG_IS_ON call site gets its own VLogSite that registers with the // global linked list of sites to asynchronously update its verbosity level on // changes to --v or --vmodule. The verbosity can also be set by manually
diff --git a/absl/memory/memory.h b/absl/memory/memory.h index 9958215..7be4983 100644 --- a/absl/memory/memory.h +++ b/absl/memory/memory.h
@@ -213,7 +213,8 @@ // `std::pointer_traits` for platforms that had not yet provided it. Those // platforms are no longer supported. New code should simply use // `std::pointer_traits`. -using std::pointer_traits; +template <typename Ptr> +using pointer_traits ABSL_DEPRECATE_AND_INLINE() = std::pointer_traits<Ptr>; // ----------------------------------------------------------------------------- // Class Template: allocator_traits @@ -223,7 +224,9 @@ // `std::allocator_traits` for platforms that had not yet provided it. Those // platforms are no longer supported. New code should simply use // `std::allocator_traits`. -using std::allocator_traits; +template <typename Alloc> +using allocator_traits ABSL_DEPRECATE_AND_INLINE() = + std::allocator_traits<Alloc>; namespace memory_internal {
diff --git a/absl/status/internal/status_internal.h b/absl/status/internal/status_internal.h index cafe160..d5aa285 100644 --- a/absl/status/internal/status_internal.h +++ b/absl/status/internal/status_internal.h
@@ -48,6 +48,7 @@ class ABSL_MUST_USE_RESULT ABSL_ATTRIBUTE_TRIVIAL_ABI Status; #endif + ABSL_NAMESPACE_END } // namespace absl #endif // !SWIG
diff --git a/absl/strings/BUILD.bazel b/absl/strings/BUILD.bazel index 79b6672..97187a5 100644 --- a/absl/strings/BUILD.bazel +++ b/absl/strings/BUILD.bazel
@@ -42,6 +42,7 @@ deps = [ "//absl/base:config", "//absl/base:core_headers", + "//absl/base:hardening", "//absl/base:nullability", ], ) @@ -106,6 +107,7 @@ "//absl/base:config", "//absl/base:core_headers", "//absl/base:endian", + "//absl/base:hardening", "//absl/base:iterator_traits_internal", "//absl/base:nullability", "//absl/base:raw_logging_internal", @@ -152,6 +154,7 @@ "//absl/base:config", "//absl/base:core_headers", "//absl/base:dynamic_annotations", + "//absl/base:hardening", "//absl/base:throw_delegate", ], ) @@ -180,6 +183,7 @@ ":resize_and_overwrite", "//absl/base:config", "//absl/base:core_headers", + "//absl/base:hardening", "//absl/base:throw_delegate", ], ) @@ -674,6 +678,7 @@ "//absl/base:config", "//absl/base:core_headers", "//absl/base:endian", + "//absl/base:hardening", "//absl/base:nullability", "//absl/base:raw_logging_internal", "//absl/cleanup", @@ -973,6 +978,7 @@ ":cord_rep_test_util", ":string_view", "//absl/base:config", + "//absl/base:hardening", "//absl/types:span", "@googletest//:gtest", "@googletest//:gtest_main",
diff --git a/absl/strings/CMakeLists.txt b/absl/strings/CMakeLists.txt index 9f7f9ef..3041e19 100644 --- a/absl/strings/CMakeLists.txt +++ b/absl/strings/CMakeLists.txt
@@ -81,6 +81,7 @@ absl::config absl::core_headers absl::endian + absl::hardening absl::int128 absl::iterator_traits_internal absl::memory @@ -179,6 +180,7 @@ absl::config absl::core_headers absl::dynamic_annotations + absl::hardening absl::throw_delegate ) @@ -206,6 +208,7 @@ DEPS absl::config absl::core_headers + absl::hardening absl::strings_resize_and_overwrite absl::throw_delegate ) @@ -1075,6 +1078,7 @@ absl::crc_cord_state absl::endian absl::function_ref + absl::hardening absl::inlined_vector absl::nullability absl::optional
diff --git a/absl/strings/cord.cc b/absl/strings/cord.cc index aa450d3..1d5e255 100644 --- a/absl/strings/cord.cc +++ b/absl/strings/cord.cc
@@ -35,6 +35,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/endian.h" +#include "absl/base/internal/hardening.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/macros.h" #include "absl/base/nullability.h" @@ -1086,8 +1087,8 @@ } Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) { - ABSL_HARDENING_ASSERT(bytes_remaining_ >= n && - "Attempted to iterate past `end()`"); + // Failure of this assertion indicates an attempt to iterate past `end()`. + absl::base_internal::HardeningAssertGE(bytes_remaining_, n); Cord subcord; auto constexpr method = CordzUpdateTracker::kCordReader; @@ -1155,7 +1156,7 @@ } char Cord::operator[](size_t i) const { - ABSL_HARDENING_ASSERT(i < size()); + absl::base_internal::HardeningAssertLT(i, size()); size_t offset = i; const CordRep* rep = contents_.tree(); if (rep == nullptr) {
diff --git a/absl/strings/cord.h b/absl/strings/cord.h index c2f1ec5..4f27fca 100644 --- a/absl/strings/cord.h +++ b/absl/strings/cord.h
@@ -75,6 +75,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/endian.h" +#include "absl/base/internal/hardening.h" #include "absl/base/macros.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" @@ -1553,8 +1554,8 @@ } inline Cord::ChunkIterator& Cord::ChunkIterator::operator++() { - ABSL_HARDENING_ASSERT(bytes_remaining_ > 0 && - "Attempted to iterate past `end()`"); + // Failure of this assertion indicates an attempt to iterate past `end()`. + absl::base_internal::HardeningAssertGT(bytes_remaining_, size_t{0}); assert(bytes_remaining_ >= current_chunk_.size()); bytes_remaining_ -= current_chunk_.size(); if (bytes_remaining_ > 0) { @@ -1583,12 +1584,12 @@ } inline Cord::ChunkIterator::reference Cord::ChunkIterator::operator*() const { - ABSL_HARDENING_ASSERT(bytes_remaining_ != 0); + absl::base_internal::HardeningAssertGT(bytes_remaining_, size_t{0}); return current_chunk_; } inline Cord::ChunkIterator::pointer Cord::ChunkIterator::operator->() const { - ABSL_HARDENING_ASSERT(bytes_remaining_ != 0); + absl::base_internal::HardeningAssertGT(bytes_remaining_, size_t{0}); return ¤t_chunk_; }
diff --git a/absl/strings/cord_buffer.h b/absl/strings/cord_buffer.h index bc0e4e4..fc806bb 100644 --- a/absl/strings/cord_buffer.h +++ b/absl/strings/cord_buffer.h
@@ -32,6 +32,7 @@ #include <utility> #include "absl/base/config.h" +#include "absl/base/internal/hardening.h" #include "absl/base/macros.h" #include "absl/numeric/bits.h" #include "absl/strings/internal/cord_internal.h" @@ -549,7 +550,7 @@ } inline void CordBuffer::SetLength(size_t length) { - ABSL_HARDENING_ASSERT(length <= capacity()); + absl::base_internal::HardeningAssertLE(length, capacity()); if (rep_.is_short()) { rep_.set_short_length(length); } else { @@ -558,7 +559,8 @@ } inline void CordBuffer::IncreaseLengthBy(size_t n) { - ABSL_HARDENING_ASSERT(n <= capacity() && length() + n <= capacity()); + absl::base_internal::HardeningAssertLE(n, capacity()); + absl::base_internal::HardeningAssertLE(length() + n, capacity()); if (rep_.is_short()) { rep_.add_short_length(n); } else {
diff --git a/absl/strings/internal/append_and_overwrite.h b/absl/strings/internal/append_and_overwrite.h index 7d4ce59..833d70d 100644 --- a/absl/strings/internal/append_and_overwrite.h +++ b/absl/strings/internal/append_and_overwrite.h
@@ -16,6 +16,7 @@ #define ABSL_STRINGS_INTERNAL_APPEND_AND_OVERWRITE_H_ #include "absl/base/config.h" +#include "absl/base/internal/hardening.h" #include "absl/base/macros.h" #include "absl/base/optimization.h" #include "absl/base/throw_delegate.h" @@ -69,9 +70,11 @@ str, resize, [old_size, append_n, do_append = std::move(append_op)]( typename T::value_type* data_ptr, typename T::size_type) mutable { - auto num_appended = - std::move(do_append)(data_ptr + old_size, append_n); - ABSL_HARDENING_ASSERT(num_appended >= 0 && num_appended <= append_n); + typename T::size_type num_appended = static_cast<typename T::size_type>( + std::move(do_append)(data_ptr + old_size, append_n)); + absl::base_internal::HardeningAssertGE(num_appended, + typename T::size_type{0}); + absl::base_internal::HardeningAssertLE(num_appended, append_n); return old_size + num_appended; });
diff --git a/absl/strings/internal/generic_printer_internal.h b/absl/strings/internal/generic_printer_internal.h index fc88b27..32a9373 100644 --- a/absl/strings/internal/generic_printer_internal.h +++ b/absl/strings/internal/generic_printer_internal.h
@@ -280,6 +280,10 @@ // Specialization for nullptr. return os << "nullptr"; + } else if constexpr (std::is_same_v<T, std::monostate>) { + // Specialization for `std::monostate`. + return os << "monostate"; + } else if constexpr (std::is_floating_point_v<T>) { // For floating point print with enough precision for a roundtrip. return PrintPreciseFP(os, v);
diff --git a/absl/strings/internal/generic_printer_test.cc b/absl/strings/internal/generic_printer_test.cc index 4093228..85c7a75 100644 --- a/absl/strings/internal/generic_printer_test.cc +++ b/absl/strings/internal/generic_printer_test.cc
@@ -373,6 +373,10 @@ generic_logging_test::Streamable{3}))); } +TEST(GenericPrinterTest, Monostate) { + EXPECT_EQ("monostate", GenericPrintToString(std::monostate{})); +} + TEST(GenericPrinterTest, Tuple) { EXPECT_EQ("<1, two, 3>", GenericPrintToString(std::make_tuple(1, "two", 3))); } @@ -396,8 +400,8 @@ } TEST(GenericPrinterTest, VariantMonostate) { - EXPECT_THAT(GenericPrintToString(std::variant<std::monostate, std::string>()), - IsUnprintable()); + EXPECT_EQ("('(index = 0)' monostate)", + GenericPrintToString(std::variant<std::monostate, std::string>())); } TEST(GenericPrinterTest, VariantNonStreamable) {
diff --git a/absl/strings/numbers.cc b/absl/strings/numbers.cc index b6a8e42..f0a8f00 100644 --- a/absl/strings/numbers.cc +++ b/absl/strings/numbers.cc
@@ -35,6 +35,7 @@ #include "absl/base/config.h" #include "absl/base/internal/endian.h" #include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" #include "absl/numeric/bits.h" @@ -400,6 +401,411 @@ return buffer; } +// Although DBL_DIG is typically 15, DBL_MAX is normally represented with 17 +// digits of precision. When converted to a string value with fewer digits +// of precision using strtod(), the result can be bigger than DBL_MAX due to +// a rounding error. Converting this value back to a double will produce an +// Inf which will trigger a SIGFPE if FP exceptions are enabled. We skip +// the precision check for sufficiently large values to avoid the SIGFPE. +static constexpr double kDoublePrecisionCheckMax = + std::numeric_limits<double>::max() / 1.000000000000001; + +char* absl_nonnull numbers_internal::RoundTripDoubleToBuffer( + double d, char* absl_nonnull buffer) { + // DBL_DIG is 15 for IEEE-754 doubles, which are used on almost all + // platforms these days. Just in case some system exists where DBL_DIG + // is significantly larger -- and risks overflowing our buffer -- we have + // this assert. + static_assert(std::numeric_limits<double>::digits10 < 20, + "std::numeric_limits<double>::digits10 is too big"); + + // We avoid snprintf for NaNs because it inconsistently outputs "-nan" + // or "nan" when given a nan with the sign bit set. + if (std::isnan(d)) { + strcpy(buffer, "nan"); // NOLINT(runtime/printf) + return buffer; + } + bool full_precision_needed = true; + if (std::abs(d) <= kDoublePrecisionCheckMax) { + int snprintf_result = + snprintf(buffer, numbers_internal::kFastToBufferSize, "%.*g", + std::numeric_limits<double>::digits10, d); + + // The snprintf should never overflow because the buffer is significantly + // larger than the precision we asked for. + ABSL_ASSERT(snprintf_result > 0 && + snprintf_result < numbers_internal::kFastToBufferSize); + + full_precision_needed = strtod(buffer, nullptr) != d; + } + + if (full_precision_needed) { + int snprintf_result = + snprintf(buffer, numbers_internal::kFastToBufferSize, "%.*g", + std::numeric_limits<double>::digits10 + 2, d); + + // Should never overflow; see above. + ABSL_ASSERT(snprintf_result > 0 && + snprintf_result < numbers_internal::kFastToBufferSize); + } + return buffer; +} + +namespace { + +// This table is used to quickly calculate the base-ten exponent of a given +// float, and then to provide a multiplier to bring that number into the +// range 1-999,999,999, that is, into uint32_t range. Finally, the exp +// string is made available so there is one less int-to-string conversion +// to be done. +struct Spec { + double min_range; + double multiplier; + const char expstr[5]; +}; + +// clang-format off +constexpr Spec kNegExpTable[] = { + Spec{1.4e-45f, 1e+55, "e-45"}, + Spec{1e-44f, 1e+54, "e-44"}, + Spec{1e-43f, 1e+53, "e-43"}, + Spec{1e-42f, 1e+52, "e-42"}, + Spec{1e-41f, 1e+51, "e-41"}, + Spec{1e-40f, 1e+50, "e-40"}, + Spec{1e-39f, 1e+49, "e-39"}, + Spec{1e-38f, 1e+48, "e-38"}, + Spec{1e-37f, 1e+47, "e-37"}, + Spec{1e-36f, 1e+46, "e-36"}, + Spec{1e-35f, 1e+45, "e-35"}, + Spec{1e-34f, 1e+44, "e-34"}, + Spec{1e-33f, 1e+43, "e-33"}, + Spec{1e-32f, 1e+42, "e-32"}, + Spec{1e-31f, 1e+41, "e-31"}, + Spec{1e-30f, 1e+40, "e-30"}, + Spec{1e-29f, 1e+39, "e-29"}, + Spec{1e-28f, 1e+38, "e-28"}, + Spec{1e-27f, 1e+37, "e-27"}, + Spec{1e-26f, 1e+36, "e-26"}, + Spec{1e-25f, 1e+35, "e-25"}, + Spec{1e-24f, 1e+34, "e-24"}, + Spec{1e-23f, 1e+33, "e-23"}, + Spec{1e-22f, 1e+32, "e-22"}, + Spec{1e-21f, 1e+31, "e-21"}, + Spec{1e-20f, 1e+30, "e-20"}, + Spec{1e-19f, 1e+29, "e-19"}, + Spec{1e-18f, 1e+28, "e-18"}, + Spec{1e-17f, 1e+27, "e-17"}, + Spec{1e-16f, 1e+26, "e-16"}, + Spec{1e-15f, 1e+25, "e-15"}, + Spec{1e-14f, 1e+24, "e-14"}, + Spec{1e-13f, 1e+23, "e-13"}, + Spec{1e-12f, 1e+22, "e-12"}, + Spec{1e-11f, 1e+21, "e-11"}, + Spec{1e-10f, 1e+20, "e-10"}, + Spec{1e-09f, 1e+19, "e-09"}, + Spec{1e-08f, 1e+18, "e-08"}, + Spec{1e-07f, 1e+17, "e-07"}, + Spec{1e-06f, 1e+16, "e-06"}, + Spec{1e-05f, 1e+15, "e-05"}, + Spec{1e-04f, 1e+14, "e-04"}, +}; +// clang-format on + +// clang-format off +constexpr Spec kPosExpTable[] = { + Spec{1e+08f, 1e+02, "e+08"}, + Spec{1e+09f, 1e+01, "e+09"}, + Spec{1e+10f, 1e+00, "e+10"}, + Spec{1e+11f, 1e-01, "e+11"}, + Spec{1e+12f, 1e-02, "e+12"}, + Spec{1e+13f, 1e-03, "e+13"}, + Spec{1e+14f, 1e-04, "e+14"}, + Spec{1e+15f, 1e-05, "e+15"}, + Spec{1e+16f, 1e-06, "e+16"}, + Spec{1e+17f, 1e-07, "e+17"}, + Spec{1e+18f, 1e-08, "e+18"}, + Spec{1e+19f, 1e-09, "e+19"}, + Spec{1e+20f, 1e-10, "e+20"}, + Spec{1e+21f, 1e-11, "e+21"}, + Spec{1e+22f, 1e-12, "e+22"}, + Spec{1e+23f, 1e-13, "e+23"}, + Spec{1e+24f, 1e-14, "e+24"}, + Spec{1e+25f, 1e-15, "e+25"}, + Spec{1e+26f, 1e-16, "e+26"}, + Spec{1e+27f, 1e-17, "e+27"}, + Spec{1e+28f, 1e-18, "e+28"}, + Spec{1e+29f, 1e-19, "e+29"}, + Spec{1e+30f, 1e-20, "e+30"}, + Spec{1e+31f, 1e-21, "e+31"}, + Spec{1e+32f, 1e-22, "e+32"}, + Spec{1e+33f, 1e-23, "e+33"}, + Spec{1e+34f, 1e-24, "e+34"}, + Spec{1e+35f, 1e-25, "e+35"}, + Spec{1e+36f, 1e-26, "e+36"}, + Spec{1e+37f, 1e-27, "e+37"}, + Spec{1e+38f, 1e-28, "e+38"}, + Spec{1e+39, 1e-29, "e+39"}, +}; +// clang-format on + +struct ExpCompare { + bool operator()(const Spec& spec, double d) const { + return spec.min_range < d; + } +}; + +} // namespace + +// Utility routine(s) for RoundTripFloatToBuffer: +// OutputNecessaryDigits takes two 11-digit numbers, whose integer portion +// represents the fractional part of a floating-point number, and outputs a +// number that is in-between them, with the fewest digits possible. For +// instance, given 12345678900 and 12345876900, it would output "0123457". +// When there are multiple final digits that would satisfy this requirement, +// this routine attempts to use a digit that would represent the average of +// lower_double and upper_double. +// +// Although the routine works using integers, all callers use doubles, so +// for their convenience this routine accepts doubles. +static char* absl_nonnull OutputNecessaryDigits(double lower_double, + double upper_double, + char* absl_nonnull out) { + assert(lower_double > 0); + assert(lower_double < upper_double - 10); + assert(upper_double < 100000000000.0); + + // Narrow the range a bit; without this bias, an input of lower=87654320010.0 + // and upper=87654320100.0 would produce an output of 876543201 + // + // We do this in three steps: first, we lower the upper bound and truncate it + // to an integer. Then, we increase the lower bound by exactly the amount we + // just decreased the upper bound by - at that point, the midpoint is exactly + // where it used to be. Then we truncate the lower bound. + + uint64_t upper64 = static_cast<uint64_t>(upper_double - (1.0 / 1024)); + double shrink = upper_double - upper64; + uint64_t lower64 = static_cast<uint64_t>(lower_double + shrink); + + // Theory of operation: we convert the lower number to ascii representation, + // two digits at a time. As we go, we remove the same digits from the upper + // number. When we see the upper number does not share those same digits, we + // know we can stop converting. When we stop, the last digit we output is + // taken from the average of upper and lower values, rounded up. + char buf[2]; + uint32_t lodigits = + static_cast<uint32_t>(lower64 / 1000000000); // 1,000,000,000 + uint64_t mul64 = lodigits * uint64_t{1000000000}; + + numbers_internal::PutTwoDigits(lodigits, out); + out += 2; + if (upper64 - mul64 >= 1000000000) { // digit mismatch! + numbers_internal::PutTwoDigits(static_cast<uint32_t>(upper64 / 1000000000), + buf); + if (out[-2] != buf[0]) { + out[-2] = static_cast<char>('0' + (upper64 + lower64 + 10000000000) / + 20000000000); + --out; + } else { + numbers_internal::PutTwoDigits( + static_cast<uint32_t>((upper64 + lower64 + 1000000000) / 2000000000), + out - 2); + } + *out = '\0'; + return out; + } + uint32_t lower = static_cast<uint32_t>(lower64 - mul64); + uint32_t upper = static_cast<uint32_t>(upper64 - mul64); + + lodigits = lower / 10000000; // 10,000,000 + uint32_t mul = lodigits * 10000000; + numbers_internal::PutTwoDigits(lodigits, out); + out += 2; + if (upper - mul >= 10000000) { // digit mismatch! + numbers_internal::PutTwoDigits(upper / 10000000, buf); + if (out[-2] != buf[0]) { + out[-2] = '0' + (upper + lower + 100000000) / 200000000; + --out; + } else { + numbers_internal::PutTwoDigits((upper + lower + 10000000) / 20000000, + out - 2); + } + *out = '\0'; + return out; + } + lower -= mul; + upper -= mul; + + lodigits = lower / 100000; // 100,000 + mul = lodigits * 100000; + numbers_internal::PutTwoDigits(lodigits, out); + out += 2; + if (upper - mul >= 100000) { // digit mismatch! + numbers_internal::PutTwoDigits(upper / 100000, buf); + if (out[-2] != buf[0]) { + out[-2] = static_cast<char>('0' + (upper + lower + 1000000) / 2000000); + --out; + } else { + numbers_internal::PutTwoDigits((upper + lower + 100000) / 200000, + out - 2); + } + *out = '\0'; + return out; + } + lower -= mul; + upper -= mul; + + lodigits = lower / 1000; + mul = lodigits * 1000; + numbers_internal::PutTwoDigits(lodigits, out); + out += 2; + if (upper - mul >= 1000) { // digit mismatch! + numbers_internal::PutTwoDigits(upper / 1000, buf); + if (out[-2] != buf[0]) { + out[-2] = static_cast<char>('0' + (upper + lower + 10000) / 20000); + --out; + } else { + numbers_internal::PutTwoDigits((upper + lower + 1000) / 2000, out - 2); + } + *out = '\0'; + return out; + } + lower -= mul; + upper -= mul; + + numbers_internal::PutTwoDigits(lower / 10, out); + out += 2; + numbers_internal::PutTwoDigits(upper / 10, buf); + if (out[-2] != buf[0]) { + out[-2] = static_cast<char>('0' + (upper + lower + 100) / 200); + --out; + } else { + numbers_internal::PutTwoDigits((upper + lower + 10) / 20, out - 2); + } + *out = '\0'; + return out; +} + +// RoundTripFloatToBuffer converts the given float into a string which, if +// passed to strtof, will produce the exact same original float. It does this +// by computing the range of possible doubles which map to the given float, and +// then examining the digits of the doubles in that range. If all the doubles +// in the range start with "2.37", then clearly our float does, too. As soon as +// they diverge, only one more digit is needed. +char* absl_nonnull numbers_internal::RoundTripFloatToBuffer( + float f, char* absl_nonnull buffer) { + static_assert(std::numeric_limits<float>::is_iec559, + "IEEE-754/IEC-559 support only"); + + // We write data to out, incrementing as we go, but FloatToBuffer always + // returns the address of the buffer passed in. + char* out = buffer; + + if (std::isnan(f)) { + strcpy(out, "nan"); // NOLINT(runtime/printf) + return buffer; + } + if (f == 0) { // +0 and -0 are handled here + if (std::signbit(f)) { + strcpy(out, "-0"); // NOLINT(runtime/printf) + } else { + strcpy(out, "0"); // NOLINT(runtime/printf) + } + return buffer; + } + if (f < 0) { + *out++ = '-'; + f = -f; + } + if (f > std::numeric_limits<float>::max()) { + strcpy(out, "inf"); // NOLINT(runtime/printf) + return buffer; + } + + double next_lower = nextafterf(f, 0.0f); + // For all doubles in the range lower_bound < f < upper_bound, the + // nearest float is f. + double lower_bound = (f + next_lower) * 0.5; + double upper_bound = f + (f - lower_bound); + // Note: because std::nextafter is slow, we calculate upper_bound + // assuming that it is the same distance from f as lower_bound is. + // For exact powers of two, upper_bound is actually twice as far + // from f as lower_bound is, but this turns out not to matter. + + // Most callers pass floats that are either 0 or within the + // range 0.0001 through 100,000,000, so handle those first, + // since they don't need exponential notation. + const Spec* spec = nullptr; + if (f < 1.0) { + if (f >= 0.0001f) { + // For fractional values, we set up the multiplier at the same + // time as we output the leading "0." / "0.0" / "0.00" / "0.000" + double multiplier = 1e+11; + *out++ = '0'; + *out++ = '.'; + if (f < 0.1f) { + multiplier = 1e+12; + *out++ = '0'; + if (f < 0.01f) { + multiplier = 1e+13; + *out++ = '0'; + if (f < 0.001f) { + multiplier = 1e+14; + *out++ = '0'; + } + } + } + OutputNecessaryDigits(lower_bound * multiplier, upper_bound * multiplier, + out); + return buffer; + } + spec = std::lower_bound(std::begin(kNegExpTable), std::end(kNegExpTable), + double{f}, ExpCompare()); + if (spec == std::end(kNegExpTable)) --spec; + } else if (f < 1e8) { + // Handling non-exponential format greater than 1.0 is similar to the above, + // but instead of 0.0 / 0.00 / 0.000, the prefix is simply the truncated + // integer part of f. + int32_t as_int = static_cast<int32_t>(f); + out = numbers_internal::FastIntToBuffer(as_int, out); + // Easy: if the integer part is within (lower_bound, upper_bound), then we + // are already done. + if (as_int > lower_bound && as_int < upper_bound) { + return buffer; + } + *out++ = '.'; + OutputNecessaryDigits((lower_bound - as_int) * 1e11, + (upper_bound - as_int) * 1e11, out); + return buffer; + } else { + spec = std::lower_bound(std::begin(kPosExpTable), std::end(kPosExpTable), + double{f}, ExpCompare()); + if (spec == std::end(kPosExpTable)) --spec; + } + // Exponential notation from here on. "spec" was computed using lower_bound, + // which means it's the first spec from the table where min_range is greater + // or equal to f. + // Unfortunately that's not quite what we want; we want a min_range that is + // less or equal. So first thing, if it was greater, back up one entry. + if (spec->min_range > f) --spec; + + // The digits might be "237000123", but we want "2.37000123", + // so we output the digits one character later, and then move the first + // digit back so we can stick the "." in. + char* start = out; + out = OutputNecessaryDigits(lower_bound * spec->multiplier, + upper_bound * spec->multiplier, start + 1); + start[0] = start[1]; + start[1] = '.'; + + // If it turns out there was only one digit output, then back up over the '.' + if (out == &start[2]) --out; + + // Now add the "e+NN" part. + memcpy(out, spec->expstr, 4); + out[4] = '\0'; + return buffer; +} + // Given a 128-bit number expressed as a pair of uint64_t, high half first, // return that number multiplied by the given 32-bit value. If the result is // too large to fit in a 128-bit number, divide it by 2 until it fits.
diff --git a/absl/strings/numbers.h b/absl/strings/numbers.h index fa552af..b4f81a3 100644 --- a/absl/strings/numbers.h +++ b/absl/strings/numbers.h
@@ -178,6 +178,10 @@ inline constexpr int kFastToBufferSize = 32; inline constexpr int kSixDigitsToBufferSize = 16; +// Helper function used to implement absl::HighPrecision(). +char* absl_nonnull RoundTripDoubleToBuffer(double d, char* absl_nonnull buffer); +char* absl_nonnull RoundTripFloatToBuffer(float f, char* absl_nonnull buffer); + // Helper function for fast formatting of floating-point values. // The result is the same as printf's "%g", a.k.a. "%.6g"; that is, six // significant digits are returned, trailing zeros are removed, and numbers
diff --git a/absl/strings/numbers_test.cc b/absl/strings/numbers_test.cc index 2eaa8c7..41f8c73 100644 --- a/absl/strings/numbers_test.cc +++ b/absl/strings/numbers_test.cc
@@ -1780,6 +1780,39 @@ } } +TEST_F(SimpleDtoaTest, ExhaustiveFloatToBuffer) { + uint64_t test_count = 0; + std::vector<float> mismatches; + ExhaustiveFloat(kFloatNumCases, [&](float f) { + if (f != f) return; // rule out NaNs + ++test_count; + char fastbuf[absl::numbers_internal::kFastToBufferSize]; + absl::numbers_internal::RoundTripFloatToBuffer(f, fastbuf); + float round_trip = strtof(fastbuf, nullptr); + if (f != round_trip) { + mismatches.push_back(f); + if (mismatches.size() < 10) { + LOG(ERROR) << "Round-trip failure with float. f=" << f << "=" + << ToNineDigits(f) << " fast=" << fastbuf + << " strtof=" << ToNineDigits(round_trip); + } + } + }); + if (!mismatches.empty()) { + EXPECT_EQ(mismatches.size(), 0); + for (size_t i = 0; i < mismatches.size(); ++i) { + if (i > 100) i = mismatches.size() - 1; + float f = mismatches[i]; + char buf[absl::numbers_internal::kFastToBufferSize]; + float rt = strtof(buf, nullptr); + LOG(ERROR) << "Mismatch #" << i << " f=" << f << " (" << ToNineDigits(f) + << ") fast='" + << absl::numbers_internal::RoundTripFloatToBuffer(f, buf) + << "' rt=" << ToNineDigits(rt); + } + } +} + TEST_F(SimpleDtoaTest, ExhaustiveDoubleToSixDigits) { uint64_t test_count = 0; std::vector<double> mismatches;
diff --git a/absl/strings/resize_and_overwrite.h b/absl/strings/resize_and_overwrite.h index 102b2bf..4d2e2b8 100644 --- a/absl/strings/resize_and_overwrite.h +++ b/absl/strings/resize_and_overwrite.h
@@ -53,6 +53,7 @@ #include "absl/base/config.h" #include "absl/base/dynamic_annotations.h" +#include "absl/base/internal/hardening.h" #include "absl/base/macros.h" #include "absl/base/optimization.h" #include "absl/base/throw_delegate.h" @@ -135,10 +136,13 @@ ABSL_ANNOTATE_MEMORY_IS_UNINITIALIZED(str.data() + old_size, n - old_size); } #endif - auto new_size = std::move(op)(str.data(), n); - ABSL_HARDENING_ASSERT(new_size >= 0 && new_size <= n); - ABSL_HARDENING_ASSERT(str.data()[n] == typename T::value_type{}); - str.erase(static_cast<typename T::size_type>(new_size)); + typename T::size_type new_size = + static_cast<typename T::size_type>(std::move(op)(str.data(), n)); + absl::base_internal::HardeningAssertGE(new_size, typename T::size_type{0}); + absl::base_internal::HardeningAssertLE(new_size, n); + absl::base_internal::HardeningAssert(str.data()[n] == + typename T::value_type{}); + str.erase(new_size); } template <typename T, typename Op>
diff --git a/absl/strings/str_cat.h b/absl/strings/str_cat.h index 1ada736..18e7058 100644 --- a/absl/strings/str_cat.h +++ b/absl/strings/str_cat.h
@@ -43,6 +43,13 @@ // Floating point numbers are formatted with six-digit precision, which is // the default for "std::cout <<" or printf "%g" (the same as "%.6g"). // +// Floating point values can also be converted to a string which, if passed to +// `strtod()`, would produce the exact same original double (except in case of +// NaN; all NaNs are considered the same value) by passing the number to +// absl::HighPrecision. HighPrecision tries to keep the string short but +// it's not guaranteed to be as short as possible. +// See http://go/faster-double-strcat +// // You can convert to hexadecimal output rather than decimal output using the // `Hex` type contained here. To do so, pass `Hex(my_int)` as a parameter to // `StrCat()` or `StrAppend()`. You may specify a minimum hex field width using @@ -305,6 +312,35 @@ }; // ----------------------------------------------------------------------------- +// HighPrecision +// ----------------------------------------------------------------------------- +// +// Converts floating point values to a string which, if passed to +// `absl::SimpleAtof`/`absl::SimpleAtod`, would produce the exact same original +// floating point value (except in case of NaN; all NaNs are considered the same +// value). Tries to keep the string short but it's not guaranteed to be as short +// as possible. +// +// HighPrecision is conisderably slower than the default formatting, so only use +// it if you need the string to convert back to the same floating-point value. + +inline strings_internal::AlphaNumBuffer<numbers_internal::kFastToBufferSize> +HighPrecision(float f) { + strings_internal::AlphaNumBuffer<numbers_internal::kFastToBufferSize> result; + result.size = + strlen(numbers_internal::RoundTripFloatToBuffer(f, &result.data[0])); + return result; +} + +inline strings_internal::AlphaNumBuffer<numbers_internal::kFastToBufferSize> +HighPrecision(double d) { + strings_internal::AlphaNumBuffer<numbers_internal::kFastToBufferSize> result; + result.size = + strlen(numbers_internal::RoundTripDoubleToBuffer(d, &result.data[0])); + return result; +} + +// ----------------------------------------------------------------------------- // AlphaNum // ----------------------------------------------------------------------------- //
diff --git a/absl/strings/str_cat_test.cc b/absl/strings/str_cat_test.cc index a3bd42c..7b21c9c 100644 --- a/absl/strings/str_cat_test.cc +++ b/absl/strings/str_cat_test.cc
@@ -140,6 +140,9 @@ result = absl::StrCat(-1); EXPECT_EQ(result, "-1"); + result = absl::StrCat(absl::HighPrecision(0.5)); + EXPECT_EQ(result, "0.5"); + result = absl::StrCat(absl::SixDigits(0.5)); EXPECT_EQ(result, "0.5"); @@ -182,16 +185,27 @@ EXPECT_EQ(result, "To output a char by ASCII/numeric value, use +: 33"); float f = 100000.5; + result = absl::StrCat("A hundred K and a half is ", absl::HighPrecision(f)); + EXPECT_EQ(result, "A hundred K and a half is 100000.5"); + result = absl::StrCat("A hundred K and a half is ", absl::SixDigits(f)); EXPECT_EQ(result, "A hundred K and a half is 100000"); f = 100001.5; + result = absl::StrCat("A hundred K and one and a half is ", + absl::HighPrecision(f)); + EXPECT_EQ(result, "A hundred K and one and a half is 100001.5"); + result = absl::StrCat("A hundred K and one and a half is ", absl::SixDigits(f)); EXPECT_EQ(result, "A hundred K and one and a half is 100002"); double d = 100000.5; d *= d; + result = absl::StrCat("A hundred K and a half squared is ", + absl::HighPrecision(d)); + EXPECT_EQ(result, "A hundred K and a half squared is 10000100000.25"); + result = absl::StrCat("A hundred K and a half squared is ", absl::SixDigits(d)); EXPECT_EQ(result, "A hundred K and a half squared is 1.00001e+10"); @@ -408,6 +422,20 @@ EXPECT_EQ(result.substr(old_size), "To output a char by ASCII/numeric value, use +: 33"); + float f = 100000.5; + old_size = result.size(); + absl::StrAppend(&result, "A hundred K and a half is ", + absl::HighPrecision(f)); + EXPECT_EQ(result.substr(old_size), "A hundred K and a half is 100000.5"); + + double d = f; + d *= d; + old_size = result.size(); + absl::StrAppend(&result, "A hundred K and a half squared is ", + absl::HighPrecision(d)); + EXPECT_EQ(result.substr(old_size), + "A hundred K and a half squared is 10000100000.25"); + // Test 9 arguments, the old maximum old_size = result.size(); absl::StrAppend(&result, 1, 22, 333, 4444, 55555, 666666, 7777777, 88888888,
diff --git a/absl/types/BUILD.bazel b/absl/types/BUILD.bazel index a3c038a..ff76679 100644 --- a/absl/types/BUILD.bazel +++ b/absl/types/BUILD.bazel
@@ -187,6 +187,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ "//absl/base:config", + "//absl/base:core_headers", "//absl/utility", ], ) @@ -203,6 +204,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ "//absl/base:config", + "//absl/base:core_headers", "//absl/utility", ], )
diff --git a/absl/types/CMakeLists.txt b/absl/types/CMakeLists.txt index f00b574..79a4c19 100644 --- a/absl/types/CMakeLists.txt +++ b/absl/types/CMakeLists.txt
@@ -138,6 +138,7 @@ ${ABSL_DEFAULT_COPTS} DEPS absl::config + absl::core_headers absl::utility PUBLIC ) @@ -151,6 +152,7 @@ ${ABSL_DEFAULT_COPTS} DEPS absl::config + absl::core_headers absl::utility PUBLIC )
diff --git a/absl/types/any.h b/absl/types/any.h index c488631..9026b1b 100644 --- a/absl/types/any.h +++ b/absl/types/any.h
@@ -30,13 +30,14 @@ // Include-what-you-use cleanup required for these headers. #include "absl/base/attributes.h" +#include "absl/base/macros.h" #include "absl/utility/utility.h" namespace absl { ABSL_NAMESPACE_BEGIN -using std::any; +using any ABSL_DEPRECATE_AND_INLINE() = std::any; using std::any_cast; -using std::bad_any_cast; +using bad_any_cast ABSL_DEPRECATE_AND_INLINE() = std::bad_any_cast; using std::make_any; ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/types/optional.h b/absl/types/optional.h index 5b68a5b..96197ab 100644 --- a/absl/types/optional.h +++ b/absl/types/optional.h
@@ -26,14 +26,21 @@ #include <optional> #include "absl/base/config.h" +#include "absl/base/macros.h" #include "absl/utility/utility.h" namespace absl { ABSL_NAMESPACE_BEGIN -using std::bad_optional_access; + +using bad_optional_access ABSL_REFACTOR_INLINE + = std::bad_optional_access; + using std::make_optional; using std::nullopt; -using std::nullopt_t; + +using nullopt_t ABSL_REFACTOR_INLINE + = std::nullopt_t; + using std::optional; ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/types/variant.h b/absl/types/variant.h index 4c50bbe..bb73cce 100644 --- a/absl/types/variant.h +++ b/absl/types/variant.h
@@ -23,23 +23,45 @@ #ifndef ABSL_TYPES_VARIANT_H_ #define ABSL_TYPES_VARIANT_H_ +#include <stddef.h> + #include <variant> #include "absl/base/config.h" +#include "absl/base/macros.h" #include "absl/utility/utility.h" namespace absl { ABSL_NAMESPACE_BEGIN -using std::bad_variant_access; + +using bad_variant_access ABSL_REFACTOR_INLINE + = std::bad_variant_access; + using std::get; using std::get_if; using std::holds_alternative; -using std::monostate; -using std::variant; -using std::variant_alternative; -using std::variant_alternative_t; + +using monostate ABSL_REFACTOR_INLINE + = std::monostate; + +template <typename... Types> +using variant ABSL_REFACTOR_INLINE + = std::variant<Types...>; + +template <size_t I, typename T> +using variant_alternative ABSL_REFACTOR_INLINE + = std::variant_alternative<I, T>; + +template <size_t I, typename T> +using variant_alternative_t ABSL_REFACTOR_INLINE + = std::variant_alternative_t<I, T>; + using std::variant_npos; -using std::variant_size; + +template <typename T> +using variant_size ABSL_REFACTOR_INLINE + = std::variant_size<T>; + using std::variant_size_v; using std::visit;
diff --git a/absl/utility/BUILD.bazel b/absl/utility/BUILD.bazel index a714b02..1994a01 100644 --- a/absl/utility/BUILD.bazel +++ b/absl/utility/BUILD.bazel
@@ -41,6 +41,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ "//absl/base:config", + "//absl/base:core_headers", "//absl/meta:type_traits", ], )
diff --git a/absl/utility/CMakeLists.txt b/absl/utility/CMakeLists.txt index 8ac87bc..ece42de 100644 --- a/absl/utility/CMakeLists.txt +++ b/absl/utility/CMakeLists.txt
@@ -23,6 +23,7 @@ ${ABSL_DEFAULT_COPTS} DEPS absl::config + absl::core_headers absl::type_traits PUBLIC )
diff --git a/absl/utility/utility.h b/absl/utility/utility.h index 4d72c31..06e5378 100644 --- a/absl/utility/utility.h +++ b/absl/utility/utility.h
@@ -21,10 +21,13 @@ #include <utility> #include "absl/base/config.h" +#include "absl/base/macros.h" // TODO(b/290784225): Include what you use cleanup required. #include "absl/meta/type_traits.h" +// TODO(b/509512528): Deprecate the C++14/C++17 symbols publicly, in all files. + namespace absl { ABSL_NAMESPACE_BEGIN @@ -37,16 +40,38 @@ using std::forward; using std::in_place; using std::in_place_index; -using std::in_place_index_t; -using std::in_place_t; + +template <size_t I> +using in_place_index_t ABSL_DEPRECATE_AND_INLINE() = std::in_place_index_t<I>; + +using in_place_t ABSL_DEPRECATE_AND_INLINE() = std::in_place_t; + using std::in_place_type; -using std::in_place_type_t; -using std::index_sequence; -using std::index_sequence_for; -using std::integer_sequence; + +template <class T> +using in_place_type_t ABSL_DEPRECATE_AND_INLINE() = std::in_place_type_t<T>; + +template <size_t... I> +using index_sequence ABSL_DEPRECATE_AND_INLINE() = std::index_sequence<I...>; + +template <class T, T... I> +using integer_sequence ABSL_DEPRECATE_AND_INLINE() = + std::integer_sequence<T, I...>; + +template <class... T> +using index_sequence_for ABSL_DEPRECATE_AND_INLINE() = + std::index_sequence_for<T...>; + using std::make_from_tuple; -using std::make_index_sequence; -using std::make_integer_sequence; + +template <size_t N> +using make_index_sequence ABSL_DEPRECATE_AND_INLINE() = + std::make_index_sequence<N>; + +template <class T, T N> +using make_integer_sequence ABSL_DEPRECATE_AND_INLINE() = + std::make_integer_sequence<T, N>; + using std::move; #if ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L