diff --git a/absl/BUILD.bazel b/absl/BUILD.bazel index 980563a..c358647 100644 --- a/absl/BUILD.bazel +++ b/absl/BUILD.bazel
@@ -45,6 +45,17 @@ ) config_setting( + name = "mingw-clang_compiler", + constraint_values = [ + "@platforms//os:windows", + ], + flag_values = { + "@bazel_tools//tools/cpp:compiler": "clang", + }, + visibility = [":__subpackages__"], +) + +config_setting( name = "fuchsia", constraint_values = [ "@platforms//os:fuchsia", @@ -57,6 +68,7 @@ match_any = [ ":mingw_unspecified_compiler", ":mingw-gcc_compiler", + ":mingw-clang_compiler", ], visibility = [":__subpackages__"], )
diff --git a/absl/base/BUILD.bazel b/absl/base/BUILD.bazel index e638bb4..9fbd476 100644 --- a/absl/base/BUILD.bazel +++ b/absl/base/BUILD.bazel
@@ -275,6 +275,7 @@ "@rules_cc//cc/compiler:msvc-cl": [], "@rules_cc//cc/compiler:clang-cl": [], "@rules_cc//cc/compiler:emscripten": [], + "@platforms//os:wasi": [], "//conditions:default": ["-pthread"], }) + ABSL_DEFAULT_LINKOPTS, visibility = [ @@ -343,6 +344,7 @@ "-ladvapi32", ], "@rules_cc//cc/compiler:emscripten": [], + "@platforms//os:wasi": [], "//conditions:default": ["-pthread"], }) + ABSL_DEFAULT_LINKOPTS, visibility = ["//visibility:public"],
diff --git a/absl/base/attributes.h b/absl/base/attributes.h index 8f2bb2c..fe544c6 100644 --- a/absl/base/attributes.h +++ b/absl/base/attributes.h
@@ -566,7 +566,24 @@ #ifdef ABSL_ATTRIBUTE_UNUSED #error "ABSL_ATTRIBUTE_UNUSED should not be defined." #endif +#if defined(__cplusplus) && defined(__clang__) && \ + ABSL_HAVE_ATTRIBUTE(diagnose_if) && !defined(SWIG) +struct [[deprecated("Use [[maybe_unused]] instead.")]] _absl_unused_macro; +// Trick: We use __attribute__((diagnose_if(...))) to refer to our own +// deprecated symbol, which then causes a deprecation message to be emitted when +// the macro is used. Since diagnose_if() isn't valid on every declaration, we +// also suppress the warning regarding that. +#define ABSL_ATTRIBUTE_UNUSED \ + [[maybe_unused]] \ + _Pragma("clang diagnostic push") /* */ \ + _Pragma("clang diagnostic ignored \"-Wignored-attributes\"") \ + __attribute__((diagnose_if( \ + sizeof(_absl_unused_macro*) == 0, "", \ + "warning"))) /* */ \ + _Pragma("clang diagnostic pop") +#else #define ABSL_ATTRIBUTE_UNUSED [[maybe_unused]] +#endif // ABSL_ATTRIBUTE_INITIAL_EXEC //
diff --git a/absl/base/internal/sysinfo.cc b/absl/base/internal/sysinfo.cc index 1f380bc..19fef81 100644 --- a/absl/base/internal/sysinfo.cc +++ b/absl/base/internal/sysinfo.cc
@@ -69,6 +69,10 @@ #include <zircon/process.h> #endif +#if defined(__FREERTOS__) +#include <task.h> +#endif + namespace absl { ABSL_NAMESPACE_BEGIN namespace base_internal { @@ -471,6 +475,12 @@ return static_cast<pid_t>(zx_thread_self()); } +#elif defined(__FREERTOS__) + +pid_t GetTID() { + return static_cast<pid_t>(uxTaskGetTaskNumber(xTaskGetCurrentTaskHandle())); +} + #else // Fallback implementation of `GetTID` using `pthread_self`.
diff --git a/absl/base/internal/thread_identity.h b/absl/base/internal/thread_identity.h index a9f26d0..385df99 100644 --- a/absl/base/internal/thread_identity.h +++ b/absl/base/internal/thread_identity.h
@@ -275,10 +275,11 @@ #define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11 #elif defined(__APPLE__) && defined(ABSL_HAVE_THREAD_LOCAL) #define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11 -#elif ABSL_PER_THREAD_TLS && defined(__GOOGLE_GRTE_VERSION__) && \ - (__GOOGLE_GRTE_VERSION__ >= 20140228L) -// Support for async-safe TLS was specifically added in GRTEv4. It's not -// present in the upstream eglibc. +#elif ABSL_PER_THREAD_TLS && \ + (!defined(__GLIBC__) || defined(__GOOGLE_GRTE_VERSION__)) +// Check that the libc implementation provides async-signal-safe TLS. Upstream +// glibc (without GRTE patches) may call malloc() in __tls_get_addr() for +// dlopen()'d modules and must use pthread_getspecific() instead. // Note: Current default for production systems. #define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_TLS #else
diff --git a/absl/container/flat_hash_map.h b/absl/container/flat_hash_map.h index 8233409..fa91949 100644 --- a/absl/container/flat_hash_map.h +++ b/absl/container/flat_hash_map.h
@@ -650,8 +650,33 @@ namespace container_internal { +// These members are common to all template instantiations, so we factor them +// out into this base class to avoid re-instantiating similar templates multiple +// times, reducing compilation time. +struct FlatHashMapPolicyBase { + template <class F, class... Args> + static decltype(absl::container_internal::DecomposePair( + std::declval<F>(), std::declval<Args>()...)) + apply(F&& f, Args&&... args) { + return absl::container_internal::DecomposePair(std::forward<F>(f), + std::forward<Args>(args)...); + } + + static size_t space_used(const void*) { return 0; } + + template <class K, class V> + static V& value(std::pair<const K, V>* kv) { + return kv->second; + } + + template <class K, class V> + static const V& value(const std::pair<const K, V>* kv) { + return kv->second; + } +}; + template <class K, class V> -struct FlatHashMapPolicy { +struct FlatHashMapPolicy : FlatHashMapPolicyBase { using slot_policy = container_internal::map_slot_policy<K, V>; using slot_type = typename slot_policy::slot_type; using key_type = K; @@ -679,14 +704,6 @@ return slot_policy::transfer(alloc, new_slot, old_slot); } - template <class F, class... Args> - static decltype(absl::container_internal::DecomposePair( - std::declval<F>(), std::declval<Args>()...)) - apply(F&& f, Args&&... args) { - return absl::container_internal::DecomposePair(std::forward<F>(f), - std::forward<Args>(args)...); - } - template <class Hash, bool kIsDefault, size_t kSeedShift> static constexpr HashSlotFn get_hash_slot_fn() { return memory_internal::IsLayoutCompatible<K, V>::value @@ -694,12 +711,7 @@ : nullptr; } - static size_t space_used(const slot_type*) { return 0; } - static std::pair<const K, V>& element(slot_type* slot) { return slot->value; } - - static V& value(std::pair<const K, V>* kv) { return kv->second; } - static const V& value(const std::pair<const K, V>* kv) { return kv->second; } }; } // namespace container_internal
diff --git a/absl/container/flat_hash_set_test.cc b/absl/container/flat_hash_set_test.cc index 6c99e8d..df448a1 100644 --- a/absl/container/flat_hash_set_test.cc +++ b/absl/container/flat_hash_set_test.cc
@@ -392,6 +392,16 @@ EXPECT_EQ(HashtableDebugAccess<flat_hash_set<int>>::kIsAbslHash, true); EXPECT_EQ(HashtableDebugAccess<flat_hash_set<std::string>>::kIsAbslHash, true); + EXPECT_EQ(HashtableDebugAccess<flat_hash_set<std::wstring>>::kIsAbslHash, + true); + EXPECT_EQ(HashtableDebugAccess<flat_hash_set<int*>>::kIsAbslHash, true); + EXPECT_EQ( + HashtableDebugAccess<flat_hash_set<std::unique_ptr<int>>>::kIsAbslHash, + true); + using TransparentStringHashSet = + flat_hash_set<std::string, + absl::TransparentHash<std::string, std::string_view>>; + EXPECT_EQ(HashtableDebugAccess<TransparentStringHashSet>::kIsAbslHash, true); struct Hash { size_t operator()(size_t i) const { return i; }
diff --git a/absl/container/internal/hash_function_defaults.h b/absl/container/internal/hash_function_defaults.h index dcf7dbd..6db9e57 100644 --- a/absl/container/internal/hash_function_defaults.h +++ b/absl/container/internal/hash_function_defaults.h
@@ -129,6 +129,14 @@ size_t operator()(std::basic_string_view<TChar> v) const { return absl::Hash<std::basic_string_view<TChar>>{}(v); } + + private: + friend struct absl::hash_internal::HashWithSeed; + + size_t hash_with_seed(std::basic_string_view<TChar> v, size_t seed) const { + return absl::hash_internal::HashWithSeed().hash( + absl::Hash<std::basic_string_view<TChar>>{}, v, seed); + } }; template <typename TChar> @@ -175,6 +183,15 @@ size_t operator()(const U& ptr) const { return absl::Hash<const T*>{}(HashEq::ToPtr(ptr)); } + + private: + friend struct absl::hash_internal::HashWithSeed; + + template <class U> + size_t hash_with_seed(const U& ptr, size_t seed) const { + return absl::hash_internal::HashWithSeed().hash(absl::Hash<const T*>{}, + HashEq::ToPtr(ptr), seed); + } }; struct Eq { using is_transparent = void;
diff --git a/absl/container/internal/hashtable_control_bytes.h b/absl/container/internal/hashtable_control_bytes.h index b70ac8c..34df1d4 100644 --- a/absl/container/internal/hashtable_control_bytes.h +++ b/absl/container/internal/hashtable_control_bytes.h
@@ -59,7 +59,7 @@ template <typename T> -uint32_t TrailingZeros(T x) { +constexpr uint32_t TrailingZeros(T x) { ABSL_ASSUME(x != 0); return static_cast<uint32_t>(countr_zero(x)); }
diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index 3b0cf42..a493b9f 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc
@@ -379,16 +379,20 @@ return result; } -HashtablezInfoHandle CommonFields::infoz_ptr() const { +HashtablezInfoHandle CommonFields::infoz_from_control(ctrl_t* ctrl) const { // growth_info is stored before control bytes. ABSL_SWISSTABLE_ASSERT(has_infoz()); HashtablezInfoHandle res; - void* src = reinterpret_cast<char*>(control()) - + void* src = reinterpret_cast<char*>(ctrl) - MetadataBeforeControlSize(/*has_infoz=*/true, capacity()); std::memcpy(&res, src, sizeof(HashtablezInfoHandle)); return res; } +HashtablezInfoHandle CommonFields::infoz_ptr() const { + return infoz_from_control(control()); +} + void CommonFields::set_infoz(HashtablezInfoHandle infoz) { ABSL_SWISSTABLE_ASSERT(has_infoz()); void* dst = reinterpret_cast<char*>(control()) - @@ -1008,10 +1012,9 @@ return total_probe_length; } -void ReportGrowthToInfozImpl(CommonFields& common, HashtablezInfoHandle infoz, - size_t hash, size_t total_probe_length, - size_t distance_from_desired) { - ABSL_SWISSTABLE_ASSERT(infoz.IsSampled()); +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void ReportGrowthToInfozImpl( + CommonFields& common, HashtablezInfoHandle infoz, size_t hash, + size_t total_probe_length, size_t distance_from_desired) { infoz.RecordStorageChanged(common.size() - 1, common.capacity()); infoz.RecordRehash(total_probe_length); infoz.RecordInsertMiss(hash, distance_from_desired); @@ -1020,25 +1023,84 @@ } // Specialization to avoid passing two 0s from hot function. -ABSL_ATTRIBUTE_NOINLINE void ReportSingleGroupTableGrowthToInfoz( - CommonFields& common, HashtablezInfoHandle infoz, size_t hash) { - ReportGrowthToInfozImpl(common, infoz, hash, /*total_probe_length=*/0, +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void +ReportSingleGroupTableGrowthToInfoz(CommonFields& common, ctrl_t* old_ctrl, + size_t hash) { + ReportGrowthToInfozImpl(common, common.infoz_from_control(old_ctrl), hash, + /*total_probe_length=*/0, /*distance_from_desired=*/0); } -ABSL_ATTRIBUTE_NOINLINE void ReportGrowthToInfoz(CommonFields& common, - HashtablezInfoHandle infoz, - size_t hash, - size_t total_probe_length, - size_t distance_from_desired) { - ReportGrowthToInfozImpl(common, infoz, hash, total_probe_length, - distance_from_desired); +// Specialization that computes hash and generates new seed. +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void +ReportSingleElementTableGrowthToInfozAndForceSample( + CommonFields& common, const PolicyFunctions& __restrict policy, + absl::FunctionRef<size_t(size_t)> get_hash) { + HashtablezInfoHandle infoz = + ForcedTrySample(policy.slot_size, policy.key_size, policy.value_size, + policy.soo_capacity()); + common.generate_new_seed(/*has_infoz=*/true); + ReportGrowthToInfozImpl(common, infoz, get_hash(common.seed().seed()), + /*total_probe_length=*/0, + /*distance_from_desired=*/0); } -ABSL_ATTRIBUTE_NOINLINE void ReportResizeToInfoz(CommonFields& common, - HashtablezInfoHandle infoz, - size_t total_probe_length) { - ABSL_SWISSTABLE_ASSERT(infoz.IsSampled()); +// Outlines cold infoz recording so callers do not inline infoz extraction +// or spill registers across SetCtrl. `common` is first and `hash` is third to +// match argument order in callers. +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void ReportInsertMissToInfoz( + CommonFields& common, size_t probe_length, size_t hash) { + common.infoz().RecordInsertMiss(hash, probe_length); +} + +// Outlines cold infoz recording with computation of hash so callers can +// avoid calling the hash function and seed extraction in the hot path. +// `common` is first and `get_hash` is third to match argument order in callers. +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void +ReportInsertMissToInfozAndComputeHash( + CommonFields& common, size_t probe_length, + absl::FunctionRef<size_t(size_t)> get_hash) { + common.infoz().RecordInsertMiss(get_hash(common.seed().seed()), probe_length); +} + +// Outlines cold infoz recording with computation of probe_length so callers can +// skip tracking of probe_length. `common` is first and `hash` is third to match +// argument order in callers. +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void +ReportInsertMissToInfozAndComputeProbeLength(CommonFields& common, + size_t target, size_t hash) { + const size_t cap = common.capacity(); + auto seq = probe(ProbeCapacity{cap}, hash); + while (((target - seq.offset()) & cap) >= Group::kWidth) { + seq.next(); + ABSL_SWISSTABLE_ASSERT(seq.index() < common.capacity()); + } + common.infoz().RecordInsertMiss(hash, seq.index()); +} + +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void ReportGrowthToInfoz( + CommonFields& common, ctrl_t* old_ctrl, size_t hash, + size_t total_probe_length, size_t distance_from_desired) { + ReportGrowthToInfozImpl(common, common.infoz_from_control(old_ctrl), hash, + total_probe_length, distance_from_desired); +} + +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void ReportResizeToInfoz( + CommonFields& common, HashtablezInfoHandle infoz, + size_t total_probe_length) { + infoz.RecordStorageChanged(common.size(), common.capacity()); + infoz.RecordRehash(total_probe_length); + common.set_has_infoz(); + common.set_infoz(infoz); +} + +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void +ReportResizeToInfozAndForceSample(CommonFields& common, + const PolicyFunctions& __restrict policy, + size_t total_probe_length) { + HashtablezInfoHandle infoz = + ForcedTrySample(policy.slot_size, policy.key_size, policy.value_size, + policy.soo_capacity()); infoz.RecordStorageChanged(common.size(), common.capacity()); infoz.RecordRehash(total_probe_length); common.set_has_infoz(); @@ -1086,14 +1148,8 @@ ABSL_SWISSTABLE_ASSERT(common.capacity() == policy.soo_capacity()); ABSL_SWISSTABLE_ASSERT(common.empty()); const size_t slot_size = policy.slot_size; - HashtablezInfoHandle infoz; - const bool should_sample = - policy.is_hashtablez_eligible && (force_infoz || ShouldSampleNextTable()); - if (ABSL_PREDICT_FALSE(should_sample)) { - infoz = ForcedTrySample(slot_size, policy.key_size, policy.value_size, - policy.soo_capacity()); - } - const bool has_infoz = infoz.IsSampled(); + const bool has_infoz = + policy.is_hashtablez_enabled && (force_infoz || ShouldSampleNextTable()); void* alloc = policy.get_char_alloc(common); common.set_capacity(new_capacity); @@ -1107,7 +1163,7 @@ ResetGrowthLeft(new_capacity, blocked_element_count, common); if (ABSL_PREDICT_FALSE(has_infoz)) { - ReportResizeToInfoz(common, infoz, 0); + ReportResizeToInfozAndForceSample(common, policy, 0); } } @@ -1167,11 +1223,12 @@ bool has_infoz = false; if (sampling_mode == ResizeFullSooTableSamplingMode::kForceSampleNoResizeIfUnsampled) { - if (ABSL_PREDICT_FALSE(policy.is_hashtablez_eligible)) { + if (policy.is_hashtablez_enabled) { infoz = ForcedTrySample(slot_size, policy.key_size, policy.value_size, policy.soo_capacity()); } + // If the table was not sampled, do not resize. if (!infoz.IsSampled()) return; has_infoz = true; } @@ -1690,12 +1747,13 @@ constexpr size_t kOldCapacity = 1; constexpr size_t kNewCapacity = NextCapacity(kOldCapacity); void* old_slots = common.slot_array(kOldCapacity); + // old_slots == old_ctrl in case of capacity == 1. + ctrl_t* old_ctrl = static_cast<ctrl_t*>(old_slots); const size_t slot_size = policy.slot_size; const size_t slot_align = policy.slot_align; void* alloc = policy.get_char_alloc(common); - HashtablezInfoHandle infoz = common.infoz(); - const bool has_infoz = infoz.IsSampled(); + const bool has_infoz = common.has_infoz(); common.set_capacity(kNewCapacity); const auto [new_ctrl, new_slots] = @@ -1722,18 +1780,18 @@ void* new_element_target_slot = SlotAddress(new_slots, offset, slot_size); SanitizerUnpoisonMemoryRegion(new_element_target_slot, slot_size); - DeallocBackingArrayImpl(alloc, kOldCapacity, - // old_slots == old_ctrl in case of capacity == 1. - static_cast<ctrl_t*>(old_slots), slot_size, - slot_align, has_infoz, - /*blocked_element_count=*/0, policy.dealloc); PrepareInsertCommon(common); ABSL_SWISSTABLE_ASSERT(common.size() == 2); common.InitGrowthLeftNoDeleted(kNewCapacity - 2, kNewCapacity); if (ABSL_PREDICT_FALSE(has_infoz)) { - ReportSingleGroupTableGrowthToInfoz(common, infoz, new_hash); + ReportSingleGroupTableGrowthToInfoz(common, old_ctrl, new_hash); } + // Deallocate after `ReportSingleGroupTableGrowthToInfoz`, because it reads + // infoz from old_ctrl. + DeallocBackingArrayImpl(alloc, kOldCapacity, old_ctrl, slot_size, slot_align, + has_infoz, + /*blocked_element_count=*/0, policy.dealloc); return new_element_target_slot; } @@ -1749,12 +1807,12 @@ ABSL_ASSUME(old_capacity > kMaxSmallCapacity); const size_t new_capacity = NextCapacity(old_capacity); + ABSL_ASSUME(new_capacity > kMaxSmallCapacity); ctrl_t* old_ctrl = common.control(); void* old_slots = common.slot_array(old_capacity); size_t old_blocked_element_count = common.blocked_element_count(); - HashtablezInfoHandle infoz = common.infoz(); - const bool has_infoz = infoz.IsSampled(); + const bool has_infoz = common.has_infoz(); common.set_capacity(new_capacity); common.set_blocked_element_count_to_zero(); const size_t slot_size = policy.slot_size; @@ -1797,15 +1855,18 @@ SetCtrlInLargeTable(common, find_info.offset, new_h2, policy.slot_size); } ABSL_SWISSTABLE_ASSERT(old_capacity > policy.soo_capacity()); - DeallocBackingArrayImpl(alloc, old_capacity, old_ctrl, slot_size, slot_align, - has_infoz, old_blocked_element_count, policy.dealloc); + PrepareInsertCommon(common); ResetGrowthLeft(new_capacity, common.size(), common); if (ABSL_PREDICT_FALSE(has_infoz)) { - ReportGrowthToInfoz(common, infoz, new_hash, total_probe_length, + ReportGrowthToInfoz(common, old_ctrl, new_hash, total_probe_length, find_info.probe_length); } + // Deallocate after `ReportGrowthToInfoz`, because it reads infoz from + // old_ctrl. + DeallocBackingArrayImpl(alloc, old_capacity, old_ctrl, slot_size, slot_align, + has_infoz, old_blocked_element_count, policy.dealloc); return SlotAddress(new_slots, find_info.offset, policy.slot_size); } @@ -1819,11 +1880,14 @@ if (common.capacity() == 1) { if (common.empty()) { IncrementSmallSizeNonSoo(common, policy); + // Compute before `RecordInsertMissCold` to avoid reloading `control_`. + void* res = common.slot_array(/*capacity=*/1); + // Call NOINLINE function to move infoz instructions out of line. if (common.has_infoz()) { - common.infoz().RecordInsertMiss(get_hash(common.seed().seed()), - /*distance_from_desired=*/0); + ReportInsertMissToInfozAndComputeHash(common, /*probe_length=*/0, + get_hash); } - return common.slot_array(/*capacity=*/1); + return res; } else { return Grow1To3AndPrepareInsert(common, policy, get_hash); } @@ -1834,14 +1898,8 @@ constexpr size_t kNewCapacity = 1; common.set_capacity(kNewCapacity); - HashtablezInfoHandle infoz; - const bool should_sample = - policy.is_hashtablez_eligible && ShouldSampleNextTable(); - if (ABSL_PREDICT_FALSE(should_sample)) { - infoz = ForcedTrySample(policy.slot_size, policy.key_size, - policy.value_size, policy.soo_capacity()); - } - const bool has_infoz = infoz.IsSampled(); + const bool has_infoz = + policy.is_hashtablez_enabled && ShouldSampleNextTable(); void* alloc = policy.get_char_alloc(common); const auto [new_ctrl, new_slots] = @@ -1853,9 +1911,8 @@ PrepareInsertCommon(common); if (ABSL_PREDICT_FALSE(has_infoz)) { - common.generate_new_seed(/*has_infoz=*/true); - ReportSingleGroupTableGrowthToInfoz(common, infoz, - get_hash(common.seed().seed())); + ReportSingleElementTableGrowthToInfozAndForceSample(common, policy, + get_hash); } return new_slots; } @@ -1864,8 +1921,8 @@ // Called whenever the table needs to vacate empty slots either by removing // tombstones via rehash or growth to next capacity. -ABSL_ATTRIBUTE_NOINLINE -void* RehashOrGrowToNextCapacityAndPrepareInsert( +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_COLD void* +RehashOrGrowToNextCapacityAndPrepareInsert( CommonFields& common, const PolicyFunctions& __restrict policy, size_t new_hash) { ABSL_SWISSTABLE_ASSERT(!common.GetGrowthInfoLowerBound().HasNoDeleted()); @@ -1949,9 +2006,14 @@ FindInfo target = find_first_non_full(common, hash); PrepareInsertCommon(common); common.OverwriteControlAsFull(common.control()[target.offset]); + // Compute before `SetCtrl*` to avoid reloading `control_`. + void* res = + SlotAddress(common.slot_array(cap), target.offset, policy.slot_size); SetCtrlInLargeTable(common, target.offset, H2(hash), policy.slot_size); - common.infoz().RecordInsertMiss(hash, target.probe_length); - return SlotAddress(common.slot_array(cap), target.offset, policy.slot_size); + if (common.has_infoz()) { + ReportInsertMissToInfoz(common, target.probe_length, hash); + } + return res; } // Resizes empty non-allocated SOO table to NextCapacity(SooCapacity()), @@ -1971,11 +2033,15 @@ PrepareInsertCommon(common); common.OverwriteEmptyAsFull(); const size_t new_hash = get_hash(common.seed().seed()); + // Compute before `SetCtrl*` to avoid reloading `control_`. + void* res = SlotAddress(common.slot_array(kNewCapacity), SooSlotIndex(), + policy.slot_size); SetCtrlInSingleGroupTable(common, SooSlotIndex(), H2(new_hash), policy.slot_size); - common.infoz().RecordInsertMiss(new_hash, /*distance_from_desired=*/0); - return SlotAddress(common.slot_array(kNewCapacity), SooSlotIndex(), - policy.slot_size); + if (common.has_infoz()) { + ReportInsertMissToInfoz(common, /*probe_length=*/0, new_hash); + } + return res; } // Returns the number of elements to block for the given capacity and reserved @@ -2074,7 +2140,7 @@ const size_t slot_size = policy.slot_size; const size_t slot_align = policy.slot_align; HashtablezInfoHandle infoz = common.infoz(); - const bool has_infoz = infoz.IsSampled(); + const bool has_infoz = common.has_infoz(); void* alloc = policy.get_char_alloc(common); common.set_capacity(new_capacity); @@ -2185,7 +2251,7 @@ } if (common.size() <= policy.soo_capacity()) { // When the table is already sampled, we keep it sampled. - if (common.infoz().IsSampled()) { + if (common.has_infoz()) { static constexpr size_t kInitialSampledCapacity = NextCapacity(SooCapacity()); if (cap > kInitialSampledCapacity) { @@ -2193,7 +2259,7 @@ kInitialSampledCapacity); } // This asserts that we didn't lose sampling coverage in `resize`. - ABSL_SWISSTABLE_ASSERT(common.infoz().IsSampled()); + ABSL_SWISSTABLE_ASSERT(common.has_infoz()); return; } ABSL_SWISSTABLE_ASSERT(slot_size <= sizeof(HeapOrSoo)); @@ -2261,7 +2327,7 @@ : SingleSlotAddress</*kSooEnabled=*/false>(common), other_slot); - if (soo_enabled && policy.is_hashtablez_eligible && + if (soo_enabled && policy.is_hashtablez_enabled && ShouldSampleNextTable()) { GrowFullSooTableToNextCapacityForceSampling(common, policy); } @@ -2325,7 +2391,7 @@ const PolicyFunctions& __restrict policy, size_t hash, Group::NonIterableBitMaskType mask_empty, - FindInfo target_group) { + size_t target_group_offset) { ABSL_SWISSTABLE_ASSERT(!common.is_small()); // When there are no deleted slots in the table // and growth_left is positive, we can insert at the first @@ -2338,27 +2404,35 @@ common.OverwriteEmptyAsFull(); const size_t cap = common.capacity(); ABSL_ASSUME(cap > kMaxSmallCapacity); - target_group.offset += mask_empty.LowestBitSet(); - target_group.offset &= cap; - SetCtrl(common, target_group.offset, H2(hash), policy.slot_size); - common.infoz().RecordInsertMiss(hash, target_group.probe_length); - return SlotAddress(common.slot_array(cap), target_group.offset, - policy.slot_size); + target_group_offset += mask_empty.LowestBitSet(); + target_group_offset &= cap; + // Compute before `SetCtrl` to avoid reloading `control_`. + void* res = SlotAddress(common.slot_array(cap), target_group_offset, + policy.slot_size); + const bool hash_infoz = common.has_infoz(); + SetCtrl(common, target_group_offset, H2(hash), policy.slot_size); + // Call NOINLINE function to move infoz instructions out of line. + if (hash_infoz) { + ReportInsertMissToInfozAndComputeProbeLength(common, target_group_offset, + hash); + } + return res; } } // namespace void* PrepareInsertLarge(CommonFields& common, const PolicyFunctions& __restrict policy, size_t hash, Group::NonIterableBitMaskType mask_empty, - FindInfo target_group) { + size_t target_group_offset) { // NOLINTNEXTLINE(misc-static-assert) ABSL_SWISSTABLE_ASSERT(!SwisstableGenerationsEnabled()); - return PrepareInsertLargeImpl(common, policy, hash, mask_empty, target_group); + return PrepareInsertLargeImpl(common, policy, hash, mask_empty, + target_group_offset); } void* PrepareInsertLargeGenerationsEnabled( CommonFields& common, const PolicyFunctions& __restrict policy, size_t hash, - Group::NonIterableBitMaskType mask_empty, FindInfo target_group, + Group::NonIterableBitMaskType mask_empty, size_t target_group_offset, absl::FunctionRef<size_t(size_t)> recompute_hash) { // NOLINTNEXTLINE(misc-static-assert) ABSL_SWISSTABLE_ASSERT(SwisstableGenerationsEnabled()); @@ -2370,10 +2444,13 @@ // Move to a different heap allocation in order to detect bugs. ResizeAllocatedTableWithSeedChange(common, policy, cap); hash = recompute_hash(common.seed().seed()); + FindInfo target_group; std::tie(target_group, mask_empty) = find_first_non_full_group(common, hash); + target_group_offset = target_group.offset; } - return PrepareInsertLargeImpl(common, policy, hash, mask_empty, target_group); + return PrepareInsertLargeImpl(common, policy, hash, mask_empty, + target_group_offset); } namespace { @@ -2401,12 +2478,6 @@ } } // namespace -// Extern template instantiation for inline function. -template size_t TryFindNewIndexWithoutProbing(size_t h1, size_t old_index, - size_t old_capacity, - ctrl_t* new_ctrl, - size_t new_capacity); - // We need to instantiate ALL possible template combinations because we define // the function in the cc file. template void* GrowSooTableToNextCapacityAndPrepareInsert<0, false>( @@ -2471,6 +2542,28 @@ template void Destruct</*kSooEnabled=*/false>(CommonFields& c, const DtorPolicy& policy); +// Extern template instantiation for inline functions. +template size_t TryFindNewIndexWithoutProbing(size_t h1, size_t old_index, + size_t old_capacity, + ctrl_t* new_ctrl, + size_t new_capacity); +template void AssertIsFull(const ctrl_t* const& ctrl, const void* slot, + GenerationType generation, + const GenerationType* generation_ptr, + const char* operation); +template void AssertIsValidForComparison(const ctrl_t* const& ctrl, + const void* slot, + GenerationType generation, + const GenerationType* generation_ptr); +template bool AreItersFromSameContainer(const ctrl_t* const& ctrl_a, + const ctrl_t* const& ctrl_b, + const void* slot_a, const void* slot_b); +template void AssertSameContainer(const ctrl_t* const& ctrl_a, + const ctrl_t* const& ctrl_b, + const void* slot_a, const void* slot_b, + const GenerationType* generation_ptr_a, + const GenerationType* generation_ptr_b); + } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index ca0e83d..7d5802d 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h
@@ -266,6 +266,20 @@ return ++generation == SentinelEmptyGeneration() ? ++generation : generation; } +#ifdef NDEBUG +inline constexpr bool kIsDebug = false; +#else +inline constexpr bool kIsDebug = true; +#endif + +constexpr bool SwisstableDebugEnabled() { +#if defined(ABSL_SWISSTABLE_ENABLE_GENERATIONS) || ABSL_OPTION_HARDENED == 1 + return true; +#else + return kIsDebug; +#endif +} + #ifdef ABSL_SWISSTABLE_ENABLE_GENERATIONS constexpr bool SwisstableGenerationsEnabled() { return true; } constexpr size_t NumGenerationBytes() { return sizeof(GenerationType); } @@ -275,10 +289,11 @@ #endif constexpr bool SwisstableGenerationsOrDebugEnabled() { -#ifndef NDEBUG - return true; -#endif - return SwisstableGenerationsEnabled(); + if constexpr (kIsDebug) { + return true; + } else { + return SwisstableGenerationsEnabled(); + } } template <typename AllocType> @@ -484,47 +499,28 @@ return (~size_t{}) >> leading_zeros; } -// The mode we store capacity in the table. -enum HashtableCapacityStorageMode { - // Capacity stored as size_t as a full number. - kCapacityByValue, - // Capacity stored as uint8_t as log2, i.e. capacity = 2^capacity_ - 1. - kCapacityByLog, -}; - // The number of slots in the backing array. This is always 2^N-1 for an // integer N. // NOTE: this class exists to simplify experiments with different ways to store // capacity within size. -// NOTE: we tried experimenting with compressing the capacity and storing it -// together with size_: (a) using 6 bits to store the corresponding power (N in -// 2^N-1), and (b) storing 2^N as the most significant bit of size_ and storing -// size in the low bits. Both of these experiments were regressions, presumably -// because we need capacity to do find operations. -template <HashtableCapacityStorageMode StorageMode> -class HashtableCapacityImpl { - using IntType = - std::conditional_t<StorageMode == kCapacityByValue, size_t, uint8_t>; - +class HashtableCapacity { public: - static constexpr HashtableCapacityImpl CreateDestroyed() { - return HashtableCapacityImpl(kDestroyed); + static constexpr HashtableCapacity CreateDestroyed() { + return HashtableCapacity(kDestroyed); } - static constexpr HashtableCapacityImpl CreateReentrance() { - return HashtableCapacityImpl(kReentrance); + static constexpr HashtableCapacity CreateReentrance() { + return HashtableCapacity(kReentrance); } - static constexpr HashtableCapacityImpl CreateMovedFrom() { - return HashtableCapacityImpl(kMovedFrom); + static constexpr HashtableCapacity CreateMovedFrom() { + return HashtableCapacity(kMovedFrom); } - static constexpr HashtableCapacityImpl CreateSelfMovedFrom() { - return HashtableCapacityImpl(kSelfMovedFrom); + static constexpr HashtableCapacity CreateSelfMovedFrom() { + return HashtableCapacity(kSelfMovedFrom); } - explicit HashtableCapacityImpl(uninitialized_tag_t) {} - explicit constexpr HashtableCapacityImpl(size_t capacity) - : capacity_data_(static_cast<IntType>( - StorageMode == kCapacityByValue ? capacity - : TrailingZeros(capacity + 1))) { + explicit HashtableCapacity(uninitialized_tag_t) {} + explicit constexpr HashtableCapacity(size_t capacity) + : capacity_data_(static_cast<uint8_t>(TrailingZeros(capacity + 1))) { ABSL_SWISSTABLE_ASSERT(capacity == 0 || IsValidCapacity(capacity)); } @@ -532,12 +528,12 @@ // This is needed to use bitfield for capacity. // At least on Windows combination uint8_t and uint64_t bitfield in one struct // is not optimized by compiler. - static HashtableCapacityImpl FromRawData(uint64_t capacity) { - auto cap = HashtableCapacityImpl(uninitialized_tag_t{}); - cap.capacity_data_ = static_cast<IntType>(capacity); + static HashtableCapacity FromRawData(uint64_t capacity) { + auto cap = HashtableCapacity(uninitialized_tag_t{}); + cap.capacity_data_ = static_cast<uint8_t>(capacity); return cap; } - IntType ToRawData() const { return capacity_data_; } + uint8_t ToRawData() const { return capacity_data_; } constexpr bool IsValid() const { return capacity_data_ <= kAboveMaxValidCapacity; @@ -553,13 +549,11 @@ constexpr size_t capacity() const { ABSL_SWISSTABLE_ASSERT(IsValid()); - return StorageMode == kCapacityByValue ? capacity_data_ - : (size_t{1} << capacity_data_) - 1; + return (size_t{1} << capacity_data_) - 1; } constexpr bool is_small() const { - // Small tables have capacity 0 or 1. This expression is valid for both - // capacity storage modes. + // Small tables have capacity 0 or 1. // Comparing capacity_data_ directly leads to a better generated code. // One byte comparison is used before computing the capacity in order to // detect small tables faster for critical path. @@ -570,10 +564,8 @@ private: // We use these sentinel capacity values in debug mode to indicate different // classes of bugs. - enum InvalidCapacity : IntType { - kAboveMaxValidCapacity = StorageMode == kCapacityByValue - ? (std::numeric_limits<IntType>::max)() - 100 - : 64 - 10, + enum InvalidCapacity : uint8_t { + kAboveMaxValidCapacity = 64 - 10, kReentrance, kDestroyed, @@ -582,18 +574,15 @@ kSelfMovedFrom, }; - explicit constexpr HashtableCapacityImpl(InvalidCapacity capacity) + explicit constexpr HashtableCapacity(InvalidCapacity capacity) : capacity_data_(capacity) { ABSL_SWISSTABLE_ASSERT(capacity_data_ > kAboveMaxValidCapacity); } - // Capacity is stored as a value or as a log2 depending on `StorageMode`. - IntType capacity_data_; + // Capacity stored as uint8_t as log2, i.e. capacity = 2^capacity_data_ - 1. + uint8_t capacity_data_; }; -template <HashtableCapacityStorageMode StorageMode> -class HashtableInlineDataImpl; - // Returns next per-table seed. uint8_t NextHashTableSeed(); @@ -606,8 +595,7 @@ size_t seed() const { return seed_; } private: - template <HashtableCapacityStorageMode StorageModeOfData> - friend class HashtableInlineDataImpl; + friend class HashtableInlineData; explicit PerTableSeed(uint64_t seed) : seed_(static_cast<uint16_t>(seed)) {} @@ -750,8 +738,7 @@ // the table. // 3) kSeedBitCount bits for the seed. (For SOO tables, the lowest bit of the // seed is repurposed to track if sampling has been tried). -template <HashtableCapacityStorageMode StorageMode> -class HashtableInlineDataImpl { +class HashtableInlineData { // The number of bits in the seed. It is big enough to ensure // non-determinism of iteration order. We store the seed inside a uint64_t // together with size and other metadata. When absl::Hash is inlined, it can @@ -759,27 +746,21 @@ static constexpr size_t kSeedBitCount = 5; public: - static constexpr HashtableCapacityStorageMode kStorageMode = StorageMode; - using HashtableCapacity = HashtableCapacityImpl<StorageMode>; static constexpr size_t kGrowthInfoLowerBoundBitCount = 8; static constexpr size_t kBlockedElementBitCount = 3; static constexpr size_t kMaxBlockedElementCount = (uint64_t{1} << kBlockedElementBitCount) - 1; - static constexpr size_t kCapacityBitCount = - StorageMode == kCapacityByValue ? sizeof(HashtableCapacity) * 8 : 6; - static constexpr size_t kCapacityBitStoredInDataCount = - StorageMode == kCapacityByValue ? 0 : kCapacityBitCount; + static constexpr size_t kCapacityBitCount = 6; static constexpr size_t kSizeBitCount = 64 - (kBlockedElementBitCount + kSeedBitCount + kGrowthInfoLowerBoundBitCount + - /*has_infoz*/ 1 + kCapacityBitStoredInDataCount); + /*has_infoz*/ 1 + kCapacityBitCount); - explicit HashtableInlineDataImpl(uninitialized_tag_t) {} - explicit HashtableInlineDataImpl(HashtableCapacity capacity, - no_seed_empty_tag_t) + explicit HashtableInlineData(uninitialized_tag_t) {} + explicit HashtableInlineData(HashtableCapacity capacity, no_seed_empty_tag_t) : capacity_internal_(capacity.ToRawData()), data_(0) {} - HashtableInlineDataImpl(HashtableCapacity capacity, full_soo_tag_t, - bool has_tried_sampling) + HashtableInlineData(HashtableCapacity capacity, full_soo_tag_t, + bool has_tried_sampling) : capacity_internal_(capacity.ToRawData()), data_(kSizeOneNoMetadata | (has_tried_sampling ? kSooHasTriedSamplingMask : 0)) {} @@ -908,16 +889,16 @@ private: // Bit layout of `data_` and `capacity_internal_` from MSB to LSB: - // (47/41 bits) : size + // (41 bits) : size // (8 bits) : growth_info_lower_bound // (3 bits) : blocked_element_count // (1 bit) : has_infoz // (5 bits) : seed - // (6 bits) : capacity (only for kCapacityByLog) + // (6 bits) : capacity // We don't split these components of `data_` into separate bit field elements // because we get worse generated code that way. - static constexpr size_t kDataBitCount = 64 - kCapacityBitStoredInDataCount; + static constexpr size_t kDataBitCount = 64 - kCapacityBitCount; static constexpr size_t kSizeShift = kDataBitCount - kSizeBitCount; static constexpr uint64_t kSizeOneNoMetadata = uint64_t{1} << kSizeShift; static constexpr uint64_t kMetadataMask = kSizeOneNoMetadata - 1; @@ -942,10 +923,10 @@ static constexpr uint8_t kSampledSeed = (1 << kSeedBitCount) - 1; static constexpr uint64_t ToPublicSeed(uint64_t seed) { - // In kCapacityByLog mode, we shift public seed to the left to keep bits of - // the seed in the original place. It allows us to use single instruction to - // access the seed (e.g., `andl $0x7c0, %r8d`). - return seed << kCapacityBitStoredInDataCount; + // We shift public seed to the left to keep bits of the seed in the original + // place. It allows us to use single instruction to access the seed (e.g., + // `andl $0x7c0, %r8d`). + return seed << kCapacityBitCount; } void set_seed(uint8_t seed) { @@ -956,23 +937,8 @@ uint64_t data_ : kDataBitCount; }; -static_assert( - sizeof(HashtableInlineDataImpl<kCapacityByValue>::HashtableCapacity) == - sizeof(size_t)); -// NOTE: some platforms have this size to be equal to 12 for two reasons: -// 1) alignof(uint64_t) == 4. -// 2) sizeof(size_t) == sizeof(HashtableCapacityImpl<kCapacityByValue>) == 4. -static_assert(sizeof(HashtableInlineDataImpl<kCapacityByValue>) <= 16); -static_assert( - sizeof(HashtableInlineDataImpl<kCapacityByLog>::HashtableCapacity) == 1); -static_assert(sizeof(HashtableInlineDataImpl<kCapacityByLog>) == 8); - -#ifndef ABSL_SWISSTABLE_INTERNAL_ENABLE_CAPACITY_BY_VALUE -using HashtableInlineData = HashtableInlineDataImpl<kCapacityByLog>; -#else -using HashtableInlineData = HashtableInlineDataImpl<kCapacityByValue>; -#endif // ABSL_SWISSTABLE_INTERNAL_ENABLE_CAPACITY_BY_VALUE -using HashtableCapacity = HashtableInlineData::HashtableCapacity; +static_assert(sizeof(HashtableCapacity) == 1); +static_assert(sizeof(HashtableInlineData) == 8); // For large tables, we limit the number of blocked elements to maintain O(1) // average case lookup complexity. @@ -1368,7 +1334,10 @@ } inline_data_.generate_new_seed(); } - void set_no_seed_for_testing() { inline_data_.set_no_seed_for_testing(); } + void set_no_seed_for_testing() { + ABSL_SWISSTABLE_ASSERT(!has_infoz()); + inline_data_.set_no_seed_for_testing(); + } HashtableCapacity capacity_impl() const { HashtableCapacity cap = inline_data_.capacity(); @@ -1435,6 +1404,7 @@ inline_data_.set_has_infoz(); } + HashtablezInfoHandle infoz_from_control(ctrl_t* ctrl) const; HashtablezInfoHandle infoz_ptr() const; HashtablezInfoHandle infoz() { @@ -1499,22 +1469,21 @@ // calls during element constructor/destructor. template <typename F> void RunWithReentrancyGuard(F f) { -#ifdef NDEBUG - f(); - return; -#endif - const HashtableCapacity cap = maybe_invalid_capacity(); - set_capacity(HashtableCapacity::CreateReentrance()); - f(); - set_capacity(cap); + if constexpr (!kIsDebug) { + f(); + } else { + const HashtableCapacity cap = maybe_invalid_capacity(); + set_capacity(HashtableCapacity::CreateReentrance()); + f(); + set_capacity(cap); + } } // Asserts that the capacity is not a sentinel invalid value. void AssertNotDebugCapacity() const { - if (!SwisstableGenerationsOrDebugEnabled()) { - return; + if constexpr (SwisstableGenerationsOrDebugEnabled()) { + AssertNotDebugCapacityImpl(); } - AssertNotDebugCapacityImpl(); } private: @@ -1586,15 +1555,6 @@ return 0; } -constexpr bool SwisstableDebugEnabled() { -#if defined(ABSL_SWISSTABLE_ENABLE_GENERATIONS) || \ - ABSL_OPTION_HARDENED == 1 || !defined(NDEBUG) - return true; -#else - return false; -#endif -} - // Dereferences `ptr`. The function is named in order to provide a helpful error // message when users see crashing stack traces. Note that this function is not // guaranteed to crash when `ptr` is invalid if sanitizer mode is not enabled. @@ -1619,6 +1579,7 @@ // so that it's not UB if they're uninitialized as long as we don't read them // (when slot is null). +template <typename = void> inline void AssertIsFull(const ctrl_t* const& ctrl, const void* slot, GenerationType generation, const GenerationType* generation_ptr, @@ -1663,6 +1624,7 @@ } // Note that for comparisons, null/end iterators are valid. +template <typename = void> inline void AssertIsValidForComparison(const ctrl_t* const& ctrl, const void* slot, GenerationType generation, @@ -1698,6 +1660,7 @@ // If the two iterators come from the same container, then their pointers will // interleave such that ctrl_a <= ctrl_b < slot_a <= slot_b or vice/versa. +template <typename = void> inline bool AreItersFromSameContainer(const ctrl_t* const& ctrl_a, const ctrl_t* const& ctrl_b, const void* slot_a, const void* slot_b) { @@ -1723,6 +1686,7 @@ // Asserts that two iterators come from the same container. // Note: we take slots by reference so that it's not UB if they're uninitialized // as long as we don't read them (when ctrl is null). +template <typename = void> inline void AssertSameContainer(const ctrl_t* const& ctrl_a, const ctrl_t* const& ctrl_b, const void* slot_a, const void* slot_b, @@ -1819,17 +1783,19 @@ size_t offset(size_t i) const { return (offset_ + i) & capacity_; } void next() { - index_ += Width; - offset_ += index_; + offset_ += next_index_; offset_ &= capacity_; + next_index_ += Width; } // 0-based probe index, a multiple of `Width`. - size_t index() const { return index_; } + size_t index() const { return next_index_ - Width; } private: size_t capacity_; size_t offset_; - size_t index_ = 0; + // We keep track of the next index rather than the current index to get better + // generated code. See e.g. https://godbolt.org/z/6jvWd135G. + size_t next_index_ = Width; }; // Begins a probing operation on `common.control`, using `hash`. @@ -1897,15 +1863,6 @@ return kProbedElementIndexSentinel; } -// Extern template for inline function keeps possibility of inlining. -// When compiler decided to not inline, no symbols will be added to the -// corresponding translation unit. -extern template size_t TryFindNewIndexWithoutProbing(size_t h1, - size_t old_index, - size_t old_capacity, - ctrl_t* new_ctrl, - size_t new_capacity); - // The HashtablezInfoHandle is stored before the control bytes. // NOTE: The growth_info is also stored before the backing array, but it doesn't // have alignment requirements. For small tables it is 1 byte, for larger tables @@ -1962,7 +1919,7 @@ uint32_t slot_size; uint16_t slot_align; bool soo_enabled; - bool is_hashtablez_eligible; + bool is_hashtablez_enabled; // Returns the pointer to the hash function stored in the set. void* (*hash_fn)(CommonFields& common); @@ -2255,13 +2212,13 @@ // at least one empty slot. void* PrepareInsertLarge(CommonFields& common, const PolicyFunctions& policy, size_t hash, Group::NonIterableBitMaskType mask_empty, - FindInfo target_group); + size_t target_group_offset); // Same as above, but with generations enabled, we may end up changing the seed, // which means we need to be able to recompute the hash. void* PrepareInsertLargeGenerationsEnabled( CommonFields& common, const PolicyFunctions& policy, size_t hash, - Group::NonIterableBitMaskType mask_empty, FindInfo target_group, + Group::NonIterableBitMaskType mask_empty, size_t target_group_offset, absl::FunctionRef<size_t(size_t)> recompute_hash); template <typename Policy, typename Hash, typename Eq, typename Alloc> @@ -2349,6 +2306,11 @@ constexpr static bool kIsAbslHash = std::is_same_v<hasher, absl::Hash<key_type>> || + (!HasAbslContainerHash<key_type>::value && + std::is_same_v<hasher, typename absl::container_internal::HashEq< + key_type>::Hash>) || + // Note: this is useful (in addition to the above case) for cases of + // `absl_container_hash = DefaultHashContainerHash<absl::string_view>`. std::is_same_v<hasher, absl::container_internal::StringHash> || // TODO(b/384509507): resolve `no header providing // "absl::hash_internal::TransparentHash" is directly included`. @@ -2358,11 +2320,10 @@ // (a) in such cases, the seed is xor'ed with the hash value rather than being // used as a seed for the hash function, (b) the seed has low bits that are // all 0s, and (c) we require random iteration order for small tables. - // In ToPublicSeed we shift the seed by kCapacityBitStoredInDataCount as - // performance optimization for default hashers. For non-default hashers, we - // shift it back. + // In ToPublicSeed we shift the seed by kCapacityBitCount as an optimization + // for default hashers. For non-default hashers, we shift it back. constexpr static size_t kSeedShift = - kIsAbslHash ? 0 : HashtableInlineData::kCapacityBitStoredInDataCount; + kIsAbslHash ? 0 : HashtableInlineData::kCapacityBitCount; constexpr static bool SooEnabled() { return PolicyTraits::soo_enabled() && @@ -2370,6 +2331,14 @@ alignof(slot_type) <= alignof(HeapOrSoo); } + constexpr static bool HashtablezForTableEnabled() { +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) + return ShouldSampleHashtablezInfoForAlloc<CharAlloc>(); +#else + return false; +#endif + } + constexpr static size_t DefaultCapacity() { return SooEnabled() ? SooCapacity() : 0; } @@ -2499,12 +2468,14 @@ } friend bool operator==(const iterator& a, const iterator& b) { - AssertIsValidForComparison(a.ctrl_, a.slot_, a.generation(), - a.generation_ptr()); - AssertIsValidForComparison(b.ctrl_, b.slot_, b.generation(), - b.generation_ptr()); - AssertSameContainer(a.ctrl_, b.ctrl_, a.slot_, b.slot_, - a.generation_ptr(), b.generation_ptr()); + if constexpr (SwisstableDebugEnabled()) { + AssertIsValidForComparison(a.ctrl_, a.slot_, a.generation(), + a.generation_ptr()); + AssertIsValidForComparison(b.ctrl_, b.slot_, b.generation(), + b.generation_ptr()); + AssertSameContainer(a.ctrl_, b.ctrl_, a.slot_, b.slot_, + a.generation_ptr(), b.generation_ptr()); + } return a.unchecked_equals(b); } friend bool operator!=(const iterator& a, const iterator& b) { @@ -3507,7 +3478,7 @@ // construction when the size can fit in SOO capacity. bool should_sample_soo() { ABSL_SWISSTABLE_ASSERT(is_soo()); - if constexpr (!ShouldSampleHashtablezInfoForAlloc<CharAlloc>()) { + if constexpr (!HashtablezForTableEnabled()) { return false; } if (common().soo_has_tried_sampling()) { @@ -3709,6 +3680,10 @@ // Aliasing can't happen here because allocs would compare equal above. assert(this != &that); destructor_impl(); + // destructor_impl() leaves common() pointing at the freed backing array + // (or at the stale small-table state), so reset it to the default state + // before re-inserting elements. + common() = CommonFields::CreateDefault<SooEnabled()>(); // We can't take over that's memory so we need to move each element. // While moving elements, this should have that's hash/eq so copy hash/eq // before moving elements. @@ -3794,12 +3769,12 @@ void* slot = SwisstableGenerationsEnabled() ? PrepareInsertLargeGenerationsEnabled( common(), GetPolicyFunctions(), hash, mask_empty, - FindInfo{target_group_offset, seq.index()}, + target_group_offset, HashKey<hasher, K, kIsAbslHash, kSeedShift>{ hash_ref(), key}) : PrepareInsertLarge( common(), GetPolicyFunctions(), hash, mask_empty, - FindInfo{target_group_offset, seq.index()}); + target_group_offset); return {to_slot(slot), true}; } seq.next(); @@ -3827,40 +3802,38 @@ // meaning that `eq(k1, k2)` implies `hash(k1)==hash(k2)`. template <class K> void AssertHashEqConsistent(const K& key) { -#ifdef NDEBUG - return; -#endif - // If the hash/eq functors are known to be consistent, then skip validation. - if (std::is_same_v<hasher, absl::container_internal::StringHash> && - std::is_same_v<key_equal, absl::container_internal::StringEq>) { - return; - } - if (std::is_scalar_v<key_type> && + // If the hash/eq functors are known to be consistent, then skip + // validation. + constexpr bool kIsStringAbslHashEq = + std::is_same_v<hasher, absl::container_internal::StringHash> && + std::is_same_v<key_equal, absl::container_internal::StringEq>; + constexpr bool kIsScalarAbslHashEq = + std::is_scalar_v<key_type> && std::is_same_v<hasher, absl::Hash<key_type>> && - std::is_same_v<key_equal, std::equal_to<key_type>>) { - return; + std::is_same_v<key_equal, std::equal_to<key_type>>; + if constexpr (kIsDebug && !kIsStringAbslHashEq && !kIsScalarAbslHashEq) { + if (empty()) return; + + const size_t hash_of_arg = hash_of(key); + const auto assert_consistent = [&](const ctrl_t*, void* slot) { + const bool is_key_equal = equal_to(key, to_slot(slot)); + if (!is_key_equal) return; + + [[maybe_unused]] const bool is_hash_equal = + hash_of_arg == hash_of(to_slot(slot)); + assert((!is_key_equal || is_hash_equal) && + "eq(k1, k2) must imply that hash(k1) == hash(k2). " + "hash/eq functors are inconsistent."); + }; + + if (is_small()) { + assert_consistent(/*unused*/ nullptr, single_slot()); + return; + } + // We only do validation for small tables so that it's constant time. + if (capacity() > 16) return; + IterateOverFullSlots(common(), sizeof(slot_type), assert_consistent); } - if (empty()) return; - - const size_t hash_of_arg = hash_of(key); - const auto assert_consistent = [&](const ctrl_t*, void* slot) { - const bool is_key_equal = equal_to(key, to_slot(slot)); - if (!is_key_equal) return; - - [[maybe_unused]] const bool is_hash_equal = - hash_of_arg == hash_of(to_slot(slot)); - assert((!is_key_equal || is_hash_equal) && - "eq(k1, k2) must imply that hash(k1) == hash(k2). " - "hash/eq functors are inconsistent."); - }; - - if (is_small()) { - assert_consistent(/*unused*/ nullptr, single_slot()); - return; - } - // We only do validation for small tables so that it's constant time. - if (capacity() > 16) return; - IterateOverFullSlots(common(), sizeof(slot_type), assert_consistent); } // Attempts to find `key` in the table; if it isn't found, returns an iterator @@ -4108,7 +4081,7 @@ static_cast<uint32_t>(sizeof(value_type)), static_cast<uint32_t>(sizeof(slot_type)), static_cast<uint16_t>(alignof(slot_type)), SooEnabled(), - ShouldSampleHashtablezInfoForAlloc<CharAlloc>(), + HashtablezForTableEnabled(), // TODO(b/328722020): try to type erase // for standard layout and alignof(Hash) <= alignof(CommonFields). std::is_empty_v<hasher> ? &GetRefForEmptyClass @@ -4310,6 +4283,31 @@ extern template void Destruct</*kSooEnabled=*/false>(CommonFields& c, const DtorPolicy& policy); +// Extern template for inline function keeps possibility of inlining. +// When compiler decided to not inline, no symbols will be added to the +// corresponding translation unit. +extern template size_t TryFindNewIndexWithoutProbing(size_t h1, + size_t old_index, + size_t old_capacity, + ctrl_t* new_ctrl, + size_t new_capacity); +extern template void AssertIsFull(const ctrl_t* const& ctrl, const void* slot, + GenerationType generation, + const GenerationType* generation_ptr, + const char* operation); +extern template void AssertIsValidForComparison( + const ctrl_t* const& ctrl, const void* slot, GenerationType generation, + const GenerationType* generation_ptr); +extern template bool AreItersFromSameContainer(const ctrl_t* const& ctrl_a, + const ctrl_t* const& ctrl_b, + const void* slot_a, + const void* slot_b); +extern template void AssertSameContainer( + const ctrl_t* const& ctrl_a, const ctrl_t* const& ctrl_b, + const void* slot_a, const void* slot_b, + const GenerationType* generation_ptr_a, + const GenerationType* generation_ptr_b); + } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/container/internal/raw_hash_set_allocator_test.cc b/absl/container/internal/raw_hash_set_allocator_test.cc index eaee003..d14f9c6 100644 --- a/absl/container/internal/raw_hash_set_allocator_test.cc +++ b/absl/container/internal/raw_hash_set_allocator_test.cc
@@ -439,6 +439,17 @@ EXPECT_EQ(0, it->num_copies()); } +TEST_F(NoPropagateOnMove, MoveAssignmentWithDifferentAllocToNonEmptyTable) { + t1.insert(0); + Table u(0, a2); + for (int32_t i = 1; i <= 16; ++i) u.insert(i); + u = std::move(t1); + EXPECT_EQ(a2, u.get_allocator()); + EXPECT_EQ(1, u.size()); + EXPECT_NE(u.find(0), u.end()); + EXPECT_EQ(u.find(1), u.end()); +} + TEST_F(PropagateOnAll, Swap) { t1.insert(0); Table u(0, a2);
diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index cc252b4..3b799c3 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc
@@ -735,22 +735,8 @@ EXPECT_THAT(offsets, ElementsAre(0, 16, 48, 96, 32, 112, 80, 64)); } -template <typename T> -class HashtableDataTest : public ::testing::Test {}; - -using StorageModes = ::testing::Types< - std::integral_constant<HashtableCapacityStorageMode, - HashtableCapacityStorageMode::kCapacityByValue>, - std::integral_constant<HashtableCapacityStorageMode, - HashtableCapacityStorageMode::kCapacityByLog>>; - -TYPED_TEST_SUITE(HashtableDataTest, StorageModes); - -TYPED_TEST(HashtableDataTest, HashtableCapacity) { - constexpr HashtableCapacityStorageMode kMode = TypeParam::value; - using Capacity = HashtableCapacityImpl<kMode>; - - Capacity cap0(0); +TEST(HashtableDataTest, HashtableCapacity) { + HashtableCapacity cap0(0); EXPECT_TRUE(cap0.IsValid()); EXPECT_EQ(cap0.capacity(), 0); EXPECT_FALSE(cap0.IsDestroyed()); @@ -759,51 +745,46 @@ EXPECT_FALSE(cap0.IsSelfMovedFrom()); for (size_t i = 0, cap = 0; i < 20; ++i, cap = NextCapacity(cap)) { - Capacity capacity(cap); + HashtableCapacity capacity(cap); ASSERT_TRUE(capacity.IsValid()); ASSERT_EQ(capacity.capacity(), cap); } - auto destroyed = Capacity::CreateDestroyed(); + auto destroyed = HashtableCapacity::CreateDestroyed(); EXPECT_FALSE(destroyed.IsValid()); EXPECT_TRUE(destroyed.IsDestroyed()); - auto reentrance = Capacity::CreateReentrance(); + auto reentrance = HashtableCapacity::CreateReentrance(); EXPECT_FALSE(reentrance.IsValid()); EXPECT_TRUE(reentrance.IsReentrance()); - auto moved_from = Capacity::CreateMovedFrom(); + auto moved_from = HashtableCapacity::CreateMovedFrom(); EXPECT_FALSE(moved_from.IsValid()); EXPECT_TRUE(moved_from.IsMovedFrom()); EXPECT_FALSE(moved_from.IsSelfMovedFrom()); - auto self_moved_from = Capacity::CreateSelfMovedFrom(); + auto self_moved_from = HashtableCapacity::CreateSelfMovedFrom(); EXPECT_FALSE(self_moved_from.IsValid()); EXPECT_TRUE(self_moved_from.IsSelfMovedFrom()); EXPECT_TRUE(self_moved_from.IsMovedFrom()); } -TYPED_TEST(HashtableDataTest, RawData) { - constexpr HashtableCapacityStorageMode kMode = TypeParam::value; - using Capacity = HashtableCapacityImpl<kMode>; - +TEST(HashtableDataTest, RawData) { for (size_t i = 0, cap = 0; i < 20; ++i, cap = NextCapacity(cap)) { - Capacity orig_capacity(cap); - Capacity capacity = Capacity::FromRawData(orig_capacity.ToRawData()); + HashtableCapacity orig_capacity(cap); + HashtableCapacity capacity = + HashtableCapacity::FromRawData(orig_capacity.ToRawData()); ASSERT_TRUE(capacity.IsValid()); ASSERT_EQ(capacity.capacity(), cap); } - auto orig_reentrance = Capacity::CreateReentrance(); - Capacity reentrance = Capacity::FromRawData(orig_reentrance.ToRawData()); + auto orig_reentrance = HashtableCapacity::CreateReentrance(); + HashtableCapacity reentrance = + HashtableCapacity::FromRawData(orig_reentrance.ToRawData()); EXPECT_TRUE(reentrance.IsReentrance()); } -TYPED_TEST(HashtableDataTest, HashtableInlineDataCapacity) { - constexpr HashtableCapacityStorageMode kMode = TypeParam::value; - using InlineData = HashtableInlineDataImpl<kMode>; - using Capacity = HashtableCapacityImpl<kMode>; - - InlineData data(Capacity(0), no_seed_empty_tag_t{}); +TEST(HashtableDataTest, HashtableInlineDataCapacity) { + HashtableInlineData data(HashtableCapacity(0), no_seed_empty_tag_t{}); EXPECT_EQ(data.capacity().capacity(), 0); EXPECT_EQ(data.size(), 0); EXPECT_TRUE(data.empty()); @@ -813,23 +794,19 @@ ASSERT_EQ(data.capacity().capacity(), cap); } - // Test overload from `Capacity` object. + // Test overload from `HashtableCapacity` object. for (size_t i = 0, cap = 0; i < 20; ++i, cap = NextCapacity(cap)) { - data.set_capacity(Capacity(cap)); + data.set_capacity(HashtableCapacity(cap)); ASSERT_EQ(data.capacity().capacity(), cap); } - auto reentrance = Capacity::CreateReentrance(); + auto reentrance = HashtableCapacity::CreateReentrance(); data.set_capacity(reentrance); EXPECT_TRUE(data.capacity().IsReentrance()); } -TYPED_TEST(HashtableDataTest, HashtableInlineDataSize) { - constexpr HashtableCapacityStorageMode kMode = TypeParam::value; - using InlineData = HashtableInlineDataImpl<kMode>; - using Capacity = HashtableCapacityImpl<kMode>; - - InlineData data(Capacity(0), no_seed_empty_tag_t{}); +TEST(HashtableDataTest, HashtableInlineDataSize) { + HashtableInlineData data(HashtableCapacity(0), no_seed_empty_tag_t{}); EXPECT_EQ(data.size(), 0); data.increment_size(); EXPECT_EQ(data.size(), 1); @@ -853,18 +830,14 @@ EXPECT_TRUE(data.has_infoz()); } -TYPED_TEST(HashtableDataTest, BlockedElementCount) { - constexpr HashtableCapacityStorageMode kMode = TypeParam::value; - using InlineData = HashtableInlineDataImpl<kMode>; - using Capacity = HashtableCapacityImpl<kMode>; - +TEST(HashtableDataTest, BlockedElementCount) { { - InlineData data(Capacity(0), no_seed_empty_tag_t{}); + HashtableInlineData data(HashtableCapacity(0), no_seed_empty_tag_t{}); EXPECT_EQ(data.blocked_element_count(), 0); } - for (size_t i = 0; i <= InlineData::kMaxBlockedElementCount; ++i) { - InlineData data(Capacity(0), no_seed_empty_tag_t{}); + for (size_t i = 0; i <= HashtableInlineData::kMaxBlockedElementCount; ++i) { + HashtableInlineData data(HashtableCapacity(0), no_seed_empty_tag_t{}); data.init_blocked_element_count(i); EXPECT_EQ(data.blocked_element_count(), i); data.set_blocked_element_count_to_zero(); @@ -872,15 +845,12 @@ } } -TYPED_TEST(HashtableDataTest, MaxStorableSize) { - constexpr HashtableCapacityStorageMode kMode = TypeParam::value; - using InlineData = HashtableInlineDataImpl<kMode>; - using Capacity = HashtableCapacityImpl<kMode>; - - InlineData data(Capacity(0), no_seed_empty_tag_t{}); +TEST(HashtableDataTest, MaxStorableSize) { + HashtableInlineData data(HashtableCapacity(0), no_seed_empty_tag_t{}); constexpr uint64_t kMaxSize = - sizeof(size_t) == 4 ? ~uint32_t{} - : (uint64_t{1} << InlineData::kSizeBitCount) - 1; + sizeof(size_t) == 4 + ? ~uint32_t{} + : (uint64_t{1} << HashtableInlineData::kSizeBitCount) - 1; data.init_blocked_element_count(3); data.increment_size(kMaxSize); EXPECT_EQ(data.size(), kMaxSize); @@ -889,12 +859,8 @@ EXPECT_EQ(data.blocked_element_count(), 3); } -TYPED_TEST(HashtableDataTest, HashtableInlineDataMetadata) { - constexpr HashtableCapacityStorageMode kMode = TypeParam::value; - using InlineData = HashtableInlineDataImpl<kMode>; - using Capacity = HashtableCapacityImpl<kMode>; - - InlineData data(Capacity(0), no_seed_empty_tag_t{}); +TEST(HashtableDataTest, HashtableInlineDataMetadata) { + HashtableInlineData data(HashtableCapacity(0), no_seed_empty_tag_t{}); // SOO sampling (test this first before seed messes with bit 0) EXPECT_FALSE(data.soo_has_tried_sampling()); @@ -913,35 +879,29 @@ EXPECT_TRUE(data.has_infoz()); } -TYPED_TEST(HashtableDataTest, HashtableInlineDataFullSooConstructor) { - constexpr HashtableCapacityStorageMode kMode = TypeParam::value; - using InlineData = HashtableInlineDataImpl<kMode>; - using Capacity = HashtableCapacityImpl<kMode>; - +TEST(HashtableDataTest, HashtableInlineDataFullSooConstructor) { { - InlineData data_soo(Capacity(1), full_soo_tag_t{}, - /*has_tried_sampling=*/true); + HashtableInlineData data_soo(HashtableCapacity(1), full_soo_tag_t{}, + /*has_tried_sampling=*/true); EXPECT_EQ(data_soo.capacity().capacity(), 1); EXPECT_EQ(data_soo.size(), 1); EXPECT_TRUE(data_soo.soo_has_tried_sampling()); } { - InlineData data_soo(Capacity(1), full_soo_tag_t{}, - /*has_tried_sampling=*/false); + HashtableInlineData data_soo(HashtableCapacity(1), full_soo_tag_t{}, + /*has_tried_sampling=*/false); EXPECT_EQ(data_soo.capacity().capacity(), 1); EXPECT_EQ(data_soo.size(), 1); EXPECT_FALSE(data_soo.soo_has_tried_sampling()); } } -TYPED_TEST(HashtableDataTest, GenerateNewSeedDoesntChangeSize) { - constexpr HashtableCapacityStorageMode kMode = TypeParam::value; - using InlineData = HashtableInlineDataImpl<kMode>; - using Capacity = HashtableCapacityImpl<kMode>; +TEST(HashtableDataTest, GenerateNewSeedDoesntChangeSize) { size_t size = 1; do { - InlineData inline_data(Capacity(15), no_seed_empty_tag_t{}); + HashtableInlineData inline_data(HashtableCapacity(15), + no_seed_empty_tag_t{}); inline_data.increment_size(size); EXPECT_EQ(inline_data.size(), size); inline_data.generate_new_seed(); @@ -1362,18 +1322,10 @@ static_assert(std::is_empty_v<std::equal_to<absl::string_view>>); static_assert(std::is_empty_v<std::allocator<int>>); - struct MockTableByValue { - size_t capacity; + struct MockTable { uint64_t size; void* ctrl; }; - struct MockTableByLog { - uint64_t size; - void* ctrl; - }; - using MockTable = - std::conditional_t<HashtableInlineData::kStorageMode == kCapacityByValue, - MockTableByValue, MockTableByLog>; struct StatelessHash { size_t operator()(absl::string_view) const { return 0; } }; @@ -1547,6 +1499,7 @@ if (SwisstableGenerationsEnabled()) { GTEST_SKIP() << "Generations enabled, so rehash happening earlier."; } + DisableSampling(); constexpr int64_t kCoef = 17; int retries = 0; for (size_t capacity = 31; capacity < 256; @@ -1629,6 +1582,7 @@ << "Note that reservation doesn't prevent rehashing since we " "are erasing one element."; } + DisableSampling(); constexpr int64_t kCoef = 17; constexpr size_t kCapacity = 31; constexpr size_t kReserveSize = @@ -4089,6 +4043,23 @@ } } +TEST(RawHashSamplerTest, NonSooTableInsertToEmpty) { + std::vector<const HashtablezInfo*> infos = + SampleNonSooMutation([](NonSooIntTable& t) { t.insert(1); }); + + for (const HashtablezInfo* info : infos) { + ASSERT_EQ(info->inline_element_size, + sizeof(typename NonSooIntTable::value_type)); + ASSERT_EQ(info->soo_capacity, 0); + ASSERT_EQ(info->capacity, 1); + ASSERT_EQ(info->size, 1); + ASSERT_EQ(info->max_reserve, 0); + ASSERT_EQ(info->num_erases, 0); + ASSERT_EQ(info->max_probe_length, 0); + ASSERT_EQ(info->total_probe_length, 0); + } +} + // Verifies that repeated insertions and erasures on an SOO table do not cause // the sampling decision to be evaluated multiple times, preventing // oversampling. @@ -5264,6 +5235,7 @@ NextCapacity(ProbedItem8Bytes::kMaxNewCapacity); #endif + DisableSampling(); absl::flat_hash_set<UnalignedInt32, ZeroHash> t(21); CommonFields& common = RawHashSetTestOnlyAccess::GetCommon(t); // Set 0 seed so that H1 is always 0.
diff --git a/absl/debugging/BUILD.bazel b/absl/debugging/BUILD.bazel index b36967a..24e5280 100644 --- a/absl/debugging/BUILD.bazel +++ b/absl/debugging/BUILD.bazel
@@ -198,6 +198,7 @@ "@rules_cc//cc/compiler:msvc-cl": [], "@rules_cc//cc/compiler:clang-cl": [], "@rules_cc//cc/compiler:emscripten": [], + "@platforms//os:wasi": [], "//conditions:default": ["-pthread"], }) + ABSL_DEFAULT_LINKOPTS, deps = [
diff --git a/absl/debugging/internal/elf_mem_image.h b/absl/debugging/internal/elf_mem_image.h index 4d31a47..f24aa26 100644 --- a/absl/debugging/internal/elf_mem_image.h +++ b/absl/debugging/internal/elf_mem_image.h
@@ -42,10 +42,20 @@ #ifdef ABSL_HAVE_ELF_MEM_IMAGE +#if __has_include(<link.h>) #include <link.h> // for ElfW +#else +#include <elf.h> +#endif -#if defined(__FreeBSD__) && !defined(ElfW) -#define ElfW(x) __ElfN(x) +#ifndef ElfW +#if defined(__FreeBSD__) +#define ElfW(type) __ElfN(type) +#elif __SIZEOF_POINTER__ == 8 +#define ElfW(type) Elf64_##type +#else +#define ElfW(type) Elf32_##type +#endif #endif namespace absl {
diff --git a/absl/debugging/internal/symbolize.h b/absl/debugging/internal/symbolize.h index fd65544..22bed14 100644 --- a/absl/debugging/internal/symbolize.h +++ b/absl/debugging/internal/symbolize.h
@@ -30,8 +30,7 @@ #ifdef ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE #error ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE cannot be directly set -#elif defined(__ELF__) && defined(__GLIBC__) && !defined(__asmjs__) \ - && !defined(__wasm__) +#elif defined(__ELF__) && defined(__linux__) && !defined(__ANDROID__) #define ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE 1 #include <elf.h>
diff --git a/absl/debugging/symbolize_elf.inc b/absl/debugging/symbolize_elf.inc index e5e171c..3fdf2ff 100644 --- a/absl/debugging/symbolize_elf.inc +++ b/absl/debugging/symbolize_elf.inc
@@ -80,10 +80,6 @@ #include "absl/debugging/internal/vdso_support.h" #include "absl/strings/string_view.h" -#if defined(__FreeBSD__) && !defined(ElfW) -#define ElfW(x) __ElfN(x) -#endif - namespace absl { ABSL_NAMESPACE_BEGIN @@ -386,11 +382,7 @@ } // namespace static size_t SymbolizerSize() { -#if defined(__wasm__) || defined(__asmjs__) - auto pagesize = static_cast<size_t>(getpagesize()); -#else auto pagesize = static_cast<size_t>(sysconf(_SC_PAGESIZE)); -#endif return ((sizeof(Symbolizer) - 1) / pagesize + 1) * pagesize; }
diff --git a/absl/debugging/symbolize_test.cc b/absl/debugging/symbolize_test.cc index f3b15e1..05de541 100644 --- a/absl/debugging/symbolize_test.cc +++ b/absl/debugging/symbolize_test.cc
@@ -424,7 +424,10 @@ static int in_data_section = 1; TEST(Symbolize, ForEachSection) { - int fd = TEMP_FAILURE_RETRY(open("/proc/self/exe", O_RDONLY)); + int fd; + do { + fd = open("/proc/self/exe", O_RDONLY); + } while (fd == -1 && errno == EINTR); ASSERT_NE(fd, -1); std::vector<std::string> sections;
diff --git a/absl/flags/BUILD.bazel b/absl/flags/BUILD.bazel index 59cae99..59771c7 100644 --- a/absl/flags/BUILD.bazel +++ b/absl/flags/BUILD.bazel
@@ -208,6 +208,7 @@ "//absl/base:core_headers", "//absl/base:fast_type_id", "//absl/base:no_destructor", + "//absl/base:nullability", "//absl/container:flat_hash_map", "//absl/strings", "//absl/strings:string_view",
diff --git a/absl/flags/CMakeLists.txt b/absl/flags/CMakeLists.txt index a96e294..0e40085 100644 --- a/absl/flags/CMakeLists.txt +++ b/absl/flags/CMakeLists.txt
@@ -174,6 +174,7 @@ absl::flags_private_handle_accessor absl::flat_hash_map absl::no_destructor + absl::nullability absl::string_view absl::strings absl::synchronization
diff --git a/absl/flags/flag.h b/absl/flags/flag.h index 2a95049..2e9a9e6 100644 --- a/absl/flags/flag.h +++ b/absl/flags/flag.h
@@ -298,10 +298,20 @@ // `default_value` and `explanation` are unused. // TODO(rogeeff): replace RETIRED_FLAGS with FLAGS once forward declarations of // retired flags are cleaned up. +#if ABSL_FLAGS_STRIP_NAMES +// When flag names are stripped, retired flags are not registered. Define an +// unused type alias instead of an object so that no runtime constructor or +// storage is emitted, the trailing semicolon is consumed, and `type` (along +// with this header) remains referenced to avoid unused-type/include warnings. +#define ABSL_RETIRED_FLAG(type, name, default_value, explanation) \ + using RETIRED_FLAGS_##name [[maybe_unused]] = \ + ::absl::flags_internal::RetiredFlag<type> +#else #define ABSL_RETIRED_FLAG(type, name, default_value, explanation) \ static absl::flags_internal::RetiredFlag<type> RETIRED_FLAGS_##name; \ [[maybe_unused]] static const auto RETIRED_FLAGS_REG_##name = \ (RETIRED_FLAGS_##name.Retire(#name), \ ::absl::flags_internal::FlagRegistrarEmpty{}) +#endif #endif // ABSL_FLAGS_FLAG_H_
diff --git a/absl/flags/flag_test.cc b/absl/flags/flag_test.cc index 7cbce88..458f1b1 100644 --- a/absl/flags/flag_test.cc +++ b/absl/flags/flag_test.cc
@@ -51,6 +51,7 @@ namespace { namespace flags = absl::flags_internal; +constexpr bool kStrippedFlagNames = ABSL_FLAGS_STRIP_NAMES != 0; std::string TestHelpMsg() { return "dynamic help"; } #if defined(_MSC_VER) && !defined(__clang__) @@ -230,9 +231,9 @@ namespace { TEST_F(FlagTest, TestFlagDeclaration) { -#if ABSL_FLAGS_STRIP_NAMES - GTEST_SKIP() << "This test requires flag names to be present"; -#endif + if constexpr (kStrippedFlagNames) { + GTEST_SKIP() << "This test requires flag names to be present"; + } // test that we can access flag objects. EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_01).Name(), "test_flag_01"); @@ -266,7 +267,7 @@ } // namespace -#if ABSL_FLAGS_STRIP_NAMES +#if ABSL_FLAGS_STRIP_NAMES && defined(__cpp_consteval) // The intent of this helper struct and an expression below is to make sure that // in the configuration where ABSL_FLAGS_STRIP_NAMES=1 registrar construction // (in cases of no Tail calls like OnUpdate) is constexpr and thus can and @@ -280,7 +281,7 @@ }; ABSL_FLAG(int, test_registrar_const_init, 0, "") + VerifyConsteval(); -#endif +#endif // ABSL_FLAGS_STRIP_NAMES && defined(__cpp_consteval) // -------------------------------------------------------------------- @@ -303,9 +304,9 @@ namespace { TEST_F(FlagTest, TestFlagDefinition) { -#if ABSL_FLAGS_STRIP_NAMES - GTEST_SKIP() << "This test requires flag names to be present"; -#endif + if constexpr (kStrippedFlagNames) { + GTEST_SKIP() << "This test requires flag names to be present"; + } absl::string_view expected_file_name = "absl/flags/flag_test.cc"; EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_01).Name(), @@ -626,9 +627,9 @@ // -------------------------------------------------------------------- TEST_F(FlagTest, TestGetViaReflection) { -#if ABSL_FLAGS_STRIP_NAMES - GTEST_SKIP() << "This test requires flag names to be present"; -#endif + if constexpr (kStrippedFlagNames) { + GTEST_SKIP() << "This test requires flag names to be present"; + } auto* handle = absl::FindCommandLineFlag("test_flag_01"); EXPECT_EQ(*handle->TryGet<bool>(), true); handle = absl::FindCommandLineFlag("test_flag_02"); @@ -663,9 +664,9 @@ // -------------------------------------------------------------------- TEST_F(FlagTest, ConcurrentSetAndGet) { -#if ABSL_FLAGS_STRIP_NAMES - GTEST_SKIP() << "This test requires flag names to be present"; -#endif + if constexpr (kStrippedFlagNames) { + GTEST_SKIP() << "This test requires flag names to be present"; + } static constexpr int kNumThreads = 8; // Two arbitrary durations. One thread will concurrently flip the flag // between these two values, while the other threads read it and verify @@ -816,9 +817,9 @@ using FlagDeathTest = FlagTest; TEST_F(FlagDeathTest, TestTypeMismatchValidations) { -#if ABSL_FLAGS_STRIP_NAMES - GTEST_SKIP() << "This test requires flag names to be present"; -#endif + if constexpr (kStrippedFlagNames) { + GTEST_SKIP() << "This test requires flag names to be present"; + } #if !defined(NDEBUG) EXPECT_DEATH_IF_SUPPORTED( static_cast<void>(absl::GetFlag(FLAGS_mistyped_int_flag)), @@ -950,14 +951,23 @@ TEST_F(FlagTest, TestRetiredFlagRegistration) { auto* handle = absl::FindCommandLineFlag("old_bool_flag"); - EXPECT_TRUE(handle->IsOfType<bool>()); - EXPECT_TRUE(handle->IsRetired()); - handle = absl::FindCommandLineFlag("old_int_flag"); - EXPECT_TRUE(handle->IsOfType<int>()); - EXPECT_TRUE(handle->IsRetired()); - handle = absl::FindCommandLineFlag("old_str_flag"); - EXPECT_TRUE(handle->IsOfType<std::string>()); - EXPECT_TRUE(handle->IsRetired()); + if constexpr (kStrippedFlagNames) { + EXPECT_EQ(handle, nullptr); + EXPECT_EQ(absl::FindCommandLineFlag("old_int_flag"), nullptr); + EXPECT_EQ(absl::FindCommandLineFlag("old_str_flag"), nullptr); + } else { + ASSERT_NE(handle, nullptr); + EXPECT_TRUE(handle->IsOfType<bool>()); + EXPECT_TRUE(handle->IsRetired()); + handle = absl::FindCommandLineFlag("old_int_flag"); + ASSERT_NE(handle, nullptr); + EXPECT_TRUE(handle->IsOfType<int>()); + EXPECT_TRUE(handle->IsRetired()); + handle = absl::FindCommandLineFlag("old_str_flag"); + ASSERT_NE(handle, nullptr); + EXPECT_TRUE(handle->IsOfType<std::string>()); + EXPECT_TRUE(handle->IsRetired()); + } } } // namespace @@ -1069,7 +1079,7 @@ absl::GetFlag(FLAGS_test_flag_ntc_udt1); EXPECT_EQ(value.c, 'A'); }, - 0); + 0, kStrippedFlagNames ? 1 : 0); TestExpectedLeaks<1>( [&] { @@ -1090,6 +1100,9 @@ } TEST_F(FlagTest, TestNonTriviallyCopyableParseSet) { + if constexpr (kStrippedFlagNames) { + GTEST_SKIP() << "This test requires flag names to be present"; + } TestExpectedLeaks<2>( [&] { const char* in_argv[] = {"testbin", "--test_flag_ntc_udt2=A"}; @@ -1115,7 +1128,7 @@ absl::SetFlag(&FLAGS_test_flag_ntc_udt3, value); EXPECT_EQ(value.c, 'B'); }, - 0); + 0, kStrippedFlagNames ? 1 : 0); } // One new instance created during initialization and stored in the flag. @@ -1125,6 +1138,9 @@ false); TEST_F(FlagTest, TestNonTriviallyCopyableGetBeforeMainParseGet) { + if constexpr (kStrippedFlagNames) { + GTEST_SKIP() << "This test requires flag names to be present"; + } TestExpectedLeaks<4>( [&] { const char* in_argv[] = {"testbin", "--test_flag_ntc_udt4=C"}; @@ -1153,6 +1169,9 @@ false); TEST_F(FlagTest, TestNonTriviallyCopyableSetParseGet) { + if constexpr (kStrippedFlagNames) { + GTEST_SKIP() << "This test requires flag names to be present"; + } TestExpectedLeaks<5>( [&] { const char* in_argv[] = {"testbin", "--test_flag_ntc_udt5=C"};
diff --git a/absl/flags/reflection.cc b/absl/flags/reflection.cc index 0729ba0..e7eeaa3 100644 --- a/absl/flags/reflection.cc +++ b/absl/flags/reflection.cc
@@ -28,6 +28,7 @@ #include "absl/base/config.h" #include "absl/base/no_destructor.h" +#include "absl/base/nullability.h" #include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" #include "absl/flags/commandlineflag.h" @@ -339,7 +340,8 @@ } // namespace flags_internal -FlagSaver::FlagSaver() : impl_(new flags_internal::FlagSaverImpl) { +FlagSaver::FlagSaver() + : impl_(std::make_unique<flags_internal::FlagSaverImpl>()) { impl_->SaveFromRegistry(); } @@ -347,12 +349,11 @@ if (!impl_) return; std::move(*impl_).RestoreToRegistry(); - delete impl_; } // -------------------------------------------------------------------- -CommandLineFlag* FindCommandLineFlag(absl::string_view name) { +CommandLineFlag* absl_nullable FindCommandLineFlag(absl::string_view name) { if (name.empty()) return nullptr; flags_internal::FlagRegistry& registry = flags_internal::FlagRegistry::GlobalRegistry(); @@ -361,7 +362,8 @@ // -------------------------------------------------------------------- -absl::flat_hash_map<absl::string_view, absl::CommandLineFlag*> GetAllFlags() { +absl::flat_hash_map<absl::string_view, absl::CommandLineFlag* absl_nonnull> +GetAllFlags() { absl::flat_hash_map<absl::string_view, absl::CommandLineFlag*> res; flags_internal::ForEachFlag([&](CommandLineFlag& flag) { if (!flag.IsRetired()) res.insert({flag.Name(), &flag});
diff --git a/absl/flags/reflection.h b/absl/flags/reflection.h index 54b3e9a..aeaf728 100644 --- a/absl/flags/reflection.h +++ b/absl/flags/reflection.h
@@ -23,9 +23,11 @@ #ifndef ABSL_FLAGS_REFLECTION_H_ #define ABSL_FLAGS_REFLECTION_H_ +#include <memory> #include <string> #include "absl/base/config.h" +#include "absl/base/nullability.h" #include "absl/container/flat_hash_map.h" #include "absl/flags/commandlineflag.h" #include "absl/flags/internal/commandlineflag.h" @@ -42,11 +44,13 @@ // Returns the reflection handle of an Abseil flag of the specified name, or // `nullptr` if not found. This function will emit a warning if the name of a // 'retired' flag is specified. -absl::CommandLineFlag* FindCommandLineFlag(absl::string_view name); +absl::CommandLineFlag* absl_nullable FindCommandLineFlag( + absl::string_view name); // Returns current state of the Flags registry in a form of mapping from flag // name to a flag reflection handle. -absl::flat_hash_map<absl::string_view, absl::CommandLineFlag*> GetAllFlags(); +absl::flat_hash_map<absl::string_view, absl::CommandLineFlag* absl_nonnull> +GetAllFlags(); //------------------------------------------------------------------------------ // FlagSaver @@ -77,10 +81,11 @@ ~FlagSaver(); FlagSaver(const FlagSaver&) = delete; - void operator=(const FlagSaver&) = delete; + FlagSaver& operator=(const FlagSaver&) = delete; private: - flags_internal::FlagSaverImpl* impl_; + // Null if no flags restoration is needed. + absl_nullable std::unique_ptr<flags_internal::FlagSaverImpl> impl_; }; //-----------------------------------------------------------------------------
diff --git a/absl/hash/hash.h b/absl/hash/hash.h index bfccf27..126c96f 100644 --- a/absl/hash/hash.h +++ b/absl/hash/hash.h
@@ -326,7 +326,7 @@ // return H::combine(std::move(h), n.first, n.last); // } // friend bool operator==(const Name& a, const Name& b); -// friend bool operator==(const Name& a, const NameView& b); +// operator NameView() const; // // using absl_container_hash = absl::TransparentHash<Name, NameView>; // };
diff --git a/absl/hash/hash_test.cc b/absl/hash/hash_test.cc index 0e98bbf..28cb108 100644 --- a/absl/hash/hash_test.cc +++ b/absl/hash/hash_test.cc
@@ -1358,36 +1358,32 @@ } struct NameView { - absl::string_view name; - absl::string_view lang; + absl::string_view first; + absl::string_view last; friend bool operator==(const NameView& lhs, const NameView& rhs) { - return lhs.name == rhs.name && lhs.lang == rhs.lang; + return lhs.first == rhs.first && lhs.last == rhs.last; } template <typename H> friend H AbslHashValue(H h, const NameView& name) { - return H::combine(std::move(h), name.name, name.lang); + return H::combine(std::move(h), name.first, name.last); } }; struct Name { - std::string name; - std::string lang; + std::string first; + std::string last; friend bool operator==(const Name& lhs, const Name& rhs) { - return lhs.name == rhs.name && lhs.lang == rhs.lang; + return lhs.first == rhs.first && lhs.last == rhs.last; } - friend bool operator==(const NameView& lhs, const Name& rhs) { - return lhs.name == rhs.name && lhs.lang == rhs.lang; - } - friend bool operator==(const Name& lhs, const NameView& rhs) { - return lhs.name == rhs.name && lhs.lang == rhs.lang; - } + + operator NameView() const { return NameView{first, last}; } // NOLINT template <typename H> friend H AbslHashValue(H h, const Name& name) { - return H::combine(std::move(h), name.name, name.lang); + return H::combine(std::move(h), name.first, name.last); } using absl_container_hash = absl::TransparentHash<NameView, Name>; @@ -1412,29 +1408,29 @@ EXPECT_FALSE((std::is_convertible_v<NameHash, absl::Hash<Name>>)); EXPECT_FALSE((std::is_convertible_v<NameHash, absl::Hash<NameView>>)); - EXPECT_EQ(NameHash{}(Name{"foo", "en"}), NameHash{}(NameView{"foo", "en"})); + EXPECT_EQ(NameHash{}(Name{"John", "Doe"}), + NameHash{}(NameView{"John", "Doe"})); - EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( - std::make_tuple(Name{"foo", "en"}, NameView{"foo", "en"}, - Name{"bar", "en"}, NameView{"bar", "en"}, - Name{"foo", "de"}, NameView{"foo", "de"}, - Name{"bar", "de"}, NameView{"bar", "de"}))); + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + Name{"John", "Doe"}, NameView{"John", "Doe"}, Name{"Jack", "Doe"}, + NameView{"Jack", "Doe"}, Name{"John", "Smith"}, NameView{"John", "Smith"}, + Name{"Jack", "Smith"}, NameView{"Jack", "Smith"}))); absl::flat_hash_set<Name, NameHash, std::equal_to<>> set; - set.insert(Name{"foo", "en"}); - EXPECT_TRUE(set.contains(NameView{"foo", "en"})); - EXPECT_TRUE(set.contains(Name{"foo", "en"})); + set.insert(Name{"John", "Doe"}); + EXPECT_TRUE(set.contains(NameView{"John", "Doe"})); + EXPECT_TRUE(set.contains(Name{"John", "Doe"})); std::unordered_set<Name, NameHash, std::equal_to<>> std_set; - std_set.insert(Name{"foo", "en"}); - EXPECT_TRUE(std_set.find(Name{"foo", "en"}) != std_set.end()); + std_set.insert(Name{"John", "Doe"}); + EXPECT_TRUE(std_set.find(Name{"John", "Doe"}) != std_set.end()); } TEST(HashTest, TransparentHashDefaultLookUp) { absl::flat_hash_set<Name> set; - set.insert(Name{"foo", "en"}); - EXPECT_TRUE(set.contains(NameView{"foo", "en"})); - EXPECT_TRUE(set.contains(Name{"foo", "en"})); + set.insert(Name{"John", "Doe"}); + EXPECT_TRUE(set.contains(NameView{"John", "Doe"})); + EXPECT_TRUE(set.contains(Name{"John", "Doe"})); } struct MyString {
diff --git a/absl/hash/internal/hash.h b/absl/hash/internal/hash.h index 590bf7f..025e57c 100644 --- a/absl/hash/internal/hash.h +++ b/absl/hash/internal/hash.h
@@ -335,8 +335,9 @@ }; }; -// `is_uniquely_represented<T>` is a trait class that indicates whether `T` -// is uniquely represented. +// `is_uniquely_represented` is like `std::has_unique_object_representations`, +// but with a few differences for legacy reasons that predate cl/989071457, such +// as disallowing pointers or Booleans. // // A type is "uniquely represented" if two equal values of that type are // guaranteed to have the same bytes in their underlying storage. In other @@ -367,24 +368,11 @@ // The Enable parameter is meaningless; it is provided as a convenience, // to support certain SFINAE techniques when defining specializations. template <typename T, typename Enable = void> -struct is_uniquely_represented : std::false_type {}; - -// unsigned char is a synonym for "byte", so it is guaranteed to be -// uniquely represented. -template <> -struct is_uniquely_represented<unsigned char> : std::true_type {}; - -// is_uniquely_represented for non-standard integral types -// -// Integral types other than bool should be uniquely represented on any -// platform that this will plausibly be ported to. -template <typename Integral> -struct is_uniquely_represented<Integral, - std::enable_if_t<std::is_integral_v<Integral>>> - : std::true_type {}; - -template <> -struct is_uniquely_represented<bool> : std::false_type {}; +struct is_uniquely_represented + : std::conditional_t<(std::is_integral_v<T> && !std::is_same_v<bool, T>) || + std::is_enum_v<T>, + std::has_unique_object_representations<T>, + std::false_type> {}; #ifdef ABSL_HAVE_INTRINSIC_INT128 // Specialize the trait for GNU extension types. @@ -395,7 +383,11 @@ #endif // ABSL_HAVE_INTRINSIC_INT128 template <typename T> -struct FitsIn64Bits : std::bool_constant<sizeof(T) <= 8> {}; +using FitsIn64Bits = + std::bool_constant<sizeof(T) <= sizeof(uint64_t) && + // Check that sizeof(T) is a power of 2, since unaligned + // operations require it. + (sizeof(T) & (sizeof(T) - 1)) == 0>; struct CombineRaw { template <typename H> @@ -1332,6 +1324,26 @@ #define ABSL_HASH_INTERNAL_SUPPORT_LEGACY_HASH_ 0 #endif +template <typename T, typename... Args> +using InvokeResultOrVoid = + typename std::conditional_t<std::is_invocable_v<T, Args...>, + std::invoke_result<T, Args...>, + std::enable_if<true>>::type; + +template <typename State, typename T, typename = void> +struct AbslHashValueResultOrVoid { + using type = void; +}; + +template <typename State, typename T> +struct AbslHashValueResultOrVoid< + State, T, + std::void_t<decltype(AbslHashValue(std::declval<State>(), + std::declval<T&>()))>> { + using type = + decltype(AbslHashValue(std::declval<State>(), std::declval<T&>())); +}; + // Type trait to select the appropriate hash implementation to use. // HashSelect::type<T> will give the proper hash implementation, to be invoked // as: @@ -1341,13 +1353,6 @@ // `false`. struct HashSelect { private: - struct WeaklyMixedIntegerProbe { - template <typename H> - static H Invoke(H state, WeaklyMixedInteger value) { - return hash_internal::hash_weakly_mixed_integer(std::move(state), value); - } - }; - struct State : HashStateBase<State> { static State combine_contiguous(State hash_state, const unsigned char*, size_t); @@ -1357,19 +1362,23 @@ WeaklyMixedInteger value); }; + struct WeaklyMixedIntegerProbe { + template <typename H> + static H Invoke(H state, WeaklyMixedInteger value) { + return hash_internal::hash_weakly_mixed_integer(std::move(state), value); + } + }; + struct UniquelyRepresentedProbe { template <typename H, typename T> - static auto Invoke(H state, const T& value) - -> std::enable_if_t<is_uniquely_represented<T>::value, H> { + static H Invoke(H state, const T& value) { return hash_internal::hash_bytes(std::move(state), value); } }; struct HashValueProbe { template <typename H, typename T> - static auto Invoke(H state, const T& value) -> std::enable_if_t< - std::is_same_v<H, decltype(AbslHashValue(std::move(state), value))>, - H> { + static H Invoke(H state, const T& value) { return AbslHashValue(std::move(state), value); } }; @@ -1377,11 +1386,7 @@ struct LegacyHashProbe { #if ABSL_HASH_INTERNAL_SUPPORT_LEGACY_HASH_ template <typename H, typename T> - static auto Invoke(H state, const T& value) -> std::enable_if_t< - std::is_convertible_v< - decltype(ABSL_INTERNAL_LEGACY_HASH_NAMESPACE::hash<T>()(value)), - size_t>, - H> { + static H Invoke(H state, const T& value) { return hash_internal::hash_bytes( std::move(state), ABSL_INTERNAL_LEGACY_HASH_NAMESPACE::hash<T>{}(value)); @@ -1391,41 +1396,39 @@ struct StdHashProbe { template <typename H, typename T> - static auto Invoke(H state, const T& value) - -> std::enable_if_t<type_traits_internal::IsHashable<T>::value, H> { + static H Invoke(H state, const T& value) { return hash_internal::hash_bytes(std::move(state), std::hash<T>{}(value)); } }; - template <typename Hash, typename T> - struct Probe : Hash { - private: - template <typename H, typename = decltype(H::Invoke( - std::declval<State>(), std::declval<const T&>()))> - static std::true_type Test(int); - template <typename U> - static std::false_type Test(char); - - public: - static constexpr bool value = decltype(Test<Hash>(0))::value; - }; - public: - // Probe each implementation in order. - // disjunction provides short circuiting wrt instantiation. template <typename T> - using Apply = std::disjunction< // - Probe<WeaklyMixedIntegerProbe, T>, // - Probe<UniquelyRepresentedProbe, T>, // - Probe<HashValueProbe, T>, // - Probe<LegacyHashProbe, T>, // - Probe<StdHashProbe, T>, // - std::false_type>; + static auto Apply() { + if constexpr (std::is_same_v<T, WeaklyMixedInteger>) { + return WeaklyMixedIntegerProbe(); + } else if constexpr (is_uniquely_represented<T>::value) { + return UniquelyRepresentedProbe(); + } else if constexpr (std::is_same_v<State, + typename AbslHashValueResultOrVoid< + State, const T&>::type>) { + return HashValueProbe(); +#if ABSL_HASH_INTERNAL_SUPPORT_LEGACY_HASH_ + } else if constexpr (std::is_convertible_v< + InvokeResultOrVoid< + ABSL_INTERNAL_LEGACY_HASH_NAMESPACE::hash<T>, + const T&>, + size_t>) { + return LegacyHashProbe(); +#endif + } else if constexpr (type_traits_internal::IsHashable<T>::value) { + return StdHashProbe(); + } + } }; template <typename T> -struct is_hashable : std::bool_constant<HashSelect::template Apply<T>::value> { -}; +struct is_hashable + : std::is_class<decltype(HashSelect::Apply<absl::remove_cvref_t<T>>())> {}; class ABSL_DLL MixingHashState : public HashStateBase<MixingHashState> { template <typename T> @@ -1461,8 +1464,12 @@ // The result should be the same as running the whole algorithm, but faster. template <typename T, std::enable_if_t<IntegralFastPath<T>::value, int> = 0> static size_t hash_with_seed(T value, size_t seed) { - return static_cast<size_t>( - CombineRawImpl(seed, static_cast<std::make_unsigned_t<T>>(value))); + return static_cast<size_t>(CombineRawImpl( + seed, + static_cast<typename std::conditional_t< + std::is_signed_v<T>, std::make_unsigned<T>, + // Special case for `bool`, which `make_unsigned` does not support. + type_identity<T>>::type>(value))); } template <typename T, std::enable_if_t<!IntegralFastPath<T>::value, int> = 0> @@ -1646,7 +1653,7 @@ template <typename H> template <typename T, typename... Ts> H HashStateBase<H>::combine(H state, const T& value, const Ts&... values) { - return H::combine(hash_internal::HashSelect::template Apply<T>::Invoke( + return H::combine(decltype(hash_internal::HashSelect::Apply<T>())::Invoke( std::move(state), value), values...); }
diff --git a/absl/log/internal/check_op.h b/absl/log/internal/check_op.h index 66050f0..ce2c76f 100644 --- a/absl/log/internal/check_op.h +++ b/absl/log/internal/check_op.h
@@ -485,8 +485,9 @@ // type. #define ABSL_LOG_INTERNAL_CHECK_OP_IMPL(name, op) \ template <typename T1, typename T2> \ - inline constexpr const char* absl_nullable name##Impl( \ - const T1& v1, const T2& v2, const char* absl_nonnull exprtext) { \ + ABSL_ATTRIBUTE_ALWAYS_INLINE inline constexpr const char* absl_nullable \ + name##Impl(const T1& v1, const T2& v2, \ + const char* absl_nonnull exprtext) { \ using U1 = CheckOpStreamType<T1, T2>; \ using U2 = CheckOpStreamType<T2, T1>; \ return ABSL_PREDICT_TRUE(v1 op v2) \ @@ -494,8 +495,8 @@ : ABSL_LOG_INTERNAL_CHECK_OP_IMPL_RESULT(U1, U2, U1(v1), \ U2(v2), exprtext); \ } \ - inline constexpr const char* absl_nullable name##Impl( \ - int v1, int v2, const char* absl_nonnull exprtext) { \ + ABSL_ATTRIBUTE_ALWAYS_INLINE inline constexpr const char* absl_nullable \ + name##Impl(int v1, int v2, const char* absl_nonnull exprtext) { \ return name##Impl<int, int>(v1, v2, exprtext); \ }
diff --git a/absl/profiling/internal/sample_recorder.h b/absl/profiling/internal/sample_recorder.h index 0d14027..8663d6c 100644 --- a/absl/profiling/internal/sample_recorder.h +++ b/absl/profiling/internal/sample_recorder.h
@@ -169,23 +169,29 @@ template <typename T> template <typename... Targs> T* SampleRecorder<T>::PopDead(Targs... args) { - absl::ReleasableMutexLock graveyard_lock(graveyard_.init_mu); + T* sample; + { + absl::MutexLock graveyard_lock(graveyard_.init_mu); - // The list is circular, so eventually it collapses down to - // graveyard_.dead == &graveyard_ - // when it is empty. - T* sample = graveyard_.dead; - if (sample == &graveyard_) return nullptr; + // The list is circular, so eventually it collapses down to + // graveyard_.dead == &graveyard_ + // when it is empty. + sample = graveyard_.dead; + if (sample == &graveyard_) return nullptr; + graveyard_.dead = ABSL_TS_UNCHECKED_READ(sample->dead); + // Release the global graveyard lock early, before the potentially slow + // preparation. + } + // `sample` is detached from the graveyard and will not be used in PopDead. + // SampleRecorder<T>::Iterate will acquire per sample lock, so there will be + // no data race either. absl::MutexLock sample_lock(sample->init_mu); - graveyard_.dead = sample->dead; - // Release the global graveyard lock early, before the potentially slow - // preparation. - graveyard_lock.Release(); - // Prepare the sample while still holding the per-sample lock. - // `Iterate` will wait for the lock to be released. - sample->dead = nullptr; + // Note: consider PrepareForSampling out of the lock. + // Currently many clients have PrepareForSampling marked as + // ABSL_EXCLUSIVE_LOCKS_REQUIRED. sample->PrepareForSampling(std::forward<Targs>(args)...); + sample->dead = nullptr; return sample; }
diff --git a/absl/random/internal/BUILD.bazel b/absl/random/internal/BUILD.bazel index efd013e..d1bf943 100644 --- a/absl/random/internal/BUILD.bazel +++ b/absl/random/internal/BUILD.bazel
@@ -189,6 +189,7 @@ "@rules_cc//cc/compiler:msvc-cl": [], "@rules_cc//cc/compiler:clang-cl": [], "@rules_cc//cc/compiler:emscripten": [], + "@platforms//os:wasi": [], "//conditions:default": ["-pthread"], }) + ABSL_DEFAULT_LINKOPTS, deps = [
diff --git a/absl/strings/internal/append_and_overwrite_test.cc b/absl/strings/internal/append_and_overwrite_test.cc index aa9c7a1..c3a7b4f 100644 --- a/absl/strings/internal/append_and_overwrite_test.cc +++ b/absl/strings/internal/append_and_overwrite_test.cc
@@ -49,7 +49,7 @@ 'b'); EXPECT_EQ(s, expected); - EXPECT_EQ(s.c_str()[s.size()], '\0'); + EXPECT_EQ(s[s.size()], '\0'); } // clang-format off
diff --git a/absl/synchronization/BUILD.bazel b/absl/synchronization/BUILD.bazel index c63ba86..977af61 100644 --- a/absl/synchronization/BUILD.bazel +++ b/absl/synchronization/BUILD.bazel
@@ -127,6 +127,7 @@ "@rules_cc//cc/compiler:msvc-cl": [], "@rules_cc//cc/compiler:clang-cl": [], "@rules_cc//cc/compiler:emscripten": [], + "@platforms//os:wasi": [], "//conditions:default": ["-pthread"], }) + ABSL_DEFAULT_LINKOPTS, visibility = ["//visibility:public"],
diff --git a/absl/time/duration.cc b/absl/time/duration.cc index 961fc61..160e253 100644 --- a/absl/time/duration.cc +++ b/absl/time/duration.cc
@@ -64,6 +64,7 @@ #include "absl/base/attributes.h" #include "absl/base/casts.h" #include "absl/base/config.h" +#include "absl/base/optimization.h" #include "absl/numeric/int128.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" @@ -417,9 +418,15 @@ Duration& Duration::operator+=(Duration rhs) { if (time_internal::IsInfiniteDuration(*this)) return *this; if (time_internal::IsInfiniteDuration(rhs)) return *this = rhs; + // Once we know the operands are finite, we can assume the low (subsecond) + // part of the representation is less than kTicksPerSecond. + ABSL_ASSUME(rep_lo_ < kTicksPerSecond); + ABSL_ASSUME(rhs.rep_lo_ < kTicksPerSecond); const int64_t orig_rep_hi = rep_hi_.Get(); rep_hi_ = DecodeTwosComp(EncodeTwosComp(rep_hi_.Get()) + EncodeTwosComp(rhs.rep_hi_.Get())); + // The ABSL_ASSUMEs above prevent the compiler from emitting unreachable carry + // logic for the next line. if (rep_lo_ >= kTicksPerSecond - rhs.rep_lo_) { rep_hi_ = DecodeTwosComp(EncodeTwosComp(rep_hi_.Get()) + 1); rep_lo_ -= kTicksPerSecond;