diff --git a/CMake/AbseilDll.cmake b/CMake/AbseilDll.cmake index 0ebeebb..96fb5c6 100644 --- a/CMake/AbseilDll.cmake +++ b/CMake/AbseilDll.cmake
@@ -215,10 +215,14 @@ "numeric/int128.h" "numeric/internal/bits.h" "numeric/internal/representation.h" + "profiling/hashtable.cc" + "profiling/hashtable.h" "profiling/internal/exponential_biased.cc" "profiling/internal/exponential_biased.h" "profiling/internal/periodic_sampler.cc" "profiling/internal/periodic_sampler.h" + "profiling/internal/profile_builder.cc" + "profiling/internal/profile_builder.h" "profiling/internal/sample_recorder.h" "random/bernoulli_distribution.h" "random/beta_distribution.h"
diff --git a/absl/container/internal/hashtablez_sampler.cc b/absl/container/internal/hashtablez_sampler.cc index c0fce87..965476a 100644 --- a/absl/container/internal/hashtablez_sampler.cc +++ b/absl/container/internal/hashtablez_sampler.cc
@@ -94,8 +94,9 @@ // The inliner makes hardcoded skip_count difficult (especially when combined // with LTO). We use the ability to exclude stacks by regex when encoding // instead. - depth = absl::GetStackTrace(stack, HashtablezInfo::kMaxStackDepth, - /* skip_count= */ 0); + depth = static_cast<unsigned>( + absl::GetStackTrace(stack, HashtablezInfo::kMaxStackDepth, + /* skip_count= */ 0)); inline_element_size = inline_element_size_value; key_size = key_size_value; value_size = value_size_value;
diff --git a/absl/container/internal/hashtablez_sampler.h b/absl/container/internal/hashtablez_sampler.h index 305dc85..55ce7ed 100644 --- a/absl/container/internal/hashtablez_sampler.h +++ b/absl/container/internal/hashtablez_sampler.h
@@ -97,7 +97,7 @@ // the lock. static constexpr int kMaxStackDepth = 64; absl::Time create_time; - int32_t depth; + uint32_t depth; // The SOO capacity for this table in elements (not bytes). Note that sampled // tables are never SOO because we need to store the infoz handle on the heap. // Tables that would be SOO if not sampled should have: soo_capacity > 0 &&
diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index bd03ac1..9f4ceff 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc
@@ -45,15 +45,9 @@ // uninitialized because reading this memory is a bug. ABSL_DLL ctrl_t kDefaultIterControl; -// We need one full byte followed by a sentinel byte for iterator::operator++ to -// work. We have a full group after kSentinel to be safe (in case operator++ is -// changed to read a full group). -ABSL_CONST_INIT ABSL_DLL const ctrl_t kSooControl[17] = { - ZeroCtrlT(), ctrl_t::kSentinel, ZeroCtrlT(), ctrl_t::kEmpty, - ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, - ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, - ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, - ctrl_t::kEmpty}; +// We need one full byte followed by a sentinel byte for iterator::operator++. +ABSL_CONST_INIT ABSL_DLL const ctrl_t kSooControl[2] = {ZeroCtrlT(), + ctrl_t::kSentinel}; namespace { @@ -521,16 +515,23 @@ } // namespace -void EraseMetaOnly(CommonFields& c, const ctrl_t* ctrl, size_t slot_size) { +void EraseMetaOnlySmall(CommonFields& c, bool soo_enabled, size_t slot_size) { + ABSL_SWISSTABLE_ASSERT(c.is_small()); + if (soo_enabled) { + c.set_empty_soo(); + return; + } + c.decrement_size(); + c.infoz().RecordErase(); + SanitizerPoisonMemoryRegion(c.slot_array(), slot_size); +} + +void EraseMetaOnlyLarge(CommonFields& c, const ctrl_t* ctrl, size_t slot_size) { + ABSL_SWISSTABLE_ASSERT(!c.is_small()); ABSL_SWISSTABLE_ASSERT(IsFull(*ctrl) && "erasing a dangling iterator"); c.decrement_size(); c.infoz().RecordErase(); - if (c.is_small()) { - SanitizerPoisonMemoryRegion(c.slot_array(), slot_size); - return; - } - size_t index = static_cast<size_t>(ctrl - c.control()); if (WasNeverFull(c, index)) {
diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 1c8e654..5fe3367 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h
@@ -368,17 +368,6 @@ std::declval<Ts>()...))>, Policy, Hash, Eq, Ts...> : std::true_type {}; -// TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. -template <class T> -constexpr bool IsNoThrowSwappable(std::true_type = {} /* is_swappable */) { - using std::swap; - return noexcept(swap(std::declval<T&>(), std::declval<T&>())); -} -template <class T> -constexpr bool IsNoThrowSwappable(std::false_type /* is_swappable */) { - return false; -} - ABSL_DLL extern ctrl_t kDefaultIterControl; // We use these sentinel capacity values in debug mode to indicate different @@ -400,7 +389,7 @@ // For use in SOO iterators. // TODO(b/289225379): we could potentially get rid of this by adding an is_soo // bit in iterators. This would add branches but reduce cache misses. -ABSL_DLL extern const ctrl_t kSooControl[17]; +ABSL_DLL extern const ctrl_t kSooControl[2]; // Returns a pointer to a full byte followed by a sentinel byte. inline ctrl_t* SooControl() { @@ -1794,8 +1783,9 @@ void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy, void* alloc, bool reuse, bool soo_enabled); -// Type-erased version of raw_hash_set::erase_meta_only. -void EraseMetaOnly(CommonFields& c, const ctrl_t* ctrl, size_t slot_size); +// Type-erased versions of raw_hash_set::erase_meta_only_{small,large}. +void EraseMetaOnlySmall(CommonFields& c, bool soo_enabled, size_t slot_size); +void EraseMetaOnlyLarge(CommonFields& c, const ctrl_t* ctrl, size_t slot_size); // For trivially relocatable types we use memcpy directly. This allows us to // share the same function body for raw_hash_set instantiations that have the @@ -2062,20 +2052,8 @@ // `slot_` until they reach one. void skip_empty_or_deleted() { while (IsEmptyOrDeleted(*ctrl_)) { - auto mask = GroupFullEmptyOrDeleted{ctrl_}.MaskFullOrSentinel(); - // Generally it is possible to compute `shift` branchless. - // This branch is useful to: - // 1. Avoid checking `IsEmptyOrDeleted` after the shift for the most - // common dense table case. - // 2. Avoid the cost of `LowestBitSet` for extremely sparse tables. - if (ABSL_PREDICT_TRUE(mask)) { - auto shift = mask.LowestBitSet(); - ctrl_ += shift; - slot_ += shift; - return; - } - ctrl_ += Group::kWidth; - slot_ += Group::kWidth; + ++ctrl_; + ++slot_; } } @@ -2333,7 +2311,7 @@ } raw_hash_set& operator=(raw_hash_set&& that) noexcept( - absl::allocator_traits<allocator_type>::is_always_equal::value && + AllocTraits::is_always_equal::value && std::is_nothrow_move_assignable<hasher>::value && std::is_nothrow_move_assignable<key_equal>::value) { // TODO(sbenza): We should only use the operations from the noexcept clause @@ -2703,7 +2681,7 @@ if (first == last) return last.inner_; if (is_small()) { destroy(single_slot()); - erase_meta_only(single_iterator()); + erase_meta_only_small(); return end(); } if (first == begin() && last == end()) { @@ -2738,12 +2716,12 @@ if (src.is_small()) { if (src.empty()) return; if (insert_slot(src.single_slot())) - src.erase_meta_only(src.single_iterator()); + src.erase_meta_only_small(); return; } for (auto it = src.begin(), e = src.end(); it != e;) { auto next = std::next(it); - if (insert_slot(it.slot())) src.erase_meta_only(it); + if (insert_slot(it.slot())) src.erase_meta_only_large(it); it = next; } } @@ -2770,9 +2748,9 @@ } void swap(raw_hash_set& that) noexcept( - IsNoThrowSwappable<hasher>() && IsNoThrowSwappable<key_equal>() && - IsNoThrowSwappable<allocator_type>( - typename AllocTraits::propagate_on_container_swap{})) { + AllocTraits::is_always_equal::value && + std::is_nothrow_swappable<hasher>::value && + std::is_nothrow_swappable<key_equal>::value) { AssertNotDebugCapacity(); that.AssertNotDebugCapacity(); using std::swap; @@ -3085,11 +3063,17 @@ // This merely updates the pertinent control byte. This can be used in // conjunction with Policy::transfer to move the object to another place. void erase_meta_only(const_iterator it) { - if (is_soo()) { - common().set_empty_soo(); + if (is_small()) { + erase_meta_only_small(); return; } - EraseMetaOnly(common(), it.control(), sizeof(slot_type)); + erase_meta_only_large(it); + } + void erase_meta_only_small() { + EraseMetaOnlySmall(common(), SooEnabled(), sizeof(slot_type)); + } + void erase_meta_only_large(const_iterator it) { + EraseMetaOnlyLarge(common(), it.control(), sizeof(slot_type)); } template <class K> @@ -3636,7 +3620,7 @@ return 0; } c->destroy(it.slot()); - c->erase_meta_only(it); + c->erase_meta_only_small(); return 1; } [[maybe_unused]] const size_t original_size_for_assert = c->size(); @@ -3648,7 +3632,7 @@ auto* slot = static_cast<SlotType*>(slot_void); if (pred(Set::PolicyTraits::element(slot))) { c->destroy(slot); - EraseMetaOnly(c->common(), ctrl, sizeof(*slot)); + EraseMetaOnlyLarge(c->common(), ctrl, sizeof(*slot)); ++num_deleted; } });
diff --git a/absl/container/internal/raw_hash_set_allocator_test.cc b/absl/container/internal/raw_hash_set_allocator_test.cc index 2e6f8f5..b268d9e 100644 --- a/absl/container/internal/raw_hash_set_allocator_test.cc +++ b/absl/container/internal/raw_hash_set_allocator_test.cc
@@ -436,13 +436,14 @@ } TEST_F(PropagateOnAll, Swap) { - auto it = t1.insert(0).first; + t1.insert(0); Table u(0, a2); u.swap(t1); EXPECT_EQ(a1, u.get_allocator()); EXPECT_EQ(a2, t1.get_allocator()); EXPECT_EQ(1, a1.num_allocs()); EXPECT_EQ(0, a2.num_allocs()); + auto it = u.begin(); EXPECT_EQ(0, it->num_moves()); EXPECT_EQ(0, it->num_copies()); }
diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index e1dafff..044e5b5 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc
@@ -2247,12 +2247,10 @@ } TEST(Table, NoThrowSwappable) { - ASSERT_TRUE( - container_internal::IsNoThrowSwappable<absl::Hash<absl::string_view>>()); - ASSERT_TRUE(container_internal::IsNoThrowSwappable< - std::equal_to<absl::string_view>>()); - ASSERT_TRUE(container_internal::IsNoThrowSwappable<std::allocator<int>>()); - EXPECT_TRUE(container_internal::IsNoThrowSwappable<StringTable>()); + ASSERT_TRUE(std::is_nothrow_swappable<absl::Hash<absl::string_view>>()); + ASSERT_TRUE(std::is_nothrow_swappable<std::equal_to<absl::string_view>>()); + ASSERT_TRUE(std::is_nothrow_swappable<std::allocator<int>>()); + EXPECT_TRUE(std::is_nothrow_swappable<StringTable>()); } TEST(Table, HeterogeneousLookup) {
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index 3194bec..ee9778d 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -137,6 +137,13 @@ // Compute a magic constant, so that multiplying by it is the same as // extending crc by length zeros. +#if defined(NDEBUG) && ABSL_HAVE_CPP_ATTRIBUTE(clang::no_sanitize) +// The array accesses in this are safe: +// length > 3, so countr_zero(length >> 2) < 62, and length & (length - 1) +// cannot introduce bits >= 62. +// The compiler cannot prove this, so manually disable bounds checking. +[[clang::no_sanitize("array-bounds")]] +#endif uint32_t CRC32AcceleratedX86ARMCombined::ComputeZeroConstant( size_t length) const { // Lowest 2 bits are handled separately in ExtendByZeroes
diff --git a/absl/debugging/stacktrace.cc b/absl/debugging/stacktrace.cc index f71e80c..30d032a 100644 --- a/absl/debugging/stacktrace.cc +++ b/absl/debugging/stacktrace.cc
@@ -124,6 +124,19 @@ internal_stacktrace::GetStackFrames(void** result, uintptr_t* frames, int* sizes, int max_depth, int skip_count) { if (internal_stacktrace::ShouldFixUpStack()) { + if constexpr (kHaveAlloca) { + // Some implementations of FixUpStack may need to be passed frame + // information from Unwind, even if the caller doesn't need that + // information. We allocate the necessary buffers for such implementations + // here. + const size_t nmax = static_cast<size_t>(max_depth); + if (frames == nullptr) { + frames = static_cast<uintptr_t*>(alloca(nmax * sizeof(*frames))); + } + if (sizes == nullptr) { + sizes = static_cast<int*>(alloca(nmax * sizeof(*sizes))); + } + } size_t depth = static_cast<size_t>(Unwind<true, true>( result, frames, sizes, max_depth, skip_count, nullptr, nullptr)); internal_stacktrace::FixUpStack(result, frames, sizes, @@ -141,6 +154,19 @@ int skip_count, const void* uc, int* min_dropped_frames) { if (internal_stacktrace::ShouldFixUpStack()) { + if constexpr (kHaveAlloca) { + // Some implementations of FixUpStack may need to be passed frame + // information from Unwind, even if the caller doesn't need that + // information. We allocate the necessary buffers for such implementations + // here. + const size_t nmax = static_cast<size_t>(max_depth); + if (frames == nullptr) { + frames = static_cast<uintptr_t*>(alloca(nmax * sizeof(*frames))); + } + if (sizes == nullptr) { + sizes = static_cast<int*>(alloca(nmax * sizeof(*sizes))); + } + } size_t depth = static_cast<size_t>(Unwind<true, true>( result, frames, sizes, max_depth, skip_count, uc, min_dropped_frames)); internal_stacktrace::FixUpStack(result, frames, sizes,
diff --git a/absl/profiling/BUILD.bazel b/absl/profiling/BUILD.bazel index f9a2a72..5afdb96 100644 --- a/absl/profiling/BUILD.bazel +++ b/absl/profiling/BUILD.bazel
@@ -144,3 +144,44 @@ "@google_benchmark//:benchmark_main", ], ) + +cc_library( + name = "profile_builder", + srcs = ["internal/profile_builder.cc"], + hdrs = ["internal/profile_builder.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + "//absl/base", + "//absl/base:config", + "//absl/base:raw_logging_internal", + "//absl/container:btree", + "//absl/container:flat_hash_map", + "//absl/strings", + "//absl/types:span", + ], +) + +cc_library( + name = "hashtable", + srcs = ["hashtable.cc"], + hdrs = ["hashtable.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":profile_builder", + "//absl/base:config", + "//absl/container:hashtablez_sampler", + "//absl/status:statusor", + "//absl/strings", + "//absl/strings:string_view", + "//absl/time", + "//absl/types:span", + ], +)
diff --git a/absl/profiling/CMakeLists.txt b/absl/profiling/CMakeLists.txt index 84b8b3b..4807f0d 100644 --- a/absl/profiling/CMakeLists.txt +++ b/absl/profiling/CMakeLists.txt
@@ -92,3 +92,40 @@ GTest::gmock_main ) +# Internal-only target, do not depend on directly +absl_cc_library( + NAME + profile_builder + HDRS + "internal/profile_builder.h" + SRCS + "internal/profile_builder.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::raw_logging_internal + absl::flat_hash_map + absl::btree + absl::strings + absl::span +) + +absl_cc_library( + NAME + hashtable_profiler + HDRS + "hashtable.h" + SRCS + "hashtable.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::profile_builder + absl::config + absl::core_headers + absl::strings + absl::span + absl::hashtablez_sampler +)
diff --git a/absl/profiling/hashtable.cc b/absl/profiling/hashtable.cc new file mode 100644 index 0000000..bfec0d5 --- /dev/null +++ b/absl/profiling/hashtable.cc
@@ -0,0 +1,124 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may +// obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/profiling/hashtable.h" + +#include <atomic> +#include <cstddef> +#include <cstdint> +#include <string> +#include <utility> +#include <vector> + +#include "absl/base/config.h" +#include "absl/container/internal/hashtablez_sampler.h" +#include "absl/profiling/internal/profile_builder.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +StatusOr<std::string> MarshalHashtableProfile() { + return debugging_internal::MarshalHashtableProfile( + container_internal::GlobalHashtablezSampler(), Now()); +} + +namespace debugging_internal { + +StatusOr<std::string> MarshalHashtableProfile( + container_internal::HashtablezSampler& sampler, Time now) { + static constexpr absl::string_view kDropFrames = + "(::)?absl::container_internal::.*|" + "(::)?absl::(flat|node)_hash_(map|set).*"; + + ProfileBuilder builder; + StringId drop_frames_id = builder.InternString(kDropFrames); + builder.set_drop_frames_id(drop_frames_id); + builder.AddSampleType(builder.InternString("capacity"), + builder.InternString("count")); + builder.set_default_sample_type_id(builder.InternString("capacity")); + + const auto capacity_id = builder.InternString("capacity"); + const auto size_id = builder.InternString("size"); + const auto num_erases_id = builder.InternString("num_erases"); + const auto num_rehashes_id = builder.InternString("num_rehashes"); + const auto max_probe_length_id = builder.InternString("max_probe_length"); + const auto total_probe_length_id = builder.InternString("total_probe_length"); + const auto stuck_bits_id = builder.InternString("stuck_bits"); + const auto inline_element_size_id = + builder.InternString("inline_element_size"); + const auto key_size_id = builder.InternString("key_size"); + const auto value_size_id = builder.InternString("value_size"); + const auto soo_capacity_id = builder.InternString("soo_capacity"); + const auto checksum_id = builder.InternString("checksum"); + const auto table_age_id = builder.InternString("table_age"); + const auto max_reserve_id = builder.InternString("max_reserve"); + + size_t dropped = + sampler.Iterate([&](const container_internal::HashtablezInfo& info) { + const size_t capacity = info.capacity.load(std::memory_order_relaxed); + std::vector<std::pair<StringId, int64_t>> labels; + + auto add_label = [&](StringId tag, uint64_t value) { + if (value == 0) { + return; + } + labels.emplace_back(tag, static_cast<int64_t>(value)); + }; + + add_label(capacity_id, capacity); + add_label(size_id, info.size.load(std::memory_order_relaxed)); + add_label(num_erases_id, + info.num_erases.load(std::memory_order_relaxed)); + add_label(num_rehashes_id, + info.num_rehashes.load(std::memory_order_relaxed)); + add_label(max_probe_length_id, + info.max_probe_length.load(std::memory_order_relaxed)); + add_label(total_probe_length_id, + info.total_probe_length.load(std::memory_order_relaxed)); + add_label(stuck_bits_id, + (info.hashes_bitwise_and.load(std::memory_order_relaxed) | + ~info.hashes_bitwise_or.load(std::memory_order_relaxed))); + add_label(inline_element_size_id, info.inline_element_size); + add_label(key_size_id, info.key_size); + add_label(value_size_id, info.value_size); + add_label(soo_capacity_id, info.soo_capacity); + add_label(checksum_id, + info.hashes_bitwise_xor.load(std::memory_order_relaxed)); + add_label( + table_age_id, + static_cast<uint64_t>(ToInt64Microseconds(now - info.create_time))); + add_label(max_reserve_id, + info.max_reserve.load(std::memory_order_relaxed)); + builder.AddSample(static_cast<int64_t>(capacity) * info.weight, + MakeSpan(info.stack, info.depth), labels); + }); + + // TODO(b/262310142): Make this more structured data. + StringId comment_id = + builder.InternString(StrCat("dropped_samples: ", dropped)); + builder.set_comment_id(comment_id); + builder.AddCurrentMappings(); + return std::move(builder).Emit(); +} + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/profiling/hashtable.h b/absl/profiling/hashtable.h new file mode 100644 index 0000000..9e490dc --- /dev/null +++ b/absl/profiling/hashtable.h
@@ -0,0 +1,40 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_PROFILING_HASHTABLE_H_ +#define ABSL_PROFILING_HASHTABLE_H_ + +#include <cstdint> +#include <string> + +#include "absl/container/internal/hashtablez_sampler.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +absl::StatusOr<std::string> MarshalHashtableProfile(); + +namespace debugging_internal { + +absl::StatusOr<std::string> MarshalHashtableProfile( + container_internal::HashtablezSampler& sampler, absl::Time now); + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_PROFILING_HASHTABLE_H_
diff --git a/absl/profiling/internal/profile_builder.cc b/absl/profiling/internal/profile_builder.cc new file mode 100644 index 0000000..2d189c8 --- /dev/null +++ b/absl/profiling/internal/profile_builder.cc
@@ -0,0 +1,462 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may +// obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/profiling/internal/profile_builder.h" + +#ifdef __linux__ +#include <elf.h> +#include <link.h> +#endif // __linux__ + +#include <cassert> +#include <cstdint> +#include <cstring> +#include <string> +#include <utility> +#include <vector> + +#include "absl/base/casts.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/escaping.h" +#include "absl/strings/str_cat.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +namespace { + +// This file contains a simplified implementation of the pprof profile builder, +// which avoids a dependency on protobuf. +// +// The canonical profile proto definition is at +// https://github.com/google/pprof/blob/master/proto/profile.proto +// +// Wire-format encoding is a simple sequence of (tag, value) pairs. The tag +// is a varint-encoded integer, where the low 3 bits are the wire type, and the +// high bits are the field number. +// +// For the fields we care about, we'll be using the following wire types: +// +// Wire Type 0: Varint-encoded integer. +// Wire Type 2: Length-delimited. Used for strings and sub-messages. +enum class WireType { + kVarint = 0, + kLengthDelimited = 2, +}; + +#ifdef __linux__ +// Returns the Phdr of the first segment of the given type. +const ElfW(Phdr) * GetFirstSegment(const dl_phdr_info* const info, + const ElfW(Word) segment_type) { + for (int i = 0; i < info->dlpi_phnum; ++i) { + if (info->dlpi_phdr[i].p_type == segment_type) { + return &info->dlpi_phdr[i]; + } + } + return nullptr; +} + +// Return DT_SONAME for the given image. If there is no PT_DYNAMIC or if +// PT_DYNAMIC does not contain DT_SONAME, return nullptr. +static const char* GetSoName(const dl_phdr_info* const info) { + const ElfW(Phdr)* const pt_dynamic = GetFirstSegment(info, PT_DYNAMIC); + if (pt_dynamic == nullptr) { + return nullptr; + } + const ElfW(Dyn)* dyn = + reinterpret_cast<ElfW(Dyn)*>(info->dlpi_addr + pt_dynamic->p_vaddr); + const ElfW(Dyn)* dt_strtab = nullptr; + const ElfW(Dyn)* dt_strsz = nullptr; + const ElfW(Dyn)* dt_soname = nullptr; + for (; dyn->d_tag != DT_NULL; ++dyn) { + if (dyn->d_tag == DT_SONAME) { + dt_soname = dyn; + } else if (dyn->d_tag == DT_STRTAB) { + dt_strtab = dyn; + } else if (dyn->d_tag == DT_STRSZ) { + dt_strsz = dyn; + } + } + if (dt_soname == nullptr) { + return nullptr; + } + ABSL_RAW_CHECK(dt_strtab != nullptr, "Unexpected nullptr"); + ABSL_RAW_CHECK(dt_strsz != nullptr, "Unexpected nullptr"); + const char* const strtab = + reinterpret_cast<char*>(info->dlpi_addr + dt_strtab->d_un.d_val); + ABSL_RAW_CHECK(dt_soname->d_un.d_val < dt_strsz->d_un.d_val, + "Unexpected order"); + return strtab + dt_soname->d_un.d_val; +} + +// Helper function to get the build ID of a shared object. +std::string GetBuildId(const dl_phdr_info* const info) { + std::string result; + + // pt_note contains entries (of type ElfW(Nhdr)) starting at + // info->dlpi_addr + pt_note->p_vaddr + // with length + // pt_note->p_memsz + // + // The length of each entry is given by + // Align(sizeof(ElfW(Nhdr)) + nhdr->n_namesz) + Align(nhdr->n_descsz) + for (int i = 0; i < info->dlpi_phnum; ++i) { + const ElfW(Phdr)* pt_note = &info->dlpi_phdr[i]; + if (pt_note->p_type != PT_NOTE) continue; + + const char* note = + reinterpret_cast<char*>(info->dlpi_addr + pt_note->p_vaddr); + const char* const last = note + pt_note->p_filesz; + const ElfW(Xword) align = pt_note->p_align; + while (note < last) { + const ElfW(Nhdr)* const nhdr = reinterpret_cast<const ElfW(Nhdr)*>(note); + if (note + sizeof(*nhdr) > last) { + // Corrupt PT_NOTE + break; + } + + // Both the start and end of the descriptor are aligned by sh_addralign + // (= p_align). + const ElfW(Xword) desc_start = + (sizeof(*nhdr) + nhdr->n_namesz + align - 1) & -align; + const ElfW(Xword) size = + desc_start + ((nhdr->n_descsz + align - 1) & -align); + + // Beware of wrap-around. + if (nhdr->n_namesz >= static_cast<ElfW(Word)>(-align) || + nhdr->n_descsz >= static_cast<ElfW(Word)>(-align) || + desc_start < sizeof(*nhdr) || size < desc_start || + size > static_cast<ElfW(Xword)>(last - note)) { + // Corrupt PT_NOTE + break; + } + + if (nhdr->n_type == NT_GNU_BUILD_ID) { + const char* const note_name = note + sizeof(*nhdr); + // n_namesz is the length of note_name. + if (nhdr->n_namesz == 4 && memcmp(note_name, "GNU\0", 4) == 0) { + if (!result.empty()) { + // Repeated build-ids. Ignore them. + return ""; + } + result = absl::BytesToHexString( + absl::string_view(note + desc_start, nhdr->n_descsz)); + } + } + note += size; + } + } + + return result; +} +#endif // __linux__ + +// A varint-encoded integer. +struct Varint { + explicit Varint(uint64_t v) : value(v) {} + explicit Varint(StringId v) : value(static_cast<uint64_t>(v)) {} + explicit Varint(LocationId v) : value(static_cast<uint64_t>(v)) {} + explicit Varint(MappingId v) : value(static_cast<uint64_t>(v)) {} + + uint64_t value; + + template <typename Sink> + friend void AbslStringify(Sink& sink, const Varint& v) { + char buf[10]; + char* p = buf; + uint64_t u = v.value; + while (u >= 0x80) { + *p++ = static_cast<char>((u & 0x7f) | 0x80); + u >>= 7; + } + *p++ = static_cast<char>(u); + sink.Append(absl::string_view(buf, static_cast<size_t>(p - buf))); + } +}; + +struct Tag { + int field_number; + WireType wire_type; + + template <typename Sink> + friend void AbslStringify(Sink& sink, const Tag& t) { + absl::Format(&sink, "%v", + Varint((static_cast<uint64_t>(t.field_number) << 3) | + static_cast<uint64_t>(t.wire_type))); + } +}; + +struct LengthDelimited { + int field_number; + absl::string_view value; + + template <typename Sink> + friend void AbslStringify(Sink& sink, const LengthDelimited& ld) { + absl::Format(&sink, "%v%v%v", + Tag{ld.field_number, WireType::kLengthDelimited}, + Varint(ld.value.size()), ld.value); + } +}; + +struct VarintField { + int field_number; + Varint value; + + template <typename Sink> + friend void AbslStringify(Sink& sink, const VarintField& vf) { + absl::Format(&sink, "%v%v", Tag{vf.field_number, WireType::kVarint}, + vf.value); + } +}; + +} // namespace + +StringId ProfileBuilder::InternString(absl::string_view str) { + if (str.empty()) return StringId(0); + return string_table_.emplace(str, StringId(string_table_.size())) + .first->second; +} + +LocationId ProfileBuilder::InternLocation(const void* address) { + return location_table_ + .emplace(absl::bit_cast<uintptr_t>(address), + LocationId(location_table_.size() + 1)) + .first->second; +} + +void ProfileBuilder::AddSample( + int64_t value, absl::Span<const void* const> stack, + absl::Span<const std::pair<StringId, int64_t>> labels) { + std::string sample_proto; + absl::StrAppend( + &sample_proto, + VarintField{SampleProto::kValue, Varint(static_cast<uint64_t>(value))}); + + for (const void* addr : stack) { + // Profile addresses are raw stack unwind addresses, so they should be + // adjusted by -1 to land inside the call instruction (although potentially + // misaligned). + absl::StrAppend( + &sample_proto, + VarintField{SampleProto::kLocationId, + Varint(InternLocation(absl::bit_cast<const void*>( + absl::bit_cast<uintptr_t>(addr) - 1)))}); + } + + for (const auto& label : labels) { + std::string label_proto = + absl::StrCat(VarintField{LabelProto::kKey, Varint(label.first)}, + VarintField{LabelProto::kNum, + Varint(static_cast<uint64_t>(label.second))}); + absl::StrAppend(&sample_proto, + LengthDelimited{SampleProto::kLabel, label_proto}); + } + samples_.push_back(std::move(sample_proto)); +} + +void ProfileBuilder::AddSampleType(StringId type, StringId unit) { + std::string sample_type_proto = + absl::StrCat(VarintField{ValueTypeProto::kType, Varint(type)}, + VarintField{ValueTypeProto::kUnit, Varint(unit)}); + sample_types_.push_back(std::move(sample_type_proto)); +} + +MappingId ProfileBuilder::AddMapping(uintptr_t memory_start, + uintptr_t memory_limit, + uintptr_t file_offset, + absl::string_view filename, + absl::string_view build_id) { + size_t index = mappings_.size() + 1; + auto [it, inserted] = mapping_table_.emplace(memory_start, index); + if (!inserted) { + return static_cast<MappingId>(it->second); + } + + Mapping m; + m.start = memory_start; + m.limit = memory_limit; + m.offset = file_offset; + m.filename = std::string(filename); + m.build_id = std::string(build_id); + + mappings_.push_back(std::move(m)); + return static_cast<MappingId>(index); +} + +std::string ProfileBuilder::Emit() && { + std::string profile_proto; + for (const auto& sample_type : sample_types_) { + absl::StrAppend(&profile_proto, + LengthDelimited{ProfileProto::kSampleType, sample_type}); + } + for (const auto& sample : samples_) { + absl::StrAppend(&profile_proto, + LengthDelimited{ProfileProto::kSample, sample}); + } + + // Build mapping table. + for (size_t i = 0, n = mappings_.size(); i < n; ++i) { + const auto& mapping = mappings_[i]; + std::string mapping_proto = absl::StrCat( + VarintField{MappingProto::kId, Varint(static_cast<uint64_t>(i + 1))}, + VarintField{MappingProto::kMemoryStart, Varint(mapping.start)}, + VarintField{MappingProto::kMemoryLimit, Varint(mapping.limit)}, + VarintField{MappingProto::kFileOffset, Varint(mapping.offset)}, + VarintField{MappingProto::kFilename, + Varint(InternString(mapping.filename))}, + VarintField{MappingProto::kBuildId, + Varint(InternString(mapping.build_id))}); + + absl::StrAppend(&profile_proto, + LengthDelimited{ProfileProto::kMapping, mapping_proto}); + } + + // Build location table. + for (const auto& [address, id] : location_table_) { + std::string location = + absl::StrCat(VarintField{LocationProto::kId, Varint(id)}, + VarintField{LocationProto::kAddress, Varint(address)}); + + if (!mappings_.empty()) { + // Find the mapping ID. + auto it = mapping_table_.upper_bound(address); + if (it != mapping_table_.begin()) { + --it; + } + + // If *it contains address, add mapping to location. + const size_t mapping_index = it->second; + const Mapping& mapping = mappings_[mapping_index - 1]; + + if (it->first <= address && address < mapping.limit) { + absl::StrAppend( + &location, + VarintField{LocationProto::kMappingId, + Varint(static_cast<uint64_t>(mapping_index))}); + } + } + + absl::StrAppend(&profile_proto, + LengthDelimited{ProfileProto::kLocation, location}); + } + + std::string string_table_proto; + std::vector<absl::string_view> sorted_strings(string_table_.size()); + for (const auto& p : string_table_) { + sorted_strings[static_cast<size_t>(p.second)] = p.first; + } + for (const auto& s : sorted_strings) { + absl::StrAppend(&string_table_proto, + LengthDelimited{ProfileProto::kStringTable, s}); + } + absl::StrAppend(&profile_proto, VarintField{ProfileProto::kDropFrames, + Varint(drop_frames_id_)}); + absl::StrAppend(&profile_proto, + VarintField{ProfileProto::kComment, Varint(comment_id_)}); + absl::StrAppend(&profile_proto, VarintField{ProfileProto::kDefaultSampleType, + Varint(default_sample_type_id_)}); + return absl::StrCat(string_table_proto, profile_proto); +} + +void ProfileBuilder::set_drop_frames_id(StringId drop_frames_id) { + drop_frames_id_ = drop_frames_id; +} + +void ProfileBuilder::set_comment_id(StringId comment_id) { + comment_id_ = comment_id; +} + +void ProfileBuilder::set_default_sample_type_id( + StringId default_sample_type_id) { + default_sample_type_id_ = default_sample_type_id; +} + +void ProfileBuilder::AddCurrentMappings() { +#ifdef __linux__ + dl_iterate_phdr( + +[](dl_phdr_info* info, size_t, void* data) { + auto& builder = *reinterpret_cast<ProfileBuilder*>(data); + + // Skip dummy entry introduced since glibc 2.18. + if (info->dlpi_phdr == nullptr && info->dlpi_phnum == 0) { + return 0; + } + + const bool is_main_executable = builder.mappings_.empty(); + + // Evaluate all the loadable segments. + for (int i = 0; i < info->dlpi_phnum; ++i) { + if (info->dlpi_phdr[i].p_type != PT_LOAD) { + continue; + } + const ElfW(Phdr)* pt_load = &info->dlpi_phdr[i]; + + ABSL_RAW_CHECK(pt_load != nullptr, "Unexpected nullptr"); + + // Extract data. + const size_t memory_start = info->dlpi_addr + pt_load->p_vaddr; + const size_t memory_limit = memory_start + pt_load->p_memsz; + const size_t file_offset = pt_load->p_offset; + + // Storage for path to executable as dlpi_name isn't populated for the + // main executable. +1 to allow for the null terminator that readlink + // does not add. + char self_filename[PATH_MAX + 1]; + const char* filename = info->dlpi_name; + if (filename == nullptr || filename[0] == '\0') { + // This is either the main executable or the VDSO. The main + // executable is always the first entry processed by callbacks. + if (is_main_executable) { + // This is the main executable. + ssize_t ret = readlink("/proc/self/exe", self_filename, + sizeof(self_filename) - 1); + if (ret >= 0 && + static_cast<size_t>(ret) < sizeof(self_filename)) { + self_filename[ret] = '\0'; + filename = self_filename; + } + } else { + // This is the VDSO. + filename = GetSoName(info); + } + } + + char resolved_path[PATH_MAX]; + absl::string_view resolved_filename; + if (realpath(filename, resolved_path)) { + resolved_filename = resolved_path; + } else { + resolved_filename = filename; + } + + const std::string build_id = GetBuildId(info); + + // Add to profile. + builder.AddMapping(memory_start, memory_limit, file_offset, + resolved_filename, build_id); + } + // Keep going. + return 0; + }, + this); +#endif // __linux__ +} + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/profiling/internal/profile_builder.h b/absl/profiling/internal/profile_builder.h new file mode 100644 index 0000000..42d4176 --- /dev/null +++ b/absl/profiling/internal/profile_builder.h
@@ -0,0 +1,137 @@ +// Copyright 2025 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may +// obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_PROFILING_INTERNAL_PROFILE_BUILDER_H_ +#define ABSL_PROFILING_INTERNAL_PROFILE_BUILDER_H_ + +#include <cstdint> +#include <string> +#include <vector> + +#include "absl/container/btree_map.h" +#include "absl/container/flat_hash_map.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// Field numbers for perftools.profiles.Profile. +// https://github.com/google/pprof/blob/master/proto/profile.proto +struct ProfileProto { + static constexpr int kSampleType = 1; + static constexpr int kSample = 2; + static constexpr int kMapping = 3; + static constexpr int kLocation = 4; + static constexpr int kStringTable = 6; + static constexpr int kDropFrames = 7; + static constexpr int kComment = 13; + static constexpr int kDefaultSampleType = 14; +}; + +struct ValueTypeProto { + static constexpr int kType = 1; + static constexpr int kUnit = 2; +}; + +struct SampleProto { + static constexpr int kLocationId = 1; + static constexpr int kValue = 2; + static constexpr int kLabel = 3; +}; + +struct LabelProto { + static constexpr int kKey = 1; + static constexpr int kStr = 2; + static constexpr int kNum = 3; + static constexpr int kNumUnit = 4; +}; + +struct MappingProto { + static constexpr int kId = 1; + static constexpr int kMemoryStart = 2; + static constexpr int kMemoryLimit = 3; + static constexpr int kFileOffset = 4; + static constexpr int kFilename = 5; + static constexpr int kBuildId = 6; +}; + +struct LocationProto { + static constexpr int kId = 1; + static constexpr int kMappingId = 2; + static constexpr int kAddress = 3; +}; + +enum class StringId : size_t {}; +enum class LocationId : size_t {}; +enum class MappingId : size_t {}; + +// A helper class to build a profile protocol buffer. +class ProfileBuilder { + public: + struct Mapping { + uint64_t start; + uint64_t limit; + uint64_t offset; + std::string filename; + std::string build_id; + }; + + StringId InternString(absl::string_view str); + + LocationId InternLocation(const void* address); + + void AddSample(int64_t value, absl::Span<const void* const> stack, + absl::Span<const std::pair<StringId, int64_t>> labels); + + void AddSampleType(StringId type, StringId unit); + + // Adds the current process mappings to the profile. + void AddCurrentMappings(); + + // Adds a single mapping to the profile and to lookup cache and returns the + // resulting ID. + MappingId AddMapping(uintptr_t memory_start, uintptr_t memory_limit, + uintptr_t file_offset, absl::string_view filename, + absl::string_view build_id); + + std::string Emit() &&; + + void set_drop_frames_id(StringId drop_frames_id); + void set_comment_id(StringId comment_id); + void set_default_sample_type_id(StringId default_sample_type_id); + + private: + absl::flat_hash_map<std::string, StringId> string_table_{{"", StringId(0)}}; + absl::flat_hash_map<uint64_t, LocationId> location_table_; + // mapping_table_ stores the start address of each mapping in mapping_ + // to its index. + absl::btree_map<uintptr_t, size_t> mapping_table_; + std::vector<Mapping> mappings_; + + std::vector<std::string> sample_types_; + std::vector<std::string> samples_; + + StringId drop_frames_id_{}; + StringId comment_id_{}; + StringId default_sample_type_id_{}; +}; + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_PROFILING_INTERNAL_PROFILE_BUILDER_H_
diff --git a/absl/profiling/internal/sample_recorder.h b/absl/profiling/internal/sample_recorder.h index 371f6c4..84843fd 100644 --- a/absl/profiling/internal/sample_recorder.h +++ b/absl/profiling/internal/sample_recorder.h
@@ -75,7 +75,7 @@ // Iterates over all the registered `StackInfo`s. Returning the number of // samples that have been dropped. - int64_t Iterate(const std::function<void(const T& stack)>& f); + size_t Iterate(const std::function<void(const T& stack)>& f); size_t GetMaxSamples() const; void SetMaxSamples(size_t max); @@ -222,7 +222,7 @@ } template <typename T> -int64_t SampleRecorder<T>::Iterate( +size_t SampleRecorder<T>::Iterate( const std::function<void(const T& stack)>& f) { T* s = all_.load(std::memory_order_acquire); while (s != nullptr) {
diff --git a/ci/absl_alternate_options.h b/ci/absl_alternate_options.h index a563859..20bf010 100644 --- a/ci/absl_alternate_options.h +++ b/ci/absl_alternate_options.h
@@ -15,13 +15,12 @@ // Alternate options.h file, used in continuous integration testing to exercise // option settings not used by default. +// SKIP_ABSL_INLINE_NAMESPACE_CHECK + #ifndef ABSL_CI_ABSL_ALTERNATE_OPTIONS_H_ #define ABSL_CI_ABSL_ALTERNATE_OPTIONS_H_ -#define ABSL_OPTION_USE_STD_ANY 0 -#define ABSL_OPTION_USE_STD_OPTIONAL 0 #define ABSL_OPTION_USE_STD_STRING_VIEW 0 -#define ABSL_OPTION_USE_STD_VARIANT 0 #define ABSL_OPTION_USE_STD_ORDERING 0 #define ABSL_OPTION_USE_INLINE_NAMESPACE 1 #define ABSL_OPTION_INLINE_NAMESPACE_NAME ns