diff --git a/MODULE.bazel b/MODULE.bazel index 405d2c6..29e624a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -39,5 +39,5 @@ # intended to be used by Abseil users depend on GoogleTest. bazel_dep( name = "googletest", - version = "1.16.0.bcr.1", + version = "1.17.0", )
diff --git a/absl/algorithm/container.h b/absl/algorithm/container.h index 3193656..913268d 100644 --- a/absl/algorithm/container.h +++ b/absl/algorithm/container.h
@@ -44,7 +44,6 @@ #include <cassert> #include <iterator> #include <numeric> -#include <random> #include <type_traits> #include <unordered_map> #include <unordered_set> @@ -847,25 +846,9 @@ typename UniformRandomBitGenerator> OutputIterator c_sample(const C& c, OutputIterator result, Distance n, UniformRandomBitGenerator&& gen) { -#if defined(__cpp_lib_sample) && __cpp_lib_sample >= 201603L return std::sample(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), result, n, std::forward<UniformRandomBitGenerator>(gen)); -#else - // Fall back to a stable selection-sampling implementation. - auto first = container_algorithm_internal::c_begin(c); - Distance unsampled_elements = c_distance(c); - n = (std::min)(n, unsampled_elements); - for (; n != 0; ++first) { - Distance r = - std::uniform_int_distribution<Distance>(0, --unsampled_elements)(gen); - if (r < n) { - *result++ = *first; - --n; - } - } - return result; -#endif } //------------------------------------------------------------------------------
diff --git a/absl/container/internal/hash_policy_testing.h b/absl/container/internal/hash_policy_testing.h index 66bb12e..e9f5757 100644 --- a/absl/container/internal/hash_policy_testing.h +++ b/absl/container/internal/hash_policy_testing.h
@@ -119,7 +119,11 @@ using propagate_on_container_swap = std::true_type; // Using old paradigm for this to ensure compatibility. - explicit Alloc(size_t id = 0) : id_(id) {} + // + // NOTE: As of 2025-05, this constructor cannot be explicit in order to work + // with the libstdc++ that ships with GCC15. + // NOLINTNEXTLINE(google-explicit-constructor) + Alloc(size_t id = 0) : id_(id) {} Alloc(const Alloc&) = default; Alloc& operator=(const Alloc&) = default;
diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index f66f486..f19e87b 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc
@@ -617,7 +617,7 @@ void ResizeNonSooImpl(CommonFields& common, const PolicyFunctions& policy, size_t new_capacity, HashtablezInfoHandle infoz) { ABSL_SWISSTABLE_ASSERT(IsValidCapacity(new_capacity)); - ABSL_SWISSTABLE_ASSERT(new_capacity > policy.soo_capacity); + ABSL_SWISSTABLE_ASSERT(new_capacity > policy.soo_capacity()); const size_t old_capacity = common.capacity(); [[maybe_unused]] ctrl_t* old_ctrl = common.control(); @@ -643,7 +643,7 @@ size_t total_probe_length = 0; ResetCtrl(common, slot_size); ABSL_SWISSTABLE_ASSERT(kMode != ResizeNonSooMode::kGuaranteedEmpty || - old_capacity == policy.soo_capacity); + old_capacity == policy.soo_capacity()); ABSL_SWISSTABLE_ASSERT(kMode != ResizeNonSooMode::kGuaranteedAllocated || old_capacity > 0); if constexpr (kMode == ResizeNonSooMode::kGuaranteedAllocated) { @@ -670,9 +670,9 @@ const PolicyFunctions& policy, size_t new_capacity, bool force_infoz) { ABSL_SWISSTABLE_ASSERT(IsValidCapacity(new_capacity)); - ABSL_SWISSTABLE_ASSERT(new_capacity > policy.soo_capacity); - ABSL_SWISSTABLE_ASSERT(!force_infoz || policy.soo_capacity > 0); - ABSL_SWISSTABLE_ASSERT(common.capacity() <= policy.soo_capacity); + ABSL_SWISSTABLE_ASSERT(new_capacity > policy.soo_capacity()); + ABSL_SWISSTABLE_ASSERT(!force_infoz || policy.soo_enabled); + ABSL_SWISSTABLE_ASSERT(common.capacity() <= policy.soo_capacity()); ABSL_SWISSTABLE_ASSERT(common.empty()); const size_t slot_size = policy.slot_size; HashtablezInfoHandle infoz; @@ -680,7 +680,7 @@ policy.is_hashtablez_eligible && (force_infoz || ShouldSampleNextTable()); if (ABSL_PREDICT_FALSE(should_sample)) { infoz = ForcedTrySample(slot_size, policy.key_size, policy.value_size, - policy.soo_capacity); + policy.soo_capacity()); } ResizeNonSooImpl<ResizeNonSooMode::kGuaranteedEmpty>(common, policy, new_capacity, infoz); @@ -694,8 +694,8 @@ const PolicyFunctions& policy, size_t hash, ctrl_t* new_ctrl, void* new_slots) { - ABSL_SWISSTABLE_ASSERT(c.size() == policy.soo_capacity); - ABSL_SWISSTABLE_ASSERT(policy.soo_capacity == SooCapacity()); + ABSL_SWISSTABLE_ASSERT(c.size() == policy.soo_capacity()); + ABSL_SWISSTABLE_ASSERT(policy.soo_enabled); size_t new_capacity = c.capacity(); c.generate_new_seed(); @@ -717,12 +717,21 @@ kForceSampleNoResizeIfUnsampled, }; +void AssertSoo([[maybe_unused]] CommonFields& common, + [[maybe_unused]] const PolicyFunctions& policy) { + ABSL_SWISSTABLE_ASSERT(policy.soo_enabled); + ABSL_SWISSTABLE_ASSERT(common.capacity() == policy.soo_capacity()); +} +void AssertFullSoo([[maybe_unused]] CommonFields& common, + [[maybe_unused]] const PolicyFunctions& policy) { + AssertSoo(common, policy); + ABSL_SWISSTABLE_ASSERT(common.size() == policy.soo_capacity()); +} + void ResizeFullSooTable(CommonFields& common, const PolicyFunctions& policy, size_t new_capacity, ResizeFullSooTableSamplingMode sampling_mode) { - ABSL_SWISSTABLE_ASSERT(common.capacity() == policy.soo_capacity); - ABSL_SWISSTABLE_ASSERT(common.size() == policy.soo_capacity); - ABSL_SWISSTABLE_ASSERT(policy.soo_capacity == SooCapacity()); + AssertFullSoo(common, policy); const size_t slot_size = policy.slot_size; const size_t slot_align = policy.slot_align; @@ -731,7 +740,7 @@ ResizeFullSooTableSamplingMode::kForceSampleNoResizeIfUnsampled) { if (ABSL_PREDICT_FALSE(policy.is_hashtablez_eligible)) { infoz = ForcedTrySample(slot_size, policy.key_size, policy.value_size, - policy.soo_capacity); + policy.soo_capacity()); } if (!infoz.IsSampled()) { @@ -1218,11 +1227,11 @@ ABSL_SWISSTABLE_ASSERT(common.growth_left() == 0); const size_t old_capacity = common.capacity(); ABSL_SWISSTABLE_ASSERT(old_capacity == 0 || - old_capacity > policy.soo_capacity); + old_capacity > policy.soo_capacity()); const size_t new_capacity = NextCapacity(old_capacity); ABSL_SWISSTABLE_ASSERT(IsValidCapacity(new_capacity)); - ABSL_SWISSTABLE_ASSERT(new_capacity > policy.soo_capacity); + ABSL_SWISSTABLE_ASSERT(new_capacity > policy.soo_capacity()); ctrl_t* old_ctrl = common.control(); void* old_slots = common.slot_array(); @@ -1238,7 +1247,7 @@ policy.is_hashtablez_eligible && ShouldSampleNextTable(); if (ABSL_PREDICT_FALSE(should_sample)) { infoz = ForcedTrySample(slot_size, policy.key_size, policy.value_size, - policy.soo_capacity); + policy.soo_capacity()); } } const bool has_infoz = infoz.IsSampled(); @@ -1291,7 +1300,7 @@ find_info = find_first_non_full(common, new_hash); SetCtrlInLargeTable(common, find_info.offset, new_h2, policy.slot_size); } - ABSL_SWISSTABLE_ASSERT(old_capacity > policy.soo_capacity); + ABSL_SWISSTABLE_ASSERT(old_capacity > policy.soo_capacity()); (*policy.dealloc)(alloc, old_capacity, old_ctrl, slot_size, slot_align, has_infoz); } @@ -1415,9 +1424,9 @@ // Resizes empty non-allocated table to the capacity to fit new_size elements. // Requires: -// 1. `c.capacity() == policy.soo_capacity`. +// 1. `c.capacity() == policy.soo_capacity()`. // 2. `c.empty()`. -// 3. `new_size > policy.soo_capacity`. +// 3. `new_size > policy.soo_capacity()`. // The table will be attempted to be sampled. void ReserveEmptyNonAllocatedTableToFitNewSize(CommonFields& common, const PolicyFunctions& policy, @@ -1435,7 +1444,7 @@ // allocated backing array. // // Requires: -// 1. `c.capacity() > policy.soo_capacity` OR `!c.empty()`. +// 1. `c.capacity() > policy.soo_capacity()` OR `!c.empty()`. // Reserving already allocated tables is considered to be a rare case. ABSL_ATTRIBUTE_NOINLINE void ReserveAllocatedTable( CommonFields& common, const PolicyFunctions& policy, size_t new_size) { @@ -1443,12 +1452,12 @@ ValidateMaxSize(new_size, policy.slot_size); ABSL_ASSUME(new_size > 0); const size_t new_capacity = SizeToCapacity(new_size); - if (cap == policy.soo_capacity) { + if (cap == policy.soo_capacity()) { ABSL_SWISSTABLE_ASSERT(!common.empty()); ResizeFullSooTable(common, policy, new_capacity, ResizeFullSooTableSamplingMode::kNoSampling); } else { - ABSL_SWISSTABLE_ASSERT(cap > policy.soo_capacity); + ABSL_SWISSTABLE_ASSERT(cap > policy.soo_capacity()); // TODO(b/382423690): consider using GrowToNextCapacity, when applicable. ResizeAllocatedTableWithSeedChange(common, policy, new_capacity); } @@ -1486,16 +1495,14 @@ const PolicyFunctions& policy, size_t new_hash, ctrl_t soo_slot_ctrl) { - ABSL_SWISSTABLE_ASSERT(common.capacity() == policy.soo_capacity); - ABSL_SWISSTABLE_ASSERT(policy.soo_capacity == SooCapacity()); + AssertSoo(common, policy); if (ABSL_PREDICT_FALSE(soo_slot_ctrl == ctrl_t::kEmpty)) { // The table is empty, it is only used for forced sampling of SOO tables. return GrowEmptySooTableToNextCapacityForceSamplingAndPrepareInsert( common, policy, new_hash); } - ABSL_SWISSTABLE_ASSERT(common.size() == policy.soo_capacity); + ABSL_SWISSTABLE_ASSERT(common.size() == policy.soo_capacity()); static constexpr size_t kNewCapacity = NextCapacity(SooCapacity()); - ABSL_SWISSTABLE_ASSERT(kNewCapacity > policy.soo_capacity); const size_t slot_size = policy.slot_size; const size_t slot_align = policy.slot_align; common.set_capacity(kNewCapacity); @@ -1560,9 +1567,7 @@ void GrowFullSooTableToNextCapacityForceSampling( CommonFields& common, const PolicyFunctions& policy) { - ABSL_SWISSTABLE_ASSERT(common.capacity() == policy.soo_capacity); - ABSL_SWISSTABLE_ASSERT(common.size() == policy.soo_capacity); - ABSL_SWISSTABLE_ASSERT(policy.soo_capacity == SooCapacity()); + AssertFullSoo(common, policy); ResizeFullSooTable( common, policy, NextCapacity(SooCapacity()), ResizeFullSooTableSamplingMode::kForceSampleNoResizeIfUnsampled); @@ -1573,18 +1578,18 @@ auto clear_backing_array = [&]() { ClearBackingArray(common, policy, policy.get_char_alloc(common), - /*reuse=*/false, policy.soo_capacity > 0); + /*reuse=*/false, policy.soo_enabled); }; const size_t slot_size = policy.slot_size; if (n == 0) { - if (cap <= policy.soo_capacity) return; + if (cap <= policy.soo_capacity()) return; if (common.empty()) { clear_backing_array(); return; } - if (common.size() <= policy.soo_capacity) { + if (common.size() <= policy.soo_capacity()) { // When the table is already sampled, we keep it sampled. if (common.infoz().IsSampled()) { static constexpr size_t kInitialSampledCapacity = @@ -1618,7 +1623,7 @@ NormalizeCapacity(n | SizeToCapacity(common.size())); // n == 0 unconditionally rehashes as per the standard. if (n == 0 || new_capacity > cap) { - if (cap == policy.soo_capacity) { + if (cap == policy.soo_capacity()) { if (common.empty()) { ResizeEmptyNonAllocatedTableImpl(common, policy, new_capacity, /*force_infoz=*/false); @@ -1640,7 +1645,7 @@ absl::FunctionRef<void(void*, const void*)> copy_fn) { const size_t size = other.size(); ABSL_SWISSTABLE_ASSERT(size > 0); - const size_t soo_capacity = policy.soo_capacity; + const size_t soo_capacity = policy.soo_capacity(); const size_t slot_size = policy.slot_size; if (size <= soo_capacity) { ABSL_SWISSTABLE_ASSERT(size == 1); @@ -1714,17 +1719,17 @@ const PolicyFunctions& policy, size_t new_size) { common.reset_reserved_growth(new_size); common.set_reservation_size(new_size); - ABSL_SWISSTABLE_ASSERT(new_size > policy.soo_capacity); + ABSL_SWISSTABLE_ASSERT(new_size > policy.soo_capacity()); const size_t cap = common.capacity(); - if (ABSL_PREDICT_TRUE(common.empty() && cap <= policy.soo_capacity)) { + if (ABSL_PREDICT_TRUE(common.empty() && cap <= policy.soo_capacity())) { return ReserveEmptyNonAllocatedTableToFitNewSize(common, policy, new_size); } - ABSL_SWISSTABLE_ASSERT(!common.empty() || cap > policy.soo_capacity); + ABSL_SWISSTABLE_ASSERT(!common.empty() || cap > policy.soo_capacity()); ABSL_SWISSTABLE_ASSERT(cap > 0); const size_t max_size_before_growth = - cap <= policy.soo_capacity ? policy.soo_capacity - : common.size() + common.growth_left(); + cap <= policy.soo_capacity() ? policy.soo_capacity() + : common.size() + common.growth_left(); if (new_size <= max_size_before_growth) { return; }
diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index f8c0731..512c946 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h
@@ -514,6 +514,8 @@ // Sets the has_infoz bit. void set_has_infoz() { data_ |= kHasInfozMask; } + void set_no_seed_for_testing() { data_ &= ~kSeedMask; } + private: static constexpr size_t kSizeShift = 64 - kSizeBitCount; static constexpr uint64_t kSizeOneNoMetadata = uint64_t{1} << kSizeShift; @@ -1037,6 +1039,7 @@ // `kGenerateSeed && !empty() && !is_single_group(capacity())` because H1 is // being changed. In such cases, we will need to rehash the table. void generate_new_seed() { size_.generate_new_seed(); } + void set_no_seed_for_testing() { size_.set_no_seed_for_testing(); } // The total number of available slots. size_t capacity() const { return capacity_; } @@ -1637,7 +1640,7 @@ uint32_t value_size; uint32_t slot_size; uint16_t slot_align; - uint8_t soo_capacity; + bool soo_enabled; bool is_hashtablez_eligible; // Returns the pointer to the hash function stored in the set. @@ -1675,6 +1678,10 @@ void* probed_storage, void (*encode_probed_element)(void* probed_storage, h2_t h2, size_t source_offset, size_t h1)); + + uint8_t soo_capacity() const { + return static_cast<uint8_t>(soo_enabled ? SooCapacity() : 0); + } }; // Returns the maximum valid size for a table with 1-byte slots. @@ -3606,8 +3613,7 @@ static_cast<uint32_t>(sizeof(key_type)), static_cast<uint32_t>(sizeof(value_type)), static_cast<uint16_t>(sizeof(slot_type)), - static_cast<uint16_t>(alignof(slot_type)), - static_cast<uint8_t>(SooEnabled() ? SooCapacity() : 0), + static_cast<uint16_t>(alignof(slot_type)), SooEnabled(), ShouldSampleHashtablezInfoForAlloc<CharAlloc>(), // TODO(b/328722020): try to type erase // for standard layout and alignof(Hash) <= alignof(CommonFields).
diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 404d1eb..9a323c4 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc
@@ -4238,9 +4238,8 @@ // artificially update growth info to force resize. absl::flat_hash_set<uint8_t, ConstUint8Hash> t(63, ConstUint8Hash{&hash}); CommonFields& common = RawHashSetTestOnlyAccess::GetCommon(t); - // Assign value to the seed, so that H1 is always 0. - // That helps to test all buffer overflows in GrowToNextCapacity. - hash = common.seed().seed() << 7; + // Set 0 seed so that H1 is always 0. + common.set_no_seed_for_testing(); ASSERT_EQ(H1(t.hash_function()(75), common.seed()), 0); uint8_t inserted_till = 210; for (uint8_t i = 0; i < inserted_till; ++i) {
diff --git a/absl/debugging/internal/stacktrace_aarch64-inl.inc b/absl/debugging/internal/stacktrace_aarch64-inl.inc index 25073f7..1746b5d 100644 --- a/absl/debugging/internal/stacktrace_aarch64-inl.inc +++ b/absl/debugging/internal/stacktrace_aarch64-inl.inc
@@ -134,17 +134,13 @@ if ((reinterpret_cast<uintptr_t>(new_frame_pointer) & 7) != 0) return nullptr; - // Check that alleged frame pointer is actually readable. This is to - // prevent "double fault" in case we hit the first fault due to e.g. - // stack corruption. - if (!absl::debugging_internal::AddressIsReadable( - new_frame_pointer)) { - return nullptr; - } - + uintptr_t new_fp_comparable = reinterpret_cast<uintptr_t>(new_frame_pointer); // Only check the size if both frames are in the same stack. - if (InsideSignalStack(new_frame_pointer, stack_info) == - InsideSignalStack(old_frame_pointer, stack_info)) { + const bool old_inside_signal_stack = + InsideSignalStack(old_frame_pointer, stack_info); + const bool new_inside_signal_stack = + InsideSignalStack(new_frame_pointer, stack_info); + if (new_inside_signal_stack == old_inside_signal_stack) { // Check frame size. In strict mode, we assume frames to be under // 100,000 bytes. In non-strict mode, we relax the limit to 1MB. const size_t max_size = STRICT_UNWINDING ? 100000 : 1000000; @@ -158,16 +154,15 @@ if (frame_size > max_size) { size_t stack_low = stack_info->stack_low; size_t stack_high = stack_info->stack_high; - if (InsideSignalStack(new_frame_pointer, stack_info)) { + if (new_inside_signal_stack) { stack_low = stack_info->sig_stack_low; stack_high = stack_info->sig_stack_high; } if (stack_high < kUnknownStackEnd && static_cast<size_t>(getpagesize()) < stack_low) { - const uintptr_t new_fp_u = - reinterpret_cast<uintptr_t>(new_frame_pointer); // Stack bounds are known. - if (!(stack_low < new_fp_u && new_fp_u <= stack_high)) { + if (!(stack_low < new_fp_comparable && + new_fp_comparable <= stack_high)) { // new_frame_pointer is not within a known stack. return nullptr; } @@ -177,8 +172,19 @@ } } } + // New frame pointer is valid if it is inside either known stack or readable. + // This assumes that everything within either known stack is readable. Outside + // either known stack but readable is unexpected, and possibly corrupt, but + // for now assume it is valid. If it isn't actually valid, the next frame will + // be corrupt and we will detect that next iteration. + if (new_inside_signal_stack || + (new_fp_comparable >= stack_info->stack_low && + new_fp_comparable < stack_info->stack_high) || + absl::debugging_internal::AddressIsReadable(new_frame_pointer)) { + return new_frame_pointer; + } - return new_frame_pointer; + return nullptr; } template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT>
diff --git a/absl/log/check.h b/absl/log/check.h index 50f633d..9e2219b 100644 --- a/absl/log/check.h +++ b/absl/log/check.h
@@ -42,7 +42,8 @@ // CHECK() // -// `CHECK` terminates the program with a fatal error if `condition` is not true. +// `CHECK` enforces that the `condition` is true. If the condition is false, +// the program is terminated with a fatal error. // // The message may include additional information such as stack traces, when // available.
diff --git a/absl/log/check_test_impl.inc b/absl/log/check_test_impl.inc index be58a3d..7a0000e 100644 --- a/absl/log/check_test_impl.inc +++ b/absl/log/check_test_impl.inc
@@ -39,6 +39,7 @@ namespace absl_log_internal { using ::testing::AllOf; +using ::testing::AnyOf; using ::testing::HasSubstr; using ::testing::Not; @@ -634,14 +635,12 @@ TEST(CHECKDeathTest, TestPointerPrintedAsNumberDespiteAbslStringify) { const auto* p = reinterpret_cast<const PointerIsStringifiable*>(0x1234); -#ifdef _MSC_VER EXPECT_DEATH( ABSL_TEST_CHECK_EQ(p, nullptr), - HasSubstr("Check failed: p == nullptr (0000000000001234 vs. (null))")); -#else // _MSC_VER - EXPECT_DEATH(ABSL_TEST_CHECK_EQ(p, nullptr), - HasSubstr("Check failed: p == nullptr (0x1234 vs. (null))")); -#endif // _MSC_VER + AnyOf( + HasSubstr("Check failed: p == nullptr (0000000000001234 vs. (null))"), + HasSubstr("Check failed: p == nullptr (0x1234 vs. (null))") + )); } // An uncopyable object with operator<<.
diff --git a/absl/strings/BUILD.bazel b/absl/strings/BUILD.bazel index 49562f7..bb152ac 100644 --- a/absl/strings/BUILD.bazel +++ b/absl/strings/BUILD.bazel
@@ -329,6 +329,7 @@ visibility = ["//visibility:private"], deps = [ ":internal", + ":string_view", "//absl/base:core_headers", "@googletest//:gtest", "@googletest//:gtest_main", @@ -1316,6 +1317,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = ["//visibility:private"], deps = [ + ":internal", ":strings", "//absl/base:config", "//absl/base:core_headers",
diff --git a/absl/strings/CMakeLists.txt b/absl/strings/CMakeLists.txt index ee73860..547ef26 100644 --- a/absl/strings/CMakeLists.txt +++ b/absl/strings/CMakeLists.txt
@@ -243,6 +243,7 @@ COPTS ${ABSL_TEST_COPTS} DEPS + absl::string_view absl::strings_internal absl::base absl::core_headers @@ -518,6 +519,7 @@ absl::utility absl::int128 absl::span + absl::strings_internal ) absl_cc_test(
diff --git a/absl/strings/internal/str_format/arg.cc b/absl/strings/internal/str_format/arg.cc index eeb2108..103c85d 100644 --- a/absl/strings/internal/str_format/arg.cc +++ b/absl/strings/internal/str_format/arg.cc
@@ -34,6 +34,7 @@ #include "absl/numeric/int128.h" #include "absl/strings/internal/str_format/extension.h" #include "absl/strings/internal/str_format/float_conversion.h" +#include "absl/strings/internal/utf8.h" #include "absl/strings/numbers.h" #include "absl/strings/string_view.h" @@ -311,68 +312,16 @@ conv.has_left_flag()); } -struct ShiftState { - bool saw_high_surrogate = false; - uint8_t bits = 0; -}; - -// Converts `v` from UTF-16 or UTF-32 to UTF-8 and writes to `buf`. `buf` is -// assumed to have enough space for the output. `s` is used to carry state -// between successive calls with a UTF-16 surrogate pair. Returns the number of -// chars written, or `static_cast<size_t>(-1)` on failure. -// -// This is basically std::wcrtomb(), but always outputting UTF-8 instead of -// respecting the current locale. -inline size_t WideToUtf8(wchar_t wc, char *buf, ShiftState &s) { - const auto v = static_cast<uint32_t>(wc); - if (v < 0x80) { - *buf = static_cast<char>(v); - return 1; - } else if (v < 0x800) { - *buf++ = static_cast<char>(0xc0 | (v >> 6)); - *buf = static_cast<char>(0x80 | (v & 0x3f)); - return 2; - } else if (v < 0xd800 || (v - 0xe000) < 0x2000) { - *buf++ = static_cast<char>(0xe0 | (v >> 12)); - *buf++ = static_cast<char>(0x80 | ((v >> 6) & 0x3f)); - *buf = static_cast<char>(0x80 | (v & 0x3f)); - return 3; - } else if ((v - 0x10000) < 0x100000) { - *buf++ = static_cast<char>(0xf0 | (v >> 18)); - *buf++ = static_cast<char>(0x80 | ((v >> 12) & 0x3f)); - *buf++ = static_cast<char>(0x80 | ((v >> 6) & 0x3f)); - *buf = static_cast<char>(0x80 | (v & 0x3f)); - return 4; - } else if (v < 0xdc00) { - s.saw_high_surrogate = true; - s.bits = static_cast<uint8_t>(v & 0x3); - const uint8_t high_bits = ((v >> 6) & 0xf) + 1; - *buf++ = static_cast<char>(0xf0 | (high_bits >> 2)); - *buf = - static_cast<char>(0x80 | static_cast<uint8_t>((high_bits & 0x3) << 4) | - static_cast<uint8_t>((v >> 2) & 0xf)); - return 2; - } else if (v < 0xe000 && s.saw_high_surrogate) { - *buf++ = static_cast<char>(0x80 | static_cast<uint8_t>(s.bits << 4) | - static_cast<uint8_t>((v >> 6) & 0xf)); - *buf = static_cast<char>(0x80 | (v & 0x3f)); - s.saw_high_surrogate = false; - s.bits = 0; - return 2; - } else { - return static_cast<size_t>(-1); - } -} - inline bool ConvertStringArg(const wchar_t *v, size_t len, const FormatConversionSpecImpl conv, FormatSinkImpl *sink) { FixedArray<char> mb(len * 4); - ShiftState s; + strings_internal::ShiftState s; size_t chars_written = 0; for (size_t i = 0; i < len; ++i) { - const size_t chars = WideToUtf8(v[i], &mb[chars_written], s); + const size_t chars = + strings_internal::WideToUtf8(v[i], &mb[chars_written], s); if (chars == static_cast<size_t>(-1)) { return false; } chars_written += chars; } @@ -382,8 +331,8 @@ bool ConvertWCharTImpl(wchar_t v, const FormatConversionSpecImpl conv, FormatSinkImpl *sink) { char mb[4]; - ShiftState s; - const size_t chars_written = WideToUtf8(v, mb, s); + strings_internal::ShiftState s; + const size_t chars_written = strings_internal::WideToUtf8(v, mb, s); return chars_written != static_cast<size_t>(-1) && !s.saw_high_surrogate && ConvertStringArg(string_view(mb, chars_written), conv, sink); }
diff --git a/absl/strings/internal/utf8.cc b/absl/strings/internal/utf8.cc index 7ecb93d..4370c7c 100644 --- a/absl/strings/internal/utf8.cc +++ b/absl/strings/internal/utf8.cc
@@ -16,6 +16,11 @@ #include "absl/strings/internal/utf8.h" +#include <cstddef> +#include <cstdint> + +#include "absl/base/config.h" + namespace absl { ABSL_NAMESPACE_BEGIN namespace strings_internal { @@ -48,6 +53,47 @@ } } +size_t WideToUtf8(wchar_t wc, char *buf, ShiftState &s) { + const auto v = static_cast<uint32_t>(wc); + if (v < 0x80) { + *buf = static_cast<char>(v); + return 1; + } else if (v < 0x800) { + *buf++ = static_cast<char>(0xc0 | (v >> 6)); + *buf = static_cast<char>(0x80 | (v & 0x3f)); + return 2; + } else if (v < 0xd800 || (v - 0xe000) < 0x2000) { + *buf++ = static_cast<char>(0xe0 | (v >> 12)); + *buf++ = static_cast<char>(0x80 | ((v >> 6) & 0x3f)); + *buf = static_cast<char>(0x80 | (v & 0x3f)); + return 3; + } else if ((v - 0x10000) < 0x100000) { + *buf++ = static_cast<char>(0xf0 | (v >> 18)); + *buf++ = static_cast<char>(0x80 | ((v >> 12) & 0x3f)); + *buf++ = static_cast<char>(0x80 | ((v >> 6) & 0x3f)); + *buf = static_cast<char>(0x80 | (v & 0x3f)); + return 4; + } else if (v < 0xdc00) { + s.saw_high_surrogate = true; + s.bits = static_cast<uint8_t>(v & 0x3); + const uint8_t high_bits = ((v >> 6) & 0xf) + 1; + *buf++ = static_cast<char>(0xf0 | (high_bits >> 2)); + *buf = + static_cast<char>(0x80 | static_cast<uint8_t>((high_bits & 0x3) << 4) | + static_cast<uint8_t>((v >> 2) & 0xf)); + return 2; + } else if (v < 0xe000 && s.saw_high_surrogate) { + *buf++ = static_cast<char>(0x80 | static_cast<uint8_t>(s.bits << 4) | + static_cast<uint8_t>((v >> 6) & 0xf)); + *buf = static_cast<char>(0x80 | (v & 0x3f)); + s.saw_high_surrogate = false; + s.bits = 0; + return 2; + } else { + return static_cast<size_t>(-1); + } +} + } // namespace strings_internal ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/strings/internal/utf8.h b/absl/strings/internal/utf8.h index 32fb109..f240408 100644 --- a/absl/strings/internal/utf8.h +++ b/absl/strings/internal/utf8.h
@@ -43,6 +43,20 @@ enum { kMaxEncodedUTF8Size = 4 }; size_t EncodeUTF8Char(char *buffer, char32_t utf8_char); +struct ShiftState { + bool saw_high_surrogate = false; + uint8_t bits = 0; +}; + +// Converts `wc` from UTF-16 or UTF-32 to UTF-8 and writes to `buf`. `buf` is +// assumed to have enough space for the output. `s` is used to carry state +// between successive calls with a UTF-16 surrogate pair. Returns the number of +// chars written, or `static_cast<size_t>(-1)` on failure. +// +// This is basically std::wcrtomb(), but always outputting UTF-8 instead of +// respecting the current locale. +size_t WideToUtf8(wchar_t wc, char *buf, ShiftState &s); + } // namespace strings_internal ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/strings/internal/utf8_test.cc b/absl/strings/internal/utf8_test.cc index 88dd503..62322dd 100644 --- a/absl/strings/internal/utf8_test.cc +++ b/absl/strings/internal/utf8_test.cc
@@ -14,14 +14,29 @@ #include "absl/strings/internal/utf8.h" +#include <cstddef> #include <cstdint> +#include <cstring> +#include <string> +#include <type_traits> #include <utility> +#include <vector> +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/base/port.h" +#include "absl/strings/string_view.h" namespace { +using ::absl::strings_internal::kMaxEncodedUTF8Size; +using ::absl::strings_internal::ShiftState; +using ::absl::strings_internal::WideToUtf8; +using ::testing::StartsWith; +using ::testing::TestParamInfo; +using ::testing::TestWithParam; +using ::testing::ValuesIn; + #if !defined(__cpp_char8_t) #if defined(__clang__) #pragma clang diagnostic push @@ -33,12 +48,12 @@ {0x00010000, u8"\U00010000"}, {0x0000FFFF, u8"\U0000FFFF"}, {0x0010FFFD, u8"\U0010FFFD"}}; - for (auto &test : tests) { + for (auto& test : tests) { char buf0[7] = {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}; char buf1[7] = {'\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF'}; - char *buf0_written = + char* buf0_written = &buf0[absl::strings_internal::EncodeUTF8Char(buf0, test.first)]; - char *buf1_written = + char* buf1_written = &buf1[absl::strings_internal::EncodeUTF8Char(buf1, test.first)]; int apparent_length = 7; while (buf0[apparent_length - 1] == '\x00' && @@ -63,4 +78,169 @@ #endif #endif // !defined(__cpp_char8_t) +struct WideToUtf8TestCase { + std::string description; + wchar_t input; + std::string expected_utf8_str; + size_t expected_bytes_written; + ShiftState initial_state = {false, 0}; + ShiftState expected_state = {false, 0}; +}; + +std::vector<WideToUtf8TestCase> GetWideToUtf8TestCases() { + constexpr size_t kError = static_cast<size_t>(-1); + std::vector<WideToUtf8TestCase> cases = { + {"ASCII_A", L'A', "A", 1}, + {"NullChar", L'\0', std::string("\0", 1), 1}, + {"ASCII_Max_7F", L'\x7F', "\x7F", 1}, + + {"TwoByte_Min_80", L'\u0080', "\xC2\x80", 2}, + {"PoundSign_A3", L'\u00A3', "\xC2\xA3", 2}, + {"TwoByte_Max_7FF", L'\u07FF', "\xDF\xBF", 2}, + + {"ThreeByte_Min_800", L'\u0800', "\xE0\xA0\x80", 3}, + {"EuroSign_20AC", L'\u20AC', "\xE2\x82\xAC", 3}, + {"BMP_MaxBeforeSurrogates_D7FF", L'\uD7FF', "\xED\x9F\xBF", 3}, + {"BMP_FFFF", L'\uFFFF', "\xEF\xBF\xBF", 3}, + + {"IsolatedHighSurr_D800", L'\xD800', "\xF0\x90", 2, {true, 0}, {true, 0}}, + {"IsolatedHighSurr_DBFF", L'\xDBFF', "\xF4\x8F", 2, {true, 3}, {true, 3}}, + + {"LowSurr_DC00_after_HighD800", L'\xDC00', "\x80\x80", 2, {true, 0}, {}}, + {"LowSurr_DFFD_after_HighDBFF", L'\xDFFD', "\xBF\xBD", 2, {true, 3}, {}}, + {"LowSurr_DC00_with_InitialState_saw_high_bits_1", + L'\xDC00', + "\x90\x80", + 2, + {true, 1}, + {}}, + + // Final state = initial on error. + {"Error_IsolatedLowSurr_DC00_NoPriorHigh", L'\xDC00', "", kError, {}, {}}, + {"Error_IsolatedLowSurr_DFFF_NoPriorHigh", L'\xDFFF', "", kError, {}, {}}, + +#if (defined(WCHAR_MAX) && WCHAR_MAX > 0xFFFF) + {"DirectSupplementaryChars_U10000", static_cast<wchar_t>(0x10000), + "\xF0\x90\x80\x80", 4}, + {"DirectSupplementaryChars_U10FFFD", static_cast<wchar_t>(0x10FFFD), + "\xF4\x8F\xBF\xBD", 4}, +#endif + }; + + wchar_t minus_one = static_cast<wchar_t>(-1); + if constexpr (sizeof(wchar_t) == 2) { + cases.push_back({"WChar_MinusOne_as_FFFF", minus_one, "\xEF\xBF\xBF", 3}); + } else { + cases.push_back( + {"Error_WChar_MinusOne_as_FFFFFFFF", minus_one, "", kError, {}, {}}); + } + + if constexpr (sizeof(wchar_t) >= 4) { +#ifdef WCHAR_MAX + if (static_cast<uintmax_t>(WCHAR_MAX) >= 0x110000UL) { + cases.push_back({"Error_OutOfRange_110000", + static_cast<wchar_t>(0x110000UL), + "", + kError, + {}, + {}}); + } +#else + cases.push_back({"Error_OutOfRange_110000_fallback", + static_cast<wchar_t>(0x110000UL), + "", + kError, + {}, + {}}); +#endif + } + return cases; +} + +class WideToUtf8ParamTest : public TestWithParam<WideToUtf8TestCase> {}; + +TEST_P(WideToUtf8ParamTest, SingleCharConversion) { + const auto& test_case = GetParam(); + ShiftState state = test_case.initial_state; + constexpr char kFillChar = '\xAB'; + std::string buffer(32, kFillChar); + + size_t bytes_written = WideToUtf8(test_case.input, buffer.data(), state); + + EXPECT_EQ(bytes_written, test_case.expected_bytes_written); + EXPECT_THAT(buffer, StartsWith(test_case.expected_utf8_str)); + + // The remaining bytes should be unchanged. + ASSERT_LT(test_case.expected_utf8_str.length(), buffer.size()); + EXPECT_EQ(buffer[test_case.expected_utf8_str.length()], kFillChar); + + EXPECT_EQ(state.saw_high_surrogate, + test_case.expected_state.saw_high_surrogate); + EXPECT_EQ(state.bits, test_case.expected_state.bits); +} + +INSTANTIATE_TEST_SUITE_P(WideCharToUtf8Conversion, WideToUtf8ParamTest, + ValuesIn(GetWideToUtf8TestCases()), + [](auto info) { return info.param.description; }); + +// Comprehensive test string for validating wchar_t to UTF-8 conversion. +// This string is designed to cover a variety of Unicode character types and +// sequences: +// 1. Basic ASCII characters (within names, numbers, and spacing). +// 2. Common 2-byte UTF-8 sequences: +// - Accented Latin characters (e.g., 'á' in "Holá"). +// - Hebrew text with combining vowel points (e.g., "שָׁלוֹם"). +// 3. Common 3-byte UTF-8 sequences: +// - Currency symbols (e.g., '€'). +// - CJK characters (e.g., "你好", "中"). +// - Components of complex emojis like the Zero Width Joiner (ZWJ) and +// Heart symbol. +// 4. Various 4-byte UTF-8 sequences (representing Supplementary Plane +// characters): +// - An emoji with a skin tone modifier ("👍🏻"). +// - A flag emoji composed of regional indicators ("🇺🇸"). +// - A complex ZWJ emoji sequence ("👩❤️💋👨") combining +// SP characters (👩, 💋, 👨) with BMP characters (ZWJ and ❤️). +// - These are critical for testing the correct handling of surrogate pairs +// when wchar_t is 2 bytes (e.g., on Windows). +// The goal is to ensure accurate conversion across a diverse set of +// characters. +// +// clang-format off +#define WIDE_STRING_LITERAL L"Holá €1 你好 שָׁלוֹם 👍🏻🇺🇸👩❤️💋👨 中" +#define UTF8_STRING_LITERAL u8"Holá €1 你好 שָׁלוֹם 👍🏻🇺🇸👩❤️💋👨 中" +// clang-format on + +absl::string_view GetUtf8TestString() { + // `u8""` forces UTF-8 encoding; MSVC will default to e.g. CP1252 (and warn) + // without it. However, the resulting character type differs between pre-C++20 + // (`char`) and C++20 (`char8_t`). So deduce the right character type for all + // C++ versions, init it with UTF-8, then `memcpy()` to get the result as a + // `char*` + static absl::string_view kUtf8TestString = [] { + using ConstChar8T = std::remove_reference_t<decltype(*u8"a")>; + constexpr ConstChar8T kOutputUtf8[] = UTF8_STRING_LITERAL; + static char output[sizeof kOutputUtf8]; + std::memcpy(output, kOutputUtf8, sizeof kOutputUtf8); + return output; + }(); + + return kUtf8TestString; +} + +TEST(WideToUtf8, FullString) { + std::string buffer(kMaxEncodedUTF8Size * sizeof(WIDE_STRING_LITERAL), '\0'); + char* buffer_ptr = buffer.data(); + + ShiftState state; + for (const wchar_t wc : WIDE_STRING_LITERAL) { + buffer_ptr += WideToUtf8(wc, buffer_ptr, state); + } + + EXPECT_THAT(buffer, StartsWith(GetUtf8TestString())); +} + +#undef WIDE_STRING_LITERAL +#undef UTF8_STRING_LITERAL + } // namespace