diff --git a/CMake/AbseilDll.cmake b/CMake/AbseilDll.cmake index c285038..18f4f33 100644 --- a/CMake/AbseilDll.cmake +++ b/CMake/AbseilDll.cmake
@@ -352,7 +352,6 @@ "strings/internal/pow10_helper.cc" "strings/internal/pow10_helper.h" "strings/internal/resize_uninitialized.h" - "strings/internal/stl_type_traits.h" "strings/internal/str_format/arg.cc" "strings/internal/str_format/arg.h" "strings/internal/str_format/bind.cc" @@ -471,12 +470,14 @@ "strings/string_view.h" ) -if(MSVC) +if(WIN32) list(APPEND ABSL_INTERNAL_DLL_FILES "time/internal/cctz/src/time_zone_name_win.cc" "time/internal/cctz/src/time_zone_name_win.h" ) -else() +endif() + +if(NOT MSVC) list(APPEND ABSL_INTERNAL_DLL_FILES "flags/commandlineflag.cc" "flags/commandlineflag.h"
diff --git a/MODULE.bazel b/MODULE.bazel index cc60d22..6dfccce 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -19,9 +19,11 @@ version = "head", ) -cc_configure = use_extension("@rules_cc//cc:extensions.bzl", - "cc_configure_extension", - dev_dependency = True) +cc_configure = use_extension( + "@rules_cc//cc:extensions.bzl", + "cc_configure_extension", + dev_dependency = True, +) use_repo(cc_configure, "local_config_cc") bazel_dep(name = "rules_cc", version = "0.2.18") @@ -40,3 +42,11 @@ name = "googletest", version = "1.17.0.bcr.2", ) + +# Note: Gloop is NOT a dev_dependency, but should never be used directly. It is only included here +# so that we can give visibility into Abseil internals. +bazel_dep( + name = "gloop", + version = "20260708.rc1", + repo_name = "do_not_use_for_gloop_visibility_only", +)
diff --git a/absl/BUILD.bazel b/absl/BUILD.bazel index e33d648..980563a 100644 --- a/absl/BUILD.bazel +++ b/absl/BUILD.bazel
@@ -16,7 +16,7 @@ load("@bazel_skylib//lib:selects.bzl", "selects") -package(default_visibility = ["//visibility:public"]) +package(default_visibility = ["//visibility:private"]) licenses(["notice"]) @@ -60,9 +60,3 @@ ], visibility = [":__subpackages__"], ) - -# Expose internals to privileged external repositories. -package_group( - name = "friends", - packages = [], -)
diff --git a/absl/algorithm/BUILD.bazel b/absl/algorithm/BUILD.bazel index f00d5d0..337f035 100644 --- a/absl/algorithm/BUILD.bazel +++ b/absl/algorithm/BUILD.bazel
@@ -24,7 +24,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -39,6 +39,7 @@ hdrs = ["algorithm.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -66,6 +67,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":algorithm", "//absl/base:config",
diff --git a/absl/base/BUILD.bazel b/absl/base/BUILD.bazel index ea4db17..2ecb1a6 100644 --- a/absl/base/BUILD.bazel +++ b/absl/base/BUILD.bazel
@@ -25,7 +25,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -56,7 +56,6 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", ], deps = [":config"], ) @@ -71,7 +70,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/util/sysinfo/topology:__subpackages__", ], deps = [ ":config", @@ -88,7 +87,8 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/util/coding:__subpackages__", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", ], deps = [ ":config", @@ -102,6 +102,7 @@ hdrs = ["log_severity.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":config", ":core_headers", @@ -113,6 +114,7 @@ hdrs = ["no_destructor.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":config", ":nullability", @@ -124,6 +126,7 @@ hdrs = ["nullability.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [":config"], ) @@ -149,7 +152,13 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", + "@do_not_use_for_gloop_visibility_only//gloop/concurrent/percpu:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/concurrent/rcu:__subpackages__", + "@do_not_use_for_gloop_visibility_only//gloop/strings:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/thread/fiber:__subpackages__", + "@do_not_use_for_gloop_visibility_only//gloop/util/hash:__subpackages__", + "@do_not_use_for_gloop_visibility_only//gloop/util/process:__subpackages__", ], deps = [ ":atomic_hook", @@ -173,8 +182,9 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/base:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/base:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/thread/fiber:__pkg__", ], deps = [ ":base_internal", @@ -192,6 +202,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], ) cc_library( @@ -204,7 +215,8 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", + "@do_not_use_for_gloop_visibility_only//gloop/util/random:__pkg__", ], deps = [ ":base_internal", @@ -222,6 +234,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":config", ":core_headers", @@ -240,6 +253,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":config", ], @@ -286,7 +300,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", ], deps = [ ":config", @@ -331,6 +345,7 @@ "@rules_cc//cc/compiler:emscripten": [], "//conditions:default": ["-pthread"], }) + ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":atomic_hook", ":base_internal", @@ -425,6 +440,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":config", ":raw_logging_internal", @@ -466,7 +482,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", ], deps = [ ":config", @@ -480,7 +496,6 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", ], ) @@ -491,6 +506,7 @@ hdrs = ["internal/exception_safety_testing.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//absl:__subpackages__"], deps = [ ":config", ":pretty_function", @@ -523,6 +539,9 @@ srcs = ["spinlock_test_common.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "@do_not_use_for_gloop_visibility_only//gloop/base:__pkg__", + ], deps = [ ":base", ":base_internal", @@ -561,7 +580,6 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/base:__pkg__", ], deps = [ @@ -597,7 +615,6 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", ], deps = [ ":base", @@ -882,7 +899,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", ], deps = [ ":config", @@ -924,6 +941,7 @@ hdrs = ["fast_type_id.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":config", ], @@ -950,6 +968,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":config", ":core_headers", @@ -981,7 +1000,6 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", ], deps = [ ":config", @@ -1042,7 +1060,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", ], deps = [ ":config", @@ -1072,7 +1090,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/perftools/tracing:__subpackages__", ], deps = [ "//absl/base:config", @@ -1085,6 +1103,10 @@ hdrs = ["internal/iterator_traits_test_helper.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + "//cloud/blockstore/metallica/util:__pkg__", + ], deps = [":config"], )
diff --git a/absl/base/attributes.h b/absl/base/attributes.h index 5887fca..525824a 100644 --- a/absl/base/attributes.h +++ b/absl/base/attributes.h
@@ -934,6 +934,23 @@ #define ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(Owner) #endif +// Internal attribute; name and documentation TBD. +// +// See the upstream documentation: +// https://clang.llvm.org/docs/AttributeReference.html#lifetime_capture_by_this +// +// Note: ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this) is deprecated. Use +// ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS instead. +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::lifetime_capture_by_this) +#define ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS \ + [[clang::lifetime_capture_by_this]] +#elif ABSL_HAVE_CPP_ATTRIBUTE(clang::lifetime_capture_by) +#define ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS \ + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this) +#else +#define ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS +#endif + // ABSL_ATTRIBUTE_VIEW indicates that a type is solely a "view" of data that it // points to, similarly to a span, string_view, or other non-owning reference // type.
diff --git a/absl/base/exception_safety_testing_test.cc b/absl/base/exception_safety_testing_test.cc index 5e98c5f..7cf8c36 100644 --- a/absl/base/exception_safety_testing_test.cc +++ b/absl/base/exception_safety_testing_test.cc
@@ -869,14 +869,14 @@ } struct LeaksIfCtorThrows : private exceptions_internal::TrackedObject { - LeaksIfCtorThrows() : TrackedObject(ABSL_PRETTY_FUNCTION) { + LeaksIfCtorThrows() : TrackedObject(ABSL_INTERNAL_PRETTY_FUNCTION) { ++counter; ThrowingValue<> v; static_cast<void>(v); --counter; } LeaksIfCtorThrows(const LeaksIfCtorThrows&) noexcept - : TrackedObject(ABSL_PRETTY_FUNCTION) {} + : TrackedObject(ABSL_INTERNAL_PRETTY_FUNCTION) {} static int counter; }; int LeaksIfCtorThrows::counter = 0; @@ -888,7 +888,7 @@ } struct Tracked : private exceptions_internal::TrackedObject { - Tracked() : TrackedObject(ABSL_PRETTY_FUNCTION) {} + Tracked() : TrackedObject(ABSL_INTERNAL_PRETTY_FUNCTION) {} }; TEST(ConstructorTrackerTest, CreatedBefore) {
diff --git a/absl/base/internal/exception_safety_testing.h b/absl/base/internal/exception_safety_testing.h index 4cccf96..330fff9 100644 --- a/absl/base/internal/exception_safety_testing.h +++ b/absl/base/internal/exception_safety_testing.h
@@ -217,7 +217,7 @@ public: ThrowingBool(bool b) noexcept : b_(b) {} // NOLINT(runtime/explicit) operator bool() const { // NOLINT - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return b_; } @@ -267,7 +267,7 @@ public: ThrowingValue() : TrackedObject(GetInstanceString(kDefaultValue)) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ = kDefaultValue; } @@ -275,7 +275,7 @@ IsSpecified(TypeSpec::kNoThrowCopy)) : TrackedObject(GetInstanceString(other.dummy_)) { if (!IsSpecified(TypeSpec::kNoThrowCopy)) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); } dummy_ = other.dummy_; } @@ -284,13 +284,13 @@ IsSpecified(TypeSpec::kNoThrowMove)) : TrackedObject(GetInstanceString(other.dummy_)) { if (!IsSpecified(TypeSpec::kNoThrowMove)) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); } dummy_ = other.dummy_; } explicit ThrowingValue(int i) : TrackedObject(GetInstanceString(i)) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ = i; } @@ -304,7 +304,7 @@ IsSpecified(TypeSpec::kNoThrowCopy)) { dummy_ = kBadValue; if (!IsSpecified(TypeSpec::kNoThrowCopy)) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); } dummy_ = other.dummy_; return *this; @@ -314,7 +314,7 @@ IsSpecified(TypeSpec::kNoThrowMove)) { dummy_ = kBadValue; if (!IsSpecified(TypeSpec::kNoThrowMove)) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); } dummy_ = other.dummy_; return *this; @@ -322,73 +322,73 @@ // Arithmetic Operators ThrowingValue operator+(const ThrowingValue& other) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(dummy_ + other.dummy_, nothrow_ctor); } ThrowingValue operator+() const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(dummy_, nothrow_ctor); } ThrowingValue operator-(const ThrowingValue& other) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(dummy_ - other.dummy_, nothrow_ctor); } ThrowingValue operator-() const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(-dummy_, nothrow_ctor); } ThrowingValue& operator++() { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); ++dummy_; return *this; } ThrowingValue operator++(int) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); auto out = ThrowingValue(dummy_, nothrow_ctor); ++dummy_; return out; } ThrowingValue& operator--() { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); --dummy_; return *this; } ThrowingValue operator--(int) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); auto out = ThrowingValue(dummy_, nothrow_ctor); --dummy_; return out; } ThrowingValue operator*(const ThrowingValue& other) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(dummy_ * other.dummy_, nothrow_ctor); } ThrowingValue operator/(const ThrowingValue& other) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(dummy_ / other.dummy_, nothrow_ctor); } ThrowingValue operator%(const ThrowingValue& other) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(dummy_ % other.dummy_, nothrow_ctor); } ThrowingValue operator<<(int shift) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(dummy_ << shift, nothrow_ctor); } ThrowingValue operator>>(int shift) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(dummy_ >> shift, nothrow_ctor); } @@ -397,129 +397,129 @@ // types/containers requires T to be convertible to bool. friend ThrowingBool operator==(const ThrowingValue& a, const ThrowingValue& b) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return a.dummy_ == b.dummy_; } friend ThrowingBool operator!=(const ThrowingValue& a, const ThrowingValue& b) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return a.dummy_ != b.dummy_; } friend ThrowingBool operator<(const ThrowingValue& a, const ThrowingValue& b) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return a.dummy_ < b.dummy_; } friend ThrowingBool operator<=(const ThrowingValue& a, const ThrowingValue& b) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return a.dummy_ <= b.dummy_; } friend ThrowingBool operator>(const ThrowingValue& a, const ThrowingValue& b) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return a.dummy_ > b.dummy_; } friend ThrowingBool operator>=(const ThrowingValue& a, const ThrowingValue& b) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return a.dummy_ >= b.dummy_; } // Logical Operators ThrowingBool operator!() const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return !dummy_; } ThrowingBool operator&&(const ThrowingValue& other) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return dummy_ && other.dummy_; } ThrowingBool operator||(const ThrowingValue& other) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return dummy_ || other.dummy_; } // Bitwise Logical Operators ThrowingValue operator~() const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(~dummy_, nothrow_ctor); } ThrowingValue operator&(const ThrowingValue& other) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(dummy_ & other.dummy_, nothrow_ctor); } ThrowingValue operator|(const ThrowingValue& other) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(dummy_ | other.dummy_, nothrow_ctor); } ThrowingValue operator^(const ThrowingValue& other) const { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return ThrowingValue(dummy_ ^ other.dummy_, nothrow_ctor); } // Compound Assignment operators ThrowingValue& operator+=(const ThrowingValue& other) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ += other.dummy_; return *this; } ThrowingValue& operator-=(const ThrowingValue& other) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ -= other.dummy_; return *this; } ThrowingValue& operator*=(const ThrowingValue& other) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ *= other.dummy_; return *this; } ThrowingValue& operator/=(const ThrowingValue& other) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ /= other.dummy_; return *this; } ThrowingValue& operator%=(const ThrowingValue& other) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ %= other.dummy_; return *this; } ThrowingValue& operator&=(const ThrowingValue& other) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ &= other.dummy_; return *this; } ThrowingValue& operator|=(const ThrowingValue& other) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ |= other.dummy_; return *this; } ThrowingValue& operator^=(const ThrowingValue& other) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ ^= other.dummy_; return *this; } ThrowingValue& operator<<=(int shift) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ <<= shift; return *this; } ThrowingValue& operator>>=(int shift) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ >>= shift; return *this; } @@ -529,12 +529,12 @@ // Stream operators friend std::ostream& operator<<(std::ostream& os, const ThrowingValue& tv) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return os << GetInstanceString(tv.dummy_); } friend std::istream& operator>>(std::istream& is, const ThrowingValue&) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return is; } @@ -542,7 +542,7 @@ static void* operator new(size_t s) noexcept( IsSpecified(TypeSpec::kNoThrowNew)) { if (!IsSpecified(TypeSpec::kNoThrowNew)) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION, true); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION, true); } return ::operator new(s); } @@ -550,7 +550,7 @@ static void* operator new[](size_t s) noexcept( IsSpecified(TypeSpec::kNoThrowNew)) { if (!IsSpecified(TypeSpec::kNoThrowNew)) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION, true); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION, true); } return ::operator new[](s); } @@ -559,7 +559,7 @@ static void* operator new(size_t s, Args&&... args) noexcept( IsSpecified(TypeSpec::kNoThrowNew)) { if (!IsSpecified(TypeSpec::kNoThrowNew)) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION, true); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION, true); } return ::operator new(s, std::forward<Args>(args)...); } @@ -568,7 +568,7 @@ static void* operator new[](size_t s, Args&&... args) noexcept( IsSpecified(TypeSpec::kNoThrowNew)) { if (!IsSpecified(TypeSpec::kNoThrowNew)) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION, true); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION, true); } return ::operator new[](s, std::forward<Args>(args)...); } @@ -654,7 +654,7 @@ using is_always_equal = std::false_type; ThrowingAllocator() : TrackedObject(GetInstanceString(next_id_)) { - exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); dummy_ = std::make_shared<const int>(next_id_++); } @@ -705,7 +705,7 @@ pointer allocate(size_type n) noexcept( IsSpecified(AllocSpec::kNoThrowAllocate)) { - ReadStateAndMaybeThrow(ABSL_PRETTY_FUNCTION); + ReadStateAndMaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return static_cast<pointer>(::operator new(n * sizeof(T))); } @@ -722,7 +722,7 @@ template <typename U, typename... Args> void construct(U* ptr, Args&&... args) noexcept( IsSpecified(AllocSpec::kNoThrowAllocate)) { - ReadStateAndMaybeThrow(ABSL_PRETTY_FUNCTION); + ReadStateAndMaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); ::new (static_cast<void*>(ptr)) U(std::forward<Args>(args)...); } @@ -738,7 +738,7 @@ ThrowingAllocator select_on_container_copy_construction() noexcept( IsSpecified(AllocSpec::kNoThrowAllocate)) { - ReadStateAndMaybeThrow(ABSL_PRETTY_FUNCTION); + ReadStateAndMaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION); return *this; }
diff --git a/absl/base/internal/pretty_function.h b/absl/base/internal/pretty_function.h index 35d5167..fbc997a 100644 --- a/absl/base/internal/pretty_function.h +++ b/absl/base/internal/pretty_function.h
@@ -15,17 +15,18 @@ #ifndef ABSL_BASE_INTERNAL_PRETTY_FUNCTION_H_ #define ABSL_BASE_INTERNAL_PRETTY_FUNCTION_H_ -// ABSL_PRETTY_FUNCTION +// ABSL_INTERNAL_PRETTY_FUNCTION // // In C++11, __func__ gives the undecorated name of the current function. That // is, "main", not "int main()". Various compilers give extra macros to get the // decorated function name, including return type and arguments, to -// differentiate between overload sets. ABSL_PRETTY_FUNCTION is a portable -// version of these macros which forwards to the correct macro on each compiler. +// differentiate between overload sets. ABSL_INTERNAL_PRETTY_FUNCTION is a +// portable version of these macros which forwards to the correct macro on each +// compiler. #if defined(_MSC_VER) -#define ABSL_PRETTY_FUNCTION __FUNCSIG__ +#define ABSL_INTERNAL_PRETTY_FUNCTION __FUNCSIG__ #elif defined(__GNUC__) -#define ABSL_PRETTY_FUNCTION __PRETTY_FUNCTION__ +#define ABSL_INTERNAL_PRETTY_FUNCTION __PRETTY_FUNCTION__ #else #error "Unsupported compiler" #endif
diff --git a/absl/base/internal/spinlock.h b/absl/base/internal/spinlock.h index d535093..81210ca 100644 --- a/absl/base/internal/spinlock.h +++ b/absl/base/internal/spinlock.h
@@ -252,7 +252,7 @@ : public std::lock_guard<SpinLock> { public: inline explicit SpinLockHolder( - SpinLock& l ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + SpinLock& l ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) ABSL_EXCLUSIVE_LOCK_FUNCTION(l) : std::lock_guard<SpinLock>(l) {} ABSL_DEPRECATE_AND_INLINE()
diff --git a/absl/base/internal/sysinfo.cc b/absl/base/internal/sysinfo.cc index cd08e51..b103f49 100644 --- a/absl/base/internal/sysinfo.cc +++ b/absl/base/internal/sysinfo.cc
@@ -378,28 +378,36 @@ // NominalCPUFrequency() may be called before main() and before malloc is // properly initialized, therefore this must not allocate memory. double NominalCPUFrequency() { - base_internal::LowLevelCallOnce( - &init_nominal_cpu_frequency_once, - []() { nominal_cpu_frequency = GetNominalCPUFrequency(); }); + base_internal::LowLevelCallOnce(&init_nominal_cpu_frequency_once, []() { + nominal_cpu_frequency = GetNominalCPUFrequency(); + }); return nominal_cpu_frequency; } #if defined(_WIN32) -pid_t GetTID() { - return pid_t{GetCurrentThreadId()}; -} +pid_t GetTID() { return pid_t{GetCurrentThreadId()}; } #elif defined(__linux__) +#ifdef __ANDROID__ +#if __ANDROID_API__ >= 21 +#define ABSL_INTERNAL_HAVE_GETTID 1 +#endif +#endif +#ifdef ABSL_INTERNAL_HAVE_GETTID +pid_t GetTID() { + return static_cast<pid_t>(gettid()); +} +#else #ifndef SYS_gettid #define SYS_gettid __NR_gettid #endif -pid_t GetTID() { - return static_cast<pid_t>(syscall(SYS_gettid)); -} +pid_t GetTID() { return static_cast<pid_t>(syscall(SYS_gettid)); } +#endif +#undef ABSL_INTERNAL_HAVE_GETTID #elif defined(__akaros__) pid_t GetTID() { @@ -419,9 +427,8 @@ // TODO(dcross): Akaros anticipates moving the thread ID to the uthread // structure at some point. We should modify this code to remove the cast // when that happens. - if (in_vcore_context()) - return 0; - return reinterpret_cast<struct pthread_tcb *>(current_uthread)->id; + if (in_vcore_context()) return 0; + return reinterpret_cast<struct pthread_tcb*>(current_uthread)->id; } #elif defined(__myriad2__) @@ -480,12 +487,23 @@ // userspace construct) to avoid unnecessary system calls. Without this caching, // it can take roughly 98ns, while it takes roughly 1ns with this caching. pid_t GetCachedTID() { -#ifdef ABSL_HAVE_THREAD_LOCAL +#ifdef __ANDROID__ +// NDK defaults to emulated TLS for API < 29, and native ELF TLS for API >= 29. +// Emulated TLS is slower than bionic's internal caching. +#if __ANDROID_API__ < 29 +#define ABSL_INTERNAL_USING_EMULATED_TLS 1 +#endif +#endif + +#if defined(ABSL_HAVE_THREAD_LOCAL) && \ + !defined(ABSL_INTERNAL_USING_EMULATED_TLS) static thread_local pid_t thread_id = GetTID(); return thread_id; #else return GetTID(); -#endif // ABSL_HAVE_THREAD_LOCAL +#endif // defined(ABSL_HAVE_THREAD_LOCAL) && + // !defined(ABSL_INTERNAL_USING_EMULATED_TLS) +#undef ABSL_INTERNAL_USING_EMULATED_TLS } } // namespace base_internal
diff --git a/absl/base/optimization.h b/absl/base/optimization.h index 9b52a92..8247ef2 100644 --- a/absl/base/optimization.h +++ b/absl/base/optimization.h
@@ -266,7 +266,8 @@ // int y = x / 16; // #if !defined(NDEBUG) -#define ABSL_ASSUME(cond) assert(cond) +#define ABSL_ASSUME(cond) \ + (ABSL_PREDICT_TRUE((cond)) ? void() : assert(false && #cond)) // NOLINT #elif ABSL_HAVE_BUILTIN(__builtin_assume) #define ABSL_ASSUME(cond) __builtin_assume(cond) #elif defined(_MSC_VER)
diff --git a/absl/cleanup/BUILD.bazel b/absl/cleanup/BUILD.bazel index 461ab37..dad0d6b 100644 --- a/absl/cleanup/BUILD.bazel +++ b/absl/cleanup/BUILD.bazel
@@ -22,7 +22,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -37,6 +37,7 @@ hdrs = ["internal/cleanup.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], # Should be private, but cleanup is required. deps = [ "//absl/base:core_headers", "//absl/base:hardening", @@ -51,6 +52,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":cleanup_internal", "//absl/base:config",
diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index a386a53..e535d9e 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel
@@ -25,7 +25,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -40,6 +40,7 @@ hdrs = ["internal/compressed_tuple.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], # Should be private, but cleanup is required. deps = [ "//absl/utility", ], @@ -65,6 +66,7 @@ hdrs = ["fixed_array.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":compressed_tuple", "//absl/algorithm", @@ -147,6 +149,7 @@ hdrs = ["inlined_vector.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":inlined_vector_internal", "//absl/algorithm", @@ -230,7 +233,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", ], deps = ["//absl/types:compare"], ) @@ -259,6 +262,7 @@ hdrs = ["flat_hash_map.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":container_memory", ":hash_container_defaults", @@ -297,6 +301,7 @@ hdrs = ["flat_hash_set.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":container_memory", ":hash_container_defaults", @@ -339,6 +344,7 @@ hdrs = ["node_hash_map.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":container_memory", ":hash_container_defaults", @@ -376,6 +382,7 @@ hdrs = ["node_hash_set.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":container_memory", ":hash_container_defaults", @@ -414,6 +421,11 @@ hdrs = ["internal/container_memory.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + "@do_not_use_for_gloop_visibility_only//gloop/strings:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__pkg__", + ], deps = [ "//absl/base:config", "//absl/hash", @@ -447,7 +459,7 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", ], deps = [ ":common", @@ -464,6 +476,7 @@ hdrs = ["hash_container_defaults.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":hash_function_defaults", "//absl/base:config", @@ -514,6 +527,9 @@ hdrs = ["internal/hash_policy_testing.h"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", + ], deps = [ "//absl/hash", "//absl/strings", @@ -594,6 +610,9 @@ hdrs = ["internal/hashtable_debug_hooks.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/container:__pkg__", + ], deps = [ "//absl/base:config", ], @@ -608,6 +627,9 @@ hdrs = ["internal/hashtablez_sampler.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], deps = [ "//absl/base", "//absl/base:config", @@ -650,6 +672,10 @@ hdrs = ["internal/node_slot_policy.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__pkg__", + ], deps = ["//absl/base:config"], ) @@ -672,6 +698,10 @@ hdrs = ["internal/raw_hash_map.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__pkg__", + ], deps = [ ":common_policy_traits", ":container_memory", @@ -688,6 +718,10 @@ hdrs = ["internal/common.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/container:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__pkg__", + ], deps = [ "//absl/meta:type_traits", "//absl/types:optional", @@ -699,6 +733,9 @@ hdrs = ["internal/hashtable_control_bytes.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/container:__pkg__", + ], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -747,6 +784,7 @@ hdrs = ["internal/raw_hash_set.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], # Should be private, but requires cleanup deps = [ ":common", ":common_policy_traits", @@ -884,6 +922,7 @@ hdrs = ["internal/layout.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], # Should be private, requires cleanup deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -948,8 +987,8 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/container:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", ], deps = [ ":hash_generator_testing", @@ -966,8 +1005,8 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/container:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", ], deps = [ ":hash_generator_testing", @@ -983,8 +1022,8 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/container:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", ], deps = [ ":hash_generator_testing", @@ -1000,8 +1039,8 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/container:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__pkg__", ], deps = [ ":hash_generator_testing", @@ -1019,7 +1058,6 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/container:__pkg__", ], deps = [ @@ -1035,7 +1073,6 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/container:__pkg__", ], deps = [ @@ -1051,8 +1088,8 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/container:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__pkg__", ], deps = [ ":hash_generator_testing", @@ -1068,8 +1105,8 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/container:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__pkg__", ], deps = [ ":hash_generator_testing", @@ -1247,7 +1284,6 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/container:__pkg__", ], deps = [ @@ -1262,6 +1298,7 @@ hdrs = ["linked_hash_set.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":common", "//absl/base:config", @@ -1278,6 +1315,7 @@ deps = [ ":heterogeneous_lookup_testing", ":linked_hash_set", + ":test_allocator", "//absl/base:config", "//absl/container:hash_generator_testing", "//absl/container:hash_policy_testing", @@ -1314,6 +1352,7 @@ hdrs = ["linked_hash_map.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":common", "//absl/base:config", @@ -1331,6 +1370,7 @@ deps = [ ":heterogeneous_lookup_testing", ":linked_hash_map", + ":test_allocator", "//absl/base:config", "//absl/base:exception_testing", "//absl/container:hash_generator_testing", @@ -1367,6 +1407,7 @@ name = "chunked_queue", srcs = ["internal/chunked_queue.h"], hdrs = ["chunked_queue.h"], + visibility = ["//visibility:public"], deps = [ ":layout", "//absl/base:config", @@ -1385,6 +1426,7 @@ ":test_allocator", "//absl/base:core_headers", "//absl/base:hardening", + "//absl/base:throw_delegate", "//absl/strings", "@googletest//:gtest", "@googletest//:gtest_main",
diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 005548f..2f89efe 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt
@@ -1155,6 +1155,7 @@ absl::hash_generator_testing absl::hash_policy_testing absl::string_view + absl::test_allocator absl::test_instance_tracker absl::unordered_set_constructor_test absl::unordered_set_lookup_test @@ -1197,6 +1198,7 @@ absl::hash_generator_testing absl::hash_policy_testing absl::string_view + absl::test_allocator absl::test_instance_tracker absl::unordered_set_constructor_test absl::unordered_set_lookup_test @@ -1239,5 +1241,6 @@ absl::hardening absl::strings absl::test_allocator + absl::throw_delegate GTest::gmock_main )
diff --git a/absl/container/chunked_queue.h b/absl/container/chunked_queue.h index 8503825..0e686ca 100644 --- a/absl/container/chunked_queue.h +++ b/absl/container/chunked_queue.h
@@ -154,8 +154,8 @@ } void IncrBy(size_t n) { - while (ptr + n > limit) { - n -= limit - ptr; + while (n > static_cast<size_t>(limit - ptr)) { + n -= static_cast<size_t>(limit - ptr); *this = iterator_common(block->next()); } ptr += n; @@ -417,6 +417,8 @@ T* storage = AllocateBack(); AllocatorTraits::construct(alloc_and_size_.allocator(), storage, std::forward<A>(args)...); + ++alloc_and_size_.size; + ++tail_.ptr; return *storage; } @@ -632,15 +634,16 @@ template <typename T, size_t BLo, size_t BHi, typename Allocator> void chunked_queue<T, BLo, BHi, Allocator>::resize(size_t new_size) { while (new_size > size()) { - ptrdiff_t to_add = new_size - size(); if (tail_.ptr == tail_.limit) { AddTailBlock(); } + size_t to_add = (std::min)(new_size - size(), + static_cast<size_t>(tail_.limit - tail_.ptr)); T* start = tail_.ptr; - T* limit = (std::min)(tail_.limit, start + to_add); + T* limit = start + to_add; Construct(start, limit); tail_.ptr = limit; - alloc_and_size_.size += limit - start; + alloc_and_size_.size += to_add; } if (size() == new_size) { return; @@ -671,8 +674,7 @@ if (tail_.ptr == tail_.limit) { AddTailBlock(); } - ++alloc_and_size_.size; - return tail_.ptr++; + return tail_.ptr; } template <typename T, size_t BLo, size_t BHi, typename Allocator>
diff --git a/absl/container/chunked_queue_test.cc b/absl/container/chunked_queue_test.cc index 4ba13c9..9540ff9 100644 --- a/absl/container/chunked_queue_test.cc +++ b/absl/container/chunked_queue_test.cc
@@ -20,8 +20,10 @@ #include <deque> #include <forward_list> #include <iterator> +#include <limits> #include <list> #include <memory> +#include <new> #include <string> #include <type_traits> #include <utility> @@ -31,6 +33,7 @@ #include "gtest/gtest.h" #include "absl/base/internal/hardening.h" #include "absl/base/macros.h" +#include "absl/base/throw_delegate.h" #include "absl/container/internal/test_allocator.h" #include "absl/strings/str_cat.h" @@ -435,6 +438,51 @@ EXPECT_EQ(2, q.size()); } +template <class T> +struct LimitedAllocator { + using value_type = T; + int* alloc_count; + int max_allocs; + + explicit LimitedAllocator(int* count, int max) + : alloc_count(count), max_allocs(max) {} + template <class U> + LimitedAllocator(const LimitedAllocator<U>& other) + : alloc_count(other.alloc_count), max_allocs(other.max_allocs) {} + + T* allocate(size_t n) { + if (*alloc_count >= max_allocs) { + absl::ThrowStdBadAlloc(); + } + ++*alloc_count; + return std::allocator<T>().allocate(n); + } + + void deallocate(T* p, size_t n) { + std::allocator<T>().deallocate(p, n); + } + + template <class U> + bool operator==(const LimitedAllocator<U>& other) const { + return alloc_count == other.alloc_count; + } + template <class U> + bool operator!=(const LimitedAllocator<U>& other) const { + return !(*this == other); + } +}; + +TEST(ChunkedQueue, ResizeOverflowSafe) { + int alloc_count = 0; + absl::chunked_queue<int64_t, 0, 0, LimitedAllocator<int64_t>> q( + LimitedAllocator<int64_t>(&alloc_count, 5)); +#ifdef ABSL_HAVE_EXCEPTIONS + EXPECT_THROW(q.resize(std::numeric_limits<size_t>::max()), std::bad_alloc); +#else + EXPECT_DEATH_IF_SUPPORTED(q.resize(std::numeric_limits<size_t>::max()), ""); +#endif +} + TEST(ChunkedQueue, MaxSize) { absl::chunked_queue<int64_t> q; EXPECT_GE(q.max_size(), @@ -767,4 +815,57 @@ EXPECT_DEATH_IF_SUPPORTED(cq.back(), ""); } +#ifdef ABSL_HAVE_EXCEPTIONS +struct ThrowingCtor { + int* ctor_count; + int* dtor_count; + int value; + + explicit ThrowingCtor(int v, bool should_throw, int* c_count, int* d_count) + : ctor_count(c_count), dtor_count(d_count), value(v) { + if (should_throw) { + throw 0; + } + ++*ctor_count; + } + ThrowingCtor(const ThrowingCtor& other) + : ctor_count(other.ctor_count), + dtor_count(other.dtor_count), + value(other.value) { + ++*ctor_count; + } + ThrowingCtor(ThrowingCtor&& other) noexcept + : ctor_count(other.ctor_count), + dtor_count(other.dtor_count), + value(other.value) { + ++*ctor_count; + } + ~ThrowingCtor() { ++*dtor_count; } +}; + +TEST(ChunkedQueue, StrongExceptionSafetyEmplaceBack) { + int ctor_count = 0; + int dtor_count = 0; + absl::chunked_queue<ThrowingCtor> q; + q.emplace_back(10, false, &ctor_count, &dtor_count); + q.emplace_back(20, false, &ctor_count, &dtor_count); + ASSERT_EQ(q.size(), 2); + ASSERT_EQ(ctor_count, 2); + ASSERT_EQ(dtor_count, 0); + + try { + q.emplace_back(30, true, &ctor_count, &dtor_count); + } catch (...) { + } + + EXPECT_EQ(q.size(), 2); + EXPECT_EQ(ctor_count, 2); + EXPECT_EQ(dtor_count, 0); + + q.clear(); + EXPECT_EQ(q.size(), 0); + EXPECT_EQ(dtor_count, 2); +} +#endif + } // namespace
diff --git a/absl/container/flat_hash_map.h b/absl/container/flat_hash_map.h index af9018d..f8c3ba7 100644 --- a/absl/container/flat_hash_map.h +++ b/absl/container/flat_hash_map.h
@@ -71,6 +71,15 @@ // * Contains a `capacity()` member function indicating the number of element // slots (open, deleted, and empty) within the hash map. // * Returns `void` from the `erase(iterator)` overload. +// * Constructors accept reservation size as an optional argument instead of +// bucket count. Reservation size is the number of elements that fits in the +// map before rehash. +// * insert/emplace and other modification functions return special iterator +// that doesn't support iteration. std::next(it) for such iterators would +// always point to the end(). +// +// TODO(b/519468416): copy redacted version of notable differences to +// node_hash_* files. // // By default, `flat_hash_map` uses the `absl::Hash` hashing framework. // All fundamental and Abseil types that support the `absl::Hash` framework have
diff --git a/absl/container/flat_hash_set.h b/absl/container/flat_hash_set.h index 60f661b..bd24dde 100644 --- a/absl/container/flat_hash_set.h +++ b/absl/container/flat_hash_set.h
@@ -71,6 +71,12 @@ // * Contains a `capacity()` member function indicating the number of element // slots (open, deleted, and empty) within the hash set. // * Returns `void` from the `erase(iterator)` overload. +// * Constructors accept reservation size as an optional argument instead of +// bucket count. Reservation size is the number of elements that fits in the +// set before rehash. +// * insert/emplace and other modification functions return special iterator +// that doesn't support iteration. std::next(it) for such iterators would +// always point to the end(). // // By default, `flat_hash_set` uses the `absl::Hash` hashing framework. All // fundamental and Abseil types that support the `absl::Hash` framework have a
diff --git a/absl/container/inlined_vector.h b/absl/container/inlined_vector.h index 8132564..2694a5d 100644 --- a/absl/container/inlined_vector.h +++ b/absl/container/inlined_vector.h
@@ -72,7 +72,7 @@ // designed to cover the same API footprint as covered by `std::vector`. template <typename T, size_t N, typename A = std::allocator<T>> class ABSL_ATTRIBUTE_WARN_UNUSED InlinedVector { - static_assert(N > 0, "`absl::InlinedVector` requires an inlined capacity."); + static_assert(N > 0, "absl::InlinedVector requires an inlined capacity."); using Storage = inlined_vector_internal::Storage<T, N, A>; @@ -135,6 +135,9 @@ explicit InlinedVector(size_type n, const allocator_type& allocator = allocator_type()) : storage_(allocator) { + if (ABSL_PREDICT_FALSE(n > max_size())) { + ThrowStdLengthError("InlinedVector::InlinedVector failed length check"); + } storage_.Initialize(DefaultValueAdapter<A>(), n); } @@ -142,6 +145,9 @@ InlinedVector(size_type n, const_reference v, const allocator_type& allocator = allocator_type()) : storage_(allocator) { + if (ABSL_PREDICT_FALSE(n > max_size())) { + ThrowStdLengthError("InlinedVector::InlinedVector failed length check"); + } storage_.Initialize(CopyValueAdapter<A>(std::addressof(v)), n); } @@ -161,8 +167,11 @@ InlinedVector(ForwardIterator first, ForwardIterator last, const allocator_type& allocator = allocator_type()) : storage_(allocator) { - storage_.Initialize(IteratorValueAdapter<A, ForwardIterator>(first), - static_cast<size_t>(std::distance(first, last))); + const size_type s = static_cast<size_type>(std::distance(first, last)); + if (ABSL_PREDICT_FALSE(s > max_size())) { + ThrowStdLengthError("InlinedVector::InlinedVector failed length check"); + } + storage_.Initialize(IteratorValueAdapter<A, ForwardIterator>(first), s); } // Creates an inlined vector with elements constructed from the provided input @@ -383,7 +392,7 @@ // in both debug and non-debug builds, `std::out_of_range` will be thrown. reference at(size_type i) ABSL_ATTRIBUTE_LIFETIME_BOUND { if (ABSL_PREDICT_FALSE(i >= size())) { - ThrowStdOutOfRange("`InlinedVector::at(size_type)` failed bounds check"); + ThrowStdOutOfRange("InlinedVector::at(size_type) failed bounds check"); } return data()[i]; } @@ -395,8 +404,7 @@ // in both debug and non-debug builds, `std::out_of_range` will be thrown. const_reference at(size_type i) const ABSL_ATTRIBUTE_LIFETIME_BOUND { if (ABSL_PREDICT_FALSE(i >= size())) { - ThrowStdOutOfRange( - "`InlinedVector::at(size_type) const` failed bounds check"); + ThrowStdOutOfRange("InlinedVector::at(size_type) failed bounds check"); } return data()[i]; } @@ -558,6 +566,9 @@ // // Replaces the contents of the inlined vector with `n` copies of `v`. void assign(size_type n, const_reference v) { + if (ABSL_PREDICT_FALSE(n > max_size())) { + ThrowStdLengthError("InlinedVector::assign failed length check"); + } storage_.Assign(CopyValueAdapter<A>(std::addressof(v)), n); } @@ -574,8 +585,11 @@ template <typename ForwardIterator, EnableIfAtLeastForwardIterator<ForwardIterator> = 0> void assign(ForwardIterator first, ForwardIterator last) { - storage_.Assign(IteratorValueAdapter<A, ForwardIterator>(first), - static_cast<size_t>(std::distance(first, last))); + const size_type s = static_cast<size_type>(std::distance(first, last)); + if (ABSL_PREDICT_FALSE(s > max_size())) { + ThrowStdLengthError("InlinedVector::assign failed length check"); + } + storage_.Assign(IteratorValueAdapter<A, ForwardIterator>(first), s); } // Overload of `InlinedVector::assign(...)` to replace the contents of the @@ -601,7 +615,9 @@ // NOTE: If `n` is smaller than `size()`, extra elements are destroyed. If `n` // is larger than `size()`, new elements are value-initialized. void resize(size_type n) { - absl::base_internal::HardeningAssertLE(n, max_size()); + if (ABSL_PREDICT_FALSE(n > max_size())) { + ThrowStdLengthError("InlinedVector::resize failed length check"); + } storage_.Resize(DefaultValueAdapter<A>(), n); } @@ -611,7 +627,9 @@ // NOTE: if `n` is smaller than `size()`, extra elements are destroyed. If `n` // is larger than `size()`, new elements are copied-constructed from `v`. void resize(size_type n, const_reference v) { - absl::base_internal::HardeningAssertLE(n, max_size()); + if (ABSL_PREDICT_FALSE(n > max_size())) { + ThrowStdLengthError("InlinedVector::resize failed length check"); + } storage_.Resize(CopyValueAdapter<A>(std::addressof(v)), n); } @@ -638,6 +656,9 @@ const_reference v) ABSL_ATTRIBUTE_LIFETIME_BOUND { absl::base_internal::HardeningAssertGE(pos, cbegin()); absl::base_internal::HardeningAssertLE(pos, cend()); + if (ABSL_PREDICT_FALSE(n > max_size() - size())) { + ThrowStdLengthError("InlinedVector::insert failed length check"); + } if (ABSL_PREDICT_TRUE(n != 0)) { value_type dealias = v; @@ -679,11 +700,14 @@ ForwardIterator last) ABSL_ATTRIBUTE_LIFETIME_BOUND { absl::base_internal::HardeningAssertGE(pos, cbegin()); absl::base_internal::HardeningAssertLE(pos, cend()); + const size_type s = static_cast<size_type>(std::distance(first, last)); + if (ABSL_PREDICT_FALSE(s > max_size() - size())) { + ThrowStdLengthError("InlinedVector::insert failed length check"); + } if (ABSL_PREDICT_TRUE(first != last)) { return storage_.Insert( - pos, IteratorValueAdapter<A, ForwardIterator>(first), - static_cast<size_type>(std::distance(first, last))); + pos, IteratorValueAdapter<A, ForwardIterator>(first), s); } else { return const_cast<iterator>(pos); } @@ -718,6 +742,9 @@ Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { absl::base_internal::HardeningAssertGE(pos, cbegin()); absl::base_internal::HardeningAssertLE(pos, cend()); + if (ABSL_PREDICT_FALSE(size() == max_size())) { + ThrowStdLengthError("InlinedVector::emplace failed length check"); + } value_type dealias(std::forward<Args>(args)...); // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102329#c2 @@ -744,6 +771,9 @@ // `end()`, returning a `reference` to the newly emplaced element. template <typename... Args> reference emplace_back(Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { + if (ABSL_PREDICT_FALSE(size() == max_size())) { + ThrowStdLengthError("InlinedVector::emplace_back failed length check"); + } return storage_.EmplaceBack(std::forward<Args>(args)...); } @@ -825,7 +855,12 @@ // `InlinedVector::reserve(...)` // // Ensures that there is enough room for at least `n` elements. - void reserve(size_type n) { storage_.Reserve(n); } + void reserve(size_type n) { + if (ABSL_PREDICT_FALSE(n > max_size())) { + ThrowStdLengthError("InlinedVector::reserve failed length check"); + } + storage_.Reserve(n); + } // `InlinedVector::shrink_to_fit()` // @@ -913,6 +948,12 @@ storage_.GetAllocator(), data(), size()); storage_.DeallocateIfAllocated(); + if constexpr (!std::is_nothrow_move_constructible_v<value_type>) { + // Reset the size to zero before moving to avoid leaking freed memory if + // an exception is thrown. + storage_.SetInlinedSize(0); + } + IteratorValueAdapter<A, MoveIterator<A>> other_values( MoveIterator<A>(other.storage_.GetInlinedData())); inlined_vector_internal::ConstructElements<A>(
diff --git a/absl/container/inlined_vector_test.cc b/absl/container/inlined_vector_test.cc index d6cc175..c9af35d 100644 --- a/absl/container/inlined_vector_test.cc +++ b/absl/container/inlined_vector_test.cc
@@ -207,6 +207,42 @@ "failed bounds check"); } +TEST(IntVec, LengthThrows) { + IntVec v = {1, 2, 3}; + ABSL_BASE_INTERNAL_EXPECT_FAIL(v.insert(v.begin(), v.max_size() + 1, 0), + std::length_error, "failed length check"); + ABSL_BASE_INTERNAL_EXPECT_FAIL( + v.insert(v.begin(), static_cast<size_t>(-1), 0), std::length_error, + "failed length check"); + ABSL_BASE_INTERNAL_EXPECT_FAIL(v.resize(v.max_size() + 1), std::length_error, + "failed length check"); + ABSL_BASE_INTERNAL_EXPECT_FAIL(v.reserve(v.max_size() + 1), std::length_error, + "failed length check"); + ABSL_BASE_INTERNAL_EXPECT_FAIL(static_cast<void>(IntVec(v.max_size() + 1, 0)), + std::length_error, "failed length check"); +} + +template <typename T> +struct SmallMaxAllocator : std::allocator<T> { + template <typename U> + struct rebind { + using other = SmallMaxAllocator<U>; + }; + size_t max_size() const noexcept { return 2; } +}; + +TEST(IntVec, EmplaceLengthThrows) { + absl::InlinedVector<int, 1, SmallMaxAllocator<int>> v = {1, 2}; + ABSL_BASE_INTERNAL_EXPECT_FAIL(v.push_back(3), std::length_error, + "failed length check"); + ABSL_BASE_INTERNAL_EXPECT_FAIL(v.emplace_back(3), std::length_error, + "failed length check"); + ABSL_BASE_INTERNAL_EXPECT_FAIL(v.insert(v.begin(), 3), std::length_error, + "failed length check"); + ABSL_BASE_INTERNAL_EXPECT_FAIL(v.emplace(v.begin(), 3), std::length_error, + "failed length check"); +} + TEST(IntVec, ReverseIterator) { for (size_t len = 0; len < 20; len++) { IntVec v; @@ -262,7 +298,6 @@ absl::base_internal::ScopedSetAbslHardeningForTesting hardener(true); EXPECT_DEATH_IF_SUPPORTED(v[10], ""); EXPECT_DEATH_IF_SUPPORTED(v[static_cast<size_t>(-1)], ""); - EXPECT_DEATH_IF_SUPPORTED(v.resize(v.max_size() + 1), ""); #endif } @@ -2278,4 +2313,74 @@ EXPECT_THAT(v, IsEmpty()); } +#ifdef ABSL_HAVE_EXCEPTIONS +struct ThrowOnMove { + static constexpr uint32_t kAlive = 0xA11FE123; + static constexpr uint32_t kDestroyed = 0xDEADBEEF; + + explicit ThrowOnMove(int* count) : alive_count(count), sentinel(kAlive) { + if (alive_count) ++(*alive_count); + } + ThrowOnMove(const ThrowOnMove&) = delete; + ThrowOnMove& operator=(const ThrowOnMove&) = delete; + ThrowOnMove(ThrowOnMove&& other) + : alive_count(other.alive_count), sentinel(kAlive) { + if (other.should_throw) throw std::runtime_error("ThrowOnMove"); + if (alive_count) ++(*alive_count); + } + ~ThrowOnMove() { + EXPECT_EQ(sentinel, kAlive) + << "Double destroy detected: destructor called twice on memory slot!"; + sentinel = kDestroyed; + if (alive_count) { + EXPECT_GT(*alive_count, 0) + << "More destructors called than constructors!"; + --(*alive_count); + } + } + + int* alive_count = nullptr; + uint32_t sentinel = kDestroyed; + bool should_throw = false; +}; + +TEST(InlinedVectorTest, SwapExceptionSafety) { + int alive_count = 0; + try { + absl::InlinedVector<ThrowOnMove, 2> a; + a.emplace_back(&alive_count); + absl::InlinedVector<ThrowOnMove, 2> b; + b.emplace_back(&alive_count); + b[0].should_throw = true; + a.swap(b); + FAIL() << "Expected swap to throw std::runtime_error"; + } catch (const std::runtime_error&) { + // Expected exception from throwing move-constructor inside swap. + } + EXPECT_EQ(alive_count, 0) + << "Mismatch between constructor and destructor calls!"; +} + +// Ensures that an exception thrown during move construction doesn't leave the +// inlined vector in a bad state. Needs ASAN for reliable detection. +TEST(InlinedVectorTest, TruncatesBeforeThrowingMoveConstruction) { + using Vec = absl::InlinedVector<ThrowOnMove, 1>; + int count = 0; + { + Vec dest; + dest.reserve(dest.capacity() + 1); // force heap allocation + dest.emplace_back(&count); + + Vec src; + src.emplace_back(&count).should_throw = true; + + try { + dest = std::move(src); + } catch (const std::runtime_error&) { + } + } + EXPECT_EQ(count, 0); +} +#endif + } // anonymous namespace
diff --git a/absl/container/internal/btree_container.h b/absl/container/internal/btree_container.h index 2bd98c6..8a4582f 100644 --- a/absl/container/internal/btree_container.h +++ b/absl/container/internal/btree_container.h
@@ -504,9 +504,9 @@ 0))> \ decltype(auto) Func( \ __VA_ARGS__ key_arg<K> KQual k ABSL_INTERNAL_IF_##KValue( \ - ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)), \ + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS), \ M MQual obj ABSL_INTERNAL_IF_##MValue( \ - ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this))) \ + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS)) \ ABSL_ATTRIBUTE_LIFETIME_BOUND { \ return ABSL_INTERNAL_IF_##KValue##_OR_##MValue( \ (this->template Func<K, M, 0>), Callee)( \ @@ -605,7 +605,7 @@ std::enable_if_t<!std::is_convertible_v<K, const_iterator>, int> = 0> \ decltype(auto) Func( \ __VA_ARGS__ key_arg<K> KQual k ABSL_INTERNAL_IF_##KValue( \ - ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)), \ + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS), \ Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { \ return ABSL_INTERNAL_IF_##KValue((this->template Func<K, 0>), Callee)( \ __VA_ARGS__ std::forward<decltype(k)>(k), \ @@ -632,7 +632,7 @@ } template <class K = key_type, int &..., EnableIf<LifetimeBoundK<K, true>> = 0> mapped_type &operator[]( - const key_arg<K> &k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + const key_arg<K> &k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->template operator[]<K, 0>(k); } @@ -641,8 +641,9 @@ return try_emplace(std::forward<key_arg<K>>(k)).first->second; } template <class K = key_type, int &..., EnableIf<LifetimeBoundK<K, true>> = 0> - mapped_type &operator[](key_arg<K> &&k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY( - this)) ABSL_ATTRIBUTE_LIFETIME_BOUND { + mapped_type &operator[]( + key_arg<K> &&k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) + ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->template operator[]<K, 0>(std::forward<key_arg<K>>(k)); }
diff --git a/absl/container/internal/hash_function_defaults.h b/absl/container/internal/hash_function_defaults.h index d0fe31c..dcf7dbd 100644 --- a/absl/container/internal/hash_function_defaults.h +++ b/absl/container/internal/hash_function_defaults.h
@@ -140,7 +140,7 @@ } }; -// Supports heterogeneous lookup for w/u16/u32 string + string_view + char*. +// Supports heterogeneous lookup for w/u8/u16/u32 string + string_view + char*. template <typename TChar> struct BasicStringHashEq { using Hash = BasicStringHash<TChar>; @@ -151,6 +151,12 @@ struct HashEq<std::wstring> : BasicStringHashEq<wchar_t> {}; template <> struct HashEq<std::wstring_view> : BasicStringHashEq<wchar_t> {}; +#ifdef __cpp_char8_t +template <> +struct HashEq<std::u8string> : BasicStringHashEq<char8_t> {}; +template <> +struct HashEq<std::u8string_view> : BasicStringHashEq<char8_t> {}; +#endif template <> struct HashEq<std::u16string> : BasicStringHashEq<char16_t> {}; template <>
diff --git a/absl/container/internal/hash_function_defaults_test.cc b/absl/container/internal/hash_function_defaults_test.cc index 4a304a9..67c853c 100644 --- a/absl/container/internal/hash_function_defaults_test.cc +++ b/absl/container/internal/hash_function_defaults_test.cc
@@ -16,6 +16,7 @@ #include <cstddef> #include <functional> +#include <string> #include <string_view> #include <type_traits> #include <utility> @@ -134,6 +135,28 @@ EXPECT_FALSE(eq(L"a", std::wstring(L"b"))); } +#ifdef __cpp_char8_t +TEST(BasicStringViewTest, U8StringEqWorks) { + hash_default_eq<std::u8string> eq; + EXPECT_TRUE(eq(u8"a", u8"a")); + EXPECT_TRUE(eq(u8"a", std::u8string_view(u8"a"))); + EXPECT_TRUE(eq(u8"a", std::u8string(u8"a"))); + EXPECT_FALSE(eq(u8"a", u8"b")); + EXPECT_FALSE(eq(u8"a", std::u8string_view(u8"b"))); + EXPECT_FALSE(eq(u8"a", std::u8string(u8"b"))); +} + +TEST(BasicStringViewTest, U8StringViewEqWorks) { + hash_default_eq<std::u8string_view> eq; + EXPECT_TRUE(eq(u8"a", u8"a")); + EXPECT_TRUE(eq(u8"a", std::u8string_view(u8"a"))); + EXPECT_TRUE(eq(u8"a", std::u8string(u8"a"))); + EXPECT_FALSE(eq(u8"a", u8"b")); + EXPECT_FALSE(eq(u8"a", std::u8string_view(u8"b"))); + EXPECT_FALSE(eq(u8"a", std::u8string(u8"b"))); +} +#endif + TEST(BasicStringViewTest, U16StringEqWorks) { hash_default_eq<std::u16string> eq; EXPECT_TRUE(eq(u"a", u"a")); @@ -192,6 +215,26 @@ EXPECT_NE(h, hash(std::wstring(L"b"))); } +#ifdef __cpp_char8_t +TEST(BasicStringViewTest, U8StringHashWorks) { + hash_default_hash<std::u8string> hash; + auto h = hash(u8"a"); + EXPECT_EQ(h, hash(std::u8string_view(u8"a"))); + EXPECT_EQ(h, hash(std::u8string(u8"a"))); + EXPECT_NE(h, hash(std::u8string_view(u8"b"))); + EXPECT_NE(h, hash(std::u8string(u8"b"))); +} + +TEST(BasicStringViewTest, U8StringViewHashWorks) { + hash_default_hash<std::u8string_view> hash; + auto h = hash(u8"a"); + EXPECT_EQ(h, hash(std::u8string_view(u8"a"))); + EXPECT_EQ(h, hash(std::u8string(u8"a"))); + EXPECT_NE(h, hash(std::u8string_view(u8"b"))); + EXPECT_NE(h, hash(std::u8string(u8"b"))); +} +#endif + TEST(BasicStringViewTest, U16StringHashWorks) { hash_default_hash<std::u16string> hash; auto h = hash(u"a");
diff --git a/absl/container/internal/hashtable_control_bytes.h b/absl/container/internal/hashtable_control_bytes.h index f2b1efe..fd4dea4 100644 --- a/absl/container/internal/hashtable_control_bytes.h +++ b/absl/container/internal/hashtable_control_bytes.h
@@ -185,6 +185,8 @@ kEmpty = -128, // 0b10000000 kDeleted = -2, // 0b11111110 kSentinel = -1, // 0b11111111 + // Special value used in the slow path of resizing. + kMarkedForSlowTransfer = -3, }; static_assert( (static_cast<int8_t>(ctrl_t::kEmpty) &
diff --git a/absl/container/internal/inlined_vector.h b/absl/container/internal/inlined_vector.h index 4e9b87e..e6491ba 100644 --- a/absl/container/internal/inlined_vector.h +++ b/absl/container/internal/inlined_vector.h
@@ -1038,10 +1038,22 @@ ValueType<A> tmp(std::move(*a)); AllocatorTraits<A>::destroy(allocator_a, a); - AllocatorTraits<A>::construct(allocator_b, a, std::move(*b)); + ABSL_INTERNAL_TRY { + AllocatorTraits<A>::construct(allocator_b, a, std::move(*b)); + } + ABSL_INTERNAL_CATCH_ANY { + AllocatorTraits<A>::construct(allocator_a, a, std::move(tmp)); + ABSL_INTERNAL_RETHROW; + } AllocatorTraits<A>::destroy(allocator_b, b); - AllocatorTraits<A>::construct(allocator_a, b, std::move(tmp)); + ABSL_INTERNAL_TRY { + AllocatorTraits<A>::construct(allocator_a, b, std::move(tmp)); + } + ABSL_INTERNAL_CATCH_ANY { + AllocatorTraits<A>::construct(allocator_b, b, std::move(*a)); + ABSL_INTERNAL_RETHROW; + } } }
diff --git a/absl/container/internal/raw_hash_map.h b/absl/container/internal/raw_hash_map.h index e039753..04e9e61 100644 --- a/absl/container/internal/raw_hash_map.h +++ b/absl/container/internal/raw_hash_map.h
@@ -130,9 +130,10 @@ 0))> \ decltype(auto) Func( \ __VA_ARGS__ key_arg<K> KQual k ABSL_INTERNAL_IF_##KValue( \ - ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)), \ - V VQual v ABSL_INTERNAL_IF_##VValue(ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY( \ - this))) ABSL_ATTRIBUTE_LIFETIME_BOUND { \ + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS), \ + V VQual v ABSL_INTERNAL_IF_##VValue( \ + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS)) \ + ABSL_ATTRIBUTE_LIFETIME_BOUND { \ return ABSL_INTERNAL_IF_##KValue##_OR_##VValue( \ (this->template Func<K, V, 0>), Callee)( \ std::forward<decltype(k)>(k), std::forward<decltype(v)>(v)) Tail; \ @@ -230,7 +231,7 @@ EnableIf<LifetimeBoundK<K, true, K*>> = 0, std::enable_if_t<!std::is_convertible_v<K, const_iterator>, int> = 0> std::pair<iterator, bool> try_emplace( - key_arg<K>&& k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + key_arg<K>&& k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS, Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->template try_emplace<K, 0>(std::forward<key_arg<K>>(k), std::forward<Args>(args)...); @@ -248,7 +249,7 @@ class K = key_type, class... Args, EnableIf<LifetimeBoundK<K, true>> = 0, std::enable_if_t<!std::is_convertible_v<K, const_iterator>, int> = 0> std::pair<iterator, bool> try_emplace( - const key_arg<K>& k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + const key_arg<K>& k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS, Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->template try_emplace<K, 0>(k, std::forward<Args>(args)...); } @@ -263,7 +264,7 @@ template <class K = key_type, class... Args, EnableIf<LifetimeBoundK<K, true, K*>> = 0> iterator try_emplace(const_iterator hint, - key_arg<K>&& k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + key_arg<K>&& k ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS, Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->template try_emplace<K, 0>(hint, std::forward<key_arg<K>>(k), std::forward<Args>(args)...); @@ -279,7 +280,7 @@ EnableIf<LifetimeBoundK<K, true>> = 0> iterator try_emplace(const_iterator hint, const key_arg<K>& k - ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS, Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->template try_emplace<K, 0>(hint, k, std::forward<Args>(args)...); @@ -317,7 +318,7 @@ template <class K = key_type, class P = Policy, int&..., EnableIf<LifetimeBoundK<K, true, K*>> = 0> MappedReference<P> operator[]( - key_arg<K>&& key ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + key_arg<K>&& key ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->template operator[]<K, P, 0>(std::forward<key_arg<K>>(key)); } @@ -334,7 +335,7 @@ template <class K = key_type, class P = Policy, int&..., EnableIf<LifetimeBoundK<K, true>> = 0> MappedReference<P> operator[]( - const key_arg<K>& key ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + const key_arg<K>& key ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->template operator[]<K, P, 0>(key); } @@ -354,9 +355,9 @@ if (res.second) { this->emplace_at(res.first, std::forward<K>(k), std::forward<V>(v)); } else { - Policy::value(&*res.first) = std::forward<V>(v); + Policy::value(&Policy::element(res.first)) = std::forward<V>(v); } - return res; + return {this->non_iterable_iterator_at_slot(res.first), res.second}; } template <class K = key_type, class... Args> @@ -368,7 +369,7 @@ std::forward_as_tuple(std::forward<K>(k)), std::forward_as_tuple(std::forward<Args>(args)...)); } - return res; + return {this->non_iterable_iterator_at_slot(res.first), res.second}; } };
diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index 80efa1e..d37c781 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc
@@ -44,13 +44,16 @@ // Represents a control byte corresponding to a full slot with arbitrary hash. constexpr ctrl_t ZeroCtrlT() { return static_cast<ctrl_t>(0); } -// A single control byte for default-constructed iterators. We leave it -// uninitialized because reading this memory is a bug. -ABSL_DLL ctrl_t kDefaultIterControl; +// A single byte for default-constructed iterators. We leave it uninitialized +// because reading this memory is a bug. +ABSL_DLL char kDefaultIterSlot; // We need one full byte followed by a sentinel byte for iterator::operator++. ABSL_CONST_INIT ABSL_DLL const ctrl_t kSooControl[2] = {ZeroCtrlT(), ctrl_t::kSentinel}; +// We need one full byte followed by a sentinel byte for iterator::operator++. +ABSL_CONST_INIT ABSL_DLL const ctrl_t kInsertIteratorControl[2] = { + ZeroCtrlT(), ctrl_t::kSentinel}; namespace { @@ -76,12 +79,14 @@ // Returns "random" seed. inline size_t RandomSeed() { + constexpr size_t kIncrement = 0xad53; #ifdef ABSL_HAVE_THREAD_LOCAL static thread_local size_t counter = 0; - size_t value = ++counter; + counter += kIncrement; + size_t value = counter; #else // ABSL_HAVE_THREAD_LOCAL static std::atomic<size_t> counter(0); - size_t value = counter.fetch_add(1, std::memory_order_relaxed); + size_t value = counter.fetch_add(kIncrement, std::memory_order_relaxed); #endif // ABSL_HAVE_THREAD_LOCAL return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter)); } @@ -92,7 +97,7 @@ // `min(1, RehashProbabilityConstant() / capacity())`. In order to do this, // we probe based on a random hash and see if the offset is less than // RehashProbabilityConstant(). - return probe(HashtableCapacity(capacity), absl::HashOf(RandomSeed())) + return probe(ProbeCapacity{capacity}, absl::HashOf(RandomSeed())) .offset() < RehashProbabilityConstant(); } @@ -138,10 +143,7 @@ // which is caused by non-constexpr initialization. uint16_t NextHashTableSeed() { static_assert(PerTableSeed::kBitCount <= 16); - thread_local uint16_t seed = - static_cast<uint16_t>(reinterpret_cast<uintptr_t>(&seed)); - seed += uint16_t{0xad53}; - return seed; + return static_cast<uint16_t>(RandomSeed()); } GenerationType* EmptyGeneration() { @@ -187,11 +189,12 @@ FindInfo find_first_non_full_from_h1(const ctrl_t* ctrl, size_t h1, HashtableCapacity capacity) { - auto seq = probe_h1(capacity, h1); + const size_t cap = capacity.capacity(); + auto seq = probe_h1(ProbeCapacity{cap}, h1); if (IsEmptyOrDeleted(ctrl[seq.offset()])) { return {seq.offset(), /*probe_length=*/0}; } - auto mask = probe_till_first_non_full_group(ctrl, seq, capacity.capacity()); + auto mask = probe_till_first_non_full_group(ctrl, seq, cap); return {seq.offset(mask.LowestBitSet()), seq.index()}; } @@ -234,9 +237,9 @@ template <class Fn> void IterateOverFullSlotsImpl(const CommonFields& c, size_t slot_size, Fn cb) { const size_t cap = c.capacity(); - ABSL_SWISSTABLE_ASSERT(!IsSmallCapacity(cap)); + ABSL_ASSUME(cap > kMaxSmallCapacity); const ctrl_t* ctrl = c.control(); - void* slot = c.slot_array(); + void* slot = c.slot_array(cap); if (is_half_group(cap)) { // Mirrored/cloned control bytes in half-group table are also located in the // first group (starting from position 0). We are taking group from position @@ -445,6 +448,23 @@ IterateOverFullSlotsImpl(c, slot_size, cb); } +HashtablezInfoHandle CommonFields::infoz_ptr() const { + // growth_info is stored before control bytes. + ABSL_SWISSTABLE_ASSERT(has_infoz()); + HashtablezInfoHandle res; + void* src = reinterpret_cast<char*>(control()) - + MetadataBeforeControlSize(/*has_infoz=*/true, capacity()); + std::memcpy(&res, src, sizeof(HashtablezInfoHandle)); + return res; +} + +void CommonFields::set_infoz(HashtablezInfoHandle infoz) { + ABSL_SWISSTABLE_ASSERT(has_infoz()); + void* dst = reinterpret_cast<char*>(control()) - + MetadataBeforeControlSize(/*has_infoz=*/true, capacity()); + std::memcpy(dst, &infoz, sizeof(HashtablezInfoHandle)); +} + namespace { void ResetGrowthLeft(GrowthInfoAccessor growth_info, size_t capacity, @@ -486,8 +506,10 @@ // Sets sanitizer poisoning for slot corresponding to control byte being set. inline void DoSanitizeOnSetCtrl(const CommonFields& c, size_t i, ctrl_t h, size_t slot_size) { - ABSL_SWISSTABLE_ASSERT(i < c.capacity()); - auto* slot_i = static_cast<const char*>(c.slot_array()) + i * slot_size; + const size_t cap = c.capacity(); + ABSL_ASSUME(cap > kMaxSmallCapacity); + ABSL_SWISSTABLE_ASSERT(i < cap); + auto* slot_i = static_cast<const char*>(c.slot_array(cap)) + i * slot_size; if (IsFull(h)) { SanitizerUnpoisonMemoryRegion(slot_i, slot_size); } else { @@ -499,45 +521,45 @@ // // Unlike setting it directly, this function will perform bounds checks and // mirror the value to the cloned tail if necessary. +inline void SetCtrlNoSanitizeImpl(const CommonFields& c, size_t i, ctrl_t h) { + ABSL_SWISSTABLE_ASSERT(i < c.capacity()); + ctrl_t* ctrl = c.control(); + const size_t cap = c.capacity(); + ctrl[i] = h; + ctrl[((i - NumClonedBytes()) & cap) + (NumClonedBytes() & cap)] = h; +} + inline void SetCtrl(const CommonFields& c, size_t i, ctrl_t h, size_t slot_size) { ABSL_SWISSTABLE_ASSERT(!c.is_small()); DoSanitizeOnSetCtrl(c, i, h, slot_size); - ctrl_t* ctrl = c.control(); - ctrl[i] = h; - ctrl[((i - NumClonedBytes()) & c.capacity()) + - (NumClonedBytes() & c.capacity())] = h; + SetCtrlNoSanitizeImpl(c, i, h); } // Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. inline void SetCtrl(const CommonFields& c, size_t i, h2_t h, size_t slot_size) { SetCtrl(c, i, static_cast<ctrl_t>(h), slot_size); } -inline void SetCtrlInSingleGroupTableNoSanitizeImpl(const CommonFields& c, - size_t i, ctrl_t h) { - ABSL_SWISSTABLE_ASSERT(!c.is_small()); - ABSL_SWISSTABLE_ASSERT(is_single_group(c.capacity())); - ctrl_t* ctrl = c.control(); - ctrl[i] = h; - ctrl[i + c.capacity() + 1] = h; -} - -// Sets `ctrl[i]` to `ctrl_t::kSentinel` in single group table. +// Sets `ctrl[i]` to `ctrl_t::kSentinel`. // // Unlike setting it directly, this function will perform bounds checks and // mirror the value to the cloned tail if necessary. -inline void BlockCtrlInSingleGroupTable(const CommonFields& c, size_t i) { - SetCtrlInSingleGroupTableNoSanitizeImpl(c, i, ctrl_t::kSentinel); +inline void BlockCtrl(const CommonFields& c, size_t i) { + ABSL_SWISSTABLE_ASSERT(!c.is_small()); + SetCtrlNoSanitizeImpl(c, i, ctrl_t::kSentinel); } // Like SetCtrl, but in a single group table, we can save some operations when // setting the cloned control byte. inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, ctrl_t h, size_t slot_size) { + const size_t cap = c.capacity(); ABSL_SWISSTABLE_ASSERT(!c.is_small()); - ABSL_SWISSTABLE_ASSERT(is_single_group(c.capacity())); + ABSL_SWISSTABLE_ASSERT(is_single_group(cap)); DoSanitizeOnSetCtrl(c, i, h, slot_size); - SetCtrlInSingleGroupTableNoSanitizeImpl(c, i, h); + ctrl_t* ctrl = c.control(); + ctrl[i] = h; + ctrl[i + cap + 1] = h; } // Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, h2_t h, @@ -561,14 +583,25 @@ SetCtrlInLargeTable(c, i, static_cast<ctrl_t>(h), slot_size); } -size_t DropDeletesWithoutResizeAndPrepareInsert( +void BlockControlBytes(CommonFields& common, size_t blocked_element_count) { + const size_t capacity = common.capacity(); + while (blocked_element_count > 0) { + BlockCtrl(common, capacity - blocked_element_count); + --blocked_element_count; + } +} + +void* DropDeletesWithoutResizeAndPrepareInsert( CommonFields& common, const PolicyFunctions& __restrict policy, size_t new_hash) { void* set = &common; - void* slot_array = common.slot_array(); const size_t capacity = common.capacity(); ABSL_SWISSTABLE_ASSERT(IsValidCapacity(capacity)); ABSL_SWISSTABLE_ASSERT(!is_single_group(capacity)); + ABSL_ASSUME(capacity > kMaxSmallCapacity); + + ctrl_t* ctrl = common.control(); + void* slot_array = common.slot_array(capacity); // Algorithm: // - mark all DELETED slots as EMPTY // - mark all FULL slots as DELETED @@ -585,8 +618,9 @@ // swap current element with target element // mark target as FULL // repeat procedure for current slot with moved from element (target) - ctrl_t* ctrl = common.control(); + const size_t blocked_element_count = common.blocked_element_count(); ConvertDeletedToEmptyAndFullToDeleted(ctrl, capacity); + BlockControlBytes(common, blocked_element_count); const void* hash_fn = policy.hash_fn(common); auto hasher = policy.hash_slot; auto transfer_n = policy.transfer_n; @@ -663,13 +697,13 @@ } // Prepare insert for the new element. PrepareInsertCommon(common); - ABSL_SWISSTABLE_ASSERT(common.blocked_element_count() == 0); - ResetGrowthLeft(common.growth_info(), capacity, common.size()); + ResetGrowthLeft(common.growth_info(), capacity, + common.size() + blocked_element_count); FindInfo find_info = find_first_non_full(common, new_hash); SetCtrlInLargeTable(common, find_info.offset, H2(new_hash), slot_size); common.infoz().RecordInsertMiss(new_hash, find_info.probe_length); common.infoz().RecordRehash(total_probe_length); - return find_info.offset; + return SlotAddress(slot_array, find_info.offset, slot_size); } bool WasNeverFull(CommonFields& c, size_t index) { @@ -714,12 +748,9 @@ capacity + 1 + NumClonedBytes()); } ctrl[capacity] = ctrl_t::kSentinel; - SanitizerPoisonMemoryRegion(common.slot_array(), + SanitizerPoisonMemoryRegion(common.slot_array(capacity), slot_size * (capacity - blocked_element_count)); - while (blocked_element_count > 0) { - BlockCtrlInSingleGroupTable(common, capacity - blocked_element_count); - --blocked_element_count; - } + BlockControlBytes(common, blocked_element_count); } // Initializes control bytes for growing from capacity 1 to 3. @@ -809,7 +840,7 @@ template <bool kSooEnabled> void* SingleSlotAddress(CommonFields& c) { - return kSooEnabled ? c.soo_data() : c.slot_array(); + return kSooEnabled ? c.soo_data() : c.slot_array(/*capacity=*/1); } template <bool kSooEnabled> @@ -831,17 +862,17 @@ } c.decrement_size(); c.infoz().RecordErase(); - SanitizerPoisonMemoryRegion(c.slot_array(), slot_size); + SanitizerPoisonMemoryRegion(SingleSlotAddress</*kSooEnabled=*/false>(c), + slot_size); } -void EraseMetaOnlyLarge(CommonFields& c, const ctrl_t* ctrl, size_t slot_size) { +void EraseMetaOnlyLarge(CommonFields& c, size_t index, size_t slot_size) { ABSL_SWISSTABLE_ASSERT(!c.is_small()); - ABSL_SWISSTABLE_ASSERT(IsFull(*ctrl) && "erasing a dangling iterator"); + ABSL_SWISSTABLE_ASSERT(IsFull(c.control()[index]) && + "erasing a dangling iterator"); c.decrement_size(); c.infoz().RecordErase(); - size_t index = static_cast<size_t>(ctrl - c.control()); - if (WasNeverFull(c, index)) { SetCtrl(c, index, ctrl_t::kEmpty, slot_size); c.growth_info().OverwriteFullAsEmpty(); @@ -855,9 +886,9 @@ void ClearBackingArray(CommonFields& c, const PolicyFunctions& __restrict policy, void* alloc, bool reuse) { - ABSL_SWISSTABLE_ASSERT(c.capacity() > MaxSmallCapacity()); + ABSL_SWISSTABLE_ASSERT(c.capacity() > kMaxSmallCapacity); if (reuse) { - size_t blocked_element_count = c.blocked_element_count(); + const size_t blocked_element_count = c.blocked_element_count(); c.set_size_to_zero(); ABSL_SWISSTABLE_ASSERT(c.capacity() > policy.soo_capacity()); ResetCtrl(c, policy.slot_size, blocked_element_count); @@ -952,7 +983,8 @@ if (destroy_slot != nullptr) { if (c.is_small()) { if (!c.empty()) { - destroy_slot(&c, c.slot_array()); + static_assert(kMaxSmallCapacity == 1); + destroy_slot(&c, c.slot_array(/*capacity=*/1)); } } else { DestroySlots(c, slot_size, destroy_slot); @@ -971,7 +1003,7 @@ size_t FindNewPositionsAndTransferSlots( CommonFields& common, const PolicyFunctions& __restrict policy, ctrl_t* old_ctrl, void* old_slots, size_t old_capacity) { - void* new_slots = common.slot_array(); + void* new_slots = common.slot_array(common.capacity()); const void* hash_fn = policy.hash_fn(common); const size_t slot_size = policy.slot_size; const size_t seed = common.seed().seed(); @@ -1094,10 +1126,10 @@ void* alloc = policy.get_char_alloc(common); common.set_capacity(new_capacity); + common.init_blocked_element_count(blocked_element_count); const auto [new_ctrl, new_slots] = AllocBackingArray( common, policy, new_capacity, has_infoz, alloc, blocked_element_count); common.set_control(new_ctrl); - common.set_slots(new_slots); common.generate_new_seed(has_infoz); ResetCtrl(common, slot_size, blocked_element_count); @@ -1120,20 +1152,19 @@ void* new_slots, bool has_infoz) { ABSL_SWISSTABLE_ASSERT(c.size() == policy.soo_capacity()); ABSL_SWISSTABLE_ASSERT(policy.soo_enabled); - size_t new_capacity = c.capacity(); + const size_t new_capacity = c.capacity(); c.generate_new_seed(has_infoz); const size_t soo_slot_hash = policy.hash_slot(policy.hash_fn(c), c.soo_data(), c.seed().seed()); - size_t offset = probe(c.capacity_impl(), soo_slot_hash).offset(); + size_t offset = probe(ProbeCapacity{new_capacity}, soo_slot_hash).offset(); offset = offset == new_capacity ? 0 : offset; SanitizerPoisonMemoryRegion(new_slots, policy.slot_size * new_capacity); void* target_slot = SlotAddress(new_slots, offset, policy.slot_size); SanitizerUnpoisonMemoryRegion(target_slot, policy.slot_size); policy.transfer_n(&c, target_slot, c.soo_data(), 1); c.set_control(new_ctrl); - c.set_slots(new_slots); ResetCtrl(c, policy.slot_size, /*blocked_element_count=*/0); SetCtrl(c, offset, H2(soo_slot_hash), policy.slot_size); } @@ -1353,7 +1384,7 @@ const ProbedItem* start, const ProbedItem* end, void* old_slots) { const HashtableCapacity new_capacity = c.capacity_impl(); - void* new_slots = c.slot_array(); + void* new_slots = c.slot_array(new_capacity.capacity()); ctrl_t* new_ctrl = c.control(); size_t total_probe_length = 0; @@ -1383,7 +1414,7 @@ constexpr size_t kNoMarkedElementsSentinel = ~size_t{}; // Process probed elements that did not fit into available buffers. -// We marked them in control bytes as kSentinel. +// We marked them in control bytes as kMarkedForSlowTransfer. // Hash recomputation and full probing is done here. // This use case should be extremely rare. ABSL_ATTRIBUTE_NOINLINE size_t ProcessProbedMarkedElements( @@ -1391,14 +1422,14 @@ void* old_slots, size_t start) { size_t old_capacity = PreviousCapacity(c.capacity()); const size_t slot_size = policy.slot_size; - void* new_slots = c.slot_array(); + void* new_slots = c.slot_array(c.capacity()); size_t total_probe_length = 0; const void* hash_fn = policy.hash_fn(c); auto hash_slot = policy.hash_slot; auto transfer_n = policy.transfer_n; const size_t seed = c.seed().seed(); for (size_t old_index = start; old_index < old_capacity; ++old_index) { - if (old_ctrl[old_index] != ctrl_t::kSentinel) { + if (old_ctrl[old_index] != ctrl_t::kMarkedForSlowTransfer) { continue; } void* src_slot = SlotAddress(old_slots, old_index, slot_size); @@ -1498,7 +1529,7 @@ "kGuaranteedFitToBuffer is true."); // We reuse GrowthInfo memory as well. return AlignToNextItem( - control_ - ControlOffset(/*has_infoz=*/false, + control_ - MetadataBeforeControlSize(/*has_infoz=*/false, NextCapacity(kMaxLocalBufferOldCapacity))); } @@ -1511,8 +1542,9 @@ // buffer is exhausted. // // If there's no space in the control buffer, we fallback to naive algorithm - // and mark probed elements as kSentinel in the control buffer. In this case, - // we will call this function for every subsequent probed element. + // and mark probed elements as kMarkedForSlowTransfer in the control buffer. + // In this case, we will call this function for every subsequent probed + // element. ABSL_ATTRIBUTE_NOINLINE void ProcessEncodeWithOverflow(ProbedItem item) { if (!local_buffer_full_) { local_buffer_full_ = true; @@ -1520,10 +1552,11 @@ } const size_t source_offset = static_cast<size_t>(item.source_offset); // We are in fallback mode so we can't reuse control buffer anymore. - // Probed elements are marked as kSentinel in the control buffer. + // Probed elements are marked as kMarkedForSlowTransfer in the control + // buffer. if (ABSL_PREDICT_FALSE(marked_elements_starting_position_ != kNoMarkedElementsSentinel)) { - control_[source_offset] = ctrl_t::kSentinel; + control_[source_offset] = ctrl_t::kMarkedForSlowTransfer; return; } // Refresh the end pointer to the new available position. @@ -1535,7 +1568,7 @@ ++pos_; return; } - control_[source_offset] = ctrl_t::kSentinel; + control_[source_offset] = ctrl_t::kMarkedForSlowTransfer; marked_elements_starting_position_ = source_offset; // Now we will always fall down to `ProcessEncodeWithOverflow`. ABSL_SWISSTABLE_ASSERT(pos_ >= end_); @@ -1665,7 +1698,8 @@ const PolicyFunctions& __restrict policy) { ABSL_SWISSTABLE_ASSERT(common.is_small()); common.increment_size(); - SanitizerUnpoisonMemoryRegion(common.slot_array(), policy.slot_size); + SanitizerUnpoisonMemoryRegion( + SingleSlotAddress</*kSooEnabled=*/false>(common), policy.slot_size); } void IncrementSmallSize(CommonFields& common, @@ -1678,9 +1712,9 @@ } } -std::pair<ctrl_t*, void*> Grow1To3AndPrepareInsert( - CommonFields& common, const PolicyFunctions& __restrict policy, - absl::FunctionRef<size_t(size_t)> get_hash) { +void* Grow1To3AndPrepareInsert(CommonFields& common, + const PolicyFunctions& __restrict policy, + absl::FunctionRef<size_t(size_t)> get_hash) { // TODO(b/413062340): Refactor to reuse more code with // GrowSooTableToNextCapacityAndPrepareInsert. ABSL_SWISSTABLE_ASSERT(common.capacity() == 1); @@ -1690,8 +1724,7 @@ ABSL_SWISSTABLE_ASSERT(common.blocked_element_count() == 0); constexpr size_t kOldCapacity = 1; constexpr size_t kNewCapacity = NextCapacity(kOldCapacity); - ctrl_t* old_ctrl = common.control(); - void* old_slots = common.slot_array(); + void* old_slots = common.slot_array(kOldCapacity); const size_t slot_size = policy.slot_size; const size_t slot_align = policy.slot_align; @@ -1704,7 +1737,6 @@ AllocBackingArray(common, policy, kNewCapacity, has_infoz, alloc, /*blocked_element_count=*/0); common.set_control(new_ctrl); - common.set_slots(new_slots); SanitizerPoisonMemoryRegion(new_slots, kNewCapacity * slot_size); if (ABSL_PREDICT_TRUE(!has_infoz)) { @@ -1725,8 +1757,10 @@ void* new_element_target_slot = SlotAddress(new_slots, offset, slot_size); SanitizerUnpoisonMemoryRegion(new_element_target_slot, slot_size); - policy.dealloc(alloc, kOldCapacity, old_ctrl, slot_size, slot_align, - has_infoz, + policy.dealloc(alloc, kOldCapacity, + // old_slots == old_ctrl in case of capacity == 1. + static_cast<ctrl_t*>(old_slots), + slot_size, slot_align, has_infoz, /*blocked_element_count=*/0); PrepareInsertCommon(common); ABSL_SWISSTABLE_ASSERT(common.size() == 2); @@ -1736,12 +1770,12 @@ if (ABSL_PREDICT_FALSE(has_infoz)) { ReportSingleGroupTableGrowthToInfoz(common, infoz, new_hash); } - return {new_ctrl + offset, new_element_target_slot}; + return new_element_target_slot; } // Grows to next capacity and prepares insert for the given new_hash. // Returns the offset of the new element. -size_t GrowToNextCapacityAndPrepareInsert( +void* GrowToNextCapacityAndPrepareInsert( CommonFields& common, const PolicyFunctions& __restrict policy, size_t new_hash) { const size_t old_capacity = common.capacity(); @@ -1749,24 +1783,25 @@ common.growth_info().GetGrowthLeftTotalSlow(old_capacity) == 0); ABSL_SWISSTABLE_ASSERT(old_capacity > policy.soo_capacity()); ABSL_SWISSTABLE_ASSERT(!IsSmallCapacity(old_capacity)); + ABSL_ASSUME(old_capacity > kMaxSmallCapacity); const size_t new_capacity = NextCapacity(old_capacity); ctrl_t* old_ctrl = common.control(); - void* old_slots = common.slot_array(); + void* old_slots = common.slot_array(old_capacity); size_t old_blocked_element_count = common.blocked_element_count(); + HashtablezInfoHandle infoz = common.infoz(); + const bool has_infoz = infoz.IsSampled(); common.set_capacity(new_capacity); + common.set_blocked_element_count_to_zero(); const size_t slot_size = policy.slot_size; const size_t slot_align = policy.slot_align; void* alloc = policy.get_char_alloc(common); - HashtablezInfoHandle infoz = common.infoz(); - const bool has_infoz = infoz.IsSampled(); const auto [new_ctrl, new_slots] = AllocBackingArray(common, policy, new_capacity, has_infoz, alloc, /*blocked_element_count=*/0); common.set_control(new_ctrl); - common.set_slots(new_slots); SanitizerPoisonMemoryRegion(new_slots, new_capacity * slot_size); h2_t new_h2 = H2(new_hash); @@ -1810,14 +1845,14 @@ ReportGrowthToInfoz(common, infoz, new_hash, total_probe_length, find_info.probe_length); } - return find_info.offset; + return SlotAddress(new_slots, find_info.offset, policy.slot_size); } } // namespace -std::pair<ctrl_t*, void*> PrepareInsertSmallNonSoo( - CommonFields& common, const PolicyFunctions& __restrict policy, - absl::FunctionRef<size_t(size_t)> get_hash) { +void* PrepareInsertSmallNonSoo(CommonFields& common, + const PolicyFunctions& __restrict policy, + absl::FunctionRef<size_t(size_t)> get_hash) { ABSL_SWISSTABLE_ASSERT(common.is_small()); ABSL_SWISSTABLE_ASSERT(!policy.soo_enabled); if (common.capacity() == 1) { @@ -1827,7 +1862,7 @@ common.infoz().RecordInsertMiss(get_hash(common.seed().seed()), /*distance_from_desired=*/0); } - return {SooControl(), common.slot_array()}; + return common.slot_array(/*capacity=*/1); } else { return Grow1To3AndPrepareInsert(common, policy, get_hash); } @@ -1852,7 +1887,6 @@ AllocBackingArray(common, policy, kNewCapacity, has_infoz, alloc, /*blocked_element_count=*/0); common.set_control(new_ctrl); - common.set_slots(new_slots); static_assert(NextCapacity(0) == 1); PrepareInsertCommon(common); @@ -1862,7 +1896,7 @@ ReportSingleGroupTableGrowthToInfoz(common, infoz, get_hash(common.seed().seed())); } - return {SooControl(), new_slots}; + return new_slots; } namespace { @@ -1870,14 +1904,16 @@ // Called whenever the table needs to vacate empty slots either by removing // tombstones via rehash or growth to next capacity. ABSL_ATTRIBUTE_NOINLINE -size_t RehashOrGrowToNextCapacityAndPrepareInsert( +void* RehashOrGrowToNextCapacityAndPrepareInsert( CommonFields& common, const PolicyFunctions& __restrict policy, size_t new_hash) { + ABSL_SWISSTABLE_ASSERT( + !common.growth_info().GetGrowthInfoLowerBound().HasNoDeleted()); const size_t cap = common.capacity(); ABSL_ASSUME(cap > 0); - if (cap > Group::kWidth && - // Do these calculations in 64-bit to avoid overflow. - common.size() * uint64_t{32} <= cap * uint64_t{25}) { + // Do these calculations in 64-bit to avoid overflow. + if (common.size() * uint64_t{32} <= + (cap - kMaxBlockedElementsForLargeTables) * uint64_t{25}) { // Squash DELETED without growing if there is enough capacity. // // Rehash in place if the current size is <= 25/32 of capacity. @@ -1929,11 +1965,12 @@ // Slow path for PrepareInsertLarge that is called when the table has deleted // slots or need to be resized or rehashed. ABSL_ATTRIBUTE_NOINLINE -size_t PrepareInsertLargeSlow(CommonFields& common, - const PolicyFunctions& __restrict policy, - size_t hash) { +void* PrepareInsertLargeSlow(CommonFields& common, + const PolicyFunctions& __restrict policy, + size_t hash) { GrowthInfoAccessor growth_info = common.growth_info(); const size_t cap = common.capacity(); + ABSL_ASSUME(cap > kMaxSmallCapacity); GrowthInfoLowerBound growth_info_lower_bound = growth_info.RebalanceGrowthLeftLowerBound(cap); if (ABSL_PREDICT_TRUE( @@ -1955,7 +1992,7 @@ growth_info.OverwriteControlAsFull(common.control()[target.offset]); SetCtrlInLargeTable(common, target.offset, H2(hash), policy.slot_size); common.infoz().RecordInsertMiss(hash, target.probe_length); - return target.offset; + return SlotAddress(common.slot_array(cap), target.offset, policy.slot_size); } // Resizes empty non-allocated SOO table to NextCapacity(SooCapacity()), @@ -1964,11 +2001,12 @@ // Requires: // 1. `c.capacity() == SooCapacity()`. // 2. `c.empty()`. -ABSL_ATTRIBUTE_NOINLINE size_t +ABSL_ATTRIBUTE_NOINLINE void* GrowEmptySooTableToNextCapacityForceSamplingAndPrepareInsert( CommonFields& common, const PolicyFunctions& __restrict policy, absl::FunctionRef<size_t(size_t)> get_hash) { - ResizeEmptyNonAllocatedTableImpl(common, policy, NextCapacity(SooCapacity()), + const size_t kNewCapacity = NextCapacity(SooCapacity()); + ResizeEmptyNonAllocatedTableImpl(common, policy, kNewCapacity, /*blocked_element_count=*/0, /*force_infoz=*/true); PrepareInsertCommon(common); @@ -1977,17 +2015,23 @@ SetCtrlInSingleGroupTable(common, SooSlotIndex(), H2(new_hash), policy.slot_size); common.infoz().RecordInsertMiss(new_hash, /*distance_from_desired=*/0); - return SooSlotIndex(); + return SlotAddress(common.slot_array(kNewCapacity), SooSlotIndex(), + policy.slot_size); } // Returns the number of elements to block for the given capacity and reserved // size. -size_t BlockedElementCount(size_t capacity, size_t reserved_size) { +size_t BlockedElementCountForReservedTable(size_t capacity, + size_t reserved_size) { if (!IsCapacityValidForBlockedElements(capacity)) { return 0; } - ABSL_SWISSTABLE_ASSERT(is_single_group(capacity)); - return CapacityToGrowth(capacity) - reserved_size; + const size_t blocked_elements = CapacityToGrowth(capacity) - reserved_size; + if (is_single_group(capacity)) { + // Single group tables never probes, so we can block all the slots. + return blocked_elements; + } + return (std::min)(blocked_elements, kMaxBlockedElementsForLargeTables); } // Resizes empty non-allocated table to the capacity to fit new_size elements. @@ -2002,9 +2046,10 @@ ValidateMaxSize(new_size, policy.key_size, policy.slot_size); ABSL_ASSUME(new_size > 0); const size_t new_capacity = SizeToCapacity(new_size); - ResizeEmptyNonAllocatedTableImpl(common, policy, new_capacity, - BlockedElementCount(new_capacity, new_size), - /*force_infoz=*/false); + ResizeEmptyNonAllocatedTableImpl( + common, policy, new_capacity, + BlockedElementCountForReservedTable(new_capacity, new_size), + /*force_infoz=*/false); // This is after resize, to ensure that we have completed the allocation // and have potentially sampled the hashtable. common.infoz().RecordReservation(new_size); @@ -2064,7 +2109,7 @@ const size_t old_capacity = common.capacity(); ctrl_t* const old_ctrl = common.control(); - void* const old_slots = common.slot_array(); + void* const old_slots = common.slot_array(old_capacity); const size_t old_blocked_element_count = common.blocked_element_count(); const size_t slot_size = policy.slot_size; @@ -2074,11 +2119,11 @@ void* alloc = policy.get_char_alloc(common); common.set_capacity(new_capacity); + common.set_blocked_element_count_to_zero(); const auto [new_ctrl, new_slots] = AllocBackingArray(common, policy, new_capacity, has_infoz, alloc, /*blocked_element_count=*/0); common.set_control(new_ctrl); - common.set_slots(new_slots); common.generate_new_seed(has_infoz); size_t total_probe_length = 0; @@ -2098,19 +2143,9 @@ } } -void ReserveEmptyNonAllocatedTableToFitBucketCount( - CommonFields& common, const PolicyFunctions& __restrict policy, - size_t bucket_count) { - size_t new_capacity = NormalizeCapacity(bucket_count); - ValidateMaxCapacity(new_capacity, policy.key_size, policy.slot_size); - ResizeEmptyNonAllocatedTableImpl(common, policy, new_capacity, - /*blocked_element_count=*/0, - /*force_infoz=*/false); -} - // Resizes a full SOO table to the NextCapacity(SooCapacity()). template <size_t SooSlotMemcpySize, bool TransferUsesMemcpy> -size_t GrowSooTableToNextCapacityAndPrepareInsert( +void* GrowSooTableToNextCapacityAndPrepareInsert( CommonFields& common, const PolicyFunctions& __restrict policy, absl::FunctionRef<size_t(size_t)> get_hash, bool force_sampling) { AssertSoo(common, policy); @@ -2168,14 +2203,13 @@ policy.transfer_n(&common, target_slot, common.soo_data(), 1); } common.set_control(new_ctrl); - common.set_slots(new_slots); // Full SOO table couldn't be sampled. If SOO table is sampled, it would // have been resized to the next capacity. ABSL_SWISSTABLE_ASSERT(!common.infoz().IsSampled()); - SanitizerUnpoisonMemoryRegion(SlotAddress(new_slots, offset, slot_size), - slot_size); - return offset; + void* new_slot = SlotAddress(new_slots, offset, slot_size); + SanitizerUnpoisonMemoryRegion(new_slot, slot_size); + return new_slot; } void Rehash(CommonFields& common, const PolicyFunctions& __restrict policy, @@ -2213,7 +2247,7 @@ size_t begin_offset = FindFirstFullSlot(0, cap, common.control()); policy.transfer_n( &common, &tmp_slot, - SlotAddress(common.slot_array(), begin_offset, slot_size), 1); + SlotAddress(common.slot_array(cap), begin_offset, slot_size), 1); clear_backing_array(); policy.transfer_n(&common, common.soo_data(), &tmp_slot, 1); common.set_full_soo(); @@ -2260,12 +2294,14 @@ const size_t other_capacity = other.capacity(); const void* other_slot = other_capacity <= soo_capacity ? other.soo_data() - : other.is_small() - ? other.slot_array() - : SlotAddress(other.slot_array(), + : IsSmallCapacity(other_capacity) + ? other.slot_array(other_capacity) + : SlotAddress(other.slot_array(other_capacity), FindFirstFullSlot(0, other_capacity, other.control()), slot_size); - copy_fn(soo_enabled ? common.soo_data() : common.slot_array(), other_slot); + copy_fn(soo_enabled ? common.soo_data() + : SingleSlotAddress</*kSooEnabled=*/false>(common), + other_slot); if (soo_enabled && policy.is_hashtablez_eligible && ShouldSampleNextTable()) { @@ -2280,10 +2316,12 @@ ABSL_SWISSTABLE_ASSERT(other.capacity() > soo_capacity); const size_t cap = common.capacity(); ABSL_SWISSTABLE_ASSERT(cap > soo_capacity); + ABSL_ASSUME(cap > kMaxSmallCapacity); size_t offset = cap; const void* hash_fn = policy.hash_fn(common); auto hasher = policy.hash_slot; const size_t seed = common.seed().seed(); + void* target_slot_array = common.slot_array(cap); IterateOverFullSlotsImpl( other, slot_size, [&](const ctrl_t*, void* that_slot) { // The table is guaranteed to be empty, so we can do faster than @@ -2293,7 +2331,7 @@ infoz.RecordInsertMiss(hash, target.probe_length); offset = target.offset; SetCtrl(common, offset, H2(hash), slot_size); - copy_fn(SlotAddress(common.slot_array(), offset, slot_size), that_slot); + copy_fn(SlotAddress(target_slot_array, offset, slot_size), that_slot); common.maybe_increment_generation_on_insert(); }); common.increment_size(size); @@ -2303,6 +2341,8 @@ void ReserveTableToFitNewSize(CommonFields& common, const PolicyFunctions& __restrict policy, size_t new_size) { + new_size = + std::min(new_size, MaxValidSize(policy.key_size, policy.slot_size)); common.reset_reserved_growth(new_size); common.set_reservation_size(new_size); ABSL_SWISSTABLE_ASSERT(new_size > policy.soo_capacity()); @@ -2324,11 +2364,11 @@ } namespace { -size_t PrepareInsertLargeImpl(CommonFields& common, - const PolicyFunctions& __restrict policy, - size_t hash, - Group::NonIterableBitMaskType mask_empty, - FindInfo target_group) { +void* PrepareInsertLargeImpl(CommonFields& common, + const PolicyFunctions& __restrict policy, + size_t hash, + Group::NonIterableBitMaskType mask_empty, + FindInfo target_group) { ABSL_SWISSTABLE_ASSERT(!common.is_small()); GrowthInfoAccessor growth_info = common.growth_info(); // When there are no deleted slots in the table @@ -2340,24 +2380,27 @@ } PrepareInsertCommon(common); growth_info.OverwriteEmptyAsFull(); + const size_t cap = common.capacity(); + ABSL_ASSUME(cap > kMaxSmallCapacity); target_group.offset += mask_empty.LowestBitSet(); - target_group.offset &= common.capacity(); + target_group.offset &= cap; SetCtrl(common, target_group.offset, H2(hash), policy.slot_size); common.infoz().RecordInsertMiss(hash, target_group.probe_length); - return target_group.offset; + return SlotAddress(common.slot_array(cap), target_group.offset, + policy.slot_size); } } // namespace -size_t PrepareInsertLarge(CommonFields& common, - const PolicyFunctions& __restrict policy, size_t hash, - Group::NonIterableBitMaskType mask_empty, - FindInfo target_group) { +void* PrepareInsertLarge(CommonFields& common, + const PolicyFunctions& __restrict policy, size_t hash, + Group::NonIterableBitMaskType mask_empty, + FindInfo target_group) { // NOLINTNEXTLINE(misc-static-assert) ABSL_SWISSTABLE_ASSERT(!SwisstableGenerationsEnabled()); return PrepareInsertLargeImpl(common, policy, hash, mask_empty, target_group); } -size_t PrepareInsertLargeGenerationsEnabled( +void* PrepareInsertLargeGenerationsEnabled( CommonFields& common, const PolicyFunctions& __restrict policy, size_t hash, Group::NonIterableBitMaskType mask_empty, FindInfo target_group, absl::FunctionRef<size_t(size_t)> recompute_hash) { @@ -2410,35 +2453,30 @@ // We need to instantiate ALL possible template combinations because we define // the function in the cc file. -template size_t GrowSooTableToNextCapacityAndPrepareInsert<0, false>( +template void* GrowSooTableToNextCapacityAndPrepareInsert<0, false>( CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>, bool); -template size_t GrowSooTableToNextCapacityAndPrepareInsert< +template void* GrowSooTableToNextCapacityAndPrepareInsert< OptimalMemcpySizeForSooSlotTransfer(1), true>( CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>, bool); static_assert(VerifyOptimalMemcpySizeForSooSlotTransferRange(2, 3)); -template size_t GrowSooTableToNextCapacityAndPrepareInsert< +template void* GrowSooTableToNextCapacityAndPrepareInsert< OptimalMemcpySizeForSooSlotTransfer(3), true>( CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>, bool); +#if UINTPTR_MAX == UINT32_MAX +static_assert(MaxSooSlotSize() == 4); +static_assert(VerifyOptimalMemcpySizeForSooSlotTransferRange(2, 4)); +#else static_assert(VerifyOptimalMemcpySizeForSooSlotTransferRange(4, 8)); -template size_t GrowSooTableToNextCapacityAndPrepareInsert< +template void* GrowSooTableToNextCapacityAndPrepareInsert< OptimalMemcpySizeForSooSlotTransfer(8), true>( CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>, bool); - -#if UINTPTR_MAX == UINT32_MAX static_assert(MaxSooSlotSize() == 8); -#else -static_assert(VerifyOptimalMemcpySizeForSooSlotTransferRange(9, 16)); -template size_t GrowSooTableToNextCapacityAndPrepareInsert< - OptimalMemcpySizeForSooSlotTransfer(16), true>( - CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>, - bool); -static_assert(MaxSooSlotSize() == 16); #endif template void* AllocateBackingArray<BackingArrayAlignment(alignof(size_t)),
diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 7119e10..e158cfc 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h
@@ -230,10 +230,6 @@ #include <ranges> // NOLINT(build/c++20) #endif -#if defined(__i386__) || defined(__x86_64__) -#include <immintrin.h> -#endif - namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { @@ -328,11 +324,11 @@ std::declval<Ts>()...))>, Policy, Hash, Eq, Ts...> : std::true_type {}; -ABSL_DLL extern ctrl_t kDefaultIterControl; +ABSL_DLL extern char kDefaultIterSlot; // Returns a pointer to a control byte that can be used by default-constructed // iterators. We don't expect this pointer to be dereferenced. -inline ctrl_t* DefaultIterControl() { return &kDefaultIterControl; } +inline void* DefaultIterSlot() { return &kDefaultIterSlot; } // For use in SOO iterators. // TODO(b/289225379): we could potentially get rid of this by adding an is_soo @@ -348,6 +344,21 @@ // Whether ctrl is from the SooControl array. inline bool IsSooControl(const ctrl_t* ctrl) { return ctrl == SooControl(); } +// For use in iterators returned by `insert` and similar. +ABSL_DLL extern const ctrl_t kInsertIteratorControl[2]; + +// Returns a pointer to a full byte followed by a sentinel byte. +inline ctrl_t* InsertIteratorControl() { + // Const must be cast away here; no uses of this function will actually write + // to it because it is only used for iterators returned by `insert` and + // similar. + return const_cast<ctrl_t*>(kInsertIteratorControl); +} +// Whether ctrl is special value for iterators returned by `insert` and similar. +inline bool IsInsertIteratorControl(const ctrl_t* ctrl) { + return ctrl == InsertIteratorControl(); +} + // Returns a pointer to a generation to use for an empty hashtable. GenerationType* EmptyGeneration(); @@ -366,11 +377,7 @@ // tables, we would need to randomize the iteration order somehow. constexpr size_t SooCapacity() { return 1; } // Maximum capacity of a table where we don't need to hash any keys. -constexpr size_t MaxSmallCapacity() { return 1; } -// Maximum capacity of a table where we can use blocked elements. -constexpr size_t MaxCapacityWithBlockedElements() { - return Group::kWidth - 1; -} +inline constexpr size_t kMaxSmallCapacity = 1; // Sentinel type to indicate SOO CommonFields construction. struct soo_tag_t {}; // Sentinel type to indicate SOO CommonFields construction with full size. @@ -389,7 +396,7 @@ // Whether a table is small enough that we don't need to hash any keys. constexpr bool IsSmallCapacity(size_t capacity) { - return capacity <= MaxSmallCapacity(); + return capacity <= kMaxSmallCapacity; } // Whether a table fits entirely into a probing group. @@ -401,7 +408,7 @@ // Whether `cap` is a valid capacity for a table that can store blocked // elements. constexpr bool IsCapacityValidForBlockedElements(size_t cap) { - return !IsSmallCapacity(cap) && cap <= MaxCapacityWithBlockedElements(); + return !IsSmallCapacity(cap); } // Converts `n` into the next valid capacity, per `IsValidCapacity`. @@ -548,23 +555,10 @@ // Comparing capacity_data_ directly leads to a better generated code. // One byte comparison is used before computing the capacity in order to // detect small tables faster for critical path. - static_assert(MaxSmallCapacity() == 1); + static_assert(kMaxSmallCapacity == 1); return capacity_data_ <= 1; } - constexpr size_t mask(size_t value) const { -#ifdef __BMI2__ - if constexpr (StorageMode == kCapacityByLog) { - if constexpr (sizeof(size_t) == 8) { - return _bzhi_u64(value, capacity_data_); - } else { - return _bzhi_u32(value, capacity_data_); - } - } -#endif // __BMI2__ - return value & capacity(); - } - private: // We use these sentinel capacity values in debug mode to indicate different // classes of bugs. @@ -628,7 +622,9 @@ // Capacity, size and also has additionally // 1) one bit that stores whether we have infoz. -// 2) PerTableSeed::kBitCount bits for the seed. (For SOO tables, the lowest +// 2) kBlockedElementsBitCount bits that stores number of blocked elements in +// the table. +// 3) PerTableSeed::kBitCount bits for the seed. (For SOO tables, the lowest // bit of the seed is repurposed to track if sampling has been tried). template <HashtableCapacityStorageMode StorageMode> class HashtableInlineDataImpl { @@ -637,10 +633,13 @@ using PerTableSeed = PerTableSeedImpl< std::conditional_t<StorageMode == kCapacityByValue, uint16_t, uint8_t>>; using HashtableCapacity = HashtableCapacityImpl<StorageMode>; + static constexpr size_t kBlockedElementBitCount = 3; + static constexpr size_t kMaxBlockedElementCount = + (uint64_t{1} << kBlockedElementBitCount) - 1; static constexpr size_t kSizeBitCount = - StorageMode == kCapacityByValue - ? 64 - PerTableSeed::kBitCount - 1 - : 64 - PerTableSeed::kBitCount - sizeof(HashtableCapacity) * 8 - 1; + 64 - + (kBlockedElementBitCount + PerTableSeed::kBitCount + /*has_infoz*/ 1 + + (StorageMode == kCapacityByValue ? 0 : sizeof(HashtableCapacity) * 8)); explicit HashtableInlineDataImpl(uninitialized_tag_t) {} explicit HashtableInlineDataImpl(HashtableCapacity capacity, @@ -706,17 +705,33 @@ // Sets the has_infoz bit. void set_has_infoz() { data_ |= kHasInfozMask; } + // Returns the number of blocked elements in the table. + size_t blocked_element_count() const { + return (data_ & kBlockedElementMask) >> kBlockedElementsShift; + } + // Initializes the number of blocked elements in the table. + // Requires: + // 1. `blocked_element_count() == 0`. + // 2. `count <= kMaxBlockedElementCount`. + void init_blocked_element_count(uint64_t count) { + ABSL_SWISSTABLE_ASSERT(blocked_element_count() == 0); + ABSL_SWISSTABLE_ASSERT(count <= kMaxBlockedElementCount); + data_ |= count << kBlockedElementsShift; + } + void set_blocked_element_count_to_zero() { data_ &= ~kBlockedElementMask; } + void set_no_seed_for_testing() { data_ &= ~kSeedMask; } private: // Bit layout of `data_` from MSB to LSB: - // (47 bits) : size + // (44 bits) : size + // (3 bits) : blocked_element_count // (1 bit) : has_infoz // (16 or 8 bits) : seed // We don't split these components of `data_` into separate bit field elements // because we get worse generated code that way. static constexpr size_t kDataBitCount = - PerTableSeed::kBitCount + 1 + kSizeBitCount; + PerTableSeed::kBitCount + 1 + kSizeBitCount + kBlockedElementBitCount; static constexpr size_t kSizeShift = kDataBitCount - kSizeBitCount; static constexpr uint64_t kSizeOneNoMetadata = uint64_t{1} << kSizeShift; static constexpr uint64_t kMetadataMask = kSizeOneNoMetadata - 1; @@ -724,6 +739,9 @@ (uint64_t{1} << PerTableSeed::kBitCount) - 1; // The next bit after the seed. static constexpr uint64_t kHasInfozMask = kSeedMask + 1; + static constexpr uint64_t kBlockedElementsShift = PerTableSeed::kBitCount + 1; + static constexpr uint64_t kBlockedElementMask = kMaxBlockedElementCount + << kBlockedElementsShift; // For SOO tables, the seed is unused, and bit 0 is repurposed to track // whether the table has already queried should_sample_soo(). static constexpr uint64_t kSooHasTriedSamplingMask = 1; @@ -755,6 +773,12 @@ using PerTableSeed = HashtableInlineData::PerTableSeed; using HashtableCapacity = HashtableInlineData::HashtableCapacity; +// For large tables, we limit the number of blocked elements to maintain O(1) +// average case lookup complexity. +constexpr size_t kMaxBlockedElementsForLargeTables = 5; +static_assert(kMaxBlockedElementsForLargeTables <= + HashtableInlineData::kMaxBlockedElementCount); + // H1 is just the low bits of the hash. inline size_t H1(size_t hash) { return hash; } @@ -1002,7 +1026,8 @@ static constexpr uint64_t kLowerBoundShift = 64 - 8; explicit GrowthInfoAccessor(void* control) - : growth_info_lower_bound_(reinterpret_cast<uint8_t*>(control) - 1) {} + : growth_info_lower_bound_(reinterpret_cast<uint8_t*>(control) - 1 - + NumGenerationBytes()) {} // Initializes the GrowthInfo assuming we can grow `growth_left` elements // and there are no kDeleted slots in the table. @@ -1090,15 +1115,16 @@ : sizeof(uint64_t); } -// Computes the offset from the start of the backing allocation of control. -// infoz and growth_info are stored at the beginning of the backing array. -constexpr size_t ControlOffset(bool has_infoz, size_t capacity) { +// Computes the size of the metadata before the control bytes. infoz, +// growth_info and generation are stored at the beginning of the backing array. +constexpr size_t MetadataBeforeControlSize(bool has_infoz, size_t capacity) { if (ABSL_PREDICT_FALSE(has_infoz)) { // We always allocate 8 bytes of growth info for sampled tables to allow // branchless access to infoz pointer. - return sizeof(HashtablezInfoHandle) + sizeof(uint64_t); + return sizeof(HashtablezInfoHandle) + sizeof(uint64_t) + + NumGenerationBytes(); } - return GrowthInfoSizeForCapacity(capacity); + return GrowthInfoSizeForCapacity(capacity) + NumGenerationBytes(); } // Returns the offset of the next item after `offset` that is aligned to `align` @@ -1113,16 +1139,21 @@ explicit RawHashSetLayout(size_t capacity, size_t slot_size, size_t slot_align, bool has_infoz, size_t blocked_element_count) - : control_offset_(ControlOffset(has_infoz, capacity)), - generation_offset_(control_offset_ + NumControlBytes(capacity)), - slot_offset_( - AlignUpTo(generation_offset_ + NumGenerationBytes(), slot_align)), - alloc_size_(slot_offset_ + - (capacity - blocked_element_count) * slot_size) { + : control_offset_(MetadataBeforeControlSize(has_infoz, capacity)), + generation_offset_(control_offset_ - NumGenerationBytes()), + slot_offset_(control_offset_ + NumControlBytes(capacity)) { ABSL_SWISSTABLE_ASSERT(IsValidCapacity(capacity)); + size_t aligned_slot_offset = AlignUpTo(slot_offset_, slot_align); + size_t slot_array_padding = aligned_slot_offset - slot_offset_; + slot_offset_ = aligned_slot_offset; ABSL_SWISSTABLE_ASSERT( slot_size <= ((std::numeric_limits<size_t>::max)() - slot_offset_) / capacity); + control_offset_ += slot_array_padding; + generation_offset_ += slot_array_padding; + ABSL_SWISSTABLE_ASSERT(!IsSmallCapacity(capacity) || + control_offset_ == slot_offset_); + alloc_size_ = slot_offset_ + (capacity - blocked_element_count) * slot_size; } // Returns precomputed offset from the start of the backing allocation of @@ -1169,11 +1200,6 @@ // Note that growth_info is stored immediately before this pointer. // May be uninitialized for small tables. MaybeInitializedPtr<ctrl_t> control; - - // The beginning of the slots, located at `SlotOffset()` bytes after - // `control`. May be uninitialized for empty tables. - // Note: we can't use `slots` because Qt defines "slots" as a macro. - MaybeInitializedPtr<void> slot_array; }; // Returns the maximum size of the SOO slot. @@ -1188,12 +1214,6 @@ MaybeInitializedPtr<ctrl_t> control() const { ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.control); } - MaybeInitializedPtr<void>& slot_array() { - ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.slot_array); - } - MaybeInitializedPtr<void> slot_array() const { - ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap.slot_array); - } void* get_soo_data() { ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(soo_data); } @@ -1264,11 +1284,16 @@ void set_control(ctrl_t* c) { heap_or_soo_.control().set(c); } // Note: we can't use slots() because Qt defines "slots" as a macro. - void* slot_array() const { return heap_or_soo_.slot_array().get(); } - MaybeInitializedPtr<void> slots_union() const { - return heap_or_soo_.slot_array(); + // Returns pointer to the slots of a table with explicit capacity that must be + // equal to the actual capacity of the table. + // Capacity is often known at compile time or already in register with some + // ABSL_ASSUME conditions. We require passing it explicitly to eliminate + // branches inside of NumControlBytes in majority of cases. + void* slot_array(size_t capacity) const { + ABSL_SWISSTABLE_ASSERT(capacity == this->capacity()); + ctrl_t* ctrl = control(); + return ctrl + NumControlBytes(capacity); } - void set_slots(void* s) { heap_or_soo_.slot_array().set(s); } // The number of filled slots. size_t size() const { return inline_data_.size(); } @@ -1346,22 +1371,12 @@ inline_data_.set_has_infoz(); } - HashtablezInfoHandle* infoz_ptr() const { - // growth_info is stored before control bytes. - ABSL_SWISSTABLE_ASSERT( - reinterpret_cast<uintptr_t>(control()) % alignof(size_t) == 0); - ABSL_SWISSTABLE_ASSERT(has_infoz()); - return reinterpret_cast<HashtablezInfoHandle*>( - control() - ControlOffset(/*has_infoz=*/true, capacity())); - } + HashtablezInfoHandle infoz_ptr() const; HashtablezInfoHandle infoz() { - return has_infoz() ? *infoz_ptr() : HashtablezInfoHandle(); + return has_infoz() ? infoz_ptr() : HashtablezInfoHandle(); } - void set_infoz(HashtablezInfoHandle infoz) { - ABSL_SWISSTABLE_ASSERT(has_infoz()); - *infoz_ptr() = infoz; - } + void set_infoz(HashtablezInfoHandle infoz); bool should_rehash_for_bug_detection_on_insert() const { if constexpr (!SwisstableGenerationsEnabled()) { @@ -1383,19 +1398,17 @@ // corresponding slots. // Control bytes are set to kSentinel for blocked elements. size_t blocked_element_count() const { - size_t cap = capacity(); - if (!IsCapacityValidForBlockedElements(cap)) { - return 0; - } - ABSL_SWISSTABLE_ASSERT(is_single_group(cap)); - // Formula is valid because MaxCapacityWithBlockedElements is less than - // group width. On erase for single group tables, we always increment the - // growth left. - ABSL_SWISSTABLE_ASSERT(cap <= - GrowthInfoLowerBound::kMaxGrowthLeftLowerBound); - return CapacityToGrowth(cap) - size() - - // We can use lower bound here because capacity is small. - growth_info().GetGrowthLeftLowerBound(); + return inline_data_.blocked_element_count(); + } + // Initializes the number of blocked elements in the table. + // Requires: + // 1. `blocked_element_count() == 0`. + // 2. `count <= kMaxBlockedElementCount`. + void init_blocked_element_count(size_t count) { + inline_data_.init_blocked_element_count(count); + } + void set_blocked_element_count_to_zero() { + inline_data_.set_blocked_element_count_to_zero(); } // The size of the backing array allocation. @@ -1456,12 +1469,9 @@ void AssertNotDebugCapacityImpl() const; - // TODO(b/289225379): we could put size_ into HeapOrSoo and make capacity_ - // encode the size in SOO case. We would be making size()/capacity() more - // expensive in order to have more SOO space. HashtableInlineData inline_data_; - // Either the control/slots pointers or the SOO slot. + // Either the heap pointer or the SOO slot. HeapOrSoo heap_or_soo_; }; @@ -1479,14 +1489,14 @@ void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity); template <class InputIter> -size_t SelectBucketCountForIterRange(InputIter first, InputIter last, - size_t bucket_count) { - if (bucket_count != 0) { - return bucket_count; +size_t SelectReservationSizeForIterRange(InputIter first, InputIter last, + size_t reservation_size) { + if (reservation_size != 0) { + return reservation_size; } if (base_internal::IsAtLeastIterator<std::random_access_iterator_tag, InputIter>()) { - return SizeToCapacity(static_cast<size_t>(std::distance(first, last))); + return static_cast<size_t>(std::distance(first, last)); } return 0; } @@ -1520,7 +1530,12 @@ return ret; } -inline void AssertIsFull(const ctrl_t* ctrl, GenerationType generation, +// Note: we take control pointers by reference in a few Assert* functions below +// so that it's not UB if they're uninitialized as long as we don't read them +// (when slot is null). + +inline void AssertIsFull(const ctrl_t* const& ctrl, const void* slot, + GenerationType generation, const GenerationType* generation_ptr, const char* operation) { if (!SwisstableDebugEnabled()) return; @@ -1529,10 +1544,10 @@ // - use `ABSL_PREDICT_FALSE()` to provide a compiler hint for code layout // - use `ABSL_RAW_LOG()` with a format string to reduce code size and improve // the chances that the hot paths will be inlined. - if (ABSL_PREDICT_FALSE(ctrl == nullptr)) { + if (ABSL_PREDICT_FALSE(slot == nullptr)) { ABSL_RAW_LOG(FATAL, "%s called on end() iterator.", operation); } - if (ABSL_PREDICT_FALSE(ctrl == DefaultIterControl())) { + if (ABSL_PREDICT_FALSE(slot == DefaultIterSlot())) { ABSL_RAW_LOG(FATAL, "%s called on default-constructed iterator.", operation); } @@ -1563,12 +1578,13 @@ } // Note that for comparisons, null/end iterators are valid. -inline void AssertIsValidForComparison(const ctrl_t* ctrl, +inline void AssertIsValidForComparison(const ctrl_t* const& ctrl, + const void* slot, GenerationType generation, const GenerationType* generation_ptr) { if (!SwisstableDebugEnabled()) return; const bool ctrl_is_valid_for_comparison = - ctrl == nullptr || ctrl == DefaultIterControl() || + slot == nullptr || slot == DefaultIterSlot() || IsFull(CrashIfIteratorIsInvalid(ctrl)); if (SwisstableGenerationsEnabled()) { if (ABSL_PREDICT_FALSE(generation != @@ -1597,33 +1613,34 @@ // If the two iterators come from the same container, then their pointers will // interleave such that ctrl_a <= ctrl_b < slot_a <= slot_b or vice/versa. -// Note: we take slots by reference so that it's not UB if they're uninitialized -// as long as we don't read them (when ctrl is null). -inline bool AreItersFromSameContainer(const ctrl_t* ctrl_a, - const ctrl_t* ctrl_b, - const void* const& slot_a, - const void* const& slot_b) { - // If either control byte is null, then we can't tell. - if (ctrl_a == nullptr || ctrl_b == nullptr) return true; +inline bool AreItersFromSameContainer(const ctrl_t* const& ctrl_a, + const ctrl_t* const& ctrl_b, + const void* slot_a, const void* slot_b) { + // If either slot is null, then we can't tell. + if (slot_a == nullptr || slot_b == nullptr) return true; + // If either slot is iterator returned by insert, then we can't tell. + if (IsInsertIteratorControl(ctrl_a) || IsInsertIteratorControl(ctrl_b)) { + return true; + } const bool a_is_soo = IsSooControl(ctrl_a); if (a_is_soo != IsSooControl(ctrl_b)) return false; if (a_is_soo) return slot_a == slot_b; - const void* low_slot = slot_a; - const void* hi_slot = slot_b; + const void* low_ctrl = ctrl_a; + const void* hi_ctrl = ctrl_b; if (ctrl_a > ctrl_b) { - std::swap(ctrl_a, ctrl_b); - std::swap(low_slot, hi_slot); + std::swap(low_ctrl, hi_ctrl); + std::swap(slot_a, slot_b); } - return ctrl_b < low_slot && low_slot <= hi_slot; + return hi_ctrl < slot_a && slot_a <= slot_b; } // Asserts that two iterators come from the same container. // Note: we take slots by reference so that it's not UB if they're uninitialized // as long as we don't read them (when ctrl is null). -inline void AssertSameContainer(const ctrl_t* ctrl_a, const ctrl_t* ctrl_b, - const void* const& slot_a, - const void* const& slot_b, +inline void AssertSameContainer(const ctrl_t* const& ctrl_a, + const ctrl_t* const& ctrl_b, const void* slot_a, + const void* slot_b, const GenerationType* generation_ptr_a, const GenerationType* generation_ptr_b) { if (!SwisstableDebugEnabled()) return; @@ -1641,8 +1658,8 @@ } }; - const bool a_is_default = ctrl_a == DefaultIterControl(); - const bool b_is_default = ctrl_b == DefaultIterControl(); + const bool a_is_default = slot_a == DefaultIterSlot(); + const bool b_is_default = slot_b == DefaultIterSlot(); if (a_is_default && b_is_default) return; fail_if(a_is_default != b_is_default, "Comparing default-constructed hashtable iterator with a " @@ -1658,8 +1675,8 @@ fail_if(a_is_empty && b_is_empty, "Comparing iterators from different empty hashtables."); - const bool a_is_end = ctrl_a == nullptr; - const bool b_is_end = ctrl_b == nullptr; + const bool a_is_end = slot_a == nullptr; + const bool b_is_end = slot_b == nullptr; fail_if(a_is_end || b_is_end, "Comparing iterator with an end() iterator from a different " "hashtable."); @@ -1678,6 +1695,10 @@ size_t probe_length; }; +struct ProbeCapacity { + size_t capacity; +}; + // The state for a probe sequence. // // Currently, the sequence is a triangular progression of the form @@ -1705,37 +1726,36 @@ // Creates a new probe sequence using `hash` as the initial value of the // sequence and `capacity` as the mask to apply to each value in the // progression. - probe_seq(HashtableCapacity capacity, size_t hash) - : capacity_(capacity), offset_(capacity.mask(hash)) {} + probe_seq(ProbeCapacity capacity, size_t hash) + : capacity_(capacity.capacity), offset_(hash & capacity_) {} // The offset within the table, i.e., the value `p(i)` above. size_t offset() const { return offset_; } - size_t offset(size_t i) const { return capacity_.mask(offset_ + i); } + size_t offset(size_t i) const { return (offset_ + i) & capacity_; } void next() { index_ += Width; offset_ += index_; - offset_ = capacity_.mask(offset_); + offset_ &= capacity_; } // 0-based probe index, a multiple of `Width`. size_t index() const { return index_; } private: - HashtableCapacity capacity_; + size_t capacity_; size_t offset_; size_t index_ = 0; }; // Begins a probing operation on `common.control`, using `hash`. -inline probe_seq<Group::kWidth> probe_h1(HashtableCapacity capacity, - size_t h1) { +inline probe_seq<Group::kWidth> probe_h1(ProbeCapacity capacity, size_t h1) { return probe_seq<Group::kWidth>(capacity, h1); } -inline probe_seq<Group::kWidth> probe(HashtableCapacity capacity, size_t hash) { +inline probe_seq<Group::kWidth> probe(ProbeCapacity capacity, size_t hash) { return probe_h1(capacity, H1(hash)); } inline probe_seq<Group::kWidth> probe(const CommonFields& common, size_t hash) { - return probe(common.capacity_impl(), hash); + return probe(ProbeCapacity{common.capacity()}, hash); } constexpr size_t kProbedElementIndexSentinel = ~size_t{}; @@ -1968,16 +1988,6 @@ void ReserveTableToFitNewSize(CommonFields& common, const PolicyFunctions& policy, size_t new_size); -// Resizes empty non-allocated table to the next valid capacity after -// `bucket_count`. Requires: -// 1. `c.capacity() == policy.soo_capacity`. -// 2. `c.empty()`. -// 3. `new_size > policy.soo_capacity`. -// 4. `bucket_count <= MaxValidCapacity()`. -// The table will be attempted to be sampled. -void ReserveEmptyNonAllocatedTableToFitBucketCount( - CommonFields& common, const PolicyFunctions& policy, size_t bucket_count); - // Type erased version of raw_hash_set::rehash. // Requires: `n <= MaxValidCapacity()`. void Rehash(CommonFields& common, const PolicyFunctions& policy, size_t n); @@ -1997,47 +2007,38 @@ // instantiations. constexpr size_t OptimalMemcpySizeForSooSlotTransfer( size_t slot_size, size_t max_soo_slot_size = MaxSooSlotSize()) { - static_assert(MaxSooSlotSize() >= 8, "unexpectedly small SOO slot size"); + static_assert(MaxSooSlotSize() >= 4, "unexpectedly small SOO slot size"); + static_assert(MaxSooSlotSize() <= 8, "unexpectedly large SOO slot size"); if (slot_size == 1) { return 1; } if (slot_size <= 3) { return 4; } + if (slot_size == max_soo_slot_size) { + return max_soo_slot_size; + } // We are merging 4 and 8 into one case because we expect them to be the // hottest cases. Copying 8 bytes is as fast on common architectures. - if (slot_size <= 8) { - return 8; - } - if (max_soo_slot_size <= 16) { - return max_soo_slot_size; - } - if (slot_size <= 16) { - return 16; - } - if (max_soo_slot_size <= 24) { - return max_soo_slot_size; - } - static_assert(MaxSooSlotSize() <= 24, "unexpectedly large SOO slot size"); - return 24; + return 8; } // Resizes SOO table to the NextCapacity(SooCapacity()) and prepares insert for -// the given new_hash. Returns the offset of the new element. +// the given new_hash. Returns the new slot. // All possible template combinations are defined in cc file to improve // compilation time. template <size_t SooSlotMemcpySize, bool TransferUsesMemcpy> -size_t GrowSooTableToNextCapacityAndPrepareInsert( +void* GrowSooTableToNextCapacityAndPrepareInsert( CommonFields& common, const PolicyFunctions& policy, absl::FunctionRef<size_t(size_t)> get_hash, bool force_sampling); // PrepareInsert for small tables (is_small()==true). -// Returns the new control and the new slot. +// Returns the new slot. // Hash is only computed if the table is sampled or grew to large size // (is_small()==false). -std::pair<ctrl_t*, void*> PrepareInsertSmallNonSoo( - CommonFields& common, const PolicyFunctions& policy, - absl::FunctionRef<size_t(size_t)> get_hash); +void* PrepareInsertSmallNonSoo(CommonFields& common, + const PolicyFunctions& policy, + absl::FunctionRef<size_t(size_t)> get_hash); // Resizes table with allocated slots and change the table seed. // Tables with SOO enabled must have capacity > policy.soo_capacity. @@ -2090,7 +2091,7 @@ // Type-erased versions of raw_hash_set::erase_meta_only_{small,large}. void EraseMetaOnlySmall(CommonFields& c, bool soo_enabled, size_t slot_size); -void EraseMetaOnlyLarge(CommonFields& c, const ctrl_t* ctrl, size_t slot_size); +void EraseMetaOnlyLarge(CommonFields& c, size_t index, size_t slot_size); // For trivially relocatable types we use memcpy directly. This allows us to // share the same function body for raw_hash_set instantiations that have the @@ -2110,8 +2111,7 @@ void* GetRefForEmptyClass(CommonFields& common); // Given the hash of a value not currently in the table and the first group with -// an empty slot in the probe sequence, finds a viable slot index to insert it -// at. +// an empty slot in the probe sequence, finds a viable slot to insert it at. // // In case there's no space left, the table can be resized or rehashed // (for tables with deleted slots, see FindInsertPositionWithGrowthOrRehash). @@ -2128,13 +2128,13 @@ // `target_group`. // REQUIRES: `target_group` is a starting position for the group that has // at least one empty slot. -size_t PrepareInsertLarge(CommonFields& common, const PolicyFunctions& policy, - size_t hash, Group::NonIterableBitMaskType mask_empty, - FindInfo target_group); +void* PrepareInsertLarge(CommonFields& common, const PolicyFunctions& policy, + size_t hash, Group::NonIterableBitMaskType mask_empty, + FindInfo target_group); // Same as above, but with generations enabled, we may end up changing the seed, // which means we need to be able to recompute the hash. -size_t PrepareInsertLargeGenerationsEnabled( +void* PrepareInsertLargeGenerationsEnabled( CommonFields& common, const PolicyFunctions& policy, size_t hash, Group::NonIterableBitMaskType mask_empty, FindInfo target_group, absl::FunctionRef<size_t(size_t)> recompute_hash); @@ -2334,7 +2334,9 @@ using pointer = std::remove_reference_t<reference>*; using difference_type = typename raw_hash_set::difference_type; - iterator() {} + // We use DefaultIterSlot() for default-constructed iterators so that + // they can be distinguished from end iterators, which have nullptr slot_. + iterator() : slot_(static_cast<slot_type*>(DefaultIterSlot())) {} // PRECONDITION: not an end() iterator. reference operator*() const { @@ -2354,7 +2356,7 @@ ++ctrl_; ++slot_; skip_empty_or_deleted(); - if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr; + if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) slot_ = nullptr; return *this; } // PRECONDITION: not an end() iterator. @@ -2365,11 +2367,13 @@ } friend bool operator==(const iterator& a, const iterator& b) { - AssertIsValidForComparison(a.ctrl_, a.generation(), a.generation_ptr()); - AssertIsValidForComparison(b.ctrl_, b.generation(), b.generation_ptr()); + AssertIsValidForComparison(a.ctrl_, a.slot_, a.generation(), + a.generation_ptr()); + AssertIsValidForComparison(b.ctrl_, b.slot_, b.generation(), + b.generation_ptr()); AssertSameContainer(a.ctrl_, b.ctrl_, a.slot_, b.slot_, a.generation_ptr(), b.generation_ptr()); - return a.ctrl_ == b.ctrl_; + return a.unchecked_equals(b); } friend bool operator!=(const iterator& a, const iterator& b) { return !(a == b); @@ -2383,26 +2387,14 @@ slot_(slot) { // This assumption helps the compiler know that any non-end iterator is // not equal to any end iterator. - ABSL_ASSUME(ctrl != nullptr); - } - // This constructor is used in begin() to avoid an MSan - // use-of-uninitialized-value error. Delegating from this constructor to - // the previous one doesn't avoid the error. - iterator(ctrl_t* ctrl, MaybeInitializedPtr<void> slot, - const GenerationType* generation_ptr) - : HashSetIteratorGenerationInfo(generation_ptr), - ctrl_(ctrl), - slot_(to_slot(slot.get())) { - // This assumption helps the compiler know that any non-end iterator is - // not equal to any end iterator. - ABSL_ASSUME(ctrl != nullptr); + ABSL_ASSUME(slot != nullptr); } // For end() iterators. explicit iterator(const GenerationType* generation_ptr) - : HashSetIteratorGenerationInfo(generation_ptr), ctrl_(nullptr) {} + : HashSetIteratorGenerationInfo(generation_ptr), slot_(nullptr) {} void assert_is_full(const char* operation) const { - AssertIsFull(ctrl_, generation(), generation_ptr(), operation); + AssertIsFull(ctrl_, slot_, generation(), generation_ptr(), operation); } // Fixes up `ctrl_` to point to a full or sentinel by advancing `ctrl_` and @@ -2418,9 +2410,7 @@ // checks. // Should be used when the lifetimes of the iterators are well-enough // understood to prove that they cannot be invalid. - bool unchecked_equals(const iterator& b) const { - return ctrl_ == b.control(); - } + bool unchecked_equals(const iterator& b) const { return slot_ == b.slot(); } // Dereferences the iterator without ABSL Hardening iterator invalidation // checks. @@ -2429,14 +2419,12 @@ ctrl_t* control() const { return ctrl_; } slot_type* slot() const { return slot_; } - // We use DefaultIterControl() for default-constructed iterators so that - // they can be distinguished from end iterators, which have nullptr ctrl_. - ctrl_t* ctrl_ = DefaultIterControl(); - // To avoid uninitialized member warnings, put slot_ in an anonymous union. + // To avoid uninitialized member warnings, put ctrl_ in an anonymous union. // The member is not initialized on singleton and end iterators. union { - slot_type* slot_; + ctrl_t* ctrl_; }; + slot_type* slot_; }; class const_iterator { @@ -2496,68 +2484,68 @@ std::is_nothrow_default_constructible_v<key_equal> && std::is_nothrow_default_constructible_v<allocator_type>) {} - explicit raw_hash_set( - size_t bucket_count, const hasher& hash = hasher(), - const key_equal& eq = key_equal(), - const allocator_type& alloc = allocator_type()) + explicit raw_hash_set(size_t reservation_size, const hasher& hash = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) : settings_(CommonFields::CreateDefault<SooEnabled()>(), hash, eq, alloc) { - if (bucket_count > DefaultCapacity()) { - ReserveEmptyNonAllocatedTableToFitBucketCount( - common(), GetPolicyFunctions(), - (std::min)(bucket_count, MaxValidCapacity())); + if (reservation_size > DefaultCapacity()) { + ReserveTableToFitNewSize(common(), GetPolicyFunctions(), + reservation_size); } } - raw_hash_set(size_t bucket_count, const hasher& hash, + raw_hash_set(size_t reservation_size, const hasher& hash, const allocator_type& alloc) - : raw_hash_set(bucket_count, hash, key_equal(), alloc) {} + : raw_hash_set(reservation_size, hash, key_equal(), alloc) {} - raw_hash_set(size_t bucket_count, const allocator_type& alloc) - : raw_hash_set(bucket_count, hasher(), key_equal(), alloc) {} + raw_hash_set(size_t reservation_size, const allocator_type& alloc) + : raw_hash_set(reservation_size, hasher(), key_equal(), alloc) {} explicit raw_hash_set(const allocator_type& alloc) : raw_hash_set(0, hasher(), key_equal(), alloc) {} template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_count = 0, + raw_hash_set(InputIter first, InputIter last, size_t reservation_size = 0, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : raw_hash_set(SelectBucketCountForIterRange(first, last, bucket_count), - hash, eq, alloc) { + : raw_hash_set( + SelectReservationSizeForIterRange(first, last, reservation_size), + hash, eq, alloc) { insert(first, last); } template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_count, + raw_hash_set(InputIter first, InputIter last, size_t reservation_size, const hasher& hash, const allocator_type& alloc) - : raw_hash_set(first, last, bucket_count, hash, key_equal(), alloc) {} + : raw_hash_set(first, last, reservation_size, hash, key_equal(), alloc) {} template <class InputIter> - raw_hash_set(InputIter first, InputIter last, size_t bucket_count, + raw_hash_set(InputIter first, InputIter last, size_t reservation_size, const allocator_type& alloc) - : raw_hash_set(first, last, bucket_count, hasher(), key_equal(), alloc) {} + : raw_hash_set(first, last, reservation_size, hasher(), key_equal(), + alloc) {} #if defined(__cpp_lib_containers_ranges) && \ __cpp_lib_containers_ranges >= 202202L template <typename R> - raw_hash_set(std::from_range_t, R&& rg, size_type bucket_count = 0, + raw_hash_set(std::from_range_t, R&& rg, size_type reservation_size = 0, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : raw_hash_set(std::begin(rg), std::end(rg), bucket_count, hash, eq, + : raw_hash_set(std::begin(rg), std::end(rg), reservation_size, hash, eq, alloc) {} template <typename R> - raw_hash_set(std::from_range_t, R&& rg, size_type bucket_count, + raw_hash_set(std::from_range_t, R&& rg, size_type reservation_size, const allocator_type& alloc) - : raw_hash_set(std::from_range, std::forward<R>(rg), bucket_count, + : raw_hash_set(std::from_range, std::forward<R>(rg), reservation_size, hasher(), key_equal(), alloc) {} template <typename R> - raw_hash_set(std::from_range_t, R&& rg, size_type bucket_count, + raw_hash_set(std::from_range_t, R&& rg, size_type reservation_size, const hasher& hash, const allocator_type& alloc) - : raw_hash_set(std::from_range, std::forward<R>(rg), bucket_count, hash, - key_equal(), alloc) {} + : raw_hash_set(std::from_range, std::forward<R>(rg), reservation_size, + hash, key_equal(), alloc) {} #endif template <class InputIter> @@ -2587,35 +2575,38 @@ // RequiresNotInit<T> is a workaround for gcc prior to 7.1. template <class T, RequiresNotInit<T> = 0, std::enable_if_t<Insertable<T>::value, int> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_count = 0, + raw_hash_set(std::initializer_list<T> init, size_t reservation_size = 0, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_count, hash, eq, alloc) {} + : raw_hash_set(init.begin(), init.end(), reservation_size, hash, eq, + alloc) {} - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count = 0, - const hasher& hash = hasher(), const key_equal& eq = key_equal(), + raw_hash_set(std::initializer_list<init_type> init, + size_t reservation_size = 0, const hasher& hash = hasher(), + const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : raw_hash_set(init.begin(), init.end(), bucket_count, hash, eq, alloc) {} + : raw_hash_set(init.begin(), init.end(), reservation_size, hash, eq, + alloc) {} template <class T, RequiresNotInit<T> = 0, std::enable_if_t<Insertable<T>::value, int> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_count, + raw_hash_set(std::initializer_list<T> init, size_t reservation_size, const hasher& hash, const allocator_type& alloc) - : raw_hash_set(init, bucket_count, hash, key_equal(), alloc) {} + : raw_hash_set(init, reservation_size, hash, key_equal(), alloc) {} - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count, + raw_hash_set(std::initializer_list<init_type> init, size_t reservation_size, const hasher& hash, const allocator_type& alloc) - : raw_hash_set(init, bucket_count, hash, key_equal(), alloc) {} + : raw_hash_set(init, reservation_size, hash, key_equal(), alloc) {} template <class T, RequiresNotInit<T> = 0, std::enable_if_t<Insertable<T>::value, int> = 0> - raw_hash_set(std::initializer_list<T> init, size_t bucket_count, + raw_hash_set(std::initializer_list<T> init, size_t reservation_size, const allocator_type& alloc) - : raw_hash_set(init, bucket_count, hasher(), key_equal(), alloc) {} + : raw_hash_set(init, reservation_size, hasher(), key_equal(), alloc) {} - raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count, + raw_hash_set(std::initializer_list<init_type> init, size_t reservation_size, const allocator_type& alloc) - : raw_hash_set(init, bucket_count, hasher(), key_equal(), alloc) {} + : raw_hash_set(init, reservation_size, hasher(), key_equal(), alloc) {} template <class T, RequiresNotInit<T> = 0, std::enable_if_t<Insertable<T>::value, int> = 0> @@ -2713,7 +2704,7 @@ iterator begin() ABSL_ATTRIBUTE_LIFETIME_BOUND { if (ABSL_PREDICT_FALSE(empty())) return end(); if (is_small()) return single_iterator(); - iterator it = {control(), common().slots_union(), + iterator it = {control(), slot_array(capacity()), common().generation_ptr()}; it.skip_empty_or_deleted(); ABSL_SWISSTABLE_ASSERT(IsFull(*it.control())); @@ -2781,7 +2772,7 @@ IsLifetimeBoundAssignmentFrom<T>::value, int> = 0> std::pair<iterator, bool> insert( - T&& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + T&& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->template insert<T, 0>(std::forward<T>(value)); } @@ -2810,7 +2801,7 @@ IsLifetimeBoundAssignmentFrom<const T&>::value, int> = 0> std::pair<iterator, bool> insert( - const T& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + const T& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->template insert<T, 0>(value); } @@ -2830,7 +2821,7 @@ } #if ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L std::pair<iterator, bool> insert( - init_type&& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + init_type&& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) ABSL_ATTRIBUTE_LIFETIME_BOUND requires(IsLifetimeBoundAssignmentFrom<init_type>::value) { @@ -2852,7 +2843,7 @@ IsLifetimeBoundAssignmentFrom<T>::value, int> = 0> iterator insert(const_iterator hint, - T&& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + T&& value ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) ABSL_ATTRIBUTE_LIFETIME_BOUND { return this->template insert<T, 0>(hint, std::forward<T>(value)); } @@ -2994,12 +2985,12 @@ F&& f) ABSL_ATTRIBUTE_LIFETIME_BOUND { auto res = find_or_prepare_insert(key); if (res.second) { - slot_type* slot = res.first.slot(); + slot_type* slot = res.first; allocator_type alloc(char_alloc_ref()); std::forward<F>(f)(constructor(&alloc, &slot)); ABSL_SWISSTABLE_ASSERT(!slot); } - return res.first; + return non_iterable_iterator_at_slot(res.first); } // Extension API: support for heterogeneous keys. @@ -3137,8 +3128,7 @@ void reserve(size_t n) { if (ABSL_PREDICT_TRUE(n > DefaultCapacity())) { - ReserveTableToFitNewSize(common(), GetPolicyFunctions(), - (std::min)(n, MaxValidSize())); + ReserveTableToFitNewSize(common(), GetPolicyFunctions(), n); } } @@ -3170,7 +3160,7 @@ if (is_small()) return; auto seq = probe(common(), hash_of(key)); PrefetchToLocalCache(control() + seq.offset()); - PrefetchToLocalCache(slot_array() + seq.offset()); + PrefetchToLocalCache(slot_array(capacity()) + seq.offset()); #endif // ABSL_HAVE_PREFETCH } @@ -3187,7 +3177,7 @@ AssertOnFind(key); if (is_small()) return find_small(key); prefetch_heap_block(); - return find_large(key, hash_of(key)); + return find_large(key); } template <class K = key_type> @@ -3300,7 +3290,7 @@ if (res.second) { s.emplace_at(res.first, std::forward<Args>(args)...); } - return res; + return {s.non_iterable_iterator_at_slot(res.first), res.second}; } raw_hash_set& s; }; @@ -3311,11 +3301,11 @@ std::pair<iterator, bool> operator()(const K& key, Args&&...) && { auto res = s.find_or_prepare_insert(key); if (res.second) { - s.transfer(res.first.slot(), &slot); + s.transfer(res.first, &slot); } else if (do_destroy) { s.destroy(&slot); } - return res; + return {s.non_iterable_iterator_at_slot(res.first), res.second}; } raw_hash_set& s; // Constructed slot. Either moved into place or destroyed. @@ -3345,29 +3335,34 @@ // TODO(b/289225379): consider having a helper class that has the impls for // SOO functionality. template <class K = key_type> - iterator find_small(const key_arg<K>& key) { + ABSL_ATTRIBUTE_ALWAYS_INLINE iterator find_small(const key_arg<K>& key) { ABSL_SWISSTABLE_ASSERT(is_small()); return empty() || !equal_to(key, single_slot()) ? end() : single_iterator(); } template <class K = key_type> - iterator find_large(const key_arg<K>& key, size_t hash) { + iterator find_large(const key_arg<K>& key) { ABSL_SWISSTABLE_ASSERT(!is_small()); - auto seq = probe(common(), hash); + const size_t cap = common().capacity(); + ABSL_ASSUME(cap > kMaxSmallCapacity); + const size_t hash = hash_of(key); + auto seq = probe(ProbeCapacity{cap}, hash); const h2_t h2 = H2(hash); - const ctrl_t* ctrl = control(); + ctrl_t* ctrl = control(); + slot_type* slot_array = to_slot(common().slot_array(cap)); while (true) { #ifndef ABSL_HAVE_MEMORY_SANITIZER - absl::PrefetchToLocalCache(slot_array() + seq.offset()); + absl::PrefetchToLocalCache(slot_array + seq.offset()); #endif Group g{ctrl + seq.offset()}; for (uint32_t i : g.Match(h2)) { - if (ABSL_PREDICT_TRUE(equal_to(key, slot_array() + seq.offset(i)))) - return iterator_at(seq.offset(i)); + const size_t offset = seq.offset(i); + if (ABSL_PREDICT_TRUE(equal_to(key, slot_array + offset))) + return iterator_at_ptr(ctrl + offset, slot_array + offset); } if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end(); seq.next(); - ABSL_SWISSTABLE_ASSERT(seq.index() <= capacity() && "full table!"); + ABSL_SWISSTABLE_ASSERT(seq.index() <= cap && "full table!"); } } @@ -3394,7 +3389,7 @@ } void clear_backing_array(bool reuse) { - ABSL_SWISSTABLE_ASSERT(capacity() > MaxSmallCapacity()); + ABSL_SWISSTABLE_ASSERT(capacity() > kMaxSmallCapacity); ClearBackingArray(common(), GetPolicyFunctions(), &char_alloc_ref(), reuse); } @@ -3446,7 +3441,11 @@ EraseMetaOnlySmall(common(), SooEnabled(), sizeof(slot_type)); } void erase_meta_only_large(const_iterator it) { - EraseMetaOnlyLarge(common(), it.control(), sizeof(slot_type)); + EraseMetaOnlyLarge(common(), + // `it` can be non-iterable iterator, so we can't use + // it.control(). + static_cast<size_t>(it.slot() - slot_array(capacity())), + sizeof(slot_type)); } template <class K> @@ -3578,92 +3577,92 @@ } template <class K> - std::pair<iterator, bool> find_or_prepare_insert_soo(const K& key) { + ABSL_ATTRIBUTE_ALWAYS_INLINE std::pair<slot_type*, bool> + find_or_prepare_insert_soo(const K& key) { ABSL_SWISSTABLE_ASSERT(is_soo()); bool force_sampling; + slot_type* slot = single_slot(); if (empty()) { if (!should_sample_soo()) { common().set_full_soo(); - return {single_iterator(), true}; + return {slot, true}; } force_sampling = true; - } else if (equal_to(key, single_slot())) { - return {single_iterator(), false}; + } else if (equal_to(key, slot)) { + return {slot, false}; } else { force_sampling = false; } ABSL_SWISSTABLE_ASSERT(capacity() == 1); constexpr bool kUseMemcpy = PolicyTraits::transfer_uses_memcpy() && SooEnabled(); - size_t index = GrowSooTableToNextCapacityAndPrepareInsert< - kUseMemcpy ? OptimalMemcpySizeForSooSlotTransfer(sizeof(slot_type)) : 0, - kUseMemcpy>(common(), GetPolicyFunctions(), - HashKey<hasher, K, kIsDefaultHash>{hash_ref(), key}, - force_sampling); - return {iterator_at(index), true}; + slot = to_slot( + GrowSooTableToNextCapacityAndPrepareInsert< + kUseMemcpy ? OptimalMemcpySizeForSooSlotTransfer(sizeof(slot_type)) + : 0, + kUseMemcpy>(common(), GetPolicyFunctions(), + HashKey<hasher, K, kIsDefaultHash>{hash_ref(), key}, + force_sampling)); + return {slot, true}; } template <class K> - std::pair<iterator, bool> find_or_prepare_insert_small(const K& key) { + ABSL_ATTRIBUTE_ALWAYS_INLINE std::pair<slot_type*, bool> + find_or_prepare_insert_small(const K& key) { ABSL_SWISSTABLE_ASSERT(is_small()); if constexpr (SooEnabled()) { return find_or_prepare_insert_soo(key); } if (!empty()) { if (equal_to(key, single_slot())) { - return {single_iterator(), false}; + return {single_slot(), false}; } } - return {iterator_at_ptr(PrepareInsertSmallNonSoo( + return {to_slot(PrepareInsertSmallNonSoo( common(), GetPolicyFunctions(), HashKey<hasher, K, kIsDefaultHash>{hash_ref(), key})), true}; } template <class K> - std::pair<iterator, bool> find_or_prepare_insert_large(const K& key) { + std::pair<slot_type*, bool> find_or_prepare_insert_large(const K& key) { ABSL_SWISSTABLE_ASSERT(!is_soo()); prefetch_heap_block(); + const size_t cap = capacity(); + ABSL_ASSUME(cap > kMaxSmallCapacity); const size_t hash = hash_of(key); - auto seq = probe(common(), hash); + auto seq = probe(ProbeCapacity{cap}, hash); const h2_t h2 = H2(hash); const ctrl_t* ctrl = control(); - size_t index; - bool inserted; - // We use a lambda function to be able to exit from the nested loop without - // duplicating generated code for the return statement (e.g. iterator_at). - [&]() ABSL_ATTRIBUTE_ALWAYS_INLINE { - while (true) { + slot_type* slot_array = to_slot(common().slot_array(cap)); + while (true) { #ifndef ABSL_HAVE_MEMORY_SANITIZER - absl::PrefetchToLocalCache(slot_array() + seq.offset()); + absl::PrefetchToLocalCache(slot_array + seq.offset()); #endif - Group g{ctrl + seq.offset()}; - for (uint32_t i : g.Match(h2)) { - if (ABSL_PREDICT_TRUE(equal_to(key, slot_array() + seq.offset(i)))) { - index = seq.offset(i); - inserted = false; - return; - } + Group g{ctrl + seq.offset()}; + for (uint32_t i : g.Match(h2)) { + slot_type* slot = slot_array + seq.offset(i); + if (ABSL_PREDICT_TRUE(equal_to(key, slot))) { + return {slot, false}; } - auto mask_empty = g.MaskEmpty(); - if (ABSL_PREDICT_TRUE(mask_empty)) { - size_t target_group_offset = seq.offset(); - index = SwisstableGenerationsEnabled() - ? PrepareInsertLargeGenerationsEnabled( - common(), GetPolicyFunctions(), hash, mask_empty, - FindInfo{target_group_offset, seq.index()}, - HashKey<hasher, K, kIsDefaultHash>{hash_ref(), key}) - : PrepareInsertLarge( - common(), GetPolicyFunctions(), hash, mask_empty, - FindInfo{target_group_offset, seq.index()}); - inserted = true; - return; - } - seq.next(); - ABSL_SWISSTABLE_ASSERT(seq.index() <= capacity() && "full table!"); } - }(); - return {iterator_at(index), inserted}; + auto mask_empty = g.MaskEmpty(); + if (ABSL_PREDICT_TRUE(mask_empty)) { + size_t target_group_offset = seq.offset(); + void* slot = + SwisstableGenerationsEnabled() + ? PrepareInsertLargeGenerationsEnabled( + common(), GetPolicyFunctions(), hash, mask_empty, + FindInfo{target_group_offset, seq.index()}, + HashKey<hasher, K, kIsDefaultHash>{hash_ref(), key}) + : PrepareInsertLarge( + common(), GetPolicyFunctions(), hash, mask_empty, + FindInfo{target_group_offset, seq.index()}); + return {to_slot(slot), true}; + } + seq.next(); + ABSL_SWISSTABLE_ASSERT(seq.index() <= capacity() && "full table!"); + } } template <class InputIt> @@ -3726,10 +3725,10 @@ // where the value can be inserted into, with the control byte already set to // `key`'s H2. Returns a bool indicating whether an insertion can take place. template <class K> - std::pair<iterator, bool> find_or_prepare_insert(const K& key) { + std::pair<slot_type*, bool> find_or_prepare_insert(const K& key) { AssertOnFind(key); - if (is_small()) return find_or_prepare_insert_small(key); - return find_or_prepare_insert_large(key); + return is_small() ? find_or_prepare_insert_small(key) + : find_or_prepare_insert_large(key); } // Constructs the value in the space pointed by the iterator. This only works @@ -3741,25 +3740,32 @@ // find_or_prepare_insert(k) was true. // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...). template <class... Args> - void emplace_at(iterator iter, Args&&... args) { - construct(iter.slot(), std::forward<Args>(args)...); + void emplace_at(slot_type* slot, Args&&... args) { + construct(slot, std::forward<Args>(args)...); // When is_small, find calls find_small and if size is 0, then it will // return an end iterator. This can happen in the raw_hash_set copy ctor. assert((is_small() || - PolicyTraits::apply(FindElement{*this}, *iter) == iter) && + PolicyTraits::apply(FindElement{*this}, PolicyTraits::element(slot)) + .slot() == slot) && "constructed value does not match the lookup key"); } + // Special iterator that can be returned by insert/emplace functions. + // It is non-iterable, meaning that std::next(it) always points to end(). + iterator non_iterable_iterator_at_slot(slot_type* slot) + ABSL_ATTRIBUTE_LIFETIME_BOUND { + return {InsertIteratorControl(), slot, common().generation_ptr()}; + } iterator iterator_at(size_t i) ABSL_ATTRIBUTE_LIFETIME_BOUND { return {control() + i, slot_array() + i, common().generation_ptr()}; } const_iterator iterator_at(size_t i) const ABSL_ATTRIBUTE_LIFETIME_BOUND { return const_cast<raw_hash_set*>(this)->iterator_at(i); } - iterator iterator_at_ptr(std::pair<ctrl_t*, void*> ptrs) + iterator iterator_at_ptr(ctrl_t* ctrl, void* slot) ABSL_ATTRIBUTE_LIFETIME_BOUND { - return {ptrs.first, to_slot(ptrs.second), common().generation_ptr()}; + return {ctrl, to_slot(slot), common().generation_ptr()}; } reference unchecked_deref(iterator it) { return it.unchecked_deref(); } @@ -3791,9 +3797,9 @@ ABSL_SWISSTABLE_ASSERT(!is_soo()); return common().control(); } - slot_type* slot_array() const { + slot_type* slot_array(size_t capacity) const { ABSL_SWISSTABLE_ASSERT(!is_soo()); - return static_cast<slot_type*>(common().slot_array()); + return static_cast<slot_type*>(common().slot_array(capacity)); } slot_type* soo_slot() { ABSL_SWISSTABLE_ASSERT(is_soo()); @@ -3806,7 +3812,9 @@ } slot_type* single_slot() { ABSL_SWISSTABLE_ASSERT(is_small()); - return SooEnabled() ? soo_slot() : slot_array(); + return SooEnabled() + ? soo_slot() + : to_slot(common().slot_array(/*capacity=*/1)); } const slot_type* single_slot() const { return const_cast<raw_hash_set*>(this)->single_slot(); @@ -3877,6 +3885,7 @@ void (*encode_probed_element)(void* probed_storage, h2_t h2, size_t source_offset, size_t h1)) { const size_t new_capacity = common.capacity(); + ABSL_ASSUME(new_capacity > kMaxSmallCapacity); const size_t old_capacity = PreviousCapacity(new_capacity); ABSL_ASSUME(old_capacity + 1 >= Group::kWidth); ABSL_ASSUME((old_capacity + 1) % Group::kWidth == 0); @@ -3884,7 +3893,7 @@ auto* set = reinterpret_cast<raw_hash_set*>(&common); slot_type* old_slots_ptr = to_slot(old_slots); ctrl_t* new_ctrl = common.control(); - slot_type* new_slots = set->slot_array(); + slot_type* new_slots = set->slot_array(new_capacity); for (size_t group_index = 0; group_index < old_capacity; group_index += Group::kWidth) { @@ -3997,7 +4006,9 @@ auto* slot = static_cast<SlotType*>(slot_void); if (pred(Set::PolicyTraits::element(slot))) { c->destroy(slot); - EraseMetaOnlyLarge(c->common(), ctrl, sizeof(*slot)); + EraseMetaOnlyLarge(c->common(), + static_cast<size_t>(ctrl - c->control()), + sizeof(*slot)); ++num_deleted; } }); @@ -4065,7 +4076,7 @@ while (true) { container_internal::Group g{ctrl + seq.offset()}; for (uint32_t i : g.Match(h2)) { - if (set.equal_to(key, set.slot_array() + seq.offset(i))) + if (set.equal_to(key, set.slot_array(set.capacity()) + seq.offset(i))) return num_probes; ++num_probes; } @@ -4097,20 +4108,17 @@ // Extern template instantiations reduce binary size and linker input size. // Function definition is in raw_hash_set.cc. -extern template size_t GrowSooTableToNextCapacityAndPrepareInsert<0, false>( +extern template void* GrowSooTableToNextCapacityAndPrepareInsert<0, false>( CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>, bool); -extern template size_t GrowSooTableToNextCapacityAndPrepareInsert<1, true>( +extern template void* GrowSooTableToNextCapacityAndPrepareInsert<1, true>( CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>, bool); -extern template size_t GrowSooTableToNextCapacityAndPrepareInsert<4, true>( - CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>, - bool); -extern template size_t GrowSooTableToNextCapacityAndPrepareInsert<8, true>( +extern template void* GrowSooTableToNextCapacityAndPrepareInsert<4, true>( CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>, bool); #if UINTPTR_MAX == UINT64_MAX -extern template size_t GrowSooTableToNextCapacityAndPrepareInsert<16, true>( +extern template void* GrowSooTableToNextCapacityAndPrepareInsert<8, true>( CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>, bool); #endif
diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 70a5952..e97648d 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc
@@ -82,8 +82,8 @@ return std::forward<C>(c).common(); } template <typename C> - static auto GetSlots(const C& c) -> decltype(c.slot_array()) { - return c.slot_array(); + static auto GetSlots(const C& c) -> decltype(c.slot_array(c.capacity())) { + return c.slot_array(c.capacity()); } template <typename C> static size_t CountTombstones(const C& c) { @@ -102,6 +102,23 @@ using ::testing::UnorderedElementsAre; using ::testing::UnorderedElementsAreArray; +// Enables sampling with 1 percent sampling rate and +// resets the rate counter for the current thread. +void SetSamplingRateTo1Percent() { + SetHashtablezEnabled(true); + SetHashtablezSampleParameter(100); // Sample ~1% of tables. + // Reset rate counter for the current thread. + TestOnlyRefreshSamplingStateForCurrentThread(); +} + +// Disables sampling and resets the rate counter for the current thread. +void DisableSampling() { + SetHashtablezEnabled(false); + SetHashtablezSampleParameter(1 << 16); + // Reset rate counter for the current thread. + TestOnlyRefreshSamplingStateForCurrentThread(); +} + // Convenience function to static cast to ctrl_t. ctrl_t CtrlT(int i) { return static_cast<ctrl_t>(i); } @@ -111,7 +128,7 @@ constexpr size_t kSlotSize = 1; RawHashSetLayout layout(1, kSlotSize, /*slot_align=*/1, /*has_infoz=*/false, /*blocked_element_count=*/0); - EXPECT_EQ(layout.control_offset(), 0); + EXPECT_EQ(layout.control_offset(), NumGenerationBytes()); EXPECT_EQ(layout.slot_offset(), NumGenerationBytes()); EXPECT_EQ(layout.alloc_size(), NumGenerationBytes() + kSlotSize); } @@ -121,7 +138,8 @@ constexpr size_t kAlignment = 4; RawHashSetLayout layout(1, kSlotSize, kAlignment, /*has_infoz=*/false, /*blocked_element_count=*/0); - EXPECT_EQ(layout.control_offset(), 0); + EXPECT_EQ(layout.control_offset(), + NumGenerationBytes() == 0 ? 0 : kAlignment); EXPECT_EQ(layout.slot_offset(), NumGenerationBytes() == 0 ? 0 : kAlignment); EXPECT_EQ(layout.alloc_size(), layout.slot_offset() + kSlotSize); } @@ -140,12 +158,12 @@ ASSERT_LE(capacity, GrowthInfoLowerBound::kMaxGrowthLeftLowerBound); RawHashSetLayout layout(capacity, slot_size, slot_align, has_infoz, blocked_element_count); - EXPECT_EQ(layout.control_offset(), 1); // 1 byte for growth_info + EXPECT_EQ(layout.control_offset(), + /*growth*/ 1 + padding + NumGenerationBytes()); size_t expected_slot_offset = - capacity + NumClonedBytes() + 1 + /*growth*/ 1 + NumGenerationBytes(); + layout.control_offset() + NumControlBytes(capacity); EXPECT_LT(padding, slot_align); - EXPECT_EQ((expected_slot_offset + padding) % slot_align, 0); - expected_slot_offset += padding; + EXPECT_EQ(expected_slot_offset % slot_align, 0); EXPECT_EQ(layout.slot_offset(), expected_slot_offset); size_t allocated_values = capacity - blocked_element_count; EXPECT_EQ(layout.alloc_size(), @@ -175,9 +193,8 @@ /*has_infoz=*/true, /*blocked_element_count=*/0); EXPECT_EQ(layout.control_offset(), // growth_info is always 8 bytes for sampled tables. - 8 + sizeof(HashtablezInfoHandle)); - EXPECT_EQ(layout.slot_offset(), - layout.control_offset() + NumGenerationBytes()); + 8 + sizeof(HashtablezInfoHandle) + NumGenerationBytes()); + EXPECT_EQ(layout.slot_offset(), layout.control_offset()); EXPECT_EQ(layout.alloc_size(), layout.slot_offset() + 1); } { @@ -187,12 +204,14 @@ RawHashSetLayout layout(kCapacity, /*slot_size=*/kSlotSize, /*slot_align=*/kAlignment, /*has_infoz=*/true, /*blocked_element_count=*/0); + size_t padding = NumGenerationBytes() == 0 ? 1 : 0; + padding += sizeof(HashtablezInfoHandle) == 4 ? 0 : 4; EXPECT_EQ(layout.control_offset(), // growth_info is always 8 bytes for sampled tables. - 8 + sizeof(HashtablezInfoHandle)); + /*growth*/ 8 + sizeof(HashtablezInfoHandle) + padding + + NumGenerationBytes()); size_t expected_slot_offset = - layout.control_offset() + kCapacity + NumClonedBytes() + 1 + - /*padding+generation*/ (sizeof(HashtablezInfoHandle) == 4 ? 1 : 5); + layout.control_offset() + NumControlBytes(kCapacity); EXPECT_EQ(expected_slot_offset % kAlignment, 0); EXPECT_EQ(layout.slot_offset(), expected_slot_offset); EXPECT_EQ(layout.alloc_size(), @@ -211,10 +230,12 @@ ASSERT_GT(capacity, GrowthInfoLowerBound::kMaxGrowthLeftLowerBound); RawHashSetLayout layout(capacity, slot_size, slot_align, has_infoz, blocked_element_count); + size_t padding = NumGenerationBytes() == 0 ? 1 : 0; EXPECT_EQ(layout.control_offset(), - has_infoz ? 8 + sizeof(HashtablezInfoHandle) : 8); - size_t expected_slot_offset = layout.control_offset() + capacity + - NumClonedBytes() + 1 + /*padding+generation*/ 1; + padding + (has_infoz ? 8 + sizeof(HashtablezInfoHandle) : 8) + + NumGenerationBytes()); + size_t expected_slot_offset = + layout.control_offset() + NumControlBytes(capacity); EXPECT_EQ(expected_slot_offset % slot_align, 0); EXPECT_EQ(layout.slot_offset(), expected_slot_offset); EXPECT_EQ( @@ -237,17 +258,24 @@ if (capacity <= GrowthInfoLowerBound::kMaxGrowthLeftLowerBound) { SanitizerPoisonMemoryRegion(control_.data(), 7); } - SanitizerPoisonMemoryRegion(control_.data() + 8, 1); + SanitizerPoisonMemoryRegion(control_.data() + kControlStart, 1); + if constexpr (NumGenerationBytes() > 0) { + SanitizerPoisonMemoryRegion( + control_.data() + kControlStart + NumGenerationBytes(), + NumGenerationBytes()); + } } GrowthInfoAccessor* operator->() { return &growth_info_; } private: + static constexpr size_t kControlStart = 8 + NumGenerationBytes(); // We allocate on heap since ASAN fails to detect access to poisoned memory // on stack. - std::vector<ctrl_t> control_ = - std::vector<ctrl_t>(9, /*garbage*/ ctrl_t::kSentinel); - GrowthInfoAccessor growth_info_ = GrowthInfoAccessor(control_.data() + 8); + std::vector<ctrl_t> control_ = std::vector<ctrl_t>( + 9 + NumGenerationBytes(), /*garbage*/ ctrl_t::kSentinel); + GrowthInfoAccessor growth_info_ = + GrowthInfoAccessor(control_.data() + kControlStart); }; TEST(GrowthInfoViewTest, GetGrowthLeft) { @@ -525,33 +553,11 @@ EXPECT_EQ(1, OptimalMemcpySizeForSooSlotTransfer(1)); ASSERT_EQ(4, OptimalMemcpySizeForSooSlotTransfer(2)); ASSERT_EQ(4, OptimalMemcpySizeForSooSlotTransfer(3)); - for (size_t slot_size = 4; slot_size <= 8; ++slot_size) { - ASSERT_EQ(8, OptimalMemcpySizeForSooSlotTransfer(slot_size)); - } - // If maximum amount of memory is 16, then we can copy up to 16 bytes. - for (size_t slot_size = 9; slot_size <= 16; ++slot_size) { - ASSERT_EQ(16, - OptimalMemcpySizeForSooSlotTransfer(slot_size, - /*max_soo_slot_size=*/16)); - ASSERT_EQ(16, - OptimalMemcpySizeForSooSlotTransfer(slot_size, - /*max_soo_slot_size=*/24)); - } - // But we shouldn't try to copy more than maximum amount of memory. - for (size_t slot_size = 9; slot_size <= 12; ++slot_size) { - ASSERT_EQ(12, OptimalMemcpySizeForSooSlotTransfer( - slot_size, /*max_soo_slot_size=*/12)); - } - for (size_t slot_size = 17; slot_size <= 24; ++slot_size) { - ASSERT_EQ(24, - OptimalMemcpySizeForSooSlotTransfer(slot_size, - /*max_soo_slot_size=*/24)); - } - // We shouldn't copy more than maximum. - for (size_t slot_size = 17; slot_size <= 20; ++slot_size) { - ASSERT_EQ(20, - OptimalMemcpySizeForSooSlotTransfer(slot_size, - /*max_soo_slot_size=*/20)); + ASSERT_EQ(4, OptimalMemcpySizeForSooSlotTransfer(4, /*max_soo_slot_size=*/4)); + if constexpr (MaxSooSlotSize() > 4) { + for (size_t slot_size = 4; slot_size <= 8; ++slot_size) { + ASSERT_EQ(8, OptimalMemcpySizeForSooSlotTransfer(slot_size)); + } } } @@ -629,8 +635,8 @@ } TEST(Util, probe_seq) { - HashtableCapacity capacity(127); - probe_seq<16> seq(capacity, /*hash=*/0); + size_t capacity = 127; + probe_seq<16> seq(ProbeCapacity{capacity}, /*hash=*/0); auto gen = [&]() { size_t res = seq.offset(); seq.next(); @@ -639,7 +645,7 @@ std::vector<size_t> offsets(8); std::generate_n(offsets.begin(), 8, gen); EXPECT_THAT(offsets, ElementsAre(0, 16, 48, 96, 32, 112, 80, 64)); - seq = probe_seq<16>(capacity, /*hash=*/128); + seq = probe_seq<16>(ProbeCapacity{capacity}, /*hash=*/128); std::generate_n(offsets.begin(), 8, gen); EXPECT_THAT(offsets, ElementsAre(0, 16, 48, 96, 32, 112, 80, 64)); } @@ -762,6 +768,42 @@ EXPECT_TRUE(data.has_infoz()); } +TYPED_TEST(HashtableDataTest, BlockedElementCount) { + constexpr HashtableCapacityStorageMode kMode = TypeParam::value; + using InlineData = HashtableInlineDataImpl<kMode>; + using Capacity = HashtableCapacityImpl<kMode>; + + { + InlineData data(Capacity(0), no_seed_empty_tag_t{}); + EXPECT_EQ(data.blocked_element_count(), 0); + } + + for (size_t i = 0; i <= InlineData::kMaxBlockedElementCount; ++i) { + InlineData data(Capacity(0), no_seed_empty_tag_t{}); + data.init_blocked_element_count(i); + EXPECT_EQ(data.blocked_element_count(), i); + data.set_blocked_element_count_to_zero(); + EXPECT_EQ(data.blocked_element_count(), 0); + } +} + +TYPED_TEST(HashtableDataTest, MaxStorableSize) { + constexpr HashtableCapacityStorageMode kMode = TypeParam::value; + using InlineData = HashtableInlineDataImpl<kMode>; + using Capacity = HashtableCapacityImpl<kMode>; + + InlineData data(Capacity(0), no_seed_empty_tag_t{}); + constexpr uint64_t kMaxSize = + sizeof(size_t) == 4 ? ~uint32_t{} + : (uint64_t{1} << InlineData::kSizeBitCount) - 1; + data.init_blocked_element_count(3); + data.increment_size(kMaxSize); + EXPECT_EQ(data.size(), kMaxSize); + // We didn't overwrite other fields. + EXPECT_FALSE(data.has_infoz()); + EXPECT_EQ(data.blocked_element_count(), 3); +} + TYPED_TEST(HashtableDataTest, HashtableInlineDataMetadata) { constexpr HashtableCapacityStorageMode kMode = TypeParam::value; using InlineData = HashtableInlineDataImpl<kMode>; @@ -937,6 +979,8 @@ bool operator==(const SizedValue& rhs) const { return **this == *rhs; } private: + static_assert(N % sizeof(int64_t) == 0); + static_assert(N >= sizeof(int64_t)); int64_t vals_[N / sizeof(int64_t)]; }; template <int N, bool kSoo> @@ -1204,9 +1248,9 @@ using Base::Base; }; -constexpr size_t kNonSooSize = sizeof(HeapOrSoo) + 8; +constexpr size_t kNonSooSize = 2 * sizeof(HeapOrSoo); using NonSooIntTableSlotType = SizedValue<kNonSooSize>; -static_assert(sizeof(NonSooIntTableSlotType) >= kNonSooSize, "too small"); +static_assert(sizeof(NonSooIntTableSlotType) > MaxSooSlotSize(), "too small"); using NonSooIntTable = ValueTable<NonSooIntTableSlotType>; using NonSooIntTableTrivialDestroy = ValueTable<NonSooIntTableSlotType, /*kTransferable=*/false, /*kSoo=*/false, @@ -1236,12 +1280,10 @@ size_t capacity; uint64_t size; void* ctrl; - void* slots; }; struct MockTableByLog { uint64_t size; void* ctrl; - void* slots; }; using MockTable = std::conditional_t<HashtableInlineData::kStorageMode == kCapacityByValue, @@ -1313,22 +1355,6 @@ raw_hash_set<P>>)); } -template <class TableType> -class SooTest : public testing::Test {}; - -using SooTableTypes = - ::testing::Types<SooIntTable, SooIntTableTrivialDestroy, NonSooIntTable, - NonSooIntTableTrivialDestroy, NonMemcpyableSooIntTable, - MemcpyableSooIntCustomAllocTable, - NonMemcpyableSooIntCustomAllocTable>; -TYPED_TEST_SUITE(SooTest, SooTableTypes); - -TYPED_TEST(SooTest, Empty) { - TypeParam t; - EXPECT_EQ(0, t.size()); - EXPECT_TRUE(t.empty()); -} - TEST(Table, Prefetch) { IntTable t; t.emplace(1); @@ -1344,38 +1370,6 @@ } } -TYPED_TEST(SooTest, LookupEmpty) { - TypeParam t; - auto it = t.find(0); - EXPECT_TRUE(it == t.end()); -} - -TYPED_TEST(SooTest, Insert1) { - TypeParam t; - EXPECT_TRUE(t.find(0) == t.end()); - auto res = t.emplace(0); - EXPECT_TRUE(res.second); - EXPECT_THAT(*res.first, 0); - EXPECT_EQ(1, t.size()); - EXPECT_THAT(*t.find(0), 0); -} - -TYPED_TEST(SooTest, Insert2) { - TypeParam t; - EXPECT_TRUE(t.find(0) == t.end()); - auto res = t.emplace(0); - EXPECT_TRUE(res.second); - EXPECT_THAT(*res.first, 0); - EXPECT_EQ(1, t.size()); - EXPECT_TRUE(t.find(1) == t.end()); - res = t.emplace(1); - EXPECT_TRUE(res.second); - EXPECT_THAT(*res.first, 1); - EXPECT_EQ(2, t.size()); - EXPECT_THAT(*t.find(0), 0); - EXPECT_THAT(*t.find(1), 1); -} - TEST(Table, InsertCollision) { BadTable t; EXPECT_TRUE(t.find(1) == t.end()); @@ -1533,6 +1527,120 @@ } } +// This test verifies that we don't rehash in place when we insert an element +// above the growth left threshold. Otherwise we may end up with zero empty +// slots. That would cause hard to debug infinite loop in `find`. +// This test do the following: +// 1. Reserve a table with `kReserveSize` elements. +// 2. Insert `Group::kWidth` elements to fill the first group (due to bad hash +// function all elements are inserted into the same group). +// 3. Erase one element to create tombstone. +// 4. Insert the same element back. But GrowthInfo still assumes that we may +// have a tombstone in the table. +// 5. Insert one more element, which should cause a rehash and growth. +TEST(Table, + ReservedTableResizeNotRehashInplaceIfInsertingElementAboveGrowthLeft) { + if (SwisstableGenerationsEnabled()) { + GTEST_SKIP() << "Generations enabled, so rehash happens earlier.\n" + << "Note that reservation doesn't prevent rehashing since we " + "are erasing one element."; + } + constexpr int64_t kCoef = 17; + constexpr size_t kCapacity = 31; + constexpr size_t kReserveSize = + CapacityToGrowth(kCapacity) - kMaxBlockedElementsForLargeTables; + + BadTwoValuesHashTable t(0, + // Negative number goes to the end of the table. + BadTwoValuesHash(kReserveSize + 2)); + // Remove seed to make table layout deterministic. + RawHashSetTestOnlyAccess::GetCommon(t).set_no_seed_for_testing(); + + t.reserve(kReserveSize); + for (int64_t i = 0; i < static_cast<int64_t>(Group::kWidth); ++i) { + ASSERT_TRUE(t.insert(i * kCoef).second); + } + EXPECT_EQ(t.erase(kCoef), 1); + EXPECT_EQ(RawHashSetTestOnlyAccess::CountTombstones(t), 1); + EXPECT_TRUE(t.insert(kCoef).second); + EXPECT_EQ(RawHashSetTestOnlyAccess::CountTombstones(t), 0); + // We want to test codepath deciding whether to rehash in place or not. + // For this we need to potentially have tombstone. + EXPECT_FALSE(RawHashSetTestOnlyAccess::GetCommon(t) + .growth_info() + .GetGrowthInfoLowerBound() + .HasNoDeleted()); + for (int64_t i = static_cast<int64_t>(Group::kWidth); + i < static_cast<int64_t>(kReserveSize); ++i) { + ASSERT_TRUE(t.insert(i * kCoef).second); + } + EXPECT_EQ(t.size(), kReserveSize); + EXPECT_EQ(t.capacity(), kCapacity); + EXPECT_TRUE(t.insert(-57).second); + EXPECT_EQ(t.size(), kReserveSize + 1); + EXPECT_EQ(RawHashSetTestOnlyAccess::CountTombstones(t), 0); + EXPECT_EQ(t.capacity(), NextCapacity(kCapacity)); + for (int64_t i = 0; i < static_cast<int64_t>(kReserveSize); ++i) { + ASSERT_TRUE(t.contains(i * kCoef)); + } + EXPECT_TRUE(t.contains(-57)); +} + +template <class TableType> +class ExtendedSooTest : public testing::Test {}; + +using ExtendedSooTableTypes = + ::testing::Types<SooIntTable, SooIntTableTrivialDestroy, NonSooIntTable, + NonSooIntTableTrivialDestroy, NonMemcpyableSooIntTable, + MemcpyableSooIntCustomAllocTable, + NonMemcpyableSooIntCustomAllocTable>; +TYPED_TEST_SUITE(ExtendedSooTest, ExtendedSooTableTypes); + +template <class TableType> +class SooTest : public testing::Test {}; + +using SooTableTypes = + ::testing::Types<SooIntTable, NonSooIntTable>; +TYPED_TEST_SUITE(SooTest, SooTableTypes); + +TYPED_TEST(SooTest, Empty) { + TypeParam t; + EXPECT_EQ(0, t.size()); + EXPECT_TRUE(t.empty()); +} + +TYPED_TEST(SooTest, LookupEmpty) { + TypeParam t; + auto it = t.find(0); + EXPECT_TRUE(it == t.end()); +} + +TYPED_TEST(SooTest, Insert1) { + TypeParam t; + EXPECT_TRUE(t.find(0) == t.end()); + auto res = t.emplace(0); + EXPECT_TRUE(res.second); + EXPECT_THAT(*res.first, 0); + EXPECT_EQ(1, t.size()); + EXPECT_THAT(*t.find(0), 0); +} + +TYPED_TEST(SooTest, Insert2) { + TypeParam t; + EXPECT_TRUE(t.find(0) == t.end()); + auto res = t.emplace(0); + EXPECT_TRUE(res.second); + EXPECT_THAT(*res.first, 0); + EXPECT_EQ(1, t.size()); + EXPECT_TRUE(t.find(1) == t.end()); + res = t.emplace(1); + EXPECT_TRUE(res.second); + EXPECT_THAT(*res.first, 1); + EXPECT_EQ(2, t.size()); + EXPECT_THAT(*t.find(0), 0); + EXPECT_THAT(*t.find(1), 1); +} + TYPED_TEST(SooTest, EraseInSmallTables) { for (int64_t size = 0; size < 64; ++size) { TypeParam t; @@ -1550,38 +1658,60 @@ } } -TYPED_TEST(SooTest, InsertWithinCapacity) { - TypeParam t; - t.reserve(10); - const size_t original_capacity = t.capacity(); - const auto addr = [&](int i) { - return reinterpret_cast<uintptr_t>(&*t.find(i)); +TYPED_TEST(ExtendedSooTest, InsertWithinCapacity) { + using TableType = TypeParam; + using ReserveFn = std::function<size_t(size_t, TableType&)>; + ReserveFn reserve_and_insert = [](size_t size, TableType& t) { + t.reserve(size); + size_t cap = t.capacity(); + return cap; }; - // Inserting an element does not change capacity. - t.insert(0); - EXPECT_THAT(t.capacity(), original_capacity); - const uintptr_t original_addr_0 = addr(0); - // Inserting another element does not rehash. - t.insert(1); - EXPECT_THAT(t.capacity(), original_capacity); - EXPECT_THAT(addr(0), original_addr_0); - // Inserting lots of duplicate elements does not rehash. - for (int i = 0; i < 100; ++i) { - t.insert(i % 10); + ReserveFn construct_and_insert = [](size_t size, TableType& t) { + t = TableType(size); + size_t cap = t.capacity(); + return cap; + }; + ReserveFn construct_inisitializer_list_with_reservation = + [](size_t size, TableType& t) { + t = TableType({}, size); + return t.capacity(); + }; + ReserveFn construct_range_iter_with_reservation = [](size_t size, + TableType& t) { + std::vector<int> v; + t = TableType(v.begin(), v.end(), size); + return t.capacity(); + }; + + std::vector<ReserveFn> reserve_and_insert_fns = { + reserve_and_insert, construct_and_insert, + construct_inisitializer_list_with_reservation, + construct_range_iter_with_reservation}; + for (size_t fn_i = 0; fn_i < reserve_and_insert_fns.size(); ++fn_i) { + ReserveFn reserve_and_insert_fn = reserve_and_insert_fns[fn_i]; + for (int size : {3, 5, 7, 11, 27, 36}) { + SCOPED_TRACE(absl::StrCat("fn_i: ", fn_i, ", size: ", size)); + TableType t; + const size_t original_capacity = + reserve_and_insert_fn(static_cast<size_t>(size), t); + const auto addr = [&](int i) { + return reinterpret_cast<uintptr_t>(&*t.find(i)); + }; + // Inserting an element does not change capacity. + t.insert(0); + ASSERT_THAT(t.capacity(), original_capacity); + const uintptr_t original_addr_0 = addr(0); + // Inserting lots of duplicate elements does not rehash. + for (int i = 0; i < size * 5; ++i) { + t.insert(i % size); + } + ASSERT_THAT(t.capacity(), original_capacity); + ASSERT_THAT(addr(0), original_addr_0); + } } - EXPECT_THAT(t.capacity(), original_capacity); - EXPECT_THAT(addr(0), original_addr_0); - // Inserting a range of duplicate elements does not rehash. - std::vector<int> dup_range; - for (int i = 0; i < 100; ++i) { - dup_range.push_back(i % 10); - } - t.insert(dup_range.begin(), dup_range.end()); - EXPECT_THAT(t.capacity(), original_capacity); - EXPECT_THAT(addr(0), original_addr_0); } -TYPED_TEST(SooTest, ClearDifferentSizes) { +TYPED_TEST(ExtendedSooTest, ClearDifferentSizes) { for (size_t size = 0; size < 32; ++size) { for (bool reserve : {false, true}) { for (bool clear_via_erase : {false, true}) { @@ -1608,7 +1738,7 @@ } } -TYPED_TEST(SooTest, ReserveTwice) { +TYPED_TEST(ExtendedSooTest, ReserveTwice) { for (int reserve_size = 0; reserve_size < 32; ++reserve_size) { for (int reserve_size2 = reserve_size; reserve_size2 < 32; ++reserve_size2) { @@ -1640,7 +1770,7 @@ } } -TYPED_TEST(SooTest, GrowAfterReserve) { +TYPED_TEST(ExtendedSooTest, GrowAfterReserve) { for (int reserve_size = 1; reserve_size <= 150; ++reserve_size) { TypeParam s; s.reserve(static_cast<size_t>(reserve_size)); @@ -1658,7 +1788,7 @@ } } -TYPED_TEST(SooTest, ClearAfterReserve) { +TYPED_TEST(ExtendedSooTest, ClearAfterReserve) { for (size_t reserve_size : std::vector<size_t>{1, 3, 4, 6, 7, 8, 13, 14, 15, 128, 150}) { TypeParam s; @@ -1681,11 +1811,777 @@ } } +TYPED_TEST(SooTest, ContainsEmpty) { + TypeParam t; + + EXPECT_FALSE(t.contains(0)); +} + +TYPED_TEST(SooTest, Contains1) { + TypeParam t; + + EXPECT_TRUE(t.insert(0).second); + EXPECT_TRUE(t.contains(0)); + EXPECT_FALSE(t.contains(1)); + + EXPECT_EQ(1, t.erase(0)); + EXPECT_FALSE(t.contains(0)); +} + +TYPED_TEST(SooTest, Contains2) { + TypeParam t; + + EXPECT_TRUE(t.insert(0).second); + EXPECT_TRUE(t.contains(0)); + EXPECT_FALSE(t.contains(1)); + + t.clear(); + EXPECT_FALSE(t.contains(0)); + + EXPECT_TRUE(t.insert(0).second); + EXPECT_TRUE(t.contains(0)); +} + +// Returns the largest m such that a table with m elements has the same number +// of buckets as a table with n elements. +size_t MaxDensitySize(size_t n) { + IntTable t; + t.reserve(n); + for (size_t i = 0; i != n; ++i) t.emplace(i); + const size_t c = t.bucket_count(); + while (c == t.bucket_count()) t.emplace(n++); + return t.size() - 1; +} + +TYPED_TEST(ExtendedSooTest, InsertEraseStressTest) { + TypeParam t; + const size_t kMinElementCount = 50; + std::deque<int> keys; + size_t i = 0; + for (; i < MaxDensitySize(kMinElementCount); ++i) { + t.emplace(static_cast<int64_t>(i)); + keys.push_back(i); + } + const size_t kNumIterations = 20000; + for (; i < kNumIterations; ++i) { + ASSERT_EQ(1, t.erase(keys.front())); + keys.pop_front(); + t.emplace(static_cast<int64_t>(i)); + keys.push_back(i); + } +} + +TEST(Table, InsertOverloads) { + StringTable t; + // These should all trigger the insert(init_type) overload. + t.insert({{}, {}}); + t.insert({"ABC", {}}); + t.insert({"DEF", "!!!"}); + + EXPECT_THAT(t, UnorderedElementsAre(Pair("", ""), Pair("ABC", ""), + Pair("DEF", "!!!"))); +} + +TYPED_TEST(SooTest, LargeTable) { + TypeParam t; + for (int64_t i = 0; i != 10000; ++i) { + t.emplace(i << 40); + ASSERT_EQ(t.size(), i + 1); + } + for (int64_t i = 0; i != 10000; ++i) + ASSERT_EQ(i << 40, static_cast<int64_t>(*t.find(i << 40))); +} + +// Timeout if copy is quadratic as it was in Rust. See b/34756399. +TYPED_TEST(SooTest, EnsureNonQuadraticAsInRust) { + static const size_t kLargeSize = 1 << 15; + + TypeParam t; + for (size_t i = 0; i != kLargeSize; ++i) { + t.insert(i); + } + + // If this is quadratic, the test will timeout. + TypeParam t2; + for (const auto& entry : t) t2.insert(entry); +} + +TYPED_TEST(SooTest, ClearBug) { + if (SwisstableGenerationsEnabled()) { + GTEST_SKIP() << "Generations being enabled causes extra rehashes."; + } + + TypeParam t; + constexpr size_t capacity = container_internal::Group::kWidth - 1; + constexpr size_t max_size = capacity / 2 + 1; + for (size_t i = 0; i < max_size; ++i) { + t.insert(i); + } + ASSERT_EQ(capacity, t.capacity()); + intptr_t original = reinterpret_cast<intptr_t>(&*t.find(2)); + t.clear(); + ASSERT_EQ(capacity, t.capacity()); + for (size_t i = 0; i < max_size; ++i) { + t.insert(i); + } + ASSERT_EQ(capacity, t.capacity()); + intptr_t second = reinterpret_cast<intptr_t>(&*t.find(2)); + // We are checking that original and second are close enough to each other + // that they are probably still in the same group. This is not strictly + // guaranteed. + EXPECT_LT(static_cast<size_t>(std::abs(original - second)), + capacity * sizeof(typename TypeParam::value_type)); +} + +TYPED_TEST(SooTest, Erase) { + TypeParam t; + EXPECT_TRUE(t.find(0) == t.end()); + auto res = t.emplace(0); + EXPECT_TRUE(res.second); + EXPECT_EQ(1, t.size()); + t.erase(res.first); + EXPECT_EQ(0, t.size()); + EXPECT_TRUE(t.find(0) == t.end()); +} + +TYPED_TEST(SooTest, EraseMaintainsValidIterator) { + TypeParam t; + const int kNumElements = 100; + for (int i = 0; i < kNumElements; i++) { + EXPECT_TRUE(t.emplace(i).second); + } + EXPECT_EQ(t.size(), kNumElements); + + int num_erase_calls = 0; + auto it = t.begin(); + while (it != t.end()) { + t.erase(it++); + num_erase_calls++; + } + + EXPECT_TRUE(t.empty()); + EXPECT_EQ(num_erase_calls, kNumElements); +} + +TYPED_TEST(SooTest, EraseBeginEnd) { + TypeParam t; + for (int i = 0; i < 10; ++i) t.insert(i); + EXPECT_EQ(t.size(), 10); + t.erase(t.begin(), t.end()); + EXPECT_EQ(t.size(), 0); +} + +TYPED_TEST(SooTest, Clear) { + TypeParam t; + EXPECT_TRUE(t.find(0) == t.end()); + t.clear(); + EXPECT_TRUE(t.find(0) == t.end()); + auto res = t.emplace(0); + EXPECT_TRUE(res.second); + EXPECT_EQ(1, t.size()); + t.clear(); + EXPECT_EQ(0, t.size()); + EXPECT_TRUE(t.find(0) == t.end()); +} + +TYPED_TEST(SooTest, Swap) { + TypeParam t; + EXPECT_TRUE(t.find(0) == t.end()); + auto res = t.emplace(0); + EXPECT_TRUE(res.second); + EXPECT_EQ(1, t.size()); + TypeParam u; + t.swap(u); + EXPECT_EQ(0, t.size()); + EXPECT_EQ(1, u.size()); + EXPECT_TRUE(t.find(0) == t.end()); + EXPECT_THAT(*u.find(0), 0); +} + +TYPED_TEST(SooTest, Rehash) { + TypeParam t; + EXPECT_TRUE(t.find(0) == t.end()); + t.emplace(0); + t.emplace(1); + EXPECT_EQ(2, t.size()); + t.rehash(128); + EXPECT_EQ(2, t.size()); + EXPECT_THAT(*t.find(0), 0); + EXPECT_THAT(*t.find(1), 1); +} + +TYPED_TEST(SooTest, RehashDoesNotRehashWhenNotNecessary) { + TypeParam t; + t.emplace(0); + t.emplace(1); + auto* p = &*t.find(0); + t.rehash(1); + EXPECT_EQ(p, &*t.find(0)); +} + +TYPED_TEST(SooTest, RehashZeroForcesRehash) { + TypeParam t; + t.emplace(0); + t.emplace(1); + auto* p = &*t.find(0); + t.rehash(0); + EXPECT_NE(p, &*t.find(0)); +} + +TYPED_TEST(SooTest, CopyConstruct) { + TypeParam t; + t.emplace(0); + EXPECT_EQ(1, t.size()); + { + TypeParam u(t); + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find(0), 0); + } + { + TypeParam u{t}; + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find(0), 0); + } + { + TypeParam u = t; + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find(0), 0); + } +} + +TYPED_TEST(SooTest, CopyAssignment) { + std::vector<size_t> sizes = {0, 1, 7, 25}; + for (size_t source_size : sizes) { + for (size_t target_size : sizes) { + SCOPED_TRACE(absl::StrCat("source_size: ", source_size, + " target_size: ", target_size)); + TypeParam source; + std::vector<int> source_elements; + for (size_t i = 0; i < source_size; ++i) { + source.emplace(static_cast<int>(i) * 2); + source_elements.push_back(static_cast<int>(i) * 2); + } + TypeParam target; + for (size_t i = 0; i < target_size; ++i) { + target.emplace(static_cast<int>(i) * 3); + } + target = source; + ASSERT_EQ(target.size(), source_size); + ASSERT_THAT(target, UnorderedElementsAreArray(source_elements)); + } + } +} + +TYPED_TEST(SooTest, CopyConstructWithSampling) { + SetSamplingRateTo1Percent(); + for (int i = 0; i < 10000; ++i) { + TypeParam t; + t.emplace(0); + EXPECT_EQ(1, t.size()); + { + TypeParam u(t); + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find(0), 0); + } + } +} + +TYPED_TEST(SooTest, CopyDifferentSizes) { + TypeParam t; + + for (int i = 0; i < 100; ++i) { + t.emplace(i); + TypeParam c = t; + for (int j = 0; j <= i; ++j) { + ASSERT_TRUE(c.find(j) != c.end()) << "i=" << i << " j=" << j; + } + // Testing find miss to verify that table is not full. + ASSERT_TRUE(c.find(-1) == c.end()); + } +} + +TYPED_TEST(ExtendedSooTest, CopyDifferentSizesWithReserve) { + for (size_t size = 0; size < 153; ++size) { + SCOPED_TRACE(absl::StrCat("size: ", size)); + TypeParam t; + t.reserve(size); + for (size_t i = 0; i < size; ++i) { + ASSERT_TRUE(t.insert(static_cast<int>(i)).second) << i; + } + auto t2 = t; + ASSERT_EQ(t2.size(), size); + for (size_t i = 0; i < size; ++i) { + ASSERT_TRUE(t2.contains(static_cast<int>(i))) << i; + } + ASSERT_TRUE(t2.insert(static_cast<int>(size)).second); + ASSERT_EQ(t2.size(), size + 1); + ASSERT_TRUE(t2.contains(static_cast<int>(size))); + } +} + +TYPED_TEST(SooTest, CopyDifferentCapacities) { + for (int cap = 1; cap < 100; cap = cap * 2 + 1) { + TypeParam t; + t.reserve(static_cast<size_t>(cap)); + for (int i = 0; i <= cap; ++i) { + t.emplace(i); + if (i != cap && i % 5 != 0) { + continue; + } + TypeParam c = t; + for (int j = 0; j <= i; ++j) { + ASSERT_TRUE(c.find(j) != c.end()) + << "cap=" << cap << " i=" << i << " j=" << j; + } + // Testing find miss to verify that table is not full. + ASSERT_TRUE(c.find(-1) == c.end()); + } + } +} + +// Invalid iterator use can trigger crashes or invalidated iterator assertions. +testing::Matcher<const std::string&> InvalidIteratorMatcher() { + return AnyOf(HasSubstr("invalidated iterator"), HasSubstr("Invalid iterator"), + HasSubstr("invalid iterator"), + HasSubstr("CrashIfIteratorIsInvalid")); +} + +TYPED_TEST(SooTest, NumDeletedRegression) { + TypeParam t; + t.emplace(0); + t.erase(t.find(0)); + // construct over a deleted slot. + t.emplace(0); + t.clear(); +} + +TYPED_TEST(SooTest, FindFullDeletedRegression) { + TypeParam t; + for (int i = 0; i < 1000; ++i) { + t.emplace(i); + t.erase(t.find(i)); + } + EXPECT_EQ(0, t.size()); +} + +TYPED_TEST(SooTest, ReplacingDeletedSlotDoesNotRehash) { + // We need to disable hashtablez to avoid issues related to SOO and sampling. + DisableSampling(); + + size_t n; + { + // Compute n such that n is the maximum number of elements before rehash. + TypeParam t; + t.emplace(0); + size_t c = t.bucket_count(); + for (n = 1; c == t.bucket_count(); ++n) t.emplace(n); + --n; + } + TypeParam t; + t.rehash(n); + const size_t c = t.bucket_count(); + for (size_t i = 0; i != n; ++i) t.emplace(i); + EXPECT_EQ(c, t.bucket_count()) << "rehashing threshold = " << n; + t.erase(0); + t.emplace(0); + EXPECT_EQ(c, t.bucket_count()) << "rehashing threshold = " << n; +} + +TYPED_TEST(SooTest, HintInsert) { + TypeParam t = {1, 2, 3}; + auto node = t.extract(1); + EXPECT_THAT(t, UnorderedElementsAre(2, 3)); + auto it = t.insert(t.begin(), std::move(node)); + EXPECT_THAT(t, UnorderedElementsAre(1, 2, 3)); + EXPECT_EQ(*it, 1); + EXPECT_FALSE(node); // NOLINT(bugprone-use-after-move) + + node = t.extract(2); + EXPECT_THAT(t, UnorderedElementsAre(1, 3)); + // reinsert 2 to make the next insert fail. + t.insert(2); + EXPECT_THAT(t, UnorderedElementsAre(1, 2, 3)); + it = t.insert(t.begin(), std::move(node)); + EXPECT_EQ(*it, 2); + // The node was not emptied by the insert call. + EXPECT_TRUE(node); // NOLINT(bugprone-use-after-move) +} + +TYPED_TEST(SooTest, RehashZeroForSmallTable) { + TypeParam t{0}; + EXPECT_EQ(t.capacity(), 1); + t.rehash(0); + EXPECT_EQ(t.capacity(), 1); + EXPECT_TRUE(t.contains(0)); + t.insert(1); + EXPECT_EQ(t.capacity(), NextCapacity(1)); + EXPECT_TRUE(t.contains(0)); + EXPECT_TRUE(t.contains(1)); +} + +TYPED_TEST(SooTest, RangeConstructorReservation) { + constexpr int kMaxSize = 25; + std::vector<int> v; + for (int size = 1; size <= kMaxSize; ++size) { + v.push_back(size); + TypeParam t(v.begin(), v.end()); + EXPECT_THAT(t, UnorderedElementsAreArray(v)); + size_t capacity = t.capacity(); + t.insert(size + 1); + auto expected_array = v; + expected_array.push_back(size + 1); + EXPECT_THAT(t, UnorderedElementsAreArray(expected_array)); + // Single group tables are making exact reservation. + if (static_cast<size_t>(size) <= CapacityToGrowth(Group::kWidth - 1)) { + EXPECT_GT(t.capacity(), capacity); + } + } + v.clear(); + v.push_back(0); + TypeParam t(v.begin(), v.end(), /*reservation_size=*/10); + EXPECT_GT(t.capacity(), 7); + EXPECT_THAT(t, UnorderedElementsAreArray(v)); +} + +template <typename T> +T MakeSimpleTable(size_t size, bool do_reserve) { + T t; + if (do_reserve) t.reserve(size); + while (t.size() < size) t.insert(t.size()); + return t; +} + +template <typename T> +std::vector<int> OrderOfIteration(const T& t) { + std::vector<int> res; + for (auto i : t) res.push_back(static_cast<int>(i)); + return res; +} + +// Generate irrelevant seeds to avoid being stuck in the same last bit +// in seed. +void GenerateIrrelevantSeeds(int cnt) { + for (int i = cnt % 17; i > 0; --i) { + NextHashTableSeed(); + } +} + +// These IterationOrderChanges tests depend on non-deterministic behavior. +// We are injecting non-determinism to the table. +// We have to retry enough times to make sure that the seed changes in bits that +// matter for the iteration order. +TYPED_TEST(SooTest, IterationOrderChangesByInstance) { + DisableSampling(); // We do not want test to pass only because of sampling. + for (bool do_reserve : {false, true}) { + for (size_t size : {2u, 6u, 12u, 20u}) { + SCOPED_TRACE(absl::StrCat("size: ", size, " do_reserve: ", do_reserve)); + const auto reference_table = MakeSimpleTable<TypeParam>(size, do_reserve); + const auto reference = OrderOfIteration(reference_table); + + bool found_difference = false; + for (int i = 0; !found_difference && i < 500; ++i) { + auto new_table = MakeSimpleTable<TypeParam>(size, do_reserve); + found_difference = OrderOfIteration(new_table) != reference; + GenerateIrrelevantSeeds(i); + } + if (!found_difference) { + FAIL() << "Iteration order remained the same across many attempts."; + } + } + } +} + +TYPED_TEST(SooTest, IterationOrderChangesOnRehash) { + DisableSampling(); // We do not want test to pass only because of sampling. + + // We test different sizes with many small numbers, because small table + // resize has a different codepath. + // Note: iteration order for size() <= 1 is always the same. + for (bool do_reserve : {false, true}) { + for (size_t size : {2u, 3u, 6u, 7u, 12u, 15u, 20u, 50u}) { + for (size_t rehash_size : { + size_t{0}, // Force rehash is guaranteed. + size * 10 // Rehash to the larger capacity is guaranteed. + }) { + SCOPED_TRACE(absl::StrCat("size: ", size, " rehash_size: ", rehash_size, + " do_reserve: ", do_reserve)); + bool ok = false; + auto t = MakeSimpleTable<TypeParam>(size, do_reserve); + const size_t original_capacity = t.capacity(); + auto reference = OrderOfIteration(t); + for (int i = 0; i < 500; ++i) { + if (i > 0 && rehash_size != 0) { + // Rehash back to original size. + t.rehash(0); + ASSERT_EQ(t.capacity(), original_capacity); + reference = OrderOfIteration(t); + } + // Force rehash. + t.rehash(rehash_size); + auto trial = OrderOfIteration(t); + if (trial != reference) { + // We are done. + ok = true; + break; + } + GenerateIrrelevantSeeds(i); + } + EXPECT_TRUE(ok) + << "Iteration order remained the same across many attempts " << size + << "->" << rehash_size << "."; + } + } + } +} + +// Verify that pointers are invalidated as soon as a second element is inserted. +// This prevents dependency on pointer stability on small tables. +TYPED_TEST(SooTest, UnstablePointers) { + // We need to disable hashtablez to avoid issues related to SOO and sampling. + DisableSampling(); + + TypeParam table; + + const auto addr = [&](int i) { + return reinterpret_cast<uintptr_t>(&*table.find(i)); + }; + + table.insert(0); + const uintptr_t old_ptr = addr(0); + + // This causes a rehash. + table.insert(1); + + EXPECT_NE(old_ptr, addr(0)); +} + +TYPED_TEST(SooTest, IteratorInvalidAssertsEqualityOperator) { + if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) + GTEST_SKIP() << "Assertions not enabled."; + + TypeParam t; + t.insert(1); + t.insert(2); + t.insert(3); + auto iter1 = t.begin(); + auto iter2 = std::next(iter1); + ASSERT_NE(iter1, t.end()); + ASSERT_NE(iter2, t.end()); + t.erase(iter1); + // Extra simple "regexp" as regexp support is highly varied across platforms. + const char* const kErasedDeathMessage = + SwisstableGenerationsEnabled() + ? "Invalid iterator comparison.*was likely erased" + : "Invalid iterator comparison.*might have been erased.*config=asan"; + EXPECT_DEATH_IF_SUPPORTED(void(iter1 == iter2), kErasedDeathMessage); + EXPECT_DEATH_IF_SUPPORTED(void(iter2 != iter1), kErasedDeathMessage); + t.erase(iter2); + EXPECT_DEATH_IF_SUPPORTED(void(iter1 == iter2), kErasedDeathMessage); + + TypeParam t1, t2; + t1.insert(0); + t2.insert(0); + iter1 = t1.begin(); + iter2 = t2.begin(); + const char* const kContainerDiffDeathMessage = + SwisstableGenerationsEnabled() + ? "Invalid iterator comparison.*iterators from different.* hashtables" + : "Invalid iterator comparison.*may be from different " + ".*containers.*config=asan"; + EXPECT_DEATH_IF_SUPPORTED(void(iter1 == iter2), kContainerDiffDeathMessage); + EXPECT_DEATH_IF_SUPPORTED(void(iter2 == iter1), kContainerDiffDeathMessage); +} + +TYPED_TEST(SooTest, IteratorInvalidAssertsEqualityOperatorRehash) { + if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) + GTEST_SKIP() << "Assertions not enabled."; +#ifdef ABSL_HAVE_THREAD_SANITIZER + GTEST_SKIP() << "ThreadSanitizer test runs fail on use-after-free even in " + "EXPECT_DEATH."; +#endif + + TypeParam t; + t.insert(0); + auto iter = t.begin(); + + // Trigger a rehash in t. + for (int i = 0; i < 10; ++i) t.insert(i); + + EXPECT_DEATH_IF_SUPPORTED(void(iter == t.begin()), InvalidIteratorMatcher()); +} + +TYPED_TEST(SooTest, IteratorInvalidAssertsEqualityOperatorMovedFrom) { + if (!SwisstableGenerationsEnabled()) + GTEST_SKIP() << "Generations not enabled."; + + TypeParam t; + for (int i = 0; i < 10; ++i) t.insert(i); + auto iter = t.begin(); + + TypeParam t2 = std::move(t); + + EXPECT_DEATH_IF_SUPPORTED(void(iter == t2.begin()), InvalidIteratorMatcher()); +} + + +TYPED_TEST(SooTest, ReservedGrowthUpdatesWhenTableDoesntGrow) { + TypeParam t; + for (int i = 0; i < 8; ++i) t.insert(i); + // Want to insert twice without invalidating iterators so reserve. + const size_t cap = t.capacity(); + t.reserve(t.size() + 2); + // We want to be testing the case in which the reserve doesn't grow the table. + ASSERT_EQ(cap, t.capacity()); + auto it = t.find(0); + t.insert(100); + t.insert(200); + // `it` shouldn't have been invalidated. + EXPECT_EQ(*it, 0); +} + +TYPED_TEST(SooTest, EraseIfAll) { + auto pred = [](const auto&) { return true; }; + for (int size = 0; size < 100; ++size) { + TypeParam t; + for (int i = 0; i < size; ++i) t.insert(i); + absl::container_internal::EraseIf(pred, &t); + ASSERT_EQ(t.size(), 0); + } +} + +TYPED_TEST(SooTest, EraseIfNone) { + auto pred = [](const auto&) { return false; }; + TypeParam t; + for (size_t size = 0; size < 100; ++size) { + absl::container_internal::EraseIf(pred, &t); + ASSERT_EQ(t.size(), size); + t.insert(size); + } +} + +TYPED_TEST(SooTest, EraseIfPartial) { + for (int mod : {0, 1}) { + auto pred = [&](const auto& x) { + return static_cast<int64_t>(x) % 2 == mod; + }; + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(absl::StrCat(mod, " ", size)); + TypeParam t; + std::vector<int64_t> expected; + for (int i = 0; i < size; ++i) { + t.insert(i); + if (i % 2 != mod) { + expected.push_back(i); + } + } + absl::container_internal::EraseIf(pred, &t); + ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); + } + } +} + +TYPED_TEST(SooTest, ForEach) { + TypeParam t; + std::vector<int64_t> expected; + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(size); + { + SCOPED_TRACE("mutable iteration"); + std::vector<int64_t> actual; + auto f = [&](auto& x) { actual.push_back(static_cast<int64_t>(x)); }; + absl::container_internal::ForEach(f, &t); + ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); + } + { + SCOPED_TRACE("const iteration"); + std::vector<int64_t> actual; + auto f = [&](auto& x) { + static_assert(std::is_const_v<std::remove_reference_t<decltype(x)>>, + "no mutable values should be passed to const ForEach"); + actual.push_back(static_cast<int64_t>(x)); + }; + const auto& ct = t; + absl::container_internal::ForEach(f, &ct); + ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); + } + t.insert(size); + expected.push_back(size); + } +} + +TEST(Table, ForEachMutate) { + StringTable t; + using ValueType = StringTable::value_type; + std::vector<ValueType> expected; + for (int size = 0; size < 100; ++size) { + SCOPED_TRACE(size); + std::vector<ValueType> actual; + auto f = [&](ValueType& x) { + actual.push_back(x); + x.second += 'a'; + }; + absl::container_internal::ForEach(f, &t); + ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); + for (ValueType& v : expected) { + v.second += 'a'; + } + ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); + t.emplace(std::to_string(size), std::to_string(size)); + expected.emplace_back(std::to_string(size), std::to_string(size)); + } +} + +TYPED_TEST(SooTest, EraseIfReentryDeath) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + auto erase_if_with_removal_reentrance = [](size_t reserve_size) { + TypeParam t; + t.reserve(reserve_size); + int64_t first_value = -1; + t.insert(1024); + t.insert(5078); + auto pred = [&](const auto& x) { + if (first_value == -1) { + first_value = static_cast<int64_t>(x); + return false; + } + // We erase on second call to `pred` to reduce the chance that assertion + // will happen in IterateOverFullSlots. + t.erase(first_value); + return true; + }; + absl::container_internal::EraseIf(pred, &t); + }; + // Removal will likely happen in a different group. + EXPECT_DEATH_IF_SUPPORTED(erase_if_with_removal_reentrance(1024 * 16), + "hash table was modified unexpectedly"); + // Removal will happen in the same group. + EXPECT_DEATH_IF_SUPPORTED( + erase_if_with_removal_reentrance(CapacityToGrowth(Group::kWidth - 1)), + "hash table was modified unexpectedly"); +} + +// This test is useful to test soo branch. +TYPED_TEST(SooTest, EraseIfReentrySingleElementDeath) { + if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; + + auto erase_if_with_removal_reentrance = []() { + TypeParam t; + t.insert(1024); + auto pred = [&](const auto& x) { + // We erase ourselves in order to confuse the erase_if. + t.erase(static_cast<int64_t>(x)); + return false; + }; + absl::container_internal::EraseIf(pred, &t); + }; + EXPECT_DEATH_IF_SUPPORTED(erase_if_with_removal_reentrance(), + "hash table was modified unexpectedly"); +} + template <class TableType> class SmallTableResizeTest : public testing::Test {}; -// TODO: b/517078510 - Speed up compilation by reducing the number of -// types. using SmallTableTypes = ::testing::Types< IntTable, TransferableIntTable, SooIntTable, // int8 @@ -1693,33 +2589,19 @@ ValueTable<int8_t, /*kTransferable=*/false, /*kSoo=*/true>, // int16 ValueTable<int16_t, /*kTransferable=*/true, /*kSoo=*/true>, - ValueTable<int16_t, /*kTransferable=*/false, /*kSoo=*/true>, // int128 ValueTable<SizedValue<16>, /*kTransferable=*/true, /*kSoo=*/true>, - ValueTable<SizedValue<16>, /*kTransferable=*/false, /*kSoo=*/true>, - // int192 - ValueTable<SizedValue<24>, /*kTransferable=*/true, /*kSoo=*/true>, - ValueTable<SizedValue<24>, /*kTransferable=*/false, /*kSoo=*/true>, // Special tables. MinimumAlignmentUint8Table, CustomAllocIntTable, ChangingSizeAllocIntTable, BadTable, // alignment 1, size 2. ValueTable<AlignedValue<uint8_t, 2>, /*kTransferable=*/true, /*kSoo=*/true>, - ValueTable<AlignedValue<uint8_t, 2>, /*kTransferable=*/false, - /*kSoo=*/true>, // alignment 1, size 7. ValueTable<AlignedValue<uint8_t, 7>, /*kTransferable=*/true, /*kSoo=*/true>, ValueTable<AlignedValue<uint8_t, 7>, /*kTransferable=*/false, /*kSoo=*/true>, // alignment 2, size 6. ValueTable<AlignedValue<uint16_t, 3>, /*kTransferable=*/true, - /*kSoo=*/true>, - ValueTable<AlignedValue<uint16_t, 3>, /*kTransferable=*/false, - /*kSoo=*/true>, - // alignment 2, size 10. - ValueTable<AlignedValue<uint16_t, 5>, /*kTransferable=*/true, - /*kSoo=*/true>, - ValueTable<AlignedValue<uint16_t, 5>, /*kTransferable=*/false, /*kSoo=*/true>>; TYPED_TEST_SUITE(SmallTableResizeTest, SmallTableTypes); @@ -1760,23 +2642,6 @@ } } -// Enables sampling with 1 percent sampling rate and -// resets the rate counter for the current thread. -void SetSamplingRateTo1Percent() { - SetHashtablezEnabled(true); - SetHashtablezSampleParameter(100); // Sample ~1% of tables. - // Reset rate counter for the current thread. - TestOnlyRefreshSamplingStateForCurrentThread(); -} - -// Disables sampling and resets the rate counter for the current thread. -void DisableSampling() { - SetHashtablezEnabled(false); - SetHashtablezSampleParameter(1 << 16); - // Reset rate counter for the current thread. - TestOnlyRefreshSamplingStateForCurrentThread(); -} - TYPED_TEST(SmallTableResizeTest, ResizeReduceSmallTables) { DisableSampling(); for (size_t source_size = 0; source_size < 32; ++source_size) { @@ -1819,37 +2684,6 @@ EXPECT_THAT(*it, Pair("abc", "ABC")); } -TYPED_TEST(SooTest, ContainsEmpty) { - TypeParam t; - - EXPECT_FALSE(t.contains(0)); -} - -TYPED_TEST(SooTest, Contains1) { - TypeParam t; - - EXPECT_TRUE(t.insert(0).second); - EXPECT_TRUE(t.contains(0)); - EXPECT_FALSE(t.contains(1)); - - EXPECT_EQ(1, t.erase(0)); - EXPECT_FALSE(t.contains(0)); -} - -TYPED_TEST(SooTest, Contains2) { - TypeParam t; - - EXPECT_TRUE(t.insert(0).second); - EXPECT_TRUE(t.contains(0)); - EXPECT_FALSE(t.contains(1)); - - t.clear(); - EXPECT_FALSE(t.contains(0)); - - EXPECT_TRUE(t.insert(0).second); - EXPECT_TRUE(t.contains(0)); -} - int decompose_constructed; int decompose_copy_constructed; int decompose_copy_assigned; @@ -2073,17 +2907,6 @@ TestDecompose<DecomposeHash, TransparentEqIntOverload>(true); } -// Returns the largest m such that a table with m elements has the same number -// of buckets as a table with n elements. -size_t MaxDensitySize(size_t n) { - IntTable t; - t.reserve(n); - for (size_t i = 0; i != n; ++i) t.emplace(i); - const size_t c = t.bucket_count(); - while (c == t.bucket_count()) t.emplace(n++); - return t.size() - 1; -} - struct Modulo1000Hash { size_t operator()(int64_t x) const { return static_cast<size_t>(x) % 1000; } }; @@ -2141,124 +2964,6 @@ } } -TYPED_TEST(SooTest, InsertEraseStressTest) { - TypeParam t; - const size_t kMinElementCount = 50; - std::deque<int> keys; - size_t i = 0; - for (; i < MaxDensitySize(kMinElementCount); ++i) { - t.emplace(static_cast<int64_t>(i)); - keys.push_back(i); - } - const size_t kNumIterations = 20000; - for (; i < kNumIterations; ++i) { - ASSERT_EQ(1, t.erase(keys.front())); - keys.pop_front(); - t.emplace(static_cast<int64_t>(i)); - keys.push_back(i); - } -} - -TEST(Table, InsertOverloads) { - StringTable t; - // These should all trigger the insert(init_type) overload. - t.insert({{}, {}}); - t.insert({"ABC", {}}); - t.insert({"DEF", "!!!"}); - - EXPECT_THAT(t, UnorderedElementsAre(Pair("", ""), Pair("ABC", ""), - Pair("DEF", "!!!"))); -} - -TYPED_TEST(SooTest, LargeTable) { - TypeParam t; - for (int64_t i = 0; i != 10000; ++i) { - t.emplace(i << 40); - ASSERT_EQ(t.size(), i + 1); - } - for (int64_t i = 0; i != 10000; ++i) - ASSERT_EQ(i << 40, static_cast<int64_t>(*t.find(i << 40))); -} - -// Timeout if copy is quadratic as it was in Rust. -TYPED_TEST(SooTest, EnsureNonQuadraticAsInRust) { - static const size_t kLargeSize = 1 << 15; - - TypeParam t; - for (size_t i = 0; i != kLargeSize; ++i) { - t.insert(i); - } - - // If this is quadratic, the test will timeout. - TypeParam t2; - for (const auto& entry : t) t2.insert(entry); -} - -TYPED_TEST(SooTest, ClearBug) { - if (SwisstableGenerationsEnabled()) { - GTEST_SKIP() << "Generations being enabled causes extra rehashes."; - } - - TypeParam t; - constexpr size_t capacity = container_internal::Group::kWidth - 1; - constexpr size_t max_size = capacity / 2 + 1; - for (size_t i = 0; i < max_size; ++i) { - t.insert(i); - } - ASSERT_EQ(capacity, t.capacity()); - intptr_t original = reinterpret_cast<intptr_t>(&*t.find(2)); - t.clear(); - ASSERT_EQ(capacity, t.capacity()); - for (size_t i = 0; i < max_size; ++i) { - t.insert(i); - } - ASSERT_EQ(capacity, t.capacity()); - intptr_t second = reinterpret_cast<intptr_t>(&*t.find(2)); - // We are checking that original and second are close enough to each other - // that they are probably still in the same group. This is not strictly - // guaranteed. - EXPECT_LT(static_cast<size_t>(std::abs(original - second)), - capacity * sizeof(typename TypeParam::value_type)); -} - -TYPED_TEST(SooTest, Erase) { - TypeParam t; - EXPECT_TRUE(t.find(0) == t.end()); - auto res = t.emplace(0); - EXPECT_TRUE(res.second); - EXPECT_EQ(1, t.size()); - t.erase(res.first); - EXPECT_EQ(0, t.size()); - EXPECT_TRUE(t.find(0) == t.end()); -} - -TYPED_TEST(SooTest, EraseMaintainsValidIterator) { - TypeParam t; - const int kNumElements = 100; - for (int i = 0; i < kNumElements; i++) { - EXPECT_TRUE(t.emplace(i).second); - } - EXPECT_EQ(t.size(), kNumElements); - - int num_erase_calls = 0; - auto it = t.begin(); - while (it != t.end()) { - t.erase(it++); - num_erase_calls++; - } - - EXPECT_TRUE(t.empty()); - EXPECT_EQ(num_erase_calls, kNumElements); -} - -TYPED_TEST(SooTest, EraseBeginEnd) { - TypeParam t; - for (int i = 0; i < 10; ++i) t.insert(i); - EXPECT_EQ(t.size(), 10); - t.erase(t.begin(), t.end()); - EXPECT_EQ(t.size(), 0); -} - // Collect N bad keys by following algorithm: // 1. Create an empty table and reserve it to 2 * N. // 2. Insert N random elements. @@ -2675,54 +3380,6 @@ .HasNoDeleted()); } -TYPED_TEST(SooTest, Clear) { - TypeParam t; - EXPECT_TRUE(t.find(0) == t.end()); - t.clear(); - EXPECT_TRUE(t.find(0) == t.end()); - auto res = t.emplace(0); - EXPECT_TRUE(res.second); - EXPECT_EQ(1, t.size()); - t.clear(); - EXPECT_EQ(0, t.size()); - EXPECT_TRUE(t.find(0) == t.end()); -} - -TYPED_TEST(SooTest, Swap) { - TypeParam t; - EXPECT_TRUE(t.find(0) == t.end()); - auto res = t.emplace(0); - EXPECT_TRUE(res.second); - EXPECT_EQ(1, t.size()); - TypeParam u; - t.swap(u); - EXPECT_EQ(0, t.size()); - EXPECT_EQ(1, u.size()); - EXPECT_TRUE(t.find(0) == t.end()); - EXPECT_THAT(*u.find(0), 0); -} - -TYPED_TEST(SooTest, Rehash) { - TypeParam t; - EXPECT_TRUE(t.find(0) == t.end()); - t.emplace(0); - t.emplace(1); - EXPECT_EQ(2, t.size()); - t.rehash(128); - EXPECT_EQ(2, t.size()); - EXPECT_THAT(*t.find(0), 0); - EXPECT_THAT(*t.find(1), 1); -} - -TYPED_TEST(SooTest, RehashDoesNotRehashWhenNotNecessary) { - TypeParam t; - t.emplace(0); - t.emplace(1); - auto* p = &*t.find(0); - t.rehash(1); - EXPECT_EQ(p, &*t.find(0)); -} - // Following two tests use non-SOO table because they test for 0 capacity. TEST(Table, RehashZeroDoesNotAllocateOnEmptyTable) { NonSooIntTable t; @@ -2739,15 +3396,6 @@ EXPECT_EQ(0, t.bucket_count()); } -TYPED_TEST(SooTest, RehashZeroForcesRehash) { - TypeParam t; - t.emplace(0); - t.emplace(1); - auto* p = &*t.find(0); - t.rehash(0); - EXPECT_NE(p, &*t.find(0)); -} - TEST(Table, ConstructFromInitList) { using P = std::pair<std::string, std::string>; struct Q { @@ -2756,117 +3404,6 @@ StringTable t = {P(), Q(), {}, {{}, {}}}; } -TYPED_TEST(SooTest, CopyConstruct) { - TypeParam t; - t.emplace(0); - EXPECT_EQ(1, t.size()); - { - TypeParam u(t); - EXPECT_EQ(1, u.size()); - EXPECT_THAT(*u.find(0), 0); - } - { - TypeParam u{t}; - EXPECT_EQ(1, u.size()); - EXPECT_THAT(*u.find(0), 0); - } - { - TypeParam u = t; - EXPECT_EQ(1, u.size()); - EXPECT_THAT(*u.find(0), 0); - } -} - -TYPED_TEST(SooTest, CopyAssignment) { - std::vector<size_t> sizes = {0, 1, 7, 25}; - for (size_t source_size : sizes) { - for (size_t target_size : sizes) { - SCOPED_TRACE(absl::StrCat("source_size: ", source_size, - " target_size: ", target_size)); - TypeParam source; - std::vector<int> source_elements; - for (size_t i = 0; i < source_size; ++i) { - source.emplace(static_cast<int>(i) * 2); - source_elements.push_back(static_cast<int>(i) * 2); - } - TypeParam target; - for (size_t i = 0; i < target_size; ++i) { - target.emplace(static_cast<int>(i) * 3); - } - target = source; - ASSERT_EQ(target.size(), source_size); - ASSERT_THAT(target, UnorderedElementsAreArray(source_elements)); - } - } -} - -TYPED_TEST(SooTest, CopyConstructWithSampling) { - SetSamplingRateTo1Percent(); - for (int i = 0; i < 10000; ++i) { - TypeParam t; - t.emplace(0); - EXPECT_EQ(1, t.size()); - { - TypeParam u(t); - EXPECT_EQ(1, u.size()); - EXPECT_THAT(*u.find(0), 0); - } - } -} - -TYPED_TEST(SooTest, CopyDifferentSizes) { - TypeParam t; - - for (int i = 0; i < 100; ++i) { - t.emplace(i); - TypeParam c = t; - for (int j = 0; j <= i; ++j) { - ASSERT_TRUE(c.find(j) != c.end()) << "i=" << i << " j=" << j; - } - // Testing find miss to verify that table is not full. - ASSERT_TRUE(c.find(-1) == c.end()); - } -} - -TYPED_TEST(SooTest, CopyDifferentSizesWithReserve) { - for (size_t size = 0; size < 153; ++size) { - SCOPED_TRACE(absl::StrCat("size: ", size)); - TypeParam t; - t.reserve(size); - for (size_t i = 0; i < size; ++i) { - ASSERT_TRUE(t.insert(static_cast<int>(i)).second) << i; - } - auto t2 = t; - ASSERT_EQ(t2.size(), size); - for (size_t i = 0; i < size; ++i) { - ASSERT_TRUE(t2.contains(static_cast<int>(i))) << i; - } - ASSERT_TRUE(t2.insert(static_cast<int>(size)).second); - ASSERT_EQ(t2.size(), size + 1); - ASSERT_TRUE(t2.contains(static_cast<int>(size))); - } -} - -TYPED_TEST(SooTest, CopyDifferentCapacities) { - for (int cap = 1; cap < 100; cap = cap * 2 + 1) { - TypeParam t; - t.reserve(static_cast<size_t>(cap)); - for (int i = 0; i <= cap; ++i) { - t.emplace(i); - if (i != cap && i % 5 != 0) { - continue; - } - TypeParam c = t; - for (int j = 0; j <= i; ++j) { - ASSERT_TRUE(c.find(j) != c.end()) - << "cap=" << cap << " i=" << i << " j=" << j; - } - // Testing find miss to verify that table is not full. - ASSERT_TRUE(c.find(-1) == c.end()); - } - } -} - TEST(Table, CopyConstructWithAlloc) { StringTable t; t.emplace("a", "b"); @@ -2999,48 +3536,6 @@ u.insert(std::begin(v2), std::end(v2)); EXPECT_NE(u, t); } - -TYPED_TEST(SooTest, NumDeletedRegression) { - TypeParam t; - t.emplace(0); - t.erase(t.find(0)); - // construct over a deleted slot. - t.emplace(0); - t.clear(); -} - -TYPED_TEST(SooTest, FindFullDeletedRegression) { - TypeParam t; - for (int i = 0; i < 1000; ++i) { - t.emplace(i); - t.erase(t.find(i)); - } - EXPECT_EQ(0, t.size()); -} - -TYPED_TEST(SooTest, ReplacingDeletedSlotDoesNotRehash) { - // We need to disable hashtablez to avoid issues related to SOO and sampling. - DisableSampling(); - - size_t n; - { - // Compute n such that n is the maximum number of elements before rehash. - TypeParam t; - t.emplace(0); - size_t c = t.bucket_count(); - for (n = 1; c == t.bucket_count(); ++n) t.emplace(n); - --n; - } - TypeParam t; - t.rehash(n); - const size_t c = t.bucket_count(); - for (size_t i = 0; i != n; ++i) t.emplace(i); - EXPECT_EQ(c, t.bucket_count()) << "rehashing threshold = " << n; - t.erase(0); - t.emplace(0); - EXPECT_EQ(c, t.bucket_count()) << "rehashing threshold = " << n; -} - TEST(Table, NoThrowMoveConstruct) { ASSERT_TRUE( std::is_nothrow_copy_constructible_v<absl::Hash<absl::string_view>>); @@ -3311,150 +3806,6 @@ EXPECT_THAT(t2, UnorderedElementsAre(Pair(k0, ""))); } -TYPED_TEST(SooTest, HintInsert) { - TypeParam t = {1, 2, 3}; - auto node = t.extract(1); - EXPECT_THAT(t, UnorderedElementsAre(2, 3)); - auto it = t.insert(t.begin(), std::move(node)); - EXPECT_THAT(t, UnorderedElementsAre(1, 2, 3)); - EXPECT_EQ(*it, 1); - EXPECT_FALSE(node); // NOLINT(bugprone-use-after-move) - - node = t.extract(2); - EXPECT_THAT(t, UnorderedElementsAre(1, 3)); - // reinsert 2 to make the next insert fail. - t.insert(2); - EXPECT_THAT(t, UnorderedElementsAre(1, 2, 3)); - it = t.insert(t.begin(), std::move(node)); - EXPECT_EQ(*it, 2); - // The node was not emptied by the insert call. - EXPECT_TRUE(node); // NOLINT(bugprone-use-after-move) -} - -TYPED_TEST(SooTest, RehashZeroForSmallTable) { - TypeParam t{0}; - EXPECT_EQ(t.capacity(), 1); - t.rehash(0); - EXPECT_EQ(t.capacity(), 1); - EXPECT_TRUE(t.contains(0)); - t.insert(1); - EXPECT_EQ(t.capacity(), NextCapacity(1)); - EXPECT_TRUE(t.contains(0)); - EXPECT_TRUE(t.contains(1)); -} - -template <typename T> -T MakeSimpleTable(size_t size, bool do_reserve) { - T t; - if (do_reserve) t.reserve(size); - while (t.size() < size) t.insert(t.size()); - return t; -} - -template <typename T> -std::vector<int> OrderOfIteration(const T& t) { - std::vector<int> res; - for (auto i : t) res.push_back(static_cast<int>(i)); - return res; -} - -// Generate irrelevant seeds to avoid being stuck in the same last bit -// in seed. -void GenerateIrrelevantSeeds(int cnt) { - for (int i = cnt % 17; i > 0; --i) { - NextHashTableSeed(); - } -} - -// These IterationOrderChanges tests depend on non-deterministic behavior. -// We are injecting non-determinism to the table. -// We have to retry enough times to make sure that the seed changes in bits that -// matter for the iteration order. -TYPED_TEST(SooTest, IterationOrderChangesByInstance) { - DisableSampling(); // We do not want test to pass only because of sampling. - for (bool do_reserve : {false, true}) { - for (size_t size : {2u, 6u, 12u, 20u}) { - SCOPED_TRACE(absl::StrCat("size: ", size, " do_reserve: ", do_reserve)); - const auto reference_table = MakeSimpleTable<TypeParam>(size, do_reserve); - const auto reference = OrderOfIteration(reference_table); - - bool found_difference = false; - for (int i = 0; !found_difference && i < 500; ++i) { - auto new_table = MakeSimpleTable<TypeParam>(size, do_reserve); - found_difference = OrderOfIteration(new_table) != reference; - GenerateIrrelevantSeeds(i); - } - if (!found_difference) { - FAIL() << "Iteration order remained the same across many attempts."; - } - } - } -} - -TYPED_TEST(SooTest, IterationOrderChangesOnRehash) { - DisableSampling(); // We do not want test to pass only because of sampling. - - // We test different sizes with many small numbers, because small table - // resize has a different codepath. - // Note: iteration order for size() <= 1 is always the same. - for (bool do_reserve : {false, true}) { - for (size_t size : {2u, 3u, 6u, 7u, 12u, 15u, 20u, 50u}) { - for (size_t rehash_size : { - size_t{0}, // Force rehash is guaranteed. - size * 10 // Rehash to the larger capacity is guaranteed. - }) { - SCOPED_TRACE(absl::StrCat("size: ", size, " rehash_size: ", rehash_size, - " do_reserve: ", do_reserve)); - bool ok = false; - auto t = MakeSimpleTable<TypeParam>(size, do_reserve); - const size_t original_capacity = t.capacity(); - auto reference = OrderOfIteration(t); - for (int i = 0; i < 500; ++i) { - if (i > 0 && rehash_size != 0) { - // Rehash back to original size. - t.rehash(0); - ASSERT_EQ(t.capacity(), original_capacity); - reference = OrderOfIteration(t); - } - // Force rehash. - t.rehash(rehash_size); - auto trial = OrderOfIteration(t); - if (trial != reference) { - // We are done. - ok = true; - break; - } - GenerateIrrelevantSeeds(i); - } - EXPECT_TRUE(ok) - << "Iteration order remained the same across many attempts " << size - << "->" << rehash_size << "."; - } - } - } -} - -// Verify that pointers are invalidated as soon as a second element is inserted. -// This prevents dependency on pointer stability on small tables. -TYPED_TEST(SooTest, UnstablePointers) { - // We need to disable hashtablez to avoid issues related to SOO and sampling. - DisableSampling(); - - TypeParam table; - - const auto addr = [&](int i) { - return reinterpret_cast<uintptr_t>(&*table.find(i)); - }; - - table.insert(0); - const uintptr_t old_ptr = addr(0); - - // This causes a rehash. - table.insert(1); - - EXPECT_NE(old_ptr, addr(0)); -} - TEST(TableDeathTest, InvalidIteratorAsserts) { if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) GTEST_SKIP() << "Assertions not enabled."; @@ -3493,81 +3844,6 @@ // the control is static constant. } -// Invalid iterator use can trigger crashes or invalidated iterator assertions. -testing::Matcher<const std::string&> InvalidIteratorMatcher() { - return AnyOf(HasSubstr("invalidated iterator"), HasSubstr("Invalid iterator"), - HasSubstr("invalid iterator"), - HasSubstr("CrashIfIteratorIsInvalid")); -} - -TYPED_TEST(SooTest, IteratorInvalidAssertsEqualityOperator) { - if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) - GTEST_SKIP() << "Assertions not enabled."; - - TypeParam t; - t.insert(1); - t.insert(2); - t.insert(3); - auto iter1 = t.begin(); - auto iter2 = std::next(iter1); - ASSERT_NE(iter1, t.end()); - ASSERT_NE(iter2, t.end()); - t.erase(iter1); - // Extra simple "regexp" as regexp support is highly varied across platforms. - const char* const kErasedDeathMessage = - SwisstableGenerationsEnabled() - ? "Invalid iterator comparison.*was likely erased" - : "Invalid iterator comparison.*might have been erased.*config=asan"; - EXPECT_DEATH_IF_SUPPORTED(void(iter1 == iter2), kErasedDeathMessage); - EXPECT_DEATH_IF_SUPPORTED(void(iter2 != iter1), kErasedDeathMessage); - t.erase(iter2); - EXPECT_DEATH_IF_SUPPORTED(void(iter1 == iter2), kErasedDeathMessage); - - TypeParam t1, t2; - t1.insert(0); - t2.insert(0); - iter1 = t1.begin(); - iter2 = t2.begin(); - const char* const kContainerDiffDeathMessage = - SwisstableGenerationsEnabled() - ? "Invalid iterator comparison.*iterators from different.* hashtables" - : "Invalid iterator comparison.*may be from different " - ".*containers.*config=asan"; - EXPECT_DEATH_IF_SUPPORTED(void(iter1 == iter2), kContainerDiffDeathMessage); - EXPECT_DEATH_IF_SUPPORTED(void(iter2 == iter1), kContainerDiffDeathMessage); -} - -TYPED_TEST(SooTest, IteratorInvalidAssertsEqualityOperatorRehash) { - if (!IsAssertEnabled() && !SwisstableGenerationsEnabled()) - GTEST_SKIP() << "Assertions not enabled."; -#ifdef ABSL_HAVE_THREAD_SANITIZER - GTEST_SKIP() << "ThreadSanitizer test runs fail on use-after-free even in " - "EXPECT_DEATH."; -#endif - - TypeParam t; - t.insert(0); - auto iter = t.begin(); - - // Trigger a rehash in t. - for (int i = 0; i < 10; ++i) t.insert(i); - - EXPECT_DEATH_IF_SUPPORTED(void(iter == t.begin()), InvalidIteratorMatcher()); -} - -TYPED_TEST(SooTest, IteratorInvalidAssertsEqualityOperatorMovedFrom) { - if (!SwisstableGenerationsEnabled()) - GTEST_SKIP() << "Generations not enabled."; - - TypeParam t; - for (int i = 0; i < 10; ++i) t.insert(i); - auto iter = t.begin(); - - TypeParam t2 = std::move(t); - - EXPECT_DEATH_IF_SUPPORTED(void(iter == t2.begin()), InvalidIteratorMatcher()); -} - #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) template <typename T> class RawHashSamplerTest : public testing::Test {}; @@ -4115,21 +4391,6 @@ #endif } -TYPED_TEST(SooTest, ReservedGrowthUpdatesWhenTableDoesntGrow) { - TypeParam t; - for (int i = 0; i < 8; ++i) t.insert(i); - // Want to insert twice without invalidating iterators so reserve. - const size_t cap = t.capacity(); - t.reserve(t.size() + 2); - // We want to be testing the case in which the reserve doesn't grow the table. - ASSERT_EQ(cap, t.capacity()); - auto it = t.find(0); - t.insert(100); - t.insert(200); - // `it` shouldn't have been invalidated. - EXPECT_EQ(*it, 0); -} - template <class TableType> class InstanceTrackerTest : public testing::Test {}; @@ -4196,146 +4457,6 @@ EXPECT_EQ(tracker.live_instances(), 0); } -TYPED_TEST(SooTest, EraseIfAll) { - auto pred = [](const auto&) { return true; }; - for (int size = 0; size < 100; ++size) { - TypeParam t; - for (int i = 0; i < size; ++i) t.insert(i); - absl::container_internal::EraseIf(pred, &t); - ASSERT_EQ(t.size(), 0); - } -} - -TYPED_TEST(SooTest, EraseIfNone) { - auto pred = [](const auto&) { return false; }; - TypeParam t; - for (size_t size = 0; size < 100; ++size) { - absl::container_internal::EraseIf(pred, &t); - ASSERT_EQ(t.size(), size); - t.insert(size); - } -} - -TYPED_TEST(SooTest, EraseIfPartial) { - for (int mod : {0, 1}) { - auto pred = [&](const auto& x) { - return static_cast<int64_t>(x) % 2 == mod; - }; - for (int size = 0; size < 100; ++size) { - SCOPED_TRACE(absl::StrCat(mod, " ", size)); - TypeParam t; - std::vector<int64_t> expected; - for (int i = 0; i < size; ++i) { - t.insert(i); - if (i % 2 != mod) { - expected.push_back(i); - } - } - absl::container_internal::EraseIf(pred, &t); - ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); - } - } -} - -TYPED_TEST(SooTest, ForEach) { - TypeParam t; - std::vector<int64_t> expected; - for (int size = 0; size < 100; ++size) { - SCOPED_TRACE(size); - { - SCOPED_TRACE("mutable iteration"); - std::vector<int64_t> actual; - auto f = [&](auto& x) { actual.push_back(static_cast<int64_t>(x)); }; - absl::container_internal::ForEach(f, &t); - ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); - } - { - SCOPED_TRACE("const iteration"); - std::vector<int64_t> actual; - auto f = [&](auto& x) { - static_assert(std::is_const_v<std::remove_reference_t<decltype(x)>>, - "no mutable values should be passed to const ForEach"); - actual.push_back(static_cast<int64_t>(x)); - }; - const auto& ct = t; - absl::container_internal::ForEach(f, &ct); - ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); - } - t.insert(size); - expected.push_back(size); - } -} - -TEST(Table, ForEachMutate) { - StringTable t; - using ValueType = StringTable::value_type; - std::vector<ValueType> expected; - for (int size = 0; size < 100; ++size) { - SCOPED_TRACE(size); - std::vector<ValueType> actual; - auto f = [&](ValueType& x) { - actual.push_back(x); - x.second += "a"; - }; - absl::container_internal::ForEach(f, &t); - ASSERT_THAT(actual, testing::UnorderedElementsAreArray(expected)); - for (ValueType& v : expected) { - v.second += "a"; - } - ASSERT_THAT(t, testing::UnorderedElementsAreArray(expected)); - t.emplace(std::to_string(size), std::to_string(size)); - expected.emplace_back(std::to_string(size), std::to_string(size)); - } -} - -TYPED_TEST(SooTest, EraseIfReentryDeath) { - if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; - - auto erase_if_with_removal_reentrance = [](size_t reserve_size) { - TypeParam t; - t.reserve(reserve_size); - int64_t first_value = -1; - t.insert(1024); - t.insert(5078); - auto pred = [&](const auto& x) { - if (first_value == -1) { - first_value = static_cast<int64_t>(x); - return false; - } - // We erase on second call to `pred` to reduce the chance that assertion - // will happen in IterateOverFullSlots. - t.erase(first_value); - return true; - }; - absl::container_internal::EraseIf(pred, &t); - }; - // Removal will likely happen in a different group. - EXPECT_DEATH_IF_SUPPORTED(erase_if_with_removal_reentrance(1024 * 16), - "hash table was modified unexpectedly"); - // Removal will happen in the same group. - EXPECT_DEATH_IF_SUPPORTED( - erase_if_with_removal_reentrance(CapacityToGrowth(Group::kWidth - 1)), - "hash table was modified unexpectedly"); -} - -// This test is useful to test soo branch. -TYPED_TEST(SooTest, EraseIfReentrySingleElementDeath) { - if (!IsAssertEnabled()) GTEST_SKIP() << "Assertions not enabled."; - - auto erase_if_with_removal_reentrance = []() { - TypeParam t; - t.insert(1024); - auto pred = [&](const auto& x) { - // We erase ourselves in order to confuse the erase_if. - t.erase(static_cast<int64_t>(x)); - return false; - }; - absl::container_internal::EraseIf(pred, &t); - }; - EXPECT_DEATH_IF_SUPPORTED(erase_if_with_removal_reentrance(), - "hash table was modified unexpectedly"); -} - TEST(Table, EraseBeginEndResetsReservedGrowth) { bool frozen = false; BadHashFreezableIntTable t{FreezableAlloc<int64_t>(&frozen)}; @@ -4608,16 +4729,10 @@ "hash table was modified unexpectedly"); } -template <typename T> -class SooTable : public testing::Test {}; -using FreezableSooTableTypes = - ::testing::Types<FreezableSizedValueSooTable<8>, - FreezableSizedValueSooTable<16>>; -TYPED_TEST_SUITE(SooTable, FreezableSooTableTypes); - -TYPED_TEST(SooTable, Basic) { +TEST(SooTable, Basic) { bool frozen = true; - TypeParam t{FreezableAlloc<typename TypeParam::value_type>(&frozen)}; + FreezableSizedValueSooTable<8> t{ + FreezableAlloc<FreezableSizedValueSooTable<8>::value_type>(&frozen)}; if (t.capacity() != SooCapacity()) { CHECK_LT(sizeof(void*), 8) << "missing SOO coverage"; GTEST_SKIP() << "not SOO on this platform"; @@ -4927,38 +5042,6 @@ } } -TEST(Table, MaxSizeOverflow) { -#ifdef ABSL_HAVE_EXCEPTIONS - GTEST_SKIP() << "Skipping test because exceptions are enabled. EXPECT_DEATH " - "doesn't work with exceptions."; -#elif defined(ABSL_HAVE_THREAD_SANITIZER) - GTEST_SKIP() << "ThreadSanitizer test runs fail on OOM even in EXPECT_DEATH."; -#else - const std::string expected_death_message = - "new failed|failed to allocate|bad_alloc|exceeds maximum supported size"; - size_t overflow = (std::numeric_limits<size_t>::max)(); - EXPECT_DEATH_IF_SUPPORTED(IntTable t(overflow), expected_death_message); - IntTable t; - EXPECT_DEATH_IF_SUPPORTED(t.reserve(overflow), expected_death_message); - EXPECT_DEATH_IF_SUPPORTED(t.rehash(overflow), expected_death_message); - size_t slightly_overflow = - MaxValidSize(sizeof(IntTable::key_type), sizeof(IntTable::value_type)) + - 1; - size_t slightly_overflow_capacity = - NextCapacity(NormalizeCapacity(slightly_overflow)); - EXPECT_DEATH_IF_SUPPORTED(IntTable t2(slightly_overflow_capacity - 10), - expected_death_message); - EXPECT_DEATH_IF_SUPPORTED(t.reserve(slightly_overflow), - expected_death_message); - EXPECT_DEATH_IF_SUPPORTED(t.rehash(slightly_overflow), - expected_death_message); - IntTable non_empty_table; - non_empty_table.insert(0); - EXPECT_DEATH_IF_SUPPORTED(non_empty_table.reserve(slightly_overflow), - expected_death_message); -#endif // defined(ABSL_HAVE_THREAD_SANITIZER) -} - // Tests that reserving enough space for more than the max number of unique keys // doesn't crash and we end up with kMaxValidCapacity. TEST(Table, MaxSizeOverflowUniqueKeys) { @@ -5038,6 +5121,19 @@ for (size_t cap = t.capacity(); cap < kTargetCapacity; cap = NextCapacity(cap)) { ASSERT_EQ(t.capacity(), cap); + // Block upto 100 elements to test that kMarkedForSlowTransfer elements do + // not conflict with blocked elements. + for (size_t i = cap - 1, + growth_left = common.growth_info().GetGrowthLeftTotalSlow(cap), + blocked = 0; + i > cap / 2; --i) { + if (common.control()[i] == ctrl_t::kEmpty && growth_left > 1) { + growth_left--; + blocked++; + common.control()[i] = ctrl_t::kSentinel; + if (blocked > 100) break; + } + } // Update growth info to force resize on the next insert. This way we avoid // having to insert many elements. common.growth_info().InitGrowthLeftNoDeleted(/*growth_left=*/0, cap);
diff --git a/absl/container/linked_hash_map.h b/absl/container/linked_hash_map.h index efc9686..7ebeaf7 100644 --- a/absl/container/linked_hash_map.h +++ b/absl/container/linked_hash_map.h
@@ -156,66 +156,69 @@ linked_hash_map() {} - explicit linked_hash_map(size_t bucket_count, const hasher& hash = hasher(), + explicit linked_hash_map(size_t reservation_size, + const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : set_(bucket_count, Wrapped<hasher>(hash), Wrapped<key_equal>(eq), + : set_(reservation_size, Wrapped<hasher>(hash), Wrapped<key_equal>(eq), alloc), list_(alloc) {} - linked_hash_map(size_t bucket_count, const hasher& hash, + linked_hash_map(size_t reservation_size, const hasher& hash, const allocator_type& alloc) - : linked_hash_map(bucket_count, hash, key_equal(), alloc) {} + : linked_hash_map(reservation_size, hash, key_equal(), alloc) {} - linked_hash_map(size_t bucket_count, const allocator_type& alloc) - : linked_hash_map(bucket_count, hasher(), key_equal(), alloc) {} + linked_hash_map(size_t reservation_size, const allocator_type& alloc) + : linked_hash_map(reservation_size, hasher(), key_equal(), alloc) {} explicit linked_hash_map(const allocator_type& alloc) : linked_hash_map(0, hasher(), key_equal(), alloc) {} template <class InputIt> - linked_hash_map(InputIt first, InputIt last, size_t bucket_count = 0, + linked_hash_map(InputIt first, InputIt last, size_t reservation_size = 0, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : linked_hash_map(bucket_count, hash, eq, alloc) { + : linked_hash_map(reservation_size, hash, eq, alloc) { insert(first, last); } template <class InputIt> - linked_hash_map(InputIt first, InputIt last, size_t bucket_count, + linked_hash_map(InputIt first, InputIt last, size_t reservation_size, const hasher& hash, const allocator_type& alloc) - : linked_hash_map(first, last, bucket_count, hash, key_equal(), alloc) {} + : linked_hash_map(first, last, reservation_size, hash, key_equal(), + alloc) {} template <class InputIt> - linked_hash_map(InputIt first, InputIt last, size_t bucket_count, + linked_hash_map(InputIt first, InputIt last, size_t reservation_size, const allocator_type& alloc) - : linked_hash_map(first, last, bucket_count, hasher(), key_equal(), + : linked_hash_map(first, last, reservation_size, hasher(), key_equal(), alloc) {} template <class InputIt> linked_hash_map(InputIt first, InputIt last, const allocator_type& alloc) - : linked_hash_map(first, last, /*bucket_count=*/0, hasher(), key_equal(), - alloc) {} + : linked_hash_map(first, last, /*reservation_size=*/0, hasher(), + key_equal(), alloc) {} linked_hash_map(std::initializer_list<value_type> init, - size_t bucket_count = 0, const hasher& hash = hasher(), + size_t reservation_size = 0, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : linked_hash_map(init.begin(), init.end(), bucket_count, hash, eq, + : linked_hash_map(init.begin(), init.end(), reservation_size, hash, eq, alloc) {} - linked_hash_map(std::initializer_list<value_type> init, size_t bucket_count, - const hasher& hash, const allocator_type& alloc) - : linked_hash_map(init, bucket_count, hash, key_equal(), alloc) {} - - linked_hash_map(std::initializer_list<value_type> init, size_t bucket_count, + linked_hash_map(std::initializer_list<value_type> init, + size_t reservation_size, const hasher& hash, const allocator_type& alloc) - : linked_hash_map(init, bucket_count, hasher(), key_equal(), alloc) {} + : linked_hash_map(init, reservation_size, hash, key_equal(), alloc) {} + + linked_hash_map(std::initializer_list<value_type> init, + size_t reservation_size, const allocator_type& alloc) + : linked_hash_map(init, reservation_size, hasher(), key_equal(), alloc) {} linked_hash_map(std::initializer_list<value_type> init, const allocator_type& alloc) - : linked_hash_map(init, /*bucket_count=*/0, hasher(), key_equal(), + : linked_hash_map(init, /*reservation_size=*/0, hasher(), key_equal(), alloc) {} linked_hash_map(const linked_hash_map& other) @@ -505,7 +508,7 @@ template <typename... Args> std::pair<iterator, bool> emplace(Args&&... args) { - ListType node_donor; + ListType node_donor(get_allocator()); auto list_iter = node_donor.emplace(node_donor.end(), std::forward<Args>(args)...); auto ins = set_.insert(list_iter); @@ -550,7 +553,7 @@ node_type extract(const_iterator position) { set_.erase(position->first); - ListType extracted_node_list; + ListType extracted_node_list(get_allocator()); extracted_node_list.splice(extracted_node_list.end(), list_, position); return node_type(std::move(extracted_node_list)); } @@ -560,7 +563,7 @@ node_type extract(const key_arg<K>& key) { auto node = set_.extract(key); if (node.empty()) return node_type(); - ListType extracted_node_list; + ListType extracted_node_list(get_allocator()); extracted_node_list.splice(extracted_node_list.end(), list_, node.value()); return node_type(std::move(extracted_node_list)); }
diff --git a/absl/container/linked_hash_map_test.cc b/absl/container/linked_hash_map_test.cc index 9f530d7..e8df807 100644 --- a/absl/container/linked_hash_map_test.cc +++ b/absl/container/linked_hash_map_test.cc
@@ -16,6 +16,8 @@ #include <algorithm> #include <cstddef> +#include <cstdint> +#include <functional> #include <memory> #include <string> #include <tuple> @@ -30,6 +32,7 @@ #include "absl/container/internal/hash_generator_testing.h" #include "absl/container/internal/hash_policy_testing.h" #include "absl/container/internal/heterogeneous_lookup_testing.h" +#include "absl/container/internal/test_allocator.h" #include "absl/container/internal/test_instance_tracker.h" #include "absl/container/internal/unordered_map_constructor_test.h" #include "absl/container/internal/unordered_map_lookup_test.h" @@ -792,6 +795,31 @@ EXPECT_THAT(m, ElementsAre(Pair(2, 9))); } +// Verify that emplacing and extracting nodes results in the same stateful +// allocator being used for splicing purposes, rather than another instance +// (say, a default-constructed one), which could otherwise silently corrupt +// memory. +TEST(LinkedHashMap, ExtractAndEmplaceUseSameStatefulAllocator) { + using Alloc = + absl::container_internal::CountingAllocator<std::pair<const int, int>>; + int64_t bytes_used = 0; + Alloc alloc(&bytes_used); + linked_hash_map<int, int, linked_hash_map<int, int>::hasher, std::equal_to<>, + Alloc> + map(alloc); + + map.emplace(1, 10); + EXPECT_GT(bytes_used, 0) << "emplace() failed to use the same allocator"; + + auto node = map.extract(map.begin()); + EXPECT_EQ(node.get_allocator(), alloc) + << "extract(iter) failed to use the same allocator"; + + map.insert(std::move(node)); + EXPECT_EQ(map.extract(1).get_allocator(), alloc) + << "extract(key) failed to use the same allocator"; +} + TEST(LinkedHashMap, Merge) { linked_hash_map<int, int> m = {{1, 7}, {3, 6}}; linked_hash_map<int, int> src = {{1, 10}, {2, 9}, {4, 16}};
diff --git a/absl/container/linked_hash_set.h b/absl/container/linked_hash_set.h index ae7819b..cda21d6 100644 --- a/absl/container/linked_hash_set.h +++ b/absl/container/linked_hash_set.h
@@ -146,66 +146,68 @@ linked_hash_set() {} - explicit linked_hash_set(size_t bucket_count, const hasher& hash = hasher(), + explicit linked_hash_set(size_t reservation_size, + const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : set_(bucket_count, Wrapped<hasher>(hash), Wrapped<key_equal>(eq), + : set_(reservation_size, Wrapped<hasher>(hash), Wrapped<key_equal>(eq), alloc), list_(alloc) {} - linked_hash_set(size_t bucket_count, const hasher& hash, + linked_hash_set(size_t reservation_size, const hasher& hash, const allocator_type& alloc) - : linked_hash_set(bucket_count, hash, key_equal(), alloc) {} + : linked_hash_set(reservation_size, hash, key_equal(), alloc) {} - linked_hash_set(size_t bucket_count, const allocator_type& alloc) - : linked_hash_set(bucket_count, hasher(), key_equal(), alloc) {} + linked_hash_set(size_t reservation_size, const allocator_type& alloc) + : linked_hash_set(reservation_size, hasher(), key_equal(), alloc) {} explicit linked_hash_set(const allocator_type& alloc) : linked_hash_set(0, hasher(), key_equal(), alloc) {} template <class InputIt> - linked_hash_set(InputIt first, InputIt last, size_t bucket_count = 0, + linked_hash_set(InputIt first, InputIt last, size_t reservation_size = 0, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : linked_hash_set(bucket_count, hash, eq, alloc) { + : linked_hash_set(reservation_size, hash, eq, alloc) { insert(first, last); } template <class InputIter> - linked_hash_set(InputIter first, InputIter last, size_t bucket_count, + linked_hash_set(InputIter first, InputIter last, size_t reservation_size, const hasher& hash, const allocator_type& alloc) - : linked_hash_set(first, last, bucket_count, hash, key_equal(), alloc) {} + : linked_hash_set(first, last, reservation_size, hash, key_equal(), + alloc) {} template <class InputIter> - linked_hash_set(InputIter first, InputIter last, size_t bucket_count, + linked_hash_set(InputIter first, InputIter last, size_t reservation_size, const allocator_type& alloc) - : linked_hash_set(first, last, bucket_count, hasher(), key_equal(), + : linked_hash_set(first, last, reservation_size, hasher(), key_equal(), alloc) {} template <class InputIt> linked_hash_set(InputIt first, InputIt last, const allocator_type& alloc) - : linked_hash_set(first, last, /*bucket_count=*/0, hasher(), key_equal(), - alloc) {} + : linked_hash_set(first, last, /*reservation_size=*/0, hasher(), + key_equal(), alloc) {} - linked_hash_set(std::initializer_list<key_type> init, size_t bucket_count = 0, - const hasher& hash = hasher(), + linked_hash_set(std::initializer_list<key_type> init, + size_t reservation_size = 0, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : linked_hash_set(init.begin(), init.end(), bucket_count, hash, eq, + : linked_hash_set(init.begin(), init.end(), reservation_size, hash, eq, alloc) {} - linked_hash_set(std::initializer_list<key_type> init, size_t bucket_count, + linked_hash_set(std::initializer_list<key_type> init, size_t reservation_size, const allocator_type& alloc) - : linked_hash_set(init, bucket_count, hasher(), key_equal(), alloc) {} + : linked_hash_set(init, reservation_size, hasher(), key_equal(), alloc) {} - linked_hash_set(std::initializer_list<key_type> init, size_t bucket_count, + linked_hash_set(std::initializer_list<key_type> init, size_t reservation_size, const hasher& hash, const allocator_type& alloc) - : linked_hash_set(init, bucket_count, hash, key_equal(), alloc) {} + : linked_hash_set(init, reservation_size, hash, key_equal(), alloc) {} linked_hash_set(std::initializer_list<key_type> init, const allocator_type& alloc) - : linked_hash_set(init, /*bucket_count=*/0, hasher(), key_equal(), + : linked_hash_set(init, /*reservation_size=*/0, hasher(), key_equal(), alloc) {} linked_hash_set(const linked_hash_set& other) @@ -448,7 +450,7 @@ node_type extract(const_iterator position) { set_.erase(position); - ListType extracted_node_list; + ListType extracted_node_list(get_allocator()); extracted_node_list.splice(extracted_node_list.end(), list_, position); return node_type(std::move(extracted_node_list)); } @@ -458,7 +460,7 @@ node_type extract(const key_arg<K>& key) { auto node = set_.extract(key); if (node.empty()) return node_type(); - ListType extracted_node_list; + ListType extracted_node_list(get_allocator()); extracted_node_list.splice(extracted_node_list.end(), list_, node.value()); return node_type(std::move(extracted_node_list)); } @@ -497,7 +499,7 @@ template <typename... Args> std::pair<iterator, bool> EmplaceInternal(const_iterator hint, Args&&... args) { - ListType node_donor; + ListType node_donor(get_allocator()); auto list_iter = node_donor.emplace(node_donor.end(), std::forward<Args>(args)...); auto ins = set_.insert(list_iter);
diff --git a/absl/container/linked_hash_set_test.cc b/absl/container/linked_hash_set_test.cc index 9a3af62..340d6e3 100644 --- a/absl/container/linked_hash_set_test.cc +++ b/absl/container/linked_hash_set_test.cc
@@ -17,6 +17,8 @@ #include <algorithm> #include <cmath> #include <cstddef> +#include <cstdint> +#include <functional> #include <memory> #include <string> #include <utility> @@ -28,6 +30,7 @@ #include "absl/container/internal/hash_generator_testing.h" #include "absl/container/internal/hash_policy_testing.h" #include "absl/container/internal/heterogeneous_lookup_testing.h" +#include "absl/container/internal/test_allocator.h" #include "absl/container/internal/test_instance_tracker.h" #include "absl/container/internal/unordered_set_constructor_test.h" #include "absl/container/internal/unordered_set_lookup_test.h" @@ -825,6 +828,29 @@ EXPECT_FALSE(s.contains(9)); } +// Verify that emplacing and extracting nodes results in the same stateful +// allocator being used for splicing purposes, rather than another instance +// (say, a default-constructed one), which could otherwise silently corrupt +// memory. +TEST(LinkedHashSet, ExtractAndEmplaceUseSameStatefulAllocator) { + using Alloc = absl::container_internal::CountingAllocator<int>; + int64_t bytes_used = 0; + Alloc alloc(&bytes_used); + linked_hash_set<int, linked_hash_set<int>::hasher, std::equal_to<>, Alloc> + set(alloc); + + set.emplace(1); + EXPECT_GT(bytes_used, 0) << "emplace() failed to use the same allocator"; + + auto node = set.extract(set.begin()); + EXPECT_EQ(node.get_allocator(), alloc) + << "extract(iter) failed to use the same allocator"; + + set.insert(std::move(node)); + EXPECT_EQ(set.extract(1).get_allocator(), alloc) + << "extract(key) failed to use the same allocator"; +} + TEST(LinkedHashSet, Merge) { linked_hash_set<int> m = {1, 7, 3, 6, 10}; linked_hash_set<int> src = {1, 2, 9, 10, 4, 16};
diff --git a/absl/debugging/BUILD.bazel b/absl/debugging/BUILD.bazel index e6f11e8..556ab9c 100644 --- a/absl/debugging/BUILD.bazel +++ b/absl/debugging/BUILD.bazel
@@ -25,7 +25,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -53,6 +53,7 @@ hdrs = ["stacktrace.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":debugging_internal", "//absl/base:config", @@ -103,6 +104,7 @@ ], "//conditions:default": [], }), + visibility = ["//visibility:public"], deps = [ ":debugging_internal", ":demangle_internal", @@ -157,8 +159,8 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/log/internal:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop:__subpackages__", ], deps = [ ":stacktrace", @@ -175,6 +177,7 @@ hdrs = ["failure_signal_handler.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":examine_stack", ":stacktrace", @@ -221,7 +224,9 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - visibility = ["//absl:friends"], + visibility = [ + "@do_not_use_for_gloop_visibility_only//gloop:__subpackages__", + ], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -238,9 +243,9 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/container:__pkg__", "//absl/debugging:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/base:__pkg__", ], deps = [ ":demangle_rust", @@ -326,6 +331,9 @@ hdrs = ["internal/demangle_rust.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "@do_not_use_for_gloop_visibility_only//gloop/util/symbolize:__pkg__", + ], deps = [ ":decode_rust_punycode", "//absl/base:config", @@ -375,6 +383,7 @@ hdrs = ["leak_check.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -421,7 +430,6 @@ hdrs = ["internal/stack_consumption.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - visibility = ["//visibility:private"], deps = [ "//absl/base:config", "//absl/base:core_headers",
diff --git a/absl/debugging/failure_signal_handler.cc b/absl/debugging/failure_signal_handler.cc index 16609f1..46cb97f 100644 --- a/absl/debugging/failure_signal_handler.cc +++ b/absl/debugging/failure_signal_handler.cc
@@ -383,10 +383,16 @@ // a bit for it to finish. If the other thread doesn't kill us, // we do so after sleeping. PortableSleepForSeconds(3); - RaiseToDefaultHandler(signo); - // The recursively raised signal may be blocked until we return. - return; + } else { + // Same thread re-entered: the handler itself faulted. Do NOT fall through + // and re-run the body (which would recurse under SA_NODEFER, resetting + // the alarm each time and consuming the unguarded altstack). Instead, + // terminate now. } + + RaiseToDefaultHandler(signo); + // The recursively raised signal may be blocked until we return. + return; } // Increase the chance that the CPU we report was the same CPU on which the
diff --git a/absl/debugging/internal/demangle.cc b/absl/debugging/internal/demangle.cc index a8d7511..5b2d623 100644 --- a/absl/debugging/internal/demangle.cc +++ b/absl/debugging/internal/demangle.cc
@@ -17,6 +17,7 @@ #include "absl/debugging/internal/demangle.h" +#include <algorithm> #include <cstddef> #include <cstdint> #include <cstdio> @@ -456,22 +457,35 @@ } // Append "str" at "out_cur_idx". If there is an overflow, out_cur_idx is -// set to out_end_idx+1. The output string is ensured to -// always terminate with '\0' as long as there is no overflow. +// set to out_end_idx+1. The output buffer is always terminated with '\0' if it +// has nonzero length. static void Append(State *state, const char *const str, const size_t length) { - for (size_t i = 0; i < length; ++i) { - if (state->parse_state.out_cur_idx + 1 < - state->out_end_idx) { // +1 for '\0' - state->out[state->parse_state.out_cur_idx++] = str[i]; - } else { - // signal overflow - state->parse_state.out_cur_idx = state->out_end_idx + 1; - break; - } + if (length == 0) { + return; } - if (state->parse_state.out_cur_idx < state->out_end_idx) { - state->out[state->parse_state.out_cur_idx] = - '\0'; // Terminate it with '\0' + + // Figure out how much space is remaining in the output buffer to copy into. + const int cap = state->out_end_idx - state->parse_state.out_cur_idx; + + // If overflow was already signaled (negative value, set further below) or + // there is zero space to write into, we cannot do anything. + if (cap <= 0) { + return; + } + + // Copy the number of characters requested, capped by the amount of space + // remaining. + std::char_traits<char>::copy(state->out + state->parse_state.out_cur_idx, str, + (std::min)(length, static_cast<size_t>(cap))); + + // Did we copy everything we needed to, with enough room to NUL-terminate? + if (length < static_cast<size_t>(cap)) { + state->parse_state.out_cur_idx += static_cast<int>(length); + state->out[state->parse_state.out_cur_idx] = '\0'; + } else { + // No, we ran out of space. Signal overflow, and NUL-terminate for safety. + state->parse_state.out_cur_idx = state->out_end_idx + 1; + state->out[state->out_end_idx - 1] = '\0'; } } @@ -893,8 +907,11 @@ ComplexityGuard guard(state); if (guard.IsTooComplex()) return false; - while (ParseOneCharToken(state, 'B')) { - ParseState copy = state->parse_state; + for (;;) { + const ParseState copy = state->parse_state; + if (!ParseOneCharToken(state, 'B')) { + break; + } MaybeAppend(state, "[abi:"); if (!ParseSourceName(state)) {
diff --git a/absl/debugging/internal/demangle_rust.cc b/absl/debugging/internal/demangle_rust.cc index f7f6713..75c46ec 100644 --- a/absl/debugging/internal/demangle_rust.cc +++ b/absl/debugging/internal/demangle_rust.cc
@@ -653,7 +653,15 @@ // A nonempty digit sequence denotes its base-62 value plus 1. int encoded_number = 0; bool overflowed = false; - while (IsAlpha(Peek()) || IsDigit(Peek())) { + for (int scanned = 0; IsAlpha(Peek()) || IsDigit(Peek()); ++scanned) { + // Cap the scan length: a u64 fits in 11 base-62 digits, and int overflows + // after ~5, so anything beyond ~16 digits is already failing to parse. + if (scanned >= 16) { + // Reject pathologically long runs so a backref cannot re-scan + // arbitrarily long stretches of input per iteration. + return false; + } + const char c = Take(); if (encoded_number >= std::numeric_limits<int>::max()/62) { // If we are close to overflowing an int, keep parsing but stop updating
diff --git a/absl/debugging/internal/demangle_rust_test.cc b/absl/debugging/internal/demangle_rust_test.cc index 8ceb1fd..110700e 100644 --- a/absl/debugging/internal/demangle_rust_test.cc +++ b/absl/debugging/internal/demangle_rust_test.cc
@@ -176,6 +176,20 @@ "crate_name::func_name::{closure#?}"); } +TEST(DemangleRust, Base62NumberScanLimit) { + // Up to 16 base-62 digits is permitted (though int overflow yields "?"). + EXPECT_DEMANGLING( + "_RNCNvCs09azAZ_10crate_name9func_names0123456789abcdef_0Cs123_12client_" + "crate", + "crate_name::func_name::{closure#?}"); + + // Beyond 16 base-62 digits is rejected to prevent excessive re-scanning. + EXPECT_DEMANGLING_FAILS( + "_RNCNvCs09azAZ_10crate_name9func_names0123456789abcdef0_0Cs123_12client_" + "crate"); + EXPECT_DEMANGLING_FAILS("_RB0123456789abcdef0_"); +} + TEST(DemangleRust, UnexpectedlyNamedClosure) { EXPECT_DEMANGLING( "_RNCNvCs123_10crate_name9func_name12closure_nameCs456_12client_crate",
diff --git a/absl/debugging/internal/demangle_test.cc b/absl/debugging/internal/demangle_test.cc index 9af2583..7238fd0 100644 --- a/absl/debugging/internal/demangle_test.cc +++ b/absl/debugging/internal/demangle_test.cc
@@ -14,6 +14,7 @@ #include "absl/debugging/internal/demangle.h" +#include <array> #include <cstdlib> #include <memory> #include <string> @@ -30,6 +31,7 @@ namespace debugging_internal { namespace { +using ::testing::Contains; using ::testing::ContainsRegex; TEST(Demangle, FunctionTemplate) { @@ -1908,6 +1910,13 @@ EXPECT_STREQ("my_crate::my_func", tmp); } +TEST(Demangle, DemanglingNulTerminatesOnParsingFailure) { + std::array buf = {'\xAA', '\xAA', '\xAA', '\xAA'}; + EXPECT_FALSE(Demangle("_ZN1xBE", std::data(buf), std::size(buf))); + // Ensure string is properly NUL-terminated despite parsing failure. + EXPECT_THAT(buf, Contains('\0')); +} + // Tests that verify that Demangle footprint is within some limit. // They are not to be run under sanitizers as the sanitizers increase // stack consumption by about 4x.
diff --git a/absl/debugging/internal/examine_stack.cc b/absl/debugging/internal/examine_stack.cc index 9a47f19..a871b04 100644 --- a/absl/debugging/internal/examine_stack.cc +++ b/absl/debugging/internal/examine_stack.cc
@@ -143,26 +143,12 @@ writer(buf, writer_arg); } -void DebugStackTraceHookLegacyAdapter(void* const stack[], int depth, - OutputWriter* writer, void* writer_arg) { - debug_stack_trace_hook(stack, depth, /*crash_pc=*/nullptr, writer, - writer_arg); -} - } // namespace void RegisterDebugStackTraceHook(SymbolizeUrlEmitter hook) { debug_stack_trace_hook = hook; } -SymbolizeUrlEmitterLegacy GetDebugStackTraceHookLegacy() { - if (debug_stack_trace_hook == nullptr) { - // No prior call to RegisterDebugStackTraceHook. - return nullptr; - } - return &DebugStackTraceHookLegacyAdapter; -} - SymbolizeUrlEmitter GetDebugStackTraceHook() { return debug_stack_trace_hook; } // Returns the program counter from signal context, nullptr if
diff --git a/absl/debugging/internal/examine_stack.h b/absl/debugging/internal/examine_stack.h index eca430f..5fa897d 100644 --- a/absl/debugging/internal/examine_stack.h +++ b/absl/debugging/internal/examine_stack.h
@@ -33,17 +33,12 @@ typedef void (*SymbolizeUrlEmitter)(void* const stack[], int depth, const void* crash_pc, OutputWriter* writer, void* writer_arg); -typedef void (*SymbolizeUrlEmitterLegacy)(void* const stack[], int depth, - OutputWriter* writer, - void* writer_arg); // Registration of SymbolizeUrlEmitter for use inside of a signal handler. // This is inherently unsafe and must be signal safe code. void RegisterDebugStackTraceHook(SymbolizeUrlEmitter hook); SymbolizeUrlEmitter GetDebugStackTraceHook(); -SymbolizeUrlEmitterLegacy GetDebugStackTraceHookLegacy(); - // Returns the program counter from signal context, or nullptr if // unknown. `vuc` is a ucontext_t*. We use void* to avoid the use of // ucontext_t on non-POSIX systems.
diff --git a/absl/debugging/internal/stacktrace_aarch64-inl.inc b/absl/debugging/internal/stacktrace_aarch64-inl.inc index 03f2294..0010644 100644 --- a/absl/debugging/internal/stacktrace_aarch64-inl.inc +++ b/absl/debugging/internal/stacktrace_aarch64-inl.inc
@@ -12,6 +12,7 @@ #include <atomic> #include <cassert> +#include <cstddef> #include <cstdint> #include <iostream> #include <limits> @@ -172,14 +173,18 @@ } } } - // New frame pointer is valid if it is inside either known stack or readable. - // This assumes that everything within either known stack is readable. Outside - // either known stack but readable is unexpected, and possibly corrupt, but - // for now assume it is valid. If it isn't actually valid, the next frame will - // be corrupt and we will detect that next iteration. - if (new_inside_signal_stack || - (new_fp_comparable >= stack_info->stack_low && - new_fp_comparable < stack_info->stack_high) || + // Verify that the candidate frame pointer lies inside a positively discovered + // and narrowed thread stack boundary. When stack boundaries are unknown (e.g. + // in open-source builds or outside Linux threads), stack_low defaults to + // getpagesize() and stack_high defaults to kUnknownStackEnd. Requiring both + // stack_high < kUnknownStackEnd and getpagesize() < stack_low ensures we only + // short-circuit AddressIsReadable() when thread stack bounds are known. + const bool new_inside_known_thread_stack = + stack_info->stack_high < kUnknownStackEnd && + static_cast<size_t>(getpagesize()) < stack_info->stack_low && + new_fp_comparable >= stack_info->stack_low && + new_fp_comparable < stack_info->stack_high; + if (new_inside_signal_stack || new_inside_known_thread_stack || absl::debugging_internal::AddressIsReadable(new_frame_pointer)) { return new_frame_pointer; }
diff --git a/absl/debugging/internal/stacktrace_powerpc-inl.inc b/absl/debugging/internal/stacktrace_powerpc-inl.inc index ade4edf..f446655 100644 --- a/absl/debugging/internal/stacktrace_powerpc-inl.inc +++ b/absl/debugging/internal/stacktrace_powerpc-inl.inc
@@ -92,6 +92,13 @@ } if ((uintptr_t)new_sp % kStackAlignment != 0) return nullptr; + // Verify readability before using new_sp. + if (!STRICT_UNWINDING && new_sp != nullptr && + !absl::debugging_internal::AddressIsReadable( + StacktracePowerPCGetLRPtr(new_sp))) { + return nullptr; + } + #if defined(__linux__) enum StackTraceKernelSymbolStatus { kNotInitialized = 0, kAddressValid, kAddressInvalid };
diff --git a/absl/debugging/internal/symbolize.h b/absl/debugging/internal/symbolize.h index a994dba..3360af5 100644 --- a/absl/debugging/internal/symbolize.h +++ b/absl/debugging/internal/symbolize.h
@@ -155,9 +155,8 @@ #ifdef __cplusplus extern "C" #endif // __cplusplus - - bool - AbslInternalGetFileMappingHint(const void** start, const void** end, - uint64_t* offset, const char** filename); + bool ABSL_INTERNAL_C_SYMBOL(AbslInternalGetFileMappingHint)( + const void** start, const void** end, uint64_t* offset, + const char** filename); #endif // ABSL_DEBUGGING_INTERNAL_SYMBOLIZE_H_
diff --git a/absl/debugging/stacktrace.cc b/absl/debugging/stacktrace.cc index ad55eef..9689337 100644 --- a/absl/debugging/stacktrace.cc +++ b/absl/debugging/stacktrace.cc
@@ -69,6 +69,7 @@ ABSL_NAMESPACE_BEGIN namespace { + typedef int (*Unwinder)(void**, int*, int, int, const void*, int*); std::atomic<Unwinder> custom;
diff --git a/absl/debugging/stacktrace.h b/absl/debugging/stacktrace.h index 8e5002a..7e0bd7b 100644 --- a/absl/debugging/stacktrace.h +++ b/absl/debugging/stacktrace.h
@@ -251,6 +251,7 @@ // information is assumed to be absent/unavailable. extern void FixUpStack(void** pcs, uintptr_t* frames, int* sizes, size_t capacity, size_t& depth); + } // namespace internal_stacktrace ABSL_NAMESPACE_END
diff --git a/absl/debugging/stacktrace_test.cc b/absl/debugging/stacktrace_test.cc index a1108b3..17552b4 100644 --- a/absl/debugging/stacktrace_test.cc +++ b/absl/debugging/stacktrace_test.cc
@@ -388,4 +388,41 @@ } +#if defined(__aarch64__) && defined(__linux__) +static void CorruptedSigStackHandler(int, siginfo_t*, void*) { + void** fp = reinterpret_cast<void**>(__builtin_frame_address(0)); + void* saved_fp = fp[0]; + fp[0] = reinterpret_cast<void*>(0x7deadbeef000ULL); // Unmapped address + + void* stack[16]; + absl::GetStackTrace(stack, 16, 0); + + fp[0] = saved_fp; +} +#endif + +TEST(StackTrace, CorruptedSignalStackFrameSafety) { +#if defined(__aarch64__) && defined(__linux__) + stack_t sigstk{}; + constexpr size_t kAltstackSize = 1 << 14; + char altstack[kAltstackSize]; + sigstk.ss_sp = altstack; + sigstk.ss_size = kAltstackSize; + sigstk.ss_flags = 0; + ASSERT_EQ(sigaltstack(&sigstk, nullptr), 0); + + struct sigaction act{}, oldact{}; + act.sa_sigaction = CorruptedSigStackHandler; + act.sa_flags = SA_SIGINFO | SA_ONSTACK; + ASSERT_EQ(sigaction(SIGUSR1, &act, &oldact), 0); + + raise(SIGUSR1); + + sigaction(SIGUSR1, &oldact, nullptr); + stack_t disable_stk{}; + disable_stk.ss_flags = SS_DISABLE; + sigaltstack(&disable_stk, nullptr); +#endif +} + } // namespace
diff --git a/absl/debugging/symbolize_darwin.inc b/absl/debugging/symbolize_darwin.inc index cf63d19..d214481 100644 --- a/absl/debugging/symbolize_darwin.inc +++ b/absl/debugging/symbolize_darwin.inc
@@ -77,11 +77,8 @@ char tmp_buf[1024]; if (debugging_internal::Demangle(symbol.c_str(), tmp_buf, sizeof(tmp_buf))) { - size_t len = strlen(tmp_buf); - if (len + 1 <= static_cast<size_t>(out_size)) { // +1 for '\0' - assert(len < sizeof(tmp_buf)); - memmove(out, tmp_buf, len + 1); - } + strncpy(out, tmp_buf, + std::min(static_cast<size_t>(out_size), std::size(tmp_buf))); } else { strncpy(out, symbol.c_str(), static_cast<size_t>(out_size)); }
diff --git a/absl/debugging/symbolize_elf.inc b/absl/debugging/symbolize_elf.inc index 14b23c1..56f31af 100644 --- a/absl/debugging/symbolize_elf.inc +++ b/absl/debugging/symbolize_elf.inc
@@ -800,7 +800,7 @@ std::min(num_remaining_symbols, buf_entries); const size_t bytes_in_chunk = entries_in_chunk * sizeof(buf[0]); const ssize_t len = file->ReadFromOffset(buf, bytes_in_chunk, offset); - SAFE_ASSERT(len >= 0); + SAFE_ASSERT(len > 0); SAFE_ASSERT(static_cast<size_t>(len) % sizeof(buf[0]) == 0); const size_t num_symbols_in_buf = static_cast<size_t>(len) / sizeof(buf[0]); SAFE_ASSERT(num_symbols_in_buf <= entries_in_chunk); @@ -1212,7 +1212,8 @@ ObjFile *obj = addr_map_.At(lo); SAFE_ASSERT(obj->end_addr > addr); if (addr >= obj->start_addr && - reinterpret_cast<const char *>(addr) + len <= obj->end_addr) + len <= static_cast<size_t>(static_cast<const char*>(obj->end_addr) - + static_cast<const char*>(addr))) return obj; } @@ -1697,10 +1698,9 @@ ABSL_NAMESPACE_END } // namespace absl -extern "C" bool AbslInternalGetFileMappingHint(const void **start, - const void **end, - uint64_t *offset, - const char **filename) { +extern "C" bool ABSL_INTERNAL_C_SYMBOL(AbslInternalGetFileMappingHint)( + const void** start, const void** end, uint64_t* offset, + const char** filename) { return absl::debugging_internal::GetFileMappingHint(start, end, offset, filename); }
diff --git a/absl/flags/BUILD.bazel b/absl/flags/BUILD.bazel index b5ecb7f..4dd3b33 100644 --- a/absl/flags/BUILD.bazel +++ b/absl/flags/BUILD.bazel
@@ -25,7 +25,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -43,8 +43,8 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/flags:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/base:__pkg__", ], deps = [ "//absl/base:config", @@ -63,9 +63,9 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/flags:__pkg__", "//absl/log:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/base:__pkg__", ], deps = [ ":path_util", @@ -88,6 +88,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":path_util", ":program_name", @@ -109,6 +110,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -131,7 +133,7 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/base:__pkg__", ], deps = [ "//absl/base:config", @@ -149,6 +151,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":commandlineflag_internal", "//absl/base:config", @@ -170,9 +173,9 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/flags:__pkg__", "//absl/flags/rust:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/base:__pkg__", ], deps = [ ":commandlineflag", @@ -193,6 +196,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":commandlineflag", ":commandlineflag_internal", @@ -248,6 +252,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":commandlineflag", ":config", @@ -271,8 +276,8 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/flags:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/base:__pkg__", ], deps = [ ":commandlineflag", @@ -301,6 +306,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":usage_internal", "//absl/base:config", @@ -320,6 +326,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":commandlineflag", ":commandlineflag_internal",
diff --git a/absl/functional/BUILD.bazel b/absl/functional/BUILD.bazel index d61eef6..c41b211 100644 --- a/absl/functional/BUILD.bazel +++ b/absl/functional/BUILD.bazel
@@ -25,7 +25,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -41,6 +41,7 @@ hdrs = ["any_invocable.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -56,6 +57,8 @@ "any_invocable_test.h", "any_invocable_test_inst1.cc", "any_invocable_test_inst2.cc", + "any_invocable_test_inst3.cc", + "any_invocable_test_inst4.cc", "internal/any_invocable.h", ], copts = ABSL_TEST_COPTS, @@ -78,6 +81,7 @@ hdrs = ["bind_back.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/container:compressed_tuple", @@ -104,6 +108,7 @@ hdrs = ["bind_front.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/container:compressed_tuple", "//absl/meta:type_traits", @@ -130,6 +135,7 @@ hdrs = ["function_ref.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":any_invocable", "//absl/base:config", @@ -161,6 +167,7 @@ hdrs = ["overload.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/meta:type_traits",
diff --git a/absl/functional/CMakeLists.txt b/absl/functional/CMakeLists.txt index 03a7596..7780174 100644 --- a/absl/functional/CMakeLists.txt +++ b/absl/functional/CMakeLists.txt
@@ -39,6 +39,8 @@ "any_invocable_test.h" "any_invocable_test_inst1.cc" "any_invocable_test_inst2.cc" + "any_invocable_test_inst3.cc" + "any_invocable_test_inst4.cc" COPTS ${ABSL_TEST_COPTS} DEPS
diff --git a/absl/functional/any_invocable_test.h b/absl/functional/any_invocable_test.h index 868ac88..689c510 100644 --- a/absl/functional/any_invocable_test.h +++ b/absl/functional/any_invocable_test.h
@@ -14,8 +14,9 @@ // To prevent compiler memory exhaustion (OOM / Killed signal terminates // cc1plus) during parallel builds with GCC, the test suite instantiations have -// been split into two separate compilation units: any_invocable_test_inst1.cc -// and any_invocable_test_inst2.cc. The test definitions remain here in this +// been split into four separate compilation units: any_invocable_test_inst1.cc, +// any_invocable_test_inst2.cc, any_invocable_test_inst3.cc, and +// any_invocable_test_inst4.cc. The test definitions remain here in this // header. // SKIP_ABSL_INLINE_NAMESPACE_CHECK
diff --git a/absl/functional/any_invocable_test_inst1.cc b/absl/functional/any_invocable_test_inst1.cc index a15722a..68a5955 100644 --- a/absl/functional/any_invocable_test_inst1.cc +++ b/absl/functional/any_invocable_test_inst1.cc
@@ -14,8 +14,7 @@ // To prevent compiler memory exhaustion (OOM / Killed signal terminates // cc1plus) during parallel builds with GCC, the test suite instantiations have -// been split into two separate compilation units: any_invocable_test_inst1.cc -// and any_invocable_test_inst2.cc. +// been split into multiple compilation units. // SKIP_ABSL_INLINE_NAMESPACE_CHECK @@ -30,17 +29,6 @@ INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallMayThrow, AnyInvTestBasic, TestParameterListRvalueQualifiersCallMayThrow); -INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestBasic, - TestParameterListRemoteNonMovable); - -INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestBasic, TestParameterListLocal); - -INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestBasic, - TestParameterListNonRvalueQualifiersNothrowCall); - -INSTANTIATE_TYPED_TEST_SUITE_P(CallNothrowRvalue, AnyInvTestBasic, - TestParameterListRvalueQualifiersNothrowCall); - INSTANTIATE_TYPED_TEST_SUITE_P( NonRvalueCallMayThrow, AnyInvTestCombinatoric, TestParameterListNonRvalueQualifiersCallMayThrow); @@ -48,18 +36,6 @@ INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallMayThrow, AnyInvTestCombinatoric, TestParameterListRvalueQualifiersCallMayThrow); -INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestCombinatoric, - TestParameterListRemoteNonMovable); - -INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestCombinatoric, - TestParameterListLocal); - -INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestCombinatoric, - TestParameterListNonRvalueQualifiersNothrowCall); - -INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallNothrow, AnyInvTestCombinatoric, - TestParameterListRvalueQualifiersNothrowCall); - INSTANTIATE_TYPED_TEST_SUITE_P( NonRvalueCallMayThrow, AnyInvTestMovable, TestParameterListNonRvalueQualifiersCallMayThrow); @@ -67,15 +43,6 @@ INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallMayThrow, AnyInvTestMovable, TestParameterListRvalueQualifiersCallMayThrow); -INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestMovable, - TestParameterListLocal); - -INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestMovable, - TestParameterListNonRvalueQualifiersNothrowCall); - -INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallNothrow, AnyInvTestMovable, - TestParameterListRvalueQualifiersNothrowCall); - INSTANTIATE_TYPED_TEST_SUITE_P( NonRvalueCallMayThrow, AnyInvTestNoexceptFalse, TestParameterListNonRvalueQualifiersCallMayThrow); @@ -83,35 +50,11 @@ INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallMayThrow, AnyInvTestNoexceptFalse, TestParameterListRvalueQualifiersCallMayThrow); -INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestNoexceptFalse, - TestParameterListRemoteNonMovable); - -INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestNoexceptFalse, - TestParameterListLocal); - -INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestNoexceptTrue, - TestParameterListNonRvalueQualifiersNothrowCall); - -INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallNothrow, AnyInvTestNoexceptTrue, - TestParameterListRvalueQualifiersNothrowCall); - INSTANTIATE_TYPED_TEST_SUITE_P( NonRvalueCallMayThrow, AnyInvTestNonRvalue, TestParameterListNonRvalueQualifiersCallMayThrow); -INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestNonRvalue, - TestParameterListRemoteNonMovable); - -INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestNonRvalue, - TestParameterListLocal); - -INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestNonRvalue, - TestParameterListNonRvalueQualifiersNothrowCall); - INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallMayThrow, AnyInvTestRvalue, TestParameterListRvalueQualifiersCallMayThrow); -INSTANTIATE_TYPED_TEST_SUITE_P(CallNothrowRvalue, AnyInvTestRvalue, - TestParameterListRvalueQualifiersNothrowCall); - } // namespace absl_any_invocable_test
diff --git a/absl/functional/any_invocable_test_inst2.cc b/absl/functional/any_invocable_test_inst2.cc index 366e1d2..5b5f22a 100644 --- a/absl/functional/any_invocable_test_inst2.cc +++ b/absl/functional/any_invocable_test_inst2.cc
@@ -14,8 +14,7 @@ // To prevent compiler memory exhaustion (OOM / Killed signal terminates // cc1plus) during parallel builds with GCC, the test suite instantiations have -// been split into two separate compilation units: any_invocable_test_inst1.cc -// and any_invocable_test_inst2.cc. +// been split into multiple compilation units. // SKIP_ABSL_INLINE_NAMESPACE_CHECK
diff --git a/absl/functional/any_invocable_test_inst3.cc b/absl/functional/any_invocable_test_inst3.cc new file mode 100644 index 0000000..ba23e63 --- /dev/null +++ b/absl/functional/any_invocable_test_inst3.cc
@@ -0,0 +1,55 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// To prevent compiler memory exhaustion (OOM / Killed signal terminates +// cc1plus) during parallel builds with GCC, the test suite instantiations have +// been split into multiple compilation units. + +// SKIP_ABSL_INLINE_NAMESPACE_CHECK + +#include "absl/functional/any_invocable_test.h" + +namespace absl_any_invocable_test { + +INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestBasic, + TestParameterListNonRvalueQualifiersNothrowCall); + +INSTANTIATE_TYPED_TEST_SUITE_P(CallNothrowRvalue, AnyInvTestBasic, + TestParameterListRvalueQualifiersNothrowCall); + +INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestCombinatoric, + TestParameterListNonRvalueQualifiersNothrowCall); + +INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallNothrow, AnyInvTestCombinatoric, + TestParameterListRvalueQualifiersNothrowCall); + +INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestMovable, + TestParameterListNonRvalueQualifiersNothrowCall); + +INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallNothrow, AnyInvTestMovable, + TestParameterListRvalueQualifiersNothrowCall); + +INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestNoexceptTrue, + TestParameterListNonRvalueQualifiersNothrowCall); + +INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallNothrow, AnyInvTestNoexceptTrue, + TestParameterListRvalueQualifiersNothrowCall); + +INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestNonRvalue, + TestParameterListNonRvalueQualifiersNothrowCall); + +INSTANTIATE_TYPED_TEST_SUITE_P(CallNothrowRvalue, AnyInvTestRvalue, + TestParameterListRvalueQualifiersNothrowCall); + +} // namespace absl_any_invocable_test
diff --git a/absl/functional/any_invocable_test_inst4.cc b/absl/functional/any_invocable_test_inst4.cc new file mode 100644 index 0000000..0549a1e --- /dev/null +++ b/absl/functional/any_invocable_test_inst4.cc
@@ -0,0 +1,51 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// To prevent compiler memory exhaustion (OOM / Killed signal terminates +// cc1plus) during parallel builds with GCC, the test suite instantiations have +// been split into multiple compilation units. + +// SKIP_ABSL_INLINE_NAMESPACE_CHECK + +#include "absl/functional/any_invocable_test.h" + +namespace absl_any_invocable_test { + +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestBasic, + TestParameterListRemoteNonMovable); + +INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestBasic, TestParameterListLocal); + +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestCombinatoric, + TestParameterListRemoteNonMovable); + +INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestCombinatoric, + TestParameterListLocal); + +INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestMovable, + TestParameterListLocal); + +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestNoexceptFalse, + TestParameterListRemoteNonMovable); + +INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestNoexceptFalse, + TestParameterListLocal); + +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestNonRvalue, + TestParameterListRemoteNonMovable); + +INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestNonRvalue, + TestParameterListLocal); + +} // namespace absl_any_invocable_test
diff --git a/absl/hash/BUILD.bazel b/absl/hash/BUILD.bazel index a2d3a02..9f12e9a 100644 --- a/absl/hash/BUILD.bazel +++ b/absl/hash/BUILD.bazel
@@ -25,7 +25,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -44,6 +44,7 @@ hdrs = ["hash.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":city", ":weakly_mixed_integer", @@ -68,6 +69,7 @@ testonly = True, hdrs = ["hash_testing.h"], linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":spy_hash_state", "//absl/meta:type_traits", @@ -168,6 +170,9 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/hash:__pkg__", + ], deps = [ "//absl/base:config", "//absl/base:core_headers",
diff --git a/absl/hash/hash.h b/absl/hash/hash.h index fc791ab..7a76771 100644 --- a/absl/hash/hash.h +++ b/absl/hash/hash.h
@@ -117,11 +117,11 @@ // * All string-like types including: // * absl::Cord // * std::string (as well as any instance of std::basic_string that -// uses one of {char, wchar_t, char16_t, char32_t} and its associated -// std::char_traits) +// uses one of {char, wchar_t, char8_t, char16_t, char32_t} and its +// associated std::char_traits) // * std::string_view (as well as any instance of std::basic_string_view -// that uses one of {char, wchar_t, char16_t, char32_t} and its associated -// std::char_traits) +// that uses one of {char, wchar_t, char8_t, char16_t, char32_t} and its +// associated std::char_traits) // * All the standard sequence containers (provided the elements are hashable) // * All the standard associative containers (provided the elements are // hashable)
diff --git a/absl/hash/hash_benchmark.cc b/absl/hash/hash_benchmark.cc index d0ebdbc..86650b3 100644 --- a/absl/hash/hash_benchmark.cc +++ b/absl/hash/hash_benchmark.cc
@@ -13,6 +13,7 @@ // limitations under the License. #include <algorithm> +#include <bitset> #include <cassert> #include <cstddef> #include <cstdint> @@ -247,6 +248,12 @@ MAKE_BENCHMARK(AbslHash, VectorDouble_10, Vector<double>(10)); MAKE_BENCHMARK(AbslHash, VectorDouble_100, Vector<double>(100)); MAKE_BENCHMARK(AbslHash, VectorDouble_1000, Vector<double>(1000)); +MAKE_BENCHMARK(AbslHash, VectorBool_10, Vector<bool>(10)); +MAKE_BENCHMARK(AbslHash, VectorBool_100, Vector<bool>(100)); +MAKE_BENCHMARK(AbslHash, VectorBool_1000, Vector<bool>(1000)); +MAKE_BENCHMARK(AbslHash, Bitset_10, std::bitset<10>()); +MAKE_BENCHMARK(AbslHash, Bitset_100, std::bitset<100>()); +MAKE_BENCHMARK(AbslHash, Bitset_1000, std::bitset<1000>()); MAKE_BENCHMARK(AbslHash, FlatHashSetInt64_10, FlatHashSet<int64_t>(10)); MAKE_BENCHMARK(AbslHash, FlatHashSetInt64_100, FlatHashSet<int64_t>(100)); MAKE_BENCHMARK(AbslHash, FlatHashSetInt64_1000, FlatHashSet<int64_t>(1000)); @@ -294,6 +301,14 @@ std::vector<double>(100, 1.1)); MAKE_BENCHMARK(TypeErasedAbslHash, VectorDouble_1000, std::vector<double>(1000, 1.1)); +MAKE_BENCHMARK(TypeErasedAbslHash, VectorBool_10, std::vector<bool>(10, true)); +MAKE_BENCHMARK(TypeErasedAbslHash, VectorBool_100, + std::vector<bool>(100, true)); +MAKE_BENCHMARK(TypeErasedAbslHash, VectorBool_1000, + std::vector<bool>(1000, true)); +MAKE_BENCHMARK(TypeErasedAbslHash, Bitset_10, std::bitset<10>()); +MAKE_BENCHMARK(TypeErasedAbslHash, Bitset_100, std::bitset<100>()); +MAKE_BENCHMARK(TypeErasedAbslHash, Bitset_1000, std::bitset<1000>()); MAKE_BENCHMARK(TypeErasedAbslHash, FlatHashSetInt64_10, FlatHashSet<int64_t>(10)); MAKE_BENCHMARK(TypeErasedAbslHash, FlatHashSetInt64_100,
diff --git a/absl/hash/hash_test.cc b/absl/hash/hash_test.cc index e128b62..63ac785 100644 --- a/absl/hash/hash_test.cc +++ b/absl/hash/hash_test.cc
@@ -192,7 +192,7 @@ constexpr size_t kMask = (1 << (kLog2NumValues + 7)) - 1; size_t stuck_bits = (~bits_or | bits_and) & kMask; int stuck_bit_count = absl::popcount(stuck_bits); - size_t max_stuck_bits = 5; + size_t max_stuck_bits = 8; EXPECT_LE(stuck_bit_count, max_stuck_bits) << "0x" << std::hex << stuck_bits; @@ -484,6 +484,17 @@ std::wstring(L"Iñtërnâtiônàlizætiøn")))); } +#ifdef __cpp_char8_t +TEST(HashValueTest, U8String) { + EXPECT_TRUE((is_hashable<std::u8string>::value)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + std::u8string(), std::u8string(u8"ABC"), std::u8string(u8"ABC"), + std::u8string(u8"Some other different string"), + std::u8string(u8"Iñtërnâtiônàlizætiøn")))); +} +#endif + TEST(HashValueTest, U16String) { EXPECT_TRUE((is_hashable<std::u16string>::value)); @@ -511,6 +522,18 @@ std::wstring_view(L"Iñtërnâtiônàlizætiøn")))); } +#ifdef __cpp_char8_t +TEST(HashValueTest, U8StringView) { + EXPECT_TRUE((is_hashable<std::u8string_view>::value)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(std::u8string_view(), std::u8string_view(u8"ABC"), + std::u8string_view(u8"ABC"), + std::u8string_view(u8"Some other different string_view"), + std::u8string_view(u8"Iñtërnâtiônàlizætiøn")))); +} +#endif + TEST(HashValueTest, U16StringView) { EXPECT_TRUE((is_hashable<std::u16string_view>::value)); @@ -558,6 +581,8 @@ std::filesystem::path("c:\\//"), std::filesystem::path("c://"), std::filesystem::path("c://\\"), + std::filesystem::path("c:/a"), + std::filesystem::path("c:\\a"), std::filesystem::path("/e/p"), std::filesystem::path("/s/../e/p"), std::filesystem::path("e/p"),
diff --git a/absl/hash/internal/hash.h b/absl/hash/internal/hash.h index 972e9bd..f68872f 100644 --- a/absl/hash/internal/hash.h +++ b/absl/hash/internal/hash.h
@@ -645,22 +645,27 @@ // // - `absl::Cord` // - `std::string` (and std::basic_string<T, std::char_traits<T>, A> for -// any allocator A and any T in {char, wchar_t, char16_t, char32_t}) +// any allocator A and any T in {char, wchar_t, char8_t, char16_t, +// char32_t}) // - `absl::string_view`, `std::string_view`, `std::wstring_view`, -// `std::u16string_view`, and `std::u32_string_view`. +// `std::u8string_view`, `std::u16string_view`, and `std::u32_string_view`. // // For simplicity, we currently support only strings built on `char`, `wchar_t`, -// `char16_t`, or `char32_t`. This support may be broadened, if necessary, but -// with some caution - this overload would misbehave in cases where the traits' -// `eq()` member isn't equivalent to `==` on the underlying character type. +// `char8_t`, `char16_t`, or `char32_t`. This support may be broadened, if +// necessary, but with some caution - this overload would misbehave in cases +// where the traits' `eq()` member isn't equivalent to `==` on the underlying +// character type. template <typename H> H AbslHashValue(H hash_state, absl::string_view str) { return H::combine_contiguous(std::move(hash_state), str.data(), str.size()); } -// Support std::wstring, std::u16string and std::u32string. +// Support std::wstring, std::u8string, std::u16string and std::u32string. template <typename Char, typename Alloc, typename H, typename = std::enable_if_t<std::is_same_v<Char, wchar_t> || +#ifdef __cpp_char8_t + std::is_same_v<Char, char8_t> || +#endif std::is_same_v<Char, char16_t> || std::is_same_v<Char, char32_t>>> H AbslHashValue( @@ -669,9 +674,13 @@ return H::combine_contiguous(std::move(hash_state), str.data(), str.size()); } -// Support std::wstring_view, std::u16string_view and std::u32string_view. +// Support std::wstring_view, std::u8string_view, std::u16string_view and +// std::u32string_view. template <typename Char, typename H, typename = std::enable_if_t<std::is_same_v<Char, wchar_t> || +#ifdef __cpp_char8_t + std::is_same_v<Char, char8_t> || +#endif std::is_same_v<Char, char16_t> || std::is_same_v<Char, char32_t>>> H AbslHashValue(H hash_state, std::basic_string_view<Char> str) { @@ -693,12 +702,28 @@ typename = std::enable_if_t< std::is_same_v<Path, std::filesystem::path>>> H AbslHashValue(H hash_state, const Path& path) { - // This is implemented by deferring to the standard library to compute the - // hash. The standard library requires that for two paths, `p1 == p2`, then - // `hash_value(p1) == hash_value(p2)`. `AbslHashValue` has the same - // requirement. Since `operator==` does platform specific matching, deferring - // to the standard library is the simplest approach. - return H::combine(std::move(hash_state), std::filesystem::hash_value(path)); + // Avoid deferring to std::filesystem::hash_value, as that makes it easy to + // generate offline collisions, bypassing per-table and per-process hash + // seeding. Instead, we hash it ourselves. + size_t count = 0; + + for (const Path& component : path) { + std::basic_string_view<typename Path::value_type> part = component.native(); + + // If this is a directory separator, pretend it is the preferred directory + // separator (rather than the alternate separator) to ensure that equal + // paths produce equal hashes. + // Analogous to LLVM commit aa427b1aae445ed46d9f60c5e2eaac61bdf76be3. + if (!part.empty() && + (*part.begin() == '/' || *part.begin() == Path::preferred_separator)) { + part = std::basic_string_view<typename Path::value_type>( + &Path::preferred_separator, 1); + } + + hash_state = H::combine(std::move(hash_state), part); + ++count; + } + return H::combine(std::move(hash_state), count); } #endif // ABSL_INTERNAL_STD_FILESYSTEM_PATH_HASH_AVAILABLE @@ -762,41 +787,50 @@ } // AbslHashValue special cases for hashing std::vector<bool> - -#if defined(ABSL_IS_BIG_ENDIAN) && \ - (defined(__GLIBCXX__) || defined(__GLIBCPP__)) - -// std::hash in libstdc++ does not work correctly with vector<bool> on Big -// Endian platforms therefore we need to implement a custom AbslHashValue for -// it. More details on the bug: -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102531 +// +// To achieve high performance without depending on private standard library +// internals, we pack bits 64 at a time into uint64_t words using a fixed +// 64-step inner loop that allows compilers to unroll bit shifts cleanly. +// +// This is slower than std::hash<std::vector<bool>> which can access private +// storage directly, but more than fast enough for the very rare case of hashing +// std::vector<bool>. In the event that higher performance is needed, a custom +// key type is likely faster than building std::vector<bool> and hashing it, +// otherwise users can just use std::hash as the hasher. template <typename H, typename T, typename Allocator> std::enable_if_t<is_hashable<T>::value && std::is_same_v<T, bool>, H> AbslHashValue(H hash_state, const std::vector<T, Allocator>& vector) { typename H::AbslInternalPiecewiseCombiner combiner; - for (const auto& i : vector) { - unsigned char c = static_cast<unsigned char>(i); - hash_state = combiner.add_buffer(std::move(hash_state), &c, sizeof(c)); + const size_t size = vector.size(); + size_t i = 0; + // Pack full 64-bit words. Fixed inner loop count enables compiler unrolling. + while (i + 64 <= size) { + uint64_t word = 0; + for (size_t j = 0; j < 64; ++j) { + word |= static_cast<uint64_t>(vector[i + j]) << j; + } + hash_state = combiner.add_buffer( + std::move(hash_state), reinterpret_cast<const unsigned char*>(&word), + sizeof(word)); + i += 64; } + // Pack remaining bits (< 64) into the final word. + if (i < size) { + uint64_t word = 0; + const size_t rem = size - i; + for (size_t j = 0; j < rem; ++j) { + word |= static_cast<uint64_t>(vector[i + j]) << j; + } + hash_state = combiner.add_buffer( + std::move(hash_state), reinterpret_cast<const unsigned char*>(&word), + (rem + 7) / 8); + } + // Mix in vector.size() to distinguish vectors with trailing false/zero bits + // (e.g. {true} vs {true, false}) that would otherwise produce identical bit + // buffers. return H::combine(combiner.finalize(std::move(hash_state)), - WeaklyMixedInteger{vector.size()}); + WeaklyMixedInteger{size}); } -#else -// When not working around the libstdc++ bug above, we still have to contend -// with the fact that std::hash<vector<bool>> is often poor quality, hashing -// directly on the internal words and on no other state. On these platforms, -// vector<bool>{1, 1} and vector<bool>{1, 1, 0} hash to the same value. -// -// Mixing in the size (as we do in our other vector<> implementations) on top -// of the library-provided hash implementation avoids this QOI issue. -template <typename H, typename T, typename Allocator> -std::enable_if_t<is_hashable<T>::value && std::is_same_v<T, bool>, H> -AbslHashValue(H hash_state, const std::vector<T, Allocator>& vector) { - return H::combine(std::move(hash_state), - std::hash<std::vector<T, Allocator>>{}(vector), - WeaklyMixedInteger{vector.size()}); -} -#endif // ----------------------------------------------------------------------------- // AbslHashValue for Ordered Associative Containers @@ -934,28 +968,44 @@ // AbslHashValue for Other Types // ----------------------------------------------------------------------------- -// AbslHashValue for hashing std::bitset is not defined on Little Endian -// platforms, for the same reason as for vector<bool> (see std::vector above): -// It does not expose the raw bytes, and a fallback to std::hash<> is most -// likely faster. - -#if defined(ABSL_IS_BIG_ENDIAN) && \ - (defined(__GLIBCXX__) || defined(__GLIBCPP__)) // AbslHashValue for hashing std::bitset // -// std::hash in libstdc++ does not work correctly with std::bitset on Big Endian -// platforms therefore we need to implement a custom AbslHashValue for it. More -// details on the bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102531 +// To achieve high performance without depending on private standard library +// internals, we pack bits 64 at a time into uint64_t words using a fixed +// 64-step inner loop that allows compilers to unroll bit shifts cleanly. +// +// This is slower than std::hash<std::bitset> which can access private storage +// directly, but more than fast enough for the very rare case of hashing +// std::bitset. In the event that higher-performance is needed, users can just +// use std::hash as the hasher. template <typename H, size_t N> H AbslHashValue(H hash_state, const std::bitset<N>& set) { typename H::AbslInternalPiecewiseCombiner combiner; - for (size_t i = 0; i < N; i++) { - unsigned char c = static_cast<unsigned char>(set[i]); - hash_state = combiner.add_buffer(std::move(hash_state), &c, sizeof(c)); + size_t i = 0; + // Pack full 64-bit words. Fixed inner loop count enables compiler unrolling. + while (i + 64 <= N) { + uint64_t word = 0; + for (size_t j = 0; j < 64; ++j) { + word |= static_cast<uint64_t>(set[i + j]) << j; + } + hash_state = combiner.add_buffer( + std::move(hash_state), reinterpret_cast<const unsigned char*>(&word), + sizeof(word)); + i += 64; + } + // Pack remaining bits (< 64) into the final word. + if (i < N) { + uint64_t word = 0; + const size_t rem = N - i; + for (size_t j = 0; j < rem; ++j) { + word |= static_cast<uint64_t>(set[i + j]) << j; + } + hash_state = combiner.add_buffer( + std::move(hash_state), reinterpret_cast<const unsigned char*>(&word), + (rem + 7) / 8); } return H::combine(combiner.finalize(std::move(hash_state)), N); } -#endif // -----------------------------------------------------------------------------
diff --git a/absl/log/BUILD.bazel b/absl/log/BUILD.bazel index b1cff4b..f44b1c0 100644 --- a/absl/log/BUILD.bazel +++ b/absl/log/BUILD.bazel
@@ -24,7 +24,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -41,6 +41,7 @@ hdrs = ["absl_check.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/log/internal:check_impl", ], @@ -51,6 +52,7 @@ hdrs = ["absl_log.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/log/internal:log_impl", ], @@ -61,6 +63,7 @@ hdrs = ["check.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/log/internal:check_impl", "//absl/log/internal:check_op", @@ -76,6 +79,7 @@ hdrs = ["die_if_null.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":log", "//absl/base:config", @@ -148,6 +152,7 @@ hdrs = ["log.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":vlog_is_on", "//absl/log/internal:log_impl", @@ -160,6 +165,7 @@ hdrs = ["log_entry.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -178,6 +184,7 @@ hdrs = ["log_sink.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":log_entry", "//absl/base:config", @@ -189,6 +196,7 @@ hdrs = ["log_sink_registry.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":log_sink", "//absl/base:config", @@ -202,6 +210,7 @@ hdrs = ["log_streamer.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":absl_log", "//absl/base:config", @@ -221,6 +230,7 @@ hdrs = ["scoped_mock_log.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":log_entry", ":log_sink", @@ -238,6 +248,7 @@ hdrs = ["structured.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -251,6 +262,7 @@ hdrs = ["absl_vlog_is_on.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -264,6 +276,7 @@ hdrs = ["vlog_is_on.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":absl_vlog_is_on", ],
diff --git a/absl/log/check_test_impl.inc b/absl/log/check_test_impl.inc index 47af1dd..c78bbd4 100644 --- a/absl/log/check_test_impl.inc +++ b/absl/log/check_test_impl.inc
@@ -727,11 +727,9 @@ TEST(CHECKDeathTest, TestPointerPrintedAsNumberDespiteAbslStringify) { const auto* p = reinterpret_cast<const PointerIsStringifiable*>(0x1234); - EXPECT_DEATH( - ABSL_TEST_CHECK_EQ(p, nullptr), - AnyOf( - HasSubstr("Check failed: p == nullptr (0000000000001234 vs. (null))"), - HasSubstr("Check failed: p == nullptr (0x1234 vs. (null))"))); + EXPECT_DEATH(ABSL_TEST_CHECK_EQ(p, nullptr), + ContainsRegex( + "Check failed: p == nullptr \\(0+x?1234 vs. \\(null\\)\\)")); } // An uncopyable object with operator<<.
diff --git a/absl/log/internal/BUILD.bazel b/absl/log/internal/BUILD.bazel index bc47330..d9cb7b2 100644 --- a/absl/log/internal/BUILD.bazel +++ b/absl/log/internal/BUILD.bazel
@@ -27,6 +27,7 @@ package( default_visibility = [ ":internal_users", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", ], features = [ "header_modules", @@ -40,7 +41,6 @@ package_group( name = "internal_users", includes = [ - "//absl:friends", ], packages = [ "//absl/log", @@ -108,8 +108,8 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/log:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", ], deps = ["//absl/base:config"], ) @@ -121,8 +121,8 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ ":internal_users", - "//absl:friends", "//absl/status:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", ], deps = [ "//absl/flags:flag", @@ -156,8 +156,8 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/log:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", ], deps = [ "//absl/base:config", @@ -189,8 +189,8 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/log:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", ], deps = [ ":append_truncated", @@ -214,7 +214,6 @@ "//absl/log:log_entry", "//absl/log:log_sink", "//absl/log:log_sink_registry", - "//absl/memory", "//absl/strings", "//absl/strings:internal", "//absl/time", @@ -310,6 +309,7 @@ visibility = [ ":internal_users", ":structured_proto_users", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", ], deps = [ ":log_message", @@ -347,7 +347,6 @@ "//absl/base:config", "//absl/strings:string_view", "//absl/types:span", - "//absl/utility", "@googletest//:gtest", "@googletest//:gtest_main", ], @@ -384,6 +383,7 @@ visibility = [ ":internal_users", ":structured_proto_users", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", ], deps = [ ":globals", @@ -405,6 +405,7 @@ visibility = [ ":internal_users", ":structured_proto_users", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", ], deps = [ ":test_helpers", @@ -467,18 +468,18 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/log:__subpackages__", "//absl/status:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/util/status:__subpackages__", ], deps = [ ":fnmatch", "//absl/base", + "//absl/base:base_internal", "//absl/base:config", "//absl/base:core_headers", "//absl/base:no_destructor", "//absl/base:nullability", - "//absl/memory", "//absl/strings", "//absl/synchronization", ], @@ -497,9 +498,7 @@ deps = [ ":vlog_config", "//absl/base:config", - "//absl/base:core_headers", "//absl/container:layout", - "//absl/memory", "//absl/random:distributions", "//absl/strings", "@google_benchmark//:benchmark_main", @@ -566,7 +565,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", ], deps = [ "//absl/base:config",
diff --git a/absl/log/internal/check_op.cc b/absl/log/internal/check_op.cc index be8ceaf..510c576 100644 --- a/absl/log/internal/check_op.cc +++ b/absl/log/internal/check_op.cc
@@ -14,6 +14,7 @@ #include "absl/log/internal/check_op.h" +#include <cstdint> #include <cstring> #include <ostream> #include <string>
diff --git a/absl/log/internal/check_op.h b/absl/log/internal/check_op.h index 9bf908e..66050f0 100644 --- a/absl/log/internal/check_op.h +++ b/absl/log/internal/check_op.h
@@ -530,29 +530,20 @@ // NOLINTBEGIN(runtime/int) // NOLINTBEGIN(google-runtime-int) template <typename T> -inline constexpr const T& GetReferenceableValue(const T& t) { +constexpr const T& GetReferenceableValue(const T& t) { return t; } -inline constexpr char GetReferenceableValue(char t) { return t; } -inline constexpr unsigned char GetReferenceableValue(unsigned char t) { - return t; -} -inline constexpr signed char GetReferenceableValue(signed char t) { return t; } -inline constexpr short GetReferenceableValue(short t) { return t; } -inline constexpr unsigned short GetReferenceableValue(unsigned short t) { - return t; -} -inline constexpr int GetReferenceableValue(int t) { return t; } -inline constexpr unsigned int GetReferenceableValue(unsigned int t) { - return t; -} -inline constexpr long GetReferenceableValue(long t) { return t; } -inline constexpr unsigned long GetReferenceableValue(unsigned long t) { - return t; -} -inline constexpr long long GetReferenceableValue(long long t) { return t; } -inline constexpr unsigned long long GetReferenceableValue( - unsigned long long t) { +constexpr char GetReferenceableValue(char t) { return t; } +constexpr unsigned char GetReferenceableValue(unsigned char t) { return t; } +constexpr signed char GetReferenceableValue(signed char t) { return t; } +constexpr short GetReferenceableValue(short t) { return t; } +constexpr unsigned short GetReferenceableValue(unsigned short t) { return t; } +constexpr int GetReferenceableValue(int t) { return t; } +constexpr unsigned int GetReferenceableValue(unsigned int t) { return t; } +constexpr long GetReferenceableValue(long t) { return t; } +constexpr unsigned long GetReferenceableValue(unsigned long t) { return t; } +constexpr long long GetReferenceableValue(long long t) { return t; } +constexpr unsigned long long GetReferenceableValue(unsigned long long t) { return t; } // NOLINTEND(google-runtime-int)
diff --git a/absl/log/internal/container_test.cc b/absl/log/internal/container_test.cc index 0a5a058..92eef74 100644 --- a/absl/log/internal/container_test.cc +++ b/absl/log/internal/container_test.cc
@@ -15,13 +15,11 @@ #include "absl/log/internal/container.h" #include <cstdint> -#include <map> #include <memory> #include <ostream> #include <set> #include <sstream> #include <string> -#include <utility> #include <vector> #include "gtest/gtest.h"
diff --git a/absl/log/internal/globals.cc b/absl/log/internal/globals.cc index 359858f..51b7245 100644 --- a/absl/log/internal/globals.cc +++ b/absl/log/internal/globals.cc
@@ -26,7 +26,7 @@ #include "absl/base/internal/raw_logging.h" #include "absl/base/log_severity.h" #include "absl/strings/string_view.h" -#include "absl/strings/strip.h" +#include "absl/strings/strip.h" // IWYU pragma: keep #include "absl/time/time.h" namespace absl {
diff --git a/absl/log/internal/log_message.cc b/absl/log/internal/log_message.cc index 4acd503..673bd8d 100644 --- a/absl/log/internal/log_message.cc +++ b/absl/log/internal/log_message.cc
@@ -32,7 +32,6 @@ #include <ostream> #include <string> #include <string_view> -#include <tuple> #include "absl/base/attributes.h" #include "absl/base/config.h" @@ -54,7 +53,6 @@ #include "absl/log/log_entry.h" #include "absl/log/log_sink.h" #include "absl/log/log_sink_registry.h" -#include "absl/memory/memory.h" #include "absl/strings/internal/utf8.h" #include "absl/strings/string_view.h" #include "absl/time/clock.h"
diff --git a/absl/log/internal/proto.cc b/absl/log/internal/proto.cc index 8e7bda9..d0c57d6 100644 --- a/absl/log/internal/proto.cc +++ b/absl/log/internal/proto.cc
@@ -33,7 +33,6 @@ #include <cstdint> #include <cstring> -#include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/types/span.h"
diff --git a/absl/log/internal/structured_proto_test.cc b/absl/log/internal/structured_proto_test.cc index 7a1b82c..80a3e7b 100644 --- a/absl/log/internal/structured_proto_test.cc +++ b/absl/log/internal/structured_proto_test.cc
@@ -25,7 +25,6 @@ #include "absl/base/config.h" #include "absl/strings/string_view.h" #include "absl/types/span.h" -#include "absl/utility/utility.h" namespace absl { ABSL_NAMESPACE_BEGIN
diff --git a/absl/log/internal/test_actions.cc b/absl/log/internal/test_actions.cc index bdfd637..5d27f14 100644 --- a/absl/log/internal/test_actions.cc +++ b/absl/log/internal/test_actions.cc
@@ -22,6 +22,8 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/log_severity.h" +#include "absl/log/log_entry.h" #include "absl/strings/escaping.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h"
diff --git a/absl/log/internal/test_helpers.cc b/absl/log/internal/test_helpers.cc index 63e9deb..9abcd87 100644 --- a/absl/log/internal/test_helpers.cc +++ b/absl/log/internal/test_helpers.cc
@@ -14,6 +14,8 @@ // #include "absl/log/internal/test_helpers.h" +#include <csignal> + #ifdef __Fuchsia__ #include <zircon/syscalls.h> #endif
diff --git a/absl/log/internal/test_matchers.cc b/absl/log/internal/test_matchers.cc index 042083d..736bc7b 100644 --- a/absl/log/internal/test_matchers.cc +++ b/absl/log/internal/test_matchers.cc
@@ -25,6 +25,7 @@ #include "gtest/gtest.h" #include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/log_severity.h" #include "absl/log/internal/test_helpers.h" #include "absl/log/log_entry.h" #include "absl/strings/string_view.h" @@ -35,7 +36,6 @@ ABSL_NAMESPACE_BEGIN namespace log_internal { namespace { -using ::testing::_; using ::testing::AllOf; using ::testing::Ge; using ::testing::HasSubstr;
diff --git a/absl/log/internal/vlog_config.cc b/absl/log/internal/vlog_config.cc index 51f003c..79eb7df 100644 --- a/absl/log/internal/vlog_config.cc +++ b/absl/log/internal/vlog_config.cc
@@ -19,7 +19,6 @@ #include <algorithm> #include <atomic> #include <functional> -#include <memory> #include <optional> #include <string> #include <utility> @@ -27,13 +26,12 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" -#include "absl/base/const_init.h" +#include "absl/base/internal/scheduling_mode.h" #include "absl/base/internal/spinlock.h" #include "absl/base/no_destructor.h" #include "absl/base/optimization.h" #include "absl/base/thread_annotations.h" #include "absl/log/internal/fnmatch.h" -#include "absl/memory/memory.h" #include "absl/strings/numbers.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h"
diff --git a/absl/log/internal/vlog_config_benchmark.cc b/absl/log/internal/vlog_config_benchmark.cc index 7700edb..2634ca3 100644 --- a/absl/log/internal/vlog_config_benchmark.cc +++ b/absl/log/internal/vlog_config_benchmark.cc
@@ -27,7 +27,6 @@ #include "absl/base/config.h" #include "absl/container/internal/layout.h" #include "absl/log/internal/vlog_config.h" -#include "absl/memory/memory.h" #include "absl/random/distributions.h" #include "absl/strings/str_cat.h" #include "benchmark/benchmark.h"
diff --git a/absl/memory/BUILD.bazel b/absl/memory/BUILD.bazel index 81e12fa..92f9372 100644 --- a/absl/memory/BUILD.bazel +++ b/absl/memory/BUILD.bazel
@@ -24,7 +24,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -42,6 +42,7 @@ "//conditions:default": [], }), linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:core_headers", "//absl/meta:type_traits",
diff --git a/absl/meta/BUILD.bazel b/absl/meta/BUILD.bazel index 6e52c6e..a4b79cc 100644 --- a/absl/meta/BUILD.bazel +++ b/absl/meta/BUILD.bazel
@@ -24,7 +24,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -42,7 +42,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", ], deps = [ "//absl/base:config", @@ -68,7 +68,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__subpackages__", ], deps = [ "//absl/base:config", @@ -92,6 +92,7 @@ hdrs = ["type_traits.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers",
diff --git a/absl/numeric/BUILD.bazel b/absl/numeric/BUILD.bazel index edfe6b6..d61606c 100644 --- a/absl/numeric/BUILD.bazel +++ b/absl/numeric/BUILD.bazel
@@ -23,7 +23,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -41,6 +41,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -88,6 +89,7 @@ hdrs = ["int128.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":bits", "//absl/base:config", @@ -139,6 +141,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//absl:__subpackages__"], deps = [ "//absl/base:config", ],
diff --git a/absl/profiling/BUILD.bazel b/absl/profiling/BUILD.bazel index 34e3d25..b88fc84 100644 --- a/absl/profiling/BUILD.bazel +++ b/absl/profiling/BUILD.bazel
@@ -40,7 +40,6 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", ], deps = [ "//absl/base:config", @@ -78,7 +77,6 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", ], deps = [ "//absl/base:config", @@ -109,7 +107,6 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", ], deps = [ ":exponential_biased",
diff --git a/absl/random/BUILD.bazel b/absl/random/BUILD.bazel index 28f6872..ab0312a 100644 --- a/absl/random/BUILD.bazel +++ b/absl/random/BUILD.bazel
@@ -27,7 +27,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -42,6 +42,7 @@ hdrs = ["random.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":distributions", ":seed_sequences", @@ -73,6 +74,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:base_internal", "//absl/base:config", @@ -96,6 +98,7 @@ hdrs = ["seed_gen_exception.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:raw_logging_internal", @@ -110,6 +113,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":seed_gen_exception", "//absl/base:config", @@ -127,6 +131,7 @@ hdrs = ["bit_gen_ref.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":mocking_access", ":random", @@ -144,6 +149,7 @@ testonly = True, hdrs = ["mock_distributions.h"], linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":distributions", ":mocking_bit_gen", @@ -160,6 +166,7 @@ "mocking_bit_gen.h", ], linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":mocking_access", ":random", @@ -176,6 +183,7 @@ cc_library( name = "mocking_access", hdrs = ["mocking_access.h"], + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:fast_type_id",
diff --git a/absl/random/internal/BUILD.bazel b/absl/random/internal/BUILD.bazel index 56377ed..df91d9c 100644 --- a/absl/random/internal/BUILD.bazel +++ b/absl/random/internal/BUILD.bazel
@@ -29,7 +29,7 @@ default_package_visibility = [ "//absl/random:__pkg__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/util/random:__subpackages__", ] package(
diff --git a/absl/status/BUILD.bazel b/absl/status/BUILD.bazel index a8e45cb..287c9e8 100644 --- a/absl/status/BUILD.bazel +++ b/absl/status/BUILD.bazel
@@ -28,7 +28,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -52,6 +52,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:atomic_hook", "//absl/base:config", @@ -116,6 +117,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":status", "//absl/base", @@ -172,6 +174,7 @@ hdrs = ["status_builder.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":status", "//absl/base:config", @@ -211,6 +214,7 @@ cc_library( name = "status_macros", hdrs = ["status_macros.h"], + visibility = ["//visibility:public"], deps = [ ":status", ":status_builder", @@ -248,6 +252,7 @@ hdrs = ["status_matchers.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":status", ":statusor",
diff --git a/absl/status/internal/status_internal.h b/absl/status/internal/status_internal.h index 2428069..767f88f 100644 --- a/absl/status/internal/status_internal.h +++ b/absl/status/internal/status_internal.h
@@ -21,6 +21,7 @@ #include <memory> #include <optional> #include <string> +#include <type_traits> #include <utility> #include <vector> @@ -62,6 +63,10 @@ enum class StatusCode : int; enum class StatusToStringMode : int; +// Forward declaration of StatusOr for Status friendship. +template <typename T> +class StatusOr; + namespace status_internal { #ifndef SWIG class StatusPrivateAccessor; @@ -86,6 +91,15 @@ message_(message_arg), payloads_(std::move(payloads_arg)) {} + template <typename String, + typename = std::enable_if_t<std::is_same_v<String, std::string>>> + StatusRep(absl::StatusCode code_arg, String&& message_arg, + std::unique_ptr<status_internal::Payloads> payloads_arg) + : ref_(int32_t{1}), + code_(code_arg), + message_(std::forward<String>(message_arg)), + payloads_(std::move(payloads_arg)) {} + absl::StatusCode code() const { return code_; } const std::string& message() const { return message_; } @@ -136,7 +150,13 @@ // As an internal implementation detail, we guarantee that if status.message() // is non-empty, then the resulting string_view is null terminated. // This is required to implement 'StatusMessageAsCStr(...)' + // + // NOTE: if most statuses are constructed with messages that are either empty + // or so long they don't fit in the std::string's local storage (small string + // optimization), replacing std::string with an entirely heap-allocated + // string might save memory at scale. std::string message_; + absl::InlinedVector<absl::SourceLocation, 1> source_locations_; std::unique_ptr<status_internal::Payloads> payloads_; };
diff --git a/absl/status/status.cc b/absl/status/status.cc index 3ef4a27..18adf51 100644 --- a/absl/status/status.cc +++ b/absl/status/status.cc
@@ -22,6 +22,8 @@ #include <memory> #include <ostream> #include <string> +#include <type_traits> +#include <utility> #include "absl/base/attributes.h" #include "absl/base/config.h" @@ -112,20 +114,36 @@ absl::StatusCode::kOk, message, nullptr))); } -uintptr_t Status::MakeRep(uintptr_t inlined_rep, absl::string_view msg, - absl::SourceLocation loc) { - bool ok = inlined_rep == CodeToInlinedRep(absl::StatusCode::kOk); +template <typename StringOrView> +uintptr_t MakeStatusRepImpl(uintptr_t inlined_rep, StringOrView msg, + absl::SourceLocation loc) { + static_assert(std::is_same_v<StringOrView, absl::string_view> || + std::is_same_v<StringOrView, std::string&&>); + bool ok = inlined_rep == Status::CodeToInlinedRep(absl::StatusCode::kOk); if (ok) return inlined_rep; if (msg.empty() ) { return inlined_rep; } - auto* rep = new status_internal::StatusRep(InlinedRepToCode(inlined_rep), msg, - nullptr); + auto* rep = + new status_internal::StatusRep(Status::InlinedRepToCode(inlined_rep), + std::forward<StringOrView>(msg), nullptr); if (loc.file_name()[0] != '\0') { rep->AddSourceLocation(loc); } - return PointerToRep(rep); + return Status::PointerToRep(rep); +} + +uintptr_t Status::MakeRepFromStringView(uintptr_t inlined_rep, + absl::string_view msg, + absl::SourceLocation loc) { + return MakeStatusRepImpl<absl::string_view>(inlined_rep, msg, loc); +} + +uintptr_t Status::MakeRepFromStringRvalue(uintptr_t inlined_rep, + std::string&& msg, + absl::SourceLocation loc) { + return MakeStatusRepImpl<std::string&&>(inlined_rep, std::move(msg), loc); } uintptr_t Status::AddSourceLocationImpl(uintptr_t rep,
diff --git a/absl/status/status.h b/absl/status/status.h index 55802cd..0b39b9b 100644 --- a/absl/status/status.h +++ b/absl/status/status.h
@@ -56,6 +56,7 @@ #include <optional> #include <ostream> #include <string> +#include <type_traits> #include <utility> #include "absl/base/attributes.h" @@ -459,6 +460,14 @@ Status(absl::StatusCode code, absl::string_view msg, absl::SourceLocation loc = SourceLocation::current()); + // Same as above but for rvalue string. + // Note: using a template to disambiguate the case of matching string_view and + // string&& (e.g. char*) as a template lowers the priority of the overload. + template <typename String, + typename = std::enable_if_t<std::is_same_v<String, std::string>>> + Status(absl::StatusCode code, String&& msg, + absl::SourceLocation loc = SourceLocation::current()); + // Create a status from a `base_status` and a `loc`. The `loc` will be // appended to the location chain of the new status, iff the `base_status` is // not ok and has non-empty msg. @@ -699,6 +708,8 @@ friend class absl::status_internal::StatusPrivateAccessor; friend class absl::status_internal::StatusPrivateAccessorForStatusBuilder; + template <typename T> + friend class absl::StatusOr; #endif // !SWIG // Creates a status in the canonical error space with the specified @@ -707,8 +718,18 @@ // Delegate factory in header that ensures CodeToInlinedRep is inlined // where possible. - static uintptr_t MakeRep(uintptr_t inlined_rep, absl::string_view msg, - absl::SourceLocation loc); + static uintptr_t MakeRepFromStringView(uintptr_t inlined_rep, + absl::string_view msg, + absl::SourceLocation loc); + + // Same as above but for rvalue string. + static uintptr_t MakeRepFromStringRvalue(uintptr_t inlined_rep, + std::string&& msg, + absl::SourceLocation loc); + + template <typename StringOrView> + friend uintptr_t MakeStatusRepImpl(uintptr_t inlined_rep, StringOrView msg, + absl::SourceLocation loc); // Underlying constructor for status from a rep_. explicit Status(uintptr_t rep) : rep_(rep) {} @@ -897,7 +918,13 @@ inline Status::Status(absl::StatusCode code, absl::string_view msg, absl::SourceLocation loc) - : Status(MakeRep(CodeToInlinedRep(code), msg, loc)) {} + : Status(MakeRepFromStringView(CodeToInlinedRep(code), msg, loc)) {} + +template <typename String, typename> +inline Status::Status(absl::StatusCode code, String&& msg, + absl::SourceLocation loc) + : Status(MakeRepFromStringRvalue(CodeToInlinedRep(code), + std::forward<String>(msg), loc)) {} inline Status::Status(const Status& x) : Status(x.rep_) { Ref(rep_); }
diff --git a/absl/status/status_benchmark.cc b/absl/status/status_benchmark.cc index 539b783..0404db1 100644 --- a/absl/status/status_benchmark.cc +++ b/absl/status/status_benchmark.cc
@@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include <string> #include <utility> #include "absl/status/status.h" @@ -46,4 +47,14 @@ } BENCHMARK(BM_AppendSourceLocation); +void BM_LongMessageRValue(benchmark::State& state) { + for (auto _ : state) { + std::string msg(100, 'X'); + benchmark::DoNotOptimize(msg); + absl::Status s(absl::StatusCode::kInvalidArgument, std::move(msg)); + benchmark::DoNotOptimize(s); + } +} +BENCHMARK(BM_LongMessageRValue); + } // namespace
diff --git a/absl/status/statusor.h b/absl/status/statusor.h index ed34bb8..8d8247f 100644 --- a/absl/status/statusor.h +++ b/absl/status/statusor.h
@@ -408,7 +408,7 @@ typename U = T, std::enable_if_t<internal_statusor::IsAssignmentValid<T, U, true>::value, int> = 0> - StatusOr& operator=(U&& v ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) { + StatusOr& operator=(U&& v ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) { this->Assign(std::forward<U>(v)); return *this; } @@ -614,11 +614,14 @@ T& emplace(Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { if (ok()) { this->Clear(); - this->MakeValue(std::forward<Args>(args)...); - } else { - this->MakeValue(std::forward<Args>(args)...); - this->status_ = absl::OkStatus(); + // Temporarily transition to a non-ok status (using the zero-allocation + // inlined representation) so that if MakeValue() throws an exception, + // ok() returns false during stack unwinding and ~StatusOrData() does not + // attempt to destroy uninitialized memory. + this->status_ = absl::Status(absl::StatusCode::kInternal); } + this->MakeValue(std::forward<Args>(args)...); + this->status_ = absl::OkStatus(); return this->data_; } @@ -630,11 +633,14 @@ Args&&... args) ABSL_ATTRIBUTE_LIFETIME_BOUND { if (ok()) { this->Clear(); - this->MakeValue(ilist, std::forward<Args>(args)...); - } else { - this->MakeValue(ilist, std::forward<Args>(args)...); - this->status_ = absl::OkStatus(); + // Temporarily transition to a non-ok status (using the zero-allocation + // inlined representation) so that if MakeValue() throws an exception, + // ok() returns false during stack unwinding and ~StatusOrData() does not + // attempt to destroy uninitialized memory. + this->status_ = absl::Status(absl::StatusCode::kInternal); } + this->MakeValue(ilist, std::forward<Args>(args)...); + this->status_ = absl::OkStatus(); return this->data_; }
diff --git a/absl/status/statusor_test.cc b/absl/status/statusor_test.cc index 2205acf..d03e8be 100644 --- a/absl/status/statusor_test.cc +++ b/absl/status/statusor_test.cc
@@ -406,6 +406,53 @@ Field(&InPlaceHelper::y, Pointee(4))))); } +#ifdef ABSL_HAVE_EXCEPTIONS +class ThrowOnEmplace { + public: + explicit ThrowOnEmplace(int* counter, int val) : destructor_calls_(counter) { + if (val < 0) { + throw std::runtime_error("expected"); + } + // While destructor_calls tracks the logic, ptr_ ensures that a double + // destruction actually results in a reliable crash. Performing a real heap + // allocation and deallocation (new/delete) guarantees that AddressSanitizer + // (ASAN) or the heap allocator will instantly catch the double-free if the + // bug regresses, rather than relying solely on the integer check. + ptr_ = new int(val); + } + + ThrowOnEmplace(const ThrowOnEmplace&) = delete; + ThrowOnEmplace& operator=(const ThrowOnEmplace&) = delete; + + ~ThrowOnEmplace() { + if (destructor_calls_) { + ++(*destructor_calls_); + } + delete ptr_; + } + + private: + int* destructor_calls_ = nullptr; + int* ptr_ = nullptr; +}; + +TEST(StatusOr, EmplaceThrowsExceptionSafety) { + int destructor_calls = 0; + { + absl::StatusOr<ThrowOnEmplace> status_or(std::in_place, &destructor_calls, + 1); + EXPECT_TRUE(status_or.ok()); + EXPECT_THROW(status_or.emplace(&destructor_calls, -1), std::runtime_error); + EXPECT_FALSE(status_or.ok()); + EXPECT_EQ(status_or.status().code(), absl::StatusCode::kInternal); + } + // Verifies that the initial object is properly destroyed by Clear() (count is + // 1), and that the exception thrown during replacement does not cause a + // second destruction (double-free) during stack unwinding. + EXPECT_EQ(destructor_calls, 1); +} +#endif // ABSL_HAVE_EXCEPTIONS + TEST(StatusOr, TestCopyCtorStatusOk) { const int kI = 4; const absl::StatusOr<int> original(kI);
diff --git a/absl/strings/BUILD.bazel b/absl/strings/BUILD.bazel index 6b96d0c..38c85cb 100644 --- a/absl/strings/BUILD.bazel +++ b/absl/strings/BUILD.bazel
@@ -24,7 +24,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -39,6 +39,7 @@ hdrs = ["string_view.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -60,7 +61,6 @@ "internal/damerau_levenshtein_distance.cc", "internal/memutil.cc", "internal/memutil.h", - "internal/stl_type_traits.h", "internal/str_join_internal.h", "internal/str_split_internal.h", "internal/stringify_sink.cc", @@ -97,6 +97,7 @@ # New code should directly depend on :string_view. "string_view.h", ], + visibility = ["//visibility:public"], deps = [ ":append_and_overwrite", ":charset", @@ -135,6 +136,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], # Logically private. Do not depend on this. deps = [ ":resize_and_overwrite", "//absl/base:config", @@ -150,6 +152,7 @@ hdrs = ["resize_and_overwrite.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -225,7 +228,9 @@ tags = ["no_test_chromiumos_x86_64"], visibility = ["//visibility:private"], deps = [ + ":charset", ":cord", + ":str_format", ":strings", "//absl/base:core_headers", "//absl/container:fixed_array", @@ -270,6 +275,7 @@ hdrs = ["has_ostream_operator.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", ], @@ -433,8 +439,8 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", "//absl/status:__pkg__", + "@do_not_use_for_gloop_visibility_only//gloop/util/status:__pkg__", ], deps = [ ":string_view", @@ -482,6 +488,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":string_view", "//absl/base:config", @@ -666,6 +673,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":append_and_overwrite", ":cord_internal", @@ -923,6 +931,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":cord", ":cord_internal", @@ -951,6 +960,9 @@ hdrs = ["cordz_test_helpers.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/strings:__pkg__", + ], deps = [ ":cord", ":cord_internal", @@ -1365,6 +1377,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":str_format_internal", ":string_view", @@ -1599,7 +1612,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/util/gtl:__pkg__", ], deps = [ ":str_format", @@ -1617,12 +1630,12 @@ linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":generic_printer", + ":str_format", ":strings", "//absl/base:config", "//absl/base:core_headers", "//absl/cleanup", "//absl/container:flat_hash_map", - "//absl/log", "//absl/status", "//absl/status:statusor", "@googletest//:gtest",
diff --git a/absl/strings/CMakeLists.txt b/absl/strings/CMakeLists.txt index ca07c3e..a9fa1f9 100644 --- a/absl/strings/CMakeLists.txt +++ b/absl/strings/CMakeLists.txt
@@ -59,7 +59,6 @@ "internal/memutil.h" "internal/stringify_sink.h" "internal/stringify_sink.cc" - "internal/stl_type_traits.h" "internal/str_join_internal.h" "internal/str_split_internal.h" "match.cc" @@ -88,7 +87,6 @@ absl::nullability absl::raw_logging_internal absl::source_location - absl::strings absl::throw_delegate absl::type_traits PUBLIC @@ -247,7 +245,9 @@ COPTS ${ABSL_TEST_COPTS} DEPS + absl::charset absl::strings + absl::str_format absl::core_headers absl::fixed_array GTest::gmock_main @@ -1319,9 +1319,9 @@ absl::config absl::flat_hash_map absl::generic_printer_internal - absl::log absl::status absl::statusor + absl::str_format absl::strings GTest::gmock_main )
diff --git a/absl/strings/cord.cc b/absl/strings/cord.cc index 584a3c6..5827d89 100644 --- a/absl/strings/cord.cc +++ b/absl/strings/cord.cc
@@ -455,6 +455,9 @@ contents_.MaybeRemoveEmptyCrcNode(); if (src.empty()) return; + ABSL_RAW_CHECK(src.contents_.size() <= + std::numeric_limits<size_t>::max() - contents_.size(), + "Cord length overflow"); if (empty()) { // Since destination is empty, we can avoid allocating a node, @@ -566,6 +569,9 @@ void Cord::Prepend(const Cord& src) { contents_.MaybeRemoveEmptyCrcNode(); if (src.empty()) return; + ABSL_RAW_CHECK(src.contents_.size() <= + std::numeric_limits<size_t>::max() - contents_.size(), + "Cord length overflow"); CordRep* src_tree = src.contents_.tree(); if (src_tree != nullptr) {
diff --git a/absl/strings/cord.h b/absl/strings/cord.h index c5b2ec4..7e80259 100644 --- a/absl/strings/cord.h +++ b/absl/strings/cord.h
@@ -1621,11 +1621,13 @@ inline Cord::ChunkIterator::reference Cord::ChunkIterator::operator*() const { absl::base_internal::HardeningAssertGT(bytes_remaining_, size_t{0}); + ABSL_ASSERT(bytes_remaining_ >= current_chunk_.size()); return current_chunk_; } inline Cord::ChunkIterator::pointer Cord::ChunkIterator::operator->() const { absl::base_internal::HardeningAssertGT(bytes_remaining_, size_t{0}); + ABSL_ASSERT(bytes_remaining_ >= current_chunk_.size()); return ¤t_chunk_; }
diff --git a/absl/strings/escaping.cc b/absl/strings/escaping.cc index ea5a958..1ed7c24 100644 --- a/absl/strings/escaping.cc +++ b/absl/strings/escaping.cc
@@ -20,7 +20,9 @@ #include <cstddef> #include <cstdint> #include <cstring> +#include <iterator> #include <limits> +#include <optional> #include <string> #include <utility> @@ -61,6 +63,12 @@ return x & 0xf; } +inline char int_to_hex_digit(int i) { + assert(i >= 0 && i <= 15); + return ((i < 10) ? (static_cast<char>(i) + '0') + : (static_cast<char>(i - 10) + 'A')); +} + inline bool IsSurrogate(char32_t c, absl::string_view src, std::string* absl_nullable error) { if (c >= 0xD800 && c <= 0xDFFF) { @@ -451,10 +459,10 @@ // We keep 3 slop bytes so that we can call `little_endian::Store32` // invariably regardless of the length of the escaped character. constexpr size_t kSlopBytes = 3; - size_t cur_dest_len = dest->size(); - size_t append_buf_len = cur_dest_len + escaped_len + kSlopBytes; - ABSL_INTERNAL_CHECK(append_buf_len > cur_dest_len, - "std::string size overflow"); + ABSL_INTERNAL_CHECK( + escaped_len <= std::numeric_limits<size_t>::max() - kSlopBytes, + "CEscape length overflow"); + size_t append_buf_len = escaped_len + kSlopBytes; strings_internal::StringAppendAndOverwrite( *dest, append_buf_len, [src, escaped_len](char* append_ptr, size_t) { for (char c : src) { @@ -1169,6 +1177,8 @@ std::string BytesToHexString(absl::string_view from) { std::string result; + ABSL_INTERNAL_CHECK(from.size() <= std::numeric_limits<size_t>::max() / 2, + "BytesToHexString() overflow"); StringResizeAndOverwrite( result, 2 * from.size(), [from](char* buf, size_t buf_size) { absl::BytesToHexStringInternal( @@ -1179,5 +1189,131 @@ return result; } +static std::string UrlEscapeInternal(absl::string_view input, + const bool escape_space_to_plus) { + // Unreserved characters from RFC 3986. + // See https://www.rfc-editor.org/info/rfc3986/#section-2.3. + static constexpr absl::CharSet kRfc3986Unreserved = + absl::CharSet::AsciiAlphanumerics() | absl::CharSet("-._~"); + + std::string output; + absl::string_view::iterator in = input.begin(); + + // Fast path for when we don't need to do any escaping. + while (in < input.end() && kRfc3986Unreserved.contains(*in)) { + ++in; + } + + std::size_t initial_portion = + static_cast<std::size_t>(std::distance(input.begin(), in)); + + if (initial_portion == input.size()) { + return std::string(input); + } + + // We need a buffer with enough space to store at most the initial portion + // plus 3 bytes for each remaining character since escapes use 3 characters. + ABSL_INTERNAL_CHECK( + (input.size() - initial_portion) <= + (std::numeric_limits<size_t>::max() - initial_portion) / 3, + "UrlEscape() overflow"); + StringResizeAndOverwrite( + output, initial_portion + 3 * (input.size() - initial_portion), + [&](char* buf, size_t) { + char* out = buf; + + // Copy the initial portion that did not need escaping. + out = std::copy(input.begin(), in, out); + + // Handle the rest of the string. + while (in < input.end()) { + char c = *in++; + if (kRfc3986Unreserved.contains(c)) { + *out++ = c; + } else if (escape_space_to_plus && c == ' ') { + *out++ = '+'; + } else { + *out++ = '%'; + *out++ = static_cast<char>( + int_to_hex_digit((static_cast<unsigned char>(c) >> 4) & 0xf)); + *out++ = static_cast<char>( + int_to_hex_digit(static_cast<unsigned char>(c) & 0xf)); + } + } + return static_cast<size_t>(std::distance(buf, out)); + }); + + return output; +} + +static std::optional<std::string> UrlUnescapeInternal( + absl::string_view input, const bool unescape_plus_to_space) { + std::string output; + + // Fast path for when we don't need to do any unescaping. + // This case includes empty input, which allows us to return 0 from the + // lambda below to signal the error case. + size_t in = + unescape_plus_to_space ? input.find_first_of("%+") : input.find('%'); + if (in == input.npos) { + return std::string(input); + } + + StringResizeAndOverwrite(output, input.size(), [&](char* buf, size_t) { + char* out = buf; + + // Copy the initial portion that did not need unescaping. + out = std::copy_n(input.data(), in, out); + + // Handle the rest of the string. + while (in < input.size()) { + char c = input[in++]; + if (unescape_plus_to_space && c == '+') { + *out++ = ' '; + } else if (c == '%') { + if (in + 1 >= input.size() || + !absl::ascii_isxdigit(static_cast<unsigned char>(input[in])) || + !absl::ascii_isxdigit(static_cast<unsigned char>(input[in + 1]))) { + return size_t{0}; // Error. + } + int x = static_cast<int>(hex_digit_to_int(input[in++])) << 4; + x += static_cast<int>(hex_digit_to_int(input[in++])); + *out++ = static_cast<char>(x); + } else { + *out++ = c; + } + } + return static_cast<size_t>(std::distance(buf, out)); + }); + + if (output.empty()) { + // Empty output is only valid if the input was empty, and that case is + // handled above. + return std::nullopt; + } + + return output; +} + +std::string UrlEscape(absl::string_view input) { + constexpr bool kEscapeSpaceToPlus = false; + return UrlEscapeInternal(input, kEscapeSpaceToPlus); +} + +std::optional<std::string> UrlUnescape(absl::string_view input) { + constexpr bool kUnescapePlusToSpace = false; + return UrlUnescapeInternal(input, kUnescapePlusToSpace); +} + +std::string UrlEscapePlus(absl::string_view input) { + constexpr bool kEscapeSpaceToPlus = true; + return UrlEscapeInternal(input, kEscapeSpaceToPlus); +} + +std::optional<std::string> UrlUnescapePlus(absl::string_view input) { + constexpr bool kUnescapePlusToSpace = true; + return UrlUnescapeInternal(input, kUnescapePlusToSpace); +} + ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/strings/escaping.h b/absl/strings/escaping.h index 4a23c13..3dbaa5b 100644 --- a/absl/strings/escaping.h +++ b/absl/strings/escaping.h
@@ -24,6 +24,7 @@ #define ABSL_STRINGS_ESCAPING_H_ #include <cstddef> +#include <optional> #include <string> #include <vector> @@ -190,6 +191,74 @@ // `2*from.size()`. std::string BytesToHexString(absl::string_view from); +// UrlEscape() +// +// Escapes a string so it can be safely used as a value in a URL component by +// replacing all characters that are not "unreserved characters" with +// percent-escapes. See https://tools.ietf.org/html/rfc3986 +// +// Usage note: URLs use "reserved characters" (like ?, &, =, /) as structural +// syntax. This function escapes these syntax characters. The correct use of +// this function is to clean individual URL components *before* assembling them +// into the final URL structure. Do not run it on a fully constructed URL, as +// this will turn structural delimiters into URL component data. +// +// Example (encoding "gift for mom & dad" as a URL query parameter): +// +// std::string url = absl::StrFormat("https://www.google.com/search?q=%s", +// absl::UrlEscape("gift for mom & dad")); +// assert(url == +// "https://www.google.com/search?q=gift%20for%20mom%20%26%20dad"); +[[nodiscard]] std::string UrlEscape(absl::string_view input); + +// UrlUnescape() +// +// Performs the inverse transformation of UrlEscape(), converting each +// percent-encoded sequence of the form "%AB" into the character with the +// hexadecimal value 0xAB. It returns `std::nullopt` if any % is not followed by +// two hexadecimal digits. +// +// UrlUnescape() is identical to UrlUnescapePlus() except that it does not +// unescape '+' to ' '. +[[nodiscard]] std::optional<std::string> UrlUnescape(absl::string_view input); + +// UrlEscapePlus() +// +// Escapes a string so it can be safely used as a value for +// application/x-www-form-urlencoded (HTML form submissions). +// +// Historically web browsers have also used this form of escaping for query +// parameters. +// +// UrlEscapePlus() differs from UrlEscape() in that space (' ') is encoded to +// plus ("+") instead of "%20". According to the URI specification (RFC 3986), +// the correct way to escape a space anywhere in a URL (including the query +// string) is "%20". Using "%20" in a query parameter will work universally. +// +// Some strict URL parsers (especially outside of web browsers/web servers) +// follow RFC 3986 strictly and will treat a literal '+' in the query string as +// a literal plus sign, rather than decoding it to a space. +// +// Recommendation: Use UrlEscapePlus() only if you are specifically implementing +// or interacting with a system that strictly expects +// "application/x-www-form-urlencoded" formatting. For general URL construction, +// UrlEscape() is the correct and safest choice. +// +// Example (encoding "gift for mom & dad" as a URL query parameter): +// +// std::string url = absl::StrFormat("https://www.google.com/search?q=%s", +// absl::UrlEscapePlus( +// "gift for mom & dad")); +// assert(url == "https://www.google.com/search?q=gift+for+mom+%26+dad"); +[[nodiscard]] std::string UrlEscapePlus(absl::string_view input); + +// UrlUnescapePlus() +// +// Performs the inverse transformation of UrlEscapePlus(). It returns +// `std::nullopt` if any % is not followed by two hexadecimal digits. +[[nodiscard]] std::optional<std::string> UrlUnescapePlus( + absl::string_view input); + ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/strings/escaping_benchmark.cc b/absl/strings/escaping_benchmark.cc index 64b5a41..1fd5b41 100644 --- a/absl/strings/escaping_benchmark.cc +++ b/absl/strings/escaping_benchmark.cc
@@ -18,6 +18,7 @@ #include <string> #include "absl/base/internal/raw_logging.h" +#include "absl/strings/ascii.h" #include "absl/strings/escaping.h" #include "absl/strings/internal/escaping_test_common.h" #include "absl/strings/str_cat.h" @@ -85,6 +86,42 @@ } BENCHMARK(BM_HexStringToBytes_Fail); +static void BM_UrlEscape(benchmark::State& state) { + std::string all; + std::string alnum; + all.reserve(256); + for (int c = 0; c < 256; ++c) { + all.push_back(c); + if (absl::ascii_isalnum(c)) { + alnum.push_back(c); + } + } + + for (auto _ : state) { + benchmark::DoNotOptimize(absl::UrlEscape(all)); + benchmark::DoNotOptimize(absl::UrlEscape(alnum)); + } +} +BENCHMARK(BM_UrlEscape); + +static void BM_UrlEscapePlus(benchmark::State& state) { + std::string all; + std::string alnum; + all.reserve(256); + for (int c = 0; c < 256; ++c) { + all.push_back(c); + if (absl::ascii_isalnum(c)) { + alnum.push_back(c); + } + } + + for (auto _ : state) { + benchmark::DoNotOptimize(absl::UrlEscapePlus(all)); + benchmark::DoNotOptimize(absl::UrlEscapePlus(alnum)); + } +} +BENCHMARK(BM_UrlEscapePlus); + // Used for the CEscape benchmarks const char kStringValueNoEscape[] = "1234567890"; const char kStringValueSomeEscaped[] = "123\n56789\xA1";
diff --git a/absl/strings/escaping_test.cc b/absl/strings/escaping_test.cc index 9651953..a564e83 100644 --- a/absl/strings/escaping_test.cc +++ b/absl/strings/escaping_test.cc
@@ -20,18 +20,24 @@ #include <cstring> #include <initializer_list> #include <memory> +#include <optional> #include <string> #include <vector> +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/log/check.h" -#include "absl/strings/str_cat.h" - +#include "absl/strings/charset.h" #include "absl/strings/internal/escaping_test_common.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" #include "absl/strings/string_view.h" namespace { +using ::testing::Eq; +using ::testing::Optional; + struct epair { std::string escaped; std::string unescaped; @@ -761,4 +767,151 @@ EXPECT_EQ(hex_only_lower, hex_result); } +TEST(UrlEscape, Basics) { + EXPECT_EQ(absl::UrlEscape(""), ""); + EXPECT_THAT(absl::UrlUnescape(""), Optional(Eq(""))); + + EXPECT_EQ(absl::UrlEscape("abc"), "abc"); + EXPECT_THAT(absl::UrlUnescape("abc"), Optional(Eq("abc"))); + + EXPECT_EQ(absl::UrlEscape("a/b"), "a%2Fb"); + EXPECT_THAT(absl::UrlUnescape("a%2Fb"), Optional(Eq("a/b"))); + + EXPECT_EQ(absl::UrlEscape("one two"), "one%20two"); + EXPECT_THAT(absl::UrlUnescape("one%20two"), Optional(Eq("one two"))); + + EXPECT_EQ(absl::UrlEscape("10%"), "10%25"); + EXPECT_THAT(absl::UrlUnescape("10%25"), Optional(Eq("10%"))); + + EXPECT_EQ(absl::UrlEscape(" ?&=#+%!<>#\"{}|\\^[]`☺\t:/@$'()*,;"), + "%20%3F%26%3D%23%2B%25%21%3C%3E%23%22%7B%7D%7C%5C%5E%5B%5D%60%E2%" + "98%BA%09%3A%2F%40%24%27%28%29%2A%2C%3B"); + EXPECT_THAT(absl::UrlUnescape("%20%3F%26%3D%23%2B%25%21%3C%3E%23%22%7B%7D%7C%" + "5C%5E%5B%5D%60%E2%98%BA%" + "09%3A%2F%40%24%27%28%29%2A%2C%3B"), + Optional(Eq(" ?&=#+%!<>#\"{}|\\^[]`☺\t:/@$'()*,;"))); + + // Test all characters. + static constexpr absl::CharSet kDoNotEscape = + absl::CharSet::AsciiAlphanumerics() | absl::CharSet("-._~"); + for (int i = 0; i < 256; ++i) { + char c = static_cast<char>(i); + std::string expected = kDoNotEscape.contains(c) + ? std::string(1, c) + : absl::StrFormat("%%%02X", c); + EXPECT_EQ(absl::UrlEscape(absl::string_view(&c, 1)), expected); + EXPECT_EQ(absl::UrlUnescape(expected), absl::string_view(&c, 1)); + } +} + +TEST(UrlUnescape, SuccessCases) { + EXPECT_THAT(absl::UrlUnescape(""), Optional(Eq(""))); + EXPECT_THAT(absl::UrlUnescape("abc"), Optional(Eq("abc"))); + EXPECT_THAT(absl::UrlUnescape("1%41"), Optional(Eq("1A"))); + EXPECT_THAT(absl::UrlUnescape("1%41%42%43"), Optional(Eq("1ABC"))); + EXPECT_THAT(absl::UrlUnescape("%4a"), Optional(Eq("J"))); + EXPECT_THAT(absl::UrlUnescape("%6F"), Optional(Eq("o"))); + EXPECT_THAT(absl::UrlUnescape("a%20b"), Optional(Eq("a b"))); + EXPECT_THAT(absl::UrlUnescape("a+b"), Optional(Eq("a+b"))); +} + +TEST(UrlUnescape, NotEnoughCharsAfterPercent) { + EXPECT_EQ(absl::UrlUnescape("%"), std::nullopt); + EXPECT_EQ(absl::UrlUnescape("%a"), std::nullopt); + EXPECT_EQ(absl::UrlUnescape("%1"), std::nullopt); + EXPECT_EQ(absl::UrlUnescape("123%45%6"), std::nullopt); +} + +TEST(UrlUnescape, InvalidHexDigits) { + EXPECT_EQ(absl::UrlUnescape("%zzzzz"), std::nullopt); +} + +TEST(UrlUnescape, NoErrorWithNoEscapeSequence) { + // Any string that does not contain '%' should not produce an error, even if + // absl::UrlEscape() would never produce a string with certain characters. + std::string no_percent; + for (int c = 0; c < 256; ++c) { + if (c != '%') { + no_percent.push_back(static_cast<char>(c)); + } + } + EXPECT_THAT(absl::UrlUnescape(no_percent), Optional(no_percent)); +} + +TEST(UrlEscapePlus, Basics) { + EXPECT_EQ(absl::UrlEscapePlus(""), ""); + EXPECT_THAT(absl::UrlUnescapePlus(""), Optional(Eq(""))); + + EXPECT_EQ(absl::UrlEscapePlus("abc"), "abc"); + EXPECT_THAT(absl::UrlUnescapePlus("abc"), Optional(Eq("abc"))); + + EXPECT_EQ(absl::UrlEscapePlus("one two"), "one+two"); + EXPECT_THAT(absl::UrlUnescapePlus("one+two"), Optional(Eq("one two"))); + + EXPECT_EQ(absl::UrlEscapePlus("gift for mom & dad"), "gift+for+mom+%26+dad"); + EXPECT_THAT(absl::UrlUnescapePlus("gift+for+mom+%26+dad"), + Optional(Eq("gift for mom & dad"))); + + EXPECT_EQ(absl::UrlEscapePlus("10%"), "10%25"); + EXPECT_THAT(absl::UrlUnescapePlus("10%25"), Optional(Eq("10%"))); + + EXPECT_EQ(absl::UrlEscapePlus(" ?&=#+%!<>#\"{}|\\^[]`☺\t:/@$'()*,;"), + "+%3F%26%3D%23%2B%25%21%3C%3E%23%22%7B%7D%7C%5C%5E%5B%5D%60%E2%" + "98%BA%09%3A%2F%40%24%27%28%29%2A%2C%3B"); + EXPECT_THAT(absl::UrlUnescapePlus("+%3F%26%3D%23%2B%25%21%3C%3E%23%22%7B%7D%" + "7C%5C%5E%5B%5D%60%E2%98%BA%" + "09%3A%2F%40%24%27%28%29%2A%2C%3B"), + Optional(Eq(" ?&=#+%!<>#\"{}|\\^[]`☺\t:/@$'()*,;"))); + + // Test all characters. + static constexpr absl::CharSet kDoNotEscape = + absl::CharSet::AsciiAlphanumerics() | absl::CharSet("-._~"); + for (int i = 0; i < 256; ++i) { + char c = static_cast<char>(i); + std::string expected = kDoNotEscape.contains(c) + ? std::string(1, c) + : absl::StrFormat("%%%02X", c); + if (c == ' ') expected = '+'; + EXPECT_EQ(absl::UrlEscapePlus(absl::string_view(&c, 1)), expected); + EXPECT_EQ(absl::UrlUnescapePlus(expected), absl::string_view(&c, 1)); + } +} + +TEST(UrlUnescapePlus, SuccessCases) { + EXPECT_THAT(absl::UrlUnescapePlus(""), Optional(Eq(""))); + EXPECT_THAT(absl::UrlUnescapePlus("abc"), Optional(Eq("abc"))); + EXPECT_THAT(absl::UrlUnescapePlus("1%41"), Optional(Eq("1A"))); + EXPECT_THAT(absl::UrlUnescapePlus("1%41%42%43"), Optional(Eq("1ABC"))); + EXPECT_THAT(absl::UrlUnescapePlus("%4a"), Optional(Eq("J"))); + EXPECT_THAT(absl::UrlUnescapePlus("%6F"), Optional(Eq("o"))); + EXPECT_THAT(absl::UrlUnescapePlus("a%20b"), Optional(Eq("a b"))); + EXPECT_THAT(absl::UrlUnescapePlus("a+b"), Optional(Eq("a b"))); +} + +TEST(UrlUnescapePlus, NotEnoughCharsAfterPercent) { + EXPECT_EQ(absl::UrlUnescapePlus("%"), std::nullopt); + EXPECT_EQ(absl::UrlUnescapePlus("%a"), std::nullopt); + EXPECT_EQ(absl::UrlUnescapePlus("%1"), std::nullopt); + EXPECT_EQ(absl::UrlUnescapePlus("123%45%6"), std::nullopt); +} + +TEST(UrlUnescapePlus, InvalidHexDigits) { + EXPECT_EQ(absl::UrlUnescapePlus("%zzzzz"), std::nullopt); +} + +TEST(UrlUnescapePlus, NoErrorWithNoEscapeSequence) { + // Any string that does not contain '%' should not produce an error, even if + // absl::UrlEscapePlus() would never produce a string with certain + // characters. + std::string no_percent; + std::string no_percent_expected; + for (int c = 0; c < 256; ++c) { + if (c != '%') { + no_percent.push_back(static_cast<char>(c)); + no_percent_expected.push_back(c != '+' ? static_cast<char>(c) : ' '); + } + } + EXPECT_THAT(absl::UrlUnescapePlus(no_percent), Optional(no_percent_expected)); +} + } // namespace
diff --git a/absl/strings/internal/charconv_parse.cc b/absl/strings/internal/charconv_parse.cc index 98823de..2766cf3 100644 --- a/absl/strings/internal/charconv_parse.cc +++ b/absl/strings/internal/charconv_parse.cc
@@ -13,12 +13,13 @@ // limitations under the License. #include "absl/strings/internal/charconv_parse.h" -#include "absl/strings/charconv.h" #include <cassert> +#include <cstddef> #include <cstdint> #include <limits> +#include "absl/strings/charconv.h" #include "absl/strings/internal/memutil.h" namespace absl { @@ -246,8 +247,9 @@ // ConsumeDigits does not protect against overflow on *out; max_digits must // be chosen with respect to type T to avoid the possibility of overflow. template <int base, typename T> -int ConsumeDigits(const char* begin, const char* end, int max_digits, T* out, - bool* dropped_nonzero_digit) { +ptrdiff_t ConsumeDigits(const char* begin, const char* end, + ptrdiff_t max_digits, T* out, + bool* dropped_nonzero_digit) { if (base == 10) { assert(max_digits <= std::numeric_limits<T>::digits10); } else if (base == 16) { @@ -282,7 +284,7 @@ *dropped_nonzero_digit = true; } *out = accumulator; - return static_cast<int>(begin - original_begin); + return begin - original_begin; } // Returns true if `v` is one of the chars allowed inside parentheses following @@ -370,24 +372,22 @@ } uint64_t mantissa = 0; - int exponent_adjustment = 0; + ptrdiff_t exponent_adjustment = 0; bool mantissa_is_inexact = false; - int pre_decimal_digits = ConsumeDigits<base>( + ptrdiff_t pre_decimal_digits = ConsumeDigits<base>( begin, end, MantissaDigitsMax<base>(), &mantissa, &mantissa_is_inexact); begin += pre_decimal_digits; - int digits_left; + ptrdiff_t digits_left; if (pre_decimal_digits >= DigitLimit<base>()) { // refuse to parse pathological inputs return result; } else if (pre_decimal_digits > MantissaDigitsMax<base>()) { // We dropped some non-fraction digits on the floor. Adjust our exponent // to compensate. - exponent_adjustment = - static_cast<int>(pre_decimal_digits - MantissaDigitsMax<base>()); + exponent_adjustment = pre_decimal_digits - MantissaDigitsMax<base>(); digits_left = 0; } else { - digits_left = - static_cast<int>(MantissaDigitsMax<base>() - pre_decimal_digits); + digits_left = MantissaDigitsMax<base>() - pre_decimal_digits; } if (begin < end && *begin == '.') { ++begin; @@ -398,14 +398,14 @@ while (begin < end && *begin == '0') { ++begin; } - int zeros_skipped = static_cast<int>(begin - begin_zeros); + ptrdiff_t zeros_skipped = begin - begin_zeros; if (zeros_skipped >= DigitLimit<base>()) { // refuse to parse pathological inputs return result; } - exponent_adjustment -= static_cast<int>(zeros_skipped); + exponent_adjustment -= zeros_skipped; } - int post_decimal_digits = ConsumeDigits<base>( + ptrdiff_t post_decimal_digits = ConsumeDigits<base>( begin, end, digits_left, &mantissa, &mantissa_is_inexact); begin += post_decimal_digits; @@ -482,15 +482,26 @@ return result; } - // Success! - result.type = strings_internal::FloatType::kNumber; if (result.mantissa > 0) { - result.exponent = result.literal_exponent + - (DigitMagnitude<base>() * exponent_adjustment); + const ptrdiff_t exponent = result.literal_exponent + + (DigitMagnitude<base>() * exponent_adjustment); + + if (exponent < (std::numeric_limits<int>::min)() || + exponent > (std::numeric_limits<int>::max)()) { + // We cannot store the exponent in int. Fail by returning a result with + // end default-initialized to nullptr. + return result; + } + + result.exponent = static_cast<int>(exponent); } else { result.exponent = 0; } result.end = begin; + + // Success! + result.type = strings_internal::FloatType::kNumber; + return result; }
diff --git a/absl/strings/internal/cord_internal.cc b/absl/strings/internal/cord_internal.cc index 57d9d38..becfc6a 100644 --- a/absl/strings/internal/cord_internal.cc +++ b/absl/strings/internal/cord_internal.cc
@@ -31,6 +31,10 @@ ABSL_CONST_INIT std::atomic<bool> shallow_subcords_enabled( kCordShallowSubcordsDefault); +void RefcountAndFlags::IncrementOverflow() { + ABSL_INTERNAL_LOG(FATAL, "refcount is too large and vulnerable to overflow"); +} + void LogFatalNodeType(CordRep* rep) { ABSL_INTERNAL_LOG(FATAL, absl::StrCat("Unexpected node type: ", static_cast<int>(rep->tag)));
diff --git a/absl/strings/internal/cord_internal.h b/absl/strings/internal/cord_internal.h index 27a8b9f..98dc4b5 100644 --- a/absl/strings/internal/cord_internal.h +++ b/absl/strings/internal/cord_internal.h
@@ -20,11 +20,13 @@ #include <cstddef> #include <cstdint> #include <cstring> +#include <limits> #include <string> #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/endian.h" +#include "absl/base/internal/raw_logging.h" #include "absl/base/macros.h" #include "absl/base/nullability.h" #include "absl/base/optimization.h" @@ -133,9 +135,18 @@ struct Immortal {}; explicit constexpr RefcountAndFlags(Immortal) : count_(kImmortalFlag) {} + static void IncrementOverflow(); + // Increments the reference count. Imposes no memory ordering. inline void Increment() { - count_.fetch_add(kRefIncrement, std::memory_order_relaxed); + const int32_t prev_count = + count_.fetch_add(kRefIncrement, std::memory_order_relaxed); + if (ABSL_PREDICT_FALSE( + prev_count >= + ((std::numeric_limits<decltype(count_)::value_type>::max)() / 3) * + 2)) { + IncrementOverflow(); + } } // Asserts that the current refcount is greater than 0. If the refcount is
diff --git a/absl/strings/internal/cord_rep_btree.cc b/absl/strings/internal/cord_rep_btree.cc index 33ea820..92e6450 100644 --- a/absl/strings/internal/cord_rep_btree.cc +++ b/absl/strings/internal/cord_rep_btree.cc
@@ -1114,7 +1114,10 @@ OpResult result = node->AddEdge<kBack>(true, edge, length); while (result.action == CordRepBtree::kPopped) { stack[height] = result.tree; - if (stack[++height] == nullptr) { + if (ABSL_PREDICT_FALSE(++height >= kMaxDepth)) { + ABSL_RAW_LOG(FATAL, "CordRepBtree::Rebuild() exceeded max depth"); + } + if (stack[height] == nullptr) { result.action = CordRepBtree::kSelf; stack[height] = CordRepBtree::New(node, result.tree); } else { @@ -1122,7 +1125,7 @@ result = node->AddEdge<kBack>(true, result.tree, length); } } - while (stack[++height] != nullptr) { + while (++height < kMaxDepth && stack[height] != nullptr) { stack[height]->length += length; } }
diff --git a/absl/strings/internal/generic_printer_test.cc b/absl/strings/internal/generic_printer_test.cc index 071bf76..be6e003 100644 --- a/absl/strings/internal/generic_printer_test.cc +++ b/absl/strings/internal/generic_printer_test.cc
@@ -15,6 +15,7 @@ #include "absl/strings/internal/generic_printer.h" #include <array> +#include <cinttypes> #include <clocale> #include <cstdint> #include <limits> @@ -38,6 +39,7 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" #include "absl/strings/substitute.h" namespace generic_logging_test { @@ -521,23 +523,26 @@ auto cp = std::make_unique<char*>(memory); EXPECT_THAT(GenericPrintToString(i), - AnyOf(Eq(absl::StrFormat("<%016X pointing to 5>", - reinterpret_cast<intptr_t>(&*i))), + AnyOf(Eq(absl::StrFormat("<%0*" PRIXPTR " pointing to 5>", + sizeof(void*) * 2, + reinterpret_cast<uintptr_t>(&*i))), Eq(absl::StrFormat("<%#x pointing to 5>", - reinterpret_cast<intptr_t>(&*i))))); + reinterpret_cast<uintptr_t>(&*i))))); EXPECT_THAT( GenericPrintToString(c), - AnyOf(HasSubstr(absl::StrFormat("<%016X pointing to 'z'", - reinterpret_cast<intptr_t>(&*c))), + AnyOf(HasSubstr(absl::StrFormat("<%0*" PRIXPTR " pointing to 'z'", + sizeof(void*) * 2, + reinterpret_cast<uintptr_t>(&*c))), HasSubstr(absl::StrFormat("<%#x pointing to 'z'", - reinterpret_cast<intptr_t>(&*c))))); + reinterpret_cast<uintptr_t>(&*c))))); EXPECT_THAT(GenericPrintToString(cp), - AnyOf(Eq(absl::StrFormat("<%016X pointing to abcdefg>", - reinterpret_cast<intptr_t>(&*cp))), + AnyOf(Eq(absl::StrFormat("<%0*" PRIXPTR " pointing to abcdefg>", + sizeof(void*) * 2, + reinterpret_cast<uintptr_t>(&*cp))), Eq(absl::StrFormat("<%#x pointing to abcdefg>", - reinterpret_cast<intptr_t>(&*cp))))); + reinterpret_cast<uintptr_t>(&*cp))))); } TEST(GenericPrinterTest, SmartPointerToArrayOnlyPrintsAddressAndHelpText) { @@ -552,18 +557,20 @@ EXPECT_THAT( GenericPrintToString(nonempty), AllOf(AnyOf(HasSubstr(absl::StrFormat( - "%016X", reinterpret_cast<intptr_t>(nonempty.get()))), + "%0*" PRIXPTR, sizeof(void*) * 2, + reinterpret_cast<uintptr_t>(nonempty.get()))), HasSubstr(absl::StrFormat( - "%#x", reinterpret_cast<intptr_t>(nonempty.get())))), + "%#x", reinterpret_cast<uintptr_t>(nonempty.get())))), HasSubstr("array"), Not(HasSubstr("to 54321")), Not(HasSubstr("to 12345")))); EXPECT_THAT( GenericPrintToString(empty), AllOf(AnyOf(HasSubstr(absl::StrFormat( - "%016X", reinterpret_cast<intptr_t>(empty.get()))), + "%0*" PRIXPTR, sizeof(void*) * 2, + reinterpret_cast<uintptr_t>(empty.get()))), HasSubstr(absl::StrFormat( - "%#x", reinterpret_cast<intptr_t>(empty.get())))), + "%#x", reinterpret_cast<uintptr_t>(empty.get())))), HasSubstr("array"))); }
diff --git a/absl/strings/internal/stl_type_traits.h b/absl/strings/internal/stl_type_traits.h deleted file mode 100644 index 6298921..0000000 --- a/absl/strings/internal/stl_type_traits.h +++ /dev/null
@@ -1,143 +0,0 @@ -// Copyright 2017 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -// The file provides the IsStrictlyBaseOfAndConvertibleToSTLContainer type -// trait metafunction to assist in working with the _GLIBCXX_DEBUG debug -// wrappers of STL containers. -// -// DO NOT INCLUDE THIS FILE DIRECTLY. Use this file by including -// absl/strings/str_split.h. -// -// IWYU pragma: private, include "absl/strings/str_split.h" - -#ifndef ABSL_STRINGS_INTERNAL_STL_TYPE_TRAITS_H_ -#define ABSL_STRINGS_INTERNAL_STL_TYPE_TRAITS_H_ - -#include <stddef.h> - -#include <array> -#include <bitset> -#include <deque> -#include <forward_list> -#include <list> -#include <map> -#include <set> -#include <type_traits> -#include <unordered_map> -#include <unordered_set> -#include <vector> - -#include "absl/meta/type_traits.h" - -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace strings_internal { - -template <typename To, typename From> -using IsCastableToDerivedSTLContainer = - std::enable_if_t<!std::is_same_v<From, To>, std::is_convertible<From, To>>; - -template <typename C> -std::false_type CastableToDerivedSTLContainer(C*); - -template <typename C, typename V, size_t N> -IsCastableToDerivedSTLContainer<C, std::array<typename C::value_type, N>> -CastableToDerivedSTLContainer(std::array<V, N>*); - -template <typename C, size_t N> -IsCastableToDerivedSTLContainer<C, std::bitset<N>> -CastableToDerivedSTLContainer(std::bitset<N>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, std::deque<typename C::value_type, typename C::allocator_type>> -CastableToDerivedSTLContainer(std::deque<U...>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, std::forward_list<typename C::value_type, typename C::allocator_type>> -CastableToDerivedSTLContainer(std::forward_list<U...>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, std::list<typename C::value_type, typename C::allocator_type>> -CastableToDerivedSTLContainer(std::list<U...>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, std::map<typename C::key_type, typename C::mapped_type, - typename C::key_compare, typename C::allocator_type>> -CastableToDerivedSTLContainer(std::map<U...>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, std::multimap<typename C::key_type, typename C::mapped_type, - typename C::key_compare, typename C::allocator_type>> -CastableToDerivedSTLContainer(std::multimap<U...>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, std::set<typename C::value_type, typename C::key_compare, - typename C::allocator_type>> -CastableToDerivedSTLContainer(std::set<U...>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, std::multiset<typename C::value_type, typename C::key_compare, - typename C::allocator_type>> -CastableToDerivedSTLContainer(std::multiset<U...>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, std::unordered_map<typename C::key_type, typename C::mapped_type, - typename C::hasher, typename C::key_equal, - typename C::allocator_type>> -CastableToDerivedSTLContainer(std::unordered_map<U...>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, std::unordered_multimap<typename C::key_type, typename C::mapped_type, - typename C::hasher, typename C::key_equal, - typename C::allocator_type>> -CastableToDerivedSTLContainer(std::unordered_multimap<U...>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, std::unordered_set<typename C::key_type, typename C::hasher, - typename C::key_equal, typename C::allocator_type>> -CastableToDerivedSTLContainer(std::unordered_set<U...>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, - std::unordered_multiset<typename C::key_type, typename C::hasher, - typename C::key_equal, typename C::allocator_type>> -CastableToDerivedSTLContainer(std::unordered_multiset<U...>*); - -template <typename C, typename... U> -IsCastableToDerivedSTLContainer< - C, std::vector<typename C::value_type, typename C::allocator_type>> -CastableToDerivedSTLContainer(std::vector<U...>*); - -template <typename C> -struct IsStrictlyBaseOfAndConvertibleToSTLContainer - : decltype(strings_internal::CastableToDerivedSTLContainer< - absl::remove_cvref_t<C>>( - std::declval<absl::remove_cvref_t<C>*>())) {}; - -} // namespace strings_internal -ABSL_NAMESPACE_END -} // namespace absl -#endif // ABSL_STRINGS_INTERNAL_STL_TYPE_TRAITS_H_
diff --git a/absl/strings/internal/str_format/arg.cc b/absl/strings/internal/str_format/arg.cc index a51f7d7..0687b4e 100644 --- a/absl/strings/internal/str_format/arg.cc +++ b/absl/strings/internal/str_format/arg.cc
@@ -25,6 +25,7 @@ #include <cstdlib> #include <cstring> #include <cwchar> +#include <limits> #include <string> #include <string_view> #include <type_traits> @@ -309,19 +310,38 @@ conv.has_left_flag()); } +inline bool IsLowSurrogate(uint32_t c) { return c >= 0xDC00 && c <= 0xDFFF; } + inline bool ConvertStringArg(const wchar_t *v, size_t len, const FormatConversionSpecImpl conv, FormatSinkImpl *sink) { - FixedArray<char> mb(len * 4); + // Each wide character may result in up to 4 bytes (UTF-8 code units). + constexpr size_t kMaxUtf8CodeUnitsPerWideChar = 4; + if (len > (std::numeric_limits<decltype(len)>::max)() / + kMaxUtf8CodeUnitsPerWideChar) { + // Size too large; we can't handle this. + return false; + } + FixedArray<char> mb(len * kMaxUtf8CodeUnitsPerWideChar); strings_internal::ShiftState s; size_t chars_written = 0; for (size_t i = 0; i < len; ++i) { + // A high surrogate must be immediately followed by a low surrogate. If it + // isn't, the UTF-16 input is malformed and WideToUtf8() would otherwise + // leave a partial sequence in the buffer. The single wchar_t path already + // rejects an unpaired surrogate, so reject it here too. + if (s.saw_high_surrogate) { + const uint32_t cu = static_cast<uint32_t>(v[i]); + if (!IsLowSurrogate(cu)) return false; + } const size_t chars = strings_internal::WideToUtf8(v[i], &mb[chars_written], s); if (chars == static_cast<size_t>(-1)) { return false; } chars_written += chars; } + // A trailing high surrogate has no low surrogate to complete it. + if (s.saw_high_surrogate) return false; return ConvertStringArg(string_view(mb.data(), chars_written), conv, sink); }
diff --git a/absl/strings/internal/str_format/convert_test.cc b/absl/strings/internal/str_format/convert_test.cc index 1c3d1a3..5e86016 100644 --- a/absl/strings/internal/str_format/convert_test.cc +++ b/absl/strings/internal/str_format/convert_test.cc
@@ -357,6 +357,44 @@ EXPECT_EQ("ABC", FormatPack(wformat2, {FormatArgImpl(wp)})); } +TEST_F(FormatConvertTest, WideStringUnpairedSurrogate) { + // The single wchar_t ("%lc") path rejects an unpaired surrogate. The wide + // string ("%ls") path should reject it too rather than emitting a partial + // UTF-8 sequence. A failed conversion yields an empty result. + auto format_ls = [](const std::wstring& ws) { + UntypedFormatSpecImpl format("%ls"); + return FormatPack(format, {FormatArgImpl(ws)}); + }; + + // A well-formed surrogate pair (U+10000) still converts. + std::wstring pair; + pair.push_back(static_cast<wchar_t>(0xD800)); + pair.push_back(static_cast<wchar_t>(0xDC00)); + EXPECT_EQ("\xF0\x90\x80\x80", format_ls(pair)); + + // Trailing high surrogate with no low surrogate to complete it. + std::wstring trailing_high; + trailing_high.push_back(static_cast<wchar_t>(0xD800)); + EXPECT_EQ("", format_ls(trailing_high)); + + // High surrogate followed by a non-surrogate. + std::wstring high_then_ascii; + high_then_ascii.push_back(static_cast<wchar_t>(0xD800)); + high_then_ascii.push_back(L'A'); + EXPECT_EQ("", format_ls(high_then_ascii)); + + // High surrogate followed by another high surrogate. + std::wstring high_then_high; + high_then_high.push_back(static_cast<wchar_t>(0xD800)); + high_then_high.push_back(static_cast<wchar_t>(0xD800)); + EXPECT_EQ("", format_ls(high_then_high)); + + // Isolated low surrogate. + std::wstring lone_low; + lone_low.push_back(static_cast<wchar_t>(0xDC00)); + EXPECT_EQ("", format_ls(lone_low)); +} + // Pointer formatting is implementation defined. This checks that the argument // can be matched to `ptr`. MATCHER_P(MatchesPointerString, ptr, "") {
diff --git a/absl/strings/internal/str_split_internal.h b/absl/strings/internal/str_split_internal.h index 1cec468..bda084b 100644 --- a/absl/strings/internal/str_split_internal.h +++ b/absl/strings/internal/str_split_internal.h
@@ -44,10 +44,6 @@ #include "absl/meta/type_traits.h" #include "absl/strings/string_view.h" -#ifdef _GLIBCXX_DEBUG -#include "absl/strings/internal/stl_type_traits.h" -#endif // _GLIBCXX_DEBUG - namespace absl { ABSL_NAMESPACE_BEGIN namespace strings_internal { @@ -231,13 +227,9 @@ struct SplitterIsConvertibleTo : SplitterIsConvertibleToImpl< C, -#ifdef _GLIBCXX_DEBUG - !IsStrictlyBaseOfAndConvertibleToSTLContainer<C>::value && -#endif // _GLIBCXX_DEBUG - !IsInitializerList<std::remove_reference_t<C>>::value && + !IsInitializerList<std::remove_reference_t<C>>::value && HasValueType<C>::value && HasConstIterator<C>::value, - HasMappedType<C>::value> { -}; + HasMappedType<C>::value> {}; template <typename StringType, typename Container, typename = void> struct ShouldUseLifetimeBound : std::false_type {};
diff --git a/absl/strings/str_cat.cc b/absl/strings/str_cat.cc index 546b8ae..b79078d 100644 --- a/absl/strings/str_cat.cc +++ b/absl/strings/str_cat.cc
@@ -26,6 +26,8 @@ #include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/nullability.h" +#include "absl/base/optimization.h" +#include "absl/base/throw_delegate.h" #include "absl/strings/internal/append_and_overwrite.h" #include "absl/strings/resize_and_overwrite.h" #include "absl/strings/string_view.h" @@ -53,6 +55,26 @@ return after; } +// Safely adds size_t values, throwing std::length_error if overflow occurs. +inline size_t SafeAdd(size_t a, size_t b) { + const uint64_t sum = static_cast<uint64_t>(a) + b; + if (ABSL_PREDICT_FALSE(sum > (std::numeric_limits<size_t>::max)())) { + ThrowStdLengthError("absl string append length overflow"); + } + return static_cast<size_t>(sum); +} + +inline size_t SafeAdd(std::initializer_list<size_t> sizes) { + uint64_t sum = 0; + for (size_t size : sizes) { + sum += size; + } + if (ABSL_PREDICT_FALSE(sum > (std::numeric_limits<size_t>::max)())) { + ThrowStdLengthError("absl string append length overflow"); + } + return static_cast<size_t>(sum); +} + } // namespace std::string StrCat(const AlphaNum& a, const AlphaNum& b) { @@ -163,7 +185,7 @@ size_t to_append = 0; for (absl::string_view piece : pieces) { ASSERT_NO_OVERLAP(*dest, piece); - to_append += piece.size(); + to_append = SafeAdd(to_append, piece.size()); } StringAppendAndOverwrite(*dest, to_append, [&pieces](char* const buf, size_t buf_size) { @@ -198,7 +220,8 @@ ASSERT_NO_OVERLAP(*dest, a); ASSERT_NO_OVERLAP(*dest, b); strings_internal::StringAppendAndOverwrite( - *dest, a.size() + b.size(), [&a, &b](char* const buf, size_t buf_size) { + *dest, SafeAdd(a.size(), b.size()), + [&a, &b](char* const buf, size_t buf_size) { char* out = buf; out = Append(out, a); out = Append(out, b); @@ -213,7 +236,7 @@ ASSERT_NO_OVERLAP(*dest, b); ASSERT_NO_OVERLAP(*dest, c); strings_internal::StringAppendAndOverwrite( - *dest, a.size() + b.size() + c.size(), + *dest, SafeAdd({a.size(), b.size(), c.size()}), [&a, &b, &c](char* const buf, size_t buf_size) { char* out = buf; out = Append(out, a); @@ -231,7 +254,7 @@ ASSERT_NO_OVERLAP(*dest, c); ASSERT_NO_OVERLAP(*dest, d); strings_internal::StringAppendAndOverwrite( - *dest, a.size() + b.size() + c.size() + d.size(), + *dest, SafeAdd({a.size(), b.size(), c.size(), d.size()}), [&a, &b, &c, &d](char* const buf, size_t buf_size) { char* out = buf; out = Append(out, a);
diff --git a/absl/strings/substitute.cc b/absl/strings/substitute.cc index f5d600b..244c8a5 100644 --- a/absl/strings/substitute.cc +++ b/absl/strings/substitute.cc
@@ -20,6 +20,7 @@ #include <cstdint> #include <limits> #include <string> +#include <type_traits> #include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" @@ -35,6 +36,19 @@ ABSL_NAMESPACE_BEGIN namespace substitute_internal { +// Checked addition to avoid overflow on 32-bit. (In 64-bit the callers wouldn't +// get anywhere close to the limit given that 2^64 bytes of memory is beyond +// anything physically possible.) +// Note that we don't declare the parameters as size_t in order to avoid +// accidental implicit conversions. +template <int&..., typename T> +[[nodiscard]] std::enable_if_t<std::is_same_v<T, size_t>, T> CheckedAdd(T a, + T b) { + ABSL_INTERNAL_CHECK(b <= (std::numeric_limits<T>::max)() - a, + "unsigned integer overflow"); + return a + b; +} + void SubstituteAndAppendArray(std::string* absl_nonnull output, absl::string_view format, const absl::string_view* absl_nullable args_array, @@ -64,10 +78,10 @@ #endif return; } - size += args_array[index].size(); + size = CheckedAdd(size, args_array[index].size()); ++i; // Skip next char. } else if (format[i + 1] == '$') { - ++size; + size = CheckedAdd(size, size_t{1}); ++i; // Skip next char. } else { #ifndef NDEBUG @@ -78,7 +92,7 @@ return; } } else { - ++size; + size = CheckedAdd(size, size_t{1}); } }
diff --git a/absl/strings/substitute_test.cc b/absl/strings/substitute_test.cc index 70f9119..e62a89a 100644 --- a/absl/strings/substitute_test.cc +++ b/absl/strings/substitute_test.cc
@@ -16,6 +16,7 @@ #include <cstdint> #include <cstring> +#include <limits> #include <string> #include <vector> @@ -283,6 +284,17 @@ "Invalid absl::Substitute\\(\\) format string: \"-\\$\""); } +TEST(SubstituteDeathTest, OverflowDeath) { + // HACK: We pretend the string_view is extremely long, in order to test an + // overflow condition that only occurs in 32-bit without using an impractical + // amount of memory. + EXPECT_DEATH(static_cast<void>(absl::Substitute( + "$0$0$0", + absl::string_view( + "abc", (std::numeric_limits<size_t>::max() / 3) + 100))), + "overflow"); +} + #endif // GTEST_HAS_DEATH_TEST } // namespace
diff --git a/absl/synchronization/BUILD.bazel b/absl/synchronization/BUILD.bazel index 583eff2..942fc03 100644 --- a/absl/synchronization/BUILD.bazel +++ b/absl/synchronization/BUILD.bazel
@@ -65,7 +65,8 @@ copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", + "@do_not_use_for_gloop_visibility_only//gloop/thread:__subpackages__", ], deps = [ "//absl/base", @@ -296,7 +297,6 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", ], deps = [ ":synchronization", @@ -345,7 +345,7 @@ copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ - "//absl:friends", + "@do_not_use_for_gloop_visibility_only//gloop/base:__subpackages__", ], deps = [ ":synchronization",
diff --git a/absl/synchronization/mutex.h b/absl/synchronization/mutex.h index ad156d4..03cce89 100644 --- a/absl/synchronization/mutex.h +++ b/absl/synchronization/mutex.h
@@ -621,7 +621,7 @@ // Calls `mu.lock()` and returns when that call returns. That is, `mu` is // guaranteed to be locked when this object is constructed. - explicit MutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + explicit MutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { this->mu_.lock(); @@ -638,7 +638,7 @@ // Like above, but calls `mu.LockWhen(cond)` instead. That is, in addition to // the above, the condition given by `cond` is also guaranteed to hold when // this object is constructed. - explicit MutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + explicit MutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS, const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { this->mu_.LockWhen(cond); @@ -667,7 +667,7 @@ // releases a shared lock on a `Mutex` via RAII. class ABSL_SCOPED_LOCKABLE ReaderMutexLock { public: - explicit ReaderMutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + explicit ReaderMutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) ABSL_SHARED_LOCK_FUNCTION(mu) : mu_(mu) { mu.lock_shared(); @@ -678,7 +678,7 @@ explicit ReaderMutexLock(Mutex* absl_nonnull mu) ABSL_SHARED_LOCK_FUNCTION(mu) : ReaderMutexLock(*mu) {} - explicit ReaderMutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + explicit ReaderMutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS, const Condition& cond) ABSL_SHARED_LOCK_FUNCTION(mu) : mu_(mu) { mu.ReaderLockWhen(cond); @@ -707,7 +707,7 @@ // releases a write (exclusive) lock on a `Mutex` via RAII. class ABSL_SCOPED_LOCKABLE WriterMutexLock { public: - explicit WriterMutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this)) + explicit WriterMutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { mu.lock(); @@ -719,7 +719,7 @@ ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : WriterMutexLock(*mu) {} - explicit WriterMutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + explicit WriterMutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS, const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { @@ -1142,8 +1142,9 @@ // mutex before destruction. `Release()` may be called at most once. class ABSL_SCOPED_LOCKABLE ReleasableMutexLock { public: - explicit ReleasableMutexLock(Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY( - this)) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) + explicit ReleasableMutexLock( + Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS) + ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(&mu) { this->mu_->lock(); } @@ -1155,7 +1156,7 @@ : ReleasableMutexLock(*mu) {} explicit ReleasableMutexLock( - Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY(this), + Mutex& mu ABSL_INTERNAL_ATTRIBUTE_CAPTURED_BY_THIS, const Condition& cond) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(&mu) { this->mu_->LockWhen(cond);
diff --git a/absl/time/BUILD.bazel b/absl/time/BUILD.bazel index a182c65..cca77da 100644 --- a/absl/time/BUILD.bazel +++ b/absl/time/BUILD.bazel
@@ -25,7 +25,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -53,6 +53,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base", "//absl/base:config", @@ -77,6 +78,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":time", "//absl/base:config", @@ -99,6 +101,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ ":clock_interface", ":time",
diff --git a/absl/time/civil_time.cc b/absl/time/civil_time.cc index 1773366..eae7e7f 100644 --- a/absl/time/civil_time.cc +++ b/absl/time/civil_time.cc
@@ -55,13 +55,20 @@ const civil_year_t y = std::strtoll(np, &endp, 10); // NOLINT(runtime/deprecated_fn) if (endp == np || errno == ERANGE) return false; - const std::string norm = StrCat(NormalizeYear(y), endp); + const civil_year_t normalized_year = NormalizeYear(y); + const std::string norm = StrCat(normalized_year, endp); const TimeZone utc = UTCTimeZone(); Time t; if (ParseTime(StrCat("%Y", fmt), norm, utc, &t, nullptr)) { const auto cs = ToCivilSecond(t, utc); - *c = CivilT(y, cs.month(), cs.day(), cs.hour(), cs.minute(), cs.second()); + // Field normalization while parsing (e.g. a ":60" leap second or an + // end-of-year rollover) can carry into the year. The other fields are taken + // from `cs`, so the same carry must be applied to the original year; + // otherwise the reconstructed value would use the wrong (un-carried) year. + const civil_year_t year = y + (cs.year() - normalized_year); + *c = + CivilT(year, cs.month(), cs.day(), cs.hour(), cs.minute(), cs.second()); return true; }
diff --git a/absl/time/civil_time.h b/absl/time/civil_time.h index d198eba..d1d0d95 100644 --- a/absl/time/civil_time.h +++ b/absl/time/civil_time.h
@@ -509,6 +509,13 @@ // absl::CivilDay d; // bool ok = absl::ParseCivilTime("2018-01-02", &d); // OK // +// Parsing tolerates the following variations from the standard format: +// * Leading and trailing whitespace is ignored. +// * The year component may be negative (prefixed with '-') and may contain +// an arbitrary number of digits. +// * Sub-year components (month, day, hour, minute, second) may consist of +// either one or two digits. +// // Note that parsing will fail if the string's format does not match the // expected type exactly. `ParseLenientCivilTime()` below is more lenient. // @@ -521,9 +528,12 @@ // ParseLenientCivilTime() // -// Parses any of the formats accepted by `absl::ParseCivilTime()`, but is more -// lenient if the format of the string does not exactly match the associated -// type. +// Parses any of the formats accepted by `absl::ParseCivilTime()`. Unlike +// `ParseCivilTime()`, the input string format does not need to match the +// target civil-time type. Discrepancies are resolved as follows: +// * Extra components in the input string are ignored. +// * Missing components are defaulted to their minimum valid values. +// This behavior is consistent with civil-time converting constructors. // // Example: //
diff --git a/absl/time/civil_time_test.cc b/absl/time/civil_time_test.cc index 59c04d3..3ad8e14 100644 --- a/absl/time/civil_time_test.cc +++ b/absl/time/civil_time_test.cc
@@ -749,7 +749,7 @@ EXPECT_EQ("2015", absl::FormatCivilTime(y)); } -TEST(CivilTime, ParseEdgeCases) { +TEST(CivilTime, ParseLenientEdgeCases) { absl::CivilSecond ss; EXPECT_TRUE( absl::ParseLenientCivilTime("9223372036854775807-12-31T23:59:59", &ss)); @@ -825,6 +825,50 @@ EXPECT_FALSE(absl::ParseLenientCivilTime("9223372036854775808", &y)) << y; } +TEST(CivilTime, ParseEdgeCases) { + absl::CivilYear y; + absl::CivilMonth m; + absl::CivilDay d; + absl::CivilSecond ss; + EXPECT_TRUE(absl::ParseCivilTime("0", &y)) << y; + EXPECT_EQ(absl::CivilYear(0), y); + EXPECT_TRUE(absl::ParseCivilTime("0-1", &m)) << m; + EXPECT_EQ(absl::CivilMonth(0, 1), m); + EXPECT_TRUE(absl::ParseCivilTime(" 2015 ", &y)) << y; + EXPECT_EQ(absl::CivilYear(2015), y); + EXPECT_TRUE(absl::ParseCivilTime( + "000000000000000000000000000000000000000000000000000000000000002015", &y)) + << y; + EXPECT_EQ(absl::CivilYear(2015), y); + EXPECT_TRUE(absl::ParseCivilTime(" 2015-6 ", &m)) << m; + EXPECT_EQ(absl::CivilMonth(2015, 6), m); + EXPECT_TRUE(absl::ParseCivilTime("0002015-6-7", &d)) << d; + EXPECT_EQ(absl::CivilDay(2015, 6, 7), d); + EXPECT_TRUE(absl::ParseCivilTime("2015-06-07T10:11:12 ", &ss)) << ss; + EXPECT_EQ(absl::CivilSecond(2015, 6, 7, 10, 11, 12), ss); + EXPECT_TRUE(absl::ParseCivilTime(" 2015-06-07T10:11:1 ", &ss)) << ss; + EXPECT_EQ(absl::CivilSecond(2015, 6, 7, 10, 11, 1), ss); + EXPECT_TRUE(absl::ParseCivilTime("-01-01", &m)) << m; + EXPECT_EQ(absl::CivilMonth(-1, 1), m); +} + +TEST(CivilTime, ParseFieldNormalizationCarriesYear) { + // When a field normalizes past the end of the year (e.g. a ":60" leap + // second on the last second of December), the carry must be reflected in + // the parsed year, so parsing agrees with direct field construction. + absl::CivilSecond ss; + EXPECT_TRUE(absl::ParseCivilTime("2020-12-31T23:59:60", &ss)) << ss; + EXPECT_EQ(absl::CivilSecond(2020, 12, 31, 23, 59, 60), ss); + EXPECT_EQ(absl::CivilSecond(2021, 1, 1, 0, 0, 0), ss); + + EXPECT_TRUE(absl::ParseLenientCivilTime("2020-12-31T23:59:60", &ss)) << ss; + EXPECT_EQ(absl::CivilSecond(2021, 1, 1, 0, 0, 0), ss); + + // The carry also works for negative years crossing zero. + EXPECT_TRUE(absl::ParseCivilTime("-1-12-31T23:59:60", &ss)) << ss; + EXPECT_EQ(absl::CivilSecond(0, 1, 1, 0, 0, 0), ss); +} + TEST(CivilTime, AbslStringify) { EXPECT_EQ("2015-01-02T03:04:05", absl::StrFormat("%v", absl::CivilSecond(2015, 1, 2, 3, 4, 5)));
diff --git a/absl/time/internal/cctz/BUILD.bazel b/absl/time/internal/cctz/BUILD.bazel index e7e2ee0..3b47877 100644 --- a/absl/time/internal/cctz/BUILD.bazel +++ b/absl/time/internal/cctz/BUILD.bazel
@@ -160,6 +160,23 @@ ], ) +cc_test( + name = "time_zone_name_win_test", + size = "small", + srcs = select({ + "@platforms//os:windows": ["src/time_zone_name_win_test.cc"], + "//conditions:default": [], + }), + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":time_zone", + "//absl/base:config", + "@googletest//:gtest", + "@googletest//:gtest_main", + ], +) + ### benchmarks cc_test(
diff --git a/absl/time/internal/cctz/src/time_zone_format.cc b/absl/time/internal/cctz/src/time_zone_format.cc index 4c8d2f9..91b4621 100644 --- a/absl/time/internal/cctz/src/time_zone_format.cc +++ b/absl/time/internal/cctz/src/time_zone_format.cc
@@ -21,7 +21,8 @@ #endif #if HAS_STRPTIME -#if !defined(_XOPEN_SOURCE) && !defined(__FreeBSD__) && !defined(__OpenBSD__) +#if !defined(_XOPEN_SOURCE) && !defined(__FreeBSD__) && \ + !defined(__OpenBSD__) && !defined(__APPLE__) #define _XOPEN_SOURCE 500 // Exposes definitions for SUSv2 (UNIX 98). #endif #endif
diff --git a/absl/time/internal/cctz/src/time_zone_name_win.cc b/absl/time/internal/cctz/src/time_zone_name_win.cc index 07aa8fa..7a1236f 100644 --- a/absl/time/internal/cctz/src/time_zone_name_win.cc +++ b/absl/time/internal/cctz/src/time_zone_name_win.cc
@@ -118,7 +118,7 @@ const auto ucal_getTimeZoneIDForWindowsIDRef = AsProcAddress<ucal_getTimeZoneIDForWindowsID_func>( icu_dll, "ucal_getTimeZoneIDForWindowsID"); - if (ucal_getTimeZoneIDForWindowsIDRef != nullptr) { + if (ucal_getTimeZoneIDForWindowsIDRef == nullptr) { g_unavailable.store(true, std::memory_order_relaxed); return nullptr; }
diff --git a/absl/time/internal/cctz/src/time_zone_name_win_test.cc b/absl/time/internal/cctz/src/time_zone_name_win_test.cc new file mode 100644 index 0000000..260c972 --- /dev/null +++ b/absl/time/internal/cctz/src/time_zone_name_win_test.cc
@@ -0,0 +1,49 @@ +// Copyright 2026 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/internal/cctz/src/time_zone_name_win.h" + +#include <windows.h> + +#include <string> + +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +TEST(TimeZoneNameWin, GetWindowsLocalTimeZone) { + // On Windows 10 1809+ (where icu.dll is available in System32), + // GetWindowsLocalTimeZone() should return a valid IANA time zone name. + // Note that LOAD_LIBRARY_SEARCH_SYSTEM32 is not sufficient to reliably load + // "icu.dll" in the production code, but it should be OK for testing purposes. + HMODULE icu_dll = + ::LoadLibraryExW(L"icu.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32); + const std::string tz = GetWindowsLocalTimeZone(); + if (icu_dll != nullptr) { + EXPECT_FALSE(tz.empty()); + ::FreeLibrary(icu_dll); + } else { + EXPECT_TRUE(tz.empty()); + } +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/types/BUILD.bazel b/absl/types/BUILD.bazel index 1d937e7..d38d4c7 100644 --- a/absl/types/BUILD.bazel +++ b/absl/types/BUILD.bazel
@@ -24,7 +24,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -39,6 +39,7 @@ hdrs = ["any.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -52,6 +53,7 @@ hdrs = ["source_location.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -84,6 +86,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/algorithm", "//absl/base:config", @@ -127,6 +130,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -182,6 +186,7 @@ hdrs = ["optional.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -194,6 +199,7 @@ hdrs = ["variant.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -219,6 +225,7 @@ hdrs = ["compare.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -246,6 +253,7 @@ hdrs = ["optional_ref.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers",
diff --git a/absl/types/any_span.h b/absl/types/any_span.h index bca47bc..04e64dc 100644 --- a/absl/types/any_span.h +++ b/absl/types/any_span.h
@@ -702,29 +702,22 @@ // subspan, but both the container and transform must remain valid. // pos must be non-negative and <= size(). // len must be non-negative and <= size() - pos, or equal to npos. - // If len == npos, the subspan continues till the end of this span. - - constexpr AnySpan subspan(size_type pos, size_type len) const { + // If len==npos, the subspan continues till the end of this span. + constexpr AnySpan subspan(size_type pos, size_type len = npos) const { const size_t this_size = size(); if (len == AnySpan<T>::npos) { len = this_size - pos; } absl::base_internal::HardeningAssertLE(pos, this_size); - absl::base_internal::HardeningAssertLE(len, - static_cast<size_type>(this_size - - pos)); + absl::base_internal::HardeningAssertLE( + len, static_cast<size_type>(this_size - pos)); return AnySpan<T>(getter_.Offset(pos), len); } - constexpr AnySpan subspan(size_type pos) const { - absl::base_internal::HardeningAssertLE(pos, size()); - return AnySpan(getter_.Offset(pos), size() - pos); - } - - // Returns a `AnySpan` containing first `len` elements. Parameter `len` - // must be non-negative and <= size(). + // Returns a `AnySpan` containing first `len` elements. Parameter `len` must + // be non-negative and <= size(). constexpr AnySpan first(size_type len) const { - absl::base_internal::HardeningAssert(len != AnySpan<T>::npos); + absl::base_internal::HardeningAssert(len != npos); return subspan(0, len); }
diff --git a/absl/utility/BUILD.bazel b/absl/utility/BUILD.bazel index 1994a01..85e1817 100644 --- a/absl/utility/BUILD.bazel +++ b/absl/utility/BUILD.bazel
@@ -22,7 +22,7 @@ ) package( - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:private"], features = [ "header_modules", "layering_check", @@ -39,6 +39,7 @@ ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], deps = [ "//absl/base:config", "//absl/base:core_headers",