diff --git a/CMake/AbseilDll.cmake b/CMake/AbseilDll.cmake index 253c73f..8ee4120 100644 --- a/CMake/AbseilDll.cmake +++ b/CMake/AbseilDll.cmake
@@ -197,16 +197,27 @@ "strings/cord.h" "strings/escaping.cc" "strings/escaping.h" + "strings/internal/charconv_bigint.cc" + "strings/internal/charconv_bigint.h" + "strings/internal/charconv_parse.cc" + "strings/internal/charconv_parse.h" "strings/internal/cord_internal.cc" "strings/internal/cord_internal.h" "strings/internal/cord_rep_flat.h" "strings/internal/cord_rep_ring.cc" "strings/internal/cord_rep_ring.h" "strings/internal/cord_rep_ring_reader.h" - "strings/internal/charconv_bigint.cc" - "strings/internal/charconv_bigint.h" - "strings/internal/charconv_parse.cc" - "strings/internal/charconv_parse.h" + "strings/internal/cordz_functions.cc" + "strings/internal/cordz_functions.h" + "strings/internal/cordz_handle.cc" + "strings/internal/cordz_handle.h" + "strings/internal/cordz_info.cc" + "strings/internal/cordz_info.h" + "strings/internal/cordz_sample_token.cc" + "strings/internal/cordz_sample_token.h" + "strings/internal/cordz_statistics.h" + "strings/internal/cordz_update_scope.h" + "strings/internal/cordz_update_tracker.h" "strings/internal/stl_type_traits.h" "strings/internal/string_constant.h" "strings/match.cc"
diff --git a/absl/algorithm/container.h b/absl/algorithm/container.h index 6398438..1652e7b 100644 --- a/absl/algorithm/container.h +++ b/absl/algorithm/container.h
@@ -905,11 +905,11 @@ // Overload of c_sort() for performing a `comp` comparison other than the // default `operator<`. -template <typename C, typename Compare> -void c_sort(C& c, Compare&& comp) { +template <typename C, typename LessThan> +void c_sort(C& c, LessThan&& comp) { std::sort(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_stable_sort() @@ -925,11 +925,11 @@ // Overload of c_stable_sort() for performing a `comp` comparison other than the // default `operator<`. -template <typename C, typename Compare> -void c_stable_sort(C& c, Compare&& comp) { +template <typename C, typename LessThan> +void c_stable_sort(C& c, LessThan&& comp) { std::stable_sort(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_is_sorted() @@ -944,11 +944,11 @@ // c_is_sorted() overload for performing a `comp` comparison other than the // default `operator<`. -template <typename C, typename Compare> -bool c_is_sorted(const C& c, Compare&& comp) { +template <typename C, typename LessThan> +bool c_is_sorted(const C& c, LessThan&& comp) { return std::is_sorted(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_partial_sort() @@ -966,14 +966,14 @@ // Overload of c_partial_sort() for performing a `comp` comparison other than // the default `operator<`. -template <typename RandomAccessContainer, typename Compare> +template <typename RandomAccessContainer, typename LessThan> void c_partial_sort( RandomAccessContainer& sequence, container_algorithm_internal::ContainerIter<RandomAccessContainer> middle, - Compare&& comp) { + LessThan&& comp) { std::partial_sort(container_algorithm_internal::c_begin(sequence), middle, container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_partial_sort_copy() @@ -994,15 +994,15 @@ // Overload of c_partial_sort_copy() for performing a `comp` comparison other // than the default `operator<`. -template <typename C, typename RandomAccessContainer, typename Compare> +template <typename C, typename RandomAccessContainer, typename LessThan> container_algorithm_internal::ContainerIter<RandomAccessContainer> c_partial_sort_copy(const C& sequence, RandomAccessContainer& result, - Compare&& comp) { + LessThan&& comp) { return std::partial_sort_copy(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), container_algorithm_internal::c_begin(result), container_algorithm_internal::c_end(result), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_is_sorted_until() @@ -1018,12 +1018,12 @@ // Overload of c_is_sorted_until() for performing a `comp` comparison other than // the default `operator<`. -template <typename C, typename Compare> +template <typename C, typename LessThan> container_algorithm_internal::ContainerIter<C> c_is_sorted_until( - C& c, Compare&& comp) { + C& c, LessThan&& comp) { return std::is_sorted_until(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_nth_element() @@ -1043,14 +1043,14 @@ // Overload of c_nth_element() for performing a `comp` comparison other than // the default `operator<`. -template <typename RandomAccessContainer, typename Compare> +template <typename RandomAccessContainer, typename LessThan> void c_nth_element( RandomAccessContainer& sequence, container_algorithm_internal::ContainerIter<RandomAccessContainer> nth, - Compare&& comp) { + LessThan&& comp) { std::nth_element(container_algorithm_internal::c_begin(sequence), nth, container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------ @@ -1072,12 +1072,12 @@ // Overload of c_lower_bound() for performing a `comp` comparison other than // the default `operator<`. -template <typename Sequence, typename T, typename Compare> +template <typename Sequence, typename T, typename LessThan> container_algorithm_internal::ContainerIter<Sequence> c_lower_bound( - Sequence& sequence, T&& value, Compare&& comp) { + Sequence& sequence, T&& value, LessThan&& comp) { return std::lower_bound(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<T>(value), std::forward<Compare>(comp)); + std::forward<T>(value), std::forward<LessThan>(comp)); } // c_upper_bound() @@ -1095,12 +1095,12 @@ // Overload of c_upper_bound() for performing a `comp` comparison other than // the default `operator<`. -template <typename Sequence, typename T, typename Compare> +template <typename Sequence, typename T, typename LessThan> container_algorithm_internal::ContainerIter<Sequence> c_upper_bound( - Sequence& sequence, T&& value, Compare&& comp) { + Sequence& sequence, T&& value, LessThan&& comp) { return std::upper_bound(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<T>(value), std::forward<Compare>(comp)); + std::forward<T>(value), std::forward<LessThan>(comp)); } // c_equal_range() @@ -1118,12 +1118,12 @@ // Overload of c_equal_range() for performing a `comp` comparison other than // the default `operator<`. -template <typename Sequence, typename T, typename Compare> +template <typename Sequence, typename T, typename LessThan> container_algorithm_internal::ContainerIterPairType<Sequence, Sequence> -c_equal_range(Sequence& sequence, T&& value, Compare&& comp) { +c_equal_range(Sequence& sequence, T&& value, LessThan&& comp) { return std::equal_range(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<T>(value), std::forward<Compare>(comp)); + std::forward<T>(value), std::forward<LessThan>(comp)); } // c_binary_search() @@ -1140,12 +1140,12 @@ // Overload of c_binary_search() for performing a `comp` comparison other than // the default `operator<`. -template <typename Sequence, typename T, typename Compare> -bool c_binary_search(Sequence&& sequence, T&& value, Compare&& comp) { +template <typename Sequence, typename T, typename LessThan> +bool c_binary_search(Sequence&& sequence, T&& value, LessThan&& comp) { return std::binary_search(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), std::forward<T>(value), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------ @@ -1166,14 +1166,14 @@ // Overload of c_merge() for performing a `comp` comparison other than // the default `operator<`. -template <typename C1, typename C2, typename OutputIterator, typename Compare> +template <typename C1, typename C2, typename OutputIterator, typename LessThan> OutputIterator c_merge(const C1& c1, const C2& c2, OutputIterator result, - Compare&& comp) { + LessThan&& comp) { return std::merge(container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), result, - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_inplace_merge() @@ -1189,13 +1189,13 @@ // Overload of c_inplace_merge() for performing a merge using a `comp` other // than `operator<`. -template <typename C, typename Compare> +template <typename C, typename LessThan> void c_inplace_merge(C& c, container_algorithm_internal::ContainerIter<C> middle, - Compare&& comp) { + LessThan&& comp) { std::inplace_merge(container_algorithm_internal::c_begin(c), middle, container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_includes() @@ -1213,13 +1213,13 @@ // Overload of c_includes() for performing a merge using a `comp` other than // `operator<`. -template <typename C1, typename C2, typename Compare> -bool c_includes(const C1& c1, const C2& c2, Compare&& comp) { +template <typename C1, typename C2, typename LessThan> +bool c_includes(const C1& c1, const C2& c2, LessThan&& comp) { return std::includes(container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_set_union() @@ -1243,7 +1243,7 @@ // Overload of c_set_union() for performing a merge using a `comp` other than // `operator<`. -template <typename C1, typename C2, typename OutputIterator, typename Compare, +template <typename C1, typename C2, typename OutputIterator, typename LessThan, typename = typename std::enable_if< !container_algorithm_internal::IsUnorderedContainer<C1>::value, void>::type, @@ -1251,12 +1251,12 @@ !container_algorithm_internal::IsUnorderedContainer<C2>::value, void>::type> OutputIterator c_set_union(const C1& c1, const C2& c2, OutputIterator output, - Compare&& comp) { + LessThan&& comp) { return std::set_union(container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), output, - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_set_intersection() @@ -1280,7 +1280,7 @@ // Overload of c_set_intersection() for performing a merge using a `comp` other // than `operator<`. -template <typename C1, typename C2, typename OutputIterator, typename Compare, +template <typename C1, typename C2, typename OutputIterator, typename LessThan, typename = typename std::enable_if< !container_algorithm_internal::IsUnorderedContainer<C1>::value, void>::type, @@ -1288,12 +1288,12 @@ !container_algorithm_internal::IsUnorderedContainer<C2>::value, void>::type> OutputIterator c_set_intersection(const C1& c1, const C2& c2, - OutputIterator output, Compare&& comp) { + OutputIterator output, LessThan&& comp) { return std::set_intersection(container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), output, - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_set_difference() @@ -1318,7 +1318,7 @@ // Overload of c_set_difference() for performing a merge using a `comp` other // than `operator<`. -template <typename C1, typename C2, typename OutputIterator, typename Compare, +template <typename C1, typename C2, typename OutputIterator, typename LessThan, typename = typename std::enable_if< !container_algorithm_internal::IsUnorderedContainer<C1>::value, void>::type, @@ -1326,12 +1326,12 @@ !container_algorithm_internal::IsUnorderedContainer<C2>::value, void>::type> OutputIterator c_set_difference(const C1& c1, const C2& c2, - OutputIterator output, Compare&& comp) { + OutputIterator output, LessThan&& comp) { return std::set_difference(container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), output, - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_set_symmetric_difference() @@ -1357,7 +1357,7 @@ // Overload of c_set_symmetric_difference() for performing a merge using a // `comp` other than `operator<`. -template <typename C1, typename C2, typename OutputIterator, typename Compare, +template <typename C1, typename C2, typename OutputIterator, typename LessThan, typename = typename std::enable_if< !container_algorithm_internal::IsUnorderedContainer<C1>::value, void>::type, @@ -1366,13 +1366,13 @@ void>::type> OutputIterator c_set_symmetric_difference(const C1& c1, const C2& c2, OutputIterator output, - Compare&& comp) { + LessThan&& comp) { return std::set_symmetric_difference( container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), output, - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------ @@ -1391,11 +1391,11 @@ // Overload of c_push_heap() for performing a push operation on a heap using a // `comp` other than `operator<`. -template <typename RandomAccessContainer, typename Compare> -void c_push_heap(RandomAccessContainer& sequence, Compare&& comp) { +template <typename RandomAccessContainer, typename LessThan> +void c_push_heap(RandomAccessContainer& sequence, LessThan&& comp) { std::push_heap(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_pop_heap() @@ -1410,11 +1410,11 @@ // Overload of c_pop_heap() for performing a pop operation on a heap using a // `comp` other than `operator<`. -template <typename RandomAccessContainer, typename Compare> -void c_pop_heap(RandomAccessContainer& sequence, Compare&& comp) { +template <typename RandomAccessContainer, typename LessThan> +void c_pop_heap(RandomAccessContainer& sequence, LessThan&& comp) { std::pop_heap(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_make_heap() @@ -1429,11 +1429,11 @@ // Overload of c_make_heap() for performing heap comparisons using a // `comp` other than `operator<` -template <typename RandomAccessContainer, typename Compare> -void c_make_heap(RandomAccessContainer& sequence, Compare&& comp) { +template <typename RandomAccessContainer, typename LessThan> +void c_make_heap(RandomAccessContainer& sequence, LessThan&& comp) { std::make_heap(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_sort_heap() @@ -1448,11 +1448,11 @@ // Overload of c_sort_heap() for performing heap comparisons using a // `comp` other than `operator<` -template <typename RandomAccessContainer, typename Compare> -void c_sort_heap(RandomAccessContainer& sequence, Compare&& comp) { +template <typename RandomAccessContainer, typename LessThan> +void c_sort_heap(RandomAccessContainer& sequence, LessThan&& comp) { std::sort_heap(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_is_heap() @@ -1467,11 +1467,11 @@ // Overload of c_is_heap() for performing heap comparisons using a // `comp` other than `operator<` -template <typename RandomAccessContainer, typename Compare> -bool c_is_heap(const RandomAccessContainer& sequence, Compare&& comp) { +template <typename RandomAccessContainer, typename LessThan> +bool c_is_heap(const RandomAccessContainer& sequence, LessThan&& comp) { return std::is_heap(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_is_heap_until() @@ -1487,12 +1487,12 @@ // Overload of c_is_heap_until() for performing heap comparisons using a // `comp` other than `operator<` -template <typename RandomAccessContainer, typename Compare> +template <typename RandomAccessContainer, typename LessThan> container_algorithm_internal::ContainerIter<RandomAccessContainer> -c_is_heap_until(RandomAccessContainer& sequence, Compare&& comp) { +c_is_heap_until(RandomAccessContainer& sequence, LessThan&& comp) { return std::is_heap_until(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------ @@ -1513,12 +1513,12 @@ // Overload of c_min_element() for performing a `comp` comparison other than // `operator<`. -template <typename Sequence, typename Compare> +template <typename Sequence, typename LessThan> container_algorithm_internal::ContainerIter<Sequence> c_min_element( - Sequence& sequence, Compare&& comp) { + Sequence& sequence, LessThan&& comp) { return std::min_element(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_max_element() @@ -1535,12 +1535,12 @@ // Overload of c_max_element() for performing a `comp` comparison other than // `operator<`. -template <typename Sequence, typename Compare> +template <typename Sequence, typename LessThan> container_algorithm_internal::ContainerIter<Sequence> c_max_element( - Sequence& sequence, Compare&& comp) { + Sequence& sequence, LessThan&& comp) { return std::max_element(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_minmax_element() @@ -1558,12 +1558,12 @@ // Overload of c_minmax_element() for performing `comp` comparisons other than // `operator<`. -template <typename C, typename Compare> +template <typename C, typename LessThan> container_algorithm_internal::ContainerIterPairType<C, C> -c_minmax_element(C& c, Compare&& comp) { +c_minmax_element(C& c, LessThan&& comp) { return std::minmax_element(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------ @@ -1588,15 +1588,15 @@ // Overload of c_lexicographical_compare() for performing a lexicographical // comparison using a `comp` operator instead of `operator<`. -template <typename Sequence1, typename Sequence2, typename Compare> +template <typename Sequence1, typename Sequence2, typename LessThan> bool c_lexicographical_compare(Sequence1&& sequence1, Sequence2&& sequence2, - Compare&& comp) { + LessThan&& comp) { return std::lexicographical_compare( container_algorithm_internal::c_begin(sequence1), container_algorithm_internal::c_end(sequence1), container_algorithm_internal::c_begin(sequence2), container_algorithm_internal::c_end(sequence2), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_next_permutation() @@ -1612,11 +1612,11 @@ // Overload of c_next_permutation() for performing a lexicographical // comparison using a `comp` operator instead of `operator<`. -template <typename C, typename Compare> -bool c_next_permutation(C& c, Compare&& comp) { +template <typename C, typename LessThan> +bool c_next_permutation(C& c, LessThan&& comp) { return std::next_permutation(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_prev_permutation() @@ -1632,11 +1632,11 @@ // Overload of c_prev_permutation() for performing a lexicographical // comparison using a `comp` operator instead of `operator<`. -template <typename C, typename Compare> -bool c_prev_permutation(C& c, Compare&& comp) { +template <typename C, typename LessThan> +bool c_prev_permutation(C& c, LessThan&& comp) { return std::prev_permutation(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------
diff --git a/absl/base/attributes.h b/absl/base/attributes.h index 294e5d0..d710f28 100644 --- a/absl/base/attributes.h +++ b/absl/base/attributes.h
@@ -281,10 +281,7 @@ // ABSL_ATTRIBUTE_RETURNS_NONNULL // // Tells the compiler that a particular function never returns a null pointer. -#if ABSL_HAVE_ATTRIBUTE(returns_nonnull) || \ - (defined(__GNUC__) && \ - (__GNUC__ > 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9)) && \ - !defined(__clang__)) +#if ABSL_HAVE_ATTRIBUTE(returns_nonnull) #define ABSL_ATTRIBUTE_RETURNS_NONNULL __attribute__((returns_nonnull)) #else #define ABSL_ATTRIBUTE_RETURNS_NONNULL @@ -622,7 +619,7 @@ #if __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") #define ABSL_FALLTHROUGH_INTENDED [[clang::fallthrough]] #endif -#elif defined(__GNUC__) && __GNUC__ >= 7 +#elif ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(7, 0) #define ABSL_FALLTHROUGH_INTENDED [[gnu::fallthrough]] #endif
diff --git a/absl/base/config.h b/absl/base/config.h index 7abc75a..0524196 100644 --- a/absl/base/config.h +++ b/absl/base/config.h
@@ -166,6 +166,22 @@ #define ABSL_HAVE_FEATURE(f) 0 #endif +// Portable check for GCC minimum version: +// https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html +#if defined(__GNUC__) && defined(__GNUC_MINOR__) +#define ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(x, y) \ + (__GNUC__ > (x) || __GNUC__ == (x) && __GNUC_MINOR__ >= (y)) +#else +#define ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(x, y) 0 +#endif + +#if defined(__clang__) && defined(__clang_major__) && defined(__clang_minor__) +#define ABSL_INTERNAL_HAVE_MIN_CLANG_VERSION(x, y) \ + (__clang_major__ > (x) || __clang_major__ == (x) && __clang_minor__ >= (y)) +#else +#define ABSL_INTERNAL_HAVE_MIN_CLANG_VERSION(x, y) 0 +#endif + // ABSL_HAVE_TLS is defined to 1 when __thread should be supported. // We assume __thread is supported on Linux when compiled with Clang or compiled // against libstdc++ with _GLIBCXX_HAVE_TLS defined. @@ -183,10 +199,9 @@ // gcc >= 4.8.1 using libstdc++, and Visual Studio. #ifdef ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE #error ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE cannot be directly set -#elif defined(_LIBCPP_VERSION) || \ - (!defined(__clang__) && defined(__GNUC__) && defined(__GLIBCXX__) && \ - (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) || \ - defined(_MSC_VER) +#elif defined(_LIBCPP_VERSION) || defined(_MSC_VER) || \ + (!defined(__clang__) && defined(__GLIBCXX__) && \ + ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(4, 8)) #define ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE 1 #endif @@ -205,10 +220,9 @@ #error ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE cannot be directly set #elif defined(ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE) #error ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE cannot directly set -#elif (defined(__clang__) && defined(_LIBCPP_VERSION)) || \ - (!defined(__clang__) && defined(__GNUC__) && \ - (__GNUC__ > 7 || (__GNUC__ == 7 && __GNUC_MINOR__ >= 4)) && \ - (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__))) || \ +#elif (defined(__clang__) && defined(_LIBCPP_VERSION)) || \ + (!defined(__clang__) && ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(7, 4) && \ + (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__))) || \ (defined(_MSC_VER) && !defined(__NVCC__)) #define ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE 1 #define ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE 1 @@ -222,7 +236,7 @@ #if ABSL_INTERNAL_HAS_KEYWORD(__builtin_LINE) && \ ABSL_INTERNAL_HAS_KEYWORD(__builtin_FILE) #define ABSL_HAVE_SOURCE_LOCATION_CURRENT 1 -#elif defined(__GNUC__) && __GNUC__ >= 5 +#elif ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(5, 0) #define ABSL_HAVE_SOURCE_LOCATION_CURRENT 1 #endif #endif @@ -319,25 +333,21 @@ // For further details, consult the compiler's documentation. #ifdef ABSL_HAVE_EXCEPTIONS #error ABSL_HAVE_EXCEPTIONS cannot be directly set. - -#elif defined(__clang__) - -#if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 6) +#elif ABSL_INTERNAL_HAVE_MIN_CLANG_VERSION(3, 6) // Clang >= 3.6 #if ABSL_HAVE_FEATURE(cxx_exceptions) #define ABSL_HAVE_EXCEPTIONS 1 #endif // ABSL_HAVE_FEATURE(cxx_exceptions) -#else +#elif defined(__clang__) // Clang < 3.6 // http://releases.llvm.org/3.6.0/tools/clang/docs/ReleaseNotes.html#the-exceptions-macro #if defined(__EXCEPTIONS) && ABSL_HAVE_FEATURE(cxx_exceptions) #define ABSL_HAVE_EXCEPTIONS 1 #endif // defined(__EXCEPTIONS) && ABSL_HAVE_FEATURE(cxx_exceptions) -#endif // __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 6) - // Handle remaining special cases and default to exceptions being supported. -#elif !(defined(__GNUC__) && (__GNUC__ < 5) && !defined(__EXCEPTIONS)) && \ - !(defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__cpp_exceptions)) && \ +#elif !(defined(__GNUC__) && (__GNUC__ < 5) && !defined(__EXCEPTIONS)) && \ + !(ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(5, 0) && \ + !defined(__cpp_exceptions)) && \ !(defined(_MSC_VER) && !defined(_CPPUNWIND)) #define ABSL_HAVE_EXCEPTIONS 1 #endif
diff --git a/absl/base/dynamic_annotations.h b/absl/base/dynamic_annotations.h index 880cbf6..03d7096 100644 --- a/absl/base/dynamic_annotations.h +++ b/absl/base/dynamic_annotations.h
@@ -471,7 +471,7 @@ __sanitizer_annotate_contiguous_container(beg, end, old_mid, new_mid) #define ABSL_ADDRESS_SANITIZER_REDZONE(name) \ struct { \ - char x[8] __attribute__((aligned(8))); \ + alignas(8) char x[8]; \ } name #else
diff --git a/absl/container/internal/hash_generator_testing.h b/absl/container/internal/hash_generator_testing.h index 6869fe4..f1f555a 100644 --- a/absl/container/internal/hash_generator_testing.h +++ b/absl/container/internal/hash_generator_testing.h
@@ -21,11 +21,13 @@ #include <stdint.h> #include <algorithm> +#include <cassert> #include <iosfwd> #include <random> #include <tuple> #include <type_traits> #include <utility> +#include <vector> #include "absl/container/internal/hash_policy_testing.h" #include "absl/memory/memory.h" @@ -153,6 +155,25 @@ typename Container::value_type, typename Container::key_type>::type>&>()()); +// Naive wrapper that performs a linear search of previous values. +// Beware this is O(SQR), which is reasonable for smaller kMaxValues. +template <class T, size_t kMaxValues = 64, class E = void> +struct UniqueGenerator { + Generator<T, E> gen; + std::vector<T> values; + + T operator()() { + assert(values.size() < kMaxValues); + for (;;) { + T value = gen(); + if (std::find(values.begin(), values.end(), value) == values.end()) { + values.push_back(value); + return value; + } + } + } +}; + } // namespace hash_internal } // namespace container_internal ABSL_NAMESPACE_END
diff --git a/absl/container/internal/unordered_map_constructor_test.h b/absl/container/internal/unordered_map_constructor_test.h index 3f90ad7..c1d20f3 100644 --- a/absl/container/internal/unordered_map_constructor_test.h +++ b/absl/container/internal/unordered_map_constructor_test.h
@@ -179,7 +179,7 @@ A alloc(0); std::vector<T> values; std::generate_n(std::back_inserter(values), 10, - hash_internal::Generator<T>()); + hash_internal::UniqueGenerator<T>()); TypeParam m(values.begin(), values.end(), 123, hasher, equal, alloc); EXPECT_EQ(m.hash_function(), hasher); EXPECT_EQ(m.key_eq(), equal); @@ -198,7 +198,7 @@ A alloc(0); std::vector<T> values; std::generate_n(std::back_inserter(values), 10, - hash_internal::Generator<T>()); + hash_internal::UniqueGenerator<T>()); TypeParam m(values.begin(), values.end(), 123, alloc); EXPECT_EQ(m.get_allocator(), alloc); EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); @@ -221,7 +221,7 @@ A alloc(0); std::vector<T> values; std::generate_n(std::back_inserter(values), 10, - hash_internal::Generator<T>()); + hash_internal::UniqueGenerator<T>()); TypeParam m(values.begin(), values.end(), 123, hasher, alloc); EXPECT_EQ(m.hash_function(), hasher); EXPECT_EQ(m.get_allocator(), alloc); @@ -241,8 +241,9 @@ H hasher; E equal; A alloc(0); + hash_internal::UniqueGenerator<T> gen; TypeParam m(123, hasher, equal, alloc); - for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + for (size_t i = 0; i != 10; ++i) m.insert(gen()); TypeParam n(m); EXPECT_EQ(m.hash_function(), n.hash_function()); EXPECT_EQ(m.key_eq(), n.key_eq()); @@ -262,8 +263,9 @@ H hasher; E equal; A alloc(0); + hash_internal::UniqueGenerator<T> gen; TypeParam m(123, hasher, equal, alloc); - for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + for (size_t i = 0; i != 10; ++i) m.insert(gen()); TypeParam n(m, A(11)); EXPECT_EQ(m.hash_function(), n.hash_function()); EXPECT_EQ(m.key_eq(), n.key_eq()); @@ -285,8 +287,9 @@ H hasher; E equal; A alloc(0); + hash_internal::UniqueGenerator<T> gen; TypeParam m(123, hasher, equal, alloc); - for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + for (size_t i = 0; i != 10; ++i) m.insert(gen()); TypeParam t(m); TypeParam n(std::move(t)); EXPECT_EQ(m.hash_function(), n.hash_function()); @@ -307,8 +310,9 @@ H hasher; E equal; A alloc(0); + hash_internal::UniqueGenerator<T> gen; TypeParam m(123, hasher, equal, alloc); - for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + for (size_t i = 0; i != 10; ++i) m.insert(gen()); TypeParam t(m); TypeParam n(std::move(t), A(1)); EXPECT_EQ(m.hash_function(), n.hash_function()); @@ -325,7 +329,7 @@ TYPED_TEST_P(ConstructorTest, InitializerListBucketHashEqualAlloc) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; using H = typename TypeParam::hasher; using E = typename TypeParam::key_equal; @@ -348,7 +352,7 @@ void InitializerListBucketAllocTest(std::true_type) { using T = hash_internal::GeneratedType<TypeParam>; using A = typename TypeParam::allocator_type; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; A alloc(0); TypeParam m(values, 123, alloc); @@ -371,7 +375,7 @@ using A = typename TypeParam::allocator_type; H hasher; A alloc(0); - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; TypeParam m(values, 123, hasher, alloc); EXPECT_EQ(m.hash_function(), hasher); @@ -392,7 +396,7 @@ H hasher; E equal; A alloc(0); - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; TypeParam m({gen(), gen(), gen()}, 123, hasher, equal, alloc); TypeParam n; n = m; @@ -412,7 +416,7 @@ H hasher; E equal; A alloc(0); - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; TypeParam m({gen(), gen(), gen()}, 123, hasher, equal, alloc); TypeParam t(m); TypeParam n; @@ -424,7 +428,7 @@ TYPED_TEST_P(ConstructorTest, AssignmentFromInitializerList) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; TypeParam m; m = values; @@ -433,7 +437,7 @@ TYPED_TEST_P(ConstructorTest, AssignmentOverwritesExisting) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; TypeParam m({gen(), gen(), gen()}); TypeParam n({gen()}); n = m; @@ -442,7 +446,7 @@ TYPED_TEST_P(ConstructorTest, MoveAssignmentOverwritesExisting) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; TypeParam m({gen(), gen(), gen()}); TypeParam t(m); TypeParam n({gen()}); @@ -452,7 +456,7 @@ TYPED_TEST_P(ConstructorTest, AssignmentFromInitializerListOverwritesExisting) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; TypeParam m; m = values; @@ -461,7 +465,7 @@ TYPED_TEST_P(ConstructorTest, AssignmentOnSelf) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; TypeParam m(values); m = *&m; // Avoid -Wself-assign
diff --git a/absl/copts/GENERATED_AbseilCopts.cmake b/absl/copts/GENERATED_AbseilCopts.cmake index 51742c9..18a1e5c 100644 --- a/absl/copts/GENERATED_AbseilCopts.cmake +++ b/absl/copts/GENERATED_AbseilCopts.cmake
@@ -71,6 +71,7 @@ "-Wformat-security" "-Wgnu-redeclared-enum" "-Winfinite-recursion" + "-Winvalid-constexpr" "-Wliteral-conversion" "-Wmissing-declarations" "-Woverlength-strings"
diff --git a/absl/copts/GENERATED_copts.bzl b/absl/copts/GENERATED_copts.bzl index 6707488..d2bd560 100644 --- a/absl/copts/GENERATED_copts.bzl +++ b/absl/copts/GENERATED_copts.bzl
@@ -72,6 +72,7 @@ "-Wformat-security", "-Wgnu-redeclared-enum", "-Winfinite-recursion", + "-Winvalid-constexpr", "-Wliteral-conversion", "-Wmissing-declarations", "-Woverlength-strings",
diff --git a/absl/copts/copts.py b/absl/copts/copts.py index cf52981..ce30df8 100644 --- a/absl/copts/copts.py +++ b/absl/copts/copts.py
@@ -87,6 +87,7 @@ "-Wformat-security", "-Wgnu-redeclared-enum", "-Winfinite-recursion", + "-Winvalid-constexpr", "-Wliteral-conversion", "-Wmissing-declarations", "-Woverlength-strings",
diff --git a/absl/copts/generate_copts.py b/absl/copts/generate_copts.py index 0e5dc9f..34be2fc 100755 --- a/absl/copts/generate_copts.py +++ b/absl/copts/generate_copts.py
@@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 """Generate Abseil compile compile option configs. Usage: <path_to_absl>/copts/generate_copts.py
diff --git a/absl/debugging/symbolize_elf.inc b/absl/debugging/symbolize_elf.inc index f4d5727..87dbd07 100644 --- a/absl/debugging/symbolize_elf.inc +++ b/absl/debugging/symbolize_elf.inc
@@ -701,6 +701,16 @@ const char *start_address = ComputeOffset(original_start_address, relocation); +#ifdef __arm__ + // ARM functions are always aligned to multiples of two bytes; the + // lowest-order bit in start_address is ignored by the CPU and indicates + // whether the function contains ARM (0) or Thumb (1) code. We don't care + // about what encoding is being used; we just want the real start address + // of the function. + start_address = reinterpret_cast<const char *>( + reinterpret_cast<uintptr_t>(start_address) & ~1); +#endif + if (deref_function_descriptor_pointer && InSection(original_start_address, opd)) { // The opd section is mapped into memory. Just dereference
diff --git a/absl/debugging/symbolize_test.cc b/absl/debugging/symbolize_test.cc index a2dd495..35de02e 100644 --- a/absl/debugging/symbolize_test.cc +++ b/absl/debugging/symbolize_test.cc
@@ -477,6 +477,46 @@ #endif } +#if defined(__arm__) && ABSL_HAVE_ATTRIBUTE(target) +// Test that we correctly identify bounds of Thumb functions on ARM. +// +// Thumb functions have the lowest-order bit set in their addresses in the ELF +// symbol table. This requires some extra logic to properly compute function +// bounds. To test this logic, nudge a Thumb function right up against an ARM +// function and try to symbolize the ARM function. +// +// A naive implementation will simply use the Thumb function's entry point as +// written in the symbol table and will therefore treat the Thumb function as +// extending one byte further in the instruction stream than it actually does. +// When asked to symbolize the start of the ARM function, it will identify an +// overlap between the Thumb and ARM functions, and it will return the name of +// the Thumb function. +// +// A correct implementation, on the other hand, will null out the lowest-order +// bit in the Thumb function's entry point. It will correctly compute the end of +// the Thumb function, it will find no overlap between the Thumb and ARM +// functions, and it will return the name of the ARM function. + +__attribute__((target("thumb"))) int ArmThumbOverlapThumb(int x) { + return x * x * x; +} + +__attribute__((target("arm"))) int ArmThumbOverlapArm(int x) { + return x * x * x; +} + +void ABSL_ATTRIBUTE_NOINLINE TestArmThumbOverlap() { +#if defined(ABSL_HAVE_ATTRIBUTE_NOINLINE) + const char *symbol = TrySymbolize((void *)&ArmThumbOverlapArm); + ABSL_RAW_CHECK(symbol != nullptr, "TestArmThumbOverlap failed"); + ABSL_RAW_CHECK(strcmp("ArmThumbOverlapArm()", symbol) == 0, + "TestArmThumbOverlap failed"); + std::cout << "TestArmThumbOverlap passed" << std::endl; +#endif +} + +#endif // defined(__arm__) && ABSL_HAVE_ATTRIBUTE(target) + #elif defined(_WIN32) #if !defined(ABSL_CONSUME_DLL) @@ -551,6 +591,9 @@ TestWithPCInsideInlineFunction(); TestWithPCInsideNonInlineFunction(); TestWithReturnAddress(); +#if defined(__arm__) && ABSL_HAVE_ATTRIBUTE(target) + TestArmThumbOverlap(); +#endif #endif return RUN_ALL_TESTS();
diff --git a/absl/numeric/int128.h b/absl/numeric/int128.h index 0dd814a..235361a 100644 --- a/absl/numeric/int128.h +++ b/absl/numeric/int128.h
@@ -817,27 +817,27 @@ return MakeUint128(hi, lo); } -inline bool operator!(uint128 val) { +constexpr inline bool operator!(uint128 val) { return !Uint128High64(val) && !Uint128Low64(val); } // Logical operators. -inline uint128 operator~(uint128 val) { +constexpr inline uint128 operator~(uint128 val) { return MakeUint128(~Uint128High64(val), ~Uint128Low64(val)); } -inline uint128 operator|(uint128 lhs, uint128 rhs) { +constexpr inline uint128 operator|(uint128 lhs, uint128 rhs) { return MakeUint128(Uint128High64(lhs) | Uint128High64(rhs), Uint128Low64(lhs) | Uint128Low64(rhs)); } -inline uint128 operator&(uint128 lhs, uint128 rhs) { +constexpr inline uint128 operator&(uint128 lhs, uint128 rhs) { return MakeUint128(Uint128High64(lhs) & Uint128High64(rhs), Uint128Low64(lhs) & Uint128Low64(rhs)); } -inline uint128 operator^(uint128 lhs, uint128 rhs) { +constexpr inline uint128 operator^(uint128 lhs, uint128 rhs) { return MakeUint128(Uint128High64(lhs) ^ Uint128High64(rhs), Uint128Low64(lhs) ^ Uint128Low64(rhs)); }
diff --git a/absl/random/internal/pool_urbg.cc b/absl/random/internal/pool_urbg.cc index 5bee530..725100a 100644 --- a/absl/random/internal/pool_urbg.cc +++ b/absl/random/internal/pool_urbg.cc
@@ -194,11 +194,10 @@ // Not all the platforms that we build for have std::aligned_alloc, however // since we never free these objects, we can over allocate and munge the // pointers to the correct alignment. - void* memory = std::malloc(sizeof(RandenPoolEntry) + kAlignment); - auto x = reinterpret_cast<intptr_t>(memory); + intptr_t x = reinterpret_cast<intptr_t>( + new char[sizeof(RandenPoolEntry) + kAlignment]); auto y = x % kAlignment; - void* aligned = - (y == 0) ? memory : reinterpret_cast<void*>(x + kAlignment - y); + void* aligned = reinterpret_cast<void*>(y == 0 ? x : (x + kAlignment - y)); return new (aligned) RandenPoolEntry(); }
diff --git a/absl/strings/BUILD.bazel b/absl/strings/BUILD.bazel index 123b5ef..68c71f0 100644 --- a/absl/strings/BUILD.bazel +++ b/absl/strings/BUILD.bazel
@@ -297,6 +297,26 @@ ) cc_library( + name = "cordz_update_tracker", + hdrs = ["internal/cordz_update_tracker.h"], + copts = ABSL_DEFAULT_COPTS, + visibility = ["//visibility:private"], + deps = ["//absl/base:config"], +) + +cc_test( + name = "cordz_update_tracker_test", + srcs = ["internal/cordz_update_tracker_test.cc"], + deps = [ + ":cordz_update_tracker", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/synchronization", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( name = "cord", srcs = [ "cord.cc", @@ -307,10 +327,16 @@ copts = ABSL_DEFAULT_COPTS, deps = [ ":cord_internal", + ":cordz_functions", + ":cordz_info", + ":cordz_statistics", + ":cordz_update_scope", + ":cordz_update_tracker", ":internal", ":str_format", ":strings", "//absl/base", + "//absl/base:config", "//absl/base:core_headers", "//absl/base:endian", "//absl/base:raw_logging_internal", @@ -323,6 +349,176 @@ ) cc_library( + name = "cordz_handle", + srcs = ["internal/cordz_handle.cc"], + hdrs = ["internal/cordz_handle.h"], + copts = ABSL_DEFAULT_COPTS, + deps = [ + "//absl/base", + "//absl/base:config", + "//absl/base:raw_logging_internal", + "//absl/synchronization", + ], +) + +cc_library( + name = "cordz_info", + srcs = ["internal/cordz_info.cc"], + hdrs = ["internal/cordz_info.h"], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":cord_internal", + ":cordz_functions", + ":cordz_handle", + ":cordz_statistics", + ":cordz_update_tracker", + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/debugging:stacktrace", + "//absl/synchronization", + "//absl/types:span", + ], +) + +cc_library( + name = "cordz_update_scope", + hdrs = ["internal/cordz_update_scope.h"], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":cord_internal", + ":cordz_info", + ":cordz_update_tracker", + "//absl/base:config", + "//absl/base:core_headers", + ], +) + +cc_test( + name = "cordz_update_scope_test", + srcs = ["internal/cordz_update_scope_test.cc"], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":cord_internal", + ":cordz_info", + ":cordz_test_helpers", + ":cordz_update_scope", + ":cordz_update_tracker", + "//absl/base:config", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "cordz_sample_token", + srcs = ["internal/cordz_sample_token.cc"], + hdrs = ["internal/cordz_sample_token.h"], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":cordz_handle", + ":cordz_info", + "//absl/base:config", + ], +) + +cc_library( + name = "cordz_functions", + srcs = ["internal/cordz_functions.cc"], + hdrs = ["internal/cordz_functions.h"], + copts = ABSL_DEFAULT_COPTS, + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:exponential_biased", + "//absl/base:raw_logging_internal", + ], +) + +cc_library( + name = "cordz_statistics", + hdrs = ["internal/cordz_statistics.h"], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":cordz_update_tracker", + "//absl/base:config", + ], +) + +cc_test( + name = "cordz_functions_test", + srcs = [ + "internal/cordz_functions_test.cc", + ], + deps = [ + ":cordz_functions", + ":cordz_test_helpers", + "//absl/base:config", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cordz_handle_test", + srcs = [ + "internal/cordz_handle_test.cc", + ], + deps = [ + ":cordz_handle", + "//absl/base:config", + "//absl/memory", + "//absl/random", + "//absl/random:distributions", + "//absl/synchronization", + "//absl/synchronization:thread_pool", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cordz_info_test", + srcs = [ + "internal/cordz_info_test.cc", + ], + deps = [ + ":cord_internal", + ":cordz_handle", + ":cordz_info", + ":cordz_statistics", + ":cordz_test_helpers", + ":cordz_update_tracker", + ":strings", + "//absl/base:config", + "//absl/debugging:stacktrace", + "//absl/debugging:symbolize", + "//absl/types:span", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cordz_sample_token_test", + srcs = [ + "internal/cordz_sample_token_test.cc", + ], + deps = [ + ":cord_internal", + ":cordz_handle", + ":cordz_info", + ":cordz_sample_token", + ":cordz_test_helpers", + "//absl/base:config", + "//absl/memory", + "//absl/random", + "//absl/synchronization", + "//absl/synchronization:thread_pool", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( name = "cord_test_helpers", testonly = 1, hdrs = [ @@ -331,6 +527,26 @@ copts = ABSL_DEFAULT_COPTS, deps = [ ":cord", + ":cord_internal", + ":strings", + ], +) + +cc_library( + name = "cordz_test_helpers", + testonly = 1, + hdrs = ["cordz_test_helpers.h"], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":cord", + ":cord_internal", + ":cordz_info", + ":cordz_sample_token", + ":cordz_statistics", + ":cordz_update_tracker", + "//absl/base:config", + "//absl/base:core_headers", + "@com_google_googletest//:gtest", ], ) @@ -343,6 +559,8 @@ deps = [ ":cord", ":cord_test_helpers", + ":cordz_functions", + ":cordz_test_helpers", ":str_format", ":strings", "//absl/base", @@ -356,6 +574,39 @@ ) cc_test( + name = "cordz_test", + size = "medium", + srcs = ["cordz_test.cc"], + copts = ABSL_TEST_COPTS, + tags = [ + "benchmark", + "no_test_android_arm", + "no_test_android_arm64", + "no_test_android_x86", + "no_test_darwin_x86_64", + "no_test_ios_x86_64", + "no_test_loonix", + "no_test_msvc_x64", + ], + visibility = ["//visibility:private"], + deps = [ + ":cord", + ":cord_test_helpers", + ":cordz_functions", + ":cordz_info", + ":cordz_sample_token", + ":cordz_statistics", + ":cordz_test_helpers", + ":cordz_update_tracker", + ":strings", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( name = "cord_ring_test", size = "medium", srcs = ["cord_ring_test.cc"],
diff --git a/absl/strings/CMakeLists.txt b/absl/strings/CMakeLists.txt index 3b7ae63..80ae2a6 100644 --- a/absl/strings/CMakeLists.txt +++ b/absl/strings/CMakeLists.txt
@@ -259,7 +259,7 @@ absl_cc_test( NAME str_join_test - SRCS +ss SRCS "str_join_test.cc" COPTS ${ABSL_TEST_COPTS} @@ -552,24 +552,264 @@ absl_cc_library( NAME + cord_internal + HDRS + "internal/cord_internal.h" + "internal/cord_rep_flat.h" + "internal/cord_rep_ring.h" + "internal/cord_rep_ring_reader.h" + SRCS + "internal/cord_internal.cc" + "internal/cord_rep_ring.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base_internal + absl::compressed_tuple + absl::config + absl::core_headers + absl::endian + absl::inlined_vector + absl::layout + absl::raw_logging_internal + absl::strings + absl::throw_delegate + absl::type_traits +) + +absl_cc_library( + NAME + cordz_update_tracker + HDRS + "internal/cordz_update_tracker.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config +) + +absl_cc_test( + NAME + cordz_update_tracker_test + SRCS + "internal/cordz_update_tracker_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::cordz_update_tracker + absl::core_headers + absl::synchronization + gmock_main +) + +absl_cc_library( + NAME + cordz_functions + HDRS + "internal/cordz_functions.h" + SRCS + "internal/cordz_functions.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::exponential_biased + absl::raw_logging_internal +) + +absl_cc_test( + NAME + cordz_functions_test + SRCS + "internal/cordz_functions_test.cc" + DEPS + absl::config + absl::cordz_functions + absl::cordz_test_helpers + gmock_main +) + +absl_cc_library( + NAME + cordz_statistics + HDRS + "internal/cordz_statistics.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::cordz_update_tracker + absl::synchronization +) + +absl_cc_library( + NAME + cordz_handle + HDRS + "internal/cordz_handle.h" + SRCS + "internal/cordz_handle.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::raw_logging_internal + absl::synchronization +) + +absl_cc_test( + NAME + cordz_handle_test + SRCS + "internal/cordz_handle_test.cc" + DEPS + absl::config + absl::cordz_handle + absl::cordz_test_helpers + absl::memory + absl::random_random + absl::random_distributions + absl::synchronization + absl::time + gmock_main +) + +absl_cc_library( + NAME + cordz_info + HDRS + "internal/cordz_info.h" + SRCS + "internal/cordz_info.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::cord_internal + absl::cordz_functions + absl::cordz_handle + absl::cordz_statistics + absl::cordz_update_tracker + absl::core_headers + absl::span + absl::raw_logging_internal + absl::stacktrace + absl::synchronization +) + +absl_cc_test( + NAME + cordz_info_test + SRCS + "internal/cordz_info_test.cc" + DEPS + absl::config + absl::cord_internal + absl::cordz_test_helpers + absl::cordz_handle + absl::cordz_info + absl::cordz_statistics + absl::cordz_test_helpers + absl::cordz_update_tracker + absl::span + absl::stacktrace + absl::symbolize + gmock_main +) + +absl_cc_library( + NAME + cordz_sample_token + HDRS + "internal/cordz_sample_token.h" + SRCS + "internal/cordz_sample_token.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::cordz_handle + absl::cordz_info +) + +absl_cc_test( + NAME + cordz_sample_token_test + SRCS + "internal/cordz_sample_token_test.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::cord_internal + absl::cordz_handle + absl::cordz_info + absl::cordz_info + absl::cordz_sample_token + absl::cordz_test_helpers + absl::memory + absl::random_random + absl::synchronization + absl::thread_pool + absl::time + gmock_main +) + +absl_cc_library( + NAME + cordz_update_scope + HDRS + "internal/cordz_update_scope.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::cord_internal + absl::cordz_info + absl::cordz_update_tracker + absl::core_headers +) + +absl_cc_test( + NAME + cordz_update_scope_test + SRCS + "internal/cordz_update_scope_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::cord_internal + absl::cordz_info + absl::cordz_test_helpers + absl::cordz_update_scope + absl::cordz_update_tracker + absl::core_headers + gmock_main +) + +absl_cc_library( + NAME cord HDRS "cord.h" SRCS "cord.cc" - "internal/cord_internal.cc" - "internal/cord_internal.h" - "internal/cord_rep_ring.h" - "internal/cord_rep_ring.cc" - "internal/cord_rep_ring_reader.h" - "internal/cord_rep_flat.h" COPTS ${ABSL_DEFAULT_COPTS} DEPS absl::base - absl::base_internal - absl::compressed_tuple absl::config + absl::cord_internal + absl::cordz_functions + absl::cordz_info + absl::cordz_update_scope + absl::cordz_update_tracker absl::core_headers absl::endian absl::fixed_array @@ -578,8 +818,6 @@ absl::optional absl::raw_logging_internal absl::strings - absl::strings_internal - absl::throw_delegate absl::type_traits PUBLIC ) @@ -593,6 +831,27 @@ ${ABSL_TEST_COPTS} DEPS absl::cord + absl::cord_internal + absl::strings + TESTONLY +) + +absl_cc_library( + NAME + cordz_test_helpers + HDRS + "cordz_test_helpers.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::cord + absl::cord_internal + absl::cordz_info + absl::cordz_sample_token + absl::cordz_statistics + absl::cordz_update_tracker + absl::core_headers TESTONLY ) @@ -609,6 +868,8 @@ absl::strings absl::base absl::config + absl::cord_test_helpers + absl::cordz_test_helpers absl::core_headers absl::endian absl::raw_logging_internal @@ -624,12 +885,12 @@ COPTS ${ABSL_TEST_COPTS} DEPS - absl::config - absl::cord - absl::strings absl::base + absl::config + absl::cord_internal absl::core_headers absl::raw_logging_internal + absl::strings gmock_main ) @@ -641,9 +902,33 @@ COPTS ${ABSL_TEST_COPTS} DEPS - absl::cord - absl::strings absl::base + absl::cord_internal absl::core_headers + absl::strings + gmock_main +) + +absl_cc_test( + NAME + cordz_test + SRCS + "cordz_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::cord + absl::cord_test_helpers + absl::cordz_test_helpers + absl::cordz_functions + absl::cordz_info + absl::cordz_sample_token + absl::cordz_statistics + absl::cordz_update_tracker + absl::base + absl::config + absl::core_headers + absl::raw_logging_internal + absl::strings gmock_main )
diff --git a/absl/strings/cord.cc b/absl/strings/cord.cc index 8f0999f..15d1733 100644 --- a/absl/strings/cord.cc +++ b/absl/strings/cord.cc
@@ -38,6 +38,9 @@ #include "absl/strings/internal/cord_internal.h" #include "absl/strings/internal/cord_rep_flat.h" #include "absl/strings/internal/cord_rep_ring.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_scope.h" +#include "absl/strings/internal/cordz_update_tracker.h" #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" @@ -53,8 +56,9 @@ using ::absl::cord_internal::CordRepFlat; using ::absl::cord_internal::CordRepRing; using ::absl::cord_internal::CordRepSubstring; -using ::absl::cord_internal::kMinFlatLength; +using ::absl::cord_internal::InlineData; using ::absl::cord_internal::kMaxFlatLength; +using ::absl::cord_internal::kMinFlatLength; using ::absl::cord_internal::CONCAT; using ::absl::cord_internal::EXTERNAL; @@ -206,7 +210,7 @@ } static CordRepFlat* CreateFlat(const char* data, size_t length, - size_t alloc_hint) { + size_t alloc_hint) { CordRepFlat* flat = CordRepFlat::New(length + alloc_hint); flat->length = length; memcpy(flat->Data(), data, length); @@ -230,9 +234,7 @@ // Create a new tree out of the specified array. // The returned node has a refcount of 1. -static CordRep* NewTree(const char* data, - size_t length, - size_t alloc_hint) { +static CordRep* NewTree(const char* data, size_t length, size_t alloc_hint) { if (length == 0) return nullptr; if (cord_ring_enabled()) { return RingNewTree(data, length, alloc_hint); @@ -299,20 +301,6 @@ return data_.as_chars(); } -inline CordRep* Cord::InlineRep::force_tree(size_t extra_hint) { - if (data_.is_tree()) { - return data_.as_tree(); - } - - size_t len = inline_size(); - CordRepFlat* result = CordRepFlat::New(len + extra_hint); - result->length = len; - static_assert(kMinFlatLength >= sizeof(data_), ""); - memcpy(result->Data(), data_.as_chars(), sizeof(data_)); - set_tree(result); - return result; -} - inline void Cord::InlineRep::reduce_size(size_t n) { size_t tag = inline_size(); assert(tag <= kMaxInline); @@ -334,25 +322,72 @@ return (rep->tag == RING) ? rep->ring() : CordRepRing::Create(rep, extra); } -void Cord::InlineRep::AppendTree(CordRep* tree) { - if (tree == nullptr) return; - if (data_.is_empty()) { - set_tree(tree); - } else if (cord_ring_enabled()) { - set_tree(CordRepRing::Append(ForceRing(force_tree(0), 1), tree)); +void Cord::InlineRep::AppendTreeToInlined(CordRep* tree, + MethodIdentifier method) { + assert(!is_tree()); + if (!data_.is_empty()) { + CordRepFlat* flat = MakeFlatWithExtraCapacity(0); + if (cord_ring_enabled()) { + tree = CordRepRing::Append(CordRepRing::Create(flat, 1), tree); + } else { + tree = Concat(flat, tree); + } + } + EmplaceTree(tree, method); +} + +void Cord::InlineRep::AppendTreeToTree(CordRep* tree, MethodIdentifier method) { + assert(is_tree()); + const CordzUpdateScope scope(data_.cordz_info(), method); + if (cord_ring_enabled()) { + tree = CordRepRing::Append(ForceRing(data_.as_tree(), 1), tree); } else { - set_tree(Concat(force_tree(0), tree)); + tree = Concat(data_.as_tree(), tree); + } + SetTree(tree, scope); +} + +void Cord::InlineRep::AppendTree(CordRep* tree, MethodIdentifier method) { + if (tree == nullptr) return; + if (data_.is_tree()) { + AppendTreeToTree(tree, method); + } else { + AppendTreeToInlined(tree, method); } } -void Cord::InlineRep::PrependTree(CordRep* tree) { - assert(tree != nullptr); - if (data_.is_empty()) { - set_tree(tree); - } else if (cord_ring_enabled()) { - set_tree(CordRepRing::Prepend(ForceRing(force_tree(0), 1), tree)); +void Cord::InlineRep::PrependTreeToInlined(CordRep* tree, + MethodIdentifier method) { + assert(!is_tree()); + if (!data_.is_empty()) { + CordRepFlat* flat = MakeFlatWithExtraCapacity(0); + if (cord_ring_enabled()) { + tree = CordRepRing::Prepend(CordRepRing::Create(flat, 1), tree); + } else { + tree = Concat(tree, flat); + } + } + EmplaceTree(tree, method); +} + +void Cord::InlineRep::PrependTreeToTree(CordRep* tree, + MethodIdentifier method) { + assert(is_tree()); + const CordzUpdateScope scope(data_.cordz_info(), method); + if (cord_ring_enabled()) { + tree = CordRepRing::Prepend(ForceRing(data_.as_tree(), 1), tree); } else { - set_tree(Concat(tree, force_tree(0))); + tree = Concat(tree, data_.as_tree()); + } + SetTree(tree, scope); +} + +void Cord::InlineRep::PrependTree(CordRep* tree, MethodIdentifier method) { + assert(tree != nullptr); + if (data_.is_tree()) { + PrependTreeToTree(tree, method); + } else { + PrependTreeToInlined(tree, method); } } @@ -404,76 +439,43 @@ return true; } +template <bool has_length> void Cord::InlineRep::GetAppendRegion(char** region, size_t* size, - size_t max_length) { - if (max_length == 0) { - *region = nullptr; - *size = 0; - return; - } + size_t length) { + auto constexpr method = CordzUpdateTracker::kGetAppendRegion; - // Try to fit in the inline buffer if possible. - if (!is_tree()) { - size_t inline_length = inline_size(); - if (max_length <= kMaxInline - inline_length) { - *region = data_.as_chars() + inline_length; - *size = max_length; - set_inline_size(inline_length + max_length); + CordRep* root = tree(); + size_t sz = root ? root->length : inline_size(); + if (root == nullptr) { + size_t available = kMaxInline - sz; + if (available >= (has_length ? length : 1)) { + *region = data_.as_chars() + sz; + *size = has_length ? length : available; + set_inline_size(has_length ? sz + length : kMaxInline); return; } } - CordRep* root = force_tree(max_length); - - if (PrepareAppendRegion(root, region, size, max_length)) { + size_t extra = has_length ? length : (std::max)(sz, kMinFlatLength); + CordRep* rep = root ? root : MakeFlatWithExtraCapacity(extra); + CordzUpdateScope scope(root ? data_.cordz_info() : nullptr, method); + if (PrepareAppendRegion(rep, region, size, length)) { + CommitTree(root, rep, scope, method); return; } // Allocate new node. - CordRepFlat* new_node = - CordRepFlat::New(std::max(static_cast<size_t>(root->length), max_length)); - new_node->length = std::min(new_node->Capacity(), max_length); + CordRepFlat* new_node = CordRepFlat::New(extra); + new_node->length = std::min(new_node->Capacity(), length); *region = new_node->Data(); *size = new_node->length; if (cord_ring_enabled()) { - replace_tree(CordRepRing::Append(ForceRing(root, 1), new_node)); - return; + rep = CordRepRing::Append(ForceRing(rep, 1), new_node); + } else { + rep = Concat(rep, new_node); } - replace_tree(Concat(root, new_node)); -} - -void Cord::InlineRep::GetAppendRegion(char** region, size_t* size) { - const size_t max_length = std::numeric_limits<size_t>::max(); - - // Try to fit in the inline buffer if possible. - if (!data_.is_tree()) { - size_t inline_length = inline_size(); - if (inline_length < kMaxInline) { - *region = data_.as_chars() + inline_length; - *size = kMaxInline - inline_length; - set_inline_size(kMaxInline); - return; - } - } - - CordRep* root = force_tree(max_length); - - if (PrepareAppendRegion(root, region, size, max_length)) { - return; - } - - // Allocate new node. - CordRepFlat* new_node = CordRepFlat::New(root->length); - new_node->length = new_node->Capacity(); - *region = new_node->Data(); - *size = new_node->length; - - if (cord_ring_enabled()) { - replace_tree(CordRepRing::Append(ForceRing(root, 1), new_node)); - return; - } - replace_tree(Concat(root, new_node)); + CommitTree(root, rep, scope, method); } // If the rep is a leaf, this will increment the value at total_mem_usage and @@ -490,18 +492,32 @@ return false; } -void Cord::InlineRep::AssignSlow(const Cord::InlineRep& src) { - UnrefTree(); +void Cord::InlineRep::UpdateCordzStatisticsSlow() { + CordRep* tree = as_tree(); + data_.cordz_info()->RecordMetrics(tree->length); +} - data_ = src.data_; - if (is_tree()) { - CordRep::Ref(tree()); - clear_cordz_info(); +void Cord::InlineRep::AssignSlow(const Cord::InlineRep& src) { + assert(&src != this); + assert(is_tree() || src.is_tree()); + auto constexpr method = CordzUpdateTracker::kAssignCord; + if (CordRep* tree = this->tree()) { + CordzUpdateScope scope(data_.cordz_info(), method); + CordRep::Unref(tree); + if (CordRep* src_tree = src.tree()) { + SetTree(CordRep::Ref(src_tree), scope); + } else { + scope.SetCordRep(nullptr); + data_ = src.data_; + } + } else { + EmplaceTree(CordRep::Ref(src.as_tree()), method); } } void Cord::InlineRep::UnrefTree() { if (is_tree()) { + CordzInfo::MaybeUntrackCord(data_.cordz_info()); CordRep::Unref(tree()); } } @@ -509,12 +525,13 @@ // -------------------------------------------------------------------- // Constructors and destructors -Cord::Cord(absl::string_view src) { +Cord::Cord(absl::string_view src) : contents_(InlineData::kDefaultInit) { const size_t n = src.size(); if (n <= InlineRep::kMaxInline) { - contents_.set_data(src.data(), n, false); + contents_.set_data(src.data(), n, true); } else { - contents_.set_tree(NewTree(src.data(), n, 0)); + CordRep* rep = NewTree(src.data(), n, 0); + contents_.EmplaceTree(rep, CordzUpdateTracker::kConstructorString); } } @@ -552,9 +569,9 @@ // The destruction code is separate so that the compiler can determine // that it does not need to call the destructor on a moved-from Cord. void Cord::DestroyCordSlow() { - if (CordRep* tree = contents_.tree()) { - CordRep::Unref(VerifyTree(tree)); - } + assert(contents_.is_tree()); + CordzInfo::MaybeUntrackCord(contents_.cordz_info()); + CordRep::Unref(VerifyTree(contents_.as_tree())); } // -------------------------------------------------------------------- @@ -567,19 +584,18 @@ } Cord& Cord::operator=(absl::string_view src) { - const char* data = src.data(); size_t length = src.size(); CordRep* tree = contents_.tree(); if (length <= InlineRep::kMaxInline) { // Embed into this->contents_ + if (tree) CordzInfo::MaybeUntrackCord(contents_.cordz_info()); contents_.set_data(data, length, true); if (tree) CordRep::Unref(tree); return *this; } if (tree != nullptr && tree->tag >= FLAT && - tree->flat()->Capacity() >= length && - tree->refcount.IsOne()) { + tree->flat()->Capacity() >= length && tree->refcount.IsOne()) { // Copy in place if the existing FLAT node is reusable. memmove(tree->flat()->Data(), data, length); tree->length = length; @@ -605,70 +621,70 @@ // TODO(sanjay): Move to Cord::InlineRep section of file. For now, // we keep it here to make diffs easier. -void Cord::InlineRep::AppendArray(const char* src_data, size_t src_size) { - if (src_size == 0) return; // memcpy(_, nullptr, 0) is undefined. +void Cord::InlineRep::AppendArray(absl::string_view src, + MethodIdentifier method) { + if (src.empty()) return; // memcpy(_, nullptr, 0) is undefined. size_t appended = 0; - CordRep* root = nullptr; - if (is_tree()) { - root = data_.as_tree(); + CordRep* rep = tree(); + const CordRep* const root = rep; + CordzUpdateScope scope(root ? cordz_info() : nullptr, method); + if (root != nullptr) { char* region; - if (PrepareAppendRegion(root, ®ion, &appended, src_size)) { - memcpy(region, src_data, appended); + if (PrepareAppendRegion(rep, ®ion, &appended, src.size())) { + memcpy(region, src.data(), appended); } } else { // Try to fit in the inline buffer if possible. size_t inline_length = inline_size(); - if (src_size <= kMaxInline - inline_length) { + if (src.size() <= kMaxInline - inline_length) { // Append new data to embedded array - memcpy(data_.as_chars() + inline_length, src_data, src_size); - set_inline_size(inline_length + src_size); + memcpy(data_.as_chars() + inline_length, src.data(), src.size()); + set_inline_size(inline_length + src.size()); return; } - // It is possible that src_data == data_, but when we transition from an + // It is possible that src.data() == data_, but when we transition from an // InlineRep to a tree we need to assign data_ = root via set_tree. To // avoid corrupting the source data before we copy it, delay calling // set_tree until after we've copied data. // We are going from an inline size to beyond inline size. Make the new size // either double the inlined size, or the added size + 10%. - const size_t size1 = inline_length * 2 + src_size; - const size_t size2 = inline_length + src_size / 10; - root = CordRepFlat::New(std::max<size_t>(size1, size2)); - appended = std::min( - src_size, root->flat()->Capacity() - inline_length); - memcpy(root->flat()->Data(), data_.as_chars(), inline_length); - memcpy(root->flat()->Data() + inline_length, src_data, appended); - root->length = inline_length + appended; - set_tree(root); + const size_t size1 = inline_length * 2 + src.size(); + const size_t size2 = inline_length + src.size() / 10; + rep = CordRepFlat::New(std::max<size_t>(size1, size2)); + appended = std::min(src.size(), rep->flat()->Capacity() - inline_length); + memcpy(rep->flat()->Data(), data_.as_chars(), inline_length); + memcpy(rep->flat()->Data() + inline_length, src.data(), appended); + rep->length = inline_length + appended; } - src_data += appended; - src_size -= appended; - if (src_size == 0) { + src.remove_prefix(appended); + if (src.empty()) { + CommitTree(root, rep, scope, method); return; } if (cord_ring_enabled()) { - absl::string_view data(src_data, src_size); - root = ForceRing(root, (data.size() - 1) / kMaxFlatLength + 1); - replace_tree(CordRepRing::Append(root->ring(), data)); - return; + rep = ForceRing(rep, (src.size() - 1) / kMaxFlatLength + 1); + rep = CordRepRing::Append(rep->ring(), src); + } else { + // Use new block(s) for any remaining bytes that were not handled above. + // Alloc extra memory only if the right child of the root of the new tree + // is going to be a FLAT node, which will permit further inplace appends. + size_t length = src.size(); + if (src.size() < kMaxFlatLength) { + // The new length is either + // - old size + 10% + // - old_size + src.size() + // This will cause a reasonable conservative step-up in size that is + // still large enough to avoid excessive amounts of small fragments + // being added. + length = std::max<size_t>(rep->length / 10, src.size()); + } + rep = Concat(rep, NewTree(src.data(), src.size(), length - src.size())); } - - // Use new block(s) for any remaining bytes that were not handled above. - // Alloc extra memory only if the right child of the root of the new tree is - // going to be a FLAT node, which will permit further inplace appends. - size_t length = src_size; - if (src_size < kMaxFlatLength) { - // The new length is either - // - old size + 10% - // - old_size + src_size - // This will cause a reasonable conservative step-up in size that is still - // large enough to avoid excessive amounts of small fragments being added. - length = std::max<size_t>(root->length / 10, src_size); - } - set_tree(Concat(root, NewTree(src_data, src_size, length - src_size))); + CommitTree(root, rep, scope, method); } inline CordRep* Cord::TakeRep() const& { @@ -683,10 +699,17 @@ template <typename C> inline void Cord::AppendImpl(C&& src) { + auto constexpr method = CordzUpdateTracker::kAppendCord; if (empty()) { - // In case of an empty destination avoid allocating a new node, do not copy - // data. - *this = std::forward<C>(src); + // Since destination is empty, we can avoid allocating a node, + if (src.contents_.is_tree()) { + // by taking the tree directly + CordRep* rep = std::forward<C>(src).TakeRep(); + contents_.EmplaceTree(rep, method); + } else { + // or copying over inline data + contents_.data_ = src.contents_.data_; + } return; } @@ -696,12 +719,12 @@ CordRep* src_tree = src.contents_.tree(); if (src_tree == nullptr) { // src has embedded data. - contents_.AppendArray(src.contents_.data(), src_size); + contents_.AppendArray({src.contents_.data(), src_size}, method); return; } if (src_tree->tag >= FLAT) { // src tree just has one flat node. - contents_.AppendArray(src_tree->flat()->Data(), src_size); + contents_.AppendArray({src_tree->flat()->Data(), src_size}, method); return; } if (&src == this) { @@ -717,7 +740,8 @@ } // Guaranteed to be a tree (kMaxBytesToCopy > kInlinedSize) - contents_.AppendTree(std::forward<C>(src).TakeRep()); + CordRep* rep = std::forward<C>(src).TakeRep(); + contents_.AppendTree(rep, CordzUpdateTracker::kAppendCord); } void Cord::Append(const Cord& src) { AppendImpl(src); } @@ -739,7 +763,7 @@ CordRep* src_tree = src.contents_.tree(); if (src_tree != nullptr) { CordRep::Ref(src_tree); - contents_.PrependTree(src_tree); + contents_.PrependTree(src_tree, CordzUpdateTracker::kPrependCord); return; } @@ -762,7 +786,8 @@ return; } } - contents_.PrependTree(NewTree(src.data(), src.size(), 0)); + CordRep* rep = NewTree(src.data(), src.size(), 0); + contents_.PrependTree(rep, CordzUpdateTracker::kPrependString); } template <typename T, Cord::EnableIfString<T>> @@ -1789,8 +1814,7 @@ *os << absl::CEscape(std::string(rep->external()->base, rep->length)); *os << "]\n"; } else if (rep->tag >= FLAT) { - *os << "FLAT cap=" << rep->flat()->Capacity() - << " ["; + *os << "FLAT cap=" << rep->flat()->Capacity() << " ["; if (include_data) *os << absl::CEscape(std::string(rep->flat()->Data(), rep->length)); *os << "]\n"; @@ -1802,7 +1826,7 @@ do { DumpNode(ring->entry_child(head), include_data, os, indent + kIndentStep); - head = ring->advance(head);; + head = ring->advance(head); } while (head != ring->tail()); } if (stack.empty()) break; @@ -1848,9 +1872,8 @@ worklist.push_back(node->concat()->left); } } else if (node->tag >= FLAT) { - ABSL_INTERNAL_CHECK( - node->length <= node->flat()->Capacity(), - ReportError(root, node)); + ABSL_INTERNAL_CHECK(node->length <= node->flat()->Capacity(), + ReportError(root, node)); } else if (node->tag == EXTERNAL) { ABSL_INTERNAL_CHECK(node->external()->base != nullptr, ReportError(root, node));
diff --git a/absl/strings/cord.h b/absl/strings/cord.h index 7ce938c..d5a13b3 100644 --- a/absl/strings/cord.h +++ b/absl/strings/cord.h
@@ -70,6 +70,7 @@ #include <string> #include <type_traits> +#include "absl/base/config.h" #include "absl/base/internal/endian.h" #include "absl/base/internal/per_thread_tls.h" #include "absl/base/macros.h" @@ -80,6 +81,11 @@ #include "absl/strings/internal/cord_internal.h" #include "absl/strings/internal/cord_rep_ring.h" #include "absl/strings/internal/cord_rep_ring_reader.h" +#include "absl/strings/internal/cordz_functions.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_scope.h" +#include "absl/strings/internal/cordz_update_tracker.h" #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/internal/string_constant.h" #include "absl/strings/string_view.h" @@ -664,10 +670,20 @@ explicit constexpr Cord(strings_internal::StringConstant<T>); private: + using CordRep = absl::cord_internal::CordRep; + using CordRepFlat = absl::cord_internal::CordRepFlat; + using CordzInfo = cord_internal::CordzInfo; + using CordzUpdateScope = cord_internal::CordzUpdateScope; + using CordzUpdateTracker = cord_internal::CordzUpdateTracker; + using InlineData = cord_internal::InlineData; + using MethodIdentifier = CordzUpdateTracker::MethodIdentifier; + friend class CordTestPeer; friend bool operator==(const Cord& lhs, const Cord& rhs); friend bool operator==(const Cord& lhs, absl::string_view rhs); + friend const CordzInfo* GetCordzInfoForTesting(const Cord& cord); + // Calls the provided function once for each cord chunk, in order. Unlike // Chunks(), this API will not allocate memory. void ForEachChunk(absl::FunctionRef<void(absl::string_view)>) const; @@ -687,6 +703,7 @@ static_assert(kMaxInline >= sizeof(absl::cord_internal::CordRep*), ""); constexpr InlineRep() : data_() {} + explicit InlineRep(InlineData::DefaultInitType init) : data_(init) {} InlineRep(const InlineRep& src); InlineRep(InlineRep&& src); InlineRep& operator=(const InlineRep& src); @@ -712,15 +729,44 @@ // Returns non-null iff was holding a pointer absl::cord_internal::CordRep* clear(); // Converts to pointer if necessary. - absl::cord_internal::CordRep* force_tree(size_t extra_hint); void reduce_size(size_t n); // REQUIRES: holding data void remove_prefix(size_t n); // REQUIRES: holding data - void AppendArray(const char* src_data, size_t src_size); + void AppendArray(absl::string_view src, MethodIdentifier method); absl::string_view FindFlatStartPiece() const; - void AppendTree(absl::cord_internal::CordRep* tree); - void PrependTree(absl::cord_internal::CordRep* tree); - void GetAppendRegion(char** region, size_t* size, size_t max_length); - void GetAppendRegion(char** region, size_t* size); + + // Creates a CordRepFlat instance from the current inlined data with `extra' + // bytes of desired additional capacity. + CordRepFlat* MakeFlatWithExtraCapacity(size_t extra); + + // Sets the tree value for this instance. `rep` must not be null. + // Requires the current instance to hold a tree, and a lock to be held on + // any CordzInfo referenced by this instance. The latter is enforced through + // the CordzUpdateScope argument. If the current instance is sampled, then + // the CordzInfo instance is updated to reference the new `rep` value. + void SetTree(CordRep* rep, const CordzUpdateScope& scope); + + // Sets the tree value for this instance, and randomly samples this cord. + // This function disregards existing contents in `data_`, and should be + // called when a Cord is 'promoted' from an 'uninitialized' or 'inlined' + // value to a non-inlined (tree / ring) value. + void EmplaceTree(CordRep* rep, MethodIdentifier method); + + // Commits the change of a newly created, or updated `rep` root value into + // this cord. `old_rep` indicates the old (inlined or tree) value of the + // cord, and determines if the commit invokes SetTree() or EmplaceTree(). + void CommitTree(const CordRep* old_rep, CordRep* rep, + const CordzUpdateScope& scope, MethodIdentifier method); + + void AppendTreeToInlined(CordRep* tree, MethodIdentifier method); + void AppendTreeToTree(CordRep* tree, MethodIdentifier method); + void AppendTree(CordRep* tree, MethodIdentifier method); + void PrependTreeToInlined(CordRep* tree, MethodIdentifier method); + void PrependTreeToTree(CordRep* tree, MethodIdentifier method); + void PrependTree(CordRep* tree, MethodIdentifier method); + + template <bool has_length> + void GetAppendRegion(char** region, size_t* size, size_t length); + bool IsSame(const InlineRep& other) const { return memcmp(&data_, &other.data_, sizeof(data_)) == 0; } @@ -772,6 +818,11 @@ // Resets the current cordz_info to null / empty. void clear_cordz_info() { data_.clear_cordz_info(); } + // Updates the cordz statistics. info may be nullptr if the CordzInfo object + // is unknown. + void UpdateCordzStatistics(); + void UpdateCordzStatisticsSlow(); + private: friend class Cord; @@ -943,6 +994,8 @@ if (is_tree()) { data_.clear_cordz_info(); absl::cord_internal::CordRep::Ref(as_tree()); + CordzInfo::MaybeTrackCord(data_, src.data_, + CordzUpdateTracker::kConstructorCord); } } @@ -1002,8 +1055,45 @@ return is_tree() ? as_tree()->length : inline_size(); } +inline cord_internal::CordRepFlat* Cord::InlineRep::MakeFlatWithExtraCapacity( + size_t extra) { + static_assert(cord_internal::kMinFlatLength >= sizeof(data_), ""); + size_t len = data_.inline_size(); + auto* result = CordRepFlat::New(len + extra); + result->length = len; + memcpy(result->Data(), data_.as_chars(), sizeof(data_)); + return result; +} + +inline void Cord::InlineRep::EmplaceTree(CordRep* rep, + MethodIdentifier method) { + data_.make_tree(rep); + CordzInfo::MaybeTrackCord(data_, method); +} + +inline void Cord::InlineRep::SetTree(CordRep* rep, + const CordzUpdateScope& scope) { + assert(rep); + assert(data_.is_tree()); + data_.set_tree(rep); + scope.SetCordRep(rep); +} + +inline void Cord::InlineRep::CommitTree(const CordRep* old_rep, CordRep* rep, + const CordzUpdateScope& scope, + MethodIdentifier method) { + if (old_rep) { + SetTree(rep, scope); + } else { + EmplaceTree(rep, method); + } +} + inline void Cord::InlineRep::set_tree(absl::cord_internal::CordRep* rep) { if (rep == nullptr) { + if (data_.is_tree()) { + CordzInfo::MaybeUntrackCord(data_.cordz_info()); + } ResetToEmpty(); } else { if (data_.is_tree()) { @@ -1013,7 +1103,9 @@ } else { // `data_` contains inlined data: initialize data_ to tree value `rep`. data_.make_tree(rep); + CordzInfo::MaybeTrackCord(data_, CordzUpdateTracker::kUnknown); } + UpdateCordzStatistics(); } } @@ -1024,9 +1116,13 @@ return; } data_.set_tree(rep); + UpdateCordzStatistics(); } inline absl::cord_internal::CordRep* Cord::InlineRep::clear() { + if (is_tree()) { + CordzInfo::MaybeUntrackCord(cordz_info()); + } absl::cord_internal::CordRep* result = tree(); ResetToEmpty(); return result; @@ -1039,6 +1135,11 @@ cord_internal::SmallMemmove(dst, data_.as_chars(), n); } +inline void Cord::InlineRep::UpdateCordzStatistics() { + if (ABSL_PREDICT_TRUE(!is_profiled())) return; + UpdateCordzStatisticsSlow(); +} + constexpr inline Cord::Cord() noexcept {} template <typename T> @@ -1113,7 +1214,7 @@ } inline void Cord::Append(absl::string_view src) { - contents_.AppendArray(src.data(), src.size()); + contents_.AppendArray(src, CordzUpdateTracker::kAppendString); } extern template void Cord::Append(std::string&& src);
diff --git a/absl/strings/cord_test.cc b/absl/strings/cord_test.cc index 710a1b4..74a9086 100644 --- a/absl/strings/cord_test.cc +++ b/absl/strings/cord_test.cc
@@ -35,6 +35,7 @@ #include "absl/base/macros.h" #include "absl/container/fixed_array.h" #include "absl/strings/cord_test_helpers.h" +#include "absl/strings/cordz_test_helpers.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" @@ -239,7 +240,6 @@ // caused crashes in production. We grow exponentially so that the code will // execute in a reasonable amount of time. absl::Cord c; - ABSL_RAW_LOG(INFO, "Made a Cord with %zu bytes!", c.size()); c.Append(from); while (c.size() < max_size) { c.Append(c);
diff --git a/absl/strings/cord_test_helpers.h b/absl/strings/cord_test_helpers.h index f1036e3..6ccccc5 100644 --- a/absl/strings/cord_test_helpers.h +++ b/absl/strings/cord_test_helpers.h
@@ -17,11 +17,50 @@ #ifndef ABSL_STRINGS_CORD_TEST_HELPERS_H_ #define ABSL_STRINGS_CORD_TEST_HELPERS_H_ +#include <cstdint> +#include <iostream> + #include "absl/strings/cord.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/string_view.h" namespace absl { ABSL_NAMESPACE_BEGIN +// Cord sizes relevant for testing +enum class TestCordSize { + kEmpty = 0, + kInlined = cord_internal::kMaxInline / 2 + 1, + kSmall = cord_internal::kMaxBytesToCopy / 2 + 1, + kMedium = cord_internal::kMaxFlatLength / 2 + 1, + kLarge = cord_internal::kMaxFlatLength * 4 +}; + +// To string helper +inline absl::string_view ToString(TestCordSize size) { + switch (size) { + case TestCordSize::kEmpty: + return "Empty"; + case TestCordSize::kInlined: + return "Inlined"; + case TestCordSize::kSmall: + return "Small"; + case TestCordSize::kMedium: + return "Medium"; + case TestCordSize::kLarge: + return "Large"; + } + return "???"; +} + +// Returns the length matching the specified size +inline size_t Length(TestCordSize size) { return static_cast<size_t>(size); } + +// Stream output helper +inline std::ostream& operator<<(std::ostream& stream, TestCordSize size) { + return stream << ToString(size); +} + // Creates a multi-segment Cord from an iterable container of strings. The // resulting Cord is guaranteed to have one segment for every string in the // container. This allows code to be unit tested with multi-segment Cord
diff --git a/absl/strings/cordz_test.cc b/absl/strings/cordz_test.cc new file mode 100644 index 0000000..b16e968 --- /dev/null +++ b/absl/strings/cordz_test.cc
@@ -0,0 +1,174 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstdint> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/strings/cord.h" +#include "absl/strings/cord_test_helpers.h" +#include "absl/strings/cordz_test_helpers.h" +#include "absl/strings/internal/cordz_functions.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/strings/internal/cordz_sample_token.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_tracker.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +#ifdef ABSL_INTERNAL_CORDZ_ENABLED + +using testing::Eq; + +namespace absl { +ABSL_NAMESPACE_BEGIN + +using cord_internal::CordzInfo; +using cord_internal::CordzSampleToken; +using cord_internal::CordzStatistics; +using cord_internal::CordzUpdateTracker; +using Method = CordzUpdateTracker::MethodIdentifier; + +// Do not print cord contents, we only care about 'size' perhaps. +// Note that this method must be inside the named namespace. +inline void PrintTo(const Cord& cord, std::ostream* s) { + if (s) *s << "Cord[" << cord.size() << "]"; +} + +namespace { + +// Returns a string_view value of the specified length +// We do this to avoid 'consuming' large strings in Cord by default. +absl::string_view MakeString(size_t size) { + thread_local std::string str; + str = std::string(size, '.'); + return str; +} + +absl::string_view MakeString(TestCordSize size) { + return MakeString(Length(size)); +} + +std::string TestParamToString(::testing::TestParamInfo<TestCordSize> size) { + return absl::StrCat("On", ToString(size.param), "Cord"); +} + +class CordzUpdateTest : public testing::TestWithParam<TestCordSize> { + public: + Cord& cord() { return cord_; } + + Method InitialOr(Method method) const { + return (GetParam() > TestCordSize::kInlined) ? Method::kConstructorString + : method; + } + + private: + CordzSamplingIntervalHelper sample_every_{1}; + Cord cord_{MakeString(GetParam())}; +}; + +INSTANTIATE_TEST_SUITE_P(WithParam, CordzUpdateTest, + testing::Values(TestCordSize::kEmpty, + TestCordSize::kInlined, + TestCordSize::kLarge), + TestParamToString); + +TEST(CordzTest, ConstructSmallString) { + CordzSamplingIntervalHelper sample_every{1}; + Cord cord(MakeString(TestCordSize::kSmall)); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); +} + +TEST(CordzTest, ConstructLargeString) { + CordzSamplingIntervalHelper sample_every{1}; + Cord cord(MakeString(TestCordSize::kLarge)); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); +} + +TEST(CordzTest, CopyConstruct) { + CordzSamplingIntervalHelper sample_every{1}; + Cord src = UnsampledCord(MakeString(TestCordSize::kLarge)); + Cord cord(src); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorCord)); +} + +TEST(CordzTest, MoveConstruct) { + CordzSamplingIntervalHelper sample_every{1}; + Cord src(MakeString(TestCordSize::kLarge)); + Cord cord(std::move(src)); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); +} + +TEST_P(CordzUpdateTest, AssignCord) { + Cord src = UnsampledCord(MakeString(TestCordSize::kLarge)); + cord() = src; + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kAssignCord))); +} + +TEST(CordzTest, AssignInlinedCord) { + CordzSampleToken token; + CordzSamplingIntervalHelper sample_every{1}; + Cord cord(MakeString(TestCordSize::kLarge)); + const CordzInfo* info = GetCordzInfoForTesting(cord); + Cord src = UnsampledCord(MakeString(TestCordSize::kInlined)); + cord = src; + EXPECT_THAT(GetCordzInfoForTesting(cord), Eq(nullptr)); + EXPECT_FALSE(CordzInfoIsListed(info)); +} + +TEST(CordzTest, MoveAssignCord) { + CordzSamplingIntervalHelper sample_every{1}; + Cord cord; + Cord src(MakeString(TestCordSize::kLarge)); + cord = std::move(src); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); +} + +TEST_P(CordzUpdateTest, AppendCord) { + Cord src = UnsampledCord(MakeString(TestCordSize::kLarge)); + cord().Append(src); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kAppendCord))); +} + +TEST_P(CordzUpdateTest, MoveAppendCord) { + cord().Append(UnsampledCord(MakeString(TestCordSize::kLarge))); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kAppendCord))); +} + +TEST_P(CordzUpdateTest, PrependCord) { + Cord src = UnsampledCord(MakeString(TestCordSize::kLarge)); + cord().Prepend(src); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kPrependCord))); +} + +TEST_P(CordzUpdateTest, AppendSmallArray) { + cord().Append(MakeString(TestCordSize::kSmall)); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kAppendString))); +} + +TEST_P(CordzUpdateTest, AppendLargeArray) { + cord().Append(MakeString(TestCordSize::kLarge)); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kAppendString))); +} + +} // namespace + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_INTERNAL_CORDZ_ENABLED
diff --git a/absl/strings/cordz_test_helpers.h b/absl/strings/cordz_test_helpers.h new file mode 100644 index 0000000..d9573c9 --- /dev/null +++ b/absl/strings/cordz_test_helpers.h
@@ -0,0 +1,134 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_CORDZ_TEST_HELPERS_H_ +#define ABSL_STRINGS_CORDZ_TEST_HELPERS_H_ + +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/macros.h" +#include "absl/strings/cord.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/strings/internal/cordz_sample_token.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_tracker.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Returns the CordzInfo for the cord, or nullptr if the cord is not sampled. +inline const cord_internal::CordzInfo* GetCordzInfoForTesting( + const Cord& cord) { + if (cord.size() <= cord_internal::kMaxInline) return nullptr; + return cord.contents_.cordz_info(); +} + +// Returns true if the provided cordz_info is in the list of sampled cords. +inline bool CordzInfoIsListed(const cord_internal::CordzInfo* cordz_info, + cord_internal::CordzSampleToken token = {}) { + for (const cord_internal::CordzInfo& info : token) { + if (cordz_info == &info) return true; + } + return false; +} + +// Matcher on Cord* that verifies all of: +// - the cord is sampled +// - the CordzInfo of the cord is listed / discoverable. +// - the reported CordzStatistics match the cord's actual properties +// - the cord has an (initial) UpdateTracker count of 1 for `method` +// This matcher accepts a const Cord* to avoid having the matcher dump +// copious amounts of cord data on failures. +MATCHER_P(HasValidCordzInfoOf, method, "CordzInfo matches cord") { + const cord_internal::CordzInfo* cord_info = GetCordzInfoForTesting(arg); + if (cord_info == nullptr) { + *result_listener << "cord is not sampled"; + return false; + } + if (!CordzInfoIsListed(cord_info)) { + *result_listener << "cord is sampled, but not listed"; + return false; + } + cord_internal::CordzStatistics stat = cord_info->GetCordzStatistics(); + if (stat.size != arg.size()) { + *result_listener << "cordz size " << stat.size + << " does not match cord size " << arg.size(); + return false; + } + if (stat.update_tracker.Value(method) != 1) { + *result_listener << "Expected method count 1 for " << method << ", found " + << stat.update_tracker.Value(method); + return false; + } + return true; +} + +// Cordz will only update with a new rate once the previously scheduled event +// has fired. When we disable Cordz, a long delay takes place where we won't +// consider profiling new Cords. CordzSampleIntervalHelper will burn through +// that interval and allow for testing that assumes that the average sampling +// interval is a particular value. +class CordzSamplingIntervalHelper { + public: + explicit CordzSamplingIntervalHelper(int32_t interval) + : orig_mean_interval_(absl::cord_internal::get_cordz_mean_interval()) { + absl::cord_internal::set_cordz_mean_interval(interval); + absl::cord_internal::cordz_set_next_sample_for_testing(interval); + } + + ~CordzSamplingIntervalHelper() { + absl::cord_internal::set_cordz_mean_interval(orig_mean_interval_); + absl::cord_internal::cordz_set_next_sample_for_testing(orig_mean_interval_); + } + + private: + int32_t orig_mean_interval_; +}; + +// Wrapper struct managing a small CordRep `rep` +struct TestCordRep { + cord_internal::CordRepFlat* rep; + + TestCordRep() { + rep = cord_internal::CordRepFlat::New(100); + rep->length = 100; + memset(rep->Data(), 1, 100); + } + ~TestCordRep() { cord_internal::CordRep::Unref(rep); } +}; + +// Wrapper struct managing a small CordRep `rep`, and +// an InlineData `data` initialized with that CordRep. +struct TestCordData { + TestCordRep rep; + cord_internal::InlineData data{rep.rep}; +}; + +// Creates a Cord that is not sampled +template <typename... Args> +Cord UnsampledCord(Args... args) { + CordzSamplingIntervalHelper never(9999); + Cord cord(std::forward<Args>(args)...); + ABSL_ASSERT(GetCordzInfoForTesting(cord) == nullptr); + return cord; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORDZ_TEST_HELPERS_H_
diff --git a/absl/strings/internal/cord_internal.h b/absl/strings/internal/cord_internal.h index 01ce70a..c7ae010 100644 --- a/absl/strings/internal/cord_internal.h +++ b/absl/strings/internal/cord_internal.h
@@ -329,6 +329,9 @@ class InlineData { public: + // DefaultInitType forces the use of the default initialization constructor. + enum DefaultInitType { kDefaultInit }; + // kNullCordzInfo holds the big endian representation of intptr_t(1) // This is the 'null' / initial value of 'cordz_info'. The null value // is specifically big endian 1 as with 64-bit pointers, the last @@ -336,6 +339,7 @@ static constexpr cordz_info_t kNullCordzInfo = BigEndianByte(1); constexpr InlineData() : as_chars_{0} {} + explicit InlineData(DefaultInitType) {} explicit constexpr InlineData(CordRep* rep) : as_tree_(rep) {} explicit constexpr InlineData(absl::string_view chars) : as_chars_{
diff --git a/absl/strings/internal/cordz_functions.cc b/absl/strings/internal/cordz_functions.cc new file mode 100644 index 0000000..6ad864f --- /dev/null +++ b/absl/strings/internal/cordz_functions.cc
@@ -0,0 +1,104 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_functions.h" + +#include <atomic> +#include <cmath> +#include <limits> +#include <random> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/exponential_biased.h" +#include "absl/base/internal/raw_logging.h" + +// TODO(b/162942788): weak 'cordz_disabled' value. +// A strong version is in the 'cordz_disabled_hack_for_odr' library which can +// be linked in to disable cordz at compile time. +extern "C" { +bool absl_internal_cordz_disabled ABSL_ATTRIBUTE_WEAK = false; +} + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +// The average interval until the next sample. A value of 0 disables profiling +// while a value of 1 will profile all Cords. +std::atomic<int> g_cordz_mean_interval(50000); + +} // namespace + +#ifdef ABSL_INTERNAL_CORDZ_ENABLED + +ABSL_CONST_INIT thread_local int64_t cordz_next_sample = 0; + +// kIntervalIfDisabled is the number of profile-eligible events need to occur +// before the code will confirm that cordz is still disabled. +constexpr int64_t kIntervalIfDisabled = 1 << 16; + +ABSL_ATTRIBUTE_NOINLINE bool cordz_should_profile_slow() { + // TODO(b/162942788): check if profiling is disabled at compile time. + if (absl_internal_cordz_disabled) { + ABSL_RAW_LOG(WARNING, "Cordz info disabled at compile time"); + // We are permanently disabled: set counter to highest possible value. + cordz_next_sample = std::numeric_limits<int64_t>::max(); + return false; + } + + thread_local absl::base_internal::ExponentialBiased + exponential_biased_generator; + int32_t mean_interval = get_cordz_mean_interval(); + + // Check if we disabled profiling. If so, set the next sample to a "large" + // number to minimize the overhead of the should_profile codepath. + if (mean_interval <= 0) { + cordz_next_sample = kIntervalIfDisabled; + return false; + } + + // Check if we're always sampling. + if (mean_interval == 1) { + cordz_next_sample = 1; + return true; + } + + if (cordz_next_sample <= 0) { + cordz_next_sample = exponential_biased_generator.GetStride(mean_interval); + return true; + } + + --cordz_next_sample; + return false; +} + +void cordz_set_next_sample_for_testing(int64_t next_sample) { + cordz_next_sample = next_sample; +} + +#endif // ABSL_INTERNAL_CORDZ_ENABLED + +int32_t get_cordz_mean_interval() { + return g_cordz_mean_interval.load(std::memory_order_acquire); +} + +void set_cordz_mean_interval(int32_t mean_interval) { + g_cordz_mean_interval.store(mean_interval, std::memory_order_release); +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/strings/internal/cordz_functions.h b/absl/strings/internal/cordz_functions.h new file mode 100644 index 0000000..c9ba145 --- /dev/null +++ b/absl/strings/internal/cordz_functions.h
@@ -0,0 +1,85 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_CORDZ_FUNCTIONS_H_ +#define ABSL_STRINGS_CORDZ_FUNCTIONS_H_ + +#include <stdint.h> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// Returns the current sample rate. This represents the average interval +// between samples. +int32_t get_cordz_mean_interval(); + +// Sets the sample rate with the average interval between samples. +void set_cordz_mean_interval(int32_t mean_interval); + +// Enable cordz unless any of the following applies: +// - no thread local support +// - MSVC build +// - Android build +// - Apple build +// - DLL build +// Hashtablez is turned off completely in opensource builds. +// MSVC's static atomics are dynamically initialized in debug mode, which breaks +// sampling. +#if defined(ABSL_HAVE_THREAD_LOCAL) && !defined(_MSC_VER) && \ + !defined(ABSL_BUILD_DLL) && !defined(ABSL_CONSUME_DLL) && \ + !defined(__ANDROID__) && !defined(__APPLE__) +#define ABSL_INTERNAL_CORDZ_ENABLED 1 +#endif + +#ifdef ABSL_INTERNAL_CORDZ_ENABLED + +// cordz_next_sample is the number of events until the next sample event. If +// the value is 1 or less, the code will check on the next event if cordz is +// enabled, and if so, will sample the Cord. cordz is only enabled when we can +// use thread locals. +ABSL_CONST_INIT extern thread_local int64_t cordz_next_sample; + +// Determines if the next sample should be profiled. If it is, the value pointed +// at by next_sample will be set with the interval until the next sample. +bool cordz_should_profile_slow(); + +// Returns true if the next cord should be sampled. +inline bool cordz_should_profile() { + if (ABSL_PREDICT_TRUE(cordz_next_sample > 1)) { + cordz_next_sample--; + return false; + } + return cordz_should_profile_slow(); +} + +// Sets the interval until the next sample (for testing only) +void cordz_set_next_sample_for_testing(int64_t next_sample); + +#else // ABSL_INTERNAL_CORDZ_ENABLED + +inline bool cordz_should_profile() { return false; } +inline void cordz_set_next_sample_for_testing(int64_t) {} + +#endif // ABSL_INTERNAL_CORDZ_ENABLED + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORDZ_FUNCTIONS_H_
diff --git a/absl/strings/internal/cordz_functions_test.cc b/absl/strings/internal/cordz_functions_test.cc new file mode 100644 index 0000000..f2cefae --- /dev/null +++ b/absl/strings/internal/cordz_functions_test.cc
@@ -0,0 +1,131 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_functions.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::Eq; +using ::testing::Ge; +using ::testing::Le; + +TEST(CordzFunctionsTest, SampleRate) { + int32_t orig_sample_rate = get_cordz_mean_interval(); + int32_t expected_sample_rate = 123; + set_cordz_mean_interval(expected_sample_rate); + EXPECT_THAT(get_cordz_mean_interval(), Eq(expected_sample_rate)); + set_cordz_mean_interval(orig_sample_rate); +} + +// Cordz is disabled when we don't have thread_local. All calls to +// should_profile will return false when cordz is diabled, so we might want to +// avoid those tests. +#ifdef ABSL_INTERNAL_CORDZ_ENABLED + +TEST(CordzFunctionsTest, ShouldProfileDisable) { + int32_t orig_sample_rate = get_cordz_mean_interval(); + + set_cordz_mean_interval(0); + cordz_set_next_sample_for_testing(0); + EXPECT_FALSE(cordz_should_profile()); + // 1 << 16 is from kIntervalIfDisabled in cordz_functions.cc. + EXPECT_THAT(cordz_next_sample, Eq(1 << 16)); + + set_cordz_mean_interval(orig_sample_rate); +} + +TEST(CordzFunctionsTest, ShouldProfileAlways) { + int32_t orig_sample_rate = get_cordz_mean_interval(); + + set_cordz_mean_interval(1); + cordz_set_next_sample_for_testing(1); + EXPECT_TRUE(cordz_should_profile()); + EXPECT_THAT(cordz_next_sample, Le(1)); + + set_cordz_mean_interval(orig_sample_rate); +} + +TEST(CordzFunctionsTest, ShouldProfileRate) { + static constexpr int kDesiredMeanInterval = 1000; + static constexpr int kSamples = 10000; + int32_t orig_sample_rate = get_cordz_mean_interval(); + + set_cordz_mean_interval(kDesiredMeanInterval); + + int64_t sum_of_intervals = 0; + for (int i = 0; i < kSamples; i++) { + // Setting next_sample to 0 will force cordz_should_profile to generate a + // new value for next_sample each iteration. + cordz_set_next_sample_for_testing(0); + cordz_should_profile(); + sum_of_intervals += cordz_next_sample; + } + + // The sum of independent exponential variables is an Erlang distribution, + // which is a gamma distribution where the shape parameter is equal to the + // number of summands. The distribution used for cordz_should_profile is + // actually floor(Exponential(1/mean)) which introduces bias. However, we can + // apply the squint-really-hard correction factor. That is, when mean is + // large, then if we squint really hard the shape of the distribution between + // N and N+1 looks like a uniform distribution. On average, each value for + // next_sample will be about 0.5 lower than we would expect from an + // exponential distribution. This squint-really-hard correction approach won't + // work when mean is smaller than about 10 but works fine when mean is 1000. + // + // We can use R to calculate a confidence interval. This + // shows how to generate a confidence interval with a false positive rate of + // one in a billion. + // + // $ R -q + // > mean = 1000 + // > kSamples = 10000 + // > errorRate = 1e-9 + // > correction = -kSamples / 2 + // > low = qgamma(errorRate/2, kSamples, 1/mean) + correction + // > high = qgamma(1 - errorRate/2, kSamples, 1/mean) + correction + // > low + // [1] 9396115 + // > high + // [1] 10618100 + EXPECT_THAT(sum_of_intervals, Ge(9396115)); + EXPECT_THAT(sum_of_intervals, Le(10618100)); + + set_cordz_mean_interval(orig_sample_rate); +} + +#else // ABSL_INTERNAL_CORDZ_ENABLED + +TEST(CordzFunctionsTest, ShouldProfileDisabled) { + int32_t orig_sample_rate = get_cordz_mean_interval(); + + set_cordz_mean_interval(1); + cordz_set_next_sample_for_testing(0); + EXPECT_FALSE(cordz_should_profile()); + + set_cordz_mean_interval(orig_sample_rate); +} + +#endif // ABSL_INTERNAL_CORDZ_ENABLED + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/strings/internal/cordz_handle.cc b/absl/strings/internal/cordz_handle.cc new file mode 100644 index 0000000..5297ec8 --- /dev/null +++ b/absl/strings/internal/cordz_handle.cc
@@ -0,0 +1,134 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "absl/strings/internal/cordz_handle.h" + +#include <atomic> + +#include "absl/base/internal/raw_logging.h" // For ABSL_RAW_CHECK +#include "absl/base/internal/spinlock.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +using ::absl::base_internal::SpinLockHolder; + +ABSL_CONST_INIT CordzHandle::Queue CordzHandle::global_queue_(absl::kConstInit); + +CordzHandle::CordzHandle(bool is_snapshot) : is_snapshot_(is_snapshot) { + if (is_snapshot) { + SpinLockHolder lock(&queue_->mutex); + CordzHandle* dq_tail = queue_->dq_tail.load(std::memory_order_acquire); + if (dq_tail != nullptr) { + dq_prev_ = dq_tail; + dq_tail->dq_next_ = this; + } + queue_->dq_tail.store(this, std::memory_order_release); + } +} + +CordzHandle::~CordzHandle() { + ODRCheck(); + if (is_snapshot_) { + std::vector<CordzHandle*> to_delete; + { + SpinLockHolder lock(&queue_->mutex); + CordzHandle* next = dq_next_; + if (dq_prev_ == nullptr) { + // We were head of the queue, delete every CordzHandle until we reach + // either the end of the list, or a snapshot handle. + while (next && !next->is_snapshot_) { + to_delete.push_back(next); + next = next->dq_next_; + } + } else { + // Another CordzHandle existed before this one, don't delete anything. + dq_prev_->dq_next_ = next; + } + if (next) { + next->dq_prev_ = dq_prev_; + } else { + queue_->dq_tail.store(dq_prev_, std::memory_order_release); + } + } + for (CordzHandle* handle : to_delete) { + delete handle; + } + } +} + +void CordzHandle::Delete(CordzHandle* handle) { + if (handle) { + handle->ODRCheck(); + Queue* const queue = handle->queue_; + if (!handle->is_snapshot_ && !queue->IsEmpty()) { + SpinLockHolder lock(&queue->mutex); + CordzHandle* dq_tail = queue->dq_tail.load(std::memory_order_acquire); + if (dq_tail != nullptr) { + handle->dq_prev_ = dq_tail; + dq_tail->dq_next_ = handle; + queue->dq_tail.store(handle, std::memory_order_release); + return; + } + } + delete handle; + } +} + +std::vector<const CordzHandle*> CordzHandle::DiagnosticsGetDeleteQueue() { + std::vector<const CordzHandle*> handles; + SpinLockHolder lock(&global_queue_.mutex); + CordzHandle* dq_tail = global_queue_.dq_tail.load(std::memory_order_acquire); + for (const CordzHandle* p = dq_tail; p; p = p->dq_prev_) { + handles.push_back(p); + } + return handles; +} + +bool CordzHandle::DiagnosticsHandleIsSafeToInspect( + const CordzHandle* handle) const { + ODRCheck(); + if (!is_snapshot_) return false; + if (handle == nullptr) return true; + if (handle->is_snapshot_) return false; + bool snapshot_found = false; + SpinLockHolder lock(&queue_->mutex); + for (const CordzHandle* p = queue_->dq_tail; p; p = p->dq_prev_) { + if (p == handle) return !snapshot_found; + if (p == this) snapshot_found = true; + } + ABSL_ASSERT(snapshot_found); // Assert that 'this' is in delete queue. + return true; +} + +std::vector<const CordzHandle*> +CordzHandle::DiagnosticsGetSafeToInspectDeletedHandles() { + ODRCheck(); + std::vector<const CordzHandle*> handles; + if (!is_snapshot()) { + return handles; + } + + SpinLockHolder lock(&queue_->mutex); + for (const CordzHandle* p = dq_next_; p != nullptr; p = p->dq_next_) { + if (!p->is_snapshot()) { + handles.push_back(p); + } + } + return handles; +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/strings/internal/cordz_handle.h b/absl/strings/internal/cordz_handle.h new file mode 100644 index 0000000..93076cf --- /dev/null +++ b/absl/strings/internal/cordz_handle.h
@@ -0,0 +1,120 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_CORDZ_HANDLE_H_ +#define ABSL_STRINGS_CORDZ_HANDLE_H_ + +#include <atomic> +#include <vector> + +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// This base class allows multiple types of object (CordzInfo and +// CordzSampleToken) to exist simultaneously on the delete queue (pointed to by +// global_dq_tail and traversed using dq_prev_ and dq_next_). The +// delete queue guarantees that once a profiler creates a CordzSampleToken and +// has gained visibility into a CordzInfo object, that CordzInfo object will not +// be deleted prematurely. This allows the profiler to inspect all CordzInfo +// objects that are alive without needing to hold a global lock. +class CordzHandle { + public: + CordzHandle() : CordzHandle(false) {} + + bool is_snapshot() const { return is_snapshot_; } + + // Deletes the provided instance, or puts it on the delete queue to be deleted + // once there are no more sample tokens (snapshot) instances potentially + // referencing the instance. `handle` may be null. + static void Delete(CordzHandle* handle); + + // Returns the current entries in the delete queue in LIFO order. + static std::vector<const CordzHandle*> DiagnosticsGetDeleteQueue(); + + // Returns true if the provided handle is nullptr or guarded by this handle. + // Since the CordzSnapshot token is itself a CordzHandle, this method will + // allow tests to check if that token is keeping an arbitrary CordzHandle + // alive. + bool DiagnosticsHandleIsSafeToInspect(const CordzHandle* handle) const; + + // Returns the current entries in the delete queue, in LIFO order, that are + // protected by this. CordzHandle objects are only placed on the delete queue + // after CordzHandle::Delete is called with them as an argument. Only + // CordzHandle objects that are not also CordzSnapshot objects will be + // included in the return vector. For each of the handles in the return + // vector, the earliest that their memory can be freed is when this + // CordzSnapshot object is deleted. + std::vector<const CordzHandle*> DiagnosticsGetSafeToInspectDeletedHandles(); + + protected: + explicit CordzHandle(bool is_snapshot); + virtual ~CordzHandle(); + + private: + // Global queue data. CordzHandle stores a pointer to the global queue + // instance to harden against ODR violations. + struct Queue { + constexpr explicit Queue(absl::ConstInitType) + : mutex(absl::kConstInit, + absl::base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL) {} + + absl::base_internal::SpinLock mutex; + std::atomic<CordzHandle*> dq_tail ABSL_GUARDED_BY(mutex){nullptr}; + + // Returns true if this delete queue is empty. This method does not acquire + // the lock, but does a 'load acquire' observation on the delete queue tail. + // It is used inside Delete() to check for the presence of a delete queue + // without holding the lock. The assumption is that the caller is in the + // state of 'being deleted', and can not be newly discovered by a concurrent + // 'being constructed' snapshot instance. Practically, this means that any + // such discovery (`find`, 'first' or 'next', etc) must have proper 'happens + // before / after' semantics and atomic fences. + bool IsEmpty() const ABSL_NO_THREAD_SAFETY_ANALYSIS { + return dq_tail.load(std::memory_order_acquire) == nullptr; + } + }; + + void ODRCheck() const { +#ifndef NDEBUG + ABSL_RAW_CHECK(queue_ == &global_queue_, "ODR violation in Cord"); +#endif + } + + ABSL_CONST_INIT static Queue global_queue_; + Queue* const queue_ = &global_queue_; + const bool is_snapshot_; + + // dq_prev_ and dq_next_ require the global queue mutex to be held. + // Unfortunately we can't use thread annotations such that the thread safety + // analysis understands that queue_ and global_queue_ are one and the same. + CordzHandle* dq_prev_ = nullptr; + CordzHandle* dq_next_ = nullptr; +}; + +class CordzSnapshot : public CordzHandle { + public: + CordzSnapshot() : CordzHandle(true) {} +}; + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORDZ_HANDLE_H_
diff --git a/absl/strings/internal/cordz_handle_test.cc b/absl/strings/internal/cordz_handle_test.cc new file mode 100644 index 0000000..c04240e --- /dev/null +++ b/absl/strings/internal/cordz_handle_test.cc
@@ -0,0 +1,253 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "absl/strings/internal/cordz_handle.h" + +#include <random> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/memory/memory.h" +#include "absl/synchronization/internal/thread_pool.h" +#include "absl/synchronization/notification.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::ElementsAre; +using ::testing::Gt; +using ::testing::IsEmpty; +using ::testing::SizeIs; + +// Local less verbose helper +std::vector<const CordzHandle*> DeleteQueue() { + return CordzHandle::DiagnosticsGetDeleteQueue(); +} + +struct CordzHandleDeleteTracker : public CordzHandle { + bool* deleted; + explicit CordzHandleDeleteTracker(bool* deleted) : deleted(deleted) {} + ~CordzHandleDeleteTracker() override { *deleted = true; } +}; + +TEST(CordzHandleTest, DeleteQueueIsEmpty) { + EXPECT_THAT(DeleteQueue(), SizeIs(0)); +} + +TEST(CordzHandleTest, CordzHandleCreateDelete) { + bool deleted = false; + auto* handle = new CordzHandleDeleteTracker(&deleted); + EXPECT_FALSE(handle->is_snapshot()); + EXPECT_THAT(DeleteQueue(), SizeIs(0)); + + CordzHandle::Delete(handle); + EXPECT_THAT(DeleteQueue(), SizeIs(0)); + EXPECT_TRUE(deleted); +} + +TEST(CordzHandleTest, CordzSnapshotCreateDelete) { + auto* snapshot = new CordzSnapshot(); + EXPECT_TRUE(snapshot->is_snapshot()); + EXPECT_THAT(DeleteQueue(), ElementsAre(snapshot)); + delete snapshot; + EXPECT_THAT(DeleteQueue(), SizeIs(0)); +} + +TEST(CordzHandleTest, CordzHandleCreateDeleteWithSnapshot) { + bool deleted = false; + auto* snapshot = new CordzSnapshot(); + auto* handle = new CordzHandleDeleteTracker(&deleted); + + CordzHandle::Delete(handle); + EXPECT_THAT(DeleteQueue(), ElementsAre(handle, snapshot)); + EXPECT_FALSE(deleted); + + delete snapshot; + EXPECT_THAT(DeleteQueue(), SizeIs(0)); + EXPECT_TRUE(deleted); +} + +TEST(CordzHandleTest, MultiSnapshot) { + bool deleted[3] = {false, false, false}; + + CordzSnapshot* snapshot[3]; + CordzHandleDeleteTracker* handle[3]; + for (int i = 0; i < 3; ++i) { + snapshot[i] = new CordzSnapshot(); + handle[i] = new CordzHandleDeleteTracker(&deleted[i]); + CordzHandle::Delete(handle[i]); + } + + EXPECT_THAT(DeleteQueue(), ElementsAre(handle[2], snapshot[2], handle[1], + snapshot[1], handle[0], snapshot[0])); + EXPECT_THAT(deleted, ElementsAre(false, false, false)); + + delete snapshot[1]; + EXPECT_THAT(DeleteQueue(), ElementsAre(handle[2], snapshot[2], handle[1], + handle[0], snapshot[0])); + EXPECT_THAT(deleted, ElementsAre(false, false, false)); + + delete snapshot[0]; + EXPECT_THAT(DeleteQueue(), ElementsAre(handle[2], snapshot[2])); + EXPECT_THAT(deleted, ElementsAre(true, true, false)); + + delete snapshot[2]; + EXPECT_THAT(DeleteQueue(), SizeIs(0)); + EXPECT_THAT(deleted, ElementsAre(true, true, deleted)); +} + +TEST(CordzHandleTest, DiagnosticsHandleIsSafeToInspect) { + CordzSnapshot snapshot1; + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(nullptr)); + + auto* handle1 = new CordzHandle(); + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(handle1)); + + CordzHandle::Delete(handle1); + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(handle1)); + + CordzSnapshot snapshot2; + auto* handle2 = new CordzHandle(); + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(handle1)); + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(handle2)); + EXPECT_FALSE(snapshot2.DiagnosticsHandleIsSafeToInspect(handle1)); + EXPECT_TRUE(snapshot2.DiagnosticsHandleIsSafeToInspect(handle2)); + + CordzHandle::Delete(handle2); + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(handle1)); +} + +TEST(CordzHandleTest, DiagnosticsGetSafeToInspectDeletedHandles) { + EXPECT_THAT(DeleteQueue(), IsEmpty()); + + auto* handle = new CordzHandle(); + auto* snapshot1 = new CordzSnapshot(); + + // snapshot1 should be able to see handle. + EXPECT_THAT(DeleteQueue(), ElementsAre(snapshot1)); + EXPECT_TRUE(snapshot1->DiagnosticsHandleIsSafeToInspect(handle)); + EXPECT_THAT(snapshot1->DiagnosticsGetSafeToInspectDeletedHandles(), + IsEmpty()); + + // This handle will be safe to inspect as long as snapshot1 is alive. However, + // since only snapshot1 can prove that it's alive, it will be hidden from + // snapshot2. + CordzHandle::Delete(handle); + + // This snapshot shouldn't be able to see handle because handle was already + // sent to Delete. + auto* snapshot2 = new CordzSnapshot(); + + // DeleteQueue elements are LIFO order. + EXPECT_THAT(DeleteQueue(), ElementsAre(snapshot2, handle, snapshot1)); + + EXPECT_TRUE(snapshot1->DiagnosticsHandleIsSafeToInspect(handle)); + EXPECT_FALSE(snapshot2->DiagnosticsHandleIsSafeToInspect(handle)); + + EXPECT_THAT(snapshot1->DiagnosticsGetSafeToInspectDeletedHandles(), + ElementsAre(handle)); + EXPECT_THAT(snapshot2->DiagnosticsGetSafeToInspectDeletedHandles(), + IsEmpty()); + + CordzHandle::Delete(snapshot1); + EXPECT_THAT(DeleteQueue(), ElementsAre(snapshot2)); + + CordzHandle::Delete(snapshot2); + EXPECT_THAT(DeleteQueue(), IsEmpty()); +} + +// Create and delete CordzHandle and CordzSnapshot objects in multiple threads +// so that tsan has some time to chew on it and look for memory problems. +TEST(CordzHandleTest, MultiThreaded) { + Notification stop; + static constexpr int kNumThreads = 4; + // Keep the number of handles relatively small so that the test will naturally + // transition to an empty delete queue during the test. If there are, say, 100 + // handles, that will virtually never happen. With 10 handles and around 50k + // iterations in each of 4 threads, the delete queue appears to become empty + // around 200 times. + static constexpr int kNumHandles = 10; + + // Each thread is going to pick a random index and atomically swap its + // CordzHandle with one in handles. This way, each thread can avoid + // manipulating a CordzHandle that might be operated upon in another thread. + std::vector<std::atomic<CordzHandle*>> handles(kNumHandles); + + absl::synchronization_internal::ThreadPool pool(kNumThreads); + + for (int i = 0; i < kNumThreads; ++i) { + pool.Schedule([&stop, &handles]() { + std::minstd_rand gen; + std::uniform_int_distribution<int> dist_type(0, 2); + std::uniform_int_distribution<int> dist_handle(0, kNumHandles - 1); + size_t max_safe_to_inspect = 0; + while (!stop.HasBeenNotified()) { + CordzHandle* handle; + switch (dist_type(gen)) { + case 0: + handle = new CordzHandle(); + break; + case 1: + handle = new CordzSnapshot(); + break; + default: + handle = nullptr; + break; + } + CordzHandle* old_handle = handles[dist_handle(gen)].exchange(handle); + if (old_handle != nullptr) { + std::vector<const CordzHandle*> safe_to_inspect = + old_handle->DiagnosticsGetSafeToInspectDeletedHandles(); + for (const CordzHandle* handle : safe_to_inspect) { + // We're in a tight loop, so don't generate too many error messages. + ASSERT_FALSE(handle->is_snapshot()); + } + if (safe_to_inspect.size() > max_safe_to_inspect) { + max_safe_to_inspect = safe_to_inspect.size(); + } + } + CordzHandle::Delete(old_handle); + } + + // Confirm that the test did *something*. This check will be satisfied as + // long as this thread has delete a CordzSnapshot object and a + // non-snapshot CordzHandle was deleted after the CordzSnapshot was + // created. This max_safe_to_inspect count will often reach around 30 + // (assuming 4 threads and 10 slots for live handles). Don't use a strict + // bound to avoid making this test flaky. + EXPECT_THAT(max_safe_to_inspect, Gt(0)); + + // Have each thread attempt to clean up everything. Some thread will be + // the last to reach this cleanup code, and it will be guaranteed to clean + // up everything because nothing remains to create new handles. + for (size_t i = 0; i < handles.size(); i++) { + CordzHandle* handle = handles[i].exchange(nullptr); + CordzHandle::Delete(handle); + } + }); + } + + // The threads will hammer away. Give it a little bit of time for tsan to + // spot errors. + absl::SleepFor(absl::Seconds(3)); + stop.Notify(); +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/strings/internal/cordz_info.cc b/absl/strings/internal/cordz_info.cc new file mode 100644 index 0000000..0461ec4 --- /dev/null +++ b/absl/strings/internal/cordz_info.cc
@@ -0,0 +1,197 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_info.h" + +#include "absl/base/config.h" +#include "absl/base/internal/spinlock.h" +#include "absl/debugging/stacktrace.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_tracker.h" +#include "absl/synchronization/mutex.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +using ::absl::base_internal::SpinLockHolder; + +constexpr int CordzInfo::kMaxStackDepth; + +ABSL_CONST_INIT CordzInfo::List CordzInfo::global_list_{absl::kConstInit}; + +CordzInfo* CordzInfo::Head(const CordzSnapshot& snapshot) { + ABSL_ASSERT(snapshot.is_snapshot()); + + // We can do an 'unsafe' load of 'head', as we are guaranteed that the + // instance it points to is kept alive by the provided CordzSnapshot, so we + // can simply return the current value using an acquire load. + // We do enforce in DEBUG builds that the 'head' value is present in the + // delete queue: ODR violations may lead to 'snapshot' and 'global_list_' + // being in different libraries / modules. + CordzInfo* head = global_list_.head.load(std::memory_order_acquire); + ABSL_ASSERT(snapshot.DiagnosticsHandleIsSafeToInspect(head)); + return head; +} + +CordzInfo* CordzInfo::Next(const CordzSnapshot& snapshot) const { + ABSL_ASSERT(snapshot.is_snapshot()); + + // Similar to the 'Head()' function, we do not need a mutex here. + CordzInfo* next = ci_next_.load(std::memory_order_acquire); + ABSL_ASSERT(snapshot.DiagnosticsHandleIsSafeToInspect(this)); + ABSL_ASSERT(snapshot.DiagnosticsHandleIsSafeToInspect(next)); + return next; +} + +void CordzInfo::TrackCord(InlineData& cord, MethodIdentifier method) { + assert(cord.is_tree()); + assert(!cord.is_profiled()); + CordzInfo* cordz_info = new CordzInfo(cord.as_tree(), nullptr, method); + cord.set_cordz_info(cordz_info); + cordz_info->Track(); +} + +void CordzInfo::TrackCord(InlineData& cord, const InlineData& src, + MethodIdentifier method) { + assert(cord.is_tree()); + assert(!cord.is_profiled()); + auto* info = src.is_tree() && src.is_profiled() ? src.cordz_info() : nullptr; + CordzInfo* cordz_info = new CordzInfo(cord.as_tree(), info, method); + cord.set_cordz_info(cordz_info); + cordz_info->Track(); +} + +CordzInfo::MethodIdentifier CordzInfo::GetParentMethod(const CordzInfo* src) { + if (src == nullptr) return MethodIdentifier::kUnknown; + return src->parent_method_ != MethodIdentifier::kUnknown ? src->parent_method_ + : src->method_; +} + +int CordzInfo::FillParentStack(const CordzInfo* src, void** stack) { + assert(stack); + if (src == nullptr) return 0; + if (src->parent_stack_depth_) { + memcpy(stack, src->parent_stack_, src->parent_stack_depth_ * sizeof(void*)); + return src->parent_stack_depth_; + } + memcpy(stack, src->stack_, src->stack_depth_ * sizeof(void*)); + return src->stack_depth_; +} + +CordzInfo::CordzInfo(CordRep* rep, const CordzInfo* src, + MethodIdentifier method) + : rep_(rep), + stack_depth_(absl::GetStackTrace(stack_, /*max_depth=*/kMaxStackDepth, + /*skip_count=*/1)), + parent_stack_depth_(FillParentStack(src, parent_stack_)), + method_(method), + parent_method_(GetParentMethod(src)), + create_time_(absl::Now()), + size_(rep->length) { + update_tracker_.LossyAdd(method); +} + +CordzInfo::~CordzInfo() { + // `rep_` is potentially kept alive if CordzInfo is included + // in a collection snapshot (which should be rare). + if (ABSL_PREDICT_FALSE(rep_)) { + CordRep::Unref(rep_); + } +} + +void CordzInfo::Track() { + SpinLockHolder l(&list_->mutex); + + CordzInfo* const head = list_->head.load(std::memory_order_acquire); + if (head != nullptr) { + head->ci_prev_.store(this, std::memory_order_release); + } + ci_next_.store(head, std::memory_order_release); + list_->head.store(this, std::memory_order_release); +} + +void CordzInfo::Untrack() { + { + // TODO(b/117940323): change this to assuming ownership instead once all + // Cord logic is properly keeping `rep_` in sync with the Cord root rep. + absl::MutexLock lock(&mutex_); + rep_ = nullptr; + } + + ODRCheck(); + { + SpinLockHolder l(&list_->mutex); + + CordzInfo* const head = list_->head.load(std::memory_order_acquire); + CordzInfo* const next = ci_next_.load(std::memory_order_acquire); + CordzInfo* const prev = ci_prev_.load(std::memory_order_acquire); + + if (next) { + ABSL_ASSERT(next->ci_prev_.load(std::memory_order_acquire) == this); + next->ci_prev_.store(prev, std::memory_order_release); + } + if (prev) { + ABSL_ASSERT(head != this); + ABSL_ASSERT(prev->ci_next_.load(std::memory_order_acquire) == this); + prev->ci_next_.store(next, std::memory_order_release); + } else { + ABSL_ASSERT(head == this); + list_->head.store(next, std::memory_order_release); + } + } + CordzHandle::Delete(this); +} + +void CordzInfo::Lock(MethodIdentifier method) + ABSL_EXCLUSIVE_LOCK_FUNCTION(mutex_) { + mutex_.Lock(); + update_tracker_.LossyAdd(method); + assert(rep_); +} + +void CordzInfo::Unlock() ABSL_UNLOCK_FUNCTION(mutex_) { + bool tracked = rep_ != nullptr; + if (rep_) { + size_.store(rep_->length); + } + mutex_.Unlock(); + if (!tracked) { + Untrack(); + } +} + +absl::Span<void* const> CordzInfo::GetStack() const { + return absl::MakeConstSpan(stack_, stack_depth_); +} + +absl::Span<void* const> CordzInfo::GetParentStack() const { + return absl::MakeConstSpan(parent_stack_, parent_stack_depth_); +} + +CordzStatistics CordzInfo::GetCordzStatistics() const { + CordzStatistics stats; + stats.method = method_; + stats.parent_method = parent_method_; + stats.update_tracker = update_tracker_; + stats.size = size_.load(std::memory_order_relaxed); + return stats; +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/strings/internal/cordz_info.h b/absl/strings/internal/cordz_info.h new file mode 100644 index 0000000..f7682cb --- /dev/null +++ b/absl/strings/internal/cordz_info.h
@@ -0,0 +1,251 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_CORDZ_INFO_H_ +#define ABSL_STRINGS_CORDZ_INFO_H_ + +#include <atomic> +#include <cstdint> +#include <functional> + +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" +#include "absl/base/thread_annotations.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cordz_functions.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_tracker.h" +#include "absl/synchronization/mutex.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// CordzInfo tracks a profiled Cord. Each of these objects can be in two places. +// If a Cord is alive, the CordzInfo will be in the global_cordz_infos map, and +// can also be retrieved via the linked list starting with +// global_cordz_infos_head and continued via the cordz_info_next() method. When +// a Cord has reached the end of its lifespan, the CordzInfo object will be +// migrated out of the global_cordz_infos list and the global_cordz_infos_map, +// and will either be deleted or appended to the global_delete_queue. If it is +// placed on the global_delete_queue, the CordzInfo object will be cleaned in +// the destructor of a CordzSampleToken object. +class ABSL_LOCKABLE CordzInfo : public CordzHandle { + public: + using MethodIdentifier = CordzUpdateTracker::MethodIdentifier; + + // TrackCord creates a CordzInfo instance which tracks important metrics of + // a sampled cord, and stores the created CordzInfo instance into `cord'. All + // CordzInfo instances are placed in a global list which is used to discover + // and snapshot all actively tracked cords. Callers are responsible for + // calling UntrackCord() before the tracked Cord instance is deleted, or to + // stop tracking the sampled Cord. Callers are also responsible for guarding + // changes to the 'tree' value of a Cord (InlineData.tree) through the Lock() + // and Unlock() calls. Any change resulting in a new tree value for the cord + // requires a call to SetCordRep() before the old tree has been unreffed + // and/or deleted. `method` identifies the Cord public API method initiating + // the cord to be sampled. + // Requires `cord` to hold a tree, and `cord.cordz_info()` to be null. + static void TrackCord(InlineData& cord, MethodIdentifier method); + + // Identical to TrackCord(), except that this function fills the + // `parent_stack` and `parent_method` properties of the returned CordzInfo + // instance from the provided `src` instance if `src` is sampled. + // This function should be used for sampling 'copy constructed' cords. + static void TrackCord(InlineData& cord, const InlineData& src, + MethodIdentifier method); + + // Maybe sample the cord identified by 'cord' for method 'method'. + // Uses `cordz_should_profile` to randomly pick cords to be sampled, and if + // so, invokes `TrackCord` to start sampling `cord`. + static void MaybeTrackCord(InlineData& cord, MethodIdentifier method); + static void MaybeTrackCord(InlineData& cord, const InlineData& src, + MethodIdentifier method); + + // Stops tracking changes for a sampled cord, and deletes the provided info. + // This function must be called before the sampled cord instance is deleted, + // and before the root cordrep of the sampled cord is unreffed. + // This function may extend the lifetime of the cordrep in cases where the + // CordInfo instance is being held by a concurrent collection thread. + void Untrack(); + + // Invokes UntrackCord() on `info` if `info` is not null. + static void MaybeUntrackCord(CordzInfo* info); + + CordzInfo() = delete; + CordzInfo(const CordzInfo&) = delete; + CordzInfo& operator=(const CordzInfo&) = delete; + + // Retrieves the oldest existing CordzInfo. + static CordzInfo* Head(const CordzSnapshot& snapshot) + ABSL_NO_THREAD_SAFETY_ANALYSIS; + + // Retrieves the next oldest existing CordzInfo older than 'this' instance. + CordzInfo* Next(const CordzSnapshot& snapshot) const + ABSL_NO_THREAD_SAFETY_ANALYSIS; + + // Locks this instance for the update identified by `method`. + // Increases the count for `method` in `update_tracker`. + void Lock(MethodIdentifier method) ABSL_EXCLUSIVE_LOCK_FUNCTION(mutex_); + + // Unlocks this instance. If the contained `rep` has been set to null + // indicating the Cord has been cleared or is otherwise no longer sampled, + // then this method will delete this CordzInfo instance. + void Unlock() ABSL_UNLOCK_FUNCTION(mutex_); + + // Asserts that this CordzInfo instance is locked. + void AssertHeld() ABSL_ASSERT_EXCLUSIVE_LOCK(mutex_); + + // Updates the `rep' property of this instance. This methods is invoked by + // Cord logic each time the root node of a sampled Cord changes, and before + // the old root reference count is deleted. This guarantees that collection + // code can always safely take a reference on the tracked cord. + // Requires a lock to be held through the `Lock()` method. + // TODO(b/117940323): annotate with ABSL_EXCLUSIVE_LOCKS_REQUIRED once all + // Cord code is in a state where this can be proven true by the compiler. + void SetCordRep(CordRep* rep); + + // Returns the current value of `rep_` for testing purposes only. + CordRep* GetCordRepForTesting() const ABSL_NO_THREAD_SAFETY_ANALYSIS { + return rep_; + } + + // Returns the stack trace for where the cord was first sampled. Cords are + // potentially sampled when they promote from an inlined cord to a tree or + // ring representation, which is not necessarily the location where the cord + // was first created. Some cords are created as inlined cords, and only as + // data is added do they become a non-inlined cord. However, typically the + // location represents reasonably well where the cord is 'created'. + absl::Span<void* const> GetStack() const; + + // Returns the stack trace for a sampled cord's 'parent stack trace'. This + // value may be set if the cord is sampled (promoted) after being created + // from, or being assigned the value of an existing (sampled) cord. + absl::Span<void* const> GetParentStack() const; + + // Retrieves the CordzStatistics associated with this Cord. The statistics + // are only updated when a Cord goes through a mutation, such as an Append + // or RemovePrefix. + CordzStatistics GetCordzStatistics() const; + + // Records size metric for this CordzInfo instance. + void RecordMetrics(int64_t size) { + size_.store(size, std::memory_order_relaxed); + } + + private: + // Global cordz info list. CordzInfo stores a pointer to the global list + // instance to harden against ODR violations. + struct List { + constexpr explicit List(absl::ConstInitType) + : mutex(absl::kConstInit, + absl::base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL) {} + + absl::base_internal::SpinLock mutex; + std::atomic<CordzInfo*> head ABSL_GUARDED_BY(mutex){nullptr}; + }; + + static constexpr int kMaxStackDepth = 64; + + explicit CordzInfo(CordRep* rep, const CordzInfo* src, + MethodIdentifier method); + ~CordzInfo() override; + + void Track(); + + // Returns the parent method from `src`, which is either `parent_method_` or + // `method_` depending on `parent_method_` being kUnknown. + // Returns kUnknown if `src` is null. + static MethodIdentifier GetParentMethod(const CordzInfo* src); + + // Fills the provided stack from `src`, copying either `parent_stack_` or + // `stack_` depending on `parent_stack_` being empty, returning the size of + // the parent stack. + // Returns 0 if `src` is null. + static int FillParentStack(const CordzInfo* src, void** stack); + + void ODRCheck() const { +#ifndef NDEBUG + ABSL_RAW_CHECK(list_ == &global_list_, "ODR violation in Cord"); +#endif + } + + ABSL_CONST_INIT static List global_list_; + List* const list_ = &global_list_; + + // ci_prev_ and ci_next_ require the global list mutex to be held. + // Unfortunately we can't use thread annotations such that the thread safety + // analysis understands that list_ and global_list_ are one and the same. + std::atomic<CordzInfo*> ci_prev_{nullptr}; + std::atomic<CordzInfo*> ci_next_{nullptr}; + + mutable absl::Mutex mutex_; + CordRep* rep_ ABSL_GUARDED_BY(mutex_); + + void* stack_[kMaxStackDepth]; + void* parent_stack_[kMaxStackDepth]; + const int stack_depth_; + const int parent_stack_depth_; + const MethodIdentifier method_; + const MethodIdentifier parent_method_; + CordzUpdateTracker update_tracker_; + const absl::Time create_time_; + + // Last recorded size for the cord. + std::atomic<int64_t> size_{0}; +}; + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CordzInfo::MaybeTrackCord( + InlineData& cord, MethodIdentifier method) { + if (ABSL_PREDICT_FALSE(cordz_should_profile())) { + TrackCord(cord, method); + } +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CordzInfo::MaybeTrackCord( + InlineData& cord, const InlineData& src, MethodIdentifier method) { + if (ABSL_PREDICT_FALSE(cordz_should_profile())) { + TrackCord(cord, src, method); + } +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CordzInfo::MaybeUntrackCord( + CordzInfo* info) { + if (ABSL_PREDICT_FALSE(info)) { + info->Untrack(); + } +} + +inline void CordzInfo::AssertHeld() ABSL_ASSERT_EXCLUSIVE_LOCK(mutex_) { +#ifndef NDEBUG + mutex_.AssertHeld(); +#endif +} + +inline void CordzInfo::SetCordRep(CordRep* rep) { + AssertHeld(); + rep_ = rep; + if (rep) { + size_.store(rep->length); + } +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORDZ_INFO_H_
diff --git a/absl/strings/internal/cordz_info_test.cc b/absl/strings/internal/cordz_info_test.cc new file mode 100644 index 0000000..5eaf4b9 --- /dev/null +++ b/absl/strings/internal/cordz_info_test.cc
@@ -0,0 +1,302 @@ +// Copyright 2019 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_info.h" + +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/debugging/stacktrace.h" +#include "absl/debugging/symbolize.h" +#include "absl/strings/cordz_test_helpers.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_tracker.h" +#include "absl/strings/str_cat.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::Ne; + +// Used test values +auto constexpr kUnknownMethod = CordzUpdateTracker::kUnknown; +auto constexpr kTrackCordMethod = CordzUpdateTracker::kConstructorString; +auto constexpr kChildMethod = CordzUpdateTracker::kConstructorCord; +auto constexpr kUpdateMethod = CordzUpdateTracker::kAppendString; + +// Local less verbose helper +std::vector<const CordzHandle*> DeleteQueue() { + return CordzHandle::DiagnosticsGetDeleteQueue(); +} + +std::string FormatStack(absl::Span<void* const> raw_stack) { + static constexpr size_t buf_size = 1 << 14; + std::unique_ptr<char[]> buf(new char[buf_size]); + std::string output; + for (void* stackp : raw_stack) { + if (absl::Symbolize(stackp, buf.get(), buf_size)) { + absl::StrAppend(&output, " ", buf.get(), "\n"); + } + } + return output; +} + +TEST(CordzInfoTest, TrackCord) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + ASSERT_THAT(info, Ne(nullptr)); + EXPECT_FALSE(info->is_snapshot()); + EXPECT_THAT(CordzInfo::Head(CordzSnapshot()), Eq(info)); + EXPECT_THAT(info->GetCordRepForTesting(), Eq(data.rep.rep)); + info->Untrack(); +} + +TEST(CordzInfoTest, UntrackCord) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + + CordzSnapshot snapshot; + info->Untrack(); + EXPECT_THAT(CordzInfo::Head(CordzSnapshot()), Eq(nullptr)); + EXPECT_THAT(info->GetCordRepForTesting(), Eq(nullptr)); + EXPECT_THAT(DeleteQueue(), ElementsAre(info, &snapshot)); +} + +TEST(CordzInfoTest, SetCordRep) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + + TestCordRep rep; + info->Lock(CordzUpdateTracker::kAppendCord); + info->SetCordRep(rep.rep); + info->Unlock(); + EXPECT_THAT(info->GetCordRepForTesting(), Eq(rep.rep)); + + info->Untrack(); +} + +TEST(CordzInfoTest, SetCordRepNullUntracksCordOnUnlock) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + + info->Lock(CordzUpdateTracker::kAppendString); + info->SetCordRep(nullptr); + EXPECT_THAT(info->GetCordRepForTesting(), Eq(nullptr)); + EXPECT_THAT(CordzInfo::Head(CordzSnapshot()), Eq(info)); + + info->Unlock(); + EXPECT_THAT(CordzInfo::Head(CordzSnapshot()), Eq(nullptr)); +} + +#if GTEST_HAS_DEATH_TEST + +TEST(CordzInfoTest, SetCordRepRequiresMutex) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + TestCordRep rep; + EXPECT_DEBUG_DEATH(info->SetCordRep(rep.rep), ".*"); + info->Untrack(); +} + +#endif // GTEST_HAS_DEATH_TEST + +TEST(CordzInfoTest, TrackUntrackHeadFirstV2) { + CordzSnapshot snapshot; + EXPECT_THAT(CordzInfo::Head(snapshot), Eq(nullptr)); + + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info1 = data.data.cordz_info(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info1)); + EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); + + TestCordData data2; + CordzInfo::TrackCord(data2.data, kTrackCordMethod); + CordzInfo* info2 = data2.data.cordz_info(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info2)); + EXPECT_THAT(info2->Next(snapshot), Eq(info1)); + EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); + + info2->Untrack(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info1)); + EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); + + info1->Untrack(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(nullptr)); +} + +TEST(CordzInfoTest, TrackUntrackTailFirstV2) { + CordzSnapshot snapshot; + EXPECT_THAT(CordzInfo::Head(snapshot), Eq(nullptr)); + + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info1 = data.data.cordz_info(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info1)); + EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); + + TestCordData data2; + CordzInfo::TrackCord(data2.data, kTrackCordMethod); + CordzInfo* info2 = data2.data.cordz_info(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info2)); + EXPECT_THAT(info2->Next(snapshot), Eq(info1)); + EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); + + info1->Untrack(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info2)); + EXPECT_THAT(info2->Next(snapshot), Eq(nullptr)); + + info2->Untrack(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(nullptr)); +} + +TEST(CordzInfoTest, StackV2) { + TestCordData data; + // kMaxStackDepth is intentionally less than 64 (which is the max depth that + // Cordz will record) because if the actual stack depth is over 64 + // (which it is on Apple platforms) then the expected_stack will end up + // catching a few frames at the end that the actual_stack didn't get and + // it will no longer be subset. At the time of this writing 58 is the max + // that will allow this test to pass (with a minimum os version of iOS 9), so + // rounded down to 50 to hopefully not run into this in the future if Apple + // makes small modifications to its testing stack. 50 is sufficient to prove + // that we got a decent stack. + static constexpr int kMaxStackDepth = 50; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + std::vector<void*> local_stack; + local_stack.resize(kMaxStackDepth); + // In some environments we don't get stack traces. For example in Android + // absl::GetStackTrace will return 0 indicating it didn't find any stack. The + // resultant formatted stack will be "", but that still equals the stack + // recorded in CordzInfo, which is also empty. The skip_count is 1 so that the + // line number of the current stack isn't included in the HasSubstr check. + local_stack.resize(absl::GetStackTrace(local_stack.data(), kMaxStackDepth, + /*skip_count=*/1)); + + std::string got_stack = FormatStack(info->GetStack()); + std::string expected_stack = FormatStack(local_stack); + // If TrackCord is inlined, got_stack should match expected_stack. If it isn't + // inlined, got_stack should include an additional frame not present in + // expected_stack. Either way, expected_stack should be a substring of + // got_stack. + EXPECT_THAT(got_stack, HasSubstr(expected_stack)); + + info->Untrack(); +} + +// Local helper functions to get different stacks for child and parent. +CordzInfo* TrackChildCord(InlineData& data, const InlineData& parent) { + CordzInfo::TrackCord(data, parent, kChildMethod); + return data.cordz_info(); +} +CordzInfo* TrackParentCord(InlineData& data) { + CordzInfo::TrackCord(data, kTrackCordMethod); + return data.cordz_info(); +} + +TEST(CordzInfoTest, GetStatistics) { + TestCordData data; + CordzInfo* info = TrackParentCord(data.data); + + CordzStatistics statistics = info->GetCordzStatistics(); + EXPECT_THAT(statistics.size, Eq(data.rep.rep->length)); + EXPECT_THAT(statistics.method, Eq(kTrackCordMethod)); + EXPECT_THAT(statistics.parent_method, Eq(kUnknownMethod)); + EXPECT_THAT(statistics.update_tracker.Value(kTrackCordMethod), Eq(1)); + + info->Untrack(); +} + +TEST(CordzInfoTest, LockCountsMethod) { + TestCordData data; + CordzInfo* info = TrackParentCord(data.data); + + info->Lock(kUpdateMethod); + info->Unlock(); + info->Lock(kUpdateMethod); + info->Unlock(); + + CordzStatistics statistics = info->GetCordzStatistics(); + EXPECT_THAT(statistics.update_tracker.Value(kUpdateMethod), Eq(2)); + + info->Untrack(); +} + +TEST(CordzInfoTest, FromParent) { + TestCordData parent; + TestCordData child; + CordzInfo* info_parent = TrackParentCord(parent.data); + CordzInfo* info_child = TrackChildCord(child.data, parent.data); + + std::string stack = FormatStack(info_parent->GetStack()); + std::string parent_stack = FormatStack(info_child->GetParentStack()); + EXPECT_THAT(stack, Eq(parent_stack)); + + CordzStatistics statistics = info_child->GetCordzStatistics(); + EXPECT_THAT(statistics.size, Eq(child.rep.rep->length)); + EXPECT_THAT(statistics.method, Eq(kChildMethod)); + EXPECT_THAT(statistics.parent_method, Eq(kTrackCordMethod)); + EXPECT_THAT(statistics.update_tracker.Value(kChildMethod), Eq(1)); + + info_parent->Untrack(); + info_child->Untrack(); +} + +TEST(CordzInfoTest, FromParentInlined) { + InlineData parent; + TestCordData child; + CordzInfo* info = TrackChildCord(child.data, parent); + EXPECT_TRUE(info->GetParentStack().empty()); + CordzStatistics statistics = info->GetCordzStatistics(); + EXPECT_THAT(statistics.size, Eq(child.rep.rep->length)); + EXPECT_THAT(statistics.method, Eq(kChildMethod)); + EXPECT_THAT(statistics.parent_method, Eq(kUnknownMethod)); + EXPECT_THAT(statistics.update_tracker.Value(kChildMethod), Eq(1)); + info->Untrack(); +} + +TEST(CordzInfoTest, RecordMetrics) { + TestCordData data; + CordzInfo* info = TrackParentCord(data.data); + + CordzStatistics expected; + expected.size = 100; + info->RecordMetrics(expected.size); + + CordzStatistics actual = info->GetCordzStatistics(); + EXPECT_EQ(actual.size, expected.size); + + info->Untrack(); +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/strings/internal/cordz_sample_token.cc b/absl/strings/internal/cordz_sample_token.cc new file mode 100644 index 0000000..ba1270d --- /dev/null +++ b/absl/strings/internal/cordz_sample_token.cc
@@ -0,0 +1,64 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_sample_token.h" + +#include "absl/base/config.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_info.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +CordzSampleToken::Iterator& CordzSampleToken::Iterator::operator++() { + if (current_) { + current_ = current_->Next(*token_); + } + return *this; +} + +CordzSampleToken::Iterator CordzSampleToken::Iterator::operator++(int) { + Iterator it(*this); + operator++(); + return it; +} + +bool operator==(const CordzSampleToken::Iterator& lhs, + const CordzSampleToken::Iterator& rhs) { + return lhs.current_ == rhs.current_ && + (lhs.current_ == nullptr || lhs.token_ == rhs.token_); +} + +bool operator!=(const CordzSampleToken::Iterator& lhs, + const CordzSampleToken::Iterator& rhs) { + return !(lhs == rhs); +} + +CordzSampleToken::Iterator::reference CordzSampleToken::Iterator::operator*() + const { + return *current_; +} + +CordzSampleToken::Iterator::pointer CordzSampleToken::Iterator::operator->() + const { + return current_; +} + +CordzSampleToken::Iterator::Iterator(const CordzSampleToken* token) + : token_(token), current_(CordzInfo::Head(*token)) {} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/strings/internal/cordz_sample_token.h b/absl/strings/internal/cordz_sample_token.h new file mode 100644 index 0000000..28a1d70 --- /dev/null +++ b/absl/strings/internal/cordz_sample_token.h
@@ -0,0 +1,97 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/config.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_info.h" + +#ifndef ABSL_STRINGS_CORDZ_SAMPLE_TOKEN_H_ +#define ABSL_STRINGS_CORDZ_SAMPLE_TOKEN_H_ + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// The existence of a CordzSampleToken guarantees that a reader can traverse the +// global_cordz_infos_head linked-list without needing to hold a mutex. When a +// CordzSampleToken exists, all CordzInfo objects that would be destroyed are +// instead appended to a deletion queue. When the CordzSampleToken is destroyed, +// it will also clean up any of these CordzInfo objects. +// +// E.g., ST are CordzSampleToken objects and CH are CordzHandle objects. +// ST1 <- CH1 <- CH2 <- ST2 <- CH3 <- global_delete_queue_tail +// +// This list tracks that CH1 and CH2 were created after ST1, so the thread +// holding ST1 might have a referece to CH1, CH2, ST2, and CH3. However, ST2 was +// created later, so the thread holding the ST2 token cannot have a reference to +// ST1, CH1, or CH2. If ST1 is cleaned up first, that thread will delete ST1, +// CH1, and CH2. If instead ST2 is cleaned up first, that thread will only +// delete ST2. +// +// If ST1 is cleaned up first, the new list will be: +// ST2 <- CH3 <- global_delete_queue_tail +// +// If ST2 is cleaned up first, the new list will be: +// ST1 <- CH1 <- CH2 <- CH3 <- global_delete_queue_tail +// +// All new CordzHandle objects are appended to the list, so if a new thread +// comes along before either ST1 or ST2 are cleaned up, the new list will be: +// ST1 <- CH1 <- CH2 <- ST2 <- CH3 <- ST3 <- global_delete_queue_tail +// +// A thread must hold the global_delete_queue_mu mutex whenever it's altering +// this list. +// +// It is safe for thread that holds a CordzSampleToken to read +// global_cordz_infos at any time since the objects it is able to retrieve will +// not be deleted while the CordzSampleToken exists. +class CordzSampleToken : public CordzSnapshot { + public: + class Iterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = const CordzInfo&; + using difference_type = ptrdiff_t; + using pointer = const CordzInfo*; + using reference = value_type; + + Iterator() = default; + + Iterator& operator++(); + Iterator operator++(int); + friend bool operator==(const Iterator& lhs, const Iterator& rhs); + friend bool operator!=(const Iterator& lhs, const Iterator& rhs); + reference operator*() const; + pointer operator->() const; + + private: + friend class CordzSampleToken; + explicit Iterator(const CordzSampleToken* token); + + const CordzSampleToken* token_ = nullptr; + pointer current_ = nullptr; + }; + + CordzSampleToken() = default; + CordzSampleToken(const CordzSampleToken&) = delete; + CordzSampleToken& operator=(const CordzSampleToken&) = delete; + + Iterator begin() { return Iterator(this); } + Iterator end() { return Iterator(); } +}; + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORDZ_SAMPLE_TOKEN_H_
diff --git a/absl/strings/internal/cordz_sample_token_test.cc b/absl/strings/internal/cordz_sample_token_test.cc new file mode 100644 index 0000000..9f54301 --- /dev/null +++ b/absl/strings/internal/cordz_sample_token_test.cc
@@ -0,0 +1,208 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_sample_token.h" + +#include <memory> +#include <type_traits> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/memory/memory.h" +#include "absl/random/random.h" +#include "absl/strings/cordz_test_helpers.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/synchronization/internal/thread_pool.h" +#include "absl/synchronization/notification.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::Ne; + +// Used test values +auto constexpr kTrackCordMethod = CordzUpdateTracker::kConstructorString; + +TEST(CordzSampleTokenTest, IteratorTraits) { + static_assert(std::is_copy_constructible<CordzSampleToken::Iterator>::value, + ""); + static_assert(std::is_copy_assignable<CordzSampleToken::Iterator>::value, ""); + static_assert(std::is_move_constructible<CordzSampleToken::Iterator>::value, + ""); + static_assert(std::is_move_assignable<CordzSampleToken::Iterator>::value, ""); + static_assert( + std::is_same< + std::iterator_traits<CordzSampleToken::Iterator>::iterator_category, + std::input_iterator_tag>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<CordzSampleToken::Iterator>::value_type, + const CordzInfo&>::value, + ""); + static_assert( + std::is_same< + std::iterator_traits<CordzSampleToken::Iterator>::difference_type, + ptrdiff_t>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<CordzSampleToken::Iterator>::pointer, + const CordzInfo*>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<CordzSampleToken::Iterator>::reference, + const CordzInfo&>::value, + ""); +} + +TEST(CordzSampleTokenTest, IteratorEmpty) { + CordzSampleToken token; + EXPECT_THAT(token.begin(), Eq(token.end())); +} + +TEST(CordzSampleTokenTest, Iterator) { + TestCordData cord1, cord2, cord3; + CordzInfo::TrackCord(cord1.data, kTrackCordMethod); + CordzInfo* info1 = cord1.data.cordz_info(); + CordzInfo::TrackCord(cord2.data, kTrackCordMethod); + CordzInfo* info2 = cord2.data.cordz_info(); + CordzInfo::TrackCord(cord3.data, kTrackCordMethod); + CordzInfo* info3 = cord3.data.cordz_info(); + + CordzSampleToken token; + std::vector<const CordzInfo*> found; + for (const CordzInfo& cord_info : token) { + found.push_back(&cord_info); + } + + EXPECT_THAT(found, ElementsAre(info3, info2, info1)); + + info1->Untrack(); + info2->Untrack(); + info3->Untrack(); +} + +TEST(CordzSampleTokenTest, IteratorEquality) { + TestCordData cord1; + TestCordData cord2; + TestCordData cord3; + CordzInfo::TrackCord(cord1.data, kTrackCordMethod); + CordzInfo* info1 = cord1.data.cordz_info(); + + CordzSampleToken token1; + // lhs starts with the CordzInfo corresponding to cord1 at the head. + CordzSampleToken::Iterator lhs = token1.begin(); + + CordzInfo::TrackCord(cord2.data, kTrackCordMethod); + CordzInfo* info2 = cord2.data.cordz_info(); + + CordzSampleToken token2; + // rhs starts with the CordzInfo corresponding to cord2 at the head. + CordzSampleToken::Iterator rhs = token2.begin(); + + CordzInfo::TrackCord(cord3.data, kTrackCordMethod); + CordzInfo* info3 = cord3.data.cordz_info(); + + // lhs is on cord1 while rhs is on cord2. + EXPECT_THAT(lhs, Ne(rhs)); + + rhs++; + // lhs and rhs are both on cord1, but they didn't come from the same + // CordzSampleToken. + EXPECT_THAT(lhs, Ne(rhs)); + + lhs++; + rhs++; + // Both lhs and rhs are done, so they are on nullptr. + EXPECT_THAT(lhs, Eq(rhs)); + + info1->Untrack(); + info2->Untrack(); + info3->Untrack(); +} + +TEST(CordzSampleTokenTest, MultiThreaded) { + Notification stop; + static constexpr int kNumThreads = 4; + static constexpr int kNumCords = 3; + static constexpr int kNumTokens = 3; + absl::synchronization_internal::ThreadPool pool(kNumThreads); + + for (int i = 0; i < kNumThreads; ++i) { + pool.Schedule([&stop]() { + absl::BitGen gen; + TestCordData cords[kNumCords]; + std::unique_ptr<CordzSampleToken> tokens[kNumTokens]; + + while (!stop.HasBeenNotified()) { + // Randomly perform one of five actions: + // 1) Untrack + // 2) Track + // 3) Iterate over Cords visible to a token. + // 4) Unsample + // 5) Sample + int index = absl::Uniform(gen, 0, kNumCords); + if (absl::Bernoulli(gen, 0.5)) { + TestCordData& cord = cords[index]; + // Track/untrack. + if (cord.data.is_profiled()) { + // 1) Untrack + cord.data.cordz_info()->Untrack(); + cord.data.clear_cordz_info();; + } else { + // 2) Track + CordzInfo::TrackCord(cord.data, kTrackCordMethod); + } + } else { + std::unique_ptr<CordzSampleToken>& token = tokens[index]; + if (token) { + if (absl::Bernoulli(gen, 0.5)) { + // 3) Iterate over Cords visible to a token. + for (const CordzInfo& info : *token) { + // This is trivial work to allow us to compile the loop. + EXPECT_THAT(info.Next(*token), Ne(&info)); + } + } else { + // 4) Unsample + token = nullptr; + } + } else { + // 5) Sample + token = absl::make_unique<CordzSampleToken>(); + } + } + } + for (TestCordData& cord : cords) { + CordzInfo::MaybeUntrackCord(cord.data.cordz_info()); + } + }); + } + // The threads will hammer away. Give it a little bit of time for tsan to + // spot errors. + absl::SleepFor(absl::Seconds(3)); + stop.Notify(); +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/strings/internal/cordz_statistics.h b/absl/strings/internal/cordz_statistics.h new file mode 100644 index 0000000..6e335c0 --- /dev/null +++ b/absl/strings/internal/cordz_statistics.h
@@ -0,0 +1,67 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORDZ_STATISTICS_H_ +#define ABSL_STRINGS_INTERNAL_CORDZ_STATISTICS_H_ + +#include <cstdint> + +#include "absl/base/config.h" +#include "absl/strings/internal/cordz_update_tracker.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// CordzStatistics captures some meta information about a Cord's shape. +struct CordzStatistics { + using MethodIdentifier = CordzUpdateTracker::MethodIdentifier; + + // The size of the cord in bytes. This matches the result of Cord::size(). + int64_t size = 0; + + // The estimated memory used by the sampled cord. This value matches the + // value as reported by Cord::EstimatedMemoryUsage(). + // A value of 0 implies the property has not been recorded. + int64_t estimated_memory_usage = 0; + + // The effective memory used by the sampled cord, inversely weighted by the + // effective indegree of each allocated node. This is a representation of the + // fair share of memory usage that should be attributed to the sampled cord. + // This value is more useful for cases where one or more nodes are referenced + // by multiple Cord instances, and for cases where a Cord includes the same + // node multiple times (either directly or indirectly). + // A value of 0 implies the property has not been recorded. + int64_t estimated_fair_share_memory_usage = 0; + + // The total number of nodes referenced by this cord. + // For ring buffer Cords, this includes the 'ring buffer' node. + // A value of 0 implies the property has not been recorded. + int64_t node_count = 0; + + // The cord method responsible for sampling the cord. + MethodIdentifier method = MethodIdentifier::kUnknown; + + // The cord method responsible for sampling the parent cord if applicable. + MethodIdentifier parent_method = MethodIdentifier::kUnknown; + + // Update tracker tracking invocation count per cord method. + CordzUpdateTracker update_tracker; +}; + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORDZ_STATISTICS_H_
diff --git a/absl/strings/internal/cordz_update_scope.h b/absl/strings/internal/cordz_update_scope.h new file mode 100644 index 0000000..57ba75d --- /dev/null +++ b/absl/strings/internal/cordz_update_scope.h
@@ -0,0 +1,71 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_SCOPE_H_ +#define ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_SCOPE_H_ + +#include "absl/base/config.h" +#include "absl/base/optimization.h" +#include "absl/base/thread_annotations.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/strings/internal/cordz_update_tracker.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// CordzUpdateScope scopes an update to the provided CordzInfo. +// The class invokes `info->Lock(method)` and `info->Unlock()` to guard +// cordrep updates. This class does nothing if `info` is null. +// See also the 'Lock`, `Unlock` and `SetCordRep` methods in `CordzInfo`. +class ABSL_SCOPED_LOCKABLE CordzUpdateScope { + public: + CordzUpdateScope(CordzInfo* info, CordzUpdateTracker::MethodIdentifier method) + ABSL_EXCLUSIVE_LOCK_FUNCTION(info) + : info_(info) { + if (ABSL_PREDICT_FALSE(info_)) { + info->Lock(method); + } + } + + // CordzUpdateScope can not be copied or assigned to. + CordzUpdateScope(CordzUpdateScope&& rhs) = delete; + CordzUpdateScope(const CordzUpdateScope&) = delete; + CordzUpdateScope& operator=(CordzUpdateScope&& rhs) = delete; + CordzUpdateScope& operator=(const CordzUpdateScope&) = delete; + + ~CordzUpdateScope() ABSL_UNLOCK_FUNCTION() { + if (ABSL_PREDICT_FALSE(info_)) { + info_->Unlock(); + } + } + + void SetCordRep(CordRep* rep) const { + if (ABSL_PREDICT_FALSE(info_)) { + info_->SetCordRep(rep); + } + } + + CordzInfo* info() const { return info_; } + + private: + CordzInfo* info_; +}; + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_SCOPE_H_
diff --git a/absl/strings/internal/cordz_update_scope_test.cc b/absl/strings/internal/cordz_update_scope_test.cc new file mode 100644 index 0000000..3d08c62 --- /dev/null +++ b/absl/strings/internal/cordz_update_scope_test.cc
@@ -0,0 +1,49 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_update_scope.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/strings/cordz_test_helpers.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/strings/internal/cordz_update_tracker.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +namespace { + +// Used test values +auto constexpr kTrackCordMethod = CordzUpdateTracker::kConstructorString; + +TEST(CordzUpdateScopeTest, ScopeNullptr) { + CordzUpdateScope scope(nullptr, kTrackCordMethod); +} + +TEST(CordzUpdateScopeTest, ScopeSampledCord) { + TestCordData cord; + CordzInfo::TrackCord(cord.data, kTrackCordMethod); + CordzUpdateScope scope(cord.data.cordz_info(), kTrackCordMethod); + cord.data.cordz_info()->SetCordRep(nullptr); +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace cord_internal + +} // namespace absl
diff --git a/absl/strings/internal/cordz_update_tracker.h b/absl/strings/internal/cordz_update_tracker.h new file mode 100644 index 0000000..741950b --- /dev/null +++ b/absl/strings/internal/cordz_update_tracker.h
@@ -0,0 +1,107 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_TRACKER_H_ +#define ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_TRACKER_H_ + +#include <atomic> +#include <cstdint> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// CordzUpdateTracker tracks counters for Cord update methods. +// +// The purpose of CordzUpdateTracker is to track the number of calls to methods +// updating Cord data for sampled cords. The class internally uses 'lossy' +// atomic operations: Cord is thread-compatible, so there is no need to +// synchronize updates. However, Cordz collection threads may call 'Value()' at +// any point, so the class needs to provide thread safe access. +// +// This class is thread-safe. But as per above comments, all non-const methods +// should be used single-threaded only: updates are thread-safe but lossy. +class CordzUpdateTracker { + public: + // Tracked update methods. + enum MethodIdentifier { + kUnknown, + kAppendCord, + kAppendExternalMemory, + kAppendString, + kAssignCord, + kAssignString, + kClear, + kConstructorCord, + kConstructorString, + kFlatten, + kGetAppendRegion, + kMoveAppendCord, + kMoveAssignCord, + kMovePrependCord, + kPrependCord, + kPrependString, + kRemovePrefix, + kRemoveSuffix, + kSubCord, + + // kNumMethods defines the number of entries: must be the last entry. + kNumMethods, + }; + + // Constructs a new instance. All counters are zero-initialized. + constexpr CordzUpdateTracker() noexcept : values_{} {} + + // Copy constructs a new instance. + CordzUpdateTracker(const CordzUpdateTracker& rhs) noexcept { *this = rhs; } + + // Assigns the provided value to this instance. + CordzUpdateTracker& operator=(const CordzUpdateTracker& rhs) noexcept { + for (int i = 0; i < kNumMethods; ++i) { + values_[i].store(rhs.values_[i].load(std::memory_order_relaxed), + std::memory_order_relaxed); + } + return *this; + } + + // Returns the value for the specified method. + int64_t Value(MethodIdentifier method) const { + return values_[method].load(std::memory_order_relaxed); + } + + // Increases the value for the specified method by `n` + void LossyAdd(MethodIdentifier method, int64_t n = 1) { + auto& value = values_[method]; + value.store(value.load(std::memory_order_relaxed) + n, + std::memory_order_relaxed); + } + + private: + // Until C++20 std::atomic is not constexpr default-constructible, so we need + // a wrapper for this class to be constexpr constructible. + class Counter : public std::atomic<int64_t> { + public: + constexpr Counter() noexcept : std::atomic<int64_t>(0) {} + }; + + Counter values_[kNumMethods]; +}; + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_TRACKER_H_
diff --git a/absl/strings/internal/cordz_update_tracker_test.cc b/absl/strings/internal/cordz_update_tracker_test.cc new file mode 100644 index 0000000..eda662f --- /dev/null +++ b/absl/strings/internal/cordz_update_tracker_test.cc
@@ -0,0 +1,141 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_update_tracker.h" + +#include <array> +#include <thread> // NOLINT + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/synchronization/notification.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::AnyOf; +using ::testing::Eq; + +using Method = CordzUpdateTracker::MethodIdentifier; +using Methods = std::array<Method, Method::kNumMethods>; + +// Returns an array of all methods defined in `MethodIdentifier` +Methods AllMethods() { + return Methods{Method::kUnknown, + Method::kAppendCord, + Method::kAppendExternalMemory, + Method::kAppendString, + Method::kAssignCord, + Method::kAssignString, + Method::kClear, + Method::kConstructorCord, + Method::kConstructorString, + Method::kFlatten, + Method::kGetAppendRegion, + Method::kMoveAppendCord, + Method::kMoveAssignCord, + Method::kMovePrependCord, + Method::kPrependCord, + Method::kPrependString, + Method::kRemovePrefix, + Method::kRemoveSuffix, + Method::kSubCord}; +} + +TEST(CordzUpdateTracker, IsConstExprAndInitializesToZero) { + constexpr CordzUpdateTracker tracker; + for (Method method : AllMethods()) { + ASSERT_THAT(tracker.Value(method), Eq(0)); + } +} + +TEST(CordzUpdateTracker, LossyAdd) { + int64_t n = 1; + CordzUpdateTracker tracker; + for (Method method : AllMethods()) { + tracker.LossyAdd(method, n); + EXPECT_THAT(tracker.Value(method), Eq(n)); + n += 2; + } +} + +TEST(CordzUpdateTracker, CopyConstructor) { + int64_t n = 1; + CordzUpdateTracker src; + for (Method method : AllMethods()) { + src.LossyAdd(method, n); + n += 2; + } + + n = 1; + CordzUpdateTracker tracker(src); + for (Method method : AllMethods()) { + EXPECT_THAT(tracker.Value(method), Eq(n)); + n += 2; + } +} + +TEST(CordzUpdateTracker, OperatorAssign) { + int64_t n = 1; + CordzUpdateTracker src; + CordzUpdateTracker tracker; + for (Method method : AllMethods()) { + src.LossyAdd(method, n); + n += 2; + } + + n = 1; + tracker = src; + for (Method method : AllMethods()) { + EXPECT_THAT(tracker.Value(method), Eq(n)); + n += 2; + } +} + +TEST(CordzUpdateTracker, ThreadSanitizedValueCheck) { + absl::Notification done; + CordzUpdateTracker tracker; + + std::thread reader([&done, &tracker] { + while (!done.HasBeenNotified()) { + int n = 1; + for (Method method : AllMethods()) { + EXPECT_THAT(tracker.Value(method), AnyOf(Eq(n), Eq(0))); + n += 2; + } + } + int n = 1; + for (Method method : AllMethods()) { + EXPECT_THAT(tracker.Value(method), Eq(n)); + n += 2; + } + }); + + int64_t n = 1; + for (Method method : AllMethods()) { + tracker.LossyAdd(method, n); + n += 2; + } + done.Notify(); + reader.join(); +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl
diff --git a/absl/synchronization/internal/waiter.cc b/absl/synchronization/internal/waiter.cc index 2123be6..28ef311 100644 --- a/absl/synchronization/internal/waiter.cc +++ b/absl/synchronization/internal/waiter.cc
@@ -79,6 +79,7 @@ // Note that, since the thread ticker is just reset, we don't need to check // whether the thread is idle on the very first pass of the loop. bool first_pass = true; + while (true) { int32_t x = futex_.load(std::memory_order_relaxed); while (x != 0) { @@ -90,7 +91,6 @@ return true; // Consumed a wakeup, we are done. } - if (!first_pass) MaybeBecomeIdle(); const int err = Futex::WaitUntil(&futex_, 0, t); if (err != 0) {
diff --git a/absl/time/duration_test.cc b/absl/time/duration_test.cc index fb28fa9..a3617e7 100644 --- a/absl/time/duration_test.cc +++ b/absl/time/duration_test.cc
@@ -1320,7 +1320,7 @@ EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(0)); // TODO(bww): Is the next one OK? - EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(0.124999999e-9)); + EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(std::nextafter(0.125e-9, 0))); EXPECT_EQ(absl::Nanoseconds(1) / 4, absl::Seconds(0.125e-9)); EXPECT_EQ(absl::Nanoseconds(1) / 4, absl::Seconds(0.250e-9)); EXPECT_EQ(absl::Nanoseconds(1) / 2, absl::Seconds(0.375e-9)); @@ -1330,7 +1330,7 @@ EXPECT_EQ(absl::Nanoseconds(1), absl::Seconds(0.875e-9)); EXPECT_EQ(absl::Nanoseconds(1), absl::Seconds(1.000e-9)); - EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(-0.124999999e-9)); + EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(std::nextafter(-0.125e-9, 0))); EXPECT_EQ(-absl::Nanoseconds(1) / 4, absl::Seconds(-0.125e-9)); EXPECT_EQ(-absl::Nanoseconds(1) / 4, absl::Seconds(-0.250e-9)); EXPECT_EQ(-absl::Nanoseconds(1) / 2, absl::Seconds(-0.375e-9));
diff --git a/absl/time/time.h b/absl/time/time.h index 2df6858..48982df 100644 --- a/absl/time/time.h +++ b/absl/time/time.h
@@ -1352,7 +1352,7 @@ inline Duration MakePosDoubleDuration(double n) { const int64_t int_secs = static_cast<int64_t>(n); const uint32_t ticks = static_cast<uint32_t>( - (n - static_cast<double>(int_secs)) * kTicksPerSecond + 0.5); + std::round((n - static_cast<double>(int_secs)) * kTicksPerSecond)); return ticks < kTicksPerSecond ? MakeDuration(int_secs, ticks) : MakeDuration(int_secs + 1, ticks - kTicksPerSecond);
diff --git a/absl/types/span.h b/absl/types/span.h index 95fe792..41db342 100644 --- a/absl/types/span.h +++ b/absl/types/span.h
@@ -243,8 +243,8 @@ // template <typename LazyT = T, typename = EnableIfConstView<LazyT>> - Span( - std::initializer_list<value_type> v) noexcept // NOLINT(runtime/explicit) + Span(std::initializer_list<value_type> v + ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept // NOLINT(runtime/explicit) : Span(v.begin(), v.size()) {} // Accessors