diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 0000000..3427eb2 --- /dev/null +++ b/.bazelrc
@@ -0,0 +1,122 @@ +# Copyright 2026 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +common --show_timestamps + +# Retry longer to avoid flakes when fetching external dependencies. +common --http_connector_attempts=10 +common --http_timeout_scaling=2.0 + +# Link GoogleTest against Abseil. +build --define=absl=1 + +test --keep_going +test --test_env=GTEST_INSTALL_FAILURE_SIGNAL_HANDLER=1 +test --test_env=TZDIR=absl/time/internal/cctz/testdata/zoneinfo +test --test_output=errors +test --test_tag_filters=-benchmark + +# Configure our Linux Docker toolchain setup. +common --extra_toolchains=@docker_toolchain//:all +build:linux --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 +build:linux --copt=-fno-exceptions +build:linux --copt=-Werror +build:linux --copt=-DGTEST_REMOVE_LEGACY_TEST_CASEAPI_=1 +build:linux --features=external_include_paths +build:linux --per_file_copt=external/.*@-w +build:linux --host_per_file_copt=external/.*@-w + +# Configure our Windows MSVC and ClangCL setup. +common:windows-msvc --enable_runfiles +build:windows-msvc --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 +build:windows-msvc --copt=/WX +build:windows-msvc --per_file_copt=external/.*@/w +build:windows-msvc --host_per_file_copt=external/.*@/w + +build:windows-clangcl --config=windows-msvc +build:windows-clangcl --compiler=clang-cl +build:windows-clangcl --copt=-Wno-microsoft-cast +build:windows-clangcl --extra_execution_platforms=//:x64_windows-clang-cl +build:windows-clangcl --extra_toolchains=@local_config_cc//:cc-toolchain-x64_windows-clang-cl + +# Configure our macOS Xcode setup. +build:macos --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 +build:macos --copt=-Werror +build:macos --copt=-DGTEST_REMOVE_LEGACY_TEST_CASEAPI_=1 +build:macos --features=external_include_paths +build:macos --per_file_copt=external/.*@-w +build:macos --host_per_file_copt=external/.*@-w + +build:gcc --config=linux --platforms=//ci:linux_gcc +build:gcc-exceptions --config=gcc --copt=-fexceptions +build:clang --config=linux --platforms=//ci:linux_clang --conlyopt=-Wno-unused-command-line-argument +build:clang-exceptions --config=clang --copt=-fexceptions +build:clang-arm --config=linux --platforms=//ci:linux_arm_clang --conlyopt=-Wno-unused-command-line-argument +build:libstdc++ --config=clang --features=libstdc++ --copt=-march=haswell + +# Sanitizer configurations. +build:ubsan --features=ubsan +build:ubsan --copt=-fsanitize=float-divide-by-zero,nullability,undefined +build:ubsan --copt=-fno-sanitize-recover=float-divide-by-zero,nullability,undefined +build:ubsan --copt=-fno-sanitize-blacklist +test:ubsan --test_env=UBSAN_OPTIONS=print_stacktrace=1 + +build:asan --config=clang --config=ubsan --features=asan +build:asan --copt=-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG +build:asan --linkopt=-fsanitize-link-c++-runtime +test:asan --test_tag_filters=-benchmark,-noasan + +build:tsan --config=clang --features=tsan +build:tsan --copt=-fno-sanitize-blacklist +build:tsan --build_tag_filters=-notsan +test:tsan --test_tag_filters=-benchmark,-notsan + +# Remote build configuration for CI. +build:remote-cache --remote_cache=https://storage.googleapis.com/absl-bazel-remote-cache +build:remote-cache --remote_timeout=1200 --bes_timeout=600s +build:remote-cache --remote_download_minimal +# cc_test puts Windows .pdb files in the pdb_file output group rather than +# runfiles, so explicitly download symbolize_test.pdb when linking hits the +# remote cache. +build:remote-cache --remote_download_regex=.*symbolize_test\.pdb$ +# Retry up for up to 15 minutes under heavy load (exponential backoff). +build:remote-cache --remote_retries=20 --remote_retry_max_delay=60s + +build:remote-base --define=EXECUTOR=remote +build:remote-base --remote_timeout=1200 --bes_timeout=600s +build:remote-base --jobs=50 +build:remote-base --remote_download_minimal +# Retry up for up to 15 minutes under heavy load (exponential backoff). +build:remote-base --remote_retries=20 --remote_retry_max_delay=60s + +build:remote --config=remote-base +build:remote --extra_execution_platforms=//ci:rbe +build:remote --host_platform=//ci:rbe +build:remote --platforms=//ci:rbe +build:remote --remote_cache=grpcs://us-east1-remotebuildexecution.googleapis.com +build:remote --remote_executor=grpcs://us-east1-remotebuildexecution.googleapis.com +build:remote --remote_instance_name=projects/google.com:absl-177019/instances/absl-ci-east + +build:remote-floor --config=remote +build:remote-floor --extra_execution_platforms=//ci:rbe-floor +build:remote-floor --host_platform=//ci:rbe-floor +build:remote-floor --platforms=//ci:rbe-floor + +build:remote-arm --config=remote-base +build:remote-arm --extra_execution_platforms=//ci:rbe-arm +build:remote-arm --host_platform=//ci:rbe-arm +build:remote-arm --platforms=//ci:rbe-arm +build:remote-arm --remote_cache=grpcs://us-central1-remotebuildexecution.googleapis.com +build:remote-arm --remote_executor=grpcs://us-central1-remotebuildexecution.googleapis.com +build:remote-arm --remote_instance_name=projects/google.com:absl-177019/instances/gloop-ci-central
diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml new file mode 100644 index 0000000..b54e910 --- /dev/null +++ b/.github/workflows/bazel.yml
@@ -0,0 +1,113 @@ +# Copyright 2026 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Bazel Tests + +on: + workflow_call: + inputs: + os: + description: "GitHub Actions runner OS (e.g. ubuntu-24.04, macos-latest, windows-2022)" + required: true + type: string + toolchain: + description: "Bazel toolchain config (--config=<toolchain>)" + required: true + type: string + remote_config: + description: "Bazel remote config (--config=<remote_config>)" + required: false + default: "remote" + type: string + container_image: + description: "Docker container image to run in (empty string for host execution)" + required: false + default: "" + type: string + alternate_options: + description: "Whether to use ci/absl_alternate_options.h" + required: false + default: false + type: boolean + stds: + description: "JSON array of C++ standards to test" + required: false + default: '["c++17", "c++20", "c++23"]' + type: string + compilation_modes: + description: "JSON array of Bazel compilation modes to test" + required: false + default: '["opt", "fastbuild"]' + type: string + +permissions: + contents: read + +jobs: + test: + runs-on: ${{ inputs.os }} + name: (${{ matrix.std }}, ${{ matrix.compilation_mode }}) + container: + image: ${{ inputs.container_image }} + options: --cap-add=SYS_PTRACE + credentials: + # TODO(b/573304451): Switch to Workload Identity Federation to avoid the need for secrets. + username: _json_key + password: ${{ secrets.GAR_SERVICE_ACCOUNT }} + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + std: ${{ fromJSON(inputs.stds) }} + compilation_mode: ${{ fromJSON(inputs.compilation_modes) }} + defaults: + run: + shell: bash + env: + BAZEL_LLVM: C:\Program Files\LLVM + MSYS2_ARG_CONV_EXCL: "*" + USE_BAZEL_VERSION: 9.x + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # 7.0.0 + + - name: Set up alternate options + if: ${{ inputs.alternate_options }} + run: cp ci/absl_alternate_options.h absl/base/options.h + + - name: Setup continuous run + if: ${{ github.event_name == 'schedule' }} + # Force builds and tests to run on scheduled runs. + run: echo "EXTRA_BAZEL_ARGS=--cache_test_results=no --action_env=FORCE_REBUILD=1" >> $GITHUB_ENV + + - name: Set up Bazel and caches + uses: bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86 # 0.19.0 + with: + cache-save: true + bazelisk-cache: true + external-cache: true + repository-cache: true + google-credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazelrc: common --config=${{ inputs.remote_config }} + + - name: Fetch all external dependencies + run: bazel fetch //... + + - name: Run tests + run: > + bazel test ... + --config=${{ inputs.toolchain }} + --cxxopt=${{ startsWith(inputs.os, 'windows') && '/std:' || '-std=' }}${{ matrix.std }} + --host_cxxopt=${{ startsWith(inputs.os, 'windows') && '/std:' || '-std=' }}${{ matrix.std }} + --compilation_mode=${{ matrix.compilation_mode }} + ${{ env.EXTRA_BAZEL_ARGS }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..9957543 --- /dev/null +++ b/.github/workflows/ci.yml
@@ -0,0 +1,215 @@ +# Copyright 2026 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Tests + +on: + # continuous runs + schedule: + # Run every hour + - cron: "0 * * * *" + + # postsubmit runs + push: + branches: + - master + - lts_* + + # presubmit runs + pull_request: + branches: + - master + - lts_* + + # manual runs + workflow_dispatch: + +permissions: + contents: read + +# Avoid running twice at the same time over the same changes. +concurrency: + group: ${{ github.event_name }}-${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: ${{ contains(fromJSON('["pull_request", "workflow_dispatch", "schedule"]'), github.event_name) }} + +jobs: + linux: + name: Linux ${{ matrix.config.name }} Bazel + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + config: + # linux_clang-latest_libcxx_bazel.cfg + - name: Clang/libc++ + toolchain: clang + # linux_clang-latest_libcxx_exceptions_bazel.cfg (fastbuild only) + - name: Clang/libc++ -fexceptions + toolchain: clang-exceptions + compilation_modes: '["fastbuild"]' + # linux_clang-latest_libcxx_alternate_options_bazel.cfg + - name: Clang/libc++ alternate options + toolchain: clang + alternate_options: true + # linux_clang-latest_libcxx_asan_bazel.cfg (fastbuild only) + - name: Clang/libc++ ASAN + toolchain: asan + compilation_modes: '["fastbuild"]' + # linux_clang-latest_libcxx_tsan_bazel.cfg (fastbuild only) + - name: Clang/libc++ TSAN + toolchain: tsan + compilation_modes: '["fastbuild"]' + # linux_clang-latest_libstdcxx_bazel.cfg + - name: Clang/libstdc++ + toolchain: libstdc++ + # linux_gcc-latest_libstdcxx_bazel.cfg + - name: GCC/libstdc++ + toolchain: gcc + # linux_gcc-latest_libstdcxx_exceptions_bazel.cfg (fastbuild only) + - name: GCC/libstdc++ -fexceptions + toolchain: gcc-exceptions + compilation_modes: '["fastbuild"]' + # linux_gcc-latest_libstdcxx_alternate_options_bazel.cfg + - name: GCC/libstdc++ alternate options + toolchain: gcc + alternate_options: true + # linux_gcc-floor_libstdcxx_bazel.cfg (c++17 only) + - name: GCC/libstdc++ (floor) + toolchain: gcc + remote_config: remote-floor + container_image: gcr.io/google.com/absl-177019/oss-linux-toolchain-floor@sha256:5a5d17e588e14d5f0d993ea828b7ba844c72b02980bcf731ccf5a4cddf8db599 + stds: '["c++17"]' + # linux_arm_clang-latest_libcxx_bazel.cfg (c++17 and c++20 only) + - name: ARM64 Clang/libc++ + os: ubuntu-24.04-arm + toolchain: clang-arm + remote_config: remote-arm + stds: '["c++17", "c++20"]' + uses: ./.github/workflows/bazel.yml + secrets: inherit + with: + os: ${{ matrix.config.os || 'ubuntu-24.04' }} + toolchain: ${{ matrix.config.toolchain }} + remote_config: ${{ matrix.config.remote_config || 'remote' }} + container_image: ${{ matrix.config.container_image || 'gcr.io/google.com/absl-177019/oss-linux-toolchain-latest@sha256:5a19067585ce5e5ee45640fd0287f05038d6b3f9574d6a5cdf08241e022d0f4a' }} + alternate_options: ${{ matrix.config.alternate_options || false }} + stds: ${{ matrix.config.stds || '["c++17", "c++20", "c++23"]' }} + compilation_modes: ${{ matrix.config.compilation_modes || '["opt", "fastbuild"]' }} + + cmake: + name: ${{ matrix.config.name }} CMake + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + config: + # linux_gcc-latest_libstdcxx_cxx17_cmake.cfg + - name: Linux GCC/libstdc++ + os: ubuntu-24.04 + container_image: gcr.io/google.com/absl-177019/oss-linux-latest@sha256:9f7f104e4857e3528e8bf5f14968183f5a0e786ca9ef63601eae4e4136f011d6 + # linux_gcc_alpine_cmake.cfg + - name: Alpine Linux GCC (musl) + os: ubuntu-24.04 + container_image: gcr.io/google.com/absl-177019/alpine@sha256:ae26b80601cebb14595056a94e070f0f865d24baaf3ce19bcd52f53be9f99c80 + build_types: '["Debug"]' + link_modes: '["static"]' + # cmake-install.cfg + - name: Linux Install + os: ubuntu-24.04 + container_image: gcr.io/google.com/absl-177019/oss-linux-latest@sha256:9f7f104e4857e3528e8bf5f14968183f5a0e786ca9ef63601eae4e4136f011d6 + build_types: '["Release"]' + link_modes: '["static", "shared"]' + install_test: true + # macos_xcode_cmake.cfg + - name: macOS Xcode + os: macos-latest + build_types: '["Debug"]' + # windows-msvc-2022-cxx17-dbg-cmake.cfg + # windows-msvc-2022-cxx17-dbg-dll-cmake.cfg + - name: Windows MSVC 2022 + os: windows-2022 + build_types: '["Debug"]' + link_modes: '["static", "shared"]' + uses: ./.github/workflows/cmake.yml + secrets: inherit + with: + os: ${{ matrix.config.os }} + container_image: ${{ matrix.config.container_image || '' }} + build_types: ${{ matrix.config.build_types || '["Debug", "Release"]' }} + link_modes: ${{ matrix.config.link_modes || '["static", "shared", "monolithic-shared"]' }} + install_test: ${{ matrix.config.install_test || false }} + + windows: + name: Windows ${{ matrix.config.name }} Bazel + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + config: + # windows-msvc-2022-cxx{17,20}-{dbg,opt}-bazel.cfg + - name: MSVC 2022 + toolchain: windows-msvc + # windows-msvc-2022-alternate-options-cxx{17,20}-dbg-bazel.cfg (dbg only) + - name: MSVC 2022 Alternate Options + toolchain: windows-msvc + alternate_options: true + compilation_modes: '["dbg"]' + # windows-clangcl-cxx17-fastbuild-bazel.cfg + - name: ClangCL + toolchain: windows-clangcl + stds: '["c++17"]' + compilation_modes: '["fastbuild"]' + uses: ./.github/workflows/bazel.yml + secrets: inherit + with: + os: windows-2022 + toolchain: ${{ matrix.config.toolchain }} + remote_config: remote-cache + alternate_options: ${{ matrix.config.alternate_options || false }} + stds: ${{ matrix.config.stds || '["c++17", "c++20"]' }} + compilation_modes: ${{ matrix.config.compilation_modes || '["dbg", "opt"]' }} + + macos: + name: macOS ${{ matrix.config.name }} Bazel + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + config: + # macos_xcode_bazel.cfg + - name: Xcode + # macos_xcode_alternate_options_bazel.cfg + - name: Xcode Alternate Options + alternate_options: true + uses: ./.github/workflows/bazel.yml + secrets: inherit + with: + os: macos-latest + toolchain: macos + remote_config: remote-cache + alternate_options: ${{ matrix.config.alternate_options || false }} + stds: ${{ matrix.config.stds || '["c++17"]' }} + compilation_modes: ${{ matrix.config.compilation_modes || '["fastbuild"]' }} + + # Provide a single predictably-named job for copybara to look for + all-blocking-tests: + name: All Blocking Tests + needs: [linux, cmake, windows, macos] + runs-on: ubuntu-24.04 + steps: + - name: Check for cancellations + if: ${{ contains(join(needs.*.result, ' '), 'cancelled') }} + run: echo "One or more tests were cancelled. If this was a presubmit job cancelled by a submit job, you can ignore this." && exit 1 + - name: Check for failures + if: ${{ contains(join(needs.*.result, ' '), 'failure') }} + run: echo "One or more tests failed. Please check the logs above and fix the issues." && exit 1 + # This workflow must run even if one or more of the dependent workflows + # failed. + if: always()
diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml new file mode 100644 index 0000000..2080af3 --- /dev/null +++ b/.github/workflows/cmake.yml
@@ -0,0 +1,148 @@ +# Copyright 2026 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: CMake Tests + +on: + workflow_call: + inputs: + os: + description: "GitHub Actions runner OS (e.g. ubuntu-24.04, macos-latest, windows-2022)" + required: true + type: string + container_image: + description: "Docker container image to run in (empty string for host execution)" + required: false + default: "" + type: string + build_types: + description: "JSON array of CMAKE_BUILD_TYPE values to test" + required: false + default: '["Debug", "Release"]' + type: string + link_modes: + description: "JSON array of link modes to test (static, shared, monolithic-shared)" + required: false + default: '["static", "shared", "monolithic-shared"]' + type: string + install_test: + description: "Whether to run the CMake installation integration test" + required: false + default: false + type: boolean + +permissions: + contents: read + +jobs: + test: + runs-on: ${{ inputs.os }} + name: (${{ matrix.build_type }}, ${{ matrix.link_mode }}) + container: + image: ${{ inputs.container_image }} + # core=1 skips kernel pipe core_pattern handlers (e.g. apport) on death tests. + options: --cap-add=SYS_PTRACE --ulimit core=1 + credentials: + # TODO(b/573304451): Switch to Workload Identity Federation to avoid the need for secrets. + username: _json_key + password: ${{ secrets.GAR_SERVICE_ACCOUNT }} + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + build_type: ${{ fromJSON(inputs.build_types) }} + link_mode: ${{ fromJSON(inputs.link_modes) }} + defaults: + run: + shell: bash + env: + CC: ${{ startsWith(inputs.os, 'ubuntu') && 'gcc' || '' }} + CFLAGS: ${{ !inputs.install_test && startsWith(inputs.os, 'ubuntu') && '-Werror' || '' }} + CXX: ${{ startsWith(inputs.os, 'ubuntu') && 'g++' || '' }} + CXXFLAGS: ${{ !inputs.install_test && (startsWith(inputs.os, 'ubuntu') && '-Werror' || startsWith(inputs.os, 'windows') && '-WX') || '' }} + CMAKE_BUILD_PARALLEL_LEVEL: 16 + CMAKE_C_COMPILER_LAUNCHER: sccache + CMAKE_CXX_COMPILER_LAUNCHER: sccache + CMAKE_GENERATOR: Ninja + CTEST_OUTPUT_ON_FAILURE: 1 + CTEST_PARALLEL_LEVEL: 16 + SCCACHE_GCS_BUCKET: absl-cmake-sccache + SCCACHE_GCS_RW_MODE: READ_WRITE + SCCACHE_IDLE_TIMEOUT: 0 + SCCACHE_IGNORE_SERVER_IO_ERROR: 1 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # 7.0.0 + + - name: Set up MSVC + if: ${{ runner.os == 'Windows' }} + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + with: + arch: x64 + vsversion: "2022" + + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + credentials_json: ${{ secrets.GAR_SERVICE_ACCOUNT }} + + - name: Set up sccache + uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + + - name: Start sccache + env: + SCCACHE_GCS_KEY_PATH: ${{ steps.auth.outputs.credentials_file_path }} + run: sccache --start-server + + - name: Run installation test + if: ${{ inputs.install_test }} + env: + LINK_TYPE: ${{ matrix.link_mode == 'shared' && 'DYNAMIC' || 'STATIC' }} + run: > + source ci/cmake_common.sh && + export ABSL_GOOGLETEST_VERSION ABSL_GOOGLETEST_DOWNLOAD_URL && + CMake/install_test_project/test.sh + + - name: Configure + if: ${{ !inputs.install_test }} + run: > + source ci/cmake_common.sh && + cmake -B build + -DABSL_BUILD_TESTING=ON + -DABSL_GOOGLETEST_DOWNLOAD_URL=${ABSL_GOOGLETEST_DOWNLOAD_URL} + -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} + -DCMAKE_C_FLAGS_DEBUG="${{ runner.os == 'Windows' && '-O1 -Ob1 -Zc:inline' || '-Og' }}" + -DCMAKE_CXX_FLAGS_DEBUG="${{ runner.os == 'Windows' && '-O1 -Ob1 -Zc:inline' || '-Og' }}" + -DCMAKE_EXE_LINKER_FLAGS_DEBUG="" + -DCMAKE_SHARED_LINKER_FLAGS_DEBUG="" + -DCMAKE_CXX_STANDARD=17 + -DBUILD_SHARED_LIBS=${{ matrix.link_mode != 'static' }} + -DABSL_BUILD_MONOLITHIC_SHARED_LIBS=${{ matrix.link_mode == 'monolithic-shared' }} + -DCMAKE_POLICY_DEFAULT_CMP0141=NEW + -DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT="" + -DCMAKE_JOB_POOLS="link=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu)" + -DCMAKE_JOB_POOL_LINK=link + + - name: Build + if: ${{ !inputs.install_test }} + run: cmake --build build + + - name: Run tests + if: ${{ !inputs.install_test }} + env: + TZDIR: ${{ github.workspace }}/absl/time/internal/cctz/testdata/zoneinfo + run: > + ctest --test-dir build + --repeat until-pass:3 + ${{ runner.os == 'Windows' && '-E "absl_lifetime_test|absl_symbolize_test"' || '' }}
diff --git a/CMake/AbseilDll.cmake b/CMake/AbseilDll.cmake index b615da8..fdd4ae1 100644 --- a/CMake/AbseilDll.cmake +++ b/CMake/AbseilDll.cmake
@@ -100,7 +100,6 @@ "container/internal/raw_hash_set.cc" "container/internal/raw_hash_set.h" "container/internal/raw_hash_set_resize_impl.h" - "container/internal/tracked.h" "container/linked_hash_map.h" "container/linked_hash_set.h" "container/node_hash_map.h" @@ -138,8 +137,6 @@ "debugging/internal/elf_mem_image.h" "debugging/internal/examine_stack.cc" "debugging/internal/examine_stack.h" - "debugging/internal/stack_consumption.cc" - "debugging/internal/stack_consumption.h" "debugging/internal/stacktrace_config.h" "debugging/internal/symbolize.h" "debugging/internal/utf8_for_code_point.cc" @@ -165,7 +162,6 @@ "hash/internal/city.h" "hash/internal/hash.cc" "hash/internal/hash.h" - "hash/internal/spy_hash_state.h" "hash/internal/weakly_mixed_integer.h" "log/absl_check.h" "log/absl_log.h" @@ -267,7 +263,6 @@ "random/internal/salted_seed_seq.h" "random/internal/seed_material.cc" "random/internal/seed_material.h" - "random/internal/sequence_urbg.h" "random/internal/traits.h" "random/internal/uniform_helper.h" "random/internal/wide_multiply.h" @@ -348,8 +343,6 @@ "strings/internal/memutil.h" "strings/internal/ostringstream.cc" "strings/internal/ostringstream.h" - "strings/internal/pow10_helper.cc" - "strings/internal/pow10_helper.h" "strings/internal/resize_uninitialized.h" "strings/internal/str_format/arg.cc" "strings/internal/str_format/arg.h" @@ -410,7 +403,6 @@ "synchronization/internal/sem_waiter.h" "synchronization/internal/stdcpp_waiter.cc" "synchronization/internal/stdcpp_waiter.h" - "synchronization/internal/thread_pool.h" "synchronization/internal/waiter.h" "synchronization/internal/waiter_base.cc" "synchronization/internal/waiter_base.h" @@ -449,8 +441,6 @@ "time/internal/cctz/src/time_zone_posix.h" "time/internal/cctz/src/tzfile.h" "time/internal/cctz/src/zone_info_source.cc" - "time/simulated_clock.cc" - "time/simulated_clock.h" "time/time.cc" "time/time.h" "types/any.h" @@ -513,186 +503,6 @@ ) endif() -set(ABSL_INTERNAL_DLL_TARGETS - "absl_check" - "absl_log" - "absl_vlog_is_on" - "algorithm" - "algorithm_container" - "any" - "any_invocable" - "atomic_hook" - "base" - "base_cpu_detect" - "base_internal" - "bind_front" - "bits" - "btree" - "check" - "city" - "civil_time" - "compare" - "compressed_tuple" - "config" - "container" - "container_common" - "container_memory" - "cord" - "cord_internal" - "cordz_functions" - "cordz_handle" - "cordz_info" - "cordz_sample_token" - "core_headers" - "counting_allocator" - "crc_cord_state" - "crc_internal" - "crc32c" - "debugging" - "debugging_internal" - "demangle_internal" - "die_if_null" - "dynamic_annotations" - "endian" - "examine_stack" - "exponential_biased" - "failure_signal_handler" - "fixed_array" - "flat_hash_map" - "flat_hash_set" - "function_ref" - "graphcycles_internal" - "hash" - "hash_function_defaults" - "hash_policy_traits" - "hashtable_debug" - "hashtable_debug_hooks" - "hashtablez_sampler" - "inlined_vector" - "inlined_vector_internal" - "int128" - "kernel_timeout_internal" - "layout" - "leak_check" - "log_internal_check_impl" - "log_internal_check_op" - "log_internal_conditions" - "log_internal_config" - "log_internal_fnmatch" - "log_internal_format" - "log_internal_globals" - "log_internal_log_impl" - "log_internal_proto" - "log_internal_message" - "log_internal_log_sink_set" - "log_internal_nullguard" - "log_internal_nullstream" - "log_internal_strip" - "log_internal_voidify" - "log_internal_append_truncated" - "log_globals" - "log_initialize" - "log" - "log_entry" - "log_sink" - "log_sink_registry" - "log_streamer" - "log_internal_structured" - "log_severity" - "log_structured" - "low_level_hash" - "malloc_internal" - "memory" - "meta" - "node_hash_map" - "node_hash_set" - "node_slot_policy" - "non_temporal_arm_intrinsics" - "non_temporal_memcpy" - "numeric" - "optional" - "periodic_sampler" - "pow10_helper" - "pretty_function" - "random_bit_gen_ref" - "random_distributions" - "random_internal_distribution_caller" - "random_internal_distributions" - "random_internal_explicit_seed_seq" - "random_internal_fastmath" - "random_internal_fast_uniform_bits" - "random_internal_generate_real" - "random_internal_iostream_state_saver" - "random_internal_nonsecure_base" - "random_internal_pcg_engine" - "random_internal_platform" - "random_internal_pool_urbg" - "random_internal_randen" - "random_internal_randen_engine" - "random_internal_randen_hwaes" - "random_internal_randen_hwaes_impl" - "random_internal_randen_slow" - "random_internal_salted_seed_seq" - "random_internal_seed_material" - "random_internal_sequence_urbg" - "random_internal_traits" - "random_internal_uniform_helper" - "random_internal_wide_multiply" - "random_random" - "random_seed_gen_exception" - "random_seed_sequences" - "raw_hash_map" - "raw_hash_set" - "raw_logging_internal" - "sample_recorder" - "scoped_set_env" - "span" - "spinlock_wait" - "spy_hash_state" - "stack_consumption" - "stacktrace" - "status" - "statusor" - "str_format" - "str_format_internal" - "strerror" - "strings" - "strings_internal" - "string_view" - "symbolize" - "synchronization" - "thread_pool" - "throw_delegate" - "time" - "time_zone" - "tracked" - "type_traits" - "utility" - "variant" - "vlog_config_internal" - "vlog_is_on" -) - -if(NOT MSVC) - list(APPEND ABSL_INTERNAL_DLL_TARGETS - "flags" - "flags_commandlineflag" - "flags_commandlineflag_internal" - "flags_config" - "flags_internal" - "flags_marshalling" - "flags_parse" - "flags_path_util" - "flags_private_handle_accessor" - "flags_program_name" - "flags_reflection" - "flags_usage" - "flags_usage_internal" - "log_internal_flags" - "log_flags" - ) -endif() - set(ABSL_INTERNAL_TEST_DLL_FILES "hash/hash_testing.h" "log/scoped_mock_log.cc" @@ -712,17 +522,6 @@ "strings/cord_test_helpers.h" ) -set(ABSL_INTERNAL_TEST_DLL_TARGETS - "cord_test_helpers" - "cordz_test_helpers" - "hash_testing" - "random_mocking_bit_gen" - "random_internal_distribution_test_util" - "random_internal_mock_overload_set" - "scoped_mock_log" - "status_matchers" -) - if(DEFINED CMAKE_CXX_STANDARD AND CMAKE_CXX_STANDARD_REQUIRED) if(CMAKE_CXX_STANDARD GREATER_EQUAL 20) set(ABSL_INTERNAL_AT_LEAST_CXX20 ON) @@ -769,63 +568,29 @@ endif() function(absl_internal_dll_contains) - cmake_parse_arguments(ABSL_INTERNAL_DLL - "" - "OUTPUT;TARGET" - "" + cmake_parse_arguments(ABSL_INTERNAL_DLL_CONTAINS + "TEST" + "OUTPUT" + "FILES" ${ARGN} ) - STRING(REGEX REPLACE "^absl::" "" _target ${ABSL_INTERNAL_DLL_TARGET}) - - if (_target IN_LIST ABSL_INTERNAL_DLL_TARGETS) - set(${ABSL_INTERNAL_DLL_OUTPUT} 1 PARENT_SCOPE) + if(ABSL_INTERNAL_DLL_CONTAINS_TEST) + set(_dll_files "${ABSL_INTERNAL_TEST_DLL_FILES}") else() - set(${ABSL_INTERNAL_DLL_OUTPUT} 0 PARENT_SCOPE) + set(_dll_files "${ABSL_INTERNAL_DLL_FILES}") endif() -endfunction() -function(absl_internal_test_dll_contains) - cmake_parse_arguments(ABSL_INTERNAL_TEST_DLL - "" - "OUTPUT;TARGET" - "" - ${ARGN} - ) - - STRING(REGEX REPLACE "^absl::" "" _target ${ABSL_INTERNAL_TEST_DLL_TARGET}) - - if (_target IN_LIST ABSL_INTERNAL_TEST_DLL_TARGETS) - set(${ABSL_INTERNAL_TEST_DLL_OUTPUT} 1 PARENT_SCOPE) - else() - set(${ABSL_INTERNAL_TEST_DLL_OUTPUT} 0 PARENT_SCOPE) - endif() -endfunction() - -function(absl_internal_dll_targets) - cmake_parse_arguments(ABSL_INTERNAL_DLL - "" - "OUTPUT" - "DEPS" - ${ARGN} - ) - - set(_deps "") - foreach(dep IN LISTS ABSL_INTERNAL_DLL_DEPS) - absl_internal_dll_contains(TARGET ${dep} OUTPUT _dll_contains) - absl_internal_test_dll_contains(TARGET ${dep} OUTPUT _test_dll_contains) - if (_dll_contains) - list(APPEND _deps abseil_dll) - elseif (_test_dll_contains) - list(APPEND _deps abseil_test_dll) - else() - list(APPEND _deps ${dep}) + file(RELATIVE_PATH _rel_dir "${absl_SOURCE_DIR}/absl" "${CMAKE_CURRENT_SOURCE_DIR}") + foreach(_file IN LISTS ABSL_INTERNAL_DLL_CONTAINS_FILES) + string(REGEX REPLACE "^\\$<\\$<[^>]+>:(.+)>$" "\\1" _file "${_file}") + if("${_rel_dir}/${_file}" IN_LIST _dll_files) + set(${ABSL_INTERNAL_DLL_CONTAINS_OUTPUT} 1 PARENT_SCOPE) + return() endif() endforeach() - # Because we may have added the DLL multiple times - list(REMOVE_DUPLICATES _deps) - set(${ABSL_INTERNAL_DLL_OUTPUT} "${_deps}" PARENT_SCOPE) + set(${ABSL_INTERNAL_DLL_CONTAINS_OUTPUT} 0 PARENT_SCOPE) endfunction() function(absl_make_dll)
diff --git a/CMake/AbseilHelpers.cmake b/CMake/AbseilHelpers.cmake index 61e1ae4..e4fda15 100644 --- a/CMake/AbseilHelpers.cmake +++ b/CMake/AbseilHelpers.cmake
@@ -130,13 +130,15 @@ # 4. "static" -- This target does not depend on the DLL and should be built # statically. if (${ABSL_BUILD_DLL}) - if(ABSL_ENABLE_INSTALL) - absl_internal_dll_contains(TARGET ${_NAME} OUTPUT _in_dll) - absl_internal_test_dll_contains(TARGET ${_NAME} OUTPUT _in_test_dll) - else() - absl_internal_dll_contains(TARGET ${ABSL_CC_LIB_NAME} OUTPUT _in_dll) - absl_internal_test_dll_contains(TARGET ${ABSL_CC_LIB_NAME} OUTPUT _in_test_dll) - endif() + absl_internal_dll_contains( + FILES ${ABSL_CC_LIB_SRCS} ${ABSL_CC_LIB_HDRS} + OUTPUT _in_dll + ) + absl_internal_dll_contains( + TEST + FILES ${ABSL_CC_LIB_SRCS} ${ABSL_CC_LIB_HDRS} + OUTPUT _in_test_dll + ) if (${_in_dll} OR ${_in_test_dll}) # This target should be replaced by the DLL set(_build_type "dll") @@ -227,17 +229,11 @@ if(NOT ABSL_CC_LIB_IS_INTERFACE) if(_build_type STREQUAL "dll_dep") - # This target depends on the DLL. When adding dependencies to this target, - # any depended-on-target which is contained inside the DLL is replaced - # with a dependency on the DLL. + # This target depends on the DLL. add_library(${_NAME} STATIC "") target_sources(${_NAME} PRIVATE ${ABSL_CC_LIB_SRCS} ${ABSL_CC_LIB_HDRS}) - absl_internal_dll_targets( - DEPS ${ABSL_CC_LIB_DEPS} - OUTPUT _dll_deps - ) target_link_libraries(${_NAME} - PUBLIC ${_dll_deps} + PUBLIC ${ABSL_CC_LIB_DEPS} PRIVATE ${ABSL_CC_LIB_LINKOPTS} ${ABSL_DEFAULT_LINKOPTS} @@ -372,6 +368,7 @@ # COPTS: List of private compile options # DEFINES: List of public defines # LINKOPTS: List of link options +# SHARD_COUNT: Number of shards to split the test into for ctest # # Note: # By default, absl_cc_test will always create a binary named absl_${NAME}. @@ -405,7 +402,7 @@ cmake_parse_arguments(ABSL_CC_TEST "" - "NAME" + "NAME;SHARD_COUNT" "SRCS;COPTS;DEFINES;LINKOPTS;DEPS" ${ARGN} ) @@ -427,16 +424,6 @@ ABSL_CONSUME_TEST_DLL GTEST_LINKED_AS_SHARED_LIBRARY=1 ) - - # Replace dependencies on targets inside the DLL with abseil_dll itself. - absl_internal_dll_targets( - DEPS ${ABSL_CC_TEST_DEPS} - OUTPUT ABSL_CC_TEST_DEPS - ) - absl_internal_dll_targets( - DEPS ${ABSL_CC_TEST_LINKOPTS} - OUTPUT ABSL_CC_TEST_LINKOPTS - ) else() target_compile_definitions(${_NAME} PUBLIC @@ -461,5 +448,16 @@ target_compile_features(${_NAME} PUBLIC ${ABSL_INTERNAL_CXX_STD_FEATURE}) endif() - add_test(NAME ${_NAME} COMMAND ${_NAME}) + if(ABSL_CC_TEST_SHARD_COUNT AND ABSL_CC_TEST_SHARD_COUNT GREATER 1) + math(EXPR _LAST_SHARD "${ABSL_CC_TEST_SHARD_COUNT} - 1") + foreach(_SHARD RANGE 0 ${_LAST_SHARD}) + add_test(NAME "${_NAME}_shard_${_SHARD}" COMMAND ${_NAME}) + set_tests_properties("${_NAME}_shard_${_SHARD}" PROPERTIES + ENVIRONMENT "GTEST_TOTAL_SHARDS=${ABSL_CC_TEST_SHARD_COUNT};GTEST_SHARD_INDEX=${_SHARD}" + COST 100 + ) + endforeach() + else() + add_test(NAME ${_NAME} COMMAND ${_NAME}) + endif() endfunction()
diff --git a/CMake/install_test_project/test.sh b/CMake/install_test_project/test.sh index 962bc8d..2d25696 100755 --- a/CMake/install_test_project/test.sh +++ b/CMake/install_test_project/test.sh
@@ -19,8 +19,9 @@ # Fail on any error. Treat unset variables an error. Print commands as executed. set -euox pipefail -absl_dir=/abseil-cpp +absl_dir="${PWD}" absl_build_dir=/buildfs +mkdir -p "${absl_build_dir}" googletest_builddir=/googletest_builddir googletest_archive="googletest-${ABSL_GOOGLETEST_VERSION}.tar.gz" project_dir="${absl_dir}/CMake/install_test_project" @@ -40,8 +41,8 @@ mkdir build pushd build cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS="${build_shared_libs}" .. -make -j $(nproc) -make install +cmake --build . +cmake --install . ldconfig popd popd @@ -57,10 +58,12 @@ -DABSL_FIND_GOOGLETEST=ON \ -DCMAKE_BUILD_TYPE=Release \ -DABSL_BUILD_TESTING=ON \ - -DBUILD_SHARED_LIBS="${build_shared_libs}" -make -j $(nproc) -ctest -j $(nproc) --output-on-failure -make install + -DBUILD_SHARED_LIBS="${build_shared_libs}" \ + -DCMAKE_JOB_POOLS="link=$(nproc)" \ + -DCMAKE_JOB_POOL_LINK=link +cmake --build . +ctest --output-on-failure --repeat until-pass:3 +cmake --install . ldconfig popd
diff --git a/MODULE.bazel b/MODULE.bazel index 00ab4c3..d4096d2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -50,3 +50,9 @@ version = "20260708.rc1", repo_name = "do_not_use_for_gloop_visibility_only", ) + +bazel_dep(name = "docker_toolchain", version = "0.1", dev_dependency = True) +local_path_override( + module_name = "docker_toolchain", + path = "ci/docker_toolchain", +)
diff --git a/absl/base/BUILD.bazel b/absl/base/BUILD.bazel index 749c54f..b242b90 100644 --- a/absl/base/BUILD.bazel +++ b/absl/base/BUILD.bazel
@@ -514,6 +514,7 @@ visibility = ["//absl:__subpackages__"], deps = [ ":config", + ":core_headers", ":pretty_function", "//absl/memory", "//absl/meta:type_traits", @@ -565,6 +566,7 @@ srcs = ["spinlock_test_common.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + shard_count = 4, tags = [ "no_test_wasm", ],
diff --git a/absl/base/CMakeLists.txt b/absl/base/CMakeLists.txt index 2f8846f..d1ce0f9 100644 --- a/absl/base/CMakeLists.txt +++ b/absl/base/CMakeLists.txt
@@ -367,6 +367,7 @@ ${ABSL_TEST_COPTS} DEPS absl::config + absl::core_headers absl::memory absl::meta absl::pretty_function @@ -533,6 +534,8 @@ absl::core_headers absl::synchronization GTest::gtest_main + SHARD_COUNT + 4 ) # Internal-only target, do not depend on directly.
diff --git a/absl/base/attributes.h b/absl/base/attributes.h index 1e4c7c3..bc080bc 100644 --- a/absl/base/attributes.h +++ b/absl/base/attributes.h
@@ -85,16 +85,16 @@ // Note: As the GCC manual states, "[s]ince non-static C++ methods // have an implicit 'this' argument, the arguments of such methods // should be counted from two, not one." -#if ABSL_HAVE_ATTRIBUTE(format) || (defined(__GNUC__) && !defined(__clang__)) -#define ABSL_PRINTF_ATTRIBUTE(string_index, first_to_check) \ - __attribute__((__format__(__printf__, string_index, first_to_check))) #if ABSL_HAVE_CPP_ATTRIBUTE(gnu::format) +#define ABSL_PRINTF_ATTRIBUTE(string_index, first_to_check) \ + [[gnu::format(printf, string_index, first_to_check)]] #define ABSL_SCANF_ATTRIBUTE(string_index, first_to_check) \ [[gnu::format(scanf, string_index, first_to_check)]] -#else +#elif ABSL_HAVE_ATTRIBUTE(format) || (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_PRINTF_ATTRIBUTE(string_index, first_to_check) \ + __attribute__((__format__(__printf__, string_index, first_to_check))) #define ABSL_SCANF_ATTRIBUTE(string_index, first_to_check) \ __attribute__((__format__(__scanf__, string_index, first_to_check))) -#endif #else #define ABSL_PRINTF_ATTRIBUTE(string_index, first_to_check) #define ABSL_SCANF_ATTRIBUTE(string_index, first_to_check) @@ -115,11 +115,7 @@ #elif defined(_MSC_VER) && \ (_MSC_VER >= 1937 || \ (_MSC_VER >= 1927 && ABSL_INTERNAL_CPLUSPLUS_LANG > 201703L)) -#define ABSL_ATTRIBUTE_ALWAYS_INLINE \ - _Pragma("warning(push)") \ - _Pragma("warning(error: 4649)") /* error on misplaced attributes */ \ - [[msvc::forceinline]] /* */ \ - _Pragma("warning(pop)") +#define ABSL_ATTRIBUTE_ALWAYS_INLINE [[msvc::forceinline]] #define ABSL_HAVE_ATTRIBUTE_ALWAYS_INLINE 1 #else #define ABSL_ATTRIBUTE_ALWAYS_INLINE @@ -131,11 +127,7 @@ #elif defined(_MSC_VER) && \ (_MSC_VER >= 1937 || \ (_MSC_VER >= 1927 && ABSL_INTERNAL_CPLUSPLUS_LANG > 201703L)) -#define ABSL_ATTRIBUTE_NOINLINE \ - _Pragma("warning(push)") \ - _Pragma("warning(error: 4649)") /* error on misplaced attributes */ \ - [[msvc::noinline]] /* */ \ - _Pragma("warning(pop)") +#define ABSL_ATTRIBUTE_NOINLINE [[msvc::noinline]] #define ABSL_HAVE_ATTRIBUTE_NOINLINE 1 #else #define ABSL_ATTRIBUTE_NOINLINE @@ -1117,4 +1109,52 @@ #define ABSL_ATTRIBUTE_WARN_UNUSED #endif +// ABSL_UNSAFE_BUFFERS() +// +// Wraps code that performs unsafe buffer accesses or pointer arithmetic. Used +// to suppress warnings when `-Wunsafe-buffer-usage` is enabled. +// +// **WARNING**: This macro should be used when unavoidable. Applying it means +// the compiler is unable to prove that your code avoids out-of-bounds reads or +// writes, forcing reliance on manual review. Over time, code evolves and +// undocumented assumptions can silently break, introducing vulnerabilities. +// Always prioritize refactoring to safer alternatives (e.g. std::array, +// absl::Span, or bounds-checked containers) over using this suppression. +// +// When using this macro is unavoidable, restrict its scope to the smallest +// possible expression or statement. You must also add a `// SAFETY: ...` +// comment explaining why the operation is actually safe. +// Justifications include: +// * Explicit bounds checks (e.g., `CHECK()`) immediately preceding the access. +// * Invariants strictly enforced by surrounding types or functions. +// * Preconditions required of the caller (though this is heavily discouraged). +// +// Example (expression): +// +// // SAFETY: `buf` is NUL-terminated so `strdup` won't read past the end. +// return ABSL_UNSAFE_BUFFERS(strdup(buf)); +// +// Example (single-line statement): +// +// // SAFETY: `offset` is bounds-checked above to be within `buffer`. +// ABSL_UNSAFE_BUFFERS(buffer += offset;) +// +// Example (multi-line statement): +// +// // SAFETY: `offset` is bounds-checked above to be within `buffer`. +// ABSL_UNSAFE_BUFFERS( +// char* p = buffer + offset; +// p[0] = 'a'; +// ) +// +// Note: When wrapping a multi-line statement, placing a semicolon after the +// closing parenthesis may produce an extra semicolon warning. +#if defined(__clang__) +#define ABSL_UNSAFE_BUFFERS(...) \ + _Pragma("clang unsafe_buffer_usage begin") \ + __VA_ARGS__ _Pragma("clang unsafe_buffer_usage end") +#else +#define ABSL_UNSAFE_BUFFERS(...) __VA_ARGS__ +#endif + #endif // ABSL_BASE_ATTRIBUTES_H_
diff --git a/absl/base/attributes_test.cc b/absl/base/attributes_test.cc index 058d1be..e87b66e 100644 --- a/absl/base/attributes_test.cc +++ b/absl/base/attributes_test.cc
@@ -54,4 +54,22 @@ Agg good1 [[maybe_unused]] = {ImplicitlyConstructible("hello")}; } +TEST(Attributes, UnsafeBuffers) { + int buffer[10] = {0}; + int offset = 1; + + // Expression example + int* p1 = ABSL_UNSAFE_BUFFERS(buffer + offset); + EXPECT_EQ(p1, &buffer[1]); + + // Single-line statement example + int* p2 = buffer; + ABSL_UNSAFE_BUFFERS(p2 += offset;) + EXPECT_EQ(p2, &buffer[1]); + + // Multi-line statement example + ABSL_UNSAFE_BUFFERS(int* p3 = buffer + offset; p3[0] = 1;) + EXPECT_EQ(buffer[1], 1); +} + } // namespace
diff --git a/absl/base/exception_safety_testing_test.cc b/absl/base/exception_safety_testing_test.cc index 24b12d4..9aa4dce 100644 --- a/absl/base/exception_safety_testing_test.cc +++ b/absl/base/exception_safety_testing_test.cc
@@ -842,7 +842,7 @@ return true; } - static unsigned char successes; + static unsigned char successes [[maybe_unused]]; }; struct {
diff --git a/absl/base/internal/exception_safety_testing.h b/absl/base/internal/exception_safety_testing.h index e600b85..b4be3ec 100644 --- a/absl/base/internal/exception_safety_testing.h +++ b/absl/base/internal/exception_safety_testing.h
@@ -40,6 +40,7 @@ #include <vector> #include "gtest/gtest.h" +#include "absl/base/attributes.h" #include "absl/base/internal/pretty_function.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" @@ -547,7 +548,7 @@ } // Memory management operators - static void* operator new(size_t s) noexcept( + ABSL_ATTRIBUTE_NOINLINE static void* operator new(size_t s) noexcept( IsSpecified(TypeSpec::kNoThrowNew)) { if (!IsSpecified(TypeSpec::kNoThrowNew)) { exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION, true); @@ -555,7 +556,7 @@ return ::operator new(s); } - static void* operator new[](size_t s) noexcept( + ABSL_ATTRIBUTE_NOINLINE static void* operator new[](size_t s) noexcept( IsSpecified(TypeSpec::kNoThrowNew)) { if (!IsSpecified(TypeSpec::kNoThrowNew)) { exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION, true); @@ -564,8 +565,8 @@ } template <typename... Args> - static void* operator new(size_t s, Args&&... args) noexcept( - IsSpecified(TypeSpec::kNoThrowNew)) { + ABSL_ATTRIBUTE_NOINLINE static void* operator new( + size_t s, Args&&... args) noexcept(IsSpecified(TypeSpec::kNoThrowNew)) { if (!IsSpecified(TypeSpec::kNoThrowNew)) { exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION, true); } @@ -573,8 +574,8 @@ } template <typename... Args> - static void* operator new[](size_t s, Args&&... args) noexcept( - IsSpecified(TypeSpec::kNoThrowNew)) { + ABSL_ATTRIBUTE_NOINLINE static void* operator new[]( + size_t s, Args&&... args) noexcept(IsSpecified(TypeSpec::kNoThrowNew)) { if (!IsSpecified(TypeSpec::kNoThrowNew)) { exceptions_internal::MaybeThrow(ABSL_INTERNAL_PRETTY_FUNCTION, true); } @@ -583,17 +584,23 @@ // Abseil doesn't support throwing overloaded operator delete. These are // provided so a throwing operator-new can clean up after itself. - void operator delete(void* p) noexcept { ::operator delete(p); } + ABSL_ATTRIBUTE_NOINLINE void operator delete(void* p) noexcept { + ::operator delete(p); + } template <typename... Args> - void operator delete(void* p, Args&&... args) noexcept { + ABSL_ATTRIBUTE_NOINLINE void operator delete(void* p, + Args&&... args) noexcept { ::operator delete(p, std::forward<Args>(args)...); } - void operator delete[](void* p) noexcept { return ::operator delete[](p); } + ABSL_ATTRIBUTE_NOINLINE void operator delete[](void* p) noexcept { + return ::operator delete[](p); + } template <typename... Args> - void operator delete[](void* p, Args&&... args) noexcept { + ABSL_ATTRIBUTE_NOINLINE void operator delete[](void* p, + Args&&... args) noexcept { return ::operator delete[](p, std::forward<Args>(args)...); }
diff --git a/absl/base/internal/low_level_alloc_test.cc b/absl/base/internal/low_level_alloc_test.cc index 2f3a3fd..4cf469d 100644 --- a/absl/base/internal/low_level_alloc_test.cc +++ b/absl/base/internal/low_level_alloc_test.cc
@@ -66,7 +66,7 @@ // Use to indicate to the malloc hooks that // this calls is from LowLevelAlloc. -static bool using_low_level_alloc = false; +static bool using_low_level_alloc [[maybe_unused]] = false; // n times, toss a coin, and based on the outcome // either allocate a new block or deallocate an old block.
diff --git a/absl/base/internal/raw_logging.cc b/absl/base/internal/raw_logging.cc index 848e69f..f705a38 100644 --- a/absl/base/internal/raw_logging.cc +++ b/absl/base/internal/raw_logging.cc
@@ -95,7 +95,7 @@ if (*size < 0) return false; int n = vsnprintf(*buf, static_cast<size_t>(*size), format, ap); bool result = true; - if (n < 0 || n > *size) { + if (n < 0 || n >= *size) { result = false; if (static_cast<size_t>(*size) > sizeof(kTruncated)) { n = *size - static_cast<int>(sizeof(kTruncated)); @@ -125,7 +125,7 @@ va_start(ap, format); int n = vsnprintf(*buf, static_cast<size_t>(*size), format, ap); va_end(ap); - if (n < 0 || n > *size) return false; + if (n < 0 || n >= *size) return false; *size -= n; *buf += n; return true;
diff --git a/absl/base/internal/spinlock.cc b/absl/base/internal/spinlock.cc index 41d2b48..9046a00 100644 --- a/absl/base/internal/spinlock.cc +++ b/absl/base/internal/spinlock.cc
@@ -62,12 +62,12 @@ namespace base_internal { ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static AtomicHook<void (*)( - const void *lock, int64_t wait_cycles)> - submit_profile_data; + const void* lock, int64_t wait_cycles)> + submit_unlock_profile_data; -void RegisterSpinLockProfiler(void (*fn)(const void *contendedlock, +void RegisterSpinLockProfiler(void (*fn)(const void* contendedlock, int64_t wait_cycles)) { - submit_profile_data.Store(fn); + submit_unlock_profile_data.Store(fn); } // Monitor the lock to see if its value changes within some time period @@ -173,7 +173,7 @@ if ((lock_value & kWaitTimeMask) != kSpinLockSleeper) { const int64_t wait_cycles = DecodeWaitCycles(lock_value); ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0); - submit_profile_data(this, wait_cycles); + submit_unlock_profile_data(this, wait_cycles); ABSL_TSAN_MUTEX_POST_DIVERT(this, 0); } }
diff --git a/absl/base/raw_logging_test.cc b/absl/base/raw_logging_test.cc index 48158c0..6d76d7b 100644 --- a/absl/base/raw_logging_test.cc +++ b/absl/base/raw_logging_test.cc
@@ -18,10 +18,17 @@ #include "absl/base/internal/raw_logging.h" +#include <cstddef> #include <string> -#include <tuple> + +#if (defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))) && \ + !defined(__EMSCRIPTEN__) +#include <unistd.h> + +#endif #include "gtest/gtest.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" namespace { @@ -75,6 +82,47 @@ kExpectedDeathOutput); } +// Raw logging writes to STDERR_FILENO via write()/syscall on POSIX platforms, +// so the truncation path can be exercised directly by capturing stderr, without +// relying on death tests. +#if (defined(__unix__) || (defined(__APPLE__) && defined(__MACH__))) && \ + !defined(__EMSCRIPTEN__) +TEST(RawLoggingTest, TruncationMarkerAtExactBufferBoundary) { + if (!absl::raw_log_internal::RawLoggingFullySupported()) { + GTEST_SKIP() << "Raw logging output is not supported on this platform."; + } + // kLogBufSize in raw_logging.cc. + constexpr int kLogBufSize = 3000; + + // The space left for the message depends on the log prefix, which a prefix + // hook can change, so try every length up to the buffer size. Each message + // must come out either whole or ending in the truncation marker. + for (int len = 1; len <= kLogBufSize; ++len) { + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + int saved_stderr = dup(STDERR_FILENO); + ASSERT_NE(saved_stderr, -1); + ASSERT_NE(dup2(fds[1], STDERR_FILENO), -1); + + const std::string msg(static_cast<size_t>(len), 'x'); + ABSL_RAW_LOG(ERROR, "%s", msg.c_str()); + + ASSERT_NE(dup2(saved_stderr, STDERR_FILENO), -1); + close(saved_stderr); + close(fds[1]); + char buf[kLogBufSize + 64]; + ssize_t n = read(fds[0], buf, sizeof(buf)); + close(fds[0]); + ASSERT_GT(n, 0); + const std::string output(buf, static_cast<size_t>(n)); + + ASSERT_TRUE(absl::StrContains(output, msg) || + absl::EndsWith(output, " ... (message truncated)\n")) + << "message length " << len; + } +} +#endif + TEST(InternalLog, CompilationTest) { ABSL_INTERNAL_LOG(INFO, "Internal Log"); std::string log_msg = "Internal Log";
diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index 850eaec..7792be0 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel
@@ -845,6 +845,7 @@ srcs = ["internal/raw_hash_set_test.cc"], copts = ABSL_TEST_COPTS, linkstatic = 1, + shard_count = 4, tags = NOTEST_TAGS_MOBILE + [ "no_test_chromiumos_arm", "no_test_chromiumos_armv7",
diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 7a2641c..82548bc 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt
@@ -49,7 +49,7 @@ absl_cc_library( NAME btree_test_common - hdrs + HDRS "btree_test.h" COPTS ${ABSL_TEST_COPTS} @@ -87,6 +87,8 @@ absl::test_allocator absl::test_instance_tracker GTest::gmock_main + SHARD_COUNT + 10 ) absl_cc_test( @@ -106,6 +108,8 @@ absl::flags absl::raw_logging_internal GTest::gmock_main + SHARD_COUNT + 10 ) # Internal-only target, do not depend on directly. @@ -871,6 +875,8 @@ absl::test_instance_tracker absl::type_traits GTest::gmock_main + SHARD_COUNT + 4 ) # Internal-only target, do not depend on directly.
diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index 3465a90..0837bf9 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc
@@ -178,9 +178,8 @@ } } -FindInfo find_first_non_full_from_h1(const ctrl_t* ctrl, size_t h1, - HashtableCapacity capacity) { - const size_t cap = capacity.capacity(); +FindInfo find_first_non_full_from_h1(size_t cap, const ctrl_t* ctrl, + size_t h1) { auto seq = probe_h1(ProbeCapacity{cap}, h1); if (IsEmptyOrDeleted(ctrl[seq.offset()])) { return {seq.offset(), /*probe_length=*/0}; @@ -196,9 +195,8 @@ // // NOTE: this function must work with tables having both empty and deleted // slots in the same group. Such tables appear during `erase()`. -FindInfo find_first_non_full(const CommonFields& common, size_t hash) { - return find_first_non_full_from_h1(common.control(), H1(hash), - common.capacity_impl()); +FindInfo find_first_non_full(size_t cap, const ctrl_t* ctrl, size_t hash) { + return find_first_non_full_from_h1(cap, ctrl, H1(hash)); } // Same as `find_first_non_full`, but returns the mask corresponding to the @@ -308,6 +306,23 @@ } } +void CommonFields::AnnotateForBugDetectionOnMoveImpl( + CommonFields& that, const PolicyFunctions& policy) { + if (!SwisstableGenerationsEnabled()) { + return; + } + that.set_capacity(this == &that ? HashtableCapacity::CreateSelfMovedFrom() + : HashtableCapacity::CreateMovedFrom()); + if (!maybe_invalid_capacity().IsValid() || + capacity() == policy.soo_capacity()) { + return; + } + increment_generation(); + if (!empty() && should_rehash_for_bug_detection_on_move()) { + ResizeAllocatedTableWithSeedChange(*this, policy, capacity()); + } +} + void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) { ABSL_SWISSTABLE_ASSERT(ctrl[capacity] == ctrl_t::kSentinel); ABSL_SWISSTABLE_ASSERT(IsValidCapacity(capacity)); @@ -501,81 +516,73 @@ // // Unlike setting it directly, this function will perform bounds checks and // mirror the value to the cloned tail if necessary. -inline void SetCtrlNoSanitizeImpl(const CommonFields& c, size_t i, ctrl_t h) { - ABSL_SWISSTABLE_ASSERT(i < c.capacity()); - ctrl_t* ctrl = c.control(); - const size_t cap = c.capacity(); +inline void SetCtrlNoSanitizeImpl(size_t cap, ctrl_t* ctrl, size_t i, + ctrl_t h) { + ABSL_SWISSTABLE_ASSERT(i < cap); + // Select the base pointer rather than the index so `ctrl + cap + 1` does not + // depend on `i`. When inlined, it can reuse `ctrl + cap` from `slot_array` + // and be hoisted out of loops. + ctrl_t* cloned_ctrl_base = i < NumClonedBytes() ? ctrl + cap + 1 : ctrl; ctrl[i] = h; - ctrl[((i - NumClonedBytes()) & cap) + (NumClonedBytes() & cap)] = h; + cloned_ctrl_base[i] = h; } -inline void SetCtrl(const CommonFields& c, size_t i, ctrl_t h, - size_t slot_size) { +inline void SetCtrl(const CommonFields& c, size_t cap, ctrl_t* ctrl, size_t i, + ctrl_t h, size_t slot_size) { ABSL_SWISSTABLE_ASSERT(!c.is_small()); DoSanitizeOnSetCtrl(c, i, h, slot_size); - SetCtrlNoSanitizeImpl(c, i, h); + SetCtrlNoSanitizeImpl(cap, ctrl, i, h); } -// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. -inline void SetCtrl(const CommonFields& c, size_t i, h2_t h, size_t slot_size) { - SetCtrl(c, i, static_cast<ctrl_t>(h), slot_size); + +inline void SetCtrl(const CommonFields& c, size_t cap, ctrl_t* ctrl, size_t i, + h2_t h, size_t slot_size) { + SetCtrl(c, cap, ctrl, i, static_cast<ctrl_t>(h), slot_size); } // Sets `ctrl[i]` to `ctrl_t::kSentinel`. // // Unlike setting it directly, this function will perform bounds checks and // mirror the value to the cloned tail if necessary. -inline void BlockCtrl(const CommonFields& c, size_t i) { - ABSL_SWISSTABLE_ASSERT(!c.is_small()); - SetCtrlNoSanitizeImpl(c, i, ctrl_t::kSentinel); +inline void BlockCtrl(size_t cap, ctrl_t* ctrl, size_t i) { + ABSL_SWISSTABLE_ASSERT(!IsSmallCapacity(cap)); + SetCtrlNoSanitizeImpl(cap, ctrl, i, ctrl_t::kSentinel); } // Like SetCtrl, but in a single group table, we can save some operations when // setting the cloned control byte. -inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, ctrl_t h, +inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t cap, + ctrl_t* ctrl, size_t i, ctrl_t h, size_t slot_size) { - const size_t cap = c.capacity(); ABSL_SWISSTABLE_ASSERT(!c.is_small()); + ABSL_SWISSTABLE_ASSERT(cap == c.capacity()); + ABSL_SWISSTABLE_ASSERT(ctrl == c.control()); ABSL_SWISSTABLE_ASSERT(is_single_group(cap)); DoSanitizeOnSetCtrl(c, i, h, slot_size); - ctrl_t* ctrl = c.control(); ctrl[i] = h; ctrl[i + cap + 1] = h; } // Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. -inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, h2_t h, +inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t cap, + ctrl_t* ctrl, size_t i, h2_t h, size_t slot_size) { - SetCtrlInSingleGroupTable(c, i, static_cast<ctrl_t>(h), slot_size); + SetCtrlInSingleGroupTable(c, cap, ctrl, i, static_cast<ctrl_t>(h), slot_size); } -// Like SetCtrl, but in a table with capacity >= Group::kWidth - 1, -// we can save some operations when setting the cloned control byte. -inline void SetCtrlInLargeTable(const CommonFields& c, size_t i, ctrl_t h, - size_t slot_size) { - ABSL_SWISSTABLE_ASSERT(c.capacity() >= Group::kWidth - 1); - DoSanitizeOnSetCtrl(c, i, h, slot_size); - ctrl_t* ctrl = c.control(); - ctrl[i] = h; - ctrl[((i - NumClonedBytes()) & c.capacity()) + NumClonedBytes()] = h; -} -// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. -inline void SetCtrlInLargeTable(const CommonFields& c, size_t i, h2_t h, - size_t slot_size) { - SetCtrlInLargeTable(c, i, static_cast<ctrl_t>(h), slot_size); -} - -void BlockControlBytes(CommonFields& common, size_t blocked_element_count) { - const size_t capacity = common.capacity(); +void BlockControlBytes(size_t capacity, ctrl_t* ctrl, + size_t blocked_element_count) { + ABSL_ASSUME(blocked_element_count <= + HashtableInlineData::kMaxBlockedElementCount); while (blocked_element_count > 0) { - BlockCtrl(common, capacity - blocked_element_count); + BlockCtrl(capacity, ctrl, capacity - blocked_element_count); --blocked_element_count; } } void* DropDeletesWithoutResizeAndPrepareInsert( CommonFields& common, const PolicyFunctions& __restrict policy, - size_t new_hash) { + size_t new_hash, size_t capacity) { void* set = &common; - const size_t capacity = common.capacity(); + ABSL_SWISSTABLE_ASSERT(common.capacity() == capacity); ABSL_SWISSTABLE_ASSERT(IsValidCapacity(capacity)); ABSL_SWISSTABLE_ASSERT(!is_single_group(capacity)); ABSL_ASSUME(capacity > kMaxSmallCapacity); @@ -600,7 +607,7 @@ // repeat procedure for current slot with moved from element (target) const size_t blocked_element_count = common.blocked_element_count(); ConvertDeletedToEmptyAndFullToDeleted(ctrl, capacity); - BlockControlBytes(common, blocked_element_count); + BlockControlBytes(capacity, ctrl, blocked_element_count); const void* hash_fn = policy.hash_fn(common); auto hasher = policy.hash_slot; auto transfer_n = policy.transfer_n; @@ -623,14 +630,14 @@ } if (!IsDeleted(ctrl[i])) continue; const size_t hash = (*hasher)(hash_fn, slot_ptr, common.seed().seed()); - const FindInfo target = find_first_non_full(common, hash); + const FindInfo target = find_first_non_full(capacity, ctrl, hash); const size_t new_i = target.offset; total_probe_length += target.probe_length; // Verify if the old and new i fall within the same group wrt the hash. // If they do, we don't need to move the object as it falls already in the // best probe we can. - const size_t probe_offset = probe(common, hash).offset(); + const size_t probe_offset = probe(ProbeCapacity{capacity}, hash).offset(); const h2_t h2 = H2(hash); const auto probe_index = [probe_offset, capacity](size_t pos) { return ((pos - probe_offset) & capacity) / Group::kWidth; @@ -638,7 +645,7 @@ // Element doesn't move. if (ABSL_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { - SetCtrlInLargeTable(common, i, h2, slot_size); + SetCtrl(common, capacity, ctrl, i, h2, slot_size); continue; } @@ -647,14 +654,14 @@ // Transfer element to the empty spot. // SetCtrl poisons/unpoisons the slots so we have to call it at the // right time. - SetCtrlInLargeTable(common, new_i, h2, slot_size); + SetCtrl(common, capacity, ctrl, new_i, h2, slot_size); (*transfer_n)(set, new_slot_ptr, slot_ptr, 1); - SetCtrlInLargeTable(common, i, ctrl_t::kEmpty, slot_size); + SetCtrl(common, capacity, ctrl, i, ctrl_t::kEmpty, slot_size); // Initialize or change empty space id. tmp_space_id = i; } else { ABSL_SWISSTABLE_ASSERT(IsDeleted(ctrl[new_i])); - SetCtrlInLargeTable(common, new_i, h2, slot_size); + SetCtrl(common, capacity, ctrl, new_i, h2, slot_size); // Until we are done rehashing, DELETED marks previously FULL slots. if (tmp_space_id == kUnknownId) { @@ -678,20 +685,20 @@ // Prepare insert for the new element. PrepareInsertCommon(common); ResetGrowthLeft(capacity, common.size() + blocked_element_count, common); - FindInfo find_info = find_first_non_full(common, new_hash); - SetCtrlInLargeTable(common, find_info.offset, H2(new_hash), slot_size); + FindInfo find_info = find_first_non_full(capacity, ctrl, new_hash); + SetCtrl(common, capacity, ctrl, find_info.offset, H2(new_hash), slot_size); common.infoz().RecordInsertMiss(new_hash, find_info.probe_length); common.infoz().RecordRehash(total_probe_length); return SlotAddress(slot_array, find_info.offset, slot_size); } -bool WasNeverFull(CommonFields& c, size_t index) { - if (is_single_group(c.capacity())) { +bool WasNeverFull(size_t capacity, ctrl_t* ctrl, size_t index) { + if (is_single_group(capacity)) { return true; } - const size_t index_before = (index - Group::kWidth) & c.capacity(); - const auto empty_after = Group(c.control() + index).MaskEmpty(); - const auto empty_before = Group(c.control() + index_before).MaskEmpty(); + const size_t index_before = (index - Group::kWidth) & capacity; + const auto empty_after = Group(ctrl + index).MaskEmpty(); + const auto empty_before = Group(ctrl + index_before).MaskEmpty(); // We count how many consecutive non empties we have to the right and to the // left of `it`. If the sum is >= kWidth then there is at least one probe @@ -706,12 +713,10 @@ // control bytes are kEmpty except for the kSentinel bytes. // If the table has blocked elements, last `blocked_element_count` are set to // kSentinel. -void ResetCtrl(CommonFields& common, size_t slot_size, - size_t blocked_element_count) { - ABSL_SWISSTABLE_ASSERT(IsCapacityValidForBlockedElements(common.capacity()) || +void ResetCtrl(size_t capacity, ctrl_t* ctrl, void* slot_array, + size_t slot_size, size_t blocked_element_count) { + ABSL_SWISSTABLE_ASSERT(IsCapacityValidForBlockedElements(capacity) || blocked_element_count == 0); - const size_t capacity = common.capacity(); - ctrl_t* ctrl = common.control(); static constexpr size_t kTwoGroupCapacity = 2 * Group::kWidth - 1; if (ABSL_PREDICT_TRUE(capacity <= kTwoGroupCapacity)) { if (IsSmallCapacity(capacity)) return; @@ -727,9 +732,9 @@ capacity + 1 + NumClonedBytes()); } ctrl[capacity] = ctrl_t::kSentinel; - SanitizerPoisonMemoryRegion(common.slot_array(capacity), + SanitizerPoisonMemoryRegion(slot_array, slot_size * (capacity - blocked_element_count)); - BlockControlBytes(common, blocked_element_count); + BlockControlBytes(capacity, ctrl, blocked_element_count); } // Initializes control bytes for growing from capacity 1 to 3. @@ -880,14 +885,16 @@ c.decrement_size(); c.infoz().RecordErase(); - if (WasNeverFull(c, index)) { - SetCtrl(c, index, ctrl_t::kEmpty, slot_size); + const size_t cap = c.capacity(); + ctrl_t* ctrl = c.control(); + if (WasNeverFull(cap, ctrl, index)) { + SetCtrl(c, cap, ctrl, index, ctrl_t::kEmpty, slot_size); c.OverwriteFullAsEmpty(); return; } c.OverwriteFullAsDeleted(); - SetCtrlInLargeTable(c, index, ctrl_t::kDeleted, slot_size); + SetCtrl(c, cap, ctrl, index, ctrl_t::kDeleted, slot_size); } void ClearBackingArray(CommonFields& c, @@ -897,9 +904,12 @@ if (reuse) { const size_t blocked_element_count = c.blocked_element_count(); c.set_size_to_zero(); - ABSL_SWISSTABLE_ASSERT(c.capacity() > policy.soo_capacity()); - ResetCtrl(c, policy.slot_size, blocked_element_count); - ResetGrowthLeft(c.capacity(), blocked_element_count, c); + size_t cap = c.capacity(); + ctrl_t* ctrl = c.control(); + void* slot_array = c.slot_array(cap); + ABSL_SWISSTABLE_ASSERT(cap > policy.soo_capacity()); + ResetCtrl(cap, ctrl, slot_array, policy.slot_size, blocked_element_count); + ResetGrowthLeft(cap, blocked_element_count, c); ABSL_SWISSTABLE_ASSERT(c.blocked_element_count() == blocked_element_count); c.infoz().RecordStorageChanged(0, c.capacity()); } else { @@ -1031,8 +1041,10 @@ if (common.is_small()) { target = FindInfo{0, 0}; } else { - target = find_first_non_full(common, hash); - SetCtrl(common, target.offset, H2(hash), slot_size); + const size_t cap = common.capacity(); + ctrl_t* ctrl = common.control(); + target = find_first_non_full(cap, ctrl, hash); + SetCtrl(common, cap, ctrl, target.offset, H2(hash), slot_size); } policy.transfer_n(&common, SlotAddress(new_slots, target.offset, slot_size), slot, 1); @@ -1103,18 +1115,23 @@ probe_length); } -// Outlines cold infoz recording with computation of probe_length so callers can -// skip tracking of probe_length. `common` is first and `hash` is third to match -// argument order in callers. -ABSL_ATTRIBUTE_COLD ABSL_ATTRIBUTE_NOINLINE void -ReportInsertMissToInfozAndComputeProbeLength(CommonFields& common, - size_t target, size_t hash) { +// Outlines cold infoz recording with computation of probe_length and slot +// address so callers can skip tracking probe_length and tail-call without +// keeping the slot address live across a call (which would cost a callee-saved +// register and a stack frame on every insert). +// (Returning a value passed in by the caller doesn't work: the compiler infers +// that and keeps the caller's copy alive across the call instead.) +ABSL_ATTRIBUTE_COLD ABSL_ATTRIBUTE_NOINLINE void* +ReportInsertMissToInfozAndGetSlot(CommonFields& common, + const PolicyFunctions& __restrict policy, + size_t hash, size_t target) { const size_t cap = common.capacity(); auto seq = probe(ProbeCapacity{cap}, hash); while (((target - seq.offset()) & cap) >= Group::kWidth) { seq.next(); } common.infoz().RecordInsertMiss(hash, seq.index()); + return SlotAddress(common.slot_array(cap), target, policy.slot_size); } ABSL_ATTRIBUTE_COLD ABSL_ATTRIBUTE_NOINLINE void ReportGrowthToInfoz( @@ -1153,10 +1170,8 @@ BackingArrayPtrs AllocBackingArray(CommonFields& common, const PolicyFunctions& __restrict policy, - size_t new_capacity, bool has_infoz, - void* alloc, size_t blocked_element_count) { - RawHashSetLayout layout(new_capacity, policy.slot_size, policy.slot_align, - has_infoz, blocked_element_count); + const RawHashSetLayout& layout, + void* alloc) { // Perform a direct call in the common case to allow for profile-guided // heap optimization (PGHO) to understand which allocation function is used. constexpr size_t kDefaultAlignment = BackingArrayAlignment(alignof(size_t)); @@ -1176,6 +1191,17 @@ mem + layout.slot_offset()}; } +BackingArrayPtrs AllocBackingArray(CommonFields& common, + const PolicyFunctions& __restrict policy, + size_t new_capacity, bool has_infoz, + void* alloc, size_t blocked_element_count) { + return AllocBackingArray( + common, policy, + RawHashSetLayout(new_capacity, policy.slot_size, policy.slot_align, + has_infoz, blocked_element_count), + alloc); +} + void ResizeEmptyNonAllocatedTableImpl(CommonFields& common, const PolicyFunctions& __restrict policy, size_t new_capacity, @@ -1198,7 +1224,8 @@ common.set_control(new_ctrl); common.generate_new_seed(has_infoz); - ResetCtrl(common, slot_size, blocked_element_count); + ResetCtrl(new_capacity, new_ctrl, new_slots, slot_size, + blocked_element_count); ResetGrowthLeft(new_capacity, blocked_element_count, common); if (ABSL_PREDICT_FALSE(has_infoz)) { @@ -1228,8 +1255,10 @@ SanitizerUnpoisonMemoryRegion(target_slot, policy.slot_size); policy.transfer_n(&c, target_slot, c.soo_data(), 1); c.set_control(new_ctrl); - ResetCtrl(c, policy.slot_size, /*blocked_element_count=*/0); - SetCtrl(c, offset, H2(soo_slot_hash), policy.slot_size); + ResetCtrl(new_capacity, new_ctrl, new_slots, policy.slot_size, + /*blocked_element_count=*/0); + SetCtrl(c, new_capacity, new_ctrl, offset, H2(soo_slot_hash), + policy.slot_size); } enum class ResizeFullSooTableSamplingMode { @@ -1447,9 +1476,9 @@ CommonFields& c, const PolicyFunctions& __restrict policy, const ProbedItem* start, const ProbedItem* end, void* old_slots) { ABSL_SWISSTABLE_ASSERT(!c.is_small()); - const HashtableCapacity new_capacity = c.capacity_impl(); + const size_t new_capacity = c.capacity(); - void* new_slots = c.slot_array(new_capacity.capacity()); + void* new_slots = c.slot_array(new_capacity); ctrl_t* new_ctrl = c.control(); size_t total_probe_length = 0; @@ -1458,18 +1487,19 @@ for (; start < end; ++start) { const FindInfo target = find_first_non_full_from_h1( - new_ctrl, static_cast<size_t>(start->h1), new_capacity); + new_capacity, new_ctrl, static_cast<size_t>(start->h1)); total_probe_length += target.probe_length; const size_t old_index = static_cast<size_t>(start->source_offset); const size_t new_i = target.offset; - ABSL_SWISSTABLE_ASSERT(old_index < new_capacity.capacity() / 2); - ABSL_SWISSTABLE_ASSERT(new_i < new_capacity.capacity()); + ABSL_SWISSTABLE_ASSERT(old_index < new_capacity / 2); + ABSL_SWISSTABLE_ASSERT(new_i < new_capacity); ABSL_SWISSTABLE_ASSERT(IsEmpty(new_ctrl[new_i])); void* src_slot = SlotAddress(old_slots, old_index, slot_size); void* dst_slot = SlotAddress(new_slots, new_i, slot_size); SanitizerUnpoisonMemoryRegion(dst_slot, slot_size); transfer_n(&c, dst_slot, src_slot, 1); - SetCtrlInLargeTable(c, new_i, static_cast<h2_t>(start->h2), slot_size); + SetCtrl(c, new_capacity, new_ctrl, new_i, static_cast<h2_t>(start->h2), + slot_size); } return total_probe_length; } @@ -1485,10 +1515,9 @@ ABSL_ATTRIBUTE_NOINLINE size_t ProcessProbedMarkedElements( CommonFields& c, const PolicyFunctions& __restrict policy, ctrl_t* old_ctrl, void* old_slots, size_t start) { - const size_t new_capacity = c.capacity(); - const size_t old_capacity = PreviousCapacity(new_capacity); + const size_t old_capacity = PreviousCapacity(c.capacity()); const size_t slot_size = policy.slot_size; - void* new_slots = c.slot_array(new_capacity); + void* new_slots = c.slot_array(c.capacity()); size_t total_probe_length = 0; const void* hash_fn = policy.hash_fn(c); auto hash_slot = policy.hash_slot; @@ -1500,11 +1529,13 @@ } void* src_slot = SlotAddress(old_slots, old_index, slot_size); const size_t hash = hash_slot(hash_fn, src_slot, seed); - const FindInfo target = find_first_non_full(c, hash); + const size_t new_capacity = c.capacity(); + ctrl_t* new_ctrl = c.control(); + const FindInfo target = find_first_non_full(new_capacity, new_ctrl, hash); total_probe_length += target.probe_length; const size_t new_i = target.offset; void* dst_slot = SlotAddress(new_slots, new_i, slot_size); - SetCtrlInLargeTable(c, new_i, H2(hash), slot_size); + SetCtrl(c, new_capacity, new_ctrl, new_i, H2(hash), slot_size); transfer_n(&c, dst_slot, src_slot, 1); } return total_probe_length; @@ -1677,17 +1708,19 @@ template <typename Encoder> size_t GrowToNextCapacity(CommonFields& common, const PolicyFunctions& __restrict policy, - ctrl_t* old_ctrl, void* old_slots) { + ctrl_t* old_ctrl, void* old_slots, + size_t new_capacity) { using ProbedItem = typename Encoder::ProbedItem; - ABSL_SWISSTABLE_ASSERT(common.capacity() <= ProbedItem::kMaxNewCapacity); + ABSL_SWISSTABLE_ASSERT(common.capacity() == new_capacity); + ABSL_SWISSTABLE_ASSERT(new_capacity <= ProbedItem::kMaxNewCapacity); Encoder encoder(old_ctrl); policy.transfer_unprobed_elements_to_next_capacity( - common, old_ctrl, old_slots, &encoder, - [](void* probed_storage, h2_t h2, size_t source_offset, size_t h1) { + common, old_ctrl, &encoder, old_slots, PreviousCapacity(new_capacity), + [](void* probed_storage, size_t hash, size_t source_offset) { auto encoder_ptr = static_cast<Encoder*>(probed_storage); - encoder_ptr->EncodeItem(ProbedItem(h2, source_offset, h1)); + encoder_ptr->EncodeItem(ProbedItem(H2(hash), source_offset, H1(hash))); }); - InitializeMirroredControlBytes(common.control(), common.capacity()); + InitializeMirroredControlBytes(common.control(), new_capacity); return encoder.DecodeAndInsertToTable(common, policy, old_slots); } @@ -1696,32 +1729,34 @@ // Returns total probe length. size_t GrowToNextCapacityThatFitsInLocalBuffer( CommonFields& common, const PolicyFunctions& __restrict policy, - ctrl_t* old_ctrl, void* old_slots) { - ABSL_SWISSTABLE_ASSERT(common.capacity() <= kMaxLocalBufferNewCapacity); + ctrl_t* old_ctrl, void* old_slots, size_t new_capacity) { + ABSL_SWISSTABLE_ASSERT(new_capacity <= kMaxLocalBufferNewCapacity); return GrowToNextCapacity< ProbedItemEncoder<ProbedItem4Bytes, /*kGuaranteedFitToBuffer=*/true>>( - common, policy, old_ctrl, old_slots); + common, policy, old_ctrl, old_slots, new_capacity); } // Grows to next capacity with different encodings. Returns total probe length. // These functions are useful to simplify profile analysis. size_t GrowToNextCapacity4BytesEncoder(CommonFields& common, const PolicyFunctions& __restrict policy, - ctrl_t* old_ctrl, void* old_slots) { + ctrl_t* old_ctrl, void* old_slots, + size_t new_capacity) { return GrowToNextCapacity<ProbedItemEncoder<ProbedItem4Bytes>>( - common, policy, old_ctrl, old_slots); + common, policy, old_ctrl, old_slots, new_capacity); } size_t GrowToNextCapacity8BytesEncoder(CommonFields& common, const PolicyFunctions& __restrict policy, - ctrl_t* old_ctrl, void* old_slots) { + ctrl_t* old_ctrl, void* old_slots, + size_t new_capacity) { return GrowToNextCapacity<ProbedItemEncoder<ProbedItem8Bytes>>( - common, policy, old_ctrl, old_slots); + common, policy, old_ctrl, old_slots, new_capacity); } size_t GrowToNextCapacity16BytesEncoder( CommonFields& common, const PolicyFunctions& __restrict policy, - ctrl_t* old_ctrl, void* old_slots) { + ctrl_t* old_ctrl, void* old_slots, size_t new_capacity) { return GrowToNextCapacity<ProbedItemEncoder<ProbedItem16Bytes>>( - common, policy, old_ctrl, old_slots); + common, policy, old_ctrl, old_slots, new_capacity); } // Grows to next capacity for tables with relatively large capacity so that we @@ -1729,16 +1764,19 @@ // total probe length. size_t GrowToNextCapacityOverflowLocalBuffer( CommonFields& common, const PolicyFunctions& __restrict policy, - ctrl_t* old_ctrl, void* old_slots) { - const size_t new_capacity = common.capacity(); + ctrl_t* old_ctrl, void* old_slots, size_t new_capacity) { + ABSL_SWISSTABLE_ASSERT(common.capacity() == new_capacity); if (ABSL_PREDICT_TRUE(new_capacity <= ProbedItem4Bytes::kMaxNewCapacity)) { - return GrowToNextCapacity4BytesEncoder(common, policy, old_ctrl, old_slots); + return GrowToNextCapacity4BytesEncoder(common, policy, old_ctrl, old_slots, + new_capacity); } if (ABSL_PREDICT_TRUE(new_capacity <= ProbedItem8Bytes::kMaxNewCapacity)) { - return GrowToNextCapacity8BytesEncoder(common, policy, old_ctrl, old_slots); + return GrowToNextCapacity8BytesEncoder(common, policy, old_ctrl, old_slots, + new_capacity); } // 16 bytes encoding supports the maximum swisstable capacity. - return GrowToNextCapacity16BytesEncoder(common, policy, old_ctrl, old_slots); + return GrowToNextCapacity16BytesEncoder(common, policy, old_ctrl, old_slots, + new_capacity); } // Dispatches to the appropriate `GrowToNextCapacity*` function based on the @@ -1746,14 +1784,15 @@ ABSL_ATTRIBUTE_NOINLINE size_t GrowToNextCapacityDispatch(CommonFields& common, const PolicyFunctions& __restrict policy, - ctrl_t* old_ctrl, void* old_slots) { - const size_t new_capacity = common.capacity(); + ctrl_t* old_ctrl, void* old_slots, + size_t new_capacity) { + ABSL_SWISSTABLE_ASSERT(common.capacity() == new_capacity); if (ABSL_PREDICT_TRUE(new_capacity <= kMaxLocalBufferNewCapacity)) { return GrowToNextCapacityThatFitsInLocalBuffer(common, policy, old_ctrl, - old_slots); + old_slots, new_capacity); } else { return GrowToNextCapacityOverflowLocalBuffer(common, policy, old_ctrl, - old_slots); + old_slots, new_capacity); } } @@ -1777,7 +1816,8 @@ void* Grow1To3AndPrepareInsert(CommonFields& common, const PolicyFunctions& __restrict policy, - void* hasher, HashForKeyFn get_hash) { + void* hasher, HashForKeyFn get_hash, void* alloc, + bool has_infoz, void* new_slots) { // TODO(b/413062340): Refactor to reuse more code with // GrowSooTableToNextCapacityAndPrepareInsert. ABSL_SWISSTABLE_ASSERT(common.capacity() == 1); @@ -1787,21 +1827,8 @@ ABSL_SWISSTABLE_ASSERT(common.blocked_element_count() == 0); constexpr size_t kOldCapacity = 1; constexpr size_t kNewCapacity = NextCapacity(kOldCapacity); - void* old_slots = common.single_non_soo_slot(); - // old_slots == old_ctrl in case of capacity == 1. - ctrl_t* old_ctrl = static_cast<ctrl_t*>(old_slots); - - const size_t slot_size = policy.slot_size; - const size_t slot_align = policy.slot_align; - void* alloc = policy.get_char_alloc(common); - const bool has_infoz = common.has_infoz(); - common.set_capacity(kNewCapacity); - - const auto [new_ctrl, new_slots] = - AllocBackingArray(common, policy, kNewCapacity, has_infoz, alloc, - /*blocked_element_count=*/0); - common.set_control(new_ctrl); - SanitizerPoisonMemoryRegion(new_slots, kNewCapacity * slot_size); + ctrl_t* new_ctrl = + static_cast<ctrl_t*>(new_slots) - NumControlBytes(kNewCapacity); if (ABSL_PREDICT_TRUE(!has_infoz)) { // When we're sampled, we already have a seed. @@ -1809,29 +1836,45 @@ } const size_t new_hash = get_hash(hasher, common.seed().seed()); h2_t new_h2 = H2(new_hash); - size_t orig_hash = - policy.hash_slot(policy.hash_fn(common), old_slots, common.seed().seed()); + // `hash_fn` is in separate statement, so the hash_slot function pointer is + // not loaded before the hash_fn call. + const void* hash_fn = policy.hash_fn(common); + size_t orig_hash = policy.hash_slot( + hash_fn, common.single_non_soo_slot(), common.seed().seed()); size_t offset = Resize1To3NewOffset(new_hash); InitializeThreeElementsControlBytes(H2(orig_hash), new_h2, offset, new_ctrl); + // Read `old_slots` and `slot_size` after the hash calls to avoid keeping + // them live across indirect calls. + void* old_slots = common.single_non_soo_slot(); + // old_slots == old_ctrl in case of capacity == 1. + ctrl_t* old_ctrl = static_cast<ctrl_t*>(old_slots); + common.set_control(new_ctrl); + const size_t slot_size = policy.slot_size; + SanitizerPoisonMemoryRegion(new_slots, kNewCapacity * slot_size); + void* old_element_target = NextSlot(new_slots, slot_size); + // Compute `new_element_target_slot` before `transfer_n` to free up registers + // of `new_slots` and `offset`. + void* new_element_target_slot = SlotAddress(new_slots, offset, slot_size); SanitizerUnpoisonMemoryRegion(old_element_target, slot_size); policy.transfer_n(&common, old_element_target, old_slots, 1); - - void* new_element_target_slot = SlotAddress(new_slots, offset, slot_size); SanitizerUnpoisonMemoryRegion(new_element_target_slot, slot_size); + // Group `common` updates together so `inline_data_` is modified in a single + // load-store sequence. + common.set_capacity(kNewCapacity); + common.InitGrowthLeftNoDeleted(kNewCapacity - 2, kNewCapacity); PrepareInsertCommon(common); ABSL_SWISSTABLE_ASSERT(common.size() == 2); - common.InitGrowthLeftNoDeleted(kNewCapacity - 2, kNewCapacity); if (ABSL_PREDICT_FALSE(has_infoz)) { ReportSingleGroupTableGrowthToInfoz(common, old_ctrl, new_hash); } // Deallocate after `ReportSingleGroupTableGrowthToInfoz`, because it reads // infoz from old_ctrl. - DeallocBackingArrayImpl(alloc, kOldCapacity, old_ctrl, slot_size, slot_align, - has_infoz, + DeallocBackingArrayImpl(alloc, kOldCapacity, old_ctrl, slot_size, + policy.slot_align, has_infoz, /*blocked_element_count=*/0, policy.dealloc); return new_element_target_slot; } @@ -1840,30 +1883,37 @@ // Returns the offset of the new element. void* GrowToNextCapacityAndPrepareInsert( CommonFields& common, const PolicyFunctions& __restrict policy, - size_t new_hash) { - const size_t old_capacity = common.capacity(); + size_t new_hash, size_t old_capacity) { + ABSL_SWISSTABLE_ASSERT(common.capacity() == old_capacity); ABSL_SWISSTABLE_ASSERT(common.GetGrowthLeftTotalSlow(old_capacity) == 0); ABSL_SWISSTABLE_ASSERT(old_capacity > policy.soo_capacity()); ABSL_SWISSTABLE_ASSERT(!IsSmallCapacity(old_capacity)); ABSL_ASSUME(old_capacity > kMaxSmallCapacity); + // Call `get_char_alloc` first so no local variables need to be spilled + // across the indirect call, which may alias `common`. + void* alloc = policy.get_char_alloc(common); const size_t new_capacity = NextCapacity(old_capacity); ABSL_ASSUME(new_capacity > kMaxSmallCapacity); + + const auto [new_ctrl, new_slots] = + AllocBackingArray(common, policy, new_capacity, common.has_infoz(), alloc, + /*blocked_element_count=*/0); + // Read old `common` fields after `AllocBackingArray` to avoid spilling them + // across the call. ctrl_t* old_ctrl = common.control(); void* old_slots = common.slot_array(old_capacity); size_t old_blocked_element_count = common.blocked_element_count(); + const size_t old_size = common.size(); - const bool has_infoz = common.has_infoz(); + // Group all `common` updates together so `inline_data_` is modified in a + // single load-store sequence. common.set_capacity(new_capacity); common.set_blocked_element_count_to_zero(); - const size_t slot_size = policy.slot_size; - const size_t slot_align = policy.slot_align; - void* alloc = policy.get_char_alloc(common); - - const auto [new_ctrl, new_slots] = - AllocBackingArray(common, policy, new_capacity, has_infoz, alloc, - /*blocked_element_count=*/0); common.set_control(new_ctrl); + PrepareInsertCommon(common); + ResetGrowthLeft(new_capacity, old_size + 1, common); + const size_t slot_size = policy.slot_size; SanitizerPoisonMemoryRegion(new_slots, new_capacity * slot_size); h2_t new_h2 = H2(new_hash); @@ -1871,7 +1921,6 @@ FindInfo find_info; if (ABSL_PREDICT_TRUE(is_single_group(new_capacity))) { size_t offset; - const size_t old_size = common.size(); GrowIntoSingleGroupShuffleControlBytes(old_ctrl, old_capacity, old_blocked_element_count, new_ctrl, new_capacity); @@ -1880,82 +1929,121 @@ offset = new_hash & 1 ? 0 : new_capacity - 1; ABSL_SWISSTABLE_ASSERT(IsEmpty(new_ctrl[offset])); - SetCtrlInSingleGroupTable(common, offset, new_h2, policy.slot_size); + SetCtrlInSingleGroupTable(common, new_capacity, new_ctrl, offset, new_h2, + slot_size); find_info = FindInfo{offset, 0}; // Single group tables have all slots full on resize. So we can transfer // all slots without checking the control bytes. - ABSL_SWISSTABLE_ASSERT(common.size() + old_blocked_element_count == + ABSL_SWISSTABLE_ASSERT(old_size + old_blocked_element_count == old_capacity); void* target = NextSlot(new_slots, slot_size); SanitizerUnpoisonMemoryRegion(target, old_size * slot_size); policy.transfer_n(&common, target, old_slots, old_size); } else { - total_probe_length = - GrowToNextCapacityDispatch(common, policy, old_ctrl, old_slots); - find_info = find_first_non_full(common, new_hash); - SetCtrlInLargeTable(common, find_info.offset, new_h2, policy.slot_size); + total_probe_length = GrowToNextCapacityDispatch(common, policy, old_ctrl, + old_slots, new_capacity); + find_info = find_first_non_full(new_capacity, new_ctrl, new_hash); + SetCtrl(common, new_capacity, new_ctrl, find_info.offset, new_h2, + slot_size); } ABSL_SWISSTABLE_ASSERT(old_capacity > policy.soo_capacity()); - PrepareInsertCommon(common); - ResetGrowthLeft(new_capacity, common.size(), common); - + // Reload `has_infoz` to avoid spilling across the indirect call. + const bool has_infoz = common.has_infoz(); if (ABSL_PREDICT_FALSE(has_infoz)) { ReportGrowthToInfoz(common, old_ctrl, new_hash, total_probe_length, find_info.probe_length); } // Deallocate after `ReportGrowthToInfoz`, because it reads infoz from // old_ctrl. - DeallocBackingArrayImpl(alloc, old_capacity, old_ctrl, slot_size, slot_align, - has_infoz, old_blocked_element_count, policy.dealloc); - return SlotAddress(new_slots, find_info.offset, policy.slot_size); + DeallocBackingArrayImpl(alloc, old_capacity, old_ctrl, slot_size, + policy.slot_align, has_infoz, + old_blocked_element_count, policy.dealloc); + return SlotAddress(new_slots, find_info.offset, slot_size); } } // namespace +RawHashSetLayout RawHashSetLayout::CreateForSmallNonSooGrowth(size_t size, + size_t slot_size, + size_t slot_align, + bool has_infoz) { + ABSL_SWISSTABLE_ASSERT(size <= 1); + static_assert(NumControlBytes(1) == 0); + const size_t num_control_bytes = size * NumControlBytes(3); + static_assert(NextCapacity(0) == 1); + static_assert(NextCapacity(1) == 3); + const size_t new_capacity = NextCapacity(size); + const size_t slots_size = new_capacity * slot_size; + + // Since `GrowthInfoSizeForCapacity` is 0 for capacities 1 and 3, compute + // `metadata_size` directly without `MetadataBeforeControlSize` to avoid its + // `ABSL_PREDICT_FALSE(has_infoz)` branch and allow a branchless shift. + static_assert(GrowthInfoSizeForCapacity(1) == 0); + static_assert(GrowthInfoSizeForCapacity(3) == 0); + const size_t metadata_size = + (has_infoz ? sizeof(HashtablezInfoHandle) + sizeof(uint64_t) : 0) + + NumGenerationBytes(); + ABSL_SWISSTABLE_ASSERT(metadata_size == + MetadataBeforeControlSize(has_infoz, new_capacity)); + + RawHashSetLayout layout(uninitialized_tag_t{}); + layout.slot_offset_ = + AlignUpTo(metadata_size + num_control_bytes, slot_align); + layout.control_offset_ = layout.slot_offset_ - num_control_bytes; + layout.generation_offset_ = layout.control_offset_ - NumGenerationBytes(); + layout.alloc_size_ = layout.slot_offset_ + slots_size; + return layout; +} + void* PrepareInsertSmallNonSoo(CommonFields& common, const PolicyFunctions& __restrict policy, void* hasher, HashForKeyFn get_hash) { ABSL_SWISSTABLE_ASSERT(common.is_small()); ABSL_SWISSTABLE_ASSERT(!policy.soo_enabled); - if (common.capacity() == 1) { - if (common.empty()) { - IncrementSmallSizeNonSoo(common, policy); - // Compute before `RecordInsertMissCold` to avoid reloading `control_`. - void* res = common.single_non_soo_slot(); - // Call NOINLINE function to move infoz instructions out of line. - if (common.has_infoz()) { - ReportInsertMissToInfozAndComputeHash(common, /*probe_length=*/0, - hasher, get_hash); - } - return res; - } else { - return Grow1To3AndPrepareInsert(common, policy, hasher, get_hash); + const bool is_empty = common.empty(); + if (common.capacity() == 1 && is_empty) { + IncrementSmallSizeNonSoo(common, policy); + // Compute before `RecordInsertMissCold` to avoid reloading `control_`. + void* res = common.single_non_soo_slot(); + // Call NOINLINE function to move infoz instructions out of line. + if (common.has_infoz()) { + ReportInsertMissToInfozAndComputeHash(common, /*probe_length=*/0, hasher, + get_hash); } + return res; } - // Growing from 0 to 1 capacity. - ABSL_SWISSTABLE_ASSERT(common.capacity() == 0); - constexpr size_t kNewCapacity = 1; - - common.set_capacity(kNewCapacity); + const size_t size = common.size(); const bool has_infoz = - policy.is_hashtablez_enabled && ShouldSampleNextTable(); + is_empty ? (policy.is_hashtablez_enabled && ShouldSampleNextTable()) + : common.has_infoz(); void* alloc = policy.get_char_alloc(common); + void* new_slots = + AllocBackingArray( + common, policy, + RawHashSetLayout::CreateForSmallNonSooGrowth( + size, policy.slot_size, policy.slot_align, has_infoz), + alloc) + .slots; + if (is_empty) { + // Growing from 0 to 1 capacity. + ABSL_SWISSTABLE_ASSERT(common.capacity() == 0); + constexpr size_t kNewCapacity = 1; + common.set_capacity(kNewCapacity); + common.set_control(static_cast<ctrl_t*>(new_slots)); - const auto [new_ctrl, new_slots] = - AllocBackingArray(common, policy, kNewCapacity, has_infoz, alloc, - /*blocked_element_count=*/0); - common.set_control(new_ctrl); + static_assert(NextCapacity(0) == 1); + PrepareInsertCommon(common); - static_assert(NextCapacity(0) == 1); - PrepareInsertCommon(common); - - if (ABSL_PREDICT_FALSE(has_infoz)) { - ReportSingleElementTableGrowthToInfozAndForceSample(common, policy, hasher, - get_hash); + if (ABSL_PREDICT_FALSE(has_infoz)) { + ReportSingleElementTableGrowthToInfozAndForceSample( + common, policy, hasher, get_hash); + } + return new_slots; } - return new_slots; + return Grow1To3AndPrepareInsert(common, policy, hasher, get_hash, alloc, + has_infoz, new_slots); } namespace { @@ -1965,9 +2053,9 @@ ABSL_ATTRIBUTE_COLD ABSL_ATTRIBUTE_NOINLINE void* RehashOrGrowToNextCapacityAndPrepareInsert( CommonFields& common, const PolicyFunctions& __restrict policy, - size_t new_hash) { + size_t new_hash, size_t cap) { ABSL_SWISSTABLE_ASSERT(!common.GetGrowthInfoLowerBound().HasNoDeleted()); - const size_t cap = common.capacity(); + ABSL_SWISSTABLE_ASSERT(common.capacity() == cap); ABSL_ASSUME(cap > 0); // Do these calculations in 64-bit to avoid overflow. if (common.size() * uint64_t{32} <= @@ -2013,10 +2101,11 @@ // 762 | 149836 0.37 13 | 148559 0.74 190 // 807 | 149736 0.39 14 | 151107 0.39 14 // 852 | 150204 0.42 15 | 151019 0.42 15 - return DropDeletesWithoutResizeAndPrepareInsert(common, policy, new_hash); + return DropDeletesWithoutResizeAndPrepareInsert(common, policy, new_hash, + cap); } else { // Otherwise grow the container. - return GrowToNextCapacityAndPrepareInsert(common, policy, new_hash); + return GrowToNextCapacityAndPrepareInsert(common, policy, new_hash, cap); } } @@ -2033,24 +2122,27 @@ if (ABSL_PREDICT_TRUE( growth_info_lower_bound.HasNoGrowthLeftAndNoDeleted())) { // Table without deleted slots (>95% cases) that needs to be resized. - return GrowToNextCapacityAndPrepareInsert(common, policy, hash); + return GrowToNextCapacityAndPrepareInsert(common, policy, hash, cap); } if (ABSL_PREDICT_FALSE( growth_info_lower_bound.HasNoGrowthLeftAndHaveDeleted())) { // Table with deleted slots that needs to be rehashed or resized. - return RehashOrGrowToNextCapacityAndPrepareInsert(common, policy, hash); + return RehashOrGrowToNextCapacityAndPrepareInsert(common, policy, hash, + cap); } // Covers two cases: // 1. Table with deleted slots that has space for the inserting element. // 2. Table without deleted slots that has space and GrowthInfoView was // rebalanced. - FindInfo target = find_first_non_full(common, hash); + const size_t slot_size = policy.slot_size; + ctrl_t* ctrl = common.control(); + FindInfo target = find_first_non_full(cap, ctrl, hash); + // Read `ctrl` before `PrepareInsertCommon` so `common` updates coalesce. + common.OverwriteControlAsFull(ctrl[target.offset]); PrepareInsertCommon(common); - common.OverwriteControlAsFull(common.control()[target.offset]); // Compute before `SetCtrl*` to avoid reloading `control_`. - void* res = - SlotAddress(common.slot_array(cap), target.offset, policy.slot_size); - SetCtrlInLargeTable(common, target.offset, H2(hash), policy.slot_size); + void* res = SlotAddress(common.slot_array(cap), target.offset, slot_size); + SetCtrl(common, cap, ctrl, target.offset, H2(hash), slot_size); if (common.has_infoz()) { ReportInsertMissToInfoz(common, target.probe_length, hash); } @@ -2077,8 +2169,8 @@ // Compute before `SetCtrl*` to avoid reloading `control_`. void* res = SlotAddress(common.slot_array(kNewCapacity), SooSlotIndex(), policy.slot_size); - SetCtrlInSingleGroupTable(common, SooSlotIndex(), H2(new_hash), - policy.slot_size); + SetCtrlInSingleGroupTable(common, kNewCapacity, common.control(), + SooSlotIndex(), H2(new_hash), policy.slot_size); if (common.has_infoz()) { ReportInsertMissToInfoz(common, /*probe_length=*/0, new_hash); } @@ -2195,7 +2287,8 @@ common.generate_new_seed(has_infoz); size_t total_probe_length = 0; - ResetCtrl(common, slot_size, /*blocked_element_count=*/0); + ResetCtrl(new_capacity, new_ctrl, new_slots, slot_size, + /*blocked_element_count=*/0); ABSL_SWISSTABLE_ASSERT(old_capacity > 0); total_probe_length = FindNewPositionsAndTransferSlots( common, policy, old_ctrl, old_slots, old_capacity); @@ -2221,10 +2314,16 @@ } ABSL_SWISSTABLE_ASSERT(common.size() == policy.soo_capacity()); static constexpr size_t kNewCapacity = NextCapacity(SooCapacity()); - const size_t slot_size = policy.slot_size; - void* alloc = policy.get_char_alloc(common); - common.set_capacity(kNewCapacity); + // Generate the seed and compute hashes before `AllocBackingArray` so + // `hasher`, `get_hash`, and `new_ctrl`/`new_slots` do not need to be spilled + // across calls. + common.generate_new_seed(/*has_infoz=*/false); + const size_t new_hash = get_hash(hasher, common.seed().seed()); + const void* hash_fn = policy.hash_fn(common); + const h2_t soo_slot_h2 = + H2(policy.hash_slot(hash_fn, common.soo_data(), common.seed().seed())); + void* alloc = policy.get_char_alloc(common); // Since the table is not empty, it will not be sampled. // The decision to sample was already made during the first insertion. // @@ -2233,21 +2332,21 @@ const auto [new_ctrl, new_slots] = AllocBackingArray( common, policy, kNewCapacity, /*has_infoz=*/false, alloc, /*blocked_element_count=*/0); - + // Group `common` updates together after `AllocBackingArray` so `inline_data_` + // is modified in a single load-store sequence. + common.set_capacity(kNewCapacity); + common.InitGrowthLeftNoDeleted(kNewCapacity - 2, kNewCapacity); PrepareInsertCommon(common); ABSL_SWISSTABLE_ASSERT(common.size() == 2); - common.InitGrowthLeftNoDeleted(kNewCapacity - 2, kNewCapacity); - common.generate_new_seed(/*has_infoz=*/false); - const h2_t soo_slot_h2 = H2(policy.hash_slot( - policy.hash_fn(common), common.soo_data(), common.seed().seed())); - const size_t new_hash = get_hash(hasher, common.seed().seed()); const size_t offset = Resize1To3NewOffset(new_hash); InitializeThreeElementsControlBytes(soo_slot_h2, H2(new_hash), offset, new_ctrl); + const size_t slot_size = policy.slot_size; SanitizerPoisonMemoryRegion(new_slots, slot_size * kNewCapacity); void* target_slot = SlotAddress(new_slots, SooSlotIndex(), slot_size); + void* new_slot = SlotAddress(new_slots, offset, slot_size); SanitizerUnpoisonMemoryRegion(target_slot, slot_size); if constexpr (TransferUsesMemcpy) { // Target slot is placed at index 1, but capacity is at @@ -2271,7 +2370,6 @@ // Full SOO table couldn't be sampled. If SOO table is sampled, it would // have been resized to the next capacity. ABSL_SWISSTABLE_ASSERT(!common.infoz().IsSampled()); - void* new_slot = SlotAddress(new_slots, offset, slot_size); SanitizerUnpoisonMemoryRegion(new_slot, slot_size); return new_slot; } @@ -2348,9 +2446,10 @@ void Copy(CommonFields& common, const PolicyFunctions& __restrict policy, const CommonFields& other, - absl::FunctionRef<void(void*, const void*)> copy_fn) { + void (*copy_fn)(void* set, void* dst, const void* src)) { + other.AssertNotDebugCapacity(); const size_t size = other.size(); - ABSL_SWISSTABLE_ASSERT(size > 0); + if (size == 0) return; const size_t soo_capacity = policy.soo_capacity(); const size_t slot_size = policy.slot_size; const bool soo_enabled = policy.soo_enabled; @@ -2368,7 +2467,8 @@ : SlotAddress(other.slot_array(other_capacity), FindFirstFullSlot(0, other_capacity, other.control()), slot_size); - copy_fn(soo_enabled ? common.soo_data() + copy_fn(&common, + soo_enabled ? common.soo_data() : SingleSlotAddress</*kSooEnabled=*/false>(common), other_slot); @@ -2396,11 +2496,13 @@ // The table is guaranteed to be empty, so we can do faster than // a full `insert`. const size_t hash = (*hasher)(hash_fn, that_slot, seed); - FindInfo target = find_first_non_full(common, hash); - infoz.RecordInsertMiss(hash, target.probe_length); + ctrl_t* ctrl = common.control(); + FindInfo target = find_first_non_full(cap, ctrl, hash); offset = target.offset; - SetCtrl(common, offset, H2(hash), slot_size); - copy_fn(SlotAddress(target_slot_array, offset, slot_size), that_slot); + SetCtrl(common, cap, ctrl, offset, H2(hash), slot_size); + infoz.RecordInsertMiss(hash, target.probe_length); + copy_fn(&common, SlotAddress(target_slot_array, offset, slot_size), + that_slot); common.maybe_increment_generation_on_insert(); }); common.increment_size(size); @@ -2454,12 +2556,12 @@ // Compute before `SetCtrl` to avoid reloading `control_`. void* res = SlotAddress(common.slot_array(cap), target_group_offset, policy.slot_size); - const bool hash_infoz = common.has_infoz(); - SetCtrl(common, target_group_offset, H2(hash), policy.slot_size); + SetCtrl(common, cap, common.control(), target_group_offset, H2(hash), + policy.slot_size); // Call NOINLINE function to move infoz instructions out of line. - if (hash_infoz) { - ReportInsertMissToInfozAndComputeProbeLength(common, target_group_offset, - hash); + if (common.has_infoz()) { + return ReportInsertMissToInfozAndGetSlot(common, policy, hash, + target_group_offset); } return res; } @@ -2587,7 +2689,7 @@ const DtorPolicy& policy); // Extern template instantiation for inline functions. -template size_t TryFindNewIndexWithoutProbing(size_t h1, size_t old_index, +template size_t TryFindNewIndexWithoutProbing(size_t hash, size_t old_index, size_t old_capacity, ctrl_t* new_ctrl, size_t new_capacity);
diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index a851710..2d43ec4 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h
@@ -260,6 +260,63 @@ assert((CONDITION) && "Try enabling sanitizers.") #endif +// ABSL_SWISSTABLE_ASSUME_SEPARATE_STORAGE(a, b) +// +// Informs the compiler that the two pointer arguments point into separately +// allocated storage (either different variable definitions or different dynamic +// storage allocations), allowing the compiler to assume they do not alias for +// optimization purposes. Differences from __restrict: +// - __restrict only applies to the pointer itself, not the pointed-to object. +// - This macro allows the compiler to optimize caller code, not just the +// pointer dereferences of a function argument within that function. +// +// WARNING: If the arguments point into the same storage, the program has +// undefined behavior. Note that the definition of "storage" here refers to the +// outermost enclosing allocation of any particular object (for example, it is +// never correct to call this macro passing the addresses of fields in the same +// struct, elements of the same array, objects on the same arena block, etc.). +// Note that in the example below, the annotation will affect callers of +// SetFirstAndGetSize as well (they may be mis-compiled if the arguments are not +// from separate allocations). +// +// See +// https://clang.llvm.org/docs/LanguageExtensions.html#builtin-assume-separate-storage +// +// Example: +// +// class Buffer { +// public: +// size_t* data() { +// // Safe because `data_` is allocated via `::operator new`, so it is +// // guaranteed to be in a separate allocation from `*this`. +// ABSL_SWISSTABLE_ASSUME_SEPARATE_STORAGE(data_, this); +// return data_; +// } +// size_t size() const { return size_; } +// +// private: +// size_t* data_ = new size_t[10]; +// size_t size_ = 10; +// }; +// +// size_t SetFirstAndGetSize(Buffer& buf) { +// size_t n = buf.size(); +// buf.data()[0] = n; +// // After inlining, the compiler knows storing to `buf.data()[0]` cannot +// // clobber `buf.size_`, and can return `n` without reloading `buf.size_`. +// return buf.size(); +// } +#if ABSL_HAVE_BUILTIN(__builtin_assume_separate_storage) +#define ABSL_SWISSTABLE_ASSUME_SEPARATE_STORAGE(a, b) \ + __builtin_assume_separate_storage(a, b) +#elif ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L +#define ABSL_SWISSTABLE_ASSUME_SEPARATE_STORAGE(a, b) \ + (decltype((void)(a), (void)(b))()) +#else +#define ABSL_SWISSTABLE_ASSUME_SEPARATE_STORAGE(a, b) \ + (false ? ((void)(a), (void)(b)) : void()) +#endif + // We use uint8_t so we don't need to worry about padding. using GenerationType = uint8_t; @@ -964,12 +1021,21 @@ #endif // Returns the `index`-th byte of `data_`. Accessing the storage through - // `unsigned char` is always allowed by the aliasing rules. Note: the reason - // we read the data this way rather than using bit manipulations as with other - // fields is that it results in better generated code (we get movzbl to read - // the seed this way). + // `unsigned char` is always allowed by the aliasing rules. Note: on x86, the + // reason we read the data this way rather than using bit manipulations as + // with other fields is that it results in better generated code (we get + // movzbl to read the seed this way). On other architectures (such as + // AArch64), byte loads prevent load-pair (ldp) of `inline_data_` with + // `control_` and introduce narrow loads after 64-bit stores to `data_`, so we + // use bit shifts on the wide word instead. uint8_t read_byte(size_t index) const { +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \ + defined(_M_IX86) return reinterpret_cast<const unsigned char*>(&data_)[index]; +#else + return static_cast<uint8_t>(little_endian::FromHost64(data_) >> + (8 * index)); +#endif } // Returns the bits of `data_` that encode `capacity`. @@ -1207,6 +1273,13 @@ alloc_size_ = slot_offset_ + (capacity - blocked_element_count) * slot_size; } + // Creates a layout for a small non-SOO table growing from capacity + // 0 -> 1 (`size == 0`) or 1 -> 3 (`size == 1`). + static RawHashSetLayout CreateForSmallNonSooGrowth(size_t size, + size_t slot_size, + size_t slot_align, + bool has_infoz); + // Returns precomputed offset from the start of the backing allocation of // control. size_t control_offset() const { return control_offset_; } @@ -1224,6 +1297,8 @@ size_t alloc_size() const { return alloc_size_; } private: + explicit RawHashSetLayout(uninitialized_tag_t) {} + size_t control_offset_; size_t generation_offset_; size_t slot_offset_; @@ -1277,6 +1352,7 @@ }; class CommonFields; +struct PolicyFunctions; // RAII object that guards against reentrant calls to hash table methods in // debug mode. @@ -1334,14 +1410,7 @@ ctrl_t* control() const { ABSL_SWISSTABLE_ASSERT(capacity() > 0); - // Assume that the control bytes don't alias `this`. - ctrl_t* ctrl = heap_or_soo_.control().get(); - [[maybe_unused]] size_t num_control_bytes = NumControlBytes(capacity()); - ABSL_ASSUME(reinterpret_cast<uintptr_t>(ctrl + num_control_bytes) <= - reinterpret_cast<uintptr_t>(this) || - reinterpret_cast<uintptr_t>(this + 1) <= - reinterpret_cast<uintptr_t>(ctrl)); - ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(ctrl); + ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(heap_or_soo_.control().get()); } void set_control(ctrl_t* c) { heap_or_soo_.control().set(c); } @@ -1501,6 +1570,15 @@ return CommonFieldsGenerationInfo::should_rehash_for_bug_detection_on_move( capacity()); } + void annotate_for_bug_detection_on_move(CommonFields& that, + const PolicyFunctions& policy) { + // We only enable moved-from validation when generations are enabled (rather + // than using NDEBUG) to avoid issues in which NDEBUG is enabled in some + // translation units but not in others. + if constexpr (SwisstableGenerationsEnabled()) { + AnnotateForBugDetectionOnMoveImpl(that, policy); + } + } void reset_reserved_growth(size_t reservation) { CommonFieldsGenerationInfo::reset_reserved_growth(reservation, size()); } @@ -1590,6 +1668,8 @@ void AssertInSooModeImpl() const; void AssertNotDebugCapacityImpl() const; + void AnnotateForBugDetectionOnMoveImpl(CommonFields& that, + const PolicyFunctions& policy); HashtableInlineData inline_data_; @@ -1881,8 +1961,8 @@ constexpr size_t kProbedElementIndexSentinel = ~size_t{}; // Implementation detail of transfer_unprobed_elements_to_next_capacity_fn. -// Tries to find the new index for an element whose hash corresponds to -// `h1` for growth to the next capacity. +// Tries to find the new index for an element with `hash` for growth to the next +// capacity. // Returns kProbedElementIndexSentinel if full probing is required. // // If element is located in the first probing group in the table before growth, @@ -1892,23 +1972,18 @@ // table. We only attempt to do so if the first probe group is already // initialized. template <typename = void> -inline size_t TryFindNewIndexWithoutProbing(size_t h1, size_t old_index, +inline size_t TryFindNewIndexWithoutProbing(size_t hash, size_t old_index, size_t old_capacity, ctrl_t* new_ctrl, size_t new_capacity) { - size_t index_diff = old_index - h1; + size_t h1 = H1(hash); // The first probe group starts with h1 & capacity. // All following groups start at (h1 + Group::kWidth * K) & capacity. - // We can find an index within the floating group as index_diff modulo - // Group::kWidth. - // Both old and new capacity are larger than Group::kWidth so we can avoid - // computing `& capacity`. - size_t in_floating_group_index = index_diff & (Group::kWidth - 1); - // By subtracting we will get the difference between the first probe group - // and the probe group corresponding to old_index. - index_diff -= in_floating_group_index; - if (ABSL_PREDICT_TRUE((index_diff & old_capacity) == 0)) { - size_t new_index = (h1 + in_floating_group_index) & new_capacity; + // If old_index is in the first probe group, (old_index - h1) & old_capacity + // is the index within the floating group (< Group::kWidth). + size_t index_diff = (old_index - h1) & old_capacity; + if (ABSL_PREDICT_TRUE(index_diff < Group::kWidth)) { + size_t new_index = (h1 + index_diff) & new_capacity; ABSL_ASSUME(new_index != kProbedElementIndexSentinel); return new_index; } @@ -2019,11 +2094,11 @@ // Different encoding is used depending on the capacity of the table. // See ProbedItem*Bytes classes for details. void (*transfer_unprobed_elements_to_next_capacity)( - CommonFields& common, const ctrl_t* old_ctrl, void* old_slots, + CommonFields& common, const ctrl_t* old_ctrl, // TODO(b/382423690): Try to use absl::FunctionRef here. - void* probed_storage, - void (*encode_probed_element)(void* probed_storage, h2_t h2, - size_t source_offset, size_t h1)); + void* probed_storage, void* old_slots, size_t old_capacity, + void (*encode_probed_element)(void* probed_storage, size_t hash, + size_t source_offset)); uint8_t soo_capacity() const { return static_cast<uint8_t>(soo_enabled ? SooCapacity() : 0); @@ -2148,7 +2223,7 @@ // Type erased version of copy constructor. void Copy(CommonFields& common, const PolicyFunctions& policy, const CommonFields& other, - absl::FunctionRef<void(void*, const void*)> copy_fn); + void (*copy_fn)(void* set, void* dst, const void* src)); // Returns the optimal size for memcpy when transferring SOO slot. // Otherwise, returns the optimal size for memcpy SOO slot transfer @@ -2899,16 +2974,7 @@ raw_hash_set(const raw_hash_set& that, const allocator_type& a) : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { - that.AssertNotDebugCapacity(); - if (that.empty()) return; - Copy(common(), GetPolicyFunctions(), that.common(), - [this](void* dst, const void* src) { - // TODO(b/413598253): type erase for trivially copyable types via - // PolicyTraits. - construct(to_slot(dst), - PolicyTraits::element( - static_cast<slot_type*>(const_cast<void*>(src)))); - }); + Copy(common(), GetPolicyFunctions(), that.common(), ©_slot_fn); } ABSL_ATTRIBUTE_NOINLINE raw_hash_set(raw_hash_set&& that) noexcept( @@ -2924,11 +2990,14 @@ : CommonFields{full_soo_tag_t{}, that.common().soo_has_tried_sampling()}, that.hash_ref(), that.eq_ref(), that.char_alloc_ref()) { - if (!PolicyTraits::transfer_uses_memcpy() && that.is_full_soo()) { - transfer(soo_slot(), that.soo_slot()); + if constexpr (SooEnabled() && !PolicyTraits::transfer_uses_memcpy()) { + if (that.is_full_soo()) { + transfer(soo_slot(), that.soo_slot()); + } } that.common() = CommonFields::CreateDefault<SooEnabled()>(); - annotate_for_bug_detection_on_move(that); + common().annotate_for_bug_detection_on_move(that.common(), + GetPolicyFunctions()); } raw_hash_set(raw_hash_set&& that, const allocator_type& a) @@ -2936,7 +3005,8 @@ that.eq_ref(), a) { if (CharAlloc(a) == that.char_alloc_ref()) { swap_common(that); - annotate_for_bug_detection_on_move(that); + common().annotate_for_bug_detection_on_move(that.common(), + GetPolicyFunctions()); } else { move_elements_allocs_unequal(std::move(that)); } @@ -3785,27 +3855,6 @@ std::move(tmp)); } - void annotate_for_bug_detection_on_move([[maybe_unused]] raw_hash_set& that) { - // We only enable moved-from validation when generations are enabled (rather - // than using NDEBUG) to avoid issues in which NDEBUG is enabled in some - // translation units but not in others. - if (SwisstableGenerationsEnabled()) { - that.common().set_capacity(this == &that - ? HashtableCapacity::CreateSelfMovedFrom() - : HashtableCapacity::CreateMovedFrom()); - } - if (!SwisstableGenerationsEnabled() || - !maybe_invalid_capacity().IsValid() || - capacity() == DefaultCapacity()) { - return; - } - common().increment_generation(); - if (!empty() && common().should_rehash_for_bug_detection_on_move()) { - ResizeAllocatedTableWithSeedChange(common(), GetPolicyFunctions(), - capacity()); - } - } - template <bool propagate_alloc> raw_hash_set& assign_impl(raw_hash_set&& that) { // We don't bother checking for this/that aliasing. We just need to avoid @@ -3818,7 +3867,8 @@ CopyAlloc(char_alloc_ref(), that.char_alloc_ref(), std::bool_constant<propagate_alloc>()); that.common() = CommonFields::CreateDefault<SooEnabled()>(); - annotate_for_bug_detection_on_move(that); + common().annotate_for_bug_detection_on_move(that.common(), + GetPolicyFunctions()); return *this; } @@ -3836,7 +3886,8 @@ &that.char_alloc_ref()); } that.common() = CommonFields::CreateDefault<SooEnabled()>(); - annotate_for_bug_detection_on_move(that); + common().annotate_for_bug_detection_on_move(that.common(), + GetPolicyFunctions()); return *this; } @@ -4003,6 +4054,8 @@ template <class K> std::pair<slot_type*, bool> find_or_prepare_insert(const K& key) { AssertOnFind(key); + // TODO(ezb): add ABSL_SWISSTABLE_ASSUME_SEPARATE_STORAGE that the returned + // slot doesn't alias `this` for non-SOO tables with default allocators. return is_small() ? find_or_prepare_insert_small(key) : find_or_prepare_insert_large<FindLargeKey<K>>(key); } @@ -4086,7 +4139,24 @@ } ctrl_t* control() const { ABSL_SWISSTABLE_ASSERT(!is_soo()); - return common().control(); + ctrl_t* ctrl = common().control(); + if constexpr (std::is_same_v<CharAlloc, std::allocator<char>>) { + // Assume that the control bytes don't alias `this`. Example benefit: when + // a caller mutates an element via an iterator returned by find(), the + // compiler can now prove that the store to the slot does not clobber + // CommonFields metadata (size, capacity, etc.). + // Example of benefitting user code: + // if (auto it = m.find(k1); it != m.end()) it->second = v; + // if (auto it = m.find(k2); it != m.end()) it->second = v; + // With this assumption, the second lookup avoids reloading capacity. + // Note: for custom allocators, `this` may be part of the same allocation + // as the control bytes so we cannot make this assumption. + // TODO(ezb): we could move this assumption into CommonFields::control() + // if we template the type-erased functions on whether the allocator is + // default or not. + ABSL_SWISSTABLE_ASSUME_SEPARATE_STORAGE(ctrl, this); + } + return ctrl; } slot_type* slot_array(size_t capacity) const { return static_cast<slot_type*>(common().slot_array(capacity)); @@ -4102,7 +4172,9 @@ } slot_type* single_slot() { ABSL_SWISSTABLE_ASSERT(is_small()); - return SooEnabled() ? soo_slot() : to_slot(common().single_non_soo_slot()); + // Note: we don't use common().single_non_soo_slot() here because + // control() is more efficient (due to separate storage assume). + return SooEnabled() ? soo_slot() : to_slot(control()); } const slot_type* single_slot() const { return const_cast<raw_hash_set*>(this)->single_slot(); @@ -4159,6 +4231,14 @@ } } + static void copy_slot_fn(void* set, void* dst, const void* src) { + // TODO(b/413598253): type erase for trivially copyable types via + // PolicyTraits. + static_cast<raw_hash_set*>(set)->construct( + to_slot(dst), + PolicyTraits::element(static_cast<slot_type*>(const_cast<void*>(src)))); + } + static void destroy_slot_fn_impl(void* set, void* slot) { auto* h = static_cast<raw_hash_set*>(set); h->destroy(to_slot(slot)); @@ -4173,15 +4253,15 @@ // by GetKey + Hash for memcpyable types. // TODO(b/382423690): Try to type erase for big slots: sizeof(slot_type) > 16. static void transfer_unprobed_elements_to_next_capacity_fn( - CommonFields& common, const ctrl_t* old_ctrl, void* old_slots, - void* probed_storage, - void (*encode_probed_element)(void* probed_storage, h2_t h2, - size_t source_offset, size_t h1)) { - const size_t new_capacity = common.capacity(); - ABSL_ASSUME(new_capacity > kMaxSmallCapacity); - const size_t old_capacity = PreviousCapacity(new_capacity); + CommonFields& common, const ctrl_t* old_ctrl, void* probed_storage, + void* old_slots, size_t old_capacity, + void (*encode_probed_element)(void* probed_storage, size_t hash, + size_t source_offset)) { ABSL_ASSUME(old_capacity + 1 >= Group::kWidth); ABSL_ASSUME((old_capacity + 1) % Group::kWidth == 0); + const size_t new_capacity = NextCapacity(old_capacity); + ABSL_ASSUME(new_capacity > kMaxSmallCapacity); + ABSL_SWISSTABLE_ASSERT(common.capacity() == new_capacity); auto* set = reinterpret_cast<raw_hash_set*>(&common); slot_type* old_slots_ptr = to_slot(old_slots); @@ -4203,19 +4283,17 @@ // TODO(b/382423690): try to avoid entire hash calculation since we need // only one new bit of h1. size_t hash = set->hash_of(old_slot); - size_t h1 = H1(hash); - h2_t h2 = H2(hash); size_t new_index = TryFindNewIndexWithoutProbing( - h1, old_index, old_capacity, new_ctrl, new_capacity); + hash, old_index, old_capacity, new_ctrl, new_capacity); // Note that encode_probed_element is allowed to use old_ctrl buffer // till and included the old_index. if (ABSL_PREDICT_FALSE(new_index == kProbedElementIndexSentinel)) { - encode_probed_element(probed_storage, h2, old_index, h1); + encode_probed_element(probed_storage, hash, old_index); continue; } ABSL_SWISSTABLE_ASSERT((new_index & old_capacity) <= old_index); ABSL_SWISSTABLE_ASSERT(IsEmpty(new_ctrl[new_index])); - new_ctrl[new_index] = static_cast<ctrl_t>(h2); + new_ctrl[new_index] = static_cast<ctrl_t>(H2(hash)); auto* new_slot = new_slots + new_index; SanitizerUnpoisonMemoryRegion(new_slot, sizeof(slot_type)); set->transfer(new_slot, old_slot); @@ -4465,7 +4543,7 @@ // Extern template for inline function keeps possibility of inlining. // When compiler decided to not inline, no symbols will be added to the // corresponding translation unit. -extern template size_t TryFindNewIndexWithoutProbing(size_t h1, +extern template size_t TryFindNewIndexWithoutProbing(size_t hash, size_t old_index, size_t old_capacity, ctrl_t* new_ctrl, @@ -4493,5 +4571,6 @@ #undef ABSL_SWISSTABLE_ENABLE_GENERATIONS #undef ABSL_SWISSTABLE_ASSERT +#undef ABSL_SWISSTABLE_ASSUME_SEPARATE_STORAGE #endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_
diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 429d7cf..ed25c05 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc
@@ -147,6 +147,26 @@ } } +TEST(RawHashSetLayout, CreateForSmallNonSooGrowth) { + for (size_t size : {0u, 1u}) { + for (size_t slot_align : {1u, 2u, 4u, 8u, 16u}) { + for (size_t slot_size : {slot_align, slot_align * 2, slot_align * 3}) { + for (bool has_infoz : {false, true}) { + RawHashSetLayout expected(NextCapacity(size), slot_size, slot_align, + has_infoz, /*blocked_element_count=*/0); + RawHashSetLayout actual = + RawHashSetLayout::CreateForSmallNonSooGrowth( + size, slot_size, slot_align, has_infoz); + EXPECT_EQ(actual.control_offset(), expected.control_offset()); + EXPECT_EQ(actual.generation_offset(), expected.generation_offset()); + EXPECT_EQ(actual.slot_offset(), expected.slot_offset()); + EXPECT_EQ(actual.alloc_size(), expected.alloc_size()); + } + } + } + } +} + void VerifyMiddleSizeTableLayout(size_t capacity, size_t slot_size, size_t slot_align, bool has_infoz, size_t blocked_element_count,
diff --git a/absl/crc/CMakeLists.txt b/absl/crc/CMakeLists.txt index fb9beea..e50b242 100644 --- a/absl/crc/CMakeLists.txt +++ b/absl/crc/CMakeLists.txt
@@ -122,6 +122,8 @@ absl::string_view absl::strings GTest::gtest_main + SHARD_COUNT + 3 ) absl_cc_test(
diff --git a/absl/crc/internal/crc_x86_arm_combined.cc b/absl/crc/internal/crc_x86_arm_combined.cc index 95221e4..98f532b 100644 --- a/absl/crc/internal/crc_x86_arm_combined.cc +++ b/absl/crc/internal/crc_x86_arm_combined.cc
@@ -18,6 +18,7 @@ #include <cstdint> #include <cstring> #include <memory> +#include <type_traits> #include <vector> #include "absl/base/attributes.h" @@ -269,52 +270,33 @@ } } - // Update partialCRC with crc of 64 byte block. Calling FinalizePclmulStream - // would produce a single crc checksum, but it is expensive. PCLMULQDQ has a - // high latency, so we run 4 128-bit partial checksums that can be reduced to - // a single value by FinalizePclmulStream later. Computing crc for arbitrary - // polynomialas with PCLMULQDQ is described in Intel paper "Fast CRC - // Computation for Generic Polynomials Using PCLMULQDQ Instruction" + // Update a single 128-bit element of partialCRC with crc of 16 byte block. + // Calling FinalizePclmulStream would produce a single crc checksum, but it is + // expensive. PCLMULQDQ has a high latency, so we run 4 128-bit partial + // checksums that can be reduced to a single value by FinalizePclmulStream + // later. Computing crc for arbitrary polynomials with PCLMULQDQ is described + // in Intel paper "Fast CRC Computation for Generic Polynomials Using + // PCLMULQDQ Instruction" // https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf // We are applying it to CRC32C polynomial. -#if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) - template <bool kUseEor3 = false> - ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesNeonPclmul( - V256 v0, V256 v1, V128* partialCRC) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE V128 Pclmul16Bytes(V128* partialCRC) const { V128 loopMultiplicands = V128_Load(reinterpret_cast<const V128*>(kFoldAcross512Bits)); - - V128 partialCRC1 = partialCRC[0]; - V128 partialCRC2 = partialCRC[1]; - V128 partialCRC3 = partialCRC[2]; - V128 partialCRC4 = partialCRC[3]; - - V128 tmp1 = V128_PMulHi(partialCRC1, loopMultiplicands); - V128 tmp2 = V128_PMulHi(partialCRC2, loopMultiplicands); - V128 tmp3 = V128_PMulHi(partialCRC3, loopMultiplicands); - V128 tmp4 = V128_PMulHi(partialCRC4, loopMultiplicands); - const V128* data_ptr0 = reinterpret_cast<const V128*>(&v0); - const V128* data_ptr1 = reinterpret_cast<const V128*>(&v1); - V128 data1 = data_ptr0[0]; - V128 data2 = data_ptr0[1]; - V128 data3 = data_ptr1[0]; - V128 data4 = data_ptr1[1]; - partialCRC1 = V128_PMulLow(partialCRC1, loopMultiplicands); - partialCRC2 = V128_PMulLow(partialCRC2, loopMultiplicands); - partialCRC3 = V128_PMulLow(partialCRC3, loopMultiplicands); - partialCRC4 = V128_PMulLow(partialCRC4, loopMultiplicands); - partialCRC1 = V128_Xor3<kUseEor3>(tmp1, partialCRC1, data1); - partialCRC2 = V128_Xor3<kUseEor3>(tmp2, partialCRC2, data2); - partialCRC3 = V128_Xor3<kUseEor3>(tmp3, partialCRC3, data3); - partialCRC4 = V128_Xor3<kUseEor3>(tmp4, partialCRC4, data4); - partialCRC[0] = partialCRC1; - partialCRC[1] = partialCRC2; - partialCRC[2] = partialCRC3; - partialCRC[3] = partialCRC4; + V128 tmp = V128_PMulHi(*partialCRC, loopMultiplicands); + *partialCRC = V128_PMulLow(*partialCRC, loopMultiplicands); + return tmp; } - // Reduce partialCRC produced by Process64BytesNeonPclmul into a single value, - // that represents crc checksum of all the processed bytes. +#if defined(ABSL_CRC_INTERNAL_HAVE_ARM_SIMD) + template <bool kUseEor3 = false> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor16BytesNeonPclmul( + V128 data, V128 tmp, V128* partialCRC) const { + *partialCRC = V128_Xor3<kUseEor3>(tmp, *partialCRC, data); + } + + // Reduce partialCRC produced by Pclmul16Bytes and Xor16BytesNeonPclmul + // into a single value, that represents crc checksum of all the processed + // bytes. template <bool kUseEor3 = false> ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizeNeonPclmulStream(V128* partialCRC) const { @@ -351,54 +333,20 @@ return crc; } - ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesPclmul(V256, V256, - V128*) const {} + ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor16BytesPclmul(V128, V128, V128*) const {} ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizePclmulStream(V128*) const { return 0; } #else - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void - Process64BytesPclmul(V256 v0, V256 v1, V128* partialCRC) const { - V128 loopMultiplicands = - V128_Load(reinterpret_cast<const V128*>(kFoldAcross512Bits)); - - V128 partialCRC1 = partialCRC[0]; - V128 partialCRC2 = partialCRC[1]; - V128 partialCRC3 = partialCRC[2]; - V128 partialCRC4 = partialCRC[3]; - - V128 tmp1 = V128_PMulHi(partialCRC1, loopMultiplicands); - V128 tmp2 = V128_PMulHi(partialCRC2, loopMultiplicands); - V128 tmp3 = V128_PMulHi(partialCRC3, loopMultiplicands); - V128 tmp4 = V128_PMulHi(partialCRC4, loopMultiplicands); - const V128* data_ptr0 = reinterpret_cast<const V128*>(&v0); - const V128* data_ptr1 = reinterpret_cast<const V128*>(&v1); - V128 data1 = data_ptr0[0]; - V128 data2 = data_ptr0[1]; - V128 data3 = data_ptr1[0]; - V128 data4 = data_ptr1[1]; - partialCRC1 = V128_PMulLow(partialCRC1, loopMultiplicands); - partialCRC2 = V128_PMulLow(partialCRC2, loopMultiplicands); - partialCRC3 = V128_PMulLow(partialCRC3, loopMultiplicands); - partialCRC4 = V128_PMulLow(partialCRC4, loopMultiplicands); - partialCRC1 = V128_Xor(tmp1, partialCRC1); - partialCRC2 = V128_Xor(tmp2, partialCRC2); - partialCRC3 = V128_Xor(tmp3, partialCRC3); - partialCRC4 = V128_Xor(tmp4, partialCRC4); - partialCRC1 = V128_Xor(partialCRC1, data1); - partialCRC2 = V128_Xor(partialCRC2, data2); - partialCRC3 = V128_Xor(partialCRC3, data3); - partialCRC4 = V128_Xor(partialCRC4, data4); - partialCRC[0] = partialCRC1; - partialCRC[1] = partialCRC2; - partialCRC[2] = partialCRC3; - partialCRC[3] = partialCRC4; + ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor16BytesPclmul(V128 data, V128 tmp, + V128* partialCRC) const { + *partialCRC = V128_Xor(V128_Xor(tmp, *partialCRC), data); } - // Reduce partialCRC produced by Process64BytesPclmul into a single value, - // that represents crc checksum of all the processed bytes. - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t + // Reduce partialCRC produced by Pclmul16Bytes and Xor16BytesPclmul into a + // single value, that represents crc checksum of all the processed bytes. + ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizePclmulStream(V128* partialCRC) const { V128 partialCRC1 = partialCRC[0]; V128 partialCRC2 = partialCRC[1]; @@ -438,7 +386,7 @@ template <bool kUseEor3 = false> ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void - Process64BytesNeonPclmul(V256, V256, V128*) const {} + Xor16BytesNeonPclmul(V128, V128, V128*) const {} template <bool kUseEor3 = false> ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t @@ -447,57 +395,21 @@ } #endif - // Update crc with 64 bytes of data from p. - ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t Process64BytesCRC(const uint8_t* p, + // Update crc with 8 bytes of data from p. + ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t Process8BytesCRC(const uint8_t* p, + uint64_t crc) const { + return CRC32_u64(static_cast<uint32_t>(crc), + absl::little_endian::Load64(p)); + } + + // Update crc with 16 bytes of data from p. + ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t Process16BytesCRC(const uint8_t* p, uint64_t crc) const { - for (int i = 0; i < 8; i++) { - crc = - CRC32_u64(static_cast<uint32_t>(crc), absl::little_endian::Load64(p)); - p += 8; - } + crc = Process8BytesCRC(p, crc); + crc = Process8BytesCRC(p + 8, crc); return crc; } - // Same as Process64BytesCRC, but just interleaved for 2 streams. - ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesCRC2Streams( - const uint8_t* p0, const uint8_t* p1, uint64_t* crc) const { - uint64_t crc0 = crc[0]; - uint64_t crc1 = crc[1]; - for (int i = 0; i < 8; i++) { - crc0 = CRC32_u64(static_cast<uint32_t>(crc0), - absl::little_endian::Load64(p0)); - crc1 = CRC32_u64(static_cast<uint32_t>(crc1), - absl::little_endian::Load64(p1)); - p0 += 8; - p1 += 8; - } - crc[0] = crc0; - crc[1] = crc1; - } - - // Same as Process64BytesCRC, but just interleaved for 3 streams. - ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesCRC3Streams( - const uint8_t* p0, const uint8_t* p1, const uint8_t* p2, - uint64_t* crc) const { - uint64_t crc0 = crc[0]; - uint64_t crc1 = crc[1]; - uint64_t crc2 = crc[2]; - for (int i = 0; i < 8; i++) { - crc0 = CRC32_u64(static_cast<uint32_t>(crc0), - absl::little_endian::Load64(p0)); - crc1 = CRC32_u64(static_cast<uint32_t>(crc1), - absl::little_endian::Load64(p1)); - crc2 = CRC32_u64(static_cast<uint32_t>(crc2), - absl::little_endian::Load64(p2)); - p0 += 8; - p1 += 8; - p2 += 8; - } - crc[0] = crc0; - crc[1] = crc1; - crc[2] = crc2; - } - #if defined(ABSL_CRC_INTERNAL_HAVE_X86_SIMD) && \ (defined(__AVX__) || defined(ABSL_INTERNAL_CAN_FORCE_AVX)) && \ (!defined(_MSC_VER) || defined(__clang__)) @@ -539,25 +451,37 @@ return crc; } - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void - Process64BytesVpclmul(V256 v0, V256 v1, V256* vpartialCRC, - V256 loopMultiplicands) const { + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE V256 + Pclmul32BytesVpclmul(V256* partialCRC) const { + V256 loopMultiplicands = + V256_Broadcast128(reinterpret_cast<const V128*>(kFoldAcross512Bits)); + V256 tmp; __asm__ volatile( - "vpclmulqdq $0x11, %3, %0, %%ymm0 \n" - "vpclmulqdq $0x11, %3, %1, %%ymm1 \n" - "vpclmulqdq $0x00, %3, %0, %0 \n" - "vpclmulqdq $0x00, %3, %1, %1 \n" - "vpxor %%ymm0, %0, %0 \n" - "vpxor %%ymm1, %1, %1 \n" + "vpclmulqdq $0x11, %2, %0, %1 \n" + "vpclmulqdq $0x00, %2, %0, %0 \n" + : "+x"(*partialCRC), "=&x"(tmp) + : "x"(loopMultiplicands)); + return tmp; + } + + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void + Xor32BytesVpclmul(const uint8_t* p, V256 tmp, V256* partialCRC) const { + V256 v = V256_LoadU(p); + __asm__ volatile( "vpxor %2, %0, %0 \n" - "vpxor %4, %1, %1 \n" - : "+x"(vpartialCRC[0]), "+x"(vpartialCRC[1]) - : "x"(v0), "x"(loopMultiplicands), "x"(v1) - : "ymm0", "ymm1"); + "vpxor %1, %0, %0 \n" + : "+x"(*partialCRC) + : "x"(v), "x"(tmp)); } #else template <typename T = V256> - ABSL_ATTRIBUTE_ALWAYS_INLINE void Process64BytesVpclmul(T, T, T*, T) const { + ABSL_ATTRIBUTE_ALWAYS_INLINE T Pclmul32BytesVpclmul(T*) const { + static_assert(sizeof(T) == 0, "Vector PCLMUL not supported"); + return T{}; + } + template <typename T = V256> + ABSL_ATTRIBUTE_ALWAYS_INLINE void Xor32BytesVpclmul(const uint8_t*, T, + T*) const { static_assert(sizeof(T) == 0, "Vector PCLMUL not supported"); } ABSL_ATTRIBUTE_ALWAYS_INLINE uint64_t FinalizeVpclmulStream(V256*) const { @@ -649,70 +573,63 @@ // For small blocks just run simple loop, because cost of combining multiple // streams is significant. - if (num_crc_streams > 1 && (length < kSmallCutoff)) { + if (num_crc_streams == 1 || (length < kSmallCutoff)) { // fallthrough; Use the same strategy as we do for processing the // remaining bytes after any other strategy. } else { l64 = l; - if (num_crc_streams > 1) { - const size_t blockSize = 32; - size_t bs = static_cast<size_t>(e - p) / kGroupsSmall / blockSize; - uint64_t l641 = 0; - uint64_t l642 = 0; - const uint8_t* p1 = p + bs * blockSize; - const uint8_t* p2 = p1 + bs * blockSize; + const size_t blockSize = 32; + size_t bs = static_cast<size_t>(e - p) / kGroupsSmall / blockSize; + uint64_t l641 = 0; + uint64_t l642 = 0; + const uint8_t* p1 = p + bs * blockSize; + const uint8_t* p2 = p1 + bs * blockSize; - for (size_t i = 0; i + 1 < bs; ++i) { - ABSL_INTERNAL_STEP8BY3(l64, l641, l642, p, p1, p2); - ABSL_INTERNAL_STEP8BY3(l64, l641, l642, p, p1, p2); - ABSL_INTERNAL_STEP8BY3(l64, l641, l642, p, p1, p2); - ABSL_INTERNAL_STEP8BY3(l64, l641, l642, p, p1, p2); - PrefetchToLocalCache( - reinterpret_cast<const char*>(p + kPrefetchHorizonMedium)); - PrefetchToLocalCache( - reinterpret_cast<const char*>(p1 + kPrefetchHorizonMedium)); - PrefetchToLocalCache( - reinterpret_cast<const char*>(p2 + kPrefetchHorizonMedium)); - } - // Don't run crc on last 8 bytes. + for (size_t i = 0; i + 1 < bs; ++i) { ABSL_INTERNAL_STEP8BY3(l64, l641, l642, p, p1, p2); ABSL_INTERNAL_STEP8BY3(l64, l641, l642, p, p1, p2); ABSL_INTERNAL_STEP8BY3(l64, l641, l642, p, p1, p2); - ABSL_INTERNAL_STEP8BY2(l64, l641, p, p1); - - // kClmulConstants stores precomputed constants for up to 32 blocks - // (bs <= 32). - constexpr size_t kMaxPrecomputedBlocks = std::size(kClmulConstants) / 2; - V128 magic; - if (bs <= kMaxPrecomputedBlocks) { - magic = V128_Load(reinterpret_cast<const V128*>(kClmulConstants) + - bs - 1); - } else { - uint32_t high = ComputeZeroConstant(32 * bs); - uint32_t low = MultiplyWithExtraX33(high, high); - alignas(16) uint64_t dynamic_magic[2] = {low, high}; - magic = V128_Load(reinterpret_cast<const V128*>(dynamic_magic)); - } - - V128 tmp = V128_From64WithZeroFill(l64); - - V128 res1 = V128_PMulLow(tmp, magic); - - tmp = V128_From64WithZeroFill(l641); - - V128 res2 = V128_PMul10(tmp, magic); - V128 x = V128_Xor(res1, res2); - l64 = static_cast<uint64_t>(V128_Low64(x)) ^ - absl::little_endian::Load64(p2); - l64 = CRC32_u64(static_cast<uint32_t>(l642), l64); - - p = p2 + 8; - } else { - while ((e - p) >= 64) { - l64 = Process64BytesCRC(p, l64); - p += 64; - } + ABSL_INTERNAL_STEP8BY3(l64, l641, l642, p, p1, p2); + PrefetchToLocalCache( + reinterpret_cast<const char*>(p + kPrefetchHorizonMedium)); + PrefetchToLocalCache( + reinterpret_cast<const char*>(p1 + kPrefetchHorizonMedium)); + PrefetchToLocalCache( + reinterpret_cast<const char*>(p2 + kPrefetchHorizonMedium)); } + // Don't run crc on last 8 bytes. + ABSL_INTERNAL_STEP8BY3(l64, l641, l642, p, p1, p2); + ABSL_INTERNAL_STEP8BY3(l64, l641, l642, p, p1, p2); + ABSL_INTERNAL_STEP8BY3(l64, l641, l642, p, p1, p2); + ABSL_INTERNAL_STEP8BY2(l64, l641, p, p1); + + // kClmulConstants stores precomputed constants for up to 32 blocks + // (bs <= 32). + constexpr size_t kMaxPrecomputedBlocks = std::size(kClmulConstants) / 2; + V128 magic; + if (bs <= kMaxPrecomputedBlocks) { + magic = + V128_Load(reinterpret_cast<const V128*>(kClmulConstants) + bs - 1); + } else { + uint32_t high = ComputeZeroConstant(32 * bs); + uint32_t low = MultiplyWithExtraX33(high, high); + alignas(16) uint64_t dynamic_magic[2] = {low, high}; + magic = V128_Load(reinterpret_cast<const V128*>(dynamic_magic)); + } + + V128 tmp = V128_From64WithZeroFill(l64); + + V128 res1 = V128_PMulLow(tmp, magic); + + tmp = V128_From64WithZeroFill(l641); + + V128 res2 = V128_PMul10(tmp, magic); + V128 x = V128_Xor(res1, res2); + l64 = static_cast<uint64_t>(V128_Low64(x)) ^ + absl::little_endian::Load64(p2); + l64 = CRC32_u64(static_cast<uint32_t>(l642), l64); + + p = p2 + 8; l = static_cast<uint32_t>(l64); } @@ -785,6 +702,7 @@ size_t bs = static_cast<size_t>(e - p) / (kNumActiveCrcStreams + kNumActivePclmulStreams) / 64; + const uint8_t* stream_start = p; const uint8_t* crc_streams[kMaxStreams]; for (size_t i = 0; i < kNumActiveCrcStreams; i++) { @@ -815,10 +733,9 @@ uint64_t l64_crc[kMaxStreams] = {l}; uint64_t l64_pclmul[kMaxStreams] = {0}; - V256 pclmul_v0_0[kMaxStreams]; - V256 pclmul_v1_0[kMaxStreams]; + V256 pclmul_v[kMaxStreams][2]; for (size_t j = 0; j < kNumActivePclmulStreams; j++) { - V256_LoadPairU(pclmul_streams[j], &pclmul_v0_0[j], &pclmul_v1_0[j]); + V256_LoadPairU(pclmul_streams[j], &pclmul_v[j][0], &pclmul_v[j][1]); } if constexpr (copy_type != CopyType::kNone) { @@ -827,31 +744,25 @@ dst_crc_streams[j] += 64; } for (size_t j = 0; j < kNumActivePclmulStreams; j++) { - Store64Bytes<copy_type>(dst_pclmul_streams[j], pclmul_v0_0[j], - pclmul_v1_0[j]); + Store64Bytes<copy_type>(dst_pclmul_streams[j], pclmul_v[j][0], + pclmul_v[j][1]); dst_pclmul_streams[j] += 64; } } - if (kNumActiveCrcStreams == 1) { - l64_crc[0] = Process64BytesCRC(crc_streams[0], l64_crc[0]); - crc_streams[0] += 16 * 4; - } else if (kNumActiveCrcStreams == 2) { - Process64BytesCRC2Streams(crc_streams[0], crc_streams[1], l64_crc); - crc_streams[0] += 16 * 4; - crc_streams[1] += 16 * 4; - } else { - Process64BytesCRC3Streams(crc_streams[0], crc_streams[1], crc_streams[2], - l64_crc); - crc_streams[0] += 16 * 4; - crc_streams[1] += 16 * 4; - crc_streams[2] += 16 * 4; + for (size_t offset = 0; offset < 64; offset += 16) { + for (size_t j = 0; j < kNumActiveCrcStreams; j++) { + l64_crc[j] = Process16BytesCRC(crc_streams[j] + offset, l64_crc[j]); + } + } + for (size_t j = 0; j < kNumActiveCrcStreams; j++) { + crc_streams[j] += 64; } // Align to 32 bytes for vpclmul implementation. alignas(32) V128 partialCRC[kMaxStreams][4]; for (size_t i = 0; i < kNumActivePclmulStreams; i++) { - InitPclmulStream(pclmul_v0_0[i], pclmul_v1_0[i], partialCRC[i]); + InitPclmulStream(pclmul_v[i][0], pclmul_v[i][1], partialCRC[i]); pclmul_streams[i] += 64; } @@ -864,32 +775,40 @@ PrefetchToLocalCache(pclmul_streams[j] + kPrefetchHorizon); } - for (size_t j = 0; j < kNumActivePclmulStreams; j++) { - if constexpr (copy_type != CopyType::kNone) { - ProcessPclmulStream<copy_type>(pclmul_streams[j], - dst_pclmul_streams[j], partialCRC[j]); + const uint8_t* pclmul_copy[kMaxStreams]; + if constexpr (copy_type != CopyType::kNone) { + for (size_t j = 0; j < kNumActivePclmulStreams; j++) { + V256_LoadPairU(pclmul_streams[j], &pclmul_v[j][0], &pclmul_v[j][1]); + Store64Bytes<copy_type>(dst_pclmul_streams[j], pclmul_v[j][0], + pclmul_v[j][1]); dst_pclmul_streams[j] += 64; - } else { - ProcessPclmulStream<CopyType::kNone>(pclmul_streams[j], nullptr, - partialCRC[j]); + pclmul_streams[j] += 64; + pclmul_copy[j] = reinterpret_cast<const uint8_t*>(pclmul_v[j]); } - pclmul_streams[j] += 64; } + const uint8_t** pclmul_src = + copy_type != CopyType::kNone ? pclmul_copy : pclmul_streams; + // Fold the four 16-byte blocks of each PCLMUL stream into + // partialCRC[0..3]. For VPCLMUL, 32 bytes are folded every other block. + ProcessAllStreams16<kNumActiveCrcStreams, kNumActivePclmulStreams, + /*kVpclmulBytes=*/32>( + crc_streams, l64_crc, pclmul_src, partialCRC, /*lane=*/0); + ProcessAllStreams16<kNumActiveCrcStreams, kNumActivePclmulStreams, + /*kVpclmulBytes=*/0>(crc_streams, l64_crc, pclmul_src, + partialCRC, /*lane=*/1); + ProcessAllStreams16<kNumActiveCrcStreams, kNumActivePclmulStreams, + /*kVpclmulBytes=*/32>( + crc_streams, l64_crc, pclmul_src, partialCRC, /*lane=*/2); + ProcessAllStreams16<kNumActiveCrcStreams, kNumActivePclmulStreams, + /*kVpclmulBytes=*/0>(crc_streams, l64_crc, pclmul_src, + partialCRC, /*lane=*/3); - if (kNumActiveCrcStreams == 1) { - l64_crc[0] = Process64BytesCRC(crc_streams[0], l64_crc[0]); - crc_streams[0] += 16 * 4; - } else if (kNumActiveCrcStreams == 2) { - Process64BytesCRC2Streams(crc_streams[0], crc_streams[1], l64_crc); - crc_streams[0] += 16 * 4; - crc_streams[1] += 16 * 4; - } else { - Process64BytesCRC3Streams(crc_streams[0], crc_streams[1], - crc_streams[2], l64_crc); - crc_streams[0] += 16 * 4; - crc_streams[1] += 16 * 4; - crc_streams[2] += 16 * 4; - } + // Copy the 64 bytes just processed only now, as whole cache lines, + // instead of 16 bytes at a time as they are processed. Otherwise, when + // the destination is slightly ahead of the source modulo 4096, the load + // of the next 16 bytes of a stream would wait for the store of the + // previous 16 bytes, since their addresses have the same lowest 12 bits + // (4K aliasing). if constexpr (copy_type != CopyType::kNone) { for (size_t j = 0; j < kNumActiveCrcStreams; j++) { Copy64Bytes<copy_type>(dst_crc_streams[j], crc_streams[j] - 64); @@ -998,11 +917,22 @@ return counts; } + // Whether PCLMUL streams are folded with 256-bit `vpclmulqdq`. + static constexpr bool kIsVpclmul = + (pclmul_stream_type == PclmulStreamType::VPCLMUL); + + // The high product of a fold, which is returned by `PclmulStream` and + // consumed by `XorPclmulStream`. Folding 32 bytes with 256-bit `vpclmulqdq` + // produces a V256; every other PclmulStreamType produces a V128. + template <size_t kVpclmulBytes> + using PclmulProduct = + std::conditional_t<kIsVpclmul && kVpclmulBytes == 32, V256, V128>; + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void InitPclmulStream(V256 v0, V256 v1, V128* partialCRC) const { if constexpr (pclmul_stream_type == PclmulStreamType::NONE) { return; - } else if constexpr (pclmul_stream_type == PclmulStreamType::VPCLMUL) { + } else if constexpr (kIsVpclmul) { V256* vpartialCRC = reinterpret_cast<V256*>(partialCRC); vpartialCRC[0] = v0; vpartialCRC[1] = v1; @@ -1016,31 +946,93 @@ } } - template <CopyType copy_type> - ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void - ProcessPclmulStream(const uint8_t* stream, uint8_t* dst_stream, - V128* partialCRC) const { - if constexpr (pclmul_stream_type == PclmulStreamType::NONE) { - return; + // Performs the polynomial multiplications to fold `partialCRC` by 512 bits, + // storing the low product in `partialCRC` and returning the high product, + // which must be passed to `XorPclmulStream` to complete the fold. + template <size_t kVpclmulBytes> + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE + PclmulProduct<kVpclmulBytes> + PclmulStream(V128* partialCRC) const { + if constexpr (pclmul_stream_type == PclmulStreamType::NONE || + (kIsVpclmul && kVpclmulBytes == 0)) { + return {}; + } else if constexpr (kIsVpclmul && kVpclmulBytes == 32) { + return Pclmul32BytesVpclmul(reinterpret_cast<V256*>(partialCRC)); } else { - const uint8_t* src_ptr = stream; - V256 v0, v1; - V256_LoadPairU(src_ptr, &v0, &v1); - Store64Bytes<copy_type>(dst_stream, v0, v1); - if constexpr (pclmul_stream_type == PclmulStreamType::VPCLMUL) { - V256 loopMultiplicands = V256_Broadcast128( - reinterpret_cast<const V128*>(kFoldAcross512Bits)); - Process64BytesVpclmul(v0, v1, reinterpret_cast<V256*>(partialCRC), - loopMultiplicands); - } else if constexpr (pclmul_stream_type == - PclmulStreamType::NEON_PCLMUL || - pclmul_stream_type == - PclmulStreamType::NEON_PCLMUL_EOR3) { + return Pclmul16Bytes(partialCRC); + } + } + + // Completes folding the next bytes at `*stream` into `partialCRC` by XORing + // the high product returned by `PclmulStream`, the low product in + // `partialCRC`, and the loaded data, and advances `*stream`. + template <size_t kVpclmulBytes> + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void XorPclmulStream( + const uint8_t** stream, const PclmulProduct<kVpclmulBytes>& high, + V128* partialCRC) const { + if constexpr (pclmul_stream_type == PclmulStreamType::NONE || + (kIsVpclmul && kVpclmulBytes == 0)) { + return; + } else if constexpr (kIsVpclmul && kVpclmulBytes == 32) { + Xor32BytesVpclmul(*stream, high, reinterpret_cast<V256*>(partialCRC)); + *stream += 32; + } else { + V128 data = V128_LoadU(reinterpret_cast<const V128*>(*stream)); + if constexpr (pclmul_stream_type == PclmulStreamType::NEON_PCLMUL || + pclmul_stream_type == PclmulStreamType::NEON_PCLMUL_EOR3) { constexpr bool kUseEor3 = (pclmul_stream_type == PclmulStreamType::NEON_PCLMUL_EOR3); - Process64BytesNeonPclmul<kUseEor3>(v0, v1, partialCRC); + Xor16BytesNeonPclmul<kUseEor3>(data, high, partialCRC); } else { - Process64BytesPclmul(v0, v1, partialCRC); + static_assert(pclmul_stream_type == PclmulStreamType::PCLMUL || + (kIsVpclmul && kVpclmulBytes == 16)); + Xor16BytesPclmul(data, high, partialCRC); + } + *stream += 16; + } + } + + // Processes the next 16 bytes of every CRC and PCLMUL stream, folding each + // PCLMUL stream into `partialCRC[lane]`, and advances the stream pointers. + // + // PCLMUL, vector XOR, and CRC32 instructions are interleaved across the two + // 8-byte halves of each 16-byte CRC step (8B CRC -> PCLMUL -> 8B CRC -> XOR) + // to keep the execution units busy. + // + // When `pclmul_stream_type` is `VPCLMUL`, a single 256-bit `vpclmulqdq` + // instruction folds 32 bytes at once across two 16-byte calls. + // `kVpclmulBytes` controls how many bytes of each VPCLMUL stream to process + // in this call: + // - 32: Fold 32 bytes into `partialCRC[lane..lane + 1]` (used on the first + // half of a 32-byte block). + // - 0: Skip the VPCLMUL streams (used on the second half of a 32-byte + // block, since the first half already folded all 32 bytes). + // - 16: Fold 16 bytes into `partialCRC[lane]` using 128-bit `pclmulqdq` + // (used for the standalone 16-byte tail). + // For non-VPCLMUL stream types, `kVpclmulBytes` is ignored and 16 bytes are + // always folded. + template <size_t kNumActiveCrcStreams, size_t kNumActivePclmulStreams, + size_t kVpclmulBytes> + ABSL_INTERNAL_ATTRIBUTE_AVX ABSL_ATTRIBUTE_ALWAYS_INLINE void + ProcessAllStreams16(const uint8_t** crc_streams, uint64_t* l64_crc, + const uint8_t** pclmul_streams, V128 (*partialCRC)[4], + size_t lane) const { + for (size_t j = 0; j < kMaxStreams; j++) { + if (j < kNumActiveCrcStreams) { + l64_crc[j] = Process8BytesCRC(crc_streams[j], l64_crc[j]); + crc_streams[j] += 8; + } + PclmulProduct<kVpclmulBytes> high; + if (j < kNumActivePclmulStreams) { + high = PclmulStream<kVpclmulBytes>(&partialCRC[j][lane]); + } + if (j < kNumActiveCrcStreams) { + l64_crc[j] = Process8BytesCRC(crc_streams[j], l64_crc[j]); + crc_streams[j] += 8; + } + if (j < kNumActivePclmulStreams) { + XorPclmulStream<kVpclmulBytes>(&pclmul_streams[j], high, + &partialCRC[j][lane]); } } } @@ -1049,7 +1041,7 @@ FinalizePclmulStream(V128* partialCRC) const { if constexpr (pclmul_stream_type == PclmulStreamType::NONE) { return 0; - } else if constexpr (pclmul_stream_type == PclmulStreamType::VPCLMUL) { + } else if constexpr (kIsVpclmul) { return FinalizeVpclmulStream(reinterpret_cast<V256*>(partialCRC)); } else if constexpr (pclmul_stream_type == PclmulStreamType::NEON_PCLMUL || pclmul_stream_type == @@ -1078,12 +1070,16 @@ CRCImpl* TryNewCRC32AcceleratedX86ARMCombined() { CpuType type = GetCpuType(); switch (type) { + case CpuType::kAmdRome: + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 3, 0, PclmulStreamType::PCLMUL>(); case CpuType::kIntelHaswell: case CpuType::kAmdNaples: return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 3, 1, PclmulStreamType::PCLMUL>(); case CpuType::kAmdMilan: case CpuType::kAmdGenoa: + case CpuType::kAmdRyzenV3000: case CpuType::kAmdSiena: case CpuType::kAmdTurin: case CpuType::kAmdVenice: @@ -1118,7 +1114,6 @@ return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 3, 2, PclmulStreamType::PCLMUL>(); // PCLMULQDQ is slow, don't use it. - case CpuType::kAmdRome: case CpuType::kIntelIvybridge: case CpuType::kIntelSandybridge: case CpuType::kIntelWestmere: @@ -1146,8 +1141,13 @@ // Not all ARM processors support the needed instructions, so check here // before trying to use an accelerated implementation. if (SupportsArmCRC32PMULL()) { +#if defined(__APPLE__) + return new CRC32AcceleratedX86ARMCombinedMultipleStreams< + 3, 3, PclmulStreamType::NEON_PCLMUL>(); +#else return new CRC32AcceleratedX86ARMCombinedMultipleStreams< 1, 1, PclmulStreamType::NEON_PCLMUL>(); +#endif } else { return nullptr; }
diff --git a/absl/debugging/BUILD.bazel b/absl/debugging/BUILD.bazel index 24e5280..265133a 100644 --- a/absl/debugging/BUILD.bazel +++ b/absl/debugging/BUILD.bazel
@@ -127,6 +127,9 @@ "@rules_cc//cc/compiler:clang-cl": ["/Z7"], "//conditions:default": [], }), + # Ensure symbolize_test.pdb is declared as a linker output on Windows even + # in opt builds so it is uploaded to the remote cache. + features = ["generate_pdb_file"], linkopts = ABSL_DEFAULT_LINKOPTS + select({ "@rules_cc//cc/compiler:msvc-cl": ["/DEBUG"], "@rules_cc//cc/compiler:clang-cl": ["/DEBUG"],
diff --git a/absl/debugging/symbolize_test.cc b/absl/debugging/symbolize_test.cc index 8fcdb84..9ee0abd 100644 --- a/absl/debugging/symbolize_test.cc +++ b/absl/debugging/symbolize_test.cc
@@ -421,7 +421,7 @@ // to optimize away the .data section if no variables live in the // section. This variable should get placed in the .data section, and // the test below checks for the existence of a .data section. -static int in_data_section = 1; +static int in_data_section [[maybe_unused]] = 1; TEST(Symbolize, ForEachSection) { int fd;
diff --git a/absl/flags/CMakeLists.txt b/absl/flags/CMakeLists.txt index 0e40085..162b654 100644 --- a/absl/flags/CMakeLists.txt +++ b/absl/flags/CMakeLists.txt
@@ -455,6 +455,8 @@ absl::flags_internal absl::time GTest::gmock_main + SHARD_COUNT + 31 ) absl_cc_test(
diff --git a/absl/flags/flag_test.cc b/absl/flags/flag_test.cc index 1d6a496..5cacf64 100644 --- a/absl/flags/flag_test.cc +++ b/absl/flags/flag_test.cc
@@ -1026,7 +1026,7 @@ default; ~NonTriviallyCopyableUDT() { s_num_instance--; } - static uint64_t s_num_instance; + static uint64_t s_num_instance [[maybe_unused]]; char c; };
diff --git a/absl/functional/BUILD.bazel b/absl/functional/BUILD.bazel index d1b46c7..0cad083 100644 --- a/absl/functional/BUILD.bazel +++ b/absl/functional/BUILD.bazel
@@ -124,6 +124,7 @@ linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":bind_front", + "//absl/base:config", "//absl/memory", "@googletest//:gtest", "@googletest//:gtest_main",
diff --git a/absl/functional/CMakeLists.txt b/absl/functional/CMakeLists.txt index 1ddfa29..231c9dd 100644 --- a/absl/functional/CMakeLists.txt +++ b/absl/functional/CMakeLists.txt
@@ -105,6 +105,7 @@ ${ABSL_DEFAULT_COPTS} DEPS absl::bind_front + absl::config absl::memory GTest::gmock_main )
diff --git a/absl/functional/bind_front.h b/absl/functional/bind_front.h index 45ca4fc..06b6e52 100644 --- a/absl/functional/bind_front.h +++ b/absl/functional/bind_front.h
@@ -21,10 +21,11 @@ // it does not require the use of argument placeholders. The simpler syntax of // `absl::bind_front()` allows you to avoid known misuses with `std::bind()`. // -// `absl::bind_front()` is meant as a drop-in replacement for C++20's upcoming -// `std::bind_front()`, which similarly resolves these issues with -// `std::bind()`. Both `bind_front()` alternatives, unlike `std::bind()`, allow -// partial function application. (See +// `absl::bind_front()` is meant as a drop-in replacement for C++20's +// `std::bind_front(func, args...)` and C++26's +// `std::bind_front<ConstFn>(args...)`, which similarly resolve these issues +// with `std::bind()`. Both `bind_front()` alternatives, unlike `std::bind()`, +// allow partial function application. (See // https://en.wikipedia.org/wiki/Partial_application). #ifndef ABSL_FUNCTIONAL_BIND_FRONT_H_ @@ -39,17 +40,40 @@ #include <functional> // For std::bind_front. #endif // defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 201907L +#include <type_traits> #include <utility> #include "absl/functional/internal/front_binder.h" #include "absl/utility/utility.h" +// ABSL_INTERNAL_HAVE_STD_NTTP_BIND_FRONT +// +// Defined if the standard library implements C++26 `std::bind_front<ConstFn>`, +// which `absl::bind_front<ConstFn>` then aliases. +// +// libc++ 17 through 22 define `__cpp_lib_bind_front` as 202306L in C++26 mode, +// but `std::bind_front<ConstFn>` was first implemented in libc++ 23: +// https://github.com/llvm/llvm-project/pull/165096#issuecomment-4487517016 +#ifdef ABSL_INTERNAL_HAVE_STD_NTTP_BIND_FRONT +#error ABSL_INTERNAL_HAVE_STD_NTTP_BIND_FRONT cannot be directly set +#elif defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 202306L && \ + (!defined(_LIBCPP_VERSION) || _LIBCPP_VERSION >= 230000) +#define ABSL_INTERNAL_HAVE_STD_NTTP_BIND_FRONT 1 +#endif + namespace absl { ABSL_NAMESPACE_BEGIN // bind_front() // // Binds the first N arguments of an invocable object and stores them by value. +// The target invocable can be passed either as a runtime argument +// `absl::bind_front(func, args...)` (C++20 `std::bind_front`) or as a +// compile-time non-type template parameter `absl::bind_front<ConstFn>(args...)` +// (C++26 `std::bind_front`). +// +// Passing `ConstFn` as a non-type template parameter avoids storing a function +// pointer or pointer-to-member at runtime. // // Like `std::bind()`, `absl::bind_front()` is implicitly convertible to // `std::function`. In particular, it may be used as a simpler replacement for @@ -57,7 +81,8 @@ // specified. More importantly, it provides more reliable correctness guarantees // than `std::bind()`; while `std::bind()` will silently ignore passing more // parameters than expected, for example, `absl::bind_front()` will report such -// mis-uses as errors. In C++20, `absl::bind_front` is replaced by +// mis-uses as errors. In C++20 (for runtime callables) and C++26 (for +// non-type template parameter callables), `absl::bind_front` aliases // `std::bind_front`. // // absl::bind_front(a...) can be seen as storing the results of @@ -70,6 +95,8 @@ // assert(absl::bind_front(Minus)(3, 2) == 3 - 2); // assert(absl::bind_front(Minus, 3)(2) == 3 - 2); // assert(absl::bind_front(Minus, 3, 2)() == 3 - 2); +// assert(absl::bind_front<&Minus>()(3, 2) == 3 - 2); +// assert(absl::bind_front<&Minus>(3)(2) == 3 - 2); // // Example: Binding a member function. // @@ -80,8 +107,10 @@ // Math math; // // assert(absl::bind_front(&Math::Double)(&math, 3) == 2 * 3); +// assert(absl::bind_front<&Math::Double>()(&math, 3) == 2 * 3); // // Stores a pointer to math inside the functor. // assert(absl::bind_front(&Math::Double, &math)(3) == 2 * 3); +// assert(absl::bind_front<&Math::Double>(&math)(3) == 2 * 3); // // Stores a copy of math inside the functor. // assert(absl::bind_front(&Math::Double, math)(3) == 2 * 3); // // Stores std::unique_ptr<Math> inside the functor. @@ -193,7 +222,23 @@ } #endif // defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 201907L +#ifndef ABSL_INTERNAL_HAVE_STD_NTTP_BIND_FRONT +template <auto ConstFn, class... BoundArgs> +constexpr auto bind_front(BoundArgs&&... args) { + using F = decltype(ConstFn); + if constexpr (std::is_pointer_v<F> || std::is_member_pointer_v<F>) { + static_assert(ConstFn != nullptr, + "bind_front: ConstFn cannot be equal to nullptr"); + } + // The runtime `bind_front` checks the requirements on `BoundArgs`. + return absl::bind_front(functional_internal::ConstFnInvoker<ConstFn>{}, + std::forward<BoundArgs>(args)...); +} +#endif // ABSL_INTERNAL_HAVE_STD_NTTP_BIND_FRONT + ABSL_NAMESPACE_END } // namespace absl +#undef ABSL_INTERNAL_HAVE_STD_NTTP_BIND_FRONT + #endif // ABSL_FUNCTIONAL_BIND_FRONT_H_
diff --git a/absl/functional/bind_front_test.cc b/absl/functional/bind_front_test.cc index 6e87d43..300466c 100644 --- a/absl/functional/bind_front_test.cc +++ b/absl/functional/bind_front_test.cc
@@ -19,15 +19,17 @@ #include <functional> #include <memory> #include <string> +#include <type_traits> #include <utility> #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/base/config.h" #include "absl/memory/memory.h" namespace { -char CharAt(const char* s, size_t index) { return s[index]; } +constexpr char CharAt(const char* s, size_t index) { return s[index]; } TEST(BindTest, Basics) { EXPECT_EQ('C', absl::bind_front(CharAt)("ABC", 2)); @@ -200,7 +202,7 @@ EXPECT_EQ(42, std::move(f)()); } -int Plus(int a, int b) { return a + b; } +constexpr int Plus(int a, int b) { return a + b; } TEST(BindTest, ConstExpr) { constexpr auto f = absl::bind_front(CharAt); @@ -223,4 +225,224 @@ absl::bind_front(ManglingCall{}, 1, 3.3)("A"); } +struct Math { + constexpr int Add(int a, int b) const noexcept { return base + a + b; } + int base = 10; +}; + +TEST(BindConstFnTest, Basics) { + EXPECT_EQ('C', absl::bind_front<&CharAt>()("ABC", 2)); + EXPECT_EQ('C', absl::bind_front<&CharAt>("ABC")(2)); + EXPECT_EQ('C', absl::bind_front<&CharAt>("ABC", 2)()); +} + +TEST(BindConstFnTest, MemberPointers) { + Math m{10}; + auto bound_ptr = absl::bind_front<&Math::Add>(&m); + EXPECT_EQ(15, bound_ptr(2, 3)); + + auto bound_ptr_and_arg = absl::bind_front<&Math::Add>(&m, 2); + EXPECT_EQ(15, bound_ptr_and_arg(3)); + + auto bound_zero_args = absl::bind_front<&Math::Add>(); + EXPECT_EQ(15, bound_zero_args(&m, 2, 3)); + EXPECT_EQ(15, bound_zero_args(m, 2, 3)); +} + +TEST(BindConstFnTest, StoreByCopyRefAndPointer) { + Struct s = {"hello"}; + auto by_copy = absl::bind_front<&Struct::value>(s); + EXPECT_EQ("hello", by_copy()); + EXPECT_NE(&s.value, &by_copy()); + + NonCopyable nc("hello"); + auto by_ref = absl::bind_front<&GetNonCopyableValue>(std::ref(nc)); + EXPECT_EQ("hello", by_ref()); + EXPECT_EQ(&nc.value, &by_ref()); + + auto by_cref = absl::bind_front<&GetNonCopyableValue>(std::cref(nc)); + EXPECT_EQ("hello", by_cref()); + EXPECT_EQ(&nc.value, &by_cref()); + + auto by_ptr = absl::bind_front<&NonCopyable::value>(&nc); + EXPECT_EQ("hello", by_ptr()); + EXPECT_EQ(&nc.value, &by_ptr()); + nc.value = "goodbye"; + EXPECT_EQ("goodbye", by_ref()); + EXPECT_EQ("goodbye", by_cref()); + EXPECT_EQ("goodbye", by_ptr()); +} + +TEST(BindConstFnTest, MoveOnlyArgsAndResult) { + EXPECT_EQ(42, absl::bind_front<&Sink>()(std::make_unique<int>(42))); + EXPECT_EQ(42, absl::bind_front<&Sink>(std::make_unique<int>(42))()); + EXPECT_THAT(absl::bind_front<&Factory>()(42), ::testing::Pointee(42)); + EXPECT_THAT(absl::bind_front<&Factory>(42)(), ::testing::Pointee(42)); + + FalseCopyable<std::unique_ptr<int>> x; + x.m = std::make_unique<int>(42); + auto f = absl::bind_front<&GetMember>(std::move(x)); + EXPECT_EQ(42, std::move(f)()); +} + +TEST(BindConstFnTest, ConstExpr) { + constexpr auto f = absl::bind_front<&CharAt>(); + EXPECT_EQ(f("ABC", 1), 'B'); + + constexpr auto plus5 = absl::bind_front<&Plus>(5); + EXPECT_EQ(plus5(1), 6); + + constexpr auto plus5_1 = absl::bind_front<&Plus>(5, 1); + EXPECT_EQ(plus5_1(), 6); + + constexpr Math m{10}; + constexpr auto add_m = absl::bind_front<&Math::Add>(m, 2); + EXPECT_EQ(add_m(3), 15); + + // Invoking the results is `constexpr` only where the runtime `bind_front` + // that produces them is `std::bind_front` (C++20). The pre-C++20 fallback's + // call operators and `std::invoke` are not `constexpr`. +#if defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 201907L + static_assert(f("ABC", 1) == 'B'); + static_assert(plus5(1) == 6); + static_assert(plus5_1() == 6); + static_assert(add_m(3) == 15); +#endif // __cpp_lib_bind_front >= 201907L +} + +// The results come from the runtime `bind_front`, which propagates `noexcept` +// where it is `std::bind_front` (C++20). +#if defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 201907L +int NoexceptPlus(int a, int b) noexcept { return a + b; } +int PotentiallyThrowingPlus(int a, int b) { return a + b; } + +TEST(BindConstFnTest, NoexceptPropagation) { + auto noexcept_0 = absl::bind_front<&NoexceptPlus>(); + static_assert(noexcept(noexcept_0(1, 2))); + + auto throwing_0 = absl::bind_front<&PotentiallyThrowingPlus>(); + static_assert(!noexcept(throwing_0(1, 2))); + + Math m{10}; + auto math_0 = absl::bind_front<&Math::Add>(); + static_assert(noexcept(math_0(m, 1, 2))); +} +#endif // __cpp_lib_bind_front >= 201907L + +// Invalid calls must be substitution failures rather than hard errors, so that +// `std::is_invocable` works on the result. +TEST(BindConstFnTest, SfinaeFriendly) { + using PlusBinder = decltype(absl::bind_front<&Plus>()); + static_assert(std::is_invocable_v<PlusBinder, int, int>); + static_assert(!std::is_invocable_v<PlusBinder, std::string, int>); + static_assert(!std::is_invocable_v<PlusBinder, int>); + + using Plus1Binder = decltype(absl::bind_front<&Plus>(1)); + static_assert(std::is_invocable_v<Plus1Binder, int>); + static_assert(!std::is_invocable_v<Plus1Binder, std::string>); +} + +// These tests pass class-type objects as template arguments (P0732R2, +// P1907R1), and `StatelessLambdaNTTP` puts a lambda in a template argument +// (P0315R4, which has no feature-test macro). +// +// We check the language version rather than +// `__cpp_nontype_template_args >= 201911L` because Clang (through at least +// 24) still defines it as 201411L: its P1907R1 support is incomplete, though +// everything used here works. Checking the feature macro would silently skip +// these tests on Clang. If a toolchain claims C++20 but can't compile these +// tests, exclude it here. +#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L +TEST(BindConstFnTest, StatelessLambdaNTTP) { + constexpr auto f = absl::bind_front<[](int x, int y) { return x + y; }>(); + EXPECT_EQ(f(2, 3), 5); + + constexpr auto f2 = absl::bind_front<[](int x, int y) { return x + y; }>(2); + EXPECT_EQ(f2(3), 5); +} + +struct Adder { + constexpr int operator()(int x) const& { return base + x; } + // `ConstFn` names a template parameter object, which is a `const` lvalue + // ([temp.param]), so it must never be invoked as an rvalue. + int operator()(int) const&& = delete; + + int base; +}; + +TEST(BindConstFnTest, StatefulObjectNTTP) { + constexpr auto add5 = absl::bind_front<Adder{5}>(); + EXPECT_EQ(add5(1), 6); + + constexpr auto add5_1 = absl::bind_front<Adder{5}>(1); + EXPECT_EQ(add5_1(), 6); +} + +struct DescribeCall { + template <class... Args> + std::string operator()(Args&&...) & { + return "&"; + } + template <class... Args> + std::string operator()(Args&&...) const& { + return "const&"; + } + template <class... Args> + std::string operator()(Args&&...) && { + return "&&"; + } + template <class... Args> + std::string operator()(Args&&...) const&& { + return "const&&"; + } +}; + +// Since `ConstFn` is a `const` lvalue, it is invoked through its `const&` +// overload whatever the value category of the result. Both zero and nonzero +// bound arguments are tested, since the standard library's +// `std::bind_front<ConstFn>` may implement them differently. +TEST(BindConstFnTest, InvokesConstFnAsConstLvalue) { + auto f0 = absl::bind_front<DescribeCall{}>(); + const auto& cf0 = f0; + EXPECT_EQ("const&", f0()); + EXPECT_EQ("const&", cf0()); + EXPECT_EQ("const&", std::move(f0)()); + EXPECT_EQ("const&", std::move(cf0)()); + + auto f1 = absl::bind_front<DescribeCall{}>(1); + const auto& cf1 = f1; + EXPECT_EQ("const&", f1()); + EXPECT_EQ("const&", cf1()); + EXPECT_EQ("const&", std::move(f1)()); + EXPECT_EQ("const&", std::move(cf1)()); +} + +struct DeletedConstLvalueCall { + template <class... Args> + void operator()(Args&&...) &; + template <class... Args> + void operator()(Args&&...) const& = delete; + template <class... Args> + void operator()(Args&&...) &&; + template <class... Args> + void operator()(Args&&...) const&&; +}; + +// If `ConstFn`'s `const&` overload is deleted, the result isn't invocable, +// rather than falling back to another overload. +TEST(BindConstFnTest, NotInvocableIfConstLvalueCallDeleted) { + using F0 = decltype(absl::bind_front<DeletedConstLvalueCall{}>()); + static_assert(!std::is_invocable_v<F0&>); + static_assert(!std::is_invocable_v<const F0&>); + static_assert(!std::is_invocable_v<F0>); + static_assert(!std::is_invocable_v<const F0>); + + using F1 = decltype(absl::bind_front<DeletedConstLvalueCall{}>(1)); + static_assert(!std::is_invocable_v<F1&>); + static_assert(!std::is_invocable_v<const F1&>); + static_assert(!std::is_invocable_v<F1>); + static_assert(!std::is_invocable_v<const F1>); +} +#endif // ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L + } // namespace
diff --git a/absl/functional/internal/front_binder.h b/absl/functional/internal/front_binder.h index fcada9c..33d4fb4 100644 --- a/absl/functional/internal/front_binder.h +++ b/absl/functional/internal/front_binder.h
@@ -18,6 +18,7 @@ #define ABSL_FUNCTIONAL_INTERNAL_FRONT_BINDER_H_ #include <cstddef> +#include <functional> #include <type_traits> #include <utility> @@ -87,6 +88,34 @@ template <class F, class... BoundArgs> using bind_front_t = FrontBinder<std::decay_t<F>, std::decay_t<BoundArgs>...>; +// Stateless invoker for the compile-time constant target `ConstFn` of +// `absl::bind_front<ConstFn>(args...)` (C++26 [func.bind.front]). +// +// `operator()` has a trailing `decltype(std::invoke(ConstFn, ...))` return type +// rather than `decltype(auto)` so that an invalid call is a substitution +// failure instead of a hard error. Deducing a `decltype(auto)` return type +// instantiates the body, so merely checking whether a call is valid would fail +// to compile. That would break `std::is_invocable` and overload resolution, +// including among the call operators of a `bind_front` result wrapping the +// invoker: e.g. calling such a result as an rvalue fails if it holds a +// move-only bound argument, because its `const&` overload is also considered. +// +// The return type is `decltype(std::invoke(ConstFn, ...))` rather than +// `std::invoke_result_t<decltype(ConstFn), ...>` because for C++20 class-type +// non-type template parameters, `decltype(ConstFn)` is the unqualified type `F` +// (which `std::invoke_result_t` treats as an rvalue `F&&`), whereas the +// template parameter expression `ConstFn` is a `const F` lvalue as specified by +// [func.bind.front]. +template <auto ConstFn> +struct ConstFnInvoker { + template <class... FreeArgs> + constexpr auto operator()(FreeArgs&&... free_args) const noexcept( + noexcept(std::invoke(ConstFn, std::forward<FreeArgs>(free_args)...))) + -> decltype(std::invoke(ConstFn, std::forward<FreeArgs>(free_args)...)) { + return std::invoke(ConstFn, std::forward<FreeArgs>(free_args)...); + } +}; + } // namespace functional_internal ABSL_NAMESPACE_END } // namespace absl
diff --git a/absl/hash/BUILD.bazel b/absl/hash/BUILD.bazel index fe76a1d..010668d 100644 --- a/absl/hash/BUILD.bazel +++ b/absl/hash/BUILD.bazel
@@ -89,6 +89,7 @@ ], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + shard_count = 4, deps = [ ":hash", ":hash_testing",
diff --git a/absl/hash/CMakeLists.txt b/absl/hash/CMakeLists.txt index 2ced4b1..f8e7afa 100644 --- a/absl/hash/CMakeLists.txt +++ b/absl/hash/CMakeLists.txt
@@ -78,6 +78,8 @@ absl::string_view absl::strings GTest::gmock_main + SHARD_COUNT + 4 ) absl_cc_test(
diff --git a/absl/log/BUILD.bazel b/absl/log/BUILD.bazel index 2c0a8ff..3e15e0b 100644 --- a/absl/log/BUILD.bazel +++ b/absl/log/BUILD.bazel
@@ -315,6 +315,7 @@ srcs = ["absl_check_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + shard_count = 4, tags = [ "no_test:os:ios", "no_test_ios", @@ -350,6 +351,7 @@ srcs = ["check_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + shard_count = 4, tags = [ "no_test:os:ios", "no_test_ios",
diff --git a/absl/log/CMakeLists.txt b/absl/log/CMakeLists.txt index ebc24f3..1da38e4 100644 --- a/absl/log/CMakeLists.txt +++ b/absl/log/CMakeLists.txt
@@ -838,6 +838,8 @@ absl::string_view absl::strings GTest::gmock_main + SHARD_COUNT + 4 ) absl_cc_test( @@ -883,6 +885,8 @@ absl::string_view absl::strings GTest::gmock_main + SHARD_COUNT + 4 ) absl_cc_test(
diff --git a/absl/random/BUILD.bazel b/absl/random/BUILD.bazel index edc4e93..d63f8f3 100644 --- a/absl/random/BUILD.bazel +++ b/absl/random/BUILD.bazel
@@ -199,6 +199,7 @@ srcs = ["bernoulli_distribution_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + shard_count = 4, deps = [ ":distributions", ":random", @@ -217,6 +218,7 @@ copts = ABSL_TEST_COPTS, flaky = 1, linkopts = ABSL_DEFAULT_LINKOPTS, + shard_count = 4, tags = [ "no_test_wasm", ], @@ -326,6 +328,7 @@ ], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + shard_count = 4, tags = [ # Too Slow. "no_test_android_arm",
diff --git a/absl/random/CMakeLists.txt b/absl/random/CMakeLists.txt index 8a31015..f9d2d60 100644 --- a/absl/random/CMakeLists.txt +++ b/absl/random/CMakeLists.txt
@@ -272,6 +272,8 @@ absl::random_random GTest::gmock GTest::gtest_main + SHARD_COUNT + 4 ) absl_cc_test( @@ -295,6 +297,8 @@ absl::strings GTest::gmock GTest::gtest_main + SHARD_COUNT + 4 ) absl_cc_test( @@ -398,6 +402,8 @@ absl::strings GTest::gmock GTest::gtest_main + SHARD_COUNT + 4 ) absl_cc_test(
diff --git a/absl/strings/BUILD.bazel b/absl/strings/BUILD.bazel index 5fe9b68..756300e 100644 --- a/absl/strings/BUILD.bazel +++ b/absl/strings/BUILD.bazel
@@ -598,6 +598,7 @@ timeout = "long", srcs = ["internal/cord_rep_btree_test.cc"], copts = ABSL_TEST_COPTS, + shard_count = 4, visibility = ["//visibility:private"], deps = [ ":cord_internal", @@ -710,10 +711,10 @@ ":cordz_info", ":cordz_update_scope", ":cordz_update_tracker", - ":internal", ":resize_and_overwrite", ":string_view", ":strings", + "//absl/algorithm:container", "//absl/base:config", "//absl/base:core_headers", "//absl/base:endian", @@ -1039,6 +1040,7 @@ timeout = "long", srcs = ["cord_test.cc"], copts = ABSL_TEST_COPTS, + shard_count = 4, visibility = ["//visibility:private"], deps = [ ":cord", @@ -1565,6 +1567,7 @@ timeout = "long", srcs = ["internal/str_format/convert_test.cc"], copts = ABSL_TEST_COPTS, + shard_count = 4, visibility = ["//visibility:private"], deps = [ ":str_format",
diff --git a/absl/strings/CMakeLists.txt b/absl/strings/CMakeLists.txt index 1461d94..8ba84b5 100644 --- a/absl/strings/CMakeLists.txt +++ b/absl/strings/CMakeLists.txt
@@ -714,6 +714,8 @@ absl::string_view absl::strings GTest::gmock_main + SHARD_COUNT + 4 ) absl_cc_test( @@ -1196,6 +1198,8 @@ absl::string_view absl::strings GTest::gmock_main + SHARD_COUNT + 4 ) absl_cc_test( @@ -1233,6 +1237,8 @@ absl::string_view absl::strings GTest::gmock_main + SHARD_COUNT + 4 ) absl_cc_test(
diff --git a/absl/strings/internal/str_format/convert_test.cc b/absl/strings/internal/str_format/convert_test.cc index 16fcaa9..9aa5671 100644 --- a/absl/strings/internal/str_format/convert_test.cc +++ b/absl/strings/internal/str_format/convert_test.cc
@@ -47,6 +47,8 @@ #include "absl/strings/internal/str_format/arg.h" #include "absl/strings/internal/str_format/bind.h" #include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/types/span.h" @@ -414,11 +416,11 @@ if (ptr == nullptr && arg == "(nil)") { return true; } - void* parsed = nullptr; - if (sscanf(arg.c_str(), "%p", &parsed) != 1) { + uintptr_t parsed = 0; + if (!absl::SimpleHexAtoi(arg, &parsed)) { LOG(FATAL) << "Could not parse " << arg; } - return ptr == parsed; + return ptr == reinterpret_cast<void*>(parsed); } TEST_F(FormatConvertTest, Pointer) { @@ -1017,6 +1019,35 @@ } } +TEST_F(FormatConvertTest, ScientificZeroPadding) { + struct Case { + absl::string_view format; + // Positive and negative expected prefixes. + absl::string_view positive; + absl::string_view negative; + } cases[] = { + {"%010.0e", "000003e", "-00003e"}, {"%010.1g", "000003e", "-00003e"}, + {"%010.0E", "000003E", "-00003E"}, {"%010.1G", "000003E", "-00003E"}, + {"%+010.0e", "+00003e", "-00003e"}, {"%+010.1g", "+00003e", "-00003e"}, + {"% 010.0e", " 00003e", "-00003e"}, {"% 010.1g", " 00003e", "-00003e"}, + {"%#010.0e", "00003.e", "-0003.e"}, {"%#010.1g", "00003.e", "-0003.e"}, + }; + + // These exact values exercise both slow scientific conversion paths. + for (double value : {0x1p128, 0x1p-128}) { + const absl::string_view exponent = value > 1 ? "+38" : "-39"; + for (const auto& c : cases) { + SCOPED_TRACE(absl::StrCat(value, ", ", c.format)); + EXPECT_EQ( + FormatPack(UntypedFormatSpecImpl(c.format), {FormatArgImpl(value)}), + absl::StrCat(c.positive, exponent)); + EXPECT_EQ( + FormatPack(UntypedFormatSpecImpl(c.format), {FormatArgImpl(-value)}), + absl::StrCat(c.negative, exponent)); + } + } +} + TEST_F(FormatConvertTest, DoubleRound) { std::string s; const auto format = [&](const char *fmt, double d) -> std::string & {
diff --git a/absl/strings/internal/str_format/float_conversion.cc b/absl/strings/internal/str_format/float_conversion.cc index 137e159..0895c31 100644 --- a/absl/strings/internal/str_format/float_conversion.cc +++ b/absl/strings/internal/str_format/float_conversion.cc
@@ -1496,6 +1496,7 @@ if (state.sign_char != '\0') { state.sink->Append(1, state.sign_char); } + state.sink->Append(padding.zeros, '0'); state.sink->Append(1, static_cast<char>(first_digit + '0')); if (precision > 0 || state.conv.has_alt_flag()) { @@ -1599,6 +1600,7 @@ if (state.sign_char != '\0') { state.sink->Append(1, state.sign_char); } + state.sink->Append(padding.zeros, '0'); state.sink->Append(1, static_cast<char>(first_digit + '0')); --digits_to_go; if (print_dot) {
diff --git a/absl/strings/match.h b/absl/strings/match.h index 431d4f5..65bd8aa 100644 --- a/absl/strings/match.h +++ b/absl/strings/match.h
@@ -80,7 +80,8 @@ if (text.size() < suffix.size()) { return false; } - absl::string_view possible_match = text.substr(text.size() - suffix.size()); + absl::string_view possible_match = + text.substr(text.size() - suffix.size(), suffix.size()); return possible_match == suffix; } // StrContainsIgnoreCase()
diff --git a/absl/strings/str_cat.h b/absl/strings/str_cat.h index 3ae1197..11e3bf6 100644 --- a/absl/strings/str_cat.h +++ b/absl/strings/str_cat.h
@@ -382,7 +382,8 @@ &digits_[0])) {} AlphaNum(float f) // NOLINT(runtime/explicit) - : piece_(digits_, numbers_internal::SixDigitsToBuffer(f, digits_)) {} + : piece_(digits_, numbers_internal::SixDigitsToBuffer( + static_cast<double>(f), digits_)) {} AlphaNum(double f) // NOLINT(runtime/explicit) : piece_(digits_, numbers_internal::SixDigitsToBuffer(f, digits_)) {} @@ -517,8 +518,8 @@ std::string result; StringResizeAndOverwrite(result, numbers_internal::kSixDigitsToBufferSize, [f](char* start, size_t buf_size) { - size_t size = - numbers_internal::SixDigitsToBuffer(f, start); + size_t size = numbers_internal::SixDigitsToBuffer( + static_cast<double>(f), start); ABSL_ASSERT(size < buf_size); return size; });
diff --git a/absl/synchronization/BUILD.bazel b/absl/synchronization/BUILD.bazel index 977af61..7acbec4 100644 --- a/absl/synchronization/BUILD.bazel +++ b/absl/synchronization/BUILD.bazel
@@ -382,6 +382,7 @@ copts = ABSL_TEST_COPTS, flaky = 1, linkopts = ABSL_DEFAULT_LINKOPTS, + shard_count = 4, deps = [ ":kernel_timeout_internal", ":synchronization",
diff --git a/absl/synchronization/CMakeLists.txt b/absl/synchronization/CMakeLists.txt index 3502d11..50129ce 100644 --- a/absl/synchronization/CMakeLists.txt +++ b/absl/synchronization/CMakeLists.txt
@@ -198,6 +198,8 @@ absl::thread_pool absl::time GTest::gmock_main + SHARD_COUNT + 25 ) absl_cc_test( @@ -278,6 +280,8 @@ absl::thread_pool absl::time GTest::gmock_main + SHARD_COUNT + 4 ) absl_cc_test(
diff --git a/absl/synchronization/mutex.cc b/absl/synchronization/mutex.cc index d6e2aac..d5c1a3c 100644 --- a/absl/synchronization/mutex.cc +++ b/absl/synchronization/mutex.cc
@@ -538,7 +538,14 @@ struct { Mutex* mu; // lock acquired int32_t count; // times acquired - GraphId id; // deadlock_graph id of acquired lock + // True iff this thread called mu->ForgetDeadlockInfo() while holding mu. + // The entry is then ignored for lock ordering until it is released. + bool forgotten; + // deadlock_graph id of mu, or InvalidGraphId() until DeadlockCheck() + // needs it, which is when this thread acquires another lock while + // holding mu. This way, acquiring a lock while holding no others, and + // releasing a lock, do not need deadlock_graph_mu. + GraphId id; } locks[40]; // If a thread overfills the array during deadlock detection, we // continue, discarding information as needed. If no overflow has @@ -559,6 +566,14 @@ return ret; } +static void RemoveHeldLockEntry(SynchLocksHeld* held_locks, int i) { + const int last = --held_locks->n; + held_locks->locks[i] = held_locks->locks[last]; + held_locks->locks[last].id = InvalidGraphId(); + held_locks->locks[last].mu = + nullptr; // clear mu to please the leak detector. +} + // Return the PerThreadSynch-struct for this thread. static PerThreadSynch* Synch_GetPerThread() { ThreadIdentity* identity = GetOrCreateCurrentThreadIdentity(); @@ -757,6 +772,20 @@ defined(ABSL_BUILD_DLL) void Mutex::Dtor() { if (kDebugMode) { + // If this thread is destroying a locked Mutex, remove any held-lock + // entries for it so their slots are not leaked. + ThreadIdentity* identity = CurrentThreadIdentityIfPresent(); + if (identity != nullptr && + identity->per_thread_synch.all_locks != nullptr) { + SynchLocksHeld* held_locks = identity->per_thread_synch.all_locks; + for (int i = 0; i < held_locks->n;) { + if (held_locks->locks[i].mu == this) { + RemoveHeldLockEntry(held_locks, i); + } else { + i++; + } + } + } this->ForgetDeadlockInfo(); } ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static); @@ -1231,28 +1260,35 @@ return deadlock_graph->GetId(mu); } -static GraphId GetGraphId(Mutex* mu) ABSL_LOCKS_EXCLUDED(deadlock_graph_mu) { - base_internal::SpinLockHolder l(deadlock_graph_mu); - GraphId id = GetGraphIdLocked(mu); - return id; +// Returns the index of the entry for mu in held_locks, or -1 if there is none. +// Prefers the entry that is not marked forgotten (there is at most one), and +// otherwise returns the first entry that is. +static int FindHeldLock(const SynchLocksHeld* held_locks, const Mutex* mu) { + int forgotten_index = -1; + for (int i = 0; i != held_locks->n; i++) { + if (held_locks->locks[i].mu == mu) { + if (!held_locks->locks[i].forgotten) return i; + if (forgotten_index == -1) forgotten_index = i; + } + } + return forgotten_index; } // Record a lock acquisition. This is used in debug mode for deadlock // detection. The held_locks pointer points to the relevant data // structure for each case. -static void LockEnter(Mutex* mu, GraphId id, SynchLocksHeld* held_locks) { +static void LockEnter(Mutex* mu, SynchLocksHeld* held_locks) { int n = held_locks->n; - int i = 0; - while (i != n && held_locks->locks[i].id != id) { - i++; - } - if (i == n) { + int i = FindHeldLock(held_locks, mu); + // A forgotten hold stays ignored, so a new hold of mu gets its own entry. + if (i == -1 || held_locks->locks[i].forgotten) { if (n == static_cast<int>(std::size(held_locks->locks))) { held_locks->overflow = true; // lost some data } else { // we have room for lock - held_locks->locks[i].mu = mu; - held_locks->locks[i].count = 1; - held_locks->locks[i].id = id; + held_locks->locks[n].mu = mu; + held_locks->locks[n].count = 1; + held_locks->locks[n].forgotten = false; + held_locks->locks[n].id = InvalidGraphId(); held_locks->n = n + 1; } } else { @@ -1260,39 +1296,28 @@ } } -// Record a lock release. Each call to LockEnter(mu, id, x) should be -// eventually followed by a call to LockLeave(mu, id, x) by the same thread. +// Record a lock release. Each call to LockEnter(mu, x) should be +// eventually followed by a call to LockLeave(mu, x) by the same thread. // It does not process the event if is not needed when deadlock detection is // disabled. -static void LockLeave(Mutex* mu, GraphId id, SynchLocksHeld* held_locks) { - int n = held_locks->n; - int i = 0; - while (i != n && held_locks->locks[i].id != id) { - i++; - } - if (i == n) { - if (!held_locks->overflow) { - // The deadlock id may have been reassigned after ForgetDeadlockInfo, - // but in that case mu should still be present. - i = 0; - while (i != n && held_locks->locks[i].mu != mu) { - i++; - } - if (i == n) { // mu missing means releasing unheld lock - SynchEvent* mu_events = GetSynchEvent(mu); - ABSL_RAW_LOG(FATAL, - "thread releasing lock it does not hold: %p %s; " - , - static_cast<void*>(mu), - mu_events == nullptr ? "" : mu_events->name); - } +static void LockLeave(Mutex* mu, SynchLocksHeld* held_locks) { + // If mu was forgotten while held, this releases the forgotten entry. + int i = FindHeldLock(held_locks, mu); + if (i == -1) { + // The thread does not hold mu, unless held_locks was full when mu was + // acquired. + if (held_locks->overflow) { + return; } - } else if (held_locks->locks[i].count == 1) { - held_locks->n = n - 1; - held_locks->locks[i] = held_locks->locks[n - 1]; - held_locks->locks[n - 1].id = InvalidGraphId(); - held_locks->locks[n - 1].mu = - nullptr; // clear mu to please the leak detector. + SynchEvent* mu_events = GetSynchEvent(mu); + ABSL_RAW_LOG(FATAL, + "thread releasing lock it does not hold: %p %s; " + , + static_cast<void*>(mu), + mu_events == nullptr ? "" : mu_events->name); + } + if (held_locks->locks[i].count == 1) { + RemoveHeldLockEntry(held_locks, i); } else { assert(held_locks->locks[i].count > 0); held_locks->locks[i].count--; @@ -1304,17 +1329,7 @@ if (kDebugMode) { if (synch_deadlock_detection.load(std::memory_order_acquire) != OnDeadlockCycle::kIgnore) { - LockEnter(mu, GetGraphId(mu), Synch_GetAllLocks()); - } - } -} - -// Call LockEnter() if in debug mode and deadlock detection is enabled. -static inline void DebugOnlyLockEnter(Mutex* mu, GraphId id) { - if (kDebugMode) { - if (synch_deadlock_detection.load(std::memory_order_acquire) != - OnDeadlockCycle::kIgnore) { - LockEnter(mu, id, Synch_GetAllLocks()); + LockEnter(mu, Synch_GetAllLocks()); } } } @@ -1324,7 +1339,7 @@ if (kDebugMode) { if (synch_deadlock_detection.load(std::memory_order_acquire) != OnDeadlockCycle::kIgnore) { - LockLeave(mu, GetGraphId(mu), Synch_GetAllLocks()); + LockLeave(mu, Synch_GetAllLocks()); } } } @@ -1384,27 +1399,56 @@ } } // anonymous namespace +// Returns true iff held_locks has an entry that counts for lock ordering, that +// is, one that is not marked forgotten. +static bool HoldsUnforgottenLock(const SynchLocksHeld* held_locks) { + for (int i = 0; i != held_locks->n; i++) { + if (!held_locks->locks[i].forgotten) return true; + } + return false; +} + +// Looks up ids for the entries in held_locks that are not marked forgotten and +// do not have a current id, because this thread has not needed it yet, or +// because another thread called ForgetDeadlockInfo() on that Mutex. +static void UpdateHeldLockIds(SynchLocksHeld* held_locks) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(deadlock_graph_mu) { + for (int i = 0; i != held_locks->n; i++) { + if (held_locks->locks[i].forgotten) continue; + const GraphId id = held_locks->locks[i].id; + if (id == InvalidGraphId() || deadlock_graph->Ptr(id) == nullptr) { + held_locks->locks[i].id = GetGraphIdLocked(held_locks->locks[i].mu); + } + } +} + // Called in debug mode when a thread is about to acquire a lock in a way that // may block. -static GraphId DeadlockCheck(Mutex* mu) { +static void DeadlockCheck(Mutex* mu) { if (synch_deadlock_detection.load(std::memory_order_acquire) == OnDeadlockCycle::kIgnore) { - return InvalidGraphId(); + return; } SynchLocksHeld* all_locks = Synch_GetAllLocks(); - absl::base_internal::SpinLockHolder lock(deadlock_graph_mu); - const GraphId mu_id = GetGraphIdLocked(mu); - - if (all_locks->n == 0) { - // There are no other locks held. Return now so that we don't need to - // call GetSynchEvent(). This way we do not record the stack trace - // for this Mutex. It's ok, since if this Mutex is involved in a deadlock, - // it can't always be the first lock acquired by a thread. - return mu_id; + if (!HoldsUnforgottenLock(all_locks)) { + // There are no other locks held, so there are no edges to add. Return now + // so that we don't need to take deadlock_graph_mu or call GetSynchEvent(). + // This way we do not record the stack trace for this Mutex. It's ok, since + // if this Mutex is involved in a deadlock, it can't always be the first + // lock acquired by a thread. + return; } + absl::base_internal::SpinLockHolder lock(deadlock_graph_mu); + + // Look up the held locks' ids before mu's, so that when both need new graph + // slots, the held lock's slot ranks below mu's and the edge between them is + // already in rank order. + UpdateHeldLockIds(all_locks); + const GraphId mu_id = GetGraphIdLocked(mu); + // We prefer to keep stack traces that show a thread holding and acquiring // as many locks as possible. This increases the chances that a given edge // in the acquires-before graph will be represented in the stack traces @@ -1413,6 +1457,7 @@ // For each other mutex already held by this thread: for (int i = 0; i != all_locks->n; i++) { + if (all_locks->locks[i].forgotten) continue; const GraphId other_node_id = all_locks->locks[i].id; const Mutex* other = static_cast<const Mutex*>(deadlock_graph->Ptr(other_node_id)); @@ -1433,6 +1478,7 @@ CurrentStackString(b->buf, sizeof (b->buf), symbolize)); size_t len = 0; for (int j = 0; j != all_locks->n; j++) { + if (all_locks->locks[j].forgotten) continue; void* pr = deadlock_graph->Ptr(all_locks->locks[j].id); if (pr != nullptr) { snprintf(b->buf + len, sizeof(b->buf) - len, " %p", pr); @@ -1467,29 +1513,37 @@ OnDeadlockCycle::kAbort) { deadlock_graph_mu.unlock(); // avoid deadlock in fatal sighandler ABSL_RAW_LOG(FATAL, "dying due to potential deadlock"); - return mu_id; } break; // report at most one potential deadlock per acquisition } } - - return mu_id; } // Invoke DeadlockCheck() iff we're in debug mode and // deadlock checking has been enabled. -static inline GraphId DebugOnlyDeadlockCheck(Mutex* mu) { +static inline void DebugOnlyDeadlockCheck(Mutex* mu) { if (kDebugMode && synch_deadlock_detection.load(std::memory_order_acquire) != OnDeadlockCycle::kIgnore) { - return DeadlockCheck(mu); - } else { - return InvalidGraphId(); + DeadlockCheck(mu); } } void Mutex::ForgetDeadlockInfo() { if (kDebugMode && synch_deadlock_detection.load(std::memory_order_acquire) != OnDeadlockCycle::kIgnore) { + // If this thread holds this Mutex, ignore that hold from now on (see + // SynchLocksHeld::forgotten). Holds by other threads keep counting: + // RemoveNode() makes their ids stale, and DeadlockCheck() looks up the + // new id when they are next needed. + ThreadIdentity* identity = CurrentThreadIdentityIfPresent(); + if (identity != nullptr && + identity->per_thread_synch.all_locks != nullptr) { + SynchLocksHeld* held_locks = identity->per_thread_synch.all_locks; + int i = FindHeldLock(held_locks, this); + if (i != -1) { + held_locks->locks[i].forgotten = true; + } + } deadlock_graph_mu.lock(); if (deadlock_graph != nullptr) { deadlock_graph->RemoveNode(this); @@ -1505,10 +1559,9 @@ (mu_.load(std::memory_order_relaxed) & (kMuWriter | kMuReader)) != 0 && synch_deadlock_detection.load(std::memory_order_acquire) != OnDeadlockCycle::kIgnore) { - GraphId id = GetGraphId(const_cast<Mutex*>(this)); SynchLocksHeld* locks = Synch_GetAllLocks(); for (int i = 0; i != locks->n; i++) { - if (locks->locks[i].id == id) { + if (locks->locks[i].mu == this) { SynchEvent* mu_events = GetSynchEvent(this); ABSL_RAW_LOG(FATAL, "thread should not hold mutex %p %s", static_cast<const void*>(this), @@ -1538,7 +1591,7 @@ void Mutex::lock() { ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); - GraphId id = DebugOnlyDeadlockCheck(this); + DebugOnlyDeadlockCheck(this); intptr_t v = mu_.load(std::memory_order_relaxed); // try fast acquire, then spin loop if (ABSL_PREDICT_FALSE((v & (kMuWriter | kMuReader | kMuEvent)) != 0) || @@ -1550,13 +1603,13 @@ this->LockSlow(kExclusive, nullptr, 0); } } - DebugOnlyLockEnter(this, id); + DebugOnlyLockEnter(this); ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); } void Mutex::lock_shared() { ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock); - GraphId id = DebugOnlyDeadlockCheck(this); + DebugOnlyDeadlockCheck(this); intptr_t v = mu_.load(std::memory_order_relaxed); for (;;) { // If there are non-readers holding the lock, use the slow loop. @@ -1572,7 +1625,7 @@ break; } } - DebugOnlyLockEnter(this, id); + DebugOnlyLockEnter(this); ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); } @@ -1581,9 +1634,9 @@ bool write) { MuHow how = write ? kExclusive : kShared; ABSL_TSAN_MUTEX_PRE_LOCK(this, TsanFlags(how)); - GraphId id = DebugOnlyDeadlockCheck(this); + DebugOnlyDeadlockCheck(this); bool res = LockSlowWithDeadline(how, &cond, t, 0); - DebugOnlyLockEnter(this, id); + DebugOnlyLockEnter(this); ABSL_TSAN_MUTEX_POST_LOCK(this, TsanFlags(how), 0); return res; }
diff --git a/absl/synchronization/mutex_test.cc b/absl/synchronization/mutex_test.cc index a96bcff..50b982e 100644 --- a/absl/synchronization/mutex_test.cc +++ b/absl/synchronization/mutex_test.cc
@@ -1357,6 +1357,288 @@ c.unlock(); } +// The tests below check what the deadlock detector reports, so they need it to +// be compiled in. Under TSAN, the lock cycles they create would also trigger +// TSAN's own reports. +#if !defined(NDEBUG) && !defined(ABSL_HAVE_THREAD_SANITIZER) +TEST(Mutex, DeadlockDetectorReportsCycle) ABSL_NO_THREAD_SAFETY_ANALYSIS { + EXPECT_DEATH_IF_SUPPORTED( + { + ScopedDisableBazelTestWarnings disable_bazel_test_warnings; + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + absl::Mutex a; + absl::Mutex b; + // Record a -> b. a is acquired with try_lock() while no other lock is + // held, so its hold must be recorded without a deadlock check. + ASSERT_TRUE(a.try_lock()); + b.lock(); + b.unlock(); + a.unlock(); + // Acquiring a while holding b closes the cycle. + b.lock(); + a.lock(); + }, + "dying due to potential deadlock"); +} + +// ForgetDeadlockInfo(), which ~Mutex() also calls, must remove the lock order +// recorded for the Mutex, including by other threads. +TEST(Mutex, ForgetDeadlockInfoForgetsLockOrder) +ABSL_NO_THREAD_SAFETY_ANALYSIS { + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + absl::Mutex a; + absl::Mutex b; + std::thread other([&]() ABSL_NO_THREAD_SAFETY_ANALYSIS { + a.lock(); + b.lock(); // Records a -> b. + b.unlock(); + a.unlock(); + }); + other.join(); + a.ForgetDeadlockInfo(); + // With a -> b forgotten, acquiring a while holding b is not a cycle. + b.lock(); + a.lock(); + a.unlock(); + b.unlock(); +} + +TEST(Mutex, ForgetDeadlockInfoWhileHeldDoesNotMaskOtherHeldLocks) +ABSL_NO_THREAD_SAFETY_ANALYSIS { + EXPECT_DEATH_IF_SUPPORTED( + { + ScopedDisableBazelTestWarnings disable_bazel_test_warnings; + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + absl::Mutex a; + absl::Mutex b; + absl::Mutex c; + // Record c -> a. + c.lock(); + a.lock(); + a.unlock(); + c.unlock(); + // Forgetting b while holding a and b must ignore only the hold of b. + a.lock(); + ASSERT_TRUE(b.try_lock_shared()); + b.ForgetDeadlockInfo(); + b.unlock_shared(); + // Acquiring c while holding a closes the cycle. + c.lock(); + }, + "dying due to potential deadlock"); +} + +// Forgetting a Mutex while another thread is blocked acquiring it must not +// hide the lock order of that thread's hold once it acquires the Mutex. +TEST(Mutex, ForgetDeadlockInfoWhileWaiterBlockedDoesNotMaskSubsequentHold) +ABSL_NO_THREAD_SAFETY_ANALYSIS { + for (bool waiter_holds_other_lock : {false, true}) { + EXPECT_DEATH_IF_SUPPORTED( + { + ScopedDisableBazelTestWarnings disable_bazel_test_warnings; + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + absl::Mutex other; + absl::Mutex a; + absl::Mutex b; + std::atomic<bool> about_to_lock{false}; + { + a.lock_shared(); + absl::synchronization_internal::ThreadPool pool(1); + pool.Schedule([&]() ABSL_NO_THREAD_SAFETY_ANALYSIS { + // If the waiter holds another lock, the deadlock check looks up + // the deadlock graph node of a before the waiter blocks. + if (waiter_holds_other_lock) other.lock(); + about_to_lock.store(true, std::memory_order_release); + a.lock(); // Blocks until a.unlock_shared() below. + b.lock(); // Records a -> b. + b.unlock(); + a.unlock(); + if (waiter_holds_other_lock) other.unlock(); + }); + while (!about_to_lock.load(std::memory_order_acquire)) { + absl::SleepFor(absl::Milliseconds(1)); + } + absl::SleepFor(absl::Milliseconds(20)); // Let the waiter block. + a.ForgetDeadlockInfo(); + a.unlock_shared(); + } + // Acquiring a while holding b closes the cycle. + b.lock(); + a.lock(); + }, + "dying due to potential deadlock"); + } +} + +// A hold by another thread keeps counting for lock ordering after +// ForgetDeadlockInfo(), even if that thread already used it to record lock +// order before the Mutex was forgotten. +TEST(Mutex, ForgetDeadlockInfoDoesNotMaskOtherThreadsHold) +ABSL_NO_THREAD_SAFETY_ANALYSIS { + EXPECT_DEATH_IF_SUPPORTED( + { + ScopedDisableBazelTestWarnings disable_bazel_test_warnings; + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + absl::Mutex a; + absl::Mutex b; + absl::Mutex c; + std::atomic<bool> recorded{false}; + std::atomic<bool> forgotten{false}; + std::thread holder([&]() ABSL_NO_THREAD_SAFETY_ANALYSIS { + a.lock(); + b.lock(); // Records a -> b. + b.unlock(); + recorded.store(true, std::memory_order_release); + while (!forgotten.load(std::memory_order_acquire)) { + absl::SleepFor(absl::Milliseconds(1)); + } + c.lock(); // Records a -> c, although a was forgotten. + c.unlock(); + a.unlock(); + }); + while (!recorded.load(std::memory_order_acquire)) { + absl::SleepFor(absl::Milliseconds(1)); + } + a.ForgetDeadlockInfo(); + forgotten.store(true, std::memory_order_release); + holder.join(); + // Acquiring a while holding c closes the cycle. + c.lock(); + a.lock(); + }, + "dying due to potential deadlock"); +} + +// Same, for a thread that waits on a CondVar: it is still considered to hold +// the Mutex while it waits. +TEST(Mutex, ForgetDeadlockInfoDuringCondVarWaitDoesNotMaskReacquiredHold) +ABSL_NO_THREAD_SAFETY_ANALYSIS { + EXPECT_DEATH_IF_SUPPORTED( + { + ScopedDisableBazelTestWarnings disable_bazel_test_warnings; + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + absl::Mutex a; + absl::Mutex b; + absl::CondVar cv; + bool ready = false; + std::atomic<bool> waiting{false}; + { + absl::synchronization_internal::ThreadPool pool(1); + pool.Schedule([&]() ABSL_NO_THREAD_SAFETY_ANALYSIS { + a.lock(); + waiting.store(true, std::memory_order_release); + while (!ready) { + cv.Wait(&a); + } + b.lock(); // Records a -> b. + b.unlock(); + a.unlock(); + }); + while (!waiting.load(std::memory_order_acquire)) { + absl::SleepFor(absl::Milliseconds(1)); + } + absl::SleepFor(absl::Milliseconds(20)); // Let the waiter wait. + a.ForgetDeadlockInfo(); + a.lock(); + ready = true; + cv.Signal(); + a.unlock(); + } + // Acquiring a while holding b closes the cycle. + b.lock(); + a.lock(); + }, + "dying due to potential deadlock"); +} + +TEST(Mutex, ForgetDeadlockInfoWhileHoldingManyLocks) +ABSL_NO_THREAD_SAFETY_ANALYSIS { + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + // Use a new thread: the list of held locks overflows and sets the thread's + // overflow flag. + std::thread thread([]() ABSL_NO_THREAD_SAFETY_ANALYSIS { + // More locks than the deadlock detector can track per thread. + constexpr int kNumLocks = 42; + absl::Mutex mutexes[kNumLocks]; + mutexes[0].lock(); + mutexes[0].ForgetDeadlockInfo(); + for (int i = 1; i < kNumLocks; ++i) { + mutexes[i].lock(); + } + for (int i = kNumLocks - 1; i >= 0; --i) { + mutexes[i].unlock(); + } + }); + thread.join(); + + // Repeatedly forgetting a held lock and unlocking it, or destroying a locked + // Mutex, must not leak slots in the thread's held-locks array. + EXPECT_DEATH_IF_SUPPORTED( + { + ScopedDisableBazelTestWarnings disable_bazel_test_warnings; + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + absl::Mutex m; + for (int i = 0; i < 50; ++i) { + m.lock(); + m.ForgetDeadlockInfo(); + m.unlock(); + } + auto destroyed_locked = std::make_unique<absl::Mutex[]>(50); + for (size_t i = 0; i < 50; ++i) { + destroyed_locked[i].lock(); + } + destroyed_locked.reset(); + absl::Mutex a; + absl::Mutex b; + a.lock(); + b.lock(); + b.unlock(); + a.unlock(); + b.lock(); + a.lock(); + }, + "dying due to potential deadlock"); +} + +TEST(Mutex, AssertNotHeld) ABSL_NO_THREAD_SAFETY_ANALYSIS { + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + absl::Mutex mu; + mu.AssertNotHeld(); + + // A hold by another thread does not count. + std::atomic<bool> held{false}; + std::atomic<bool> release{false}; + std::thread holder([&]() ABSL_NO_THREAD_SAFETY_ANALYSIS { + mu.lock(); + held.store(true, std::memory_order_release); + while (!release.load(std::memory_order_acquire)) { + absl::SleepFor(absl::Milliseconds(1)); + } + mu.unlock(); + }); + while (!held.load(std::memory_order_acquire)) { + absl::SleepFor(absl::Milliseconds(1)); + } + mu.AssertNotHeld(); + release.store(true, std::memory_order_release); + holder.join(); + + EXPECT_DEATH_IF_SUPPORTED( + { + mu.lock(); + mu.AssertNotHeld(); + }, + "thread should not hold mutex"); + EXPECT_DEATH_IF_SUPPORTED( + { + mu.lock(); + mu.ForgetDeadlockInfo(); + mu.AssertNotHeld(); + }, + "thread should not hold mutex"); +} +#endif // !defined(NDEBUG) && !defined(ABSL_HAVE_THREAD_SANITIZER) + // -------------------------------------------------------- // Test for timeouts/deadlines on condition waits that are specified using // absl::Duration and absl::Time. For each waiting function we test with
diff --git a/absl/time/BUILD.bazel b/absl/time/BUILD.bazel index eaea38a..347e24b 100644 --- a/absl/time/BUILD.bazel +++ b/absl/time/BUILD.bazel
@@ -181,6 +181,7 @@ ], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + shard_count = 4, deps = [ ":clock_interface", ":simulated_clock",
diff --git a/absl/time/CMakeLists.txt b/absl/time/CMakeLists.txt index b141b62..79b99e8 100644 --- a/absl/time/CMakeLists.txt +++ b/absl/time/CMakeLists.txt
@@ -202,6 +202,8 @@ absl::synchronization absl::time GTest::gmock_main + SHARD_COUNT + 4 ) absl_cc_test(
diff --git a/absl/time/clock_interface.h b/absl/time/clock_interface.h index a14d852..ef3be13 100644 --- a/absl/time/clock_interface.h +++ b/absl/time/clock_interface.h
@@ -55,7 +55,7 @@ // MyTestClock test_clock(SATURDAY); // IsWeekend(test_clock); // -class Clock { +class ABSL_DLL Clock { public: // Returns a reference to the global realtime clock. // The returned clock is thread-safe.
diff --git a/ci/BUILD.bazel b/ci/BUILD.bazel new file mode 100644 index 0000000..c83d079 --- /dev/null +++ b/ci/BUILD.bazel
@@ -0,0 +1,84 @@ +# Copyright 2026 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//visibility:private"]) + +platform( + name = "rbe", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + exec_properties = { + "OSArch": "amd64", + "cache-silo-key": "n2-highcpu-2-2026-10-14", + "container-image": "docker://gcr.io/google.com/absl-177019/oss-linux-latest@sha256:9f7f104e4857e3528e8bf5f14968183f5a0e786ca9ef63601eae4e4136f011d6", + "dockerRunAsRoot": "true", + }, +) + +platform( + name = "rbe-floor", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + exec_properties = { + "OSArch": "amd64", + "cache-silo-key": "n2-highcpu-2-2026-10-14", + "container-image": "docker://gcr.io/google.com/absl-177019/oss-linux-floor@sha256:60898eb3b0dbcf85cd60e346f103fbf3169d966d391dcb2362b243d528b1183b", + "dockerRunAsRoot": "true", + }, +) + +platform( + name = "rbe-arm", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + exec_properties = { + "OSArch": "arm64", + "cache-silo-key": "t2a-standard-4-2026-10-14", + "container-image": "docker://gcr.io/google.com/absl-177019/oss-linux-latest@sha256:9f7f104e4857e3528e8bf5f14968183f5a0e786ca9ef63601eae4e4136f011d6", + "dockerRunAsRoot": "true", + }, +) + +platform( + name = "linux_gcc", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + "@bazel_tools//tools/cpp:gcc", + ], +) + +platform( + name = "linux_clang", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + "@bazel_tools//tools/cpp:clang", + ], +) + +platform( + name = "linux_arm_clang", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + "@bazel_tools//tools/cpp:clang", + ], +)
diff --git a/ci/docker_toolchain/BUILD.bazel b/ci/docker_toolchain/BUILD.bazel new file mode 100644 index 0000000..ae482b0 --- /dev/null +++ b/ci/docker_toolchain/BUILD.bazel
@@ -0,0 +1,21 @@ +# Copyright 2026 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Standalone fallback toolchain module when building outside the Docker container. +# Inside the container, this module is overridden via /etc/bazel.bazelrc to point to an actual +# toolchain. + +package(default_visibility = ["//visibility:public"]) + +filegroup(name = "empty")
diff --git a/ci/docker_toolchain/MODULE.bazel b/ci/docker_toolchain/MODULE.bazel new file mode 100644 index 0000000..10d8650 --- /dev/null +++ b/ci/docker_toolchain/MODULE.bazel
@@ -0,0 +1,18 @@ +# Copyright 2026 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module( + name = "docker_toolchain", + version = "0.1", +)
diff --git a/ci/macos_xcode_bazel.sh b/ci/macos_xcode_bazel.sh index 54d06eb..a45d353 100755 --- a/ci/macos_xcode_bazel.sh +++ b/ci/macos_xcode_bazel.sh
@@ -38,6 +38,12 @@ # Use Bazel Vendor mode to reduce reliance on external dependencies. if [[ ${KOKORO_GFILE_DIR:-} ]] && [[ -f "${KOKORO_GFILE_DIR}/distdir/abseil-cpp_vendor.tar.gz" ]]; then tar -xf "${KOKORO_GFILE_DIR}/distdir/abseil-cpp_vendor.tar.gz" -C "${HOME}/" + # Pin all vendored repositories so Bazel uses them without checking their + # marker files, which were generated on Linux and have OS-specific hashes. + for marker in "${HOME}/abseil-cpp_vendor"/@*.marker; do + repo="$(basename "${marker}" .marker)" + echo "pin('@${repo}')" >> "${HOME}/abseil-cpp_vendor/VENDOR.bazel" + done BAZEL_EXTRA_ARGS="--vendor_dir=${HOME}/abseil-cpp_vendor ${BAZEL_EXTRA_ARGS:-}" fi