Merge pull request #991 from Cyan4973/dev
v0.8.3
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9e9d1ad..d806dff 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -30,7 +30,7 @@
# | Item | Section in the right pane |
# | ------------------------- | ------------------------------------- |
# | OS, VM | Set up job |
-# | git repo, commit hash | Run actions/checkout@v3 |
+# | git repo, commit hash | Run actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 |
# | gcc, tools | Environment info |
#
# - To fail earlier, order of tests in the same job are roughly sorted by
@@ -72,34 +72,33 @@
# pkgs : apt-get package names. It can include multiple package names which are delimited by space.
# cc : C compiler executable.
# cxx : C++ compiler executable for `make ctocpptest`.
- # avx512 : Set 'true' if compiler supports avx512. Otherwise, set 'false'.
# os : GitHub Actions YAML workflow label. See https://github.com/actions/virtual-environments#available-environments
# cc
- { pkgs: '', cc: cc, cxx: c++, avx512: 'true', os: ubuntu-latest, },
+ { pkgs: '', cc: cc, cxx: c++, os: ubuntu-latest, },
# gcc
- { pkgs: '', cc: gcc, cxx: g++, avx512: 'true', os: ubuntu-latest, },
- { pkgs: 'gcc-13 g++-13 lib32gcc-13-dev', cc: gcc-13, cxx: g++-13, avx512: 'true', os: ubuntu-22.04, },
- { pkgs: 'gcc-12 g++-12 lib32gcc-12-dev', cc: gcc-12, cxx: g++-12, avx512: 'true', os: ubuntu-22.04, },
- { pkgs: 'gcc-11 g++-11 lib32gcc-11-dev', cc: gcc-11, cxx: g++-11, avx512: 'true', os: ubuntu-22.04, },
- { pkgs: 'gcc-10 g++-10 lib32gcc-10-dev', cc: gcc-10, cxx: g++-10, avx512: 'true', os: ubuntu-22.04, },
- { pkgs: 'gcc-9 g++-9 lib32gcc-9-dev', cc: gcc-9, cxx: g++-9, avx512: 'true', os: ubuntu-22.04, },
- { pkgs: 'gcc-8 g++-8 lib32gcc-8-dev', cc: gcc-8, cxx: g++-8, avx512: 'true', os: ubuntu-20.04, },
- { pkgs: 'gcc-7 g++-7 lib32gcc-7-dev', cc: gcc-7, cxx: g++-7, avx512: 'true', os: ubuntu-20.04, },
+ { pkgs: '', cc: gcc, cxx: g++, os: ubuntu-latest, },
+ # { pkgs: 'gcc-13 g++-13 lib32gcc-13-dev', cc: gcc-13, cxx: g++-13, os: ubuntu-22.04, }, # currently failing
+ { pkgs: 'gcc-12 g++-12 lib32gcc-12-dev', cc: gcc-12, cxx: g++-12, os: ubuntu-22.04, },
+ { pkgs: 'gcc-11 g++-11 lib32gcc-11-dev', cc: gcc-11, cxx: g++-11, os: ubuntu-22.04, },
+ { pkgs: 'gcc-10 g++-10 lib32gcc-10-dev', cc: gcc-10, cxx: g++-10, os: ubuntu-22.04, },
+ { pkgs: 'gcc-9 g++-9 lib32gcc-9-dev', cc: gcc-9, cxx: g++-9, os: ubuntu-22.04, },
+ { pkgs: 'gcc-8 g++-8 lib32gcc-8-dev', cc: gcc-8, cxx: g++-8, os: ubuntu-20.04, },
+ { pkgs: 'gcc-7 g++-7 lib32gcc-7-dev', cc: gcc-7, cxx: g++-7, os: ubuntu-20.04, },
# clang
- { pkgs: '', cc: clang, cxx: clang++, avx512: 'true', os: ubuntu-latest, },
- { pkgs: 'clang-15', cc: clang-15, cxx: clang++-15, avx512: 'true', os: ubuntu-22.04, },
- { pkgs: 'clang-14', cc: clang-14, cxx: clang++-14, avx512: 'true', os: ubuntu-22.04, },
- { pkgs: 'clang-13', cc: clang-13, cxx: clang++-13, avx512: 'true', os: ubuntu-22.04, },
- { pkgs: 'clang-12', cc: clang-12, cxx: clang++-12, avx512: 'true', os: ubuntu-22.04, },
- { pkgs: 'clang-11', cc: clang-11, cxx: clang++-11, avx512: 'true', os: ubuntu-22.04, },
- { pkgs: 'clang-10', cc: clang-10, cxx: clang++-10, avx512: 'true', os: ubuntu-20.04, },
- { pkgs: 'clang-9', cc: clang-9, cxx: clang++-9, avx512: 'true', os: ubuntu-20.04, },
- { pkgs: 'clang-8', cc: clang-8, cxx: clang++-8, avx512: 'true', os: ubuntu-20.04, },
- { pkgs: 'clang-7', cc: clang-7, cxx: clang++-7, avx512: 'true', os: ubuntu-20.04, },
- { pkgs: 'clang-6.0', cc: clang-6.0, cxx: clang++-6.0, avx512: 'true', os: ubuntu-20.04, },
+ { pkgs: '', cc: clang, cxx: clang++, os: ubuntu-latest, },
+ { pkgs: 'clang-15', cc: clang-15, cxx: clang++-15, os: ubuntu-22.04, },
+ { pkgs: 'clang-14', cc: clang-14, cxx: clang++-14, os: ubuntu-22.04, },
+ { pkgs: 'clang-13', cc: clang-13, cxx: clang++-13, os: ubuntu-22.04, },
+ { pkgs: 'clang-12', cc: clang-12, cxx: clang++-12, os: ubuntu-22.04, },
+ { pkgs: 'clang-11', cc: clang-11, cxx: clang++-11, os: ubuntu-22.04, },
+ { pkgs: 'clang-10', cc: clang-10, cxx: clang++-10, os: ubuntu-20.04, },
+ { pkgs: 'clang-9', cc: clang-9, cxx: clang++-9, os: ubuntu-20.04, },
+ { pkgs: 'clang-8', cc: clang-8, cxx: clang++-8, os: ubuntu-20.04, },
+ { pkgs: 'clang-7', cc: clang-7, cxx: clang++-7, os: ubuntu-20.04, },
+ { pkgs: 'clang-6.0', cc: clang-6.0, cxx: clang++-6.0, os: ubuntu-20.04, },
]
runs-on: ${{ matrix.os }}
@@ -108,7 +107,7 @@
CC: ${{ matrix.cc }}
CXX: ${{ matrix.cxx }}
steps:
- - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: apt-get install
run: |
@@ -163,7 +162,6 @@
make clean noxxh3test
- name: make avx512f
- if: ${{ matrix.avx512 == 'true' }}
run: |
CFLAGS="-O1 -mavx512f -Werror" make clean default
@@ -177,7 +175,7 @@
name: Linux x64 check results consistency
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Environment info
run: |
@@ -225,23 +223,23 @@
CC: emcc
steps:
- - uses: actions/checkout@v3 # https://github.com/actions/checkout
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup cache
id: cache-system-libraries
- uses: actions/cache@v3
+ uses: actions/cache@v4
with:
path: ${{env.EM_CACHE_FOLDER}}
key: em${{env.EM_VERSION}}-node${{ matrix.node-version }}-${{ runner.os }}
- name: Setup emsdk
- uses: mymindstorm/setup-emsdk@v12
+ uses: mymindstorm/setup-emsdk@v14
with:
version: ${{env.EM_VERSION}}
actions-cache-folder: ${{env.EM_CACHE_FOLDER}}
- name: Use Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v3
+ uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
@@ -269,7 +267,7 @@
name: Linux x64 misc tests
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: apt-get install
run: |
@@ -316,7 +314,7 @@
name: Linux x64 cmake unofficial build test
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Environment info
run: |
@@ -345,26 +343,26 @@
cat build/libxxhash.pc | grep "libdir=/usr/lib"
cat build/libxxhash.pc | grep "includedir=/usr/include"
- - name: cmake minimum version v2.8.12 test
+ - name: cmake minimum version v3.10 test
run: |
mkdir -p cmake_bins
cd cmake_bins
- wget https://cmake.org/files/v2.8/cmake-2.8.12.2-Linux-i386.tar.gz
- tar xzf cmake-2.8.12.2-Linux-i386.tar.gz
+ wget https://cmake.org/files/v3.10/cmake-3.10.0-Linux-x86_64.tar.gz
+ tar xzf cmake-3.10.0-Linux-x86_64.tar.gz
cd ../cmake_unofficial
rm -rf build
pwd
ls
mkdir -p build
cd build
- ../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake --version
- ../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake ..
- ../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake --build .
+ ../../cmake_bins/cmake-3.10.0-Linux-x86_64/bin/cmake --version
+ ../../cmake_bins/cmake-3.10.0-Linux-x86_64/bin/cmake ..
+ ../../cmake_bins/cmake-3.10.0-Linux-x86_64/bin/cmake --build .
mkdir -p test_install_dir
- DESTDIR=test_install_dir ../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake --install .
+ DESTDIR=test_install_dir ../../cmake_bins/cmake-3.10.0-Linux-x86_64/bin/cmake --install .
rm -rf *
- ../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake -DCMAKE_BUILD_TYPE=Debug ..
- ../../cmake_bins/cmake-2.8.12.2-Linux-i386/bin/cmake --build .
+ ../../cmake_bins/cmake-3.10.0-Linux-x86_64/bin/cmake -DCMAKE_BUILD_TYPE=Debug ..
+ ../../cmake_bins/cmake-3.10.0-Linux-x86_64/bin/cmake --build .
@@ -386,6 +384,9 @@
{ name: 'MIPS', xcc_pkg: gcc-mips-linux-gnu, xcc: mips-linux-gnu-gcc, xemu_pkg: qemu-system-mips, xemu: qemu-mips-static, os: ubuntu-latest, },
{ name: 'M68K', xcc_pkg: gcc-m68k-linux-gnu, xcc: m68k-linux-gnu-gcc, xemu_pkg: qemu-system-m68k, xemu: qemu-m68k-static, os: ubuntu-latest, },
{ name: 'RISC-V', xcc_pkg: gcc-riscv64-linux-gnu, xcc: riscv64-linux-gnu-gcc, xemu_pkg: qemu-system-riscv64,xemu: qemu-riscv64-static, os: ubuntu-latest, },
+ # SPARC64 qemu emulation seems broken on Ubuntu-22
+ { name: 'SPARC', xcc_pkg: gcc-sparc64-linux-gnu, xcc: sparc64-linux-gnu-gcc, xemu_pkg: qemu-system-sparc, xemu: qemu-sparc64-static, os: ubuntu-20.04, },
+ { name: 'LoongArch', xcc_pkg: gcc-14-loongarch64-linux-gnu, xcc: loongarch64-linux-gnu-gcc-14, xemu_pkg: qemu-system-loongarch64, xemu: qemu-loongarch64-static, os: ubuntu-24.04, },
{ name: 'ARM, gcc-10', xcc_pkg: gcc-10-arm-linux-gnueabi, xcc: arm-linux-gnueabi-gcc-10, xemu_pkg: qemu-system-arm, xemu: qemu-arm-static, os: ubuntu-20.04, },
{ name: 'AARCH64, gcc-10', xcc_pkg: gcc-10-aarch64-linux-gnu, xcc: aarch64-linux-gnu-gcc-10, xemu_pkg: qemu-system-arm, xemu: qemu-aarch64-static, os: ubuntu-20.04, },
@@ -416,7 +417,7 @@
XEMU: ${{ matrix.xemu }}
MOREFLAGS: -Werror
steps:
- - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: apt update & install (1)
run: |
sudo apt-get update
@@ -443,6 +444,8 @@
run: |
CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
CPPFLAGS="-DXXH_VECTOR=XXH_NEON" CFLAGS="-O3 -march=armv7-a -fPIC -mfloat-abi=softfp -mfpu=neon-vfpv4" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
+ make -C tests clean
+ CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make -C tests sanity_test_vectors.h test_sanity
- name: AARCH64 (XXH_VECTOR=[ scalar, NEON, SVE ])
if: ${{ startsWith(matrix.name, 'AARCH64') }}
@@ -467,13 +470,18 @@
CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
CPPFLAGS=-DXXH_VECTOR=XXH_VSX CFLAGS="-O3 -march=arch11 -mzvector" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
- - name: MIPS-M68K-RISCV (XXH_VECTOR=[ scalar ])
- if: ${{ startsWith(matrix.name, 'MIPS') || startsWith(matrix.name, 'M68K') || startsWith(matrix.name, 'RISC-V') }}
+ - name: MIPS-M68K-RISCV-SPARC (XXH_VECTOR=[ scalar ])
+ if: ${{ startsWith(matrix.name, 'MIPS') || startsWith(matrix.name, 'M68K') || startsWith(matrix.name, 'RISC-V') || startsWith(matrix.name, 'SPARC') }}
run: |
- LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
+ make clean; LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make check
+ - name: LoongArch (XXH_VECTOR=[ scalar, LSX ])
+ if: ${{ startsWith(matrix.name, 'LoongArch') }}
+ run: |
+ CPPFLAGS="-DXXH_VECTOR=XXH_SCALAR" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
+ CPPFLAGS=-DXXH_VECTOR=XXH_LSX CFLAGS="-O3 -march=la464 -mlsx" LDFLAGS="-static" CC=$XCC RUN_ENV=$XEMU make clean check
- # macOS, { 11 }
+ # macOS
macos-general:
name: ${{ matrix.system.os }}
@@ -482,11 +490,13 @@
fail-fast: false # 'false' means Don't stop matrix workflows even if some matrix failed.
matrix:
system: [
- { os: macos-11 },
- { os: macos-12 },
+ { os: macos-13 },
+ { os: macos-14 },
+ { os: macos-15 },
+ { os: macos-latest },
]
steps:
- - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Environment info
run: |
@@ -503,7 +513,7 @@
make clean test MOREFLAGS='-Werror' | tee
- # Windows, { VC++2022, VC++2019, VC++2017 } x { x64, Win32, ARM, ARM64 }
+ # Windows, { VC++2022, VC++2019 } x { x64, Win32, ARM, ARM64 }
#
# - Default shell for Windows environment is PowerShell Core.
# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
@@ -524,17 +534,17 @@
{ os: windows-2022, vc: "VC++ 2022", clangcl: 'true', },
{ os: windows-2019, vc: "VC++ 2019", clangcl: 'true', },
]
- arch: [ x64, Win32, ARM, ARM64 ]
+ arch: [ x64, Win32, ARM64 ]
steps:
- - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Build ${{ matrix.system.os }}, ${{ matrix.arch }}
run: |
cd cmake_unofficial
mkdir build
cd build
- cmake .. -DCMAKE_BUILD_TYPE=Release -A ${{ matrix.arch }} -DXXHASH_C_FLAGS="/WX"
+ cmake .. -DCMAKE_BUILD_TYPE=Release -A ${{ matrix.arch }} -DCMAKE_C_FLAGS="/W4 /WX"
cmake --build . --config Release
- name: Test
@@ -543,6 +553,24 @@
run: |
.\cmake_unofficial\build\Release\xxhsum.exe -bi1
+ - name: Build ${{ matrix.system.os }}, ${{ matrix.arch }}, with DISPATCH
+ # DISPATCH only if target arch is x64 or Win32.
+ if: ${{ ( matrix.arch == 'x64' || matrix.arch == 'Win32' ) }}
+ run: |
+ cd cmake_unofficial
+ mkdir build-visual-dispatch
+ cd build-visual-dispatch
+ cmake .. -DCMAKE_BUILD_TYPE=Release -DDISPATCH=ON -A x64 -DCMAKE_C_FLAGS="/W4 /WX"
+ cmake --build . --config Release
+
+ - name: Runtime Test (DISPATCH)
+ # Run benchmark for testing only if target arch is x64 or Win32.
+ if: ${{ ( matrix.arch == 'x64' || matrix.arch == 'Win32' ) }}
+ run: |
+ .\cmake_unofficial\build-visual-dispatch\Release\xxhsum.exe -V | grep autoVec
+ .\cmake_unofficial\build-visual-dispatch\Release\xxhsum.exe -bi1
+
+
- name: Build ${{ matrix.system.os }}, clang-cl, ${{ matrix.arch }}
if: ${{ matrix.system.clangcl == 'true' }}
run: |
@@ -558,22 +586,6 @@
run: |
.\cmake_unofficial\build-clang-cl\Release\xxhsum.exe -bi1
- - name: Build ${{ matrix.system.os }}, clang-cl, ${{ matrix.arch }}, with DISPATCH
- # DISPATCH only if target arch is x64 or Win32.
- if: ${{ matrix.system.clangcl == 'true' && ( matrix.arch == 'x64' || matrix.arch == 'Win32' ) }}
- run: |
- cd cmake_unofficial
- mkdir build-clang-cl-dispatch
- cd build-clang-cl-dispatch
- cmake .. -DCMAKE_BUILD_TYPE=Release -DDISPATCH=ON -A x64 -DCMAKE_GENERATOR_TOOLSET=ClangCL
- cmake --build . --config Release
-
- - name: Runtime Test (clang-cl + DISPATCH)
- # Run benchmark for testing only if target arch is x64 or Win32.
- if: ${{ matrix.system.clangcl == 'true' && ( matrix.arch == 'x64' || matrix.arch == 'Win32' ) }}
- run: |
- .\cmake_unofficial\build-clang-cl-dispatch\Release\xxhsum.exe -V | grep autoVec
- .\cmake_unofficial\build-clang-cl-dispatch\Release\xxhsum.exe -bi1
# Windows, { mingw64, mingw32 }
@@ -597,7 +609,7 @@
run:
shell: msys2 {0}
steps:
- - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: msys2/setup-msys2@7efe20baefed56359985e327d329042cde2434ff # v2
with:
msystem: MSYS
@@ -612,10 +624,11 @@
- name: mingw64
if: ${{ matrix.msystem == 'mingw64' }}
run: |
- PATH=/mingw64/bin:$PATH /mingw32/bin/mingw32-make clean test MOREFLAGS=-Werror
+ PATH=/mingw64/bin:$PATH /mingw32/bin/mingw32-make clean test gcc-og-test MOREFLAGS=-Werror
PATH=/mingw64/bin:$PATH /mingw32/bin/mingw32-make -C tests/bench
# Abort if result of "file ./xxhsum.exe" doesn't contain 'x86-64'.
# Expected output is "./xxhsum.exe: PE32+ executable (console) x86-64, for MS Windows"
+ file ./xxhsum.exe
file ./xxhsum.exe | grep -q 'x86-64' || $(exit 1)
./xxhsum.exe --version
@@ -625,6 +638,7 @@
PATH=/mingw32/bin:$PATH /mingw32/bin/mingw32-make.exe clean test MOREFLAGS=-Werror
PATH=/mingw32/bin:$PATH /mingw32/bin/mingw32-make.exe -C tests/bench
# Abort if result of "file ./xxhsum.exe" doesn't contain '80386'.
- # Expected output is "./xxhsum.exe: PE32 executable (console) Intel 80386, for MS Windows"
- file ./xxhsum.exe | grep -q '80386' || $(exit 1)
+ # Expected output is "./xxhsum.exe: PE32 executable (console) Intel i386, for MS Windows"
+ file ./xxhsum.exe
+ file ./xxhsum.exe | grep -q '386' || $(exit 1)
./xxhsum.exe --version
diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml
new file mode 100644
index 0000000..c44e0b2
--- /dev/null
+++ b/.github/workflows/scorecard.yml
@@ -0,0 +1,72 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecard supply-chain security
+on:
+ # For Branch-Protection check. Only the default branch is supported. See
+ # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+ branch_protection_rule:
+ # To guarantee Maintained check is occasionally updated. See
+ # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+ schedule:
+ - cron: '35 19 * * 2'
+ push:
+ branches: [ "dev" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+ analysis:
+ name: Scorecard analysis
+ runs-on: ubuntu-latest
+ permissions:
+ # Needed to upload the results to code-scanning dashboard.
+ security-events: write
+ # Needed to publish results and get a badge (see publish_results below).
+ id-token: write
+ # Uncomment the permissions below if installing in a private repository.
+ # contents: read
+ # actions: read
+
+ steps:
+ - name: "Checkout code"
+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+
+ - name: "Run analysis"
+ uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
+ with:
+ results_file: results.sarif
+ results_format: sarif
+ # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+ # - you want to enable the Branch-Protection check on a *public* repository, or
+ # - you are installing Scorecard on a *private* repository
+ # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+ # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+ # Public repositories:
+ # - Publish results to OpenSSF REST API for easy access by consumers
+ # - Allows the repository to include the Scorecard badge.
+ # - See https://github.com/ossf/scorecard-action#publishing-results.
+ # For private repositories:
+ # - `publish_results` will always be set to `false`, regardless
+ # of the value entered here.
+ publish_results: true
+
+ # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+ # format to the repository Actions tab.
+ - name: "Upload artifact"
+ uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+ with:
+ name: SARIF file
+ path: results.sarif
+ retention-days: 5
+
+ # Upload the results to GitHub's code scanning dashboard.
+ - name: "Upload to code-scanning"
+ uses: github/codeql-action/upload-sarif@c6c77c8c2d62cfd5b2e8d548817fd3d1582ac744 # v2.14.5
+ with:
+ sarif_file: results.sarif
diff --git a/.gitignore b/.gitignore
index 823bc83..b88c1cc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@
xxh32sum
xxh64sum
xxh128sum
+xxh3sum
xxhsum
xxhsum32
xxhsum_privateXXH
@@ -20,9 +21,7 @@
tests/generate_unicode_test
tests/sanity_test
tests/sanity_test_vectors_generator
-
-# local conf
-.clang_complete
+fuzzer
# Mac OS-X artefacts
*.dSYM
@@ -47,6 +46,16 @@
tmp*
tests/*.unicode
tests/unicode_test*
+*.txt
+!CMakeLists.txt
+*.xxhsum
+
+# editor artifacts
+.clang_complete
+.clangd
+*.swp
+.vscode/
+.vs/
# Doxygen
doxygen/
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 9f9e42c..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,142 +0,0 @@
-language: c
-
-# Dump CPU info before start
-before_install:
- - cat /proc/cpuinfo || echo /proc/cpuinfo is not present
-
-matrix:
- fast_finish: true
- include:
-
- - name: General linux x64 tests
- arch: amd64
- addons:
- apt:
- packages:
- - g++-multilib
- - gcc-multilib
- - cppcheck
- script:
- - make -B test-all
- - make clean
- - CFLAGS="-Werror" MOREFLAGS="-Wno-sign-conversion" make dispatch # removing sign conversion warnings due to a bug in gcc-5's definition of some AVX512 intrinsics
- - make clean
- - CFLAGS="-O1 -mavx512f -Werror" make
- - make clean
- - CFLAGS="-Wall -Wextra -Werror" make DISPATCH=1
- - make clean
- - CFLAGS="-std=c90 -pedantic -Wno-long-long -Werror" make xxhsum # check C90 + long long compliance
- - make c90test # strict c90, with no long long support; resulting in no XXH64_* symbol
- - make noxxh3test # check library can be compiled with XXH_NO_XXH3, resulting in no XXH3_* symbol
-
-
- - name: Check results consistency on x64
- arch: amd64
- script:
- - CPPFLAGS=-DXXH_VECTOR=XXH_SCALAR make check # Scalar code path
- - make clean
- - CPPFLAGS=-DXXH_VECTOR=XXH_SSE2 make check # SSE2 code path
- - make clean
- - CPPFLAGS="-mavx2 -DXXH_VECTOR=XXH_AVX2" make check # AVX2 code path
- - make clean
- - CPPFLAGS="-mavx512f -DXXH_VECTOR=XXH_AVX512" make check # AVX512 code path
- - make clean
- - CPPFLAGS=-DXXH_REROLL=1 make check # reroll code path (#240)
- - make -C tests/bench
-
- - name: macOS General Test
- os: osx
- compiler: clang
- script:
- - CFLAGS="-Werror" make # test library build
- - make clean
- - make test MOREFLAGS='-Werror' | tee # test scenario where `stdout` is not the console
-
- - name: ARM compilation and consistency checks (Qemu)
- dist: xenial
- arch: amd64
- addons:
- apt:
- packages:
- - qemu-system-arm
- - qemu-user-static
- - gcc-arm-linux-gnueabi
- - libc6-dev-armel-cross
- script:
- # arm (32-bit)
- - CC=arm-linux-gnueabi-gcc CPPFLAGS=-DXXH_VECTOR=XXH_SCALAR LDFLAGS=-static RUN_ENV=qemu-arm-static make check # Scalar code path
- - make clean
- # NEON (32-bit)
- - CC=arm-linux-gnueabi-gcc CPPFLAGS=-DXXH_VECTOR=XXH_NEON CFLAGS="-O3 -march=armv7-a -fPIC -mfloat-abi=softfp -mfpu=neon-vfpv4" LDFLAGS=-static RUN_ENV=qemu-arm-static make check # NEON code path
-
- - name: aarch64 compilation and consistency checks
- dist: xenial
- arch: arm64
- script:
- # aarch64
- - CPPFLAGS=-DXXH_VECTOR=XXH_SCALAR make check # Scalar code path
- # NEON (64-bit)
- - make clean
- - CPPFLAGS=-DXXH_VECTOR=XXH_NEON make check # NEON code path
- # clang
- - make clean
- - CC=clang CPPFLAGS=-DXXH_VECTOR=XXH_SCALAR make check # Scalar code path
- # clang + NEON
- - make clean
- - CC=clang CPPFLAGS=-DXXH_VECTOR=XXH_NEON make check # NEON code path
-
- # We need Bionic here because the QEMU versions shipped in the older repos
- # do not support POWER8 emulation, and compiling QEMU from source is a pain.
- - name: PowerPC + PPC64 compilation and consistency checks (Qemu on Bionic)
- dist: bionic
- arch: amd64
- addons:
- apt:
- packages:
- - qemu-system-ppc
- - qemu-user-static
- - gcc-powerpc-linux-gnu
- - gcc-powerpc64-linux-gnu
- - libc6-dev-powerpc-cross
- - libc6-dev-ppc64-cross
- script:
- - CC=powerpc-linux-gnu-gcc RUN_ENV=qemu-ppc-static LDFLAGS=-static make check # Scalar code path
- - make clean
- - CC=powerpc64-linux-gnu-gcc RUN_ENV=qemu-ppc64-static CPPFLAGS=-DXXH_VECTOR=XXH_SCALAR CFLAGS="-O3" LDFLAGS="-static -m64" make check # Scalar code path
- # VSX code
- - make clean
- - CC=powerpc64-linux-gnu-gcc RUN_ENV="qemu-ppc64-static -cpu power8" CPPFLAGS=-DXXH_VECTOR=XXH_VSX CFLAGS="-O3 -maltivec -mvsx -mcpu=power8 -mpower8-vector" LDFLAGS="-static -m64" make check # VSX code path
- # altivec.h redefinition issue #426
- - make clean
- - CC=powerpc64-linux-gnu-gcc CPPFLAGS=-DXXH_VECTOR=XXH_VSX CFLAGS="-maltivec -mvsx -mcpu=power8 -mpower8-vector" make -C tests test_ppc_redefine
-
- - name: PPC64LE compilation and consistency checks
- dist: xenial
- arch: ppc64le
- script:
- # Scalar (universal) code path
- - CPPFLAGS=-DXXH_VECTOR=XXH_SCALAR LDFLAGS=-static make check
- # VSX code path (64-bit)
- - make clean
- - CPPFLAGS=-DXXH_VECTOR=XXH_VSX CFLAGS="-O3 -maltivec -mvsx -mpower8-vector -mcpu=power8" LDFLAGS="-static" make check
- # altivec.h redefinition issue #426
- - make clean
- - CPPFLAGS=-DXXH_VECTOR=XXH_VSX CFLAGS="-maltivec -mvsx -mcpu=power8 -mpower8-vector" make -C tests test_ppc_redefine
-
- - name: IBM s390x compilation and consistency checks
- dist: bionic
- arch: s390x
- script:
- # Scalar (universal) code path
- - CPPFLAGS=-DXXH_VECTOR=XXH_SCALAR LDFLAGS=-static make check
- # s390x code path (64-bit)
- - make clean
- - CPPFLAGS=-DXXH_VECTOR=XXH_VSX CFLAGS="-O3 -march=arch11 -mzvector" LDFLAGS="-static" make check
-
- - name: cmake build test
- script:
- - cd cmake_unofficial
- - mkdir build
- - cd build
- - cmake ..
- - CFLAGS=-Werror make
diff --git a/CHANGELOG b/CHANGELOG
index 3040855..0eaba61 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,12 @@
+v0.8.3
+- fix : variant `XXH3_128bits_withSecretandSeed()` could produce an invalid result in some specific set of conditions, #894 by @hltj
+- cli : vector extension detected at runtime on x86/x64, enabled by default
+- cli : new commands `--filelist` and `--files-from`, by @Ian-Clowes
+- cli : XXH3 64-bits GNU format can now be generated and checked (command `-H3`)
+- portability: LoongArch SX SIMD extension, by @lrzlin
+- portability: can build on AIX, suggested by @likema
+- portability: validated for SPARC cpus
+
v0.8.2
- fix : XXH3 S390x vector implementation (@hzhuang1)
- fix : PowerPC vector compilation with IBM XL compiler (@MaxiBoether)
diff --git a/Doxyfile b/Doxyfile
index 940cb23..a2597db 100644
--- a/Doxyfile
+++ b/Doxyfile
@@ -38,6 +38,7 @@
# Predefine some macros to clean up the output.
PREDEFINED = "XXH_DOXYGEN=" \
"XXH_PUBLIC_API=" \
+ "XXH_NOESCAPE=" \
"XXH_FORCE_INLINE=static inline" \
"XXH_NO_INLINE=static" \
"XXH_RESTRICT=restrict" \
diff --git a/Doxyfile-internal b/Doxyfile-internal
index a15037d..2efcef5 100644
--- a/Doxyfile-internal
+++ b/Doxyfile-internal
@@ -38,6 +38,7 @@
# Predefine some macros to clean up the output.
PREDEFINED = "XXH_DOXYGEN=" \
"XXH_PUBLIC_API=" \
+ "XXH_NOESCAPE=" \
"XXH_FORCE_INLINE=static inline" \
"XXH_NO_INLINE=static" \
"XXH_RESTRICT=restrict" \
diff --git a/Makefile b/Makefile
index 7b0fe3d..5b7b8ff 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
# ################################################################
# xxHash Makefile
-# Copyright (C) 2012-2021 Yann Collet
+# Copyright (C) 2012-2024 Yann Collet
#
# GPL v2 License
#
@@ -55,6 +55,13 @@
EXT =
endif
+# automatically enable runtime vector dispatch on x86/64 targets
+detect_x86_arch = $(shell $(CC) -dumpmachine | grep -E 'i[3-6]86|x86_64')
+ifneq ($(strip $(call detect_x86_arch)),)
+ #note: can be overridden at compile time, by setting DISPATCH=0
+ DISPATCH ?= 1
+endif
+
ifeq ($(NODE_JS),1)
# Link in unrestricted filesystem support
LDFLAGS += -sNODERAWFS
@@ -81,6 +88,7 @@
XXHSUM_SRC_DIR = cli
XXHSUM_SPLIT_SRCS = $(XXHSUM_SRC_DIR)/xxhsum.c \
$(XXHSUM_SRC_DIR)/xsum_os_specific.c \
+ $(XXHSUM_SRC_DIR)/xsum_arch.c \
$(XXHSUM_SRC_DIR)/xsum_output.c \
$(XXHSUM_SRC_DIR)/xsum_sanity_check.c \
$(XXHSUM_SRC_DIR)/xsum_bench.c
@@ -123,9 +131,9 @@
xxh_x86dispatch.o: xxh_x86dispatch.c xxh_x86dispatch.h xxhash.h
.PHONY: xxhsum_and_links
-xxhsum_and_links: xxhsum xxh32sum xxh64sum xxh128sum
+xxhsum_and_links: xxhsum xxh32sum xxh64sum xxh128sum xxh3sum
-xxh32sum xxh64sum xxh128sum: xxhsum
+xxh32sum xxh64sum xxh128sum xxh3sum: xxhsum
ln -sf $<$(EXT) $@$(EXT)
xxhsum_inlinedXXH: CPPFLAGS += -DXXH_INLINE_ALL
@@ -143,7 +151,7 @@
ifeq (,$(filter Windows%,$(OS)))
$(LIBXXH): CFLAGS += -fPIC
endif
-ifeq ($(DISPATCH),1)
+ifeq ($(LIBXXH_DISPATCH),1)
$(LIBXXH): xxh_x86dispatch.c
endif
$(LIBXXH): xxhash.c
@@ -182,7 +190,8 @@
$(Q)$(RM) core *.o *.obj *.$(SHARED_EXT) *.$(SHARED_EXT).* *.a libxxhash.pc
$(Q)$(RM) xxhsum$(EXT) xxhsum32$(EXT) xxhsum_inlinedXXH$(EXT) dispatch$(EXT)
$(Q)$(RM) xxhsum.wasm xxhsum.js xxhsum.html
- $(Q)$(RM) xxh32sum$(EXT) xxh64sum$(EXT) xxh128sum$(EXT)
+ $(Q)$(RM) xxh32sum$(EXT) xxh64sum$(EXT) xxh128sum$(EXT) xxh3sum$(EXT)
+ $(Q)$(RM) fuzzer
$(Q)$(RM) $(XXHSUM_SRC_DIR)/*.o $(XXHSUM_SRC_DIR)/*.obj
$(MAKE) -C tests clean
$(MAKE) -C tests/bench clean
@@ -288,14 +297,14 @@
./xxhsum --tag -H1 xxhsum* | $(GREP) XXH64
./xxhsum --tag -H2 xxhsum* | $(GREP) XXH128
./xxhsum --tag -H3 xxhsum* | $(GREP) XXH3
- ./xxhsum -H3 xxhsum* | $(GREP) XXH3 # --tag is implicit for H3
+ ./xxhsum -H3 xxhsum* | $(GREP) XXH3_ # prefix for GNU format
./xxhsum --tag -H32 xxhsum* | $(GREP) XXH32
./xxhsum --tag -H64 xxhsum* | $(GREP) XXH64
./xxhsum --tag -H128 xxhsum* | $(GREP) XXH128
./xxhsum --tag -H0 --little-endian xxhsum* | $(GREP) XXH32_LE
./xxhsum --tag -H1 --little-endian xxhsum* | $(GREP) XXH64_LE
./xxhsum --tag -H2 --little-endian xxhsum* | $(GREP) XXH128_LE
- ./xxhsum -H3 --little-endian xxhsum* | $(GREP) XXH3_LE
+ ./xxhsum --tag -H3 --little-endian xxhsum* | $(GREP) XXH3_LE
./xxhsum --tag -H32 --little-endian xxhsum* | $(GREP) XXH32_LE
./xxhsum --tag -H64 --little-endian xxhsum* | $(GREP) XXH64_LE
./xxhsum --tag -H128 --little-endian xxhsum* | $(GREP) XXH128_LE
@@ -313,8 +322,20 @@
# Expects "FAILED open or read"
echo "0000000000000000 test-expects-file-not-found" | ./xxhsum -c -; test $$? -eq 1
echo "00000000 test-expects-file-not-found" | ./xxhsum -c -; test $$? -eq 1
+ # --filelist
+ echo xxhash.c > .test.filenames
+ $(RUN_ENV) ./xxhsum$(EXT) --filelist .test.filenames
+ # --filelist from stdin
+ cat .test.filenames | $(RUN_ENV) ./xxhsum$(EXT) --filelist
@$(RM) .test.*
+LIB_FUZZING_ENGINE?="-fsanitize=fuzzer"
+CC_VERSION := $(shell $(CC) --version)
+ifneq (,$(findstring clang,$(CC_VERSION)))
+fuzzer: libxxhash.a fuzz/fuzzer.c
+ $(CC) $(CFLAGS) $(LIB_FUZZING_ENGINE) -I. -o fuzzer fuzz/fuzzer.c -L. -Wl,-Bstatic -lxxhash -Wl,-Bdynamic
+endif
+
.PHONY: test-filename-escape
test-filename-escape:
$(MAKE) -C tests test_filename_escape
@@ -340,13 +361,15 @@
.PHONY: gcc-og-test
gcc-og-test: clean
@echo ---- test gcc -Og compilation ----
- CFLAGS="-Og -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror -fPIC" MOREFLAGS="-Werror" $(MAKE) all
+ CFLAGS="-Og -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror -fPIC" CPPFLAGS="-DXXH_NO_INLINE_HINTS" MOREFLAGS="-Werror" $(MAKE) all
.PHONY: cxxtest
cxxtest: clean
@echo ---- test C++ compilation ----
CC="$(CXX) -Wno-deprecated" $(MAKE) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror -fPIC"
+# In strict C90 mode, there is no `long long` type support,
+# consequently, only XXH32 can be compiled.
.PHONY: c90test
ifeq ($(NO_C90_TEST),true)
c90test:
@@ -440,10 +463,20 @@
test: DEBUGFLAGS += -DXXH_DEBUGLEVEL=1
test: all namespaceTest check test-xxhsum-c c90test test-tools noxxh3test nostdlibtest
-.PHONY: test-inline
-test-inline:
+# this test checks that including "xxhash.h" multiple times and with different directives still compiles properly
+.PHONY: test-multiInclude
+test-multiInclude:
$(MAKE) -C tests test_multiInclude
+.PHONY: test-inline-notexposed
+test-inline-notexposed: xxhsum_inlinedXXH
+ $(NM) xxhsum_inlinedXXH | $(GREP) "t _XXH32_" ; test $$? -eq 1 # no XXH32 symbol should be left
+ $(NM) xxhsum_inlinedXXH | $(GREP) "t _XXH64_" ; test $$? -eq 1 # no XXH64 symbol should be left
+
+.PHONY: test-inline
+test-inline: test-inline-notexposed test-multiInclude
+
+
.PHONY: test-all
test-all: CFLAGS += -Werror
test-all: test test32 test-unicode clangtest gcc-og-test cxxtest usan test-inline listL120 trailingWhitespace test-xxh-nnn-sums
@@ -451,7 +484,7 @@
.PHONY: test-tools
test-tools:
CFLAGS=-Werror $(MAKE) -C tests/bench
- CFLAGS=-Werror $(MAKE) -C tests/collisions
+ CFLAGS=-Werror $(MAKE) -C tests/collisions check
.PHONY: test-xxh-nnn-sums
test-xxh-nnn-sums: xxhsum_and_links
@@ -459,26 +492,37 @@
./xxh32sum README.md > tmp.xxh32sum.out
./xxh64sum README.md > tmp.xxh64sum.out
./xxh128sum README.md > tmp.xxh128sum.out
+ ./xxh3sum README.md > tmp.xxh3sum.out
cat tmp.xxhsum.out
cat tmp.xxh32sum.out
cat tmp.xxh64sum.out
cat tmp.xxh128sum.out
+ cat tmp.xxh3sum.out
./xxhsum -c tmp.xxhsum.out
./xxhsum -c tmp.xxh32sum.out
./xxhsum -c tmp.xxh64sum.out
./xxhsum -c tmp.xxh128sum.out
+ ./xxhsum -c tmp.xxh3sum.out
./xxh32sum -c tmp.xxhsum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh32sum -c tmp.xxh32sum.out
./xxh32sum -c tmp.xxh64sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh32sum -c tmp.xxh128sum.out ; test $$? -eq 1 # expects "no properly formatted"
+ ./xxh32sum -c tmp.xxh3sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh64sum -c tmp.xxhsum.out
./xxh64sum -c tmp.xxh32sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh64sum -c tmp.xxh64sum.out
./xxh64sum -c tmp.xxh128sum.out ; test $$? -eq 1 # expects "no properly formatted"
+ ./xxh64sum -c tmp.xxh3sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh128sum -c tmp.xxhsum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh128sum -c tmp.xxh32sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh128sum -c tmp.xxh64sum.out ; test $$? -eq 1 # expects "no properly formatted"
./xxh128sum -c tmp.xxh128sum.out
+ ./xxh128sum -c tmp.xxh3sum.out ; test $$? -eq 1 # expects "no properly formatted"
+ ./xxh3sum -c tmp.xxhsum.out ; test $$? -eq 1 # expects "no properly formatted"
+ ./xxh3sum -c tmp.xxh32sum.out ; test $$? -eq 1 # expects "no properly formatted"
+ ./xxh3sum -c tmp.xxh64sum.out ; test $$? -eq 1 # expects "no properly formatted"
+ ./xxh3sum -c tmp.xxh128sum.out ; test $$? -eq 1 # expects "no properly formatted"
+ ./xxh3sum -c tmp.xxh3sum.out
.PHONY: listL120
listL120: # extract lines >= 120 characters in *.{c,h}, by Takayuki Matsuoka (note: $$, for Makefile compatibility)
@@ -535,7 +579,7 @@
INSTALL_PROGRAM ?= $(INSTALL)
INSTALL_DATA ?= $(INSTALL) -m 644
-INSTALL_DIR ?= $(INSTALL) -d -m 755
+MAKE_DIR ?= $(INSTALL) -d -m 755
# Escape special symbols by putting each character into its separate class
@@ -574,12 +618,12 @@
install_libxxhash.a: libxxhash.a
@echo Installing libxxhash.a
- $(Q)$(INSTALL_DIR) $(DESTDIR)$(LIBDIR)
+ $(Q)$(MAKE_DIR) $(DESTDIR)$(LIBDIR)
$(Q)$(INSTALL_DATA) libxxhash.a $(DESTDIR)$(LIBDIR)
install_libxxhash: libxxhash
@echo Installing libxxhash
- $(Q)$(INSTALL_DIR) $(DESTDIR)$(LIBDIR)
+ $(Q)$(MAKE_DIR) $(DESTDIR)$(LIBDIR)
$(Q)$(INSTALL_PROGRAM) $(LIBXXH) $(DESTDIR)$(LIBDIR)
$(Q)ln -sf $(LIBXXH) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT_MAJOR)
$(Q)ln -sf $(LIBXXH) $(DESTDIR)$(LIBDIR)/libxxhash.$(SHARED_EXT)
@@ -588,33 +632,36 @@
$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR) # includes
$(Q)$(INSTALL_DATA) xxhash.h $(DESTDIR)$(INCLUDEDIR)
$(Q)$(INSTALL_DATA) xxh3.h $(DESTDIR)$(INCLUDEDIR) # for compatibility, will be removed in v0.9.0
-ifeq ($(DISPATCH),1)
+ifeq ($(LIBXXH_DISPATCH),1)
$(Q)$(INSTALL_DATA) xxh_x86dispatch.h $(DESTDIR)$(INCLUDEDIR)
endif
install_libxxhash.pc: libxxhash.pc
@echo Installing pkgconfig
- $(Q)$(INSTALL_DIR) $(DESTDIR)$(PKGCONFIGDIR)/
+ $(Q)$(MAKE_DIR) $(DESTDIR)$(PKGCONFIGDIR)/
$(Q)$(INSTALL_DATA) libxxhash.pc $(DESTDIR)$(PKGCONFIGDIR)/
install_xxhsum: xxhsum
@echo Installing xxhsum
- $(Q)$(INSTALL_DIR) $(DESTDIR)$(BINDIR)/
+ $(Q)$(MAKE_DIR) $(DESTDIR)$(BINDIR)/
$(Q)$(INSTALL_PROGRAM) xxhsum $(DESTDIR)$(BINDIR)/xxhsum
$(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh32sum
$(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh64sum
$(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh128sum
+ $(Q)ln -sf xxhsum $(DESTDIR)$(BINDIR)/xxh3sum
install_man:
@echo Installing man pages
- $(Q)$(INSTALL_DIR) $(DESTDIR)$(MANDIR)/
+ $(Q)$(MAKE_DIR) $(DESTDIR)$(MANDIR)/
$(Q)$(INSTALL_DATA) $(MAN) $(DESTDIR)$(MANDIR)/xxhsum.1
$(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh32sum.1
$(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh64sum.1
$(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh128sum.1
+ $(Q)ln -sf xxhsum.1 $(DESTDIR)$(MANDIR)/xxh3sum.1
.PHONY: install
-install: install_libxxhash.a install_libxxhash install_libxxhash.includes install_libxxhash.pc install_xxhsum install_man ## install libraries, CLI, links and man page
+## install libraries, CLI, links and man pages
+install: install_libxxhash.a install_libxxhash install_libxxhash.includes install_libxxhash.pc install_xxhsum install_man
@echo xxhash installation completed
.PHONY: uninstall
@@ -630,10 +677,12 @@
$(Q)$(RM) $(DESTDIR)$(BINDIR)/xxh32sum
$(Q)$(RM) $(DESTDIR)$(BINDIR)/xxh64sum
$(Q)$(RM) $(DESTDIR)$(BINDIR)/xxh128sum
+ $(Q)$(RM) $(DESTDIR)$(BINDIR)/xxh3sum
$(Q)$(RM) $(DESTDIR)$(BINDIR)/xxhsum
$(Q)$(RM) $(DESTDIR)$(MANDIR)/xxh32sum.1
$(Q)$(RM) $(DESTDIR)$(MANDIR)/xxh64sum.1
$(Q)$(RM) $(DESTDIR)$(MANDIR)/xxh128sum.1
+ $(Q)$(RM) $(DESTDIR)$(MANDIR)/xxh3sum.1
$(Q)$(RM) $(DESTDIR)$(MANDIR)/xxhsum.1
@echo xxhsum successfully uninstalled
diff --git a/README.md b/README.md
index eea13e2..a634d1a 100644
--- a/README.md
+++ b/README.md
@@ -93,15 +93,17 @@
### Build modifiers
-The following macros can be set at compilation time to modify libxxhash's behavior. They are generally disabled by default.
+The following macros can be set at compilation time to modify `libxxhash`'s behavior. They are generally disabled by default.
-- `XXH_INLINE_ALL`: Make all functions `inline`, with implementations being directly included within `xxhash.h`.
- Inlining functions is beneficial for speed on small keys.
- It's _extremely effective_ when key length is expressed as _a compile time constant_,
+- `XXH_INLINE_ALL`: Make all functions `inline`, implementation is directly included within `xxhash.h`.
+ Inlining functions is beneficial for speed, notably for small keys.
+ It's _extremely effective_ when key's length is expressed as _a compile time constant_,
with performance improvements observed in the +200% range .
See [this article](https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html) for details.
- `XXH_PRIVATE_API`: same outcome as `XXH_INLINE_ALL`. Still available for legacy support.
The name underlines that `XXH_*` symbol names will not be exported.
+- `XXH_STATIC_LINKING_ONLY`: gives access to internal state declaration, required for static allocation.
+ Incompatible with dynamic linking, due to risks of ABI changes.
- `XXH_NAMESPACE`: Prefixes all symbols with the value of `XXH_NAMESPACE`.
This macro can only use compilable character set.
Useful to evade symbol naming collisions,
@@ -109,50 +111,66 @@
Client applications still use the regular function names,
as symbols are automatically translated through `xxhash.h`.
- `XXH_FORCE_ALIGN_CHECK`: Use a faster direct read path when input is aligned.
- This option can result in dramatic performance improvement when input to hash is aligned on 32 or 64-bit boundaries,
- when running on architectures unable to load memory from unaligned addresses, or suffering a performance penalty from it.
+ This option can result in dramatic performance improvement on architectures unable to load memory from unaligned addresses
+ when input to hash happens to be aligned on 32 or 64-bit boundaries.
It is (slightly) detrimental on platform with good unaligned memory access performance (same instruction for both aligned and unaligned accesses).
This option is automatically disabled on `x86`, `x64` and `aarch64`, and enabled on all other platforms.
- `XXH_FORCE_MEMORY_ACCESS`: The default method `0` uses a portable `memcpy()` notation.
Method `1` uses a gcc-specific `packed` attribute, which can provide better performance for some targets.
Method `2` forces unaligned reads, which is not standard compliant, but might sometimes be the only way to extract better read performance.
Method `3` uses a byteshift operation, which is best for old compilers which don't inline `memcpy()` or big-endian systems without a byteswap instruction.
-- `XXH_VECTOR` : manually select a vector instruction set (default: auto-selected at compilation time). Available instruction sets are `XXH_SCALAR`, `XXH_SSE2`, `XXH_AVX2`, `XXH_AVX512`, `XXH_NEON` and `XXH_VSX`. Compiler may require additional flags to ensure proper support (for example, `gcc` on linux will require `-mavx2` for `AVX2`, and `-mavx512f` for `AVX512`).
-- `XXH_NO_PREFETCH` : disable prefetching. Some platforms or situations may perform better without prefetching. XXH3 only.
-- `XXH_PREFETCH_DIST` : select prefetching distance. For close-to-metal adaptation to specific hardware platforms. XXH3 only.
-- `XXH_NO_STREAM`: Disables the streaming API, limiting it to single shot variants only.
-- `XXH_SIZE_OPT`: `0`: default, optimize for speed
- `1`: default for `-Os` and `-Oz`: disables some speed hacks for size optimization
- `2`: makes code as small as possible, performance may cry
-- `XXH_NO_INLINE_HINTS`: By default, xxHash uses `__attribute__((always_inline))` and `__forceinline` to improve performance at the cost of code size.
- Defining this macro to 1 will mark all internal functions as `static`, allowing the compiler to decide whether to inline a function or not.
- This is very useful when optimizing for smallest binary size,
- and is automatically defined when compiling with `-O0`, `-Os`, `-Oz`, or `-fno-inline` on GCC and Clang.
- This may also increase performance depending on compiler and architecture.
+- `XXH_CPU_LITTLE_ENDIAN`: By default, endianness is determined by a runtime test resolved at compile time.
+ If, for some reason, the compiler cannot simplify the runtime test, it can cost performance.
+ It's possible to skip auto-detection and simply state that the architecture is little-endian by setting this macro to 1.
+ Setting it to 0 states big-endian.
+- `XXH_ENABLE_AUTOVECTORIZE`: Auto-vectorization may be triggered for XXH32 and XXH64, depending on cpu vector capabilities and compiler version.
+ Note: auto-vectorization tends to be triggered more easily with recent versions of `clang`.
+ For XXH32, SSE4.1 or equivalent (NEON) is enough, while XXH64 requires AVX512.
+ Unfortunately, auto-vectorization is generally detrimental to XXH performance.
+ For this reason, the xxhash source code tries to prevent auto-vectorization by default.
+ That being said, systems evolve, and this conclusion is not forthcoming.
+ For example, it has been reported that recent Zen4 cpus are more likely to improve performance with vectorization.
+ Therefore, should you prefer or want to test vectorized code, you can enable this flag:
+ it will remove the no-vectorization protection code, thus making it more likely for XXH32 and XXH64 to be auto-vectorized.
- `XXH32_ENDJMP`: Switch multi-branch finalization stage of XXH32 by a single jump.
This is generally undesirable for performance, especially when hashing inputs of random sizes.
But depending on exact architecture and compiler, a jump might provide slightly better performance on small inputs. Disabled by default.
+- `XXH_IMPORT`: MSVC specific: should only be defined for dynamic linking, as it prevents linkage errors.
- `XXH_NO_STDLIB`: Disable invocation of `<stdlib.h>` functions, notably `malloc()` and `free()`.
`libxxhash`'s `XXH*_createState()` will always fail and return `NULL`.
But one-shot hashing (like `XXH32()`) or streaming using statically allocated states
still work as expected.
This build flag is useful for embedded environments without dynamic allocation.
-- `XXH_STATIC_LINKING_ONLY`: gives access to internal state declaration, required for static allocation.
- Incompatible with dynamic linking, due to risks of ABI changes.
-- `XXH_NO_XXH3` : removes symbols related to `XXH3` (both 64 & 128 bits) from generated binary.
- Useful to reduce binary size, notably for applications which do not employ `XXH3`.
-- `XXH_NO_LONG_LONG`: removes compilation of algorithms relying on 64-bit types (`XXH3` and `XXH64`). Only `XXH32` will be compiled.
- Useful for targets (architectures and compilers) without 64-bit support.
-- `XXH_IMPORT`: MSVC specific: should only be defined for dynamic linking, as it prevents linkage errors.
-- `XXH_CPU_LITTLE_ENDIAN`: By default, endianness is determined by a runtime test resolved at compile time.
- If, for some reason, the compiler cannot simplify the runtime test, it can cost performance.
- It's possible to skip auto-detection and simply state that the architecture is little-endian by setting this macro to 1.
- Setting it to 0 states big-endian.
- `XXH_DEBUGLEVEL` : When set to any value >= 1, enables `assert()` statements.
This (slightly) slows down execution, but may help finding bugs during debugging sessions.
+#### Binary size control
+- `XXH_NO_XXH3` : removes symbols related to `XXH3` (both 64 & 128 bits) from generated binary.
+ `XXH3` is by far the largest contributor to `libxxhash` size,
+ so it's useful to reduce binary size for applications which do not employ `XXH3`.
+- `XXH_NO_LONG_LONG`: removes compilation of algorithms relying on 64-bit `long long` types
+ which include `XXH3` and `XXH64`.
+ Only `XXH32` will be compiled.
+ Useful for targets (architectures and compilers) without 64-bit support.
+- `XXH_NO_STREAM`: Disables the streaming API, limiting the library to single shot variants only.
+- `XXH_NO_INLINE_HINTS`: By default, xxHash uses `__attribute__((always_inline))` and `__forceinline` to improve performance at the cost of code size.
+ Defining this macro to 1 will mark all internal functions as `static`, allowing the compiler to decide whether to inline a function or not.
+ This is very useful when optimizing for smallest binary size,
+ and is automatically defined when compiling with `-O0`, `-Os`, `-Oz`, or `-fno-inline` on GCC and Clang.
+ It may also be required to successfully compile using `-Og`, depending on compiler version.
+- `XXH_SIZE_OPT`: `0`: default, optimize for speed
+ `1`: default for `-Os` and `-Oz`: disables some speed hacks for size optimization
+ `2`: makes code as small as possible, performance may cry
+
+#### Build modifiers specific for XXH3
+- `XXH_VECTOR` : manually select a vector instruction set (default: auto-selected at compilation time). Available instruction sets are `XXH_SCALAR`, `XXH_SSE2`, `XXH_AVX2`, `XXH_AVX512`, `XXH_NEON` and `XXH_VSX`. Compiler may require additional flags to ensure proper support (for example, `gcc` on x86_64 requires `-mavx2` for `AVX2`, or `-mavx512f` for `AVX512`).
+- `XXH_PREFETCH_DIST` : select prefetching distance. For close-to-metal adaptation to specific hardware platforms. XXH3 only.
+- `XXH_NO_PREFETCH` : disable prefetching. Some platforms or situations may perform better without prefetching. XXH3 only.
+
+#### Makefile variables
When compiling the Command Line Interface `xxhsum` using `make`, the following environment variables can also be set :
-- `DISPATCH=1` : use `xxh_x86dispatch.c`, to automatically select between `scalar`, `sse2`, `avx2` or `avx512` instruction set at runtime, depending on local host. This option is only valid for `x86`/`x64` systems.
+- `DISPATCH=1` : use `xxh_x86dispatch.c`, select at runtime between `scalar`, `sse2`, `avx2` or `avx512` instruction set. This option is only valid for `x86`/`x64` systems. It is enabled by default when target `x86`/`x64` is detected. It can be forcefully turned off using `DISPATCH=0`.
+- `LIBXXH_DISPATCH=1` : same idea, implemented a runtime vector extension detector, but within `libxxhash`. This parameter is disabled by default. When enabled (only valid for `x86`/`x64` systems), new symbols published in `xxh_x86dispatch.h` become accessible. At the time of this writing, it's required to include `xxh_x86dispatch.h` in order to access the symbols with runtime vector extension detection.
- `XXH_1ST_SPEED_TARGET` : select an initial speed target, expressed in MB/s, for the first speed test in benchmark mode. Benchmark will adjust the target at subsequent iterations, but the first test is made "blindly" by targeting this speed. Currently conservatively set to 10 MB/s, to support very slow (emulated) platforms.
- `NODE_JS=1` : When compiling `xxhsum` for Node.js with Emscripten, this links the `NODERAWFS` library for unrestricted filesystem access and patches `isatty` to make the command line utility correctly detect the terminal. This does make the binary specific to Node.js.
diff --git a/cli/xsum_arch.c b/cli/xsum_arch.c
new file mode 100644
index 0000000..3540af4
--- /dev/null
+++ b/cli/xsum_arch.c
@@ -0,0 +1,51 @@
+/*
+ * xxhsum - Command line interface for xxhash algorithms
+ * Copyright (C) 2013-2024 Yann Collet
+ *
+ * GPL v2 License
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * You can contact the author at:
+ * - xxHash homepage: https://www.xxhash.com
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
+ */
+
+int g_xsumarch_avoid_empty_unit = 0;
+
+#if ((defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) && !defined(_M_ARM64EC)) || defined(__i386__) || defined(_M_IX86) || defined(_M_IX86_FP)
+#if defined(XXHSUM_DISPATCH)
+
+#include "../xxh_x86dispatch.h"
+
+const char* XSUM_autox86(void)
+{
+ int vecVersion = XXH_featureTest();
+ switch(vecVersion) {
+ case XXH_SCALAR:
+ return "x86 autoVec (scalar: no vector extension detected)";
+ case XXH_SSE2:
+ return "x86 autoVec (SSE2 detected)";
+ case XXH_AVX2:
+ return "x86 autoVec (AVX2 detected)";
+ case XXH_AVX512:
+ return "x86 autoVec (AVX512 detected)";
+ default:;
+ }
+ return " autoVec (error detecting vector extension)";
+}
+
+#endif /* XXHSUM_DISPATCH */
+#endif /* x86 */
diff --git a/cli/xsum_arch.h b/cli/xsum_arch.h
index f025f34..aae7eaa 100644
--- a/cli/xsum_arch.h
+++ b/cli/xsum_arch.h
@@ -89,7 +89,8 @@
/* Try to detect the architecture. */
#if defined(XSUM_ARCH_X86)
# if defined(XXHSUM_DISPATCH)
-# define XSUM_ARCH XSUM_ARCH_X86 " autoVec"
+ const char* XSUM_autox86(void);
+# define XSUM_ARCH XSUM_autox86()
# elif defined(__AVX512F__)
# define XSUM_ARCH XSUM_ARCH_X86 " + AVX512"
# elif defined(__AVX2__)
@@ -161,6 +162,8 @@
# else
# define XSUM_ARCH "wasm/asmjs"
# endif
+#elif defined(__loongarch_lp64)
+# define XSUM_ARCH "loongarch"
#else
# define XSUM_ARCH "unknown"
#endif
diff --git a/cli/xsum_bench.c b/cli/xsum_bench.c
index dfea66b..7f67969 100644
--- a/cli/xsum_bench.c
+++ b/cli/xsum_bench.c
@@ -340,9 +340,13 @@
/*!
* XSUM_benchMem():
- * buffer: Must be 16-byte aligned.
- * The real allocated size of buffer is supposed to be >= (bufferSize+3).
- * returns: 0 on success, 1 if error (invalid mode selected)
+ * Benchmark provided content up to twice per function:
+ * - once at provided aligned memory address (%16)
+ * - second time at unaligned memory address (+3)
+ * Enabled functions and modes are provided via @g_hashesToBench global variable.
+ * @buffer: Must be 16-byte aligned.
+ * The allocated size of underlying @buffer must be >= (@bufferSize+3).
+ * This function also fills @g_benchSecretBuf, to bench XXH3's _withSecret() variants.
*/
static void XSUM_benchMem(const void* buffer, size_t bufferSize)
{
diff --git a/cli/xsum_os_specific.c b/cli/xsum_os_specific.c
index 0f115fd..8438d0c 100644
--- a/cli/xsum_os_specific.c
+++ b/cli/xsum_os_specific.c
@@ -69,12 +69,17 @@
|| defined(__DJGPP__) \
|| defined(__MSYS__) \
|| defined(__HAIKU__)
+# ifdef __OpenBSD__
+# include <errno.h> /* errno */
+# include <string.h> /* strerror */
+# include "xsum_output.h" /* XSUM_log */
+# endif
# include <unistd.h> /* isatty */
# define XSUM_IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#elif defined(MSDOS) || defined(OS2)
# include <io.h> /* _isatty */
# define XSUM_IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
-#elif defined(WIN32) || defined(_WIN32)
+#elif defined(_WIN32)
# include <io.h> /* _isatty */
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
# include <stdio.h> /* FILE */
@@ -87,7 +92,7 @@
# define XSUM_IS_CONSOLE(stdStream) 0
#endif
-#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
+#if defined(MSDOS) || defined(OS2) || defined(_WIN32)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _fileno, _get_osfhandle */
# if !defined(__DJGPP__)
@@ -135,6 +140,16 @@
#ifndef XSUM_NO_MAIN
int main(int argc, const char* argv[])
{
+#ifdef __OpenBSD__
+ /*
+ * xxhsum(1) does not create or write files, permit reading only.
+ */
+ if (pledge("stdio rpath", NULL) == -1) {
+ XSUM_log("pledge: %s\n", strerror(errno));
+ return 1;
+ }
+#endif
+
return XSUM_main(argc, argv);
}
#endif
diff --git a/cli/xsum_sanity_check.c b/cli/xsum_sanity_check.c
index 624b93a..98014d5 100644
--- a/cli/xsum_sanity_check.c
+++ b/cli/xsum_sanity_check.c
@@ -390,7 +390,17 @@
XXH3_generateSecret_fromSeed(secret, seed);
{ XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
XSUM_checkResult64(Dresult, Nresult);
- } }
+ }
+
+ /* check that XXH3_64bits_withSecretandSeed()
+ * results in exactly the same return value as XXH3_64bits_withSeed()
+ * when len <= XXH3_MIDSIZE_MAX, whatever the content of @secret */
+ memset(secret, 0x99, 9);
+ if (len <= XXH3_MIDSIZE_MAX) {
+ XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
+ XSUM_checkResult64(Dresult, Nresult);
+ }
+ }
/* streaming API test */
{ XXH3_state_t* const state = XXH3_createState();
@@ -423,6 +433,17 @@
(void)XXH3_64bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
(void)XXH3_64bits_update(state, data, len);
XSUM_checkResult64(XXH3_64bits_digest(state), Nresult);
+
+ /* check that XXH3_64bits_withSecretandSeed()
+ * results in exactly the same return value as XXH3_64bits_withSeed()
+ * when len <= XXH3_MIDSIZE_MAX, whatever the content of @secret */
+ if (len <= XXH3_MIDSIZE_MAX) {
+ /* single ingestion */
+ memset(secret, 0x99, 9);
+ (void)XXH3_64bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
+ (void)XXH3_64bits_update(state, data, len);
+ XSUM_checkResult64(XXH3_64bits_digest(state), Nresult);
+ }
}
XXH3_freeState(state);
@@ -431,10 +452,6 @@
}
-#ifndef XXH3_MIDSIZE_MAX
-# define XXH3_MIDSIZE_MAX 240
-#endif
-
static void XSUM_testXXH3_withSecret(const void* data, const void* secret,
size_t secretSize, const XSUM_testdata64_t* testData)
{
@@ -478,10 +495,11 @@
}
/* check that XXH3_64bits_reset_withSecretandSeed()
- * results in exactly the same return value as XXH3_64bits_reset_withSecret() */
+ * results in exactly the same return value as XXH3_64bits_reset_withSecret()
+ * when len > XXH3_MIDSIZE_MAX, whatever the value of @seed */
if (len > XXH3_MIDSIZE_MAX) {
/* single ingestion */
- (void)XXH3_64bits_reset_withSecretandSeed(state, secret, secretSize, 0);
+ (void)XXH3_64bits_reset_withSecretandSeed(state, secret, secretSize, 17);
(void)XXH3_64bits_update(state, data, len);
XSUM_checkResult64(XXH3_64bits_digest(state), Nresult);
}
@@ -524,7 +542,17 @@
XXH3_generateSecret_fromSeed(secret, seed);
{ XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
XSUM_checkResult128(Dresult, Nresult);
- } }
+ }
+
+ /* check that XXH3_128bits_withSecretandSeed()
+ * results in exactly the same return value as XXH3_128bits_withSeed()
+ * if len <= XXH3_MIDSIZE_MAX, whatever the content of @secret */
+ memset(secret, 0x99, 9);
+ if (len <= XXH3_MIDSIZE_MAX) {
+ XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
+ XSUM_checkResult128(Dresult, Nresult);
+ }
+ }
/* streaming API test */
{ XXH3_state_t *state = XXH3_createState();
@@ -558,6 +586,17 @@
(void)XXH3_128bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
(void)XXH3_128bits_update(state, data, len);
XSUM_checkResult128(XXH3_128bits_digest(state), Nresult);
+
+ /* check that XXH3_128bits_reset_withSecretandSeed()
+ * results in exactly the same return value as XXH3_128bits_reset_withSeed()
+ * if len <= XXH3_MIDSIZE_MAX, whatever the content of @secret */
+ if (len <= XXH3_MIDSIZE_MAX) {
+ /* single ingestion */
+ memset(secret, 0x99, 9);
+ (void)XXH3_128bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
+ (void)XXH3_128bits_update(state, data, len);
+ XSUM_checkResult128(XXH3_128bits_digest(state), Nresult);
+ }
}
XXH3_freeState(state);
diff --git a/cli/xxhsum.1 b/cli/xxhsum.1
index c7938a9..dd5fdeb 100644
--- a/cli/xxhsum.1
+++ b/cli/xxhsum.1
@@ -1,120 +1,181 @@
-.TH "XXHSUM" "1" "July 2023" "xxhsum 0.8.2" "User Commands"
+.
+.TH "XXHSUM" "1" "May 2024" "xxhsum 0.8.3" "User Commands"
+.
.SH "NAME"
\fBxxhsum\fR \- print or check xxHash non\-cryptographic checksums
+.
.SH "SYNOPSIS"
-\fBxxhsum\fR [\fIOPTION\fR]\|\.\|\.\|\. [\fIFILE\fR]\|\.\|\.\|\.
-.br
-\fBxxhsum \-b\fR [\fIOPTION\fR]\|\.\|\.\|\.
+\fBxxhsum\fR [\fIOPTION\fR]\.\.\. [\fIFILE\fR]\.\.\. \fBxxhsum \-b\fR [\fIOPTION\fR]\.\.\.
+.
.P
-\fBxxh32sum\fR is equivalent to \fBxxhsum \-H0\fR, \fBxxh64sum\fR is equivalent to \fBxxhsum \-H1\fR, \fBxxh128sum\fR is equivalent to \fBxxhsum \-H2\fR\.
+\fBxxh32sum\fR is equivalent to \fBxxhsum \-H0\fR, \fBxxh64sum\fR is equivalent to \fBxxhsum \-H1\fR, \fBxxh128sum\fR is equivalent to \fBxxhsum \-H2\fR, \fBxxh3sum\fR is equivalent to \fBxxhsum \-H3\fR\.
+.
.SH "DESCRIPTION"
-Print or check xxHash (32, 64 or 128 bits) checksums\.
-.br
-When no \fIFILE\fR, read standard input, except if it's the console\. When \fIFILE\fR is \fB\-\fR, read standard input even if it's the console\.
+Print or check xxHash (32, 64 or 128 bits) checksums\. When no \fIFILE\fR, read standard input, except if it\'s the console\. When \fIFILE\fR is \fB\-\fR, read standard input even if it\'s the console\.
+.
.P
\fBxxhsum\fR supports a command line syntax similar but not identical to md5sum(1)\. Differences are:
-.IP "\[ci]" 4
-\fBxxhsum\fR doesn't have text mode switch (\fB\-t\fR)
-.IP "\[ci]" 4
-\fBxxhsum\fR doesn't have short binary mode switch (\fB\-b\fR)
-.IP "\[ci]" 4
+.
+.IP "\(bu" 4
+\fBxxhsum\fR doesn\'t have text mode switch (\fB\-t\fR)
+.
+.IP "\(bu" 4
+\fBxxhsum\fR doesn\'t have short binary mode switch (\fB\-b\fR)
+.
+.IP "\(bu" 4
\fBxxhsum\fR always treats files as binary file
-.IP "\[ci]" 4
+.
+.IP "\(bu" 4
\fBxxhsum\fR has a hash selection switch (\fB\-H\fR)
+.
.IP "" 0
+.
.P
As xxHash is a fast non\-cryptographic checksum algorithm, \fBxxhsum\fR should not be used for security related purposes\.
+.
.P
\fBxxhsum \-b\fR invokes benchmark mode\. See OPTIONS and EXAMPLES for details\.
+.
.SH "OPTIONS"
+.
.TP
\fB\-V\fR, \fB\-\-version\fR
Displays xxhsum version and exits
+.
.TP
\fB\-H\fR\fIHASHTYPE\fR
-Hash selection\. \fIHASHTYPE\fR means \fB0\fR=XXH32, \fB1\fR=XXH64, \fB2\fR=XXH128, \fB3\fR=XXH3\. Note that \fB\-H3\fR triggers \fB\-\-tag\fR, which can't be skipped (this is to reduce risks of confusion with \fB\-H2\fR (\fBXXH64\fR))\. Alternatively, \fIHASHTYPE\fR \fB32\fR=XXH32, \fB64\fR=XXH64, \fB128\fR=XXH128\. Default value is \fB1\fR (XXH64)
+Hash selection\. \fIHASHTYPE\fR means \fB0\fR=XXH32, \fB1\fR=XXH64, \fB2\fR=XXH128, \fB3\fR=XXH3\. Alternatively, \fIHASHTYPE\fR \fB32\fR=XXH32, \fB64\fR=XXH64, \fB128\fR=XXH128\. Default value is \fB1\fR (XXH64)
+.
.TP
\fB\-\-binary\fR
Read in binary mode\.
+.
.TP
\fB\-\-tag\fR
Output in the BSD style\.
+.
.TP
\fB\-\-little\-endian\fR
Set output hexadecimal checksum value as little endian convention\. By default, value is displayed as big endian\.
+.
.TP
\fB\-h\fR, \fB\-\-help\fR
Displays help and exits
+.
.SS "The following options are useful only when verifying checksums (\-c):"
+.
.TP
\fB\-c\fR, \fB\-\-check\fR \fIFILE\fR
Read xxHash sums from \fIFILE\fR and check them
+.
.TP
\fB\-q\fR, \fB\-\-quiet\fR
-Don't print OK for each successfully verified file
+Don\'t print OK for each successfully verified file
+.
.TP
\fB\-\-strict\fR
Return an error code if any line in the file is invalid, not just if some checksums are wrong\. This policy is disabled by default, though UI will prompt an informational message if any line in the file is detected invalid\.
+.
.TP
\fB\-\-status\fR
-Don't output anything\. Status code shows success\.
+Don\'t output anything\. Status code shows success\.
+.
.TP
\fB\-w\fR, \fB\-\-warn\fR
Emit a warning message about each improperly formatted checksum line\.
+.
.SS "The following options are useful only benchmark purpose:"
+.
.TP
\fB\-b\fR
Benchmark mode\. See EXAMPLES for details\.
+.
.TP
\fB\-b#\fR
-Specify ID of variant to be tested\. Multiple variants can be selected, separated by a ',' comma\.
+Specify ID of variant to be tested\. Multiple variants can be selected, separated by a \',\' comma\.
+.
.TP
\fB\-B\fR\fIBLOCKSIZE\fR
-Only useful for benchmark mode (\fB\-b\fR)\. See \fIEXAMPLES\fR for details\. \fIBLOCKSIZE\fR specifies benchmark mode's test data block size in bytes\. Default value is 102400
+Only useful for benchmark mode (\fB\-b\fR)\. See \fIEXAMPLES\fR for details\. \fIBLOCKSIZE\fR specifies benchmark mode\'s test data block size in bytes\. Default value is 102400
+.
.TP
\fB\-i\fR\fIITERATIONS\fR
Only useful for benchmark mode (\fB\-b\fR)\. See \fIEXAMPLES\fR for details\. \fIITERATIONS\fR specifies number of iterations in benchmark\. Single iteration lasts approximately 1000 milliseconds\. Default value is 3
+.
.SH "EXIT STATUS"
-\fBxxhsum\fR exit \fB0\fR on success, \fB1\fR if at least one file couldn't be read or doesn't have the same checksum as the \fB\-c\fR option\.
+\fBxxhsum\fR exit \fB0\fR on success, \fB1\fR if at least one file couldn\'t be read or doesn\'t have the same checksum as the \fB\-c\fR option\.
+.
.SH "EXAMPLES"
Output xxHash (64bit) checksum values of specific files to standard output
+.
.IP "" 4
+.
.nf
+
$ xxhsum \-H1 foo bar baz
+.
.fi
+.
.IP "" 0
+.
.P
Output xxHash (32bit and 64bit) checksum values of specific files to standard output, and redirect it to \fBxyz\.xxh32\fR and \fBqux\.xxh64\fR
+.
.IP "" 4
+.
.nf
+
$ xxhsum \-H0 foo bar baz > xyz\.xxh32
$ xxhsum \-H1 foo bar baz > qux\.xxh64
+.
.fi
+.
.IP "" 0
+.
.P
Read xxHash sums from specific files and check them
+.
.IP "" 4
+.
.nf
+
$ xxhsum \-c xyz\.xxh32 qux\.xxh64
+.
.fi
+.
.IP "" 0
+.
.P
Benchmark xxHash algorithm\. By default, \fBxxhsum\fR benchmarks xxHash main variants on a synthetic sample of 100 KB, and print results into standard output\. The first column is the algorithm, the second column is the source data size in bytes, the third column is the number of hashes generated per second (throughput), and finally the last column translates speed in megabytes per second\.
+.
.IP "" 4
+.
.nf
+
$ xxhsum \-b
+.
.fi
+.
.IP "" 0
+.
.P
In the following example, the sample to hash is set to 16384 bytes, the variants to be benched are selected by their IDs, and each benchmark test is repeated 10 times, for increased accuracy\.
+.
.IP "" 4
+.
.nf
+
$ xxhsum \-b1,2,3 \-i10 \-B16384
+.
.fi
+.
.IP "" 0
+.
.SH "BUGS"
Report bugs at: https://github\.com/Cyan4973/xxHash/issues/
+.
.SH "AUTHOR"
Yann Collet
+.
.SH "SEE ALSO"
md5sum(1)
diff --git a/cli/xxhsum.1.md b/cli/xxhsum.1.md
index 613ccee..7f5c76d 100644
--- a/cli/xxhsum.1.md
+++ b/cli/xxhsum.1.md
@@ -4,18 +4,20 @@
SYNOPSIS
--------
-`xxhsum` [*OPTION*]... [*FILE*]...
+`xxhsum` [*OPTION*]... [*FILE*]...
+
`xxhsum -b` [*OPTION*]...
`xxh32sum` is equivalent to `xxhsum -H0`,
`xxh64sum` is equivalent to `xxhsum -H1`,
-`xxh128sum` is equivalent to `xxhsum -H2`.
+`xxh128sum` is equivalent to `xxhsum -H2`,
+`xxh3sum` is equivalent to `xxhsum -H3`.
DESCRIPTION
-----------
-Print or check xxHash (32, 64 or 128 bits) checksums.
+Print or check xxHash (32, 64 or 128 bits) checksums.
When no *FILE*, read standard input, except if it's the console.
When *FILE* is `-`, read standard input even if it's the console.
@@ -34,13 +36,8 @@
OPTIONS
-------
-* `-V`, `--version`:
- Displays xxhsum version and exits
-
* `-H`*HASHTYPE*:
Hash selection. *HASHTYPE* means `0`=XXH32, `1`=XXH64, `2`=XXH128, `3`=XXH3.
- Note that `-H3` triggers `--tag`, which can't be skipped
- (this is to reduce risks of confusion with `-H2` (`XXH64`)).
Alternatively, *HASHTYPE* `32`=XXH32, `64`=XXH64, `128`=XXH128.
Default value is `1` (XXH64)
@@ -54,29 +51,42 @@
Set output hexadecimal checksum value as little endian convention.
By default, value is displayed as big endian.
+* `-V`, `--version`:
+ Displays xxhsum version and exits
+
* `-h`, `--help`:
Displays help and exits
-### The following options are useful only when verifying checksums (-c):
+### Advanced file input options
+
+* `--files-from`, `--filelist` *FILE*:
+ Read filenames from *FILE* and generate hashes for them.
+ `stdin` is also a valid way to provide filenames (when no *FILE* or `-` provided).
+ Valid format is one filename per line, which can include embedded spaces, etc with no need for quotes, escapes, etc.
+ A line commencing with '\\' will enable the convention used in the encoding of filenames against output hashes,
+ whereby subsequent \\\\, \n and \r seqeuences are converted to the single
+ character 0x5C, 0x0A and 0x0D respectively.
+
+### The following options are useful only for checksum verification:
* `-c`, `--check` *FILE*:
Read xxHash sums from *FILE* and check them
-* `-q`, `--quiet`:
- Don't print OK for each successfully verified file
-
* `--strict`:
- Return an error code if any line in the file is invalid,
+ Return an error code if any line in *FILE* is invalid,
not just if some checksums are wrong.
This policy is disabled by default,
though UI will prompt an informational message
if any line in the file is detected invalid.
-* `--status`:
- Don't output anything. Status code shows success.
-
* `-w`, `--warn`:
- Emit a warning message about each improperly formatted checksum line.
+ Emit a warning message about each improperly formatted line in *FILE*.
+
+* `-q`, `--quiet`:
+ Don't print OK for each successfully verified hash
+
+* `--status`:
+ Don't output anything. Only generate a Status code to show success.
### The following options are useful only benchmark purpose:
@@ -120,6 +130,19 @@
$ xxhsum -c xyz.xxh32 qux.xxh64
+Produce a list of files, then generate hashes for that list
+
+ $ find . -type f -name '*.[ch]' > c-files.txt
+ $ xxhsum --files-from c-files.txt
+
+Read the list of files from standard input to avoid the need for an intermediate file
+
+ $ find . -type f -name '*.[ch]' | xxhsum --files-from -
+
+Note that if shell expansion, length of argument list, clarity of use of spaces in filenames, etc allow it then the same output as the previous example can be generated like this
+
+ $ xxhsum `find . -name '*.[ch]'`
+
Benchmark xxHash algorithm.
By default, `xxhsum` benchmarks xxHash main variants
on a synthetic sample of 100 KB,
diff --git a/cli/xxhsum.c b/cli/xxhsum.c
index 6a5b028..046070d 100644
--- a/cli/xxhsum.c
+++ b/cli/xxhsum.c
@@ -1,6 +1,6 @@
/*
* xxhsum - Command line interface for xxhash algorithms
- * Copyright (C) 2013-2021 Yann Collet
+ * Copyright (C) 2013-2023 Yann Collet
*
* GPL v2 License
*
@@ -347,6 +347,14 @@
XSUM_printLine_BSD_internal(filename, canonicalHash, hashType, XSUM_algoName, XSUM_display_BigEndian);
}
+static void XSUM_displayPrefix(const AlgoSelected hashType)
+{
+ assert(0 <= hashType && (size_t)hashType <= XSUM_TABLE_ELT_SIZE(XSUM_algoName));
+ if (hashType == algo_xxh3) {
+ XSUM_output("XXH3_");
+ }
+}
+
static void XSUM_printLine_GNU_internal(const char* filename,
const void* canonicalHash, const AlgoSelected hashType,
XSUM_displayHash_f f_displayHash)
@@ -357,6 +365,7 @@
if (needsEscape) {
XSUM_output("%c", '\\');
}
+ XSUM_displayPrefix(hashType);
f_displayHash(canonicalHash, hashLength);
XSUM_output(" ");
XSUM_printFilename(filename, needsEscape);
@@ -375,27 +384,35 @@
XSUM_printLine_GNU_internal(filename, canonicalHash, hashType, XSUM_display_LittleEndian);
}
-typedef enum { big_endian, little_endian} Display_endianess;
+typedef enum { big_endian, little_endian} Display_endianness;
typedef enum { display_gnu, display_bsd } Display_convention;
typedef void (*XSUM_displayLine_f)(const char*, const void*, AlgoSelected); /* line display signature */
+typedef enum {
+ LineStatus_hashOk,
+ LineStatus_hashFailed,
+ LineStatus_failedToOpen,
+ LineStatus_isDirectory,
+ LineStatus_memoryError
+} LineStatus;
+
static XSUM_displayLine_f XSUM_kDisplayLine_fTable[2][2] = {
{ XSUM_printLine_GNU, XSUM_printLine_GNU_LE },
{ XSUM_printLine_BSD, XSUM_printLine_BSD_LE }
};
-static int XSUM_hashFile(const char* fileName,
+static LineStatus XSUM_hashFile(const char* fileName,
const AlgoSelected hashType,
- const Display_endianess displayEndianess,
+ const Display_endianness displayEndianness,
const Display_convention convention)
{
size_t const blockSize = 64 KB;
- XSUM_displayLine_f const f_displayLine = XSUM_kDisplayLine_fTable[convention][displayEndianess];
+ XSUM_displayLine_f const f_displayLine = XSUM_kDisplayLine_fTable[convention][displayEndianness];
FILE* inFile;
Multihash hashValue;
- assert(displayEndianess==big_endian || displayEndianess==little_endian);
+ assert(displayEndianness==big_endian || displayEndianness==little_endian);
assert(convention==display_gnu || convention==display_bsd);
/* Check file existence */
@@ -405,13 +422,11 @@
XSUM_setBinaryMode(stdin);
} else {
if (XSUM_isDirectory(fileName)) {
- XSUM_log("xxhsum: %s: Is a directory \n", fileName);
- return 1;
+ return LineStatus_isDirectory;
}
inFile = XSUM_fopen( fileName, "rb" );
if (inFile==NULL) {
- XSUM_log("Error: Could not open '%s': %s. \n", fileName, strerror(errno));
- return 1;
+ return LineStatus_failedToOpen;
} }
/* Memory allocation & streaming */
@@ -419,7 +434,7 @@
if (buffer == NULL) {
XSUM_log("\nError: Out of memory.\n");
fclose(inFile);
- return 1;
+ return LineStatus_memoryError;
}
/* Stream file & update hash */
@@ -460,7 +475,7 @@
assert(0); /* not possible */
}
- return 0;
+ return LineStatus_hashOk;
}
@@ -470,18 +485,59 @@
*/
static int XSUM_hashFiles(const char* fnList[], int fnTotal,
AlgoSelected hashType,
- Display_endianess displayEndianess,
+ Display_endianness displayEndianness,
Display_convention convention)
{
int fnNb;
int result = 0;
- if (fnTotal==0)
- return XSUM_hashFile(stdinName, hashType, displayEndianess, convention);
+ if (fnTotal == 0)
+ {
+ LineStatus filestatus = XSUM_hashFile(stdinName, hashType, displayEndianness, convention);
+ switch (filestatus)
+ {
+ case LineStatus_hashOk:
+ case LineStatus_hashFailed:
+ break;
+ case LineStatus_isDirectory:
+ XSUM_log("xxhsum: %s: Is a directory \n", stdinName);
+ break;
+ case LineStatus_failedToOpen:
+ XSUM_log("Error: Could not open '%s': %s. \n", stdinName, strerror(errno));
+ break;
+ case LineStatus_memoryError:
+ XSUM_log("\nError: Out of memory.\n");
+ break;
+ }
- for (fnNb=0; fnNb<fnTotal; fnNb++)
- result |= XSUM_hashFile(fnList[fnNb], hashType, displayEndianess, convention);
- XSUM_logVerbose(2, "\r%70s\r", "");
+ if (filestatus != LineStatus_hashOk)
+ result = 1;
+ }
+
+
+ for (fnNb = 0; fnNb < fnTotal; fnNb++)
+ {
+ LineStatus filestatus = XSUM_hashFile(fnList[fnNb], hashType, displayEndianness, convention);
+ switch (filestatus)
+ {
+ case LineStatus_hashOk:
+ case LineStatus_hashFailed:
+ break;
+ case LineStatus_isDirectory:
+ XSUM_log("xxhsum: %s: Is a directory \n", fnList[fnNb]);
+ break;
+ case LineStatus_failedToOpen:
+ XSUM_log("Error: Could not open '%s': %s. \n", fnList[fnNb], strerror(errno));
+ break;
+ case LineStatus_memoryError:
+ XSUM_log("\nError: Out of memory.\n");
+ break;
+ }
+
+ if (filestatus != LineStatus_hashOk)
+ result = 1;
+ }
+
return result;
}
@@ -504,12 +560,6 @@
ParseLine_invalidFormat
} ParseLineResult;
-typedef enum {
- LineStatus_hashOk,
- LineStatus_hashFailed,
- LineStatus_failedToOpen
-} LineStatus;
-
typedef union {
XXH32_canonical_t xxh32;
XXH64_canonical_t xxh64;
@@ -534,18 +584,18 @@
} ParseFileReport;
typedef struct {
- const char* inFileName;
- FILE* inFile;
- int lineMax;
- char* lineBuf;
- size_t blockSize;
- char* blockBuf;
- XSUM_U32 strictMode;
- XSUM_U32 statusOnly;
- XSUM_U32 ignoreMissing;
- XSUM_U32 warn;
- XSUM_U32 quiet;
- XSUM_U32 algoBitmask;
+ const char* inFileName;
+ FILE* inFile;
+ int lineMax;
+ char* lineBuf;
+ size_t blockSize;
+ char* blockBuf;
+ int strictMode;
+ int statusOnly;
+ int ignoreMissing;
+ int warn;
+ int quiet;
+ XSUM_U32 algoBitmask;
ParseFileReport report;
} ParseFileArg;
@@ -707,6 +757,7 @@
hash_len = (size_t)(firstSpace - line);
if (hash_len==8) parsedLine->algo = algo_xxh32;
if (hash_len==16) parsedLine->algo = algo_xxh64;
+ if (hash_len==21) parsedLine->algo = algo_xxh3;
if (hash_len==32) parsedLine->algo = algo_xxh128;
}
@@ -737,6 +788,18 @@
break;
}
+ case 21:
+ if (parsedLine->algo != algo_xxh3) return ParseLine_invalidFormat;
+ /* check XXH3 prefix*/
+ if (memcmp(hash_ptr, "XXH3_", 5) != 0) return ParseLine_invalidFormat;
+ { XXH64_canonical_t* xxh64c = &parsedLine->canonical.xxh64;
+ if (XSUM_canonicalFromString(xxh64c->digest, sizeof(xxh64c->digest), (const char*)hash_ptr+5, rev)
+ != CanonicalFromString_ok) {
+ return ParseLine_invalidFormat;
+ }
+ break;
+ }
+
case 32:
if (parsedLine->algo != algo_xxh128) return ParseLine_invalidFormat;
{ XXH128_canonical_t* xxh128c = &parsedLine->canonical.xxh128;
@@ -802,9 +865,10 @@
break;
}
- { GetLineResult const XSUM_getLineResult = XSUM_getLine(&XSUM_parseFileArg->lineBuf,
- &XSUM_parseFileArg->lineMax,
- XSUM_parseFileArg->inFile);
+ { GetLineResult const XSUM_getLineResult =
+ XSUM_getLine(&XSUM_parseFileArg->lineBuf,
+ &XSUM_parseFileArg->lineMax,
+ XSUM_parseFileArg->inFile);
/* Ignore comment lines */
if (XSUM_getLineResult == GetLine_comment) {
@@ -898,6 +962,11 @@
report->quit = 1;
break;
+ case LineStatus_memoryError:
+ case LineStatus_isDirectory:
+ assert(0); /* Never happens on these paths */
+ break;
+
case LineStatus_failedToOpen:
if (XSUM_parseFileArg->ignoreMissing) {
report->nMissing++;
@@ -952,12 +1021,12 @@
* - (strict mode) All lines in checksum file are consistent and well formatted.
*/
static int XSUM_checkFile(const char* inFileName,
- const Display_endianess displayEndianess,
- XSUM_U32 strictMode,
- XSUM_U32 statusOnly,
- XSUM_U32 ignoreMissing,
- XSUM_U32 warn,
- XSUM_U32 quiet,
+ const Display_endianness displayEndianness,
+ int strictMode,
+ int statusOnly,
+ int ignoreMissing,
+ int warn,
+ int quiet,
XSUM_U32 algoBitmask)
{
int result = 0;
@@ -1001,7 +1070,7 @@
XSUM_log("Error: : memory allocation failed \n");
exit(1);
}
- XSUM_parseFile1(XSUM_parseFileArg, displayEndianess != big_endian);
+ XSUM_parseFile1(XSUM_parseFileArg, displayEndianness != big_endian);
free(XSUM_parseFileArg->blockBuf);
free(XSUM_parseFileArg->lineBuf);
@@ -1049,12 +1118,12 @@
static int XSUM_checkFiles(const char* fnList[], int fnTotal,
- const Display_endianess displayEndianess,
- XSUM_U32 strictMode,
- XSUM_U32 statusOnly,
- XSUM_U32 ignoreMissing,
- XSUM_U32 warn,
- XSUM_U32 quiet,
+ const Display_endianness displayEndianness,
+ int strictMode,
+ int statusOnly,
+ int ignoreMissing,
+ int warn,
+ int quiet,
XSUM_U32 algoBitmask)
{
int ok = 1;
@@ -1062,11 +1131,255 @@
/* Special case for stdinName "-",
* note: stdinName is not a string. It's special pointer. */
if (fnTotal==0) {
- ok &= XSUM_checkFile(stdinName, displayEndianess, strictMode, statusOnly, ignoreMissing, warn, quiet, algoBitmask);
+ ok &= XSUM_checkFile(stdinName, displayEndianness, strictMode, statusOnly, ignoreMissing, warn, quiet, algoBitmask);
} else {
int fnNb;
for (fnNb=0; fnNb<fnTotal; fnNb++)
- ok &= XSUM_checkFile(fnList[fnNb], displayEndianess, strictMode, statusOnly, ignoreMissing, warn, quiet, algoBitmask);
+ ok &= XSUM_checkFile(fnList[fnNb], displayEndianness, strictMode, statusOnly, ignoreMissing, warn, quiet, algoBitmask);
+ }
+ return ok ? 0 : 1;
+}
+
+
+/*
+*
+* Parse single filename from list to generate hashes for.
+* Returns ParseLine_invalidFormat if the filename is not well formatted.
+* Returns ParseLine_ok if the filename is parsed successfully.
+*/
+static ParseLineResult XSUM_parseGenLine(ParsedLine * parsedLine,
+ char* filename)
+{
+ if (XSUM_lineNeedsUnescape(filename)) {
+ size_t filenameLen;
+ ++filename;
+ filenameLen = strlen(filename);
+
+ if (XSUM_filenameUnescape(filename, filenameLen) == NULL) {
+ parsedLine->filename = NULL;
+ return ParseLine_invalidFormat;
+ }
+ }
+
+ parsedLine->filename = filename;
+
+ return ParseLine_ok;
+}
+
+/*
+ * Parse gen source file.
+ */
+static void XSUM_parseGenFile1(ParseFileArg* XSUM_parseGenArg,
+ AlgoSelected hashType,
+ Display_endianness displayEndianness,
+ Display_convention convention)
+{
+ const char* const inFileName = XSUM_parseGenArg->inFileName;
+ ParseFileReport* const report = &XSUM_parseGenArg->report;
+
+ unsigned long lineNumber = 0;
+ memset(report, 0, sizeof(*report));
+
+ while (!report->quit) {
+ LineStatus lineStatus = LineStatus_hashFailed;
+ ParsedLine parsedLine;
+ memset(&parsedLine, 0, sizeof(parsedLine));
+
+ lineNumber++;
+ if (lineNumber == 0) {
+ /* This is unlikely happen, but md5sum.c has this error check. */
+ XSUM_log("%s: Error: Too many generate lines\n", inFileName);
+ report->quit = 1;
+ break;
+ }
+
+ { GetLineResult const XSUM_getLineResult = XSUM_getLine(&XSUM_parseGenArg->lineBuf,
+ &XSUM_parseGenArg->lineMax,
+ XSUM_parseGenArg->inFile);
+
+ /* Ignore comment lines */
+ if (XSUM_getLineResult == GetLine_comment) {
+ continue;
+ }
+
+ if (XSUM_getLineResult != GetLine_ok) {
+ if (XSUM_getLineResult == GetLine_eof) break;
+
+ switch (XSUM_getLineResult)
+ {
+ case GetLine_ok:
+ case GetLine_comment:
+ case GetLine_eof:
+ /* These cases never happen. See above XSUM_getLineResult related "if"s.
+ They exist just for make gcc's -Wswitch-enum happy. */
+ assert(0);
+ break;
+
+ default:
+ XSUM_log("%s:%lu: Error: Unknown error.\n", inFileName, lineNumber);
+ break;
+
+ case GetLine_exceedMaxLineLength:
+ XSUM_log("%s:%lu: Error: Line too long.\n", inFileName, lineNumber);
+ break;
+
+ case GetLine_outOfMemory:
+ XSUM_log("%s:%lu: Error: Out of memory.\n", inFileName, lineNumber);
+ break;
+ }
+ report->quit = 1;
+ break;
+ } }
+
+ if (XSUM_parseGenLine(&parsedLine, XSUM_parseGenArg->lineBuf) != ParseLine_ok) {
+ report->nImproperlyFormattedLines++;
+ if (XSUM_parseGenArg->warn) {
+ XSUM_log("%s:%lu: Error: Improperly formatted line.\n",
+ inFileName, lineNumber);
+ }
+ continue;
+ }
+
+ report->nProperlyFormattedLines++;
+
+ lineStatus = XSUM_hashFile(parsedLine.filename, hashType, displayEndianness, convention);
+
+ switch (lineStatus)
+ {
+ default:
+ XSUM_log("%s: Error: Unknown error.\n", parsedLine.filename);
+ report->quit = 1;
+ break;
+
+ case LineStatus_memoryError:
+ XSUM_log("\nError: Out of memory.\n");
+ break;
+
+ case LineStatus_failedToOpen:
+ case LineStatus_isDirectory:
+ if (XSUM_parseGenArg->ignoreMissing) {
+ report->nMissing++;
+ }
+ else {
+ report->nOpenOrReadFailures++;
+ if (!XSUM_parseGenArg->statusOnly) {
+ XSUM_output(
+ lineStatus == LineStatus_failedToOpen ?
+ "%s:%lu: Could not open or read '%s': %s.\n" :
+ "%s:%lu: Target is a directory '%s'.\n", /* Leaves errno argument unconsumed */
+ inFileName, lineNumber, parsedLine.filename, strerror(errno));
+ }
+ }
+ break;
+
+ case LineStatus_hashOk:
+ case LineStatus_hashFailed:
+ break;
+ }
+ } /* while (!report->quit) */
+}
+
+
+/* Parse text file for list of targets.
+ */
+static int XSUM_generateFile(const char* inFileName,
+ AlgoSelected hashType,
+ Display_endianness displayEndianness,
+ Display_convention convention,
+ int statusOnly,
+ int ignoreMissing,
+ int warn)
+{
+ int result = 0;
+ FILE* inFile = NULL;
+ ParseFileArg XSUM_parseGenArgBody;
+ ParseFileArg* const XSUM_parseGenArg = &XSUM_parseGenArgBody;
+ ParseFileReport* const report = &XSUM_parseGenArg->report;
+
+ /* note: stdinName is special constant pointer. It is not a string. */
+ if (inFileName == stdinName) {
+ /*
+ * Note: Since we expect text input for xxhash -c mode,
+ * we don't set binary mode for stdin.
+ */
+ inFileName = stdinFileName; /* "stdin" */
+ inFile = stdin;
+ }
+ else {
+ inFile = XSUM_fopen(inFileName, "rt");
+ }
+
+ if (inFile == NULL) {
+ XSUM_log("Error: Could not open '%s': %s\n", inFileName, strerror(errno));
+ return 0;
+ }
+
+ XSUM_parseGenArg->inFileName = inFileName;
+ XSUM_parseGenArg->inFile = inFile;
+ XSUM_parseGenArg->lineMax = DEFAULT_LINE_LENGTH;
+ XSUM_parseGenArg->lineBuf = (char*)malloc((size_t)XSUM_parseGenArg->lineMax);
+ XSUM_parseGenArg->blockSize = 64 * 1024;
+ XSUM_parseGenArg->blockBuf = (char*)malloc(XSUM_parseGenArg->blockSize);
+ XSUM_parseGenArg->statusOnly = statusOnly;
+ XSUM_parseGenArg->ignoreMissing = ignoreMissing;
+ XSUM_parseGenArg->warn = warn;
+
+ if ((XSUM_parseGenArg->lineBuf == NULL)
+ || (XSUM_parseGenArg->blockBuf == NULL)) {
+ XSUM_log("Error: : memory allocation failed \n");
+ exit(1);
+ }
+ XSUM_parseGenFile1(XSUM_parseGenArg, hashType, displayEndianness, convention);
+
+ free(XSUM_parseGenArg->blockBuf);
+ free(XSUM_parseGenArg->lineBuf);
+
+ if (inFile != stdin) fclose(inFile);
+
+ /* Show error/warning messages. All messages are copied from md5sum.c
+ */
+ if (report->nProperlyFormattedLines == 0) {
+ XSUM_log("%s: no properly formatted filename lines found\n", inFileName);
+ }
+ if (report->nImproperlyFormattedLines) {
+ XSUM_output("%lu %s improperly formatted\n"
+ , report->nImproperlyFormattedLines
+ , report->nImproperlyFormattedLines == 1 ? "line is" : "lines are");
+ }
+ if (report->nOpenOrReadFailures) {
+ XSUM_output("%lu listed %s could not be read\n"
+ , report->nOpenOrReadFailures
+ , report->nOpenOrReadFailures == 1 ? "file" : "files");
+ }
+ /* Result (exit) code logic is copied from
+ * gnu coreutils/src/md5sum.c digest_check() */
+ result = report->nProperlyFormattedLines != 0
+ && report->nOpenOrReadFailures == 0
+ && (report->nImproperlyFormattedLines == 0)
+ && report->quit == 0;
+
+ return result;
+}
+
+static int XSUM_generateFiles(const char* fnList[], int fnTotal,
+ AlgoSelected hashType,
+ Display_endianness displayEndianness,
+ Display_convention convention,
+ int statusOnly,
+ int ignoreMissing,
+ int warn)
+{
+ int ok = 1;
+
+ /* Special case for stdinName "-",
+ * note: stdinName is not a string. It's special pointer. */
+ if (fnTotal == 0) {
+ ok &= XSUM_generateFile(stdinName, hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn);
+ }
+ else {
+ int fnNb;
+ for (fnNb = 0; fnNb < fnTotal; fnNb++)
+ ok &= XSUM_generateFile(fnList[fnNb], hashType, displayEndianness, convention, statusOnly, ignoreMissing, warn);
}
return ok ? 0 : 1;
}
@@ -1079,13 +1392,18 @@
static int XSUM_usage(const char* exename)
{
XSUM_log( WELCOME_MESSAGE(exename) );
- XSUM_log( "Print or verify checksums using fast non-cryptographic algorithm xxHash \n\n" );
+ XSUM_log( "Create or verify checksums using fast non-cryptographic algorithm xxHash \n\n" );
XSUM_log( "Usage: %s [options] [files] \n\n", exename);
XSUM_log( "When no filename provided or when '-' is provided, uses stdin as input. \n");
- XSUM_log( "Options: \n");
- XSUM_log( " -H# algorithm selection: 0,1,2,3 or 32,64,128 (default: %i) \n", (int)g_defaultAlgo);
- XSUM_log( " -c, --check read xxHash checksum from [files] and check them \n");
- XSUM_log( " -h, --help display a long help page about advanced options \n");
+ XSUM_log( "\nOptions: \n");
+ XSUM_log( " -H# select an xxhash algorithm (default: %i) \n", (int)g_defaultAlgo);
+ XSUM_log( " 0: XXH32 \n");
+ XSUM_log( " 1: XXH64 \n");
+ XSUM_log( " 2: XXH128 (also called XXH3_128bits) \n");
+ XSUM_log( " 3: XXH3 (also called XXH3_64bits) \n");
+ XSUM_log( " -c, --check read xxHash checksum from [files] and check them \n");
+ XSUM_log( " --filelist generate hashes for files listed in [files] \n");
+ XSUM_log( " -h, --help display a long help page about advanced options \n");
return 0;
}
@@ -1093,7 +1411,7 @@
static int XSUM_usage_advanced(const char* exename)
{
XSUM_usage(exename);
- XSUM_log( "Advanced :\n");
+ XSUM_log( "\nAdvanced :\n");
XSUM_log( " -V, --version Display version information \n");
XSUM_log( " --tag Produce BSD-style checksum lines \n");
XSUM_log( " --little-endian Checksum values use little endian convention (default: big endian) \n");
@@ -1103,11 +1421,11 @@
XSUM_log( " -i# Number of times to run the benchmark (default: %i) \n", NBLOOPS_DEFAULT);
XSUM_log( " -q, --quiet Don't display version header in benchmark mode \n");
XSUM_log( "\n");
- XSUM_log( "The following five options are useful only when verifying checksums (-c): \n");
- XSUM_log( " -q, --quiet Don't print OK for each successfully verified file \n");
+ XSUM_log( "The following five options are useful only when using lists in [files] to verify or generate checksums: \n");
+ XSUM_log( " -q, --quiet Don't print OK for each successfully verified hash \n");
XSUM_log( " --status Don't output anything, status code shows success \n");
- XSUM_log( " --strict Exit non-zero for improperly formatted checksum lines \n");
- XSUM_log( " --warn Warn about improperly formatted checksum lines \n");
+ XSUM_log( " --strict Exit non-zero for improperly formatted lines in [files] \n");
+ XSUM_log( " --warn Warn about improperly formatted lines in [files] \n");
XSUM_log( " --ignore-missing Don't fail or report status for missing files \n");
return 0;
}
@@ -1186,19 +1504,20 @@
{
int i, filenamesStart = 0;
const char* const exename = XSUM_lastNameFromPath(argv[0]);
- XSUM_U32 benchmarkMode = 0;
- XSUM_U32 fileCheckMode = 0;
- XSUM_U32 strictMode = 0;
- XSUM_U32 statusOnly = 0;
- XSUM_U32 warn = 0;
- XSUM_U32 ignoreMissing = 0;
+ int benchmarkMode = 0;
+ int fileCheckMode = 0;
+ int readFilenamesMode = 0;
+ int strictMode = 0;
+ int statusOnly = 0;
+ int warn = 0;
+ int ignoreMissing = 0;
XSUM_U32 algoBitmask = algo_bitmask_all;
int explicitStdin = 0;
XSUM_U32 selectBenchIDs= 0; /* 0 == use default k_testIDs_default, kBenchAll == bench all */
static const XSUM_U32 kBenchAll = 99;
size_t keySize = XSUM_DEFAULT_SAMPLE_SIZE;
AlgoSelected algo = g_defaultAlgo;
- Display_endianess displayEndianess = big_endian;
+ Display_endianness displayEndianness = big_endian;
Display_convention convention = display_gnu;
int nbIterations = NBLOOPS_DEFAULT;
@@ -1206,16 +1525,19 @@
if (strstr(exename, "xxh32sum") != NULL) { algo = g_defaultAlgo = algo_xxh32; algoBitmask = algo_bitmask_xxh32; }
if (strstr(exename, "xxh64sum") != NULL) { algo = g_defaultAlgo = algo_xxh64; algoBitmask = algo_bitmask_xxh64; }
if (strstr(exename, "xxh128sum") != NULL) { algo = g_defaultAlgo = algo_xxh128; algoBitmask = algo_bitmask_xxh128; }
+ if (strstr(exename, "xxh3sum") != NULL) { algo = g_defaultAlgo = algo_xxh3; algoBitmask = algo_bitmask_xxh3; }
for (i=1; i<argc; i++) {
const char* argument = argv[i];
assert(argument != NULL);
if (!strcmp(argument, "--check")) { fileCheckMode = 1; continue; }
+ if (!strcmp(argument, "--files-from")) { readFilenamesMode = 1; continue; }
+ if (!strcmp(argument, "--filelist")) { readFilenamesMode = 1; continue; }
if (!strcmp(argument, "--benchmark-all")) { benchmarkMode = 1; selectBenchIDs = kBenchAll; continue; }
if (!strcmp(argument, "--bench-all")) { benchmarkMode = 1; selectBenchIDs = kBenchAll; continue; }
if (!strcmp(argument, "--quiet")) { XSUM_logLevel--; continue; }
- if (!strcmp(argument, "--little-endian")) { displayEndianess = little_endian; continue; }
+ if (!strcmp(argument, "--little-endian")) { displayEndianness = little_endian; continue; }
if (!strcmp(argument, "--strict")) { strictMode = 1; continue; }
if (!strcmp(argument, "--status")) { statusOnly = 1; continue; }
if (!strcmp(argument, "--warn")) { warn = 1; continue; }
@@ -1235,7 +1557,7 @@
}
/* command selection */
- argument++; /* note: *argument=='-' */
+ argument++; /* note: argument[0] was =='-' */
if (*argument == 0) explicitStdin = 1;
while (*argument != 0) {
@@ -1260,10 +1582,8 @@
case 64: algo = algo_xxh64; break;
case 2 :
case 128: algo = algo_xxh128; break;
- case 3 : /* xxh3 - necessarily uses BSD convention to avoid confusion with XXH64 */
- algo = algo_xxh3;
- convention = display_bsd;
- break;
+ case 3 :
+ algo = algo_xxh3; break;
default:
return XSUM_badusage(exename);
}
@@ -1275,6 +1595,12 @@
argument++;
break;
+ /* Generate hash mode (tar style short form of --files-from)
+ case 'T':
+ readFilenamesMode = 2;
+ argument++;
+ break; */
+
/* Warning mode (file check mode only, alias of "--warning") */
case 'w':
warn=1;
@@ -1336,10 +1662,16 @@
return XSUM_badusage(exename);
if (filenamesStart==0) filenamesStart = argc;
+
if (fileCheckMode) {
return XSUM_checkFiles(argv+filenamesStart, argc-filenamesStart,
- displayEndianess, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/, algoBitmask);
- } else {
- return XSUM_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianess, convention);
+ displayEndianness, strictMode, statusOnly, ignoreMissing, warn, (XSUM_logLevel < 2) /*quiet*/, algoBitmask);
}
+
+ if (readFilenamesMode) {
+ return XSUM_generateFiles(argv + filenamesStart, argc - filenamesStart, algo, displayEndianness, convention, statusOnly, ignoreMissing, warn);
+ }
+
+ return XSUM_hashFiles(argv+filenamesStart, argc-filenamesStart, algo, displayEndianness, convention);
+
}
diff --git a/clib.json b/clib.json
new file mode 100644
index 0000000..242343c
--- /dev/null
+++ b/clib.json
@@ -0,0 +1,12 @@
+{
+ "name": "xxhash",
+ "version": "0.8.2",
+ "repo": "Cyan4973/xxhash",
+ "description": "Extremely fast non-cryptographic hash algorithm",
+ "keywords": ["xxhash", "hashing"],
+ "license": "BSD-2-Clause",
+ "src": [
+ "xxhash.c",
+ "xxhash.h"
+ ]
+}
diff --git a/cmake_unofficial/CMakeLists.txt b/cmake_unofficial/CMakeLists.txt
index e8fbb53..89242ec 100644
--- a/cmake_unofficial/CMakeLists.txt
+++ b/cmake_unofficial/CMakeLists.txt
@@ -5,7 +5,7 @@
#
# For details, see <https://creativecommons.org/publicdomain/zero/1.0/>.
-cmake_minimum_required (VERSION 2.8.12 FATAL_ERROR)
+cmake_minimum_required (VERSION 3.10 FATAL_ERROR)
set(XXHASH_DIR "${CMAKE_CURRENT_SOURCE_DIR}/..")
@@ -25,14 +25,10 @@
else()
cmake_policy (SET CMP0077 NEW)
endif()
-if("${CMAKE_VERSION}" VERSION_LESS "3.0")
- project(xxHash C)
-else()
- cmake_policy (SET CMP0048 NEW)
- project(xxHash
- VERSION ${XXHASH_VERSION_STRING}
- LANGUAGES C)
-endif()
+cmake_policy (SET CMP0048 NEW)
+project(xxHash
+ VERSION ${XXHASH_VERSION_STRING}
+ LANGUAGES C)
if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Project build type" FORCE)
@@ -75,12 +71,9 @@
include(CMakeDependentOption)
CMAKE_DEPENDENT_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON "NOT XXHASH_BUNDLED_MODE" OFF)
-if("${CMAKE_VERSION}" VERSION_LESS "3.10")
- # Can not enable DISPATCH mode since it fails to recognize architecture.
-else()
- CMAKE_HOST_SYSTEM_INFORMATION(RESULT PLATFORM QUERY OS_PLATFORM)
- message(STATUS "Architecture: ${PLATFORM}")
-endif()
+# detect architecture for DISPATCH mode
+CMAKE_HOST_SYSTEM_INFORMATION(RESULT PLATFORM QUERY OS_PLATFORM)
+message(STATUS "Architecture: ${PLATFORM}")
# libxxhash
if((DEFINED DISPATCH) AND (DEFINED PLATFORM))
@@ -119,6 +112,7 @@
list(APPEND XXHSUM_SOURCES "${XXHASH_DIR}/xxh_x86dispatch.c")
endif()
list(APPEND XXHSUM_SOURCES "${XXHSUM_DIR}/xxhsum.c"
+ "${XXHSUM_DIR}/xsum_arch.c"
"${XXHSUM_DIR}/xsum_os_specific.c"
"${XXHSUM_DIR}/xsum_output.c"
"${XXHSUM_DIR}/xsum_sanity_check.c"
@@ -174,19 +168,9 @@
${PROJECT_SOURCE_DIR}/xxHashConfig.cmake.in
${xxHash_PROJECT_CONFIG}
INSTALL_DESTINATION ${xxHash_CONFIG_INSTALL_DIR})
- if("${CMAKE_VERSION}" VERSION_LESS "3.0")
- set(XXHASH_EXPORT_SET xxhash)
- if(XXHASH_BUILD_XXHSUM)
- set(XXHASH_EXPORT_SET ${XXHASH_EXPORT_SET} xxhsum)
- endif()
- export(TARGETS ${XXHASH_EXPORT_SET}
- FILE ${xxHash_TARGETS_CONFIG}
- NAMESPACE ${PROJECT_NAME}::)
- else()
export(EXPORT xxHashTargets
FILE ${xxHash_TARGETS_CONFIG}
NAMESPACE ${PROJECT_NAME}::)
- endif()
install(FILES ${xxHash_PROJECT_CONFIG} ${xxHash_VERSION_CONFIG}
DESTINATION ${xxHash_CONFIG_INSTALL_DIR})
diff --git a/fuzz/fuzzer.c b/fuzz/fuzzer.c
new file mode 100644
index 0000000..731ba4d
--- /dev/null
+++ b/fuzz/fuzzer.c
@@ -0,0 +1,11 @@
+#include <stdint.h>
+#include "xxhash.h"
+
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+ volatile XXH64_hash_t hash64 = XXH64(data, size, 0);
+ (void)hash64;
+ volatile XXH32_hash_t hash32 = XXH32(data, size, 0);
+ (void)hash32;
+ return 0;
+}
diff --git a/tests/bench/.gitignore b/tests/bench/.gitignore
index ede2d58..93f3e07 100644
--- a/tests/bench/.gitignore
+++ b/tests/bench/.gitignore
@@ -4,6 +4,7 @@
benchHash
benchHash32
benchHash_avx2
+benchHash_avx512
benchHash_hw
# test files
diff --git a/tests/bench/Makefile b/tests/bench/Makefile
index 4eb7467..a7edd48 100644
--- a/tests/bench/Makefile
+++ b/tests/bench/Makefile
@@ -49,11 +49,14 @@
benchHash_avx2: CFLAGS += -mavx2
benchHash_avx2: CXXFLAGS += -mavx2
+benchHash_avx512: CFLAGS += -mavx512f
+benchHash_avx512: CXXFLAGS += -mavx512f
+
benchHash_hw: CPPFLAGS += -DHARDWARE_SUPPORT
benchHash_hw: CFLAGS += -mavx2 -maes
benchHash_hw: CXXFLAGS += -mavx2 -mpclmul -std=c++14
-benchHash benchHash32 benchHash_avx2 benchHash_nosimd benchHash_hw: $(OBJ_LIST)
+benchHash benchHash32 benchHash_avx2 benchHash_avx512 benchHash_nosimd benchHash_hw: $(OBJ_LIST)
$(CXX) $(CPPFLAGS) $(CXXFLAGS) $^ $(LDFLAGS) -o $@
@@ -65,4 +68,4 @@
clean:
- $(RM) *.o benchHash benchHash32 benchHash_avx2 benchHash_hw
+ $(RM) *.o benchHash benchHash32 benchHash_avx2 benchHash_avx512 benchHash_hw
diff --git a/tests/collisions/Makefile b/tests/collisions/Makefile
index 7191b70..eee4b59 100644
--- a/tests/collisions/Makefile
+++ b/tests/collisions/Makefile
@@ -27,11 +27,11 @@
VPATH = $(SRC_DIRS)
CPPFLAGS += $(addprefix -I ,$(SRC_DIRS))
CFLAGS += -Wall -Wextra -Wconversion \
- -std=c99
+ -std=c11
CXXFLAGS += -Wall -Wextra -Wconversion \
-std=c++11
LDFLAGS += -pthread
-TESTHASHES = 110000000
+TESTHASHES = 3200000
HASH_SRC := $(sort $(wildcard allcodecs/*.c allcodecs/*.cc))
HASH_OBJ := $(patsubst %.c,%.o,$(HASH_SRC))
@@ -62,11 +62,14 @@
.PHONY: test
test: debug
@echo ""
- @echo "## $(TESTHASHES) hashes with original and 0 threads"
+ @echo "## $(TESTHASHES) hashes"
@time ./collisionsTest --nbh=$(TESTHASHES)
@echo ""
- @echo "## $(TESTHASHES) hashes with original and 4 threads"
- @time ./collisionsTest --nbh=$(TESTHASHES) --threadlog=2
+ @echo "## $(TESTHASHES) hashes with filter"
+ @time ./collisionsTest --nbh=$(TESTHASHES) --filter
+ @echo ""
+ @echo "## $(TESTHASHES) hashes with 2 threads"
+ @time ./collisionsTest --nbh=$(TESTHASHES) --threadlog=1
@echo ""
.PHONY: clean
diff --git a/tests/collisions/main.c b/tests/collisions/main.c
index cc36ee7..30aee4a 100644
--- a/tests/collisions/main.c
+++ b/tests/collisions/main.c
@@ -54,12 +54,22 @@
#include "sort.hh" /* sort64 */
+#ifdef __MINGW32__
+/* MINGW claims C11 complians, yet doesn't provide aligned_alloc().
+ * _aligned_malloc() isn't a good workaround,
+ * as it seems incompatible with realloc().
+ * Let's use malloc() instead, array will not be fully aligned,
+ * resulting in some performance degradation, but nothing major.
+ * This policy can be updated once MINGW becomes really C11 compliant.
+ */
+# define aligned_alloc(a,s) ((void)(a), malloc(s))
+#endif
typedef enum { ht32, ht64, ht128 } Htype_e;
/* === Debug === */
-#define EXIT(...) { printf(__VA_ARGS__); printf("\n"); exit(1); }
+#define EXIT(...) { fprintf(stderr, __VA_ARGS__); fprintf(stderr, "\n"); fflush(NULL); exit(1); }
static void hexRaw(const void* buffer, size_t size)
{
@@ -332,10 +342,14 @@
Filter* create_Filter(int bflog)
{
assert(bflog < 64 && bflog > 1);
- size_t bfsize = (size_t)1 << bflog;
- Filter* bf = malloc(bfsize);
- assert(((void)"Filter creation failed", bf));
- memset(bf, 0, bfsize);
+ size_t const bfsize = (size_t)1 << bflog;
+ size_t const cacheline_size = 64;
+ Filter* const bf = aligned_alloc(cacheline_size, bfsize); // requires C11
+ if (bf==NULL) {
+ fprintf(stderr, "Filter creation failed (need %zu MB) \n", bfsize >> 20);
+ exit(1);
+ }
+ memset(bf, 0, bfsize); // we want memory to be actually used
return bf;
}
@@ -404,72 +418,82 @@
* Attach hash to 2 slots
* return: Nb of potential candidates detected
* 0: position not yet occupied
- * 2: position previously occupied by a single candidate (at most)
- * 1: position already occupied by multiple candidates
+ * 1: position may be already occupied
*/
static inline int Filter_insert(Filter* bf, int bflog, uint64_t hash)
- {
- hash = avalanche64(hash);
- unsigned const slot1 = hash & 255;
- hash >>= 8;
- unsigned const slot2 = hash & 255;
- hash >>= 8;
+{
+ hash = avalanche64(hash);
+ unsigned const slot1 = hash & 255;
+ hash >>= 8;
+ unsigned const slot2 = hash & 255;
+ hash >>= 8;
- size_t const fclmask = ((size_t)1 << (bflog-6)) - 1;
- size_t const cacheLineNb = (size_t)hash & fclmask;
+ size_t const fclmask = ((size_t)1 << (bflog-6)) - 1;
+ size_t const cacheLineNb = (size_t)hash & fclmask;
- size_t const pos1 = (cacheLineNb << 6) + (slot1 >> 2);
- unsigned const shift1 = (slot1 & 3) * 2;
- unsigned const ex1 = (bf[pos1] >> shift1) & 3;
+ size_t const pos1 = (cacheLineNb << 6) + (slot1 >> 3);
+ unsigned const shift1 = slot1 & 7;
+ unsigned const bit1 = 1U << shift1;
+ unsigned present1 = bf[pos1] & bit1;
- size_t const pos2 = (cacheLineNb << 6) + (slot2 >> 2);
- unsigned const shift2 = (slot2 & 3) * 2;
- unsigned const ex2 = (bf[pos2] >> shift2) & 3;
+ size_t const pos2 = (cacheLineNb << 6) + (slot2 >> 3);
+ unsigned const shift2 = slot2 & 7;
+ unsigned const bit2 = 1U << shift2;
+ unsigned present2 = bf[pos2] & bit2;
- unsigned const existing = MIN(ex1, ex2);
+ unsigned const maybePresent = (present1 >> shift1) & (present2 >> shift2);
+ present1 &= -maybePresent;
+ present2 &= -maybePresent;
- static const int addCandidates[4] = { 0, 2, 1, 1 };
- static const unsigned nextValue[4] = { 1, 2, 3, 3 };
+ // Write presence
+ bf[pos1 + 32] |= (Filter)present1;
+ bf[pos2 + 32] |= (Filter)present2;
+ bf[pos1] |= (Filter)bit1;
+ bf[pos2] |= (Filter)bit2;
- bf[pos1] &= (Filter)(~(3 << shift1)); /* erase previous value */
- unsigned const max1 = MAX(ex1, nextValue[existing]);
- bf[pos1] |= (Filter)(max1 << shift1);
- unsigned const max2 = MAX(ex2, nextValue[existing]);
- bf[pos2] |= (Filter)(max2 << shift2);
+ return (int)maybePresent;
+}
- return addCandidates[existing];
- }
+static Filter* Filter_reduce(Filter* bf, int bflog)
+{
+ assert(6 < bflog && bflog < 64);
+ size_t const nbLines = (size_t)1 << (bflog - 6);
+ for (size_t lineNb = 0; lineNb < nbLines; lineNb++) {
+ memcpy(bf + lineNb*32, bf + lineNb*64 + 32, 32);
+ }
+ return realloc(bf, nbLines << 5);
+}
/*
* Check if provided 64-bit hash is a collision candidate
- * Requires the slot to be occupied by at least 2 candidates.
+ * Requires both bit positions to be set.
* return >0 if hash is a collision candidate
* 0 otherwise (slot unoccupied, or only one candidate)
* note: unoccupied slots should not happen in this algorithm,
* since all hashes are supposed to have been inserted at least once.
*/
static inline int Filter_check(const Filter* bf, int bflog, uint64_t hash)
- {
- hash = avalanche64(hash);
- unsigned const slot1 = hash & 255;
- hash >>= 8;
- unsigned const slot2 = hash & 255;
- hash >>= 8;
+{
+ hash = avalanche64(hash);
+ unsigned const slot1 = hash & 255;
+ hash >>= 8;
+ unsigned const slot2 = hash & 255;
+ hash >>= 8;
- size_t const fclmask = ((size_t)1 << (bflog-6)) - 1;
- size_t const cacheLineNb = (size_t)hash & fclmask;
+ size_t const fclmask = ((size_t)1 << (bflog-6)) - 1;
+ size_t const lineNb = (size_t)hash & fclmask;
- size_t const pos1 = (cacheLineNb << 6) + (slot1 >> 2);
- unsigned const shift1 = (slot1 & 3) * 2;
- unsigned const ex1 = (bf[pos1] >> shift1) & 3;
+ size_t const pos1 = (lineNb << 5) + (slot1 >> 3);
+ unsigned const shift1 = slot1 & 7;
+ unsigned const present1 = (bf[pos1] >> shift1) & 1;
- size_t const pos2 = (cacheLineNb << 6) + (slot2 >> 2);
- unsigned const shift2 = (slot2 & 3) * 2;
- unsigned const ex2 = (bf[pos2] >> shift2) & 3;
+ size_t const pos2 = (lineNb << 5) + (slot2 >> 3);
+ unsigned const shift2 = slot2 & 7;
+ unsigned const present2 = (bf[pos2] >> shift2) & 1;
- return (ex1 >= 2) && (ex2 >= 2);
- }
+ return (int)(present1 & present2);
+}
#endif // FILTER_1_PROBE
@@ -669,7 +693,7 @@
/* === filter hashes (optional) === */
Filter* bf = NULL;
- uint64_t nbPresents = totalH;
+ uint64_t maxNbH = totalH;
if (filter) {
time_t const filterTBegin = time(NULL);
@@ -677,10 +701,9 @@
bf = create_Filter(bflog);
if (!bf) EXIT("not enough memory for filter");
-
DISPLAY(" Generate %llu hashes from samples of %u bytes \n",
(unsigned long long)totalH, (unsigned)sampleSize);
- nbPresents = 0;
+ maxNbH = 0;
for (uint64_t n=0; n < totalH; n++) {
if (display && ((n&0xFFFFF) == 1) )
@@ -690,10 +713,10 @@
UniHash const h = hfunction(sf->buffer, sampleSize);
if ((h.h64 & hMask) != hSelector) continue;
- nbPresents += (uint64_t)Filter_insert(bf, bflog, h.h64);
+ maxNbH += 2 * (uint64_t)Filter_insert(bf, bflog, h.h64);
}
- if (nbPresents==0) {
+ if (maxNbH==0) {
DISPLAY(" Analysis completed: No collision detected \n");
if (param.resultPtr) param.resultPtr->nbCollisions = 0;
free_Filter(bf);
@@ -703,16 +726,19 @@
{ double const filterDelay = difftime(time(NULL), filterTBegin);
DISPLAY(" Generation and filter completed in %s, detected up to %llu candidates \n",
- displayDelay(filterDelay), (unsigned long long) nbPresents);
- } }
+ displayDelay(filterDelay), (unsigned long long) maxNbH);
+ }
+ bf = Filter_reduce(bf, bflog);
+ if (bf == NULL) EXIT("filter reduction failed");
+ }
/* === store hash candidates: duplicates will be present here === */
time_t const storeTBegin = time(NULL);
size_t const hashByteSize = (htype == ht128) ? 16 : 8;
- size_t const tableSize = (size_t)((nbPresents+1) * hashByteSize);
- assert(tableSize > nbPresents); /* check tableSize calculation overflow */
+ size_t const tableSize = (size_t)((maxNbH+1) * hashByteSize);
+ assert(tableSize > maxNbH); /* check tableSize calculation overflow */
DISPLAY(" Storing hash candidates (%i MB) \n", (int)(tableSize >> 20));
/* Generate and store hashes */
@@ -729,15 +755,16 @@
if (filter) {
if (Filter_check(bf, bflog, h.h64)) {
- assert(nbCandidates < nbPresents);
+ //printf("found %zu candidates / %llu generated \n", nbCandidates, n);
+ assert(nbCandidates < maxNbH);
addHashCandidate(hashCandidates, h, htype, nbCandidates++);
}
} else {
- assert(nbCandidates < nbPresents);
+ assert(nbCandidates < maxNbH);
addHashCandidate(hashCandidates, h, htype, nbCandidates++);
}
}
- if (nbCandidates < nbPresents) {
+ if (nbCandidates < maxNbH) {
/* Try to mitigate gnuc_quicksort behavior, by reducing allocated memory,
* since gnuc_quicksort uses a lot of additional memory for mergesort */
void* const checkPtr = realloc(hashCandidates, nbCandidates * hashByteSize);
@@ -985,7 +1012,7 @@
int main(int argc, const char** argv)
{
- if (sizeof(size_t) < 8) return 1; // cannot work on systems without ability to allocate objects >= 4 GB
+ printf(" *** Collision tester for 64+ bit hashes *** \n\n");
assert(argc > 0);
const char* const exeName = argv[0];
@@ -1031,11 +1058,10 @@
if (bflog == 0) bflog = highestBitSet(totalH) + 1; /* auto-size filter */
if (!filter) bflog = -1; // disable filter
- if (sizeof(size_t) < 8)
- EXIT("This program has not been validated on architectures other than "
- "64bit \n");
+ if (sizeof(size_t) < 8) {
+ printf("warning: in 32-bit mode, the program is more likely going to lack address space \n");
+ }
- printf(" *** Collision tester for 64+ bit hashes *** \n\n");
printf("Testing %s algorithm (%i-bit) \n", hname, hwidth);
printf("This program will allocate a lot of memory,\n");
printf("generate %llu %i-bit hashes from samples of %u bytes, \n",
@@ -1074,9 +1100,9 @@
time_t const programTBegin = time(NULL);
POOL_ctx* const pt = POOL_create((size_t)nbThreads, 1);
if (!pt) EXIT("not enough memory for threads");
- searchCollisions_results* const MTresults = calloc (sizeof(searchCollisions_results), (size_t)nbThreads);
+ searchCollisions_results* const MTresults = calloc ((size_t)nbThreads, sizeof(searchCollisions_results));
if (!MTresults) EXIT("not enough memory");
- searchCollisions_parameters* const MTparams = calloc (sizeof(searchCollisions_parameters), (size_t)nbThreads);
+ searchCollisions_parameters* const MTparams = calloc ((size_t)nbThreads, sizeof(searchCollisions_parameters));
if (!MTparams) EXIT("not enough memory");
/* distribute jobs */
diff --git a/tests/sanity_test.c b/tests/sanity_test.c
index 4c59bf3..3e3d4f1 100644
--- a/tests/sanity_test.c
+++ b/tests/sanity_test.c
@@ -318,6 +318,8 @@
/**/
static void testXXH3(
const void* data,
+ const void* secret,
+ size_t secretSize,
const XSUM_testdata64_t* testData,
XSUM_U64* pRandSeed,
const char* testName,
@@ -346,12 +348,19 @@
* XXH3_generateSecret_fromSeed() and XXH3_64bits_withSecretandSeed()
* results in exactly the same hash generation as XXH3_64bits_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
- char* const secret = secretBuffer + 1; /* intentional unalignment */
- XXH3_generateSecret_fromSeed(secret, seed);
- { XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
+ char* const secretFromSeed = secretBuffer + 1; /* intentional unalignment */
+ XXH3_generateSecret_fromSeed(secretFromSeed, seed);
+ { XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secretFromSeed, XXH3_SECRET_DEFAULT_SIZE, seed);
checkResult64(Dresult, Nresult, testName, testNb, __LINE__);
} }
+ /* check that XXH3_64bits_withSecretandSeed()
+ * results in exactly the same return value as XXH3_64bits_withSeed() */
+ if (len <= XXH3_MIDSIZE_MAX) {
+ XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, secretSize, seed);
+ checkResult64(Dresult, Nresult, testName, testNb, __LINE__);
+ }
+
/* streaming API test */
{ XXH3_state_t* const state = XXH3_createState();
assert(state != NULL);
@@ -377,10 +386,19 @@
* XXH3_generateSecret_fromSeed() and XXH3_64bits_reset_withSecretandSeed()
* results in exactly the same hash generation as XXH3_64bits_reset_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
- char* const secret = secretBuffer + 1; /* intentional unalignment */
- XXH3_generateSecret_fromSeed(secret, seed);
+ char* const secretFromSeed = secretBuffer + 1; /* intentional unalignment */
+ XXH3_generateSecret_fromSeed(secretFromSeed, seed);
/* single ingestion */
- (void)XXH3_64bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
+ (void)XXH3_64bits_reset_withSecretandSeed(state, secretFromSeed, XXH3_SECRET_DEFAULT_SIZE, seed);
+ (void)XXH3_64bits_update(state, data, len);
+ checkResult64(XXH3_64bits_digest(state), Nresult, testName, testNb, __LINE__);
+ }
+
+ /* check that XXH3_64bits_withSecretandSeed()
+ * results in exactly the same return value as XXH3_64bits_withSeed() */
+ if (len <= XXH3_MIDSIZE_MAX) {
+ /* single ingestion */
+ (void)XXH3_64bits_reset_withSecretandSeed(state, secret, secretSize, seed);
(void)XXH3_64bits_update(state, data, len);
checkResult64(XXH3_64bits_digest(state), Nresult, testName, testNb, __LINE__);
}
@@ -459,6 +477,8 @@
/**/
static void testXXH128(
const void* data,
+ const void* secret,
+ size_t secretSize,
const XSUM_testdata128_t* testData,
XSUM_U64* pRandSeed,
const char* testName,
@@ -493,12 +513,19 @@
* XXH3_generateSecret_fromSeed() and XXH3_128bits_withSecretandSeed()
* results in exactly the same hash generation as XXH3_64bits_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
- char* const secret = secretBuffer + 1; /* intentional unalignment */
- XXH3_generateSecret_fromSeed(secret, seed);
- { XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
+ char* const secretFromSeed = secretBuffer + 1; /* intentional unalignment */
+ XXH3_generateSecret_fromSeed(secretFromSeed, seed);
+ { XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secretFromSeed, XXH3_SECRET_DEFAULT_SIZE, seed);
checkResult128(Dresult, Nresult, testName, testNb, __LINE__);
} }
+ /* check that XXH3_128bits_withSecretandSeed()
+ * results in exactly the same return value as XXH3_128bits_withSeed() */
+ if (len <= XXH3_MIDSIZE_MAX) {
+ XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, secretSize, seed);
+ checkResult128(Dresult, Nresult, testName, testNb, __LINE__);
+ }
+
/* streaming API test */
{ XXH3_state_t * const state = XXH3_createState();
assert(state != NULL);
@@ -525,10 +552,19 @@
* XXH3_generateSecret_fromSeed() and XXH3_128bits_reset_withSecretandSeed()
* results in exactly the same hash generation as XXH3_128bits_reset_withSeed() */
{ char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1];
- char* const secret = secretBuffer + 1; /* intentional unalignment */
- XXH3_generateSecret_fromSeed(secret, seed);
+ char* const secretFromSeed = secretBuffer + 1; /* intentional unalignment */
+ XXH3_generateSecret_fromSeed(secretFromSeed, seed);
/* single ingestion */
- (void)XXH3_128bits_reset_withSecretandSeed(state, secret, XXH3_SECRET_DEFAULT_SIZE, seed);
+ (void)XXH3_128bits_reset_withSecretandSeed(state, secretFromSeed, XXH3_SECRET_DEFAULT_SIZE, seed);
+ (void)XXH3_128bits_update(state, data, len);
+ checkResult128(XXH3_128bits_digest(state), Nresult, testName, testNb, __LINE__);
+ }
+
+ /* check that XXH3_128bits_reset_withSecretandSeed()
+ * results in exactly the same return value as XXH3_128bits_reset_withSeed() */
+ if (len <= XXH3_MIDSIZE_MAX) {
+ /* single ingestion */
+ (void)XXH3_128bits_reset_withSecretandSeed(state, secret, secretSize, seed);
(void)XXH3_128bits_update(state, data, len);
checkResult128(XXH3_128bits_digest(state), Nresult, testName, testNb, __LINE__);
}
@@ -676,6 +712,8 @@
for (i = 0; i < n; ++i, ++testCount) {
testXXH3(
sanityBuffer,
+ secret,
+ secretSize,
&XSUM_XXH3_testdata[i],
&randSeed,
"XSUM_XXH3_testdata",
@@ -713,6 +751,8 @@
for (i = 0; i < n; ++i, ++testCount) {
testXXH128(
sanityBuffer,
+ secret,
+ secretSize,
&XSUM_XXH128_testdata[i],
&randSeed,
"XSUM_XXH128_testdata",
diff --git a/tests/sanity_test_vectors_generator.c b/tests/sanity_test_vectors_generator.c
index 196995d..f910462 100644
--- a/tests/sanity_test_vectors_generator.c
+++ b/tests/sanity_test_vectors_generator.c
@@ -138,11 +138,11 @@
}
static void fprintf_XSUM_testdata64_t(FILE* fp, XSUM_testdata64_t const v) {
- fprintf(fp, "{ %4d, 0x%016zXULL, 0x%016zXULL },", v.len, v.seed, v.Nresult);
+ fprintf(fp, "{ %4d, 0x%016llXULL, 0x%016llXULL },", v.len, v.seed, v.Nresult);
}
static void fprintf_XSUM_testdata128_t(FILE* fp, XSUM_testdata128_t const v) {
- fprintf(fp, "{ %4d, 0x%016zXULL, { 0x%016zXULL, 0x%016zXULL } },",
+ fprintf(fp, "{ %4d, 0x%016llXULL, { 0x%016llXULL, 0x%016llXULL } },",
v.len, v.seed, v.Nresult.low64, v.Nresult.high64);
}
@@ -208,7 +208,7 @@
size_t index = 0;
for(size_t len = 0; len < maxLen; ++len) {
- static const size_t seeds[] = { 0, PRIME32 };
+ static const uint64_t seeds[] = { 0, PRIME32 };
for(size_t iSeed = 0; iSeed < sizeof(seeds)/sizeof(seeds[0]); ++iSeed) {
size_t const seed = seeds[iSeed];
XSUM_testdata32_t const v = tvgen_XXH32(sanityBuffer, len, seed);
@@ -234,7 +234,7 @@
size_t index = 0;
for(size_t len = 0; len < maxLen; ++len) {
- static const size_t seeds[] = { 0, PRIME32 };
+ static const uint64_t seeds[] = { 0, PRIME32 };
for(size_t iSeed = 0; iSeed < sizeof(seeds)/sizeof(seeds[0]); ++iSeed) {
size_t const seed = seeds[iSeed];
XSUM_testdata64_t const v = tvgen_XXH64(sanityBuffer, len, seed);
@@ -260,7 +260,7 @@
size_t index = 0;
for(size_t len = 0; len < maxLen; ++len) {
- static const size_t seeds[] = { 0, PRIME64 };
+ static const uint64_t seeds[] = { 0, PRIME64 };
for(size_t iSeed = 0; iSeed < sizeof(seeds)/sizeof(seeds[0]); ++iSeed) {
size_t const seed = seeds[iSeed];
XSUM_testdata64_t const v = tvgen_XXH3_64bits_withSeed(sanityBuffer, len, seed);
@@ -312,7 +312,7 @@
size_t index = 0;
for(size_t len = 0; len < maxLen; ++len) {
- static const size_t seeds[] = { 0, PRIME32, PRIME64 };
+ static const uint64_t seeds[] = { 0, PRIME32, PRIME64 };
for(size_t iSeed = 0; iSeed < sizeof(seeds)/sizeof(seeds[0]); ++iSeed) {
XSUM_U64 const seed = seeds[iSeed];
XSUM_testdata128_t const v = tvgen_XXH3_128bits_withSeed(sanityBuffer, len, seed);
diff --git a/xxh_x86dispatch.c b/xxh_x86dispatch.c
index 2489e15..03e7dc4 100644
--- a/xxh_x86dispatch.c
+++ b/xxh_x86dispatch.c
@@ -40,9 +40,9 @@
*
* Optional add-on.
*
- * **Compile this file with the default flags for your target.** Do not compile
- * with flags like `-mavx*`, `-march=native`, or `/arch:AVX*`, there will be
- * an error. See @ref XXH_X86DISPATCH_ALLOW_AVX for details.
+ * **Compile this file with the default flags for your target.**
+ * Note that compiling with flags like `-mavx*`, `-march=native`, or `/arch:AVX*`
+ * will make the resulting binary incompatible with cpus not supporting the requested instruction set.
*
* @defgroup dispatch x86 Dispatcher
* @{
@@ -59,7 +59,11 @@
/*! @cond Doxygen ignores this part */
#ifndef XXH_HAS_INCLUDE
# ifdef __has_include
-# define XXH_HAS_INCLUDE(x) __has_include(x)
+/*
+ * Not defined as XXH_HAS_INCLUDE(x) (function-like) because
+ * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion)
+ */
+# define XXH_HAS_INCLUDE __has_include
# else
# define XXH_HAS_INCLUDE(x) 0
# endif
@@ -67,34 +71,6 @@
/*! @endcond */
/*!
- * @def XXH_X86DISPATCH_ALLOW_AVX
- * @brief Disables the AVX sanity check.
- *
- * xxh_x86dispatch.c is intended to be compiled for the minimum target, and
- * it selectively enables SSE2, AVX2, and AVX512 when it is needed.
- *
- * Compiling with options like `-mavx*`, `-march=native`, or `/arch:AVX*`
- * _globally_ will always enable this feature, and therefore makes it
- * undefined behavior to execute on any CPU without said feature.
- *
- * Even if the source code isn't directly using AVX intrinsics in a function,
- * the compiler can still generate AVX code from autovectorization and by
- * "upgrading" SSE2 intrinsics to use the VEX prefixes (a.k.a. AVX128).
- *
- * Define XXH_X86DISPATCH_ALLOW_AVX to ignore this check,
- * thus accepting that the produced binary will not work correctly
- * on any CPU with less features than the ones stated at compilation time.
- */
-#ifdef XXH_DOXYGEN
-# define XXH_X86DISPATCH_ALLOW_AVX
-#endif
-
-#if defined(__AVX__) && !defined(XXH_X86DISPATCH_ALLOW_AVX)
-# error "Error: if xxh_x86dispatch.c is compiled with AVX enabled, the resulting binary will crash on sse2-only cpus !! " \
- "If you nonetheless want to do that, please enable the XXH_X86DISPATCH_ALLOW_AVX build variable"
-#endif
-
-/*!
* @def XXH_DISPATCH_SCALAR
* @brief Enables/dispatching the scalar code path.
*
@@ -364,7 +340,7 @@
* @return The best @ref XXH_VECTOR implementation.
* @see XXH_VECTOR_TYPES
*/
-static int XXH_featureTest(void)
+int XXH_featureTest(void)
{
xxh_u32 abcd[4];
xxh_u32 max_leaves;
@@ -725,8 +701,8 @@
/*! @cond Doxygen ignores this part */
static XXH64_hash_t
-XXH3_hashLong_64b_defaultSecret_selection(const void* input, size_t len,
- XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
+XXH3_hashLong_64b_defaultSecret_selection(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
{
(void)seed64; (void)secret; (void)secretLen;
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_default == NULL)
@@ -740,8 +716,8 @@
}
static XXH64_hash_t
-XXH3_hashLong_64b_withSeed_selection(const void* input, size_t len,
- XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
+XXH3_hashLong_64b_withSeed_selection(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
{
(void)secret; (void)secretLen;
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_seed == NULL)
@@ -755,8 +731,8 @@
}
static XXH64_hash_t
-XXH3_hashLong_64b_withSecret_selection(const void* input, size_t len,
- XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen)
+XXH3_hashLong_64b_withSecret_selection(const void* XXH_RESTRICT input, size_t len,
+ XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
{
(void)seed64;
if (XXH_DISPATCH_MAYBE_NULL && XXH_g_dispatch.hashLong64_secret == NULL)
diff --git a/xxh_x86dispatch.h b/xxh_x86dispatch.h
index b87cea9..7085221 100644
--- a/xxh_x86dispatch.h
+++ b/xxh_x86dispatch.h
@@ -1,6 +1,6 @@
/*
* xxHash - XXH3 Dispatcher for x86-based targets
- * Copyright (C) 2020-2021 Yann Collet
+ * Copyright (C) 2020-2024 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*
@@ -41,6 +41,14 @@
extern "C" {
#endif
+/*!
+ * @brief Returns the best XXH3 implementation for x86
+ *
+ * @return The best @ref XXH_VECTOR implementation.
+ * @see XXH_VECTOR_TYPES
+ */
+XXH_PUBLIC_API int XXH_featureTest(void);
+
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_dispatch(XXH_NOESCAPE const void* input, size_t len);
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed);
XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret_dispatch(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretLen);
diff --git a/xxhash.c b/xxhash.c
index 083b039..e60cc37 100644
--- a/xxhash.c
+++ b/xxhash.c
@@ -1,6 +1,6 @@
/*
* xxHash - Extremely Fast Hash algorithm
- * Copyright (C) 2012-2021 Yann Collet
+ * Copyright (C) 2012-2023 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*
@@ -32,12 +32,11 @@
* - xxHash source repository: https://github.com/Cyan4973/xxHash
*/
-
/*
* xxhash.c instantiates functions defined in xxhash.h
*/
-#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
-#define XXH_IMPLEMENTATION /* access definitions */
+#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */
+#define XXH_IMPLEMENTATION /* access definitions */
#include "xxhash.h"
diff --git a/xxhash.h b/xxhash.h
index a18e8c7..78fc2e8 100644
--- a/xxhash.h
+++ b/xxhash.h
@@ -1,7 +1,7 @@
/*
* xxHash - Extremely Fast Hash algorithm
* Header File
- * Copyright (C) 2012-2021 Yann Collet
+ * Copyright (C) 2012-2023 Yann Collet
*
* BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
*
@@ -130,6 +130,7 @@
* }
* @endcode
*
+ *
* @anchor streaming_example
* **Streaming**
*
@@ -165,6 +166,77 @@
* }
* @endcode
*
+ * Streaming functions generate the xxHash value from an incremental input.
+ * This method is slower than single-call functions, due to state management.
+ * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
+ *
+ * An XXH state must first be allocated using `XXH*_createState()`.
+ *
+ * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
+ *
+ * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
+ *
+ * The function returns an error code, with 0 meaning OK, and any other value
+ * meaning there is an error.
+ *
+ * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
+ * This function returns the nn-bits hash as an int or long long.
+ *
+ * It's still possible to continue inserting input into the hash state after a
+ * digest, and generate new hash values later on by invoking `XXH*_digest()`.
+ *
+ * When done, release the state using `XXH*_freeState()`.
+ *
+ *
+ * @anchor canonical_representation_example
+ * **Canonical Representation**
+ *
+ * The default return values from XXH functions are unsigned 32, 64 and 128 bit
+ * integers.
+ * This the simplest and fastest format for further post-processing.
+ *
+ * However, this leaves open the question of what is the order on the byte level,
+ * since little and big endian conventions will store the same number differently.
+ *
+ * The canonical representation settles this issue by mandating big-endian
+ * convention, the same convention as human-readable numbers (large digits first).
+ *
+ * When writing hash values to storage, sending them over a network, or printing
+ * them, it's highly recommended to use the canonical representation to ensure
+ * portability across a wider range of systems, present and future.
+ *
+ * The following functions allow transformation of hash values to and from
+ * canonical format.
+ *
+ * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(),
+ * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(),
+ * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(),
+ *
+ * @code{.c}
+ * #include <stdio.h>
+ * #include "xxhash.h"
+ *
+ * // Example for a function which prints XXH32_hash_t in human readable format
+ * void printXxh32(XXH32_hash_t hash)
+ * {
+ * XXH32_canonical_t cano;
+ * XXH32_canonicalFromHash(&cano, hash);
+ * size_t i;
+ * for(i = 0; i < sizeof(cano.digest); ++i) {
+ * printf("%02x", cano.digest[i]);
+ * }
+ * printf("\n");
+ * }
+ *
+ * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t
+ * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano)
+ * {
+ * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano);
+ * return hash;
+ * }
+ * @endcode
+ *
+ *
* @file xxhash.h
* xxHash prototypes and implementation
*/
@@ -261,7 +333,7 @@
/* make all functions private */
# undef XXH_PUBLIC_API
# if defined(__GNUC__)
-# define XXH_PUBLIC_API static __inline __attribute__((unused))
+# define XXH_PUBLIC_API static __inline __attribute__((__unused__))
# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define XXH_PUBLIC_API static inline
# elif defined(_MSC_VER)
@@ -373,7 +445,7 @@
/*! @brief Marks a global symbol. */
#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
-# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
+# if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
# ifdef XXH_EXPORT
# define XXH_PUBLIC_API __declspec(dllexport)
# elif XXH_IMPORT
@@ -449,7 +521,7 @@
/* specific declaration modes for Windows */
#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
-# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
+# if defined(_WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
# ifdef XXH_EXPORT
# define XXH_PUBLIC_API __declspec(dllexport)
# elif XXH_IMPORT
@@ -461,9 +533,9 @@
#endif
#if defined (__GNUC__)
-# define XXH_CONSTF __attribute__((const))
-# define XXH_PUREF __attribute__((pure))
-# define XXH_MALLOCF __attribute__((malloc))
+# define XXH_CONSTF __attribute__((__const__))
+# define XXH_PUREF __attribute__((__pure__))
+# define XXH_MALLOCF __attribute__((__malloc__))
#else
# define XXH_CONSTF /* disable */
# define XXH_PUREF
@@ -475,7 +547,7 @@
***************************************/
#define XXH_VERSION_MAJOR 0
#define XXH_VERSION_MINOR 8
-#define XXH_VERSION_RELEASE 2
+#define XXH_VERSION_RELEASE 3
/*! @brief Version number, encoded as two digits each */
#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
@@ -517,7 +589,11 @@
#elif !defined (__VMS) \
&& (defined (__cplusplus) \
|| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include <stdint.h>
+# ifdef _AIX
+# include <inttypes.h>
+# else
+# include <stdint.h>
+# endif
typedef uint32_t XXH32_hash_t;
#else
@@ -551,10 +627,6 @@
/*!
* @brief Calculates the 32-bit hash of @p input using xxHash32.
*
- * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s
- *
- * See @ref single_shot_example "Single Shot Example" for an example.
- *
* @param input The block of data to be hashed, at least @p length bytes in size.
* @param length The length of @p input, in bytes.
* @param seed The 32-bit seed to alter the hash's output predictably.
@@ -564,63 +636,44 @@
* readable, contiguous memory. However, if @p length is `0`, @p input may be
* `NULL`. In C++, this also must be *TriviallyCopyable*.
*
- * @return The calculated 32-bit hash value.
+ * @return The calculated 32-bit xxHash32 value.
*
- * @see
- * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
- * Direct equivalents for the other variants of xxHash.
- * @see
- * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version.
+ * @see @ref single_shot_example "Single Shot Example" for an example.
*/
XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);
#ifndef XXH_NO_STREAM
/*!
- * Streaming functions generate the xxHash value from an incremental input.
- * This method is slower than single-call functions, due to state management.
- * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
- *
- * An XXH state must first be allocated using `XXH*_createState()`.
- *
- * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
- *
- * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
- *
- * The function returns an error code, with 0 meaning OK, and any other value
- * meaning there is an error.
- *
- * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
- * This function returns the nn-bits hash as an int or long long.
- *
- * It's still possible to continue inserting input into the hash state after a
- * digest, and generate new hash values later on by invoking `XXH*_digest()`.
- *
- * When done, release the state using `XXH*_freeState()`.
- *
- * @see streaming_example at the top of @ref xxhash.h for an example.
- */
-
-/*!
* @typedef struct XXH32_state_s XXH32_state_t
* @brief The opaque state struct for the XXH32 streaming API.
*
* @see XXH32_state_s for details.
+ * @see @ref streaming_example "Streaming Example"
*/
typedef struct XXH32_state_s XXH32_state_t;
/*!
* @brief Allocates an @ref XXH32_state_t.
*
- * Must be freed with XXH32_freeState().
- * @return An allocated XXH32_state_t on success, `NULL` on failure.
+ * @return An allocated pointer of @ref XXH32_state_t on success.
+ * @return `NULL` on failure.
+ *
+ * @note Must be freed with XXH32_freeState().
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void);
/*!
* @brief Frees an @ref XXH32_state_t.
*
- * Must be allocated with XXH32_createState().
* @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState().
- * @return XXH_OK.
+ *
+ * @return @ref XXH_OK.
+ *
+ * @note @p statePtr must be allocated with XXH32_createState().
+ *
+ * @see @ref streaming_example "Streaming Example"
+ *
*/
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
/*!
@@ -636,23 +689,24 @@
/*!
* @brief Resets an @ref XXH32_state_t to begin a new hash.
*
- * This function resets and seeds a state. Call it before @ref XXH32_update().
- *
* @param statePtr The state struct to reset.
* @param seed The 32-bit seed to alter the hash result predictably.
*
* @pre
* @p statePtr must not be `NULL`.
*
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note This function resets and seeds a state. Call it before @ref XXH32_update().
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed);
/*!
* @brief Consumes a block of @p input to an @ref XXH32_state_t.
*
- * Call this to incrementally consume blocks of data.
- *
* @param statePtr The state struct to update.
* @param input The block of data to be hashed, at least @p length bytes in size.
* @param length The length of @p input, in bytes.
@@ -664,48 +718,36 @@
* readable, contiguous memory. However, if @p length is `0`, @p input may be
* `NULL`. In C++, this also must be *TriviallyCopyable*.
*
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note Call this to incrementally consume blocks of data.
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
/*!
* @brief Returns the calculated hash value from an @ref XXH32_state_t.
*
- * @note
- * Calling XXH32_digest() will not affect @p statePtr, so you can update,
- * digest, and update again.
- *
* @param statePtr The state struct to calculate the hash from.
*
* @pre
* @p statePtr must not be `NULL`.
*
- * @return The calculated xxHash32 value from that state.
+ * @return The calculated 32-bit xxHash32 value from that state.
+ *
+ * @note
+ * Calling XXH32_digest() will not affect @p statePtr, so you can update,
+ * digest, and update again.
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
#endif /* !XXH_NO_STREAM */
/******* Canonical representation *******/
-/*
- * The default return values from XXH functions are unsigned 32 and 64 bit
- * integers.
- * This the simplest and fastest format for further post-processing.
- *
- * However, this leaves open the question of what is the order on the byte level,
- * since little and big endian conventions will store the same number differently.
- *
- * The canonical representation settles this issue by mandating big-endian
- * convention, the same convention as human-readable numbers (large digits first).
- *
- * When writing hash values to storage, sending them over a network, or printing
- * them, it's highly recommended to use the canonical representation to ensure
- * portability across a wider range of systems, present and future.
- *
- * The following functions allow transformation of hash values to and from
- * canonical format.
- */
-
/*!
* @brief Canonical (big endian) representation of @ref XXH32_hash_t.
*/
@@ -716,11 +758,13 @@
/*!
* @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t.
*
- * @param dst The @ref XXH32_canonical_t pointer to be stored to.
+ * @param dst The @ref XXH32_canonical_t pointer to be stored to.
* @param hash The @ref XXH32_hash_t to be converted.
*
* @pre
* @p dst must not be `NULL`.
+ *
+ * @see @ref canonical_representation_example "Canonical Representation Example"
*/
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
@@ -733,6 +777,8 @@
* @p src must not be `NULL`.
*
* @return The converted hash.
+ *
+ * @see @ref canonical_representation_example "Canonical Representation Example"
*/
XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
@@ -794,7 +840,7 @@
* As of writing this, only supported by clang.
*/
#if XXH_HAS_ATTRIBUTE(noescape)
-# define XXH_NOESCAPE __attribute__((noescape))
+# define XXH_NOESCAPE __attribute__((__noescape__))
#else
# define XXH_NOESCAPE
#endif
@@ -821,7 +867,11 @@
#elif !defined (__VMS) \
&& (defined (__cplusplus) \
|| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include <stdint.h>
+# ifdef _AIX
+# include <inttypes.h>
+# else
+# include <stdint.h>
+# endif
typedef uint64_t XXH64_hash_t;
#else
# include <limits.h>
@@ -851,9 +901,6 @@
/*!
* @brief Calculates the 64-bit hash of @p input using xxHash64.
*
- * This function usually runs faster on 64-bit systems, but slower on 32-bit
- * systems (see benchmark).
- *
* @param input The block of data to be hashed, at least @p length bytes in size.
* @param length The length of @p input, in bytes.
* @param seed The 64-bit seed to alter the hash's output predictably.
@@ -863,13 +910,9 @@
* readable, contiguous memory. However, if @p length is `0`, @p input may be
* `NULL`. In C++, this also must be *TriviallyCopyable*.
*
- * @return The calculated 64-bit hash.
+ * @return The calculated 64-bit xxHash64 value.
*
- * @see
- * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
- * Direct equivalents for the other variants of xxHash.
- * @see
- * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version.
+ * @see @ref single_shot_example "Single Shot Example" for an example.
*/
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
@@ -879,23 +922,32 @@
* @brief The opaque state struct for the XXH64 streaming API.
*
* @see XXH64_state_s for details.
+ * @see @ref streaming_example "Streaming Example"
*/
typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
/*!
* @brief Allocates an @ref XXH64_state_t.
*
- * Must be freed with XXH64_freeState().
- * @return An allocated XXH64_state_t on success, `NULL` on failure.
+ * @return An allocated pointer of @ref XXH64_state_t on success.
+ * @return `NULL` on failure.
+ *
+ * @note Must be freed with XXH64_freeState().
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void);
/*!
* @brief Frees an @ref XXH64_state_t.
*
- * Must be allocated with XXH64_createState().
* @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState().
- * @return XXH_OK.
+ *
+ * @return @ref XXH_OK.
+ *
+ * @note @p statePtr must be allocated with XXH64_createState().
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
@@ -912,23 +964,24 @@
/*!
* @brief Resets an @ref XXH64_state_t to begin a new hash.
*
- * This function resets and seeds a state. Call it before @ref XXH64_update().
- *
* @param statePtr The state struct to reset.
* @param seed The 64-bit seed to alter the hash result predictably.
*
* @pre
* @p statePtr must not be `NULL`.
*
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note This function resets and seeds a state. Call it before @ref XXH64_update().
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed);
/*!
* @brief Consumes a block of @p input to an @ref XXH64_state_t.
*
- * Call this to incrementally consume blocks of data.
- *
* @param statePtr The state struct to update.
* @param input The block of data to be hashed, at least @p length bytes in size.
* @param length The length of @p input, in bytes.
@@ -940,23 +993,30 @@
* readable, contiguous memory. However, if @p length is `0`, @p input may be
* `NULL`. In C++, this also must be *TriviallyCopyable*.
*
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note Call this to incrementally consume blocks of data.
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
/*!
* @brief Returns the calculated hash value from an @ref XXH64_state_t.
*
- * @note
- * Calling XXH64_digest() will not affect @p statePtr, so you can update,
- * digest, and update again.
- *
* @param statePtr The state struct to calculate the hash from.
*
* @pre
* @p statePtr must not be `NULL`.
*
- * @return The calculated xxHash64 value from that state.
+ * @return The calculated 64-bit xxHash64 value from that state.
+ *
+ * @note
+ * Calling XXH64_digest() will not affect @p statePtr, so you can update,
+ * digest, and update again.
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr);
#endif /* !XXH_NO_STREAM */
@@ -975,6 +1035,8 @@
*
* @pre
* @p dst must not be `NULL`.
+ *
+ * @see @ref canonical_representation_example "Canonical Representation Example"
*/
XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash);
@@ -987,6 +1049,8 @@
* @p src must not be `NULL`.
*
* @return The converted hash.
+ *
+ * @see @ref canonical_representation_example "Canonical Representation Example"
*/
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src);
@@ -1046,40 +1110,74 @@
*
* The API supports one-shot hashing, streaming mode, and custom secrets.
*/
+
+/*!
+ * @ingroup tuning
+ * @brief Possible values for @ref XXH_VECTOR.
+ *
+ * Unless set explicitly, determined automatically.
+ */
+# define XXH_SCALAR 0 /*!< Portable scalar version */
+# define XXH_SSE2 1 /*!< SSE2 for Pentium 4, Opteron, all x86_64. */
+# define XXH_AVX2 2 /*!< AVX2 for Haswell and Bulldozer */
+# define XXH_AVX512 3 /*!< AVX512 for Skylake and Icelake */
+# define XXH_NEON 4 /*!< NEON for most ARMv7-A, all AArch64, and WASM SIMD128 */
+# define XXH_VSX 5 /*!< VSX and ZVector for POWER8/z13 (64-bit) */
+# define XXH_SVE 6 /*!< SVE for some ARMv8-A and ARMv9-A */
+# define XXH_LSX 7 /*!< LSX (128-bit SIMD) for LoongArch64 */
+
+
/*-**********************************************************************
* XXH3 64-bit variant
************************************************************************/
/*!
- * @brief 64-bit unseeded variant of XXH3.
+ * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input.
*
- * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of 0, however
- * it may have slightly better performance due to constant propagation of the
- * defaults.
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
*
- * @see
- * XXH32(), XXH64(), XXH3_128bits(): equivalent for the other xxHash algorithms
+ * @pre
+ * The memory between @p input and @p input + @p length must be valid,
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 64-bit XXH3 hash value.
+ *
+ * @note
+ * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however
+ * it may have slightly better performance due to constant propagation of the
+ * defaults.
+ *
* @see
* XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants
- * @see
- * XXH3_64bits_reset(), XXH3_64bits_update(), XXH3_64bits_digest(): Streaming version.
+ * @see @ref single_shot_example "Single Shot Example" for an example.
*/
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length);
/*!
- * @brief 64-bit seeded variant of XXH3
+ * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input.
*
- * This variant generates a custom secret on the fly based on default secret
- * altered using the `seed` value.
+ * @param input The block of data to be hashed, at least @p length bytes in size.
+ * @param length The length of @p input, in bytes.
+ * @param seed The 64-bit seed to alter the hash result predictably.
*
- * While this operation is decently fast, note that it's not completely free.
+ * @pre
+ * The memory between @p input and @p input + @p length must be valid,
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 64-bit XXH3 hash value.
*
* @note
* seed == 0 produces the same results as @ref XXH3_64bits().
*
- * @param input The data to hash
- * @param length The length
- * @param seed The 64-bit seed to alter the state.
+ * This variant generates a custom secret on the fly based on default secret
+ * altered using the @p seed value.
+ *
+ * While this operation is decently fast, note that it's not completely free.
+ *
+ * @see @ref single_shot_example "Single Shot Example" for an example.
*/
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
@@ -1093,22 +1191,36 @@
#define XXH3_SECRET_SIZE_MIN 136
/*!
- * @brief 64-bit variant of XXH3 with a custom "secret".
+ * @brief Calculates 64-bit variant of XXH3 with a custom "secret".
+ *
+ * @param data The block of data to be hashed, at least @p len bytes in size.
+ * @param len The length of @p data, in bytes.
+ * @param secret The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ *
+ * @return The calculated 64-bit XXH3 hash value.
+ *
+ * @pre
+ * The memory between @p data and @p data + @p len must be valid,
+ * readable, contiguous memory. However, if @p length is `0`, @p data may be
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
*
* It's possible to provide any blob of bytes as a "secret" to generate the hash.
* This makes it more difficult for an external actor to prepare an intentional collision.
- * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN).
+ * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN).
* However, the quality of the secret impacts the dispersion of the hash algorithm.
* Therefore, the secret _must_ look like a bunch of random bytes.
* Avoid "trivial" or structured data such as repeated sequences or a text document.
* Whenever in doubt about the "randomness" of the blob of bytes,
- * consider employing "XXH3_generateSecret()" instead (see below).
+ * consider employing @ref XXH3_generateSecret() instead (see below).
* It will generate a proper high entropy secret derived from the blob of bytes.
* Another advantage of using XXH3_generateSecret() is that
* it guarantees that all bits within the initial blob of bytes
* will impact every bit of the output.
* This is not necessarily the case when using the blob of bytes directly
* because, when hashing _small_ inputs, only a portion of the secret is employed.
+ *
+ * @see @ref single_shot_example "Single Shot Example" for an example.
*/
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
@@ -1123,9 +1235,10 @@
*/
/*!
- * @brief The state struct for the XXH3 streaming API.
+ * @brief The opaque state struct for the XXH3 streaming API.
*
* @see XXH3_state_s for details.
+ * @see @ref streaming_example "Streaming Example"
*/
typedef struct XXH3_state_s XXH3_state_t;
XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void);
@@ -1144,15 +1257,20 @@
/*!
* @brief Resets an @ref XXH3_state_t to begin a new hash.
*
- * This function resets `statePtr` and generate a secret with default parameters. Call it before @ref XXH3_64bits_update().
- * Digest will be equivalent to `XXH3_64bits()`.
- *
* @param statePtr The state struct to reset.
*
* @pre
* @p statePtr must not be `NULL`.
*
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note
+ * - This function resets `statePtr` and generate a secret with default parameters.
+ * - Call this function before @ref XXH3_64bits_update().
+ * - Digest will be equivalent to `XXH3_64bits()`.
+ *
+ * @see @ref streaming_example "Streaming Example"
*
*/
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
@@ -1160,36 +1278,54 @@
/*!
* @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
*
- * This function resets `statePtr` and generate a secret from `seed`. Call it before @ref XXH3_64bits_update().
- * Digest will be equivalent to `XXH3_64bits_withSeed()`.
- *
* @param statePtr The state struct to reset.
- * @param seed The 64-bit seed to alter the state.
+ * @param seed The 64-bit seed to alter the hash result predictably.
*
* @pre
* @p statePtr must not be `NULL`.
*
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note
+ * - This function resets `statePtr` and generate a secret from `seed`.
+ * - Call this function before @ref XXH3_64bits_update().
+ * - Digest will be equivalent to `XXH3_64bits_withSeed()`.
+ *
+ * @see @ref streaming_example "Streaming Example"
*
*/
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
/*!
- * XXH3_64bits_reset_withSecret():
- * `secret` is referenced, it _must outlive_ the hash streaming session.
- * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`,
+ * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
+ *
+ * @param statePtr The state struct to reset.
+ * @param secret The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ *
+ * @pre
+ * @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note
+ * `secret` is referenced, it _must outlive_ the hash streaming session.
+ *
+ * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN,
* and the quality of produced hash values depends on secret's entropy
* (secret's content should look like a bunch of random bytes).
* When in doubt about the randomness of a candidate `secret`,
* consider employing `XXH3_generateSecret()` instead (see below).
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
/*!
* @brief Consumes a block of @p input to an @ref XXH3_state_t.
*
- * Call this to incrementally consume blocks of data.
- *
* @param statePtr The state struct to update.
* @param input The block of data to be hashed, at least @p length bytes in size.
* @param length The length of @p input, in bytes.
@@ -1201,23 +1337,30 @@
* readable, contiguous memory. However, if @p length is `0`, @p input may be
* `NULL`. In C++, this also must be *TriviallyCopyable*.
*
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note Call this to incrementally consume blocks of data.
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
/*!
* @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t.
*
- * @note
- * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update,
- * digest, and update again.
- *
* @param statePtr The state struct to calculate the hash from.
*
* @pre
* @p statePtr must not be `NULL`.
*
* @return The calculated XXH3 64-bit hash value from that state.
+ *
+ * @note
+ * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update,
+ * digest, and update again.
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
#endif /* !XXH_NO_STREAM */
@@ -1242,26 +1385,71 @@
} XXH128_hash_t;
/*!
- * @brief Unseeded 128-bit variant of XXH3
+ * @brief Calculates 128-bit unseeded variant of XXH3 of @p data.
+ *
+ * @param data The block of data to be hashed, at least @p length bytes in size.
+ * @param len The length of @p data, in bytes.
+ *
+ * @return The calculated 128-bit variant of XXH3 value.
*
* The 128-bit variant of XXH3 has more strength, but it has a bit of overhead
* for shorter inputs.
*
- * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of 0, however
+ * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however
* it may have slightly better performance due to constant propagation of the
* defaults.
*
- * @see
- * XXH32(), XXH64(), XXH3_64bits(): equivalent for the other xxHash algorithms
- * @see
- * XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants
- * @see
- * XXH3_128bits_reset(), XXH3_128bits_update(), XXH3_128bits_digest(): Streaming version.
+ * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants
+ * @see @ref single_shot_example "Single Shot Example" for an example.
*/
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len);
-/*! @brief Seeded 128-bit variant of XXH3. @see XXH3_64bits_withSeed(). */
+/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data.
+ *
+ * @param data The block of data to be hashed, at least @p length bytes in size.
+ * @param len The length of @p data, in bytes.
+ * @param seed The 64-bit seed to alter the hash result predictably.
+ *
+ * @return The calculated 128-bit variant of XXH3 value.
+ *
+ * @note
+ * seed == 0 produces the same results as @ref XXH3_64bits().
+ *
+ * This variant generates a custom secret on the fly based on default secret
+ * altered using the @p seed value.
+ *
+ * While this operation is decently fast, note that it's not completely free.
+ *
+ * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants
+ * @see @ref single_shot_example "Single Shot Example" for an example.
+ */
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
-/*! @brief Custom secret 128-bit variant of XXH3. @see XXH3_64bits_withSecret(). */
+/*!
+ * @brief Calculates 128-bit variant of XXH3 with a custom "secret".
+ *
+ * @param data The block of data to be hashed, at least @p len bytes in size.
+ * @param len The length of @p data, in bytes.
+ * @param secret The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ *
+ * @return The calculated 128-bit variant of XXH3 value.
+ *
+ * It's possible to provide any blob of bytes as a "secret" to generate the hash.
+ * This makes it more difficult for an external actor to prepare an intentional collision.
+ * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN).
+ * However, the quality of the secret impacts the dispersion of the hash algorithm.
+ * Therefore, the secret _must_ look like a bunch of random bytes.
+ * Avoid "trivial" or structured data such as repeated sequences or a text document.
+ * Whenever in doubt about the "randomness" of the blob of bytes,
+ * consider employing @ref XXH3_generateSecret() instead (see below).
+ * It will generate a proper high entropy secret derived from the blob of bytes.
+ * Another advantage of using XXH3_generateSecret() is that
+ * it guarantees that all bits within the initial blob of bytes
+ * will impact every bit of the output.
+ * This is not necessarily the case when using the blob of bytes directly
+ * because, when hashing _small_ inputs, only a portion of the secret is employed.
+ *
+ * @see @ref single_shot_example "Single Shot Example" for an example.
+ */
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
/******* Streaming *******/
@@ -1281,36 +1469,65 @@
/*!
* @brief Resets an @ref XXH3_state_t to begin a new hash.
*
- * This function resets `statePtr` and generate a secret with default parameters. Call it before @ref XXH3_128bits_update().
- * Digest will be equivalent to `XXH3_128bits()`.
- *
* @param statePtr The state struct to reset.
*
* @pre
* @p statePtr must not be `NULL`.
*
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
*
+ * @note
+ * - This function resets `statePtr` and generate a secret with default parameters.
+ * - Call it before @ref XXH3_128bits_update().
+ * - Digest will be equivalent to `XXH3_128bits()`.
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
/*!
* @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
*
- * This function resets `statePtr` and generate a secret from `seed`. Call it before @ref XXH3_128bits_update().
- * Digest will be equivalent to `XXH3_128bits_withSeed()`.
- *
* @param statePtr The state struct to reset.
- * @param seed The 64-bit seed to alter the state.
+ * @param seed The 64-bit seed to alter the hash result predictably.
*
* @pre
* @p statePtr must not be `NULL`.
*
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
*
+ * @note
+ * - This function resets `statePtr` and generate a secret from `seed`.
+ * - Call it before @ref XXH3_128bits_update().
+ * - Digest will be equivalent to `XXH3_128bits_withSeed()`.
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
-/*! @brief Custom secret 128-bit variant of XXH3. @see XXH_64bits_reset_withSecret(). */
+/*!
+ * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
+ *
+ * @param statePtr The state struct to reset.
+ * @param secret The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ *
+ * @pre
+ * @p statePtr must not be `NULL`.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * `secret` is referenced, it _must outlive_ the hash streaming session.
+ * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN,
+ * and the quality of produced hash values depends on secret's entropy
+ * (secret's content should look like a bunch of random bytes).
+ * When in doubt about the randomness of a candidate `secret`,
+ * consider employing `XXH3_generateSecret()` instead (see below).
+ *
+ * @see @ref streaming_example "Streaming Example"
+ */
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
/*!
@@ -1324,28 +1541,32 @@
*
* @pre
* @p statePtr must not be `NULL`.
- * @pre
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @note
* The memory between @p input and @p input + @p length must be valid,
* readable, contiguous memory. However, if @p length is `0`, @p input may be
* `NULL`. In C++, this also must be *TriviallyCopyable*.
*
- * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
*/
XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
/*!
* @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t.
*
- * @note
- * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update,
- * digest, and update again.
- *
* @param statePtr The state struct to calculate the hash from.
*
* @pre
* @p statePtr must not be `NULL`.
*
* @return The calculated XXH3 128-bit hash value from that state.
+ *
+ * @note
+ * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update,
+ * digest, and update again.
+ *
*/
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
#endif /* !XXH_NO_STREAM */
@@ -1355,18 +1576,27 @@
* Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */
/*!
- * XXH128_isEqual():
- * Return: 1 if `h1` and `h2` are equal, 0 if they are not.
+ * @brief Check equality of two XXH128_hash_t values
+ *
+ * @param h1 The 128-bit hash value.
+ * @param h2 Another 128-bit hash value.
+ *
+ * @return `1` if `h1` and `h2` are equal.
+ * @return `0` if they are not.
*/
XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);
/*!
* @brief Compares two @ref XXH128_hash_t
+ *
* This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
*
- * @return: >0 if *h128_1 > *h128_2
- * =0 if *h128_1 == *h128_2
- * <0 if *h128_1 < *h128_2
+ * @param h128_1 Left-hand side value
+ * @param h128_2 Right-hand side value
+ *
+ * @return >0 if @p h128_1 > @p h128_2
+ * @return =0 if @p h128_1 == @p h128_2
+ * @return <0 if @p h128_1 < @p h128_2
*/
XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2);
@@ -1378,11 +1608,12 @@
/*!
* @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t.
*
- * @param dst The @ref XXH128_canonical_t pointer to be stored to.
+ * @param dst The @ref XXH128_canonical_t pointer to be stored to.
* @param hash The @ref XXH128_hash_t to be converted.
*
* @pre
* @p dst must not be `NULL`.
+ * @see @ref canonical_representation_example "Canonical Representation Example"
*/
XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash);
@@ -1395,6 +1626,7 @@
* @p src must not be `NULL`.
*
* @return The converted hash.
+ * @see @ref canonical_representation_example "Canonical Representation Example"
*/
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src);
@@ -1440,9 +1672,9 @@
struct XXH32_state_s {
XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */
XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */
- XXH32_hash_t v[4]; /*!< Accumulator lanes */
- XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */
- XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */
+ XXH32_hash_t acc[4]; /*!< Accumulator lanes */
+ unsigned char buffer[16]; /*!< Internal buffer for partial reads. */
+ XXH32_hash_t bufferedSize; /*!< Amount of data in @ref buffer */
XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */
}; /* typedef'd to XXH32_state_t */
@@ -1463,9 +1695,9 @@
*/
struct XXH64_state_s {
XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */
- XXH64_hash_t v[4]; /*!< Accumulator lanes */
- XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */
- XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */
+ XXH64_hash_t acc[4]; /*!< Accumulator lanes */
+ unsigned char buffer[32]; /*!< Internal buffer for partial reads.. */
+ XXH32_hash_t bufferedSize; /*!< Amount of data in @ref buffer */
XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/
XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */
}; /* typedef'd to XXH64_state_t */
@@ -1473,8 +1705,7 @@
#ifndef XXH_NO_XXH3
#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */
-# include <stdalign.h>
-# define XXH_ALIGN(n) alignas(n)
+# define XXH_ALIGN(n) _Alignas(n)
#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
/* In C++ alignas() is a keyword */
# define XXH_ALIGN(n) alignas(n)
@@ -1587,7 +1818,20 @@
/*!
- * simple alias to pre-selected XXH3_128bits variant
+ * @brief Calculates the 128-bit hash of @p data using XXH3.
+ *
+ * @param data The block of data to be hashed, at least @p len bytes in size.
+ * @param len The length of @p data, in bytes.
+ * @param seed The 64-bit seed to alter the hash's output predictably.
+ *
+ * @pre
+ * The memory between @p data and @p data + @p len must be valid,
+ * readable, contiguous memory. However, if @p len is `0`, @p data may be
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
+ *
+ * @return The calculated 128-bit XXH3 value.
+ *
+ * @see @ref single_shot_example "Single Shot Example" for an example.
*/
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
@@ -1596,9 +1840,16 @@
/* Symbols defined below must be considered tied to a specific library version. */
/*!
- * XXH3_generateSecret():
+ * @brief Derive a high-entropy secret from any user-defined content, named customSeed.
*
- * Derive a high-entropy secret from any user-defined content, named customSeed.
+ * @param secretBuffer A writable buffer for derived high-entropy secret data.
+ * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_SIZE_MIN.
+ * @param customSeed A user-defined content.
+ * @param customSeedSize Size of customSeed, in bytes.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
* The generated secret can be used in combination with `*_withSecret()` functions.
* The `_withSecret()` variants are useful to provide a higher level of protection
* than 64-bit seed, as it becomes much more difficult for an external actor to
@@ -1651,6 +1902,9 @@
/*!
* @brief Generate the same secret as the _withSeed() variants.
*
+ * @param secretBuffer A writable buffer of @ref XXH3_SECRET_DEFAULT_SIZE bytes
+ * @param seed The 64-bit seed to alter the hash result predictably.
+ *
* The generated secret can be used in combination with
*`*_withSecret()` and `_withSecretandSeed()` variants.
*
@@ -1670,7 +1924,7 @@
* };
* // Fast, caches the seeded secret for future uses.
* class HashFast {
- * unsigned char secret[XXH3_SECRET_SIZE_MIN];
+ * unsigned char secret[XXH3_SECRET_DEFAULT_SIZE];
* public:
* HashFast(XXH64_hash_t s) {
* XXH3_generateSecret_fromSeed(secret, seed);
@@ -1682,15 +1936,26 @@
* }
* };
* @endcode
- * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes
- * @param seed The seed to seed the state.
*/
XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed);
/*!
- * These variants generate hash values using either
- * @p seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes)
- * or @p secret for "large" keys (>= XXH3_MIDSIZE_MAX).
+ * @brief Maximum size of "short" key in bytes.
+ */
+#define XXH3_MIDSIZE_MAX 240
+
+/*!
+ * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data.
+ *
+ * @param data The block of data to be hashed, at least @p len bytes in size.
+ * @param len The length of @p data, in bytes.
+ * @param secret The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ * @param seed The 64-bit seed to alter the hash result predictably.
+ *
+ * These variants generate hash values using either:
+ * - @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes)
+ * - @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX).
*
* This generally benefits speed, compared to `_withSeed()` or `_withSecret()`.
* `_withSeed()` has to generate the secret on the fly for "large" keys.
@@ -1717,22 +1982,71 @@
XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len,
XXH_NOESCAPE const void* secret, size_t secretSize,
XXH64_hash_t seed);
-/*! @copydoc XXH3_64bits_withSecretandSeed() */
+
+/*!
+ * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data.
+ *
+ * @param data The memory segment to be hashed, at least @p len bytes in size.
+ * @param length The length of @p data, in bytes.
+ * @param secret The secret used to alter hash result predictably.
+ * @param secretSize The length of @p secret, in bytes (must be >= XXH3_SECRET_SIZE_MIN)
+ * @param seed64 The 64-bit seed to alter the hash result predictably.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @see XXH3_64bits_withSecretandSeed(): contract is the same.
+ */
XXH_PUBLIC_API XXH_PUREF XXH128_hash_t
XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length,
XXH_NOESCAPE const void* secret, size_t secretSize,
XXH64_hash_t seed64);
+
#ifndef XXH_NO_STREAM
-/*! @copydoc XXH3_64bits_withSecretandSeed() */
+/*!
+ * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
+ *
+ * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
+ * @param secret The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ * @param seed64 The 64-bit seed to alter the hash result predictably.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @see XXH3_64bits_withSecretandSeed(). Contract is identical.
+ */
XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
XXH_NOESCAPE const void* secret, size_t secretSize,
XXH64_hash_t seed64);
-/*! @copydoc XXH3_64bits_withSecretandSeed() */
+
+/*!
+ * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
+ *
+ * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
+ * @param secret The secret data.
+ * @param secretSize The length of @p secret, in bytes.
+ * @param seed64 The 64-bit seed to alter the hash result predictably.
+ *
+ * @return @ref XXH_OK on success.
+ * @return @ref XXH_ERROR on failure.
+ *
+ * @see XXH3_64bits_withSecretandSeed(). Contract is identical.
+ *
+ * Note: there was a bug in an earlier version of this function (<= v0.8.2)
+ * that would make it generate an incorrect hash value
+ * when @p seed == 0 and @p length < XXH3_MIDSIZE_MAX
+ * and @p secret is different from XXH3_generateSecret_fromSeed().
+ * As stated in the contract, the correct hash result must be
+ * the same as XXH3_128bits_withSeed() when @p length <= XXH3_MIDSIZE_MAX.
+ * Results generated by this older version are wrong, hence not comparable.
+ */
XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
XXH_NOESCAPE const void* secret, size_t secretSize,
XXH64_hash_t seed64);
+
#endif /* !XXH_NO_STREAM */
#endif /* !XXH_NO_XXH3 */
@@ -2100,15 +2414,15 @@
#if XXH_NO_INLINE_HINTS /* disable inlining hints */
# if defined(__GNUC__) || defined(__clang__)
-# define XXH_FORCE_INLINE static __attribute__((unused))
+# define XXH_FORCE_INLINE static __attribute__((__unused__))
# else
# define XXH_FORCE_INLINE static
# endif
# define XXH_NO_INLINE static
/* enable inlining hints */
#elif defined(__GNUC__) || defined(__clang__)
-# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused))
-# define XXH_NO_INLINE static __attribute__((noinline))
+# define XXH_FORCE_INLINE static __inline__ __attribute__((__always_inline__, __unused__))
+# define XXH_NO_INLINE static __attribute__((__noinline__))
#elif defined(_MSC_VER) /* Visual Studio */
# define XXH_FORCE_INLINE static __forceinline
# define XXH_NO_INLINE static __declspec(noinline)
@@ -2121,12 +2435,34 @@
# define XXH_NO_INLINE static
#endif
+#if defined(XXH_INLINE_ALL)
+# define XXH_STATIC XXH_FORCE_INLINE
+#else
+# define XXH_STATIC static
+#endif
+
#if XXH3_INLINE_SECRET
# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE
#else
# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE
#endif
+#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
+# define XXH_RESTRICT /* disable */
+#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */
+# define XXH_RESTRICT restrict
+#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \
+ || (defined (__clang__)) \
+ || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \
+ || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300))
+/*
+ * There are a LOT more compilers that recognize __restrict but this
+ * covers the major ones.
+ */
+# define XXH_RESTRICT __restrict
+#else
+# define XXH_RESTRICT /* disable */
+#endif
/* *************************************
* Debug
@@ -2206,10 +2542,14 @@
#if !defined (__VMS) \
&& (defined (__cplusplus) \
|| (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
-# include <stdint.h>
- typedef uint8_t xxh_u8;
+# ifdef _AIX
+# include <inttypes.h>
+# else
+# include <stdint.h>
+# endif
+ typedef uint8_t xxh_u8;
#else
- typedef unsigned char xxh_u8;
+ typedef unsigned char xxh_u8;
#endif
typedef XXH32_hash_t xxh_u32;
@@ -2295,11 +2635,11 @@
* https://gcc.godbolt.org/z/xYez1j67Y.
*/
#ifdef XXH_OLD_NAMES
-typedef union { xxh_u32 u32; } __attribute__((packed)) unalign;
+typedef union { xxh_u32 u32; } __attribute__((__packed__)) unalign;
#endif
static xxh_u32 XXH_read32(const void* ptr)
{
- typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32;
+ typedef __attribute__((__aligned__(1))) xxh_u32 xxh_unalign32;
return *((const xxh_unalign32*)ptr);
}
@@ -2445,6 +2785,9 @@
&& XXH_HAS_BUILTIN(__builtin_rotateleft64)
# define XXH_rotl32 __builtin_rotateleft32
# define XXH_rotl64 __builtin_rotateleft64
+#elif XXH_HAS_BUILTIN(__builtin_stdc_rotate_left)
+# define XXH_rotl32 __builtin_stdc_rotate_left
+# define XXH_rotl64 __builtin_stdc_rotate_left
/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
#elif defined(_MSC_VER)
# define XXH_rotl32(x,r) _rotl(x,r)
@@ -2590,7 +2933,7 @@
#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
/*
* UGLY HACK:
- * A compiler fence is the only thing that prevents GCC and Clang from
+ * A compiler fence is used to prevent GCC and Clang from
* autovectorizing the XXH32 loop (pragmas and attributes don't work for some
* reason) without globally disabling SSE4.1.
*
@@ -2653,6 +2996,61 @@
/*!
* @internal
+ * @brief Sets up the initial accumulator state for XXH32().
+ */
+XXH_FORCE_INLINE void
+XXH32_initAccs(xxh_u32 *acc, xxh_u32 seed)
+{
+ XXH_ASSERT(acc != NULL);
+ acc[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
+ acc[1] = seed + XXH_PRIME32_2;
+ acc[2] = seed + 0;
+ acc[3] = seed - XXH_PRIME32_1;
+}
+
+/*!
+ * @internal
+ * @brief Consumes a block of data for XXH32().
+ *
+ * @return the end input pointer.
+ */
+XXH_FORCE_INLINE const xxh_u8 *
+XXH32_consumeLong(
+ xxh_u32 *XXH_RESTRICT acc,
+ xxh_u8 const *XXH_RESTRICT input,
+ size_t len,
+ XXH_alignment align
+)
+{
+ const xxh_u8* const bEnd = input + len;
+ const xxh_u8* const limit = bEnd - 15;
+ XXH_ASSERT(acc != NULL);
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(len >= 16);
+ do {
+ acc[0] = XXH32_round(acc[0], XXH_get32bits(input)); input += 4;
+ acc[1] = XXH32_round(acc[1], XXH_get32bits(input)); input += 4;
+ acc[2] = XXH32_round(acc[2], XXH_get32bits(input)); input += 4;
+ acc[3] = XXH32_round(acc[3], XXH_get32bits(input)); input += 4;
+ } while (input < limit);
+
+ return input;
+}
+
+/*!
+ * @internal
+ * @brief Merges the accumulator lanes together for XXH32()
+ */
+XXH_FORCE_INLINE XXH_PUREF xxh_u32
+XXH32_mergeAccs(const xxh_u32 *acc)
+{
+ XXH_ASSERT(acc != NULL);
+ return XXH_rotl32(acc[0], 1) + XXH_rotl32(acc[1], 7)
+ + XXH_rotl32(acc[2], 12) + XXH_rotl32(acc[3], 18);
+}
+
+/*!
+ * @internal
* @brief Processes the last 0-15 bytes of @p ptr.
*
* There may be up to 15 bytes remaining to consume from the input.
@@ -2763,22 +3161,12 @@
if (input==NULL) XXH_ASSERT(len == 0);
if (len>=16) {
- const xxh_u8* const bEnd = input + len;
- const xxh_u8* const limit = bEnd - 15;
- xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
- xxh_u32 v2 = seed + XXH_PRIME32_2;
- xxh_u32 v3 = seed + 0;
- xxh_u32 v4 = seed - XXH_PRIME32_1;
+ xxh_u32 acc[4];
+ XXH32_initAccs(acc, seed);
- do {
- v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4;
- v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4;
- v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4;
- v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4;
- } while (input < limit);
+ input = XXH32_consumeLong(acc, input, len, align);
- h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7)
- + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+ h32 = XXH32_mergeAccs(acc);
} else {
h32 = seed + XXH_PRIME32_5;
}
@@ -2834,10 +3222,7 @@
{
XXH_ASSERT(statePtr != NULL);
memset(statePtr, 0, sizeof(*statePtr));
- statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
- statePtr->v[1] = seed + XXH_PRIME32_2;
- statePtr->v[2] = seed + 0;
- statePtr->v[3] = seed - XXH_PRIME32_1;
+ XXH32_initAccs(statePtr->acc, seed);
return XXH_OK;
}
@@ -2851,45 +3236,37 @@
return XXH_OK;
}
- { const xxh_u8* p = (const xxh_u8*)input;
- const xxh_u8* const bEnd = p + len;
+ state->total_len_32 += (XXH32_hash_t)len;
+ state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
- state->total_len_32 += (XXH32_hash_t)len;
- state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
+ XXH_ASSERT(state->bufferedSize < sizeof(state->buffer));
+ if (len < sizeof(state->buffer) - state->bufferedSize) { /* fill in tmp buffer */
+ XXH_memcpy(state->buffer + state->bufferedSize, input, len);
+ state->bufferedSize += (XXH32_hash_t)len;
+ return XXH_OK;
+ }
- if (state->memsize + len < 16) { /* fill in tmp buffer */
- XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len);
- state->memsize += (XXH32_hash_t)len;
- return XXH_OK;
+ { const xxh_u8* xinput = (const xxh_u8*)input;
+ const xxh_u8* const bEnd = xinput + len;
+
+ if (state->bufferedSize) { /* non-empty buffer: complete first */
+ XXH_memcpy(state->buffer + state->bufferedSize, xinput, sizeof(state->buffer) - state->bufferedSize);
+ xinput += sizeof(state->buffer) - state->bufferedSize;
+ /* then process one round */
+ (void)XXH32_consumeLong(state->acc, state->buffer, sizeof(state->buffer), XXH_aligned);
+ state->bufferedSize = 0;
}
- if (state->memsize) { /* some data left from previous update */
- XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize);
- { const xxh_u32* p32 = state->mem32;
- state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++;
- state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++;
- state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++;
- state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32));
- }
- p += 16-state->memsize;
- state->memsize = 0;
+ XXH_ASSERT(xinput <= bEnd);
+ if ((size_t)(bEnd - xinput) >= sizeof(state->buffer)) {
+ /* Process the remaining data */
+ xinput = XXH32_consumeLong(state->acc, xinput, (size_t)(bEnd - xinput), XXH_unaligned);
}
- if (p <= bEnd-16) {
- const xxh_u8* const limit = bEnd - 16;
-
- do {
- state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4;
- state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4;
- state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4;
- state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4;
- } while (p<=limit);
-
- }
-
- if (p < bEnd) {
- XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
- state->memsize = (unsigned)(bEnd-p);
+ if (xinput < bEnd) {
+ /* Copy the leftover to the tmp buffer */
+ XXH_memcpy(state->buffer, xinput, (size_t)(bEnd-xinput));
+ state->bufferedSize = (unsigned)(bEnd-xinput);
}
}
@@ -2903,36 +3280,20 @@
xxh_u32 h32;
if (state->large_len) {
- h32 = XXH_rotl32(state->v[0], 1)
- + XXH_rotl32(state->v[1], 7)
- + XXH_rotl32(state->v[2], 12)
- + XXH_rotl32(state->v[3], 18);
+ h32 = XXH32_mergeAccs(state->acc);
} else {
- h32 = state->v[2] /* == seed */ + XXH_PRIME32_5;
+ h32 = state->acc[2] /* == seed */ + XXH_PRIME32_5;
}
h32 += state->total_len_32;
- return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned);
+ return XXH32_finalize(h32, state->buffer, state->bufferedSize, XXH_aligned);
}
#endif /* !XXH_NO_STREAM */
/******* Canonical representation *******/
-/*!
- * @ingroup XXH32_family
- * The default return values from XXH functions are unsigned 32 and 64 bit
- * integers.
- *
- * The canonical representation uses big endian convention, the same convention
- * as human-readable numbers (large digits first).
- *
- * This way, hash values can be written into a file or buffer, remaining
- * comparable across different systems.
- *
- * The following functions allow transformation of hash values to and from their
- * canonical format.
- */
+/*! @ingroup XXH32_family */
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
{
XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
@@ -2987,11 +3348,11 @@
* https://gcc.godbolt.org/z/xYez1j67Y.
*/
#ifdef XXH_OLD_NAMES
-typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64;
+typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((__packed__)) unalign64;
#endif
static xxh_u64 XXH_read64(const void* ptr)
{
- typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64;
+ typedef __attribute__((__aligned__(1))) xxh_u64 xxh_unalign64;
return *((const xxh_unalign64*)ptr);
}
@@ -3110,6 +3471,23 @@
acc += input * XXH_PRIME64_2;
acc = XXH_rotl64(acc, 31);
acc *= XXH_PRIME64_1;
+#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
+ /*
+ * DISABLE AUTOVECTORIZATION:
+ * A compiler fence is used to prevent GCC and Clang from
+ * autovectorizing the XXH64 loop (pragmas and attributes don't work for some
+ * reason) without globally disabling AVX512.
+ *
+ * Autovectorization of XXH64 tends to be detrimental,
+ * though the exact outcome may change depending on exact cpu and compiler version.
+ * For information, it has been reported as detrimental for Skylake-X,
+ * but possibly beneficial for Zen4.
+ *
+ * The default is to disable auto-vectorization,
+ * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable.
+ */
+ XXH_COMPILER_GUARD(acc);
+#endif
return acc;
}
@@ -3137,6 +3515,85 @@
/*!
* @internal
+ * @brief Sets up the initial accumulator state for XXH64().
+ */
+XXH_FORCE_INLINE void
+XXH64_initAccs(xxh_u64 *acc, xxh_u64 seed)
+{
+ XXH_ASSERT(acc != NULL);
+ acc[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
+ acc[1] = seed + XXH_PRIME64_2;
+ acc[2] = seed + 0;
+ acc[3] = seed - XXH_PRIME64_1;
+}
+
+/*!
+ * @internal
+ * @brief Consumes a block of data for XXH64().
+ *
+ * @return the end input pointer.
+ */
+XXH_FORCE_INLINE const xxh_u8 *
+XXH64_consumeLong(
+ xxh_u64 *XXH_RESTRICT acc,
+ xxh_u8 const *XXH_RESTRICT input,
+ size_t len,
+ XXH_alignment align
+)
+{
+ const xxh_u8* const bEnd = input + len;
+ const xxh_u8* const limit = bEnd - 31;
+ XXH_ASSERT(acc != NULL);
+ XXH_ASSERT(input != NULL);
+ XXH_ASSERT(len >= 32);
+ do {
+ /* reroll on 32-bit */
+ if (sizeof(void *) < sizeof(xxh_u64)) {
+ size_t i;
+ for (i = 0; i < 4; i++) {
+ acc[i] = XXH64_round(acc[i], XXH_get64bits(input));
+ input += 8;
+ }
+ } else {
+ acc[0] = XXH64_round(acc[0], XXH_get64bits(input)); input += 8;
+ acc[1] = XXH64_round(acc[1], XXH_get64bits(input)); input += 8;
+ acc[2] = XXH64_round(acc[2], XXH_get64bits(input)); input += 8;
+ acc[3] = XXH64_round(acc[3], XXH_get64bits(input)); input += 8;
+ }
+ } while (input < limit);
+
+ return input;
+}
+
+/*!
+ * @internal
+ * @brief Merges the accumulator lanes together for XXH64()
+ */
+XXH_FORCE_INLINE XXH_PUREF xxh_u64
+XXH64_mergeAccs(const xxh_u64 *acc)
+{
+ XXH_ASSERT(acc != NULL);
+ {
+ xxh_u64 h64 = XXH_rotl64(acc[0], 1) + XXH_rotl64(acc[1], 7)
+ + XXH_rotl64(acc[2], 12) + XXH_rotl64(acc[3], 18);
+ /* reroll on 32-bit */
+ if (sizeof(void *) < sizeof(xxh_u64)) {
+ size_t i;
+ for (i = 0; i < 4; i++) {
+ h64 = XXH64_mergeRound(h64, acc[i]);
+ }
+ } else {
+ h64 = XXH64_mergeRound(h64, acc[0]);
+ h64 = XXH64_mergeRound(h64, acc[1]);
+ h64 = XXH64_mergeRound(h64, acc[2]);
+ h64 = XXH64_mergeRound(h64, acc[3]);
+ }
+ return h64;
+ }
+}
+
+/*!
+ * @internal
* @brief Processes the last 0-31 bytes of @p ptr.
*
* There may be up to 31 bytes remaining to consume from the input.
@@ -3150,7 +3607,7 @@
* @return The finalized hash
* @see XXH32_finalize().
*/
-static XXH_PUREF xxh_u64
+XXH_STATIC XXH_PUREF xxh_u64
XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
{
if (ptr==NULL) XXH_ASSERT(len == 0);
@@ -3200,27 +3657,13 @@
xxh_u64 h64;
if (input==NULL) XXH_ASSERT(len == 0);
- if (len>=32) {
- const xxh_u8* const bEnd = input + len;
- const xxh_u8* const limit = bEnd - 31;
- xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
- xxh_u64 v2 = seed + XXH_PRIME64_2;
- xxh_u64 v3 = seed + 0;
- xxh_u64 v4 = seed - XXH_PRIME64_1;
+ if (len>=32) { /* Process a large block of data */
+ xxh_u64 acc[4];
+ XXH64_initAccs(acc, seed);
- do {
- v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8;
- v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8;
- v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8;
- v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8;
- } while (input<limit);
+ input = XXH64_consumeLong(acc, input, len, align);
- h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
- h64 = XXH64_mergeRound(h64, v1);
- h64 = XXH64_mergeRound(h64, v2);
- h64 = XXH64_mergeRound(h64, v3);
- h64 = XXH64_mergeRound(h64, v4);
-
+ h64 = XXH64_mergeAccs(acc);
} else {
h64 = seed + XXH_PRIME64_5;
}
@@ -3276,10 +3719,7 @@
{
XXH_ASSERT(statePtr != NULL);
memset(statePtr, 0, sizeof(*statePtr));
- statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
- statePtr->v[1] = seed + XXH_PRIME64_2;
- statePtr->v[2] = seed + 0;
- statePtr->v[3] = seed - XXH_PRIME64_1;
+ XXH64_initAccs(statePtr->acc, seed);
return XXH_OK;
}
@@ -3292,42 +3732,36 @@
return XXH_OK;
}
- { const xxh_u8* p = (const xxh_u8*)input;
- const xxh_u8* const bEnd = p + len;
+ state->total_len += len;
- state->total_len += len;
+ XXH_ASSERT(state->bufferedSize <= sizeof(state->buffer));
+ if (len < sizeof(state->buffer) - state->bufferedSize) { /* fill in tmp buffer */
+ XXH_memcpy(state->buffer + state->bufferedSize, input, len);
+ state->bufferedSize += (XXH32_hash_t)len;
+ return XXH_OK;
+ }
- if (state->memsize + len < 32) { /* fill in tmp buffer */
- XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len);
- state->memsize += (xxh_u32)len;
- return XXH_OK;
+ { const xxh_u8* xinput = (const xxh_u8*)input;
+ const xxh_u8* const bEnd = xinput + len;
+
+ if (state->bufferedSize) { /* non-empty buffer => complete first */
+ XXH_memcpy(state->buffer + state->bufferedSize, xinput, sizeof(state->buffer) - state->bufferedSize);
+ xinput += sizeof(state->buffer) - state->bufferedSize;
+ /* and process one round */
+ (void)XXH64_consumeLong(state->acc, state->buffer, sizeof(state->buffer), XXH_aligned);
+ state->bufferedSize = 0;
}
- if (state->memsize) { /* tmp buffer is full */
- XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize);
- state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0));
- state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1));
- state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2));
- state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3));
- p += 32 - state->memsize;
- state->memsize = 0;
+ XXH_ASSERT(xinput <= bEnd);
+ if ((size_t)(bEnd - xinput) >= sizeof(state->buffer)) {
+ /* Process the remaining data */
+ xinput = XXH64_consumeLong(state->acc, xinput, (size_t)(bEnd - xinput), XXH_unaligned);
}
- if (p+32 <= bEnd) {
- const xxh_u8* const limit = bEnd - 32;
-
- do {
- state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8;
- state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8;
- state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8;
- state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8;
- } while (p<=limit);
-
- }
-
- if (p < bEnd) {
- XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
- state->memsize = (unsigned)(bEnd-p);
+ if (xinput < bEnd) {
+ /* Copy the leftover to the tmp buffer */
+ XXH_memcpy(state->buffer, xinput, (size_t)(bEnd-xinput));
+ state->bufferedSize = (unsigned)(bEnd-xinput);
}
}
@@ -3341,18 +3775,14 @@
xxh_u64 h64;
if (state->total_len >= 32) {
- h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18);
- h64 = XXH64_mergeRound(h64, state->v[0]);
- h64 = XXH64_mergeRound(h64, state->v[1]);
- h64 = XXH64_mergeRound(h64, state->v[2]);
- h64 = XXH64_mergeRound(h64, state->v[3]);
+ h64 = XXH64_mergeAccs(state->acc);
} else {
- h64 = state->v[2] /*seed*/ + XXH_PRIME64_5;
+ h64 = state->acc[2] /*seed*/ + XXH_PRIME64_5;
}
h64 += (xxh_u64) state->total_len;
- return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned);
+ return XXH64_finalize(h64, state->buffer, (size_t)state->total_len, XXH_aligned);
}
#endif /* !XXH_NO_STREAM */
@@ -3387,22 +3817,6 @@
/* === Compiler specifics === */
-#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
-# define XXH_RESTRICT /* disable */
-#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */
-# define XXH_RESTRICT restrict
-#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \
- || (defined (__clang__)) \
- || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \
- || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300))
-/*
- * There are a LOT more compilers that recognize __restrict but this
- * covers the major ones.
- */
-# define XXH_RESTRICT __restrict
-#else
-# define XXH_RESTRICT /* disable */
-#endif
#if (defined(__GNUC__) && (__GNUC__ >= 3)) \
|| (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
@@ -3416,7 +3830,11 @@
#ifndef XXH_HAS_INCLUDE
# ifdef __has_include
-# define XXH_HAS_INCLUDE(x) __has_include(x)
+/*
+ * Not defined as XXH_HAS_INCLUDE(x) (function-like) because
+ * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion)
+ */
+# define XXH_HAS_INCLUDE __has_include
# else
# define XXH_HAS_INCLUDE(x) 0
# endif
@@ -3437,6 +3855,8 @@
# include <immintrin.h>
# elif defined(__SSE2__)
# include <emmintrin.h>
+# elif defined(__loongarch_sx)
+# include <lsxintrin.h>
# endif
#endif
@@ -3535,33 +3955,6 @@
# define XXH_VECTOR XXH_SCALAR
/*!
* @ingroup tuning
- * @brief Possible values for @ref XXH_VECTOR.
- *
- * Note that these are actually implemented as macros.
- *
- * If this is not defined, it is detected automatically.
- * internal macro XXH_X86DISPATCH overrides this.
- */
-enum XXH_VECTOR_TYPE /* fake enum */ {
- XXH_SCALAR = 0, /*!< Portable scalar version */
- XXH_SSE2 = 1, /*!<
- * SSE2 for Pentium 4, Opteron, all x86_64.
- *
- * @note SSE2 is also guaranteed on Windows 10, macOS, and
- * Android x86.
- */
- XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */
- XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */
- XXH_NEON = 4, /*!<
- * NEON for most ARMv7-A, all AArch64, and WASM SIMD128
- * via the SIMDeverywhere polyfill provided with the
- * Emscripten SDK.
- */
- XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */
- XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */
-};
-/*!
- * @ingroup tuning
* @brief Selects the minimum alignment for XXH3's accumulators.
*
* When using SIMD, this should match the alignment required for said vector
@@ -3574,13 +3967,6 @@
/* Actual definition */
#ifndef XXH_DOXYGEN
-# define XXH_SCALAR 0
-# define XXH_SSE2 1
-# define XXH_AVX2 2
-# define XXH_AVX512 3
-# define XXH_NEON 4
-# define XXH_VSX 5
-# define XXH_SVE 6
#endif
#ifndef XXH_VECTOR /* can be defined on command line */
@@ -3605,6 +3991,8 @@
|| (defined(__s390x__) && defined(__VEC__)) \
&& defined(__GNUC__) /* TODO: IBM XL */
# define XXH_VECTOR XXH_VSX
+# elif defined(__loongarch_sx)
+# define XXH_VECTOR XXH_LSX
# else
# define XXH_VECTOR XXH_SCALAR
# endif
@@ -3642,6 +4030,8 @@
# define XXH_ACC_ALIGN 64
# elif XXH_VECTOR == XXH_SVE /* sve */
# define XXH_ACC_ALIGN 64
+# elif XXH_VECTOR == XXH_LSX /* lsx */
+# define XXH_ACC_ALIGN 64
# endif
#endif
@@ -3655,7 +4045,7 @@
#endif
#if defined(__GNUC__) || defined(__clang__)
-# define XXH_ALIASING __attribute__((may_alias))
+# define XXH_ALIASING __attribute__((__may_alias__))
#else
# define XXH_ALIASING /* nothing */
#endif
@@ -4408,8 +4798,6 @@
}
}
-#define XXH3_MIDSIZE_MAX 240
-
XXH_NO_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
@@ -5281,6 +5669,71 @@
#endif
+#if (XXH_VECTOR == XXH_LSX)
+#define _LSX_SHUFFLE(z, y, x, w) (((z) << 6) | ((y) << 4) | ((x) << 2) | (w))
+
+XXH_FORCE_INLINE void
+XXH3_accumulate_512_lsx( void* XXH_RESTRICT acc,
+ const void* XXH_RESTRICT input,
+ const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+ {
+ __m128i* const xacc = (__m128i *) acc;
+ const __m128i* const xinput = (const __m128i *) input;
+ const __m128i* const xsecret = (const __m128i *) secret;
+
+ for (size_t i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) {
+ /* data_vec = xinput[i]; */
+ __m128i const data_vec = __lsx_vld(xinput + i, 0);
+ /* key_vec = xsecret[i]; */
+ __m128i const key_vec = __lsx_vld(xsecret + i, 0);
+ /* data_key = data_vec ^ key_vec; */
+ __m128i const data_key = __lsx_vxor_v(data_vec, key_vec);
+ /* data_key_lo = data_key >> 32; */
+ __m128i const data_key_lo = __lsx_vsrli_d(data_key, 32);
+ // __m128i const data_key_lo = __lsx_vsrli_d(data_key, 32);
+ /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
+ __m128i const product = __lsx_vmulwev_d_wu(data_key, data_key_lo);
+ /* xacc[i] += swap(data_vec); */
+ __m128i const data_swap = __lsx_vshuf4i_w(data_vec, _LSX_SHUFFLE(1, 0, 3, 2));
+ __m128i const sum = __lsx_vadd_d(xacc[i], data_swap);
+ /* xacc[i] += product; */
+ xacc[i] = __lsx_vadd_d(product, sum);
+ }
+ }
+}
+XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(lsx)
+
+XXH_FORCE_INLINE void
+XXH3_scrambleAcc_lsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
+{
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
+ {
+ __m128i* const xacc = (__m128i*) acc;
+ const __m128i* const xsecret = (const __m128i *) secret;
+ const __m128i prime32 = __lsx_vreplgr2vr_w((int)XXH_PRIME32_1);
+
+ for (size_t i = 0; i < XXH_STRIPE_LEN / sizeof(__m128i); i++) {
+ /* xacc[i] ^= (xacc[i] >> 47) */
+ __m128i const acc_vec = xacc[i];
+ __m128i const shifted = __lsx_vsrli_d(acc_vec, 47);
+ __m128i const data_vec = __lsx_vxor_v(acc_vec, shifted);
+ /* xacc[i] ^= xsecret[i]; */
+ __m128i const key_vec = __lsx_vld(xsecret + i, 0);
+ __m128i const data_key = __lsx_vxor_v(data_vec, key_vec);
+
+ /* xacc[i] *= XXH_PRIME32_1; */
+ __m128i const data_key_hi = __lsx_vsrli_d(data_key, 32);
+ __m128i const prod_lo = __lsx_vmulwev_d_wu(data_key, prime32);
+ __m128i const prod_hi = __lsx_vmulwev_d_wu(data_key_hi, prime32);
+ xacc[i] = __lsx_vadd_d(prod_lo, __lsx_vslli_d(prod_hi, 32));
+ }
+ }
+}
+
+#endif
+
/* scalar variants - universal */
#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__))
@@ -5511,6 +5964,12 @@
#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar
#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+#elif (XXH_VECTOR == XXH_LSX)
+#define XXH3_accumulate_512 XXH3_accumulate_512_lsx
+#define XXH3_accumulate XXH3_accumulate_lsx
+#define XXH3_scrambleAcc XXH3_scrambleAcc_lsx
+#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
+
#else /* scalar */
#define XXH3_accumulate_512 XXH3_accumulate_512_scalar
@@ -5566,7 +6025,7 @@
acc[1] ^ XXH_readLE64(secret+8) );
}
-static XXH64_hash_t
+static XXH_PUREF XXH64_hash_t
XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
{
xxh_u64 result64 = start;
@@ -5593,6 +6052,15 @@
return XXH3_avalanche(result64);
}
+/* do not align on 8, so that the secret is different from the accumulator */
+#define XXH_SECRET_MERGEACCS_START 11
+
+static XXH_PUREF XXH64_hash_t
+XXH3_finalizeLong_64b(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 len)
+{
+ return XXH3_mergeAccs(acc, secret + XXH_SECRET_MERGEACCS_START, len * XXH_PRIME64_1);
+}
+
#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
@@ -5608,10 +6076,8 @@
/* converge into final hash */
XXH_STATIC_ASSERT(sizeof(acc) == 64);
- /* do not align on 8, so that the secret is different from the accumulator */
-#define XXH_SECRET_MERGEACCS_START 11
XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
- return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1);
+ return XXH3_finalizeLong_64b(acc, (const xxh_u8*)secret, (xxh_u64)len);
}
/*
@@ -5747,7 +6213,7 @@
/* === XXH3 streaming === */
#ifndef XXH_NO_STREAM
/*
- * Malloc's a pointer that is always aligned to align.
+ * Malloc's a pointer that is always aligned to @align.
*
* This must be freed with `XXH_alignedFree()`.
*
@@ -5815,8 +6281,12 @@
/*!
* @brief Allocate an @ref XXH3_state_t.
*
- * Must be freed with XXH3_freeState().
- * @return An allocated XXH3_state_t on success, `NULL` on failure.
+ * @return An allocated pointer of @ref XXH3_state_t on success.
+ * @return `NULL` on failure.
+ *
+ * @note Must be freed with XXH3_freeState().
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
{
@@ -5830,9 +6300,13 @@
/*!
* @brief Frees an @ref XXH3_state_t.
*
- * Must be allocated with XXH3_createState().
* @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
- * @return XXH_OK.
+ *
+ * @return @ref XXH_OK.
+ *
+ * @note Must be allocated with XXH3_createState().
+ *
+ * @see @ref streaming_example "Streaming Example"
*/
XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
{
@@ -6111,9 +6585,7 @@
if (state->totalLen > XXH3_MIDSIZE_MAX) {
XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
XXH3_digest_long(acc, state, secret);
- return XXH3_mergeAccs(acc,
- secret + XXH_SECRET_MERGEACCS_START,
- (xxh_u64)state->totalLen * XXH_PRIME64_1);
+ return XXH3_finalizeLong_64b(acc, secret, (xxh_u64)state->totalLen);
}
/* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
if (state->useSeed)
@@ -6405,6 +6877,17 @@
}
}
+static XXH_PUREF XXH128_hash_t
+XXH3_finalizeLong_128b(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, size_t secretSize, xxh_u64 len)
+{
+ XXH128_hash_t h128;
+ h128.low64 = XXH3_finalizeLong_64b(acc, secret, len);
+ h128.high64 = XXH3_mergeAccs(acc, secret + secretSize
+ - XXH_STRIPE_LEN - XXH_SECRET_MERGEACCS_START,
+ ~(len * XXH_PRIME64_2));
+ return h128;
+}
+
XXH_FORCE_INLINE XXH128_hash_t
XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len,
const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
@@ -6418,16 +6901,7 @@
/* converge into final hash */
XXH_STATIC_ASSERT(sizeof(acc) == 64);
XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
- { XXH128_hash_t h128;
- h128.low64 = XXH3_mergeAccs(acc,
- secret + XXH_SECRET_MERGEACCS_START,
- (xxh_u64)len * XXH_PRIME64_1);
- h128.high64 = XXH3_mergeAccs(acc,
- secret + secretSize
- - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
- ~((xxh_u64)len * XXH_PRIME64_2));
- return h128;
- }
+ return XXH3_finalizeLong_128b(acc, secret, secretSize, (xxh_u64)len);
}
/*
@@ -6610,19 +7084,10 @@
XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
XXH3_digest_long(acc, state, secret);
XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
- { XXH128_hash_t h128;
- h128.low64 = XXH3_mergeAccs(acc,
- secret + XXH_SECRET_MERGEACCS_START,
- (xxh_u64)state->totalLen * XXH_PRIME64_1);
- h128.high64 = XXH3_mergeAccs(acc,
- secret + state->secretLimit + XXH_STRIPE_LEN
- - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
- ~((xxh_u64)state->totalLen * XXH_PRIME64_2));
- return h128;
- }
+ return XXH3_finalizeLong_128b(acc, secret, state->secretLimit + XXH_STRIPE_LEN, (xxh_u64)state->totalLen);
}
/* len <= XXH3_MIDSIZE_MAX : short code */
- if (state->seed)
+ if (state->useSeed)
return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen),
secret, state->secretLimit + XXH_STRIPE_LEN);
