Yann Collet's `xxhash`. [[ xxhash Fast Hash Algorithm; https://cyan4973.github.io/xxHash/ ]]

Collection Info

View Source
Collection
core
Path
hash/xxhash
Entries
169

Source Files

Constants

35

XXH_DISABLE_PREFETCH #

Source
XXH_DISABLE_PREFETCH :: #config(XXH_DISABLE_PREFETCH, true)

XXH_FORCE_ALIGN_CHECK #

Source
XXH_FORCE_ALIGN_CHECK :: #config(XXH_FORCE_ALIGN_CHECK, false)

`false` - Use this on platforms where unaligned reads are fast `true` - Use this on platforms where unaligned reads are slow

XXH_FORCE_MEMORY_ACCESS #

Source
XXH_FORCE_MEMORY_ACCESS :: #config(XXH_FORCE_MEMORY_ACCESS, 2)

0 - Use memcopy, for platforms where unaligned reads are a problem 2 - Direct cast, for platforms where unaligned are allowed (default)

XXH_MAX_WIDTH #

Source
XXH_MAX_WIDTH :: #config(XXH_MAX_WIDTH, 512) / 64

XXH_NATIVE_WIDTH #

Source
XXH_NATIVE_WIDTH :: min(XXH_MAX_WIDTH, 8 when intrinsics.has_target_feature(AVX512_FEATURES) else 4 when intrinsics.has_target_feature(AVX2_FEATURES) else 2 when intrinsics.has_target_feature(SSE2_FEATURES) else 1)

XXH_PRIME32_1 #

Source
XXH_PRIME32_1 :: 0x9E3779B1

!< 0b10011110001101110111100110110001

XXH_PRIME32_2 #

Source
XXH_PRIME32_2 :: 0x85EBCA77

!< 0b10000101111010111100101001110111

XXH_PRIME32_3 #

Source
XXH_PRIME32_3 :: 0xC2B2AE3D

!< 0b11000010101100101010111000111101

XXH_PRIME32_4 #

Source
XXH_PRIME32_4 :: 0x27D4EB2F

!< 0b00100111110101001110101100101111

XXH_PRIME32_5 #

Source
XXH_PRIME32_5 :: 0x165667B1

!< 0b00010110010101100110011110110001

XXH_PRIME64_1 #

Source
XXH_PRIME64_1 :: 0x9E3779B185EBCA87

!< 0b1001111000110111011110011011000110000101111010111100101010000111

XXH_PRIME64_2 #

Source
XXH_PRIME64_2 :: 0xC2B2AE3D27D4EB4F

!< 0b1100001010110010101011100011110100100111110101001110101101001111

XXH_PRIME64_3 #

Source
XXH_PRIME64_3 :: 0x165667B19E3779F9

!< 0b0001011001010110011001111011000110011110001101110111100111111001

XXH_PRIME64_4 #

Source
XXH_PRIME64_4 :: 0x85EBCA77C2B2AE63

!< 0b1000010111101011110010100111011111000010101100101010111001100011

XXH_PRIME64_5 #

Source
XXH_PRIME64_5 :: 0x27D4EB2F165667C5

!< 0b0010011111010100111010110010111100010110010101100110011111000101

XXH_SECRET_CONSUME_RATE #

Source
XXH_SECRET_CONSUME_RATE :: 8

nb of secret bytes consumed at each accumulation

XXH_SECRET_DEFAULT_SIZE #

Source
XXH_SECRET_DEFAULT_SIZE :: max(XXH3_SECRET_SIZE_MIN, #config(XXH_SECRET_DEFAULT_SIZE, 192))

Custom secrets have a default length of 192, but can be set to a different size. The minimum secret size is 136 bytes. It must also be a multiple of 64.

XXH_SECRET_LASTACC_START #

Source
XXH_SECRET_LASTACC_START :: 7

not aligned on 8, last secret is different from acc & scrambler

XXH_VERSION_NUMBER #

Source
XXH_VERSION_NUMBER :: XXH_VERSION_MAJOR * 100 * 100 + XXH_VERSION_MINOR * 100 + XXH_VERSION_RELEASE

XXH3_INIT_ACC #

Source
XXH3_INIT_ACC :: [XXH_ACC_NB]xxh_u64{XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1}

XXH3_INTERNAL_BUFFER_SIZE #

Source
XXH3_INTERNAL_BUFFER_SIZE :: 256

This is the optimal update size for incremental hashing.

XXH3_SECRET_SIZE_MIN #

Source
XXH3_SECRET_SIZE_MIN :: 136

Do not change this constant.

Config Values

4

XXH_DISABLE_PREFETCH #

Source
XXH_DISABLE_PREFETCH :: #config(XXH_DISABLE_PREFETCH, true)

XXH_FORCE_ALIGN_CHECK #

Source
XXH_FORCE_ALIGN_CHECK :: #config(XXH_FORCE_ALIGN_CHECK, false)

`false` - Use this on platforms where unaligned reads are fast `true` - Use this on platforms where unaligned reads are slow

XXH_FORCE_MEMORY_ACCESS #

Source
XXH_FORCE_MEMORY_ACCESS :: #config(XXH_FORCE_MEMORY_ACCESS, 2)

0 - Use memcopy, for platforms where unaligned reads are a problem 2 - Direct cast, for platforms where unaligned are allowed (default)

XXH_MAX_WIDTH #

Source
XXH_MAX_WIDTH :: #config(XXH_MAX_WIDTH, 512) / 64

Types

20

xxh_u128 #

Source
xxh_u128 :: u128

Stored in little endian order, although the fields themselves are in native endianness.

XXH3_state #

Source
XXH3_state :: XXH3_state

Streaming state. IMPORTANT: This structure has a strict alignment requirement of 64 bytes!! ** Default allocators will align it correctly if created via `new`, as will placing this struct on the stack, but if using a custom allocator make sure that it handles the alignment correctly!

Procedures

107

XXH_mul_64_to_128_fold_64 #

Source
@(optimization_mode="favor_size")
XXH_mul_64_to_128_fold_64 :: proc(lhs, rhs: u64) -> (res: u64) {…}

The reason for the separate function is to prevent passing too many structs around by value. This will hopefully inline the multiply, but we don't force it. @param lhs, rhs The 64-bit integers to multiply @return The low 64 bits of the product XOR'd by the high 64 bits.

XXH_rotl32 #

Source
@(optimization_mode="favor_size")
XXH_rotl32 :: proc(x, r: u32) -> (res: u32) {…}

XXH_rotl64 #

Source
@(optimization_mode="favor_size")
XXH_rotl64 :: proc(x, r: u64) -> (res: u64) {…}

XXH_writeLE64 #

Source
@(optimization_mode="favor_size")
XXH_writeLE64 :: proc(dst: []u8, v64: u64le) {…}

XXH_xorshift_64 #

Source
@(optimization_mode="favor_size")
XXH_xorshift_64 :: proc(v: u64, #any_int shift: uint) -> (res: u64) {…}

XXH128_mix32B #

Source
@(optimization_mode="favor_size")
XXH128_mix32B :: proc(acc: u128, input_1: []u8, input_2: []u8, secret: []u8, seed: u64) -> (res: u128) {…}

A bit slower than XXH3_mix16B, but handles multiply by zero better.

XXH3_128_default #

Source
@(optimization_mode="favor_size")
XXH3_128_default :: proc(input: []u8) -> (hash: u128) {…}

=== Public XXH128 API ===

XXH3_128_reset #

Source
XXH3_128_reset :: proc(state: ^XXH3_state) -> (err: Error) {…}

=== XXH3 128-bit streaming === All the functions are actually the same as for 64-bit streaming variant. The only difference is the finalization routine.

XXH3_128_with_secret #

Source
@(optimization_mode="favor_size")
XXH3_128_with_secret :: proc(input, secret: []u8) -> (hash: u128) {…}

XXH3_128_with_seed #

Source
@(optimization_mode="favor_size")
XXH3_128_with_seed :: proc(input: []u8, seed: u64) -> (hash: u128) {…}

XXH3_64_default #

Source
@(optimization_mode="favor_size")
XXH3_64_default :: proc(input: []u8) -> (hash: u64) {…}

=== Public entry point ===

XXH3_64_reset #

Source
XXH3_64_reset :: proc(state: ^XXH3_state) -> (err: Error) {…}

=== XXH3 128-bit streaming === All the functions are actually the same as for 64-bit streaming variant. The only difference is the finalization routine.

XXH3_64_with_secret #

Source
@(optimization_mode="favor_size")
XXH3_64_with_secret :: proc(input, secret: []u8) -> (hash: u64) {…}

XXH3_64_with_seed #

Source
@(optimization_mode="favor_size")
XXH3_64_with_seed :: proc(input: []u8, seed: u64) -> (hash: u64) {…}

XXH3_accumulate #

Source
@(optimization_mode="favor_size")
XXH3_accumulate :: proc(acc: []u64, input: []u8, secret: []u8, nbStripes: uint, f_acc512: XXH3_accumulate_512_f) {…}

* XXH3_accumulate() * Loops over XXH3_accumulate_512(). * Assumption: nbStripes will not overflow the secret size

XXH3_accumulate_512_scalar #

Source
@(optimization_mode="favor_size")
XXH3_accumulate_512_scalar :: proc(acc: []u64, input: []u8, secret: []u8) {…}

scalar variants - universal

XXH3_accumulate_512_simd_generic #

Source
XXH3_accumulate_512_simd_generic :: proc(acc: []u64, input: []u8, secret: []u8, $W: uint) {…}

generalized SIMD variants

XXH3_avalanche #

Source
@(optimization_mode="favor_size")
XXH3_avalanche :: proc(h64: u64) -> (res: u64) {…}

This is a fast avalanche stage, suitable when input bits are already partially mixed

XXH3_generate_secret #

Source
XXH3_generate_secret :: proc(secret_buffer: []u8, custom_seed: []u8) {…}

XXH3_hashLong_128b_default #

Source
XXH3_hashLong_128b_default :: proc(input: []u8, seed: u64, secret: []u8) -> (res: u128) {…}

* It's important for performance that XXH3_hashLong is not inlined.

XXH3_hashLong_128b_withSecret #

Source
XXH3_hashLong_128b_withSecret :: proc(input: []u8, seed: u64, secret: []u8) -> (res: u128) {…}

* It's important for performance that XXH3_hashLong is not inlined.

XXH3_hashLong_128b_withSeed #

Source
XXH3_hashLong_128b_withSeed :: proc(input: []u8, seed: u64, secret: []u8) -> (res: u128) {…}

* It's important for performance that XXH3_hashLong is not inlined.

XXH3_hashLong_64b_default #

Source
XXH3_hashLong_64b_default :: proc(input: []u8, seed64: u64, secret: []u8) -> (hash: u64) {…}

It's important for performance that XXH3_hashLong is not inlined. Since the function is not inlined, the compiler may not be able to understand that, in some scenarios, its `secret` argument is actually a compile time constant. This variant enforces that the compiler can detect that, and uses this opportunity to streamline the generated code for better performance.

XXH3_hashLong_64b_withSecret #

Source
XXH3_hashLong_64b_withSecret :: proc(input: []u8, seed64: u64, secret: []u8) -> (hash: u64) {…}

It's important for performance that XXH3_hashLong is not inlined.

XXH3_hashLong_64b_withSeed #

Source
XXH3_hashLong_64b_withSeed :: proc(input: []u8, seed: u64, secret: []u8) -> (hash: u64) {…}

XXH3_hashLong_64b_withSeed(): Generate a custom key based on alteration of default XXH3_kSecret with the seed, and then use this key for long mode hashing. This operation is decently fast but nonetheless costs a little bit of time. Try to avoid it whenever possible (typically when seed==0). It's important for performance that XXH3_hashLong is not inlined. Not sure why (uop cache maybe?), but the difference is large and easily measurable.

XXH3_init_custom_secret #

Source
XXH3_init_custom_secret :: proc(custom_secret: []u8, seed64: u64) {…}

XXH3_init_custom_secret_scalar #

Source
@(optimization_mode="favor_size")
XXH3_init_custom_secret_scalar :: proc(custom_secret: []u8, seed64: u64) {…}

XXH3_init_custom_secret_simd_generic #

Source
XXH3_init_custom_secret_simd_generic :: proc(custom_secret: []u8, seed64: u64, $W: uint) {…}

XXH3_len_0to16_128b #

Source
@(optimization_mode="favor_size")
XXH3_len_0to16_128b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u128) {…}

Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN

XXH3_len_0to16_64b #

Source
@(optimization_mode="favor_size")
XXH3_len_0to16_64b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u64) {…}

XXH3_len_129to240_128b #

Source
@(optimization_mode="favor_size")
XXH3_len_129to240_128b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u128) {…}

XXH3_len_129to240_64b #

Source
@(optimization_mode="favor_size")
XXH3_len_129to240_64b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u64) {…}

XXH3_len_17to128_128b #

Source
@(optimization_mode="favor_size")
XXH3_len_17to128_128b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u128) {…}

XXH3_len_17to128_64b #

Source
@(optimization_mode="favor_size")
XXH3_len_17to128_64b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u64) {…}

For mid range keys, XXH3 uses a Mum-hash variant.

XXH3_len_1to3_128b #

Source
@(optimization_mode="favor_size")
XXH3_len_1to3_128b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u128) {…}

XXH3_len_1to3_64b #

Source
@(optimization_mode="favor_size")
XXH3_len_1to3_64b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u64) {…}

At very short lengths, there isn't enough input to fully hide secrets, or use the entire secret. There is also only a limited amount of mixing we can do before significantly impacting performance. Therefore, we use different sections of the secret and always mix two secret samples with an XOR. This should have no effect on performance on the seedless or withSeed variants because everything _should_ be constant folded by modern compilers. The XOR mixing hides individual parts of the secret and increases entropy. This adds an extra layer of strength for custom secrets.

XXH3_len_4to8_128b #

Source
@(optimization_mode="favor_size")
XXH3_len_4to8_128b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u128) {…}

XXH3_len_4to8_64b #

Source
@(optimization_mode="favor_size")
XXH3_len_4to8_64b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u64) {…}

XXH3_len_9to16_128b #

Source
@(optimization_mode="favor_size")
XXH3_len_9to16_128b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u128) {…}

XXH3_len_9to16_64b #

Source
@(optimization_mode="favor_size")
XXH3_len_9to16_64b :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u64) {…}

XXH3_mergeAccs #

Source
@(optimization_mode="favor_size")
XXH3_mergeAccs :: proc(acc: []u64, secret: []u8, start: u64) -> (res: u64) {…}

XXH3_mix16B #

Source
@(optimization_mode="favor_size")
XXH3_mix16B :: proc(input: []u8, secret: []u8, seed: u64) -> (res: u64) {…}

DISCLAIMER: There are known *seed-dependent* multicollisions here due to multiplication by zero, affecting hashes of lengths 17 to 240. However, they are very unlikely. Keep this in mind when using the unseeded XXH3_64bits() variant: As with all unseeded non-cryptographic hashes, it does not attempt to defend itself against specially crafted inputs, only random inputs. Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes cancelling out the secret is taken an arbitrary number of times (addressed in XXH3_accumulate_512), this collision is very unlikely with random inputs and/or proper seeding: This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a function that is only called up to 16 times per hash with up to 240 bytes of input. This is not too bad for a non-cryptographic hash function, especially with only 64 bit outputs. The 128-bit variant (which trades some speed for strength) is NOT affected by this, although it is always a good idea to use a proper seed if you care about strength.

XXH3_mix2Accs #

Source
@(optimization_mode="favor_size")
XXH3_mix2Accs :: proc(acc: []u64, secret: []u8) -> (res: u64) {…}

XXH3_rrmxmx #

Source
@(optimization_mode="favor_size")
XXH3_rrmxmx :: proc(h64, length: u64) -> (res: u64) {…}

This is a stronger avalanche, inspired by Pelle Evensen's rrmxmx preferable when input has not been previously mixed

XXH3_scramble_accumulator #

Source
XXH3_scramble_accumulator :: proc(acc: []u64, secret: []u8) {…}

XXH3_scramble_accumulator_scalar #

Source
@(optimization_mode="favor_size")
XXH3_scramble_accumulator_scalar :: proc(acc: []u64, secret: []u8) {…}

XXH3_scramble_accumulator_simd_generic #

Source
XXH3_scramble_accumulator_simd_generic :: proc(acc: []u64, secret: []u8, $W: uint) {…}

XXH32_avalanche #

Source
@(optimization_mode="favor_size")
XXH32_avalanche :: proc(h32: u32) -> (res: u32) {…}

Mix all bits

XXH32_canonical_from_hash #

Source
XXH32_canonical_from_hash :: proc(hash: u32) -> (canonical: XXH32_canonical) {…}

****** Canonical representation ****** The default return values from XXH functions are unsigned 32 and 64 bit integers. The canonical representation uses big endian convention, the same convention as human-readable numbers (large digits first). This way, hash values can be written into a file or buffer, remaining comparable across different systems. The following functions allow transformation of hash values to and from their canonical format.

XXH32_create_state #

Source
XXH32_create_state :: proc(allocator := context.allocator) -> (res: ^XXH32_state, err: Error) {…}

****** Hash streaming ******

XXH32_endian_align #

Source
@(optimization_mode="favor_size")
XXH32_endian_align :: proc(input: []u8, seed: u32 = XXH32_DEFAULT_SEED, alignment: Alignment) -> (res: u32) {…}

XXH32_read32 #

Source
@(optimization_mode="favor_size")
XXH32_read32 :: proc(buf: []u8, alignment: Alignment = Alignment.Unaligned) -> (res: u32) {…}

XXH32_round #

Source
@(optimization_mode="favor_size")
XXH32_round :: proc(seed, input: u32) -> (res: u32) {…}

XXH64_avalanche #

Source
@(optimization_mode="favor_size")
XXH64_avalanche :: proc(h64: u64) -> (res: u64) {…}

XXH64_canonical_from_hash #

Source
XXH64_canonical_from_hash :: proc(hash: u64) -> (canonical: XXH64_canonical) {…}

****** Canonical representation ****** The default return values from XXH functions are unsigned 32 and 64 bit integers. The canonical representation uses big endian convention, the same convention as human-readable numbers (large digits first). This way, hash values can be written into a file or buffer, remaining comparable across different systems. The following functions allow transformation of hash values to and from their canonical format.

XXH64_create_state #

Source
XXH64_create_state :: proc(allocator := context.allocator) -> (res: ^XXH64_state, err: Error) {…}

****** Hash Streaming ******

XXH64_endian_align #

Source
@(optimization_mode="favor_size")
XXH64_endian_align :: proc(input: []u8, seed: u64 = XXH64_DEFAULT_SEED, alignment: Alignment = Alignment.Unaligned) -> (res: u64) {…}

XXH64_mergeRound #

Source
@(optimization_mode="favor_size")
XXH64_mergeRound :: proc(acc, val: u64) -> (res: u64) {…}

XXH64_read64 #

Source
@(optimization_mode="favor_size")
XXH64_read64 :: proc(buf: []u8, alignment: Alignment = Alignment.Unaligned) -> (res: u64) {…}

XXH64_round #

Source
@(optimization_mode="favor_size")
XXH64_round :: proc(acc, input: u64) -> (res: u64) {…}

XXH64_write64_simd #

Source
XXH64_write64_simd :: proc(buf: []$E, value: $V/#simd[0]u64, alignment: Alignment = Alignment.Unaligned) {…}

Procedure Groups

2

Variables

1

XXH3_kSecret #

Source
XXH3_kSecret: [192]u8 = [XXH_SECRET_DEFAULT_SIZE]u8{0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e}