binius_field/arch/x86_64/
packed_ghash_128.rs

1// Copyright 2024-2025 Irreducible Inc.
2// Copyright 2026 The Binius Developers
3
4//! PCLMULQDQ-accelerated implementation of GHASH for x86_64.
5//!
6//! This module provides optimized GHASH multiplication using the PCLMULQDQ instruction
7//! available on modern x86_64 processors. The implementation follows the algorithm
8//! described in the GHASH specification with polynomial x^128 + x^7 + x^2 + x + 1.
9
10use super::m128::M128;
11#[cfg(not(target_feature = "pclmulqdq"))]
12use crate::arch::portable::univariate_mul_utils_128::{Underlier128bLanes, spread_bits_64};
13use crate::arch::x86_64::arithmetic::ghash::{self, GhashLanes};
14
15/// Widening-multiply wrapper used by the GHASH packing: the reduction-deferring
16/// `GhashClMulWideMul` when PCLMULQDQ is available, otherwise the portable `GhashWideMul` which
17/// also defers reduction for deferred-reduction sum-of-products.
18#[cfg(target_feature = "pclmulqdq")]
19pub type GhashWideMul1x<T> = ghash::GhashClMulWideMul<T>;
20#[cfg(not(target_feature = "pclmulqdq"))]
21pub type GhashWideMul1x<T> = crate::arch::portable::arithmetic::ghash::GhashWideMul<T>;
22
23/// Square wrapper for the `PackedGhash1x128b` packing: the CLMUL square `GhashClMul` when
24/// PCLMULQDQ is available, otherwise the shared software square `GhashSoftMul`.
25#[cfg(target_feature = "pclmulqdq")]
26pub type GhashSquare1x<T> = ghash::GhashClMul<T>;
27#[cfg(not(target_feature = "pclmulqdq"))]
28pub type GhashSquare1x<T> = crate::arch::portable::arithmetic::ghash::GhashSoftMul<T>;
29
30/// Invert wrapper for the `PackedGhash1x128b` packing: the shared Itoh-Tsujii inversion
31/// (there is no CLMUL inverse).
32pub type GhashInvert1x<T> = crate::arch::portable::arithmetic::itoh_tsujii::GhashItohTsujii<T>;
33
34/// `Underlier128bLanes` for x86_64 `M128` — required for the portable `GhashWideMul`/`GhashSoftMul`
35/// fallbacks.
36///
37/// Delegates through `u128` (SSE2 load/store) since this path is only active on targets without
38/// PCLMULQDQ, where SIMD lane extraction intrinsics are not necessarily available.
39#[cfg(not(target_feature = "pclmulqdq"))]
40impl Underlier128bLanes for M128 {
41	type U64 = u64;
42
43	#[inline(always)]
44	fn split_hi_lo_64(self) -> (u64, u64) {
45		u128::from(self).split_hi_lo_64()
46	}
47
48	#[inline(always)]
49	fn join_u64s(high: u64, low: u64) -> Self {
50		Self::from(u128::join_u64s(high, low))
51	}
52
53	#[inline(always)]
54	fn broadcast_64(val: u64) -> Self {
55		Self::from(u128::broadcast_64(val))
56	}
57
58	#[inline(always)]
59	fn spread_bits_128(self) -> (Self, Self) {
60		let (hi, lo) = self.split_hi_lo_64();
61		(Self::from(spread_bits_64(hi)), Self::from(spread_bits_64(lo)))
62	}
63}
64
65impl GhashLanes for M128 {
66	#[inline]
67	fn move_64_to_hi(a: Self) -> Self {
68		unsafe { std::arch::x86_64::_mm_slli_si128::<8>(a.into()) }.into()
69	}
70
71	#[inline]
72	fn shl_1_epi64(a: Self) -> Self {
73		unsafe { std::arch::x86_64::_mm_slli_epi64::<1>(a.into()) }.into()
74	}
75
76	#[inline]
77	fn shr_63_epi64(a: Self) -> Self {
78		unsafe { std::arch::x86_64::_mm_srli_epi64::<63>(a.into()) }.into()
79	}
80
81	#[inline]
82	fn broadcast_bit_127(a: Self) -> Self {
83		// Bit 127 is the sign bit of the lane's top 32-bit word.
84		// Copying that word over the whole lane, then shifting each word right by 31 as a signed
85		// value, leaves every bit equal to it.
86		unsafe {
87			let top_word = std::arch::x86_64::_mm_shuffle_epi32::<0xff>(a.into());
88			std::arch::x86_64::_mm_srai_epi32::<31>(top_word)
89		}
90		.into()
91	}
92}
93
94/// Scaling wrapper for the `PackedGhash1x128b` packing: the vector sequence, which needs
95/// only SSE2.
96pub type GhashMulX1x<T> = ghash::GhashMulX<T>;
97
98#[cfg(target_feature = "pclmulqdq")]
99impl ghash::ClMulUnderlier for M128 {
100	#[inline]
101	fn clmulepi64<const IMM8: i32>(a: Self, b: Self) -> Self {
102		// Safety: the `pclmulqdq` gate on this impl is exactly what the intrinsic requires.
103		unsafe { std::arch::x86_64::_mm_clmulepi64_si128::<IMM8>(a.into(), b.into()) }.into()
104	}
105}
binius_field/arch/x86_64/packed_ghash_128.rs

binius_field/arch/x86_64/
packed_ghash_128.rs
