binius_field/arch/x86_64/
packed_ghash_128.rs

1// Copyright 2024-2025 Irreducible Inc.
2
3//! PCLMULQDQ-accelerated implementation of GHASH for x86_64.
4//!
5//! This module provides optimized GHASH multiplication using the PCLMULQDQ instruction
6//! available on modern x86_64 processors. The implementation follows the algorithm
7//! described in the GHASH specification with polynomial x^128 + x^7 + x^2 + x + 1.
8
9use cfg_if::cfg_if;
10
11use super::m128::M128;
12use crate::{
13	BinaryField128bGhash,
14	arch::portable::packed_macros::{portable_macros::*, *},
15	arithmetic_traits::{
16		InvertOrZero, TaggedInvertOrZero, TaggedMul, TaggedSquare, impl_invert_with, impl_mul_with,
17		impl_square_with,
18	},
19};
20
21#[cfg(target_feature = "pclmulqdq")]
22impl crate::arch::shared::ghash::ClMulUnderlier for M128 {
23	#[inline]
24	fn clmulepi64<const IMM8: i32>(a: Self, b: Self) -> Self {
25		unsafe { std::arch::x86_64::_mm_clmulepi64_si128::<IMM8>(a.into(), b.into()) }.into()
26	}
27
28	#[inline]
29	fn move_64_to_hi(a: Self) -> Self {
30		unsafe { std::arch::x86_64::_mm_slli_si128::<8>(a.into()) }.into()
31	}
32}
33
34/// Strategy for x86_64 GHASH field arithmetic operations.
35pub struct GhashStrategy;
36
37// Define PackedBinaryGhash1x128b using the macro
38define_packed_binary_field!(
39	PackedBinaryGhash1x128b,
40	BinaryField128bGhash,
41	M128,
42	(GhashStrategy),
43	(GhashStrategy),
44	(GhashStrategy),
45	(None)
46);
47
48// Implement TaggedMul for GhashStrategy
49cfg_if! {
50	if #[cfg(target_feature = "pclmulqdq")] {
51		impl TaggedMul<GhashStrategy> for PackedBinaryGhash1x128b {
52			#[inline]
53			fn mul(self, rhs: Self) -> Self {
54				Self::from_underlier(crate::arch::shared::ghash::mul_clmul(
55					self.to_underlier(),
56					rhs.to_underlier(),
57				))
58			}
59		}
60	} else {
61		impl TaggedMul<GhashStrategy> for PackedBinaryGhash1x128b {
62			#[inline]
63			fn mul(self, rhs: Self) -> Self {
64				use super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b as PortablePackedBinaryGhash1x128b;
65
66				let portable_lhs = PortablePackedBinaryGhash1x128b::from(u128::from(self.to_underlier()));
67				let portable_rhs = PortablePackedBinaryGhash1x128b::from(u128::from(rhs.to_underlier()));
68
69				Self::from_underlier(std::ops::Mul::mul(portable_lhs, portable_rhs).to_underlier().into())
70			}
71		}
72	}
73}
74
75// Implement TaggedSquare for GhashStrategy
76cfg_if! {
77	if #[cfg(target_feature = "pclmulqdq")] {
78		impl TaggedSquare<GhashStrategy> for PackedBinaryGhash1x128b {
79			#[inline]
80			fn square(self) -> Self {
81				Self::from_underlier(crate::arch::shared::ghash::square_clmul(
82					self.to_underlier(),
83				))
84			}
85		}
86	} else {
87		impl TaggedSquare<GhashStrategy> for PackedBinaryGhash1x128b {
88			#[inline]
89			fn square(self) -> Self {
90				use super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b as PortablePackedBinaryGhash1x128b;
91
92				let portable_val = PortablePackedBinaryGhash1x128b::from(u128::from(self.to_underlier()));
93
94				Self::from_underlier(crate::arithmetic_traits::Square::square(portable_val).to_underlier().into())
95			}
96		}
97	}
98}
99
100// Implement TaggedInvertOrZero for GhashStrategy (always uses portable fallback)
101impl TaggedInvertOrZero<GhashStrategy> for PackedBinaryGhash1x128b {
102	fn invert_or_zero(self) -> Self {
103		let portable = super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b::from(
104			u128::from(self.to_underlier()),
105		);
106
107		Self::from_underlier(InvertOrZero::invert_or_zero(portable).to_underlier().into())
108	}
109}
binius_field/arch/x86_64/packed_ghash_128.rs

binius_field/arch/x86_64/
packed_ghash_128.rs
