binius_field/arch/x86_64/
packed_ghash_128.rs

1// Copyright 2024-2025 Irreducible Inc.
2// Copyright 2026 The Binius Developers
3
4//! PCLMULQDQ-accelerated implementation of GHASH for x86_64.
5//!
6//! This module provides optimized GHASH multiplication using the PCLMULQDQ instruction
7//! available on modern x86_64 processors. The implementation follows the algorithm
8//! described in the GHASH specification with polynomial x^128 + x^7 + x^2 + x + 1.
9
10use cfg_if::cfg_if;
11
12use super::m128::M128;
13use crate::{
14	BinaryField128bGhash,
15	arch::portable::packed_macros::{portable_macros::*, *},
16	arithmetic_traits::{
17		InvertOrZero, TaggedInvertOrZero, TaggedMul, TaggedSquare, impl_invert_with, impl_mul_with,
18		impl_square_with,
19	},
20};
21// Only used by the CLMUL-accelerated `ClMulUnderlier` and `WideMul` impls below.
22#[cfg(target_feature = "pclmulqdq")]
23use crate::{arch::shared::ghash, arithmetic_traits::WideMul};
24
25#[cfg(target_feature = "pclmulqdq")]
26impl ghash::ClMulUnderlier for M128 {
27	#[inline]
28	fn clmulepi64<const IMM8: i32>(a: Self, b: Self) -> Self {
29		unsafe { std::arch::x86_64::_mm_clmulepi64_si128::<IMM8>(a.into(), b.into()) }.into()
30	}
31
32	#[inline]
33	fn move_64_to_hi(a: Self) -> Self {
34		unsafe { std::arch::x86_64::_mm_slli_si128::<8>(a.into()) }.into()
35	}
36}
37
38/// Strategy for x86_64 GHASH field arithmetic operations.
39pub struct GhashStrategy;
40
41// Define PackedBinaryGhash1x128b using the macro
42define_packed_binary_field!(
43	PackedBinaryGhash1x128b,
44	BinaryField128bGhash,
45	M128,
46	(GhashStrategy),
47	(GhashStrategy),
48	(GhashStrategy),
49	(None)
50);
51
52// Implement TaggedMul for GhashStrategy
53cfg_if! {
54	if #[cfg(target_feature = "pclmulqdq")] {
55		impl TaggedMul<GhashStrategy> for PackedBinaryGhash1x128b {
56			#[inline]
57			fn mul(self, rhs: Self) -> Self {
58				Self::from_underlier(crate::arch::shared::ghash::mul_clmul(
59					self.to_underlier(),
60					rhs.to_underlier(),
61				))
62			}
63		}
64	} else {
65		impl TaggedMul<GhashStrategy> for PackedBinaryGhash1x128b {
66			#[inline]
67			fn mul(self, rhs: Self) -> Self {
68				use super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b as PortablePackedBinaryGhash1x128b;
69
70				let portable_lhs = PortablePackedBinaryGhash1x128b::from(u128::from(self.to_underlier()));
71				let portable_rhs = PortablePackedBinaryGhash1x128b::from(u128::from(rhs.to_underlier()));
72
73				Self::from_underlier(std::ops::Mul::mul(portable_lhs, portable_rhs).to_underlier().into())
74			}
75		}
76	}
77}
78
79// Implement TaggedSquare for GhashStrategy
80cfg_if! {
81	if #[cfg(target_feature = "pclmulqdq")] {
82		impl TaggedSquare<GhashStrategy> for PackedBinaryGhash1x128b {
83			#[inline]
84			fn square(self) -> Self {
85				Self::from_underlier(crate::arch::shared::ghash::square_clmul(
86					self.to_underlier(),
87				))
88			}
89		}
90	} else {
91		impl TaggedSquare<GhashStrategy> for PackedBinaryGhash1x128b {
92			#[inline]
93			fn square(self) -> Self {
94				use super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b as PortablePackedBinaryGhash1x128b;
95
96				let portable_val = PortablePackedBinaryGhash1x128b::from(u128::from(self.to_underlier()));
97
98				Self::from_underlier(crate::arithmetic_traits::Square::square(portable_val).to_underlier().into())
99			}
100		}
101	}
102}
103
104// Implement WideMul
105cfg_if! {
106	if #[cfg(target_feature = "pclmulqdq")] {
107		impl WideMul for PackedBinaryGhash1x128b {
108			type Output = ghash::WideGhashProduct<M128>;
109
110			#[inline]
111			fn wide_mul(a: Self, b: Self) -> Self::Output {
112				ghash::WideGhashProduct::wide_mul(a.to_underlier(), b.to_underlier())
113			}
114
115			#[inline]
116			fn reduce(wide: Self::Output) -> Self {
117				Self::from_underlier(wide.reduce())
118			}
119		}
120	} else {
121		crate::arithmetic_traits::impl_trivial_wide_mul!(PackedBinaryGhash1x128b);
122	}
123}
124
125// Implement TaggedInvertOrZero for GhashStrategy (always uses portable fallback)
126impl TaggedInvertOrZero<GhashStrategy> for PackedBinaryGhash1x128b {
127	fn invert_or_zero(self) -> Self {
128		let portable = super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b::from(
129			u128::from(self.to_underlier()),
130		);
131
132		Self::from_underlier(InvertOrZero::invert_or_zero(portable).to_underlier().into())
133	}
134}
binius_field/arch/x86_64/packed_ghash_128.rs

binius_field/arch/x86_64/
packed_ghash_128.rs
