binius_field/arch/x86_64/
packed_ghash_128.rs

1// Copyright 2024-2025 Irreducible Inc.
2
3//! PCLMULQDQ-accelerated implementation of GHASH for x86_64.
4//!
5//! This module provides optimized GHASH multiplication using the PCLMULQDQ instruction
6//! available on modern x86_64 processors. The implementation follows the algorithm
7//! described in the GHASH specification with polynomial x^128 + x^7 + x^2 + x + 1.
8
9use cfg_if::cfg_if;
10
11use super::m128::M128;
12use crate::{
13	BinaryField128bGhash,
14	arch::portable::packed_macros::{portable_macros::*, *},
15	arithmetic_traits::{
16		InvertOrZero, TaggedInvertOrZero, TaggedMul, TaggedSquare, impl_invert_with, impl_mul_with,
17		impl_square_with,
18	},
19};
20
21#[cfg(target_feature = "pclmulqdq")]
22impl crate::arch::shared::ghash::ClMulUnderlier for M128 {
23	#[inline]
24	fn clmulepi64<const IMM8: i32>(a: Self, b: Self) -> Self {
25		unsafe { std::arch::x86_64::_mm_clmulepi64_si128::<IMM8>(a.into(), b.into()) }.into()
26	}
27
28	#[inline]
29	fn move_64_to_hi(a: Self) -> Self {
30		unsafe { std::arch::x86_64::_mm_slli_si128::<8>(a.into()) }.into()
31	}
32}
33
34/// Strategy for x86_64 GHASH field arithmetic operations.
35pub struct GhashStrategy;
36
37// Define PackedBinaryGhash1x128b using the macro
38define_packed_binary_field!(
39	PackedBinaryGhash1x128b,
40	BinaryField128bGhash,
41	M128,
42	(GhashStrategy),
43	(GhashStrategy),
44	(GhashStrategy),
45	(None),
46	(None)
47);
48
49// Implement TaggedMul for GhashStrategy
50cfg_if! {
51	if #[cfg(target_feature = "pclmulqdq")] {
52		impl TaggedMul<GhashStrategy> for PackedBinaryGhash1x128b {
53			#[inline]
54			fn mul(self, rhs: Self) -> Self {
55				Self::from_underlier(crate::arch::shared::ghash::mul_clmul(
56					self.to_underlier(),
57					rhs.to_underlier(),
58				))
59			}
60		}
61	} else {
62		impl TaggedMul<GhashStrategy> for PackedBinaryGhash1x128b {
63			#[inline]
64			fn mul(self, rhs: Self) -> Self {
65				use super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b as PortablePackedBinaryGhash1x128b;
66
67				let portable_lhs = PortablePackedBinaryGhash1x128b::from(u128::from(self.to_underlier()));
68				let portable_rhs = PortablePackedBinaryGhash1x128b::from(u128::from(rhs.to_underlier()));
69
70				Self::from_underlier(std::ops::Mul::mul(portable_lhs, portable_rhs).to_underlier().into())
71			}
72		}
73	}
74}
75
76// Implement TaggedSquare for GhashStrategy
77cfg_if! {
78	if #[cfg(target_feature = "pclmulqdq")] {
79		impl TaggedSquare<GhashStrategy> for PackedBinaryGhash1x128b {
80			#[inline]
81			fn square(self) -> Self {
82				Self::from_underlier(crate::arch::shared::ghash::square_clmul(
83					self.to_underlier(),
84				))
85			}
86		}
87	} else {
88		impl TaggedSquare<GhashStrategy> for PackedBinaryGhash1x128b {
89			#[inline]
90			fn square(self) -> Self {
91				use super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b as PortablePackedBinaryGhash1x128b;
92
93				let portable_val = PortablePackedBinaryGhash1x128b::from(u128::from(self.to_underlier()));
94
95				Self::from_underlier(crate::arithmetic_traits::Square::square(portable_val).to_underlier().into())
96			}
97		}
98	}
99}
100
101// Implement TaggedInvertOrZero for GhashStrategy (always uses portable fallback)
102impl TaggedInvertOrZero<GhashStrategy> for PackedBinaryGhash1x128b {
103	fn invert_or_zero(self) -> Self {
104		let portable = super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b::from(
105			u128::from(self.to_underlier()),
106		);
107
108		Self::from_underlier(InvertOrZero::invert_or_zero(portable).to_underlier().into())
109	}
110}
binius_field/arch/x86_64/packed_ghash_128.rs

binius_field/arch/x86_64/
packed_ghash_128.rs
