binius_field/arch/x86_64/
packed_ghash_128.rs

1// Copyright 2024-2025 Irreducible Inc.
2
3//! PCLMULQDQ-accelerated implementation of GHASH for x86_64.
4//!
5//! This module provides optimized GHASH multiplication using the PCLMULQDQ instruction
6//! available on modern x86_64 processors. The implementation follows the algorithm
7//! described in the GHASH specification with polynomial x^128 + x^7 + x^2 + x + 1.
8
9use std::ops::Mul;
10
11use cfg_if::cfg_if;
12
13use super::{super::portable::packed::PackedPrimitiveType, m128::M128};
14use crate::{
15	BinaryField128bGhash,
16	arch::portable::packed_macros::impl_serialize_deserialize_for_packed_binary_field,
17	arithmetic_traits::{InvertOrZero, Square},
18	packed::PackedField,
19};
20
21#[cfg(target_feature = "pclmulqdq")]
22impl crate::arch::shared::ghash::ClMulUnderlier for M128 {
23	#[inline]
24	fn clmulepi64<const IMM8: i32>(a: Self, b: Self) -> Self {
25		unsafe { std::arch::x86_64::_mm_clmulepi64_si128::<IMM8>(a.into(), b.into()) }.into()
26	}
27
28	#[inline]
29	fn move_64_to_hi(a: Self) -> Self {
30		unsafe { std::arch::x86_64::_mm_slli_si128::<8>(a.into()) }.into()
31	}
32}
33
34pub type PackedBinaryGhash1x128b = PackedPrimitiveType<M128, BinaryField128bGhash>;
35
36// Define multiply
37cfg_if! {
38	if #[cfg(target_feature = "pclmulqdq")] {
39		impl Mul for PackedBinaryGhash1x128b {
40			type Output = Self;
41
42			#[inline]
43			fn mul(self, rhs: Self) -> Self::Output {
44				crate::tracing::trace_multiplication!(PackedBinaryGhash1x128b);
45
46				Self::from_underlier(crate::arch::shared::ghash::mul_clmul(
47					self.to_underlier(),
48					rhs.to_underlier(),
49				))
50			}
51		}
52
53		impl Square for PackedBinaryGhash1x128b {
54			#[inline]
55			fn square(self) -> Self {
56				Self::from_underlier(crate::arch::shared::ghash::square_clmul(
57					self.to_underlier(),
58				))
59			}
60		}
61
62	} else {
63		impl Mul for PackedBinaryGhash1x128b {
64			type Output = Self;
65
66			#[inline]
67			fn mul(self, rhs: Self) -> Self::Output {
68				use super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b as PortablePackedBinaryGhash1x128b;
69
70				crate::tracing::trace_multiplication!(PackedBinaryGhash1x128b);
71
72				let portable_lhs = PortablePackedBinaryGhash1x128b::from(u128::from(self.to_underlier()));
73				let portable_rhs = PortablePackedBinaryGhash1x128b::from(u128::from(rhs.to_underlier()));
74
75				Self::from_underlier(Mul::mul(portable_lhs, portable_rhs).to_underlier().into())
76			}
77		}
78
79		impl Square for PackedBinaryGhash1x128b {
80			#[inline]
81			fn square(self) -> Self {
82				use super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b as PortablePackedBinaryGhash1x128b;
83
84				let portable_val = PortablePackedBinaryGhash1x128b::from(u128::from(self.to_underlier()));
85
86				Self::from_underlier(Square::square(portable_val).to_underlier().into())
87			}
88		}
89	}
90}
91
92// Define invert
93impl InvertOrZero for PackedBinaryGhash1x128b {
94	fn invert_or_zero(self) -> Self {
95		let portable = super::super::portable::packed_ghash_128::PackedBinaryGhash1x128b::from(
96			u128::from(self.to_underlier()),
97		);
98
99		Self::from_underlier(PackedField::invert_or_zero(portable).to_underlier().into())
100	}
101}
102
103cfg_if! {
104	if #[cfg(target_feature = "gfni")] {
105		use crate::arch::x86_64::gfni::gfni_arithmetics::impl_transformation_with_gfni_nxn;
106		impl_transformation_with_gfni_nxn!(PackedBinaryGhash1x128b, 16);
107	} else {
108		crate::arithmetic_traits::impl_transformation_with_strategy!(
109			PackedBinaryGhash1x128b,
110			crate::arch::SimdStrategy
111		);
112	}
113}
114
115// Define (de)serialize
116impl_serialize_deserialize_for_packed_binary_field!(PackedBinaryGhash1x128b);
binius_field/arch/x86_64/packed_ghash_128.rs

binius_field/arch/x86_64/
packed_ghash_128.rs
