Skip to main content

binius_math/field_buffer/
mod.rs

1// Copyright 2025 Irreducible Inc.
2// Copyright 2026 The Binius Developers
3
4//! A power-of-two-sized buffer of packed field elements.
5//!
6//! The buffer lives in this file, with its methods and the guard its halving methods hand out.
7//! `SplitMut` is built in exactly one place, so it needs no visibility beyond this module.
8//!
9//! The rest sits in modules of their own:
10//!
11//! ```text
12//! chunks      the buffer's chunk methods, and the iterators and guard they hand out
13//! structured  a buffer that may hold one aligned block explicitly, and zero elsewhere
14//! view        the borrowed aliases, which name the buffer rather than being produced by it
15//! ```
16//!
17//! # Why the backing store is a type parameter
18//!
19//! A store's length fixes the element count only once the buffer fills a whole packed word.
20//! Below one word, the same single word backs any length:
21//!
22//! ```text
23//! WIDTH = 4 lanes per word
24//!
25//! 1 element   ->  [ x . . . ]   one word
26//! 2 elements  ->  [ x x . . ]   one word
27//! 4 elements  ->  [ x x x x ]   one word
28//! ```
29//!
30//! So the logical length is irreducible extra state, carried alongside the store.
31//! That rules out a transparent wrapper around a packed slice.
32//! With it goes the deref-coercion design a vector and its two slice types enjoy.
33
34use std::{
35	mem::MaybeUninit,
36	ops::{Deref, DerefMut},
37	slice,
38};
39
40use binius_compute::{Allocator, BufferData, VecLike};
41use binius_field::{
42	Field, PackedField,
43	packed::{get_packed_slice_unchecked, set_packed_slice_unchecked},
44};
45use binius_utils::{
46	checked_arithmetics::strict_log_2,
47	rayon::{
48		prelude::*,
49		task_size::{IndexedParallelIteratorExt, task_chunk_len},
50	},
51};
52use bytemuck::zeroed_vec;
53
54mod chunks;
55mod structured;
56mod view;
57
58pub use chunks::{ChunkMut, Chunks, ChunksMut};
59pub use structured::StructuredBuffer;
60pub use view::{FieldSlice, FieldSliceData, FieldSliceMut, FieldVec};
61
62/// A power-of-two-sized buffer containing field elements, stored in packed fields.
63///
64/// The backing store length is fully determined by `log_len`:
65///
66/// ```text
67/// words.len() == 1 << log_len.saturating_sub(P::LOG_WIDTH)
68/// ```
69///
70/// A buffer shorter than one packed word still occupies a whole word.
71/// The lanes at and past the logical length are then not elements of the buffer:
72///
73/// ```text
74/// WIDTH = 4, log_len = 1
75///
76/// word:  [ s_0, s_1, dead, dead ]
77///          ^^^^^^^^  live prefix
78/// ```
79///
80/// Nothing reads a dead lane as an element.
81/// Equality and the halving routines touch the live prefix only.
82///
83/// Truncation additionally zeros the dead lanes it creates.
84/// Other constructors may leave whatever the caller's store held there.
85// The buffer's methods are grouped by topic, one module per group, so its inherent impls span
86// more than one file.
87#[allow(clippy::multiple_inherent_impl)]
88#[derive(Debug, Clone, Eq)]
89pub struct FieldBuffer<P: PackedField, Data: Deref<Target = [P]> = Vec<P>> {
90	/// log2 the number over elements in the buffer.
91	log_len: usize,
92	/// The packed words.
93	words: Data,
94}
95
96impl<P: PackedField, Data: Deref<Target = [P]> + Copy> Copy for FieldBuffer<P, Data> {}
97
98impl<P: PackedField, Data: Deref<Target = [P]>> PartialEq for FieldBuffer<P, Data> {
99	fn eq(&self, other: &Self) -> bool {
100		// Equality compares only the live scalars, never the raw packed backing store.
101		//
102		// Invariant: buffers of different lengths are never equal.
103		//
104		// The length-below-width branch compares only the live prefix of a single packed word,
105		// never the dead lanes past it: shorter and longer buffers sharing that prefix would
106		// otherwise compare equal, and the dead lanes may hold unrelated data.
107		if self.log_len != other.log_len {
108			return false;
109		}
110		if self.log_len < P::LOG_WIDTH {
111			let iter_1 = self
112				.words
113				.first()
114				.expect("len >= 1")
115				.iter()
116				.take(1 << self.log_len);
117			let iter_2 = other
118				.words
119				.first()
120				.expect("len >= 1")
121				.iter()
122				.take(1 << self.log_len);
123			iter_1.eq(iter_2)
124		} else {
125			let prefix = 1 << (self.log_len - P::LOG_WIDTH);
126			self.words[..prefix] == other.words[..prefix]
127		}
128	}
129}
130
131impl<P: PackedField> FieldBuffer<P> {
132	/// Create a new FieldBuffer from a vector of values.
133	///
134	/// # Preconditions
135	///
136	/// * `values.len()` must be a power of two.
137	#[track_caller]
138	pub fn from_values(values: &[P::Scalar]) -> Self {
139		let log_len =
140			strict_log_2(values.len()).expect("precondition: values.len() must be a power of two");
141
142		let packed_len = 1 << log_len.saturating_sub(P::LOG_WIDTH);
143		let mut words = Vec::with_capacity(packed_len);
144		words.extend(
145			values
146				.chunks(P::WIDTH)
147				.map(|chunk| P::from_scalars(chunk.iter().copied())),
148		);
149
150		Self { log_len, words }
151	}
152
153	/// Builds a buffer of `2^log_len` zeros.
154	pub fn zeros(log_len: usize) -> Self {
155		let packed_len = 1 << log_len.saturating_sub(P::LOG_WIDTH);
156		let words = zeroed_vec(packed_len);
157		Self { log_len, words }
158	}
159
160	/// Builds a one-element buffer holding `value`, with room reserved to grow.
161	///
162	/// The store is sized for `2^log_capacity` elements up front.
163	/// Growing the buffer to that many therefore never reallocates.
164	pub fn scalar_with_capacity(value: P::Scalar, log_capacity: usize) -> Self {
165		let mut words = Vec::with_capacity(1 << log_capacity.saturating_sub(P::LOG_WIDTH));
166		words.push(P::from_scalars([value]));
167		Self { log_len: 0, words }
168	}
169}
170
171impl<P: PackedField, Data: VecLike<P>> FieldBuffer<P, Data> {
172	/// Builds a zeroed buffer of `2^log_len` elements, backed by memory drawn from `alloc`.
173	///
174	/// The allocator-aware counterpart to the plain zeroed constructor.
175	/// Under a pool the result is a recyclable buffer rather than a fresh allocation.
176	pub fn zeros_in<A>(alloc: &A, log_len: usize) -> Self
177	where
178		A: Allocator<Vec<P> = Data>,
179	{
180		let packed_len = 1 << log_len.saturating_sub(P::LOG_WIDTH);
181		// An allocator hands out an empty buffer, so the words are both created and zeroed here.
182		let mut words = alloc.alloc::<P>(packed_len);
183		words.resize(packed_len, P::default());
184		FieldBuffer::new(log_len, words)
185	}
186
187	/// Copies a borrowed buffer into memory drawn from `alloc`.
188	///
189	/// Whole packed words are copied, dead lanes and all, so the copy is bit-identical.
190	/// A long source spreads across workers.
191	pub fn from_view_in<A>(alloc: &A, src: FieldSlice<'_, P>) -> Self
192	where
193		A: Allocator<Vec<P> = Data>,
194	{
195		Self::from_view_with_capacity_in(alloc, src, src.log_len())
196	}
197
198	/// Copies a borrowed buffer into memory drawn from `alloc`, with room reserved to grow.
199	///
200	/// The buffer spans `src.log_len()` elements and its store is sized for `2^log_capacity`.
201	/// Growing it to that many elements therefore never reallocates.
202	/// [`Self::repeat_extend`] is the growth this reserves for.
203	///
204	/// Whole packed words are copied, dead lanes and all, so the copy is bit-identical.
205	/// A long source spreads across workers.
206	///
207	/// # Panics
208	///
209	/// Panics if `log_capacity` is shorter than what `src` spans.
210	#[track_caller]
211	pub fn from_view_with_capacity_in<A>(
212		alloc: &A,
213		src: FieldSlice<'_, P>,
214		log_capacity: usize,
215	) -> Self
216	where
217		A: Allocator<Vec<P> = Data>,
218	{
219		assert!(
220			log_capacity >= src.log_len(),
221			"precondition: log_capacity must be at least src.log_len()"
222		);
223
224		let mut words = alloc.alloc::<P>(1 << log_capacity.saturating_sub(P::LOG_WIDTH));
225
226		// The allocator rounds its blocks up, so the fill is bounded to the words that are live.
227		let source = src.as_ref();
228		let head = &mut words.spare_capacity_mut()[..source.len()];
229
230		// The floor holds a worker's share at one task's byte budget, so a short source stays put.
231		(head.par_iter_mut(), source.par_iter())
232			.into_par_iter()
233			.with_min_task_bytes::<P>()
234			.for_each(|(dst, src)| {
235				dst.write(*src);
236			});
237
238		// SAFETY: the loop above wrote every word of the source.
239		unsafe { words.set_len(source.len()) };
240
241		FieldBuffer::new(src.log_len(), words)
242	}
243
244	/// Grows the buffer to span `2^log_len` elements, repeating what it already holds.
245	///
246	/// ```text
247	///     [x]  ->  [x | x | x | x]
248	/// ```
249	///
250	/// This is the buffer's [`Vec::extend_from_within`], split across workers.
251	/// Every copy is drawn from the live prefix, so one pass fills the whole buffer.
252	///
253	/// Returns the buffer untouched when it already spans that many elements.
254	///
255	/// # Panics
256	///
257	/// Panics if `log_len` is shorter than what the buffer already spans.
258	/// Panics if the store has no room reserved for `2^log_len` elements.
259	#[track_caller]
260	pub fn repeat_extend(&mut self, log_len: usize) {
261		assert!(
262			log_len >= self.log_len,
263			"precondition: log_len must be at least the buffer's own log_len"
264		);
265		if log_len == self.log_len {
266			return;
267		}
268
269		let total = 1 << log_len.saturating_sub(P::LOG_WIDTH);
270		assert!(
271			total <= self.words.capacity(),
272			"precondition: the store must have room reserved for 2^log_len elements"
273		);
274
275		if self.log_len < P::LOG_WIDTH {
276			// The live elements share one word, so repeating them cycles lanes, not words.
277			let word = P::from_scalars(self.iter_scalars().cycle());
278			self.words.clear();
279			self.words.resize(total, word);
280		} else {
281			let prefix = self.words.len();
282
283			// A run is the words one worker copies at a time.
284			// It is a power of two at most the prefix, so it divides the prefix evenly.
285			let run = prefix.min(task_chunk_len::<P>().next_power_of_two());
286
287			// One borrow has to span both the prefix the copies read and the room they fill.
288			// Emptying the buffer first is what puts the whole store behind that single borrow.
289			//
290			// SAFETY: a packed field has no destructor, so forgetting the live words is a no-op.
291			unsafe { self.words.set_len(0) };
292			let store = &mut self.words.spare_capacity_mut()[..total];
293			let (head, tail) = store.split_at_mut(prefix);
294
295			// SAFETY: the split point is the length the buffer just had.
296			// So every word of `head` was written before this call.
297			let head = unsafe { &*(head as *const [MaybeUninit<P>] as *const [P]) };
298
299			repeat_words(head, tail, run);
300
301			// SAFETY: the call above wrote every word between the prefix and `total`.
302			unsafe { self.words.set_len(total) };
303		}
304
305		self.log_len = log_len;
306	}
307
308	/// Builds a buffer from scalar values, directly into memory drawn from `alloc`.
309	///
310	/// The allocator-aware counterpart to building from a scalar slice.
311	/// Packing happens straight into the allocator's buffer, so no intermediate vector is copied.
312	/// Under a pool the result is a recyclable buffer.
313	///
314	/// # Preconditions
315	///
316	/// * `values.len()` must be a power of two.
317	#[track_caller]
318	pub fn from_values_in<A>(alloc: &A, values: &[P::Scalar]) -> Self
319	where
320		A: Allocator<Vec<P> = Data>,
321	{
322		let log_len =
323			strict_log_2(values.len()).expect("precondition: values.len() must be a power of two");
324
325		let packed_len = 1 << log_len.saturating_sub(P::LOG_WIDTH);
326		let mut words = alloc.alloc::<P>(packed_len);
327		words.extend(
328			values
329				.chunks(P::WIDTH)
330				.map(|chunk| P::from_scalars(chunk.iter().copied())),
331		);
332
333		FieldBuffer::new(log_len, words)
334	}
335
336	/// Grows the buffer to span `log_len` variables, padding the new positions with zeros.
337	///
338	/// Returns the buffer untouched when it already spans that many.
339	/// Padding the shorter of two buffers to match the longer hits that case when both are equal.
340	///
341	/// New memory comes from `alloc`.
342	/// A pooled buffer therefore stays pooled, instead of escaping the pool on a reallocation.
343	///
344	/// # Panics
345	///
346	/// Panics if `log_len` is shorter than what the buffer already spans.
347	/// Shrinking is a separate operation.
348	/// Silently returning a narrower buffer than asked for would hide the mistake.
349	#[track_caller]
350	pub fn zero_extend_in<A>(self, alloc: &A, log_len: usize) -> Self
351	where
352		A: Allocator<Vec<P> = Data>,
353	{
354		assert!(
355			log_len >= self.log_len,
356			"precondition: log_len must be at least the buffer's own log_len"
357		);
358		// Nothing to pad, so the caller's buffer is handed straight back with no copy.
359		if log_len == self.log_len {
360			return self;
361		}
362
363		// The target starts fully zeroed, so only the occupied words need writing.
364		let mut extended = Self::zeros_in(alloc, log_len);
365
366		if self.log_len < P::LOG_WIDTH {
367			// The source's single word also spans lanes that the padding covers.
368			// Nothing sitting there is an element of it, so only the live scalars carry over.
369			//
370			//     source   [ s_0, s_1, x, y ]
371			//     result   [ s_0, s_1, 0, 0 ] [ 0, 0, 0, 0 ]
372			extended.as_mut()[0] = P::from_scalars(self.iter_scalars());
373		} else {
374			// Every lane of every source word is live, so whole words move across untouched.
375			extended.as_mut()[..self.as_ref().len()].copy_from_slice(self.as_ref());
376		}
377
378		extended
379	}
380}
381
382#[allow(clippy::len_without_is_empty)]
383impl<P: PackedField, Data: Deref<Target = [P]>> FieldBuffer<P, Data> {
384	/// Create a new FieldBuffer from a slice of packed words.
385	///
386	/// # Preconditions
387	///
388	/// * `words.len()` must equal the expected packed length for `log_len`.
389	#[track_caller]
390	pub fn new(log_len: usize, words: Data) -> Self {
391		let expected_packed_len = 1 << log_len.saturating_sub(P::LOG_WIDTH);
392		assert!(
393			words.len() == expected_packed_len,
394			"precondition: words.len() must equal expected packed length"
395		);
396
397		Self { log_len, words }
398	}
399
400	/// Consumes the buffer and returns its backing data store.
401	pub fn into_inner(self) -> Data {
402		self.words
403	}
404
405	/// Returns log2 the number of field elements.
406	pub const fn log_len(&self) -> usize {
407		self.log_len
408	}
409
410	/// Returns the number of field elements.
411	pub const fn len(&self) -> usize {
412		1 << self.log_len
413	}
414
415	/// Borrows the whole buffer as a shared view.
416	pub fn as_view(&self) -> FieldSlice<'_, P> {
417		FieldSlice::from_slice(self.log_len, self.as_ref())
418	}
419
420	/// Get a field element at the given index.
421	///
422	/// # Preconditions
423	///
424	/// * the index is in the range `0..self.len()`
425	#[track_caller]
426	pub fn get(&self, index: usize) -> P::Scalar {
427		assert!(
428			index < self.len(),
429			"precondition: index {index} must be less than len {}",
430			self.len()
431		);
432
433		// Safety: bound check on index performed above. The buffer length is at least
434		// `self.len() >> P::LOG_WIDTH` by struct invariant.
435		unsafe { get_packed_slice_unchecked(&self.words, index) }
436	}
437
438	/// Returns an iterator over the scalar elements in the buffer.
439	pub fn iter_scalars(&self) -> impl Iterator<Item = P::Scalar> + Send + Clone + '_ {
440		P::iter_slice(self.as_ref()).take(self.len())
441	}
442
443	/// Returns an iterator over the packed words the elements occupy.
444	///
445	/// The run covers the live words only, never a store's spare room past them.
446	/// A buffer shorter than one packed word yields that one word whole, dead lanes and all.
447	#[inline]
448	pub fn iter_packed(&self) -> slice::Iter<'_, P> {
449		self.as_ref().iter()
450	}
451
452	/// Splits the buffer in half and returns a pair of borrowed slices.
453	///
454	/// # Preconditions
455	///
456	/// * `self.log_len()` must be greater than 0.
457	#[track_caller]
458	pub fn split_half(&self) -> (FieldSlice<'_, P>, FieldSlice<'_, P>) {
459		assert!(self.log_len > 0, "precondition: cannot split a buffer of length 1");
460
461		let new_log_len = self.log_len - 1;
462		if new_log_len < P::LOG_WIDTH {
463			// The result will be two Single variants
464			// We have exactly one packed element that needs to be split
465			let packed = self.words[0];
466			let zeros = P::default();
467
468			let (first_half, second_half) = packed.interleave(zeros, new_log_len);
469
470			let first = FieldBuffer {
471				log_len: new_log_len,
472				words: FieldSliceData::Single(first_half),
473			};
474			let second = FieldBuffer {
475				log_len: new_log_len,
476				words: FieldSliceData::Single(second_half),
477			};
478
479			(first, second)
480		} else {
481			// Split the packed word slice in half
482			let half_len = 1 << (new_log_len - P::LOG_WIDTH);
483			let (first_half, second_half) = self.words.split_at(half_len);
484			let second_half = &second_half[..half_len];
485
486			let first = FieldBuffer {
487				log_len: new_log_len,
488				words: FieldSliceData::Slice(first_half),
489			};
490			let second = FieldBuffer {
491				log_len: new_log_len,
492				words: FieldSliceData::Slice(second_half),
493			};
494
495			(first, second)
496		}
497	}
498}
499
500impl<P: PackedField, Data: DerefMut<Target = [P]>> FieldBuffer<P, Data> {
501	/// Borrows the whole buffer as a mutable view.
502	pub fn as_mut_view(&mut self) -> FieldSliceMut<'_, P> {
503		FieldSliceMut::from_slice(self.log_len, self.as_mut())
504	}
505
506	/// Set a field element at the given index.
507	///
508	/// # Preconditions
509	///
510	/// * the index is in the range `0..self.len()`
511	#[track_caller]
512	pub fn set(&mut self, index: usize, value: P::Scalar) {
513		assert!(
514			index < self.len(),
515			"precondition: index {index} must be less than len {}",
516			self.len()
517		);
518
519		// Safety: bound check on index performed above. The buffer length is at least
520		// `self.len() >> P::LOG_WIDTH` by struct invariant.
521		unsafe { set_packed_slice_unchecked(&mut self.words, index, value) };
522	}
523
524	/// Returns a mutable iterator over the packed words the elements occupy.
525	///
526	/// The mutable counterpart of the shared word iterator, covering the same run of words.
527	/// A final word below the packing width is lent out whole, dead lanes included.
528	#[inline]
529	pub fn iter_packed_mut(&mut self) -> slice::IterMut<'_, P> {
530		self.as_mut().iter_mut()
531	}
532
533	/// Consumes the buffer and halves it, returning a guard that owns the store.
534	///
535	/// The guard lends out the two halves mutably.
536	///
537	/// When both halves live inside one packed word, the guard holds detached copies of them.
538	/// Those are merged back into the original word when it drops.
539	///
540	/// # Preconditions
541	///
542	/// * `self.log_len()` must be greater than 0.
543	#[track_caller]
544	pub fn into_split_half(self) -> SplitMut<P, Data> {
545		assert!(self.log_len > 0, "precondition: cannot split a buffer of length 1");
546
547		// Each half holds one variable fewer than the buffer, and the guard decides from that
548		// alone whether the halves have to be detached from the store.
549		SplitMut::new(self.log_len - 1, self.words)
550	}
551
552	/// Halves the buffer through a mutable borrow, rather than consuming it.
553	///
554	/// Equivalent to borrowing the buffer as a mutable view and halving that.
555	///
556	/// # Preconditions
557	///
558	/// * `self.log_len()` must be greater than 0.
559	#[track_caller]
560	pub fn split_half_mut(&mut self) -> SplitMut<P, &'_ mut [P]> {
561		self.as_mut_view().into_split_half()
562	}
563}
564
565impl<P: PackedField, Data: BufferData<P>> FieldBuffer<P, Data> {
566	/// Truncates the buffer to a shorter length, shrinking the backing store to match.
567	///
568	/// Asking for a length at or above the current one does nothing.
569	///
570	/// A result shorter than one packed word has the dead lanes of its final word zeroed.
571	/// That upholds the invariant that lanes past the logical length are zero.
572	pub fn truncate(&mut self, new_log_len: usize) {
573		if new_log_len >= self.log_len {
574			return;
575		}
576		self.log_len = new_log_len;
577
578		// Zero the lanes past the new logical length in the final word, so a sub-packing-width
579		// result carries no stale scalars. Wider results drop whole words and need no masking.
580		if new_log_len < P::LOG_WIDTH {
581			for i in 1 << new_log_len..P::WIDTH {
582				self.words[0].set(i, <P::Scalar as Field>::ZERO);
583			}
584		}
585
586		self.words
587			.truncate(1 << new_log_len.saturating_sub(P::LOG_WIDTH));
588	}
589}
590
591impl<P: PackedField, Data: Deref<Target = [P]>> AsRef<[P]> for FieldBuffer<P, Data> {
592	#[inline]
593	fn as_ref(&self) -> &[P] {
594		&self.words[..1 << self.log_len.saturating_sub(P::LOG_WIDTH)]
595	}
596}
597
598impl<P: PackedField, Data: DerefMut<Target = [P]>> AsMut<[P]> for FieldBuffer<P, Data> {
599	#[inline]
600	fn as_mut(&mut self) -> &mut [P] {
601		&mut self.words[..1 << self.log_len.saturating_sub(P::LOG_WIDTH)]
602	}
603}
604
605impl<P: PackedField> FromIterator<P::Scalar> for FieldBuffer<P> {
606	/// Builds a buffer over a fresh vector, packing the elements as they arrive.
607	///
608	/// # Panics
609	///
610	/// Panics unless the number of elements is a power of two.
611	/// An empty iterator panics too, matching what building from an empty scalar slice does.
612	/// The count is known only once the iterator runs dry, so a bad one can only panic.
613	#[track_caller]
614	fn from_iter<I: IntoIterator<Item = P::Scalar>>(iter: I) -> Self {
615		let mut iter = iter.into_iter();
616		// The lower bound is the whole count whenever the iterator knows its own length.
617		let mut words = Vec::with_capacity(iter.size_hint().0.div_ceil(P::WIDTH));
618
619		// The length check needs the total, which only the elements themselves can give.
620		let mut len = 0usize;
621		loop {
622			// Fill one word from at most one packing width of elements.
623			// That bound is a constant, so the lanes are written at constant indices rather
624			// than at a running offset into the stream.
625			let mut filled = 0usize;
626			let word = P::from_scalars(iter.by_ref().take(P::WIDTH).inspect(|_| filled += 1));
627
628			// A dry iterator yields an all-zero word belonging to no element.
629			if filled == 0 {
630				break;
631			}
632			words.push(word);
633			len += filled;
634
635			// A short group can only be the last, and its unwritten lanes are already zero,
636			// which is what the invariant on lanes past the logical length asks for.
637			if filled < P::WIDTH {
638				break;
639			}
640		}
641
642		let log_len =
643			strict_log_2(len).expect("precondition: element count must be a power of two");
644
645		Self { log_len, words }
646	}
647}
648
649/// Fills `tail` with copies of `head`, `run` words at a time.
650///
651/// ```text
652///     head            tail
653///     [x]      ->     [x | x | x]
654/// ```
655///
656/// Every destination run draws from one contiguous run of `head`.
657/// So a worker copying one run never reads outside it.
658///
659/// # Preconditions
660///
661/// * `head.len()` is a power of two, and `run` is a power of two at most that
662/// * `tail.len()` is a multiple of `head.len()`
663fn repeat_words<P: PackedField>(head: &[P], tail: &mut [MaybeUninit<P>], run: usize) {
664	debug_assert!(head.len().is_power_of_two());
665	debug_assert!(run.is_power_of_two() && run <= head.len());
666	debug_assert_eq!(tail.len() % head.len(), 0);
667
668	tail.par_chunks_mut(run).enumerate().for_each(|(i, dst)| {
669		// Run `i` of the tail sits one whole head past run `i` of the buffer.
670		// The head length is a power of two, so masking that position gives its source.
671		let source = (i * run) & (head.len() - 1);
672		dst.write_copy_of_slice(&head[source..source + dst.len()]);
673	});
674}
675
676/// Guards the two halves of a buffer that was split along its highest variable.
677///
678/// Holds the parent store, and lends each half out as a mutable view.
679#[derive(Debug)]
680pub struct SplitMut<P: PackedField, Data: DerefMut<Target = [P]>> {
681	/// Element count of each half, as a base-2 logarithm.
682	log_len: usize,
683	/// The detached halves, present only when a half is narrower than a packed word.
684	singles: Option<[P; 2]>,
685	/// The store both halves come from, and the one a detached pair merges back into.
686	data: Data,
687}
688
689impl<P: PackedField, Data: DerefMut<Target = [P]>> SplitMut<P, Data> {
690	/// Guards the two halves of `data`, each holding `2^log_len` elements.
691	fn new(log_len: usize, data: Data) -> Self {
692		// Halves this narrow share word 0, so neither can be lent out as a slice of that store.
693		// Interleaving against zero lifts each half into a word of its own, starting at lane 0.
694		let singles = (log_len < P::LOG_WIDTH).then(|| {
695			let (lo_half, hi_half) = data[0].interleave(P::default(), log_len);
696			[lo_half, hi_half]
697		});
698
699		Self {
700			log_len,
701			singles,
702			data,
703		}
704	}
705
706	/// Lends the two halves out as mutable views, the low half first.
707	///
708	/// A half of whole words is a view straight onto the store, so edits land at once.
709	/// A narrower half is a view onto a detached word, so edits land when this guard drops.
710	pub fn halves(&mut self) -> (FieldSliceMut<'_, P>, FieldSliceMut<'_, P>) {
711		match &mut self.singles {
712			Some([lo_half, hi_half]) => (
713				FieldBuffer {
714					log_len: self.log_len,
715					words: slice::from_mut(lo_half),
716				},
717				FieldBuffer {
718					log_len: self.log_len,
719					words: slice::from_mut(hi_half),
720				},
721			),
722			None => {
723				let half_len = 1 << (self.log_len - P::LOG_WIDTH);
724				let (lo_half, hi_half) = self.data.split_at_mut(half_len);
725				(
726					FieldBuffer {
727						log_len: self.log_len,
728						words: lo_half,
729					},
730					FieldBuffer {
731						log_len: self.log_len,
732						words: hi_half,
733					},
734				)
735			}
736		}
737	}
738}
739
740impl<P: PackedField, Data: DerefMut<Target = [P]>> Drop for SplitMut<P, Data> {
741	fn drop(&mut self) {
742		// Detached halves are the only shape with anything to write back.
743		if let Some([lo_half, hi_half]) = self.singles {
744			(self.data[0], _) = lo_half.interleave(hi_half, self.log_len);
745		}
746	}
747}
748
749#[cfg(test)]
750mod tests {
751	use binius_compute::{BufferPool, GlobalAllocator};
752	use binius_field::packed::get_packed_slice;
753	use binius_utils::rayon::task_size::min_len_for_bytes;
754	use proptest::prelude::*;
755	use rand::{SeedableRng, rngs::StdRng};
756
757	use super::*;
758	use crate::test_utils::{B128, Packed128b, random_field_buffer};
759
760	type P = Packed128b;
761	type F = B128;
762
763	/// Runs every allocator-backed constructor and the zero-padding grow against one allocator.
764	///
765	/// Both a plain heap allocator and a recycling pool must satisfy the same contract.
766	fn check_alloc_constructors<A: Allocator>(alloc: &A) {
767		// Fixture state: a packed word holds 4 lanes, so lengths map to backing words as
768		//
769		//     16 scalars -> log_len 4 -> 4 packed words, every lane live
770		//      2 scalars -> log_len 1 -> 1 packed word,  2 lanes live and 2 dead
771		let scalars: Vec<F> = (0..16).map(F::new).collect();
772		let src = FieldBuffer::<P>::from_values(&scalars);
773
774		// A copy drawn from the allocator carries over both the length and the scalars.
775		let cloned: FieldVec<P, A> = FieldBuffer::from_view_in(alloc, src.as_view());
776		assert_eq!(cloned.log_len(), 4);
777		assert_eq!(cloned.as_view(), src.as_view());
778
779		// Copying a source shorter than one packed word keeps the two live lanes, not four.
780		//
781		// Invariant: a lane past the logical length is not an element, whatever it holds.
782		//
783		// Fixture state: two live scalars, and two dead lanes carrying unrelated values.
784		//
785		//     word   [ 0, 1 | 0xdead, 0xbeef ]
786		//              ^^^^  live prefix
787		//
788		// Packing from scalars would zero those lanes, so the word is supplied whole instead.
789		let small = FieldBuffer::<P>::new(
790			1,
791			vec![P::from_scalars([
792				scalars[0],
793				scalars[1],
794				F::new(0xdead),
795				F::new(0xbeef),
796			])],
797		);
798		let cloned_small: FieldVec<P, A> = FieldBuffer::from_view_in(alloc, small.as_view());
799		assert_eq!(cloned_small.log_len(), 1);
800		assert_eq!(cloned_small.as_view(), small.as_view());
801
802		// 32 elements requested, 32 zeros readable: the buffer arrives at full length, not empty.
803		let mut zeros: FieldVec<P, A> = FieldBuffer::zeros_in(alloc, 5);
804		assert_eq!(zeros.log_len(), 5);
805		assert!(zeros.iter_scalars().all(|scalar| scalar == F::ZERO));
806
807		// Index 31 is the last live element, so writing it stays inside the allocated words.
808		zeros.set(31, F::new(7));
809		assert_eq!(zeros.get(31), F::new(7));
810
811		// Two elements still need the one word that spans them, rounded up from half a word.
812		let zeros_small: FieldVec<P, A> = FieldBuffer::zeros_in(alloc, 1);
813		assert_eq!(zeros_small.as_ref().len(), 1);
814		assert!(zeros_small.iter_scalars().all(|scalar| scalar == F::ZERO));
815
816		// Padding to a wider length keeps the live scalars and zeros every position above them.
817		//
818		//     source (log_len 4)    [0 .. 16]
819		//     result (log_len 5)    [0 .. 16] unchanged, [16 .. 32] zero
820		//
821		// Under a pool the target may sit on memory a prior buffer dirtied.
822		// So the zeros above the copied words have to be written, never assumed.
823		let src_vec: FieldVec<P, A> = FieldBuffer::from_view_in(alloc, src.as_view());
824		let extended = src_vec.zero_extend_in(alloc, 5);
825		assert_eq!(extended.log_len(), 5);
826		for i in 0..16 {
827			assert_eq!(extended.get(i), F::new(i as u128));
828		}
829		assert!((16..32).all(|i| extended.get(i) == F::ZERO));
830
831		// An already-wide-enough buffer comes back unchanged, with no copy taken.
832		let same: FieldVec<P, A> = FieldBuffer::from_view_in(alloc, src.as_view());
833		let same = same.zero_extend_in(alloc, 4);
834		assert_eq!(same.log_len(), 4);
835		assert_eq!(same.as_view(), src.as_view());
836
837		// A source narrower than one packed word pads out of that word's dead lanes.
838		//
839		//     source (log_len 1)    [0, 1 | dead, dead]
840		//     result (log_len 3)    [0, 1 | 0, 0] [0, 0 | 0, 0]
841		//
842		// The two dead lanes become live, so they read back as zeros, not stale scalars.
843		let small_vec: FieldVec<P, A> = FieldBuffer::from_view_in(alloc, small.as_view());
844		let widened = small_vec.zero_extend_in(alloc, 3);
845		assert_eq!(widened.log_len(), 3);
846		assert_eq!(widened.get(0), F::new(0));
847		assert_eq!(widened.get(1), F::new(1));
848		assert!((2..8).all(|i| widened.get(i) == F::ZERO));
849
850		// Reserving room copies the source and leaves the buffer at the source's own length.
851		let reserved: FieldVec<P, A> =
852			FieldBuffer::from_view_with_capacity_in(alloc, src.as_view(), 6);
853		assert_eq!(reserved.log_len(), 4);
854		assert_eq!(reserved.as_view(), src.as_view());
855
856		// Growing into that reserved room repeats the source, four copies over.
857		let mut repeated = reserved;
858		repeated.repeat_extend(6);
859		assert_eq!(repeated.log_len(), 6);
860		assert!((0..64).all(|i| repeated.get(i) == F::new((i % 16) as u128)));
861
862		// A source narrower than one packed word repeats its live lanes, never its dead ones.
863		//
864		//     source (log_len 1)    [0, 1 | dead, dead]
865		//     result (log_len 3)    [0, 1 | 0, 1] [0, 1 | 0, 1]
866		let mut cycled: FieldVec<P, A> =
867			FieldBuffer::from_view_with_capacity_in(alloc, small.as_view(), 3);
868		cycled.repeat_extend(3);
869		assert_eq!(cycled.log_len(), 3);
870		assert!((0..8).all(|i| cycled.get(i) == F::new((i % 2) as u128)));
871	}
872
873	/// Pins `repeat_words` against the definition: word `i` of the tail is word `i mod n` of head.
874	fn check_repeat_words(log_head: usize, log_copies: usize) {
875		let head: Vec<P> = (0..1 << log_head)
876			.map(|i| P::broadcast(F::new(i)))
877			.collect();
878
879		// Every run width the caller can pass, from one word up to the whole head.
880		// A run under the head length is what splits a fill across workers.
881		for log_run in 0..=log_head {
882			let mut tail = vec![MaybeUninit::uninit(); head.len() * ((1 << log_copies) - 1)];
883			repeat_words(&head, &mut tail, 1 << log_run);
884
885			for (i, word) in tail.iter().enumerate() {
886				// SAFETY: the call above wrote every word of the tail.
887				let word = unsafe { word.assume_init() };
888				assert_eq!(word, head[i % head.len()], "log_run={log_run} i={i}");
889			}
890		}
891	}
892
893	#[test]
894	fn repeat_words_tiles_the_head_at_every_run_width() {
895		// Heads of 1 to 16 words, each repeated 1 to 8 times over.
896		for log_head in 0..5 {
897			for log_copies in 0..4 {
898				check_repeat_words(log_head, log_copies);
899			}
900		}
901	}
902
903	/// Pins `repeat_extend` against the definition, in scalars rather than words.
904	fn check_repeat_extend(log_src: usize, log_dst: usize) {
905		let mut rng = StdRng::seed_from_u64(0);
906		let src = random_field_buffer::<P>(&mut rng, log_src);
907
908		let mut buffer: FieldVec<P, GlobalAllocator> =
909			FieldBuffer::from_view_with_capacity_in(&GlobalAllocator, src.as_view(), log_dst);
910		buffer.repeat_extend(log_dst);
911
912		assert_eq!(buffer.log_len(), log_dst);
913		for i in 0..1 << log_dst {
914			let expected = src.get(i % (1 << log_src));
915			assert_eq!(buffer.get(i), expected, "log_src={log_src} log_dst={log_dst} i={i}");
916		}
917	}
918
919	#[test]
920	fn repeat_extend_repeats_the_live_scalars() {
921		// A packed word holds 4 lanes here, so `log_src` 0 and 1 exercise the sub-word path,
922		// and a `log_dst` that stays under the width exercises repeating inside one word.
923		for log_src in 0..5 {
924			for log_dst in log_src..7 {
925				check_repeat_extend(log_src, log_dst);
926			}
927		}
928	}
929
930	#[test]
931	#[should_panic(expected = "precondition: log_len must be at least the buffer's own log_len")]
932	fn repeat_extend_rejects_shrinking() {
933		let mut buffer: FieldVec<P, GlobalAllocator> = FieldBuffer::from_view_with_capacity_in(
934			&GlobalAllocator,
935			FieldBuffer::<P>::zeros(4).as_view(),
936			5,
937		);
938		buffer.repeat_extend(3);
939	}
940
941	#[test]
942	#[should_panic(expected = "precondition: the store must have room reserved")]
943	fn repeat_extend_rejects_a_store_without_room() {
944		let mut buffer: FieldVec<P, GlobalAllocator> = FieldBuffer::from_view_with_capacity_in(
945			&GlobalAllocator,
946			FieldBuffer::<P>::zeros(4).as_view(),
947			4,
948		);
949		buffer.repeat_extend(5);
950	}
951
952	#[test]
953	fn a_copy_reproduces_every_word_on_both_sides_of_the_task_floor() {
954		// The floor is the fewest words a worker takes, set by one task's byte budget.
955		//
956		//     at the floor   [--------]              one task
957		//     above it       [--------][--------]    split across workers
958		let floor = min_len_for_bytes::<P>();
959		for words in [floor, 2 * floor] {
960			let log_len = words.ilog2() as usize + P::LOG_WIDTH;
961			let src = random_field_buffer::<P>(&mut StdRng::seed_from_u64(0), log_len);
962			let copy: FieldVec<P, GlobalAllocator> =
963				FieldBuffer::from_view_in(&GlobalAllocator, src.as_view());
964			assert_eq!(copy.log_len(), log_len);
965			assert_eq!(copy.as_ref(), src.as_ref(), "words={words}");
966		}
967	}
968
969	#[test]
970	#[should_panic(expected = "precondition: log_capacity must be at least src.log_len()")]
971	fn from_view_with_capacity_rejects_a_capacity_below_the_source() {
972		let src = FieldBuffer::<P>::zeros(4);
973		let _: FieldVec<P, GlobalAllocator> =
974			FieldBuffer::from_view_with_capacity_in(&GlobalAllocator, src.as_view(), 3);
975	}
976
977	#[test]
978	fn zeros() {
979		// Make a buffer with `zeros()` and check that all elements are zero.
980		// Test with log_len >= LOG_WIDTH
981		let buffer = FieldBuffer::<P>::zeros(6); // 64 elements
982		assert_eq!(buffer.log_len(), 6);
983		assert_eq!(buffer.len(), 64);
984
985		// Check all elements are zero
986		for i in 0..64 {
987			assert_eq!(buffer.get(i), F::ZERO);
988		}
989
990		// Test with log_len < LOG_WIDTH
991		let buffer = FieldBuffer::<P>::zeros(1); // 2 elements
992		assert_eq!(buffer.log_len(), 1);
993		assert_eq!(buffer.len(), 2);
994
995		// Check all elements are zero
996		for i in 0..2 {
997			assert_eq!(buffer.get(i), F::ZERO);
998		}
999	}
1000
1001	#[test]
1002	fn alloc_constructors_global() {
1003		// Every allocation is an independent heap buffer, freed on drop.
1004		check_alloc_constructors(&GlobalAllocator);
1005	}
1006
1007	#[test]
1008	fn alloc_constructors_pooled() {
1009		// A pool reuses freed blocks, so a buffer may start life on memory a prior buffer dirtied.
1010		let pool = BufferPool::new();
1011		check_alloc_constructors(&&pool);
1012	}
1013
1014	#[test]
1015	fn from_values_below_packing_width() {
1016		// Make a buffer using `from_values()`, where the number of scalars is below the packing
1017		// width
1018		// P::LOG_WIDTH = 2, so P::WIDTH = 4
1019		let values = vec![F::new(1), F::new(2)]; // 2 elements < 4
1020		let buffer = FieldBuffer::<P>::from_values(&values);
1021
1022		assert_eq!(buffer.log_len(), 1); // log2(2) = 1
1023		assert_eq!(buffer.len(), 2);
1024
1025		// Verify the values
1026		assert_eq!(buffer.get(0), F::new(1));
1027		assert_eq!(buffer.get(1), F::new(2));
1028	}
1029
1030	#[test]
1031	fn from_values_above_packing_width() {
1032		// Make a buffer using `from_values()`, where the number of scalars is above the packing
1033		// width
1034		// P::LOG_WIDTH = 2, so P::WIDTH = 4
1035		let values: Vec<F> = (0..16).map(F::new).collect(); // 16 elements > 4
1036		let buffer = FieldBuffer::<P>::from_values(&values);
1037
1038		assert_eq!(buffer.log_len(), 4); // log2(16) = 4
1039		assert_eq!(buffer.len(), 16);
1040
1041		// Verify all values
1042		for i in 0..16 {
1043			assert_eq!(buffer.get(i), F::new(i as u128));
1044		}
1045	}
1046
1047	#[test]
1048	#[should_panic(expected = "power of two")]
1049	fn from_values_non_power_of_two() {
1050		let values: Vec<F> = (0..7).map(F::new).collect(); // 7 is not a power of two
1051		let _ = FieldBuffer::<P>::from_values(&values);
1052	}
1053
1054	#[test]
1055	#[should_panic(expected = "power of two")]
1056	fn from_values_empty() {
1057		let values: Vec<F> = vec![];
1058		let _ = FieldBuffer::<P>::from_values(&values);
1059	}
1060
1061	#[test]
1062	fn new_below_packing_width() {
1063		// Make a buffer using `new()`, where the number of scalars is below the packing
1064		// width
1065		// P::LOG_WIDTH = 2, so P::WIDTH = 4
1066		// For log_len = 1 (2 elements), we need 1 packed value
1067		let mut packed_values = vec![P::default()];
1068		let mut buffer = FieldBuffer::new(1, packed_values.as_mut_slice());
1069
1070		assert_eq!(buffer.log_len(), 1);
1071		assert_eq!(buffer.len(), 2);
1072
1073		// Set and verify values
1074		buffer.set(0, F::new(10));
1075		buffer.set(1, F::new(20));
1076		assert_eq!(buffer.get(0), F::new(10));
1077		assert_eq!(buffer.get(1), F::new(20));
1078	}
1079
1080	#[test]
1081	fn new_above_packing_width() {
1082		// Make a buffer using `new()`, where the number of scalars is above the packing
1083		// width
1084		// P::LOG_WIDTH = 2, so P::WIDTH = 4
1085		// For log_len = 4 (16 elements), we need 4 packed values
1086		let mut packed_values = vec![P::default(); 4];
1087		let mut buffer = FieldBuffer::new(4, packed_values.as_mut_slice());
1088
1089		assert_eq!(buffer.log_len(), 4);
1090		assert_eq!(buffer.len(), 16);
1091
1092		// Set and verify values
1093		for i in 0..16 {
1094			buffer.set(i, F::new(i as u128 * 10));
1095		}
1096		for i in 0..16 {
1097			assert_eq!(buffer.get(i), F::new(i as u128 * 10));
1098		}
1099	}
1100
1101	#[test]
1102	#[should_panic(expected = "precondition")]
1103	fn new_wrong_packed_length() {
1104		let packed_values = vec![P::default(); 3]; // Wrong: should be 4 for log_len=4
1105		let _ = FieldBuffer::new(4, packed_values.as_slice());
1106	}
1107
1108	#[test]
1109	fn get_set() {
1110		let mut buffer = FieldBuffer::<P>::zeros(3); // 8 elements
1111
1112		// Set some values
1113		for i in 0..8 {
1114			buffer.set(i, F::new(i as u128));
1115		}
1116
1117		// Get them back
1118		for i in 0..8 {
1119			assert_eq!(buffer.get(i), F::new(i as u128));
1120		}
1121	}
1122
1123	#[test]
1124	#[should_panic(expected = "precondition")]
1125	fn get_out_of_bounds() {
1126		let buffer = FieldBuffer::<P>::zeros(3); // 8 elements
1127		let _ = buffer.get(8);
1128	}
1129
1130	#[test]
1131	#[should_panic(expected = "precondition")]
1132	fn set_out_of_bounds() {
1133		let mut buffer = FieldBuffer::<P>::zeros(3); // 8 elements
1134		buffer.set(8, F::new(0));
1135	}
1136
1137	#[test]
1138	fn borrowed_views() {
1139		let mut buffer = FieldBuffer::<P>::zeros(3);
1140
1141		// Test the shared view
1142		let slice_ref = buffer.as_view();
1143		assert_eq!(slice_ref.len(), buffer.len());
1144		assert_eq!(slice_ref.log_len(), buffer.log_len());
1145		assert_eq!(slice_ref.as_ref().len(), 1 << slice_ref.log_len().saturating_sub(P::LOG_WIDTH));
1146
1147		// Test the mutable view
1148		let mut slice_mut = buffer.as_mut_view();
1149		slice_mut.set(0, F::new(123));
1150		assert_eq!(slice_mut.as_mut().len(), 1 << slice_mut.log_len().saturating_sub(P::LOG_WIDTH));
1151		assert_eq!(buffer.get(0), F::new(123));
1152	}
1153
1154	#[test]
1155	fn iter_scalars() {
1156		// Test with buffer size below packing width
1157		// P::LOG_WIDTH = 2, so P::WIDTH = 4
1158		let values = vec![F::new(10), F::new(20)]; // 2 elements < 4
1159		let buffer = FieldBuffer::<P>::from_values(&values);
1160
1161		let collected: Vec<F> = buffer.iter_scalars().collect();
1162		assert_eq!(collected, values);
1163
1164		// Verify it matches individual get calls
1165		for (i, &val) in collected.iter().enumerate() {
1166			assert_eq!(val, buffer.get(i));
1167		}
1168
1169		// Test with buffer size equal to packing width
1170		let values = vec![F::new(1), F::new(2), F::new(3), F::new(4)]; // 4 elements = P::WIDTH
1171		let buffer = FieldBuffer::<P>::from_values(&values);
1172
1173		let collected: Vec<F> = buffer.iter_scalars().collect();
1174		assert_eq!(collected, values);
1175
1176		// Test with buffer size above packing width
1177		let values: Vec<F> = (0..16).map(F::new).collect(); // 16 elements > 4
1178		let buffer = FieldBuffer::<P>::from_values(&values);
1179
1180		let collected: Vec<F> = buffer.iter_scalars().collect();
1181		assert_eq!(collected, values);
1182
1183		// Verify it matches individual get calls
1184		for (i, &val) in collected.iter().enumerate() {
1185			assert_eq!(val, buffer.get(i));
1186		}
1187
1188		// Test with single element buffer
1189		let values = vec![F::new(42)];
1190		let buffer = FieldBuffer::<P>::from_values(&values);
1191
1192		let collected: Vec<F> = buffer.iter_scalars().collect();
1193		assert_eq!(collected, values);
1194
1195		// Test with large buffer
1196		let values: Vec<F> = (0..256).map(F::new).collect();
1197		let buffer = FieldBuffer::<P>::from_values(&values);
1198
1199		let collected: Vec<F> = buffer.iter_scalars().collect();
1200		assert_eq!(collected, values);
1201
1202		// Test that iterator is cloneable and can be used multiple times
1203		let values: Vec<F> = (0..8).map(F::new).collect();
1204		let buffer = FieldBuffer::<P>::from_values(&values);
1205
1206		let iter1 = buffer.iter_scalars();
1207		let iter2 = iter1.clone();
1208
1209		let collected1: Vec<F> = iter1.collect();
1210		let collected2: Vec<F> = iter2.collect();
1211		assert_eq!(collected1, collected2);
1212		assert_eq!(collected1, values);
1213	}
1214
1215	#[test]
1216	fn from_iter_below_packing_width() {
1217		// One element in a 4-lane word: the buffer spans 1 element, not the word's 4.
1218		let buffer: FieldBuffer<P> = std::iter::once(F::new(9)).collect();
1219		assert_eq!(buffer.log_len(), 0);
1220		assert_eq!(buffer.len(), 1);
1221		assert_eq!(buffer.get(0), F::new(9));
1222
1223		// The one word packed keeps zeros in the 3 lanes past the element.
1224		let data = buffer.into_inner();
1225		assert_eq!(data.len(), 1);
1226		assert!((1..P::WIDTH).all(|lane| get_packed_slice(&data[..], lane) == F::ZERO));
1227	}
1228
1229	#[test]
1230	#[should_panic(expected = "power of two")]
1231	fn from_iter_non_power_of_two() {
1232		// 7 elements: the count is only known once the iterator runs dry, and then it panics.
1233		let _: FieldBuffer<P> = (0..7).map(F::new).collect();
1234	}
1235
1236	#[test]
1237	#[should_panic(expected = "power of two")]
1238	fn from_iter_empty() {
1239		// Zero elements is not a power of two, so this panics rather than making a 0-length buffer.
1240		let _: FieldBuffer<P> = std::iter::empty::<F>().collect();
1241	}
1242
1243	#[test]
1244	fn iter_packed_covers_the_live_words() {
1245		// Room for 32 elements but 1 live: iteration follows the live count, not the room.
1246		let buffer = FieldBuffer::<P>::scalar_with_capacity(F::new(5), 5);
1247		let live: Vec<P> = buffer.iter_packed().copied().collect();
1248		assert_eq!(live.len(), 1);
1249		assert_eq!(get_packed_slice(&live[..], 0), F::new(5));
1250
1251		// 16 elements over 4-lane words: 4 words, holding the elements in order.
1252		let values: Vec<F> = (0..16).map(F::new).collect();
1253		let buffer = FieldBuffer::<P>::from_values(&values);
1254		let words: Vec<P> = buffer.iter_packed().copied().collect();
1255		assert_eq!(words.len(), 4);
1256		assert_eq!(P::iter_slice(&words).collect::<Vec<_>>(), values);
1257
1258		// A buffer truncated to 2 elements iterates the 1 word they sit in, not the original 4.
1259		let mut truncated = FieldBuffer::<P>::from_values(&values);
1260		truncated.truncate(1);
1261		assert_eq!(truncated.iter_packed().count(), 1);
1262	}
1263
1264	#[test]
1265	fn iter_packed_mut_writes_the_live_words() {
1266		// 8 elements over 2 words: writing both words writes every element.
1267		let mut buffer = FieldBuffer::<P>::zeros(3);
1268		assert_eq!(buffer.iter_packed_mut().count(), 2);
1269		for word in buffer.iter_packed_mut() {
1270			*word = P::broadcast(F::new(3));
1271		}
1272		assert!(buffer.iter_scalars().all(|scalar| scalar == F::new(3)));
1273
1274		// Sub-packing-width: 2 elements share one word, which is lent out whole.
1275		let mut small = FieldBuffer::<P>::zeros(1);
1276		assert_eq!(small.iter_packed_mut().count(), 1);
1277		for word in small.iter_packed_mut() {
1278			*word = P::broadcast(F::new(7));
1279		}
1280		// Only the 2 live lanes are elements, however many lanes the write touched.
1281		assert_eq!(small.iter_scalars().collect::<Vec<_>>(), vec![F::new(7); 2]);
1282	}
1283
1284	#[test]
1285	fn truncate_vec_backing() {
1286		// P::LOG_WIDTH = 2, P::WIDTH = 4.
1287		let make = || FieldBuffer::<P>::from_values(&(0..16).map(F::new).collect::<Vec<_>>());
1288
1289		// Above-width result: the backing Vec shrinks to the new packed length.
1290		let mut buffer = make();
1291		buffer.truncate(3); // 8 elements -> 2 packed words
1292		assert_eq!(buffer.log_len(), 3);
1293		assert_eq!(buffer.len(), 8);
1294		for i in 0..8 {
1295			assert_eq!(buffer.get(i), F::new(i as u128));
1296		}
1297		assert_eq!(buffer.into_inner().len(), 2);
1298
1299		// Sub-packing-width result: one word retained, live prefix kept, dead lanes zeroed.
1300		let mut buffer = make();
1301		buffer.truncate(1); // 2 elements
1302		assert_eq!(buffer.len(), 2);
1303		assert_eq!(buffer.get(0), F::new(0));
1304		assert_eq!(buffer.get(1), F::new(1));
1305		let data = buffer.into_inner();
1306		assert_eq!(data.len(), 1);
1307		assert_eq!(get_packed_slice(&data[..], 2), F::new(0));
1308		assert_eq!(get_packed_slice(&data[..], 3), F::new(0));
1309
1310		// No-op when the requested length is not smaller.
1311		let mut buffer = FieldBuffer::<P>::from_values(&(0..4).map(F::new).collect::<Vec<_>>());
1312		buffer.truncate(5);
1313		assert_eq!(buffer.log_len(), 2);
1314		assert_eq!(buffer.into_inner().len(), 1);
1315	}
1316
1317	#[test]
1318	fn truncate_slice_backing() {
1319		// Truncating a `&mut [P]` backing reslices it and zeros the sub-width dead lanes.
1320		let mut storage = vec![P::default(); 4]; // 16 elements at log_len 4
1321		let mut buffer = FieldSliceMut::from_slice(4, storage.as_mut_slice());
1322		for i in 0..16 {
1323			buffer.set(i, F::new(i as u128));
1324		}
1325
1326		buffer.truncate(1); // 2 elements, sub-width
1327		assert_eq!(buffer.len(), 2);
1328		assert_eq!(buffer.get(0), F::new(0));
1329		assert_eq!(buffer.get(1), F::new(1));
1330
1331		let data = buffer.into_inner();
1332		assert_eq!(data.len(), 1);
1333		assert_eq!(get_packed_slice(&data[..], 2), F::new(0));
1334		assert_eq!(get_packed_slice(&data[..], 3), F::new(0));
1335	}
1336
1337	#[test]
1338	fn split_half() {
1339		// Test with buffer size > P::WIDTH (multiple packed elements)
1340		let values: Vec<F> = (0..16).map(F::new).collect();
1341		let buffer = FieldBuffer::<P>::from_values(&values);
1342
1343		let (first, second) = buffer.split_half();
1344		assert_eq!(first.len(), 8);
1345		assert_eq!(second.len(), 8);
1346
1347		// Verify values
1348		for i in 0..8 {
1349			assert_eq!(first.get(i), F::new(i as u128));
1350			assert_eq!(second.get(i), F::new((i + 8) as u128));
1351		}
1352
1353		// Test with buffer size = P::WIDTH (single packed element)
1354		// P::LOG_WIDTH = 2, so P::WIDTH = 4
1355		let values: Vec<F> = (0..4).map(F::new).collect();
1356		let buffer = FieldBuffer::<P>::from_values(&values);
1357
1358		let (first, second) = buffer.split_half();
1359		assert_eq!(first.len(), 2);
1360		assert_eq!(second.len(), 2);
1361
1362		// Verify we got Single variants
1363		match &first.words {
1364			FieldSliceData::Single(_) => {}
1365			_ => panic!("Expected Single variant for first half"),
1366		}
1367		match &second.words {
1368			FieldSliceData::Single(_) => {}
1369			_ => panic!("Expected Single variant for second half"),
1370		}
1371
1372		// Verify values
1373		assert_eq!(first.get(0), F::new(0));
1374		assert_eq!(first.get(1), F::new(1));
1375		assert_eq!(second.get(0), F::new(2));
1376		assert_eq!(second.get(1), F::new(3));
1377
1378		// Test with buffer size = 2 (less than P::WIDTH)
1379		let values: Vec<F> = vec![F::new(10), F::new(20)];
1380		let buffer = FieldBuffer::<P>::from_values(&values);
1381
1382		let (first, second) = buffer.split_half();
1383		assert_eq!(first.len(), 1);
1384		assert_eq!(second.len(), 1);
1385
1386		// Verify we got Single variants
1387		match &first.words {
1388			FieldSliceData::Single(_) => {}
1389			_ => panic!("Expected Single variant for first half"),
1390		}
1391		match &second.words {
1392			FieldSliceData::Single(_) => {}
1393			_ => panic!("Expected Single variant for second half"),
1394		}
1395
1396		assert_eq!(first.get(0), F::new(10));
1397		assert_eq!(second.get(0), F::new(20));
1398	}
1399
1400	#[test]
1401	#[should_panic(expected = "precondition")]
1402	fn split_half_size_one() {
1403		let values = vec![F::new(42)];
1404		let buffer = FieldBuffer::<P>::from_values(&values);
1405		let _ = buffer.split_half();
1406	}
1407
1408	#[test]
1409	fn split_half_mut_no_closure() {
1410		// Test with buffer size > P::WIDTH (multiple packed elements)
1411		let mut buffer = FieldBuffer::<P>::zeros(4); // 16 elements
1412
1413		// Fill with test data
1414		for i in 0..16 {
1415			buffer.set(i, F::new(i as u128));
1416		}
1417
1418		{
1419			let mut split = buffer.split_half_mut();
1420			let (mut first, mut second) = split.halves();
1421
1422			assert_eq!(first.len(), 8);
1423			assert_eq!(second.len(), 8);
1424
1425			// Modify through the split halves
1426			for i in 0..8 {
1427				first.set(i, F::new((i * 10) as u128));
1428				second.set(i, F::new((i * 20) as u128));
1429			}
1430			// split drops here and writes back the changes
1431		}
1432
1433		// Verify changes were made to original buffer
1434		for i in 0..8 {
1435			assert_eq!(buffer.get(i), F::new((i * 10) as u128));
1436			assert_eq!(buffer.get(i + 8), F::new((i * 20) as u128));
1437		}
1438
1439		// Test with buffer size = P::WIDTH (single packed element)
1440		// P::LOG_WIDTH = 2, so a buffer with log_len = 2 (4 elements) can now be split
1441		let mut buffer = FieldBuffer::<P>::zeros(2); // 4 elements
1442
1443		// Fill with test data
1444		for i in 0..4 {
1445			buffer.set(i, F::new(i as u128));
1446		}
1447
1448		{
1449			let mut split = buffer.split_half_mut();
1450			let (mut first, mut second) = split.halves();
1451
1452			assert_eq!(first.len(), 2);
1453			assert_eq!(second.len(), 2);
1454
1455			// Modify values
1456			first.set(0, F::new(100));
1457			first.set(1, F::new(101));
1458			second.set(0, F::new(200));
1459			second.set(1, F::new(201));
1460			// split drops here and writes back the changes using interleave
1461		}
1462
1463		// Verify changes were written back
1464		assert_eq!(buffer.get(0), F::new(100));
1465		assert_eq!(buffer.get(1), F::new(101));
1466		assert_eq!(buffer.get(2), F::new(200));
1467		assert_eq!(buffer.get(3), F::new(201));
1468
1469		// Test with buffer size = 2
1470		let mut buffer = FieldBuffer::<P>::zeros(1); // 2 elements
1471
1472		buffer.set(0, F::new(10));
1473		buffer.set(1, F::new(20));
1474
1475		{
1476			let mut split = buffer.split_half_mut();
1477			let (mut first, mut second) = split.halves();
1478
1479			assert_eq!(first.len(), 1);
1480			assert_eq!(second.len(), 1);
1481
1482			// Modify values
1483			first.set(0, F::new(30));
1484			second.set(0, F::new(40));
1485			// split drops here and writes back the changes using interleave
1486		}
1487
1488		// Verify changes
1489		assert_eq!(buffer.get(0), F::new(30));
1490		assert_eq!(buffer.get(1), F::new(40));
1491	}
1492
1493	#[test]
1494	#[should_panic(expected = "precondition")]
1495	fn split_half_mut_size_one() {
1496		let mut buffer = FieldBuffer::<P>::zeros(0); // 1 element
1497		let _ = buffer.split_half_mut();
1498	}
1499
1500	proptest! {
1501		#[test]
1502		fn unequal_length_buffers_are_never_equal(
1503			log_len_a in 0usize..=6,
1504			log_len_b in 0usize..=6,
1505			fill in any::<u128>(),
1506		) {
1507			// Invariant: buffers are equal iff same length and same scalars.
1508			let value = F::new(fill);
1509			let buf_a = FieldBuffer::<P>::from_values(&vec![value; 1 << log_len_a]);
1510			let buf_b = FieldBuffer::<P>::from_values(&vec![value; 1 << log_len_b]);
1511
1512			// - Same length -> equal;
1513			// - Different length -> unequal despite matching scalars.
1514			if log_len_a == log_len_b {
1515				prop_assert_eq!(buf_a, buf_b);
1516			} else {
1517				prop_assert_ne!(buf_a, buf_b);
1518			}
1519		}
1520
1521		#[test]
1522		fn from_iter_matches_from_values(log_len in 0usize..=6) {
1523			// The sweep straddles the packing width: a sub-word store, then multi-word ones.
1524			let values: Vec<F> = (0..1u128 << log_len).map(F::new).collect();
1525
1526			// Collecting the scalars must rebuild what packing the slice builds, length included.
1527			let collected: FieldBuffer<P> = values.iter().copied().collect();
1528			prop_assert_eq!(collected.log_len(), log_len);
1529			prop_assert_eq!(&collected, &FieldBuffer::<P>::from_values(&values));
1530
1531			// Round trip: iterating the collected buffer hands the scalars back, in order.
1532			prop_assert_eq!(&collected.iter_scalars().collect::<Vec<_>>(), &values);
1533		}
1534	}
1535}