Bit-sliced storage for finite-field vectors

ext/crates/fp/src/field/field_internal.rs

@@ -133,6 +133,124 @@ pub trait FieldInternal:
         }
     }
 
+    // # Group layout (bit-sliced storage)
+    //
+    // Storage is organized into *groups*: a group holds [`entries_per_group`] = 64 consecutive
+    // entries and occupies [`limbs_per_group`] = `k` consecutive limbs, the *bit-planes*. Plane
+    // `j` of a group holds bit `j` of all 64 entries, so entry `i` lives at bit `i` of each of
+    // the `k` planes. Every field uses this layout, with `k = ceil(log2 q)` (the bits needed to
+    // store an encoded value in `0..q`). For `q = 2` this is `k = 1`, which coincides exactly
+    // with the old packed layout — so `F_2` (and its SIMD / matrix machinery) is byte-identical
+    // and unaffected. Entry access goes through [`gather`]/[`scatter`]; the sizing helpers
+    // [`number`]/[`range`] are expressed in terms of groups.
+    //
+    // [`entries_per_group`]: FieldInternal::entries_per_group
+    // [`limbs_per_group`]: FieldInternal::limbs_per_group
+    // [`gather`]: FieldInternal::gather
+    // [`scatter`]: FieldInternal::scatter
+    // [`number`]: FieldInternal::number
+    // [`range`]: FieldInternal::range
+
+    /// The number of entries stored in a single group: one per bit of a [`Limb`].
+    fn entries_per_group(self) -> usize {
+        BITS_PER_LIMB
+    }
+
+    /// The number of bit-planes per group, `k = ceil(log2 q)`. Each field defines this; `q = 2`
+    /// gives `k = 1` (packed-compatible).
+    fn limbs_per_group(self) -> usize;
+
+    /// The index of the group containing entry `idx`.
+    fn group_of(self, idx: usize) -> usize {
+        idx / self.entries_per_group()
+    }
+
+    /// The position of entry `idx` within its group, in `0..entries_per_group()`.
+    fn lane_of(self, idx: usize) -> usize {
+        idx % self.entries_per_group()
+    }
+
+    /// Read entry `lane` (in `0..entries_per_group()`) out of a single group's `k` planes (a
+    /// slice of length [`limbs_per_group`](FieldInternal::limbs_per_group)) by reassembling its
+    /// bit from each plane.
+    fn gather(self, group: &[Limb], lane: usize) -> FieldElement<Self> {
+        let mut value: Limb = 0;
+        for (j, plane) in group.iter().enumerate() {
+            value |= ((plane >> lane) & 1) << j;
+        }
+        self.decode(value)
+    }
+
+    /// Write `value` into entry `lane` of a single group's `k` planes, dispersing the encoded
+    /// value's bits one per plane. Assumes the stored value fits in `k` bits.
+    fn scatter(self, group: &mut [Limb], lane: usize, value: FieldElement<Self>) {
+        let encoded = self.encode(value);
+        let lane_mask: Limb = 1 << lane;
+        for (j, plane) in group.iter_mut().enumerate() {
+            let bit = (encoded >> j) & 1;
+            *plane = (*plane & !lane_mask) | (bit << lane);
+        }
+    }
+
+    /// Whether this field uses a genuinely multi-plane layout (`k > 1`). Only `F_2` has `k = 1`,
+    /// where the bit-sliced layout coincides with the packed one and the `F_2`-specific fast
+    /// paths (`offset`, `limb_masks`, SIMD, m4ri) apply.
+    fn is_bitsliced(self) -> bool {
+        self.limbs_per_group() > 1
+    }
+
+    /// `dst += coeff * src` (mod p) over a span of whole groups (`dst` and `src` have equal,
+    /// group-aligned length). Both are assumed reduced; the result is reduced.
+    ///
+    /// Default: element-wise over lanes via [`Self::gather`]/[`Self::scatter`] and the field's own
+    /// arithmetic — correct for any bit-sliced field (used by [`SmallFq`](super::SmallFq)). The
+    /// prime fields [`Fp`](super::Fp) override this with a branch-free plane circuit.
+    fn add_groups(self, dst: &mut [Limb], src: &[Limb], coeff: FieldElement<Self>) {
+        let lpg = self.limbs_per_group();
+        let epg = self.entries_per_group();
+        for (dgroup, sgroup) in dst.chunks_exact_mut(lpg).zip(src.chunks_exact(lpg)) {
+            for lane in 0..epg {
+                let a = self.gather(dgroup, lane);
+                let b = self.gather(sgroup, lane);
+                let result = self.add(a, self.mul(coeff.clone(), b));
+                self.scatter(dgroup, lane, result);
+            }
+        }
+    }
+
+    /// `dst *= coeff` (mod p) over a span of whole groups. Default: element-wise; overridden by
+    /// [`Fp`](super::Fp).
+    fn scale_groups(self, dst: &mut [Limb], coeff: FieldElement<Self>) {
+        let lpg = self.limbs_per_group();
+        let epg = self.entries_per_group();
+        for dgroup in dst.chunks_exact_mut(lpg) {
+            for lane in 0..epg {
+                let a = self.gather(dgroup, lane);
+                self.scatter(dgroup, lane, self.mul(a, coeff.clone()));
+            }
+        }
+    }
+
+    /// `dst += coeff * src` (mod p) for a single group (each `limbs_per_group()` limbs),
+    /// restricted to the lanes set in `lane_mask`; other lanes are unchanged. Used for the
+    /// partial boundary groups of a slice add. Default: element-wise; overridden by
+    /// [`Fp`](super::Fp) with a masked plane circuit.
+    fn add_group_masked(
+        self,
+        dst: &mut [Limb],
+        src: &[Limb],
+        coeff: FieldElement<Self>,
+        lane_mask: Limb,
+    ) {
+        for lane in 0..self.entries_per_group() {
+            if (lane_mask >> lane) & 1 == 1 {
+                let a = self.gather(dst, lane);
+                let b = self.gather(src, lane);
+                self.scatter(dst, lane, self.add(a, self.mul(coeff.clone(), b)));
+            }
+        }
+    }
+
     /// Check whether or not a limb is reduced. This may potentially not be faster than calling
     /// [`reduce`](FieldInternal::reduce) directly.
     fn is_reduced(self, limb: Limb) -> bool {
@@ -166,17 +284,20 @@ pub trait FieldInternal:
 
     /// Return the number of limbs required to hold `dim` entries.
     fn number(self, dim: usize) -> usize {
-        if dim == 0 {
-            0
-        } else {
-            self.limb_bit_index_pair(dim - 1).limb + 1
-        }
+        // Whole groups needed to hold `dim` entries, times the limbs in each group.
+        self.limbs_per_group() * dim.div_ceil(self.entries_per_group())
     }
 
-    /// Return the `Range<usize>` starting at the index of the limb containing the `start`th entry, and
-    /// ending at the index of the limb containing the `end`th entry (including the latter).
+    /// Return the `Range<usize>` of limbs spanning entries `start..end`: from the first limb of
+    /// the group containing `start` to the last limb of the group containing `end - 1`.
     fn range(self, start: usize, end: usize) -> Range<usize> {
-        let min = self.limb_bit_index_pair(start).limb;
+        debug_assert!(start <= end);
+        let min = self.group_of(start) * self.limbs_per_group();
+        if start == end {
+            // An empty entry range maps to an empty limb range; otherwise callers that guard
+            // on `limb_range.is_empty()` would touch the (unrelated) containing group.
+            return min..min;
+        }
         let max = self.number(end);
         min..max
     }

ext/crates/fp/src/field/fp.rs

@@ -142,6 +142,54 @@ impl<P: Prime> FieldInternal for Fp<P> {
             _ => self.pack(self.unpack(limb)),
         }
     }
+
+    // # Bit-sliced layout
+    //
+    // Prime-field vectors use the bit-sliced group layout (see [`FieldInternal`]). The packed
+    // limb helpers above are not used to lay out `FqVector<Fp<_>>` storage, but are retained
+    // because `decode`/`encode`/`reduce` are still called when constructing elements. The
+    // uniform `gather`/`scatter` defaults handle entry access; `Fp` only overrides the bulk
+    // kernels with a branch-free plane circuit.
+
+    fn limbs_per_group(self) -> usize {
+        crate::field::bitslice::planes(self.characteristic().as_u32())
+    }
+
+    fn add_groups(self, dst: &mut [Limb], src: &[Limb], coeff: FieldElement<Self>) {
+        crate::field::bitslice::add_groups(
+            self.characteristic().as_u32(),
+            self.limbs_per_group(),
+            dst,
+            src,
+            self.encode(coeff) as u32,
+        );
+    }
+
+    fn scale_groups(self, dst: &mut [Limb], coeff: FieldElement<Self>) {
+        crate::field::bitslice::scale_groups(
+            self.characteristic().as_u32(),
+            self.limbs_per_group(),
+            dst,
+            self.encode(coeff) as u32,
+        );
+    }
+
+    fn add_group_masked(
+        self,
+        dst: &mut [Limb],
+        src: &[Limb],
+        coeff: FieldElement<Self>,
+        lane_mask: Limb,
+    ) {
+        crate::field::bitslice::add_group_masked(
+            self.characteristic().as_u32(),
+            self.limbs_per_group(),
+            dst,
+            src,
+            self.encode(coeff) as u32,
+            lane_mask,
+        );
+    }
 }
 
 #[cfg(feature = "proptest")]

ext/crates/fp/src/field/mod.rs

@@ -4,6 +4,7 @@ use crate::prime::Prime;
 pub mod element;
 pub(crate) mod field_internal;
 
+pub(crate) mod bitslice;
 pub mod fp;
 pub mod smallfq;
 

ext/crates/fp/src/field/smallfq.rs

@@ -282,6 +282,13 @@ impl<P: Prime> FieldInternal for SmallFq<P> {
         BITS_PER_LIMB - (self.q() - 1).leading_zeros() as usize + 1
     }
 
+    fn limbs_per_group(self) -> usize {
+        // Bit-sliced layout: one plane per bit of the encoded value. `encode` maps a^n to the
+        // odd number `2n + 1` (and zero to `0`), which occupies exactly `bit_length()` bits.
+        // The default element-wise `add_groups`/`scale_groups` use Zech-log field arithmetic.
+        self.bit_length()
+    }
+
     fn fma_limb(self, limb_a: Limb, limb_b: Limb, coeff: FieldElement<Self>) -> Limb {
         let bit_length = self.bit_length();
         let mut result: Limb = 0;

ext/crates/fp/src/matrix/matrix_inner.rs

@@ -322,8 +322,26 @@ impl Matrix {
             return Self::new(p, 0, 0);
         }
         let columns = input[0].len();
+        assert!(
+            input.iter().all(|row| row.len() == columns),
+            "all rows must have the same length"
+        );
         let stride = fp.number(columns);
         let physical_rows = get_physical_rows(p, rows);
+
+        if fp.is_bitsliced() {
+            // The bit-sliced layout interleaves an entry's bits across planes, so build each
+            // row through the (dispatching) row slice rather than by packing contiguous chunks.
+            let mut matrix = Self::new(p, rows, columns);
+            for (i, row) in input.iter().enumerate() {
+                let mut target = matrix.row_mut(i);
+                for (j, &x) in row.iter().enumerate() {
+                    target.set_entry(j, x);
+                }
+            }
+            return matrix;
+        }
+
         let mut data = AVec::with_capacity(0, physical_rows * stride);
         for row in input {
             for chunk in row.chunks(fp.entries_per_limb()) {
@@ -352,6 +370,14 @@ impl Matrix {
     /// assert_eq!(Matrix::from_vec(TWO, &matrix_vec).to_vec(), matrix_vec);
     /// ```
     pub fn to_vec(&self) -> Vec<Vec<u32>> {
+        if self.fp.is_bitsliced() {
+            return (0..self.rows())
+                .map(|i| {
+                    let row = self.row(i);
+                    (0..self.columns()).map(|j| row.entry(j)).collect()
+                })
+                .collect();
+        }
         self.data
             .iter()
             .chunks(self.stride)

ext/crates/fp/src/vector/fp_wrapper/mod.rs

@@ -112,14 +112,13 @@ impl FpVector {
         v
     }
 
-    // Convenient for some matrix methods
-    pub(crate) fn num_limbs(p: ValidPrime, len: usize) -> usize {
-        Fp::new(p).number(len)
-    }
-
-    // Convenient for some matrix methods
+    // Round `len` up to a whole number of groups, so that an augmented-matrix segment of this
+    // length ends on a group boundary and the next segment starts on one. A group spans 64
+    // entries (across several limbs in the bit-sliced layout), and segments must align to those
+    // 64-entry boundaries or a single group would straddle two segments.
     pub(crate) fn padded_len(p: ValidPrime, len: usize) -> usize {
-        Self::num_limbs(p, len) * Fp::new(p).entries_per_limb()
+        let entries_per_group = Fp::new(p).entries_per_group();
+        len.div_ceil(entries_per_group) * entries_per_group
     }
 }
 

ext/crates/fp/src/vector/impl_fqslice.rs

@@ -33,12 +33,11 @@ impl<'a, F: Field> FqSlice<'a, F> {
             index,
             self.len()
         );
-        let bit_mask = self.fq().bitmask();
-        let limb_index = self.fq().limb_bit_index_pair(index + self.start());
-        let mut result = self.limbs()[limb_index.limb];
-        result >>= limb_index.bit_index;
-        result &= bit_mask;
-        self.fq().decode(result)
+        let fq = self.fq();
+        let idx = index + self.start();
+        let lpg = fq.limbs_per_group();
+        let base = fq.group_of(idx) * lpg;
+        fq.gather(&self.limbs()[base..base + lpg], fq.lane_of(idx))
     }
 
     /// TODO: implement prime 2 version
@@ -55,6 +54,9 @@ impl<'a, F: Field> FqSlice<'a, F> {
     }
 
     pub fn is_zero(&self) -> bool {
+        if self.fq().is_bitsliced() {
+            return self.first_nonzero().is_none();
+        }
         let limb_range = self.limb_range();
         if limb_range.is_empty() {
             return true;
@@ -90,7 +92,7 @@ impl<'a, F: Field> FqSlice<'a, F> {
     #[must_use]
     pub fn to_owned(self) -> FqVector<F> {
         let mut new = FqVector::new(self.fq(), self.len());
-        if self.start().is_multiple_of(self.fq().entries_per_limb()) {
+        if !self.fq().is_bitsliced() && self.start().is_multiple_of(self.fq().entries_per_limb()) {
             let limb_range = self.limb_range();
             new.limbs_mut()[0..limb_range.len()].copy_from_slice(&self.limbs()[limb_range]);
             if !new.limbs().is_empty() {