Barretenberg
The ZK-SNARK library at the core of Aztec
Loading...
Searching...
No Matches
vector_field_wasm.cpp
Go to the documentation of this file.
1// Out-of-line explicit specializations of `VectorField<Params>::operator*` for
2// WASM. Compiled as a separate TU so LLVM emits each per-Params specialization
3// as a standalone function (not inlined into callers), giving it the same
4// register-allocation scope the gist's hand-written WAT function gets —
5// inlining across ~2400 lines of carefully ordered ops lets LLVM's instruction
6// scheduler re-coalesce locals and recreate the stream-reorder problem the
7// asm barriers solve.
8//
9// The kernel body is hand-maintained (no generator script) in
10// vector_field_mont_mul_body.inl.hpp and #included once per specialization
11// (Bn254FrParams, Bn254FqParams). The macros it expands (BB_VF_LOAD_LIMBS,
12// BB_VF_KARATSUBA_STAGES_1_4, BB_VF_RUN_STAGES_6_THROUGH_10) reference
13// unqualified R_INV_WASM, P_WASM, R_INV_MOD_2_29 — these resolve in the
14// enclosing class scope to whichever Params constants the specialization
15// was bound to.
16
17#include "barretenberg/ecc/curves/bn254/fq.hpp"
18#include "barretenberg/ecc/curves/bn254/fr.hpp"
19#include "barretenberg/ecc/fields/vector_field.hpp"
20
21#if defined(__wasm_simd128__)
22
23namespace bb {
24
25// =====================================================================
26// Karatsuba Stage 1..4 shared macro.
27//
28// Textually stamps the interleaved scalar + quad computation of
29// pl0..pl8 (P_lo = left[0..4] * right[0..4], 25 muls)
30// ph0..ph6 (P_hi = left[5..8] * right[5..8], 16 muls)
31// ssl/ssr/qsl/qsr 0..4 (cross sums, i32x4 add — NOT i64x2)
32// pc0..pc8 (P_cross = sl * sr, 25 muls)
33// from the named-local inputs `sl0..sl8`, `sri0..sri8`, `ql0..ql8`,
34// `qri0..qri8` that the caller must define.
35//
36// This is a MACRO, not a function: if it were inlined through a function
37// boundary, the struct-return + array-intermediate would defeat LLVM's
38// SSA register allocator and lose ~5% on the mul bench (see commit
39// history). The macro form stamps the body directly so the compiler
40// sees the inputs and outputs as fresh locals in each caller.
41//
42// Every `bb_vf_barrier_sqq` / `bb_vf_barrier_sq` is load-bearing: see
43// vector_field.hpp for why. DO NOT remove.
44//
45// Stage 3 "break the ql4==ql4 / qri4==qri4 alias" (via bb_vf_barrier_sq
46// on sl4/ql4/sri4/qri4) prevents LLVM from CSE-ing pc8's extmul pair
47// with pl8 = l4*r4.
48// =====================================================================
49// clang-format off
50#define BB_VF_KARATSUBA_STAGES_1_4() \
51 /* Stage 1: P_lo (25 muls, pl0..pl8). */ \
52 uint64_t pl0 = sl0 * sri0; \
53 v128_t pl0_lo = wasm_u64x2_extmul_low_u32x4(ql0, qri0); \
54 v128_t pl0_hi = wasm_u64x2_extmul_high_u32x4(ql0, qri0); \
55 vector_field_detail::bb_vf_barrier_sqq(pl0, pl0_lo, pl0_hi); \
56 uint64_t pl1 = sl0 * sri1; \
57 v128_t pl1_lo = wasm_u64x2_extmul_low_u32x4(ql0, qri1); \
58 v128_t pl1_hi = wasm_u64x2_extmul_high_u32x4(ql0, qri1); \
59 vector_field_detail::bb_vf_barrier_sqq(pl1, pl1_lo, pl1_hi); \
60 pl1 += sl1 * sri0; \
61 pl1_lo = wasm_i64x2_add(pl1_lo, wasm_u64x2_extmul_low_u32x4(ql1, qri0)); \
62 pl1_hi = wasm_i64x2_add(pl1_hi, wasm_u64x2_extmul_high_u32x4(ql1, qri0)); \
63 vector_field_detail::bb_vf_barrier_sqq(pl1, pl1_lo, pl1_hi); \
64 uint64_t pl2 = sl0 * sri2; \
65 v128_t pl2_lo = wasm_u64x2_extmul_low_u32x4(ql0, qri2); \
66 v128_t pl2_hi = wasm_u64x2_extmul_high_u32x4(ql0, qri2); \
67 vector_field_detail::bb_vf_barrier_sqq(pl2, pl2_lo, pl2_hi); \
68 pl2 += sl1 * sri1; \
69 pl2_lo = wasm_i64x2_add(pl2_lo, wasm_u64x2_extmul_low_u32x4(ql1, qri1)); \
70 pl2_hi = wasm_i64x2_add(pl2_hi, wasm_u64x2_extmul_high_u32x4(ql1, qri1)); \
71 vector_field_detail::bb_vf_barrier_sqq(pl2, pl2_lo, pl2_hi); \
72 pl2 += sl2 * sri0; \
73 pl2_lo = wasm_i64x2_add(pl2_lo, wasm_u64x2_extmul_low_u32x4(ql2, qri0)); \
74 pl2_hi = wasm_i64x2_add(pl2_hi, wasm_u64x2_extmul_high_u32x4(ql2, qri0)); \
75 vector_field_detail::bb_vf_barrier_sqq(pl2, pl2_lo, pl2_hi); \
76 uint64_t pl3 = sl0 * sri3; \
77 v128_t pl3_lo = wasm_u64x2_extmul_low_u32x4(ql0, qri3); \
78 v128_t pl3_hi = wasm_u64x2_extmul_high_u32x4(ql0, qri3); \
79 vector_field_detail::bb_vf_barrier_sqq(pl3, pl3_lo, pl3_hi); \
80 pl3 += sl1 * sri2; \
81 pl3_lo = wasm_i64x2_add(pl3_lo, wasm_u64x2_extmul_low_u32x4(ql1, qri2)); \
82 pl3_hi = wasm_i64x2_add(pl3_hi, wasm_u64x2_extmul_high_u32x4(ql1, qri2)); \
83 vector_field_detail::bb_vf_barrier_sqq(pl3, pl3_lo, pl3_hi); \
84 pl3 += sl2 * sri1; \
85 pl3_lo = wasm_i64x2_add(pl3_lo, wasm_u64x2_extmul_low_u32x4(ql2, qri1)); \
86 pl3_hi = wasm_i64x2_add(pl3_hi, wasm_u64x2_extmul_high_u32x4(ql2, qri1)); \
87 vector_field_detail::bb_vf_barrier_sqq(pl3, pl3_lo, pl3_hi); \
88 pl3 += sl3 * sri0; \
89 pl3_lo = wasm_i64x2_add(pl3_lo, wasm_u64x2_extmul_low_u32x4(ql3, qri0)); \
90 pl3_hi = wasm_i64x2_add(pl3_hi, wasm_u64x2_extmul_high_u32x4(ql3, qri0)); \
91 vector_field_detail::bb_vf_barrier_sqq(pl3, pl3_lo, pl3_hi); \
92 uint64_t pl4 = sl0 * sri4; \
93 v128_t pl4_lo = wasm_u64x2_extmul_low_u32x4(ql0, qri4); \
94 v128_t pl4_hi = wasm_u64x2_extmul_high_u32x4(ql0, qri4); \
95 vector_field_detail::bb_vf_barrier_sqq(pl4, pl4_lo, pl4_hi); \
96 pl4 += sl1 * sri3; \
97 pl4_lo = wasm_i64x2_add(pl4_lo, wasm_u64x2_extmul_low_u32x4(ql1, qri3)); \
98 pl4_hi = wasm_i64x2_add(pl4_hi, wasm_u64x2_extmul_high_u32x4(ql1, qri3)); \
99 vector_field_detail::bb_vf_barrier_sqq(pl4, pl4_lo, pl4_hi); \
100 pl4 += sl2 * sri2; \
101 pl4_lo = wasm_i64x2_add(pl4_lo, wasm_u64x2_extmul_low_u32x4(ql2, qri2)); \
102 pl4_hi = wasm_i64x2_add(pl4_hi, wasm_u64x2_extmul_high_u32x4(ql2, qri2)); \
103 vector_field_detail::bb_vf_barrier_sqq(pl4, pl4_lo, pl4_hi); \
104 pl4 += sl3 * sri1; \
105 pl4_lo = wasm_i64x2_add(pl4_lo, wasm_u64x2_extmul_low_u32x4(ql3, qri1)); \
106 pl4_hi = wasm_i64x2_add(pl4_hi, wasm_u64x2_extmul_high_u32x4(ql3, qri1)); \
107 vector_field_detail::bb_vf_barrier_sqq(pl4, pl4_lo, pl4_hi); \
108 pl4 += sl4 * sri0; \
109 pl4_lo = wasm_i64x2_add(pl4_lo, wasm_u64x2_extmul_low_u32x4(ql4, qri0)); \
110 pl4_hi = wasm_i64x2_add(pl4_hi, wasm_u64x2_extmul_high_u32x4(ql4, qri0)); \
111 vector_field_detail::bb_vf_barrier_sqq(pl4, pl4_lo, pl4_hi); \
112 uint64_t pl5 = sl1 * sri4; \
113 v128_t pl5_lo = wasm_u64x2_extmul_low_u32x4(ql1, qri4); \
114 v128_t pl5_hi = wasm_u64x2_extmul_high_u32x4(ql1, qri4); \
115 vector_field_detail::bb_vf_barrier_sqq(pl5, pl5_lo, pl5_hi); \
116 pl5 += sl2 * sri3; \
117 pl5_lo = wasm_i64x2_add(pl5_lo, wasm_u64x2_extmul_low_u32x4(ql2, qri3)); \
118 pl5_hi = wasm_i64x2_add(pl5_hi, wasm_u64x2_extmul_high_u32x4(ql2, qri3)); \
119 vector_field_detail::bb_vf_barrier_sqq(pl5, pl5_lo, pl5_hi); \
120 pl5 += sl3 * sri2; \
121 pl5_lo = wasm_i64x2_add(pl5_lo, wasm_u64x2_extmul_low_u32x4(ql3, qri2)); \
122 pl5_hi = wasm_i64x2_add(pl5_hi, wasm_u64x2_extmul_high_u32x4(ql3, qri2)); \
123 vector_field_detail::bb_vf_barrier_sqq(pl5, pl5_lo, pl5_hi); \
124 pl5 += sl4 * sri1; \
125 pl5_lo = wasm_i64x2_add(pl5_lo, wasm_u64x2_extmul_low_u32x4(ql4, qri1)); \
126 pl5_hi = wasm_i64x2_add(pl5_hi, wasm_u64x2_extmul_high_u32x4(ql4, qri1)); \
127 vector_field_detail::bb_vf_barrier_sqq(pl5, pl5_lo, pl5_hi); \
128 uint64_t pl6 = sl2 * sri4; \
129 v128_t pl6_lo = wasm_u64x2_extmul_low_u32x4(ql2, qri4); \
130 v128_t pl6_hi = wasm_u64x2_extmul_high_u32x4(ql2, qri4); \
131 vector_field_detail::bb_vf_barrier_sqq(pl6, pl6_lo, pl6_hi); \
132 pl6 += sl3 * sri3; \
133 pl6_lo = wasm_i64x2_add(pl6_lo, wasm_u64x2_extmul_low_u32x4(ql3, qri3)); \
134 pl6_hi = wasm_i64x2_add(pl6_hi, wasm_u64x2_extmul_high_u32x4(ql3, qri3)); \
135 vector_field_detail::bb_vf_barrier_sqq(pl6, pl6_lo, pl6_hi); \
136 pl6 += sl4 * sri2; \
137 pl6_lo = wasm_i64x2_add(pl6_lo, wasm_u64x2_extmul_low_u32x4(ql4, qri2)); \
138 pl6_hi = wasm_i64x2_add(pl6_hi, wasm_u64x2_extmul_high_u32x4(ql4, qri2)); \
139 vector_field_detail::bb_vf_barrier_sqq(pl6, pl6_lo, pl6_hi); \
140 uint64_t pl7 = sl3 * sri4; \
141 v128_t pl7_lo = wasm_u64x2_extmul_low_u32x4(ql3, qri4); \
142 v128_t pl7_hi = wasm_u64x2_extmul_high_u32x4(ql3, qri4); \
143 vector_field_detail::bb_vf_barrier_sqq(pl7, pl7_lo, pl7_hi); \
144 pl7 += sl4 * sri3; \
145 pl7_lo = wasm_i64x2_add(pl7_lo, wasm_u64x2_extmul_low_u32x4(ql4, qri3)); \
146 pl7_hi = wasm_i64x2_add(pl7_hi, wasm_u64x2_extmul_high_u32x4(ql4, qri3)); \
147 vector_field_detail::bb_vf_barrier_sqq(pl7, pl7_lo, pl7_hi); \
148 uint64_t pl8 = sl4 * sri4; \
149 v128_t pl8_lo = wasm_u64x2_extmul_low_u32x4(ql4, qri4); \
150 v128_t pl8_hi = wasm_u64x2_extmul_high_u32x4(ql4, qri4); \
151 vector_field_detail::bb_vf_barrier_sqq(pl8, pl8_lo, pl8_hi); \
152 /* Stage 2: P_hi (16 muls, ph0..ph6). */ \
153 uint64_t ph0 = sl5 * sri5; \
154 v128_t ph0_lo = wasm_u64x2_extmul_low_u32x4(ql5, qri5); \
155 v128_t ph0_hi = wasm_u64x2_extmul_high_u32x4(ql5, qri5); \
156 vector_field_detail::bb_vf_barrier_sqq(ph0, ph0_lo, ph0_hi); \
157 uint64_t ph1 = sl5 * sri6; \
158 v128_t ph1_lo = wasm_u64x2_extmul_low_u32x4(ql5, qri6); \
159 v128_t ph1_hi = wasm_u64x2_extmul_high_u32x4(ql5, qri6); \
160 vector_field_detail::bb_vf_barrier_sqq(ph1, ph1_lo, ph1_hi); \
161 ph1 += sl6 * sri5; \
162 ph1_lo = wasm_i64x2_add(ph1_lo, wasm_u64x2_extmul_low_u32x4(ql6, qri5)); \
163 ph1_hi = wasm_i64x2_add(ph1_hi, wasm_u64x2_extmul_high_u32x4(ql6, qri5)); \
164 vector_field_detail::bb_vf_barrier_sqq(ph1, ph1_lo, ph1_hi); \
165 uint64_t ph2 = sl5 * sri7; \
166 v128_t ph2_lo = wasm_u64x2_extmul_low_u32x4(ql5, qri7); \
167 v128_t ph2_hi = wasm_u64x2_extmul_high_u32x4(ql5, qri7); \
168 vector_field_detail::bb_vf_barrier_sqq(ph2, ph2_lo, ph2_hi); \
169 ph2 += sl6 * sri6; \
170 ph2_lo = wasm_i64x2_add(ph2_lo, wasm_u64x2_extmul_low_u32x4(ql6, qri6)); \
171 ph2_hi = wasm_i64x2_add(ph2_hi, wasm_u64x2_extmul_high_u32x4(ql6, qri6)); \
172 vector_field_detail::bb_vf_barrier_sqq(ph2, ph2_lo, ph2_hi); \
173 ph2 += sl7 * sri5; \
174 ph2_lo = wasm_i64x2_add(ph2_lo, wasm_u64x2_extmul_low_u32x4(ql7, qri5)); \
175 ph2_hi = wasm_i64x2_add(ph2_hi, wasm_u64x2_extmul_high_u32x4(ql7, qri5)); \
176 vector_field_detail::bb_vf_barrier_sqq(ph2, ph2_lo, ph2_hi); \
177 uint64_t ph3 = sl5 * sri8; \
178 v128_t ph3_lo = wasm_u64x2_extmul_low_u32x4(ql5, qri8); \
179 v128_t ph3_hi = wasm_u64x2_extmul_high_u32x4(ql5, qri8); \
180 vector_field_detail::bb_vf_barrier_sqq(ph3, ph3_lo, ph3_hi); \
181 ph3 += sl6 * sri7; \
182 ph3_lo = wasm_i64x2_add(ph3_lo, wasm_u64x2_extmul_low_u32x4(ql6, qri7)); \
183 ph3_hi = wasm_i64x2_add(ph3_hi, wasm_u64x2_extmul_high_u32x4(ql6, qri7)); \
184 vector_field_detail::bb_vf_barrier_sqq(ph3, ph3_lo, ph3_hi); \
185 ph3 += sl7 * sri6; \
186 ph3_lo = wasm_i64x2_add(ph3_lo, wasm_u64x2_extmul_low_u32x4(ql7, qri6)); \
187 ph3_hi = wasm_i64x2_add(ph3_hi, wasm_u64x2_extmul_high_u32x4(ql7, qri6)); \
188 vector_field_detail::bb_vf_barrier_sqq(ph3, ph3_lo, ph3_hi); \
189 ph3 += sl8 * sri5; \
190 ph3_lo = wasm_i64x2_add(ph3_lo, wasm_u64x2_extmul_low_u32x4(ql8, qri5)); \
191 ph3_hi = wasm_i64x2_add(ph3_hi, wasm_u64x2_extmul_high_u32x4(ql8, qri5)); \
192 vector_field_detail::bb_vf_barrier_sqq(ph3, ph3_lo, ph3_hi); \
193 uint64_t ph4 = sl6 * sri8; \
194 v128_t ph4_lo = wasm_u64x2_extmul_low_u32x4(ql6, qri8); \
195 v128_t ph4_hi = wasm_u64x2_extmul_high_u32x4(ql6, qri8); \
196 vector_field_detail::bb_vf_barrier_sqq(ph4, ph4_lo, ph4_hi); \
197 ph4 += sl7 * sri7; \
198 ph4_lo = wasm_i64x2_add(ph4_lo, wasm_u64x2_extmul_low_u32x4(ql7, qri7)); \
199 ph4_hi = wasm_i64x2_add(ph4_hi, wasm_u64x2_extmul_high_u32x4(ql7, qri7)); \
200 vector_field_detail::bb_vf_barrier_sqq(ph4, ph4_lo, ph4_hi); \
201 ph4 += sl8 * sri6; \
202 ph4_lo = wasm_i64x2_add(ph4_lo, wasm_u64x2_extmul_low_u32x4(ql8, qri6)); \
203 ph4_hi = wasm_i64x2_add(ph4_hi, wasm_u64x2_extmul_high_u32x4(ql8, qri6)); \
204 vector_field_detail::bb_vf_barrier_sqq(ph4, ph4_lo, ph4_hi); \
205 uint64_t ph5 = sl7 * sri8; \
206 v128_t ph5_lo = wasm_u64x2_extmul_low_u32x4(ql7, qri8); \
207 v128_t ph5_hi = wasm_u64x2_extmul_high_u32x4(ql7, qri8); \
208 vector_field_detail::bb_vf_barrier_sqq(ph5, ph5_lo, ph5_hi); \
209 ph5 += sl8 * sri7; \
210 ph5_lo = wasm_i64x2_add(ph5_lo, wasm_u64x2_extmul_low_u32x4(ql8, qri7)); \
211 ph5_hi = wasm_i64x2_add(ph5_hi, wasm_u64x2_extmul_high_u32x4(ql8, qri7)); \
212 vector_field_detail::bb_vf_barrier_sqq(ph5, ph5_lo, ph5_hi); \
213 uint64_t ph6 = sl8 * sri8; \
214 v128_t ph6_lo = wasm_u64x2_extmul_low_u32x4(ql8, qri8); \
215 v128_t ph6_hi = wasm_u64x2_extmul_high_u32x4(ql8, qri8); \
216 vector_field_detail::bb_vf_barrier_sqq(ph6, ph6_lo, ph6_hi); \
217 /* Stage 4: P_cross (25 muls, pc0..pc8). */ \
218 uint64_t pc0 = (sl0 + sl5) * (sri0 + sri5); \
219 v128_t pc0_lo = wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql0, ql5), wasm_i32x4_add(qri0, qri5)); \
220 v128_t pc0_hi = wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql0, ql5), wasm_i32x4_add(qri0, qri5)); \
221 vector_field_detail::bb_vf_barrier_sqq(pc0, pc0_lo, pc0_hi); \
222 uint64_t pc1 = (sl0 + sl5) * (sri1 + sri6); \
223 v128_t pc1_lo = wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql0, ql5), wasm_i32x4_add(qri1, qri6)); \
224 v128_t pc1_hi = wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql0, ql5), wasm_i32x4_add(qri1, qri6)); \
225 vector_field_detail::bb_vf_barrier_sqq(pc1, pc1_lo, pc1_hi); \
226 pc1 += (sl1 + sl6) * (sri0 + sri5); \
227 pc1_lo = wasm_i64x2_add(pc1_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql1, ql6), wasm_i32x4_add(qri0, qri5))); \
228 pc1_hi = wasm_i64x2_add(pc1_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql1, ql6), wasm_i32x4_add(qri0, qri5))); \
229 vector_field_detail::bb_vf_barrier_sqq(pc1, pc1_lo, pc1_hi); \
230 uint64_t pc2 = (sl0 + sl5) * (sri2 + sri7); \
231 v128_t pc2_lo = wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql0, ql5), wasm_i32x4_add(qri2, qri7)); \
232 v128_t pc2_hi = wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql0, ql5), wasm_i32x4_add(qri2, qri7)); \
233 vector_field_detail::bb_vf_barrier_sqq(pc2, pc2_lo, pc2_hi); \
234 pc2 += (sl1 + sl6) * (sri1 + sri6); \
235 pc2_lo = wasm_i64x2_add(pc2_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql1, ql6), wasm_i32x4_add(qri1, qri6))); \
236 pc2_hi = wasm_i64x2_add(pc2_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql1, ql6), wasm_i32x4_add(qri1, qri6))); \
237 vector_field_detail::bb_vf_barrier_sqq(pc2, pc2_lo, pc2_hi); \
238 pc2 += (sl2 + sl7) * (sri0 + sri5); \
239 pc2_lo = wasm_i64x2_add(pc2_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql2, ql7), wasm_i32x4_add(qri0, qri5))); \
240 pc2_hi = wasm_i64x2_add(pc2_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql2, ql7), wasm_i32x4_add(qri0, qri5))); \
241 vector_field_detail::bb_vf_barrier_sqq(pc2, pc2_lo, pc2_hi); \
242 uint64_t pc3 = (sl0 + sl5) * (sri3 + sri8); \
243 v128_t pc3_lo = wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql0, ql5), wasm_i32x4_add(qri3, qri8)); \
244 v128_t pc3_hi = wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql0, ql5), wasm_i32x4_add(qri3, qri8)); \
245 vector_field_detail::bb_vf_barrier_sqq(pc3, pc3_lo, pc3_hi); \
246 pc3 += (sl1 + sl6) * (sri2 + sri7); \
247 pc3_lo = wasm_i64x2_add(pc3_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql1, ql6), wasm_i32x4_add(qri2, qri7))); \
248 pc3_hi = wasm_i64x2_add(pc3_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql1, ql6), wasm_i32x4_add(qri2, qri7))); \
249 vector_field_detail::bb_vf_barrier_sqq(pc3, pc3_lo, pc3_hi); \
250 pc3 += (sl2 + sl7) * (sri1 + sri6); \
251 pc3_lo = wasm_i64x2_add(pc3_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql2, ql7), wasm_i32x4_add(qri1, qri6))); \
252 pc3_hi = wasm_i64x2_add(pc3_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql2, ql7), wasm_i32x4_add(qri1, qri6))); \
253 vector_field_detail::bb_vf_barrier_sqq(pc3, pc3_lo, pc3_hi); \
254 pc3 += (sl3 + sl8) * (sri0 + sri5); \
255 pc3_lo = wasm_i64x2_add(pc3_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql3, ql8), wasm_i32x4_add(qri0, qri5))); \
256 pc3_hi = wasm_i64x2_add(pc3_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql3, ql8), wasm_i32x4_add(qri0, qri5))); \
257 vector_field_detail::bb_vf_barrier_sqq(pc3, pc3_lo, pc3_hi); \
258 uint64_t pc4 = (sl0 + sl5) * sri4; \
259 v128_t pc4_lo = wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql0, ql5), qri4); \
260 v128_t pc4_hi = wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql0, ql5), qri4); \
261 vector_field_detail::bb_vf_barrier_sqq(pc4, pc4_lo, pc4_hi); \
262 pc4 += (sl1 + sl6) * (sri3 + sri8); \
263 pc4_lo = wasm_i64x2_add(pc4_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql1, ql6), wasm_i32x4_add(qri3, qri8))); \
264 pc4_hi = wasm_i64x2_add(pc4_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql1, ql6), wasm_i32x4_add(qri3, qri8))); \
265 vector_field_detail::bb_vf_barrier_sqq(pc4, pc4_lo, pc4_hi); \
266 pc4 += (sl2 + sl7) * (sri2 + sri7); \
267 pc4_lo = wasm_i64x2_add(pc4_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql2, ql7), wasm_i32x4_add(qri2, qri7))); \
268 pc4_hi = wasm_i64x2_add(pc4_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql2, ql7), wasm_i32x4_add(qri2, qri7))); \
269 vector_field_detail::bb_vf_barrier_sqq(pc4, pc4_lo, pc4_hi); \
270 pc4 += (sl3 + sl8) * (sri1 + sri6); \
271 pc4_lo = wasm_i64x2_add(pc4_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql3, ql8), wasm_i32x4_add(qri1, qri6))); \
272 pc4_hi = wasm_i64x2_add(pc4_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql3, ql8), wasm_i32x4_add(qri1, qri6))); \
273 vector_field_detail::bb_vf_barrier_sqq(pc4, pc4_lo, pc4_hi); \
274 pc4 += sl4 * (sri0 + sri5); \
275 pc4_lo = wasm_i64x2_add(pc4_lo, wasm_u64x2_extmul_low_u32x4(ql4, wasm_i32x4_add(qri0, qri5))); \
276 pc4_hi = wasm_i64x2_add(pc4_hi, wasm_u64x2_extmul_high_u32x4(ql4, wasm_i32x4_add(qri0, qri5))); \
277 vector_field_detail::bb_vf_barrier_sqq(pc4, pc4_lo, pc4_hi); \
278 uint64_t pc5 = (sl1 + sl6) * sri4; \
279 v128_t pc5_lo = wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql1, ql6), qri4); \
280 v128_t pc5_hi = wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql1, ql6), qri4); \
281 vector_field_detail::bb_vf_barrier_sqq(pc5, pc5_lo, pc5_hi); \
282 pc5 += (sl2 + sl7) * (sri3 + sri8); \
283 pc5_lo = wasm_i64x2_add(pc5_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql2, ql7), wasm_i32x4_add(qri3, qri8))); \
284 pc5_hi = wasm_i64x2_add(pc5_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql2, ql7), wasm_i32x4_add(qri3, qri8))); \
285 vector_field_detail::bb_vf_barrier_sqq(pc5, pc5_lo, pc5_hi); \
286 pc5 += (sl3 + sl8) * (sri2 + sri7); \
287 pc5_lo = wasm_i64x2_add(pc5_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql3, ql8), wasm_i32x4_add(qri2, qri7))); \
288 pc5_hi = wasm_i64x2_add(pc5_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql3, ql8), wasm_i32x4_add(qri2, qri7))); \
289 vector_field_detail::bb_vf_barrier_sqq(pc5, pc5_lo, pc5_hi); \
290 pc5 += sl4 * (sri1 + sri6); \
291 pc5_lo = wasm_i64x2_add(pc5_lo, wasm_u64x2_extmul_low_u32x4(ql4, wasm_i32x4_add(qri1, qri6))); \
292 pc5_hi = wasm_i64x2_add(pc5_hi, wasm_u64x2_extmul_high_u32x4(ql4, wasm_i32x4_add(qri1, qri6))); \
293 vector_field_detail::bb_vf_barrier_sqq(pc5, pc5_lo, pc5_hi); \
294 uint64_t pc6 = (sl2 + sl7) * sri4; \
295 v128_t pc6_lo = wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql2, ql7), qri4); \
296 v128_t pc6_hi = wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql2, ql7), qri4); \
297 vector_field_detail::bb_vf_barrier_sqq(pc6, pc6_lo, pc6_hi); \
298 pc6 += (sl3 + sl8) * (sri3 + sri8); \
299 pc6_lo = wasm_i64x2_add(pc6_lo, wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql3, ql8), wasm_i32x4_add(qri3, qri8))); \
300 pc6_hi = wasm_i64x2_add(pc6_hi, wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql3, ql8), wasm_i32x4_add(qri3, qri8))); \
301 vector_field_detail::bb_vf_barrier_sqq(pc6, pc6_lo, pc6_hi); \
302 pc6 += sl4 * (sri2 + sri7); \
303 pc6_lo = wasm_i64x2_add(pc6_lo, wasm_u64x2_extmul_low_u32x4(ql4, wasm_i32x4_add(qri2, qri7))); \
304 pc6_hi = wasm_i64x2_add(pc6_hi, wasm_u64x2_extmul_high_u32x4(ql4, wasm_i32x4_add(qri2, qri7))); \
305 vector_field_detail::bb_vf_barrier_sqq(pc6, pc6_lo, pc6_hi); \
306 uint64_t pc7 = (sl3 + sl8) * sri4; \
307 v128_t pc7_lo = wasm_u64x2_extmul_low_u32x4(wasm_i32x4_add(ql3, ql8), qri4); \
308 v128_t pc7_hi = wasm_u64x2_extmul_high_u32x4(wasm_i32x4_add(ql3, ql8), qri4); \
309 vector_field_detail::bb_vf_barrier_sqq(pc7, pc7_lo, pc7_hi); \
310 pc7 += sl4 * (sri3 + sri8); \
311 pc7_lo = wasm_i64x2_add(pc7_lo, wasm_u64x2_extmul_low_u32x4(ql4, wasm_i32x4_add(qri3, qri8))); \
312 pc7_hi = wasm_i64x2_add(pc7_hi, wasm_u64x2_extmul_high_u32x4(ql4, wasm_i32x4_add(qri3, qri8))); \
313 vector_field_detail::bb_vf_barrier_sqq(pc7, pc7_lo, pc7_hi); \
314 uint64_t pc8 = sl4 * sri4; \
315 v128_t pc8_lo = wasm_u64x2_extmul_low_u32x4(ql4, qri4); \
316 v128_t pc8_hi = wasm_u64x2_extmul_high_u32x4(ql4, qri4); \
317 vector_field_detail::bb_vf_barrier_sqq(pc8, pc8_lo, pc8_hi)
318
319// Load the 9-scalar-limb + 9-quad-limb input from LEFT and RIGHT references
320// into named locals used by BB_VF_KARATSUBA_STAGES_1_4.
321#define BB_VF_LOAD_LIMBS(LEFT, RIGHT) \
322 const uint64_t sl0 = (LEFT).scalar_data[0], sl1 = (LEFT).scalar_data[1], sl2 = (LEFT).scalar_data[2], \
323 sl3 = (LEFT).scalar_data[3], sl4 = (LEFT).scalar_data[4], sl5 = (LEFT).scalar_data[5], \
324 sl6 = (LEFT).scalar_data[6], sl7 = (LEFT).scalar_data[7], sl8 = (LEFT).scalar_data[8]; \
325 const uint64_t sri0 = (RIGHT).scalar_data[0], sri1 = (RIGHT).scalar_data[1], sri2 = (RIGHT).scalar_data[2], \
326 sri3 = (RIGHT).scalar_data[3], sri4 = (RIGHT).scalar_data[4], sri5 = (RIGHT).scalar_data[5], \
327 sri6 = (RIGHT).scalar_data[6], sri7 = (RIGHT).scalar_data[7], sri8 = (RIGHT).scalar_data[8]; \
328 const v128_t ql0 = (LEFT).quad_data[0], ql1 = (LEFT).quad_data[1], ql2 = (LEFT).quad_data[2], \
329 ql3 = (LEFT).quad_data[3], ql4 = (LEFT).quad_data[4], ql5 = (LEFT).quad_data[5], \
330 ql6 = (LEFT).quad_data[6], ql7 = (LEFT).quad_data[7], ql8 = (LEFT).quad_data[8]; \
331 const v128_t qri0 = (RIGHT).quad_data[0], qri1 = (RIGHT).quad_data[1], qri2 = (RIGHT).quad_data[2], \
332 qri3 = (RIGHT).quad_data[3], qri4 = (RIGHT).quad_data[4], qri5 = (RIGHT).quad_data[5], \
333 qri6 = (RIGHT).quad_data[6], qri7 = (RIGHT).quad_data[7], qri8 = (RIGHT).quad_data[8]
334// clang-format on
335
336// Body of Stages 6..10. Takes 17 `temp_m` accumulators (scalar + tlo + thi) by
337// named local reference, runs 8 Yuval reductions, 1 wasm_reduce, carry
338// propagation, and stores into `result`. The Yuval barriers / asm "+r"
339// barriers on km_q and r_inv_* are load-bearing: they force LLVM to emit fast
340// extmul_low/high_u32x4 (pmuludq) instead of slow i64x2.mul.
341// clang-format off
342#define BB_VF_RUN_STAGES_6_THROUGH_10() \
343 do { \
344 constexpr uint64_t MASK29 = 0x1fffffffULL; \
345 const v128_t mask29_i32x4 = wasm_i32x4_splat(0x1fffffff); \
346 \
347 v128_t r_inv0 = wasm_i32x4_splat(static_cast<int32_t>(R_INV_WASM[0])); \
348 v128_t r_inv1 = wasm_i32x4_splat(static_cast<int32_t>(R_INV_WASM[1])); \
349 v128_t r_inv2 = wasm_i32x4_splat(static_cast<int32_t>(R_INV_WASM[2])); \
350 v128_t r_inv3 = wasm_i32x4_splat(static_cast<int32_t>(R_INV_WASM[3])); \
351 v128_t r_inv4 = wasm_i32x4_splat(static_cast<int32_t>(R_INV_WASM[4])); \
352 v128_t r_inv5 = wasm_i32x4_splat(static_cast<int32_t>(R_INV_WASM[5])); \
353 v128_t r_inv6 = wasm_i32x4_splat(static_cast<int32_t>(R_INV_WASM[6])); \
354 v128_t r_inv7 = wasm_i32x4_splat(static_cast<int32_t>(R_INV_WASM[7])); \
355 v128_t r_inv8 = wasm_i32x4_splat(static_cast<int32_t>(R_INV_WASM[8])); \
356 asm volatile("" : "+r"(r_inv0), "+r"(r_inv1), "+r"(r_inv2), "+r"(r_inv3), "+r"(r_inv4)); \
357 asm volatile("" : "+r"(r_inv5), "+r"(r_inv6), "+r"(r_inv7), "+r"(r_inv8)); \
358 \
359 BB_VF_DP_YUVAL(0, 1, 2, 3, 4, 5, 6, 7, 8, 9) \
360 BB_VF_DP_YUVAL(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) \
361 BB_VF_DP_YUVAL(2, 3, 4, 5, 6, 7, 8, 9, 10, 11) \
362 BB_VF_DP_YUVAL(3, 4, 5, 6, 7, 8, 9, 10, 11, 12) \
363 BB_VF_DP_YUVAL(4, 5, 6, 7, 8, 9, 10, 11, 12, 13) \
364 BB_VF_DP_YUVAL(5, 6, 7, 8, 9, 10, 11, 12, 13, 14) \
365 BB_VF_DP_YUVAL(6, 7, 8, 9, 10, 11, 12, 13, 14, 15) \
366 BB_VF_DP_YUVAL(7, 8, 9, 10, 11, 12, 13, 14, 15, 16) \
367 \
368 { \
369 const uint64_t rk_s = (temp_8 * R_INV_MOD_2_29) & MASK29; \
370 const v128_t rinv_splat = wasm_i32x4_splat(static_cast<int32_t>(R_INV_MOD_2_29)); \
371 const v128_t t8_i32x4 = \
372 wasm_i8x16_shuffle(tlo_8, thi_8, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27); \
373 const v128_t rk_q = wasm_v128_and(wasm_i32x4_mul(t8_i32x4, rinv_splat), mask29_i32x4); \
374 \
375 v128_t p0_splat = wasm_i32x4_splat(static_cast<int32_t>(P_WASM[0])); \
376 v128_t p1_splat = wasm_i32x4_splat(static_cast<int32_t>(P_WASM[1])); \
377 v128_t p2_splat = wasm_i32x4_splat(static_cast<int32_t>(P_WASM[2])); \
378 v128_t p3_splat = wasm_i32x4_splat(static_cast<int32_t>(P_WASM[3])); \
379 v128_t p4_splat = wasm_i32x4_splat(static_cast<int32_t>(P_WASM[4])); \
380 v128_t p5_splat = wasm_i32x4_splat(static_cast<int32_t>(P_WASM[5])); \
381 v128_t p6_splat = wasm_i32x4_splat(static_cast<int32_t>(P_WASM[6])); \
382 v128_t p7_splat = wasm_i32x4_splat(static_cast<int32_t>(P_WASM[7])); \
383 v128_t p8_splat = wasm_i32x4_splat(static_cast<int32_t>(P_WASM[8])); \
384 asm volatile("" : "+r"(p0_splat), "+r"(p1_splat), "+r"(p2_splat), "+r"(p3_splat), "+r"(p4_splat)); \
385 asm volatile("" : "+r"(p5_splat), "+r"(p6_splat), "+r"(p7_splat), "+r"(p8_splat)); \
386 \
387 temp_8 += rk_s * P_WASM[0]; \
388 tlo_8 = wasm_i64x2_add(tlo_8, wasm_u64x2_extmul_low_u32x4(rk_q, p0_splat)); \
389 thi_8 = wasm_i64x2_add(thi_8, wasm_u64x2_extmul_high_u32x4(rk_q, p0_splat)); \
390 vector_field_detail::bb_vf_barrier_sqq(temp_8, tlo_8, thi_8); \
391 \
392 temp_9 += rk_s * P_WASM[1] + (temp_8 >> 29); \
393 tlo_9 = wasm_i64x2_add(wasm_i64x2_add(tlo_9, wasm_u64x2_extmul_low_u32x4(rk_q, p1_splat)), \
394 wasm_u64x2_shr(tlo_8, 29)); \
395 thi_9 = wasm_i64x2_add(wasm_i64x2_add(thi_9, wasm_u64x2_extmul_high_u32x4(rk_q, p1_splat)), \
396 wasm_u64x2_shr(thi_8, 29)); \
397 vector_field_detail::bb_vf_barrier_sqq(temp_9, tlo_9, thi_9); \
398 \
399 temp_10 += rk_s * P_WASM[2]; \
400 tlo_10 = wasm_i64x2_add(tlo_10, wasm_u64x2_extmul_low_u32x4(rk_q, p2_splat)); \
401 thi_10 = wasm_i64x2_add(thi_10, wasm_u64x2_extmul_high_u32x4(rk_q, p2_splat)); \
402 vector_field_detail::bb_vf_barrier_sqq(temp_10, tlo_10, thi_10); \
403 \
404 temp_11 += rk_s * P_WASM[3]; \
405 tlo_11 = wasm_i64x2_add(tlo_11, wasm_u64x2_extmul_low_u32x4(rk_q, p3_splat)); \
406 thi_11 = wasm_i64x2_add(thi_11, wasm_u64x2_extmul_high_u32x4(rk_q, p3_splat)); \
407 vector_field_detail::bb_vf_barrier_sqq(temp_11, tlo_11, thi_11); \
408 \
409 temp_12 += rk_s * P_WASM[4]; \
410 tlo_12 = wasm_i64x2_add(tlo_12, wasm_u64x2_extmul_low_u32x4(rk_q, p4_splat)); \
411 thi_12 = wasm_i64x2_add(thi_12, wasm_u64x2_extmul_high_u32x4(rk_q, p4_splat)); \
412 vector_field_detail::bb_vf_barrier_sqq(temp_12, tlo_12, thi_12); \
413 \
414 temp_13 += rk_s * P_WASM[5]; \
415 tlo_13 = wasm_i64x2_add(tlo_13, wasm_u64x2_extmul_low_u32x4(rk_q, p5_splat)); \
416 thi_13 = wasm_i64x2_add(thi_13, wasm_u64x2_extmul_high_u32x4(rk_q, p5_splat)); \
417 vector_field_detail::bb_vf_barrier_sqq(temp_13, tlo_13, thi_13); \
418 \
419 temp_14 += rk_s * P_WASM[6]; \
420 tlo_14 = wasm_i64x2_add(tlo_14, wasm_u64x2_extmul_low_u32x4(rk_q, p6_splat)); \
421 thi_14 = wasm_i64x2_add(thi_14, wasm_u64x2_extmul_high_u32x4(rk_q, p6_splat)); \
422 vector_field_detail::bb_vf_barrier_sqq(temp_14, tlo_14, thi_14); \
423 \
424 temp_15 += rk_s * P_WASM[7]; \
425 tlo_15 = wasm_i64x2_add(tlo_15, wasm_u64x2_extmul_low_u32x4(rk_q, p7_splat)); \
426 thi_15 = wasm_i64x2_add(thi_15, wasm_u64x2_extmul_high_u32x4(rk_q, p7_splat)); \
427 vector_field_detail::bb_vf_barrier_sqq(temp_15, tlo_15, thi_15); \
428 \
429 temp_16 += rk_s * P_WASM[8]; \
430 tlo_16 = wasm_i64x2_add(tlo_16, wasm_u64x2_extmul_low_u32x4(rk_q, p8_splat)); \
431 thi_16 = wasm_i64x2_add(thi_16, wasm_u64x2_extmul_high_u32x4(rk_q, p8_splat)); \
432 vector_field_detail::bb_vf_barrier_sqq(temp_16, tlo_16, thi_16); \
433 } \
434 \
435 temp_10 += temp_9 >> 29; \
436 tlo_10 = wasm_i64x2_add(tlo_10, wasm_u64x2_shr(tlo_9, 29)); \
437 thi_10 = wasm_i64x2_add(thi_10, wasm_u64x2_shr(thi_9, 29)); \
438 temp_11 += temp_10 >> 29; \
439 tlo_11 = wasm_i64x2_add(tlo_11, wasm_u64x2_shr(tlo_10, 29)); \
440 thi_11 = wasm_i64x2_add(thi_11, wasm_u64x2_shr(thi_10, 29)); \
441 temp_12 += temp_11 >> 29; \
442 tlo_12 = wasm_i64x2_add(tlo_12, wasm_u64x2_shr(tlo_11, 29)); \
443 thi_12 = wasm_i64x2_add(thi_12, wasm_u64x2_shr(thi_11, 29)); \
444 temp_13 += temp_12 >> 29; \
445 tlo_13 = wasm_i64x2_add(tlo_13, wasm_u64x2_shr(tlo_12, 29)); \
446 thi_13 = wasm_i64x2_add(thi_13, wasm_u64x2_shr(thi_12, 29)); \
447 temp_14 += temp_13 >> 29; \
448 tlo_14 = wasm_i64x2_add(tlo_14, wasm_u64x2_shr(tlo_13, 29)); \
449 thi_14 = wasm_i64x2_add(thi_14, wasm_u64x2_shr(thi_13, 29)); \
450 temp_15 += temp_14 >> 29; \
451 tlo_15 = wasm_i64x2_add(tlo_15, wasm_u64x2_shr(tlo_14, 29)); \
452 thi_15 = wasm_i64x2_add(thi_15, wasm_u64x2_shr(thi_14, 29)); \
453 temp_16 += temp_15 >> 29; \
454 tlo_16 = wasm_i64x2_add(tlo_16, wasm_u64x2_shr(tlo_15, 29)); \
455 thi_16 = wasm_i64x2_add(thi_16, wasm_u64x2_shr(thi_15, 29)); \
456 \
457 const uint64_t temp_17 = temp_16 >> 29; \
458 const v128_t tlo_17 = wasm_u64x2_shr(tlo_16, 29); \
459 const v128_t thi_17 = wasm_u64x2_shr(thi_16, 29); \
460 \
461 result.scalar_data[0] = static_cast<uint32_t>(temp_9) & static_cast<uint32_t>(MASK29); \
462 result.quad_data[0] = wasm_v128_and( \
463 wasm_i8x16_shuffle(tlo_9, thi_9, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27), mask29_i32x4); \
464 result.scalar_data[1] = static_cast<uint32_t>(temp_10) & static_cast<uint32_t>(MASK29); \
465 result.quad_data[1] = wasm_v128_and( \
466 wasm_i8x16_shuffle(tlo_10, thi_10, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27), \
467 mask29_i32x4); \
468 result.scalar_data[2] = static_cast<uint32_t>(temp_11) & static_cast<uint32_t>(MASK29); \
469 result.quad_data[2] = wasm_v128_and( \
470 wasm_i8x16_shuffle(tlo_11, thi_11, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27), \
471 mask29_i32x4); \
472 result.scalar_data[3] = static_cast<uint32_t>(temp_12) & static_cast<uint32_t>(MASK29); \
473 result.quad_data[3] = wasm_v128_and( \
474 wasm_i8x16_shuffle(tlo_12, thi_12, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27), \
475 mask29_i32x4); \
476 result.scalar_data[4] = static_cast<uint32_t>(temp_13) & static_cast<uint32_t>(MASK29); \
477 result.quad_data[4] = wasm_v128_and( \
478 wasm_i8x16_shuffle(tlo_13, thi_13, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27), \
479 mask29_i32x4); \
480 result.scalar_data[5] = static_cast<uint32_t>(temp_14) & static_cast<uint32_t>(MASK29); \
481 result.quad_data[5] = wasm_v128_and( \
482 wasm_i8x16_shuffle(tlo_14, thi_14, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27), \
483 mask29_i32x4); \
484 result.scalar_data[6] = static_cast<uint32_t>(temp_15) & static_cast<uint32_t>(MASK29); \
485 result.quad_data[6] = wasm_v128_and( \
486 wasm_i8x16_shuffle(tlo_15, thi_15, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27), \
487 mask29_i32x4); \
488 result.scalar_data[7] = static_cast<uint32_t>(temp_16) & static_cast<uint32_t>(MASK29); \
489 result.quad_data[7] = wasm_v128_and( \
490 wasm_i8x16_shuffle(tlo_16, thi_16, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27), \
491 mask29_i32x4); \
492 result.scalar_data[8] = static_cast<uint32_t>(temp_17); \
493 result.quad_data[8] = \
494 wasm_i8x16_shuffle(tlo_17, thi_17, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27); \
495 } while (0)
496
497// Single Yuval reduction step. `lo`, `p1..p9` are the temp/tlo/thi indices.
498// `p1 == lo+1`, ..., `p9 == lo+9`, passed explicitly so the macro token-pastes
499// cleanly. km_q must be kept as i32x4: without the asm "+r"(km_q) barriers
500// between the 9 multiplier uses, LLVM pre-extends km_q once to i64x2 and emits
501// 9x slow i64x2.mul instead of 9x fast extmul_low/high_u32x4 (pmuludq).
502#define BB_VF_DP_YUVAL(lo, p1, p2, p3, p4, p5, p6, p7, p8, p9) \
503 { \
504 const uint64_t km_s = temp_##lo & MASK29; \
505 const uint64_t carry_s = temp_##lo >> 29; \
506 v128_t km_q_raw = \
507 wasm_i8x16_shuffle(tlo_##lo, thi_##lo, 0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27); \
508 v128_t km_q = wasm_v128_and(km_q_raw, mask29_i32x4); \
509 v128_t carry_q_lo = wasm_u64x2_shr(tlo_##lo, 29); \
510 v128_t carry_q_hi = wasm_u64x2_shr(thi_##lo, 29); \
511 temp_##p1 += km_s * R_INV_WASM[0] + carry_s; \
512 tlo_##p1 = wasm_i64x2_add(wasm_i64x2_add(tlo_##p1, wasm_u64x2_extmul_low_u32x4(km_q, r_inv0)), carry_q_lo); \
513 thi_##p1 = wasm_i64x2_add(wasm_i64x2_add(thi_##p1, wasm_u64x2_extmul_high_u32x4(km_q, r_inv0)), carry_q_hi); \
514 vector_field_detail::bb_vf_barrier_sqq(temp_##p1, tlo_##p1, thi_##p1); \
515 asm volatile("" : "+r"(km_q)); \
516 temp_##p2 += km_s * R_INV_WASM[1]; \
517 tlo_##p2 = wasm_i64x2_add(tlo_##p2, wasm_u64x2_extmul_low_u32x4(km_q, r_inv1)); \
518 thi_##p2 = wasm_i64x2_add(thi_##p2, wasm_u64x2_extmul_high_u32x4(km_q, r_inv1)); \
519 asm volatile("" : "+r"(km_q)); \
520 temp_##p3 += km_s * R_INV_WASM[2]; \
521 tlo_##p3 = wasm_i64x2_add(tlo_##p3, wasm_u64x2_extmul_low_u32x4(km_q, r_inv2)); \
522 thi_##p3 = wasm_i64x2_add(thi_##p3, wasm_u64x2_extmul_high_u32x4(km_q, r_inv2)); \
523 vector_field_detail::bb_vf_barrier_sqq(temp_##p3, tlo_##p3, thi_##p3); \
524 asm volatile("" : "+r"(km_q)); \
525 temp_##p4 += km_s * R_INV_WASM[3]; \
526 tlo_##p4 = wasm_i64x2_add(tlo_##p4, wasm_u64x2_extmul_low_u32x4(km_q, r_inv3)); \
527 thi_##p4 = wasm_i64x2_add(thi_##p4, wasm_u64x2_extmul_high_u32x4(km_q, r_inv3)); \
528 asm volatile("" : "+r"(km_q)); \
529 temp_##p5 += km_s * R_INV_WASM[4]; \
530 tlo_##p5 = wasm_i64x2_add(tlo_##p5, wasm_u64x2_extmul_low_u32x4(km_q, r_inv4)); \
531 thi_##p5 = wasm_i64x2_add(thi_##p5, wasm_u64x2_extmul_high_u32x4(km_q, r_inv4)); \
532 vector_field_detail::bb_vf_barrier_sqq(temp_##p5, tlo_##p5, thi_##p5); \
533 asm volatile("" : "+r"(km_q)); \
534 temp_##p6 += km_s * R_INV_WASM[5]; \
535 tlo_##p6 = wasm_i64x2_add(tlo_##p6, wasm_u64x2_extmul_low_u32x4(km_q, r_inv5)); \
536 thi_##p6 = wasm_i64x2_add(thi_##p6, wasm_u64x2_extmul_high_u32x4(km_q, r_inv5)); \
537 asm volatile("" : "+r"(km_q)); \
538 temp_##p7 += km_s * R_INV_WASM[6]; \
539 tlo_##p7 = wasm_i64x2_add(tlo_##p7, wasm_u64x2_extmul_low_u32x4(km_q, r_inv6)); \
540 thi_##p7 = wasm_i64x2_add(thi_##p7, wasm_u64x2_extmul_high_u32x4(km_q, r_inv6)); \
541 vector_field_detail::bb_vf_barrier_sqq(temp_##p7, tlo_##p7, thi_##p7); \
542 asm volatile("" : "+r"(km_q)); \
543 temp_##p8 += km_s * R_INV_WASM[7]; \
544 tlo_##p8 = wasm_i64x2_add(tlo_##p8, wasm_u64x2_extmul_low_u32x4(km_q, r_inv7)); \
545 thi_##p8 = wasm_i64x2_add(thi_##p8, wasm_u64x2_extmul_high_u32x4(km_q, r_inv7)); \
546 asm volatile("" : "+r"(km_q)); \
547 temp_##p9 += km_s * R_INV_WASM[8]; \
548 tlo_##p9 = wasm_i64x2_add(tlo_##p9, wasm_u64x2_extmul_low_u32x4(km_q, r_inv8)); \
549 thi_##p9 = wasm_i64x2_add(thi_##p9, wasm_u64x2_extmul_high_u32x4(km_q, r_inv8)); \
550 vector_field_detail::bb_vf_barrier_sqq(temp_##p9, tlo_##p9, thi_##p9); \
551 }
552// clang-format on
553// operator* specializations. The body is shared via .inl.hpp include so that
554// each specialization stamps the same kernel against its own Params constants
555// (R_INV_WASM, P_WASM, R_INV_MOD_2_29 resolve in the enclosing class scope).
556//
557// Each must be its own explicit specialization so LLVM/Clang emit each kernel
558// as a standalone function in this TU — preserving the register-allocation
559// scope that lets V8 reproduce the gist's hand-scheduled WAT layout.
560
561template <>
562VectorField<Bn254FrParams> VectorField<Bn254FrParams>::operator*(const VectorField<Bn254FrParams>& other) const noexcept
563{
564#include "barretenberg/ecc/fields/vector_field_mont_mul_body.inl.hpp"
565}
566
567template <>
568VectorField<Bn254FqParams> VectorField<Bn254FqParams>::operator*(const VectorField<Bn254FqParams>& other) const noexcept
569{
570#include "barretenberg/ecc/fields/vector_field_mont_mul_body.inl.hpp"
571}
572
573#undef BB_VF_RUN_STAGES_6_THROUGH_10
574#undef BB_VF_KARATSUBA_STAGES_1_4
575#undef BB_VF_LOAD_LIMBS
576
577} // namespace bb
578
579#endif // __wasm_simd128__
bb
Entry point for Barretenberg command-line interface.
Definition api.hpp:5
bb::VectorField::operator*
friend VectorField operator*(VectorField v, const Field &s) noexcept
Definition vector_field.hpp:364
vector_field.hpp
vector_field_mont_mul_body.inl.hpp