Skip to content

Commit b08df58

Browse files
martin-frbgmartin-frbg
authored
Merge pull request #5972 from Orcina-Ltd/asum-alignment-determinism
Title: kernel/x86_64: make AVX-512 asum/sum kernels independent of buffer alignment
2 parents 3777530 + d793be8 commit b08df58

6 files changed

Lines changed: 142 additions & 180 deletions

‎kernel/x86_64/casum_microk_skylakex-2.c‎

Lines changed: 31 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
130130
accum_2 = _mm512_setzero_ps();
131131
accum_3 = _mm512_setzero_ps();
132132

133-
// alignment has side-effect when the size of input array is not large enough
133+
// shorter unrolled path for small inputs
134134
if (n2 < 256) {
135135
if (n2 >= 128) {
136136
x00 = _mm512_loadu_ps(&x1[ 0]);
@@ -217,29 +217,22 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
217217

218218
sumf = _mm512_reduce_add_ps(accum_0);
219219
}
220-
// n2 >= 256, doing alignment
220+
/* n2 >= 256. Deliberately no peeling to an alignment boundary:
221+
peeling makes the grouping of the sum into accumulators - and
222+
hence the rounding of the result - depend on the buffer address.
223+
Unaligned loads keep the result a function of n alone, and cost
224+
the same as aligned loads when the address happens to be
225+
aligned. */
221226
else {
222227

223-
int align_header = ((64 - ((uintptr_t)x1 & (uintptr_t)0x3f)) >> 2) & 0xf;
224-
225-
if (0 != align_header) {
226-
uint16_t align_mask16 = (((uint16_t)0xffff) >> (16 - align_header));
227-
x00 = _mm512_maskz_loadu_ps(*((__mmask16*) &align_mask16), &x1[0]);
228-
x00 = _mm512_and_ps(x00, abs_mask);
229-
accum_0 = _mm512_add_ps(accum_0, x00);
230-
231-
n2 -= align_header;
232-
x1 += align_header;
233-
}
234-
235-
x00 = _mm512_load_ps(&x1[ 0]);
236-
x01 = _mm512_load_ps(&x1[ 16]);
237-
x02 = _mm512_load_ps(&x1[ 32]);
238-
x03 = _mm512_load_ps(&x1[ 48]);
239-
x04 = _mm512_load_ps(&x1[ 64]);
240-
x05 = _mm512_load_ps(&x1[ 80]);
241-
x06 = _mm512_load_ps(&x1[ 96]);
242-
x07 = _mm512_load_ps(&x1[112]);
228+
x00 = _mm512_loadu_ps(&x1[ 0]);
229+
x01 = _mm512_loadu_ps(&x1[ 16]);
230+
x02 = _mm512_loadu_ps(&x1[ 32]);
231+
x03 = _mm512_loadu_ps(&x1[ 48]);
232+
x04 = _mm512_loadu_ps(&x1[ 64]);
233+
x05 = _mm512_loadu_ps(&x1[ 80]);
234+
x06 = _mm512_loadu_ps(&x1[ 96]);
235+
x07 = _mm512_loadu_ps(&x1[112]);
243236

244237
n2 -= 128;
245238
x1 += 128;
@@ -251,26 +244,26 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
251244
x03 = _mm512_and_ps(x03, abs_mask);
252245

253246
accum_0 = _mm512_add_ps(accum_0, x00);
254-
x00 = _mm512_load_ps(&x1[ 0]);
247+
x00 = _mm512_loadu_ps(&x1[ 0]);
255248
accum_1 = _mm512_add_ps(accum_1, x01);
256-
x01 = _mm512_load_ps(&x1[ 16]);
249+
x01 = _mm512_loadu_ps(&x1[ 16]);
257250
accum_2 = _mm512_add_ps(accum_2, x02);
258-
x02 = _mm512_load_ps(&x1[ 32]);
251+
x02 = _mm512_loadu_ps(&x1[ 32]);
259252
accum_3 = _mm512_add_ps(accum_3, x03);
260-
x03 = _mm512_load_ps(&x1[ 48]);
253+
x03 = _mm512_loadu_ps(&x1[ 48]);
261254

262255
x04 = _mm512_and_ps(x04, abs_mask);
263256
x05 = _mm512_and_ps(x05, abs_mask);
264257
x06 = _mm512_and_ps(x06, abs_mask);
265258
x07 = _mm512_and_ps(x07, abs_mask);
266259
accum_0 = _mm512_add_ps(accum_0, x04);
267-
x04 = _mm512_load_ps(&x1[ 64]);
260+
x04 = _mm512_loadu_ps(&x1[ 64]);
268261
accum_1 = _mm512_add_ps(accum_1, x05);
269-
x05 = _mm512_load_ps(&x1[ 80]);
262+
x05 = _mm512_loadu_ps(&x1[ 80]);
270263
accum_2 = _mm512_add_ps(accum_2, x06);
271-
x06 = _mm512_load_ps(&x1[ 96]);
264+
x06 = _mm512_loadu_ps(&x1[ 96]);
272265
accum_3 = _mm512_add_ps(accum_3, x07);
273-
x07 = _mm512_load_ps(&x1[112]);
266+
x07 = _mm512_loadu_ps(&x1[112]);
274267

275268
n2 -= 128;
276269
x1 += 128;
@@ -296,10 +289,10 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
296289
accum_3 = _mm512_add_ps(accum_3, x07);
297290

298291
if (n2 >= 64) {
299-
x00 = _mm512_load_ps(&x1[ 0]);
300-
x01 = _mm512_load_ps(&x1[16]);
301-
x02 = _mm512_load_ps(&x1[32]);
302-
x03 = _mm512_load_ps(&x1[48]);
292+
x00 = _mm512_loadu_ps(&x1[ 0]);
293+
x01 = _mm512_loadu_ps(&x1[16]);
294+
x02 = _mm512_loadu_ps(&x1[32]);
295+
x03 = _mm512_loadu_ps(&x1[48]);
303296
x00 = _mm512_and_ps(x00, abs_mask);
304297
x01 = _mm512_and_ps(x01, abs_mask);
305298
x02 = _mm512_and_ps(x02, abs_mask);
@@ -314,8 +307,8 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
314307
}
315308

316309
if (n2 >= 32) {
317-
x00 = _mm512_load_ps(&x1[ 0]);
318-
x01 = _mm512_load_ps(&x1[16]);
310+
x00 = _mm512_loadu_ps(&x1[ 0]);
311+
x01 = _mm512_loadu_ps(&x1[16]);
319312
x00 = _mm512_and_ps(x00, abs_mask);
320313
x01 = _mm512_and_ps(x01, abs_mask);
321314
accum_0 = _mm512_add_ps(accum_0, x00);
@@ -326,7 +319,7 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
326319
}
327320

328321
if (n2 >= 16) {
329-
x00 = _mm512_load_ps(&x1[ 0]);
322+
x00 = _mm512_loadu_ps(&x1[ 0]);
330323
x00 = _mm512_and_ps(x00, abs_mask);
331324
accum_0 = _mm512_add_ps(accum_0, x00);
332325

@@ -336,7 +329,7 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
336329

337330
if (n2) {
338331
uint16_t tail_mask16 = (((uint16_t) 0xffff) >> (16 - n2));
339-
x00 = _mm512_maskz_load_ps(*((__mmask16*) &tail_mask16), &x1[ 0]);
332+
x00 = _mm512_maskz_loadu_ps(*((__mmask16*) &tail_mask16), &x1[ 0]);
340333
x00 = _mm512_and_ps(x00, abs_mask);
341334
accum_0 = _mm512_add_ps(accum_0, x00);
342335
}

‎kernel/x86_64/csum_microk_skylakex-2.c‎

Lines changed: 31 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
109109
accum_2 = _mm512_setzero_ps();
110110
accum_3 = _mm512_setzero_ps();
111111

112-
// alignment has side-effect when the size of input array is not large enough
112+
// shorter unrolled path for small inputs
113113
if (n2 < 256) {
114114
if (n2 >= 128) {
115115
x00 = _mm512_loadu_ps(&x1[ 0]);
@@ -178,51 +178,45 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
178178

179179
sumf = _mm512_reduce_add_ps(accum_0);
180180
}
181-
// n2 >= 256, doing alignment
181+
/* n2 >= 256. Deliberately no peeling to an alignment boundary:
182+
peeling makes the grouping of the sum into accumulators - and
183+
hence the rounding of the result - depend on the buffer address.
184+
Unaligned loads keep the result a function of n alone, and cost
185+
the same as aligned loads when the address happens to be
186+
aligned. */
182187
else {
183188

184-
int align_header = ((64 - ((uintptr_t)x1 & (uintptr_t)0x3f)) >> 2) & 0xf;
185-
186-
if (0 != align_header) {
187-
uint16_t align_mask16 = (((uint16_t)0xffff) >> (16 - align_header));
188-
x00 = _mm512_maskz_loadu_ps(*((__mmask16*) &align_mask16), &x1[0]);
189-
accum_0 = _mm512_add_ps(accum_0, x00);
190-
191-
n2 -= align_header;
192-
x1 += align_header;
193-
}
194-
195-
x00 = _mm512_load_ps(&x1[ 0]);
196-
x01 = _mm512_load_ps(&x1[ 16]);
197-
x02 = _mm512_load_ps(&x1[ 32]);
198-
x03 = _mm512_load_ps(&x1[ 48]);
199-
x04 = _mm512_load_ps(&x1[ 64]);
200-
x05 = _mm512_load_ps(&x1[ 80]);
201-
x06 = _mm512_load_ps(&x1[ 96]);
202-
x07 = _mm512_load_ps(&x1[112]);
189+
x00 = _mm512_loadu_ps(&x1[ 0]);
190+
x01 = _mm512_loadu_ps(&x1[ 16]);
191+
x02 = _mm512_loadu_ps(&x1[ 32]);
192+
x03 = _mm512_loadu_ps(&x1[ 48]);
193+
x04 = _mm512_loadu_ps(&x1[ 64]);
194+
x05 = _mm512_loadu_ps(&x1[ 80]);
195+
x06 = _mm512_loadu_ps(&x1[ 96]);
196+
x07 = _mm512_loadu_ps(&x1[112]);
203197

204198
n2 -= 128;
205199
x1 += 128;
206200

207201
while (n2 >= 128) {
208202

209203
accum_0 = _mm512_add_ps(accum_0, x00);
210-
x00 = _mm512_load_ps(&x1[ 0]);
204+
x00 = _mm512_loadu_ps(&x1[ 0]);
211205
accum_1 = _mm512_add_ps(accum_1, x01);
212-
x01 = _mm512_load_ps(&x1[ 16]);
206+
x01 = _mm512_loadu_ps(&x1[ 16]);
213207
accum_2 = _mm512_add_ps(accum_2, x02);
214-
x02 = _mm512_load_ps(&x1[ 32]);
208+
x02 = _mm512_loadu_ps(&x1[ 32]);
215209
accum_3 = _mm512_add_ps(accum_3, x03);
216-
x03 = _mm512_load_ps(&x1[ 48]);
210+
x03 = _mm512_loadu_ps(&x1[ 48]);
217211

218212
accum_0 = _mm512_add_ps(accum_0, x04);
219-
x04 = _mm512_load_ps(&x1[ 64]);
213+
x04 = _mm512_loadu_ps(&x1[ 64]);
220214
accum_1 = _mm512_add_ps(accum_1, x05);
221-
x05 = _mm512_load_ps(&x1[ 80]);
215+
x05 = _mm512_loadu_ps(&x1[ 80]);
222216
accum_2 = _mm512_add_ps(accum_2, x06);
223-
x06 = _mm512_load_ps(&x1[ 96]);
217+
x06 = _mm512_loadu_ps(&x1[ 96]);
224218
accum_3 = _mm512_add_ps(accum_3, x07);
225-
x07 = _mm512_load_ps(&x1[112]);
219+
x07 = _mm512_loadu_ps(&x1[112]);
226220

227221
n2 -= 128;
228222
x1 += 128;
@@ -239,10 +233,10 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
239233
accum_3 = _mm512_add_ps(accum_3, x07);
240234

241235
if (n2 >= 64) {
242-
x00 = _mm512_load_ps(&x1[ 0]);
243-
x01 = _mm512_load_ps(&x1[16]);
244-
x02 = _mm512_load_ps(&x1[32]);
245-
x03 = _mm512_load_ps(&x1[48]);
236+
x00 = _mm512_loadu_ps(&x1[ 0]);
237+
x01 = _mm512_loadu_ps(&x1[16]);
238+
x02 = _mm512_loadu_ps(&x1[32]);
239+
x03 = _mm512_loadu_ps(&x1[48]);
246240
accum_0 = _mm512_add_ps(accum_0, x00);
247241
accum_1 = _mm512_add_ps(accum_1, x01);
248242
accum_2 = _mm512_add_ps(accum_2, x02);
@@ -253,8 +247,8 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
253247
}
254248

255249
if (n2 >= 32) {
256-
x00 = _mm512_load_ps(&x1[ 0]);
257-
x01 = _mm512_load_ps(&x1[16]);
250+
x00 = _mm512_loadu_ps(&x1[ 0]);
251+
x01 = _mm512_loadu_ps(&x1[16]);
258252
accum_0 = _mm512_add_ps(accum_0, x00);
259253
accum_1 = _mm512_add_ps(accum_1, x01);
260254

@@ -263,7 +257,7 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
263257
}
264258

265259
if (n2 >= 16) {
266-
x00 = _mm512_load_ps(&x1[ 0]);
260+
x00 = _mm512_loadu_ps(&x1[ 0]);
267261
accum_0 = _mm512_add_ps(accum_0, x00);
268262

269263
n2 -= 16;
@@ -272,7 +266,7 @@ static FLOAT casum_kernel(BLASLONG n, FLOAT *x)
272266

273267
if (n2) {
274268
uint16_t tail_mask16 = (((uint16_t) 0xffff) >> (16 - n2));
275-
x00 = _mm512_maskz_load_ps(*((__mmask16*) &tail_mask16), &x1[ 0]);
269+
x00 = _mm512_maskz_loadu_ps(*((__mmask16*) &tail_mask16), &x1[ 0]);
276270
accum_0 = _mm512_add_ps(accum_0, x00);
277271
}
278272

‎kernel/x86_64/dasum_microk_skylakex-2.c‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,14 @@ static FLOAT dasum_kernel(BLASLONG n, FLOAT *x1)
1919
BLASLONG i = 0;
2020
FLOAT sumf = 0.0;
2121

22-
if (n >= 256) {
23-
BLASLONG align_512 = ((64 - ((uintptr_t)x1 & (uintptr_t)0x3f)) >> 3) & 0x7;
24-
25-
for (i = 0; i < align_512; i++) {
26-
sumf += ABS_K(x1[i]);
27-
}
28-
29-
n -= align_512;
30-
x1 += align_512;
31-
}
32-
3322
BLASLONG tail_index_SSE = n&(~7);
3423
BLASLONG tail_index_AVX512 = n&(~255);
3524

36-
//
25+
/* Deliberately no peeling to an alignment boundary: peeling makes the
26+
grouping of the sum into accumulators - and hence the rounding of the
27+
result - depend on the buffer address. Unaligned loads keep the result
28+
a function of n alone, and cost the same as aligned loads when the
29+
address happens to be aligned. */
3730
if ( n >= 256 ) {
3831

3932
__m512d accum_0, accum_1, accum_2, accum_3;
@@ -42,10 +35,10 @@ static FLOAT dasum_kernel(BLASLONG n, FLOAT *x1)
4235
accum_2 = _mm512_setzero_pd();
4336
accum_3 = _mm512_setzero_pd();
4437
for (i = 0; i < tail_index_AVX512; i += 32) {
45-
accum_0 += _mm512_abs_pd(_mm512_load_pd(&x1[i + 0]));
46-
accum_1 += _mm512_abs_pd(_mm512_load_pd(&x1[i + 8]));
47-
accum_2 += _mm512_abs_pd(_mm512_load_pd(&x1[i +16]));
48-
accum_3 += _mm512_abs_pd(_mm512_load_pd(&x1[i +24]));
38+
accum_0 += _mm512_abs_pd(_mm512_loadu_pd(&x1[i + 0]));
39+
accum_1 += _mm512_abs_pd(_mm512_loadu_pd(&x1[i + 8]));
40+
accum_2 += _mm512_abs_pd(_mm512_loadu_pd(&x1[i +16]));
41+
accum_3 += _mm512_abs_pd(_mm512_loadu_pd(&x1[i +24]));
4942
}
5043

5144
accum_0 = accum_0 + accum_1 + accum_2 + accum_3;

‎kernel/x86_64/sasum_microk_skylakex-2.c‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,14 @@ static FLOAT sasum_kernel(BLASLONG n, FLOAT *x1)
1818
BLASLONG i = 0;
1919
FLOAT sumf = 0.0;
2020

21-
if (n >= 256) {
22-
BLASLONG align_512 = ((64 - ((uintptr_t)x1 & (uintptr_t)0x3f)) >> 2) & 0xf;
23-
24-
for (i = 0; i < align_512; i++) {
25-
sumf += ABS_K(x1[i]);
26-
}
27-
n -= align_512;
28-
x1 += align_512;
29-
}
30-
3121
BLASLONG tail_index_SSE = n&(~7);
3222
BLASLONG tail_index_AVX512 = n&(~255);
3323

24+
/* Deliberately no peeling to an alignment boundary: peeling makes the
25+
grouping of the sum into accumulators - and hence the rounding of the
26+
result - depend on the buffer address. Unaligned loads keep the result
27+
a function of n alone, and cost the same as aligned loads when the
28+
address happens to be aligned. */
3429
if (n >= 256) {
3530
__m512 accum_0, accum_1, accum_2, accum_3;
3631
accum_0 = _mm512_setzero_ps();
@@ -39,10 +34,10 @@ static FLOAT sasum_kernel(BLASLONG n, FLOAT *x1)
3934
accum_3 = _mm512_setzero_ps();
4035

4136
for (i = 0; i < tail_index_AVX512; i += 64) {
42-
accum_0 += _mm512_abs_ps(_mm512_load_ps(&x1[i + 0]));
43-
accum_1 += _mm512_abs_ps(_mm512_load_ps(&x1[i +16]));
44-
accum_2 += _mm512_abs_ps(_mm512_load_ps(&x1[i +32]));
45-
accum_3 += _mm512_abs_ps(_mm512_load_ps(&x1[i +48]));
37+
accum_0 += _mm512_abs_ps(_mm512_loadu_ps(&x1[i + 0]));
38+
accum_1 += _mm512_abs_ps(_mm512_loadu_ps(&x1[i +16]));
39+
accum_2 += _mm512_abs_ps(_mm512_loadu_ps(&x1[i +32]));
40+
accum_3 += _mm512_abs_ps(_mm512_loadu_ps(&x1[i +48]));
4641
}
4742

4843
accum_0 = accum_0 + accum_1 + accum_2 + accum_3;

0 commit comments

Comments
 (0)