diff --git a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_x86_64.inc b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_x86_64.inc
index d42e5ce..c9814ff 100644
--- a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_x86_64.inc
+++ b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSse2_x86_64.inc
@@ -10,11 +10,7 @@
// weight bytes via pmaddwd (SSE2), producing the same 4 x i32 weighted
// sums per 16-byte half that pmaddubsw + pmaddwd would yield.
- sub rsp, 64
- movdqu oword [rsp], xmm6
- movdqu oword [rsp + $10], xmm7
- movdqu oword [rsp + $20], xmm8
- movdqu oword [rsp + $30], xmm9
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
// Zero constant
pxor xmm3, xmm3
@@ -102,8 +98,4 @@
mov dword [r8], eax
mov dword [r8 + 4], r10d
- movdqu xmm6, oword [rsp]
- movdqu xmm7, oword [rsp + $10]
- movdqu xmm8, oword [rsp + $20]
- movdqu xmm9, oword [rsp + $30]
- add rsp, 64
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_x86_64.inc b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_x86_64.inc
index 0e07a73..0cd7015 100644
--- a/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_x86_64.inc
+++ b/HashLib/src/Include/Simd/Adler32/Adler32BlocksSsse3_x86_64.inc
@@ -5,10 +5,7 @@
// Processes num_blocks x 32-byte blocks. Does NOT apply mod 65521 (caller does it).
// Uses xmm0-xmm8; xmm6-xmm8 are MS x64 non-volatile (saved/restored).
- sub rsp, 48
- movdqu oword [rsp], xmm6
- movdqu oword [rsp + $10], xmm7
- movdqu oword [rsp + $20], xmm8
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
// Load constants
movdqu xmm4, oword [r9]
@@ -79,7 +76,4 @@
mov dword [r8], eax
mov dword [r8 + 4], r10d
- movdqu xmm6, oword [rsp]
- movdqu xmm7, oword [rsp + $10]
- movdqu xmm8, oword [rsp + $20]
- add rsp, 48
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockAvx2_x86_64.inc b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockAvx2_x86_64.inc
index 6c3597e..bf0d523 100644
--- a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockAvx2_x86_64.inc
+++ b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockAvx2_x86_64.inc
@@ -15,17 +15,14 @@
// Diagonalize/Undiagonalize via vpermq (cross-lane 64-bit permute).
// vzeroupper required before return.
- sub rsp, 2184
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
- db $C5, $FE, $7F, $34, $24 // vmovdqu yword [rsp], ymm6
- db $C5, $FE, $7F, $7C, $24, $20 // vmovdqu yword [rsp + $20], ymm7
- db $C5, $7E, $7F, $44, $24, $40 // vmovdqu yword [rsp + $40], ymm8
- db $C5, $7E, $7F, $4C, $24, $60 // vmovdqu yword [rsp + $60], ymm9
+ sub rsp, 2056
// =========================================================================
// Step 1: Compute R_buf = Left XOR Right, store at [rsp+128]
// =========================================================================
- lea rax, [rsp + 128]
+ lea rax, [rsp]
xor r10, r10
@xor_loop:
db $C4, $A1, $7E, $6F, $04, $11 // vmovdqu ymm0, yword [rcx + r10]
@@ -38,7 +35,7 @@
// =========================================================================
// Step 2: Copy R_buf to Z_buf at [rsp+1152]
// =========================================================================
- lea r11, [rsp + 1152]
+ lea r11, [rsp + 1024]
xor r10, r10
@copy_loop:
db $C4, $A1, $7E, $6F, $04, $10 // vmovdqu ymm0, yword [rax + r10]
@@ -326,10 +323,7 @@
jb @final_xor_loop
@epilogue:
- db $C5, $FE, $6F, $34, $24 // vmovdqu ymm6, yword [rsp]
- db $C5, $FE, $6F, $7C, $24, $20 // vmovdqu ymm7, yword [rsp + $20]
- db $C5, $7E, $6F, $44, $24, $40 // vmovdqu ymm8, yword [rsp + $40]
- db $C5, $7E, $6F, $4C, $24, $60 // vmovdqu ymm9, yword [rsp + $60]
-
- add rsp, 2184
+ add rsp, 2056
db $C5, $F8, $77 // vzeroupper
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_x86_64.inc b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_x86_64.inc
index 56557e7..d307e4a 100644
--- a/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_x86_64.inc
+++ b/HashLib/src/Include/Simd/Argon2/Argon2FillBlockSse2_x86_64.inc
@@ -16,17 +16,14 @@
// register pairs. Right-rotates require a 3-movdqa swap since shufpd
// naturally places results in the opposite register for that direction.
- sub rsp, 2120
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
- movdqu oword [rsp], xmm6
- movdqu oword [rsp + $10], xmm7
- movdqu oword [rsp + $20], xmm8
- movdqu oword [rsp + $30], xmm9
+ sub rsp, 2056
// =========================================================================
// Step 1: Compute R_buf = Left XOR Right, store at [rsp+64]
// =========================================================================
- lea rax, [rsp + 64]
+ lea rax, [rsp]
xor r10, r10
@xor_loop:
movdqu xmm0, oword [rcx + r10]
@@ -40,7 +37,7 @@
// =========================================================================
// Step 2: Copy R_buf to Z_buf at [rsp+1088]
// =========================================================================
- lea r11, [rsp + 1088]
+ lea r11, [rsp + 1024]
xor r10, r10
@copy_loop:
movdqu xmm0, oword [rax + r10]
@@ -556,9 +553,6 @@
jb @final_xor_loop
@epilogue:
- movdqu xmm6, oword [rsp]
- movdqu xmm7, oword [rsp + $10]
- movdqu xmm8, oword [rsp + $20]
- movdqu xmm9, oword [rsp + $30]
+ add rsp, 2056
- add rsp, 2120
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_x86_64.inc b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_x86_64.inc
index 0f4b66b..8f70d6c 100644
--- a/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_x86_64.inc
+++ b/HashLib/src/Include/Simd/Blake2B/Blake2BCompressSse2_x86_64.inc
@@ -5,11 +5,7 @@
// xmm4-5 = row3 (v8-11), xmm6-7 = row4 (v12-15), xmm8-9 = temps.
// Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves.
- sub rsp, 64
- movdqu oword [rsp], xmm6
- movdqu oword [rsp + $10], xmm7
- movdqu oword [rsp + $20], xmm8
- movdqu oword [rsp + $30], xmm9
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
// Initialize working vector
movdqu xmm0, oword [rcx]
@@ -1993,8 +1989,4 @@
movdqu oword [rcx + $20], xmm2
movdqu oword [rcx + $30], xmm3
- movdqu xmm6, oword [rsp]
- movdqu xmm7, oword [rsp + $10]
- movdqu xmm8, oword [rsp + $20]
- movdqu xmm9, oword [rsp + $30]
- add rsp, 64
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx2_x86_64.inc b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx2_x86_64.inc
index 7e45c52..7caa820 100644
--- a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx2_x86_64.inc
+++ b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressAvx2_x86_64.inc
@@ -7,8 +7,7 @@
// AVX/AVX2 instructions are db-encoded for broad assembler compatibility.
// Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves.
- sub rsp, 16
- db $C5, $FA, $7F, $34, $24 // vmovdqu oword [rsp], xmm6
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
// Initialize working vector
db $C5, $FA, $6F, $01 // vmovdqu xmm0, oword [rcx]
@@ -992,7 +991,6 @@
db $C5, $FA, $7F, $01 // vmovdqu oword [rcx], xmm0
db $C5, $FA, $7F, $49, $10 // vmovdqu oword [rcx + $10], xmm1
- db $C5, $FA, $6F, $34, $24 // vmovdqu xmm6, oword [rsp]
- add rsp, 16
-
db $C5, $F8, $77 // vzeroupper
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_x86_64.inc b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_x86_64.inc
index 0d1cd36..bf3debf 100644
--- a/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_x86_64.inc
+++ b/HashLib/src/Include/Simd/Blake2S/Blake2SCompressSse2_x86_64.inc
@@ -8,8 +8,7 @@
// Message loads via movd + punpcklqdq + shufps.
// Reference: BLAKE2/BLAKE2 sse/ by Samuel Neves.
- sub rsp, 16
- movdqu oword [rsp], xmm6
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
// Initialize working vector
movdqu xmm0, oword [rcx]
@@ -1073,5 +1072,4 @@
movdqu oword [rcx], xmm0
movdqu oword [rcx + $10], xmm1
- movdqu xmm6, oword [rsp]
- add rsp, 16
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/Blake3/Blake3CompressAvx2_x86_64.inc b/HashLib/src/Include/Simd/Blake3/Blake3CompressAvx2_x86_64.inc
index 8f6ba71..e41cd89 100644
--- a/HashLib/src/Include/Simd/Blake3/Blake3CompressAvx2_x86_64.inc
+++ b/HashLib/src/Include/Simd/Blake3/Blake3CompressAvx2_x86_64.inc
@@ -16,23 +16,19 @@
// [rsp + 96..111]: IV staging area (loaded into xmm2)
// [rsp +112..119]: padding
- sub rsp, 120
- db $C5, $F9, $7F, $34, $24 // vmovdqa oword [rsp], xmm6
- db $C5, $F9, $7F, $7C, $24, $10 // vmovdqa oword [rsp + $10], xmm7
- db $C5, $79, $7F, $44, $24, $20 // vmovdqa oword [rsp + $20], xmm8
- db $C5, $79, $7F, $4C, $24, $30 // vmovdqa oword [rsp + $30], xmm9
- db $C5, $79, $7F, $5C, $24, $40 // vmovdqa oword [rsp + $40], xmm11
- db $C5, $79, $7F, $74, $24, $50 // vmovdqa oword [rsp + $50], xmm14
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 24
mov eax, $6A09E667
- mov dword ptr [rsp + $60], eax
+ mov dword ptr [rsp + $00], eax
mov eax, $BB67AE85
- mov dword ptr [rsp + $64], eax
+ mov dword ptr [rsp + $04], eax
mov eax, $3C6EF372
- mov dword ptr [rsp + $68], eax
+ mov dword ptr [rsp + $08], eax
mov eax, $A54FF53A
- mov dword ptr [rsp + $6C], eax
- db $C5, $F9, $6F, $54, $24, $60 // vmovdqa xmm2, oword [rsp + $60]
+ mov dword ptr [rsp + $0C], eax
+ db $C5, $F9, $6F, $54, $24, $00 // vmovdqa xmm2, oword [rsp + $00]
// Initialize state
db $C4, $C1, $7A, $6F, $00 // vmovdqu xmm0, oword [r8]
@@ -586,10 +582,6 @@
db $C5, $FA, $7F, $51, $20 // vmovdqu oword [rcx + $20], xmm2
db $C5, $FA, $7F, $59, $30 // vmovdqu oword [rcx + $30], xmm3
- db $C5, $F9, $6F, $34, $24 // vmovdqa xmm6, oword [rsp]
- db $C5, $F9, $6F, $7C, $24, $10 // vmovdqa xmm7, oword [rsp + $10]
- db $C5, $79, $6F, $44, $24, $20 // vmovdqa xmm8, oword [rsp + $20]
- db $C5, $79, $6F, $4C, $24, $30 // vmovdqa xmm9, oword [rsp + $30]
- db $C5, $79, $6F, $5C, $24, $40 // vmovdqa xmm11, oword [rsp + $40]
- db $C5, $79, $6F, $74, $24, $50 // vmovdqa xmm14, oword [rsp + $50]
- add rsp, 120
+ add rsp, 24
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_x86_64.inc b/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_x86_64.inc
index 36a2821..8aaecdb 100644
--- a/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_x86_64.inc
+++ b/HashLib/src/Include/Simd/Blake3/Blake3CompressSse2_x86_64.inc
@@ -16,23 +16,19 @@
// [rsp + 96..111]: IV staging area (loaded into xmm2)
// [rsp +112..119]: padding
- sub rsp, 120
- movdqa oword [rsp], xmm6
- movdqa oword [rsp + $10], xmm7
- movdqa oword [rsp + $20], xmm8
- movdqa oword [rsp + $30], xmm9
- movdqa oword [rsp + $40], xmm11
- movdqa oword [rsp + $50], xmm14
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 24
mov eax, $6A09E667
- mov dword ptr [rsp + $60], eax
+ mov dword ptr [rsp + $00], eax
mov eax, $BB67AE85
- mov dword ptr [rsp + $64], eax
+ mov dword ptr [rsp + $04], eax
mov eax, $3C6EF372
- mov dword ptr [rsp + $68], eax
+ mov dword ptr [rsp + $08], eax
mov eax, $A54FF53A
- mov dword ptr [rsp + $6C], eax
- movdqa xmm2, oword [rsp + $60]
+ mov dword ptr [rsp + $0C], eax
+ movdqa xmm2, oword [rsp + $00]
// Initialize state: a=CV[0..3], b=CV[4..7], c=IV[0..3], d=CounterFlags
movdqu xmm0, oword [r8]
@@ -639,10 +635,6 @@
movdqu oword [rcx + $20], xmm2
movdqu oword [rcx + $30], xmm3
- movdqa xmm6, oword [rsp]
- movdqa xmm7, oword [rsp + $10]
- movdqa xmm8, oword [rsp + $20]
- movdqa xmm9, oword [rsp + $30]
- movdqa xmm11, oword [rsp + $40]
- movdqa xmm14, oword [rsp + $50]
- add rsp, 120
+ add rsp, 24
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_x86_64.inc b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_x86_64.inc
index bbfc6cd..24a548e 100644
--- a/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_x86_64.inc
+++ b/HashLib/src/Include/Simd/Blake3/Blake3Hash4Sse2_x86_64.inc
@@ -18,15 +18,9 @@
push rbx
push rbp
- sub rsp, 712
- movdqa oword [rsp + 576], xmm6
- movdqa oword [rsp + 592], xmm7
- movdqa oword [rsp + 608], xmm8
- movdqa oword [rsp + 624], xmm9
- movdqa oword [rsp + 640], xmm10
- movdqa oword [rsp + 656], xmm11
- movdqa oword [rsp + 672], xmm12
- movdqa oword [rsp + 688], xmm13
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 584
mov qword ptr [rsp + 560], r8
mov dword ptr [rsp + 568], r11d
@@ -1832,15 +1826,9 @@
movdqu oword [rax + 80], xmm4
movdqu oword [rax + 112], xmm3
- movdqa xmm6, oword [rsp + 576]
- movdqa xmm7, oword [rsp + 592]
- movdqa xmm8, oword [rsp + 608]
- movdqa xmm9, oword [rsp + 624]
- movdqa xmm10, oword [rsp + 640]
- movdqa xmm11, oword [rsp + 656]
- movdqa xmm12, oword [rsp + 672]
- movdqa xmm13, oword [rsp + 688]
- add rsp, 712
+ add rsp, 584
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
pop rbp
pop rbx
diff --git a/HashLib/src/Include/Simd/Blake3/Blake3Hash8Avx2_x86_64.inc b/HashLib/src/Include/Simd/Blake3/Blake3Hash8Avx2_x86_64.inc
index 3127f49..d57a409 100644
--- a/HashLib/src/Include/Simd/Blake3/Blake3Hash8Avx2_x86_64.inc
+++ b/HashLib/src/Include/Simd/Blake3/Blake3Hash8Avx2_x86_64.inc
@@ -18,6 +18,8 @@
//
// MS x64 non-volatile saves: xmm6-xmm13.
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
push rbx
push rbp
push r12
@@ -25,15 +27,7 @@
mov r12, rsp
and rsp, -32
- sub rsp, 1280
- db $C5, $FA, $7F, $B4, $24, $80, $04, $00, $00 // vmovdqu [rsp+1152], xmm6
- db $C5, $FA, $7F, $BC, $24, $90, $04, $00, $00 // vmovdqu [rsp+1168], xmm7
- db $C5, $7A, $7F, $84, $24, $A0, $04, $00, $00 // vmovdqu [rsp+1184], xmm8
- db $C5, $7A, $7F, $8C, $24, $B0, $04, $00, $00 // vmovdqu [rsp+1200], xmm9
- db $C5, $7A, $7F, $94, $24, $C0, $04, $00, $00 // vmovdqu [rsp+1216], xmm10
- db $C5, $7A, $7F, $9C, $24, $D0, $04, $00, $00 // vmovdqu [rsp+1232], xmm11
- db $C5, $7A, $7F, $A4, $24, $E0, $04, $00, $00 // vmovdqu [rsp+1248], xmm12
- db $C5, $7A, $7F, $AC, $24, $F0, $04, $00, $00 // vmovdqu [rsp+1264], xmm13
+ sub rsp, 1152
mov qword ptr [rsp + 1136], r12
mov qword ptr [rsp + 1120], r8
@@ -1677,18 +1671,11 @@
db $C5, $7E, $7F, $A0, $C0, $00, $00, $00 // vmovdqu [rax+192], ymm12
db $C5, $7E, $7F, $A8, $E0, $00, $00, $00 // vmovdqu [rax+224], ymm13
- db $C5, $FA, $6F, $B4, $24, $80, $04, $00, $00 // vmovdqu xmm6, [rsp+1152]
- db $C5, $FA, $6F, $BC, $24, $90, $04, $00, $00 // vmovdqu xmm7, [rsp+1168]
- db $C5, $7A, $6F, $84, $24, $A0, $04, $00, $00 // vmovdqu xmm8, [rsp+1184]
- db $C5, $7A, $6F, $8C, $24, $B0, $04, $00, $00 // vmovdqu xmm9, [rsp+1200]
- db $C5, $7A, $6F, $94, $24, $C0, $04, $00, $00 // vmovdqu xmm10, [rsp+1216]
- db $C5, $7A, $6F, $9C, $24, $D0, $04, $00, $00 // vmovdqu xmm11, [rsp+1232]
- db $C5, $7A, $6F, $A4, $24, $E0, $04, $00, $00 // vmovdqu xmm12, [rsp+1248]
- db $C5, $7A, $6F, $AC, $24, $F0, $04, $00, $00 // vmovdqu xmm13, [rsp+1264]
-
db $C5, $F8, $77 // vzeroupper
mov rsp, qword ptr [rsp + 1136]
pop r12
pop rbp
pop rbx
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldForwardPclmul_x86_64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldForwardPclmul_x86_64.inc
index 0c3ab09..a72d6b1 100644
--- a/HashLib/src/Include/Simd/CRC/CRCFoldForwardPclmul_x86_64.inc
+++ b/HashLib/src/Include/Simd/CRC/CRCFoldForwardPclmul_x86_64.inc
@@ -29,6 +29,8 @@
// pshufb xmm_dst, xmm_src (SSSE3):
// 66 0F 38 00 ModRM
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
// BswapMask is always needed (kept in xmm6 throughout)
movdqu xmm6, dqword ptr [r9 + 64] // BswapMask
@@ -192,3 +194,5 @@
// movq rax, xmm0: 66 48 0F 7E C0
db $66, $48, $0F, $7E, $C0
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldForwardVpclmul_x86_64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldForwardVpclmul_x86_64.inc
index 2f9e8c9..3abd489 100644
--- a/HashLib/src/Include/Simd/CRC/CRCFoldForwardVpclmul_x86_64.inc
+++ b/HashLib/src/Include/Simd/CRC/CRCFoldForwardVpclmul_x86_64.inc
@@ -26,6 +26,8 @@
// 2-byte: C5 [R.vvvv.L.pp]
// 3-byte: C4 [R.X.B.mmmmm] [W.vvvv.L.pp]
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
cmp edx, 128
jb @xmm_path
@@ -237,3 +239,5 @@
db $C5, $F8, $77 // vzeroupper
// movq rax, xmm0: 66 48 0F 7E C0
db $66, $48, $0F, $7E, $C0
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPclmul_x86_64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPclmul_x86_64.inc
index ee9366a..a4fd5ac 100644
--- a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPclmul_x86_64.inc
+++ b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedPclmul_x86_64.inc
@@ -26,6 +26,8 @@
// xmm0=000 xmm1=001 xmm2=010 xmm3=011 xmm4=100
// xmm5=101 xmm6=110 xmm7=111
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
// fold_1x128 is always needed (fold-by-1 loop and Barrett step 1)
movdqu xmm7, dqword ptr [r9 + 16] // fold_1x128
@@ -191,3 +193,5 @@
// Extract high qword (bits 64..127) into rax.
// pextrq rax, xmm0, 1: 66 48 0F 3A 16 C0 01
db $66, $48, $0F, $3A, $16, $C0, $01
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedVpclmul_x86_64.inc b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedVpclmul_x86_64.inc
index d5809fd..8bd13b9 100644
--- a/HashLib/src/Include/Simd/CRC/CRCFoldReflectedVpclmul_x86_64.inc
+++ b/HashLib/src/Include/Simd/CRC/CRCFoldReflectedVpclmul_x86_64.inc
@@ -25,6 +25,8 @@
// 2-byte: C5 [R.vvvv.L.pp]
// 3-byte: C4 [R.X.B.mmmmm] [W.vvvv.L.pp]
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
cmp edx, 128
jb @xmm_path
@@ -213,3 +215,5 @@
// Extract high qword (bits 64..127) into rax and return.
db $C5, $F8, $77 // vzeroupper
db $C4, $E3, $F9, $16, $C0, $01 // vpextrq rax, xmm0, 1
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/Common/SimdNonVolatileRestore_x86_64.inc b/HashLib/src/Include/Simd/Common/SimdNonVolatileRestore_x86_64.inc
new file mode 100644
index 0000000..bd4f12e
--- /dev/null
+++ b/HashLib/src/Include/Simd/Common/SimdNonVolatileRestore_x86_64.inc
@@ -0,0 +1,19 @@
+// Shared nonvolatile-XMM restore (x86-64). Mirror of SimdNonVolatileSave_x86_64.inc:
+// restores xmm6..xmm15 and frees the 160-byte save frame. Include it at the
+// kernel's restore point (after the last use of xmm6..15, and after any vzeroupper,
+// before returning), in any procedure that included SimdNonVolatileSave_x86_64.inc.
+// Legacy movups here is safe because a ymm kernel has already issued vzeroupper,
+// and a VEX-128-only kernel leaves the upper halves clean. No-op on SysV x86-64.
+{$IFNDEF HASHLIB_SYSV_X64_ABI}
+ movups xmm6, [rsp]
+ movups xmm7, [rsp + 16]
+ movups xmm8, [rsp + 32]
+ movups xmm9, [rsp + 48]
+ movups xmm10, [rsp + 64]
+ movups xmm11, [rsp + 80]
+ movups xmm12, [rsp + 96]
+ movups xmm13, [rsp + 112]
+ movups xmm14, [rsp + 128]
+ movups xmm15, [rsp + 144]
+ add rsp, 160
+{$ENDIF}
diff --git a/HashLib/src/Include/Simd/Common/SimdNonVolatileSave_x86_64.inc b/HashLib/src/Include/Simd/Common/SimdNonVolatileSave_x86_64.inc
new file mode 100644
index 0000000..d7e3c44
--- /dev/null
+++ b/HashLib/src/Include/Simd/Common/SimdNonVolatileSave_x86_64.inc
@@ -0,0 +1,29 @@
+// Shared nonvolatile-XMM save (x86-64). Opt-in by inclusion: a kernel that uses
+// any of xmm6..xmm15 includes this at its save point (before it first writes
+// those registers), and the matching SimdNonVolatileRestore_x86_64.inc at its restore
+// point (after the last use, before returning). Light kernels that touch only
+// xmm0..xmm5 do NOT include it.
+//
+// The Microsoft x64 ABI makes xmm6..xmm15 callee-saved (nonvolatile); a kernel
+// that clobbers them must preserve them for the caller. All ten are saved with
+// one fixed 160-byte frame (a multiple of 16, so it does not disturb a kernel's
+// own stack usage). AVX2 kernels only need the low 128 bits preserved (the upper
+// halves of ymm are always volatile on Win64), so this 128-bit save covers them
+// too. Save with legacy movups is safe: it runs before the first VEX instruction,
+// and the matching restore runs after vzeroupper (see the restore include).
+//
+// On SysV x86-64 (Linux/macOS) xmm6..xmm15 are volatile, so nothing is emitted.
+// Save and restore MUST be paired within the same procedure.
+{$IFNDEF HASHLIB_SYSV_X64_ABI}
+ sub rsp, 160
+ movups [rsp], xmm6
+ movups [rsp + 16], xmm7
+ movups [rsp + 32], xmm8
+ movups [rsp + 48], xmm9
+ movups [rsp + 64], xmm10
+ movups [rsp + 80], xmm11
+ movups [rsp + 96], xmm12
+ movups [rsp + 112], xmm13
+ movups [rsp + 128], xmm14
+ movups [rsp + 144], xmm15
+{$ENDIF}
diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressAvx2_x86_64.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressAvx2_x86_64.inc
index f59f4c0..9595bf5 100644
--- a/HashLib/src/Include/Simd/SHA1/SHA1CompressAvx2_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressAvx2_x86_64.inc
@@ -14,24 +14,24 @@
//
// Stack layout (sub rsp, 440): same as SSSE3 version
- sub rsp, 440
-
- db $C5, $FA, $7F, $34, $24 // vmovdqu oword [rsp], xmm6
- db $C5, $FA, $7F, $7C, $24, $10 // vmovdqu oword [rsp + $10], xmm7
- mov [rsp + $30], rdi
- mov [rsp + $38], rsi
- mov [rsp + $20], rbx
- mov [rsp + $28], rbp
- mov [rsp + $40], r12
- mov [rsp + $48], r13
- mov [rsp + $50], r14
- mov [rsp + $58], r15
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 408
+
+ mov [rsp + $10], rdi
+ mov [rsp + $18], rsi
+ mov [rsp + $00], rbx
+ mov [rsp + $08], rbp
+ mov [rsp + $20], r12
+ mov [rsp + $28], r13
+ mov [rsp + $30], r14
+ mov [rsp + $38], r15
mov r12, rcx
mov r13, rdx
mov r14d, r8d
- mov [rsp + $60], r9
- mov [rsp + $68], r10
+ mov [rsp + $40], r9
+ mov [rsp + $48], r10
test r14d, r14d
jz @avx2_sha1_done
@@ -40,11 +40,11 @@
// ========== Phase 1: SIMD message schedule ==========
- mov rax, [rsp + $68]
+ mov rax, [rsp + $48]
db $C5, $FA, $6F, $38 // vmovdqu xmm7, oword [rax]
- mov rax, [rsp + $60]
+ mov rax, [rsp + $40]
- lea r15, [rsp + $70]
+ lea r15, [rsp + $50]
db $C4, $C1, $7A, $6F, $45, $00 // vmovdqu xmm0, oword [r13]
db $C4, $E2, $79, $00, $C7 // vpshufb xmm0, xmm0, xmm7
@@ -138,7 +138,7 @@
mov edx, [r12 + $0C]
mov r8d, [r12 + $10]
- lea rbp, [rsp + $70]
+ lea rbp, [rsp + $50]
// ---------- Rounds 0-19: Ch, K = $5A827999 ----------
@@ -433,16 +433,16 @@
@avx2_sha1_done:
- mov rbx, [rsp + $20]
- mov rbp, [rsp + $28]
- mov r12, [rsp + $40]
- mov r13, [rsp + $48]
- mov r14, [rsp + $50]
- mov r15, [rsp + $58]
- mov rdi, [rsp + $30]
- mov rsi, [rsp + $38]
- db $C5, $FA, $6F, $34, $24 // vmovdqu xmm6, oword [rsp]
- db $C5, $FA, $6F, $7C, $24, $10 // vmovdqu xmm7, oword [rsp + $10]
- add rsp, 440
+ mov rbx, [rsp + $00]
+ mov rbp, [rsp + $08]
+ mov r12, [rsp + $20]
+ mov r13, [rsp + $28]
+ mov r14, [rsp + $30]
+ mov r15, [rsp + $38]
+ mov rdi, [rsp + $10]
+ mov rsi, [rsp + $18]
+ add rsp, 408
db $C5, $F8, $77 // vzeroupper
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressShaNi_x86_64.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressShaNi_x86_64.inc
index e4f916f..cab6baf 100644
--- a/HashLib/src/Include/Simd/SHA1/SHA1CompressShaNi_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressShaNi_x86_64.inc
@@ -22,10 +22,7 @@
// xmm7 = BSWAP mask, then E_SAVE
// xmm8 = ABCD_SAVE
- sub rsp, 56
- movdqu oword [rsp], xmm6
- movdqu oword [rsp + $10], xmm7
- movdqu oword [rsp + $20], xmm8
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
test r8d, r8d
jz @sha1ni_done
@@ -228,7 +225,4 @@
@sha1ni_done:
- movdqu xmm6, oword [rsp]
- movdqu xmm7, oword [rsp + $10]
- movdqu xmm8, oword [rsp + $20]
- add rsp, 56
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_x86_64.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_x86_64.inc
index d081b34..f2dbe58 100644
--- a/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressSse2_x86_64.inc
@@ -28,17 +28,18 @@
// [rsp + 88.. 95]: padding
// [rsp + 96..415]: W buffer (320 bytes = 80 x 4, 16-byte aligned)
- sub rsp, 424
-
- movdqu oword [rsp], xmm6
- mov [rsp + $20], rdi
- mov [rsp + $28], rsi
- mov [rsp + $10], rbx
- mov [rsp + $18], rbp
- mov [rsp + $30], r12
- mov [rsp + $38], r13
- mov [rsp + $40], r14
- mov [rsp + $48], r15
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 408
+
+ mov [rsp + $10], rdi
+ mov [rsp + $18], rsi
+ mov [rsp + $00], rbx
+ mov [rsp + $08], rbp
+ mov [rsp + $20], r12
+ mov [rsp + $28], r13
+ mov [rsp + $30], r14
+ mov [rsp + $38], r15
mov r12, rcx
mov r13, rdx
@@ -51,7 +52,7 @@
// ========== Phase 1: SIMD message schedule ==========
- lea r15, [rsp + $60]
+ lea r15, [rsp + $50]
// SSE2 32-bit byte-swap: pshuflw/pshufhw swap 16-bit words, then swap bytes within words
movdqu xmm0, oword [r13]
@@ -166,7 +167,7 @@
mov edx, [r12 + $0C]
mov r8d, [r12 + $10]
- lea rbp, [rsp + $60]
+ lea rbp, [rsp + $50]
// ---------- Rounds 0-19: Ch, K = $5A827999 ----------
@@ -461,13 +462,14 @@
@sse2_sha1_done:
- mov rbx, [rsp + $10]
- mov rbp, [rsp + $18]
- mov r12, [rsp + $30]
- mov r13, [rsp + $38]
- mov r14, [rsp + $40]
- mov r15, [rsp + $48]
- mov rdi, [rsp + $20]
- mov rsi, [rsp + $28]
- movdqu xmm6, oword [rsp]
- add rsp, 424
+ mov rbx, [rsp + $00]
+ mov rbp, [rsp + $08]
+ mov r12, [rsp + $20]
+ mov r13, [rsp + $28]
+ mov r14, [rsp + $30]
+ mov r15, [rsp + $38]
+ mov rdi, [rsp + $10]
+ mov rsi, [rsp + $18]
+ add rsp, 408
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/SHA1/SHA1CompressSsse3_x86_64.inc b/HashLib/src/Include/Simd/SHA1/SHA1CompressSsse3_x86_64.inc
index a14b0e9..334220f 100644
--- a/HashLib/src/Include/Simd/SHA1/SHA1CompressSsse3_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA1/SHA1CompressSsse3_x86_64.inc
@@ -30,24 +30,24 @@
// [rsp + 104..111]: BSWAP mask ptr save
// [rsp + 112..431]: W buffer (320 bytes = 80 x 4, 16-byte aligned)
- sub rsp, 440
-
- movdqu oword [rsp], xmm6
- movdqu oword [rsp + $10], xmm7
- mov [rsp + $30], rdi
- mov [rsp + $38], rsi
- mov [rsp + $20], rbx
- mov [rsp + $28], rbp
- mov [rsp + $40], r12
- mov [rsp + $48], r13
- mov [rsp + $50], r14
- mov [rsp + $58], r15
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 408
+
+ mov [rsp + $10], rdi
+ mov [rsp + $18], rsi
+ mov [rsp + $00], rbx
+ mov [rsp + $08], rbp
+ mov [rsp + $20], r12
+ mov [rsp + $28], r13
+ mov [rsp + $30], r14
+ mov [rsp + $38], r15
mov r12, rcx
mov r13, rdx
mov r14d, r8d
- mov [rsp + $60], r9
- mov [rsp + $68], r10
+ mov [rsp + $40], r9
+ mov [rsp + $48], r10
test r14d, r14d
jz @ssse3_sha1_done
@@ -56,11 +56,11 @@
// ========== Phase 1: SIMD message schedule ==========
- mov rax, [rsp + $68]
+ mov rax, [rsp + $48]
movdqu xmm7, oword [rax]
- mov rax, [rsp + $60]
+ mov rax, [rsp + $40]
- lea r15, [rsp + $70]
+ lea r15, [rsp + $50]
movdqu xmm0, oword [r13]
pshufb xmm0, xmm7
@@ -157,7 +157,7 @@
mov edx, [r12 + $0C]
mov r8d, [r12 + $10]
- lea rbp, [rsp + $70]
+ lea rbp, [rsp + $50]
// ---------- Rounds 0-19: Ch, K = $5A827999 ----------
@@ -452,14 +452,14 @@
@ssse3_sha1_done:
- mov rbx, [rsp + $20]
- mov rbp, [rsp + $28]
- mov r12, [rsp + $40]
- mov r13, [rsp + $48]
- mov r14, [rsp + $50]
- mov r15, [rsp + $58]
- mov rdi, [rsp + $30]
- mov rsi, [rsp + $38]
- movdqu xmm6, oword [rsp]
- movdqu xmm7, oword [rsp + $10]
- add rsp, 440
+ mov rbx, [rsp + $00]
+ mov rbp, [rsp + $08]
+ mov r12, [rsp + $20]
+ mov r13, [rsp + $28]
+ mov r14, [rsp + $30]
+ mov r15, [rsp + $38]
+ mov rdi, [rsp + $10]
+ mov rsi, [rsp + $18]
+ add rsp, 408
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressAvx2_x86_64.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressAvx2_x86_64.inc
index e7f1c84..51b651b 100644
--- a/HashLib/src/Include/Simd/SHA256/SHA256CompressAvx2_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressAvx2_x86_64.inc
@@ -9,41 +9,40 @@
// Phase 1 (VEX-128 SIMD): Compute W+K[0..63] using message schedule, store to stack
// Phase 2 (GPR): Run 64 compression rounds reading W+K from stack
//
-// MS x64 non-volatile saves: xmm6-xmm7, rbx, rbp, rdi, rsi, r12-r15.
+// MS x64 non-volatile saves: xmm6-xmm15 via the shared SimdNonVolatileSave/Restore
+// include (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame.
//
-// Stack layout (sub rsp, 376): same as SSSE3 version
-// [rsp + 0.. 15]: xmm6 save
-// [rsp + 16.. 31]: xmm7 save
-// [rsp + 32.. 39]: rbx save
-// [rsp + 40.. 47]: rbp save
-// [rsp + 48.. 55]: rdi save
-// [rsp + 56.. 63]: rsi save
-// [rsp + 64.. 71]: r12 save
-// [rsp + 72.. 79]: r13 save
-// [rsp + 80.. 87]: r14 save
-// [rsp + 88.. 95]: r15 save
-// [rsp + 96..103]: K256 ptr save
-// [rsp + 104..111]: BSWAP mask ptr save
-// [rsp + 112..367]: W+K buffer (256 bytes, 16-byte aligned)
-
- sub rsp, 376
-
- db $C5, $FA, $7F, $34, $24 // vmovdqu oword [rsp], xmm6
- db $C5, $FA, $7F, $7C, $24, $10 // vmovdqu oword [rsp + $10], xmm7
- mov [rsp + $30], rdi
- mov [rsp + $38], rsi
- mov [rsp + $20], rbx
- mov [rsp + $28], rbp
- mov [rsp + $40], r12
- mov [rsp + $48], r13
- mov [rsp + $50], r14
- mov [rsp + $58], r15
+// Stack layout (sub rsp, 344): same as SSSE3 version
+// [rsp + 0.. 7]: rbx save
+// [rsp + 8.. 15]: rbp save
+// [rsp + 16.. 23]: rdi save
+// [rsp + 24.. 31]: rsi save
+// [rsp + 32.. 39]: r12 save
+// [rsp + 40.. 47]: r13 save
+// [rsp + 48.. 55]: r14 save
+// [rsp + 56.. 63]: r15 save
+// [rsp + 64.. 71]: K256 ptr save
+// [rsp + 72.. 79]: BSWAP mask ptr save
+// [rsp + 80..335]: W+K buffer (256 bytes)
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 344
+
+ mov [rsp + $10], rdi
+ mov [rsp + $18], rsi
+ mov [rsp + $00], rbx
+ mov [rsp + $08], rbp
+ mov [rsp + $20], r12
+ mov [rsp + $28], r13
+ mov [rsp + $30], r14
+ mov [rsp + $38], r15
mov r12, rcx
mov r13, rdx
mov r14d, r8d
- mov [rsp + $60], r9
- mov [rsp + $68], r10
+ mov [rsp + $40], r9
+ mov [rsp + $48], r10
test r14d, r14d
jz @avx2_done
@@ -52,9 +51,9 @@
// ========== Phase 1: Message Schedule (VEX-128) ==========
- mov rax, [rsp + $68]
+ mov rax, [rsp + $48]
db $C5, $FA, $6F, $38 // vmovdqu xmm7, oword [rax]
- mov rax, [rsp + $60]
+ mov rax, [rsp + $40]
db $C4, $C1, $7A, $6F, $45, $00 // vmovdqu xmm0, oword [r13]
db $C4, $E2, $79, $00, $C7 // vpshufb xmm0, xmm0, xmm7
@@ -65,7 +64,7 @@
db $C4, $C1, $7A, $6F, $5D, $30 // vmovdqu xmm3, oword [r13 + $30]
db $C4, $E2, $61, $00, $DF // vpshufb xmm3, xmm3, xmm7
- lea rbp, [rsp + $70]
+ lea rbp, [rsp + $50]
db $C5, $FA, $6F, $20 // vmovdqu xmm4, oword [rax]
db $C5, $F9, $FE, $E4 // vpaddd xmm4, xmm0, xmm4
@@ -160,8 +159,8 @@
mov r10d, [r12 + $18]
mov r11d, [r12 + $1C]
- lea rbp, [rsp + $70]
- lea r15, [rsp + $70 + $100]
+ lea rbp, [rsp + $50]
+ lea r15, [rsp + $50 + $100]
@avx2_round_loop:
@@ -441,16 +440,16 @@
@avx2_done:
- mov rbx, [rsp + $20]
- mov rbp, [rsp + $28]
- mov r12, [rsp + $40]
- mov r13, [rsp + $48]
- mov r14, [rsp + $50]
- mov r15, [rsp + $58]
- mov rdi, [rsp + $30]
- mov rsi, [rsp + $38]
- db $C5, $FA, $6F, $34, $24 // vmovdqu xmm6, oword [rsp]
- db $C5, $FA, $6F, $7C, $24, $10 // vmovdqu xmm7, oword [rsp + $10]
- add rsp, 376
+ mov rbx, [rsp + $00]
+ mov rbp, [rsp + $08]
+ mov r12, [rsp + $20]
+ mov r13, [rsp + $28]
+ mov r14, [rsp + $30]
+ mov r15, [rsp + $38]
+ mov rdi, [rsp + $10]
+ mov rsi, [rsp + $18]
+ add rsp, 344
db $C5, $F8, $77 // vzeroupper
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressShaNi_x86_64.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressShaNi_x86_64.inc
index 0dbc4a7..a2dd444 100644
--- a/HashLib/src/Include/Simd/SHA256/SHA256CompressShaNi_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressShaNi_x86_64.inc
@@ -21,11 +21,7 @@
// xmm8 = ABEF_SAVE
// xmm9 = CDGH_SAVE
- sub rsp, 72
- movdqu oword [rsp], xmm6
- movdqu oword [rsp + $10], xmm7
- movdqu oword [rsp + $20], xmm8
- movdqu oword [rsp + $30], xmm9
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
test r8d, r8d
jz @shani_done
@@ -250,8 +246,4 @@
@shani_done:
- movdqu xmm6, oword [rsp]
- movdqu xmm7, oword [rsp + $10]
- movdqu xmm8, oword [rsp + $20]
- movdqu xmm9, oword [rsp + $30]
- add rsp, 72
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_x86_64.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_x86_64.inc
index 6e1c3f1..15a7b13 100644
--- a/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressSse2_x86_64.inc
@@ -12,27 +12,28 @@
// Phase 1 (SIMD): Compute W+K[0..63] using SSE2 message schedule, store to stack
// Phase 2 (GPR): Run 64 compression rounds reading W+K from stack
//
-// MS x64 non-volatile saves: xmm6-xmm7, rbx, rbp, rdi, rsi, r12-r15.
+// MS x64 non-volatile saves: xmm6-xmm15 via the shared SimdNonVolatileSave/Restore
+// include (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame.
//
-// Stack layout (sub rsp, 376): same as SSSE3 version
+// Stack layout (sub rsp, 344): GPR saves at [rsp+$00..$47], W+K schedule at [rsp+$50..]
- sub rsp, 376
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
- movdqu oword [rsp], xmm6
- movdqu oword [rsp + $10], xmm7
- mov [rsp + $30], rdi
- mov [rsp + $38], rsi
- mov [rsp + $20], rbx
- mov [rsp + $28], rbp
- mov [rsp + $40], r12
- mov [rsp + $48], r13
- mov [rsp + $50], r14
- mov [rsp + $58], r15
+ sub rsp, 344
+
+ mov [rsp + $10], rdi
+ mov [rsp + $18], rsi
+ mov [rsp + $00], rbx
+ mov [rsp + $08], rbp
+ mov [rsp + $20], r12
+ mov [rsp + $28], r13
+ mov [rsp + $30], r14
+ mov [rsp + $38], r15
mov r12, rcx
mov r13, rdx
mov r14d, r8d
- mov [rsp + $60], r9
+ mov [rsp + $40], r9
test r14d, r14d
jz @sse2_256_done
@@ -41,7 +42,7 @@
// ========== Phase 1: Message Schedule ==========
- mov rax, [rsp + $60]
+ mov rax, [rsp + $40]
// SSE2 32-bit byte-swap for W[0..15]
movdqu xmm0, oword [r13]
@@ -76,7 +77,7 @@
psllw xmm4, 8
por xmm3, xmm4
- lea rbp, [rsp + $70]
+ lea rbp, [rsp + $50]
movdqa xmm4, xmm0
movdqu xmm5, oword [rax]
@@ -191,8 +192,8 @@
mov r10d, [r12 + $18]
mov r11d, [r12 + $1C]
- lea rbp, [rsp + $70]
- lea r15, [rsp + $70 + $100]
+ lea rbp, [rsp + $50]
+ lea r15, [rsp + $50 + $100]
@sse2_256_round_loop:
@@ -472,14 +473,14 @@
@sse2_256_done:
- mov rbx, [rsp + $20]
- mov rbp, [rsp + $28]
- mov r12, [rsp + $40]
- mov r13, [rsp + $48]
- mov r14, [rsp + $50]
- mov r15, [rsp + $58]
- mov rdi, [rsp + $30]
- mov rsi, [rsp + $38]
- movdqu xmm6, oword [rsp]
- movdqu xmm7, oword [rsp + $10]
- add rsp, 376
+ mov rbx, [rsp + $00]
+ mov rbp, [rsp + $08]
+ mov r12, [rsp + $20]
+ mov r13, [rsp + $28]
+ mov r14, [rsp + $30]
+ mov r15, [rsp + $38]
+ mov rdi, [rsp + $10]
+ mov rsi, [rsp + $18]
+ add rsp, 344
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/SHA256/SHA256CompressSsse3_x86_64.inc b/HashLib/src/Include/Simd/SHA256/SHA256CompressSsse3_x86_64.inc
index b03e8c9..81d22a1 100644
--- a/HashLib/src/Include/Simd/SHA256/SHA256CompressSsse3_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA256/SHA256CompressSsse3_x86_64.inc
@@ -7,41 +7,40 @@
// Phase 1 (SIMD): Compute W+K[0..63] using SSSE3 message schedule, store to stack
// Phase 2 (GPR): Run 64 compression rounds reading W+K from stack
//
-// MS x64 non-volatile saves: xmm6-xmm7, rbx, rbp, rdi, rsi, r12-r15.
+// MS x64 non-volatile saves: xmm6-xmm15 via the shared SimdNonVolatileSave/Restore
+// include (no-op on SysV); rbx, rbp, rdi, rsi, r12-r15 in the local frame.
//
-// Stack layout (sub rsp, 376):
-// [rsp + 0.. 15]: xmm6 save
-// [rsp + 16.. 31]: xmm7 save
-// [rsp + 32.. 39]: rbx save
-// [rsp + 40.. 47]: rbp save
-// [rsp + 48.. 55]: rdi save
-// [rsp + 56.. 63]: rsi save
-// [rsp + 64.. 71]: r12 save
-// [rsp + 72.. 79]: r13 save
-// [rsp + 80.. 87]: r14 save
-// [rsp + 88.. 95]: r15 save
-// [rsp + 96..103]: K256 ptr save
-// [rsp + 104..111]: BSWAP mask ptr save
-// [rsp + 112..367]: W+K buffer (256 bytes, 16-byte aligned)
-
- sub rsp, 376
-
- movdqu oword [rsp], xmm6
- movdqu oword [rsp + $10], xmm7
- mov [rsp + $30], rdi
- mov [rsp + $38], rsi
- mov [rsp + $20], rbx
- mov [rsp + $28], rbp
- mov [rsp + $40], r12
- mov [rsp + $48], r13
- mov [rsp + $50], r14
- mov [rsp + $58], r15
+// Stack layout (sub rsp, 344):
+// [rsp + 0.. 7]: rbx save
+// [rsp + 8.. 15]: rbp save
+// [rsp + 16.. 23]: rdi save
+// [rsp + 24.. 31]: rsi save
+// [rsp + 32.. 39]: r12 save
+// [rsp + 40.. 47]: r13 save
+// [rsp + 48.. 55]: r14 save
+// [rsp + 56.. 63]: r15 save
+// [rsp + 64.. 71]: K256 ptr save
+// [rsp + 72.. 79]: BSWAP mask ptr save
+// [rsp + 80..335]: W+K buffer (256 bytes)
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 344
+
+ mov [rsp + $10], rdi
+ mov [rsp + $18], rsi
+ mov [rsp + $00], rbx
+ mov [rsp + $08], rbp
+ mov [rsp + $20], r12
+ mov [rsp + $28], r13
+ mov [rsp + $30], r14
+ mov [rsp + $38], r15
mov r12, rcx
mov r13, rdx
mov r14d, r8d
- mov [rsp + $60], r9
- mov [rsp + $68], r10
+ mov [rsp + $40], r9
+ mov [rsp + $48], r10
test r14d, r14d
jz @ssse3_done
@@ -50,9 +49,9 @@
// ========== Phase 1: Message Schedule ==========
- mov rax, [rsp + $68]
+ mov rax, [rsp + $48]
movdqu xmm7, oword [rax]
- mov rax, [rsp + $60]
+ mov rax, [rsp + $40]
movdqu xmm0, oword [r13]
pshufb xmm0, xmm7
@@ -63,7 +62,7 @@
movdqu xmm3, oword [r13 + $30]
pshufb xmm3, xmm7
- lea rbp, [rsp + $70]
+ lea rbp, [rsp + $50]
movdqa xmm4, xmm0
movdqu xmm5, oword [rax]
@@ -172,8 +171,8 @@
mov r10d, [r12 + $18]
mov r11d, [r12 + $1C]
- lea rbp, [rsp + $70]
- lea r15, [rsp + $70 + $100]
+ lea rbp, [rsp + $50]
+ lea r15, [rsp + $50 + $100]
@ssse3_round_loop:
@@ -455,14 +454,14 @@
@ssse3_done:
- mov rbx, [rsp + $20]
- mov rbp, [rsp + $28]
- mov r12, [rsp + $40]
- mov r13, [rsp + $48]
- mov r14, [rsp + $50]
- mov r15, [rsp + $58]
- mov rdi, [rsp + $30]
- mov rsi, [rsp + $38]
- movdqu xmm6, oword [rsp]
- movdqu xmm7, oword [rsp + $10]
- add rsp, 376
+ mov rbx, [rsp + $00]
+ mov rbp, [rsp + $08]
+ mov r12, [rsp + $20]
+ mov r13, [rsp + $28]
+ mov r14, [rsp + $30]
+ mov r15, [rsp + $38]
+ mov rdi, [rsp + $10]
+ mov rsi, [rsp + $18]
+ add rsp, 344
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2Absorb_x86_64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2Absorb_x86_64.inc
index 25ec748..0f9002f 100644
--- a/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2Absorb_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2Absorb_x86_64.inc
@@ -52,19 +52,9 @@
push rdi
push rsi
- sub rsp, 424
-
- // Save XMM6-XMM15
- db $C5, $FA, $7F, $B4, $24, $00, $01, $00, $00 // vmovdqu [rsp+256], xmm6
- db $C5, $FA, $7F, $BC, $24, $10, $01, $00, $00 // vmovdqu [rsp+272], xmm7
- db $C4, $61, $7A, $7F, $84, $24, $20, $01, $00, $00 // vmovdqu [rsp+288], xmm8
- db $C4, $61, $7A, $7F, $8C, $24, $30, $01, $00, $00 // vmovdqu [rsp+304], xmm9
- db $C4, $61, $7A, $7F, $94, $24, $40, $01, $00, $00 // vmovdqu [rsp+320], xmm10
- db $C4, $61, $7A, $7F, $9C, $24, $50, $01, $00, $00 // vmovdqu [rsp+336], xmm11
- db $C4, $61, $7A, $7F, $A4, $24, $60, $01, $00, $00 // vmovdqu [rsp+352], xmm12
- db $C4, $61, $7A, $7F, $AC, $24, $70, $01, $00, $00 // vmovdqu [rsp+368], xmm13
- db $C4, $61, $7A, $7F, $B4, $24, $80, $01, $00, $00 // vmovdqu [rsp+384], xmm14
- db $C4, $61, $7A, $7F, $BC, $24, $90, $01, $00, $00 // vmovdqu [rsp+400], xmm15
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 264
// Save parameters to callee-saved registers
mov r12, rcx
@@ -401,19 +391,9 @@
// ===== Epilogue =====
db $C5, $F8, $77 // vzeroupper
- // Restore XMM6-XMM15
- db $C5, $FA, $6F, $B4, $24, $00, $01, $00, $00 // vmovdqu xmm6, [rsp+256]
- db $C5, $FA, $6F, $BC, $24, $10, $01, $00, $00 // vmovdqu xmm7, [rsp+272]
- db $C4, $61, $7A, $6F, $84, $24, $20, $01, $00, $00 // vmovdqu xmm8, [rsp+288]
- db $C4, $61, $7A, $6F, $8C, $24, $30, $01, $00, $00 // vmovdqu xmm9, [rsp+304]
- db $C4, $61, $7A, $6F, $94, $24, $40, $01, $00, $00 // vmovdqu xmm10, [rsp+320]
- db $C4, $61, $7A, $6F, $9C, $24, $50, $01, $00, $00 // vmovdqu xmm11, [rsp+336]
- db $C4, $61, $7A, $6F, $A4, $24, $60, $01, $00, $00 // vmovdqu xmm12, [rsp+352]
- db $C4, $61, $7A, $6F, $AC, $24, $70, $01, $00, $00 // vmovdqu xmm13, [rsp+368]
- db $C4, $61, $7A, $6F, $B4, $24, $80, $01, $00, $00 // vmovdqu xmm14, [rsp+384]
- db $C4, $61, $7A, $6F, $BC, $24, $90, $01, $00, $00 // vmovdqu xmm15, [rsp+400]
-
- add rsp, 424
+ add rsp, 264
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
pop rsi
pop rdi
diff --git a/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2_x86_64.inc b/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2_x86_64.inc
index 80d7ec3..7e649bb 100644
--- a/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA3/KeccakF1600Avx2_x86_64.inc
@@ -24,18 +24,9 @@
// [rsp+ 32..191]: xmm6-xmm15 save area
// [rsp+192..223]: padding
- sub rsp, 224
-
- db $C5, $FA, $7F, $74, $24, $20 // vmovdqu [rsp+32], xmm6
- db $C5, $FA, $7F, $7C, $24, $30 // vmovdqu [rsp+48], xmm7
- db $C4, $61, $7A, $7F, $44, $24, $40 // vmovdqu [rsp+64], xmm8
- db $C4, $61, $7A, $7F, $4C, $24, $50 // vmovdqu [rsp+80], xmm9
- db $C4, $61, $7A, $7F, $54, $24, $60 // vmovdqu [rsp+96], xmm10
- db $C4, $61, $7A, $7F, $5C, $24, $70 // vmovdqu [rsp+112], xmm11
- db $C4, $61, $7A, $7F, $A4, $24, $80, $00, $00, $00 // vmovdqu [rsp+128], xmm12
- db $C4, $61, $7A, $7F, $AC, $24, $90, $00, $00, $00 // vmovdqu [rsp+144], xmm13
- db $C4, $61, $7A, $7F, $B4, $24, $A0, $00, $00, $00 // vmovdqu [rsp+160], xmm14
- db $C4, $61, $7A, $7F, $BC, $24, $B0, $00, $00, $00 // vmovdqu [rsp+176], xmm15
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 64
// Set up table pointers (biased by +96 for compact displacements)
lea r8, [rdx + 96]
@@ -281,15 +272,6 @@
db $C5, $F8, $77 // vzeroupper
- db $C5, $FA, $6F, $74, $24, $20 // vmovdqu xmm6, [rsp+32]
- db $C5, $FA, $6F, $7C, $24, $30 // vmovdqu xmm7, [rsp+48]
- db $C4, $61, $7A, $6F, $44, $24, $40 // vmovdqu xmm8, [rsp+64]
- db $C4, $61, $7A, $6F, $4C, $24, $50 // vmovdqu xmm9, [rsp+80]
- db $C4, $61, $7A, $6F, $54, $24, $60 // vmovdqu xmm10, [rsp+96]
- db $C4, $61, $7A, $6F, $5C, $24, $70 // vmovdqu xmm11, [rsp+112]
- db $C4, $61, $7A, $6F, $A4, $24, $80, $00, $00, $00 // vmovdqu xmm12, [rsp+128]
- db $C4, $61, $7A, $6F, $AC, $24, $90, $00, $00, $00 // vmovdqu xmm13, [rsp+144]
- db $C4, $61, $7A, $6F, $B4, $24, $A0, $00, $00, $00 // vmovdqu xmm14, [rsp+160]
- db $C4, $61, $7A, $6F, $BC, $24, $B0, $00, $00, $00 // vmovdqu xmm15, [rsp+176]
-
- add rsp, 224
+ add rsp, 64
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/SHA512/SHA512CompressAvx2_x86_64.inc b/HashLib/src/Include/Simd/SHA512/SHA512CompressAvx2_x86_64.inc
index 4c3c7e7..57fd8ef 100644
--- a/HashLib/src/Include/Simd/SHA512/SHA512CompressAvx2_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA512/SHA512CompressAvx2_x86_64.inc
@@ -23,24 +23,24 @@
// [rsp + 112..751]: W+K buffer (640 bytes = 80 * 8, 16-byte aligned)
// [rsp + 752..879]: W circular buffer (128 bytes = 16 * 8)
- sub rsp, 1016
-
- db $C5, $FA, $7F, $34, $24 // vmovdqu oword [rsp], xmm6
- db $C5, $FA, $7F, $7C, $24, $10 // vmovdqu oword [rsp + $10], xmm7
- mov [rsp + $30], rdi
- mov [rsp + $38], rsi
- mov [rsp + $20], rbx
- mov [rsp + $28], rbp
- mov [rsp + $40], r12
- mov [rsp + $48], r13
- mov [rsp + $50], r14
- mov [rsp + $58], r15
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 984
+
+ mov [rsp + $10], rdi
+ mov [rsp + $18], rsi
+ mov [rsp + $00], rbx
+ mov [rsp + $08], rbp
+ mov [rsp + $20], r12
+ mov [rsp + $28], r13
+ mov [rsp + $30], r14
+ mov [rsp + $38], r15
mov r12, rcx
mov r13, rdx
mov r14d, r8d
- mov [rsp + $60], r9
- mov [rsp + $68], r10
+ mov [rsp + $40], r9
+ mov [rsp + $48], r10
test r14d, r14d
jz @avx2_512_done
@@ -49,12 +49,12 @@
// ========== Phase 1: Load, byte-swap (VEX), expand ==========
- mov rax, [rsp + $68]
+ mov rax, [rsp + $48]
db $C5, $FA, $6F, $38 // vmovdqu xmm7, oword [rax]
- mov rax, [rsp + $60]
+ mov rax, [rsp + $40]
- lea rbp, [rsp + $70]
- lea r15, [rsp + $2F0]
+ lea rbp, [rsp + $50]
+ lea r15, [rsp + $2D0]
// Load, byte-swap W[0..15], store to W circular buffer and compute W+K
db $C4, $C1, $7A, $6F, $45, $00 // vmovdqu xmm0, oword [r13]
@@ -463,16 +463,16 @@
@avx2_512_done:
- mov rbx, [rsp + $20]
- mov rbp, [rsp + $28]
- mov r12, [rsp + $40]
- mov r13, [rsp + $48]
- mov r14, [rsp + $50]
- mov r15, [rsp + $58]
- mov rdi, [rsp + $30]
- mov rsi, [rsp + $38]
- db $C5, $FA, $6F, $34, $24 // vmovdqu xmm6, oword [rsp]
- db $C5, $FA, $6F, $7C, $24, $10 // vmovdqu xmm7, oword [rsp + $10]
- add rsp, 1016
+ mov rbx, [rsp + $00]
+ mov rbp, [rsp + $08]
+ mov r12, [rsp + $20]
+ mov r13, [rsp + $28]
+ mov r14, [rsp + $30]
+ mov r15, [rsp + $38]
+ mov rdi, [rsp + $10]
+ mov rsi, [rsp + $18]
+ add rsp, 984
db $C5, $F8, $77 // vzeroupper
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Include/Simd/SHA512/SHA512CompressSsse3_x86_64.inc b/HashLib/src/Include/Simd/SHA512/SHA512CompressSsse3_x86_64.inc
index be17545..1969b06 100644
--- a/HashLib/src/Include/Simd/SHA512/SHA512CompressSsse3_x86_64.inc
+++ b/HashLib/src/Include/Simd/SHA512/SHA512CompressSsse3_x86_64.inc
@@ -25,24 +25,24 @@
// [rsp + 112..751]: W+K buffer (640 bytes = 80 * 8, 16-byte aligned)
// [rsp + 752..879]: W circular buffer (128 bytes = 16 * 8)
- sub rsp, 1016
-
- movdqu oword [rsp], xmm6
- movdqu oword [rsp + $10], xmm7
- mov [rsp + $30], rdi
- mov [rsp + $38], rsi
- mov [rsp + $20], rbx
- mov [rsp + $28], rbp
- mov [rsp + $40], r12
- mov [rsp + $48], r13
- mov [rsp + $50], r14
- mov [rsp + $58], r15
+ {$I ..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc}
+
+ sub rsp, 984
+
+ mov [rsp + $10], rdi
+ mov [rsp + $18], rsi
+ mov [rsp + $00], rbx
+ mov [rsp + $08], rbp
+ mov [rsp + $20], r12
+ mov [rsp + $28], r13
+ mov [rsp + $30], r14
+ mov [rsp + $38], r15
mov r12, rcx
mov r13, rdx
mov r14d, r8d
- mov [rsp + $60], r9
- mov [rsp + $68], r10
+ mov [rsp + $40], r9
+ mov [rsp + $48], r10
test r14d, r14d
jz @ssse3_512_done
@@ -51,12 +51,12 @@
// ========== Phase 1: Load, byte-swap, expand message schedule ==========
- mov rax, [rsp + $68]
+ mov rax, [rsp + $48]
movdqu xmm7, oword [rax]
- mov rax, [rsp + $60]
+ mov rax, [rsp + $40]
- lea rbp, [rsp + $70]
- lea r15, [rsp + $2F0]
+ lea rbp, [rsp + $50]
+ lea r15, [rsp + $2D0]
// Load, byte-swap W[0..15], store to W circular buffer and compute W+K
movdqu xmm0, oword [r13]
@@ -470,14 +470,14 @@
@ssse3_512_done:
- mov rbx, [rsp + $20]
- mov rbp, [rsp + $28]
- mov r12, [rsp + $40]
- mov r13, [rsp + $48]
- mov r14, [rsp + $50]
- mov r15, [rsp + $58]
- mov rdi, [rsp + $30]
- mov rsi, [rsp + $38]
- movdqu xmm6, oword [rsp]
- movdqu xmm7, oword [rsp + $10]
- add rsp, 1016
+ mov rbx, [rsp + $00]
+ mov rbp, [rsp + $08]
+ mov r12, [rsp + $20]
+ mov r13, [rsp + $28]
+ mov r14, [rsp + $30]
+ mov r15, [rsp + $38]
+ mov rdi, [rsp + $10]
+ mov rsi, [rsp + $18]
+ add rsp, 984
+
+ {$I ..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc}
diff --git a/HashLib/src/Packages/FPC/HashLib4PascalPackage.lpk b/HashLib/src/Packages/FPC/HashLib4PascalPackage.lpk
index 2c02401..d3300a5 100644
--- a/HashLib/src/Packages/FPC/HashLib4PascalPackage.lpk
+++ b/HashLib/src/Packages/FPC/HashLib4PascalPackage.lpk
@@ -9,7 +9,7 @@
-
+