diff --git a/CryptoLib.Benchmark/Delphi/CryptoLib.BenchmarkConsole.dpr b/CryptoLib.Benchmark/Delphi/CryptoLib.BenchmarkConsole.dpr index 7bb4c6a9..f1ccdfd7 100644 --- a/CryptoLib.Benchmark/Delphi/CryptoLib.BenchmarkConsole.dpr +++ b/CryptoLib.Benchmark/Delphi/CryptoLib.BenchmarkConsole.dpr @@ -691,14 +691,18 @@ uses ClpIFusedCcmKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCcmKernel.pas', ClpIFusedEaxKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedEaxKernel.pas', ClpIFusedGcmSivKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedGcmSivKernel.pas', + ClpIFusedCtrKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCtrKernel.pas', + ClpIFusedCbcKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCbcKernel.pas', ClpFusedKernelRegistry in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpFusedKernelRegistry.pas', ClpAesFusedAeadSimd in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesFusedAeadSimd.pas', - ClpAesFusedAeadX86Backend in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesFusedAeadX86Backend.pas', + ClpAesNiFusedX86Backend in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesNiFusedX86Backend.pas', ClpBinPolySimd in '..\..\CryptoLib\src\Math\BinPoly\ClpBinPolySimd.pas', ClpAesNiOcbKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiOcbKernel.pas', ClpAesNiCcmKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCcmKernel.pas', ClpAesNiEaxKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiEaxKernel.pas', ClpAesNiGcmKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiGcmKernel.pas', + ClpAesNiCtrKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCtrKernel.pas', + ClpAesNiCbcKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCbcKernel.pas', ClpPclmulGcmSivKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpPclmulGcmSivKernel.pas', ClpFusedKernelDefaults in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpFusedKernelDefaults.pas', ClpAeadParameters in '..\..\CryptoLib\src\Crypto\Parameters\ClpAeadParameters.pas', @@ -728,6 +732,8 @@ uses ClpCfbBlockCipherMac in '..\..\CryptoLib\src\Crypto\Macs\ClpCfbBlockCipherMac.pas', ClpICfbBlockCipherMac in '..\..\CryptoLib\src\Interfaces\Crypto\Macs\ClpICfbBlockCipherMac.pas', ClpByteUtilities in '..\..\CryptoLib\src\NumberUtilities\ClpByteUtilities.pas', + ClpByteXorSimd in '..\..\CryptoLib\src\NumberUtilities\ClpByteXorSimd.pas', + ClpByteXorX86Backend in '..\..\CryptoLib\src\NumberUtilities\ClpByteXorX86Backend.pas', ClpInt16Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt16Utilities.pas', ClpInt32Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt32Utilities.pas', ClpInt64Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt64Utilities.pas', diff --git a/CryptoLib.Examples/Delphi.Examples/CryptoLib.Examples.dpr b/CryptoLib.Examples/Delphi.Examples/CryptoLib.Examples.dpr index f48218a3..a370cb5d 100644 --- a/CryptoLib.Examples/Delphi.Examples/CryptoLib.Examples.dpr +++ b/CryptoLib.Examples/Delphi.Examples/CryptoLib.Examples.dpr @@ -707,14 +707,18 @@ uses ClpIFusedCcmKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCcmKernel.pas', ClpIFusedEaxKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedEaxKernel.pas', ClpIFusedGcmSivKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedGcmSivKernel.pas', + ClpIFusedCtrKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCtrKernel.pas', + ClpIFusedCbcKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCbcKernel.pas', ClpFusedKernelRegistry in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpFusedKernelRegistry.pas', ClpAesFusedAeadSimd in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesFusedAeadSimd.pas', - ClpAesFusedAeadX86Backend in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesFusedAeadX86Backend.pas', + ClpAesNiFusedX86Backend in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesNiFusedX86Backend.pas', ClpBinPolySimd in '..\..\CryptoLib\src\Math\BinPoly\ClpBinPolySimd.pas', ClpAesNiOcbKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiOcbKernel.pas', ClpAesNiCcmKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCcmKernel.pas', ClpAesNiEaxKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiEaxKernel.pas', ClpAesNiGcmKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiGcmKernel.pas', + ClpAesNiCtrKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCtrKernel.pas', + ClpAesNiCbcKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCbcKernel.pas', ClpPclmulGcmSivKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpPclmulGcmSivKernel.pas', ClpFusedKernelDefaults in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpFusedKernelDefaults.pas', ClpAeadParameters in '..\..\CryptoLib\src\Crypto\Parameters\ClpAeadParameters.pas', @@ -744,6 +748,8 @@ uses ClpCfbBlockCipherMac in '..\..\CryptoLib\src\Crypto\Macs\ClpCfbBlockCipherMac.pas', ClpICfbBlockCipherMac in '..\..\CryptoLib\src\Interfaces\Crypto\Macs\ClpICfbBlockCipherMac.pas', ClpByteUtilities in '..\..\CryptoLib\src\NumberUtilities\ClpByteUtilities.pas', + ClpByteXorSimd in '..\..\CryptoLib\src\NumberUtilities\ClpByteXorSimd.pas', + ClpByteXorX86Backend in '..\..\CryptoLib\src\NumberUtilities\ClpByteXorX86Backend.pas', ClpInt16Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt16Utilities.pas', ClpInt32Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt32Utilities.pas', ClpInt64Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt64Utilities.pas', diff --git a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dpr b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dpr index c4398a7c..2e7edcaf 100644 --- a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dpr +++ b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dpr @@ -688,14 +688,18 @@ uses ClpIFusedCcmKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCcmKernel.pas', ClpIFusedEaxKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedEaxKernel.pas', ClpIFusedGcmSivKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedGcmSivKernel.pas', + ClpIFusedCtrKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCtrKernel.pas', + ClpIFusedCbcKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCbcKernel.pas', ClpFusedKernelRegistry in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpFusedKernelRegistry.pas', ClpAesFusedAeadSimd in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesFusedAeadSimd.pas', - ClpAesFusedAeadX86Backend in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesFusedAeadX86Backend.pas', + ClpAesNiFusedX86Backend in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesNiFusedX86Backend.pas', ClpBinPolySimd in '..\..\CryptoLib\src\Math\BinPoly\ClpBinPolySimd.pas', ClpAesNiOcbKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiOcbKernel.pas', ClpAesNiCcmKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCcmKernel.pas', ClpAesNiEaxKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiEaxKernel.pas', ClpAesNiGcmKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiGcmKernel.pas', + ClpAesNiCtrKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCtrKernel.pas', + ClpAesNiCbcKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCbcKernel.pas', ClpPclmulGcmSivKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpPclmulGcmSivKernel.pas', ClpFusedKernelDefaults in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpFusedKernelDefaults.pas', ClpAeadParameters in '..\..\CryptoLib\src\Crypto\Parameters\ClpAeadParameters.pas', @@ -725,6 +729,8 @@ uses ClpCfbBlockCipherMac in '..\..\CryptoLib\src\Crypto\Macs\ClpCfbBlockCipherMac.pas', ClpICfbBlockCipherMac in '..\..\CryptoLib\src\Interfaces\Crypto\Macs\ClpICfbBlockCipherMac.pas', ClpByteUtilities in '..\..\CryptoLib\src\NumberUtilities\ClpByteUtilities.pas', + ClpByteXorSimd in '..\..\CryptoLib\src\NumberUtilities\ClpByteXorSimd.pas', + ClpByteXorX86Backend in '..\..\CryptoLib\src\NumberUtilities\ClpByteXorX86Backend.pas', ClpInt16Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt16Utilities.pas', ClpInt32Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt32Utilities.pas', ClpInt64Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt64Utilities.pas', @@ -865,6 +871,7 @@ uses SicBulkParityTests in '..\src\Crypto\SicBulkParityTests.pas', EcbBulkParityTests in '..\src\Crypto\EcbBulkParityTests.pas', CbcBulkParityTests in '..\src\Crypto\CbcBulkParityTests.pas', + GcmSivBulkParityTests in '..\src\Crypto\GcmSivBulkParityTests.pas', SpeckBlockCipherTestBase in '..\src\Crypto\SpeckBlockCipherTestBase.pas', SpeckLegacyTests in '..\src\Crypto\SpeckLegacyTests.pas', SpeckTests in '..\src\Crypto\SpeckTests.pas', diff --git a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dproj b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dproj index 2814605e..91ad8636 100644 --- a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dproj +++ b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.Mobile.dproj @@ -1031,14 +1031,18 @@ + + - + + + @@ -1068,6 +1072,8 @@ + + @@ -1204,6 +1210,7 @@ + diff --git a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.dpr b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.dpr index 8eaf0505..9bb65b5f 100644 --- a/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.dpr +++ b/CryptoLib.Tests/Delphi.Tests/CryptoLib.Tests.dpr @@ -707,14 +707,18 @@ uses ClpIFusedCcmKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCcmKernel.pas', ClpIFusedEaxKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedEaxKernel.pas', ClpIFusedGcmSivKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedGcmSivKernel.pas', + ClpIFusedCtrKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCtrKernel.pas', + ClpIFusedCbcKernel in '..\..\CryptoLib\src\Interfaces\Crypto\Modes\Fused\ClpIFusedCbcKernel.pas', ClpFusedKernelRegistry in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpFusedKernelRegistry.pas', ClpAesFusedAeadSimd in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesFusedAeadSimd.pas', - ClpAesFusedAeadX86Backend in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesFusedAeadX86Backend.pas', + ClpAesNiFusedX86Backend in '..\..\CryptoLib\src\Crypto\Modes\Fused\Internal\ClpAesNiFusedX86Backend.pas', ClpBinPolySimd in '..\..\CryptoLib\src\Math\BinPoly\ClpBinPolySimd.pas', ClpAesNiOcbKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiOcbKernel.pas', ClpAesNiCcmKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCcmKernel.pas', ClpAesNiEaxKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiEaxKernel.pas', ClpAesNiGcmKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiGcmKernel.pas', + ClpAesNiCtrKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCtrKernel.pas', + ClpAesNiCbcKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpAesNiCbcKernel.pas', ClpPclmulGcmSivKernel in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpPclmulGcmSivKernel.pas', ClpFusedKernelDefaults in '..\..\CryptoLib\src\Crypto\Modes\Fused\ClpFusedKernelDefaults.pas', ClpAeadParameters in '..\..\CryptoLib\src\Crypto\Parameters\ClpAeadParameters.pas', @@ -744,6 +748,8 @@ uses ClpCfbBlockCipherMac in '..\..\CryptoLib\src\Crypto\Macs\ClpCfbBlockCipherMac.pas', ClpICfbBlockCipherMac in '..\..\CryptoLib\src\Interfaces\Crypto\Macs\ClpICfbBlockCipherMac.pas', ClpByteUtilities in '..\..\CryptoLib\src\NumberUtilities\ClpByteUtilities.pas', + ClpByteXorSimd in '..\..\CryptoLib\src\NumberUtilities\ClpByteXorSimd.pas', + ClpByteXorX86Backend in '..\..\CryptoLib\src\NumberUtilities\ClpByteXorX86Backend.pas', ClpInt16Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt16Utilities.pas', ClpInt32Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt32Utilities.pas', ClpInt64Utilities in '..\..\CryptoLib\src\NumberUtilities\ClpInt64Utilities.pas', @@ -884,6 +890,7 @@ uses SicBulkParityTests in '..\src\Crypto\SicBulkParityTests.pas', EcbBulkParityTests in '..\src\Crypto\EcbBulkParityTests.pas', CbcBulkParityTests in '..\src\Crypto\CbcBulkParityTests.pas', + GcmSivBulkParityTests in '..\src\Crypto\GcmSivBulkParityTests.pas', SpeckBlockCipherTestBase in '..\src\Crypto\SpeckBlockCipherTestBase.pas', SpeckLegacyTests in '..\src\Crypto\SpeckLegacyTests.pas', SpeckTests in '..\src\Crypto\SpeckTests.pas', diff --git a/CryptoLib.Tests/FreePascal.Tests/CryptoLib.Tests.lpi b/CryptoLib.Tests/FreePascal.Tests/CryptoLib.Tests.lpi index eedf4965..9c15e648 100644 --- a/CryptoLib.Tests/FreePascal.Tests/CryptoLib.Tests.lpi +++ b/CryptoLib.Tests/FreePascal.Tests/CryptoLib.Tests.lpi @@ -79,7 +79,7 @@ - + @@ -821,6 +821,10 @@ + + + + diff --git a/CryptoLib.Tests/FreePascal.Tests/CryptoLib.lpr b/CryptoLib.Tests/FreePascal.Tests/CryptoLib.lpr index c2338fa4..2cb3a95b 100644 --- a/CryptoLib.Tests/FreePascal.Tests/CryptoLib.lpr +++ b/CryptoLib.Tests/FreePascal.Tests/CryptoLib.lpr @@ -11,7 +11,7 @@ NamedCurveTests, SignerUtilitiesTests, SecureRandomTests, DigestRandomNumberTests, FixedPointTests, AESTests, AesLightTests, FusedExternalRegistrationTests, AesHardwareEngineTests, AESSICTests, - SicBulkParityTests, EcbBulkParityTests, CbcBulkParityTests, + SicBulkParityTests, EcbBulkParityTests, CbcBulkParityTests, GcmSivBulkParityTests, SpeckBlockCipherTestBase, SpeckLegacyTests, SpeckTests, IESCipherTests, HMacTests, Pkcs5Tests, HkdfGeneratorTests, ECIESTests, PascalCoinECIESTests, ECNRTests, PrimesTests, ECEncodingTests, GF256AesTests, PaddingTests, diff --git a/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpi b/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpi index b6ea61f6..213a0319 100644 --- a/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpi +++ b/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpi @@ -39,7 +39,7 @@ - + @@ -780,6 +780,10 @@ + + + + diff --git a/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpr b/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpr index 3807da20..7dcf180a 100644 --- a/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpr +++ b/CryptoLib.Tests/FreePascal.Tests/CryptoLibConsole.lpr @@ -12,7 +12,7 @@ NamedCurveTests, SignerUtilitiesTests, SecureRandomTests, DigestRandomNumberTests, FixedPointTests, AESTests, AesLightTests, FusedExternalRegistrationTests, AesHardwareEngineTests, AESSICTests, - SicBulkParityTests, EcbBulkParityTests, CbcBulkParityTests, + SicBulkParityTests, EcbBulkParityTests, CbcBulkParityTests, GcmSivBulkParityTests, SpeckBlockCipherTestBase, SpeckLegacyTests, SpeckTests, IESCipherTests, HMacTests, Pkcs5Tests, HkdfGeneratorTests, ECIESTests, PascalCoinECIESTests, ECNRTests, PrimesTests, ECEncodingTests, GF256AesTests, PaddingTests, diff --git a/CryptoLib.Tests/src/Crypto/AESSICTests.pas b/CryptoLib.Tests/src/Crypto/AESSICTests.pas index e427d4e8..b7fe6765 100644 --- a/CryptoLib.Tests/src/Crypto/AESSICTests.pas +++ b/CryptoLib.Tests/src/Crypto/AESSICTests.pas @@ -59,6 +59,15 @@ TTestAESSIC = class(TCryptoLibAlgorithmTestCase) published procedure TestAESSIC; + /// + /// Exercises the AES-NI 8-wide CTR fast path (ICtrBlockCipher) on buffers + /// large enough to hit it, cross-checked against an independent + /// ECB-of-counter-blocks keystream reference, plus a decrypt round-trip. + /// Covers all three key sizes and block counts that span below-batch, + /// exact-batch, batch+tail and multi-batch, with an IV that forces + /// big-endian counter carries across byte boundaries. + /// + procedure TestAESCTRLargeBufferMatchesEcbReference; end; @@ -201,6 +210,92 @@ procedure TTestAESSIC.TestAESSIC; end; +// Big-endian whole-counter increment, identical to TSicBlockCipher's. +procedure IncCounterBE(const ACounter: TBytes); +var + Lk: Int32; +begin + Lk := System.Length(ACounter) - 1; + while Lk >= 0 do + begin + ACounter[Lk] := Byte(ACounter[Lk] + 1); + if ACounter[Lk] <> 0 then + Break; + System.Dec(Lk); + end; +end; + +procedure TTestAESSIC.TestAESCTRLargeBufferMatchesEcbReference; +const + BLOCK = 16; + // Below one 8-block batch, exactly one batch, batch + tail, multi-batch exact. + CBlockCounts: array [0 .. 3] of Int32 = (7, 8, 61, 64); +var + LKeyBytes: array [0 .. 2] of Int32; + LKi, LNi, LN, Li, Lj, LLen: Int32; + LKey, LIV, LPlain, LCounters, LKeystream, LRef, LCtr, LBack, LCtrBlk: TBytes; + LKeyParam: IKeyParameter; + LEcb, LCtrCipher: IBufferedCipher; +begin + LKeyBytes[0] := 16; + LKeyBytes[1] := 24; + LKeyBytes[2] := 32; + + for LKi := 0 to 2 do + begin + System.SetLength(LKey, LKeyBytes[LKi]); + for Li := 0 to System.High(LKey) do + LKey[Li] := Byte((Li * 11) + (LKi * 7) + 1); + LKeyParam := TParameterUtilities.CreateKeyParameter('AES', LKey); + + // Full 16-byte IV whose low bytes force per-block carries across byte + // boundaries (and through the low-32 region) as the counter advances. + LIV := DecodeHex('000102030405060708090A0B0CFFFFFE'); + + for LNi := 0 to System.High(CBlockCounts) do + begin + LN := CBlockCounts[LNi]; + LLen := LN * BLOCK; + + System.SetLength(LPlain, LLen); + for Li := 0 to LLen - 1 do + LPlain[Li] := Byte((Li * 13) xor (Li shr 3) xor (LN * 5)); + + // Independent reference: keystream = ECB(counter_i), ct = pt xor keystream. + System.SetLength(LCounters, LLen); + LCtrBlk := System.Copy(LIV, 0, BLOCK); + for Li := 0 to LN - 1 do + begin + System.Move(LCtrBlk[0], LCounters[Li * BLOCK], BLOCK); + IncCounterBE(LCtrBlk); + end; + LEcb := TCipherUtilities.GetCipher('AES/ECB/NoPadding'); + LEcb.Init(True, LKeyParam); + LKeystream := LEcb.DoFinal(LCounters); + System.SetLength(LRef, LLen); + for Lj := 0 to LLen - 1 do + LRef[Lj] := Byte(LPlain[Lj] xor LKeystream[Lj]); + + // CTR fast path over the whole buffer (hits the 8-wide kernel for LN >= 8). + LCtrCipher := TCipherUtilities.GetCipher('AES/CTR/NoPadding'); + LCtrCipher.Init(True, TParametersWithIV.Create(LKeyParam, LIV) + as IParametersWithIV); + LCtr := LCtrCipher.DoFinal(LPlain); + if not AreEqual(LCtr, LRef) then + Fail(Format('CTR fast path != ECB reference (key %d bytes, %d blocks)', + [LKeyBytes[LKi], LN])); + + // Decrypt round-trip must recover the plaintext. + LCtrCipher.Init(False, TParametersWithIV.Create(LKeyParam, LIV) + as IParametersWithIV); + LBack := LCtrCipher.DoFinal(LCtr); + if not AreEqual(LBack, LPlain) then + Fail(Format('CTR round-trip failed (key %d bytes, %d blocks)', + [LKeyBytes[LKi], LN])); + end; + end; +end; + initialization {$IFDEF FPC} diff --git a/CryptoLib.Tests/src/Crypto/GcmSivBulkParityTests.pas b/CryptoLib.Tests/src/Crypto/GcmSivBulkParityTests.pas new file mode 100644 index 00000000..06536473 --- /dev/null +++ b/CryptoLib.Tests/src/Crypto/GcmSivBulkParityTests.pas @@ -0,0 +1,185 @@ +{ *********************************************************************************** } +{ * CryptoLib Library * } +{ * Author - Ugochukwu Mmaduekwe * } +{ * Github Repository * } +{ * * } +{ * Distributed under the MIT software license, see the accompanying file LICENSE * } +{ * or visit http://www.opensource.org/licenses/mit-license.php. * } +{ * * } +{ * Acknowledgements: * } +{ * * } +{ * Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring * } +{ * the development of this library * } +{ * ******************************************************************************* * } + +(* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) + +unit GcmSivBulkParityTests; + +{$I ..\..\..\CryptoLib\src\Include\CryptoLib.inc} + +interface + +uses + SysUtils, +{$IFDEF FPC} + fpcunit, + testregistry, +{$ELSE} + TestFramework, +{$ENDIF FPC} + ClpGcmSivBlockCipher, + ClpIGcmSivBlockCipher, + ClpAeadParameters, + ClpKeyParameter, + ClpIKeyParameter, + ClpICipherParameters, + ClpFusedKernelRegistry, + ClpSecureRandom, + ClpISecureRandom, + ClpConverters, + ClpCryptoLibTypes, + CryptoLibTestBase; + +type + /// + /// AES-GCM-SIV POLYVAL parity: the fused POLYVAL kernel (used for hashed spans + /// >= 128 bytes) must produce byte-identical ciphertext+tag to the scalar + /// fallback across AAD/plaintext lengths around the 8-block boundary. + /// Cross-comparing fused vs scalar output catches a symmetric POLYVAL bug that + /// a round-trip (matching wrong tag on both sides) cannot. + /// + TTestGcmSivBulkParity = class(TCryptoLibAlgorithmTestCase) + strict private + function GcmSivEncrypt(const AKey, ANonce, AAad, APlain: TBytes; + AUseFused: Boolean): TBytes; + function GcmSivDecrypt(const AKey, ANonce, AAad, ACipher: TBytes; + AUseFused: Boolean): TBytes; + procedure RunParity(AKeyBytes: Int32; const ALabel: String); + published + procedure TestGcmSiv128FusedVsScalarParity; + procedure TestGcmSiv256FusedVsScalarParity; + end; + +implementation + +{ TTestGcmSivBulkParity } + +function TTestGcmSivBulkParity.GcmSivEncrypt(const AKey, ANonce, AAad, + APlain: TBytes; AUseFused: Boolean): TBytes; +var + LCipher: IGcmSivBlockCipher; + LSaved: Boolean; + LLen: Int32; +begin + LSaved := TFusedKernelGate.ForceDisabled; + // The fused POLYVAL kernel is resolved during Init (DeriveKeys); the gate must + // therefore be set BEFORE Init to select the fused or scalar path. + TFusedKernelGate.ForceDisabled := not AUseFused; + try + LCipher := TGcmSivBlockCipher.Create() as IGcmSivBlockCipher; + LCipher.Init(True, TAeadParameters.Create(TKeyParameter.Create(AKey) + as IKeyParameter, 128, ANonce, AAad) as ICipherParameters); + System.SetLength(Result, LCipher.GetOutputSize(System.Length(APlain))); + LLen := LCipher.ProcessBytes(APlain, 0, System.Length(APlain), nil, 0); + LLen := LLen + LCipher.DoFinal(Result, 0); + if LLen <> System.Length(Result) then + System.SetLength(Result, LLen); + finally + TFusedKernelGate.ForceDisabled := LSaved; + end; +end; + +function TTestGcmSivBulkParity.GcmSivDecrypt(const AKey, ANonce, AAad, + ACipher: TBytes; AUseFused: Boolean): TBytes; +var + LCipher: IGcmSivBlockCipher; + LSaved: Boolean; + LLen: Int32; +begin + LSaved := TFusedKernelGate.ForceDisabled; + TFusedKernelGate.ForceDisabled := not AUseFused; + try + LCipher := TGcmSivBlockCipher.Create() as IGcmSivBlockCipher; + LCipher.Init(False, TAeadParameters.Create(TKeyParameter.Create(AKey) + as IKeyParameter, 128, ANonce, AAad) as ICipherParameters); + System.SetLength(Result, LCipher.GetOutputSize(System.Length(ACipher))); + LLen := LCipher.ProcessBytes(ACipher, 0, System.Length(ACipher), nil, 0); + LLen := LLen + LCipher.DoFinal(Result, 0); + if LLen <> System.Length(Result) then + System.SetLength(Result, LLen); + finally + TFusedKernelGate.ForceDisabled := LSaved; + end; +end; + +procedure TTestGcmSivBulkParity.RunParity(AKeyBytes: Int32; const ALabel: String); +const + // Lengths straddling the 8-block (128-byte) fused batch boundary, the 16-byte + // tail, and multi-batch spans. + Lengths: array [0 .. 20] of Int32 = (0, 1, 15, 16, 17, 31, 96, 112, 127, 128, + 129, 143, 144, 240, 255, 256, 257, 383, 384, 512, 1000); +var + LRnd: ISecureRandom; + LKey, LNonce, LAad, LPlain, LEncFused, LEncScalar, LDec: TBytes; + LAi, LPi, LIter: Int32; +begin + LRnd := TSecureRandom.GetInstance('SHA256PRNG'); + LRnd.SetSeed(TConverters.ConvertStringToBytes( + 'GcmSivBulkParitySeed-' + ALabel, TEncoding.ASCII)); + + System.SetLength(LKey, AKeyBytes); + System.SetLength(LNonce, 12); + + for LAi := 0 to System.Length(Lengths) - 1 do + for LPi := 0 to System.Length(Lengths) - 1 do + for LIter := 0 to 1 do + begin + LRnd.NextBytes(LKey); + LRnd.NextBytes(LNonce); + + System.SetLength(LAad, Lengths[LAi]); + if System.Length(LAad) > 0 then + LRnd.NextBytes(LAad); + System.SetLength(LPlain, Lengths[LPi]); + if System.Length(LPlain) > 0 then + LRnd.NextBytes(LPlain); + + LEncFused := GcmSivEncrypt(LKey, LNonce, LAad, LPlain, True); + LEncScalar := GcmSivEncrypt(LKey, LNonce, LAad, LPlain, False); + + // The whole point: fused and scalar POLYVAL must agree byte-for-byte. + if not AreEqual(LEncFused, LEncScalar) then + Fail(Format('%s: fused vs scalar ciphertext+tag mismatch ' + + '(aad=%d plain=%d iter=%d)', + [ALabel, Lengths[LAi], Lengths[LPi], LIter])); + + // Cross-mode round-trip: the fused encryption must decrypt (and verify) + // under the scalar path back to the original plaintext. + LDec := GcmSivDecrypt(LKey, LNonce, LAad, LEncFused, False); + if not AreEqual(LPlain, LDec) then + Fail(Format('%s: scalar decrypt of fused ciphertext mismatch ' + + '(aad=%d plain=%d iter=%d)', + [ALabel, Lengths[LAi], Lengths[LPi], LIter])); + end; +end; + +procedure TTestGcmSivBulkParity.TestGcmSiv128FusedVsScalarParity; +begin + RunParity(16, 'AES-128-GCM-SIV'); +end; + +procedure TTestGcmSivBulkParity.TestGcmSiv256FusedVsScalarParity; +begin + RunParity(32, 'AES-256-GCM-SIV'); +end; + +initialization + +{$IFDEF FPC} + RegisterTest(TTestGcmSivBulkParity); +{$ELSE} + RegisterTest(TTestGcmSivBulkParity.Suite); +{$ENDIF FPC} + +end. diff --git a/CryptoLib/src/Crypto/Macs/ClpCMac.pas b/CryptoLib/src/Crypto/Macs/ClpCMac.pas index bc0b397a..b49737c6 100644 --- a/CryptoLib/src/Crypto/Macs/ClpCMac.pas +++ b/CryptoLib/src/Crypto/Macs/ClpCMac.pas @@ -34,6 +34,7 @@ interface ClpIISO7816d4Padding, ClpArrayUtilities, ClpBitOperations, + ClpByteUtilities, ClpCryptoLibTypes; resourcestring @@ -229,7 +230,7 @@ procedure TCMac.BlockUpdate(const AInput: TCryptoLibByteArray; function TCMac.DoFinal(const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; var - LBlockSize, LI: Int32; + LBlockSize: Int32; LLu: TCryptoLibByteArray; LPadding: IISO7816d4Padding; begin @@ -246,8 +247,7 @@ function TCMac.DoFinal(const AOutput: TCryptoLibByteArray; LLu := FLu2; end; - for LI := 0 to System.Pred(System.Length(FMac)) do - FBuf[LI] := FBuf[LI] xor LLu[LI]; + TByteUtilities.XorTo(System.Length(FMac), PByte(@LLu[0]), PByte(@FBuf[0])); FCipherMode.ProcessBlock(FBuf, 0, FMac, 0); System.Move(FMac[0], AOutput[AOutOff], FMacSize * System.SizeOf(Byte)); diff --git a/CryptoLib/src/Crypto/Modes/ClpCbcBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpCbcBlockCipher.pas index d96d02c4..805b84cf 100644 --- a/CryptoLib/src/Crypto/Modes/ClpCbcBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpCbcBlockCipher.pas @@ -30,7 +30,11 @@ interface ClpICipherParameters, ClpIParametersWithIV, ClpBlockCipherBulkUtilities, + ClpFusedKernelTypes, + ClpIFusedCbcKernel, + ClpFusedKernelRegistry, ClpArrayUtilities, + ClpByteUtilities, ClpCryptoLibTypes; resourcestring @@ -67,6 +71,12 @@ TCbcBlockCipher = class sealed(TInterfacedObject, ICbcBlockCipher, // one interface dispatch per block. Any bulk-capable block cipher // lights up both paths automatically by implementing the interface. FBulkCipher: IBulkBlockCipher; + // Cached on encrypt Init only. Non-nil when a registered accelerator + // claims the underlying cipher. Runs the whole serial CBC chain + // C[i] = E(P[i] xor C[i-1]) in one call with the chaining value held in a + // register, replacing the per-block dispatch loop. nil otherwise -- + // CbcEncryptBulk keeps its per-block path. + FCbcKernel: IFusedCbcKernel; function EncryptBlock(const AInput: TCryptoLibByteArray; AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32): Int32; @@ -152,15 +162,14 @@ constructor TCbcBlockCipher.Create(const ACipher: IBlockCipher); function TCbcBlockCipher.DecryptBlock(const AInput: TCryptoLibByteArray; AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32): Int32; var - LLength, LI: Int32; + LLength: Int32; LTmp: TCryptoLibByteArray; begin if ((AInOff + FBlockSize) > System.Length(AInput)) then raise EDataLengthCryptoLibException.CreateRes(@SInputBufferTooShort); System.Move(AInput[AInOff], FCbcNextV[0], FBlockSize * System.SizeOf(Byte)); LLength := FCipher.ProcessBlock(AInput, AInOff, AOutBytes, AOutOff); - for LI := 0 to System.Pred(FBlockSize) do - AOutBytes[AOutOff + LI] := AOutBytes[AOutOff + LI] xor FCbcV[LI]; + TByteUtilities.XorTo(FBlockSize, PByte(@FCbcV[0]), PByte(@AOutBytes[AOutOff])); LTmp := FCbcV; FCbcV := FCbcNextV; FCbcNextV := LTmp; @@ -170,12 +179,11 @@ function TCbcBlockCipher.DecryptBlock(const AInput: TCryptoLibByteArray; function TCbcBlockCipher.EncryptBlock(const AInput: TCryptoLibByteArray; AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32): Int32; var - LI, LLen: Int32; + LLen: Int32; begin if ((AInOff + FBlockSize) > System.Length(AInput)) then raise EDataLengthCryptoLibException.CreateRes(@SInputBufferTooShort); - for LI := 0 to System.Pred(FBlockSize) do - FCbcV[LI] := FCbcV[LI] xor AInput[AInOff + LI]; + TByteUtilities.XorTo(FBlockSize, PByte(@AInput[AInOff]), PByte(@FCbcV[0])); LLen := FCipher.ProcessBlock(FCbcV, 0, AOutBytes, AOutOff); System.Move(AOutBytes[AOutOff], FCbcV[0], System.Length(FCbcV) * System.SizeOf(Byte)); Result := LLen; @@ -186,11 +194,22 @@ procedure TCbcBlockCipher.CbcEncryptBulk(const AInBuf: TCryptoLibByteArray; AOutOff: Int32); var LPCbcV, LPIn, LPOut: PByte; - LI: Int32; begin // 16-byte CBC encryption: C[i] = ENC(P[i] XOR C[i-1]). Serial chain. // We mutate FCbcV in place across the batch; the final value equals the // last produced ciphertext block, exactly matching sequential EncryptBlock. + + // Fast path: a registered accelerator runs the whole serial chain in one + // call, holding the chaining value in a register (no per-block dispatch, + // XOR helper call, or FCbcV store/reload). It updates FCbcV in place, so the + // post-state matches the per-block path exactly. + if FCbcKernel <> nil then + begin + FCbcKernel.ProcessCbcBlocks(@AInBuf[AInOff], @AOutBuf[AOutOff], + @FCbcV[0], ABlockCount); + Exit; + end; + // Dispatches via FBulkCipher.ProcessBlock (inherited from IBlockCipher); // this is the same vtable entry the engine already services for single- // block calls, so no extra probe per iteration. @@ -199,8 +218,7 @@ procedure TCbcBlockCipher.CbcEncryptBulk(const AInBuf: TCryptoLibByteArray; begin LPIn := @AInBuf[AInOff]; LPOut := @AOutBuf[AOutOff]; - for LI := 0 to 15 do - LPCbcV[LI] := LPCbcV[LI] xor LPIn[LI]; + TByteUtilities.XorTo(16, LPIn, LPCbcV); FBulkCipher.ProcessBlock(FCbcV, 0, AOutBuf, AOutOff); System.Move(LPOut^, LPCbcV^, 16); System.Inc(AInOff, 16); @@ -214,40 +232,44 @@ procedure TCbcBlockCipher.CbcDecryptBulk(const AInBuf: TCryptoLibByteArray; AOutOff: Int32); var LCtStage: array [0 .. 127] of Byte; - LPOut, LPStage, LPCbcV: PByte; - LI, LJ: Int32; + LPIn, LPOut: PByte; + LTotal: Int32; begin // 16-byte CBC decryption: P[i] = DEC(C[i]) XOR C[i-1] (with C[-1] = IV - // carried in FCbcV). The inverse calls across i are independent, so we - // batch 8 through the engine's bulk path, stage the raw ciphertext up - // front (so in-place aliasing cannot corrupt the XOR feed), then apply - // the chain XOR. 1..7 block residue goes per-block below -- the - // engine's internal 4/1 ladder would kick in if we routed residue - // through it too, but per-block is already engine-fast and the residue - // is <= 112 bytes per message so the lost 4-wide parallelism is sub- - // noise. + // carried in FCbcV). DEC(C[i]) are all independent (8-wide parallel), and + // every XOR feed C[i-1] is *input* ciphertext -- so there is no serial + // dependency; the whole run parallelises. + LPIn := @AInBuf[AInOff]; + LPOut := @AOutBuf[AOutOff]; + LTotal := ABlockCount * 16; + + // Fast path: input and output regions do not overlap (the usual case -- the + // buffered cipher hands us separate buffers). Decrypt the entire run in one + // engine call (full 8-wide throughput, = ECB decrypt), then apply the chain + // XOR in a single pass reading the still-intact input: out[0] ^= FCbcV, + // out[i] ^= C[i-1] (= input block i-1). No per-batch staging. + if (NativeUInt(LPIn) + NativeUInt(LTotal) <= NativeUInt(LPOut)) or + (NativeUInt(LPOut) + NativeUInt(LTotal) <= NativeUInt(LPIn)) then + begin + FBulkCipher.ProcessBlocks(AInBuf, AInOff, ABlockCount, AOutBuf, AOutOff); + TByteUtilities.&Xor(16, LPOut, @FCbcV[0], LPOut); + if ABlockCount > 1 then + TByteUtilities.&Xor((ABlockCount - 1) * 16, LPOut + 16, LPIn, LPOut + 16); + System.Move((LPIn + (ABlockCount - 1) * 16)^, FCbcV[0], 16); + Exit; + end; + + // Overlapping / in-place path: stage each 8-block batch's ciphertext up front + // (so the in-place decrypt cannot corrupt the XOR feed), then chain-XOR. while ABlockCount >= 8 do begin - // Snapshot ciphertext so the engine can safely run in-place; the - // chain XOR then has guaranteed access to the original bytes. System.Move(AInBuf[AInOff], LCtStage[0], 128); FBulkCipher.ProcessBlocks(AInBuf, AInOff, 8, AOutBuf, AOutOff); LPOut := @AOutBuf[AOutOff]; - LPCbcV := @FCbcV[0]; - // First block XORs against the previous-batch chain state (FCbcV = IV - // on the very first batch, or the last ciphertext of the prior batch). - for LI := 0 to 15 do - LPOut[LI] := LPOut[LI] xor LPCbcV[LI]; - // Blocks 1..7 XOR against the staged ciphertext of the previous block - // within this batch. - LPStage := @LCtStage[0]; - for LJ := 1 to 7 do - for LI := 0 to 15 do - LPOut[LJ * 16 + LI] := LPOut[LJ * 16 + LI] xor - LPStage[(LJ - 1) * 16 + LI]; - - // Chain state -> last ciphertext block of this batch. + TByteUtilities.&Xor(16, LPOut, @FCbcV[0], LPOut); + TByteUtilities.&Xor(112, LPOut + 16, @LCtStage[0], LPOut + 16); + System.Move(LCtStage[7 * 16], FCbcV[0], 16); System.Inc(AInOff, 128); System.Inc(AOutOff, 128); @@ -320,6 +342,15 @@ procedure TCbcBlockCipher.Init(AForEncryption: Boolean; // guard in ProcessBlocks keeps us correct if a future non-16-byte bulk // engine ever surfaces. TBlockCipherBulkUtilities.TryResolveBulkCipher(FCipher, FBulkCipher); + + // CBC encrypt is serial; acquire the in-register-chain accelerator when a + // provider claims FCipher. Encrypt-only: CBC decrypt is parallel and already + // served by the bulk engine's 8-wide path. + if FEncrypting then + TFusedKernelRegistry.TryAcquireCbc(FCipher, TFusedModeDirection.Encrypt, + FCbcKernel) + else + FCbcKernel := nil; end; function TCbcBlockCipher.ProcessBlock(const AInput: TCryptoLibByteArray; diff --git a/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas index b1e3d50e..facb24ba 100644 --- a/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpCcmBlockCipher.pas @@ -44,6 +44,7 @@ interface ClpParametersWithIV, ClpCheck, ClpArrayUtilities, + ClpByteUtilities, ClpStreamUtilities, ClpCryptoLibTypes; @@ -61,6 +62,30 @@ interface SCannotReuseNonce = 'cannot reuse nonce for %s encryption'; type + // Per-packet inputs a body decryptor needs (Dest/DestOff set by RunDecrypt). + TCcmDecryptCtx = record + Input: TCryptoLibByteArray; + InOff: Int32; + OutputLen: Int32; + Dest: TCryptoLibByteArray; + DestOff: Int32; + Iv: TCryptoLibByteArray; + CtrCipher: ISicBlockCipher; + BulkCtr: IBulkBlockCipherMode; + end; + + // Per-packet inputs an encrypt body needs. + TCcmEncryptCtx = record + Input: TCryptoLibByteArray; + InOff: Int32; + InLen: Int32; + Output: TCryptoLibByteArray; + OutOff: Int32; + Iv: TCryptoLibByteArray; + CtrCipher: ISicBlockCipher; + BulkCtr: IBulkBlockCipherMode; + end; + TCcmBlockCipher = class(TInterfacedObject, ICcmBlockCipher, IAeadBlockCipher, IAeadCipher) @@ -78,7 +103,19 @@ TCcmBlockCipher = class(TInterfacedObject, ICcmBlockCipher, FKeyParam: ICipherParameters; FLastKey: TCryptoLibByteArray; FAssociatedText: TMemoryStream; - FData: TMemoryStream; + // Accumulated body input (plaintext on encrypt, ciphertext+tag on decrypt). + // CCM needs the total length before processing (B_0 encodes it), so input is + // buffered here and consumed once in DoFinal. Capacity grows by doubling and + // is reused across packets (FDataLen is the used prefix); DoFinal hands this + // buffer straight to ProcessPacket, avoiding the readback copy a stream needs. + FData: TCryptoLibByteArray; + FDataLen: Int32; + // Reused decrypt staging buffer. CCM must not release unverified plaintext, + // so decrypt lands here first; the tag is verified, and only on success is + // the plaintext copied to the caller's output (which stays untouched on + // failure). Grown by doubling and reused across packets, and wiped after + // every packet (in a finally) so no recovered plaintext lingers. + FPlainScratch: TCryptoLibByteArray; // Cached once per Init; non-nil when the registry resolved a fused // CCM kernel for the underlying cipher and current direction. FCcmKernel: IFusedCcmKernel; @@ -95,18 +132,16 @@ TCcmBlockCipher = class(TInterfacedObject, ICcmBlockCipher, // AMacState. Matches the scalar CalculateMac contract. procedure ComputePostHeaderMacState(AInLen: Int32; const AMacState: TCryptoLibByteArray); - // Fused 2-wide CTR + CBC-MAC body path. Returns False if the fused - // kernel cannot be invoked. On success, writes the ciphertext, - // scalar tail, tag, and FMacBlock (raw pre-encryption MAC). - function ProcessPacketEncryptFused(const AInput: TCryptoLibByteArray; - AInOff, AInLen: Int32; const AOutput: TCryptoLibByteArray; - AOutOff: Int32; const AIV: TCryptoLibByteArray): Boolean; - // Fused decrypt twin. Raises on tag mismatch, matching the scalar - // path. On success FMacBlock holds the zero-padded received MAC. - function ProcessPacketDecryptFused(const AInput: TCryptoLibByteArray; - AInOff, AInLen, AOutputLen: Int32; - const AOutput: TCryptoLibByteArray; AOutOff: Int32; - const AIV: TCryptoLibByteArray): Boolean; + // Encrypt bodies (fused kernel / scalar SIC). + procedure EncryptBodyFused(const ACtx: TCcmEncryptCtx); + procedure EncryptBodyScalar(const ACtx: TCcmEncryptCtx); + // Decrypt bodies (fused kernel / scalar SIC); return whether the tag verifies. + function DecryptBodyFused(const ACtx: TCcmDecryptCtx): Boolean; + function DecryptBodyScalar(const ACtx: TCcmDecryptCtx): Boolean; + + // Decrypts into scratch, verifies, releases to output only on success. + function RunDecrypt(AUseFused: Boolean; var ACtx: TCcmDecryptCtx; + const AOutput: TCryptoLibByteArray; AOutOff: Int32): Boolean; strict protected function GetAlgorithmName: String; virtual; @@ -152,7 +187,9 @@ constructor TCcmBlockCipher.Create(const ACipher: IBlockCipher); FCipher := ACipher; System.SetLength(FMacBlock, BlockSize); FAssociatedText := TMemoryStream.Create; - FData := TMemoryStream.Create; + FData := nil; + FDataLen := 0; + FPlainScratch := nil; if (ACipher.GetBlockSize() <> BlockSize) then raise EArgumentCryptoLibException.CreateResFmt(@SCipherRequired, [BlockSize]); @@ -161,7 +198,6 @@ constructor TCcmBlockCipher.Create(const ACipher: IBlockCipher); destructor TCcmBlockCipher.Destroy; begin FAssociatedText.Free; - FData.Free; inherited Destroy; end; @@ -264,7 +300,7 @@ procedure TCcmBlockCipher.ProcessAadBytes(const AInput: TCryptoLibByteArray; function TCcmBlockCipher.ProcessByte(AInput: Byte; const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; begin - FData.WriteByte(AInput); + TArrayUtilities.AppendTo(FData, FDataLen, AInput); Result := 0; end; @@ -272,25 +308,16 @@ function TCcmBlockCipher.ProcessBytes(const AInput: TCryptoLibByteArray; AInOff, ALen: Int32; const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; begin TCheck.DataLength(AInput, AInOff, ALen, SInputBufferTooShort); - FData.WriteBuffer(AInput[AInOff], ALen); + TArrayUtilities.AppendTo(FData, FDataLen, AInput, AInOff, ALen); Result := 0; end; function TCcmBlockCipher.DoFinal(const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; -var - LInput: TCryptoLibByteArray; - LInLen: Int32; begin - LInLen := Int32(FData.Size); - System.SetLength(LInput, LInLen); - if LInLen > 0 then - begin - FData.Position := 0; - FData.ReadBuffer(LInput[0], LInLen); - end; - - Result := ProcessPacket(LInput, 0, LInLen, AOutput, AOutOff); + // The accumulator is already a contiguous byte array, so hand its used + // prefix straight to ProcessPacket (no readback copy / staging alloc). + Result := ProcessPacket(FData, 0, FDataLen, AOutput, AOutOff); Reset(); end; @@ -298,7 +325,7 @@ function TCcmBlockCipher.DoFinal(const AOutput: TCryptoLibByteArray; procedure TCcmBlockCipher.Reset; begin FAssociatedText.Size := 0; - FData.Size := 0; + FDataLen := 0; end; function TCcmBlockCipher.GetMac: TCryptoLibByteArray; @@ -315,7 +342,7 @@ function TCcmBlockCipher.GetOutputSize(ALen: Int32): Int32; var LTotalData: Int32; begin - LTotalData := Int32(FData.Size) + ALen; + LTotalData := FDataLen + ALen; if FForEncryption then begin @@ -356,14 +383,16 @@ function TCcmBlockCipher.ProcessPacket(const AInput: TCryptoLibByteArray; AInOff, AInLen: Int32; const AOutput: TCryptoLibByteArray; AOutOff: Int32): Int32; var - LN, LQ, LLimitLen, LInputAdjustment, LOutputLen, LInIndex, LOutIndex, LI, - LBulkBlocks, LBulkBytes: Int32; - LIV, LEncMac, LBlock, LCalculatedMacBlock, LPlain: TCryptoLibByteArray; + LN, LQ, LLimitLen, LInputAdjustment, LOutputLen: Int32; + LIV: TCryptoLibByteArray; LCtrCipher: ISicBlockCipher; // Cached IBulkBlockCipherMode view of LCtrCipher. TSicBlockCipher always // implements IBulkBlockCipherMode, so this is non-nil in practice; the // Supports() guard keeps us robust to a future SIC variant that opts out. LBulkCtr: IBulkBlockCipherMode; + LEncCtx: TCcmEncryptCtx; + LDecCtx: TCcmDecryptCtx; + LUseFused: Boolean; begin TCheck.DataLength(AInput, AInOff, AInLen, SInputBufferTooShort); @@ -395,62 +424,26 @@ function TCcmBlockCipher.ProcessPacket(const AInput: TCryptoLibByteArray; LCtrCipher.Init(FForEncryption, TParametersWithIV.Create(FKeyParam, LIV) as IParametersWithIV); TBlockCipherBulkUtilities.TryResolveBulkCipherMode(LCtrCipher, LBulkCtr); - LInIndex := AInOff; - LOutIndex := AOutOff; - if FForEncryption then begin LOutputLen := AInLen + FMacSize; TCheck.OutputLength(AOutput, AOutOff, LOutputLen, SOutputBufferTooShort); - // Fused fast path folds CTR and CBC-MAC into one sweep; the scalar - // path handles the 1..16-byte tail and the tag encryption. - if (FCcmKernel <> nil) - and ((AInLen - 1) div BlockSize > 0) - and ProcessPacketEncryptFused(AInput, AInOff, AInLen, AOutput, - AOutOff, LIV) then - begin - Result := LOutputLen; - Exit; - end; - - CalculateMac(AInput, AInOff, AInLen, FMacBlock); - - System.SetLength(LEncMac, BlockSize); - LCtrCipher.ProcessBlock(FMacBlock, 0, LEncMac, 0); - - // Number of whole 16-byte blocks that the classic loop would have - // consumed. The tail (1..BlockSize bytes) is always handled via the - // LBlock scratch path below, so we intentionally hold back the last - // (possibly full) block and let the per-block tail finish it. This - // preserves byte-identical behaviour with the pre-bulk code. - LBulkBlocks := (AInLen - 1) div BlockSize; - if (LBulkCtr <> nil) and (LBulkBlocks > 0) then - begin - LBulkBytes := LBulkCtr.ProcessBlocks(AInput, LInIndex, LBulkBlocks, - AOutput, LOutIndex); - LInIndex := LInIndex + LBulkBytes; - LOutIndex := LOutIndex + LBulkBytes; - end + LEncCtx.Input := AInput; + LEncCtx.InOff := AInOff; + LEncCtx.InLen := AInLen; + LEncCtx.Output := AOutput; + LEncCtx.OutOff := AOutOff; + LEncCtx.Iv := LIV; + LEncCtx.CtrCipher := LCtrCipher; + LEncCtx.BulkCtr := LBulkCtr; + + // Fused kernel when it can cover at least one stride; scalar otherwise. + if (FCcmKernel <> nil) and + ((AInLen - 1) div BlockSize >= FCcmKernel.MinimumBlockCount) then + EncryptBodyFused(LEncCtx) else - begin - while (LInIndex < (AInOff + AInLen - BlockSize)) do - begin - LCtrCipher.ProcessBlock(AInput, LInIndex, AOutput, LOutIndex); - LOutIndex := LOutIndex + BlockSize; - LInIndex := LInIndex + BlockSize; - end; - end; - - System.SetLength(LBlock, BlockSize); - - System.Move(AInput[LInIndex], LBlock[0], AInLen + AInOff - LInIndex); - - LCtrCipher.ProcessBlock(LBlock, 0, LBlock, 0); - - System.Move(LBlock[0], AOutput[LOutIndex], AInLen + AInOff - LInIndex); - - System.Move(LEncMac[0], AOutput[AOutOff + AInLen], FMacSize); + EncryptBodyScalar(LEncCtx); end else begin @@ -460,72 +453,18 @@ function TCcmBlockCipher.ProcessPacket(const AInput: TCryptoLibByteArray; LOutputLen := AInLen - FMacSize; TCheck.OutputLength(AOutput, AOutOff, LOutputLen, SOutputBufferTooShort); - // Fused decrypt twin. Scalar tail handles the trailing 1..16-byte - // block plus the FixedTime tag compare. - if (FCcmKernel <> nil) - and ((LOutputLen - 1) div BlockSize > 0) - and ProcessPacketDecryptFused(AInput, AInOff, AInLen, LOutputLen, - AOutput, AOutOff, LIV) then - begin - Result := LOutputLen; - Exit; - end; - - System.Move(AInput[AInOff + LOutputLen], FMacBlock[0], FMacSize); - - LCtrCipher.ProcessBlock(FMacBlock, 0, FMacBlock, 0); - - for LI := FMacSize to System.Pred(System.Length(FMacBlock)) do - begin - FMacBlock[LI] := 0; - end; - - System.SetLength(LPlain, LOutputLen); - try - LOutIndex := 0; - - // Same LBulkBlocks / tail split as encrypt, but over LOutputLen - // (ciphertext minus trailing tag) rather than AInLen. The last - // (possibly full) 16-byte block is held back for the LBlock scratch - // path so behaviour matches the pre-bulk loop byte-for-byte. - LBulkBlocks := (LOutputLen - 1) div BlockSize; - if (LBulkCtr <> nil) and (LBulkBlocks > 0) then - begin - LBulkBytes := LBulkCtr.ProcessBlocks(AInput, LInIndex, LBulkBlocks, - LPlain, LOutIndex); - LInIndex := LInIndex + LBulkBytes; - LOutIndex := LOutIndex + LBulkBytes; - end - else - begin - while (LInIndex < (AInOff + LOutputLen - BlockSize)) do - begin - LCtrCipher.ProcessBlock(AInput, LInIndex, LPlain, LOutIndex); - LOutIndex := LOutIndex + BlockSize; - LInIndex := LInIndex + BlockSize; - end; - end; + LDecCtx.Input := AInput; + LDecCtx.InOff := AInOff; + LDecCtx.OutputLen := LOutputLen; + LDecCtx.Iv := LIV; + LDecCtx.CtrCipher := LCtrCipher; + LDecCtx.BulkCtr := LBulkCtr; - System.SetLength(LBlock, BlockSize); + // Fused kernel when it can cover at least one stride; scalar otherwise. + LUseFused := (FCcmKernel <> nil) and + ((LOutputLen - 1) div BlockSize >= FCcmKernel.MinimumBlockCount); - System.Move(AInput[LInIndex], LBlock[0], LOutputLen - (LInIndex - AInOff)); - - LCtrCipher.ProcessBlock(LBlock, 0, LBlock, 0); - - System.Move(LBlock[0], LPlain[LOutIndex], LOutputLen - (LInIndex - AInOff)); - - System.SetLength(LCalculatedMacBlock, BlockSize); - - CalculateMac(LPlain, 0, LOutputLen, LCalculatedMacBlock); - - if (not TArrayUtilities.FixedTimeEquals(FMacBlock, LCalculatedMacBlock)) then - raise EInvalidCipherTextCryptoLibException.CreateResFmt(@SMacCheckFailed, ['CCM']); - - System.Move(LPlain[0], AOutput[AOutOff], LOutputLen); - finally - if LPlain <> nil then - TArrayUtilities.Fill(LPlain, 0, System.Length(LPlain), 0); - end; + RunDecrypt(LUseFused, LDecCtx, AOutput, AOutOff); end; Result := LOutputLen; @@ -718,139 +657,220 @@ procedure TCcmBlockCipher.ComputePostHeaderMacState(AInLen: Int32; LOffset := 0; while LOffset < LHeaderLen do begin - for LI := 0 to BlockSize - 1 do - LBlock[LI] := AMacState[LI] xor LHeader[LOffset + LI]; + TByteUtilities.&Xor(BlockSize, PByte(@AMacState[0]), PByte(@LHeader[LOffset]), + PByte(@LBlock[0])); FCipher.ProcessBlock(LBlock, 0, AMacState, 0); System.Inc(LOffset, BlockSize); end; end; -function TCcmBlockCipher.ProcessPacketEncryptFused( - const AInput: TCryptoLibByteArray; AInOff, AInLen: Int32; - const AOutput: TCryptoLibByteArray; AOutOff: Int32; - const AIV: TCryptoLibByteArray): Boolean; +procedure TCcmBlockCipher.EncryptBodyFused(const ACtx: TCcmEncryptCtx); var - LBulkBlocks, LTailLen, LI, LTailStart: Int32; + LBulkBlocks, LTailLen, LTailStart: Int32; LS0, LMacState, LCtrBlock, LTailBlock: TCryptoLibByteArray; begin - Result := False; - if FCcmKernel = nil then - Exit; - - LBulkBlocks := (AInLen - 1) div BlockSize; - if LBulkBlocks < FCcmKernel.MinimumBlockCount then - Exit; + LBulkBlocks := (ACtx.InLen - 1) div BlockSize; // S_0 = E_K(J_0); XOR with the final MAC to emit the tag. System.SetLength(LS0, BlockSize); - FCipher.ProcessBlock(AIV, 0, LS0, 0); + FCipher.ProcessBlock(ACtx.Iv, 0, LS0, 0); System.SetLength(LMacState, BlockSize); - ComputePostHeaderMacState(AInLen, LMacState); + ComputePostHeaderMacState(ACtx.InLen, LMacState); // Body counter block (counter = 1). System.SetLength(LCtrBlock, BlockSize); - System.Move(AIV[0], LCtrBlock[0], BlockSize); + System.Move(ACtx.Iv[0], LCtrBlock[0], BlockSize); LCtrBlock[BlockSize - 1] := LCtrBlock[BlockSize - 1] or 1; - FCcmKernel.ProcessBody(@AInput[AInOff], @AOutput[AOutOff], + // Fused CTR + CBC-MAC over the body. + FCcmKernel.ProcessBody(@ACtx.Input[ACtx.InOff], @ACtx.Output[ACtx.OutOff], @LCtrBlock[0], @LMacState[0], LBulkBlocks); // Scalar tail: kernel held back the last 1..16 bytes. LCtrBlock now // carries counter_{1 + LBulkBlocks}. - LTailLen := AInLen - LBulkBlocks * BlockSize; - LTailStart := AInOff + LBulkBlocks * BlockSize; + LTailLen := ACtx.InLen - LBulkBlocks * BlockSize; + LTailStart := ACtx.InOff + LBulkBlocks * BlockSize; System.SetLength(LTailBlock, BlockSize); - System.Move(AInput[LTailStart], LTailBlock[0], LTailLen); - for LI := 0 to BlockSize - 1 do - LMacState[LI] := LMacState[LI] xor LTailBlock[LI]; + System.Move(ACtx.Input[LTailStart], LTailBlock[0], LTailLen); + TByteUtilities.XorTo(BlockSize, PByte(@LTailBlock[0]), PByte(@LMacState[0])); FCipher.ProcessBlock(LMacState, 0, LMacState, 0); FCipher.ProcessBlock(LCtrBlock, 0, LTailBlock, 0); - for LI := 0 to LTailLen - 1 do - AOutput[AOutOff + LBulkBlocks * BlockSize + LI] := - AInput[LTailStart + LI] xor LTailBlock[LI]; + TByteUtilities.&Xor(LTailLen, PByte(@ACtx.Input[LTailStart]), PByte(@LTailBlock[0]), + PByte(@ACtx.Output[ACtx.OutOff + LBulkBlocks * BlockSize])); // FMacBlock holds the raw pre-encryption MAC (GetMac contract). System.Move(LMacState[0], FMacBlock[0], BlockSize); - for LI := 0 to FMacSize - 1 do - AOutput[AOutOff + AInLen + LI] := LMacState[LI] xor LS0[LI]; + TByteUtilities.&Xor(FMacSize, PByte(@LMacState[0]), PByte(@LS0[0]), + PByte(@ACtx.Output[ACtx.OutOff + ACtx.InLen])); +end; + +procedure TCcmBlockCipher.EncryptBodyScalar(const ACtx: TCcmEncryptCtx); +var + LInIndex, LOutIndex, LBulkBlocks, LBulkBytes: Int32; + LEncMac, LBlock: TCryptoLibByteArray; +begin + CalculateMac(ACtx.Input, ACtx.InOff, ACtx.InLen, FMacBlock); + + // Encrypt the tag with S_0 (CtrCipher's first block); this advances CtrCipher + // to counter 1, which the body CTR below continues from. + System.SetLength(LEncMac, BlockSize); + ACtx.CtrCipher.ProcessBlock(FMacBlock, 0, LEncMac, 0); + + LInIndex := ACtx.InOff; + LOutIndex := ACtx.OutOff; + + // The last (possibly full) 16-byte block is held back for the LBlock scratch + // path so behaviour matches the pre-bulk loop byte-for-byte. + LBulkBlocks := (ACtx.InLen - 1) div BlockSize; + if (ACtx.BulkCtr <> nil) and (LBulkBlocks > 0) then + begin + LBulkBytes := ACtx.BulkCtr.ProcessBlocks(ACtx.Input, LInIndex, LBulkBlocks, + ACtx.Output, LOutIndex); + LInIndex := LInIndex + LBulkBytes; + LOutIndex := LOutIndex + LBulkBytes; + end + else + begin + while (LInIndex < (ACtx.InOff + ACtx.InLen - BlockSize)) do + begin + ACtx.CtrCipher.ProcessBlock(ACtx.Input, LInIndex, ACtx.Output, LOutIndex); + LOutIndex := LOutIndex + BlockSize; + LInIndex := LInIndex + BlockSize; + end; + end; + + System.SetLength(LBlock, BlockSize); + System.Move(ACtx.Input[LInIndex], LBlock[0], ACtx.InLen + ACtx.InOff - LInIndex); + ACtx.CtrCipher.ProcessBlock(LBlock, 0, LBlock, 0); + System.Move(LBlock[0], ACtx.Output[LOutIndex], ACtx.InLen + ACtx.InOff - LInIndex); - Result := True; + System.Move(LEncMac[0], ACtx.Output[ACtx.OutOff + ACtx.InLen], FMacSize); end; -function TCcmBlockCipher.ProcessPacketDecryptFused( - const AInput: TCryptoLibByteArray; AInOff, AInLen, AOutputLen: Int32; - const AOutput: TCryptoLibByteArray; AOutOff: Int32; - const AIV: TCryptoLibByteArray): Boolean; +function TCcmBlockCipher.DecryptBodyFused(const ACtx: TCcmDecryptCtx): Boolean; var LBulkBlocks, LTailLen, LI, LTailStart: Int32; LS0, LMacState, LCtrBlock, LTailBlock, LReceivedRawMac, - LComputedMac, LPlain: TCryptoLibByteArray; + LComputedMac: TCryptoLibByteArray; begin - Result := False; - if FCcmKernel = nil then - Exit; - - LBulkBlocks := (AOutputLen - 1) div BlockSize; - if LBulkBlocks < FCcmKernel.MinimumBlockCount then - Exit; + LBulkBlocks := (ACtx.OutputLen - 1) div BlockSize; System.SetLength(LS0, BlockSize); - FCipher.ProcessBlock(AIV, 0, LS0, 0); + FCipher.ProcessBlock(ACtx.Iv, 0, LS0, 0); // Decrypt the received MAC: R = (enc_tag || 0..) XOR S_0 truncated. System.SetLength(LReceivedRawMac, BlockSize); - System.Move(AInput[AInOff + AOutputLen], LReceivedRawMac[0], FMacSize); - for LI := 0 to FMacSize - 1 do - LReceivedRawMac[LI] := LReceivedRawMac[LI] xor LS0[LI]; + System.Move(ACtx.Input[ACtx.InOff + ACtx.OutputLen], LReceivedRawMac[0], FMacSize); + TByteUtilities.XorTo(FMacSize, PByte(@LS0[0]), PByte(@LReceivedRawMac[0])); System.Move(LReceivedRawMac[0], FMacBlock[0], BlockSize); System.SetLength(LMacState, BlockSize); - ComputePostHeaderMacState(AOutputLen, LMacState); + ComputePostHeaderMacState(ACtx.OutputLen, LMacState); // Body counter block (counter = 1). System.SetLength(LCtrBlock, BlockSize); - System.Move(AIV[0], LCtrBlock[0], BlockSize); + System.Move(ACtx.Iv[0], LCtrBlock[0], BlockSize); LCtrBlock[BlockSize - 1] := LCtrBlock[BlockSize - 1] or 1; - System.SetLength(LPlain, AOutputLen); + // Fused CTR + CBC-MAC over the body, straight into the caller-chosen dest. + FCcmKernel.ProcessBody(@ACtx.Input[ACtx.InOff], @ACtx.Dest[ACtx.DestOff], + @LCtrBlock[0], @LMacState[0], LBulkBlocks); + + // Scalar tail: decrypt via keystream XOR, then fold into the MAC. + LTailLen := ACtx.OutputLen - LBulkBlocks * BlockSize; + LTailStart := ACtx.InOff + LBulkBlocks * BlockSize; + + System.SetLength(LTailBlock, BlockSize); + FCipher.ProcessBlock(LCtrBlock, 0, LTailBlock, 0); + TByteUtilities.&Xor(LTailLen, PByte(@ACtx.Input[LTailStart]), PByte(@LTailBlock[0]), + PByte(@ACtx.Dest[ACtx.DestOff + LBulkBlocks * BlockSize])); + + // Zero-pad plaintext tail and fold one last CBC step. + System.FillChar(LTailBlock[0], BlockSize, 0); + for LI := 0 to LTailLen - 1 do + LTailBlock[LI] := ACtx.Dest[ACtx.DestOff + LBulkBlocks * BlockSize + LI]; + TByteUtilities.XorTo(BlockSize, PByte(@LTailBlock[0]), PByte(@LMacState[0])); + FCipher.ProcessBlock(LMacState, 0, LMacState, 0); + + System.SetLength(LComputedMac, BlockSize); + System.Move(LMacState[0], LComputedMac[0], FMacSize); + + Result := TArrayUtilities.FixedTimeEquals(LReceivedRawMac, LComputedMac); +end; + +function TCcmBlockCipher.DecryptBodyScalar(const ACtx: TCcmDecryptCtx): Boolean; +var + LInIndex, LOutIndex, LI, LBulkBlocks, LBulkBytes: Int32; + LBlock, LCalculatedMacBlock: TCryptoLibByteArray; +begin + // Expected MAC: received tag XOR S_0 keystream (CtrCipher's first block), + // then zero-pad. Consuming S_0 here advances CtrCipher to counter 1, which + // the body CTR below continues from. + System.Move(ACtx.Input[ACtx.InOff + ACtx.OutputLen], FMacBlock[0], FMacSize); + ACtx.CtrCipher.ProcessBlock(FMacBlock, 0, FMacBlock, 0); + for LI := FMacSize to System.Pred(System.Length(FMacBlock)) do + FMacBlock[LI] := 0; + + LInIndex := ACtx.InOff; + LOutIndex := ACtx.DestOff; + + // LBulkBlocks / tail split: the last (possibly full) 16-byte block is held + // back for the LBlock scratch path so behaviour matches the pre-bulk loop. + LBulkBlocks := (ACtx.OutputLen - 1) div BlockSize; + if (ACtx.BulkCtr <> nil) and (LBulkBlocks > 0) then + begin + LBulkBytes := ACtx.BulkCtr.ProcessBlocks(ACtx.Input, LInIndex, LBulkBlocks, + ACtx.Dest, LOutIndex); + LInIndex := LInIndex + LBulkBytes; + LOutIndex := LOutIndex + LBulkBytes; + end + else + begin + while (LInIndex < (ACtx.InOff + ACtx.OutputLen - BlockSize)) do + begin + ACtx.CtrCipher.ProcessBlock(ACtx.Input, LInIndex, ACtx.Dest, LOutIndex); + LOutIndex := LOutIndex + BlockSize; + LInIndex := LInIndex + BlockSize; + end; + end; + + System.SetLength(LBlock, BlockSize); + System.Move(ACtx.Input[LInIndex], LBlock[0], + ACtx.OutputLen - (LInIndex - ACtx.InOff)); + ACtx.CtrCipher.ProcessBlock(LBlock, 0, LBlock, 0); + System.Move(LBlock[0], ACtx.Dest[LOutIndex], + ACtx.OutputLen - (LInIndex - ACtx.InOff)); + + System.SetLength(LCalculatedMacBlock, BlockSize); + CalculateMac(ACtx.Dest, ACtx.DestOff, ACtx.OutputLen, LCalculatedMacBlock); + + Result := TArrayUtilities.FixedTimeEquals(FMacBlock, LCalculatedMacBlock); +end; + +function TCcmBlockCipher.RunDecrypt(AUseFused: Boolean; + var ACtx: TCcmDecryptCtx; const AOutput: TCryptoLibByteArray; + AOutOff: Int32): Boolean; +var + LOk: Boolean; +begin + TArrayUtilities.EnsureCapacity(FPlainScratch, ACtx.OutputLen); + ACtx.Dest := FPlainScratch; + ACtx.DestOff := 0; try - FCcmKernel.ProcessBody(@AInput[AInOff], @LPlain[0], - @LCtrBlock[0], @LMacState[0], LBulkBlocks); - - // Scalar tail: decrypt via keystream XOR, then fold into the MAC. - LTailLen := AOutputLen - LBulkBlocks * BlockSize; - LTailStart := AInOff + LBulkBlocks * BlockSize; - - System.SetLength(LTailBlock, BlockSize); - FCipher.ProcessBlock(LCtrBlock, 0, LTailBlock, 0); - for LI := 0 to LTailLen - 1 do - LPlain[LBulkBlocks * BlockSize + LI] := - AInput[LTailStart + LI] xor LTailBlock[LI]; - - // Zero-pad plaintext tail and fold one last CBC step. - System.FillChar(LTailBlock[0], BlockSize, 0); - for LI := 0 to LTailLen - 1 do - LTailBlock[LI] := LPlain[LBulkBlocks * BlockSize + LI]; - for LI := 0 to BlockSize - 1 do - LMacState[LI] := LMacState[LI] xor LTailBlock[LI]; - FCipher.ProcessBlock(LMacState, 0, LMacState, 0); - - System.SetLength(LComputedMac, BlockSize); - System.Move(LMacState[0], LComputedMac[0], FMacSize); - - if not TArrayUtilities.FixedTimeEquals(LReceivedRawMac, LComputedMac) then + if AUseFused then + LOk := DecryptBodyFused(ACtx) + else + LOk := DecryptBodyScalar(ACtx); + if not LOk then raise EInvalidCipherTextCryptoLibException.CreateResFmt(@SMacCheckFailed, ['CCM']); - - System.Move(LPlain[0], AOutput[AOutOff], AOutputLen); + System.Move(FPlainScratch[0], AOutput[AOutOff], ACtx.OutputLen); Result := True; finally - if LPlain <> nil then - TArrayUtilities.Fill(LPlain, 0, System.Length(LPlain), 0); + TArrayUtilities.Fill(FPlainScratch, 0, ACtx.OutputLen, Byte(0)); end; end; - end. diff --git a/CryptoLib/src/Crypto/Modes/ClpCfbBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpCfbBlockCipher.pas index 7a4c4e62..ac596a71 100644 --- a/CryptoLib/src/Crypto/Modes/ClpCfbBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpCfbBlockCipher.pas @@ -142,7 +142,7 @@ constructor TCfbBlockCipher.Create(const ACipher: IBlockCipher; function TCfbBlockCipher.DecryptBlock(const AInput: TCryptoLibByteArray; AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32): Int32; var - LI, LCount: Int32; + LCount: Int32; begin if ((AInOff + FBlockSize) > System.Length(AInput)) then raise EDataLengthCryptoLibException.CreateRes(@SInputBufferTooShort); @@ -159,8 +159,8 @@ function TCfbBlockCipher.DecryptBlock(const AInput: TCryptoLibByteArray; System.Move(AInput[AInOff], FCfbV[(System.Length(FCfbV) - FBlockSize)], FBlockSize * System.SizeOf(Byte)); - for LI := 0 to System.Pred(FBlockSize) do - AOutBytes[AOutOff + LI] := Byte(FCfbOutV[LI] xor AInput[AInOff + LI]); + TByteUtilities.&Xor(FBlockSize, PByte(@FCfbOutV[0]), PByte(@AInput[AInOff]), + PByte(@AOutBytes[AOutOff])); Result := FBlockSize; end; @@ -168,7 +168,7 @@ function TCfbBlockCipher.DecryptBlock(const AInput: TCryptoLibByteArray; function TCfbBlockCipher.EncryptBlock(const AInput: TCryptoLibByteArray; AInOff: Int32; const AOutBytes: TCryptoLibByteArray; AOutOff: Int32): Int32; var - LI, LCount: Int32; + LCount: Int32; begin if ((AInOff + FBlockSize) > System.Length(AInput)) then raise EDataLengthCryptoLibException.CreateRes(@SInputBufferTooShort); @@ -178,8 +178,8 @@ function TCfbBlockCipher.EncryptBlock(const AInput: TCryptoLibByteArray; FCipher.ProcessBlock(FCfbV, 0, FCfbOutV, 0); - for LI := 0 to System.Pred(FBlockSize) do - AOutBytes[AOutOff + LI] := Byte(FCfbOutV[LI] xor AInput[AInOff + LI]); + TByteUtilities.&Xor(FBlockSize, PByte(@FCfbOutV[0]), PByte(@AInput[AInOff]), + PByte(@AOutBytes[AOutOff])); LCount := (System.Length(FCfbV) - FBlockSize) * System.SizeOf(Byte); if LCount > 0 then @@ -254,7 +254,7 @@ function TCfbBlockCipher.ProcessBlocks(const AInBuf: TCryptoLibByteArray; AInOff, ABlockCount: Int32; const AOutBuf: TCryptoLibByteArray; AOutOff: Int32): Int32; var - LI, LBS, LTotalBytes, LK: Int32; + LI, LBS, LTotalBytes: Int32; LScratch: TCryptoLibByteArray; begin LBS := FBlockSize; @@ -287,8 +287,7 @@ function TCfbBlockCipher.ProcessBlocks(const AInBuf: TCryptoLibByteArray; // on previously-seen ciphertext, never on the not-yet-produced plaintext, // so we can stage them contiguously and let the bulk engine encrypt // them all in one SIMD-accelerated call. Plaintext is then - // P_k = E_K(FCfbV_k) xor C_k, done via the 128 / 64-byte triple-XOR - // primitives in TBlockCipherBulkUtilities. + // P_k = E_K(FCfbV_k) xor C_k, applied over the whole run by TByteUtilities. System.SetLength(LScratch, LTotalBytes); System.Move(FCfbV[0], LScratch[0], LBS); if ABlockCount > 1 then @@ -302,21 +301,8 @@ function TCfbBlockCipher.ProcessBlocks(const AInBuf: TCryptoLibByteArray; FBulkCipher.ProcessBlocks(LScratch, 0, ABlockCount, LScratch, 0); - LK := 0; - while LK + 128 <= LTotalBytes do - begin - TByteUtilities.&Xor(128, PByte(LScratch) + LK, PByte(AInBuf) + AInOff + LK, - PByte(AOutBuf) + AOutOff + LK); - LK := LK + 128; - end; - while LK + 64 <= LTotalBytes do - begin - TByteUtilities.&Xor(64, PByte(LScratch) + LK, PByte(AInBuf) + AInOff + LK, - PByte(AOutBuf) + AOutOff + LK); - LK := LK + 64; - end; - for LI := LK to LTotalBytes - 1 do - AOutBuf[AOutOff + LI] := LScratch[LI] xor AInBuf[AInOff + LI]; + TByteUtilities.&Xor(LTotalBytes, PByte(LScratch), PByte(AInBuf) + AInOff, + PByte(AOutBuf) + AOutOff); Result := LTotalBytes; end; diff --git a/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas index 9127ab92..7cce28bc 100644 --- a/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpGcmBlockCipher.pas @@ -152,11 +152,10 @@ TGcmBlockCipher = class(TInterfacedObject, IGcmBlockCipher, procedure GhashFourShuffledBlocks(PC0, PC16, PC32, PC48: PByte); procedure GhashEightShuffledBlocks(PBase: PByte); /// - /// Shared big-endian counter-word packing used by both `FillNextCtrBlocks8Raw` - /// and `GetNextCtrBlocks8`'s SIMD fast path. Advances `ACounter32` by 8 and - /// writes the eight 16-byte counter blocks (pre-AES form) into `ABlocks[0..127]`. - /// Also mutates `ACounter[12..15]` in place (the block-index tail) as a side - /// effect of the byte-packing strategy. + /// Big-endian counter-word packing for `GetNextCtrBlocks8`'s SIMD fast path. + /// Advances `ACounter32` by 8 and writes the eight 16-byte counter blocks + /// (pre-AES form) into `ABlocks[0..127]`. Also mutates `ACounter[12..15]` in + /// place (the block-index tail) as a side effect of the byte-packing strategy. /// class procedure FillCtr8BlocksRaw(const ACounter: TCryptoLibByteArray; var ACounter32: UInt32; const ABlocks: TCryptoLibByteArray); static; @@ -201,8 +200,6 @@ TGcmBlockCipher = class(TInterfacedObject, IGcmBlockCipher, // how it is scheduled against the available vector-register budget is a // backend detail hidden entirely behind the IFusedGcmKernel surface, so // this driver only ever sees the kernel interface. - /// Fills ABlocks[0..127] with eight 16-byte counter blocks (pre-cipher form). Used by the FusedILP pipeline where the block-cipher keystream is produced inside the fused kernel. - procedure FillNextCtrBlocks8Raw(const ABlocks: TCryptoLibByteArray); /// /// Pipelined GCM path driven by FGcmKernel (when a fused kernel is /// registered). Active when a fused CTR+GHASH kernel was acquired at Init; otherwise the @@ -1390,11 +1387,6 @@ class procedure TGcmBlockCipher.FillCtr8BlocksRaw( // interface, summarised on the matching banner in the class declaration. // ======================================================================= -procedure TGcmBlockCipher.FillNextCtrBlocks8Raw(const ABlocks: TCryptoLibByteArray); -begin - FillCtr8BlocksRaw(FCounter, FCounter32, ABlocks); -end; - // Fused block-cipher keystream + 8-way GHASH pipeline. The cipher engine is // always in encrypt mode (CTR keystream) regardless of GCM direction. // AForEncrypt selects the per-direction bookkeeping only: @@ -1419,8 +1411,9 @@ procedure TGcmBlockCipher.ProcessBlocks8FusedILP(const AInBuf: TCryptoLibByteArr var AInOff: Int32; var ALen: Int32; const AOutBuf: TCryptoLibByteArray; var AOutOff: Int32; ALimit: Int32; AForEncrypt: Boolean); var - LCurrCtrs, LNextCtrs: TCryptoLibByteArray; + LCurrCtrs: TCryptoLibByteArray; LPrevCipher, LPOut, LPIn: PByte; + LBatches: Int32; begin if FGcmKernel = nil then Exit; @@ -1428,7 +1421,6 @@ procedure TGcmBlockCipher.ProcessBlocks8FusedILP(const AInBuf: TCryptoLibByteArr Exit; LCurrCtrs := FWorkCtr; - LNextCtrs := FWorkCtrAhead; // Prime batch 0: regular 8-wide keystream into LCurrCtrs (now holds keystream), // XOR with plaintext/ciphertext at LPOut, defer GHASH of batch 0. @@ -1445,24 +1437,25 @@ procedure TGcmBlockCipher.ProcessBlocks8FusedILP(const AInBuf: TCryptoLibByteArr AOutOff := AOutOff + (BlockSize * 8); ALen := ALen - (BlockSize * 8); - while ALen >= ALimit + (BlockSize * 8) * 2 do + // Bulk: process all full batches in one kernel call (counters and the + // lagging GHASH pipeline run internally), seeded with the prime batch's + // ciphertext and leaving the final batch for the drain below. + LBatches := 0; + if ALen >= ALimit + (BlockSize * 8) * 2 then + LBatches := (ALen - ALimit - (BlockSize * 8)) div (BlockSize * 8); + if LBatches > 0 then begin - // Fill raw (pre-AES) counter blocks; the kernel AES-encrypts them in-place. - FillNextCtrBlocks8Raw(LNextCtrs); - LPIn := PByte(AInBuf) + AInOff; LPOut := PByte(AOutBuf) + AOutOff; - - FGcmKernel.ProcessCtrGhashBatch(LPIn, LPOut, @LNextCtrs[0], LPrevCipher, - @FS[0], FGcmKernelMinBlocks); - + FCounter32 := FGcmKernel.ProcessCtrGhashBatches(LPIn, LPOut, LPrevCipher, + @FS[0], @FJ0[0], FCounter32, LBatches, AForEncrypt); if AForEncrypt then - LPrevCipher := LPOut + LPrevCipher := LPOut + (LBatches - 1) * (BlockSize * 8) else - LPrevCipher := LPIn; - AInOff := AInOff + (BlockSize * 8); - AOutOff := AOutOff + (BlockSize * 8); - ALen := ALen - (BlockSize * 8); + LPrevCipher := LPIn + (LBatches - 1) * (BlockSize * 8); + AInOff := AInOff + LBatches * (BlockSize * 8); + AOutOff := AOutOff + LBatches * (BlockSize * 8); + ALen := ALen - LBatches * (BlockSize * 8); end; // Tail: GHASH the last pending ciphertext, then produce and GHASH the final batch. @@ -1548,69 +1541,23 @@ procedure TGcmBlockCipher.CipherBlock(const AInBuf: TCryptoLibByteArray; AForEncrypt: Boolean); var LCtrBlock: TCryptoLibByteArray; - LI: Int32; - LC0, LC1, LC2, LC3: Byte; begin LCtrBlock := nil; System.SetLength(LCtrBlock, BlockSize); GetNextCtrBlock(LCtrBlock); - if TGhashSimd.IsBlockXorSupported then - begin - if AForEncrypt then - begin - System.Move(LCtrBlock[0], AOutBuf[AOutOff], BlockSize); - TGhashSimd.BlockXor128(@AOutBuf[AOutOff], @AInBuf[AInOff]); - TGhashSimd.BlockXor128(@FS[0], @AOutBuf[AOutOff]); - end - else - begin - System.Move(AInBuf[AInOff], AOutBuf[AOutOff], BlockSize); - TGhashSimd.BlockXor128(@AOutBuf[AOutOff], @LCtrBlock[0]); - TGhashSimd.BlockXor128(@FS[0], @AInBuf[AInOff]); - end; - FMultiplier.MultiplyH(FS); - Exit; - end; - + // Encrypt: C := keystream xor In; FS := FS xor C; Out := C. + // Decrypt: FS := FS xor In (= ciphertext); Out := In xor keystream. if AForEncrypt then begin - for LI := 0 to (BlockSize - 1) div 4 do - begin - LC0 := Byte(LCtrBlock[(LI * 4) + 0] xor AInBuf[AInOff + (LI * 4) + 0]); - LC1 := Byte(LCtrBlock[(LI * 4) + 1] xor AInBuf[AInOff + (LI * 4) + 1]); - LC2 := Byte(LCtrBlock[(LI * 4) + 2] xor AInBuf[AInOff + (LI * 4) + 2]); - LC3 := Byte(LCtrBlock[(LI * 4) + 3] xor AInBuf[AInOff + (LI * 4) + 3]); - - FS[(LI * 4) + 0] := FS[(LI * 4) + 0] xor LC0; - FS[(LI * 4) + 1] := FS[(LI * 4) + 1] xor LC1; - FS[(LI * 4) + 2] := FS[(LI * 4) + 2] xor LC2; - FS[(LI * 4) + 3] := FS[(LI * 4) + 3] xor LC3; - - AOutBuf[AOutOff + (LI * 4) + 0] := LC0; - AOutBuf[AOutOff + (LI * 4) + 1] := LC1; - AOutBuf[AOutOff + (LI * 4) + 2] := LC2; - AOutBuf[AOutOff + (LI * 4) + 3] := LC3; - end; + System.Move(LCtrBlock[0], AOutBuf[AOutOff], BlockSize); + TByteUtilities.XorTo(BlockSize, PByte(@AInBuf[AInOff]), PByte(@AOutBuf[AOutOff])); + TByteUtilities.XorTo(BlockSize, PByte(@AOutBuf[AOutOff]), PByte(@FS[0])); end else begin - for LI := 0 to (BlockSize - 1) div 4 do - begin - LC0 := AInBuf[AInOff + (LI * 4) + 0]; - LC1 := AInBuf[AInOff + (LI * 4) + 1]; - LC2 := AInBuf[AInOff + (LI * 4) + 2]; - LC3 := AInBuf[AInOff + (LI * 4) + 3]; - - FS[(LI * 4) + 0] := FS[(LI * 4) + 0] xor LC0; - FS[(LI * 4) + 1] := FS[(LI * 4) + 1] xor LC1; - FS[(LI * 4) + 2] := FS[(LI * 4) + 2] xor LC2; - FS[(LI * 4) + 3] := FS[(LI * 4) + 3] xor LC3; - - AOutBuf[AOutOff + (LI * 4) + 0] := Byte(LC0 xor LCtrBlock[(LI * 4) + 0]); - AOutBuf[AOutOff + (LI * 4) + 1] := Byte(LC1 xor LCtrBlock[(LI * 4) + 1]); - AOutBuf[AOutOff + (LI * 4) + 2] := Byte(LC2 xor LCtrBlock[(LI * 4) + 2]); - AOutBuf[AOutOff + (LI * 4) + 3] := Byte(LC3 xor LCtrBlock[(LI * 4) + 3]); - end; + System.Move(AInBuf[AInOff], AOutBuf[AOutOff], BlockSize); + TByteUtilities.XorTo(BlockSize, PByte(@LCtrBlock[0]), PByte(@AOutBuf[AOutOff])); + TByteUtilities.XorTo(BlockSize, PByte(@AInBuf[AInOff]), PByte(@FS[0])); end; FMultiplier.MultiplyH(FS); end; @@ -1670,78 +1617,25 @@ procedure TGcmBlockCipher.CipherBlocks2(const AInBuf: TCryptoLibByteArray; AForEncrypt: Boolean); var LCtrBlock: TCryptoLibByteArray; - LI, LB: Int32; - LC0, LC1, LC2, LC3: Byte; + LB: Int32; begin LCtrBlock := nil; System.SetLength(LCtrBlock, BlockSize); - if TGhashSimd.IsBlockXorSupported then - begin - for LB := 0 to 1 do - begin - GetNextCtrBlock(LCtrBlock); - if AForEncrypt then - begin - System.Move(LCtrBlock[0], AOutBuf[AOutOff], BlockSize); - TGhashSimd.BlockXor128(@AOutBuf[AOutOff], @AInBuf[AInOff]); - TGhashSimd.BlockXor128(@FS[0], @AOutBuf[AOutOff]); - end - else - begin - System.Move(AInBuf[AInOff], AOutBuf[AOutOff], BlockSize); - TGhashSimd.BlockXor128(@AOutBuf[AOutOff], @LCtrBlock[0]); - TGhashSimd.BlockXor128(@FS[0], @AInBuf[AInOff]); - end; - FMultiplier.MultiplyH(FS); - AInOff := AInOff + BlockSize; - AOutOff := AOutOff + BlockSize; - end; - Exit; - end; - for LB := 0 to 1 do begin GetNextCtrBlock(LCtrBlock); if AForEncrypt then begin - for LI := 0 to (BlockSize - 1) div 4 do - begin - LC0 := Byte(LCtrBlock[(LI * 4) + 0] xor AInBuf[AInOff + (LI * 4) + 0]); - LC1 := Byte(LCtrBlock[(LI * 4) + 1] xor AInBuf[AInOff + (LI * 4) + 1]); - LC2 := Byte(LCtrBlock[(LI * 4) + 2] xor AInBuf[AInOff + (LI * 4) + 2]); - LC3 := Byte(LCtrBlock[(LI * 4) + 3] xor AInBuf[AInOff + (LI * 4) + 3]); - - FS[(LI * 4) + 0] := FS[(LI * 4) + 0] xor LC0; - FS[(LI * 4) + 1] := FS[(LI * 4) + 1] xor LC1; - FS[(LI * 4) + 2] := FS[(LI * 4) + 2] xor LC2; - FS[(LI * 4) + 3] := FS[(LI * 4) + 3] xor LC3; - - AOutBuf[AOutOff + (LI * 4) + 0] := LC0; - AOutBuf[AOutOff + (LI * 4) + 1] := LC1; - AOutBuf[AOutOff + (LI * 4) + 2] := LC2; - AOutBuf[AOutOff + (LI * 4) + 3] := LC3; - end; + System.Move(LCtrBlock[0], AOutBuf[AOutOff], BlockSize); + TByteUtilities.XorTo(BlockSize, PByte(@AInBuf[AInOff]), PByte(@AOutBuf[AOutOff])); + TByteUtilities.XorTo(BlockSize, PByte(@AOutBuf[AOutOff]), PByte(@FS[0])); end else begin - for LI := 0 to (BlockSize - 1) div 4 do - begin - LC0 := AInBuf[AInOff + (LI * 4) + 0]; - LC1 := AInBuf[AInOff + (LI * 4) + 1]; - LC2 := AInBuf[AInOff + (LI * 4) + 2]; - LC3 := AInBuf[AInOff + (LI * 4) + 3]; - - FS[(LI * 4) + 0] := FS[(LI * 4) + 0] xor LC0; - FS[(LI * 4) + 1] := FS[(LI * 4) + 1] xor LC1; - FS[(LI * 4) + 2] := FS[(LI * 4) + 2] xor LC2; - FS[(LI * 4) + 3] := FS[(LI * 4) + 3] xor LC3; - - AOutBuf[AOutOff + (LI * 4) + 0] := Byte(LC0 xor LCtrBlock[(LI * 4) + 0]); - AOutBuf[AOutOff + (LI * 4) + 1] := Byte(LC1 xor LCtrBlock[(LI * 4) + 1]); - AOutBuf[AOutOff + (LI * 4) + 2] := Byte(LC2 xor LCtrBlock[(LI * 4) + 2]); - AOutBuf[AOutOff + (LI * 4) + 3] := Byte(LC3 xor LCtrBlock[(LI * 4) + 3]); - end; + System.Move(AInBuf[AInOff], AOutBuf[AOutOff], BlockSize); + TByteUtilities.XorTo(BlockSize, PByte(@LCtrBlock[0]), PByte(@AOutBuf[AOutOff])); + TByteUtilities.XorTo(BlockSize, PByte(@AInBuf[AInOff]), PByte(@FS[0])); end; FMultiplier.MultiplyH(FS); AInOff := AInOff + BlockSize; diff --git a/CryptoLib/src/Crypto/Modes/ClpGcmSivBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpGcmSivBlockCipher.pas index de7f71d9..e75fe249 100644 --- a/CryptoLib/src/Crypto/Modes/ClpGcmSivBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpGcmSivBlockCipher.pas @@ -73,9 +73,6 @@ TGcmSivBlockCipher = class(TInterfacedObject, IGcmSivBlockCipher, strict private type - TGcmSivCache = class(TMemoryStream) - end; - TGcmSivHasher = class(TObject) strict private FBuffer: TCryptoLibByteArray; @@ -115,8 +112,12 @@ TGcmSivHasher = class(TObject) FTheReverse: TCryptoLibByteArray; FTheAEADHasher: TGcmSivHasher; FTheDataHasher: TGcmSivHasher; - FThePlain: TGcmSivCache; - FTheEncData: TGcmSivCache; + // Reused input accumulators (plaintext on encrypt, ciphertext on decrypt); + // FThePlain also stages recovered plaintext on decrypt. Grown by doubling. + FThePlain: TCryptoLibByteArray; + FThePlainLen: Int32; + FTheEncData: TCryptoLibByteArray; + FTheEncDataLen: Int32; FForEncryption: Boolean; FTheInitialAEAD: TCryptoLibByteArray; FTheNonce: TCryptoLibByteArray; @@ -250,7 +251,10 @@ procedure TGcmSivBlockCipher.TGcmSivHasher.UpdateHash(const ABuffer: TCryptoLibB FNumActive := 0; end; - // Fused POLYVAL Horner-by-8 fast path for full 128-byte batches. + // Fused POLYVAL Horner-by-8 fast path for full 128-byte batches. The kernel + // consumes the blocks in natural memory order and performs the POLYVAL + // byte-reversal internally (it skips its input byte-reverse, so the raw block + // is the field operand), matching the scalar path's FillReverse + GHASH fold. if (FParent.FGcmSivKernel <> nil) and (LMyRemaining >= FParent.FGcmSivKernelBatchBytes) then begin @@ -337,8 +341,6 @@ destructor TGcmSivBlockCipher.Destroy; begin FTheAEADHasher.Free; FTheDataHasher.Free; - FThePlain.Free; - FTheEncData.Free; inherited Destroy; end; @@ -420,11 +422,11 @@ procedure TGcmSivBlockCipher.CheckStatus(ALen: Int32); end; LDataLimit := MAX_DATALEN; - LCurrBytes := FThePlain.Size; + LCurrBytes := FThePlainLen; if not FForEncryption then begin LDataLimit := LDataLimit + BUFLEN; - LCurrBytes := FTheEncData.Size; + LCurrBytes := FTheEncDataLen; end; if TInt64Utilities.CompareUnsigned(LCurrBytes, LDataLimit - ALen) > 0 then @@ -452,13 +454,11 @@ function TGcmSivBlockCipher.ProcessByte(AInput: Byte; if FForEncryption then begin - FThePlain.WriteByte(AInput); + TArrayUtilities.AppendTo(FThePlain, FThePlainLen, AInput); FTheDataHasher.UpdateHash(AInput); end else - begin - FTheEncData.WriteByte(AInput); - end; + TArrayUtilities.AppendTo(FTheEncData, FTheEncDataLen, AInput); Result := 0; end; @@ -471,13 +471,11 @@ function TGcmSivBlockCipher.ProcessBytes(const AInput: TCryptoLibByteArray; if FForEncryption then begin - FThePlain.Write(AInput[AInOff], ALen); + TArrayUtilities.AppendTo(FThePlain, FThePlainLen, AInput, AInOff, ALen); FTheDataHasher.UpdateHash(AInput, AInOff, ALen); end else - begin - FTheEncData.Write(AInput[AInOff], ALen); - end; + TArrayUtilities.AppendTo(FTheEncData, FTheEncDataLen, AInput, AInOff, ALen); Result := 0; end; @@ -497,7 +495,7 @@ function TGcmSivBlockCipher.DoFinal(const AOutput: TCryptoLibByteArray; LMyDataLen := BUFLEN + EncryptPlain(LMyTag, AOutput, AOutOff); - System.Move(LMyTag[0], AOutput[AOutOff + FThePlain.Size], BUFLEN); + System.Move(LMyTag[0], AOutput[AOutOff + FThePlainLen], BUFLEN); ResetStreams(); Result := LMyDataLen; @@ -506,7 +504,10 @@ function TGcmSivBlockCipher.DoFinal(const AOutput: TCryptoLibByteArray; begin DecryptPlain(); - LMyDataLen := TStreamUtilities.WriteBufTo(FThePlain, AOutput, AOutOff); + // Release the verified plaintext (staged in FThePlain by DecryptPlain). + LMyDataLen := FThePlainLen; + if LMyDataLen > 0 then + System.Move(FThePlain[0], AOutput[AOutOff], LMyDataLen); ResetStreams(); Result := LMyDataLen; @@ -529,11 +530,11 @@ function TGcmSivBlockCipher.GetOutputSize(ALen: Int32): Int32; begin if FForEncryption then begin - Result := ALen + Int32(FThePlain.Size) + BUFLEN; + Result := ALen + FThePlainLen + BUFLEN; end else begin - LMyCurr := ALen + Int32(FTheEncData.Size); + LMyCurr := ALen + FTheEncDataLen; if LMyCurr > BUFLEN then Result := LMyCurr - BUFLEN else @@ -547,29 +548,17 @@ procedure TGcmSivBlockCipher.Reset; end; procedure TGcmSivBlockCipher.ResetStreams; -var - LCount: Int32; begin - if FThePlain <> nil then - begin - LCount := Int32(FThePlain.Size); - if LCount > 0 then - FillChar(PByte(FThePlain.Memory)^, LCount, 0); - FThePlain.Size := 0; - end; + // Wipe held/recovered plaintext (FTheEncData holds only ciphertext); reset + // used lengths but keep the reused buffers' capacity. + if FThePlainLen > 0 then + System.FillChar(FThePlain[0], FThePlainLen, 0); + FThePlainLen := 0; + FTheEncDataLen := 0; FTheAEADHasher.Reset(); FTheDataHasher.Reset(); - FThePlain.Free; - FThePlain := TGcmSivCache.Create(); - - FTheEncData.Free; - if FForEncryption then - FTheEncData := nil - else - FTheEncData := TGcmSivCache.Create(); - FTheFlags := FTheFlags and (not AEAD_COMPLETE); TArrayUtilities.Fill(FTheGHash, 0, System.Length(FTheGHash), Byte(0)); @@ -620,18 +609,14 @@ procedure TGcmSivBlockCipher.ProcessEightBlocksSivCtr( function TGcmSivBlockCipher.EncryptPlain(const ACounter: TCryptoLibByteArray; const ATarget: TCryptoLibByteArray; AOffset: Int32): Int32; var - LThePlainBuf: TCryptoLibByteArray; LThePlainLen: Int32; LMySrc, LMyCounter, LMyMask, LMyCounters: TCryptoLibByteArray; LMyRemaining: Int64; LMyOff, LMyLen: Int32; begin - System.SetLength(LThePlainBuf, FThePlain.Size); - FThePlain.Position := 0; - FThePlain.ReadBuffer(LThePlainBuf[0], System.Length(LThePlainBuf)); - LThePlainLen := System.Length(LThePlainBuf); - - LMySrc := LThePlainBuf; + // Read the buffered plaintext directly from the accumulator (no readback). + LThePlainLen := FThePlainLen; + LMySrc := FThePlain; LMyCounter := System.Copy(ACounter); LMyCounter[BUFLEN - 1] := LMyCounter[BUFLEN - 1] or MASK; System.SetLength(LMyMask, BUFLEN); @@ -675,18 +660,17 @@ function TGcmSivBlockCipher.EncryptPlain(const ACounter: TCryptoLibByteArray; procedure TGcmSivBlockCipher.DecryptPlain; var - LTheEncDataBuf: TCryptoLibByteArray; LTheEncDataLen: Int32; LMySrc, LMyExpected, LMyCounter, LMyMask, LMyTag, LMyCounters, LMyScratch128: TCryptoLibByteArray; LMyRemaining, LMyOff, LMyLen: Int32; begin - System.SetLength(LTheEncDataBuf, FTheEncData.Size); - FTheEncData.Position := 0; - FTheEncData.ReadBuffer(LTheEncDataBuf[0], System.Length(LTheEncDataBuf)); - LTheEncDataLen := System.Length(LTheEncDataBuf); + // Read the buffered ciphertext directly from the accumulator (no readback); + // recovered plaintext is appended to FThePlain and released only after the + // tag verifies below. + LTheEncDataLen := FTheEncDataLen; + LMySrc := FTheEncData; - LMySrc := LTheEncDataBuf; LMyRemaining := LTheEncDataLen - BUFLEN; if LMyRemaining < 0 then @@ -698,8 +682,8 @@ procedure TGcmSivBlockCipher.DecryptPlain; System.SetLength(LMyMask, BUFLEN); LMyOff := 0; - // Bulk 8-wide CTR path. In contrast with the scalar loop (which called - // FThePlain.Write + FTheDataHasher.UpdateHash eight times per 128-byte + // Bulk 8-wide CTR path. In contrast with the scalar loop (which appended to + // FThePlain + called FTheDataHasher.UpdateHash eight times per 128-byte // batch), the bulk variant materialises the plaintext into a 128-byte // scratch and makes ONE UpdateHash call per batch. That collapses the // per-batch PolyVal bookkeeping (FNumActive tests, FillReverse inner @@ -714,7 +698,7 @@ procedure TGcmSivBlockCipher.DecryptPlain; ProcessEightBlocksSivCtr(LMyCounter, LMyCounters); TByteUtilities.&Xor(128, PByte(LMyCounters), PByte(LMySrc) + LMyOff, PByte(LMyScratch128)); - FThePlain.Write(LMyScratch128[0], 128); + TArrayUtilities.AppendTo(FThePlain, FThePlainLen, LMyScratch128, 0, 128); FTheDataHasher.UpdateHash(LMyScratch128, 0, 128); LMyRemaining := LMyRemaining - 128; LMyOff := LMyOff + 128; @@ -728,7 +712,7 @@ procedure TGcmSivBlockCipher.DecryptPlain; LMyLen := Math.Min(BUFLEN, LMyRemaining); XorBlock(LMyMask, LMySrc, LMyOff, LMyLen); - FThePlain.Write(LMyMask[0], LMyLen); + TArrayUtilities.AppendTo(FThePlain, FThePlainLen, LMyMask, 0, LMyLen); FTheDataHasher.UpdateHash(LMyMask, 0, LMyLen); LMyRemaining := LMyRemaining - LMyLen; @@ -803,20 +787,14 @@ class procedure TGcmSivBlockCipher.FillReverse(const AInput: TCryptoLibByteArray end; class procedure TGcmSivBlockCipher.XorBlock(const ALeft, ARight: TCryptoLibByteArray); -var - LI: Int32; begin - for LI := 0 to 15 do - ALeft[LI] := ALeft[LI] xor ARight[LI]; + TByteUtilities.XorTo(16, ARight, ALeft); end; class procedure TGcmSivBlockCipher.XorBlock(const ALeft, ARight: TCryptoLibByteArray; AOffset, ALength: Int32); -var - LI: Int32; begin - for LI := 0 to ALength - 1 do - ALeft[LI] := ALeft[LI] xor ARight[LI + AOffset]; + TByteUtilities.XorTo(ALength, ARight, AOffset, ALeft, 0); end; class procedure TGcmSivBlockCipher.IncrementCounter(const ACounter: TCryptoLibByteArray); diff --git a/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas index 44af0ece..4627079b 100644 --- a/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpOcbBlockCipher.pas @@ -65,21 +65,20 @@ TOcbBlockCipher = class(TInterfacedObject, IOcbBlockCipher, const BLOCK_SIZE = 16; - // Maximum blocks staged per fused-kernel dispatch. LCM(4, 8) = 8, - // kept at 96 to match the previous batch cadence (96 is a multiple - // of both the i386 4-wide kernel stride and the x86_64 8-wide - // stride). The fused kernel now owns the offset ladder, L-table - // lookup and checksum fold internally, so this batch only governs - // how many pre-computed ntz bytes (and scratch lookahead bytes on - // the decrypt path) the mode stages on the stack per dispatch. + // Block cap for the ONE fused path that stages a per-batch stack + // scratch buffer: truncated-MAC decrypt (FMacSize < BLOCK_SIZE). The + // encrypt and full-MAC-decrypt paths need no scratch and are dispatched + // as a single whole-span call, so this bound does not apply to them. + // Must be a multiple of the fused kernel's MinimumBlockCount; Init + // rejects any kernel whose stride does not divide it. FUSED_BATCH_BLOCKS = 96; - // Number of 16-byte L-table entries the mode materialises - // contiguously for the kernel per call. An OCB session is bounded - // by the security proof to well under 2^48 blocks per nonce, and - // OCB_ntz of any Int64 block count is at most 63, so 64 entries - // covers every practically reachable max ntz. At BLOCK_SIZE bytes - // each the stack footprint is 1 KiB. + // Number of 16-byte L-table entries the mode materialises contiguously + // for the kernel (FLTableFlat, cached across calls). An OCB session is + // bounded by the security proof to well under 2^48 blocks per nonce, and + // OCB_ntz of any Int64 block count is at most 63, so 64 entries covers + // every practically reachable max ntz. At BLOCK_SIZE bytes each the + // footprint is 1 KiB. FUSED_LTABLE_ENTRIES = 64; var @@ -93,6 +92,14 @@ TOcbBlockCipher = class(TInterfacedObject, IOcbBlockCipher, FL: TList; FL_Asterisk, FL_Dollar: TCryptoLibByteArray; + // Cached contiguous snapshot of FL[0..FLTableFlatCount-1] handed to the + // fused kernel. FL is key-stable and only ever grows, so the flat copy is + // materialised lazily (EnsureLTableFlat) and reused across every batch and + // ProcessBytes call; it is invalidated only on Init, when FL is rebuilt for + // a new key. + FLTableFlat: TCryptoLibByteArray; + FLTableFlatCount: Int32; + FKTopInput: TCryptoLibByteArray; FStretch: TCryptoLibByteArray; FOffsetMAIN_0: TCryptoLibByteArray; @@ -134,12 +141,16 @@ TOcbBlockCipher = class(TInterfacedObject, IOcbBlockCipher, AInOff: Int32; const AOutput: TCryptoLibByteArray; AOutOff: Int32; ABlockCount: Int32); + // Grow FL and the cached flat L-table so entries [0..AMaxNtz] are present + // in FLTableFlat, extending only the not-yet-materialised tail. + procedure EnsureLTableFlat(AMaxNtz: Int32); + procedure ProcessEightBlocksBulk(const AInput: TCryptoLibByteArray; AInOff: Int32; const AOutput: TCryptoLibByteArray; AOutOff: Int32); class function OCB_double(const ABlock: TCryptoLibByteArray): TCryptoLibByteArray; static; class procedure OCB_extend(const ABlock: TCryptoLibByteArray; APos: Int32); static; - class function OCB_ntz(AX: Int64): Int32; static; + class function OCB_ntz(AX: Int64): Int32; static; inline; class function ShiftLeft(const ABlock, AOutput: TCryptoLibByteArray): Int32; static; strict protected @@ -206,6 +217,8 @@ constructor TOcbBlockCipher.Create(const AHashCipher, AMainCipher: IBlockCipher) System.SetLength(FOffsetMAIN_0, 16); System.SetLength(FOffsetMAIN, 16); FL := TList.Create; + System.SetLength(FLTableFlat, FUSED_LTABLE_ENTRIES * BLOCK_SIZE); + FLTableFlatCount := 0; end; destructor TOcbBlockCipher.Destroy; @@ -214,6 +227,11 @@ destructor TOcbBlockCipher.Destroy; inherited Destroy; end; +class function TOcbBlockCipher.OCB_ntz(AX: Int64): Int32; +begin + Result := TBitOperations.NumberOfTrailingZeros64(UInt64(AX)); +end; + function TOcbBlockCipher.GetAlgorithmName: String; begin Result := FMainCipher.AlgorithmName + '/OCB'; @@ -348,6 +366,7 @@ procedure TOcbBlockCipher.Init(AForEncryption: Boolean; FL.Clear; FL.Add(OCB_double(FL_Dollar)); + FLTableFlatCount := 0; // FL rebuilt for the new key; drop the flattened cache LBottom := ProcessNonce(LN); @@ -524,21 +543,25 @@ function TOcbBlockCipher.ProcessBytes(const AInput: TCryptoLibByteArray; while (LI < ALen) do begin - // Fused-kernel fast path: accelerator-provided AEAD kernel - // (e.g. AES-NI; other accelerators pluggable via the - // registry). FOcbKernel is nil when no factory accepted this + // Fused-kernel fast path: a hardware-accelerated AEAD kernel resolved + // from the registry (any accelerator whose factory accepts the cipher / + // direction). FOcbKernel is nil when no factory accepted this // cipher / direction (always so off-SIMD), in which case this branch // is skipped and the 8-wide bulk / scalar paths below run unchanged. // Takes priority over the 8-wide bulk-cipher path below whenever at - // least one kernel-stride batch fits the steady-state window. A single - // dispatch stages up to FUSED_BATCH_BLOCKS worth of offsets and lets - // the kernel loop internally in MinimumBlockCount strides, amortising - // per-call overhead across the whole batch. + // least one kernel-stride batch fits the steady-state window. The kernel + // loops internally in MinimumBlockCount strides and owns the offset ladder, + // ntz derivation and checksum fold, so encrypt and full-MAC decrypt hand it + // the ENTIRE remaining whole-block span in one dispatch (no per-batch + // marshalling). Only the truncated-MAC decrypt path stages a sliding + // scratch buffer, so it stays bounded by FUSED_BATCH_BLOCKS. if (FOcbKernel <> nil) and (FMainBlockPos = LSteadyPos) and ((ALen - LI) >= FOcbKernelMinBlocks * BLOCK_SIZE) then begin LRemaining := (ALen - LI) div BLOCK_SIZE; - if LRemaining > FUSED_BATCH_BLOCKS then + if FForEncryption or (FMacSize = BLOCK_SIZE) then + LBatchBlocks := LRemaining // whole-span dispatch (no scratch needed) + else if LRemaining > FUSED_BATCH_BLOCKS then LBatchBlocks := FUSED_BATCH_BLOCKS else LBatchBlocks := LRemaining; @@ -584,48 +607,32 @@ procedure TOcbBlockCipher.ProcessFusedBulk(const AInput: TCryptoLibByteArray; AInOff: Int32; const AOutput: TCryptoLibByteArray; AOutOff: Int32; ABlockCount: Int32); var - // One byte per block holding ntz(FMainBlockCount+1..+ABlockCount). - // The kernel consumes these directly (movzx + shl 4) to index - // LTableFlat for the per-block L-XOR ladder update. - LNtz: array [0 .. FUSED_BATCH_BLOCKS - 1] of Byte; - // Contiguous snapshot of FL[0..FUSED_LTABLE_ENTRIES-1]. GetLSub grows - // FL on demand up through LMaxNtz before the Move loop copies the - // live entries in; bytes past LMaxNtz remain undefined but the kernel - // never addresses them (ntz value bounds the index by construction). - LTableFlat: array [0 .. FUSED_LTABLE_ENTRIES * BLOCK_SIZE - 1] of Byte; // Decrypt fallback (FMacSize < BLOCK_SIZE): FMacSize-byte lookahead - // followed by fresh AInput bytes. The fast path (FMacSize = BLOCK_SIZE, - // the common case) bypasses this buffer entirely and hands the kernel - // two pointers -- one at FMainBlock for iter-0 block 0 and one at - // AInput - BLOCK_SIZE for the main stream -- saving a full batch - // memcpy per decrypt call. + // followed by fresh AInput bytes. This path is the ONLY per-batch scratch + // user, so ProcessBytes bounds its ABlockCount by FUSED_BATCH_BLOCKS; the + // encrypt and full-MAC-decrypt fast paths bypass this buffer entirely (they + // hand the kernel raw pointers) and are dispatched as a single whole-span + // call. LScratch: array [0 .. FUSED_BATCH_BLOCKS * BLOCK_SIZE - 1] of Byte; - LI, LBatchBytes, LNtzVal, LMaxNtz: Int32; + LBatchBytes, LMaxNtz: Int32; + LStartBlockCount: UInt64; LInPtr, LBlock0Ptr: Pointer; begin LBatchBytes := ABlockCount * BLOCK_SIZE; - // Pre-compute ntz for every block in the batch and track the largest - // index so we can grow FL exactly once before materialising LTableFlat. - // Moving ntz out of the kernel keeps i386 GPR pressure sane and lets - // both kernels share a single byte-indexed lookup path. - LMaxNtz := 0; - for LI := 0 to ABlockCount - 1 do - begin - System.Inc(FMainBlockCount); - LNtzVal := OCB_ntz(FMainBlockCount); - LNtz[LI] := Byte(LNtzVal); - if LNtzVal > LMaxNtz then - LMaxNtz := LNtzVal; - end; + // Block count consumed just before the first block of this batch. The kernel + // seeds a running counter from it and derives ntz per block itself, so there + // is no per-block ntz precompute loop. + LStartBlockCount := UInt64(FMainBlockCount); + System.Inc(FMainBlockCount, ABlockCount); + // Upper bound on ntz over (start+1..start+ABlockCount): floor(log2(endCount)) + // = index of the most significant set bit. The kernel indexes L up to this; a + // slightly loose bound just flattens a few extra entries. + LMaxNtz := 63 - TBitOperations.NumberOfLeadingZeros64(UInt64(FMainBlockCount)); - // Grow FL to cover LMaxNtz (side effect of GetLSub) then flatten the - // live L entries into a contiguous kernel-friendly buffer. FL entries - // are themselves 16-byte arrays; a tight Move loop is cheaper than a - // per-block pointer chase from the kernel would be. - GetLSub(LMaxNtz); - for LI := 0 to LMaxNtz do - System.Move(FL[LI][0], LTableFlat[LI * BLOCK_SIZE], BLOCK_SIZE); + // Materialise FLTableFlat through LMaxNtz (lazy, cached across calls) so the + // kernel can index the L-table with a single contiguous load per block. + EnsureLTableFlat(LMaxNtz); if FForEncryption then begin @@ -637,8 +644,8 @@ procedure TOcbBlockCipher.ProcessFusedBulk(const AInput: TCryptoLibByteArray; LInPtr := @AInput[AInOff]; LBlock0Ptr := LInPtr; FOcbKernel.ProcessBlocks(LInPtr, @AOutput[AOutOff], - @FOffsetMAIN[0], @FChecksum[0], @LTableFlat[0], @LNtz[0], - LBlock0Ptr, ABlockCount); + @FOffsetMAIN[0], @FChecksum[0], @FLTableFlat[0], + LBlock0Ptr, ABlockCount, LStartBlockCount); end else if FMacSize = BLOCK_SIZE then begin @@ -655,16 +662,15 @@ procedure TOcbBlockCipher.ProcessFusedBulk(const AInput: TCryptoLibByteArray; // eliminates the LBatchBytes - BLOCK_SIZE byte memcpy the old // LScratch reshape had to do on every fused decrypt batch. // - // The `InPtr - BLOCK_SIZE` pointer is never dereferenced at - // offset 0 by the kernel (verified phase-by-phase in - // AesOcbFusedWide_x86_64.inc / AesOcbFusedWide_i386.inc); the address - // itself only feeds register arithmetic and the `[InPtr + 16..]` - // loads, all of which resolve inside AInput. + // The `InPtr - BLOCK_SIZE` pointer is never dereferenced at offset 0 by + // the kernel (guaranteed by the IFusedOcbKernel.ProcessBlocks contract for + // Block0Ptr); the address itself only feeds register arithmetic and the + // `[InPtr + 16..]` loads, all of which resolve inside AInput. LBlock0Ptr := @FMainBlock[0]; LInPtr := PByte(@AInput[AInOff]) - BLOCK_SIZE; FOcbKernel.ProcessBlocks(LInPtr, @AOutput[AOutOff], - @FOffsetMAIN[0], @FChecksum[0], @LTableFlat[0], @LNtz[0], - LBlock0Ptr, ABlockCount); + @FOffsetMAIN[0], @FChecksum[0], @FLTableFlat[0], + LBlock0Ptr, ABlockCount, LStartBlockCount); // Refresh the BLOCK_SIZE lookahead from the tail of the consumed // AInput window so subsequent calls (fused, 8-wide, or scalar) @@ -687,8 +693,8 @@ procedure TOcbBlockCipher.ProcessFusedBulk(const AInput: TCryptoLibByteArray; LInPtr := @LScratch[0]; LBlock0Ptr := LInPtr; FOcbKernel.ProcessBlocks(LInPtr, @AOutput[AOutOff], - @FOffsetMAIN[0], @FChecksum[0], @LTableFlat[0], @LNtz[0], - LBlock0Ptr, ABlockCount); + @FOffsetMAIN[0], @FChecksum[0], @FLTableFlat[0], + LBlock0Ptr, ABlockCount, LStartBlockCount); System.Move(AInput[AInOff + LBatchBytes - FMacSize], FMainBlock[0], FMacSize); @@ -864,6 +870,18 @@ function TOcbBlockCipher.GetLSub(AN: Int32): TCryptoLibByteArray; Result := FL[AN]; end; +procedure TOcbBlockCipher.EnsureLTableFlat(AMaxNtz: Int32); +var + LI: Int32; +begin + if (AMaxNtz < FLTableFlatCount) then + Exit; // already materialised through AMaxNtz + GetLSub(AMaxNtz); // grow FL to cover AMaxNtz + for LI := FLTableFlatCount to AMaxNtz do + System.Move(FL[LI][0], FLTableFlat[LI * BLOCK_SIZE], BLOCK_SIZE); + FLTableFlatCount := AMaxNtz + 1; +end; + procedure TOcbBlockCipher.ProcessHashBlock; begin System.Inc(FHashBlockCount); @@ -957,11 +975,6 @@ class procedure TOcbBlockCipher.OCB_extend(const ABlock: TCryptoLibByteArray; end; end; -class function TOcbBlockCipher.OCB_ntz(AX: Int64): Int32; -begin - Result := TBitOperations.NumberOfTrailingZeros64(UInt64(AX)); -end; - class function TOcbBlockCipher.ShiftLeft(const ABlock, AOutput: TCryptoLibByteArray): Int32; var diff --git a/CryptoLib/src/Crypto/Modes/ClpSicBlockCipher.pas b/CryptoLib/src/Crypto/Modes/ClpSicBlockCipher.pas index 26d68314..bcedd528 100644 --- a/CryptoLib/src/Crypto/Modes/ClpSicBlockCipher.pas +++ b/CryptoLib/src/Crypto/Modes/ClpSicBlockCipher.pas @@ -26,6 +26,9 @@ interface ClpIBlockCipher, ClpIBlockCipherMode, ClpIBulkBlockCipher, + ClpIFusedCtrKernel, + ClpFusedKernelTypes, + ClpFusedKernelRegistry, ClpIBulkBlockCipherMode, ClpBlockCipherBulkUtilities, ClpByteUtilities, @@ -67,6 +70,11 @@ TSicBlockCipher = class(TInterfacedObject, ISicBlockCipher, // bulk-capable engine plugs in unchanged by implementing the // interface -- the mode does not care which cipher is underneath. FBulkCipher: IBulkBlockCipher; + // Acquired on Init from the fused-kernel registry when an accelerated + // counter-mode kernel is available for FCipher: a single AES-NI pass that + // fuses counter generation, encryption and XOR. Preferred over FBulkCipher + // for the batch-aligned bulk; nil (with FBulkCipher fallback) otherwise. + FCtrKernel: IFusedCtrKernel; /// /// Snapshot FCounter into APlainCounters and advance FCounter by ABlockCount @@ -206,6 +214,11 @@ procedure TSicBlockCipher.Init(AForEncryption: Boolean; // interface the batched path dispatches straight through FBulkCipher; // otherwise we stay on the per-block path. TBlockCipherBulkUtilities.TryResolveBulkCipher(FCipher, FBulkCipher); + // Acquire the fused counter-mode kernel if an accelerator is registered for + // FCipher (sets FCtrKernel nil on miss). Direction is irrelevant for CTR + // keystream (always AES-encrypt of the counter), so request Encrypt. + TFusedKernelRegistry.TryAcquireCtr(FCipher, TFusedModeDirection.Encrypt, + FCtrKernel); Reset(); end; @@ -242,7 +255,7 @@ function TSicBlockCipher.ProcessBlocks(const AInBuf: TCryptoLibByteArray; AInOff, ABlockCount: Int32; const AOutBuf: TCryptoLibByteArray; AOutOff: Int32): Int32; var - LTotalBytes: Int32; + LTotalBytes, LDone: Int32; begin if ABlockCount <= 0 then begin @@ -258,6 +271,24 @@ function TSicBlockCipher.ProcessBlocks(const AInBuf: TCryptoLibByteArray; if ((AOutOff < 0) or ((AOutOff + LTotalBytes) > System.Length(AOutBuf))) then raise EDataLengthCryptoLibException.CreateRes(@SOutputBufferTooShort); + // Preferred fast path: the fused CTR kernel does counter generation + AES-NI + // + XOR in a single pass over the batch-aligned bulk, advancing FCounter in + // place. It consumes a whole multiple of its batch granularity; the residue + // drops to the per-block tail below. When no CTR kernel was acquired + // (FCtrKernel = nil), nothing is consumed here and the generic FBulkCipher + // path runs instead. + if (FCtrKernel <> nil) and (FBlockSize = 16) and + (ABlockCount >= FCtrKernel.BatchBlockCount) then + begin + LDone := (ABlockCount div FCtrKernel.BatchBlockCount) * + FCtrKernel.BatchBlockCount; + FCtrKernel.ProcessCtrBlocks(@AInBuf[AInOff], @AOutBuf[AOutOff], + @FCounter[0], LDone); + System.Inc(AInOff, LDone * FBlockSize); + System.Inc(AOutOff, LDone * FBlockSize); + System.Dec(ABlockCount, LDone); + end; + // Fast path: 128-byte (8-block) batches through the bulk engine. // FBulkCipher is only assigned in Init when the underlying cipher // implements IBulkBlockCipher. The ProcessEightBlocksBulk helper diff --git a/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiCbcKernel.pas b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiCbcKernel.pas new file mode 100644 index 00000000..229d1ea6 --- /dev/null +++ b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiCbcKernel.pas @@ -0,0 +1,207 @@ +{ *********************************************************************************** } +{ * CryptoLib Library * } +{ * Author - Ugochukwu Mmaduekwe * } +{ * Github Repository * } +{ * * } +{ * Distributed under the MIT software license, see the accompanying file LICENSE * } +{ * or visit http://www.opensource.org/licenses/mit-license.php. * } +{ * * } +{ * Acknowledgements: * } +{ * * } +{ * Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring * } +{ * the development of this library * } +{ * ******************************************************************************* * } + +(* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) + +unit ClpAesNiCbcKernel; + +{$I ..\..\..\Include\CryptoLib.inc} + +interface + +uses + SysUtils, + ClpIBlockCipher, + ClpIAesEngineX86, + ClpFusedKernelTypes, + ClpIFusedCbcKernel, + ClpFusedKernelRegistry, + ClpAesNiFusedX86Backend; + +type + /// + /// AES-NI implementation of IFusedCbcKernel: the serial CBC-encrypt chain + /// C_i = E_K(P_i xor C_{i-1}) applied over a whole run in one call, with the + /// chaining value held in a register between blocks (kernel body in + /// Include\Simd\Aes\Cbc). Reuses the shared 1-wide AES round chain + /// (AesNiOneRoundsOnly). Available on x86_64 (CRYPTOLIB_X86_64_ASM) and i386 + /// (CRYPTOLIB_I386_ASM), gated collectively by CRYPTOLIB_X86_SIMD. When + /// unavailable the factory returns nil and TCbcBlockCipher keeps its existing + /// per-block bulk path. + /// + TAesNiCbcKernel = class sealed(TInterfacedObject, IFusedCbcKernel) + strict private + // FEngine is retained so the round-key buffer FKeys points into stays alive + // for the kernel's lifetime. + FEngine: IAesEngineX86; + FKeys: Pointer; + FRounds: Int32; + public + constructor Create(const AEngine: IAesEngineX86; AKeys: Pointer; + ARounds: Int32); + procedure ProcessCbcBlocks(AInPtr, AOutPtr, AIvPtr: Pointer; + ABlockCount: NativeInt); + end; + + TAesNiCbcKernelFactory = class sealed(TInterfacedObject, IFusedCbcKernelFactory) + public + function ProviderName: String; + function Priority: TFusedKernelPriority; + function TryCreate(const ACipher: IBlockCipher; + ADirection: TFusedModeDirection; + out AKernel: IFusedCbcKernel): Boolean; + end; + +implementation + +{$IFDEF CRYPTOLIB_X86_SIMD} + +type + // Context handed to the fused AES-NI CBC-encrypt kernel. Field offsets match + // the [rcx + N] / [ebx + N] accesses in AesNiCbcEncryptSerial_x86_64.inc and + // AesNiCbcEncryptSerial_i386.inc (8-byte fields on x86_64, 4-byte on i386). The + // kernel loops internally over BlockCount blocks, advancing InPtr/OutPtr and + // updating the 16-byte chaining value at IvPtr in place. + TCbcEncryptFusedCtx = record + InPtr: Pointer; + OutPtr: Pointer; + KeysPtr: Pointer; + IvPtr: Pointer; + BlockCount: NativeUInt; + end; + +// Fused AES-NI CBC encryption (C_i = E_K(P_i xor C_{i-1})). Each proc processes +// BlockCount blocks, looping internally, and writes the final ciphertext block +// back to the chaining slot at IvPtr. +procedure AesNiCbcEncrypt128(PCtx: Pointer); +{$DEFINE CRYPTOLIB_AESNI_KEY128} +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcEncryptSerial_x86_64.inc} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_i386.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcEncryptSerial_i386.inc} +{$ENDIF} +{$UNDEF CRYPTOLIB_AESNI_KEY128} +end; + +procedure AesNiCbcEncrypt192(PCtx: Pointer); +{$DEFINE CRYPTOLIB_AESNI_KEY192} +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcEncryptSerial_x86_64.inc} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_i386.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcEncryptSerial_i386.inc} +{$ENDIF} +{$UNDEF CRYPTOLIB_AESNI_KEY192} +end; + +procedure AesNiCbcEncrypt256(PCtx: Pointer); +{$DEFINE CRYPTOLIB_AESNI_KEY256} +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcEncryptSerial_x86_64.inc} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_i386.inc} +{$I ..\..\..\Include\Simd\Aes\Cbc\AesNiCbcEncryptSerial_i386.inc} +{$ENDIF} +{$UNDEF CRYPTOLIB_AESNI_KEY256} +end; + +{$ENDIF CRYPTOLIB_X86_SIMD} + +{ TAesNiCbcKernel } + +constructor TAesNiCbcKernel.Create(const AEngine: IAesEngineX86; + AKeys: Pointer; ARounds: Int32); +begin + inherited Create; + FEngine := AEngine; + FKeys := AKeys; + FRounds := ARounds; +end; + +procedure TAesNiCbcKernel.ProcessCbcBlocks(AInPtr, AOutPtr, AIvPtr: Pointer; + ABlockCount: NativeInt); +{$IFDEF CRYPTOLIB_X86_SIMD} +var + LCtx: TCbcEncryptFusedCtx; +{$ENDIF CRYPTOLIB_X86_SIMD} +begin +{$IFDEF CRYPTOLIB_X86_SIMD} + if ABlockCount <= 0 then + Exit; + LCtx.InPtr := AInPtr; + LCtx.OutPtr := AOutPtr; + LCtx.KeysPtr := FKeys; + LCtx.IvPtr := AIvPtr; + LCtx.BlockCount := NativeUInt(ABlockCount); + case FRounds of + 10: AesNiCbcEncrypt128(@LCtx); + 12: AesNiCbcEncrypt192(@LCtx); + else + AesNiCbcEncrypt256(@LCtx); + end; +{$ENDIF CRYPTOLIB_X86_SIMD} +end; + +{ TAesNiCbcKernelFactory } + +function TAesNiCbcKernelFactory.ProviderName: String; +begin + Result := 'AES-NI'; +end; + +function TAesNiCbcKernelFactory.Priority: TFusedKernelPriority; +begin + Result := TFusedKernelPriority.Baseline; +end; + +function TAesNiCbcKernelFactory.TryCreate(const ACipher: IBlockCipher; + ADirection: TFusedModeDirection; out AKernel: IFusedCbcKernel): Boolean; +var + LEngine: IAesEngineX86; + LKeys: PByte; + LRounds: Int32; +begin + AKernel := nil; + Result := False; + if ADirection <> TFusedModeDirection.Encrypt then + Exit; // only encrypt is implemented; decrypt not yet supported + try +{$IFDEF CRYPTOLIB_X86_SIMD} + if not TAesNiFusedX86Backend.TryResolveEngine(ACipher, LEngine) then + Exit; + if not LEngine.TryGetEncKeysPtr(LKeys, LRounds) then + Exit; + if not (LRounds in [10, 12, 14]) then + Exit; + AKernel := TAesNiCbcKernel.Create(LEngine, LKeys, LRounds); + Result := True; +{$ENDIF CRYPTOLIB_X86_SIMD} + except + AKernel := nil; + Result := False; + end; +end; + +initialization + TFusedKernelRegistry.RegisterCbcFactory( + TAesNiCbcKernelFactory.Create() as IFusedCbcKernelFactory); + +end. diff --git a/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiCcmKernel.pas b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiCcmKernel.pas index 16a2dde9..3f4bd174 100644 --- a/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiCcmKernel.pas +++ b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiCcmKernel.pas @@ -28,7 +28,7 @@ interface ClpIFusedCcmKernel, ClpFusedKernelRegistry, ClpAesFusedAeadSimd, - ClpAesFusedAeadX86Backend; + ClpAesNiFusedX86Backend; type /// @@ -290,7 +290,7 @@ function TAesNiCcmKernelFactory.TryCreate(const ACipher: IBlockCipher; try if not TAesFusedAeadSimd.CpuSupports then Exit; - if not TAesFusedAeadX86Backend.TryResolveEngine(ACipher, LEngine) then + if not TAesNiFusedX86Backend.TryResolveEngine(ACipher, LEngine) then Exit; // CCM drives CTR and CBC-MAC lanes from the same forward-encrypt // schedule for both directions. LKeys is consumed only to probe diff --git a/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiCtrKernel.pas b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiCtrKernel.pas new file mode 100644 index 00000000..d839f04a --- /dev/null +++ b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiCtrKernel.pas @@ -0,0 +1,211 @@ +{ *********************************************************************************** } +{ * CryptoLib Library * } +{ * Author - Ugochukwu Mmaduekwe * } +{ * Github Repository * } +{ * * } +{ * Distributed under the MIT software license, see the accompanying file LICENSE * } +{ * or visit http://www.opensource.org/licenses/mit-license.php. * } +{ * * } +{ * Acknowledgements: * } +{ * * } +{ * Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring * } +{ * the development of this library * } +{ * ******************************************************************************* * } + +(* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) + +unit ClpAesNiCtrKernel; + +{$I ..\..\..\Include\CryptoLib.inc} + +interface + +uses + SysUtils, + ClpIBlockCipher, + ClpIAesEngineX86, + ClpFusedKernelTypes, + ClpIFusedCtrKernel, + ClpFusedKernelRegistry, + ClpAesNiFusedX86Backend; + +type + /// + /// AES-NI implementation of IFusedCtrKernel: the fused counter-mode + /// keystream + XOR body used by TSicBlockCipher's bulk path. The MAC-free + /// base of the fused-kernel family; reuses the plain 8-wide AES round chain + /// (AesNiEightRoundsOnly) since CTR has no extra per-block state. + /// Available on x86_64 (CRYPTOLIB_X86_64_ASM) and i386 (CRYPTOLIB_I386_ASM); + /// both arms gated collectively by CRYPTOLIB_X86_SIMD. When unavailable the + /// factory returns nil and TSicBlockCipher keeps its existing bulk path. + /// + TAesNiCtrKernel = class sealed(TInterfacedObject, IFusedCtrKernel) + strict private + const + FUSED_CTR_BATCH_BLOCKS = 8; + strict private + // FEngine is retained so the round-key buffer FKeys points into stays alive + // for the kernel's lifetime. + FEngine: IAesEngineX86; + FKeys: Pointer; + FRounds: Int32; + public + constructor Create(const AEngine: IAesEngineX86; AKeys: Pointer; + ARounds: Int32); + function BatchBlockCount: Int32; + procedure ProcessCtrBlocks(AInPtr, AOutPtr, ACounter: Pointer; + ABlockCount: NativeInt); + end; + + TAesNiCtrKernelFactory = class sealed(TInterfacedObject, IFusedCtrKernelFactory) + public + function ProviderName: String; + function Priority: TFusedKernelPriority; + function TryCreate(const ACipher: IBlockCipher; + ADirection: TFusedModeDirection; + out AKernel: IFusedCtrKernel): Boolean; + end; + +implementation + +{$IFDEF CRYPTOLIB_X86_SIMD} + +type + // Context handed to the fused AES-NI CTR keystream + XOR kernel. Field offsets + // match the [rcx + N] / [ebx + N] accesses in AesNiCtrEight_x86_64.inc and + // AesNiCtrEight_i386.inc (8-byte fields on x86_64, 4-byte on i386). The kernel + // loops internally over BlockCount blocks, advancing InPtr/OutPtr and the + // 16-byte big-endian counter at CounterPtr in place. + TCtrFusedCtx = record + InPtr: Pointer; + OutPtr: Pointer; + KeysPtr: Pointer; + CounterPtr: Pointer; + BlockCount: NativeUInt; + end; + +// Fused AES-NI CTR keystream + XOR (encrypt-only; CTR keystream is the same for +// encrypt and decrypt). Each proc processes BlockCount blocks (a multiple of 8), +// looping internally, and advances the 16-byte big-endian counter in place. +procedure AesNiCtrEnc128(PCtx: Pointer); +{$DEFINE CRYPTOLIB_AESNI_KEY128} +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Aes\Ctr\AesNiCtrEight_x86_64.inc} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_i386.inc} +{$I ..\..\..\Include\Simd\Aes\Ctr\AesNiCtrEight_i386.inc} +{$ENDIF} +{$UNDEF CRYPTOLIB_AESNI_KEY128} +end; + +procedure AesNiCtrEnc192(PCtx: Pointer); +{$DEFINE CRYPTOLIB_AESNI_KEY192} +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Aes\Ctr\AesNiCtrEight_x86_64.inc} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_i386.inc} +{$I ..\..\..\Include\Simd\Aes\Ctr\AesNiCtrEight_i386.inc} +{$ENDIF} +{$UNDEF CRYPTOLIB_AESNI_KEY192} +end; + +procedure AesNiCtrEnc256(PCtx: Pointer); +{$DEFINE CRYPTOLIB_AESNI_KEY256} +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_x86_64.inc} +{$I ..\..\..\Include\Simd\Aes\Ctr\AesNiCtrEight_x86_64.inc} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\..\..\Include\Simd\Common\SimdProc1Begin_i386.inc} +{$I ..\..\..\Include\Simd\Aes\Ctr\AesNiCtrEight_i386.inc} +{$ENDIF} +{$UNDEF CRYPTOLIB_AESNI_KEY256} +end; + +{$ENDIF CRYPTOLIB_X86_SIMD} + +{ TAesNiCtrKernel } + +constructor TAesNiCtrKernel.Create(const AEngine: IAesEngineX86; AKeys: Pointer; + ARounds: Int32); +begin + inherited Create; + FEngine := AEngine; + FKeys := AKeys; + FRounds := ARounds; +end; + +function TAesNiCtrKernel.BatchBlockCount: Int32; +begin + Result := FUSED_CTR_BATCH_BLOCKS; +end; + +procedure TAesNiCtrKernel.ProcessCtrBlocks(AInPtr, AOutPtr, ACounter: Pointer; + ABlockCount: NativeInt); +{$IFDEF CRYPTOLIB_X86_SIMD} +var + LCtx: TCtrFusedCtx; +{$ENDIF CRYPTOLIB_X86_SIMD} +begin +{$IFDEF CRYPTOLIB_X86_SIMD} + LCtx.InPtr := AInPtr; + LCtx.OutPtr := AOutPtr; + LCtx.KeysPtr := FKeys; + LCtx.CounterPtr := ACounter; + LCtx.BlockCount := NativeUInt(ABlockCount); + case FRounds of + 10: AesNiCtrEnc128(@LCtx); + 12: AesNiCtrEnc192(@LCtx); + else + AesNiCtrEnc256(@LCtx); + end; +{$ENDIF CRYPTOLIB_X86_SIMD} +end; + +{ TAesNiCtrKernelFactory } + +function TAesNiCtrKernelFactory.ProviderName: String; +begin + Result := 'AES-NI'; +end; + +function TAesNiCtrKernelFactory.Priority: TFusedKernelPriority; +begin + Result := TFusedKernelPriority.Baseline; +end; + +function TAesNiCtrKernelFactory.TryCreate(const ACipher: IBlockCipher; + ADirection: TFusedModeDirection; out AKernel: IFusedCtrKernel): Boolean; +var + LEngine: IAesEngineX86; + LKeys: PByte; + LRounds: Int32; +begin + AKernel := nil; + Result := False; + try +{$IFDEF CRYPTOLIB_X86_SIMD} + if not TAesNiFusedX86Backend.TryResolveEngine(ACipher, LEngine) then + Exit; + if not LEngine.TryGetEncKeysPtr(LKeys, LRounds) then + Exit; + if not (LRounds in [10, 12, 14]) then + Exit; + AKernel := TAesNiCtrKernel.Create(LEngine, LKeys, LRounds); + Result := True; +{$ENDIF CRYPTOLIB_X86_SIMD} + except + AKernel := nil; + Result := False; + end; +end; + +initialization + TFusedKernelRegistry.RegisterCtrFactory( + TAesNiCtrKernelFactory.Create() as IFusedCtrKernelFactory); + +end. diff --git a/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiEaxKernel.pas b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiEaxKernel.pas index 8efa8dea..8b17386e 100644 --- a/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiEaxKernel.pas +++ b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiEaxKernel.pas @@ -28,7 +28,7 @@ interface ClpIFusedEaxKernel, ClpFusedKernelRegistry, ClpAesFusedAeadSimd, - ClpAesFusedAeadX86Backend; + ClpAesNiFusedX86Backend; type /// @@ -281,7 +281,7 @@ function TAesNiEaxKernelFactory.TryCreate(const ACipher: IBlockCipher; try if not TAesFusedAeadSimd.CpuSupports then Exit; - if not TAesFusedAeadX86Backend.TryResolveEngine(ACipher, LEngine) then + if not TAesNiFusedX86Backend.TryResolveEngine(ACipher, LEngine) then Exit; // EAX drives CTR and OMAC lanes from the same forward-encrypt // schedule for both directions. diff --git a/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiGcmKernel.pas b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiGcmKernel.pas index a89b840e..722d1f02 100644 --- a/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiGcmKernel.pas +++ b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiGcmKernel.pas @@ -28,7 +28,7 @@ interface ClpIFusedGcmKernel, ClpFusedKernelRegistry, ClpAesFusedAeadSimd, - ClpAesFusedAeadX86Backend; + ClpAesNiFusedX86Backend; type /// @@ -37,10 +37,12 @@ interface /// (CRYPTOLIB_I386_ASM); both arms gated collectively by /// CRYPTOLIB_X86_SIMD. /// Direction-independent: the fused AES-CTR keystream is identical - /// for GCM encrypt and decrypt; the Pascal wrapper supplies the - /// correct PXorIn / POut / PPrevCipher triple per call. - /// The kernel processes exactly one stride (8 blocks = 128 bytes) - /// per call; the mode layer drives the outer loop. + /// for GCM encrypt and decrypt; ProcessCtrGhashBatches supplies the + /// direction via AForEncrypt (which buffer feeds GHASH). + /// The kernel loops over a whole run of 8-block (128-byte) batches in + /// one call, generating the GCM counters and rotating the GHASH + /// pipeline internally; the mode primes the first batch and drains the + /// last. /// TAesNiGcmKernel = class sealed(TInterfacedObject, IFusedGcmKernel) strict private @@ -56,8 +58,9 @@ TAesNiGcmKernel = class sealed(TInterfacedObject, IFusedGcmKernel) constructor Create(const AEngine: IAesEngineX86; AKeys: Pointer; ARounds: Int32; AHPow128, AMask: Pointer); function MinimumBlockCount: Int32; - procedure ProcessCtrGhashBatch(AInPtr, AOutPtr, ARawCtrs, APrevCipher, - AGhashState: Pointer; ABlockCount: Int32); + function ProcessCtrGhashBatches(AInPtr, AOutPtr, APrevInit, AGhashState, + AJ0Template: Pointer; ACounter32: UInt32; ABatchCount: NativeInt; + AForEncrypt: Boolean): UInt32; end; TAesNiGcmKernelFactory = class sealed(TInterfacedObject, @@ -75,19 +78,27 @@ implementation {$IFDEF CRYPTOLIB_X86_SIMD} type - // Context handed to the fused AES-NI keystream + 8-way GHASH kernel. - // Natural pointer-sized alignment, no padding: matches the - // [rcx / ebx + offset] accesses in AesGcmFusedCtrGhashEight_x86_64.inc and - // AesGcmFusedCtrGhashEight_i386.inc. - TGcmFusedBatchCtx = record + // Context handed to the fused AES-NI CTR keystream + 8-way GHASH loop kernel. + // Natural pointer-sized alignment, no padding: the field offsets match the + // [rcx + N] / [ebx + N] accesses in AesGcmFusedCtrGhashEight_x86_64.inc and + // AesGcmFusedCtrGhashEight_i386.inc (they differ with pointer / NativeUInt + // width: 8-byte fields on x86_64, 4-byte on i386). The kernel advances + // PXorIn/POut/PPrevCipher and Counter32 in place across the batch run and + // builds each batch's counter blocks in-register from Counter32 + PJ0Template + // (both arches); the GHASH-from-output flag is 1 for encrypt (fold the output + // ciphertext) and 0 for decrypt (fold the input). + TGcmFusedCtx = record PXorIn: Pointer; POut: Pointer; - PCtrCurr: Pointer; PPrevCipher: Pointer; PRoundKeys: Pointer; PHPow128: Pointer; PFS: Pointer; PMask: Pointer; + Counter32: NativeUInt; + PJ0Template: Pointer; + BatchCount: NativeUInt; + GhashFromOutput: NativeUInt; end; procedure GcmFusedAesEnc128GhashEight(PCtx: Pointer); @@ -155,31 +166,35 @@ function TAesNiGcmKernel.MinimumBlockCount: Int32; Result := FUSED_GCM_MIN_BLOCKS; end; -procedure TAesNiGcmKernel.ProcessCtrGhashBatch(AInPtr, AOutPtr, ARawCtrs, - APrevCipher, AGhashState: Pointer; ABlockCount: Int32); +function TAesNiGcmKernel.ProcessCtrGhashBatches(AInPtr, AOutPtr, APrevInit, + AGhashState, AJ0Template: Pointer; ACounter32: UInt32; ABatchCount: NativeInt; + AForEncrypt: Boolean): UInt32; {$IFDEF CRYPTOLIB_X86_SIMD} var - LCtx: TGcmFusedBatchCtx; + LCtx: TGcmFusedCtx; {$ENDIF CRYPTOLIB_X86_SIMD} begin {$IFDEF CRYPTOLIB_X86_SIMD} - if ABlockCount <> FUSED_GCM_MIN_BLOCKS then - Exit; LCtx.PXorIn := AInPtr; LCtx.POut := AOutPtr; - LCtx.PCtrCurr := ARawCtrs; - LCtx.PPrevCipher := APrevCipher; + LCtx.PPrevCipher := APrevInit; LCtx.PRoundKeys := FKeys; LCtx.PHPow128 := FHPow128; LCtx.PFS := AGhashState; LCtx.PMask := FMask; + LCtx.Counter32 := ACounter32; + LCtx.PJ0Template := AJ0Template; + LCtx.BatchCount := NativeUInt(ABatchCount); + LCtx.GhashFromOutput := NativeUInt(Ord(AForEncrypt)); case FRounds of 10: GcmFusedAesEnc128GhashEight(@LCtx); 12: GcmFusedAesEnc192GhashEight(@LCtx); else GcmFusedAesEnc256GhashEight(@LCtx); end; + Exit(UInt32(LCtx.Counter32)); {$ENDIF CRYPTOLIB_X86_SIMD} + Result := 0; end; { TAesNiGcmKernelFactory } @@ -210,7 +225,7 @@ function TAesNiGcmKernelFactory.TryCreate(const ACipher: IBlockCipher; {$IFDEF CRYPTOLIB_X86_SIMD} if not TAesFusedAeadSimd.CpuSupports then Exit; - if not TAesFusedAeadX86Backend.TryResolveEngine(ACipher, LEngine) then + if not TAesNiFusedX86Backend.TryResolveEngine(ACipher, LEngine) then Exit; if not LEngine.TryGetEncKeysPtr(LKeys, LRounds) then Exit; diff --git a/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiOcbKernel.pas b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiOcbKernel.pas index 8d275e08..bb97b856 100644 --- a/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiOcbKernel.pas +++ b/CryptoLib/src/Crypto/Modes/Fused/ClpAesNiOcbKernel.pas @@ -28,7 +28,7 @@ interface ClpIFusedOcbKernel, ClpFusedKernelRegistry, ClpAesFusedAeadSimd, - ClpAesFusedAeadX86Backend; + ClpAesNiFusedX86Backend; type /// @@ -62,7 +62,8 @@ TAesNiOcbKernel = class sealed(TInterfacedObject, IFusedOcbKernel) ARounds: Int32; ADirection: TFusedModeDirection); function MinimumBlockCount: Int32; procedure ProcessBlocks(AInPtr, AOutPtr, AOffsetPtr, AChecksumPtr, - ALTablePtr, ANtzPtr, ABlock0Ptr: Pointer; ABlockCount: Int32); + ALTablePtr, ABlock0Ptr: Pointer; ABlockCount: Int32; + AStartBlockCount: UInt64); end; TAesNiOcbKernelFactory = class sealed(TInterfacedObject, @@ -84,9 +85,10 @@ implementation /// Context record shared with the assembly body. All live OCB state /// the kernel touches is addressed through this record: seven /// pointers (Keys, In, Out, OffsetState, ChecksumState, LTable, - /// NtzArray, Block0) plus the NativeUInt BlockCount. Field offsets - /// match the kernel's displacement accesses documented in - /// AesOcbFusedWide_x86_64.inc and AesOcbFusedWide_i386.inc. + /// Block0) plus the NativeUInt BlockCount and the UInt64 + /// StartBlockCount. Field offsets match the kernel's displacement + /// accesses documented in AesOcbFusedWide_x86_64.inc and + /// AesOcbFusedWide_i386.inc. /// TOcbFusedKernelCtx = record Keys: Pointer; // AES expanded schedule (enc / dec+invMC) @@ -95,11 +97,14 @@ TOcbFusedKernelCtx = record OffsetPtr: Pointer; // 16-byte live FOffsetMAIN state (r/w) ChecksumPtr: Pointer; // 16-byte live FChecksum state (r/w) LTablePtr: Pointer; // flat L[0..LMax] * 16 bytes (read-only) - NtzPtr: Pointer; // ABlockCount bytes of pre-computed ntz Block0Ptr: Pointer; // 16-byte source of iter-0 block 0 (may // alias InPtr or point at an unrelated // 16-byte buffer such as FMainBlock) - BlockCount: NativeUInt; // positive multiple of MinimumBlockCount + BlockCount: NativeUInt; // positive multiple of MinimumBlockCount + StartBlockCount: UInt64; // OCB block count before this span; the kernel + // seeds a running counter here and derives ntz + // per block in-asm (bsf). UInt64 on both arches so + // the count is exact past 2^32 on i386 too. end; procedure OcbFusedEncWide128(PCtx: Pointer); @@ -206,7 +211,8 @@ function TAesNiOcbKernel.MinimumBlockCount: Int32; end; procedure TAesNiOcbKernel.ProcessBlocks(AInPtr, AOutPtr, AOffsetPtr, - AChecksumPtr, ALTablePtr, ANtzPtr, ABlock0Ptr: Pointer; ABlockCount: Int32); + AChecksumPtr, ALTablePtr, ABlock0Ptr: Pointer; ABlockCount: Int32; + AStartBlockCount: UInt64); {$IFDEF CRYPTOLIB_X86_SIMD} var LCtx: TOcbFusedKernelCtx; @@ -222,9 +228,9 @@ procedure TAesNiOcbKernel.ProcessBlocks(AInPtr, AOutPtr, AOffsetPtr, LCtx.OffsetPtr := AOffsetPtr; LCtx.ChecksumPtr := AChecksumPtr; LCtx.LTablePtr := ALTablePtr; - LCtx.NtzPtr := ANtzPtr; LCtx.Block0Ptr := ABlock0Ptr; LCtx.BlockCount := NativeUInt(ABlockCount); + LCtx.StartBlockCount := AStartBlockCount; if FDirection = TFusedModeDirection.Encrypt then begin case FRounds of @@ -271,7 +277,7 @@ function TAesNiOcbKernelFactory.TryCreate(const ACipher: IBlockCipher; try if not TAesFusedAeadSimd.CpuSupports then Exit; - if not TAesFusedAeadX86Backend.TryResolveEngine(ACipher, LEngine) then + if not TAesNiFusedX86Backend.TryResolveEngine(ACipher, LEngine) then Exit; if ADirection = TFusedModeDirection.Encrypt then LHasSchedule := LEngine.TryGetEncKeysPtr(LKeys, LRounds) diff --git a/CryptoLib/src/Crypto/Modes/Fused/ClpFusedKernelDefaults.pas b/CryptoLib/src/Crypto/Modes/Fused/ClpFusedKernelDefaults.pas index 4d95894b..0f2ef3f6 100644 --- a/CryptoLib/src/Crypto/Modes/Fused/ClpFusedKernelDefaults.pas +++ b/CryptoLib/src/Crypto/Modes/Fused/ClpFusedKernelDefaults.pas @@ -34,6 +34,8 @@ implementation ClpAesNiOcbKernel, ClpAesNiCcmKernel, ClpAesNiEaxKernel, + ClpAesNiCtrKernel, + ClpAesNiCbcKernel, ClpPclmulGcmSivKernel; end. diff --git a/CryptoLib/src/Crypto/Modes/Fused/ClpFusedKernelRegistry.pas b/CryptoLib/src/Crypto/Modes/Fused/ClpFusedKernelRegistry.pas index 4c1497ef..587b95ed 100644 --- a/CryptoLib/src/Crypto/Modes/Fused/ClpFusedKernelRegistry.pas +++ b/CryptoLib/src/Crypto/Modes/Fused/ClpFusedKernelRegistry.pas @@ -31,7 +31,9 @@ interface ClpIFusedOcbKernel, ClpIFusedCcmKernel, ClpIFusedEaxKernel, - ClpIFusedGcmSivKernel; + ClpIFusedGcmSivKernel, + ClpIFusedCtrKernel, + ClpIFusedCbcKernel; type /// @@ -69,12 +71,16 @@ TFusedKernelRegistry = class sealed(TObject) class var FCcmFactories: TList; class var FEaxFactories: TList; class var FGcmSivFactories: TList; + class var FCtrFactories: TList; + class var FCbcFactories: TList; class function SnapshotGcmFactories: TCryptoLibGenericArray; static; class function SnapshotOcbFactories: TCryptoLibGenericArray; static; class function SnapshotCcmFactories: TCryptoLibGenericArray; static; class function SnapshotEaxFactories: TCryptoLibGenericArray; static; class function SnapshotGcmSivFactories: TCryptoLibGenericArray; static; + class function SnapshotCtrFactories: TCryptoLibGenericArray; static; + class function SnapshotCbcFactories: TCryptoLibGenericArray; static; class constructor Create; class destructor Destroy; @@ -88,6 +94,8 @@ TFusedKernelRegistry = class sealed(TObject) class procedure RegisterCcmFactory(const AFactory: IFusedCcmKernelFactory); static; class procedure RegisterEaxFactory(const AFactory: IFusedEaxKernelFactory); static; class procedure RegisterGcmSivFactory(const AFactory: IFusedGcmSivKernelFactory); static; + class procedure RegisterCtrFactory(const AFactory: IFusedCtrKernelFactory); static; + class procedure RegisterCbcFactory(const AFactory: IFusedCbcKernelFactory); static; /// Remove AFactory. No-op if not registered. class procedure UnregisterGcmFactory(const AFactory: IFusedGcmKernelFactory); static; @@ -95,6 +103,8 @@ TFusedKernelRegistry = class sealed(TObject) class procedure UnregisterCcmFactory(const AFactory: IFusedCcmKernelFactory); static; class procedure UnregisterEaxFactory(const AFactory: IFusedEaxKernelFactory); static; class procedure UnregisterGcmSivFactory(const AFactory: IFusedGcmSivKernelFactory); static; + class procedure UnregisterCtrFactory(const AFactory: IFusedCtrKernelFactory); static; + class procedure UnregisterCbcFactory(const AFactory: IFusedCbcKernelFactory); static; /// Walk the factory list (highest priority first) and /// return the first kernel whose TryCreate succeeds. Returns False @@ -121,6 +131,18 @@ TFusedKernelRegistry = class sealed(TObject) ADirection: TFusedModeDirection; AHPowers: Pointer; out AKernel: IFusedGcmSivKernel): Boolean; static; + /// CTR variant: no auxiliary state (MAC-free). Returns False (and + /// AKernel nil) when the gate is ForceDisabled, no factory is registered, + /// or every TryCreate returns False. + class function TryAcquireCtr(const ACipher: IBlockCipher; + ADirection: TFusedModeDirection; + out AKernel: IFusedCtrKernel): Boolean; static; + /// CBC-encrypt variant: no auxiliary state. Returns False (and + /// AKernel nil) when the gate is ForceDisabled, no factory is registered, + /// or every TryCreate returns False. + class function TryAcquireCbc(const ACipher: IBlockCipher; + ADirection: TFusedModeDirection; + out AKernel: IFusedCbcKernel): Boolean; static; /// Snapshot of ProviderName strings in current priority /// order. Used for diagnostics and test assertions. @@ -129,6 +151,8 @@ TFusedKernelRegistry = class sealed(TObject) class function GetRegisteredCcmProviders: TCryptoLibStringArray; static; class function GetRegisteredEaxProviders: TCryptoLibStringArray; static; class function GetRegisteredGcmSivProviders: TCryptoLibStringArray; static; + class function GetRegisteredCtrProviders: TCryptoLibStringArray; static; + class function GetRegisteredCbcProviders: TCryptoLibStringArray; static; end; implementation @@ -143,10 +167,14 @@ implementation FCcmFactories := TList.Create; FEaxFactories := TList.Create; FGcmSivFactories := TList.Create; + FCtrFactories := TList.Create; + FCbcFactories := TList.Create; end; class destructor TFusedKernelRegistry.Destroy; begin + FCbcFactories.Free; + FCtrFactories.Free; FGcmSivFactories.Free; FEaxFactories.Free; FCcmFactories.Free; @@ -302,6 +330,64 @@ class procedure TFusedKernelRegistry.RegisterGcmSivFactory( end; end; +class procedure TFusedKernelRegistry.RegisterCtrFactory( + const AFactory: IFusedCtrKernelFactory); +var + LI: Int32; + LInserted: Boolean; + LPriority: Int32; +begin + if AFactory = nil then Exit; + FLock.Enter; + try + if FCtrFactories.IndexOf(AFactory) >= 0 then Exit; + LPriority := Ord(AFactory.Priority); + LInserted := False; + for LI := 0 to FCtrFactories.Count - 1 do + begin + if LPriority > Ord(FCtrFactories[LI].Priority) then + begin + FCtrFactories.Insert(LI, AFactory); + LInserted := True; + Break; + end; + end; + if not LInserted then + FCtrFactories.Add(AFactory); + finally + FLock.Leave; + end; +end; + +class procedure TFusedKernelRegistry.RegisterCbcFactory( + const AFactory: IFusedCbcKernelFactory); +var + LI: Int32; + LInserted: Boolean; + LPriority: Int32; +begin + if AFactory = nil then Exit; + FLock.Enter; + try + if FCbcFactories.IndexOf(AFactory) >= 0 then Exit; + LPriority := Ord(AFactory.Priority); + LInserted := False; + for LI := 0 to FCbcFactories.Count - 1 do + begin + if LPriority > Ord(FCbcFactories[LI].Priority) then + begin + FCbcFactories.Insert(LI, AFactory); + LInserted := True; + Break; + end; + end; + if not LInserted then + FCbcFactories.Add(AFactory); + finally + FLock.Leave; + end; +end; + { ---- Unregister ------------------------------------------------------------- } class procedure TFusedKernelRegistry.UnregisterGcmFactory( @@ -384,6 +470,38 @@ class procedure TFusedKernelRegistry.UnregisterGcmSivFactory( end; end; +class procedure TFusedKernelRegistry.UnregisterCtrFactory( + const AFactory: IFusedCtrKernelFactory); +var + LIdx: Int32; +begin + if AFactory = nil then Exit; + FLock.Enter; + try + LIdx := FCtrFactories.IndexOf(AFactory); + if LIdx >= 0 then + FCtrFactories.Delete(LIdx); + finally + FLock.Leave; + end; +end; + +class procedure TFusedKernelRegistry.UnregisterCbcFactory( + const AFactory: IFusedCbcKernelFactory); +var + LIdx: Int32; +begin + if AFactory = nil then Exit; + FLock.Enter; + try + LIdx := FCbcFactories.IndexOf(AFactory); + if LIdx >= 0 then + FCbcFactories.Delete(LIdx); + finally + FLock.Leave; + end; +end; + { ---- Snapshots -------------------------------------------------------------- } class function TFusedKernelRegistry.SnapshotGcmFactories: TCryptoLibGenericArray; @@ -461,6 +579,36 @@ class function TFusedKernelRegistry.SnapshotGcmSivFactories: TCryptoLibGenericAr end; end; +class function TFusedKernelRegistry.SnapshotCtrFactories: TCryptoLibGenericArray; +var + LI, LCount: Int32; +begin + FLock.Enter; + try + LCount := FCtrFactories.Count; + System.SetLength(Result, LCount); + for LI := 0 to LCount - 1 do + Result[LI] := FCtrFactories[LI]; + finally + FLock.Leave; + end; +end; + +class function TFusedKernelRegistry.SnapshotCbcFactories: TCryptoLibGenericArray; +var + LI, LCount: Int32; +begin + FLock.Enter; + try + LCount := FCbcFactories.Count; + System.SetLength(Result, LCount); + for LI := 0 to LCount - 1 do + Result[LI] := FCbcFactories[LI]; + finally + FLock.Leave; + end; +end; + { ---- TryAcquire ------------------------------------------------------------- } class function TFusedKernelRegistry.TryAcquireGcm(const ACipher: IBlockCipher; @@ -576,6 +724,50 @@ class function TFusedKernelRegistry.TryAcquireGcmSiv(const ACipher: IBlockCipher AKernel := nil; end; +class function TFusedKernelRegistry.TryAcquireCtr(const ACipher: IBlockCipher; + ADirection: TFusedModeDirection; out AKernel: IFusedCtrKernel): Boolean; +var + LSnapshot: TCryptoLibGenericArray; + LI: Int32; +begin + AKernel := nil; + Result := False; + if TFusedKernelGate.ForceDisabled then Exit; + if ACipher = nil then Exit; + LSnapshot := SnapshotCtrFactories; + for LI := 0 to System.Length(LSnapshot) - 1 do + begin + if LSnapshot[LI].TryCreate(ACipher, ADirection, AKernel) and (AKernel <> nil) then + begin + Result := True; + Exit; + end; + end; + AKernel := nil; +end; + +class function TFusedKernelRegistry.TryAcquireCbc(const ACipher: IBlockCipher; + ADirection: TFusedModeDirection; out AKernel: IFusedCbcKernel): Boolean; +var + LSnapshot: TCryptoLibGenericArray; + LI: Int32; +begin + AKernel := nil; + Result := False; + if TFusedKernelGate.ForceDisabled then Exit; + if ACipher = nil then Exit; + LSnapshot := SnapshotCbcFactories; + for LI := 0 to System.Length(LSnapshot) - 1 do + begin + if LSnapshot[LI].TryCreate(ACipher, ADirection, AKernel) and (AKernel <> nil) then + begin + Result := True; + Exit; + end; + end; + AKernel := nil; +end; + { ---- Provider queries ------------------------------------------------------- } class function TFusedKernelRegistry.GetRegisteredGcmProviders: TCryptoLibStringArray; @@ -633,4 +825,26 @@ class function TFusedKernelRegistry.GetRegisteredGcmSivProviders: TCryptoLibStri Result[LI] := LSnapshot[LI].ProviderName; end; +class function TFusedKernelRegistry.GetRegisteredCtrProviders: TCryptoLibStringArray; +var + LSnapshot: TCryptoLibGenericArray; + LI: Int32; +begin + LSnapshot := SnapshotCtrFactories; + System.SetLength(Result, System.Length(LSnapshot)); + for LI := 0 to System.Length(LSnapshot) - 1 do + Result[LI] := LSnapshot[LI].ProviderName; +end; + +class function TFusedKernelRegistry.GetRegisteredCbcProviders: TCryptoLibStringArray; +var + LSnapshot: TCryptoLibGenericArray; + LI: Int32; +begin + LSnapshot := SnapshotCbcFactories; + System.SetLength(Result, System.Length(LSnapshot)); + for LI := 0 to System.Length(LSnapshot) - 1 do + Result[LI] := LSnapshot[LI].ProviderName; +end; + end. diff --git a/CryptoLib/src/Crypto/Modes/Fused/Internal/ClpAesFusedAeadSimd.pas b/CryptoLib/src/Crypto/Modes/Fused/Internal/ClpAesFusedAeadSimd.pas index e8b1c8b7..a9404478 100644 --- a/CryptoLib/src/Crypto/Modes/Fused/Internal/ClpAesFusedAeadSimd.pas +++ b/CryptoLib/src/Crypto/Modes/Fused/Internal/ClpAesFusedAeadSimd.pas @@ -22,7 +22,7 @@ interface {$IF DEFINED(CRYPTOLIB_X86_SIMD)} uses - ClpAesFusedAeadX86Backend; + ClpAesNiFusedX86Backend; {$IFEND} type @@ -35,7 +35,7 @@ interface /// /// Engine resolution deliberately does NOT live here: it hands back an /// instruction-set-specific round-key schedule (see IAesEngineX86), so it - /// belongs on the per-arch backend (TAesFusedAeadX86Backend.TryResolveEngine) + /// belongs on the per-arch backend (TAesNiFusedX86Backend.TryResolveEngine) /// that the matching per-arch kernels call. Not exported via the public /// interface surface and never imported by mode units, which stay /// cipher-agnostic. @@ -53,7 +53,7 @@ implementation class function TAesFusedAeadSimd.CpuSupports: Boolean; begin {$IF DEFINED(CRYPTOLIB_X86_SIMD)} - Result := TAesFusedAeadX86Backend.CpuSupports; + Result := TAesNiFusedX86Backend.CpuSupports; {$ELSE} Result := False; {$IFEND} diff --git a/CryptoLib/src/Crypto/Modes/Fused/Internal/ClpAesFusedAeadX86Backend.pas b/CryptoLib/src/Crypto/Modes/Fused/Internal/ClpAesNiFusedX86Backend.pas similarity index 74% rename from CryptoLib/src/Crypto/Modes/Fused/Internal/ClpAesFusedAeadX86Backend.pas rename to CryptoLib/src/Crypto/Modes/Fused/Internal/ClpAesNiFusedX86Backend.pas index bb5e8aab..844a800e 100644 --- a/CryptoLib/src/Crypto/Modes/Fused/Internal/ClpAesFusedAeadX86Backend.pas +++ b/CryptoLib/src/Crypto/Modes/Fused/Internal/ClpAesNiFusedX86Backend.pas @@ -14,7 +14,7 @@ (* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) -unit ClpAesFusedAeadX86Backend; +unit ClpAesNiFusedX86Backend; {$I ..\..\..\..\Include\CryptoLib.inc} @@ -30,15 +30,18 @@ interface type /// - /// x86 backend for the fused AES-NI AEAD pipeline: owns the CPU/build gate and - /// the concrete IAesEngineX86 resolution used by the in-tree AES-NI AEAD - /// kernel factories. Compiles on every target - CpuSupports is - /// False off x86 and TryResolveEngine then finds no engine. + /// x86 backend shared by the in-tree AES-NI fused kernel factories (the AEAD + /// family and the standalone CTR kernel): owns the AEAD CPU/build gate and the + /// concrete IAesEngineX86 resolution (CTR uses only the latter). + /// Compiles on every target - CpuSupports is False off x86 and + /// TryResolveEngine then finds no engine. /// - TAesFusedAeadX86Backend = class sealed + TAesNiFusedX86Backend = class sealed public - /// True only when the build defines CRYPTOLIB_X86_SIMD, the CPU - /// exposes hardware AES + carryless multiply, and the SIMD intrinsics layout is packed. + /// AEAD gate: True only when the build defines CRYPTOLIB_X86_SIMD, + /// the CPU exposes hardware AES + carryless multiply + SSSE3, and the SIMD + /// intrinsics layout is packed (the AEAD kernels pass a packed context + /// record to asm). class function CpuSupports: Boolean; static; /// Probe ACipher for IAesEngineX86, handling both the direct case @@ -50,9 +53,9 @@ TAesFusedAeadX86Backend = class sealed implementation -{ TAesFusedAeadX86Backend } +{ TAesNiFusedX86Backend } -class function TAesFusedAeadX86Backend.CpuSupports: Boolean; +class function TAesNiFusedX86Backend.CpuSupports: Boolean; begin {$IFDEF CRYPTOLIB_X86_SIMD} Result := TCpuFeatures.X86.HasAESNI and TCpuFeatures.X86.HasPCLMULQDQ and @@ -62,7 +65,7 @@ class function TAesFusedAeadX86Backend.CpuSupports: Boolean; {$ENDIF CRYPTOLIB_X86_SIMD} end; -class function TAesFusedAeadX86Backend.TryResolveEngine(const ACipher: IBlockCipher; +class function TAesNiFusedX86Backend.TryResolveEngine(const ACipher: IBlockCipher; out AEngine: IAesEngineX86): Boolean; var LMode: IBlockCipherMode; diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas index 5e019e04..6adee681 100644 --- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas +++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashSimd.pas @@ -53,12 +53,8 @@ TGhashSimd = class sealed /// True when the shuffled/fused GHASH path (4-/8-way) is usable on this build/CPU. Gates the batch dispatch and the H-power precompute. class function IsShuffledGhashSupported: Boolean; static; - /// True when the packed 128-bit block-XOR fast path is usable. - class function IsBlockXorSupported: Boolean; static; /// True when a hardware carryless (polynomial) multiply is available (selects the carryless-multiply GCM multiplier over the 4K-table one). class function HasCarrylessMultiply: Boolean; static; - /// XOR one 128-bit block: PDst := PDst xor PSrc. Precondition: IsBlockXorSupported. - class procedure BlockXor128(PDst, PSrc: PByte); static; /// Full byte-reverse of one 128-bit block from PSrc into PDst; False when unavailable. class function TryBlockReverse128(PDst, PSrc: PByte): Boolean; static; end; @@ -130,15 +126,6 @@ class function TGhashSimd.IsShuffledGhashSupported: Boolean; {$IFEND} end; -class function TGhashSimd.IsBlockXorSupported: Boolean; -begin -{$IF DEFINED(CRYPTOLIB_X86_SIMD)} - Result := TGhashX86Backend.IsBlockXorSupported; -{$ELSE} - Result := False; -{$IFEND} -end; - class function TGhashSimd.HasCarrylessMultiply: Boolean; begin {$IF DEFINED(CRYPTOLIB_X86_SIMD)} @@ -148,13 +135,6 @@ class function TGhashSimd.HasCarrylessMultiply: Boolean; {$IFEND} end; -class procedure TGhashSimd.BlockXor128(PDst, PSrc: PByte); -begin -{$IF DEFINED(CRYPTOLIB_X86_SIMD)} - TGhashX86Backend.BlockXor128(PDst, PSrc); -{$IFEND} -end; - class function TGhashSimd.TryBlockReverse128(PDst, PSrc: PByte): Boolean; begin {$IF DEFINED(CRYPTOLIB_X86_SIMD)} diff --git a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas index d63cea50..b6315427 100644 --- a/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas +++ b/CryptoLib/src/Crypto/Modes/Gcm/ClpGhashX86Backend.pas @@ -52,12 +52,8 @@ TGhashX86Backend = class sealed /// True when the fused shuffled-GHASH path is usable on this CPU (needs packed vector layout). Gates the 4-/8-way batch dispatch and the H-power precompute. class function IsShuffledGhashSupported: Boolean; static; - /// True when the packed 128-bit block XOR fast path is usable. - class function IsBlockXorSupported: Boolean; static; /// True when the PCLMULQDQ carryless multiply is available (selects the carryless-multiply GCM multiplier over the 4K-table one). class function HasCarrylessMultiply: Boolean; static; - /// SIMD XOR of one 128-bit block: PDst := PDst xor PSrc. Precondition: IsBlockXorSupported. - class procedure BlockXor128(PDst, PSrc: PByte); static; /// Full byte-reverse of one 128-bit block from PSrc into PDst; returns False when unavailable on this CPU. class function TryBlockReverse128(PDst, PSrc: PByte): Boolean; static; end; @@ -156,17 +152,6 @@ procedure GcmGhashEightFull(PFS, PC0, PHPow128, PMask: Pointer); ReverseBytesMask: packed array[0..15] of Byte = ( $0F, $0E, $0D, $0C, $0B, $0A, $09, $08, $07, $06, $05, $04, $03, $02, $01, $00); -procedure GcmBlockXor128Sse2(PDst, PSrc: PByte); -{$IFDEF CRYPTOLIB_X86_64_ASM} -{$I ..\..\..\Include\Simd\Common\SimdProc2Begin_x86_64.inc} -{$I ..\..\..\Include\Simd\Gcm\GcmBlockXor128Sse2_x86_64.inc} -{$ENDIF} -{$IFDEF CRYPTOLIB_I386_ASM} -{$I ..\..\..\Include\Simd\Common\SimdProc2Begin_i386.inc} -{$I ..\..\..\Include\Simd\Gcm\GcmBlockXor128Sse2_i386.inc} -{$ENDIF} -end; - procedure GcmBlockReverse128Ssse3(PDst, PSrc, PMask: PByte); {$IFDEF CRYPTOLIB_X86_64_ASM} {$I ..\..\..\Include\Simd\Common\SimdProc3Begin_x86_64.inc} @@ -295,15 +280,6 @@ class function TGhashX86Backend.IsShuffledGhashSupported: Boolean; {$ENDIF} end; -class function TGhashX86Backend.IsBlockXorSupported: Boolean; -begin -{$IFDEF CRYPTOLIB_X86_SIMD} - Result := TCpuFeatures.X86.HasSSE2 and TIntrinsicsVector.IsPacked; -{$ELSE} - Result := False; -{$ENDIF} -end; - class function TGhashX86Backend.HasCarrylessMultiply: Boolean; begin {$IFDEF CRYPTOLIB_X86_SIMD} @@ -313,13 +289,6 @@ class function TGhashX86Backend.HasCarrylessMultiply: Boolean; {$ENDIF} end; -class procedure TGhashX86Backend.BlockXor128(PDst, PSrc: PByte); -begin -{$IFDEF CRYPTOLIB_X86_SIMD} - GcmBlockXor128Sse2(PDst, PSrc); -{$ENDIF} -end; - class function TGhashX86Backend.TryBlockReverse128(PDst, PSrc: PByte): Boolean; begin {$IFDEF CRYPTOLIB_X86_SIMD} diff --git a/CryptoLib/src/GeneralUtilities/ClpArrayUtilities.pas b/CryptoLib/src/GeneralUtilities/ClpArrayUtilities.pas index 1eb48045..81be5767 100644 --- a/CryptoLib/src/GeneralUtilities/ClpArrayUtilities.pas +++ b/CryptoLib/src/GeneralUtilities/ClpArrayUtilities.pas @@ -40,6 +40,11 @@ TArrayUtilities = class sealed(TObject) /// Processes ACount bytes starting at AData. /// class function HashCore(AData: PByte; ACount: Int32): Int32; static; + + // Per-element fill shared by Fill and the typed Fill fallbacks. + // Callers validate ABuf / range first; the loop assumes AFrom < ATo. + class procedure FillCore(ABuf: TCryptoLibGenericArray; + AFrom, ATo: Int32; const AFiller: T); static; public class function AreEqual(const A, B: TCryptoLibByteArray): Boolean; overload; static; @@ -109,7 +114,29 @@ TArrayUtilities = class sealed(TObject) : Boolean; overload; static; class procedure Fill(ABuf: TCryptoLibGenericArray; AFrom, ATo: Int32; - const AFiller: T); static; + const AFiller: T); overload; static; + + class procedure Fill(const ABuf: TCryptoLibByteArray; AFrom, ATo: Int32; + AFiller: Byte); overload; static; inline; + class procedure Fill(const ABuf: TCryptoLibUInt32Array; AFrom, ATo: Int32; + AFiller: UInt32); overload; static; inline; + class procedure Fill(const ABuf: TCryptoLibUInt64Array; AFrom, ATo: Int32; + AFiller: UInt64); overload; static; inline; + + /// + /// Grow ABuf's capacity to at least ANeeded by doubling; never shrinks. + /// + class procedure EnsureCapacity(var ABuf: TCryptoLibByteArray; + ANeeded: Int32); static; + + /// + /// Grow ABuf if needed, write at index ALen, and advance ALen by the number + /// of bytes written. + /// + class procedure AppendTo(var ABuf: TCryptoLibByteArray; var ALen: Int32; + AValue: Byte); overload; static; + class procedure AppendTo(var ABuf: TCryptoLibByteArray; var ALen: Int32; + const ASrc: TCryptoLibByteArray; AOff, ACount: Int32); overload; static; /// /// Deep-clone an array using ACloneFunc. Returns nil if AData is nil. @@ -304,15 +331,88 @@ class function TArrayUtilities.CopyOfRange(const AData: TCryptoLibGenericArra Result[LI] := AData[AFrom + LI]; end; +class procedure TArrayUtilities.FillCore(ABuf: TCryptoLibGenericArray; + AFrom, ATo: Int32; const AFiller: T); +begin + while AFrom < ATo do + begin + ABuf[AFrom] := AFiller; + Inc(AFrom); + end; +end; + class procedure TArrayUtilities.Fill(ABuf: TCryptoLibGenericArray; AFrom, ATo: Int32; const AFiller: T); +begin + if (ABuf = nil) or (ATo <= AFrom) then + Exit; + FillCore(ABuf, AFrom, ATo, AFiller); +end; + +class procedure TArrayUtilities.Fill(const ABuf: TCryptoLibByteArray; + AFrom, ATo: Int32; AFiller: Byte); +begin + if (ABuf <> nil) and (ATo > AFrom) then + System.FillChar(ABuf[AFrom], ATo - AFrom, AFiller); +end; + +class procedure TArrayUtilities.Fill(const ABuf: TCryptoLibUInt32Array; + AFrom, ATo: Int32; AFiller: UInt32); +begin + if (ABuf = nil) or (ATo <= AFrom) then + Exit; +{$IFDEF FPC} + System.FillDWord(ABuf[AFrom], ATo - AFrom, AFiller); +{$ELSE} + FillCore(ABuf, AFrom, ATo, AFiller); +{$ENDIF FPC} +end; + +class procedure TArrayUtilities.Fill(const ABuf: TCryptoLibUInt64Array; + AFrom, ATo: Int32; AFiller: UInt64); +begin + if (ABuf = nil) or (ATo <= AFrom) then + Exit; +{$IFDEF FPC} + System.FillQWord(ABuf[AFrom], ATo - AFrom, AFiller); +{$ELSE} + FillCore(ABuf, AFrom, ATo, AFiller); +{$ENDIF FPC} +end; + +class procedure TArrayUtilities.EnsureCapacity(var ABuf: TCryptoLibByteArray; + ANeeded: Int32); var - LI: Int32; + LCap: Int32; begin - if ABuf = nil then + LCap := System.Length(ABuf); + if ANeeded <= LCap then + Exit; + if LCap = 0 then + LCap := 64; + while (LCap < ANeeded) and (LCap > 0) do + LCap := LCap * 2; + if LCap < ANeeded then // Int32 overflow guard for very large packets + LCap := ANeeded; + System.SetLength(ABuf, LCap); +end; + +class procedure TArrayUtilities.AppendTo(var ABuf: TCryptoLibByteArray; + var ALen: Int32; AValue: Byte); +begin + EnsureCapacity(ABuf, ALen + 1); + ABuf[ALen] := AValue; + System.Inc(ALen); +end; + +class procedure TArrayUtilities.AppendTo(var ABuf: TCryptoLibByteArray; + var ALen: Int32; const ASrc: TCryptoLibByteArray; AOff, ACount: Int32); +begin + if ACount <= 0 then Exit; - for LI := AFrom to ATo - 1 do - ABuf[LI] := AFiller; + EnsureCapacity(ABuf, ALen + ACount); + System.Move(ASrc[AOff], ABuf[ALen], ACount); + System.Inc(ALen, ACount); end; class function TArrayUtilities.GetArrayHashCode(const AData: TCryptoLibByteArray): Int32; diff --git a/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_i386.inc b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_i386.inc new file mode 100644 index 00000000..5edcd989 --- /dev/null +++ b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_i386.inc @@ -0,0 +1,56 @@ +// AES-NI 1-wide CBC encryption, serial in-register chain kernel (i386). +// ------------------------------------------------------------------- +// +// i386 counterpart of AesNiCbcEncryptSerial_x86_64.inc: C_i = E_K(P_i xor C_{i-1}), +// the serial CBC chaining mode, for TCbcBlockCipher's bulk path. CBC encrypt is +// inherently serial, so the win is orchestration only: the chaining value stays +// in xmm1 across the whole run -- no per-block store/reload, no XOR helper call, +// no per-block dispatch/validation. +// +// The AES rounds reuse the shared 1-wide chain (AesNiOneRoundsOnly_i386.inc), +// which consumes round keys as [edi + K] memory operands and touches only xmm0. +// i386 has only xmm0..7 (all volatile); the chain uses xmm1. ebx (pushed by the +// prologue), esi and edi are callee-saved and restored before return. +// +// Entry (after SimdProc1Begin_i386.inc): ebx = pointer to TCbcEncryptFusedCtx. The +// kernel pushes esi/edi and unpacks the record: +// [ebx + 0] InPtr -> esi : input (advances +16 per block) +// [ebx + 4] OutPtr -> edx : output (advances +16) +// [ebx + 8] KeysPtr -> edi : AES round-key schedule ([edi + K]) +// [ebx + 12] IvPtr -> eax : 16-byte chaining value (seed + writeback) +// [ebx + 16] BlockCount -> ecx : number of blocks, caller guarantees >= 1 +// +// Exactly one of CRYPTOLIB_AESNI_KEY128 / KEY192 / KEY256 must be defined; the +// direction is always encrypt, so CRYPTOLIB_AESNI_DECRYPT must NOT be defined. +// +// AES-NI instructions and the register pxor forms are db-encoded for broad +// assembler compatibility. + // ---- unpack TCbcEncryptFusedCtx (ebx = PCtx) ---- + push esi + push edi + mov esi, [ebx + 0] // InPtr + mov edx, [ebx + 4] // OutPtr + mov edi, [ebx + 8] // KeysPtr + mov eax, [ebx + 12] // IvPtr + mov ecx, [ebx + 16] // BlockCount + + movdqu xmm1, [eax] // chain = C_{-1} (the IV) + +@@CbcEncryptSerialLoopI386: + movdqu xmm0, [esi] // P_i + db $66, $0F, $EF, $C1 // pxor xmm0, xmm1 (P_i xor chain) +{$I ..\..\..\Include\Simd\Aes\AesNiOneRoundsOnly_i386.inc} // xmm0 = E_K(xmm0) + movdqu [edx], xmm0 // C_i + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 (chain = C_i) + add esi, 16 + add edx, 16 + sub ecx, 1 + jnz @@CbcEncryptSerialLoopI386 + + movdqu [eax], xmm1 // write final chain (= last C) back + db $66, $0F, $EF, $C0 // pxor xmm0, xmm0 (scrub) + db $66, $0F, $EF, $C9 // pxor xmm1, xmm1 (scrub) + + pop edi + pop esi + pop ebx diff --git a/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_x86_64.inc new file mode 100644 index 00000000..78e2693c --- /dev/null +++ b/CryptoLib/src/Include/Simd/Aes/Cbc/AesNiCbcEncryptSerial_x86_64.inc @@ -0,0 +1,50 @@ +// AES-NI 1-wide CBC encryption, serial in-register chain kernel (x86-64). +// ------------------------------------------------------------------- +// +// Fused CBC-encrypt body for TCbcBlockCipher's bulk path: C_i = E_K(P_i xor +// C_{i-1}), the serial chaining mode. CBC encrypt cannot be block-parallelised +// (each block needs the previous block's ciphertext), so the win here is purely +// in orchestration: the chaining value lives in xmm1 across the whole run -- no +// per-block store/reload, no XOR helper call, no per-block dispatch/validation. +// +// The AES rounds reuse the shared 1-wide chain (AesNiOneRoundsOnly_x86_64.inc), +// which consumes round keys as [r8 + K] memory operands and touches only xmm0. +// Every working register (rcx,rdx,r8,r9,r10,r11,xmm0,xmm1) is volatile on the +// MS x64 ABI, so this stays a true leaf: no xmm6-15 save, no callee-saved push. +// +// Entry (after SimdProc1Begin_x86_64.inc): rcx = pointer to TCbcEncryptFusedCtx. +// The kernel unpacks it (InPtr loaded into r11): +// [rcx + 0] InPtr -> r11 : input (advances +16 per block) +// [rcx + 8] OutPtr -> rdx : output (advances +16) +// [rcx + 16] KeysPtr -> r8 : AES encrypt round-key schedule ([r8 + K]) +// [rcx + 24] IvPtr -> r9 : 16-byte chaining value (seed + writeback) +// [rcx + 32] BlockCount -> r10 : number of blocks, caller guarantees >= 1 +// +// Exactly one of CRYPTOLIB_AESNI_KEY128 / KEY192 / KEY256 must be defined; the +// direction is always encrypt, so CRYPTOLIB_AESNI_DECRYPT must NOT be defined. +// +// AES-NI instructions and the register pxor forms are db-encoded for broad +// assembler compatibility. + // ---- unpack TCbcEncryptFusedCtx (rcx = PCtx); load InPtr into r11 last ---- + mov r8, [rcx + 16] // KeysPtr + mov r9, [rcx + 24] // IvPtr + mov r10, [rcx + 32] // BlockCount + mov rdx, [rcx + 8] // OutPtr + mov r11, [rcx + 0] // InPtr + + movdqu xmm1, [r9] // chain = C_{-1} (the IV) + +@@CbcEncryptSerialLoop: + movdqu xmm0, [r11] // P_i + db $66, $0F, $EF, $C1 // pxor xmm0, xmm1 (P_i xor chain) +{$I ..\..\..\Include\Simd\Aes\AesNiOneRoundsOnly_x86_64.inc} // xmm0 = E_K(xmm0) + movdqu [rdx], xmm0 // C_i + db $66, $0F, $6F, $C8 // movdqa xmm1, xmm0 (chain = C_i) + add r11, 16 + add rdx, 16 + sub r10, 1 + jnz @@CbcEncryptSerialLoop + + movdqu [r9], xmm1 // write final chain (= last C) back + db $66, $0F, $EF, $C0 // pxor xmm0, xmm0 (scrub) + db $66, $0F, $EF, $C9 // pxor xmm1, xmm1 (scrub) diff --git a/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_i386.inc b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_i386.inc new file mode 100644 index 00000000..c9414a2d --- /dev/null +++ b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_i386.inc @@ -0,0 +1,473 @@ +// AES-NI 8-wide CTR keystream + XOR, interleaved fused kernel (i386). +// ------------------------------------------------------------------- +// +// i386 counterpart of AesNiCtrEight_x86_64.inc: processes eight AES-CTR blocks +// per iteration for TSicBlockCipher's bulk path, XORing the keystream with the +// input in one pass. The eight cipher lanes live in xmm0..7 across the whole +// round chain (round keys are consumed as [edi + K] memory operands, so no key +// register is needed), and the NEXT batch's eight counter blocks are staged into +// a 128-byte stack scratch *interleaved between the aesenc rounds* of the current +// batch - the scalar counter build overlaps the aesenc throughput instead of +// serialising ahead of it. +// +// Entry (after SimdProc1Begin_i386.inc): ebx = pointer to TCtrFusedCtx. The +// kernel pushes esi/edi and unpacks the record (InPtr loaded into ebx last): +// [ebx + 0] InPtr -> ebx : input (advances +128 per 8-block iteration) +// [ebx + 4] OutPtr -> esi : output (advances +128) +// [ebx + 8] KeysPtr -> edi : AES round-key schedule (consumed as [edi+K]) +// [ebx + 12] CounterPtr -> eax : 16-byte big-endian counter (advanced in place) +// [ebx + 16] BlockCount -> ecx : positive multiple of 8 (caller guarantees it) +// +// Exactly one of CRYPTOLIB_AESNI_KEY128 / KEY192 / KEY256 must be defined; the +// direction is always encrypt (CTR keystream), so CRYPTOLIB_AESNI_DECRYPT must +// NOT be defined when including this file. +// +// Constraints: i386 has only xmm0..7 (all volatile) and, after unpacking the +// context, only edx as a free scratch GP register. The 128-bit counter lives in +// memory at [eax] and is bumped in place with a byte-wise big-endian carry (edx = +// byte index). Each staged block copies the current [eax] to the scratch (four +// dword moves through edx, since no xmm is free mid-round) then advances. The +// pipeline builds one batch ahead, so [eax] over-runs by exactly one batch (8) - +// undone before returning. The keystream^plaintext XOR has no free xmm for the +// plaintext (nor can it use a memory-operand pxor, which would require the input +// to be 16-byte aligned), so it spills lane 7 to a scratch slot above the counter +// area and reuses xmm7 for the movdqu plaintext load. AES-NI and register pxor +// forms are db-encoded for broad assembler compatibility. + // ---- unpack TCtrFusedCtx (ebx = PCtx); load InPtr into ebx last ---- + push esi + push edi + mov esi, [ebx + 4] // OutPtr + mov edi, [ebx + 8] // KeysPtr + mov eax, [ebx + 12] // CounterPtr + mov ecx, [ebx + 16] // BlockCount + mov ebx, [ebx + 0] // InPtr (overwrites PCtx) + + sub esp, 144 // next-batch counters [esp+0..127]; ks7 spill [esp+128..143] + + // ---- stage batch 0 into the scratch (xmm0 is free here), then load it ---- + movdqu xmm0, [eax] + movdqu [esp + 0], xmm0 + mov edx, 15 +@@PreA0: + inc byte ptr [eax + edx] + jnz @@PreD0 + dec edx + jns @@PreA0 +@@PreD0: + movdqu xmm0, [eax] + movdqu [esp + 16], xmm0 + mov edx, 15 +@@PreA1: + inc byte ptr [eax + edx] + jnz @@PreD1 + dec edx + jns @@PreA1 +@@PreD1: + movdqu xmm0, [eax] + movdqu [esp + 32], xmm0 + mov edx, 15 +@@PreA2: + inc byte ptr [eax + edx] + jnz @@PreD2 + dec edx + jns @@PreA2 +@@PreD2: + movdqu xmm0, [eax] + movdqu [esp + 48], xmm0 + mov edx, 15 +@@PreA3: + inc byte ptr [eax + edx] + jnz @@PreD3 + dec edx + jns @@PreA3 +@@PreD3: + movdqu xmm0, [eax] + movdqu [esp + 64], xmm0 + mov edx, 15 +@@PreA4: + inc byte ptr [eax + edx] + jnz @@PreD4 + dec edx + jns @@PreA4 +@@PreD4: + movdqu xmm0, [eax] + movdqu [esp + 80], xmm0 + mov edx, 15 +@@PreA5: + inc byte ptr [eax + edx] + jnz @@PreD5 + dec edx + jns @@PreA5 +@@PreD5: + movdqu xmm0, [eax] + movdqu [esp + 96], xmm0 + mov edx, 15 +@@PreA6: + inc byte ptr [eax + edx] + jnz @@PreD6 + dec edx + jns @@PreA6 +@@PreD6: + movdqu xmm0, [eax] + movdqu [esp + 112], xmm0 + mov edx, 15 +@@PreA7: + inc byte ptr [eax + edx] + jnz @@PreD7 + dec edx + jns @@PreA7 +@@PreD7: + + movdqu xmm0, [esp + 0] + movdqu xmm1, [esp + 16] + movdqu xmm2, [esp + 32] + movdqu xmm3, [esp + 48] + movdqu xmm4, [esp + 64] + movdqu xmm5, [esp + 80] + movdqu xmm6, [esp + 96] + movdqu xmm7, [esp + 112] + +@@CtrEightLoopI386: + // ---- round 0: whiten with K_0 ---- + db $66, $0F, $EF, $07 // pxor xmm0, [edi+0] + db $66, $0F, $EF, $0F // pxor xmm1, [edi+0] + db $66, $0F, $EF, $17 // pxor xmm2, [edi+0] + db $66, $0F, $EF, $1F // pxor xmm3, [edi+0] + db $66, $0F, $EF, $27 // pxor xmm4, [edi+0] + db $66, $0F, $EF, $2F // pxor xmm5, [edi+0] + db $66, $0F, $EF, $37 // pxor xmm6, [edi+0] + db $66, $0F, $EF, $3F // pxor xmm7, [edi+0] + + // ---- rounds 1..8: aesenc, each interleaved with staging one next counter ---- + db $66, $0F, $38, $DC, $47, $10 // aesenc xmm0, [edi+16] + db $66, $0F, $38, $DC, $4F, $10 // aesenc xmm1, [edi+16] + db $66, $0F, $38, $DC, $57, $10 // aesenc xmm2, [edi+16] + db $66, $0F, $38, $DC, $5F, $10 // aesenc xmm3, [edi+16] + db $66, $0F, $38, $DC, $67, $10 // aesenc xmm4, [edi+16] + db $66, $0F, $38, $DC, $6F, $10 // aesenc xmm5, [edi+16] + db $66, $0F, $38, $DC, $77, $10 // aesenc xmm6, [edi+16] + db $66, $0F, $38, $DC, $7F, $10 // aesenc xmm7, [edi+16] + mov edx, [eax] + mov [esp + 0], edx + mov edx, [eax + 4] + mov [esp + 4], edx + mov edx, [eax + 8] + mov [esp + 8], edx + mov edx, [eax + 12] + mov [esp + 12], edx + mov edx, 15 +@@LpA0: + inc byte ptr [eax + edx] + jnz @@LpD0 + dec edx + jns @@LpA0 +@@LpD0: + + db $66, $0F, $38, $DC, $47, $20 // aesenc xmm0, [edi+32] + db $66, $0F, $38, $DC, $4F, $20 // aesenc xmm1, [edi+32] + db $66, $0F, $38, $DC, $57, $20 // aesenc xmm2, [edi+32] + db $66, $0F, $38, $DC, $5F, $20 // aesenc xmm3, [edi+32] + db $66, $0F, $38, $DC, $67, $20 // aesenc xmm4, [edi+32] + db $66, $0F, $38, $DC, $6F, $20 // aesenc xmm5, [edi+32] + db $66, $0F, $38, $DC, $77, $20 // aesenc xmm6, [edi+32] + db $66, $0F, $38, $DC, $7F, $20 // aesenc xmm7, [edi+32] + mov edx, [eax] + mov [esp + 16], edx + mov edx, [eax + 4] + mov [esp + 20], edx + mov edx, [eax + 8] + mov [esp + 24], edx + mov edx, [eax + 12] + mov [esp + 28], edx + mov edx, 15 +@@LpA1: + inc byte ptr [eax + edx] + jnz @@LpD1 + dec edx + jns @@LpA1 +@@LpD1: + + db $66, $0F, $38, $DC, $47, $30 // aesenc xmm0, [edi+48] + db $66, $0F, $38, $DC, $4F, $30 // aesenc xmm1, [edi+48] + db $66, $0F, $38, $DC, $57, $30 // aesenc xmm2, [edi+48] + db $66, $0F, $38, $DC, $5F, $30 // aesenc xmm3, [edi+48] + db $66, $0F, $38, $DC, $67, $30 // aesenc xmm4, [edi+48] + db $66, $0F, $38, $DC, $6F, $30 // aesenc xmm5, [edi+48] + db $66, $0F, $38, $DC, $77, $30 // aesenc xmm6, [edi+48] + db $66, $0F, $38, $DC, $7F, $30 // aesenc xmm7, [edi+48] + mov edx, [eax] + mov [esp + 32], edx + mov edx, [eax + 4] + mov [esp + 36], edx + mov edx, [eax + 8] + mov [esp + 40], edx + mov edx, [eax + 12] + mov [esp + 44], edx + mov edx, 15 +@@LpA2: + inc byte ptr [eax + edx] + jnz @@LpD2 + dec edx + jns @@LpA2 +@@LpD2: + + db $66, $0F, $38, $DC, $47, $40 // aesenc xmm0, [edi+64] + db $66, $0F, $38, $DC, $4F, $40 // aesenc xmm1, [edi+64] + db $66, $0F, $38, $DC, $57, $40 // aesenc xmm2, [edi+64] + db $66, $0F, $38, $DC, $5F, $40 // aesenc xmm3, [edi+64] + db $66, $0F, $38, $DC, $67, $40 // aesenc xmm4, [edi+64] + db $66, $0F, $38, $DC, $6F, $40 // aesenc xmm5, [edi+64] + db $66, $0F, $38, $DC, $77, $40 // aesenc xmm6, [edi+64] + db $66, $0F, $38, $DC, $7F, $40 // aesenc xmm7, [edi+64] + mov edx, [eax] + mov [esp + 48], edx + mov edx, [eax + 4] + mov [esp + 52], edx + mov edx, [eax + 8] + mov [esp + 56], edx + mov edx, [eax + 12] + mov [esp + 60], edx + mov edx, 15 +@@LpA3: + inc byte ptr [eax + edx] + jnz @@LpD3 + dec edx + jns @@LpA3 +@@LpD3: + + db $66, $0F, $38, $DC, $47, $50 // aesenc xmm0, [edi+80] + db $66, $0F, $38, $DC, $4F, $50 // aesenc xmm1, [edi+80] + db $66, $0F, $38, $DC, $57, $50 // aesenc xmm2, [edi+80] + db $66, $0F, $38, $DC, $5F, $50 // aesenc xmm3, [edi+80] + db $66, $0F, $38, $DC, $67, $50 // aesenc xmm4, [edi+80] + db $66, $0F, $38, $DC, $6F, $50 // aesenc xmm5, [edi+80] + db $66, $0F, $38, $DC, $77, $50 // aesenc xmm6, [edi+80] + db $66, $0F, $38, $DC, $7F, $50 // aesenc xmm7, [edi+80] + mov edx, [eax] + mov [esp + 64], edx + mov edx, [eax + 4] + mov [esp + 68], edx + mov edx, [eax + 8] + mov [esp + 72], edx + mov edx, [eax + 12] + mov [esp + 76], edx + mov edx, 15 +@@LpA4: + inc byte ptr [eax + edx] + jnz @@LpD4 + dec edx + jns @@LpA4 +@@LpD4: + + db $66, $0F, $38, $DC, $47, $60 // aesenc xmm0, [edi+96] + db $66, $0F, $38, $DC, $4F, $60 // aesenc xmm1, [edi+96] + db $66, $0F, $38, $DC, $57, $60 // aesenc xmm2, [edi+96] + db $66, $0F, $38, $DC, $5F, $60 // aesenc xmm3, [edi+96] + db $66, $0F, $38, $DC, $67, $60 // aesenc xmm4, [edi+96] + db $66, $0F, $38, $DC, $6F, $60 // aesenc xmm5, [edi+96] + db $66, $0F, $38, $DC, $77, $60 // aesenc xmm6, [edi+96] + db $66, $0F, $38, $DC, $7F, $60 // aesenc xmm7, [edi+96] + mov edx, [eax] + mov [esp + 80], edx + mov edx, [eax + 4] + mov [esp + 84], edx + mov edx, [eax + 8] + mov [esp + 88], edx + mov edx, [eax + 12] + mov [esp + 92], edx + mov edx, 15 +@@LpA5: + inc byte ptr [eax + edx] + jnz @@LpD5 + dec edx + jns @@LpA5 +@@LpD5: + + db $66, $0F, $38, $DC, $47, $70 // aesenc xmm0, [edi+112] + db $66, $0F, $38, $DC, $4F, $70 // aesenc xmm1, [edi+112] + db $66, $0F, $38, $DC, $57, $70 // aesenc xmm2, [edi+112] + db $66, $0F, $38, $DC, $5F, $70 // aesenc xmm3, [edi+112] + db $66, $0F, $38, $DC, $67, $70 // aesenc xmm4, [edi+112] + db $66, $0F, $38, $DC, $6F, $70 // aesenc xmm5, [edi+112] + db $66, $0F, $38, $DC, $77, $70 // aesenc xmm6, [edi+112] + db $66, $0F, $38, $DC, $7F, $70 // aesenc xmm7, [edi+112] + mov edx, [eax] + mov [esp + 96], edx + mov edx, [eax + 4] + mov [esp + 100], edx + mov edx, [eax + 8] + mov [esp + 104], edx + mov edx, [eax + 12] + mov [esp + 108], edx + mov edx, 15 +@@LpA6: + inc byte ptr [eax + edx] + jnz @@LpD6 + dec edx + jns @@LpA6 +@@LpD6: + + db $66, $0F, $38, $DC, $87, $80, $00, $00, $00 // aesenc xmm0, [edi+128] + db $66, $0F, $38, $DC, $8F, $80, $00, $00, $00 // aesenc xmm1, [edi+128] + db $66, $0F, $38, $DC, $97, $80, $00, $00, $00 // aesenc xmm2, [edi+128] + db $66, $0F, $38, $DC, $9F, $80, $00, $00, $00 // aesenc xmm3, [edi+128] + db $66, $0F, $38, $DC, $A7, $80, $00, $00, $00 // aesenc xmm4, [edi+128] + db $66, $0F, $38, $DC, $AF, $80, $00, $00, $00 // aesenc xmm5, [edi+128] + db $66, $0F, $38, $DC, $B7, $80, $00, $00, $00 // aesenc xmm6, [edi+128] + db $66, $0F, $38, $DC, $BF, $80, $00, $00, $00 // aesenc xmm7, [edi+128] + mov edx, [eax] + mov [esp + 112], edx + mov edx, [eax + 4] + mov [esp + 116], edx + mov edx, [eax + 8] + mov [esp + 120], edx + mov edx, [eax + 12] + mov [esp + 124], edx + mov edx, 15 +@@LpA7: + inc byte ptr [eax + edx] + jnz @@LpD7 + dec edx + jns @@LpA7 +@@LpD7: + + // ---- round 9 (all key sizes): aesenc, no more staging ---- + db $66, $0F, $38, $DC, $87, $90, $00, $00, $00 // aesenc xmm0, [edi+144] + db $66, $0F, $38, $DC, $8F, $90, $00, $00, $00 // aesenc xmm1, [edi+144] + db $66, $0F, $38, $DC, $97, $90, $00, $00, $00 // aesenc xmm2, [edi+144] + db $66, $0F, $38, $DC, $9F, $90, $00, $00, $00 // aesenc xmm3, [edi+144] + db $66, $0F, $38, $DC, $A7, $90, $00, $00, $00 // aesenc xmm4, [edi+144] + db $66, $0F, $38, $DC, $AF, $90, $00, $00, $00 // aesenc xmm5, [edi+144] + db $66, $0F, $38, $DC, $B7, $90, $00, $00, $00 // aesenc xmm6, [edi+144] + db $66, $0F, $38, $DC, $BF, $90, $00, $00, $00 // aesenc xmm7, [edi+144] + +{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)} + db $66, $0F, $38, $DC, $87, $A0, $00, $00, $00 // aesenc xmm0, [edi+160] + db $66, $0F, $38, $DC, $8F, $A0, $00, $00, $00 // aesenc xmm1, [edi+160] + db $66, $0F, $38, $DC, $97, $A0, $00, $00, $00 // aesenc xmm2, [edi+160] + db $66, $0F, $38, $DC, $9F, $A0, $00, $00, $00 // aesenc xmm3, [edi+160] + db $66, $0F, $38, $DC, $A7, $A0, $00, $00, $00 // aesenc xmm4, [edi+160] + db $66, $0F, $38, $DC, $AF, $A0, $00, $00, $00 // aesenc xmm5, [edi+160] + db $66, $0F, $38, $DC, $B7, $A0, $00, $00, $00 // aesenc xmm6, [edi+160] + db $66, $0F, $38, $DC, $BF, $A0, $00, $00, $00 // aesenc xmm7, [edi+160] + db $66, $0F, $38, $DC, $87, $B0, $00, $00, $00 // aesenc xmm0, [edi+176] + db $66, $0F, $38, $DC, $8F, $B0, $00, $00, $00 // aesenc xmm1, [edi+176] + db $66, $0F, $38, $DC, $97, $B0, $00, $00, $00 // aesenc xmm2, [edi+176] + db $66, $0F, $38, $DC, $9F, $B0, $00, $00, $00 // aesenc xmm3, [edi+176] + db $66, $0F, $38, $DC, $A7, $B0, $00, $00, $00 // aesenc xmm4, [edi+176] + db $66, $0F, $38, $DC, $AF, $B0, $00, $00, $00 // aesenc xmm5, [edi+176] + db $66, $0F, $38, $DC, $B7, $B0, $00, $00, $00 // aesenc xmm6, [edi+176] + db $66, $0F, $38, $DC, $BF, $B0, $00, $00, $00 // aesenc xmm7, [edi+176] +{$IFEND} + +{$IFDEF CRYPTOLIB_AESNI_KEY256} + db $66, $0F, $38, $DC, $87, $C0, $00, $00, $00 // aesenc xmm0, [edi+192] + db $66, $0F, $38, $DC, $8F, $C0, $00, $00, $00 // aesenc xmm1, [edi+192] + db $66, $0F, $38, $DC, $97, $C0, $00, $00, $00 // aesenc xmm2, [edi+192] + db $66, $0F, $38, $DC, $9F, $C0, $00, $00, $00 // aesenc xmm3, [edi+192] + db $66, $0F, $38, $DC, $A7, $C0, $00, $00, $00 // aesenc xmm4, [edi+192] + db $66, $0F, $38, $DC, $AF, $C0, $00, $00, $00 // aesenc xmm5, [edi+192] + db $66, $0F, $38, $DC, $B7, $C0, $00, $00, $00 // aesenc xmm6, [edi+192] + db $66, $0F, $38, $DC, $BF, $C0, $00, $00, $00 // aesenc xmm7, [edi+192] + db $66, $0F, $38, $DC, $87, $D0, $00, $00, $00 // aesenc xmm0, [edi+208] + db $66, $0F, $38, $DC, $8F, $D0, $00, $00, $00 // aesenc xmm1, [edi+208] + db $66, $0F, $38, $DC, $97, $D0, $00, $00, $00 // aesenc xmm2, [edi+208] + db $66, $0F, $38, $DC, $9F, $D0, $00, $00, $00 // aesenc xmm3, [edi+208] + db $66, $0F, $38, $DC, $A7, $D0, $00, $00, $00 // aesenc xmm4, [edi+208] + db $66, $0F, $38, $DC, $AF, $D0, $00, $00, $00 // aesenc xmm5, [edi+208] + db $66, $0F, $38, $DC, $B7, $D0, $00, $00, $00 // aesenc xmm6, [edi+208] + db $66, $0F, $38, $DC, $BF, $D0, $00, $00, $00 // aesenc xmm7, [edi+208] +{$ENDIF} + + // ---- final round: aesenclast on the last round key ---- +{$IFDEF CRYPTOLIB_AESNI_KEY128} + db $66, $0F, $38, $DD, $87, $A0, $00, $00, $00 // aesenclast xmm0, [edi+160] + db $66, $0F, $38, $DD, $8F, $A0, $00, $00, $00 // aesenclast xmm1, [edi+160] + db $66, $0F, $38, $DD, $97, $A0, $00, $00, $00 // aesenclast xmm2, [edi+160] + db $66, $0F, $38, $DD, $9F, $A0, $00, $00, $00 // aesenclast xmm3, [edi+160] + db $66, $0F, $38, $DD, $A7, $A0, $00, $00, $00 // aesenclast xmm4, [edi+160] + db $66, $0F, $38, $DD, $AF, $A0, $00, $00, $00 // aesenclast xmm5, [edi+160] + db $66, $0F, $38, $DD, $B7, $A0, $00, $00, $00 // aesenclast xmm6, [edi+160] + db $66, $0F, $38, $DD, $BF, $A0, $00, $00, $00 // aesenclast xmm7, [edi+160] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY192} + db $66, $0F, $38, $DD, $87, $C0, $00, $00, $00 // aesenclast xmm0, [edi+192] + db $66, $0F, $38, $DD, $8F, $C0, $00, $00, $00 // aesenclast xmm1, [edi+192] + db $66, $0F, $38, $DD, $97, $C0, $00, $00, $00 // aesenclast xmm2, [edi+192] + db $66, $0F, $38, $DD, $9F, $C0, $00, $00, $00 // aesenclast xmm3, [edi+192] + db $66, $0F, $38, $DD, $A7, $C0, $00, $00, $00 // aesenclast xmm4, [edi+192] + db $66, $0F, $38, $DD, $AF, $C0, $00, $00, $00 // aesenclast xmm5, [edi+192] + db $66, $0F, $38, $DD, $B7, $C0, $00, $00, $00 // aesenclast xmm6, [edi+192] + db $66, $0F, $38, $DD, $BF, $C0, $00, $00, $00 // aesenclast xmm7, [edi+192] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY256} + db $66, $0F, $38, $DD, $87, $E0, $00, $00, $00 // aesenclast xmm0, [edi+224] + db $66, $0F, $38, $DD, $8F, $E0, $00, $00, $00 // aesenclast xmm1, [edi+224] + db $66, $0F, $38, $DD, $97, $E0, $00, $00, $00 // aesenclast xmm2, [edi+224] + db $66, $0F, $38, $DD, $9F, $E0, $00, $00, $00 // aesenclast xmm3, [edi+224] + db $66, $0F, $38, $DD, $A7, $E0, $00, $00, $00 // aesenclast xmm4, [edi+224] + db $66, $0F, $38, $DD, $AF, $E0, $00, $00, $00 // aesenclast xmm5, [edi+224] + db $66, $0F, $38, $DD, $B7, $E0, $00, $00, $00 // aesenclast xmm6, [edi+224] + db $66, $0F, $38, $DD, $BF, $E0, $00, $00, $00 // aesenclast xmm7, [edi+224] +{$ENDIF} + + // ---- keystream ^ plaintext -> output. Spill ks7 to the scratch slot above + // the counters, reuse xmm7 as the movdqu plaintext scratch for blocks + // 0..6, then reload ks7 into the consumed xmm0 for block 7. ---- + movdqu [esp + 128], xmm7 // spill ks7 + movdqu xmm7, [ebx + 0] + db $66, $0F, $EF, $F8 // pxor xmm7, xmm0 + movdqu [esi + 0], xmm7 + movdqu xmm7, [ebx + 16] + db $66, $0F, $EF, $F9 // pxor xmm7, xmm1 + movdqu [esi + 16], xmm7 + movdqu xmm7, [ebx + 32] + db $66, $0F, $EF, $FA // pxor xmm7, xmm2 + movdqu [esi + 32], xmm7 + movdqu xmm7, [ebx + 48] + db $66, $0F, $EF, $FB // pxor xmm7, xmm3 + movdqu [esi + 48], xmm7 + movdqu xmm7, [ebx + 64] + db $66, $0F, $EF, $FC // pxor xmm7, xmm4 + movdqu [esi + 64], xmm7 + movdqu xmm7, [ebx + 80] + db $66, $0F, $EF, $FD // pxor xmm7, xmm5 + movdqu [esi + 80], xmm7 + movdqu xmm7, [ebx + 96] + db $66, $0F, $EF, $FE // pxor xmm7, xmm6 + movdqu [esi + 96], xmm7 + movdqu xmm0, [esp + 128] // reload ks7 into consumed xmm0 + movdqu xmm7, [ebx + 112] + db $66, $0F, $EF, $F8 // pxor xmm7, xmm0 + movdqu [esi + 112], xmm7 + + // ---- load the staged next-batch counters into xmm0..7 ---- + movdqu xmm0, [esp + 0] + movdqu xmm1, [esp + 16] + movdqu xmm2, [esp + 32] + movdqu xmm3, [esp + 48] + movdqu xmm4, [esp + 64] + movdqu xmm5, [esp + 80] + movdqu xmm6, [esp + 96] + movdqu xmm7, [esp + 112] + + add ebx, 128 + add esi, 128 + sub ecx, 8 + jnz @@CtrEightLoopI386 + + // ---- undo the one-batch build-ahead: counter (at [eax]) -= 8, big-endian ---- + sub byte ptr [eax + 15], 8 + jnc @@CtrWbDone + mov edx, 14 +@@CtrWbBorrow: + sub byte ptr [eax + edx], 1 + jnc @@CtrWbDone + dec edx + jns @@CtrWbBorrow +@@CtrWbDone: + + add esp, 144 + pop edi + pop esi + pop ebx diff --git a/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc new file mode 100644 index 00000000..6c8ba096 --- /dev/null +++ b/CryptoLib/src/Include/Simd/Aes/Ctr/AesNiCtrEight_x86_64.inc @@ -0,0 +1,412 @@ +// AES-NI 8-wide CTR keystream + XOR, interleaved fused kernel (x86-64). +// ------------------------------------------------------------------- +// +// Fused counter-mode body for TSicBlockCipher's bulk path: processes eight +// AES-CTR blocks per iteration, XORing the keystream with the input in one pass. +// +// Technique (Shay Gueron, "Intel AES New Instructions"): the eight cipher lanes +// live in xmm0..7 across the whole AES round chain, and the NEXT batch's eight +// counter blocks are materialised (to a 128-byte stack scratch) *interleaved +// between the aesenc rounds* of the current batch. The counter build is scalar +// (mov / bswap / add / adc on the integer ports) so it overlaps the aesenc +// throughput (vector port) instead of serialising ahead of it. The XOR with +// plaintext reuses xmm8 (the round-key scratch) after the final round. +// +// Entry (after SimdProc1Begin_x86_64.inc): rcx = pointer to TCtrFusedCtx. The +// kernel unpacks it (InPtr loaded into rcx last) into the working registers: +// [rcx + 0] InPtr -> rcx : input (advances +128 per 8-block iteration) +// [rcx + 8] OutPtr -> rdx : output (advances +128) +// [rcx + 16] KeysPtr -> r8 : AES encrypt round-key schedule (16-byte aligned) +// [rcx + 24] CounterPtr -> r9 : 16-byte big-endian counter (updated in place) +// [rcx + 32] BlockCount -> r10 : positive multiple of 8 (caller guarantees it) +// +// Exactly one of CRYPTOLIB_AESNI_KEY128 / KEY192 / KEY256 must be defined; the +// direction is always encrypt (CTR keystream), so CRYPTOLIB_AESNI_DECRYPT must +// NOT be defined when including this file. +// +// The 128-bit counter is held natively as r11 (high 64 bits) : rbx (low 64 bits) +// across the whole span; the low half is bumped with add/adc so carry propagates +// through the full 128 bits, matching SIC's big-endian whole-counter increment. +// Each staged block writes bswap(r11) || bswap(rbx) (big-endian) to the scratch, +// then advances. The pipeline builds one batch ahead, so on exit the counter has +// over-run by exactly one batch (8) - undone before write-back. +// +// Register/stack notes: rbx is callee-saved (push/pop). xmm6-15 are saved once +// via the shared core (xmm8 is the round-key / plaintext-XOR scratch). A 128-byte +// stack scratch below the xmm save area holds the next batch's counter blocks. +// AES-NI instructions and pxor forms touching xmm8 are db-encoded for broad +// assembler compatibility. + push rbx +{$I ..\..\..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc} + sub rsp, 128 // next-batch counter scratch: [rsp+0..127] + + // ---- unpack TCtrFusedCtx (rcx = PCtx); load InPtr into rcx last ---- + mov r8, [rcx + 16] // KeysPtr + mov r9, [rcx + 24] // CounterPtr + mov r10, [rcx + 32] // BlockCount + mov rdx, [rcx + 8] // OutPtr + mov rcx, [rcx + 0] // InPtr (overwrites PCtx) + + mov r11, [r9] // counter high 64 bits (big-endian) + bswap r11 + mov rbx, [r9 + 8] // counter low 64 bits (big-endian) + bswap rbx + + // ---- stage batch 0 into the scratch, then load it into xmm0..7 ---- + mov rax, r11 + bswap rax + mov [rsp + 0], rax + mov rax, rbx + bswap rax + mov [rsp + 8], rax + add rbx, 1 + adc r11, 0 + mov rax, r11 + bswap rax + mov [rsp + 16], rax + mov rax, rbx + bswap rax + mov [rsp + 24], rax + add rbx, 1 + adc r11, 0 + mov rax, r11 + bswap rax + mov [rsp + 32], rax + mov rax, rbx + bswap rax + mov [rsp + 40], rax + add rbx, 1 + adc r11, 0 + mov rax, r11 + bswap rax + mov [rsp + 48], rax + mov rax, rbx + bswap rax + mov [rsp + 56], rax + add rbx, 1 + adc r11, 0 + mov rax, r11 + bswap rax + mov [rsp + 64], rax + mov rax, rbx + bswap rax + mov [rsp + 72], rax + add rbx, 1 + adc r11, 0 + mov rax, r11 + bswap rax + mov [rsp + 80], rax + mov rax, rbx + bswap rax + mov [rsp + 88], rax + add rbx, 1 + adc r11, 0 + mov rax, r11 + bswap rax + mov [rsp + 96], rax + mov rax, rbx + bswap rax + mov [rsp + 104], rax + add rbx, 1 + adc r11, 0 + mov rax, r11 + bswap rax + mov [rsp + 112], rax + mov rax, rbx + bswap rax + mov [rsp + 120], rax + add rbx, 1 + adc r11, 0 + + movdqu xmm0, [rsp] + movdqu xmm1, [rsp + 16] + movdqu xmm2, [rsp + 32] + movdqu xmm3, [rsp + 48] + movdqu xmm4, [rsp + 64] + movdqu xmm5, [rsp + 80] + movdqu xmm6, [rsp + 96] + movdqu xmm7, [rsp + 112] + +@@CtrEightLoop: + // ---- round 0: whiten with K_0 ---- + movdqa xmm8, [r8] + db $66, $41, $0F, $EF, $C0 // pxor xmm0, xmm8 + db $66, $41, $0F, $EF, $C8 // pxor xmm1, xmm8 + db $66, $41, $0F, $EF, $D0 // pxor xmm2, xmm8 + db $66, $41, $0F, $EF, $D8 // pxor xmm3, xmm8 + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + + // ---- rounds 1..8: aesenc, each interleaved with staging one next counter ---- + movdqa xmm8, [r8 + 16] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + mov rax, r11 + bswap rax + mov [rsp + 0], rax + mov rax, rbx + bswap rax + mov [rsp + 8], rax + add rbx, 1 + adc r11, 0 + + movdqa xmm8, [r8 + 32] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + mov rax, r11 + bswap rax + mov [rsp + 16], rax + mov rax, rbx + bswap rax + mov [rsp + 24], rax + add rbx, 1 + adc r11, 0 + + movdqa xmm8, [r8 + 48] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + mov rax, r11 + bswap rax + mov [rsp + 32], rax + mov rax, rbx + bswap rax + mov [rsp + 40], rax + add rbx, 1 + adc r11, 0 + + movdqa xmm8, [r8 + 64] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + mov rax, r11 + bswap rax + mov [rsp + 48], rax + mov rax, rbx + bswap rax + mov [rsp + 56], rax + add rbx, 1 + adc r11, 0 + + movdqa xmm8, [r8 + 80] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + mov rax, r11 + bswap rax + mov [rsp + 64], rax + mov rax, rbx + bswap rax + mov [rsp + 72], rax + add rbx, 1 + adc r11, 0 + + movdqa xmm8, [r8 + 96] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + mov rax, r11 + bswap rax + mov [rsp + 80], rax + mov rax, rbx + bswap rax + mov [rsp + 88], rax + add rbx, 1 + adc r11, 0 + + movdqa xmm8, [r8 + 112] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + mov rax, r11 + bswap rax + mov [rsp + 96], rax + mov rax, rbx + bswap rax + mov [rsp + 104], rax + add rbx, 1 + adc r11, 0 + + movdqa xmm8, [r8 + 128] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + mov rax, r11 + bswap rax + mov [rsp + 112], rax + mov rax, rbx + bswap rax + mov [rsp + 120], rax + add rbx, 1 + adc r11, 0 + + // ---- round 9 (all key sizes): aesenc, no more staging ---- + movdqa xmm8, [r8 + 144] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + +{$IF DEFINED(CRYPTOLIB_AESNI_KEY192) OR DEFINED(CRYPTOLIB_AESNI_KEY256)} + movdqa xmm8, [r8 + 160] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + movdqa xmm8, [r8 + 176] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 +{$IFEND} + +{$IFDEF CRYPTOLIB_AESNI_KEY256} + movdqa xmm8, [r8 + 192] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 + movdqa xmm8, [r8 + 208] + db $66, $41, $0F, $38, $DC, $C0 // aesenc xmm0, xmm8 + db $66, $41, $0F, $38, $DC, $C8 // aesenc xmm1, xmm8 + db $66, $41, $0F, $38, $DC, $D0 // aesenc xmm2, xmm8 + db $66, $41, $0F, $38, $DC, $D8 // aesenc xmm3, xmm8 + db $66, $41, $0F, $38, $DC, $E0 // aesenc xmm4, xmm8 + db $66, $41, $0F, $38, $DC, $E8 // aesenc xmm5, xmm8 + db $66, $41, $0F, $38, $DC, $F0 // aesenc xmm6, xmm8 + db $66, $41, $0F, $38, $DC, $F8 // aesenc xmm7, xmm8 +{$ENDIF} + + // ---- final round: aesenclast on the last round key ---- +{$IFDEF CRYPTOLIB_AESNI_KEY128} + movdqa xmm8, [r8 + 160] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY192} + movdqa xmm8, [r8 + 192] +{$ENDIF} +{$IFDEF CRYPTOLIB_AESNI_KEY256} + movdqa xmm8, [r8 + 224] +{$ENDIF} + db $66, $41, $0F, $38, $DD, $C0 // aesenclast xmm0, xmm8 + db $66, $41, $0F, $38, $DD, $C8 // aesenclast xmm1, xmm8 + db $66, $41, $0F, $38, $DD, $D0 // aesenclast xmm2, xmm8 + db $66, $41, $0F, $38, $DD, $D8 // aesenclast xmm3, xmm8 + db $66, $41, $0F, $38, $DD, $E0 // aesenclast xmm4, xmm8 + db $66, $41, $0F, $38, $DD, $E8 // aesenclast xmm5, xmm8 + db $66, $41, $0F, $38, $DD, $F0 // aesenclast xmm6, xmm8 + db $66, $41, $0F, $38, $DD, $F8 // aesenclast xmm7, xmm8 + + // ---- keystream ^ plaintext -> output (xmm8 = unaligned plaintext load) ---- + movdqu xmm8, [rcx] + db $66, $41, $0F, $EF, $C0 // pxor xmm0, xmm8 + movdqu [rdx], xmm0 + movdqu xmm8, [rcx + 16] + db $66, $41, $0F, $EF, $C8 // pxor xmm1, xmm8 + movdqu [rdx + 16], xmm1 + movdqu xmm8, [rcx + 32] + db $66, $41, $0F, $EF, $D0 // pxor xmm2, xmm8 + movdqu [rdx + 32], xmm2 + movdqu xmm8, [rcx + 48] + db $66, $41, $0F, $EF, $D8 // pxor xmm3, xmm8 + movdqu [rdx + 48], xmm3 + movdqu xmm8, [rcx + 64] + db $66, $41, $0F, $EF, $E0 // pxor xmm4, xmm8 + movdqu [rdx + 64], xmm4 + movdqu xmm8, [rcx + 80] + db $66, $41, $0F, $EF, $E8 // pxor xmm5, xmm8 + movdqu [rdx + 80], xmm5 + movdqu xmm8, [rcx + 96] + db $66, $41, $0F, $EF, $F0 // pxor xmm6, xmm8 + movdqu [rdx + 96], xmm6 + movdqu xmm8, [rcx + 112] + db $66, $41, $0F, $EF, $F8 // pxor xmm7, xmm8 + movdqu [rdx + 112], xmm7 + + // ---- load the staged next-batch counters into xmm0..7 ---- + movdqu xmm0, [rsp] + movdqu xmm1, [rsp + 16] + movdqu xmm2, [rsp + 32] + movdqu xmm3, [rsp + 48] + movdqu xmm4, [rsp + 64] + movdqu xmm5, [rsp + 80] + movdqu xmm6, [rsp + 96] + movdqu xmm7, [rsp + 112] + + add rcx, 128 + add rdx, 128 + sub r10, 8 + jnz @@CtrEightLoop + + // ---- undo the one-batch build-ahead, write the counter back (big-endian) ---- + sub rbx, 8 + sbb r11, 0 + mov rax, r11 + bswap rax + mov [r9], rax + mov rax, rbx + bswap rax + mov [r9 + 8], rax + + add rsp, 128 +{$I ..\..\..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc} + pop rbx diff --git a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc index a8c9edd5..3f713aca 100644 --- a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc +++ b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_i386.inc @@ -5,17 +5,19 @@ // Skylake+/Zen+) and the PCLMUL port (p5) operate in parallel, hiding // GHASH latency inside the AES critical path. // -// Processes one pipelined batch: AES encrypts eight counter blocks to -// produce a keystream, XORs it with PXorIn to produce POut, and -// concurrently GHASHes the PREVIOUS batch's 8 ciphertext blocks into -// the running authentication state. AES inside GCM is always CTR -// keystream construction; the Pascal wrapper picks which buffers -// become PXorIn / POut / PPrevCipher for encrypt vs decrypt. The 8-xmm -// budget does not fit a single-pass 8-wide layout, so the kernel -// processes the batch as two back-to-back 4-block halves sharing the -// same running Z0/Z1/Z2 accumulators; each half packs 4 GHASH iters -// into AES rounds 1..4 to keep both ports busy, rounds 5..N-1 are -// AES-only, and the 3-limb reduction runs once after half 2. +// Loops over BatchCount pipelined batches. Each batch AES-encrypts eight +// counter blocks to produce a keystream, XORs it with PXorIn to produce POut, +// and concurrently GHASHes the PREVIOUS batch's 8 ciphertext blocks into the +// running authentication state (the pipeline lags by one batch). AES inside GCM +// is always CTR keystream construction; GhashFromOutput picks which buffer feeds +// GHASH (output on encrypt, input on decrypt). The 8-xmm budget does not fit a +// single-pass 8-wide layout, so the body processes the batch as two back-to-back +// 4-block halves sharing the same running Z0/Z1/Z2 accumulators; each half packs +// 4 GHASH iters into AES rounds 1..4 to keep both ports busy, rounds 5..N-1 are +// AES-only, and the 3-limb reduction runs once after half 2. Only 4 counter +// blocks are live per half, so each half builds its four in-register (eax/edx +// are free there) - no counter scratch buffer is needed. The mode primes batch 0 +// and drains the last. // // Variant selection (set EXACTLY ONE): // GCM_FUSED_AES_ROUNDS_10 -> AES-128 (schedule = 11 x 16 bytes). @@ -24,16 +26,18 @@ // // On entry (after SimdProc1Begin_i386.inc): ebx = PCtx. // -// Context layout (8 pointers = 32 bytes, 4-byte aligned). See -// TGcmFusedBatchCtx in ClpAesNiGcmKernel.pas. -// [ebx + 0] PXorIn : 128-byte input to XOR with keystream -// [ebx + 4] POut : 128-byte output buffer (may alias PXorIn) -// [ebx + 8] PCtrCurr : 8 counter blocks (128 bytes) to encrypt -// [ebx + 12] PPrevCipher : 128 bytes prev-batch ciphertext for GHASH -// [ebx + 16] PRoundKeys : AES encrypt round-key schedule (16-byte aligned) -// [ebx + 20] PHPow128 : H^8..H^1, eight 16-byte limbs (field form) -// [ebx + 24] PFS : 16-byte running GHASH state (in/out, canonical) -// [ebx + 28] PMask : 16-byte PSHUFB byte-reverse mask +// Context layout (TGcmFusedCtx in ClpAesNiGcmKernel.pas; 4-byte fields). +// [ebx + 0] PXorIn : 128-byte input to XOR with keystream (advances) +// [ebx + 4] POut : 128-byte output (advances; may alias PXorIn) +// [ebx + 8] PPrevCipher : prev-batch ciphertext for GHASH (rotated per batch) +// [ebx + 12] PRoundKeys : AES encrypt round-key schedule (16-byte aligned) +// [ebx + 16] PHPow128 : H^8..H^1, eight 16-byte limbs (field form) +// [ebx + 20] PFS : 16-byte running GHASH state (in/out, canonical) +// [ebx + 24] PMask : 16-byte PSHUFB byte-reverse mask +// [ebx + 28] Counter32 : 32-bit GCM counter, last-used value (advances by 8) +// [ebx + 32] PJ0Template : 16-byte J0 (upper 96 bits reused per counter block) +// [ebx + 36] BatchCount : number of 8-block batches to process +// [ebx + 40] GhashFromOutput : 1 = GHASH output (encrypt), 0 = input (decrypt) // // XMM (8/8 used; i386 has only xmm0..xmm7): // xmm0..xmm3 : 4 AES keystream lanes (per half; reloaded between halves) @@ -48,8 +52,8 @@ // [esp + 32..47] Z2 accumulator (updated 8x per batch) // [esp + 48..63] byte-reversed PFS (computed once, folded into iter 0) // -// AES-NI (aesenc / aesenclast) and PCLMULQDQ instructions are db-encoded -// for broad assembler compatibility. +// AES-NI (aesenc / aesenclast), PCLMULQDQ, and PINSRD (counter build) +// instructions are db-encoded for broad assembler compatibility. // ------------------------------------------------------------------- // --- Callee-save prologue. SimdProc1Begin_i386.inc already pushed ebx @@ -60,11 +64,20 @@ push ebp sub esp, 64 + // --- Batch loop: process BatchCount batches, generating GCM counters and + // rotating the GHASH pipeline in-asm. i386 has no spare GP registers, so + // the loop state lives in the context: the 32-bit counter [ebx+28], the + // J0 template [ebx+32], the batch count [ebx+36], and the + // "GHASH-from-output" flag [ebx+40] (1=encrypt). Each batch's 8 counter + // blocks are built in-register, 4 per half, right before that half runs + // (see the per-half counter builds below). --- +@@GcmFusedLoop: + // --- Load context pointers. --- - mov edi, [ebx + 16] // edi = PRoundKeys (pinned) - mov esi, [ebx + 12] // esi = PPrevCipher (pinned base) - mov ebp, [ebx + 20] // ebp = PHPow128 (pinned base) - mov ecx, [ebx + 28] // ecx = PMask (pinned) + mov edi, [ebx + 12] // edi = PRoundKeys (pinned) + mov esi, [ebx + 8] // esi = PPrevCipher (pinned base) + mov ebp, [ebx + 16] // ebp = PHPow128 (pinned base) + mov ecx, [ebx + 24] // ecx = PMask (pinned) // --- Zero Z0 / Z1 / Z2 accumulator slots. --- pxor xmm0, xmm0 @@ -73,18 +86,35 @@ movdqu [esp + 32], xmm0 // Z2 = 0 // --- Compute byte-reversed running state once, stash for iter-0 fold. --- - mov eax, [ebx + 24] // eax = PFS + mov eax, [ebx + 20] // eax = PFS movdqu xmm0, [eax] // load canonical PFS movdqu xmm7, [ecx] // mask pshufb xmm0, xmm7 // byte-reverse movdqu [esp + 48], xmm0 // stash byte-reversed PFS - // --- Load 4 counter blocks for HALF 1 (blocks 0..3). --- - mov eax, [ebx + 8] // eax = PCtrCurr - movdqu xmm0, [eax + 0] - movdqu xmm1, [eax + 16] - movdqu xmm2, [eax + 32] - movdqu xmm3, [eax + 48] + // --- Build 4 counter blocks for HALF 1 (blocks 0..3): J0[0..11] || + // BE(counter32+1..+4). GCM's counter is the last-used value, so the batch + // based at counter32 spans counter32+1..counter32+8 (matching the scalar + // GetNextCtrBlocks8). Load J0 into each lane, then PINSRD (db-encoded) the + // big-endian counter word into dword 3; eax/edx are free here. --- + mov eax, [ebx + 32] // PJ0Template + movdqu xmm0, [eax] + movdqu xmm1, [eax] + movdqu xmm2, [eax] + movdqu xmm3, [eax] + mov eax, [ebx + 28] // counter32 + lea edx, [eax + 1] + bswap edx + db $66, $0F, $3A, $22, $C2, $03 // pinsrd xmm0, edx, 3 + lea edx, [eax + 2] + bswap edx + db $66, $0F, $3A, $22, $CA, $03 // pinsrd xmm1, edx, 3 + lea edx, [eax + 3] + bswap edx + db $66, $0F, $3A, $22, $D2, $03 // pinsrd xmm2, edx, 3 + lea edx, [eax + 4] + bswap edx + db $66, $0F, $3A, $22, $DA, $03 // pinsrd xmm3, edx, 3 // ====================================================================== // HALF 1: blocks 0..3, GHASH iters 0..3 (prev_cipher[0..3], H^8..H^5) @@ -340,12 +370,26 @@ // no state fold (already consumed by iter 0 of half 1). // ====================================================================== - // --- Load 4 counter blocks for half 2 (blocks 4..7). --- - mov eax, [ebx + 8] // eax = PCtrCurr - movdqu xmm0, [eax + 64] - movdqu xmm1, [eax + 80] - movdqu xmm2, [eax + 96] - movdqu xmm3, [eax + 112] + // --- Build 4 counter blocks for half 2 (blocks 4..7): counter32+5..+8. The + // pinned regs (edi/esi/ebp/ecx) are untouched; eax/edx are free here. --- + mov eax, [ebx + 32] // PJ0Template + movdqu xmm0, [eax] + movdqu xmm1, [eax] + movdqu xmm2, [eax] + movdqu xmm3, [eax] + mov eax, [ebx + 28] // counter32 + lea edx, [eax + 5] + bswap edx + db $66, $0F, $3A, $22, $C2, $03 // pinsrd xmm0, edx, 3 + lea edx, [eax + 6] + bswap edx + db $66, $0F, $3A, $22, $CA, $03 // pinsrd xmm1, edx, 3 + lea edx, [eax + 7] + bswap edx + db $66, $0F, $3A, $22, $D2, $03 // pinsrd xmm2, edx, 3 + lea edx, [eax + 8] + bswap edx + db $66, $0F, $3A, $22, $DA, $03 // pinsrd xmm3, edx, 3 // --- Round 0: AddRoundKey. --- pxor xmm0, [edi + 0] @@ -663,9 +707,24 @@ por xmm0, xmm6 movdqu xmm7, [ecx] // mask reload pshufb xmm0, xmm7 // byte-reverse -> canonical - mov eax, [ebx + 24] // PFS + mov eax, [ebx + 20] // PFS movdqu [eax], xmm0 + // --- Advance the context for the next batch, then loop. PPrevCipher becomes + // this batch's ciphertext (output on encrypt, input on decrypt); the + // counter advances by 8 in place so the caller reads the final value. --- + mov eax, [ebx + 4] // POut (encrypt: prev = output) + cmp dword ptr [ebx + 40], 0 // GHASH-from-output flag (1=encrypt) + jne @@GcmPrevSet + mov eax, [ebx + 0] // PXorIn (decrypt: prev = input) +@@GcmPrevSet: + mov [ebx + 8], eax // PPrevCipher = this batch's ciphertext + add dword ptr [ebx + 0], 128 // PXorIn += 128 + add dword ptr [ebx + 4], 128 // POut += 128 + add dword ptr [ebx + 28], 8 // counter32 += 8 + dec dword ptr [ebx + 36] // batch count + jnz @@GcmFusedLoop + // --- Epilogue: unwind stack scratch and pop callee-saved GPRs. ebx // is popped last because SimdProc1Begin_i386.inc pushed it first. --- add esp, 64 diff --git a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc index fb9fa667..13c6d9a6 100644 --- a/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Aes/Gcm/AesGcmFusedCtrGhashEight_x86_64.inc @@ -5,14 +5,15 @@ // Skylake+/Zen+) and the PCLMUL port (p5) operate in parallel, hiding // GHASH latency inside the AES critical path. // -// Processes one pipelined batch: AES encrypts eight counter blocks to -// produce a keystream, XORs it with PXorIn to produce POut, and -// concurrently GHASHes the PREVIOUS batch's 8 ciphertext blocks into -// the running authentication state. AES inside GCM is always CTR -// keystream construction; the Pascal wrapper picks which buffers -// become PXorIn / POut / PPrevCipher for encrypt vs decrypt. Rounds -// 0..8 (with all 8 GHASH iters) are shared across key sizes; only -// rounds 9..N-1 (extra aesenc) and round N (aesenclast) depend on N. +// Loops over BatchCount pipelined batches. Each batch AES-encrypts eight +// counter blocks (generated in-register from Counter32 + PJ0Template) to +// produce a keystream, XORs it with PXorIn to produce POut, and concurrently +// GHASHes the PREVIOUS batch's 8 ciphertext blocks into the running +// authentication state (the pipeline lags by one batch). AES inside GCM is +// always CTR keystream construction; GhashFromOutput picks which buffer feeds +// GHASH (output on encrypt, input on decrypt). Rounds 0..8 (with all 8 GHASH +// iters) are shared across key sizes; only rounds 9..N-1 (extra aesenc) and +// round N (aesenclast) depend on N. The mode primes batch 0 and drains the last. // // Variant selection (set EXACTLY ONE): // GCM_FUSED_AES_ROUNDS_10 -> AES-128 (schedule = 11 x 16 bytes). @@ -21,16 +22,18 @@ // // On entry (after SimdProc1Begin_x86_64.inc): rcx = PCtx. // -// Context layout (8 pointers = 64 bytes, 8-byte aligned). See -// TGcmFusedBatchCtx in ClpAesNiGcmKernel.pas. -// [rcx + 0] PXorIn : 128-byte input to XOR with keystream -// [rcx + 8] POut : 128-byte output buffer (may alias PXorIn) -// [rcx + 16] PCtrCurr : 8 counter blocks (128 bytes) to encrypt -// [rcx + 24] PPrevCipher : 128 bytes prev-batch ciphertext for GHASH -// [rcx + 32] PRoundKeys : AES encrypt round-key schedule (16-byte aligned) -// [rcx + 40] PHPow128 : H^8..H^1, eight 16-byte limbs (field form) -// [rcx + 48] PFS : 16-byte running GHASH state (in/out, canonical) -// [rcx + 56] PMask : 16-byte PSHUFB byte-reverse mask +// Context layout (TGcmFusedCtx in ClpAesNiGcmKernel.pas; 8-byte fields). +// [rcx + 0] PXorIn : 128-byte input to XOR with keystream (advances) +// [rcx + 8] POut : 128-byte output (advances; may alias PXorIn) +// [rcx + 16] PPrevCipher : prev-batch ciphertext for GHASH (rotated per batch) +// [rcx + 24] PRoundKeys : AES encrypt round-key schedule (16-byte aligned) +// [rcx + 32] PHPow128 : H^8..H^1, eight 16-byte limbs (field form) +// [rcx + 40] PFS : 16-byte running GHASH state (in/out, canonical) +// [rcx + 48] PMask : 16-byte PSHUFB byte-reverse mask +// [rcx + 56] Counter32 : 32-bit GCM counter, last-used value (advances by 8) +// [rcx + 64] PJ0Template : 16-byte J0 (upper 96 bits reused per counter block) +// [rcx + 72] BatchCount : number of 8-block batches to process +// [rcx + 80] GhashFromOutput : 1 = GHASH output (encrypt), 0 = input (decrypt) // // XMM (16/16 used): // xmm0..xmm7 : 8 AES keystream lanes (persistent across rounds) @@ -47,23 +50,57 @@ // ------------------------------------------------------------------- {$I ..\..\..\Include\Simd\Common\SimdNonVolatileSave_x86_64.inc} - // --- Prologue: load pointers and initial state --------------------- - mov rax, [rcx + 16] // rax = PCtrCurr - movdqu xmm0, [rax] - movdqu xmm1, [rax + 16] - movdqu xmm2, [rax + 32] - movdqu xmm3, [rax + 48] - movdqu xmm4, [rax + 64] - movdqu xmm5, [rax + 80] - movdqu xmm6, [rax + 96] - movdqu xmm7, [rax + 112] - - mov rax, [rcx + 56] // rax = PMask + // --- Batch loop. The context (rcx) carries advancing PXorIn/POut/PPrevCipher, + // the 32-bit GCM counter [rcx+64], the J0 template [rcx+72], the batch + // count [rcx+80], and the "GHASH-from-output" flag [rcx+88] (1=encrypt). + // r8d = counter32, r9 = J0 template (persist across the loop; both are + // volatile and untouched by the shared body). --- + mov r8d, [rcx + 56] // counter32 + mov r9, [rcx + 64] // PJ0Template +@@GcmFusedLoop: + // Build 8 counter blocks into xmm0..7: block j = J0[0..11] || BE(counter32+j+1). + // GCM's 32-bit counter holds the last-used value, so the batch based at + // counter32 spans counter32+1..counter32+8 (matching FillCtr8BlocksRaw); only + // bytes 12-15 vary. PINSRD is db-encoded. + movdqu xmm0, [r9] + lea eax, [r8 + 1] + bswap eax + db $66, $0F, $3A, $22, $C0, $03 // pinsrd xmm0, eax, 3 + movdqu xmm1, [r9] + lea eax, [r8 + 2] + bswap eax + db $66, $0F, $3A, $22, $C8, $03 // pinsrd xmm1, eax, 3 + movdqu xmm2, [r9] + lea eax, [r8 + 3] + bswap eax + db $66, $0F, $3A, $22, $D0, $03 // pinsrd xmm2, eax, 3 + movdqu xmm3, [r9] + lea eax, [r8 + 4] + bswap eax + db $66, $0F, $3A, $22, $D8, $03 // pinsrd xmm3, eax, 3 + movdqu xmm4, [r9] + lea eax, [r8 + 5] + bswap eax + db $66, $0F, $3A, $22, $E0, $03 // pinsrd xmm4, eax, 3 + movdqu xmm5, [r9] + lea eax, [r8 + 6] + bswap eax + db $66, $0F, $3A, $22, $E8, $03 // pinsrd xmm5, eax, 3 + movdqu xmm6, [r9] + lea eax, [r8 + 7] + bswap eax + db $66, $0F, $3A, $22, $F0, $03 // pinsrd xmm6, eax, 3 + movdqu xmm7, [r9] + lea eax, [r8 + 8] + bswap eax + db $66, $0F, $3A, $22, $F8, $03 // pinsrd xmm7, eax, 3 + + mov rax, [rcx + 48] // rax = PMask movdqu xmm15, [rax] // xmm15 = byte-reverse mask - mov rdx, [rcx + 32] // rdx = PRoundKeys - mov r10, [rcx + 24] // r10 = PPrevCipher - mov r11, [rcx + 40] // r11 = PHPow128 + mov rdx, [rcx + 24] // rdx = PRoundKeys + mov r10, [rcx + 16] // r10 = PPrevCipher + mov r11, [rcx + 32] // r11 = PHPow128 // --- Round 0: AddRoundKey + zero Z0/Z1/Z2 ---------------------------- movdqa xmm8, [rdx] // rk[0] @@ -85,7 +122,7 @@ // GHASH iter 0 setup: load block 0, byte-reverse, fold running state movdqu xmm12, [r10] // prev cipher block 0 pshufb xmm12, xmm15 // byte-reverse - mov rax, [rcx + 48] // rax = PFS + mov rax, [rcx + 40] // rax = PFS movdqu xmm14, [rax] // load running state pshufb xmm14, xmm15 // byte-reverse state db $66, $45, $0F, $EF, $E6 // pxor xmm12, xmm14 (fold) @@ -534,6 +571,21 @@ pslldq xmm6, 8 por xmm0, xmm6 // xmm0 = [LZ0 | LZ1] pshufb xmm0, xmm15 // byte-reverse -> canonical - mov rax, [rcx + 48] // PFS + mov rax, [rcx + 40] // PFS movdqu [rax], xmm0 + + // --- Advance the context for the next batch, then loop. PPrevCipher becomes + // this batch's ciphertext (output on encrypt, input on decrypt). --- + mov rax, [rcx + 8] // POut (out[i]); encrypt: prev = output + cmp qword ptr [rcx + 80], 0 // GHASH-from-output flag (1 = encrypt) + jne @@GcmPrevSet + mov rax, [rcx + 0] // PXorIn (in[i]); decrypt: prev = input +@@GcmPrevSet: + mov [rcx + 16], rax // PPrevCipher = this batch's ciphertext + add qword ptr [rcx + 0], 128 // PXorIn += 128 + add qword ptr [rcx + 8], 128 // POut += 128 + add r8d, 8 // counter32 += 8 + dec qword ptr [rcx + 72] // batch count + jnz @@GcmFusedLoop + mov [rcx + 56], r8d // store advanced counter32 {$I ..\..\..\Include\Simd\Common\SimdNonVolatileRestore_x86_64.inc} diff --git a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc index f6cc1ce0..8ac411f5 100644 --- a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc +++ b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_i386.inc @@ -7,9 +7,10 @@ // scratch are reserved; memory-form aesenc / aesdec is therefore // used throughout against the 16-byte-aligned round-key schedule. // The kernel owns all per-block OCB bookkeeping across the batch: -// on entry it seeds xmm_offset from OffsetPtr and xmm_checksum from -// ChecksumPtr; the loop walks a pre-computed ntz byte array, XORs -// the L-table entry into xmm_offset to derive each Delta, runs +// on entry it seeds xmm_offset from OffsetPtr, xmm_checksum from +// ChecksumPtr and a running block counter from StartBlockCount; the +// loop derives ntz per block itself (bsf on the incremented counter), +// XORs the L-table entry into xmm_offset to derive each Delta, runs // P_i XOR delta XOR K_0 through the AES rounds, XORs delta into // the post-aesenclast state to produce C_i, and folds plaintext // into xmm_checksum; on exit it writes xmm_offset and xmm_checksum @@ -21,7 +22,7 @@ // // On entry (after SimdProc1Begin_i386.inc): ebx = PCtx. // -// Context layout (8 pointers + 1 NativeUInt = 36 bytes, 4-byte aligned). +// Context layout (7 pointers + NativeUInt + UInt64 = 40 bytes, 4-byte aligned). // See TOcbFusedKernelCtx in ClpAesNiOcbKernel.pas. // [ebx + 0] Keys : AES schedule (encrypt or inv-MixColumns; 16-byte aligned) // [ebx + 4] InPtr : base of the main 16-byte-block stream @@ -29,9 +30,10 @@ // [ebx + 12] OffsetPtr : 16-byte live FOffsetMAIN (r/w) // [ebx + 16] ChecksumPtr : 16-byte live FChecksum (r/w) // [ebx + 20] LTablePtr : flat L[0..LMax] * 16 bytes (read-only; cached on stack) -// [ebx + 24] NtzPtr : BlockCount bytes of pre-computed ntz values -// [ebx + 28] Block0Ptr : 16-byte source of iteration-0 block 0 (may alias InPtr) -// [ebx + 32] BlockCount : NativeUInt; multiple of 4 +// [ebx + 24] Block0Ptr : 16-byte source of iteration-0 block 0 (may alias InPtr) +// [ebx + 28] BlockCount : NativeUInt; multiple of 4 +// [ebx + 32] StartBlockCount : UInt64; block count before this span (ntz counter seed). +// lo at +32, hi at +36. // // XMM (7/8 used; i386 has only xmm0..xmm7): // xmm0..xmm3 : 4 data lanes @@ -40,9 +42,15 @@ // xmm6 : scratch (L-table loads; stack-offset reloads during whitening) // xmm7 : free // -// Stack scratch (68 bytes, base unaligned - movdqu only): +// GPR: ebx=PCtx, edi=Keys, edx=InPtr, esi=OutPtr, ecx=BlockCount, +// ebp=block counter low dword (bsf source), eax=scratch. The counter +// high dword lives on the stack (only touched on a 2^32 low-dword wrap), +// so the full 64-bit ntz is exact yet the hot path stays register-only. +// +// Stack scratch (72 bytes, base unaligned - movdqu only): // [esp + 0..63] 4 x 16-byte per-block offset stashes // [esp + 64..67] LTablePtr cache (one dword; GPR budget cannot pin it) +// [esp + 68..71] block counter high dword // // AES-NI instructions are db-encoded for broad assembler compatibility. // ------------------------------------------------------------------- @@ -53,14 +61,19 @@ push edi push esi push ebp - sub esp, 68 + sub esp, 72 // --- Load context pointers and seed accumulators. --- mov edi, [ebx + 0] // edi = Keys (pinned) mov edx, [ebx + 4] // edx = InPtr (advancing) mov esi, [ebx + 8] // esi = OutPtr (advancing) - mov ebp, [ebx + 24] // ebp = NtzPtr (advancing) - mov ecx, [ebx + 32] // ecx = BlockCount (>= 4, multiple of 4) + mov ecx, [ebx + 28] // ecx = BlockCount (>= 4, multiple of 4) + + // Seed the running block counter (64-bit) from StartBlockCount: low + // dword in ebp (bsf source), high dword in the [esp + 68] stack slot. + mov ebp, [ebx + 32] // ebp = StartBlockCount low + mov eax, [ebx + 36] // eax = StartBlockCount high + mov [esp + 68], eax // Stash LTablePtr into the stack slot at [esp + 64]; the inner loop // re-adds it to ntz*16 to form each L-table entry address. @@ -77,7 +90,7 @@ // of each iteration (before Phase 1 clobbers it), and refreshed to // edx at the bottom of every iteration so iter >= 1 transparently // reads block 0 from the advancing main stream at offset 0. - mov eax, [ebx + 28] // eax = Block0Ptr + mov eax, [ebx + 24] // eax = Block0Ptr @@ocb_i386_loop: @@ -94,38 +107,65 @@ // ------------------------------------------------------------------ // Phase 1: offset ladder. For each block i in 0..3: - // eax = ntz[i] (movzx from [ebp + i]) + // advance the 64-bit block counter (ebp low + [esp+68] high) by one + // eax = ntz(counter) (bsf ebp; on a low-dword wrap, 32 + bsf hi) // eax = eax * 16 (shl eax, 4) // eax = LTablePtr + eax (add from stashed slot) - // xmm6 = L[ntz[i]] (movdqu from [eax]) + // xmm6 = L[ntz] (movdqu from [eax]) // xmm4 ^= xmm6 (pxor) // [esp + 16*i] <- xmm4 (movdqu stash of offset_i) // After this phase xmm4 holds offset_3 which is the Delta carried // into the next iteration, and [esp + 0..63] holds offsets_0..3. + // `add ebp, 1` (not inc) sets CF so the rare high-dword carry + // propagates; the bsf-high branch is essentially never taken. // ------------------------------------------------------------------ // Block 0 - movzx eax, byte [ebp + 0] + add ebp, 1 + adc dword [esp + 68], 0 + bsf eax, ebp + jnz @@ocb_i386_ntz0 + bsf eax, [esp + 68] + add eax, 32 +@@ocb_i386_ntz0: shl eax, 4 add eax, [esp + 64] movdqu xmm6, [eax] pxor xmm4, xmm6 movdqu [esp + 0], xmm4 // Block 1 - movzx eax, byte [ebp + 1] + add ebp, 1 + adc dword [esp + 68], 0 + bsf eax, ebp + jnz @@ocb_i386_ntz1 + bsf eax, [esp + 68] + add eax, 32 +@@ocb_i386_ntz1: shl eax, 4 add eax, [esp + 64] movdqu xmm6, [eax] pxor xmm4, xmm6 movdqu [esp + 16], xmm4 // Block 2 - movzx eax, byte [ebp + 2] + add ebp, 1 + adc dword [esp + 68], 0 + bsf eax, ebp + jnz @@ocb_i386_ntz2 + bsf eax, [esp + 68] + add eax, 32 +@@ocb_i386_ntz2: shl eax, 4 add eax, [esp + 64] movdqu xmm6, [eax] pxor xmm4, xmm6 movdqu [esp + 32], xmm4 // Block 3 - movzx eax, byte [ebp + 3] + add ebp, 1 + adc dword [esp + 68], 0 + bsf eax, ebp + jnz @@ocb_i386_ntz3 + bsf eax, [esp + 68] + add eax, 32 +@@ocb_i386_ntz3: shl eax, 4 add eax, [esp + 64] movdqu xmm6, [eax] @@ -383,10 +423,10 @@ movdqu [esi + 32], xmm2 movdqu [esi + 48], xmm3 - // --- Advance pointers and decrement loop counter. --- + // --- Advance pointers and decrement loop counter. The block counter + // (ebp + [esp+68]) is advanced per block inside the ladder above. --- add edx, 64 add esi, 64 - add ebp, 4 mov eax, edx // next iter's block-0 source // tracks the main stream sub ecx, 4 @@ -400,7 +440,7 @@ mov eax, [ebx + 16] // eax = ChecksumPtr movdqu [eax], xmm5 - add esp, 68 + add esp, 72 pop ebp pop esi pop edi diff --git a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc index 8c75f1e2..ae4b95ca 100644 --- a/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Aes/Ocb/AesOcbFusedWide_x86_64.inc @@ -4,8 +4,9 @@ // Processes ABlockCount consecutive 16-byte OCB body blocks in chunks // of 8 blocks (128 bytes) per iteration. The kernel owns all per-block // OCB bookkeeping across the batch: on entry it seeds xmm_offset from -// OffsetPtr and xmm_checksum from ChecksumPtr; the loop walks a -// pre-computed ntz byte array, XORs the L-table entry into xmm_offset +// OffsetPtr and xmm_checksum from ChecksumPtr and a running block counter +// from StartBlockCount; the loop derives ntz per block in-register (bsf on +// the incremented counter), XORs the L-table entry into xmm_offset // to derive each Delta, runs P_i XOR delta XOR K_0 through the AES // rounds, XORs delta into the post-aesenclast state to produce C_i, // and folds plaintext into xmm_checksum; on exit it writes xmm_offset @@ -17,17 +18,17 @@ // // On entry (after SimdProc1Begin_x86_64.inc): rcx = PCtx. // -// Context layout (8 pointers + 1 NativeUInt = 72 bytes, 8-byte aligned). +// Context layout (7 pointers + NativeUInt + UInt64 = 72 bytes, 8-byte aligned). // See TOcbFusedKernelCtx in ClpAesNiOcbKernel.pas. -// [rcx + 0] Keys : AES schedule (encrypt or inv-MixColumns; 16-byte aligned) -// [rcx + 8] InPtr : base of the main 16-byte-block stream -// [rcx + 16] OutPtr : base of BlockCount output blocks (may alias InPtr) -// [rcx + 24] OffsetPtr : 16-byte live FOffsetMAIN (r/w) -// [rcx + 32] ChecksumPtr : 16-byte live FChecksum (r/w) -// [rcx + 40] LTablePtr : flat L[0..LMax] * 16 bytes (read-only) -// [rcx + 48] NtzPtr : BlockCount bytes of pre-computed ntz values -// [rcx + 56] Block0Ptr : 16-byte source of iteration-0 block 0 (may alias InPtr) -// [rcx + 64] BlockCount : NativeUInt; multiple of 8 +// [rcx + 0] Keys : AES schedule (encrypt or inv-MixColumns; 16-byte aligned) +// [rcx + 8] InPtr : base of the main 16-byte-block stream +// [rcx + 16] OutPtr : base of BlockCount output blocks (may alias InPtr) +// [rcx + 24] OffsetPtr : 16-byte live FOffsetMAIN (r/w) +// [rcx + 32] ChecksumPtr : 16-byte live FChecksum (r/w) +// [rcx + 40] LTablePtr : flat L[0..LMax] * 16 bytes (read-only) +// [rcx + 48] Block0Ptr : 16-byte source of iteration-0 block 0 (may alias InPtr) +// [rcx + 56] BlockCount : NativeUInt; multiple of 8 +// [rcx + 64] StartBlockCount : UInt64; block count before this span (ntz counter seed) // // XMM (12/16 used): // xmm0..xmm7 : 8 data lanes @@ -55,8 +56,10 @@ mov rbx, [rcx + 8] // rbx = InPtr mov r8, [rcx + 16] // r8 = OutPtr mov r9, [rcx + 40] // r9 = LTablePtr - mov r11, [rcx + 48] // r11 = NtzPtr - mov r10, [rcx + 64] // r10 = BlockCount + mov r11, [rcx + 64] // r11 = running block counter + // (seeded from StartBlockCount); + // ntz derived per block via bsf + mov r10, [rcx + 56] // r10 = BlockCount shr r10, 3 // r10 /= 8 (>= 1) // --- Seed the persistent accumulators from memory (one-shot). @@ -72,7 +75,7 @@ // every iteration's bottom refreshes it to rbx (the main // stream advance) so iter >= 1 loads block 0 from the // normal +0 offset transparently. --- - mov rax, [rcx + 56] // rax = Block0Ptr + mov rax, [rcx + 48] // rax = Block0Ptr @@ocb_x64_loop: @@ -111,49 +114,57 @@ // movdqa [rsp + 16*i], xmm8 ; stash offset_i (db) // ------------------------------------------------------------------ // Block 0 - movzx eax, byte [r11 + 0] + inc r11 + bsf rax, r11 // rax = ntz(block counter) shl rax, 4 movdqu xmm11, [r9 + rax] db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 db $66, $44, $0F, $7F, $44, $24, $00 // movdqa [rsp+0], xmm8 // Block 1 - movzx eax, byte [r11 + 1] + inc r11 + bsf rax, r11 // rax = ntz(block counter) shl rax, 4 movdqu xmm11, [r9 + rax] db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 db $66, $44, $0F, $7F, $44, $24, $10 // movdqa [rsp+16], xmm8 // Block 2 - movzx eax, byte [r11 + 2] + inc r11 + bsf rax, r11 // rax = ntz(block counter) shl rax, 4 movdqu xmm11, [r9 + rax] db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 db $66, $44, $0F, $7F, $44, $24, $20 // movdqa [rsp+32], xmm8 // Block 3 - movzx eax, byte [r11 + 3] + inc r11 + bsf rax, r11 // rax = ntz(block counter) shl rax, 4 movdqu xmm11, [r9 + rax] db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 db $66, $44, $0F, $7F, $44, $24, $30 // movdqa [rsp+48], xmm8 // Block 4 - movzx eax, byte [r11 + 4] + inc r11 + bsf rax, r11 // rax = ntz(block counter) shl rax, 4 movdqu xmm11, [r9 + rax] db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 db $66, $44, $0F, $7F, $44, $24, $40 // movdqa [rsp+64], xmm8 // Block 5 - movzx eax, byte [r11 + 5] + inc r11 + bsf rax, r11 // rax = ntz(block counter) shl rax, 4 movdqu xmm11, [r9 + rax] db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 db $66, $44, $0F, $7F, $44, $24, $50 // movdqa [rsp+80], xmm8 // Block 6 - movzx eax, byte [r11 + 6] + inc r11 + bsf rax, r11 // rax = ntz(block counter) shl rax, 4 movdqu xmm11, [r9 + rax] db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 db $66, $44, $0F, $7F, $44, $24, $60 // movdqa [rsp+96], xmm8 // Block 7 - movzx eax, byte [r11 + 7] + inc r11 + bsf rax, r11 // rax = ntz(block counter) shl rax, 4 movdqu xmm11, [r9 + rax] db $66, $45, $0F, $EF, $C3 // pxor xmm8, xmm11 @@ -602,7 +613,6 @@ // --- Advance pointers and decrement loop counter. --- add rbx, 128 add r8, 128 - add r11, 8 mov rax, rbx // next iter's block-0 source // tracks the main stream dec r10 diff --git a/CryptoLib/src/Include/Simd/ByteXor/ByteXorSse2_i386.inc b/CryptoLib/src/Include/Simd/ByteXor/ByteXorSse2_i386.inc new file mode 100644 index 00000000..b0b61ae5 --- /dev/null +++ b/CryptoLib/src/Include/Simd/ByteXor/ByteXorSse2_i386.inc @@ -0,0 +1,34 @@ +// SSE2 variable-length XOR utility (i386). +// After SimdProc4Begin_i386.inc: ebx = ALen, esi = PX, edi = PY, eax = PZ. +// Computes PZ[i] := PX[i] xor PY[i] for i in [0, ALen). PZ may alias PX or PY. +// 16-byte movdqu/pxor chunks, then a byte tail (dl scratch). Callee-saved +// EBX/ESI/EDI are popped in the epilogue below (pushed by SimdProc4Begin_i386). +// +@@ByteXorLoop16: + cmp ebx, 16 + jb @@ByteXorTail + movdqu xmm0, [esi] + movdqu xmm1, [edi] + pxor xmm0, xmm1 + movdqu [eax], xmm0 + add esi, 16 + add edi, 16 + add eax, 16 + sub ebx, 16 + jmp @@ByteXorLoop16 +@@ByteXorTail: + test ebx, ebx + jz @@ByteXorDone +@@ByteXorTailLoop: + mov dl, [esi] + xor dl, [edi] + mov [eax], dl + inc esi + inc edi + inc eax + dec ebx + jnz @@ByteXorTailLoop +@@ByteXorDone: + pop edi + pop esi + pop ebx diff --git a/CryptoLib/src/Include/Simd/ByteXor/ByteXorSse2_x86_64.inc b/CryptoLib/src/Include/Simd/ByteXor/ByteXorSse2_x86_64.inc new file mode 100644 index 00000000..831189e8 --- /dev/null +++ b/CryptoLib/src/Include/Simd/ByteXor/ByteXorSse2_x86_64.inc @@ -0,0 +1,31 @@ +// SSE2 variable-length XOR utility (x86-64). +// After SimdProc4Begin_x86_64.inc: rcx = ALen, rdx = PX, r8 = PY, r9 = PZ. +// Computes PZ[i] := PX[i] xor PY[i] for i in [0, ALen). PZ may alias PX or PY. +// 16-byte movdqu/pxor chunks, then a byte tail. Uses xmm0, xmm1, al - all +// volatile on both ABIs, so no register save is needed (leaf, nostackframe). +// +@@ByteXorLoop16: + cmp rcx, 16 + jb @@ByteXorTail + movdqu xmm0, [rdx] + movdqu xmm1, [r8] + pxor xmm0, xmm1 + movdqu [r9], xmm0 + add rdx, 16 + add r8, 16 + add r9, 16 + sub rcx, 16 + jmp @@ByteXorLoop16 +@@ByteXorTail: + test rcx, rcx + jz @@ByteXorDone +@@ByteXorTailLoop: + mov al, [rdx] + xor al, [r8] + mov [r9], al + inc rdx + inc r8 + inc r9 + dec rcx + jnz @@ByteXorTailLoop +@@ByteXorDone: diff --git a/CryptoLib/src/Include/Simd/Gcm/GcmBlockXor128Sse2_i386.inc b/CryptoLib/src/Include/Simd/Gcm/GcmBlockXor128Sse2_i386.inc deleted file mode 100644 index 421f8ecb..00000000 --- a/CryptoLib/src/Include/Simd/Gcm/GcmBlockXor128Sse2_i386.inc +++ /dev/null @@ -1,11 +0,0 @@ -// SSE2 16-byte XOR utility (i386). -// After SimdProc2Begin_i386.inc: ebx = PDst, esi = PSrc. -// Computes [ebx] ^= [esi]. -// Uses xmm0, xmm1. -// - movdqu xmm0, [ebx] - movdqu xmm1, [esi] - pxor xmm0, xmm1 - movdqu [ebx], xmm0 - pop esi - pop ebx diff --git a/CryptoLib/src/Include/Simd/Gcm/GcmBlockXor128Sse2_x86_64.inc b/CryptoLib/src/Include/Simd/Gcm/GcmBlockXor128Sse2_x86_64.inc deleted file mode 100644 index 8400f01d..00000000 --- a/CryptoLib/src/Include/Simd/Gcm/GcmBlockXor128Sse2_x86_64.inc +++ /dev/null @@ -1,9 +0,0 @@ -// SSE2 16-byte XOR utility (x86-64). -// After SimdProc2Begin_x86_64.inc: rcx = PDst, rdx = PSrc. -// Computes [rcx] ^= [rdx]. -// Uses xmm0, xmm1. -// - movdqu xmm0, [rcx] - movdqu xmm1, [rdx] - pxor xmm0, xmm1 - movdqu [rcx], xmm0 diff --git a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc index 3b5c6b4d..6ddb692d 100644 --- a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc +++ b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_i386.inc @@ -10,13 +10,20 @@ // registers so the PSHUFB mask cannot stay pinned; it is reloaded // from [eax] on every use. // -// Variant selection (set EXACTLY ONE): +// Variant selection (set EXACTLY ONE of the block counts): // GCM_GHASH_FULL_BLOCKS_4 -> 4-block (64-byte) batch, H^4..H^1. // GCM_GHASH_FULL_BLOCKS_8 -> 8-block (128-byte) batch, H^8..H^1. +// Optional: +// GCM_GHASH_RAW_INPUT -> the data blocks are already in field form, so +// skip the per-block input byte-reverse. Used by the +// GCM-SIV POLYVAL kernel, which supplies raw blocks; +// the running-state load and the final store are +// still byte-reversed. Unset for GCM (canonical +// input), leaving that path byte-identical. // // On entry (after SimdProc4Begin_i386.inc): // ebx = PFS (16-byte running GHASH state, canonical; in/out) -// esi = PC0 (64 or 128 bytes ciphertext, canonical form) +// esi = PC0 (64 or 128 bytes data; canonical, or field form with RAW_INPUT) // edi = PHPow (H^N..H^1 as 16-byte limbs, field form) // eax = PMask (16-byte PSHUFB byte-reverse control) // @@ -37,7 +44,9 @@ // Block 0: byte-reverse + fold running state movdqu xmm3, [esi] movdqu xmm7, [eax] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm7 +{$ENDIF} movdqu xmm5, [ebx] pshufb xmm5, xmm7 pxor xmm3, xmm5 @@ -58,7 +67,9 @@ // Block 1 movdqu xmm3, [esi + 16] movdqu xmm7, [eax] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm7 +{$ENDIF} movdqu xmm4, [edi + 16] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -76,7 +87,9 @@ // Block 2 movdqu xmm3, [esi + 32] movdqu xmm7, [eax] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm7 +{$ENDIF} movdqu xmm4, [edi + 32] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -94,7 +107,9 @@ // Block 3 movdqu xmm3, [esi + 48] movdqu xmm7, [eax] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm7 +{$ENDIF} movdqu xmm4, [edi + 48] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -113,7 +128,9 @@ // Block 4 movdqu xmm3, [esi + 64] movdqu xmm7, [eax] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm7 +{$ENDIF} movdqu xmm4, [edi + 64] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -131,7 +148,9 @@ // Block 5 movdqu xmm3, [esi + 80] movdqu xmm7, [eax] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm7 +{$ENDIF} movdqu xmm4, [edi + 80] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -149,7 +168,9 @@ // Block 6 movdqu xmm3, [esi + 96] movdqu xmm7, [eax] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm7 +{$ENDIF} movdqu xmm4, [edi + 96] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -167,7 +188,9 @@ // Block 7 movdqu xmm3, [esi + 112] movdqu xmm7, [eax] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm7 +{$ENDIF} movdqu xmm4, [edi + 112] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 diff --git a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc index fcd375cb..f263a76f 100644 --- a/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc +++ b/CryptoLib/src/Include/Simd/Gcm/GcmGhashFull_x86_64.inc @@ -11,13 +11,20 @@ // multiply-accumulate, 3-limb reduction, byte-reverse back) with a // single kernel. // -// Variant selection (set EXACTLY ONE): +// Variant selection (set EXACTLY ONE of the block counts): // GCM_GHASH_FULL_BLOCKS_4 -> 4-block (64-byte) batch, H^4..H^1. // GCM_GHASH_FULL_BLOCKS_8 -> 8-block (128-byte) batch, H^8..H^1. +// Optional: +// GCM_GHASH_RAW_INPUT -> the data blocks are already in field form, so +// skip the per-block input byte-reverse. Used by the +// GCM-SIV POLYVAL kernel, which supplies raw blocks; +// the running-state load and the final store are +// still byte-reversed. Unset for GCM (canonical +// input), leaving that path byte-identical. // // On entry (after SimdProc4Begin_x86_64.inc): // rcx = PFS (16-byte running GHASH state, canonical; in/out) -// rdx = PC0 (64 or 128 bytes ciphertext, canonical form) +// rdx = PC0 (64 or 128 bytes data; canonical, or field form with RAW_INPUT) // r8 = PHPow (H^N..H^1 as 16-byte limbs, field form) // r9 = PMask (16-byte PSHUFB byte-reverse control) // @@ -38,7 +45,9 @@ // Block 0: byte-reverse + fold running state movdqu xmm3, [rdx] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 +{$ENDIF} movdqu xmm5, [rcx] pshufb xmm5, xmm8 pxor xmm3, xmm5 @@ -58,7 +67,9 @@ // Block 1 movdqu xmm3, [rdx + 16] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 +{$ENDIF} movdqu xmm4, [r8 + 16] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -75,7 +86,9 @@ // Block 2 movdqu xmm3, [rdx + 32] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 +{$ENDIF} movdqu xmm4, [r8 + 32] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -92,7 +105,9 @@ // Block 3 movdqu xmm3, [rdx + 48] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 +{$ENDIF} movdqu xmm4, [r8 + 48] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -110,7 +125,9 @@ {$IFDEF GCM_GHASH_FULL_BLOCKS_8} // Block 4 movdqu xmm3, [rdx + 64] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 +{$ENDIF} movdqu xmm4, [r8 + 64] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -127,7 +144,9 @@ // Block 5 movdqu xmm3, [rdx + 80] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 +{$ENDIF} movdqu xmm4, [r8 + 80] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -144,7 +163,9 @@ // Block 6 movdqu xmm3, [rdx + 96] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 +{$ENDIF} movdqu xmm4, [r8 + 96] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 @@ -161,7 +182,9 @@ // Block 7 movdqu xmm3, [rdx + 112] +{$IFNDEF GCM_GHASH_RAW_INPUT} pshufb xmm3, xmm8 +{$ENDIF} movdqu xmm4, [r8 + 112] movdqa xmm5, xmm3 db $66, $0F, $3A, $44, $EC, $00 // pclmulqdq xmm5, xmm4, 0 diff --git a/CryptoLib/src/Include/Simd/GcmSiv/PolyvalHornerEight_i386.inc b/CryptoLib/src/Include/Simd/GcmSiv/PolyvalHornerEight_i386.inc index 21917754..e70610b7 100644 --- a/CryptoLib/src/Include/Simd/GcmSiv/PolyvalHornerEight_i386.inc +++ b/CryptoLib/src/Include/Simd/GcmSiv/PolyvalHornerEight_i386.inc @@ -17,10 +17,16 @@ // mulX_GHASH(ByteReverse(H_polyval))). // * PFS (running hash state) is kept in GHASH canonical // (byte-reversed) form across calls. -// * PC0 points to 128 bytes of plaintext in natural memory order; -// the PSHUFB pair inside the delegated body supplies the POLYVAL -// ByteReverse on load and the inverse on final store. +// * PC0 points to 128 bytes of data blocks in natural memory order. The kernel +// folds them to the same field element the scalar POLYVAL path does: the +// scalar path byte-reverses each block (FillReverse) before the GHASH +// multiply, and the kernel reaches the identical operand by consuming the +// raw block with the per-block input byte-reverse skipped +// (GCM_GHASH_RAW_INPUT). The running-state load and final store stay +// byte-reversed via the PMask pshufb. // {$DEFINE GCM_GHASH_FULL_BLOCKS_8} +{$DEFINE GCM_GHASH_RAW_INPUT} {$I ..\..\..\Include\Simd\Gcm\GcmGhashFull_i386.inc} +{$UNDEF GCM_GHASH_RAW_INPUT} {$UNDEF GCM_GHASH_FULL_BLOCKS_8} diff --git a/CryptoLib/src/Include/Simd/GcmSiv/PolyvalHornerEight_x86_64.inc b/CryptoLib/src/Include/Simd/GcmSiv/PolyvalHornerEight_x86_64.inc index a861264c..52eff882 100644 --- a/CryptoLib/src/Include/Simd/GcmSiv/PolyvalHornerEight_x86_64.inc +++ b/CryptoLib/src/Include/Simd/GcmSiv/PolyvalHornerEight_x86_64.inc @@ -15,12 +15,18 @@ // mulX_GHASH(ByteReverse(H_polyval))). // * PFS (running hash state) is kept in GHASH canonical // (byte-reversed) form across calls. -// * PC0 points to 128 bytes of plaintext in natural memory order; -// the PSHUFB pair inside the delegated body supplies the POLYVAL -// ByteReverse on load and the inverse on final store. +// * PC0 points to 128 bytes of data blocks in natural memory order. The kernel +// folds them to the same field element the scalar POLYVAL path does: the +// scalar path byte-reverses each block (FillReverse) before the GHASH +// multiply, and the kernel reaches the identical operand by consuming the +// raw block with the per-block input byte-reverse skipped +// (GCM_GHASH_RAW_INPUT). The running-state load and final store stay +// byte-reversed via the PMask pshufb. // // Nonvolatile xmm6-15 save/restore is inherited from the delegated GcmGhashFull // body; this shell adds no asm of its own. {$DEFINE GCM_GHASH_FULL_BLOCKS_8} +{$DEFINE GCM_GHASH_RAW_INPUT} {$I ..\..\..\Include\Simd\Gcm\GcmGhashFull_x86_64.inc} +{$UNDEF GCM_GHASH_RAW_INPUT} {$UNDEF GCM_GHASH_FULL_BLOCKS_8} diff --git a/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedCbcKernel.pas b/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedCbcKernel.pas new file mode 100644 index 00000000..ac483a1d --- /dev/null +++ b/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedCbcKernel.pas @@ -0,0 +1,84 @@ +{ *********************************************************************************** } +{ * CryptoLib Library * } +{ * Author - Ugochukwu Mmaduekwe * } +{ * Github Repository * } +{ * * } +{ * Distributed under the MIT software license, see the accompanying file LICENSE * } +{ * or visit http://www.opensource.org/licenses/mit-license.php. * } +{ * * } +{ * Acknowledgements: * } +{ * * } +{ * Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring * } +{ * the development of this library * } +{ * ******************************************************************************* * } + +(* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) + +unit ClpIFusedCbcKernel; + +{$I ..\..\..\..\Include\CryptoLib.inc} + +interface + +uses + ClpIBlockCipher, + ClpCryptoLibTypes, + ClpFusedKernelTypes; + +type + /// + /// Mode-specific contract for a fused CBC body kernel, direction-bound at + /// construction (like the OCB / CCM kernels). Applies the CBC chain over a + /// whole run in one call, keeping the chaining value in a register between + /// blocks. Encrypt is inherently serial (C_i = E_K(P_i xor C_{i-1})), so its + /// win is orchestration, not parallelism; decrypt (P_i = D_K(C_i) xor + /// C_{i-1}) parallelises and is not yet implemented. Cipher state lives + /// inside the implementation; the mode sees only this interface. + /// + IFusedCbcKernel = interface + ['{898E71F2-B3C8-4B44-A3AB-2A493B4AE126}'] + + /// + /// Process ABlockCount consecutive 16-byte CBC blocks in one pass, in the + /// direction fixed at construction. Encrypt: AOutPtr[i] := E_K(AInPtr[i] xor + /// chain), chain = AIvPtr for i = 0 else the previous ciphertext; the final + /// ciphertext block is written back to AIvPtr. ABlockCount MUST be >= 1. + /// AInPtr and AOutPtr are identical (in-place) or fully disjoint; partial + /// overlap is not supported. + /// + procedure ProcessCbcBlocks(AInPtr, AOutPtr, AIvPtr: Pointer; + ABlockCount: NativeInt); + end; + + /// + /// Factory contract for CBC-encrypt kernel providers. Registered with + /// TFusedKernelRegistry; the registry walks the factory list (highest + /// priority first) and returns the first kernel whose TryCreate succeeds. + /// Factories self-probe (CPU features, cipher identity) and wrap construction + /// in try/except; TryCreate MUST return False on failure rather than + /// propagating. + /// + IFusedCbcKernelFactory = interface + ['{A4FBAB88-8E80-45A0-86E1-B95B6AFBA9A2}'] + + /// Stable human-readable provider label (diagnostics / tests). + function ProviderName: String; + + /// Priority class controlling factory order inside the registry; + /// see TFusedKernelPriority. + function Priority: TFusedKernelPriority; + + /// + /// Attempt to construct a CBC kernel bound to ACipher for ADirection. + /// Only Encrypt is implemented today; a Decrypt request returns False + /// (the mode then falls back). Returns False with AKernel = nil on any + /// failure; never raises. + /// + function TryCreate(const ACipher: IBlockCipher; + ADirection: TFusedModeDirection; + out AKernel: IFusedCbcKernel): Boolean; + end; + +implementation + +end. diff --git a/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedCtrKernel.pas b/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedCtrKernel.pas new file mode 100644 index 00000000..c3ddcd59 --- /dev/null +++ b/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedCtrKernel.pas @@ -0,0 +1,90 @@ +{ *********************************************************************************** } +{ * CryptoLib Library * } +{ * Author - Ugochukwu Mmaduekwe * } +{ * Github Repository * } +{ * * } +{ * Distributed under the MIT software license, see the accompanying file LICENSE * } +{ * or visit http://www.opensource.org/licenses/mit-license.php. * } +{ * * } +{ * Acknowledgements: * } +{ * * } +{ * Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring * } +{ * the development of this library * } +{ * ******************************************************************************* * } + +(* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) + +unit ClpIFusedCtrKernel; + +{$I ..\..\..\..\Include\CryptoLib.inc} + +interface + +uses + ClpIBlockCipher, + ClpCryptoLibTypes, + ClpFusedKernelTypes; + +type + /// + /// Mode-specific contract for a fused counter (CTR / SIC) body kernel: + /// generate AES-CTR keystream and XOR it into the payload in a single pass, + /// advancing the counter. It is the non-authenticated, MAC-free member of + /// the fused-kernel family - the shared core the AEAD kernels are built on + /// (GCM = CTR + GHASH, CCM = CTR + CBC-MAC, EAX = CTR + OMAC). All cipher + /// state (the key schedule) lives inside the implementation; the mode sees + /// only this interface. + /// + IFusedCtrKernel = interface + ['{2A9F4C71-6E38-4B0D-9C57-1F3B8E26D4A5}'] + + /// The batch granularity: ProcessCtrBlocks requires ABlockCount to + /// be a positive multiple of this (the kernel processes the whole run in + /// one call, looping internally). + function BatchBlockCount: Int32; + + /// + /// Encrypt-and-XOR ABlockCount consecutive counter blocks in one pass: + /// for each block i, AOutPtr[i] := AInPtr[i] xor E_K(ACounter + i), then + /// advance ACounter. ACounter is a 16-byte big-endian value incremented + /// by one per block with carry across all 16 bytes (NIST SP 800-38A CTR, + /// matching TSicBlockCipher's per-block increment), updated in place. + /// ABlockCount MUST be a positive multiple of BatchBlockCount. AInPtr and + /// AOutPtr are identical (in-place) or fully disjoint. + /// + procedure ProcessCtrBlocks(AInPtr, AOutPtr, ACounter: Pointer; + ABlockCount: NativeInt); + end; + + /// + /// Factory contract for CTR kernel providers. Registered with + /// TFusedKernelRegistry; the registry walks the per-mode factory list + /// (highest-priority first) and returns the first kernel whose TryCreate + /// succeeds. Factories self-probe (CPU features, cipher identity) and wrap + /// construction in try/except; TryCreate MUST return False on failure + /// rather than propagating. + /// + IFusedCtrKernelFactory = interface + ['{7D1E6B02-4A9C-4F58-8B3D-2C57F1E96A4D}'] + + /// Stable human-readable provider label (diagnostics / tests). + function ProviderName: String; + + /// Priority class controlling factory order inside the registry; + /// see TFusedKernelPriority. + function Priority: TFusedKernelPriority; + + /// + /// Attempt to construct a CTR kernel bound to ACipher. ADirection is + /// accepted for signature symmetry with the AEAD factories but ignored: + /// CTR keystream is direction-independent (both encrypt and decrypt XOR + /// the same E_K(counter) stream). + /// + function TryCreate(const ACipher: IBlockCipher; + ADirection: TFusedModeDirection; + out AKernel: IFusedCtrKernel): Boolean; + end; + +implementation + +end. diff --git a/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedGcmKernel.pas b/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedGcmKernel.pas index 03e68551..51c0c6a4 100644 --- a/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedGcmKernel.pas +++ b/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedGcmKernel.pas @@ -36,29 +36,24 @@ interface IFusedGcmKernel = interface ['{D4D7F5F0-3C56-44E0-8BDD-944AC05E4D2E}'] - /// Number of 16-byte blocks consumed per - /// ProcessCtrGhashBatch invocation. + /// Number of 16-byte blocks per batch consumed by + /// ProcessCtrGhashBatches. function MinimumBlockCount: Int32; /// - /// Process exactly MinimumBlockCount blocks of the GCM body. - /// Per-call buffers: - /// AInPtr MinimumBlockCount * 16 bytes of input. - /// AOutPtr MinimumBlockCount * 16 bytes of output; may - /// alias AInPtr. - /// ARawCtrs MinimumBlockCount * 16 bytes of pre-populated - /// raw counter blocks (encrypted in place by the - /// kernel to produce keystream). - /// APrevCipher MinimumBlockCount * 16 bytes of the previous - /// batch's ciphertext, folded into GHASH by this - /// call. The mode seeds this from the prime batch - /// and rotates it between iterations. - /// AGhashState 16-byte running GHASH accumulator, updated in - /// place. - /// ABlockCount MUST equal MinimumBlockCount. + /// Process ABatchCount consecutive 8-block batches in one call: generate + /// the GCM keystream (32-bit counter starting at ACounter32, J0 upper 96 + /// bits from AJ0Template), XOR AInPtr into AOutPtr, and fold each batch's + /// ciphertext into the running GHASH state (AGhashState). APrevInit is the + /// ciphertext of the batch immediately before this run (the prime batch); + /// the pipeline GHASHes it first and lags by one, leaving the final batch + /// for the caller's drain. AForEncrypt selects whether the GHASHed + /// ciphertext is the output (encrypt) or the input (decrypt). Returns the + /// advanced 32-bit counter (ACounter32 + 8*ABatchCount). /// - procedure ProcessCtrGhashBatch(AInPtr, AOutPtr, ARawCtrs, APrevCipher, - AGhashState: Pointer; ABlockCount: Int32); + function ProcessCtrGhashBatches(AInPtr, AOutPtr, APrevInit, AGhashState, + AJ0Template: Pointer; ACounter32: UInt32; ABatchCount: NativeInt; + AForEncrypt: Boolean): UInt32; end; /// diff --git a/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedOcbKernel.pas b/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedOcbKernel.pas index 5f3627cb..132129d3 100644 --- a/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedOcbKernel.pas +++ b/CryptoLib/src/Interfaces/Crypto/Modes/Fused/ClpIFusedOcbKernel.pas @@ -61,14 +61,9 @@ interface /// /// ALTablePtr points at a read-only contiguous buffer of L-table /// entries L[0], L[1], ..., L[LMax], each 16 bytes. The caller - /// guarantees it covers every ntz value referenced by ANtzPtr. - /// - /// ANtzPtr points at a read-only ABlockCount-byte array where - /// ANtzPtr[i] = OCB_ntz(FMainBlockCount + i + 1). The caller - /// pre-computes these so the kernel can index the L-table with a - /// single byte load per block (cheaper and simpler than doing - /// BSF + 64-bit counter bookkeeping inside the kernel, especially - /// on i386). + /// guarantees it covers every ntz value the kernel derives across + /// the span (the kernel computes ntz per block itself via a running + /// block counter seeded from AStartBlockCount). /// /// ABlock0Ptr points at the 16-byte source of block 0 of the very /// first kernel iteration. It MAY equal AInPtr (the common case, @@ -81,8 +76,12 @@ interface /// transparently transitions from the prefix source to the main /// stream after iteration 0. /// + /// AStartBlockCount is the OCB block count before this span (= the value + /// consumed as `FMainBlockCount` just before the first block); the + /// kernel seeds its per-block ntz counter from it. procedure ProcessBlocks(AInPtr, AOutPtr, AOffsetPtr, AChecksumPtr, - ALTablePtr, ANtzPtr, ABlock0Ptr: Pointer; ABlockCount: Int32); + ALTablePtr, ABlock0Ptr: Pointer; ABlockCount: Int32; + AStartBlockCount: UInt64); end; IFusedOcbKernelFactory = interface diff --git a/CryptoLib/src/Misc/ClpBitOperations.pas b/CryptoLib/src/Misc/ClpBitOperations.pas index 94dfa436..368d13fc 100644 --- a/CryptoLib/src/Misc/ClpBitOperations.pas +++ b/CryptoLib/src/Misc/ClpBitOperations.pas @@ -169,37 +169,37 @@ TBitOperations = class sealed(TObject) /// Counts leading zero bits from the MSB down to the highest set bit. /// Returns 32 when AValue is zero. /// - class function NumberOfLeadingZeros32(AValue: UInt32): Int32; static; + class function NumberOfLeadingZeros32(AValue: UInt32): Int32; static; inline; /// /// Counts leading zero bits from the MSB down to the highest set bit. /// Returns 64 when AValue is zero. /// - class function NumberOfLeadingZeros64(AValue: UInt64): Int32; static; + class function NumberOfLeadingZeros64(AValue: UInt64): Int32; static; inline; /// /// Counts trailing zero bits from the LSB up to the lowest set bit. /// Returns 32 when AValue is zero. /// - class function NumberOfTrailingZeros32(AValue: UInt32): Int32; static; + class function NumberOfTrailingZeros32(AValue: UInt32): Int32; static; inline; /// /// Counts trailing zero bits from the LSB up to the lowest set bit. /// Returns 64 when AValue is zero. /// - class function NumberOfTrailingZeros64(AValue: UInt64): Int32; static; + class function NumberOfTrailingZeros64(AValue: UInt64): Int32; static; inline; /// /// Population count (Hamming weight) of a 32-bit value. /// Returns the number of bits set to 1. /// - class function PopCount32(AValue: UInt32): Int32; static; + class function PopCount32(AValue: UInt32): Int32; static; inline; /// /// Population count (Hamming weight) of a 64-bit value. /// Returns the number of bits set to 1. /// - class function PopCount64(AValue: UInt64): Int32; static; + class function PopCount64(AValue: UInt64): Int32; static; inline; end; implementation diff --git a/CryptoLib/src/NumberUtilities/ClpByteUtilities.pas b/CryptoLib/src/NumberUtilities/ClpByteUtilities.pas index d302df4f..9436be5e 100644 --- a/CryptoLib/src/NumberUtilities/ClpByteUtilities.pas +++ b/CryptoLib/src/NumberUtilities/ClpByteUtilities.pas @@ -23,6 +23,7 @@ interface uses ClpNat, ClpBinaryPrimitives, + ClpByteXorSimd, ClpCryptoLibTypes; type @@ -67,6 +68,9 @@ class procedure TByteUtilities.&Xor(ALen: Int32; AX, AY, AZ: PByte); if ALen <= 0 then Exit; + if (ALen >= 128) and TByteXorSimd.TryXor(ALen, AX, AY, AZ) then + Exit; + PSrcA := AX; PSrcB := AY; PDst := AZ; @@ -105,29 +109,9 @@ class procedure TByteUtilities.&Xor(ALen: Int32; const AX: TCryptoLibByteArray; end; class procedure TByteUtilities.XorTo(ALen: Int32; AX, AZ: PByte); -var - LI, LQwords: Int32; - PSrc, PDst: PByte; begin - if ALen <= 0 then - Exit; - - PSrc := AX; - PDst := AZ; - - if (ALen >= 8) and - ((NativeUInt(PSrc) or NativeUInt(PDst) or NativeUInt(ALen)) and 7 = 0) then - begin - LQwords := ALen shr 3; - for LI := 0 to LQwords - 1 do - TBinaryPrimitives.StoreUInt64(PUInt64(PDst + (LI * 8)), - TBinaryPrimitives.LoadUInt64(PUInt64(PDst + (LI * 8))) xor - TBinaryPrimitives.LoadUInt64(PUInt64(PSrc + (LI * 8)))); - Exit; - end; - - for LI := 0 to ALen - 1 do - PDst[LI] := PDst[LI] xor PSrc[LI]; + // In-place AZ := AZ xor AX is the special case of &Xor with AY = AZ. + &Xor(ALen, AZ, AX, AZ); end; class procedure TByteUtilities.XorTo(ALen: Int32; AX, AZ: PByte; AXOff, AZOff: Integer); diff --git a/CryptoLib/src/NumberUtilities/ClpByteXorSimd.pas b/CryptoLib/src/NumberUtilities/ClpByteXorSimd.pas new file mode 100644 index 00000000..1a42b3a3 --- /dev/null +++ b/CryptoLib/src/NumberUtilities/ClpByteXorSimd.pas @@ -0,0 +1,55 @@ +{ *********************************************************************************** } +{ * CryptoLib Library * } +{ * Author - Ugochukwu Mmaduekwe * } +{ * Github Repository * } +{ * * } +{ * Distributed under the MIT software license, see the accompanying file LICENSE * } +{ * or visit http://www.opensource.org/licenses/mit-license.php. * } +{ * * } +{ * Acknowledgements: * } +{ * * } +{ * Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring * } +{ * the development of this library * } +{ * ******************************************************************************* * } + +(* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) + +unit ClpByteXorSimd; + +{$I ..\Include\CryptoLib.inc} + +interface + +uses +{$IFDEF CRYPTOLIB_X86_SIMD} + ClpByteXorX86Backend, +{$ENDIF} + ClpCryptoLibTypes; + +type + /// + /// Arch-agnostic SIMD facade for the byte-array XOR primitives behind + /// TByteUtilities. Selects the per-arch backend at compile time; on a + /// build with no SIMD backend (or when the active SIMD level is too low) every + /// entry point returns False, leaving the caller on its scalar + /// qword/byte path. + /// + TByteXorSimd = class sealed + public + /// AZ[i] := AX[i] xor AY[i] over ALen bytes; AZ may alias AX/AY + /// (so the in-place XorTo case is TryXor with AY = AZ). False if unavailable. + class function TryXor(ALen: NativeInt; AX, AY, AZ: PByte): Boolean; static; + end; + +implementation + +class function TByteXorSimd.TryXor(ALen: NativeInt; AX, AY, AZ: PByte): Boolean; +begin +{$IFDEF CRYPTOLIB_X86_SIMD} + Result := TByteXorX86Backend.TryXor(ALen, AX, AY, AZ); +{$ELSE} + Result := False; +{$ENDIF} +end; + +end. diff --git a/CryptoLib/src/NumberUtilities/ClpByteXorX86Backend.pas b/CryptoLib/src/NumberUtilities/ClpByteXorX86Backend.pas new file mode 100644 index 00000000..3c6b0ccd --- /dev/null +++ b/CryptoLib/src/NumberUtilities/ClpByteXorX86Backend.pas @@ -0,0 +1,73 @@ +{ *********************************************************************************** } +{ * CryptoLib Library * } +{ * Author - Ugochukwu Mmaduekwe * } +{ * Github Repository * } +{ * * } +{ * Distributed under the MIT software license, see the accompanying file LICENSE * } +{ * or visit http://www.opensource.org/licenses/mit-license.php. * } +{ * * } +{ * Acknowledgements: * } +{ * * } +{ * Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring * } +{ * the development of this library * } +{ * ******************************************************************************* * } + +(* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) + +unit ClpByteXorX86Backend; + +{$I ..\Include\CryptoLib.inc} + +interface + +uses + ClpCpuFeatures, + ClpCryptoLibTypes; + +type + /// + /// x86 SIMD backend for the byte-array XOR primitives behind + /// TByteUtilities: an SSE2 movdqu/pxor kernel (body in + /// Include\Simd\ByteXor\) plus the runtime capability gate. + /// Compiles on every target - the entry points return False (leaving the + /// caller on its scalar qword/byte path) when built without x86 SIMD or when + /// the active SIMD level is below SSE2 (e.g. CRYPTOLIB_FORCE_SCALAR). Works on + /// raw pointers only; unaligned operands are handled by movdqu. + /// + TByteXorX86Backend = class sealed + public + /// AZ[i] := AX[i] xor AY[i] over ALen bytes; AZ may alias AX or AY + /// (so the in-place XorTo case is TryXor with AY = AZ). False if unavailable. + class function TryXor(ALen: NativeInt; AX, AY, AZ: PByte): Boolean; static; + end; + +implementation + +{$IFDEF CRYPTOLIB_X86_SIMD} +procedure ByteXorSse2(ALen: NativeInt; AX, AY, AZ: Pointer); +{$IFDEF CRYPTOLIB_X86_64_ASM} +{$I ..\Include\Simd\Common\SimdProc4Begin_x86_64.inc} +{$I ..\Include\Simd\ByteXor\ByteXorSse2_x86_64.inc} +{$ENDIF} +{$IFDEF CRYPTOLIB_I386_ASM} +{$I ..\Include\Simd\Common\SimdProc4Begin_i386.inc} +{$I ..\Include\Simd\ByteXor\ByteXorSse2_i386.inc} +{$ENDIF} +end; +{$ENDIF CRYPTOLIB_X86_SIMD} + +{ TByteXorX86Backend } + +class function TByteXorX86Backend.TryXor(ALen: NativeInt; AX, AY, AZ: PByte): Boolean; +begin +{$IFDEF CRYPTOLIB_X86_SIMD} + if TCpuFeatures.X86.HasSSE2 then + begin + ByteXorSse2(ALen, AX, AY, AZ); + Exit(True); + end; +{$ENDIF} + Result := False; +end; + +end. diff --git a/CryptoLib/src/Packages/Delphi/CryptoLib4PascalPackage.dpk b/CryptoLib/src/Packages/Delphi/CryptoLib4PascalPackage.dpk index e40523c7..e4a1fc52 100644 --- a/CryptoLib/src/Packages/Delphi/CryptoLib4PascalPackage.dpk +++ b/CryptoLib/src/Packages/Delphi/CryptoLib4PascalPackage.dpk @@ -690,14 +690,18 @@ contains ClpIFusedCcmKernel in '..\..\Interfaces\Crypto\Modes\Fused\ClpIFusedCcmKernel.pas', ClpIFusedEaxKernel in '..\..\Interfaces\Crypto\Modes\Fused\ClpIFusedEaxKernel.pas', ClpIFusedGcmSivKernel in '..\..\Interfaces\Crypto\Modes\Fused\ClpIFusedGcmSivKernel.pas', + ClpIFusedCtrKernel in '..\..\Interfaces\Crypto\Modes\Fused\ClpIFusedCtrKernel.pas', + ClpIFusedCbcKernel in '..\..\Interfaces\Crypto\Modes\Fused\ClpIFusedCbcKernel.pas', ClpFusedKernelRegistry in '..\..\Crypto\Modes\Fused\ClpFusedKernelRegistry.pas', ClpAesFusedAeadSimd in '..\..\Crypto\Modes\Fused\Internal\ClpAesFusedAeadSimd.pas', - ClpAesFusedAeadX86Backend in '..\..\Crypto\Modes\Fused\Internal\ClpAesFusedAeadX86Backend.pas', + ClpAesNiFusedX86Backend in '..\..\Crypto\Modes\Fused\Internal\ClpAesNiFusedX86Backend.pas', ClpBinPolySimd in '..\..\Math\BinPoly\ClpBinPolySimd.pas', ClpAesNiOcbKernel in '..\..\Crypto\Modes\Fused\ClpAesNiOcbKernel.pas', ClpAesNiCcmKernel in '..\..\Crypto\Modes\Fused\ClpAesNiCcmKernel.pas', ClpAesNiEaxKernel in '..\..\Crypto\Modes\Fused\ClpAesNiEaxKernel.pas', ClpAesNiGcmKernel in '..\..\Crypto\Modes\Fused\ClpAesNiGcmKernel.pas', + ClpAesNiCtrKernel in '..\..\Crypto\Modes\Fused\ClpAesNiCtrKernel.pas', + ClpAesNiCbcKernel in '..\..\Crypto\Modes\Fused\ClpAesNiCbcKernel.pas', ClpPclmulGcmSivKernel in '..\..\Crypto\Modes\Fused\ClpPclmulGcmSivKernel.pas', ClpFusedKernelDefaults in '..\..\Crypto\Modes\Fused\ClpFusedKernelDefaults.pas', ClpAeadParameters in '..\..\Crypto\Parameters\ClpAeadParameters.pas', @@ -727,6 +731,8 @@ contains ClpCfbBlockCipherMac in '..\..\Crypto\Macs\ClpCfbBlockCipherMac.pas', ClpICfbBlockCipherMac in '..\..\Interfaces\Crypto\Macs\ClpICfbBlockCipherMac.pas', ClpByteUtilities in '..\..\NumberUtilities\ClpByteUtilities.pas', + ClpByteXorSimd in '..\..\NumberUtilities\ClpByteXorSimd.pas', + ClpByteXorX86Backend in '..\..\NumberUtilities\ClpByteXorX86Backend.pas', ClpInt16Utilities in '..\..\NumberUtilities\ClpInt16Utilities.pas', ClpInt32Utilities in '..\..\NumberUtilities\ClpInt32Utilities.pas', ClpInt64Utilities in '..\..\NumberUtilities\ClpInt64Utilities.pas', diff --git a/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.lpk b/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.lpk index 8ba7e913..c5296e79 100644 --- a/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.lpk +++ b/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.lpk @@ -31,7 +31,7 @@ Acknowledgements: Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring the development of this library "/> - + @@ -3178,8 +3178,8 @@ Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring the devel - - + + @@ -3189,6 +3189,30 @@ Thanks to Sphere 10 Software (http://www.sphere10.com/) for sponsoring the devel + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.pas b/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.pas index 42a0be66..4e35c4b3 100644 --- a/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.pas +++ b/CryptoLib/src/Packages/FPC/CryptoLib4PascalPackage.pas @@ -255,7 +255,9 @@ interface ClpIBackingHashProvider, ClpChaChaSimd, ClpChaChaX86Backend, ClpSalsaSimd, ClpSalsaX86Backend, ClpPoly1305State, ClpPoly1305Simd, ClpPoly1305X86Backend, ClpGhashSimd, ClpGhashX86Backend, ClpGcmSivSimd, - ClpGcmSivX86Backend, ClpAesFusedAeadX86Backend, ClpBinPolySimd, ClpAesSimd; + ClpGcmSivX86Backend, ClpAesNiFusedX86Backend, ClpBinPolySimd, ClpAesSimd, + ClpIFusedCtrKernel, ClpAesNiCtrKernel, ClpByteXorSimd, ClpByteXorX86Backend, + ClpIFusedCbcKernel, ClpAesNiCbcKernel; implementation