From 94e51e5a5ba268ff4f42edc45462ff294918228d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 9 Jul 2026 11:23:17 +0000
Subject: [PATCH] chore(deps): bump golang.org/x/crypto from 0.51.0 to 0.52.0

Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.51.0 to 0.52.0.
- [Commits](https://github.com/golang/crypto/compare/v0.51.0...v0.52.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-version: 0.52.0
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod                                        |    2 +-
 go.sum                                        |    4 +-
 .../OvyFlash/telegram-bot-api/CODEOWNERS      |    2 +-
 vendor/golang.org/x/crypto/blake2b/go125.go   |   11 -
 .../chacha20poly1305_amd64.go                 |    6 +-
 .../chacha20poly1305/chacha20poly1305_amd64.s | 8664 ++++-------------
 vendor/golang.org/x/crypto/ssh/certs.go       |   18 +-
 vendor/golang.org/x/crypto/ssh/channel.go     |   65 +-
 vendor/golang.org/x/crypto/ssh/cipher.go      |    2 +-
 vendor/golang.org/x/crypto/ssh/keys.go        |   58 +
 vendor/golang.org/x/crypto/ssh/mux.go         |   36 +-
 vendor/golang.org/x/crypto/ssh/server.go      |  127 +-
 vendor/modules.txt                            |    2 +-
 13 files changed, 2355 insertions(+), 6642 deletions(-)
 delete mode 100644 vendor/golang.org/x/crypto/blake2b/go125.go

diff --git a/go.mod b/go.mod
index 87cb84cd..6de58b48 100644
--- a/go.mod
+++ b/go.mod
@@ -129,7 +129,7 @@ require (
 	go.opentelemetry.io/otel v1.43.0 // indirect
 	go.opentelemetry.io/otel/metric v1.43.0 // indirect
 	go.opentelemetry.io/otel/trace v1.43.0 // indirect
-	golang.org/x/crypto v0.51.0 // indirect
+	golang.org/x/crypto v0.52.0 // indirect
 	golang.org/x/net v0.55.0 // indirect
 	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.45.0 // indirect
diff --git a/go.sum b/go.sum
index d98dedf2..7c2b6694 100644
--- a/go.sum
+++ b/go.sum
@@ -310,8 +310,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI=
-golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8=
+golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
+golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo=
 golang.org/x/image v0.41.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA=
diff --git a/vendor/github.com/OvyFlash/telegram-bot-api/CODEOWNERS b/vendor/github.com/OvyFlash/telegram-bot-api/CODEOWNERS
index 511f002d..eda9b16e 100644
--- a/vendor/github.com/OvyFlash/telegram-bot-api/CODEOWNERS
+++ b/vendor/github.com/OvyFlash/telegram-bot-api/CODEOWNERS
@@ -1 +1 @@
-* @iamwavecut @zhuk-kk
+* @iamwavecut @zhuk-kk
diff --git a/vendor/golang.org/x/crypto/blake2b/go125.go b/vendor/golang.org/x/crypto/blake2b/go125.go
deleted file mode 100644
index 67e990b7..00000000
--- a/vendor/golang.org/x/crypto/blake2b/go125.go
+++ /dev/null
@@ -1,11 +0,0 @@
-// Copyright 2025 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-//go:build go1.25
-
-package blake2b
-
-import "hash"
-
-var _ hash.XOF = (*xof)(nil)
diff --git a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go
index b850e772..bfe546b6 100644
--- a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go
+++ b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go
@@ -20,7 +20,7 @@ func chacha20Poly1305Open(dst []byte, key []uint32, src, ad []byte) bool
 func chacha20Poly1305Seal(dst []byte, key []uint32, src, ad []byte)
 
 var (
-	useAVX2 = cpu.X86.HasAVX2 && cpu.X86.HasBMI2
+	useAVX2 = cpu.X86.HasSSSE3 && cpu.X86.HasAVX2 && cpu.X86.HasBMI2
 )
 
 // setupState writes a ChaCha20 input matrix to state. See
@@ -47,7 +47,7 @@ func setupState(state *[16]uint32, key *[32]byte, nonce []byte) {
 }
 
 func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []byte {
-	if !cpu.X86.HasSSSE3 {
+	if !useAVX2 {
 		return c.sealGeneric(dst, nonce, plaintext, additionalData)
 	}
 
@@ -66,7 +66,7 @@ func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []
 }
 
 func (c *chacha20poly1305) open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
-	if !cpu.X86.HasSSSE3 {
+	if !useAVX2 {
 		return c.openGeneric(dst, nonce, ciphertext, additionalData)
 	}
 
diff --git a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s
index fd5ee845..c703c134 100644
--- a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s
+++ b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.s
@@ -6,27 +6,6 @@
 
 // func polyHashADInternal<>()
 TEXT polyHashADInternal<>(SB), NOSPLIT, $0
-	// Hack: Must declare #define macros inside of a function due to Avo constraints
-	// ROL rotates the uint32s in register R left by N bits, using temporary T.
-	#define ROL(N, R, T) \
-		MOVO R, T; \
-		PSLLL $(N), T; \
-		PSRLL $(32-(N)), R; \
-		PXOR T, R
-
-	// ROL8 rotates the uint32s in register R left by 8, using temporary T if needed.
-	#ifdef GOAMD64_v2
-		#define ROL8(R, T) PSHUFB ·rol8<>(SB), R
-	#else
-		#define ROL8(R, T) ROL(8, R, T)
-	#endif
-
-	// ROL16 rotates the uint32s in register R left by 16, using temporary T if needed.
-	#ifdef GOAMD64_v2
-		#define ROL16(R, T) PSHUFB ·rol16<>(SB), R
-	#else
-		#define ROL16(R, T) ROL(16, R, T)
-	#endif
 	XORQ  R10, R10
 	XORQ  R11, R11
 	XORQ  R12, R12
@@ -192,676 +171,112 @@ hashADDone:
 // Requires: AVX, AVX2, BMI2, CMOV, SSE2
 TEXT ·chacha20Poly1305Open(SB), $288-97
 	// For aligned stack access
-	MOVQ SP, BP
-	ADDQ $0x20, BP
-	ANDQ $-32, BP
-	MOVQ dst_base+0(FP), DI
-	MOVQ key_base+24(FP), R8
-	MOVQ src_base+48(FP), SI
-	MOVQ src_len+56(FP), BX
-	MOVQ ad_base+72(FP), CX
-
-	// Check for AVX2 support
-	CMPB ·useAVX2+0(SB), $0x01
-	JE   chacha20Poly1305Open_AVX2
+	MOVQ           SP, BP
+	ADDQ           $0x20, BP
+	ANDQ           $-32, BP
+	MOVQ           dst_base+0(FP), DI
+	MOVQ           key_base+24(FP), R8
+	MOVQ           src_base+48(FP), SI
+	MOVQ           src_len+56(FP), BX
+	MOVQ           ad_base+72(FP), CX
+	VZEROUPPER
+	VMOVDQU        ·chacha20Constants<>+0(SB), Y0
+	VBROADCASTI128 16(R8), Y14
+	VBROADCASTI128 32(R8), Y12
+	VBROADCASTI128 48(R8), Y4
+	VPADDD         ·avx2InitMask<>+0(SB), Y4, Y4
 
 	// Special optimization, for very short buffers
-	CMPQ BX, $0x80
-	JBE  openSSE128
-
-	// For long buffers, prepare the poly key first
-	MOVOU ·chacha20Constants<>+0(SB), X0
-	MOVOU 16(R8), X3
-	MOVOU 32(R8), X6
-	MOVOU 48(R8), X9
-	MOVO  X9, X13
-
-	// Store state on stack for future use
-	MOVO X3, 32(BP)
-	MOVO X6, 48(BP)
-	MOVO X9, 128(BP)
-	MOVQ $0x0000000a, R9
-
-openSSEPreparePolyKey:
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	DECQ  R9
-	JNE   openSSEPreparePolyKey
+	CMPQ BX, $0xc0
+	JBE  openAVX2192
+	CMPQ BX, $0x00000140
+	JBE  openAVX2320
+
+	// For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream
+	VMOVDQA Y14, 32(BP)
+	VMOVDQA Y12, 64(BP)
+	VMOVDQA Y4, 192(BP)
+	MOVQ    $0x0000000a, R9
+
+openAVX2PreparePolyKey:
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	DECQ       R9
+	JNE        openAVX2PreparePolyKey
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     192(BP), Y4, Y4
+	VPERM2I128 $0x02, Y0, Y14, Y3
 
-	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
-	PADDL ·chacha20Constants<>+0(SB), X0
-	PADDL 32(BP), X3
+	// Clamp and store poly key
+	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA Y3, (BP)
 
-	// Clamp and store the key
-	PAND ·polyClampMask<>+0(SB), X0
-	MOVO X0, (BP)
-	MOVO X3, 16(BP)
+	// Stream for the first 64 bytes
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
 
-	// Hash AAD
+	// Hash AD + first 64 bytes
 	MOVQ ad_len+80(FP), R9
 	CALL polyHashADInternal<>(SB)
+	XORQ CX, CX
 
-openSSEMainLoop:
-	CMPQ BX, $0x00000100
-	JB   openSSEMainLoopDone
-
-	// Load state, increment counter blocks
-	MOVO  ·chacha20Constants<>+0(SB), X0
-	MOVO  32(BP), X3
-	MOVO  48(BP), X6
-	MOVO  128(BP), X9
-	PADDL ·sseIncMask<>+0(SB), X9
-	MOVO  X0, X1
-	MOVO  X3, X4
-	MOVO  X6, X7
-	MOVO  X9, X10
-	PADDL ·sseIncMask<>+0(SB), X10
-	MOVO  X1, X2
-	MOVO  X4, X5
-	MOVO  X7, X8
-	MOVO  X10, X11
-	PADDL ·sseIncMask<>+0(SB), X11
-	MOVO  X2, X12
-	MOVO  X5, X13
-	MOVO  X8, X14
-	MOVO  X11, X15
-	PADDL ·sseIncMask<>+0(SB), X15
-
-	// Store counters
-	MOVO X9, 80(BP)
-	MOVO X10, 96(BP)
-	MOVO X11, 112(BP)
-	MOVO X15, 128(BP)
-
-	// There are 10 ChaCha20 iterations of 2QR each, so for 6 iterations we hash
-	// 2 blocks, and for the remaining 4 only 1 block - for a total of 16
-	MOVQ $0x00000004, CX
-	MOVQ SI, R9
-
-openSSEInternalLoop:
-	MOVO  X14, 64(BP)
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X3
-	PXOR  X14, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X3
-	PXOR  X14, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X4
-	PXOR  X14, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X4
-	PXOR  X14, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X5
-	PXOR  X14, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X5
-	PXOR  X14, X5
-	MOVO  64(BP), X14
-	MOVO  X7, 64(BP)
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL16(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x0c, X7
-	PSRLL $0x14, X13
-	PXOR  X7, X13
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL8(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x07, X7
-	PSRLL $0x19, X13
-	PXOR  X7, X13
-	MOVO  64(BP), X7
-	ADDQ  (R9), R10
-	ADCQ  8(R9), R11
-	ADCQ  $0x01, R12
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x0c
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	LEAQ  16(R9), R9
-	MOVO  X14, 64(BP)
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X3
-	PXOR  X14, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X3
-	PXOR  X14, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X4
-	PXOR  X14, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X4
-	PXOR  X14, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X5
-	PXOR  X14, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X5
-	PXOR  X14, X5
-	MOVO  64(BP), X14
-	MOVO  X7, 64(BP)
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL16(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x0c, X7
-	PSRLL $0x14, X13
-	PXOR  X7, X13
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL8(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x07, X7
-	PSRLL $0x19, X13
-	PXOR  X7, X13
-	MOVO  64(BP), X7
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x04
-	DECQ  CX
-	JGE   openSSEInternalLoop
-	ADDQ  (R9), R10
-	ADCQ  8(R9), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	LEAQ  16(R9), R9
-	CMPQ  CX, $-6
-	JG    openSSEInternalLoop
-
-	// Add in the state
-	PADDD ·chacha20Constants<>+0(SB), X0
-	PADDD ·chacha20Constants<>+0(SB), X1
-	PADDD ·chacha20Constants<>+0(SB), X2
-	PADDD ·chacha20Constants<>+0(SB), X12
-	PADDD 32(BP), X3
-	PADDD 32(BP), X4
-	PADDD 32(BP), X5
-	PADDD 32(BP), X13
-	PADDD 48(BP), X6
-	PADDD 48(BP), X7
-	PADDD 48(BP), X8
-	PADDD 48(BP), X14
-	PADDD 80(BP), X9
-	PADDD 96(BP), X10
-	PADDD 112(BP), X11
-	PADDD 128(BP), X15
-
-	// Load - xor - store
-	MOVO  X15, 64(BP)
-	MOVOU (SI), X15
-	PXOR  X15, X0
-	MOVOU X0, (DI)
-	MOVOU 16(SI), X15
-	PXOR  X15, X3
-	MOVOU X3, 16(DI)
-	MOVOU 32(SI), X15
-	PXOR  X15, X6
-	MOVOU X6, 32(DI)
-	MOVOU 48(SI), X15
-	PXOR  X15, X9
-	MOVOU X9, 48(DI)
-	MOVOU 64(SI), X9
-	PXOR  X9, X1
-	MOVOU X1, 64(DI)
-	MOVOU 80(SI), X9
-	PXOR  X9, X4
-	MOVOU X4, 80(DI)
-	MOVOU 96(SI), X9
-	PXOR  X9, X7
-	MOVOU X7, 96(DI)
-	MOVOU 112(SI), X9
-	PXOR  X9, X10
-	MOVOU X10, 112(DI)
-	MOVOU 128(SI), X9
-	PXOR  X9, X2
-	MOVOU X2, 128(DI)
-	MOVOU 144(SI), X9
-	PXOR  X9, X5
-	MOVOU X5, 144(DI)
-	MOVOU 160(SI), X9
-	PXOR  X9, X8
-	MOVOU X8, 160(DI)
-	MOVOU 176(SI), X9
-	PXOR  X9, X11
-	MOVOU X11, 176(DI)
-	MOVOU 192(SI), X9
-	PXOR  X9, X12
-	MOVOU X12, 192(DI)
-	MOVOU 208(SI), X9
-	PXOR  X9, X13
-	MOVOU X13, 208(DI)
-	MOVOU 224(SI), X9
-	PXOR  X9, X14
-	MOVOU X14, 224(DI)
-	MOVOU 240(SI), X9
-	PXOR  64(BP), X9
-	MOVOU X9, 240(DI)
-	LEAQ  256(SI), SI
-	LEAQ  256(DI), DI
-	SUBQ  $0x00000100, BX
-	JMP   openSSEMainLoop
-
-openSSEMainLoopDone:
-	// Handle the various tail sizes efficiently
-	TESTQ BX, BX
-	JE    openSSEFinalize
-	CMPQ  BX, $0x40
-	JBE   openSSETail64
-	CMPQ  BX, $0x80
-	JBE   openSSETail128
-	CMPQ  BX, $0xc0
-	JBE   openSSETail192
-	JMP   openSSETail256
-
-openSSEFinalize:
-	// Hash in the PT, AAD lengths
-	ADDQ  ad_len+80(FP), R10
-	ADCQ  src_len+56(FP), R11
+openAVX2InitialHash64:
+	ADDQ  (SI)(CX*1), R10
+	ADCQ  8(SI)(CX*1), R11
 	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
 	IMULQ R12, R15
+	MULXQ R11, AX, DX
 	ADDQ  AX, R14
 	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
 	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
 	ADCQ  DX, R8
 	MOVQ  R13, R10
 	MOVQ  R14, R11
@@ -878,3187 +293,494 @@ openSSEFinalize:
 	ADDQ  R15, R10
 	ADCQ  R8, R11
 	ADCQ  $0x00, R12
+	ADDQ  $0x10, CX
+	CMPQ  CX, $0x40
+	JNE   openAVX2InitialHash64
 
-	// Final reduce
-	MOVQ    R10, R13
-	MOVQ    R11, R14
-	MOVQ    R12, R15
-	SUBQ    $-5, R10
-	SBBQ    $-1, R11
-	SBBQ    $0x03, R12
-	CMOVQCS R13, R10
-	CMOVQCS R14, R11
-	CMOVQCS R15, R12
-
-	// Add in the "s" part of the key
-	ADDQ 16(BP), R10
-	ADCQ 24(BP), R11
-
-	// Finally, constant time compare to the tag at the end of the message
-	XORQ    AX, AX
-	MOVQ    $0x00000001, DX
-	XORQ    (SI), R10
-	XORQ    8(SI), R11
-	ORQ     R11, R10
-	CMOVQEQ DX, AX
+	// Decrypt the first 64 bytes
+	VPXOR   (SI), Y0, Y0
+	VPXOR   32(SI), Y14, Y14
+	VMOVDQU Y0, (DI)
+	VMOVDQU Y14, 32(DI)
+	LEAQ    64(SI), SI
+	LEAQ    64(DI), DI
+	SUBQ    $0x40, BX
 
-	// Return true iff tags are equal
-	MOVB AX, ret+96(FP)
-	RET
+openAVX2MainLoop:
+	CMPQ BX, $0x00000200
+	JB   openAVX2MainLoopDone
 
-openSSE128:
-	MOVOU ·chacha20Constants<>+0(SB), X0
-	MOVOU 16(R8), X3
-	MOVOU 32(R8), X6
-	MOVOU 48(R8), X9
-	MOVO  X0, X1
-	MOVO  X3, X4
-	MOVO  X6, X7
-	MOVO  X9, X10
-	PADDL ·sseIncMask<>+0(SB), X10
-	MOVO  X1, X2
-	MOVO  X4, X5
-	MOVO  X7, X8
-	MOVO  X10, X11
-	PADDL ·sseIncMask<>+0(SB), X11
-	MOVO  X3, X13
-	MOVO  X6, X14
-	MOVO  X10, X15
-	MOVQ  $0x0000000a, R9
-
-openSSE128InnerCipherLoop:
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X5
-	PXOR  X12, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X5
-	PXOR  X12, X5
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X5
-	PXOR  X12, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X5
-	PXOR  X12, X5
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	DECQ  R9
-	JNE   openSSE128InnerCipherLoop
-
-	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
-	PADDL ·chacha20Constants<>+0(SB), X0
-	PADDL ·chacha20Constants<>+0(SB), X1
-	PADDL ·chacha20Constants<>+0(SB), X2
-	PADDL X13, X3
-	PADDL X13, X4
-	PADDL X13, X5
-	PADDL X14, X7
-	PADDL X14, X8
-	PADDL X15, X10
-	PADDL ·sseIncMask<>+0(SB), X15
-	PADDL X15, X11
-
-	// Clamp and store the key
-	PAND  ·polyClampMask<>+0(SB), X0
-	MOVOU X0, (BP)
-	MOVOU X3, 16(BP)
+	// Load state, increment counter blocks, store the incremented counters
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
+	XORQ    CX, CX
 
-	// Hash
-	MOVQ ad_len+80(FP), R9
-	CALL polyHashADInternal<>(SB)
-
-openSSE128Open:
-	CMPQ BX, $0x10
-	JB   openSSETail16
-	SUBQ $0x10, BX
-
-	// Load for hashing
-	ADDQ (SI), R10
-	ADCQ 8(SI), R11
-	ADCQ $0x01, R12
-
-	// Load for decryption
-	MOVOU (SI), X12
-	PXOR  X12, X1
-	MOVOU X1, (DI)
-	LEAQ  16(SI), SI
-	LEAQ  16(DI), DI
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-
-	// Shift the stream "left"
-	MOVO X4, X1
-	MOVO X7, X4
-	MOVO X10, X7
-	MOVO X2, X10
-	MOVO X5, X2
-	MOVO X8, X5
-	MOVO X11, X8
-	JMP  openSSE128Open
-
-openSSETail16:
-	TESTQ BX, BX
-	JE    openSSEFinalize
-
-	// We can safely load the CT from the end, because it is padded with the MAC
-	MOVQ  BX, R9
-	SHLQ  $0x04, R9
-	LEAQ  ·andMask<>+0(SB), R13
-	MOVOU (SI), X12
-	ADDQ  BX, SI
-	PAND  -16(R13)(R9*1), X12
-	MOVO  X12, 64(BP)
-	MOVQ  X12, R13
-	MOVQ  72(BP), R14
-	PXOR  X1, X12
-
-	// We can only store one byte at a time, since plaintext can be shorter than 16 bytes
-openSSETail16Store:
-	MOVQ   X12, R8
-	MOVB   R8, (DI)
-	PSRLDQ $0x01, X12
-	INCQ   DI
-	DECQ   BX
-	JNE    openSSETail16Store
-	ADDQ   R13, R10
-	ADCQ   R14, R11
-	ADCQ   $0x01, R12
-	MOVQ   (BP), AX
-	MOVQ   AX, R15
-	MULQ   R10
-	MOVQ   AX, R13
-	MOVQ   DX, R14
-	MOVQ   (BP), AX
-	MULQ   R11
-	IMULQ  R12, R15
-	ADDQ   AX, R14
-	ADCQ   DX, R15
-	MOVQ   8(BP), AX
-	MOVQ   AX, R8
-	MULQ   R10
-	ADDQ   AX, R14
-	ADCQ   $0x00, DX
-	MOVQ   DX, R10
-	MOVQ   8(BP), AX
-	MULQ   R11
-	ADDQ   AX, R15
-	ADCQ   $0x00, DX
-	IMULQ  R12, R8
-	ADDQ   R10, R15
-	ADCQ   DX, R8
-	MOVQ   R13, R10
-	MOVQ   R14, R11
-	MOVQ   R15, R12
-	ANDQ   $0x03, R12
-	MOVQ   R15, R13
-	ANDQ   $-4, R13
-	MOVQ   R8, R14
-	SHRQ   $0x02, R8, R15
-	SHRQ   $0x02, R8
-	ADDQ   R13, R10
-	ADCQ   R14, R11
-	ADCQ   $0x00, R12
-	ADDQ   R15, R10
-	ADCQ   R8, R11
-	ADCQ   $0x00, R12
-	JMP    openSSEFinalize
-
-openSSETail64:
-	MOVO  ·chacha20Constants<>+0(SB), X0
-	MOVO  32(BP), X3
-	MOVO  48(BP), X6
-	MOVO  128(BP), X9
-	PADDL ·sseIncMask<>+0(SB), X9
-	MOVO  X9, 80(BP)
-	XORQ  R9, R9
-	MOVQ  BX, CX
-	CMPQ  CX, $0x10
-	JB    openSSETail64LoopB
-
-openSSETail64LoopA:
-	ADDQ  (SI)(R9*1), R10
-	ADCQ  8(SI)(R9*1), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	SUBQ  $0x10, CX
-
-openSSETail64LoopB:
-	ADDQ  $0x10, R9
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	CMPQ  CX, $0x10
-	JAE   openSSETail64LoopA
-	CMPQ  R9, $0xa0
-	JNE   openSSETail64LoopB
-	PADDL ·chacha20Constants<>+0(SB), X0
-	PADDL 32(BP), X3
-	PADDL 48(BP), X6
-	PADDL 80(BP), X9
-
-openSSETail64DecLoop:
-	CMPQ  BX, $0x10
-	JB    openSSETail64DecLoopDone
-	SUBQ  $0x10, BX
-	MOVOU (SI), X12
-	PXOR  X12, X0
-	MOVOU X0, (DI)
-	LEAQ  16(SI), SI
-	LEAQ  16(DI), DI
-	MOVO  X3, X0
-	MOVO  X6, X3
-	MOVO  X9, X6
-	JMP   openSSETail64DecLoop
-
-openSSETail64DecLoopDone:
-	MOVO X0, X1
-	JMP  openSSETail16
-
-openSSETail128:
-	MOVO  ·chacha20Constants<>+0(SB), X1
-	MOVO  32(BP), X4
-	MOVO  48(BP), X7
-	MOVO  128(BP), X10
-	PADDL ·sseIncMask<>+0(SB), X10
-	MOVO  X10, 80(BP)
-	MOVO  X1, X0
-	MOVO  X4, X3
-	MOVO  X7, X6
-	MOVO  X10, X9
-	PADDL ·sseIncMask<>+0(SB), X9
-	MOVO  X9, 96(BP)
-	XORQ  R9, R9
-	MOVQ  BX, CX
-	ANDQ  $-16, CX
-
-openSSETail128LoopA:
-	ADDQ  (SI)(R9*1), R10
-	ADCQ  8(SI)(R9*1), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-
-openSSETail128LoopB:
-	ADDQ  $0x10, R9
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x0c
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x04
-	CMPQ  R9, CX
-	JB    openSSETail128LoopA
-	CMPQ  R9, $0xa0
-	JNE   openSSETail128LoopB
-	PADDL ·chacha20Constants<>+0(SB), X0
-	PADDL ·chacha20Constants<>+0(SB), X1
-	PADDL 32(BP), X3
-	PADDL 32(BP), X4
-	PADDL 48(BP), X6
-	PADDL 48(BP), X7
-	PADDL 96(BP), X9
-	PADDL 80(BP), X10
-	MOVOU (SI), X12
-	MOVOU 16(SI), X13
-	MOVOU 32(SI), X14
-	MOVOU 48(SI), X15
-	PXOR  X12, X1
-	PXOR  X13, X4
-	PXOR  X14, X7
-	PXOR  X15, X10
-	MOVOU X1, (DI)
-	MOVOU X4, 16(DI)
-	MOVOU X7, 32(DI)
-	MOVOU X10, 48(DI)
-	SUBQ  $0x40, BX
-	LEAQ  64(SI), SI
-	LEAQ  64(DI), DI
-	JMP   openSSETail64DecLoop
-
-openSSETail192:
-	MOVO    ·chacha20Constants<>+0(SB), X2
-	MOVO    32(BP), X5
-	MOVO    48(BP), X8
-	MOVO    128(BP), X11
-	PADDL   ·sseIncMask<>+0(SB), X11
-	MOVO    X11, 80(BP)
-	MOVO    X2, X1
-	MOVO    X5, X4
-	MOVO    X8, X7
-	MOVO    X11, X10
-	PADDL   ·sseIncMask<>+0(SB), X10
-	MOVO    X10, 96(BP)
-	MOVO    X1, X0
-	MOVO    X4, X3
-	MOVO    X7, X6
-	MOVO    X10, X9
-	PADDL   ·sseIncMask<>+0(SB), X9
-	MOVO    X9, 112(BP)
-	MOVQ    BX, CX
-	MOVQ    $0x000000a0, R9
-	CMPQ    CX, $0xa0
-	CMOVQGT R9, CX
-	ANDQ    $-16, CX
-	XORQ    R9, R9
-
-openSSLTail192LoopA:
-	ADDQ  (SI)(R9*1), R10
-	ADCQ  8(SI)(R9*1), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-
-openSSLTail192LoopB:
-	ADDQ  $0x10, R9
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X5
-	PXOR  X12, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X5
-	PXOR  X12, X5
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X5
-	PXOR  X12, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X5
-	PXOR  X12, X5
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	CMPQ  R9, CX
-	JB    openSSLTail192LoopA
-	CMPQ  R9, $0xa0
-	JNE   openSSLTail192LoopB
-	CMPQ  BX, $0xb0
-	JB    openSSLTail192Store
-	ADDQ  160(SI), R10
-	ADCQ  168(SI), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	CMPQ  BX, $0xc0
-	JB    openSSLTail192Store
-	ADDQ  176(SI), R10
-	ADCQ  184(SI), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-
-openSSLTail192Store:
-	PADDL ·chacha20Constants<>+0(SB), X0
-	PADDL ·chacha20Constants<>+0(SB), X1
-	PADDL ·chacha20Constants<>+0(SB), X2
-	PADDL 32(BP), X3
-	PADDL 32(BP), X4
-	PADDL 32(BP), X5
-	PADDL 48(BP), X6
-	PADDL 48(BP), X7
-	PADDL 48(BP), X8
-	PADDL 112(BP), X9
-	PADDL 96(BP), X10
-	PADDL 80(BP), X11
-	MOVOU (SI), X12
-	MOVOU 16(SI), X13
-	MOVOU 32(SI), X14
-	MOVOU 48(SI), X15
-	PXOR  X12, X2
-	PXOR  X13, X5
-	PXOR  X14, X8
-	PXOR  X15, X11
-	MOVOU X2, (DI)
-	MOVOU X5, 16(DI)
-	MOVOU X8, 32(DI)
-	MOVOU X11, 48(DI)
-	MOVOU 64(SI), X12
-	MOVOU 80(SI), X13
-	MOVOU 96(SI), X14
-	MOVOU 112(SI), X15
-	PXOR  X12, X1
-	PXOR  X13, X4
-	PXOR  X14, X7
-	PXOR  X15, X10
-	MOVOU X1, 64(DI)
-	MOVOU X4, 80(DI)
-	MOVOU X7, 96(DI)
-	MOVOU X10, 112(DI)
-	SUBQ  $0x80, BX
-	LEAQ  128(SI), SI
-	LEAQ  128(DI), DI
-	JMP   openSSETail64DecLoop
-
-openSSETail256:
-	MOVO  ·chacha20Constants<>+0(SB), X0
-	MOVO  32(BP), X3
-	MOVO  48(BP), X6
-	MOVO  128(BP), X9
-	PADDL ·sseIncMask<>+0(SB), X9
-	MOVO  X0, X1
-	MOVO  X3, X4
-	MOVO  X6, X7
-	MOVO  X9, X10
-	PADDL ·sseIncMask<>+0(SB), X10
-	MOVO  X1, X2
-	MOVO  X4, X5
-	MOVO  X7, X8
-	MOVO  X10, X11
-	PADDL ·sseIncMask<>+0(SB), X11
-	MOVO  X2, X12
-	MOVO  X5, X13
-	MOVO  X8, X14
-	MOVO  X11, X15
-	PADDL ·sseIncMask<>+0(SB), X15
-
-	// Store counters
-	MOVO X9, 80(BP)
-	MOVO X10, 96(BP)
-	MOVO X11, 112(BP)
-	MOVO X15, 128(BP)
-	XORQ R9, R9
-
-openSSETail256Loop:
-	ADDQ  (SI)(R9*1), R10
-	ADCQ  8(SI)(R9*1), R11
-	ADCQ  $0x01, R12
-	MOVO  X14, 64(BP)
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X3
-	PXOR  X14, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X3
-	PXOR  X14, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X4
-	PXOR  X14, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X4
-	PXOR  X14, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X5
-	PXOR  X14, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X5
-	PXOR  X14, X5
-	MOVO  64(BP), X14
-	MOVO  X7, 64(BP)
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL16(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x0c, X7
-	PSRLL $0x14, X13
-	PXOR  X7, X13
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL8(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x07, X7
-	PSRLL $0x19, X13
-	PXOR  X7, X13
-	MOVO  64(BP), X7
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x0c
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	MOVO  X14, 64(BP)
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X3
-	PXOR  X14, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X3
-	PXOR  X14, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X4
-	PXOR  X14, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X4
-	PXOR  X14, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X5
-	PXOR  X14, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X5
-	PXOR  X14, X5
-	MOVO  64(BP), X14
-	MOVO  X7, 64(BP)
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL16(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x0c, X7
-	PSRLL $0x14, X13
-	PXOR  X7, X13
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL8(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x07, X7
-	PSRLL $0x19, X13
-	PXOR  X7, X13
-	MOVO  64(BP), X7
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x04
-	ADDQ  $0x10, R9
-	CMPQ  R9, $0xa0
-	JB    openSSETail256Loop
-	MOVQ  BX, CX
-	ANDQ  $-16, CX
-
-openSSETail256HashLoop:
-	ADDQ  (SI)(R9*1), R10
-	ADCQ  8(SI)(R9*1), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	ADDQ  $0x10, R9
-	CMPQ  R9, CX
-	JB    openSSETail256HashLoop
-
-	// Add in the state
-	PADDD ·chacha20Constants<>+0(SB), X0
-	PADDD ·chacha20Constants<>+0(SB), X1
-	PADDD ·chacha20Constants<>+0(SB), X2
-	PADDD ·chacha20Constants<>+0(SB), X12
-	PADDD 32(BP), X3
-	PADDD 32(BP), X4
-	PADDD 32(BP), X5
-	PADDD 32(BP), X13
-	PADDD 48(BP), X6
-	PADDD 48(BP), X7
-	PADDD 48(BP), X8
-	PADDD 48(BP), X14
-	PADDD 80(BP), X9
-	PADDD 96(BP), X10
-	PADDD 112(BP), X11
-	PADDD 128(BP), X15
-	MOVO  X15, 64(BP)
-
-	// Load - xor - store
-	MOVOU (SI), X15
-	PXOR  X15, X0
-	MOVOU 16(SI), X15
-	PXOR  X15, X3
-	MOVOU 32(SI), X15
-	PXOR  X15, X6
-	MOVOU 48(SI), X15
-	PXOR  X15, X9
-	MOVOU X0, (DI)
-	MOVOU X3, 16(DI)
-	MOVOU X6, 32(DI)
-	MOVOU X9, 48(DI)
-	MOVOU 64(SI), X0
-	MOVOU 80(SI), X3
-	MOVOU 96(SI), X6
-	MOVOU 112(SI), X9
-	PXOR  X0, X1
-	PXOR  X3, X4
-	PXOR  X6, X7
-	PXOR  X9, X10
-	MOVOU X1, 64(DI)
-	MOVOU X4, 80(DI)
-	MOVOU X7, 96(DI)
-	MOVOU X10, 112(DI)
-	MOVOU 128(SI), X0
-	MOVOU 144(SI), X3
-	MOVOU 160(SI), X6
-	MOVOU 176(SI), X9
-	PXOR  X0, X2
-	PXOR  X3, X5
-	PXOR  X6, X8
-	PXOR  X9, X11
-	MOVOU X2, 128(DI)
-	MOVOU X5, 144(DI)
-	MOVOU X8, 160(DI)
-	MOVOU X11, 176(DI)
-	LEAQ  192(SI), SI
-	LEAQ  192(DI), DI
-	SUBQ  $0xc0, BX
-	MOVO  X12, X0
-	MOVO  X13, X3
-	MOVO  X14, X6
-	MOVO  64(BP), X9
-	JMP   openSSETail64DecLoop
-
-chacha20Poly1305Open_AVX2:
-	VZEROUPPER
-	VMOVDQU ·chacha20Constants<>+0(SB), Y0
-	BYTE    $0xc4
-	BYTE    $0x42
-	BYTE    $0x7d
-	BYTE    $0x5a
-	BYTE    $0x70
-	BYTE    $0x10
-	BYTE    $0xc4
-	BYTE    $0x42
-	BYTE    $0x7d
-	BYTE    $0x5a
-	BYTE    $0x60
-	BYTE    $0x20
-	BYTE    $0xc4
-	BYTE    $0xc2
-	BYTE    $0x7d
-	BYTE    $0x5a
-	BYTE    $0x60
-	BYTE    $0x30
-	VPADDD  ·avx2InitMask<>+0(SB), Y4, Y4
-
-	// Special optimization, for very short buffers
-	CMPQ BX, $0xc0
-	JBE  openAVX2192
-	CMPQ BX, $0x00000140
-	JBE  openAVX2320
-
-	// For the general key prepare the key first - as a byproduct we have 64 bytes of cipher stream
-	VMOVDQA Y14, 32(BP)
-	VMOVDQA Y12, 64(BP)
-	VMOVDQA Y4, 192(BP)
-	MOVQ    $0x0000000a, R9
-
-openAVX2PreparePolyKey:
-	VPADDD     Y14, Y0, Y0
-	VPXOR      Y0, Y4, Y4
-	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
-	VPADDD     Y4, Y12, Y12
-	VPXOR      Y12, Y14, Y14
-	VPSLLD     $0x0c, Y14, Y3
-	VPSRLD     $0x14, Y14, Y14
-	VPXOR      Y3, Y14, Y14
-	VPADDD     Y14, Y0, Y0
-	VPXOR      Y0, Y4, Y4
-	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
-	VPADDD     Y4, Y12, Y12
-	VPXOR      Y12, Y14, Y14
-	VPSLLD     $0x07, Y14, Y3
-	VPSRLD     $0x19, Y14, Y14
-	VPXOR      Y3, Y14, Y14
-	VPALIGNR   $0x04, Y14, Y14, Y14
-	VPALIGNR   $0x08, Y12, Y12, Y12
-	VPALIGNR   $0x0c, Y4, Y4, Y4
-	VPADDD     Y14, Y0, Y0
-	VPXOR      Y0, Y4, Y4
-	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
-	VPADDD     Y4, Y12, Y12
-	VPXOR      Y12, Y14, Y14
-	VPSLLD     $0x0c, Y14, Y3
-	VPSRLD     $0x14, Y14, Y14
-	VPXOR      Y3, Y14, Y14
-	VPADDD     Y14, Y0, Y0
-	VPXOR      Y0, Y4, Y4
-	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
-	VPADDD     Y4, Y12, Y12
-	VPXOR      Y12, Y14, Y14
-	VPSLLD     $0x07, Y14, Y3
-	VPSRLD     $0x19, Y14, Y14
-	VPXOR      Y3, Y14, Y14
-	VPALIGNR   $0x0c, Y14, Y14, Y14
-	VPALIGNR   $0x08, Y12, Y12, Y12
-	VPALIGNR   $0x04, Y4, Y4, Y4
-	DECQ       R9
-	JNE        openAVX2PreparePolyKey
-	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
-	VPADDD     32(BP), Y14, Y14
-	VPADDD     64(BP), Y12, Y12
-	VPADDD     192(BP), Y4, Y4
-	VPERM2I128 $0x02, Y0, Y14, Y3
-
-	// Clamp and store poly key
-	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
-	VMOVDQA Y3, (BP)
-
-	// Stream for the first 64 bytes
-	VPERM2I128 $0x13, Y0, Y14, Y0
-	VPERM2I128 $0x13, Y12, Y4, Y14
-
-	// Hash AD + first 64 bytes
-	MOVQ ad_len+80(FP), R9
-	CALL polyHashADInternal<>(SB)
-	XORQ CX, CX
-
-openAVX2InitialHash64:
-	ADDQ  (SI)(CX*1), R10
-	ADCQ  8(SI)(CX*1), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), DX
-	MOVQ  DX, R15
-	MULXQ R10, R13, R14
-	IMULQ R12, R15
-	MULXQ R11, AX, DX
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), DX
-	MULXQ R10, R10, AX
-	ADDQ  R10, R14
-	MULXQ R11, R11, R8
-	ADCQ  R11, R15
-	ADCQ  $0x00, R8
-	IMULQ R12, DX
-	ADDQ  AX, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	ADDQ  $0x10, CX
-	CMPQ  CX, $0x40
-	JNE   openAVX2InitialHash64
-
-	// Decrypt the first 64 bytes
-	VPXOR   (SI), Y0, Y0
-	VPXOR   32(SI), Y14, Y14
-	VMOVDQU Y0, (DI)
-	VMOVDQU Y14, 32(DI)
-	LEAQ    64(SI), SI
-	LEAQ    64(DI), DI
-	SUBQ    $0x40, BX
-
-openAVX2MainLoop:
-	CMPQ BX, $0x00000200
-	JB   openAVX2MainLoopDone
-
-	// Load state, increment counter blocks, store the incremented counters
-	VMOVDQU ·chacha20Constants<>+0(SB), Y0
-	VMOVDQA Y0, Y5
-	VMOVDQA Y0, Y6
-	VMOVDQA Y0, Y7
-	VMOVDQA 32(BP), Y14
-	VMOVDQA Y14, Y9
-	VMOVDQA Y14, Y10
-	VMOVDQA Y14, Y11
-	VMOVDQA 64(BP), Y12
-	VMOVDQA Y12, Y13
-	VMOVDQA Y12, Y8
-	VMOVDQA Y12, Y15
-	VMOVDQA 192(BP), Y4
-	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
-	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
-	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
-	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
-	VMOVDQA Y4, 96(BP)
-	VMOVDQA Y1, 128(BP)
-	VMOVDQA Y2, 160(BP)
-	VMOVDQA Y3, 192(BP)
-	XORQ    CX, CX
-
-openAVX2InternalLoop:
-	ADDQ     (SI)(CX*1), R10
-	ADCQ     8(SI)(CX*1), R11
-	ADCQ     $0x01, R12
-	VPADDD   Y14, Y0, Y0
-	VPADDD   Y9, Y5, Y5
-	VPADDD   Y10, Y6, Y6
-	VPADDD   Y11, Y7, Y7
-	MOVQ     (BP), DX
-	MOVQ     DX, R15
-	MULXQ    R10, R13, R14
-	IMULQ    R12, R15
-	MULXQ    R11, AX, DX
-	ADDQ     AX, R14
-	ADCQ     DX, R15
-	VPXOR    Y0, Y4, Y4
-	VPXOR    Y5, Y1, Y1
-	VPXOR    Y6, Y2, Y2
-	VPXOR    Y7, Y3, Y3
-	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
-	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
-	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
-	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
-	MOVQ     8(BP), DX
-	MULXQ    R10, R10, AX
-	ADDQ     R10, R14
-	MULXQ    R11, R11, R8
-	ADCQ     R11, R15
-	ADCQ     $0x00, R8
-	VPADDD   Y4, Y12, Y12
-	VPADDD   Y1, Y13, Y13
-	VPADDD   Y2, Y8, Y8
-	VPADDD   Y3, Y15, Y15
-	VPXOR    Y12, Y14, Y14
-	VPXOR    Y13, Y9, Y9
-	VPXOR    Y8, Y10, Y10
-	VPXOR    Y15, Y11, Y11
-	IMULQ    R12, DX
-	ADDQ     AX, R15
-	ADCQ     DX, R8
-	VMOVDQA  Y15, 224(BP)
-	VPSLLD   $0x0c, Y14, Y15
-	VPSRLD   $0x14, Y14, Y14
-	VPXOR    Y15, Y14, Y14
-	VPSLLD   $0x0c, Y9, Y15
-	VPSRLD   $0x14, Y9, Y9
-	VPXOR    Y15, Y9, Y9
-	VPSLLD   $0x0c, Y10, Y15
-	VPSRLD   $0x14, Y10, Y10
-	VPXOR    Y15, Y10, Y10
-	VPSLLD   $0x0c, Y11, Y15
-	VPSRLD   $0x14, Y11, Y11
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  224(BP), Y15
-	MOVQ     R13, R10
-	MOVQ     R14, R11
-	MOVQ     R15, R12
-	ANDQ     $0x03, R12
-	MOVQ     R15, R13
-	ANDQ     $-4, R13
-	MOVQ     R8, R14
-	SHRQ     $0x02, R8, R15
-	SHRQ     $0x02, R8
-	ADDQ     R13, R10
-	ADCQ     R14, R11
-	ADCQ     $0x00, R12
-	ADDQ     R15, R10
-	ADCQ     R8, R11
-	ADCQ     $0x00, R12
-	VPADDD   Y14, Y0, Y0
-	VPADDD   Y9, Y5, Y5
-	VPADDD   Y10, Y6, Y6
-	VPADDD   Y11, Y7, Y7
-	VPXOR    Y0, Y4, Y4
-	VPXOR    Y5, Y1, Y1
-	VPXOR    Y6, Y2, Y2
-	VPXOR    Y7, Y3, Y3
-	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
-	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
-	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
-	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
-	ADDQ     16(SI)(CX*1), R10
-	ADCQ     24(SI)(CX*1), R11
-	ADCQ     $0x01, R12
-	VPADDD   Y4, Y12, Y12
-	VPADDD   Y1, Y13, Y13
-	VPADDD   Y2, Y8, Y8
-	VPADDD   Y3, Y15, Y15
-	MOVQ     (BP), DX
-	MOVQ     DX, R15
-	MULXQ    R10, R13, R14
-	IMULQ    R12, R15
-	MULXQ    R11, AX, DX
-	ADDQ     AX, R14
-	ADCQ     DX, R15
-	VPXOR    Y12, Y14, Y14
-	VPXOR    Y13, Y9, Y9
-	VPXOR    Y8, Y10, Y10
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  Y15, 224(BP)
-	VPSLLD   $0x07, Y14, Y15
-	VPSRLD   $0x19, Y14, Y14
-	VPXOR    Y15, Y14, Y14
-	VPSLLD   $0x07, Y9, Y15
-	VPSRLD   $0x19, Y9, Y9
-	VPXOR    Y15, Y9, Y9
-	VPSLLD   $0x07, Y10, Y15
-	VPSRLD   $0x19, Y10, Y10
-	VPXOR    Y15, Y10, Y10
-	VPSLLD   $0x07, Y11, Y15
-	VPSRLD   $0x19, Y11, Y11
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  224(BP), Y15
-	MOVQ     8(BP), DX
-	MULXQ    R10, R10, AX
-	ADDQ     R10, R14
-	MULXQ    R11, R11, R8
-	ADCQ     R11, R15
-	ADCQ     $0x00, R8
-	VPALIGNR $0x04, Y14, Y14, Y14
-	VPALIGNR $0x04, Y9, Y9, Y9
-	VPALIGNR $0x04, Y10, Y10, Y10
-	VPALIGNR $0x04, Y11, Y11, Y11
-	VPALIGNR $0x08, Y12, Y12, Y12
-	VPALIGNR $0x08, Y13, Y13, Y13
-	VPALIGNR $0x08, Y8, Y8, Y8
-	VPALIGNR $0x08, Y15, Y15, Y15
-	VPALIGNR $0x0c, Y4, Y4, Y4
-	VPALIGNR $0x0c, Y1, Y1, Y1
-	VPALIGNR $0x0c, Y2, Y2, Y2
-	VPALIGNR $0x0c, Y3, Y3, Y3
-	VPADDD   Y14, Y0, Y0
-	VPADDD   Y9, Y5, Y5
-	VPADDD   Y10, Y6, Y6
-	VPADDD   Y11, Y7, Y7
-	IMULQ    R12, DX
-	ADDQ     AX, R15
-	ADCQ     DX, R8
-	VPXOR    Y0, Y4, Y4
-	VPXOR    Y5, Y1, Y1
-	VPXOR    Y6, Y2, Y2
-	VPXOR    Y7, Y3, Y3
-	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
-	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
-	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
-	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
-	MOVQ     R13, R10
-	MOVQ     R14, R11
-	MOVQ     R15, R12
-	ANDQ     $0x03, R12
-	MOVQ     R15, R13
-	ANDQ     $-4, R13
-	MOVQ     R8, R14
-	SHRQ     $0x02, R8, R15
-	SHRQ     $0x02, R8
-	ADDQ     R13, R10
-	ADCQ     R14, R11
-	ADCQ     $0x00, R12
-	ADDQ     R15, R10
-	ADCQ     R8, R11
-	ADCQ     $0x00, R12
-	VPADDD   Y4, Y12, Y12
-	VPADDD   Y1, Y13, Y13
-	VPADDD   Y2, Y8, Y8
-	VPADDD   Y3, Y15, Y15
-	VPXOR    Y12, Y14, Y14
-	VPXOR    Y13, Y9, Y9
-	VPXOR    Y8, Y10, Y10
-	VPXOR    Y15, Y11, Y11
-	ADDQ     32(SI)(CX*1), R10
-	ADCQ     40(SI)(CX*1), R11
-	ADCQ     $0x01, R12
-	LEAQ     48(CX), CX
-	VMOVDQA  Y15, 224(BP)
-	VPSLLD   $0x0c, Y14, Y15
-	VPSRLD   $0x14, Y14, Y14
-	VPXOR    Y15, Y14, Y14
-	VPSLLD   $0x0c, Y9, Y15
-	VPSRLD   $0x14, Y9, Y9
-	VPXOR    Y15, Y9, Y9
-	VPSLLD   $0x0c, Y10, Y15
-	VPSRLD   $0x14, Y10, Y10
-	VPXOR    Y15, Y10, Y10
-	VPSLLD   $0x0c, Y11, Y15
-	VPSRLD   $0x14, Y11, Y11
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  224(BP), Y15
-	MOVQ     (BP), DX
-	MOVQ     DX, R15
-	MULXQ    R10, R13, R14
-	IMULQ    R12, R15
-	MULXQ    R11, AX, DX
-	ADDQ     AX, R14
-	ADCQ     DX, R15
-	VPADDD   Y14, Y0, Y0
-	VPADDD   Y9, Y5, Y5
-	VPADDD   Y10, Y6, Y6
-	VPADDD   Y11, Y7, Y7
-	VPXOR    Y0, Y4, Y4
-	VPXOR    Y5, Y1, Y1
-	VPXOR    Y6, Y2, Y2
-	VPXOR    Y7, Y3, Y3
-	MOVQ     8(BP), DX
-	MULXQ    R10, R10, AX
-	ADDQ     R10, R14
-	MULXQ    R11, R11, R8
-	ADCQ     R11, R15
-	ADCQ     $0x00, R8
-	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
-	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
-	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
-	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
-	VPADDD   Y4, Y12, Y12
-	VPADDD   Y1, Y13, Y13
-	VPADDD   Y2, Y8, Y8
-	VPADDD   Y3, Y15, Y15
-	IMULQ    R12, DX
-	ADDQ     AX, R15
-	ADCQ     DX, R8
-	VPXOR    Y12, Y14, Y14
-	VPXOR    Y13, Y9, Y9
-	VPXOR    Y8, Y10, Y10
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  Y15, 224(BP)
-	VPSLLD   $0x07, Y14, Y15
-	VPSRLD   $0x19, Y14, Y14
-	VPXOR    Y15, Y14, Y14
-	VPSLLD   $0x07, Y9, Y15
-	VPSRLD   $0x19, Y9, Y9
-	VPXOR    Y15, Y9, Y9
-	VPSLLD   $0x07, Y10, Y15
-	VPSRLD   $0x19, Y10, Y10
-	VPXOR    Y15, Y10, Y10
-	VPSLLD   $0x07, Y11, Y15
-	VPSRLD   $0x19, Y11, Y11
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  224(BP), Y15
-	MOVQ     R13, R10
-	MOVQ     R14, R11
-	MOVQ     R15, R12
-	ANDQ     $0x03, R12
-	MOVQ     R15, R13
-	ANDQ     $-4, R13
-	MOVQ     R8, R14
-	SHRQ     $0x02, R8, R15
-	SHRQ     $0x02, R8
-	ADDQ     R13, R10
-	ADCQ     R14, R11
-	ADCQ     $0x00, R12
-	ADDQ     R15, R10
-	ADCQ     R8, R11
-	ADCQ     $0x00, R12
-	VPALIGNR $0x0c, Y14, Y14, Y14
-	VPALIGNR $0x0c, Y9, Y9, Y9
-	VPALIGNR $0x0c, Y10, Y10, Y10
-	VPALIGNR $0x0c, Y11, Y11, Y11
-	VPALIGNR $0x08, Y12, Y12, Y12
-	VPALIGNR $0x08, Y13, Y13, Y13
-	VPALIGNR $0x08, Y8, Y8, Y8
-	VPALIGNR $0x08, Y15, Y15, Y15
-	VPALIGNR $0x04, Y4, Y4, Y4
-	VPALIGNR $0x04, Y1, Y1, Y1
-	VPALIGNR $0x04, Y2, Y2, Y2
-	VPALIGNR $0x04, Y3, Y3, Y3
-	CMPQ     CX, $0x000001e0
-	JNE      openAVX2InternalLoop
-	VPADDD   ·chacha20Constants<>+0(SB), Y0, Y0
-	VPADDD   ·chacha20Constants<>+0(SB), Y5, Y5
-	VPADDD   ·chacha20Constants<>+0(SB), Y6, Y6
-	VPADDD   ·chacha20Constants<>+0(SB), Y7, Y7
-	VPADDD   32(BP), Y14, Y14
-	VPADDD   32(BP), Y9, Y9
-	VPADDD   32(BP), Y10, Y10
-	VPADDD   32(BP), Y11, Y11
-	VPADDD   64(BP), Y12, Y12
-	VPADDD   64(BP), Y13, Y13
-	VPADDD   64(BP), Y8, Y8
-	VPADDD   64(BP), Y15, Y15
-	VPADDD   96(BP), Y4, Y4
-	VPADDD   128(BP), Y1, Y1
-	VPADDD   160(BP), Y2, Y2
-	VPADDD   192(BP), Y3, Y3
-	VMOVDQA  Y15, 224(BP)
-
-	// We only hashed 480 of the 512 bytes available - hash the remaining 32 here
-	ADDQ       480(SI), R10
-	ADCQ       488(SI), R11
-	ADCQ       $0x01, R12
-	MOVQ       (BP), DX
-	MOVQ       DX, R15
-	MULXQ      R10, R13, R14
-	IMULQ      R12, R15
-	MULXQ      R11, AX, DX
-	ADDQ       AX, R14
-	ADCQ       DX, R15
-	MOVQ       8(BP), DX
-	MULXQ      R10, R10, AX
-	ADDQ       R10, R14
-	MULXQ      R11, R11, R8
-	ADCQ       R11, R15
-	ADCQ       $0x00, R8
-	IMULQ      R12, DX
-	ADDQ       AX, R15
-	ADCQ       DX, R8
-	MOVQ       R13, R10
-	MOVQ       R14, R11
-	MOVQ       R15, R12
-	ANDQ       $0x03, R12
-	MOVQ       R15, R13
-	ANDQ       $-4, R13
-	MOVQ       R8, R14
-	SHRQ       $0x02, R8, R15
-	SHRQ       $0x02, R8
-	ADDQ       R13, R10
-	ADCQ       R14, R11
-	ADCQ       $0x00, R12
-	ADDQ       R15, R10
-	ADCQ       R8, R11
-	ADCQ       $0x00, R12
-	VPERM2I128 $0x02, Y0, Y14, Y15
-	VPERM2I128 $0x13, Y0, Y14, Y14
-	VPERM2I128 $0x02, Y12, Y4, Y0
-	VPERM2I128 $0x13, Y12, Y4, Y12
-	VPXOR      (SI), Y15, Y15
-	VPXOR      32(SI), Y0, Y0
-	VPXOR      64(SI), Y14, Y14
-	VPXOR      96(SI), Y12, Y12
-	VMOVDQU    Y15, (DI)
-	VMOVDQU    Y0, 32(DI)
-	VMOVDQU    Y14, 64(DI)
-	VMOVDQU    Y12, 96(DI)
-	VPERM2I128 $0x02, Y5, Y9, Y0
-	VPERM2I128 $0x02, Y13, Y1, Y14
-	VPERM2I128 $0x13, Y5, Y9, Y12
-	VPERM2I128 $0x13, Y13, Y1, Y4
-	VPXOR      128(SI), Y0, Y0
-	VPXOR      160(SI), Y14, Y14
-	VPXOR      192(SI), Y12, Y12
-	VPXOR      224(SI), Y4, Y4
-	VMOVDQU    Y0, 128(DI)
-	VMOVDQU    Y14, 160(DI)
-	VMOVDQU    Y12, 192(DI)
-	VMOVDQU    Y4, 224(DI)
-
-	// and here
-	ADDQ       496(SI), R10
-	ADCQ       504(SI), R11
-	ADCQ       $0x01, R12
-	MOVQ       (BP), DX
-	MOVQ       DX, R15
-	MULXQ      R10, R13, R14
-	IMULQ      R12, R15
-	MULXQ      R11, AX, DX
-	ADDQ       AX, R14
-	ADCQ       DX, R15
-	MOVQ       8(BP), DX
-	MULXQ      R10, R10, AX
-	ADDQ       R10, R14
-	MULXQ      R11, R11, R8
-	ADCQ       R11, R15
-	ADCQ       $0x00, R8
-	IMULQ      R12, DX
-	ADDQ       AX, R15
-	ADCQ       DX, R8
-	MOVQ       R13, R10
-	MOVQ       R14, R11
-	MOVQ       R15, R12
-	ANDQ       $0x03, R12
-	MOVQ       R15, R13
-	ANDQ       $-4, R13
-	MOVQ       R8, R14
-	SHRQ       $0x02, R8, R15
-	SHRQ       $0x02, R8
-	ADDQ       R13, R10
-	ADCQ       R14, R11
-	ADCQ       $0x00, R12
-	ADDQ       R15, R10
-	ADCQ       R8, R11
-	ADCQ       $0x00, R12
-	VPERM2I128 $0x02, Y6, Y10, Y0
-	VPERM2I128 $0x02, Y8, Y2, Y14
-	VPERM2I128 $0x13, Y6, Y10, Y12
-	VPERM2I128 $0x13, Y8, Y2, Y4
-	VPXOR      256(SI), Y0, Y0
-	VPXOR      288(SI), Y14, Y14
-	VPXOR      320(SI), Y12, Y12
-	VPXOR      352(SI), Y4, Y4
-	VMOVDQU    Y0, 256(DI)
-	VMOVDQU    Y14, 288(DI)
-	VMOVDQU    Y12, 320(DI)
-	VMOVDQU    Y4, 352(DI)
-	VPERM2I128 $0x02, Y7, Y11, Y0
-	VPERM2I128 $0x02, 224(BP), Y3, Y14
-	VPERM2I128 $0x13, Y7, Y11, Y12
-	VPERM2I128 $0x13, 224(BP), Y3, Y4
-	VPXOR      384(SI), Y0, Y0
-	VPXOR      416(SI), Y14, Y14
-	VPXOR      448(SI), Y12, Y12
-	VPXOR      480(SI), Y4, Y4
-	VMOVDQU    Y0, 384(DI)
-	VMOVDQU    Y14, 416(DI)
-	VMOVDQU    Y12, 448(DI)
-	VMOVDQU    Y4, 480(DI)
-	LEAQ       512(SI), SI
-	LEAQ       512(DI), DI
-	SUBQ       $0x00000200, BX
-	JMP        openAVX2MainLoop
-
-openAVX2MainLoopDone:
-	// Handle the various tail sizes efficiently
-	TESTQ BX, BX
-	JE    openSSEFinalize
-	CMPQ  BX, $0x80
-	JBE   openAVX2Tail128
-	CMPQ  BX, $0x00000100
-	JBE   openAVX2Tail256
-	CMPQ  BX, $0x00000180
-	JBE   openAVX2Tail384
-	JMP   openAVX2Tail512
-
-openAVX2192:
-	VMOVDQA Y0, Y5
-	VMOVDQA Y14, Y9
-	VMOVDQA Y12, Y13
-	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
-	VMOVDQA Y0, Y6
-	VMOVDQA Y14, Y10
-	VMOVDQA Y12, Y8
-	VMOVDQA Y4, Y2
-	VMOVDQA Y1, Y15
-	MOVQ    $0x0000000a, R9
-
-openAVX2192InnerCipherLoop:
-	VPADDD     Y14, Y0, Y0
-	VPXOR      Y0, Y4, Y4
-	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
-	VPADDD     Y4, Y12, Y12
-	VPXOR      Y12, Y14, Y14
-	VPSLLD     $0x0c, Y14, Y3
-	VPSRLD     $0x14, Y14, Y14
-	VPXOR      Y3, Y14, Y14
-	VPADDD     Y14, Y0, Y0
-	VPXOR      Y0, Y4, Y4
-	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
-	VPADDD     Y4, Y12, Y12
-	VPXOR      Y12, Y14, Y14
-	VPSLLD     $0x07, Y14, Y3
-	VPSRLD     $0x19, Y14, Y14
-	VPXOR      Y3, Y14, Y14
-	VPADDD     Y9, Y5, Y5
-	VPXOR      Y5, Y1, Y1
-	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
-	VPADDD     Y1, Y13, Y13
-	VPXOR      Y13, Y9, Y9
-	VPSLLD     $0x0c, Y9, Y3
-	VPSRLD     $0x14, Y9, Y9
-	VPXOR      Y3, Y9, Y9
-	VPADDD     Y9, Y5, Y5
-	VPXOR      Y5, Y1, Y1
-	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
-	VPADDD     Y1, Y13, Y13
-	VPXOR      Y13, Y9, Y9
-	VPSLLD     $0x07, Y9, Y3
-	VPSRLD     $0x19, Y9, Y9
-	VPXOR      Y3, Y9, Y9
-	VPALIGNR   $0x04, Y14, Y14, Y14
-	VPALIGNR   $0x04, Y9, Y9, Y9
-	VPALIGNR   $0x08, Y12, Y12, Y12
-	VPALIGNR   $0x08, Y13, Y13, Y13
-	VPALIGNR   $0x0c, Y4, Y4, Y4
-	VPALIGNR   $0x0c, Y1, Y1, Y1
-	VPADDD     Y14, Y0, Y0
-	VPXOR      Y0, Y4, Y4
-	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
-	VPADDD     Y4, Y12, Y12
-	VPXOR      Y12, Y14, Y14
-	VPSLLD     $0x0c, Y14, Y3
-	VPSRLD     $0x14, Y14, Y14
-	VPXOR      Y3, Y14, Y14
-	VPADDD     Y14, Y0, Y0
-	VPXOR      Y0, Y4, Y4
-	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
-	VPADDD     Y4, Y12, Y12
-	VPXOR      Y12, Y14, Y14
-	VPSLLD     $0x07, Y14, Y3
-	VPSRLD     $0x19, Y14, Y14
-	VPXOR      Y3, Y14, Y14
-	VPADDD     Y9, Y5, Y5
-	VPXOR      Y5, Y1, Y1
-	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
-	VPADDD     Y1, Y13, Y13
-	VPXOR      Y13, Y9, Y9
-	VPSLLD     $0x0c, Y9, Y3
-	VPSRLD     $0x14, Y9, Y9
-	VPXOR      Y3, Y9, Y9
-	VPADDD     Y9, Y5, Y5
-	VPXOR      Y5, Y1, Y1
-	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
-	VPADDD     Y1, Y13, Y13
-	VPXOR      Y13, Y9, Y9
-	VPSLLD     $0x07, Y9, Y3
-	VPSRLD     $0x19, Y9, Y9
-	VPXOR      Y3, Y9, Y9
-	VPALIGNR   $0x0c, Y14, Y14, Y14
-	VPALIGNR   $0x0c, Y9, Y9, Y9
-	VPALIGNR   $0x08, Y12, Y12, Y12
-	VPALIGNR   $0x08, Y13, Y13, Y13
-	VPALIGNR   $0x04, Y4, Y4, Y4
-	VPALIGNR   $0x04, Y1, Y1, Y1
-	DECQ       R9
-	JNE        openAVX2192InnerCipherLoop
-	VPADDD     Y6, Y0, Y0
-	VPADDD     Y6, Y5, Y5
-	VPADDD     Y10, Y14, Y14
-	VPADDD     Y10, Y9, Y9
-	VPADDD     Y8, Y12, Y12
-	VPADDD     Y8, Y13, Y13
-	VPADDD     Y2, Y4, Y4
-	VPADDD     Y15, Y1, Y1
-	VPERM2I128 $0x02, Y0, Y14, Y3
-
-	// Clamp and store poly key
-	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
-	VMOVDQA Y3, (BP)
-
-	// Stream for up to 192 bytes
-	VPERM2I128 $0x13, Y0, Y14, Y0
-	VPERM2I128 $0x13, Y12, Y4, Y14
-	VPERM2I128 $0x02, Y5, Y9, Y12
-	VPERM2I128 $0x02, Y13, Y1, Y4
-	VPERM2I128 $0x13, Y5, Y9, Y5
-	VPERM2I128 $0x13, Y13, Y1, Y9
-
-openAVX2ShortOpen:
-	// Hash
-	MOVQ ad_len+80(FP), R9
-	CALL polyHashADInternal<>(SB)
-
-openAVX2ShortOpenLoop:
-	CMPQ BX, $0x20
-	JB   openAVX2ShortTail32
-	SUBQ $0x20, BX
-
-	// Load for hashing
-	ADDQ  (SI), R10
-	ADCQ  8(SI), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), DX
-	MOVQ  DX, R15
-	MULXQ R10, R13, R14
-	IMULQ R12, R15
-	MULXQ R11, AX, DX
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), DX
-	MULXQ R10, R10, AX
-	ADDQ  R10, R14
-	MULXQ R11, R11, R8
-	ADCQ  R11, R15
-	ADCQ  $0x00, R8
-	IMULQ R12, DX
-	ADDQ  AX, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	ADDQ  16(SI), R10
-	ADCQ  24(SI), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), DX
-	MOVQ  DX, R15
-	MULXQ R10, R13, R14
-	IMULQ R12, R15
-	MULXQ R11, AX, DX
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), DX
-	MULXQ R10, R10, AX
-	ADDQ  R10, R14
-	MULXQ R11, R11, R8
-	ADCQ  R11, R15
-	ADCQ  $0x00, R8
-	IMULQ R12, DX
-	ADDQ  AX, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-
-	// Load for decryption
-	VPXOR   (SI), Y0, Y0
-	VMOVDQU Y0, (DI)
-	LEAQ    32(SI), SI
-	LEAQ    32(DI), DI
-
-	// Shift stream left
-	VMOVDQA Y14, Y0
-	VMOVDQA Y12, Y14
-	VMOVDQA Y4, Y12
-	VMOVDQA Y5, Y4
-	VMOVDQA Y9, Y5
-	VMOVDQA Y13, Y9
-	VMOVDQA Y1, Y13
-	VMOVDQA Y6, Y1
-	VMOVDQA Y10, Y6
-	JMP     openAVX2ShortOpenLoop
-
-openAVX2ShortTail32:
-	CMPQ    BX, $0x10
-	VMOVDQA X0, X1
-	JB      openAVX2ShortDone
-	SUBQ    $0x10, BX
-
-	// Load for hashing
-	ADDQ  (SI), R10
-	ADCQ  8(SI), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), DX
-	MOVQ  DX, R15
-	MULXQ R10, R13, R14
-	IMULQ R12, R15
-	MULXQ R11, AX, DX
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), DX
-	MULXQ R10, R10, AX
-	ADDQ  R10, R14
-	MULXQ R11, R11, R8
-	ADCQ  R11, R15
-	ADCQ  $0x00, R8
-	IMULQ R12, DX
-	ADDQ  AX, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-
-	// Load for decryption
-	VPXOR      (SI), X0, X12
-	VMOVDQU    X12, (DI)
-	LEAQ       16(SI), SI
-	LEAQ       16(DI), DI
-	VPERM2I128 $0x11, Y0, Y0, Y0
-	VMOVDQA    X0, X1
-
-openAVX2ShortDone:
-	VZEROUPPER
-	JMP openSSETail16
-
-openAVX2320:
-	VMOVDQA Y0, Y5
-	VMOVDQA Y14, Y9
-	VMOVDQA Y12, Y13
-	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
-	VMOVDQA Y0, Y6
-	VMOVDQA Y14, Y10
-	VMOVDQA Y12, Y8
-	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
-	VMOVDQA Y14, Y7
-	VMOVDQA Y12, Y11
-	VMOVDQA Y4, Y15
-	MOVQ    $0x0000000a, R9
-
-openAVX2320InnerCipherLoop:
-	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x0c, Y14, Y3
-	VPSRLD   $0x14, Y14, Y14
-	VPXOR    Y3, Y14, Y14
-	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x07, Y14, Y3
-	VPSRLD   $0x19, Y14, Y14
-	VPXOR    Y3, Y14, Y14
-	VPADDD   Y9, Y5, Y5
-	VPXOR    Y5, Y1, Y1
-	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
-	VPADDD   Y1, Y13, Y13
-	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x0c, Y9, Y3
-	VPSRLD   $0x14, Y9, Y9
-	VPXOR    Y3, Y9, Y9
-	VPADDD   Y9, Y5, Y5
-	VPXOR    Y5, Y1, Y1
-	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
-	VPADDD   Y1, Y13, Y13
-	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x07, Y9, Y3
-	VPSRLD   $0x19, Y9, Y9
-	VPXOR    Y3, Y9, Y9
-	VPADDD   Y10, Y6, Y6
-	VPXOR    Y6, Y2, Y2
-	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
-	VPADDD   Y2, Y8, Y8
-	VPXOR    Y8, Y10, Y10
-	VPSLLD   $0x0c, Y10, Y3
-	VPSRLD   $0x14, Y10, Y10
-	VPXOR    Y3, Y10, Y10
-	VPADDD   Y10, Y6, Y6
-	VPXOR    Y6, Y2, Y2
-	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
-	VPADDD   Y2, Y8, Y8
-	VPXOR    Y8, Y10, Y10
-	VPSLLD   $0x07, Y10, Y3
-	VPSRLD   $0x19, Y10, Y10
-	VPXOR    Y3, Y10, Y10
-	VPALIGNR $0x04, Y14, Y14, Y14
-	VPALIGNR $0x04, Y9, Y9, Y9
-	VPALIGNR $0x04, Y10, Y10, Y10
-	VPALIGNR $0x08, Y12, Y12, Y12
-	VPALIGNR $0x08, Y13, Y13, Y13
-	VPALIGNR $0x08, Y8, Y8, Y8
-	VPALIGNR $0x0c, Y4, Y4, Y4
-	VPALIGNR $0x0c, Y1, Y1, Y1
-	VPALIGNR $0x0c, Y2, Y2, Y2
-	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x0c, Y14, Y3
-	VPSRLD   $0x14, Y14, Y14
-	VPXOR    Y3, Y14, Y14
-	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x07, Y14, Y3
-	VPSRLD   $0x19, Y14, Y14
-	VPXOR    Y3, Y14, Y14
-	VPADDD   Y9, Y5, Y5
-	VPXOR    Y5, Y1, Y1
-	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
-	VPADDD   Y1, Y13, Y13
-	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x0c, Y9, Y3
-	VPSRLD   $0x14, Y9, Y9
-	VPXOR    Y3, Y9, Y9
-	VPADDD   Y9, Y5, Y5
-	VPXOR    Y5, Y1, Y1
-	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
-	VPADDD   Y1, Y13, Y13
-	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x07, Y9, Y3
-	VPSRLD   $0x19, Y9, Y9
-	VPXOR    Y3, Y9, Y9
-	VPADDD   Y10, Y6, Y6
-	VPXOR    Y6, Y2, Y2
-	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
-	VPADDD   Y2, Y8, Y8
-	VPXOR    Y8, Y10, Y10
-	VPSLLD   $0x0c, Y10, Y3
-	VPSRLD   $0x14, Y10, Y10
-	VPXOR    Y3, Y10, Y10
-	VPADDD   Y10, Y6, Y6
-	VPXOR    Y6, Y2, Y2
-	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
-	VPADDD   Y2, Y8, Y8
-	VPXOR    Y8, Y10, Y10
-	VPSLLD   $0x07, Y10, Y3
-	VPSRLD   $0x19, Y10, Y10
-	VPXOR    Y3, Y10, Y10
-	VPALIGNR $0x0c, Y14, Y14, Y14
-	VPALIGNR $0x0c, Y9, Y9, Y9
-	VPALIGNR $0x0c, Y10, Y10, Y10
-	VPALIGNR $0x08, Y12, Y12, Y12
-	VPALIGNR $0x08, Y13, Y13, Y13
-	VPALIGNR $0x08, Y8, Y8, Y8
-	VPALIGNR $0x04, Y4, Y4, Y4
-	VPALIGNR $0x04, Y1, Y1, Y1
-	VPALIGNR $0x04, Y2, Y2, Y2
-	DECQ     R9
-	JNE      openAVX2320InnerCipherLoop
-	VMOVDQA  ·chacha20Constants<>+0(SB), Y3
-	VPADDD   Y3, Y0, Y0
-	VPADDD   Y3, Y5, Y5
-	VPADDD   Y3, Y6, Y6
-	VPADDD   Y7, Y14, Y14
-	VPADDD   Y7, Y9, Y9
-	VPADDD   Y7, Y10, Y10
-	VPADDD   Y11, Y12, Y12
-	VPADDD   Y11, Y13, Y13
-	VPADDD   Y11, Y8, Y8
-	VMOVDQA  ·avx2IncMask<>+0(SB), Y3
-	VPADDD   Y15, Y4, Y4
-	VPADDD   Y3, Y15, Y15
-	VPADDD   Y15, Y1, Y1
-	VPADDD   Y3, Y15, Y15
-	VPADDD   Y15, Y2, Y2
-
-	// Clamp and store poly key
-	VPERM2I128 $0x02, Y0, Y14, Y3
-	VPAND      ·polyClampMask<>+0(SB), Y3, Y3
-	VMOVDQA    Y3, (BP)
-
-	// Stream for up to 320 bytes
-	VPERM2I128 $0x13, Y0, Y14, Y0
-	VPERM2I128 $0x13, Y12, Y4, Y14
-	VPERM2I128 $0x02, Y5, Y9, Y12
-	VPERM2I128 $0x02, Y13, Y1, Y4
-	VPERM2I128 $0x13, Y5, Y9, Y5
-	VPERM2I128 $0x13, Y13, Y1, Y9
-	VPERM2I128 $0x02, Y6, Y10, Y13
-	VPERM2I128 $0x02, Y8, Y2, Y1
-	VPERM2I128 $0x13, Y6, Y10, Y6
-	VPERM2I128 $0x13, Y8, Y2, Y10
-	JMP        openAVX2ShortOpen
-
-openAVX2Tail128:
-	// Need to decrypt up to 128 bytes - prepare two blocks
-	VMOVDQA ·chacha20Constants<>+0(SB), Y5
-	VMOVDQA 32(BP), Y9
-	VMOVDQA 64(BP), Y13
-	VMOVDQA 192(BP), Y1
-	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y1
-	VMOVDQA Y1, Y4
-	XORQ    R9, R9
-	MOVQ    BX, CX
-	ANDQ    $-16, CX
-	TESTQ   CX, CX
-	JE      openAVX2Tail128LoopB
-
-openAVX2Tail128LoopA:
-	ADDQ  (SI)(R9*1), R10
-	ADCQ  8(SI)(R9*1), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), DX
-	MOVQ  DX, R15
-	MULXQ R10, R13, R14
-	IMULQ R12, R15
-	MULXQ R11, AX, DX
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), DX
-	MULXQ R10, R10, AX
-	ADDQ  R10, R14
-	MULXQ R11, R11, R8
-	ADCQ  R11, R15
-	ADCQ  $0x00, R8
-	IMULQ R12, DX
-	ADDQ  AX, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-
-openAVX2Tail128LoopB:
-	ADDQ       $0x10, R9
-	VPADDD     Y9, Y5, Y5
-	VPXOR      Y5, Y1, Y1
-	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
-	VPADDD     Y1, Y13, Y13
-	VPXOR      Y13, Y9, Y9
-	VPSLLD     $0x0c, Y9, Y3
-	VPSRLD     $0x14, Y9, Y9
-	VPXOR      Y3, Y9, Y9
-	VPADDD     Y9, Y5, Y5
-	VPXOR      Y5, Y1, Y1
-	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
-	VPADDD     Y1, Y13, Y13
-	VPXOR      Y13, Y9, Y9
-	VPSLLD     $0x07, Y9, Y3
-	VPSRLD     $0x19, Y9, Y9
-	VPXOR      Y3, Y9, Y9
-	VPALIGNR   $0x04, Y9, Y9, Y9
-	VPALIGNR   $0x08, Y13, Y13, Y13
-	VPALIGNR   $0x0c, Y1, Y1, Y1
-	VPADDD     Y9, Y5, Y5
-	VPXOR      Y5, Y1, Y1
-	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
-	VPADDD     Y1, Y13, Y13
-	VPXOR      Y13, Y9, Y9
-	VPSLLD     $0x0c, Y9, Y3
-	VPSRLD     $0x14, Y9, Y9
-	VPXOR      Y3, Y9, Y9
-	VPADDD     Y9, Y5, Y5
-	VPXOR      Y5, Y1, Y1
-	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
-	VPADDD     Y1, Y13, Y13
-	VPXOR      Y13, Y9, Y9
-	VPSLLD     $0x07, Y9, Y3
-	VPSRLD     $0x19, Y9, Y9
-	VPXOR      Y3, Y9, Y9
-	VPALIGNR   $0x0c, Y9, Y9, Y9
-	VPALIGNR   $0x08, Y13, Y13, Y13
-	VPALIGNR   $0x04, Y1, Y1, Y1
-	CMPQ       R9, CX
-	JB         openAVX2Tail128LoopA
-	CMPQ       R9, $0xa0
-	JNE        openAVX2Tail128LoopB
-	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
-	VPADDD     32(BP), Y9, Y9
-	VPADDD     64(BP), Y13, Y13
-	VPADDD     Y4, Y1, Y1
-	VPERM2I128 $0x02, Y5, Y9, Y0
-	VPERM2I128 $0x02, Y13, Y1, Y14
-	VPERM2I128 $0x13, Y5, Y9, Y12
-	VPERM2I128 $0x13, Y13, Y1, Y4
-
-openAVX2TailLoop:
-	CMPQ BX, $0x20
-	JB   openAVX2Tail
-	SUBQ $0x20, BX
-
-	// Load for decryption
-	VPXOR   (SI), Y0, Y0
-	VMOVDQU Y0, (DI)
-	LEAQ    32(SI), SI
-	LEAQ    32(DI), DI
-	VMOVDQA Y14, Y0
-	VMOVDQA Y12, Y14
-	VMOVDQA Y4, Y12
-	JMP     openAVX2TailLoop
-
-openAVX2Tail:
-	CMPQ    BX, $0x10
-	VMOVDQA X0, X1
-	JB      openAVX2TailDone
-	SUBQ    $0x10, BX
-
-	// Load for decryption
-	VPXOR      (SI), X0, X12
-	VMOVDQU    X12, (DI)
-	LEAQ       16(SI), SI
-	LEAQ       16(DI), DI
-	VPERM2I128 $0x11, Y0, Y0, Y0
-	VMOVDQA    X0, X1
-
-openAVX2TailDone:
-	VZEROUPPER
-	JMP openSSETail16
-
-openAVX2Tail256:
-	VMOVDQA ·chacha20Constants<>+0(SB), Y0
-	VMOVDQA Y0, Y5
-	VMOVDQA 32(BP), Y14
-	VMOVDQA Y14, Y9
-	VMOVDQA 64(BP), Y12
-	VMOVDQA Y12, Y13
-	VMOVDQA 192(BP), Y4
-	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
-	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
-	VMOVDQA Y4, Y7
-	VMOVDQA Y1, Y11
-
-	// Compute the number of iterations that will hash data
-	MOVQ    BX, 224(BP)
-	MOVQ    BX, CX
-	SUBQ    $0x80, CX
-	SHRQ    $0x04, CX
-	MOVQ    $0x0000000a, R9
-	CMPQ    CX, $0x0a
-	CMOVQGT R9, CX
-	MOVQ    SI, BX
-	XORQ    R9, R9
-
-openAVX2Tail256LoopA:
-	ADDQ  (BX), R10
-	ADCQ  8(BX), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), DX
-	MOVQ  DX, R15
-	MULXQ R10, R13, R14
-	IMULQ R12, R15
-	MULXQ R11, AX, DX
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), DX
-	MULXQ R10, R10, AX
-	ADDQ  R10, R14
-	MULXQ R11, R11, R8
-	ADCQ  R11, R15
-	ADCQ  $0x00, R8
-	IMULQ R12, DX
-	ADDQ  AX, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	LEAQ  16(BX), BX
-
-openAVX2Tail256LoopB:
-	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x0c, Y14, Y3
-	VPSRLD   $0x14, Y14, Y14
-	VPXOR    Y3, Y14, Y14
+openAVX2InternalLoop:
+	ADDQ     (SI)(CX*1), R10
+	ADCQ     8(SI)(CX*1), R11
+	ADCQ     $0x01, R12
 	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x07, Y14, Y3
-	VPSRLD   $0x19, Y14, Y14
-	VPXOR    Y3, Y14, Y14
 	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPXOR    Y0, Y4, Y4
 	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
 	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPADDD   Y4, Y12, Y12
 	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
 	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x0c, Y9, Y3
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
 	VPSRLD   $0x14, Y9, Y9
-	VPXOR    Y3, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPADDD   Y14, Y0, Y0
 	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
 	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
 	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	ADDQ     16(SI)(CX*1), R10
+	ADCQ     24(SI)(CX*1), R11
+	ADCQ     $0x01, R12
+	VPADDD   Y4, Y12, Y12
 	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPXOR    Y12, Y14, Y14
 	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x07, Y9, Y3
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
 	VPSRLD   $0x19, Y9, Y9
-	VPXOR    Y3, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
 	VPALIGNR $0x04, Y14, Y14, Y14
 	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x04, Y11, Y11, Y11
 	VPALIGNR $0x08, Y12, Y12, Y12
 	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
 	VPALIGNR $0x0c, Y4, Y4, Y4
 	VPALIGNR $0x0c, Y1, Y1, Y1
-	INCQ     R9
-	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x0c, Y14, Y3
-	VPSRLD   $0x14, Y14, Y14
-	VPXOR    Y3, Y14, Y14
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPALIGNR $0x0c, Y3, Y3, Y3
 	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x07, Y14, Y3
-	VPSRLD   $0x19, Y14, Y14
-	VPXOR    Y3, Y14, Y14
 	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VPXOR    Y0, Y4, Y4
 	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
 	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPADDD   Y4, Y12, Y12
 	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
 	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x0c, Y9, Y3
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	ADDQ     32(SI)(CX*1), R10
+	ADCQ     40(SI)(CX*1), R11
+	ADCQ     $0x01, R12
+	LEAQ     48(CX), CX
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
 	VPSRLD   $0x14, Y9, Y9
-	VPXOR    Y3, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	VPADDD   Y14, Y0, Y0
 	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
 	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
 	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
 	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	VPXOR    Y12, Y14, Y14
 	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x07, Y9, Y3
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
 	VPSRLD   $0x19, Y9, Y9
-	VPXOR    Y3, Y9, Y9
-	VPALIGNR $0x0c, Y14, Y14, Y14
-	VPALIGNR $0x0c, Y9, Y9, Y9
-	VPALIGNR $0x08, Y12, Y12, Y12
-	VPALIGNR $0x08, Y13, Y13, Y13
-	VPALIGNR $0x04, Y4, Y4, Y4
-	VPALIGNR $0x04, Y1, Y1, Y1
-	CMPQ     R9, CX
-	JB       openAVX2Tail256LoopA
-	CMPQ     R9, $0x0a
-	JNE      openAVX2Tail256LoopB
-	MOVQ     BX, R9
-	SUBQ     SI, BX
-	MOVQ     BX, CX
-	MOVQ     224(BP), BX
-
-openAVX2Tail256Hash:
-	ADDQ  $0x10, CX
-	CMPQ  CX, BX
-	JGT   openAVX2Tail256HashEnd
-	ADDQ  (R9), R10
-	ADCQ  8(R9), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), DX
-	MOVQ  DX, R15
-	MULXQ R10, R13, R14
-	IMULQ R12, R15
-	MULXQ R11, AX, DX
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), DX
-	MULXQ R10, R10, AX
-	ADDQ  R10, R14
-	MULXQ R11, R11, R8
-	ADCQ  R11, R15
-	ADCQ  $0x00, R8
-	IMULQ R12, DX
-	ADDQ  AX, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	LEAQ  16(R9), R9
-	JMP   openAVX2Tail256Hash
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x0c, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	VPALIGNR $0x04, Y3, Y3, Y3
+	CMPQ     CX, $0x000001e0
+	JNE      openAVX2InternalLoop
+	VPADDD   ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD   ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD   ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD   ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD   32(BP), Y14, Y14
+	VPADDD   32(BP), Y9, Y9
+	VPADDD   32(BP), Y10, Y10
+	VPADDD   32(BP), Y11, Y11
+	VPADDD   64(BP), Y12, Y12
+	VPADDD   64(BP), Y13, Y13
+	VPADDD   64(BP), Y8, Y8
+	VPADDD   64(BP), Y15, Y15
+	VPADDD   96(BP), Y4, Y4
+	VPADDD   128(BP), Y1, Y1
+	VPADDD   160(BP), Y2, Y2
+	VPADDD   192(BP), Y3, Y3
+	VMOVDQA  Y15, 224(BP)
 
-openAVX2Tail256HashEnd:
-	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
-	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
-	VPADDD     32(BP), Y14, Y14
-	VPADDD     32(BP), Y9, Y9
-	VPADDD     64(BP), Y12, Y12
-	VPADDD     64(BP), Y13, Y13
-	VPADDD     Y7, Y4, Y4
-	VPADDD     Y11, Y1, Y1
-	VPERM2I128 $0x02, Y0, Y14, Y6
-	VPERM2I128 $0x02, Y12, Y4, Y10
-	VPERM2I128 $0x13, Y0, Y14, Y8
-	VPERM2I128 $0x13, Y12, Y4, Y2
+	// We only hashed 480 of the 512 bytes available - hash the remaining 32 here
+	ADDQ       480(SI), R10
+	ADCQ       488(SI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPERM2I128 $0x02, Y0, Y14, Y15
+	VPERM2I128 $0x13, Y0, Y14, Y14
+	VPERM2I128 $0x02, Y12, Y4, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y12
+	VPXOR      (SI), Y15, Y15
+	VPXOR      32(SI), Y0, Y0
+	VPXOR      64(SI), Y14, Y14
+	VPXOR      96(SI), Y12, Y12
+	VMOVDQU    Y15, (DI)
+	VMOVDQU    Y0, 32(DI)
+	VMOVDQU    Y14, 64(DI)
+	VMOVDQU    Y12, 96(DI)
 	VPERM2I128 $0x02, Y5, Y9, Y0
 	VPERM2I128 $0x02, Y13, Y1, Y14
 	VPERM2I128 $0x13, Y5, Y9, Y12
 	VPERM2I128 $0x13, Y13, Y1, Y4
-	VPXOR      (SI), Y6, Y6
-	VPXOR      32(SI), Y10, Y10
-	VPXOR      64(SI), Y8, Y8
-	VPXOR      96(SI), Y2, Y2
-	VMOVDQU    Y6, (DI)
-	VMOVDQU    Y10, 32(DI)
-	VMOVDQU    Y8, 64(DI)
-	VMOVDQU    Y2, 96(DI)
-	LEAQ       128(SI), SI
-	LEAQ       128(DI), DI
-	SUBQ       $0x80, BX
-	JMP        openAVX2TailLoop
+	VPXOR      128(SI), Y0, Y0
+	VPXOR      160(SI), Y14, Y14
+	VPXOR      192(SI), Y12, Y12
+	VPXOR      224(SI), Y4, Y4
+	VMOVDQU    Y0, 128(DI)
+	VMOVDQU    Y14, 160(DI)
+	VMOVDQU    Y12, 192(DI)
+	VMOVDQU    Y4, 224(DI)
 
-openAVX2Tail384:
-	// Need to decrypt up to 384 bytes - prepare six blocks
-	VMOVDQA ·chacha20Constants<>+0(SB), Y0
-	VMOVDQA Y0, Y5
-	VMOVDQA Y0, Y6
-	VMOVDQA 32(BP), Y14
-	VMOVDQA Y14, Y9
-	VMOVDQA Y14, Y10
-	VMOVDQA 64(BP), Y12
-	VMOVDQA Y12, Y13
-	VMOVDQA Y12, Y8
-	VMOVDQA 192(BP), Y4
-	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
-	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
-	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
-	VMOVDQA Y4, 96(BP)
-	VMOVDQA Y1, 128(BP)
-	VMOVDQA Y2, 160(BP)
+	// and here
+	ADDQ       496(SI), R10
+	ADCQ       504(SI), R11
+	ADCQ       $0x01, R12
+	MOVQ       (BP), DX
+	MOVQ       DX, R15
+	MULXQ      R10, R13, R14
+	IMULQ      R12, R15
+	MULXQ      R11, AX, DX
+	ADDQ       AX, R14
+	ADCQ       DX, R15
+	MOVQ       8(BP), DX
+	MULXQ      R10, R10, AX
+	ADDQ       R10, R14
+	MULXQ      R11, R11, R8
+	ADCQ       R11, R15
+	ADCQ       $0x00, R8
+	IMULQ      R12, DX
+	ADDQ       AX, R15
+	ADCQ       DX, R8
+	MOVQ       R13, R10
+	MOVQ       R14, R11
+	MOVQ       R15, R12
+	ANDQ       $0x03, R12
+	MOVQ       R15, R13
+	ANDQ       $-4, R13
+	MOVQ       R8, R14
+	SHRQ       $0x02, R8, R15
+	SHRQ       $0x02, R8
+	ADDQ       R13, R10
+	ADCQ       R14, R11
+	ADCQ       $0x00, R12
+	ADDQ       R15, R10
+	ADCQ       R8, R11
+	ADCQ       $0x00, R12
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      256(SI), Y0, Y0
+	VPXOR      288(SI), Y14, Y14
+	VPXOR      320(SI), Y12, Y12
+	VPXOR      352(SI), Y4, Y4
+	VMOVDQU    Y0, 256(DI)
+	VMOVDQU    Y14, 288(DI)
+	VMOVDQU    Y12, 320(DI)
+	VMOVDQU    Y4, 352(DI)
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, 224(BP), Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, 224(BP), Y3, Y4
+	VPXOR      384(SI), Y0, Y0
+	VPXOR      416(SI), Y14, Y14
+	VPXOR      448(SI), Y12, Y12
+	VPXOR      480(SI), Y4, Y4
+	VMOVDQU    Y0, 384(DI)
+	VMOVDQU    Y14, 416(DI)
+	VMOVDQU    Y12, 448(DI)
+	VMOVDQU    Y4, 480(DI)
+	LEAQ       512(SI), SI
+	LEAQ       512(DI), DI
+	SUBQ       $0x00000200, BX
+	JMP        openAVX2MainLoop
 
-	// Compute the number of iterations that will hash two blocks of data
-	MOVQ    BX, 224(BP)
-	MOVQ    BX, CX
-	SUBQ    $0x00000100, CX
-	SHRQ    $0x04, CX
-	ADDQ    $0x06, CX
-	MOVQ    $0x0000000a, R9
-	CMPQ    CX, $0x0a
-	CMOVQGT R9, CX
-	MOVQ    SI, BX
-	XORQ    R9, R9
+openAVX2MainLoopDone:
+	// Handle the various tail sizes efficiently
+	TESTQ BX, BX
+	JE    openSSEFinalize
+	CMPQ  BX, $0x80
+	JBE   openAVX2Tail128
+	CMPQ  BX, $0x00000100
+	JBE   openAVX2Tail256
+	CMPQ  BX, $0x00000180
+	JBE   openAVX2Tail384
+	JMP   openAVX2Tail512
 
-openAVX2Tail384LoopB:
-	ADDQ  (BX), R10
-	ADCQ  8(BX), R11
+openSSEFinalize:
+	// Hash in the PT, AAD lengths
+	ADDQ  ad_len+80(FP), R10
+	ADCQ  src_len+56(FP), R11
 	ADCQ  $0x01, R12
-	MOVQ  (BP), DX
-	MOVQ  DX, R15
-	MULXQ R10, R13, R14
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
 	IMULQ R12, R15
-	MULXQ R11, AX, DX
 	ADDQ  AX, R14
 	ADCQ  DX, R15
-	MOVQ  8(BP), DX
-	MULXQ R10, R10, AX
-	ADDQ  R10, R14
-	MULXQ R11, R11, R8
-	ADCQ  R11, R15
-	ADCQ  $0x00, R8
-	IMULQ R12, DX
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
 	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
 	ADCQ  DX, R8
 	MOVQ  R13, R10
 	MOVQ  R14, R11
@@ -4075,174 +797,261 @@ openAVX2Tail384LoopB:
 	ADDQ  R15, R10
 	ADCQ  R8, R11
 	ADCQ  $0x00, R12
-	LEAQ  16(BX), BX
 
-openAVX2Tail384LoopA:
-	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x0c, Y14, Y3
-	VPSRLD   $0x14, Y14, Y14
-	VPXOR    Y3, Y14, Y14
-	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x07, Y14, Y3
-	VPSRLD   $0x19, Y14, Y14
-	VPXOR    Y3, Y14, Y14
-	VPADDD   Y9, Y5, Y5
-	VPXOR    Y5, Y1, Y1
-	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
-	VPADDD   Y1, Y13, Y13
-	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x0c, Y9, Y3
-	VPSRLD   $0x14, Y9, Y9
-	VPXOR    Y3, Y9, Y9
-	VPADDD   Y9, Y5, Y5
-	VPXOR    Y5, Y1, Y1
-	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
-	VPADDD   Y1, Y13, Y13
-	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x07, Y9, Y3
-	VPSRLD   $0x19, Y9, Y9
-	VPXOR    Y3, Y9, Y9
-	VPADDD   Y10, Y6, Y6
-	VPXOR    Y6, Y2, Y2
-	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
-	VPADDD   Y2, Y8, Y8
-	VPXOR    Y8, Y10, Y10
-	VPSLLD   $0x0c, Y10, Y3
-	VPSRLD   $0x14, Y10, Y10
-	VPXOR    Y3, Y10, Y10
-	VPADDD   Y10, Y6, Y6
-	VPXOR    Y6, Y2, Y2
-	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
-	VPADDD   Y2, Y8, Y8
-	VPXOR    Y8, Y10, Y10
-	VPSLLD   $0x07, Y10, Y3
-	VPSRLD   $0x19, Y10, Y10
-	VPXOR    Y3, Y10, Y10
-	VPALIGNR $0x04, Y14, Y14, Y14
-	VPALIGNR $0x04, Y9, Y9, Y9
-	VPALIGNR $0x04, Y10, Y10, Y10
-	VPALIGNR $0x08, Y12, Y12, Y12
-	VPALIGNR $0x08, Y13, Y13, Y13
-	VPALIGNR $0x08, Y8, Y8, Y8
-	VPALIGNR $0x0c, Y4, Y4, Y4
-	VPALIGNR $0x0c, Y1, Y1, Y1
-	VPALIGNR $0x0c, Y2, Y2, Y2
-	ADDQ     (BX), R10
-	ADCQ     8(BX), R11
-	ADCQ     $0x01, R12
-	MOVQ     (BP), DX
-	MOVQ     DX, R15
-	MULXQ    R10, R13, R14
-	IMULQ    R12, R15
-	MULXQ    R11, AX, DX
-	ADDQ     AX, R14
-	ADCQ     DX, R15
-	MOVQ     8(BP), DX
-	MULXQ    R10, R10, AX
-	ADDQ     R10, R14
-	MULXQ    R11, R11, R8
-	ADCQ     R11, R15
-	ADCQ     $0x00, R8
-	IMULQ    R12, DX
-	ADDQ     AX, R15
-	ADCQ     DX, R8
-	MOVQ     R13, R10
-	MOVQ     R14, R11
-	MOVQ     R15, R12
-	ANDQ     $0x03, R12
-	MOVQ     R15, R13
-	ANDQ     $-4, R13
-	MOVQ     R8, R14
-	SHRQ     $0x02, R8, R15
-	SHRQ     $0x02, R8
-	ADDQ     R13, R10
-	ADCQ     R14, R11
-	ADCQ     $0x00, R12
-	ADDQ     R15, R10
-	ADCQ     R8, R11
-	ADCQ     $0x00, R12
-	LEAQ     16(BX), BX
-	INCQ     R9
-	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x0c, Y14, Y3
-	VPSRLD   $0x14, Y14, Y14
-	VPXOR    Y3, Y14, Y14
-	VPADDD   Y14, Y0, Y0
-	VPXOR    Y0, Y4, Y4
-	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
-	VPADDD   Y4, Y12, Y12
-	VPXOR    Y12, Y14, Y14
-	VPSLLD   $0x07, Y14, Y3
-	VPSRLD   $0x19, Y14, Y14
-	VPXOR    Y3, Y14, Y14
-	VPADDD   Y9, Y5, Y5
-	VPXOR    Y5, Y1, Y1
-	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
-	VPADDD   Y1, Y13, Y13
-	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x0c, Y9, Y3
-	VPSRLD   $0x14, Y9, Y9
-	VPXOR    Y3, Y9, Y9
-	VPADDD   Y9, Y5, Y5
-	VPXOR    Y5, Y1, Y1
-	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
-	VPADDD   Y1, Y13, Y13
-	VPXOR    Y13, Y9, Y9
-	VPSLLD   $0x07, Y9, Y3
-	VPSRLD   $0x19, Y9, Y9
-	VPXOR    Y3, Y9, Y9
-	VPADDD   Y10, Y6, Y6
-	VPXOR    Y6, Y2, Y2
-	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
-	VPADDD   Y2, Y8, Y8
-	VPXOR    Y8, Y10, Y10
-	VPSLLD   $0x0c, Y10, Y3
-	VPSRLD   $0x14, Y10, Y10
-	VPXOR    Y3, Y10, Y10
-	VPADDD   Y10, Y6, Y6
-	VPXOR    Y6, Y2, Y2
-	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
-	VPADDD   Y2, Y8, Y8
-	VPXOR    Y8, Y10, Y10
-	VPSLLD   $0x07, Y10, Y3
-	VPSRLD   $0x19, Y10, Y10
-	VPXOR    Y3, Y10, Y10
-	VPALIGNR $0x0c, Y14, Y14, Y14
-	VPALIGNR $0x0c, Y9, Y9, Y9
-	VPALIGNR $0x0c, Y10, Y10, Y10
-	VPALIGNR $0x08, Y12, Y12, Y12
-	VPALIGNR $0x08, Y13, Y13, Y13
-	VPALIGNR $0x08, Y8, Y8, Y8
-	VPALIGNR $0x04, Y4, Y4, Y4
-	VPALIGNR $0x04, Y1, Y1, Y1
-	VPALIGNR $0x04, Y2, Y2, Y2
-	CMPQ     R9, CX
-	JB       openAVX2Tail384LoopB
-	CMPQ     R9, $0x0a
-	JNE      openAVX2Tail384LoopA
-	MOVQ     BX, R9
-	SUBQ     SI, BX
-	MOVQ     BX, CX
-	MOVQ     224(BP), BX
+	// Final reduce
+	MOVQ    R10, R13
+	MOVQ    R11, R14
+	MOVQ    R12, R15
+	SUBQ    $-5, R10
+	SBBQ    $-1, R11
+	SBBQ    $0x03, R12
+	CMOVQCS R13, R10
+	CMOVQCS R14, R11
+	CMOVQCS R15, R12
+
+	// Add in the "s" part of the key
+	ADDQ 16(BP), R10
+	ADCQ 24(BP), R11
+
+	// Finally, constant time compare to the tag at the end of the message
+	XORQ    AX, AX
+	MOVQ    $0x00000001, DX
+	XORQ    (SI), R10
+	XORQ    8(SI), R11
+	ORQ     R11, R10
+	CMOVQEQ DX, AX
+
+	// Return true iff tags are equal
+	MOVB AX, ret+96(FP)
+	RET
+
+openSSETail16:
+	TESTQ BX, BX
+	JE    openSSEFinalize
+
+	// We can safely load the CT from the end, because it is padded with the MAC
+	MOVQ  BX, R9
+	SHLQ  $0x04, R9
+	LEAQ  ·andMask<>+0(SB), R13
+	MOVOU (SI), X12
+	ADDQ  BX, SI
+	PAND  -16(R13)(R9*1), X12
+	MOVO  X12, 64(BP)
+	MOVQ  X12, R13
+	MOVQ  72(BP), R14
+	PXOR  X1, X12
+
+	// We can only store one byte at a time, since plaintext can be shorter than 16 bytes
+openSSETail16Store:
+	MOVQ   X12, R8
+	MOVB   R8, (DI)
+	PSRLDQ $0x01, X12
+	INCQ   DI
+	DECQ   BX
+	JNE    openSSETail16Store
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x01, R12
+	MOVQ   (BP), AX
+	MOVQ   AX, R15
+	MULQ   R10
+	MOVQ   AX, R13
+	MOVQ   DX, R14
+	MOVQ   (BP), AX
+	MULQ   R11
+	IMULQ  R12, R15
+	ADDQ   AX, R14
+	ADCQ   DX, R15
+	MOVQ   8(BP), AX
+	MOVQ   AX, R8
+	MULQ   R10
+	ADDQ   AX, R14
+	ADCQ   $0x00, DX
+	MOVQ   DX, R10
+	MOVQ   8(BP), AX
+	MULQ   R11
+	ADDQ   AX, R15
+	ADCQ   $0x00, DX
+	IMULQ  R12, R8
+	ADDQ   R10, R15
+	ADCQ   DX, R8
+	MOVQ   R13, R10
+	MOVQ   R14, R11
+	MOVQ   R15, R12
+	ANDQ   $0x03, R12
+	MOVQ   R15, R13
+	ANDQ   $-4, R13
+	MOVQ   R8, R14
+	SHRQ   $0x02, R8, R15
+	SHRQ   $0x02, R8
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x00, R12
+	ADDQ   R15, R10
+	ADCQ   R8, R11
+	ADCQ   $0x00, R12
+	JMP    openSSEFinalize
 
-openAVX2Tail384Hash:
-	ADDQ  $0x10, CX
-	CMPQ  CX, BX
-	JGT   openAVX2Tail384HashEnd
-	ADDQ  (R9), R10
-	ADCQ  8(R9), R11
+openAVX2192:
+	VMOVDQA Y0, Y5
+	VMOVDQA Y14, Y9
+	VMOVDQA Y12, Y13
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y0, Y6
+	VMOVDQA Y14, Y10
+	VMOVDQA Y12, Y8
+	VMOVDQA Y4, Y2
+	VMOVDQA Y1, Y15
+	MOVQ    $0x0000000a, R9
+
+openAVX2192InnerCipherLoop:
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x04, Y14, Y14, Y14
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y4, Y4, Y4
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol16<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x0c, Y14, Y3
+	VPSRLD     $0x14, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y14, Y0, Y0
+	VPXOR      Y0, Y4, Y4
+	VPSHUFB    ·rol8<>+0(SB), Y4, Y4
+	VPADDD     Y4, Y12, Y12
+	VPXOR      Y12, Y14, Y14
+	VPSLLD     $0x07, Y14, Y3
+	VPSRLD     $0x19, Y14, Y14
+	VPXOR      Y3, Y14, Y14
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x0c, Y14, Y14, Y14
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y12, Y12, Y12
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y4, Y4, Y4
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	DECQ       R9
+	JNE        openAVX2192InnerCipherLoop
+	VPADDD     Y6, Y0, Y0
+	VPADDD     Y6, Y5, Y5
+	VPADDD     Y10, Y14, Y14
+	VPADDD     Y10, Y9, Y9
+	VPADDD     Y8, Y12, Y12
+	VPADDD     Y8, Y13, Y13
+	VPADDD     Y2, Y4, Y4
+	VPADDD     Y15, Y1, Y1
+	VPERM2I128 $0x02, Y0, Y14, Y3
+
+	// Clamp and store poly key
+	VPAND   ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA Y3, (BP)
+
+	// Stream for up to 192 bytes
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
+	VPERM2I128 $0x02, Y5, Y9, Y12
+	VPERM2I128 $0x02, Y13, Y1, Y4
+	VPERM2I128 $0x13, Y5, Y9, Y5
+	VPERM2I128 $0x13, Y13, Y1, Y9
+
+openAVX2ShortOpen:
+	// Hash
+	MOVQ ad_len+80(FP), R9
+	CALL polyHashADInternal<>(SB)
+
+openAVX2ShortOpenLoop:
+	CMPQ BX, $0x20
+	JB   openAVX2ShortTail32
+	SUBQ $0x20, BX
+
+	// Load for hashing
+	ADDQ  (SI), R10
+	ADCQ  8(SI), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
+	IMULQ R12, R15
+	MULXQ R11, AX, DX
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
+	ADDQ  AX, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	ADDQ  16(SI), R10
+	ADCQ  24(SI), R11
 	ADCQ  $0x01, R12
 	MOVQ  (BP), DX
 	MOVQ  DX, R15
@@ -4275,83 +1084,34 @@ openAVX2Tail384Hash:
 	ADDQ  R15, R10
 	ADCQ  R8, R11
 	ADCQ  $0x00, R12
-	LEAQ  16(R9), R9
-	JMP   openAVX2Tail384Hash
 
-openAVX2Tail384HashEnd:
-	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
-	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
-	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
-	VPADDD     32(BP), Y14, Y14
-	VPADDD     32(BP), Y9, Y9
-	VPADDD     32(BP), Y10, Y10
-	VPADDD     64(BP), Y12, Y12
-	VPADDD     64(BP), Y13, Y13
-	VPADDD     64(BP), Y8, Y8
-	VPADDD     96(BP), Y4, Y4
-	VPADDD     128(BP), Y1, Y1
-	VPADDD     160(BP), Y2, Y2
-	VPERM2I128 $0x02, Y0, Y14, Y3
-	VPERM2I128 $0x02, Y12, Y4, Y7
-	VPERM2I128 $0x13, Y0, Y14, Y11
-	VPERM2I128 $0x13, Y12, Y4, Y15
-	VPXOR      (SI), Y3, Y3
-	VPXOR      32(SI), Y7, Y7
-	VPXOR      64(SI), Y11, Y11
-	VPXOR      96(SI), Y15, Y15
-	VMOVDQU    Y3, (DI)
-	VMOVDQU    Y7, 32(DI)
-	VMOVDQU    Y11, 64(DI)
-	VMOVDQU    Y15, 96(DI)
-	VPERM2I128 $0x02, Y5, Y9, Y3
-	VPERM2I128 $0x02, Y13, Y1, Y7
-	VPERM2I128 $0x13, Y5, Y9, Y11
-	VPERM2I128 $0x13, Y13, Y1, Y15
-	VPXOR      128(SI), Y3, Y3
-	VPXOR      160(SI), Y7, Y7
-	VPXOR      192(SI), Y11, Y11
-	VPXOR      224(SI), Y15, Y15
-	VMOVDQU    Y3, 128(DI)
-	VMOVDQU    Y7, 160(DI)
-	VMOVDQU    Y11, 192(DI)
-	VMOVDQU    Y15, 224(DI)
-	VPERM2I128 $0x02, Y6, Y10, Y0
-	VPERM2I128 $0x02, Y8, Y2, Y14
-	VPERM2I128 $0x13, Y6, Y10, Y12
-	VPERM2I128 $0x13, Y8, Y2, Y4
-	LEAQ       256(SI), SI
-	LEAQ       256(DI), DI
-	SUBQ       $0x00000100, BX
-	JMP        openAVX2TailLoop
+	// Load for decryption
+	VPXOR   (SI), Y0, Y0
+	VMOVDQU Y0, (DI)
+	LEAQ    32(SI), SI
+	LEAQ    32(DI), DI
 
-openAVX2Tail512:
-	VMOVDQU ·chacha20Constants<>+0(SB), Y0
-	VMOVDQA Y0, Y5
-	VMOVDQA Y0, Y6
-	VMOVDQA Y0, Y7
-	VMOVDQA 32(BP), Y14
-	VMOVDQA Y14, Y9
-	VMOVDQA Y14, Y10
-	VMOVDQA Y14, Y11
-	VMOVDQA 64(BP), Y12
-	VMOVDQA Y12, Y13
-	VMOVDQA Y12, Y8
-	VMOVDQA Y12, Y15
-	VMOVDQA 192(BP), Y4
-	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
-	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
-	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
-	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
-	VMOVDQA Y4, 96(BP)
-	VMOVDQA Y1, 128(BP)
-	VMOVDQA Y2, 160(BP)
-	VMOVDQA Y3, 192(BP)
-	XORQ    CX, CX
-	MOVQ    SI, R9
+	// Shift stream left
+	VMOVDQA Y14, Y0
+	VMOVDQA Y12, Y14
+	VMOVDQA Y4, Y12
+	VMOVDQA Y5, Y4
+	VMOVDQA Y9, Y5
+	VMOVDQA Y13, Y9
+	VMOVDQA Y1, Y13
+	VMOVDQA Y6, Y1
+	VMOVDQA Y10, Y6
+	JMP     openAVX2ShortOpenLoop
+
+openAVX2ShortTail32:
+	CMPQ    BX, $0x10
+	VMOVDQA X0, X1
+	JB      openAVX2ShortDone
+	SUBQ    $0x10, BX
 
-openAVX2Tail512LoopB:
-	ADDQ  (R9), R10
-	ADCQ  8(R9), R11
+	// Load for hashing
+	ADDQ  (SI), R10
+	ADCQ  8(SI), R11
 	ADCQ  $0x01, R12
 	MOVQ  (BP), DX
 	MOVQ  DX, R15
@@ -4384,252 +1144,202 @@ openAVX2Tail512LoopB:
 	ADDQ  R15, R10
 	ADCQ  R8, R11
 	ADCQ  $0x00, R12
-	LEAQ  16(R9), R9
 
-openAVX2Tail512LoopA:
+	// Load for decryption
+	VPXOR      (SI), X0, X12
+	VMOVDQU    X12, (DI)
+	LEAQ       16(SI), SI
+	LEAQ       16(DI), DI
+	VPERM2I128 $0x11, Y0, Y0, Y0
+	VMOVDQA    X0, X1
+
+openAVX2ShortDone:
+	VZEROUPPER
+	JMP openSSETail16
+
+openAVX2320:
+	VMOVDQA Y0, Y5
+	VMOVDQA Y14, Y9
+	VMOVDQA Y12, Y13
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y0, Y6
+	VMOVDQA Y14, Y10
+	VMOVDQA Y12, Y8
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y14, Y7
+	VMOVDQA Y12, Y11
+	VMOVDQA Y4, Y15
+	MOVQ    $0x0000000a, R9
+
+openAVX2320InnerCipherLoop:
 	VPADDD   Y14, Y0, Y0
-	VPADDD   Y9, Y5, Y5
-	VPADDD   Y10, Y6, Y6
-	VPADDD   Y11, Y7, Y7
 	VPXOR    Y0, Y4, Y4
-	VPXOR    Y5, Y1, Y1
-	VPXOR    Y6, Y2, Y2
-	VPXOR    Y7, Y3, Y3
 	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
-	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
-	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
-	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
 	VPADDD   Y4, Y12, Y12
-	VPADDD   Y1, Y13, Y13
-	VPADDD   Y2, Y8, Y8
-	VPADDD   Y3, Y15, Y15
 	VPXOR    Y12, Y14, Y14
-	VPXOR    Y13, Y9, Y9
-	VPXOR    Y8, Y10, Y10
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  Y15, 224(BP)
-	VPSLLD   $0x0c, Y14, Y15
+	VPSLLD   $0x0c, Y14, Y3
 	VPSRLD   $0x14, Y14, Y14
-	VPXOR    Y15, Y14, Y14
-	VPSLLD   $0x0c, Y9, Y15
-	VPSRLD   $0x14, Y9, Y9
-	VPXOR    Y15, Y9, Y9
-	VPSLLD   $0x0c, Y10, Y15
-	VPSRLD   $0x14, Y10, Y10
-	VPXOR    Y15, Y10, Y10
-	VPSLLD   $0x0c, Y11, Y15
-	VPSRLD   $0x14, Y11, Y11
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  224(BP), Y15
-	ADDQ     (R9), R10
-	ADCQ     8(R9), R11
-	ADCQ     $0x01, R12
-	MOVQ     (BP), DX
-	MOVQ     DX, R15
-	MULXQ    R10, R13, R14
-	IMULQ    R12, R15
-	MULXQ    R11, AX, DX
-	ADDQ     AX, R14
-	ADCQ     DX, R15
-	MOVQ     8(BP), DX
-	MULXQ    R10, R10, AX
-	ADDQ     R10, R14
-	MULXQ    R11, R11, R8
-	ADCQ     R11, R15
-	ADCQ     $0x00, R8
-	IMULQ    R12, DX
-	ADDQ     AX, R15
-	ADCQ     DX, R8
-	MOVQ     R13, R10
-	MOVQ     R14, R11
-	MOVQ     R15, R12
-	ANDQ     $0x03, R12
-	MOVQ     R15, R13
-	ANDQ     $-4, R13
-	MOVQ     R8, R14
-	SHRQ     $0x02, R8, R15
-	SHRQ     $0x02, R8
-	ADDQ     R13, R10
-	ADCQ     R14, R11
-	ADCQ     $0x00, R12
-	ADDQ     R15, R10
-	ADCQ     R8, R11
-	ADCQ     $0x00, R12
+	VPXOR    Y3, Y14, Y14
 	VPADDD   Y14, Y0, Y0
-	VPADDD   Y9, Y5, Y5
-	VPADDD   Y10, Y6, Y6
-	VPADDD   Y11, Y7, Y7
 	VPXOR    Y0, Y4, Y4
-	VPXOR    Y5, Y1, Y1
-	VPXOR    Y6, Y2, Y2
-	VPXOR    Y7, Y3, Y3
 	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
-	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
-	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
-	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
 	VPADDD   Y4, Y12, Y12
-	VPADDD   Y1, Y13, Y13
-	VPADDD   Y2, Y8, Y8
-	VPADDD   Y3, Y15, Y15
 	VPXOR    Y12, Y14, Y14
-	VPXOR    Y13, Y9, Y9
-	VPXOR    Y8, Y10, Y10
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  Y15, 224(BP)
-	VPSLLD   $0x07, Y14, Y15
+	VPSLLD   $0x07, Y14, Y3
 	VPSRLD   $0x19, Y14, Y14
-	VPXOR    Y15, Y14, Y14
-	VPSLLD   $0x07, Y9, Y15
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
 	VPSRLD   $0x19, Y9, Y9
-	VPXOR    Y15, Y9, Y9
-	VPSLLD   $0x07, Y10, Y15
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
 	VPSRLD   $0x19, Y10, Y10
-	VPXOR    Y15, Y10, Y10
-	VPSLLD   $0x07, Y11, Y15
-	VPSRLD   $0x19, Y11, Y11
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  224(BP), Y15
+	VPXOR    Y3, Y10, Y10
 	VPALIGNR $0x04, Y14, Y14, Y14
 	VPALIGNR $0x04, Y9, Y9, Y9
 	VPALIGNR $0x04, Y10, Y10, Y10
-	VPALIGNR $0x04, Y11, Y11, Y11
 	VPALIGNR $0x08, Y12, Y12, Y12
 	VPALIGNR $0x08, Y13, Y13, Y13
 	VPALIGNR $0x08, Y8, Y8, Y8
-	VPALIGNR $0x08, Y15, Y15, Y15
 	VPALIGNR $0x0c, Y4, Y4, Y4
 	VPALIGNR $0x0c, Y1, Y1, Y1
 	VPALIGNR $0x0c, Y2, Y2, Y2
-	VPALIGNR $0x0c, Y3, Y3, Y3
 	VPADDD   Y14, Y0, Y0
-	VPADDD   Y9, Y5, Y5
-	VPADDD   Y10, Y6, Y6
-	VPADDD   Y11, Y7, Y7
 	VPXOR    Y0, Y4, Y4
-	VPXOR    Y5, Y1, Y1
-	VPXOR    Y6, Y2, Y2
-	VPXOR    Y7, Y3, Y3
 	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
-	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
-	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
-	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
 	VPADDD   Y4, Y12, Y12
-	VPADDD   Y1, Y13, Y13
-	VPADDD   Y2, Y8, Y8
-	VPADDD   Y3, Y15, Y15
 	VPXOR    Y12, Y14, Y14
-	VPXOR    Y13, Y9, Y9
-	VPXOR    Y8, Y10, Y10
-	VPXOR    Y15, Y11, Y11
-	ADDQ     16(R9), R10
-	ADCQ     24(R9), R11
-	ADCQ     $0x01, R12
-	MOVQ     (BP), DX
-	MOVQ     DX, R15
-	MULXQ    R10, R13, R14
-	IMULQ    R12, R15
-	MULXQ    R11, AX, DX
-	ADDQ     AX, R14
-	ADCQ     DX, R15
-	MOVQ     8(BP), DX
-	MULXQ    R10, R10, AX
-	ADDQ     R10, R14
-	MULXQ    R11, R11, R8
-	ADCQ     R11, R15
-	ADCQ     $0x00, R8
-	IMULQ    R12, DX
-	ADDQ     AX, R15
-	ADCQ     DX, R8
-	MOVQ     R13, R10
-	MOVQ     R14, R11
-	MOVQ     R15, R12
-	ANDQ     $0x03, R12
-	MOVQ     R15, R13
-	ANDQ     $-4, R13
-	MOVQ     R8, R14
-	SHRQ     $0x02, R8, R15
-	SHRQ     $0x02, R8
-	ADDQ     R13, R10
-	ADCQ     R14, R11
-	ADCQ     $0x00, R12
-	ADDQ     R15, R10
-	ADCQ     R8, R11
-	ADCQ     $0x00, R12
-	LEAQ     32(R9), R9
-	VMOVDQA  Y15, 224(BP)
-	VPSLLD   $0x0c, Y14, Y15
+	VPSLLD   $0x0c, Y14, Y3
 	VPSRLD   $0x14, Y14, Y14
-	VPXOR    Y15, Y14, Y14
-	VPSLLD   $0x0c, Y9, Y15
-	VPSRLD   $0x14, Y9, Y9
-	VPXOR    Y15, Y9, Y9
-	VPSLLD   $0x0c, Y10, Y15
-	VPSRLD   $0x14, Y10, Y10
-	VPXOR    Y15, Y10, Y10
-	VPSLLD   $0x0c, Y11, Y15
-	VPSRLD   $0x14, Y11, Y11
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  224(BP), Y15
+	VPXOR    Y3, Y14, Y14
 	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
 	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
 	VPADDD   Y10, Y6, Y6
-	VPADDD   Y11, Y7, Y7
-	VPXOR    Y0, Y4, Y4
-	VPXOR    Y5, Y1, Y1
 	VPXOR    Y6, Y2, Y2
-	VPXOR    Y7, Y3, Y3
-	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
-	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
 	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
-	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
-	VPADDD   Y4, Y12, Y12
-	VPADDD   Y1, Y13, Y13
 	VPADDD   Y2, Y8, Y8
-	VPADDD   Y3, Y15, Y15
-	VPXOR    Y12, Y14, Y14
-	VPXOR    Y13, Y9, Y9
 	VPXOR    Y8, Y10, Y10
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  Y15, 224(BP)
-	VPSLLD   $0x07, Y14, Y15
-	VPSRLD   $0x19, Y14, Y14
-	VPXOR    Y15, Y14, Y14
-	VPSLLD   $0x07, Y9, Y15
-	VPSRLD   $0x19, Y9, Y9
-	VPXOR    Y15, Y9, Y9
-	VPSLLD   $0x07, Y10, Y15
+	VPSLLD   $0x07, Y10, Y3
 	VPSRLD   $0x19, Y10, Y10
-	VPXOR    Y15, Y10, Y10
-	VPSLLD   $0x07, Y11, Y15
-	VPSRLD   $0x19, Y11, Y11
-	VPXOR    Y15, Y11, Y11
-	VMOVDQA  224(BP), Y15
+	VPXOR    Y3, Y10, Y10
 	VPALIGNR $0x0c, Y14, Y14, Y14
 	VPALIGNR $0x0c, Y9, Y9, Y9
 	VPALIGNR $0x0c, Y10, Y10, Y10
-	VPALIGNR $0x0c, Y11, Y11, Y11
 	VPALIGNR $0x08, Y12, Y12, Y12
 	VPALIGNR $0x08, Y13, Y13, Y13
 	VPALIGNR $0x08, Y8, Y8, Y8
-	VPALIGNR $0x08, Y15, Y15, Y15
 	VPALIGNR $0x04, Y4, Y4, Y4
 	VPALIGNR $0x04, Y1, Y1, Y1
 	VPALIGNR $0x04, Y2, Y2, Y2
-	VPALIGNR $0x04, Y3, Y3, Y3
-	INCQ     CX
-	CMPQ     CX, $0x04
-	JLT      openAVX2Tail512LoopB
-	CMPQ     CX, $0x0a
-	JNE      openAVX2Tail512LoopA
-	MOVQ     BX, CX
-	SUBQ     $0x00000180, CX
-	ANDQ     $-16, CX
+	DECQ     R9
+	JNE      openAVX2320InnerCipherLoop
+	VMOVDQA  ·chacha20Constants<>+0(SB), Y3
+	VPADDD   Y3, Y0, Y0
+	VPADDD   Y3, Y5, Y5
+	VPADDD   Y3, Y6, Y6
+	VPADDD   Y7, Y14, Y14
+	VPADDD   Y7, Y9, Y9
+	VPADDD   Y7, Y10, Y10
+	VPADDD   Y11, Y12, Y12
+	VPADDD   Y11, Y13, Y13
+	VPADDD   Y11, Y8, Y8
+	VMOVDQA  ·avx2IncMask<>+0(SB), Y3
+	VPADDD   Y15, Y4, Y4
+	VPADDD   Y3, Y15, Y15
+	VPADDD   Y15, Y1, Y1
+	VPADDD   Y3, Y15, Y15
+	VPADDD   Y15, Y2, Y2
 
-openAVX2Tail512HashLoop:
-	TESTQ CX, CX
-	JE    openAVX2Tail512HashEnd
-	ADDQ  (R9), R10
-	ADCQ  8(R9), R11
+	// Clamp and store poly key
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPAND      ·polyClampMask<>+0(SB), Y3, Y3
+	VMOVDQA    Y3, (BP)
+
+	// Stream for up to 320 bytes
+	VPERM2I128 $0x13, Y0, Y14, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y14
+	VPERM2I128 $0x02, Y5, Y9, Y12
+	VPERM2I128 $0x02, Y13, Y1, Y4
+	VPERM2I128 $0x13, Y5, Y9, Y5
+	VPERM2I128 $0x13, Y13, Y1, Y9
+	VPERM2I128 $0x02, Y6, Y10, Y13
+	VPERM2I128 $0x02, Y8, Y2, Y1
+	VPERM2I128 $0x13, Y6, Y10, Y6
+	VPERM2I128 $0x13, Y8, Y2, Y10
+	JMP        openAVX2ShortOpen
+
+openAVX2Tail128:
+	// Need to decrypt up to 128 bytes - prepare two blocks
+	VMOVDQA ·chacha20Constants<>+0(SB), Y5
+	VMOVDQA 32(BP), Y9
+	VMOVDQA 64(BP), Y13
+	VMOVDQA 192(BP), Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y1
+	VMOVDQA Y1, Y4
+	XORQ    R9, R9
+	MOVQ    BX, CX
+	ANDQ    $-16, CX
+	TESTQ   CX, CX
+	JE      openAVX2Tail128LoopB
+
+openAVX2Tail128LoopA:
+	ADDQ  (SI)(R9*1), R10
+	ADCQ  8(SI)(R9*1), R11
 	ADCQ  $0x01, R12
 	MOVQ  (BP), DX
 	MOVQ  DX, R15
@@ -4662,884 +1372,137 @@ openAVX2Tail512HashLoop:
 	ADDQ  R15, R10
 	ADCQ  R8, R11
 	ADCQ  $0x00, R12
-	LEAQ  16(R9), R9
-	SUBQ  $0x10, CX
-	JMP   openAVX2Tail512HashLoop
 
-openAVX2Tail512HashEnd:
-	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+openAVX2Tail128LoopB:
+	ADDQ       $0x10, R9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x04, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x0c, Y1, Y1, Y1
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol16<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x0c, Y9, Y3
+	VPSRLD     $0x14, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPADDD     Y9, Y5, Y5
+	VPXOR      Y5, Y1, Y1
+	VPSHUFB    ·rol8<>+0(SB), Y1, Y1
+	VPADDD     Y1, Y13, Y13
+	VPXOR      Y13, Y9, Y9
+	VPSLLD     $0x07, Y9, Y3
+	VPSRLD     $0x19, Y9, Y9
+	VPXOR      Y3, Y9, Y9
+	VPALIGNR   $0x0c, Y9, Y9, Y9
+	VPALIGNR   $0x08, Y13, Y13, Y13
+	VPALIGNR   $0x04, Y1, Y1, Y1
+	CMPQ       R9, CX
+	JB         openAVX2Tail128LoopA
+	CMPQ       R9, $0xa0
+	JNE        openAVX2Tail128LoopB
 	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
-	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
-	VPADDD     ·chacha20Constants<>+0(SB), Y7, Y7
-	VPADDD     32(BP), Y14, Y14
 	VPADDD     32(BP), Y9, Y9
-	VPADDD     32(BP), Y10, Y10
-	VPADDD     32(BP), Y11, Y11
-	VPADDD     64(BP), Y12, Y12
 	VPADDD     64(BP), Y13, Y13
-	VPADDD     64(BP), Y8, Y8
-	VPADDD     64(BP), Y15, Y15
-	VPADDD     96(BP), Y4, Y4
-	VPADDD     128(BP), Y1, Y1
-	VPADDD     160(BP), Y2, Y2
-	VPADDD     192(BP), Y3, Y3
-	VMOVDQA    Y15, 224(BP)
-	VPERM2I128 $0x02, Y0, Y14, Y15
-	VPERM2I128 $0x13, Y0, Y14, Y14
-	VPERM2I128 $0x02, Y12, Y4, Y0
-	VPERM2I128 $0x13, Y12, Y4, Y12
-	VPXOR      (SI), Y15, Y15
-	VPXOR      32(SI), Y0, Y0
-	VPXOR      64(SI), Y14, Y14
-	VPXOR      96(SI), Y12, Y12
-	VMOVDQU    Y15, (DI)
-	VMOVDQU    Y0, 32(DI)
-	VMOVDQU    Y14, 64(DI)
-	VMOVDQU    Y12, 96(DI)
-	VPERM2I128 $0x02, Y5, Y9, Y0
-	VPERM2I128 $0x02, Y13, Y1, Y14
-	VPERM2I128 $0x13, Y5, Y9, Y12
-	VPERM2I128 $0x13, Y13, Y1, Y4
-	VPXOR      128(SI), Y0, Y0
-	VPXOR      160(SI), Y14, Y14
-	VPXOR      192(SI), Y12, Y12
-	VPXOR      224(SI), Y4, Y4
-	VMOVDQU    Y0, 128(DI)
-	VMOVDQU    Y14, 160(DI)
-	VMOVDQU    Y12, 192(DI)
-	VMOVDQU    Y4, 224(DI)
-	VPERM2I128 $0x02, Y6, Y10, Y0
-	VPERM2I128 $0x02, Y8, Y2, Y14
-	VPERM2I128 $0x13, Y6, Y10, Y12
-	VPERM2I128 $0x13, Y8, Y2, Y4
-	VPXOR      256(SI), Y0, Y0
-	VPXOR      288(SI), Y14, Y14
-	VPXOR      320(SI), Y12, Y12
-	VPXOR      352(SI), Y4, Y4
-	VMOVDQU    Y0, 256(DI)
-	VMOVDQU    Y14, 288(DI)
-	VMOVDQU    Y12, 320(DI)
-	VMOVDQU    Y4, 352(DI)
-	VPERM2I128 $0x02, Y7, Y11, Y0
-	VPERM2I128 $0x02, 224(BP), Y3, Y14
-	VPERM2I128 $0x13, Y7, Y11, Y12
-	VPERM2I128 $0x13, 224(BP), Y3, Y4
-	LEAQ       384(SI), SI
-	LEAQ       384(DI), DI
-	SUBQ       $0x00000180, BX
-	JMP        openAVX2TailLoop
-
-DATA ·chacha20Constants<>+0(SB)/4, $0x61707865
-DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e
-DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32
-DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574
-DATA ·chacha20Constants<>+16(SB)/4, $0x61707865
-DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e
-DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32
-DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574
-GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32
-
-DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff
-DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc
-DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff
-DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff
-GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32
-
-DATA ·sseIncMask<>+0(SB)/8, $0x0000000000000001
-DATA ·sseIncMask<>+8(SB)/8, $0x0000000000000000
-GLOBL ·sseIncMask<>(SB), RODATA|NOPTR, $16
-
-DATA ·andMask<>+0(SB)/8, $0x00000000000000ff
-DATA ·andMask<>+8(SB)/8, $0x0000000000000000
-DATA ·andMask<>+16(SB)/8, $0x000000000000ffff
-DATA ·andMask<>+24(SB)/8, $0x0000000000000000
-DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff
-DATA ·andMask<>+40(SB)/8, $0x0000000000000000
-DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff
-DATA ·andMask<>+56(SB)/8, $0x0000000000000000
-DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff
-DATA ·andMask<>+72(SB)/8, $0x0000000000000000
-DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff
-DATA ·andMask<>+88(SB)/8, $0x0000000000000000
-DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff
-DATA ·andMask<>+104(SB)/8, $0x0000000000000000
-DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+120(SB)/8, $0x0000000000000000
-DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+136(SB)/8, $0x00000000000000ff
-DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+152(SB)/8, $0x000000000000ffff
-DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff
-DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff
-DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff
-DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff
-DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff
-DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff
-GLOBL ·andMask<>(SB), RODATA|NOPTR, $240
+	VPADDD     Y4, Y1, Y1
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
 
-DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000
-DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000
-DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001
-DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000
-GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32
+openAVX2TailLoop:
+	CMPQ BX, $0x20
+	JB   openAVX2Tail
+	SUBQ $0x20, BX
 
-DATA ·rol16<>+0(SB)/8, $0x0504070601000302
-DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a
-DATA ·rol16<>+16(SB)/8, $0x0504070601000302
-DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a
-GLOBL ·rol16<>(SB), RODATA|NOPTR, $32
+	// Load for decryption
+	VPXOR   (SI), Y0, Y0
+	VMOVDQU Y0, (DI)
+	LEAQ    32(SI), SI
+	LEAQ    32(DI), DI
+	VMOVDQA Y14, Y0
+	VMOVDQA Y12, Y14
+	VMOVDQA Y4, Y12
+	JMP     openAVX2TailLoop
 
-DATA ·rol8<>+0(SB)/8, $0x0605040702010003
-DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b
-DATA ·rol8<>+16(SB)/8, $0x0605040702010003
-DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b
-GLOBL ·rol8<>(SB), RODATA|NOPTR, $32
+openAVX2Tail:
+	CMPQ    BX, $0x10
+	VMOVDQA X0, X1
+	JB      openAVX2TailDone
+	SUBQ    $0x10, BX
 
-DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002
-DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000
-DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002
-DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000
-GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32
+	// Load for decryption
+	VPXOR      (SI), X0, X12
+	VMOVDQU    X12, (DI)
+	LEAQ       16(SI), SI
+	LEAQ       16(DI), DI
+	VPERM2I128 $0x11, Y0, Y0, Y0
+	VMOVDQA    X0, X1
 
-// func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte)
-// Requires: AVX, AVX2, BMI2, CMOV, SSE2
-TEXT ·chacha20Poly1305Seal(SB), $288-96
-	MOVQ SP, BP
-	ADDQ $0x20, BP
-	ANDQ $-32, BP
-	MOVQ dst_base+0(FP), DI
-	MOVQ key_base+24(FP), R8
-	MOVQ src_base+48(FP), SI
-	MOVQ src_len+56(FP), BX
-	MOVQ ad_base+72(FP), CX
-	CMPB ·useAVX2+0(SB), $0x01
-	JE   chacha20Poly1305Seal_AVX2
+openAVX2TailDone:
+	VZEROUPPER
+	JMP openSSETail16
 
-	// Special optimization, for very short buffers
-	CMPQ BX, $0x80
-	JBE  sealSSE128
-
-	// In the seal case - prepare the poly key + 3 blocks of stream in the first iteration
-	MOVOU ·chacha20Constants<>+0(SB), X0
-	MOVOU 16(R8), X3
-	MOVOU 32(R8), X6
-	MOVOU 48(R8), X9
-
-	// Store state on stack for future use
-	MOVO X3, 32(BP)
-	MOVO X6, 48(BP)
-
-	// Load state, increment counter blocks
-	MOVO  X0, X1
-	MOVO  X3, X4
-	MOVO  X6, X7
-	MOVO  X9, X10
-	PADDL ·sseIncMask<>+0(SB), X10
-	MOVO  X1, X2
-	MOVO  X4, X5
-	MOVO  X7, X8
-	MOVO  X10, X11
-	PADDL ·sseIncMask<>+0(SB), X11
-	MOVO  X2, X12
-	MOVO  X5, X13
-	MOVO  X8, X14
-	MOVO  X11, X15
-	PADDL ·sseIncMask<>+0(SB), X15
-
-	// Store counters
-	MOVO X9, 80(BP)
-	MOVO X10, 96(BP)
-	MOVO X11, 112(BP)
-	MOVO X15, 128(BP)
-	MOVQ $0x0000000a, R9
-
-sealSSEIntroLoop:
-	MOVO  X14, 64(BP)
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X3
-	PXOR  X14, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X3
-	PXOR  X14, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X4
-	PXOR  X14, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X4
-	PXOR  X14, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X5
-	PXOR  X14, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X5
-	PXOR  X14, X5
-	MOVO  64(BP), X14
-	MOVO  X7, 64(BP)
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL16(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x0c, X7
-	PSRLL $0x14, X13
-	PXOR  X7, X13
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL8(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x07, X7
-	PSRLL $0x19, X13
-	PXOR  X7, X13
-	MOVO  64(BP), X7
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x0c
-	MOVO  X14, 64(BP)
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X3
-	PXOR  X14, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X3
-	PXOR  X14, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X4
-	PXOR  X14, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X4
-	PXOR  X14, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X5
-	PXOR  X14, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X5
-	PXOR  X14, X5
-	MOVO  64(BP), X14
-	MOVO  X7, 64(BP)
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL16(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x0c, X7
-	PSRLL $0x14, X13
-	PXOR  X7, X13
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL8(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x07, X7
-	PSRLL $0x19, X13
-	PXOR  X7, X13
-	MOVO  64(BP), X7
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x04
-	DECQ  R9
-	JNE   sealSSEIntroLoop
-
-	// Add in the state
-	PADDD ·chacha20Constants<>+0(SB), X0
-	PADDD ·chacha20Constants<>+0(SB), X1
-	PADDD ·chacha20Constants<>+0(SB), X2
-	PADDD ·chacha20Constants<>+0(SB), X12
-	PADDD 32(BP), X3
-	PADDD 32(BP), X4
-	PADDD 32(BP), X5
-	PADDD 32(BP), X13
-	PADDD 48(BP), X7
-	PADDD 48(BP), X8
-	PADDD 48(BP), X14
-	PADDD 96(BP), X10
-	PADDD 112(BP), X11
-	PADDD 128(BP), X15
-
-	// Clamp and store the key
-	PAND ·polyClampMask<>+0(SB), X0
-	MOVO X0, (BP)
-	MOVO X3, 16(BP)
-
-	// Hash AAD
-	MOVQ  ad_len+80(FP), R9
-	CALL  polyHashADInternal<>(SB)
-	MOVOU (SI), X0
-	MOVOU 16(SI), X3
-	MOVOU 32(SI), X6
-	MOVOU 48(SI), X9
-	PXOR  X0, X1
-	PXOR  X3, X4
-	PXOR  X6, X7
-	PXOR  X9, X10
-	MOVOU X1, (DI)
-	MOVOU X4, 16(DI)
-	MOVOU X7, 32(DI)
-	MOVOU X10, 48(DI)
-	MOVOU 64(SI), X0
-	MOVOU 80(SI), X3
-	MOVOU 96(SI), X6
-	MOVOU 112(SI), X9
-	PXOR  X0, X2
-	PXOR  X3, X5
-	PXOR  X6, X8
-	PXOR  X9, X11
-	MOVOU X2, 64(DI)
-	MOVOU X5, 80(DI)
-	MOVOU X8, 96(DI)
-	MOVOU X11, 112(DI)
-	MOVQ  $0x00000080, CX
-	SUBQ  $0x80, BX
-	LEAQ  128(SI), SI
-	MOVO  X12, X1
-	MOVO  X13, X4
-	MOVO  X14, X7
-	MOVO  X15, X10
-	CMPQ  BX, $0x40
-	JBE   sealSSE128SealHash
-	MOVOU (SI), X0
-	MOVOU 16(SI), X3
-	MOVOU 32(SI), X6
-	MOVOU 48(SI), X9
-	PXOR  X0, X12
-	PXOR  X3, X13
-	PXOR  X6, X14
-	PXOR  X9, X15
-	MOVOU X12, 128(DI)
-	MOVOU X13, 144(DI)
-	MOVOU X14, 160(DI)
-	MOVOU X15, 176(DI)
-	ADDQ  $0x40, CX
-	SUBQ  $0x40, BX
-	LEAQ  64(SI), SI
-	MOVQ  $0x00000002, CX
-	MOVQ  $0x00000008, R9
-	CMPQ  BX, $0x40
-	JBE   sealSSETail64
-	CMPQ  BX, $0x80
-	JBE   sealSSETail128
-	CMPQ  BX, $0xc0
-	JBE   sealSSETail192
-
-sealSSEMainLoop:
-	// Load state, increment counter blocks
-	MOVO  ·chacha20Constants<>+0(SB), X0
-	MOVO  32(BP), X3
-	MOVO  48(BP), X6
-	MOVO  128(BP), X9
-	PADDL ·sseIncMask<>+0(SB), X9
-	MOVO  X0, X1
-	MOVO  X3, X4
-	MOVO  X6, X7
-	MOVO  X9, X10
-	PADDL ·sseIncMask<>+0(SB), X10
-	MOVO  X1, X2
-	MOVO  X4, X5
-	MOVO  X7, X8
-	MOVO  X10, X11
-	PADDL ·sseIncMask<>+0(SB), X11
-	MOVO  X2, X12
-	MOVO  X5, X13
-	MOVO  X8, X14
-	MOVO  X11, X15
-	PADDL ·sseIncMask<>+0(SB), X15
-
-	// Store counters
-	MOVO X9, 80(BP)
-	MOVO X10, 96(BP)
-	MOVO X11, 112(BP)
-	MOVO X15, 128(BP)
-
-sealSSEInnerLoop:
-	MOVO  X14, 64(BP)
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X3
-	PXOR  X14, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X3
-	PXOR  X14, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X4
-	PXOR  X14, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X4
-	PXOR  X14, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X5
-	PXOR  X14, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X5
-	PXOR  X14, X5
-	MOVO  64(BP), X14
-	MOVO  X7, 64(BP)
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL16(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x0c, X7
-	PSRLL $0x14, X13
-	PXOR  X7, X13
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL8(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x07, X7
-	PSRLL $0x19, X13
-	PXOR  X7, X13
-	MOVO  64(BP), X7
-	ADDQ  (DI), R10
-	ADCQ  8(DI), R11
+openAVX2Tail256:
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VMOVDQA Y4, Y7
+	VMOVDQA Y1, Y11
+
+	// Compute the number of iterations that will hash data
+	MOVQ    BX, 224(BP)
+	MOVQ    BX, CX
+	SUBQ    $0x80, CX
+	SHRQ    $0x04, CX
+	MOVQ    $0x0000000a, R9
+	CMPQ    CX, $0x0a
+	CMOVQGT R9, CX
+	MOVQ    SI, BX
+	XORQ    R9, R9
+
+openAVX2Tail256LoopA:
+	ADDQ  (BX), R10
+	ADCQ  8(BX), R11
 	ADCQ  $0x01, R12
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x0c
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
 	IMULQ R12, R15
+	MULXQ R11, AX, DX
 	ADDQ  AX, R14
 	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
 	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	LEAQ  16(DI), DI
-	MOVO  X14, 64(BP)
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X3
-	PXOR  X14, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X14)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X3
-	PXOR  X14, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X4
-	PXOR  X14, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X14)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X4
-	PXOR  X14, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x0c, X14
-	PSRLL $0x14, X5
-	PXOR  X14, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X14)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X14
-	PSLLL $0x07, X14
-	PSRLL $0x19, X5
-	PXOR  X14, X5
-	MOVO  64(BP), X14
-	MOVO  X7, 64(BP)
-	IMULQ R12, R8
-	ADDQ  R10, R15
 	ADCQ  DX, R8
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL16(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x0c, X7
-	PSRLL $0x14, X13
-	PXOR  X7, X13
-	PADDD X13, X12
-	PXOR  X12, X15
-	ROL8(X15, X7)
-	PADDD X15, X14
-	PXOR  X14, X13
-	MOVO  X13, X7
-	PSLLL $0x07, X7
-	PSRLL $0x19, X13
-	PXOR  X7, X13
-	MOVO  64(BP), X7
 	MOVQ  R13, R10
 	MOVQ  R14, R11
 	MOVQ  R15, R12
@@ -5555,112 +1518,117 @@ sealSSEInnerLoop:
 	ADDQ  R15, R10
 	ADCQ  R8, R11
 	ADCQ  $0x00, R12
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x04
-	DECQ  R9
-	JGE   sealSSEInnerLoop
-	ADDQ  (DI), R10
-	ADCQ  8(DI), R11
+	LEAQ  16(BX), BX
+
+openAVX2Tail256LoopB:
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	INCQ     R9
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	CMPQ     R9, CX
+	JB       openAVX2Tail256LoopA
+	CMPQ     R9, $0x0a
+	JNE      openAVX2Tail256LoopB
+	MOVQ     BX, R9
+	SUBQ     SI, BX
+	MOVQ     BX, CX
+	MOVQ     224(BP), BX
+
+openAVX2Tail256Hash:
+	ADDQ  $0x10, CX
+	CMPQ  CX, BX
+	JGT   openAVX2Tail256HashEnd
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
 	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
 	IMULQ R12, R15
+	MULXQ R11, AX, DX
 	ADDQ  AX, R14
 	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
 	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
 	ADCQ  DX, R8
 	MOVQ  R13, R10
 	MOVQ  R14, R11
@@ -5672,143 +1640,94 @@ sealSSEInnerLoop:
 	SHRQ  $0x02, R8, R15
 	SHRQ  $0x02, R8
 	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	LEAQ  16(DI), DI
-	DECQ  CX
-	JG    sealSSEInnerLoop
-
-	// Add in the state
-	PADDD ·chacha20Constants<>+0(SB), X0
-	PADDD ·chacha20Constants<>+0(SB), X1
-	PADDD ·chacha20Constants<>+0(SB), X2
-	PADDD ·chacha20Constants<>+0(SB), X12
-	PADDD 32(BP), X3
-	PADDD 32(BP), X4
-	PADDD 32(BP), X5
-	PADDD 32(BP), X13
-	PADDD 48(BP), X6
-	PADDD 48(BP), X7
-	PADDD 48(BP), X8
-	PADDD 48(BP), X14
-	PADDD 80(BP), X9
-	PADDD 96(BP), X10
-	PADDD 112(BP), X11
-	PADDD 128(BP), X15
-	MOVO  X15, 64(BP)
-
-	// Load - xor - store
-	MOVOU (SI), X15
-	PXOR  X15, X0
-	MOVOU 16(SI), X15
-	PXOR  X15, X3
-	MOVOU 32(SI), X15
-	PXOR  X15, X6
-	MOVOU 48(SI), X15
-	PXOR  X15, X9
-	MOVOU X0, (DI)
-	MOVOU X3, 16(DI)
-	MOVOU X6, 32(DI)
-	MOVOU X9, 48(DI)
-	MOVO  64(BP), X15
-	MOVOU 64(SI), X0
-	MOVOU 80(SI), X3
-	MOVOU 96(SI), X6
-	MOVOU 112(SI), X9
-	PXOR  X0, X1
-	PXOR  X3, X4
-	PXOR  X6, X7
-	PXOR  X9, X10
-	MOVOU X1, 64(DI)
-	MOVOU X4, 80(DI)
-	MOVOU X7, 96(DI)
-	MOVOU X10, 112(DI)
-	MOVOU 128(SI), X0
-	MOVOU 144(SI), X3
-	MOVOU 160(SI), X6
-	MOVOU 176(SI), X9
-	PXOR  X0, X2
-	PXOR  X3, X5
-	PXOR  X6, X8
-	PXOR  X9, X11
-	MOVOU X2, 128(DI)
-	MOVOU X5, 144(DI)
-	MOVOU X8, 160(DI)
-	MOVOU X11, 176(DI)
-	ADDQ  $0xc0, SI
-	MOVQ  $0x000000c0, CX
-	SUBQ  $0xc0, BX
-	MOVO  X12, X1
-	MOVO  X13, X4
-	MOVO  X14, X7
-	MOVO  X15, X10
-	CMPQ  BX, $0x40
-	JBE   sealSSE128SealHash
-	MOVOU (SI), X0
-	MOVOU 16(SI), X3
-	MOVOU 32(SI), X6
-	MOVOU 48(SI), X9
-	PXOR  X0, X12
-	PXOR  X3, X13
-	PXOR  X6, X14
-	PXOR  X9, X15
-	MOVOU X12, 192(DI)
-	MOVOU X13, 208(DI)
-	MOVOU X14, 224(DI)
-	MOVOU X15, 240(DI)
-	LEAQ  64(SI), SI
-	SUBQ  $0x40, BX
-	MOVQ  $0x00000006, CX
-	MOVQ  $0x00000004, R9
-	CMPQ  BX, $0xc0
-	JG    sealSSEMainLoop
-	MOVQ  BX, CX
-	TESTQ BX, BX
-	JE    sealSSE128SealHash
-	MOVQ  $0x00000006, CX
-	CMPQ  BX, $0x40
-	JBE   sealSSETail64
-	CMPQ  BX, $0x80
-	JBE   sealSSETail128
-	JMP   sealSSETail192
-
-sealSSETail64:
-	MOVO  ·chacha20Constants<>+0(SB), X1
-	MOVO  32(BP), X4
-	MOVO  48(BP), X7
-	MOVO  128(BP), X10
-	PADDL ·sseIncMask<>+0(SB), X10
-	MOVO  X10, 80(BP)
-
-sealSSETail64LoopA:
-	ADDQ  (DI), R10
-	ADCQ  8(DI), R11
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
+	JMP   openAVX2Tail256Hash
+
+openAVX2Tail256HashEnd:
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     Y7, Y4, Y4
+	VPADDD     Y11, Y1, Y1
+	VPERM2I128 $0x02, Y0, Y14, Y6
+	VPERM2I128 $0x02, Y12, Y4, Y10
+	VPERM2I128 $0x13, Y0, Y14, Y8
+	VPERM2I128 $0x13, Y12, Y4, Y2
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      (SI), Y6, Y6
+	VPXOR      32(SI), Y10, Y10
+	VPXOR      64(SI), Y8, Y8
+	VPXOR      96(SI), Y2, Y2
+	VMOVDQU    Y6, (DI)
+	VMOVDQU    Y10, 32(DI)
+	VMOVDQU    Y8, 64(DI)
+	VMOVDQU    Y2, 96(DI)
+	LEAQ       128(SI), SI
+	LEAQ       128(DI), DI
+	SUBQ       $0x80, BX
+	JMP        openAVX2TailLoop
+
+openAVX2Tail384:
+	// Need to decrypt up to 384 bytes - prepare six blocks
+	VMOVDQA ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
+
+	// Compute the number of iterations that will hash two blocks of data
+	MOVQ    BX, 224(BP)
+	MOVQ    BX, CX
+	SUBQ    $0x00000100, CX
+	SHRQ    $0x04, CX
+	ADDQ    $0x06, CX
+	MOVQ    $0x0000000a, R9
+	CMPQ    CX, $0x0a
+	CMOVQGT R9, CX
+	MOVQ    SI, BX
+	XORQ    R9, R9
+
+openAVX2Tail384LoopB:
+	ADDQ  (BX), R10
+	ADCQ  8(BX), R11
 	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
 	IMULQ R12, R15
+	MULXQ R11, AX, DX
 	ADDQ  AX, R14
 	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
 	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
 	ADCQ  DX, R8
 	MOVQ  R13, R10
 	MOVQ  R14, R11
@@ -5825,175 +1744,190 @@ sealSSETail64LoopA:
 	ADDQ  R15, R10
 	ADCQ  R8, R11
 	ADCQ  $0x00, R12
-	LEAQ  16(DI), DI
+	LEAQ  16(BX), BX
+
+openAVX2Tail384LoopA:
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	ADDQ     (BX), R10
+	ADCQ     8(BX), R11
+	ADCQ     $0x01, R12
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	LEAQ     16(BX), BX
+	INCQ     R9
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x0c, Y14, Y3
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y14, Y0, Y0
+	VPXOR    Y0, Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPADDD   Y4, Y12, Y12
+	VPXOR    Y12, Y14, Y14
+	VPSLLD   $0x07, Y14, Y3
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y3, Y14, Y14
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x0c, Y9, Y3
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y9, Y5, Y5
+	VPXOR    Y5, Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPADDD   Y1, Y13, Y13
+	VPXOR    Y13, Y9, Y9
+	VPSLLD   $0x07, Y9, Y3
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y3, Y9, Y9
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x0c, Y10, Y3
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPADDD   Y10, Y6, Y6
+	VPXOR    Y6, Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPADDD   Y2, Y8, Y8
+	VPXOR    Y8, Y10, Y10
+	VPSLLD   $0x07, Y10, Y3
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y3, Y10, Y10
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	CMPQ     R9, CX
+	JB       openAVX2Tail384LoopB
+	CMPQ     R9, $0x0a
+	JNE      openAVX2Tail384LoopA
+	MOVQ     BX, R9
+	SUBQ     SI, BX
+	MOVQ     BX, CX
+	MOVQ     224(BP), BX
 
-sealSSETail64LoopB:
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X13)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X13
-	PSLLL $0x0c, X13
-	PSRLL $0x14, X4
-	PXOR  X13, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X13)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X13
-	PSLLL $0x07, X13
-	PSRLL $0x19, X4
-	PXOR  X13, X4
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x0c
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X13)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X13
-	PSLLL $0x0c, X13
-	PSRLL $0x14, X4
-	PXOR  X13, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X13)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X13
-	PSLLL $0x07, X13
-	PSRLL $0x19, X4
-	PXOR  X13, X4
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x04
-	ADDQ  (DI), R10
-	ADCQ  8(DI), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	LEAQ  16(DI), DI
-	DECQ  CX
-	JG    sealSSETail64LoopA
-	DECQ  R9
-	JGE   sealSSETail64LoopB
-	PADDL ·chacha20Constants<>+0(SB), X1
-	PADDL 32(BP), X4
-	PADDL 48(BP), X7
-	PADDL 80(BP), X10
-	JMP   sealSSE128Seal
-
-sealSSETail128:
-	MOVO  ·chacha20Constants<>+0(SB), X0
-	MOVO  32(BP), X3
-	MOVO  48(BP), X6
-	MOVO  128(BP), X9
-	PADDL ·sseIncMask<>+0(SB), X9
-	MOVO  X9, 80(BP)
-	MOVO  X0, X1
-	MOVO  X3, X4
-	MOVO  X6, X7
-	MOVO  X9, X10
-	PADDL ·sseIncMask<>+0(SB), X10
-	MOVO  X10, 96(BP)
-
-sealSSETail128LoopA:
-	ADDQ  (DI), R10
-	ADCQ  8(DI), R11
+openAVX2Tail384Hash:
+	ADDQ  $0x10, CX
+	CMPQ  CX, BX
+	JGT   openAVX2Tail384HashEnd
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
 	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
 	IMULQ R12, R15
+	MULXQ R11, AX, DX
 	ADDQ  AX, R14
 	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
 	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
 	ADCQ  DX, R8
 	MOVQ  R13, R10
 	MOVQ  R14, R11
@@ -6010,430 +1944,99 @@ sealSSETail128LoopA:
 	ADDQ  R15, R10
 	ADCQ  R8, R11
 	ADCQ  $0x00, R12
-	LEAQ  16(DI), DI
+	LEAQ  16(R9), R9
+	JMP   openAVX2Tail384Hash
 
-sealSSETail128LoopB:
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x0c
-	ADDQ  (DI), R10
-	ADCQ  8(DI), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	LEAQ  16(DI), DI
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x04
-	DECQ  CX
-	JG    sealSSETail128LoopA
-	DECQ  R9
-	JGE   sealSSETail128LoopB
-	PADDL ·chacha20Constants<>+0(SB), X0
-	PADDL ·chacha20Constants<>+0(SB), X1
-	PADDL 32(BP), X3
-	PADDL 32(BP), X4
-	PADDL 48(BP), X6
-	PADDL 48(BP), X7
-	PADDL 80(BP), X9
-	PADDL 96(BP), X10
-	MOVOU (SI), X12
-	MOVOU 16(SI), X13
-	MOVOU 32(SI), X14
-	MOVOU 48(SI), X15
-	PXOR  X12, X0
-	PXOR  X13, X3
-	PXOR  X14, X6
-	PXOR  X15, X9
-	MOVOU X0, (DI)
-	MOVOU X3, 16(DI)
-	MOVOU X6, 32(DI)
-	MOVOU X9, 48(DI)
-	MOVQ  $0x00000040, CX
-	LEAQ  64(SI), SI
-	SUBQ  $0x40, BX
-	JMP   sealSSE128SealHash
-
-sealSSETail192:
-	MOVO  ·chacha20Constants<>+0(SB), X0
-	MOVO  32(BP), X3
-	MOVO  48(BP), X6
-	MOVO  128(BP), X9
-	PADDL ·sseIncMask<>+0(SB), X9
-	MOVO  X9, 80(BP)
-	MOVO  X0, X1
-	MOVO  X3, X4
-	MOVO  X6, X7
-	MOVO  X9, X10
-	PADDL ·sseIncMask<>+0(SB), X10
-	MOVO  X10, 96(BP)
-	MOVO  X1, X2
-	MOVO  X4, X5
-	MOVO  X7, X8
-	MOVO  X10, X11
-	PADDL ·sseIncMask<>+0(SB), X11
-	MOVO  X11, 112(BP)
-
-sealSSETail192LoopA:
-	ADDQ  (DI), R10
-	ADCQ  8(DI), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
-	LEAQ  16(DI), DI
+openAVX2Tail384HashEnd:
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     96(BP), Y4, Y4
+	VPADDD     128(BP), Y1, Y1
+	VPADDD     160(BP), Y2, Y2
+	VPERM2I128 $0x02, Y0, Y14, Y3
+	VPERM2I128 $0x02, Y12, Y4, Y7
+	VPERM2I128 $0x13, Y0, Y14, Y11
+	VPERM2I128 $0x13, Y12, Y4, Y15
+	VPXOR      (SI), Y3, Y3
+	VPXOR      32(SI), Y7, Y7
+	VPXOR      64(SI), Y11, Y11
+	VPXOR      96(SI), Y15, Y15
+	VMOVDQU    Y3, (DI)
+	VMOVDQU    Y7, 32(DI)
+	VMOVDQU    Y11, 64(DI)
+	VMOVDQU    Y15, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y3
+	VPERM2I128 $0x02, Y13, Y1, Y7
+	VPERM2I128 $0x13, Y5, Y9, Y11
+	VPERM2I128 $0x13, Y13, Y1, Y15
+	VPXOR      128(SI), Y3, Y3
+	VPXOR      160(SI), Y7, Y7
+	VPXOR      192(SI), Y11, Y11
+	VPXOR      224(SI), Y15, Y15
+	VMOVDQU    Y3, 128(DI)
+	VMOVDQU    Y7, 160(DI)
+	VMOVDQU    Y11, 192(DI)
+	VMOVDQU    Y15, 224(DI)
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	LEAQ       256(SI), SI
+	LEAQ       256(DI), DI
+	SUBQ       $0x00000100, BX
+	JMP        openAVX2TailLoop
 
-sealSSETail192LoopB:
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X5
-	PXOR  X12, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X5
-	PXOR  X12, X5
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	ADDQ  (DI), R10
-	ADCQ  8(DI), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
+openAVX2Tail512:
+	VMOVDQU ·chacha20Constants<>+0(SB), Y0
+	VMOVDQA Y0, Y5
+	VMOVDQA Y0, Y6
+	VMOVDQA Y0, Y7
+	VMOVDQA 32(BP), Y14
+	VMOVDQA Y14, Y9
+	VMOVDQA Y14, Y10
+	VMOVDQA Y14, Y11
+	VMOVDQA 64(BP), Y12
+	VMOVDQA Y12, Y13
+	VMOVDQA Y12, Y8
+	VMOVDQA Y12, Y15
+	VMOVDQA 192(BP), Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y4
+	VPADDD  ·avx2IncMask<>+0(SB), Y4, Y1
+	VPADDD  ·avx2IncMask<>+0(SB), Y1, Y2
+	VPADDD  ·avx2IncMask<>+0(SB), Y2, Y3
+	VMOVDQA Y4, 96(BP)
+	VMOVDQA Y1, 128(BP)
+	VMOVDQA Y2, 160(BP)
+	VMOVDQA Y3, 192(BP)
+	XORQ    CX, CX
+	MOVQ    SI, R9
+
+openAVX2Tail512LoopB:
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
 	IMULQ R12, R15
+	MULXQ R11, AX, DX
 	ADDQ  AX, R14
 	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
 	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
 	ADCQ  DX, R8
 	MOVQ  R13, R10
 	MOVQ  R14, R11
@@ -6450,465 +2053,268 @@ sealSSETail192LoopB:
 	ADDQ  R15, R10
 	ADCQ  R8, R11
 	ADCQ  $0x00, R12
-	LEAQ  16(DI), DI
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X5
-	PXOR  X12, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X5
-	PXOR  X12, X5
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	DECQ  CX
-	JG    sealSSETail192LoopA
-	DECQ  R9
-	JGE   sealSSETail192LoopB
-	PADDL ·chacha20Constants<>+0(SB), X0
-	PADDL ·chacha20Constants<>+0(SB), X1
-	PADDL ·chacha20Constants<>+0(SB), X2
-	PADDL 32(BP), X3
-	PADDL 32(BP), X4
-	PADDL 32(BP), X5
-	PADDL 48(BP), X6
-	PADDL 48(BP), X7
-	PADDL 48(BP), X8
-	PADDL 80(BP), X9
-	PADDL 96(BP), X10
-	PADDL 112(BP), X11
-	MOVOU (SI), X12
-	MOVOU 16(SI), X13
-	MOVOU 32(SI), X14
-	MOVOU 48(SI), X15
-	PXOR  X12, X0
-	PXOR  X13, X3
-	PXOR  X14, X6
-	PXOR  X15, X9
-	MOVOU X0, (DI)
-	MOVOU X3, 16(DI)
-	MOVOU X6, 32(DI)
-	MOVOU X9, 48(DI)
-	MOVOU 64(SI), X12
-	MOVOU 80(SI), X13
-	MOVOU 96(SI), X14
-	MOVOU 112(SI), X15
-	PXOR  X12, X1
-	PXOR  X13, X4
-	PXOR  X14, X7
-	PXOR  X15, X10
-	MOVOU X1, 64(DI)
-	MOVOU X4, 80(DI)
-	MOVOU X7, 96(DI)
-	MOVOU X10, 112(DI)
-	MOVO  X2, X1
-	MOVO  X5, X4
-	MOVO  X8, X7
-	MOVO  X11, X10
-	MOVQ  $0x00000080, CX
-	LEAQ  128(SI), SI
-	SUBQ  $0x80, BX
-	JMP   sealSSE128SealHash
-
-sealSSE128:
-	MOVOU ·chacha20Constants<>+0(SB), X0
-	MOVOU 16(R8), X3
-	MOVOU 32(R8), X6
-	MOVOU 48(R8), X9
-	MOVO  X0, X1
-	MOVO  X3, X4
-	MOVO  X6, X7
-	MOVO  X9, X10
-	PADDL ·sseIncMask<>+0(SB), X10
-	MOVO  X1, X2
-	MOVO  X4, X5
-	MOVO  X7, X8
-	MOVO  X10, X11
-	PADDL ·sseIncMask<>+0(SB), X11
-	MOVO  X3, X13
-	MOVO  X6, X14
-	MOVO  X10, X15
-	MOVQ  $0x0000000a, R9
-
-sealSSE128InnerCipherLoop:
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X5
-	PXOR  X12, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X5
-	PXOR  X12, X5
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL16(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X3
-	PXOR  X12, X3
-	PADDD X3, X0
-	PXOR  X0, X9
-	ROL8(X9, X12)
-	PADDD X9, X6
-	PXOR  X6, X3
-	MOVO  X3, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X3
-	PXOR  X12, X3
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL16(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X4
-	PXOR  X12, X4
-	PADDD X4, X1
-	PXOR  X1, X10
-	ROL8(X10, X12)
-	PADDD X10, X7
-	PXOR  X7, X4
-	MOVO  X4, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X4
-	PXOR  X12, X4
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL16(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x0c, X12
-	PSRLL $0x14, X5
-	PXOR  X12, X5
-	PADDD X5, X2
-	PXOR  X2, X11
-	ROL8(X11, X12)
-	PADDD X11, X8
-	PXOR  X8, X5
-	MOVO  X5, X12
-	PSLLL $0x07, X12
-	PSRLL $0x19, X5
-	PXOR  X12, X5
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xe4
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xed
-	BYTE  $0x0c
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xf6
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xff
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc0
-	BYTE  $0x08
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xc9
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xd2
-	BYTE  $0x04
-	BYTE  $0x66
-	BYTE  $0x45
-	BYTE  $0x0f
-	BYTE  $0x3a
-	BYTE  $0x0f
-	BYTE  $0xdb
-	BYTE  $0x04
-	DECQ  R9
-	JNE   sealSSE128InnerCipherLoop
-
-	// A0|B0 hold the Poly1305 32-byte key, C0,D0 can be discarded
-	PADDL ·chacha20Constants<>+0(SB), X0
-	PADDL ·chacha20Constants<>+0(SB), X1
-	PADDL ·chacha20Constants<>+0(SB), X2
-	PADDL X13, X3
-	PADDL X13, X4
-	PADDL X13, X5
-	PADDL X14, X7
-	PADDL X14, X8
-	PADDL X15, X10
-	PADDL ·sseIncMask<>+0(SB), X15
-	PADDL X15, X11
-	PAND  ·polyClampMask<>+0(SB), X0
-	MOVOU X0, (BP)
-	MOVOU X3, 16(BP)
+	LEAQ  16(R9), R9
 
-	// Hash
-	MOVQ ad_len+80(FP), R9
-	CALL polyHashADInternal<>(SB)
-	XORQ CX, CX
+openAVX2Tail512LoopA:
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	ADDQ     (R9), R10
+	ADCQ     8(R9), R11
+	ADCQ     $0x01, R12
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	VPALIGNR $0x04, Y14, Y14, Y14
+	VPALIGNR $0x04, Y9, Y9, Y9
+	VPALIGNR $0x04, Y10, Y10, Y10
+	VPALIGNR $0x04, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x0c, Y4, Y4, Y4
+	VPALIGNR $0x0c, Y1, Y1, Y1
+	VPALIGNR $0x0c, Y2, Y2, Y2
+	VPALIGNR $0x0c, Y3, Y3, Y3
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol16<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol16<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol16<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol16<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	ADDQ     16(R9), R10
+	ADCQ     24(R9), R11
+	ADCQ     $0x01, R12
+	MOVQ     (BP), DX
+	MOVQ     DX, R15
+	MULXQ    R10, R13, R14
+	IMULQ    R12, R15
+	MULXQ    R11, AX, DX
+	ADDQ     AX, R14
+	ADCQ     DX, R15
+	MOVQ     8(BP), DX
+	MULXQ    R10, R10, AX
+	ADDQ     R10, R14
+	MULXQ    R11, R11, R8
+	ADCQ     R11, R15
+	ADCQ     $0x00, R8
+	IMULQ    R12, DX
+	ADDQ     AX, R15
+	ADCQ     DX, R8
+	MOVQ     R13, R10
+	MOVQ     R14, R11
+	MOVQ     R15, R12
+	ANDQ     $0x03, R12
+	MOVQ     R15, R13
+	ANDQ     $-4, R13
+	MOVQ     R8, R14
+	SHRQ     $0x02, R8, R15
+	SHRQ     $0x02, R8
+	ADDQ     R13, R10
+	ADCQ     R14, R11
+	ADCQ     $0x00, R12
+	ADDQ     R15, R10
+	ADCQ     R8, R11
+	ADCQ     $0x00, R12
+	LEAQ     32(R9), R9
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x0c, Y14, Y15
+	VPSRLD   $0x14, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x0c, Y9, Y15
+	VPSRLD   $0x14, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x0c, Y10, Y15
+	VPSRLD   $0x14, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x0c, Y11, Y15
+	VPSRLD   $0x14, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	VPADDD   Y14, Y0, Y0
+	VPADDD   Y9, Y5, Y5
+	VPADDD   Y10, Y6, Y6
+	VPADDD   Y11, Y7, Y7
+	VPXOR    Y0, Y4, Y4
+	VPXOR    Y5, Y1, Y1
+	VPXOR    Y6, Y2, Y2
+	VPXOR    Y7, Y3, Y3
+	VPSHUFB  ·rol8<>+0(SB), Y4, Y4
+	VPSHUFB  ·rol8<>+0(SB), Y1, Y1
+	VPSHUFB  ·rol8<>+0(SB), Y2, Y2
+	VPSHUFB  ·rol8<>+0(SB), Y3, Y3
+	VPADDD   Y4, Y12, Y12
+	VPADDD   Y1, Y13, Y13
+	VPADDD   Y2, Y8, Y8
+	VPADDD   Y3, Y15, Y15
+	VPXOR    Y12, Y14, Y14
+	VPXOR    Y13, Y9, Y9
+	VPXOR    Y8, Y10, Y10
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  Y15, 224(BP)
+	VPSLLD   $0x07, Y14, Y15
+	VPSRLD   $0x19, Y14, Y14
+	VPXOR    Y15, Y14, Y14
+	VPSLLD   $0x07, Y9, Y15
+	VPSRLD   $0x19, Y9, Y9
+	VPXOR    Y15, Y9, Y9
+	VPSLLD   $0x07, Y10, Y15
+	VPSRLD   $0x19, Y10, Y10
+	VPXOR    Y15, Y10, Y10
+	VPSLLD   $0x07, Y11, Y15
+	VPSRLD   $0x19, Y11, Y11
+	VPXOR    Y15, Y11, Y11
+	VMOVDQA  224(BP), Y15
+	VPALIGNR $0x0c, Y14, Y14, Y14
+	VPALIGNR $0x0c, Y9, Y9, Y9
+	VPALIGNR $0x0c, Y10, Y10, Y10
+	VPALIGNR $0x0c, Y11, Y11, Y11
+	VPALIGNR $0x08, Y12, Y12, Y12
+	VPALIGNR $0x08, Y13, Y13, Y13
+	VPALIGNR $0x08, Y8, Y8, Y8
+	VPALIGNR $0x08, Y15, Y15, Y15
+	VPALIGNR $0x04, Y4, Y4, Y4
+	VPALIGNR $0x04, Y1, Y1, Y1
+	VPALIGNR $0x04, Y2, Y2, Y2
+	VPALIGNR $0x04, Y3, Y3, Y3
+	INCQ     CX
+	CMPQ     CX, $0x04
+	JLT      openAVX2Tail512LoopB
+	CMPQ     CX, $0x0a
+	JNE      openAVX2Tail512LoopA
+	MOVQ     BX, CX
+	SUBQ     $0x00000180, CX
+	ANDQ     $-16, CX
 
-sealSSE128SealHash:
-	CMPQ  CX, $0x10
-	JB    sealSSE128Seal
-	ADDQ  (DI), R10
-	ADCQ  8(DI), R11
+openAVX2Tail512HashLoop:
+	TESTQ CX, CX
+	JE    openAVX2Tail512HashEnd
+	ADDQ  (R9), R10
+	ADCQ  8(R9), R11
 	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
+	MOVQ  (BP), DX
+	MOVQ  DX, R15
+	MULXQ R10, R13, R14
 	IMULQ R12, R15
+	MULXQ R11, AX, DX
 	ADDQ  AX, R14
 	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
+	MOVQ  8(BP), DX
+	MULXQ R10, R10, AX
+	ADDQ  R10, R14
+	MULXQ R11, R11, R8
+	ADCQ  R11, R15
+	ADCQ  $0x00, R8
+	IMULQ R12, DX
 	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
 	ADCQ  DX, R8
 	MOVQ  R13, R10
 	MOVQ  R14, R11
@@ -6925,238 +2331,162 @@ sealSSE128SealHash:
 	ADDQ  R15, R10
 	ADCQ  R8, R11
 	ADCQ  $0x00, R12
+	LEAQ  16(R9), R9
 	SUBQ  $0x10, CX
-	ADDQ  $0x10, DI
-	JMP   sealSSE128SealHash
-
-sealSSE128Seal:
-	CMPQ BX, $0x10
-	JB   sealSSETail
-	SUBQ $0x10, BX
-
-	// Load for decryption
-	MOVOU (SI), X12
-	PXOR  X12, X1
-	MOVOU X1, (DI)
-	LEAQ  16(SI), SI
-	LEAQ  16(DI), DI
-
-	// Extract for hashing
-	MOVQ   X1, R13
-	PSRLDQ $0x08, X1
-	MOVQ   X1, R14
-	ADDQ   R13, R10
-	ADCQ   R14, R11
-	ADCQ   $0x01, R12
-	MOVQ   (BP), AX
-	MOVQ   AX, R15
-	MULQ   R10
-	MOVQ   AX, R13
-	MOVQ   DX, R14
-	MOVQ   (BP), AX
-	MULQ   R11
-	IMULQ  R12, R15
-	ADDQ   AX, R14
-	ADCQ   DX, R15
-	MOVQ   8(BP), AX
-	MOVQ   AX, R8
-	MULQ   R10
-	ADDQ   AX, R14
-	ADCQ   $0x00, DX
-	MOVQ   DX, R10
-	MOVQ   8(BP), AX
-	MULQ   R11
-	ADDQ   AX, R15
-	ADCQ   $0x00, DX
-	IMULQ  R12, R8
-	ADDQ   R10, R15
-	ADCQ   DX, R8
-	MOVQ   R13, R10
-	MOVQ   R14, R11
-	MOVQ   R15, R12
-	ANDQ   $0x03, R12
-	MOVQ   R15, R13
-	ANDQ   $-4, R13
-	MOVQ   R8, R14
-	SHRQ   $0x02, R8, R15
-	SHRQ   $0x02, R8
-	ADDQ   R13, R10
-	ADCQ   R14, R11
-	ADCQ   $0x00, R12
-	ADDQ   R15, R10
-	ADCQ   R8, R11
-	ADCQ   $0x00, R12
-
-	// Shift the stream "left"
-	MOVO X4, X1
-	MOVO X7, X4
-	MOVO X10, X7
-	MOVO X2, X10
-	MOVO X5, X2
-	MOVO X8, X5
-	MOVO X11, X8
-	JMP  sealSSE128Seal
+	JMP   openAVX2Tail512HashLoop
 
-sealSSETail:
-	TESTQ BX, BX
-	JE    sealSSEFinalize
+openAVX2Tail512HashEnd:
+	VPADDD     ·chacha20Constants<>+0(SB), Y0, Y0
+	VPADDD     ·chacha20Constants<>+0(SB), Y5, Y5
+	VPADDD     ·chacha20Constants<>+0(SB), Y6, Y6
+	VPADDD     ·chacha20Constants<>+0(SB), Y7, Y7
+	VPADDD     32(BP), Y14, Y14
+	VPADDD     32(BP), Y9, Y9
+	VPADDD     32(BP), Y10, Y10
+	VPADDD     32(BP), Y11, Y11
+	VPADDD     64(BP), Y12, Y12
+	VPADDD     64(BP), Y13, Y13
+	VPADDD     64(BP), Y8, Y8
+	VPADDD     64(BP), Y15, Y15
+	VPADDD     96(BP), Y4, Y4
+	VPADDD     128(BP), Y1, Y1
+	VPADDD     160(BP), Y2, Y2
+	VPADDD     192(BP), Y3, Y3
+	VMOVDQA    Y15, 224(BP)
+	VPERM2I128 $0x02, Y0, Y14, Y15
+	VPERM2I128 $0x13, Y0, Y14, Y14
+	VPERM2I128 $0x02, Y12, Y4, Y0
+	VPERM2I128 $0x13, Y12, Y4, Y12
+	VPXOR      (SI), Y15, Y15
+	VPXOR      32(SI), Y0, Y0
+	VPXOR      64(SI), Y14, Y14
+	VPXOR      96(SI), Y12, Y12
+	VMOVDQU    Y15, (DI)
+	VMOVDQU    Y0, 32(DI)
+	VMOVDQU    Y14, 64(DI)
+	VMOVDQU    Y12, 96(DI)
+	VPERM2I128 $0x02, Y5, Y9, Y0
+	VPERM2I128 $0x02, Y13, Y1, Y14
+	VPERM2I128 $0x13, Y5, Y9, Y12
+	VPERM2I128 $0x13, Y13, Y1, Y4
+	VPXOR      128(SI), Y0, Y0
+	VPXOR      160(SI), Y14, Y14
+	VPXOR      192(SI), Y12, Y12
+	VPXOR      224(SI), Y4, Y4
+	VMOVDQU    Y0, 128(DI)
+	VMOVDQU    Y14, 160(DI)
+	VMOVDQU    Y12, 192(DI)
+	VMOVDQU    Y4, 224(DI)
+	VPERM2I128 $0x02, Y6, Y10, Y0
+	VPERM2I128 $0x02, Y8, Y2, Y14
+	VPERM2I128 $0x13, Y6, Y10, Y12
+	VPERM2I128 $0x13, Y8, Y2, Y4
+	VPXOR      256(SI), Y0, Y0
+	VPXOR      288(SI), Y14, Y14
+	VPXOR      320(SI), Y12, Y12
+	VPXOR      352(SI), Y4, Y4
+	VMOVDQU    Y0, 256(DI)
+	VMOVDQU    Y14, 288(DI)
+	VMOVDQU    Y12, 320(DI)
+	VMOVDQU    Y4, 352(DI)
+	VPERM2I128 $0x02, Y7, Y11, Y0
+	VPERM2I128 $0x02, 224(BP), Y3, Y14
+	VPERM2I128 $0x13, Y7, Y11, Y12
+	VPERM2I128 $0x13, 224(BP), Y3, Y4
+	LEAQ       384(SI), SI
+	LEAQ       384(DI), DI
+	SUBQ       $0x00000180, BX
+	JMP        openAVX2TailLoop
 
-	// We can only load the PT one byte at a time to avoid read after end of buffer
-	MOVQ BX, R9
-	SHLQ $0x04, R9
-	LEAQ ·andMask<>+0(SB), R13
-	MOVQ BX, CX
-	LEAQ -1(SI)(BX*1), SI
-	XORQ R15, R15
-	XORQ R8, R8
-	XORQ AX, AX
+DATA ·chacha20Constants<>+0(SB)/4, $0x61707865
+DATA ·chacha20Constants<>+4(SB)/4, $0x3320646e
+DATA ·chacha20Constants<>+8(SB)/4, $0x79622d32
+DATA ·chacha20Constants<>+12(SB)/4, $0x6b206574
+DATA ·chacha20Constants<>+16(SB)/4, $0x61707865
+DATA ·chacha20Constants<>+20(SB)/4, $0x3320646e
+DATA ·chacha20Constants<>+24(SB)/4, $0x79622d32
+DATA ·chacha20Constants<>+28(SB)/4, $0x6b206574
+GLOBL ·chacha20Constants<>(SB), RODATA|NOPTR, $32
 
-sealSSETailLoadLoop:
-	SHLQ   $0x08, R15, R8
-	SHLQ   $0x08, R15
-	MOVB   (SI), AX
-	XORQ   AX, R15
-	LEAQ   -1(SI), SI
-	DECQ   CX
-	JNE    sealSSETailLoadLoop
-	MOVQ   R15, 64(BP)
-	MOVQ   R8, 72(BP)
-	PXOR   64(BP), X1
-	MOVOU  X1, (DI)
-	MOVOU  -16(R13)(R9*1), X12
-	PAND   X12, X1
-	MOVQ   X1, R13
-	PSRLDQ $0x08, X1
-	MOVQ   X1, R14
-	ADDQ   R13, R10
-	ADCQ   R14, R11
-	ADCQ   $0x01, R12
-	MOVQ   (BP), AX
-	MOVQ   AX, R15
-	MULQ   R10
-	MOVQ   AX, R13
-	MOVQ   DX, R14
-	MOVQ   (BP), AX
-	MULQ   R11
-	IMULQ  R12, R15
-	ADDQ   AX, R14
-	ADCQ   DX, R15
-	MOVQ   8(BP), AX
-	MOVQ   AX, R8
-	MULQ   R10
-	ADDQ   AX, R14
-	ADCQ   $0x00, DX
-	MOVQ   DX, R10
-	MOVQ   8(BP), AX
-	MULQ   R11
-	ADDQ   AX, R15
-	ADCQ   $0x00, DX
-	IMULQ  R12, R8
-	ADDQ   R10, R15
-	ADCQ   DX, R8
-	MOVQ   R13, R10
-	MOVQ   R14, R11
-	MOVQ   R15, R12
-	ANDQ   $0x03, R12
-	MOVQ   R15, R13
-	ANDQ   $-4, R13
-	MOVQ   R8, R14
-	SHRQ   $0x02, R8, R15
-	SHRQ   $0x02, R8
-	ADDQ   R13, R10
-	ADCQ   R14, R11
-	ADCQ   $0x00, R12
-	ADDQ   R15, R10
-	ADCQ   R8, R11
-	ADCQ   $0x00, R12
-	ADDQ   BX, DI
+DATA ·avx2InitMask<>+0(SB)/8, $0x0000000000000000
+DATA ·avx2InitMask<>+8(SB)/8, $0x0000000000000000
+DATA ·avx2InitMask<>+16(SB)/8, $0x0000000000000001
+DATA ·avx2InitMask<>+24(SB)/8, $0x0000000000000000
+GLOBL ·avx2InitMask<>(SB), RODATA|NOPTR, $32
 
-sealSSEFinalize:
-	// Hash in the buffer lengths
-	ADDQ  ad_len+80(FP), R10
-	ADCQ  src_len+56(FP), R11
-	ADCQ  $0x01, R12
-	MOVQ  (BP), AX
-	MOVQ  AX, R15
-	MULQ  R10
-	MOVQ  AX, R13
-	MOVQ  DX, R14
-	MOVQ  (BP), AX
-	MULQ  R11
-	IMULQ R12, R15
-	ADDQ  AX, R14
-	ADCQ  DX, R15
-	MOVQ  8(BP), AX
-	MOVQ  AX, R8
-	MULQ  R10
-	ADDQ  AX, R14
-	ADCQ  $0x00, DX
-	MOVQ  DX, R10
-	MOVQ  8(BP), AX
-	MULQ  R11
-	ADDQ  AX, R15
-	ADCQ  $0x00, DX
-	IMULQ R12, R8
-	ADDQ  R10, R15
-	ADCQ  DX, R8
-	MOVQ  R13, R10
-	MOVQ  R14, R11
-	MOVQ  R15, R12
-	ANDQ  $0x03, R12
-	MOVQ  R15, R13
-	ANDQ  $-4, R13
-	MOVQ  R8, R14
-	SHRQ  $0x02, R8, R15
-	SHRQ  $0x02, R8
-	ADDQ  R13, R10
-	ADCQ  R14, R11
-	ADCQ  $0x00, R12
-	ADDQ  R15, R10
-	ADCQ  R8, R11
-	ADCQ  $0x00, R12
+DATA ·rol16<>+0(SB)/8, $0x0504070601000302
+DATA ·rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a
+DATA ·rol16<>+16(SB)/8, $0x0504070601000302
+DATA ·rol16<>+24(SB)/8, $0x0d0c0f0e09080b0a
+GLOBL ·rol16<>(SB), RODATA|NOPTR, $32
 
-	// Final reduce
-	MOVQ    R10, R13
-	MOVQ    R11, R14
-	MOVQ    R12, R15
-	SUBQ    $-5, R10
-	SBBQ    $-1, R11
-	SBBQ    $0x03, R12
-	CMOVQCS R13, R10
-	CMOVQCS R14, R11
-	CMOVQCS R15, R12
+DATA ·rol8<>+0(SB)/8, $0x0605040702010003
+DATA ·rol8<>+8(SB)/8, $0x0e0d0c0f0a09080b
+DATA ·rol8<>+16(SB)/8, $0x0605040702010003
+DATA ·rol8<>+24(SB)/8, $0x0e0d0c0f0a09080b
+GLOBL ·rol8<>(SB), RODATA|NOPTR, $32
 
-	// Add in the "s" part of the key
-	ADDQ 16(BP), R10
-	ADCQ 24(BP), R11
+DATA ·polyClampMask<>+0(SB)/8, $0x0ffffffc0fffffff
+DATA ·polyClampMask<>+8(SB)/8, $0x0ffffffc0ffffffc
+DATA ·polyClampMask<>+16(SB)/8, $0xffffffffffffffff
+DATA ·polyClampMask<>+24(SB)/8, $0xffffffffffffffff
+GLOBL ·polyClampMask<>(SB), RODATA|NOPTR, $32
 
-	// Finally store the tag at the end of the message
-	MOVQ R10, (DI)
-	MOVQ R11, 8(DI)
-	RET
+DATA ·avx2IncMask<>+0(SB)/8, $0x0000000000000002
+DATA ·avx2IncMask<>+8(SB)/8, $0x0000000000000000
+DATA ·avx2IncMask<>+16(SB)/8, $0x0000000000000002
+DATA ·avx2IncMask<>+24(SB)/8, $0x0000000000000000
+GLOBL ·avx2IncMask<>(SB), RODATA|NOPTR, $32
+
+DATA ·andMask<>+0(SB)/8, $0x00000000000000ff
+DATA ·andMask<>+8(SB)/8, $0x0000000000000000
+DATA ·andMask<>+16(SB)/8, $0x000000000000ffff
+DATA ·andMask<>+24(SB)/8, $0x0000000000000000
+DATA ·andMask<>+32(SB)/8, $0x0000000000ffffff
+DATA ·andMask<>+40(SB)/8, $0x0000000000000000
+DATA ·andMask<>+48(SB)/8, $0x00000000ffffffff
+DATA ·andMask<>+56(SB)/8, $0x0000000000000000
+DATA ·andMask<>+64(SB)/8, $0x000000ffffffffff
+DATA ·andMask<>+72(SB)/8, $0x0000000000000000
+DATA ·andMask<>+80(SB)/8, $0x0000ffffffffffff
+DATA ·andMask<>+88(SB)/8, $0x0000000000000000
+DATA ·andMask<>+96(SB)/8, $0x00ffffffffffffff
+DATA ·andMask<>+104(SB)/8, $0x0000000000000000
+DATA ·andMask<>+112(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+120(SB)/8, $0x0000000000000000
+DATA ·andMask<>+128(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+136(SB)/8, $0x00000000000000ff
+DATA ·andMask<>+144(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+152(SB)/8, $0x000000000000ffff
+DATA ·andMask<>+160(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+168(SB)/8, $0x0000000000ffffff
+DATA ·andMask<>+176(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+184(SB)/8, $0x00000000ffffffff
+DATA ·andMask<>+192(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+200(SB)/8, $0x000000ffffffffff
+DATA ·andMask<>+208(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+216(SB)/8, $0x0000ffffffffffff
+DATA ·andMask<>+224(SB)/8, $0xffffffffffffffff
+DATA ·andMask<>+232(SB)/8, $0x00ffffffffffffff
+GLOBL ·andMask<>(SB), RODATA|NOPTR, $240
 
-chacha20Poly1305Seal_AVX2:
+// func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte)
+// Requires: AVX, AVX2, BMI2, CMOV, SSE2
+TEXT ·chacha20Poly1305Seal(SB), $288-96
+	MOVQ           SP, BP
+	ADDQ           $0x20, BP
+	ANDQ           $-32, BP
+	MOVQ           dst_base+0(FP), DI
+	MOVQ           key_base+24(FP), R8
+	MOVQ           src_base+48(FP), SI
+	MOVQ           src_len+56(FP), BX
+	MOVQ           ad_base+72(FP), CX
 	VZEROUPPER
-	VMOVDQU ·chacha20Constants<>+0(SB), Y0
-	BYTE    $0xc4
-	BYTE    $0x42
-	BYTE    $0x7d
-	BYTE    $0x5a
-	BYTE    $0x70
-	BYTE    $0x10
-	BYTE    $0xc4
-	BYTE    $0x42
-	BYTE    $0x7d
-	BYTE    $0x5a
-	BYTE    $0x60
-	BYTE    $0x20
-	BYTE    $0xc4
-	BYTE    $0xc2
-	BYTE    $0x7d
-	BYTE    $0x5a
-	BYTE    $0x60
-	BYTE    $0x30
-	VPADDD  ·avx2InitMask<>+0(SB), Y4, Y4
+	VMOVDQU        ·chacha20Constants<>+0(SB), Y0
+	VBROADCASTI128 16(R8), Y14
+	VBROADCASTI128 32(R8), Y12
+	VBROADCASTI128 48(R8), Y4
+	VPADDD         ·avx2InitMask<>+0(SB), Y4, Y4
 
 	// Special optimizations, for very short buffers
 	CMPQ BX, $0x000000c0
@@ -8170,6 +3500,144 @@ sealAVX2InternalLoopStart:
 	JBE   sealAVX2Tail384
 	JMP   sealAVX2Tail512
 
+sealSSETail:
+	TESTQ BX, BX
+	JE    sealSSEFinalize
+
+	// We can only load the PT one byte at a time to avoid read after end of buffer
+	MOVQ BX, R9
+	SHLQ $0x04, R9
+	LEAQ ·andMask<>+0(SB), R13
+	MOVQ BX, CX
+	LEAQ -1(SI)(BX*1), SI
+	XORQ R15, R15
+	XORQ R8, R8
+	XORQ AX, AX
+
+sealSSETailLoadLoop:
+	SHLQ   $0x08, R15, R8
+	SHLQ   $0x08, R15
+	MOVB   (SI), AX
+	XORQ   AX, R15
+	LEAQ   -1(SI), SI
+	DECQ   CX
+	JNE    sealSSETailLoadLoop
+	MOVQ   R15, 64(BP)
+	MOVQ   R8, 72(BP)
+	PXOR   64(BP), X1
+	MOVOU  X1, (DI)
+	MOVOU  -16(R13)(R9*1), X12
+	PAND   X12, X1
+	MOVQ   X1, R13
+	PSRLDQ $0x08, X1
+	MOVQ   X1, R14
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x01, R12
+	MOVQ   (BP), AX
+	MOVQ   AX, R15
+	MULQ   R10
+	MOVQ   AX, R13
+	MOVQ   DX, R14
+	MOVQ   (BP), AX
+	MULQ   R11
+	IMULQ  R12, R15
+	ADDQ   AX, R14
+	ADCQ   DX, R15
+	MOVQ   8(BP), AX
+	MOVQ   AX, R8
+	MULQ   R10
+	ADDQ   AX, R14
+	ADCQ   $0x00, DX
+	MOVQ   DX, R10
+	MOVQ   8(BP), AX
+	MULQ   R11
+	ADDQ   AX, R15
+	ADCQ   $0x00, DX
+	IMULQ  R12, R8
+	ADDQ   R10, R15
+	ADCQ   DX, R8
+	MOVQ   R13, R10
+	MOVQ   R14, R11
+	MOVQ   R15, R12
+	ANDQ   $0x03, R12
+	MOVQ   R15, R13
+	ANDQ   $-4, R13
+	MOVQ   R8, R14
+	SHRQ   $0x02, R8, R15
+	SHRQ   $0x02, R8
+	ADDQ   R13, R10
+	ADCQ   R14, R11
+	ADCQ   $0x00, R12
+	ADDQ   R15, R10
+	ADCQ   R8, R11
+	ADCQ   $0x00, R12
+	ADDQ   BX, DI
+
+sealSSEFinalize:
+	// Hash in the buffer lengths
+	ADDQ  ad_len+80(FP), R10
+	ADCQ  src_len+56(FP), R11
+	ADCQ  $0x01, R12
+	MOVQ  (BP), AX
+	MOVQ  AX, R15
+	MULQ  R10
+	MOVQ  AX, R13
+	MOVQ  DX, R14
+	MOVQ  (BP), AX
+	MULQ  R11
+	IMULQ R12, R15
+	ADDQ  AX, R14
+	ADCQ  DX, R15
+	MOVQ  8(BP), AX
+	MOVQ  AX, R8
+	MULQ  R10
+	ADDQ  AX, R14
+	ADCQ  $0x00, DX
+	MOVQ  DX, R10
+	MOVQ  8(BP), AX
+	MULQ  R11
+	ADDQ  AX, R15
+	ADCQ  $0x00, DX
+	IMULQ R12, R8
+	ADDQ  R10, R15
+	ADCQ  DX, R8
+	MOVQ  R13, R10
+	MOVQ  R14, R11
+	MOVQ  R15, R12
+	ANDQ  $0x03, R12
+	MOVQ  R15, R13
+	ANDQ  $-4, R13
+	MOVQ  R8, R14
+	SHRQ  $0x02, R8, R15
+	SHRQ  $0x02, R8
+	ADDQ  R13, R10
+	ADCQ  R14, R11
+	ADCQ  $0x00, R12
+	ADDQ  R15, R10
+	ADCQ  R8, R11
+	ADCQ  $0x00, R12
+
+	// Final reduce
+	MOVQ    R10, R13
+	MOVQ    R11, R14
+	MOVQ    R12, R15
+	SUBQ    $-5, R10
+	SBBQ    $-1, R11
+	SBBQ    $0x03, R12
+	CMOVQCS R13, R10
+	CMOVQCS R14, R11
+	CMOVQCS R15, R12
+
+	// Add in the "s" part of the key
+	ADDQ 16(BP), R10
+	ADCQ 24(BP), R11
+
+	// Finally store the tag at the end of the message
+	MOVQ R10, (DI)
+	MOVQ R11, 8(DI)
+	RET
+
 seal192AVX2:
 	VMOVDQA Y0, Y5
 	VMOVDQA Y14, Y9
diff --git a/vendor/golang.org/x/crypto/ssh/certs.go b/vendor/golang.org/x/crypto/ssh/certs.go
index 139fa31e..6f75d77e 100644
--- a/vendor/golang.org/x/crypto/ssh/certs.go
+++ b/vendor/golang.org/x/crypto/ssh/certs.go
@@ -348,6 +348,9 @@ func (c *CertChecker) CheckHostKey(addr string, remote net.Addr, key PublicKey)
 	if cert.CertType != HostCert {
 		return fmt.Errorf("ssh: certificate presented as a host key has type %d", cert.CertType)
 	}
+	if c.IsHostAuthority == nil {
+		return errors.New("ssh: cannot verify certificate, IsHostAuthority not set")
+	}
 	if !c.IsHostAuthority(cert.SignatureKey, addr) {
 		return fmt.Errorf("ssh: no authorities for hostname: %v", addr)
 	}
@@ -375,6 +378,9 @@ func (c *CertChecker) Authenticate(conn ConnMetadata, pubKey PublicKey) (*Permis
 	if cert.CertType != UserCert {
 		return nil, fmt.Errorf("ssh: cert has type %d", cert.CertType)
 	}
+	if c.IsUserAuthority == nil {
+		return nil, errors.New("ssh: cannot verify certificate, IsUserAuthority not set")
+	}
 	if !c.IsUserAuthority(cert.SignatureKey) {
 		return nil, fmt.Errorf("ssh: certificate signed by unrecognized authority")
 	}
@@ -438,7 +444,17 @@ func (c *CertChecker) CheckCert(principal string, cert *Certificate) error {
 	if before := int64(cert.ValidBefore); cert.ValidBefore != uint64(CertTimeInfinity) && (unixNow >= before || before < 0) {
 		return fmt.Errorf("ssh: cert has expired")
 	}
-	if err := cert.SignatureKey.Verify(cert.bytesForSigning(), cert.Signature); err != nil {
+	// Match OpenSSH: the SK user-presence flag is never enforced on a
+	// certificate's CA signature. OpenSSH calls sshkey_verify with
+	// detailsp==NULL in sshkey.c:cert_parse, so the UP/UV flags are
+	// not even extracted. The UP bit on a CA signature reflects the
+	// CA operator's presence at signing time, which has no bearing on
+	// whether the user being authenticated is present now; enforcing
+	// it here would only break interop with certificates issued by
+	// non-interactive SK CAs. skKeyWithoutUP is a no-op for non-SK
+	// keys (the common case).
+	caKey := skKeyWithoutUP(cert.SignatureKey)
+	if err := caKey.Verify(cert.bytesForSigning(), cert.Signature); err != nil {
 		return fmt.Errorf("ssh: certificate signature does not verify")
 	}
 
diff --git a/vendor/golang.org/x/crypto/ssh/channel.go b/vendor/golang.org/x/crypto/ssh/channel.go
index cc0bb7ab..67379966 100644
--- a/vendor/golang.org/x/crypto/ssh/channel.go
+++ b/vendor/golang.org/x/crypto/ssh/channel.go
@@ -11,6 +11,7 @@ import (
 	"io"
 	"log"
 	"sync"
+	"sync/atomic"
 )
 
 const (
@@ -131,11 +132,17 @@ func (r RejectionReason) String() string {
 	return fmt.Sprintf("unknown reason %d", int(r))
 }
 
-func min(a uint32, b int) uint32 {
-	if a < uint32(b) {
-		return a
+// minPayloadSize returns min(limit, length) clamped to a uint32. It is used
+// to compute the size of the next channel data packet from the remaining
+// payload. The comparison is done in int64 because length is an int — on
+// 64-bit systems len(data) can exceed 2^32, and a direct uint32(length)
+// cast would silently truncate to 0 at every multiple of 2^32, causing
+// WriteExtended's loop to spin without making progress.
+func minPayloadSize(limit uint32, length int) uint32 {
+	if int64(length) > int64(limit) {
+		return limit
 	}
-	return uint32(b)
+	return uint32(length)
 }
 
 type channelDirection uint8
@@ -177,6 +184,12 @@ type channel struct {
 	// with WantReply=true outstanding.  This lock is held by a
 	// goroutine that has such an outgoing request pending.
 	sentRequestMu sync.Mutex
+	// sentRequestPending is set to true while a SendRequest call with
+	// WantReply=true is in flight. handlePacket uses it as a gate: responses
+	// arriving while no request is pending are dropped to prevent a
+	// misbehaving peer from stalling the mux read loop by filling ch.msg
+	// with unsolicited channelRequestSuccess/Failure messages.
+	sentRequestPending atomic.Bool
 
 	incomingRequests chan *Request
 
@@ -251,7 +264,7 @@ func (ch *channel) WriteExtended(data []byte, extendedCode uint32) (n int, err e
 	ch.writeMu.Unlock()
 
 	for len(data) > 0 {
-		space := min(ch.maxRemotePayload, len(data))
+		space := minPayloadSize(ch.maxRemotePayload, len(data))
 		if space, err = ch.remoteWin.reserve(space); err != nil {
 			return n, err
 		}
@@ -460,6 +473,18 @@ func (ch *channel) handlePacket(packet []byte) error {
 		}
 
 		ch.incomingRequests <- &req
+	case *channelRequestSuccessMsg, *channelRequestFailureMsg:
+		// Drop responses that arrive when no SendRequest is waiting, to
+		// prevent a malicious peer from filling ch.msg and stalling the
+		// mux read loop. The non-blocking send additionally protects the
+		// loop if a well-behaved caller is slow to read.
+		if !ch.sentRequestPending.Load() {
+			return nil
+		}
+		select {
+		case ch.msg <- msg:
+		default:
+		}
 	default:
 		ch.msg <- msg
 	}
@@ -530,7 +555,17 @@ func (ch *channel) Reject(reason RejectionReason, message string) error {
 		Language: "en",
 	}
 	ch.decided = true
-	return ch.sendMessage(reject)
+	err := ch.sendMessage(reject)
+
+	// Remove the channel from the mux to prevent memory leaks.
+	// Do not call ch.close() here: no goroutine holds a reference to a
+	// rejected channel's internal channels (msg, incomingRequests), so
+	// removing it from chanList is sufficient for GC. Calling close()
+	// would race with the mux loop goroutine (handlePacket or dropAll),
+	// causing a panic from closing an already-closed channel.
+	ch.mux.chanList.remove(ch.localId)
+
+	return err
 }
 
 func (ch *channel) Read(data []byte) (int, error) {
@@ -586,6 +621,24 @@ func (ch *channel) SendRequest(name string, wantReply bool, payload []byte) (boo
 	if wantReply {
 		ch.sentRequestMu.Lock()
 		defer ch.sentRequestMu.Unlock()
+
+		// Open the gate so that responses arriving while this request is in
+		// flight are allowed to reach ch.msg. Responses arriving while no
+		// request is pending are dropped by handlePacket.
+		ch.sentRequestPending.Store(true)
+		defer ch.sentRequestPending.Store(false)
+
+		// Drain any spurious responses that may have been buffered. This
+		// prevents a previously buffered unexpected response from being
+		// consumed instead of the actual response for this request.
+	drain:
+		for {
+			select {
+			case <-ch.msg:
+			default:
+				break drain
+			}
+		}
 	}
 
 	msg := channelRequestMsg{
diff --git a/vendor/golang.org/x/crypto/ssh/cipher.go b/vendor/golang.org/x/crypto/ssh/cipher.go
index ad2b3705..48d01995 100644
--- a/vendor/golang.org/x/crypto/ssh/cipher.go
+++ b/vendor/golang.org/x/crypto/ssh/cipher.go
@@ -407,7 +407,7 @@ func (c *gcmCipher) readCipherPacket(seqNum uint32, r io.Reader) ([]byte, error)
 		return nil, fmt.Errorf("ssh: illegal padding %d", padding)
 	}
 
-	if int(padding+1) >= len(plain) {
+	if int(padding)+1 >= len(plain) {
 		return nil, fmt.Errorf("ssh: padding %d too large", padding)
 	}
 	plain = plain[1 : length-uint32(padding)]
diff --git a/vendor/golang.org/x/crypto/ssh/keys.go b/vendor/golang.org/x/crypto/ssh/keys.go
index 47a07539..3482c4d2 100644
--- a/vendor/golang.org/x/crypto/ssh/keys.go
+++ b/vendor/golang.org/x/crypto/ssh/keys.go
@@ -469,6 +469,12 @@ func parseRSA(in []byte) (out PublicKey, rest []byte, err error) {
 		return nil, nil, err
 	}
 
+	// 8192 bits is also the maximum RSA key size accepted by crypto/tls for
+	// signature verification:
+	// https://github.com/golang/go/blob/69801b25/src/crypto/tls/handshake_client.go#L1096
+	if w.N.BitLen() > 8192 {
+		return nil, nil, errors.New("ssh: rsa modulus too large")
+	}
 	if w.E.BitLen() > 24 {
 		return nil, nil, errors.New("ssh: exponent too large")
 	}
@@ -574,6 +580,24 @@ func checkDSAParams(param *dsa.Parameters) error {
 		return fmt.Errorf("ssh: unsupported DSA key size %d", l)
 	}
 
+	// FIPS 186-2 specifies that Q must be exactly 160 bits. We must enforce
+	// this to prevent DoS attacks where an attacker sends a huge Q which makes
+	// verification slow.
+	if l := param.Q.BitLen(); l != 160 {
+		return fmt.Errorf("ssh: unsupported DSA sub-prime size %d", l)
+	}
+
+	// The generator G is an element of the group, so it must be strictly less
+	// than the modulus P.
+	if param.G.Cmp(param.P) >= 0 {
+		return errors.New("ssh: DSA generator larger than modulus")
+	}
+
+	// G must be positive.
+	if param.G.Sign() <= 0 {
+		return errors.New("ssh: DSA generator must be positive")
+	}
+
 	return nil
 }
 
@@ -596,6 +620,14 @@ func parseDSA(in []byte) (out PublicKey, rest []byte, err error) {
 		return nil, nil, err
 	}
 
+	// The public value Y must be a non-zero element of the group, i.e.
+	// strictly between 0 and P. crypto/dsa.Verify does not range-check Y,
+	// so we reject out-of-range values here to prevent a maliciously
+	// oversized Y from slowing verification.
+	if w.Y.Sign() <= 0 || w.Y.Cmp(w.P) >= 0 {
+		return nil, nil, errors.New("ssh: DSA public value Y out of range")
+	}
+
 	key := &dsaPublicKey{
 		Parameters: param,
 		Y:          w.Y,
@@ -869,11 +901,25 @@ type skFields struct {
 	Counter uint32
 }
 
+// flagUserPresence is the "user present" bit (UP) in the SK signature
+// flags, matching the FIDO CTAP2 authenticatorData UP flag. See
+// openssh/PROTOCOL.u2f.
+const flagUserPresence = 0x01
+
+// errSKMissingUserPresence is returned by SK key Verify methods when
+// the signature does not assert user presence and the key was not
+// marked as no-touch-required.
+var errSKMissingUserPresence = errors.New("ssh: signature missing required user presence flag")
+
 type skECDSAPublicKey struct {
 	// application is a URL-like string, typically "ssh:" for SSH.
 	// see openssh/PROTOCOL.u2f for details.
 	application string
 	ecdsa.PublicKey
+	// noTouchRequired, when true, disables the default user-presence
+	// check in Verify. It is set by skKeyWithoutUP on a clone of the
+	// key, never on an instance shared across authentication attempts.
+	noTouchRequired bool
 }
 
 func (k *skECDSAPublicKey) Type() string {
@@ -959,6 +1005,10 @@ func (k *skECDSAPublicKey) Verify(data []byte, sig *Signature) error {
 		return err
 	}
 
+	if skf.Flags&flagUserPresence == 0 && !k.noTouchRequired {
+		return errSKMissingUserPresence
+	}
+
 	blob := struct {
 		ApplicationDigest []byte `ssh:"rest"`
 		Flags             byte
@@ -992,6 +1042,10 @@ type skEd25519PublicKey struct {
 	// see openssh/PROTOCOL.u2f for details.
 	application string
 	ed25519.PublicKey
+	// noTouchRequired, when true, disables the default user-presence
+	// check in Verify. It is set by skKeyWithoutUP on a clone of the
+	// key, never on an instance shared across authentication attempts.
+	noTouchRequired bool
 }
 
 func (k *skEd25519PublicKey) Type() string {
@@ -1066,6 +1120,10 @@ func (k *skEd25519PublicKey) Verify(data []byte, sig *Signature) error {
 		return err
 	}
 
+	if skf.Flags&flagUserPresence == 0 && !k.noTouchRequired {
+		return errSKMissingUserPresence
+	}
+
 	blob := struct {
 		ApplicationDigest []byte `ssh:"rest"`
 		Flags             byte
diff --git a/vendor/golang.org/x/crypto/ssh/mux.go b/vendor/golang.org/x/crypto/ssh/mux.go
index d2d24c63..3bc4afbd 100644
--- a/vendor/golang.org/x/crypto/ssh/mux.go
+++ b/vendor/golang.org/x/crypto/ssh/mux.go
@@ -91,9 +91,10 @@ type mux struct {
 
 	incomingChannels chan NewChannel
 
-	globalSentMu     sync.Mutex
-	globalResponses  chan interface{}
-	incomingRequests chan *Request
+	globalSentMu      sync.Mutex
+	globalSentPending atomic.Bool
+	globalResponses   chan interface{}
+	incomingRequests  chan *Request
 
 	errCond *sync.Cond
 	err     error
@@ -141,6 +142,24 @@ func (m *mux) SendRequest(name string, wantReply bool, payload []byte) (bool, []
 	if wantReply {
 		m.globalSentMu.Lock()
 		defer m.globalSentMu.Unlock()
+
+		// Open the gate so that responses arriving while this request is in
+		// flight are allowed to reach globalResponses. Any response arriving
+		// while no request is pending is dropped by handleGlobalPacket.
+		m.globalSentPending.Store(true)
+		defer m.globalSentPending.Store(false)
+
+		// Drain any spurious responses that may have been buffered. This prevents
+		// a previously buffered unexpected response from being consumed instead
+		// of the actual response for this request.
+	drain:
+		for {
+			select {
+			case <-m.globalResponses:
+			default:
+				break drain
+			}
+		}
 	}
 
 	if err := m.sendMessage(globalRequestMsg{
@@ -267,7 +286,16 @@ func (m *mux) handleGlobalPacket(packet []byte) error {
 			mux:       m,
 		}
 	case *globalRequestSuccessMsg, *globalRequestFailureMsg:
-		m.globalResponses <- msg
+		// Drop responses that arrive when no SendRequest is waiting, to
+		// prevent a malicious peer from staging responses for a future
+		// caller.
+		if !m.globalSentPending.Load() {
+			return nil
+		}
+		select {
+		case m.globalResponses <- msg:
+		default:
+		}
 	default:
 		panic(fmt.Sprintf("not a global message %#v", msg))
 	}
diff --git a/vendor/golang.org/x/crypto/ssh/server.go b/vendor/golang.org/x/crypto/ssh/server.go
index 064dcbaf..0192a675 100644
--- a/vendor/golang.org/x/crypto/ssh/server.go
+++ b/vendor/golang.org/x/crypto/ssh/server.go
@@ -34,15 +34,20 @@ type Permissions struct {
 	// or not supported.
 	CriticalOptions map[string]string
 
-	// Extensions are extra functionality that the server may
-	// offer on authenticated connections. Lack of support for an
-	// extension does not preclude authenticating a user. Common
-	// extensions are "permit-agent-forwarding",
-	// "permit-X11-forwarding". The Go SSH library currently does
-	// not act on any extension, and it is up to server
-	// implementations to honor them. Extensions can be used to
-	// pass data from the authentication callbacks to the server
-	// application layer.
+	// Extensions are extra functionality that the server may offer on
+	// authenticated connections. Lack of support for an extension does not
+	// preclude authenticating a user. Common extensions are
+	// "permit-agent-forwarding", "permit-X11-forwarding". In general the Go
+	// SSH library does not act on extensions and it is up to server
+	// implementations to honor them; extensions can also be used to pass data
+	// from the authentication callbacks to the server application layer.
+	//
+	// The one extension acted upon by this library is "no-touch-required",
+	// which applies only to security-key public keys
+	// (sk-ecdsa-sha2-nistp256@openssh.com and sk-ssh-ed25519@openssh.com).
+	// When present, it waives the default requirement that SK signatures
+	// assert user presence (i.e. a physical touch of the authenticator)
+	// during signature verification.
 	Extensions map[string]string
 
 	// ExtraData allows to store user defined data.
@@ -84,6 +89,79 @@ type ServerPreAuthConn interface {
 	SendAuthBanner(string) error
 }
 
+// noTouchRequiredExtension is the extension name used by OpenSSH in
+// authorized_keys options and certificate extensions to mark keys
+// whose signatures do not need to assert user presence (touch). See
+// ssh-keygen(1) and sshd(8).
+const noTouchRequiredExtension = "no-touch-required"
+
+// noTouchAllowed reports whether the user presence requirement on
+// SK signatures should be waived for this authentication attempt. The
+// requirement is waived when the "no-touch-required" extension is
+// present either in the Permissions returned by the auth callback
+// (authorized_keys-level opt-out) or in the certificate's own
+// Extensions (CA-level opt-out), matching OpenSSH behavior. OpenSSH
+// reads the per-key opt-out only from cert Extensions and
+// authorized_keys options (never from CriticalOptions); we follow the
+// same rule.
+func noTouchAllowed(pubKey PublicKey, perms *Permissions) bool {
+	if perms != nil {
+		if _, ok := perms.Extensions[noTouchRequiredExtension]; ok {
+			return true
+		}
+	}
+	if cert, ok := pubKey.(*Certificate); ok {
+		if _, ok := cert.Extensions[noTouchRequiredExtension]; ok {
+			return true
+		}
+	}
+	return false
+}
+
+// skKeyWithoutUP returns a PublicKey equivalent to pubKey but whose
+// Verify accepts SK signatures with the user-presence flag clear. If
+// pubKey is not (and does not wrap) an SK key, pubKey is returned
+// unchanged. The returned value never mutates pubKey: for SK keys a
+// shallow copy is made so that the noTouchRequired flag is set only on
+// the clone.
+//
+// The implementation is iterative rather than recursive. When pubKey
+// is a *Certificate we unwrap exactly one level to look at the inner
+// key. The SSH cert format forbids Certificate.Key from being another
+// Certificate (parseCert rejects it), but nothing stops callers from
+// constructing such a value directly in Go; a recursive descent could
+// otherwise be driven to unbounded depth by a hand-crafted or cyclic
+// Certificate. A malformed input of that shape simply returns
+// unchanged here.
+func skKeyWithoutUP(pubKey PublicKey) PublicKey {
+	cert, isCert := pubKey.(*Certificate)
+	target := pubKey
+	if isCert {
+		target = cert.Key
+	}
+	var cloned PublicKey
+	switch k := target.(type) {
+	case *skECDSAPublicKey:
+		c := *k
+		c.noTouchRequired = true
+		cloned = &c
+	case *skEd25519PublicKey:
+		c := *k
+		c.noTouchRequired = true
+		cloned = &c
+	default:
+		// Not an SK key (or a pathological *Certificate wrapping
+		// another *Certificate): pubKey is already usable for Verify.
+		return pubKey
+	}
+	if !isCert {
+		return cloned
+	}
+	c := *cert
+	c.Key = cloned
+	return &c
+}
+
 // ServerConfig holds server specific configuration data.
 type ServerConfig struct {
 	// Config contains configuration shared between client and server.
@@ -242,8 +320,10 @@ func (c *pubKeyCache) add(candidate cachedPubKey) {
 type ServerConn struct {
 	Conn
 
-	// If the succeeding authentication callback returned a
-	// non-nil Permissions pointer, it is stored here.
+	// If the succeeding authentication callback returned a non-nil Permissions
+	// pointer, it is stored here. These are the permissions from the final,
+	// successful authentication method. Permissions returned by callbacks that
+	// return PartialSuccessError are not preserved and must be nil.
 	Permissions *Permissions
 }
 
@@ -737,8 +817,15 @@ userAuthLoop:
 				}
 
 				signedData := buildDataSignedForAuth(sessionID, userAuthReq, algo, pubKeyData)
-
-				if err := pubKey.Verify(signedData, sig); err != nil {
+				// pubKey is reused below for VerifiedPublicKeyCallback and
+				// must remain the key as presented by the client; derive a
+				// separate value for Verify that carries any applicable
+				// no-touch-required opt-out.
+				pubKeyForVerify := pubKey
+				if noTouchAllowed(pubKey, candidate.perms) {
+					pubKeyForVerify = skKeyWithoutUP(pubKey)
+				}
+				if err := pubKeyForVerify.Verify(signedData, sig); err != nil {
 					return nil, err
 				}
 
@@ -750,6 +837,13 @@ userAuthLoop:
 					// considered verified and the callback must not run.
 					perms, authErr = config.VerifiedPublicKeyCallback(s, pubKey, perms, algo)
 				}
+				if authErr == nil && perms != nil && perms.CriticalOptions != nil {
+					if saco := perms.CriticalOptions[sourceAddressCriticalOption]; saco != "" {
+						if err := checkSourceAddress(s.RemoteAddr(), saco); err != nil {
+							authErr = err
+						}
+					}
+				}
 			}
 		case "gssapi-with-mic":
 			if authConfig.GSSAPIWithMICConfig == nil {
@@ -824,6 +918,13 @@ userAuthLoop:
 		var failureMsg userAuthFailureMsg
 
 		if partialSuccess, ok := authErr.(*PartialSuccessError); ok {
+			// Permissions are not preserved between authentication steps. To
+			// avoid confusion about the final state of the connection, we
+			// disallow returning non-nil Permissions combined with
+			// PartialSuccessError.
+			if perms != nil {
+				return nil, errors.New("ssh: permissions must be nil when returning PartialSuccessError")
+			}
 			// After a partial success error we don't allow changing the user
 			// name and execute the NoClientAuthCallback.
 			partialSuccessReturned = true
diff --git a/vendor/modules.txt b/vendor/modules.txt
index cdc072d1..bdcd2737 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -627,7 +627,7 @@ go.opentelemetry.io/otel/trace
 go.opentelemetry.io/otel/trace/embedded
 go.opentelemetry.io/otel/trace/internal/telemetry
 go.opentelemetry.io/otel/trace/noop
-# golang.org/x/crypto v0.51.0
+# golang.org/x/crypto v0.52.0
 ## explicit; go 1.25.0
 golang.org/x/crypto/argon2
 golang.org/x/crypto/bcrypt