diff --git a/go.mod b/go.mod index 91b81bc54..11a57a8d8 100644 --- a/go.mod +++ b/go.mod @@ -5,28 +5,29 @@ go 1.25.9 require ( github.com/DataDog/datadog-go/v5 v5.8.3 github.com/go-ini/ini v1.67.0 - github.com/go-mysql-org/go-mysql v1.11.0 + github.com/go-mysql-org/go-mysql v1.15.0 github.com/go-sql-driver/mysql v1.8.1 github.com/google/uuid v1.6.0 github.com/hashicorp/go-version v1.7.0 github.com/openark/golib v0.0.0-20210531070646-355f37940af8 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go v0.37.0 github.com/testcontainers/testcontainers-go/modules/mysql v0.37.0 - golang.org/x/sync v0.18.0 + golang.org/x/sync v0.20.0 golang.org/x/term v0.37.0 - golang.org/x/text v0.31.0 + golang.org/x/text v0.36.0 ) require ( dario.cat/mergo v1.0.1 // indirect - filippo.io/edwards25519 v1.1.0 // indirect + filippo.io/edwards25519 v1.2.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect + github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect @@ -38,10 +39,10 @@ require ( github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/goccy/go-json v0.10.2 // indirect + github.com/goccy/go-json v0.10.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.18.6 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -53,14 +54,14 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb // indirect - github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 // indirect - github.com/pingcap/tidb/pkg/parser v0.0.0-20241118164214-4f047be191be // indirect + github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 // indirect + github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 // indirect + github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/shirou/gopsutil/v4 v4.25.1 // indirect - github.com/shopspring/decimal v1.2.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -74,7 +75,7 @@ require ( go.opentelemetry.io/otel/trace v1.35.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect + go.uber.org/zap v1.28.0 // indirect golang.org/x/crypto v0.45.0 // indirect golang.org/x/sys v0.38.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect diff --git a/go.sum b/go.sum index e39b587e8..dcb552f90 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -21,6 +23,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -49,12 +53,16 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-mysql-org/go-mysql v1.11.0 h1:Y0ooXu2UtbjsgpfjFBXZEvidEl1q8n0ESxej0zZ78Zc= github.com/go-mysql-org/go-mysql v1.11.0/go.mod h1:y/7aggbs+Io8rPVerIjTe1+nMgt8q5tBIxIc+qQnE0k= +github.com/go-mysql-org/go-mysql v1.15.0 h1:bZeRUc9yNVbFEyote79Q4j8SV+q8Ls32AYXRl2QjUoc= +github.com/go-mysql-org/go-mysql v1.15.0/go.mod h1:VjBTZTTDKL8OMXUAhNbg3VHaVVq9HOXJEBLpAKBFIfE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= @@ -71,6 +79,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -105,10 +115,16 @@ github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgr github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb h1:3pSi4EDG6hg0orE1ndHkXvX6Qdq2cZn8gAPir8ymKZk= github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 h1:cLgCk5mwDG9lDH+dPK8TmEliTjyGJwwKN0qevWAl8IY= +github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2/go.mod h1:ktAJCA9lxrHHjVyVl2pKJFvzBnq2eZbb+CUOjBRPlXo= github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 h1:2SOzvGvE8beiC1Y4g9Onkvu6UmuBBOeWRGQEjJaT/JY= github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= +github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 h1:A2bEfgSb7hLwR9mxDszgGKweF+xY9YoTDG+8RjdFjDE= +github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1/go.mod h1:pxfz2oJfAuhwrb3/rcLqD//GS/5gRP4gD022iP3cEO0= github.com/pingcap/tidb/pkg/parser v0.0.0-20241118164214-4f047be191be h1:t5EkCmZpxLCig5GQA0AZG47aqsuL5GTsJeeUD+Qfies= github.com/pingcap/tidb/pkg/parser v0.0.0-20241118164214-4f047be191be/go.mod h1:Hju1TEWZvrctQKbztTRwXH7rd41Yq0Pgmq4PrEKcq7o= +github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17 h1:cfAVPis6GP6lxQgm1WGaNGi4rVXTB4KDvYf96LjqRCM= +github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17/go.mod h1:zDLDsfNBU5+L6T4J9/OgWAHc/WZvMUjbpgHqQ/t3yKo= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -122,6 +138,8 @@ github.com/shirou/gopsutil/v4 v4.25.1 h1:QSWkTc+fu9LTAWfkZwZ6j8MSUk4A2LV7rbH0Zqm github.com/shirou/gopsutil/v4 v4.25.1/go.mod h1:RoUCUpndaJFtT+2zsZzzmhvbfGoDCJ7nFXKJf8GqJbI= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -132,6 +150,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -139,8 +158,11 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/testcontainers/testcontainers-go v0.37.0 h1:L2Qc0vkTw2EHWQ08djon0D2uw7Z/PtHS/QzZZ5Ra/hg= github.com/testcontainers/testcontainers-go v0.37.0/go.mod h1:QPzbxZhQ6Bclip9igjLFj6z0hs01bU8lrl2dHQmgFGM= github.com/testcontainers/testcontainers-go/modules/mysql v0.37.0 h1:LqUos1oR5iuuzorFnSvxsHNdYdCHB/DfI82CuT58wbI= @@ -187,6 +209,8 @@ go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -210,6 +234,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -233,6 +259,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/go/binlog/gomysql_reader.go b/go/binlog/gomysql_reader.go index 189a5f399..06203be90 100644 --- a/go/binlog/gomysql_reader.go +++ b/go/binlog/gomysql_reader.go @@ -16,6 +16,7 @@ import ( "time" "context" + gomysql "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/replication" uuid "github.com/google/uuid" @@ -165,8 +166,11 @@ func (gmr *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesChan gmr.currentCoordinates = gmr.LastTrxCoords.Clone() } coords := gmr.currentCoordinates.(*mysql.GTIDBinlogCoordinates) - trxGset := gomysql.NewUUIDSet(sid, gomysql.Interval{Start: event.GNO, Stop: event.GNO + 1}) - coords.GTIDSet.AddSet(trxGset) + if coords.GTIDSet == nil { + gtidSet := gomysql.NewMysqlGTIDSet() + coords.GTIDSet = >idSet + } + coords.GTIDSet.AddGTID(sid, event.GNO) gmr.currentCoordinatesMutex.Unlock() case *replication.RotateEvent: if gmr.migrationContext.UseGTIDs { diff --git a/go/mysql/binlog_gtid.go b/go/mysql/binlog_gtid.go index edd027a73..505ac92d1 100644 --- a/go/mysql/binlog_gtid.go +++ b/go/mysql/binlog_gtid.go @@ -12,7 +12,6 @@ import ( // GTIDBinlogCoordinates describe binary log coordinates in MySQL GTID format. type GTIDBinlogCoordinates struct { GTIDSet *gomysql.MysqlGTIDSet - UUIDSet *gomysql.UUIDSet } // NewGTIDBinlogCoordinates parses a MySQL GTID set into a *GTIDBinlogCoordinates struct. @@ -25,9 +24,6 @@ func NewGTIDBinlogCoordinates(gtidSet string) (*GTIDBinlogCoordinates, error) { // DisplayString returns a user-friendly string representation of these current UUID set or the full GTID set. func (coord *GTIDBinlogCoordinates) DisplayString() string { - if coord.UUIDSet != nil { - return coord.UUIDSet.String() - } return coord.String() } @@ -80,8 +76,5 @@ func (coord *GTIDBinlogCoordinates) Clone() BinlogCoordinates { if coord.GTIDSet != nil { out.GTIDSet = coord.GTIDSet.Clone().(*gomysql.MysqlGTIDSet) } - if coord.UUIDSet != nil { - out.UUIDSet = coord.UUIDSet.Clone() - } return out } diff --git a/vendor/filippo.io/edwards25519/README.md b/vendor/filippo.io/edwards25519/README.md index 24e2457d8..dcdd8d85f 100644 --- a/vendor/filippo.io/edwards25519/README.md +++ b/vendor/filippo.io/edwards25519/README.md @@ -7,8 +7,10 @@ import "filippo.io/edwards25519" This library implements the edwards25519 elliptic curve, exposing the necessary APIs to build a wide array of higher-level primitives. Read the docs at [pkg.go.dev/filippo.io/edwards25519](https://pkg.go.dev/filippo.io/edwards25519). -The code is originally derived from Adam Langley's internal implementation in the Go standard library, and includes George Tankersley's [performance improvements](https://golang.org/cl/71950). It was then further developed by Henry de Valence for use in ristretto255, and was finally [merged back into the Go standard library](https://golang.org/cl/276272) as of Go 1.17. It now tracks the upstream codebase and extends it with additional functionality. +The package tracks the upstream standard library package `crypto/internal/fips140/edwards25519` and extends it with additional functionality. -Most users don't need this package, and should instead use `crypto/ed25519` for signatures, `golang.org/x/crypto/curve25519` for Diffie-Hellman, or `github.com/gtank/ristretto255` for prime order group logic. However, for anyone currently using a fork of `crypto/internal/edwards25519`/`crypto/ed25519/internal/edwards25519` or `github.com/agl/edwards25519`, this package should be a safer, faster, and more powerful alternative. +The code is originally derived from Adam Langley's internal implementation in the Go standard library, and includes George Tankersley's [performance improvements](https://golang.org/cl/71950). It was then further developed by Henry de Valence for use in ristretto255, and was finally [merged back into the Go standard library](https://golang.org/cl/276272) as of Go 1.17. + +Most users don't need this package, and should instead use `crypto/ed25519` for signatures, `crypto/ecdh` for Diffie-Hellman, or `github.com/gtank/ristretto255` for prime order group logic. However, for anyone currently using a fork of the internal `edwards25519` package or of `github.com/agl/edwards25519`, this package should be a safer, faster, and more powerful alternative. Since this package is meant to curb proliferation of edwards25519 implementations in the Go ecosystem, it welcomes requests for new APIs or reviewable performance improvements. diff --git a/vendor/filippo.io/edwards25519/doc.go b/vendor/filippo.io/edwards25519/doc.go index ab6aaebc0..dd2deb644 100644 --- a/vendor/filippo.io/edwards25519/doc.go +++ b/vendor/filippo.io/edwards25519/doc.go @@ -10,11 +10,11 @@ // the curve used by the Ed25519 signature scheme. // // Most users don't need this package, and should instead use crypto/ed25519 for -// signatures, golang.org/x/crypto/curve25519 for Diffie-Hellman, or -// github.com/gtank/ristretto255 for prime order group logic. +// signatures, crypto/ecdh for Diffie-Hellman, or github.com/gtank/ristretto255 +// for prime order group logic. // // However, developers who do need to interact with low-level edwards25519 // operations can use this package, which is an extended version of -// crypto/internal/edwards25519 from the standard library repackaged as +// crypto/internal/fips140/edwards25519 from the standard library repackaged as // an importable module. package edwards25519 diff --git a/vendor/filippo.io/edwards25519/extra.go b/vendor/filippo.io/edwards25519/extra.go index d152d68ff..ee9b5ca5b 100644 --- a/vendor/filippo.io/edwards25519/extra.go +++ b/vendor/filippo.io/edwards25519/extra.go @@ -9,6 +9,7 @@ package edwards25519 import ( "errors" + "slices" "filippo.io/edwards25519/field" ) @@ -100,13 +101,15 @@ func (v *Point) bytesMontgomery(buf *[32]byte) []byte { // // u = (1 + y) / (1 - y) // - // where y = Y / Z. + // where y = Y / Z and therefore + // + // u = (Z + Y) / (Z - Y) - var y, recip, u field.Element + var n, r, u field.Element - y.Multiply(&v.y, y.Invert(&v.z)) // y = Y / Z - recip.Invert(recip.Subtract(feOne, &y)) // r = 1/(1 - y) - u.Multiply(u.Add(feOne, &y), &recip) // u = (1 + y)*r + n.Add(&v.z, &v.y) // n = Z + Y + r.Invert(r.Subtract(&v.z, &v.y)) // r = 1 / (Z - Y) + u.Multiply(&n, &r) // u = n * r return copyFieldElement(buf, &u) } @@ -124,7 +127,7 @@ func (v *Point) MultByCofactor(p *Point) *Point { return v.fromP1xP1(&result) } -// Given k > 0, set s = s**(2*i). +// Given k > 0, set s = s**(2*k). func (s *Scalar) pow2k(k int) { for i := 0; i < k; i++ { s.Multiply(s, s) @@ -250,12 +253,14 @@ func (v *Point) MultiScalarMult(scalars []*Scalar, points []*Point) *Point { // between each point in the multiscalar equation. // Build lookup tables for each point - tables := make([]projLookupTable, len(points)) + tables := make([]projLookupTable, 0, 2) // avoid allocation for small sizes + tables = slices.Grow(tables, len(points))[:len(points)] for i := range tables { tables[i].FromP3(points[i]) } // Compute signed radix-16 digits for each scalar - digits := make([][64]int8, len(scalars)) + digits := make([][64]int8, 0, 2) // avoid allocation for small sizes + digits = slices.Grow(digits, len(scalars))[:len(scalars)] for i := range digits { digits[i] = scalars[i].signedRadix16() } @@ -265,6 +270,7 @@ func (v *Point) MultiScalarMult(scalars []*Scalar, points []*Point) *Point { tmp1 := &projP1xP1{} tmp2 := &projP2{} // Lookup-and-add the appropriate multiple of each input point + v.Set(NewIdentityPoint()) for j := range tables { tables[j].SelectInto(multiple, digits[j][63]) tmp1.Add(v, multiple) // tmp1 = v + x_(j,63)*Q in P1xP1 coords @@ -347,3 +353,49 @@ func (v *Point) VarTimeMultiScalarMult(scalars []*Scalar, points []*Point) *Poin v.fromP2(tmp2) return v } + +// Select sets v to a if cond == 1 and to b if cond == 0. +func (v *Point) Select(a, b *Point, cond int) *Point { + checkInitialized(a, b) + v.x.Select(&a.x, &b.x, cond) + v.y.Select(&a.y, &b.y, cond) + v.z.Select(&a.z, &b.z, cond) + v.t.Select(&a.t, &b.t, cond) + return v +} + +// Double sets v = p + p, and returns v. +func (v *Point) Double(p *Point) *Point { + checkInitialized(p) + + pp := new(projP2).FromP3(p) + p1 := new(projP1xP1).Double(pp) + return v.fromP1xP1(p1) +} + +func (v *Point) addCached(p *Point, qCached *projCached) *Point { + result := new(projP1xP1).Add(p, qCached) + return v.fromP1xP1(result) +} + +// ScalarMultSlow sets v = x * q, and returns v. It doesn't precompute a large +// table, so it is considerably slower, but requires less memory. +// +// The scalar multiplication is done in constant time. +func (v *Point) ScalarMultSlow(x *Scalar, q *Point) *Point { + checkInitialized(q) + + s := x.Bytes() + qCached := new(projCached).FromP3(q) + v.Set(NewIdentityPoint()) + t := new(Point) + + for i := 255; i >= 0; i-- { + v.Double(v) + t.addCached(v, qCached) + cond := (s[i/8] >> (i % 8)) & 1 + v.Select(t, v, int(cond)) + } + + return v +} diff --git a/vendor/filippo.io/edwards25519/field/fe.go b/vendor/filippo.io/edwards25519/field/fe.go index 5518ef2b9..4d52cc10d 100644 --- a/vendor/filippo.io/edwards25519/field/fe.go +++ b/vendor/filippo.io/edwards25519/field/fe.go @@ -90,11 +90,7 @@ func (v *Element) Add(a, b *Element) *Element { v.l2 = a.l2 + b.l2 v.l3 = a.l3 + b.l3 v.l4 = a.l4 + b.l4 - // Using the generic implementation here is actually faster than the - // assembly. Probably because the body of this function is so simple that - // the compiler can figure out better optimizations by inlining the carry - // propagation. - return v.carryPropagateGeneric() + return v.carryPropagate() } // Subtract sets v = a - b, and returns v. @@ -232,18 +228,22 @@ func (v *Element) bytes(out *[32]byte) []byte { t := *v t.reduce() - var buf [8]byte - for i, l := range [5]uint64{t.l0, t.l1, t.l2, t.l3, t.l4} { - bitsOffset := i * 51 - binary.LittleEndian.PutUint64(buf[:], l<= len(out) { - break - } - out[off] |= bb - } - } + // Pack five 51-bit limbs into four 64-bit words: + // + // 255 204 153 102 51 0 + // ├──l4──┼──l3──┼──l2──┼──l1──┼──l0──┤ + // ├───u3───┼───u2───┼───u1───┼───u0───┤ + // 256 192 128 64 0 + + u0 := t.l1<<51 | t.l0 + u1 := t.l2<<(102-64) | t.l1>>(64-51) + u2 := t.l3<<(153-128) | t.l2>>(128-102) + u3 := t.l4<<(204-192) | t.l3>>(192-153) + + binary.LittleEndian.PutUint64(out[0*8:], u0) + binary.LittleEndian.PutUint64(out[1*8:], u1) + binary.LittleEndian.PutUint64(out[2*8:], u2) + binary.LittleEndian.PutUint64(out[3*8:], u3) return out[:] } diff --git a/vendor/filippo.io/edwards25519/field/fe_amd64.go b/vendor/filippo.io/edwards25519/field/fe_amd64.go index edcf163c4..00bf8f447 100644 --- a/vendor/filippo.io/edwards25519/field/fe_amd64.go +++ b/vendor/filippo.io/edwards25519/field/fe_amd64.go @@ -1,7 +1,6 @@ // Code generated by command: go run fe_amd64_asm.go -out ../fe_amd64.s -stubs ../fe_amd64.go -pkg field. DO NOT EDIT. -//go:build amd64 && gc && !purego -// +build amd64,gc,!purego +//go:build !purego package field diff --git a/vendor/filippo.io/edwards25519/field/fe_amd64.s b/vendor/filippo.io/edwards25519/field/fe_amd64.s index 293f013c9..5e06e242e 100644 --- a/vendor/filippo.io/edwards25519/field/fe_amd64.s +++ b/vendor/filippo.io/edwards25519/field/fe_amd64.s @@ -1,7 +1,6 @@ // Code generated by command: go run fe_amd64_asm.go -out ../fe_amd64.s -stubs ../fe_amd64.go -pkg field. DO NOT EDIT. -//go:build amd64 && gc && !purego -// +build amd64,gc,!purego +//go:build !purego #include "textflag.h" @@ -17,32 +16,36 @@ TEXT ·feMul(SB), NOSPLIT, $0-24 MOVQ DX, SI // r0 += 19×a1×b4 - MOVQ 8(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 32(BX) - ADDQ AX, DI - ADCQ DX, SI + MOVQ 8(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 32(BX) + ADDQ AX, DI + ADCQ DX, SI // r0 += 19×a2×b3 - MOVQ 16(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 24(BX) - ADDQ AX, DI - ADCQ DX, SI + MOVQ 16(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 24(BX) + ADDQ AX, DI + ADCQ DX, SI // r0 += 19×a3×b2 - MOVQ 24(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 16(BX) - ADDQ AX, DI - ADCQ DX, SI + MOVQ 24(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 16(BX) + ADDQ AX, DI + ADCQ DX, SI // r0 += 19×a4×b1 - MOVQ 32(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 8(BX) - ADDQ AX, DI - ADCQ DX, SI + MOVQ 32(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 8(BX) + ADDQ AX, DI + ADCQ DX, SI // r1 = a0×b1 MOVQ (CX), AX @@ -57,25 +60,28 @@ TEXT ·feMul(SB), NOSPLIT, $0-24 ADCQ DX, R8 // r1 += 19×a2×b4 - MOVQ 16(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 32(BX) - ADDQ AX, R9 - ADCQ DX, R8 + MOVQ 16(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 32(BX) + ADDQ AX, R9 + ADCQ DX, R8 // r1 += 19×a3×b3 - MOVQ 24(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 24(BX) - ADDQ AX, R9 - ADCQ DX, R8 + MOVQ 24(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 24(BX) + ADDQ AX, R9 + ADCQ DX, R8 // r1 += 19×a4×b2 - MOVQ 32(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 16(BX) - ADDQ AX, R9 - ADCQ DX, R8 + MOVQ 32(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 16(BX) + ADDQ AX, R9 + ADCQ DX, R8 // r2 = a0×b2 MOVQ (CX), AX @@ -96,18 +102,20 @@ TEXT ·feMul(SB), NOSPLIT, $0-24 ADCQ DX, R10 // r2 += 19×a3×b4 - MOVQ 24(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 32(BX) - ADDQ AX, R11 - ADCQ DX, R10 + MOVQ 24(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 32(BX) + ADDQ AX, R11 + ADCQ DX, R10 // r2 += 19×a4×b3 - MOVQ 32(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 24(BX) - ADDQ AX, R11 - ADCQ DX, R10 + MOVQ 32(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 24(BX) + ADDQ AX, R11 + ADCQ DX, R10 // r3 = a0×b3 MOVQ (CX), AX @@ -134,11 +142,12 @@ TEXT ·feMul(SB), NOSPLIT, $0-24 ADCQ DX, R12 // r3 += 19×a4×b4 - MOVQ 32(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 32(BX) - ADDQ AX, R13 - ADCQ DX, R12 + MOVQ 32(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 32(BX) + ADDQ AX, R13 + ADCQ DX, R12 // r4 = a0×b4 MOVQ (CX), AX @@ -232,18 +241,22 @@ TEXT ·feSquare(SB), NOSPLIT, $0-16 MOVQ DX, BX // r0 += 38×l1×l4 - MOVQ 8(CX), AX - IMUL3Q $0x26, AX, AX - MULQ 32(CX) - ADDQ AX, SI - ADCQ DX, BX + MOVQ 8(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + SHLQ $0x01, AX + MULQ 32(CX) + ADDQ AX, SI + ADCQ DX, BX // r0 += 38×l2×l3 - MOVQ 16(CX), AX - IMUL3Q $0x26, AX, AX - MULQ 24(CX) - ADDQ AX, SI - ADCQ DX, BX + MOVQ 16(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + SHLQ $0x01, AX + MULQ 24(CX) + ADDQ AX, SI + ADCQ DX, BX // r1 = 2×l0×l1 MOVQ (CX), AX @@ -253,18 +266,21 @@ TEXT ·feSquare(SB), NOSPLIT, $0-16 MOVQ DX, DI // r1 += 38×l2×l4 - MOVQ 16(CX), AX - IMUL3Q $0x26, AX, AX - MULQ 32(CX) - ADDQ AX, R8 - ADCQ DX, DI + MOVQ 16(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + SHLQ $0x01, AX + MULQ 32(CX) + ADDQ AX, R8 + ADCQ DX, DI // r1 += 19×l3×l3 - MOVQ 24(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 24(CX) - ADDQ AX, R8 - ADCQ DX, DI + MOVQ 24(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 24(CX) + ADDQ AX, R8 + ADCQ DX, DI // r2 = 2×l0×l2 MOVQ (CX), AX @@ -280,11 +296,13 @@ TEXT ·feSquare(SB), NOSPLIT, $0-16 ADCQ DX, R9 // r2 += 38×l3×l4 - MOVQ 24(CX), AX - IMUL3Q $0x26, AX, AX - MULQ 32(CX) - ADDQ AX, R10 - ADCQ DX, R9 + MOVQ 24(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + SHLQ $0x01, AX + MULQ 32(CX) + ADDQ AX, R10 + ADCQ DX, R9 // r3 = 2×l0×l3 MOVQ (CX), AX @@ -294,18 +312,19 @@ TEXT ·feSquare(SB), NOSPLIT, $0-16 MOVQ DX, R11 // r3 += 2×l1×l2 - MOVQ 8(CX), AX - IMUL3Q $0x02, AX, AX - MULQ 16(CX) - ADDQ AX, R12 - ADCQ DX, R11 + MOVQ 8(CX), AX + SHLQ $0x01, AX + MULQ 16(CX) + ADDQ AX, R12 + ADCQ DX, R11 // r3 += 19×l4×l4 - MOVQ 32(CX), AX - IMUL3Q $0x13, AX, AX - MULQ 32(CX) - ADDQ AX, R12 - ADCQ DX, R11 + MOVQ 32(CX), DX + LEAQ (DX)(DX*8), AX + LEAQ (DX)(AX*2), AX + MULQ 32(CX) + ADDQ AX, R12 + ADCQ DX, R11 // r4 = 2×l0×l4 MOVQ (CX), AX @@ -315,11 +334,11 @@ TEXT ·feSquare(SB), NOSPLIT, $0-16 MOVQ DX, R13 // r4 += 2×l1×l3 - MOVQ 8(CX), AX - IMUL3Q $0x02, AX, AX - MULQ 24(CX) - ADDQ AX, R14 - ADCQ DX, R13 + MOVQ 8(CX), AX + SHLQ $0x01, AX + MULQ 24(CX) + ADDQ AX, R14 + ADCQ DX, R13 // r4 += l2×l2 MOVQ 16(CX), AX diff --git a/vendor/filippo.io/edwards25519/field/fe_amd64_noasm.go b/vendor/filippo.io/edwards25519/field/fe_amd64_noasm.go index ddb6c9b8f..4b81f25d1 100644 --- a/vendor/filippo.io/edwards25519/field/fe_amd64_noasm.go +++ b/vendor/filippo.io/edwards25519/field/fe_amd64_noasm.go @@ -2,8 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !amd64 || !gc || purego -// +build !amd64 !gc purego +//go:build !amd64 || purego package field diff --git a/vendor/filippo.io/edwards25519/field/fe_arm64.go b/vendor/filippo.io/edwards25519/field/fe_arm64.go deleted file mode 100644 index af459ef51..000000000 --- a/vendor/filippo.io/edwards25519/field/fe_arm64.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build arm64 && gc && !purego -// +build arm64,gc,!purego - -package field - -//go:noescape -func carryPropagate(v *Element) - -func (v *Element) carryPropagate() *Element { - carryPropagate(v) - return v -} diff --git a/vendor/filippo.io/edwards25519/field/fe_arm64.s b/vendor/filippo.io/edwards25519/field/fe_arm64.s deleted file mode 100644 index 3126a4341..000000000 --- a/vendor/filippo.io/edwards25519/field/fe_arm64.s +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build arm64 && gc && !purego - -#include "textflag.h" - -// carryPropagate works exactly like carryPropagateGeneric and uses the -// same AND, ADD, and LSR+MADD instructions emitted by the compiler, but -// avoids loading R0-R4 twice and uses LDP and STP. -// -// See https://golang.org/issues/43145 for the main compiler issue. -// -// func carryPropagate(v *Element) -TEXT ·carryPropagate(SB),NOFRAME|NOSPLIT,$0-8 - MOVD v+0(FP), R20 - - LDP 0(R20), (R0, R1) - LDP 16(R20), (R2, R3) - MOVD 32(R20), R4 - - AND $0x7ffffffffffff, R0, R10 - AND $0x7ffffffffffff, R1, R11 - AND $0x7ffffffffffff, R2, R12 - AND $0x7ffffffffffff, R3, R13 - AND $0x7ffffffffffff, R4, R14 - - ADD R0>>51, R11, R11 - ADD R1>>51, R12, R12 - ADD R2>>51, R13, R13 - ADD R3>>51, R14, R14 - // R4>>51 * 19 + R10 -> R10 - LSR $51, R4, R21 - MOVD $19, R22 - MADD R22, R10, R21, R10 - - STP (R10, R11), 0(R20) - STP (R12, R13), 16(R20) - MOVD R14, 32(R20) - - RET diff --git a/vendor/filippo.io/edwards25519/field/fe_arm64_noasm.go b/vendor/filippo.io/edwards25519/field/fe_arm64_noasm.go deleted file mode 100644 index 234a5b2e5..000000000 --- a/vendor/filippo.io/edwards25519/field/fe_arm64_noasm.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !arm64 || !gc || purego -// +build !arm64 !gc purego - -package field - -func (v *Element) carryPropagate() *Element { - return v.carryPropagateGeneric() -} diff --git a/vendor/filippo.io/edwards25519/field/fe_generic.go b/vendor/filippo.io/edwards25519/field/fe_generic.go index 86f5fd955..ef1f15a5d 100644 --- a/vendor/filippo.io/edwards25519/field/fe_generic.go +++ b/vendor/filippo.io/edwards25519/field/fe_generic.go @@ -12,20 +12,42 @@ type uint128 struct { lo, hi uint64 } -// mul64 returns a * b. -func mul64(a, b uint64) uint128 { +// mul returns a * b. +func mul(a, b uint64) uint128 { hi, lo := bits.Mul64(a, b) return uint128{lo, hi} } -// addMul64 returns v + a * b. -func addMul64(v uint128, a, b uint64) uint128 { +// addMul returns v + a * b. +func addMul(v uint128, a, b uint64) uint128 { hi, lo := bits.Mul64(a, b) lo, c := bits.Add64(lo, v.lo, 0) hi, _ = bits.Add64(hi, v.hi, c) return uint128{lo, hi} } +// mul19 returns v * 19. +func mul19(v uint64) uint64 { + // Using this approach seems to yield better optimizations than *19. + return v + (v+v<<3)<<1 +} + +// addMul19 returns v + 19 * a * b, where a and b are at most 52 bits. +func addMul19(v uint128, a, b uint64) uint128 { + hi, lo := bits.Mul64(mul19(a), b) + lo, c := bits.Add64(lo, v.lo, 0) + hi, _ = bits.Add64(hi, v.hi, c) + return uint128{lo, hi} +} + +// addMul38 returns v + 38 * a * b, where a and b are at most 52 bits. +func addMul38(v uint128, a, b uint64) uint128 { + hi, lo := bits.Mul64(mul19(a), b*2) + lo, c := bits.Add64(lo, v.lo, 0) + hi, _ = bits.Add64(hi, v.hi, c) + return uint128{lo, hi} +} + // shiftRightBy51 returns a >> 51. a is assumed to be at most 115 bits. func shiftRightBy51(a uint128) uint64 { return (a.hi << (64 - 51)) | (a.lo >> 51) @@ -76,45 +98,40 @@ func feMulGeneric(v, a, b *Element) { // // Finally we add up the columns into wide, overlapping limbs. - a1_19 := a1 * 19 - a2_19 := a2 * 19 - a3_19 := a3 * 19 - a4_19 := a4 * 19 - // r0 = a0×b0 + 19×(a1×b4 + a2×b3 + a3×b2 + a4×b1) - r0 := mul64(a0, b0) - r0 = addMul64(r0, a1_19, b4) - r0 = addMul64(r0, a2_19, b3) - r0 = addMul64(r0, a3_19, b2) - r0 = addMul64(r0, a4_19, b1) + r0 := mul(a0, b0) + r0 = addMul19(r0, a1, b4) + r0 = addMul19(r0, a2, b3) + r0 = addMul19(r0, a3, b2) + r0 = addMul19(r0, a4, b1) // r1 = a0×b1 + a1×b0 + 19×(a2×b4 + a3×b3 + a4×b2) - r1 := mul64(a0, b1) - r1 = addMul64(r1, a1, b0) - r1 = addMul64(r1, a2_19, b4) - r1 = addMul64(r1, a3_19, b3) - r1 = addMul64(r1, a4_19, b2) + r1 := mul(a0, b1) + r1 = addMul(r1, a1, b0) + r1 = addMul19(r1, a2, b4) + r1 = addMul19(r1, a3, b3) + r1 = addMul19(r1, a4, b2) // r2 = a0×b2 + a1×b1 + a2×b0 + 19×(a3×b4 + a4×b3) - r2 := mul64(a0, b2) - r2 = addMul64(r2, a1, b1) - r2 = addMul64(r2, a2, b0) - r2 = addMul64(r2, a3_19, b4) - r2 = addMul64(r2, a4_19, b3) + r2 := mul(a0, b2) + r2 = addMul(r2, a1, b1) + r2 = addMul(r2, a2, b0) + r2 = addMul19(r2, a3, b4) + r2 = addMul19(r2, a4, b3) // r3 = a0×b3 + a1×b2 + a2×b1 + a3×b0 + 19×a4×b4 - r3 := mul64(a0, b3) - r3 = addMul64(r3, a1, b2) - r3 = addMul64(r3, a2, b1) - r3 = addMul64(r3, a3, b0) - r3 = addMul64(r3, a4_19, b4) + r3 := mul(a0, b3) + r3 = addMul(r3, a1, b2) + r3 = addMul(r3, a2, b1) + r3 = addMul(r3, a3, b0) + r3 = addMul19(r3, a4, b4) // r4 = a0×b4 + a1×b3 + a2×b2 + a3×b1 + a4×b0 - r4 := mul64(a0, b4) - r4 = addMul64(r4, a1, b3) - r4 = addMul64(r4, a2, b2) - r4 = addMul64(r4, a3, b1) - r4 = addMul64(r4, a4, b0) + r4 := mul(a0, b4) + r4 = addMul(r4, a1, b3) + r4 = addMul(r4, a2, b2) + r4 = addMul(r4, a3, b1) + r4 = addMul(r4, a4, b0) // After the multiplication, we need to reduce (carry) the five coefficients // to obtain a result with limbs that are at most slightly larger than 2⁵¹, @@ -149,7 +166,7 @@ func feMulGeneric(v, a, b *Element) { c3 := shiftRightBy51(r3) c4 := shiftRightBy51(r4) - rr0 := r0.lo&maskLow51Bits + c4*19 + rr0 := r0.lo&maskLow51Bits + mul19(c4) rr1 := r1.lo&maskLow51Bits + c0 rr2 := r2.lo&maskLow51Bits + c1 rr3 := r3.lo&maskLow51Bits + c2 @@ -158,8 +175,12 @@ func feMulGeneric(v, a, b *Element) { // Now all coefficients fit into 64-bit registers but are still too large to // be passed around as an Element. We therefore do one last carry chain, // where the carries will be small enough to fit in the wiggle room above 2⁵¹. - *v = Element{rr0, rr1, rr2, rr3, rr4} - v.carryPropagate() + + v.l0 = rr0&maskLow51Bits + mul19(rr4>>51) + v.l1 = rr1&maskLow51Bits + rr0>>51 + v.l2 = rr2&maskLow51Bits + rr1>>51 + v.l3 = rr3&maskLow51Bits + rr2>>51 + v.l4 = rr4&maskLow51Bits + rr3>>51 } func feSquareGeneric(v, a *Element) { @@ -190,44 +211,31 @@ func feSquareGeneric(v, a *Element) { // l0l4 19×l4l4 19×l3l4 19×l2l4 19×l1l4 = // -------------------------------------- // r4 r3 r2 r1 r0 - // - // With precomputed 2×, 19×, and 2×19× terms, we can compute each limb with - // only three Mul64 and four Add64, instead of five and eight. - - l0_2 := l0 * 2 - l1_2 := l1 * 2 - - l1_38 := l1 * 38 - l2_38 := l2 * 38 - l3_38 := l3 * 38 - - l3_19 := l3 * 19 - l4_19 := l4 * 19 // r0 = l0×l0 + 19×(l1×l4 + l2×l3 + l3×l2 + l4×l1) = l0×l0 + 19×2×(l1×l4 + l2×l3) - r0 := mul64(l0, l0) - r0 = addMul64(r0, l1_38, l4) - r0 = addMul64(r0, l2_38, l3) + r0 := mul(l0, l0) + r0 = addMul38(r0, l1, l4) + r0 = addMul38(r0, l2, l3) // r1 = l0×l1 + l1×l0 + 19×(l2×l4 + l3×l3 + l4×l2) = 2×l0×l1 + 19×2×l2×l4 + 19×l3×l3 - r1 := mul64(l0_2, l1) - r1 = addMul64(r1, l2_38, l4) - r1 = addMul64(r1, l3_19, l3) + r1 := mul(l0*2, l1) + r1 = addMul38(r1, l2, l4) + r1 = addMul19(r1, l3, l3) // r2 = l0×l2 + l1×l1 + l2×l0 + 19×(l3×l4 + l4×l3) = 2×l0×l2 + l1×l1 + 19×2×l3×l4 - r2 := mul64(l0_2, l2) - r2 = addMul64(r2, l1, l1) - r2 = addMul64(r2, l3_38, l4) + r2 := mul(l0*2, l2) + r2 = addMul(r2, l1, l1) + r2 = addMul38(r2, l3, l4) // r3 = l0×l3 + l1×l2 + l2×l1 + l3×l0 + 19×l4×l4 = 2×l0×l3 + 2×l1×l2 + 19×l4×l4 - r3 := mul64(l0_2, l3) - r3 = addMul64(r3, l1_2, l2) - r3 = addMul64(r3, l4_19, l4) + r3 := mul(l0*2, l3) + r3 = addMul(r3, l1*2, l2) + r3 = addMul19(r3, l4, l4) // r4 = l0×l4 + l1×l3 + l2×l2 + l3×l1 + l4×l0 = 2×l0×l4 + 2×l1×l3 + l2×l2 - r4 := mul64(l0_2, l4) - r4 = addMul64(r4, l1_2, l3) - r4 = addMul64(r4, l2, l2) + r4 := mul(l0*2, l4) + r4 = addMul(r4, l1*2, l3) + r4 = addMul(r4, l2, l2) c0 := shiftRightBy51(r0) c1 := shiftRightBy51(r1) @@ -235,32 +243,30 @@ func feSquareGeneric(v, a *Element) { c3 := shiftRightBy51(r3) c4 := shiftRightBy51(r4) - rr0 := r0.lo&maskLow51Bits + c4*19 + rr0 := r0.lo&maskLow51Bits + mul19(c4) rr1 := r1.lo&maskLow51Bits + c0 rr2 := r2.lo&maskLow51Bits + c1 rr3 := r3.lo&maskLow51Bits + c2 rr4 := r4.lo&maskLow51Bits + c3 - *v = Element{rr0, rr1, rr2, rr3, rr4} - v.carryPropagate() + v.l0 = rr0&maskLow51Bits + mul19(rr4>>51) + v.l1 = rr1&maskLow51Bits + rr0>>51 + v.l2 = rr2&maskLow51Bits + rr1>>51 + v.l3 = rr3&maskLow51Bits + rr2>>51 + v.l4 = rr4&maskLow51Bits + rr3>>51 } -// carryPropagateGeneric brings the limbs below 52 bits by applying the reduction +// carryPropagate brings the limbs below 52 bits by applying the reduction // identity (a * 2²⁵⁵ + b = a * 19 + b) to the l4 carry. -func (v *Element) carryPropagateGeneric() *Element { - c0 := v.l0 >> 51 - c1 := v.l1 >> 51 - c2 := v.l2 >> 51 - c3 := v.l3 >> 51 - c4 := v.l4 >> 51 - - // c4 is at most 64 - 51 = 13 bits, so c4*19 is at most 18 bits, and +func (v *Element) carryPropagate() *Element { + // (l4>>51) is at most 64 - 51 = 13 bits, so (l4>>51)*19 is at most 18 bits, and // the final l0 will be at most 52 bits. Similarly for the rest. - v.l0 = v.l0&maskLow51Bits + c4*19 - v.l1 = v.l1&maskLow51Bits + c0 - v.l2 = v.l2&maskLow51Bits + c1 - v.l3 = v.l3&maskLow51Bits + c2 - v.l4 = v.l4&maskLow51Bits + c3 + l0 := v.l0 + v.l0 = v.l0&maskLow51Bits + mul19(v.l4>>51) + v.l4 = v.l4&maskLow51Bits + v.l3>>51 + v.l3 = v.l3&maskLow51Bits + v.l2>>51 + v.l2 = v.l2&maskLow51Bits + v.l1>>51 + v.l1 = v.l1&maskLow51Bits + l0>>51 return v } diff --git a/vendor/filippo.io/edwards25519/pull.sh b/vendor/filippo.io/edwards25519/pull.sh new file mode 100644 index 000000000..f6217c96e --- /dev/null +++ b/vendor/filippo.io/edwards25519/pull.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +TAG="$1" +TMPDIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMPDIR" +} +trap cleanup EXIT + +command -v git >/dev/null +command -v git-filter-repo >/dev/null + +if [ -d "$HOME/go/.git" ]; then + REFERENCE=(--reference "$HOME/go" --dissociate) +else + REFERENCE=() +fi + +git -c advice.detachedHead=false clone --no-checkout "${REFERENCE[@]}" \ + -b "$TAG" https://go.googlesource.com/go.git "$TMPDIR" + +# Simplify the history graph by removing the dev.boringcrypto branches, whose +# merges end up empty after grafting anyway. This also fixes a weird quirk +# (maybe a git-filter-repo bug?) where only one file from an old path, +# src/crypto/ed25519/internal/edwards25519/const.go, would still exist in the +# filtered repo. +git -C "$TMPDIR" replace --graft f771edd7f9 99f1bf54eb +git -C "$TMPDIR" replace --graft 109c13b64f c2f96e686f +git -C "$TMPDIR" replace --graft aa4da4f189 912f075047 + +git -C "$TMPDIR" filter-repo --force \ + --paths-from-file /dev/stdin \ + --prune-empty always \ + --prune-degenerate always \ + --tag-callback 'tag.skip()' <<'EOF' +src/crypto/internal/fips140/edwards25519 +src/crypto/internal/edwards25519 +src/crypto/ed25519/internal/edwards25519 +EOF + +git fetch "$TMPDIR" +git update-ref "refs/heads/upstream/$TAG" FETCH_HEAD + +echo +echo "Fetched upstream history up to $TAG. Merge with:" +echo -e "\tgit merge --no-ff --no-commit --allow-unrelated-histories upstream/$TAG" diff --git a/vendor/filippo.io/edwards25519/scalar.go b/vendor/filippo.io/edwards25519/scalar.go index 3fd165387..f08b26245 100644 --- a/vendor/filippo.io/edwards25519/scalar.go +++ b/vendor/filippo.io/edwards25519/scalar.go @@ -7,6 +7,7 @@ package edwards25519 import ( "encoding/binary" "errors" + "math/bits" ) // A Scalar is an integer modulo @@ -179,15 +180,23 @@ func isReduced(s []byte) bool { return false } - for i := len(s) - 1; i >= 0; i-- { - switch { - case s[i] > scalarMinusOneBytes[i]: - return false - case s[i] < scalarMinusOneBytes[i]: - return true - } - } - return true + s0 := binary.LittleEndian.Uint64(s[:8]) + s1 := binary.LittleEndian.Uint64(s[8:16]) + s2 := binary.LittleEndian.Uint64(s[16:24]) + s3 := binary.LittleEndian.Uint64(s[24:]) + + l0 := binary.LittleEndian.Uint64(scalarMinusOneBytes[:8]) + l1 := binary.LittleEndian.Uint64(scalarMinusOneBytes[8:16]) + l2 := binary.LittleEndian.Uint64(scalarMinusOneBytes[16:24]) + l3 := binary.LittleEndian.Uint64(scalarMinusOneBytes[24:]) + + // Do a constant time subtraction chain scalarMinusOneBytes - s. If there is + // a borrow at the end, then s > scalarMinusOneBytes. + _, b := bits.Sub64(l0, s0, 0) + _, b = bits.Sub64(l1, s1, b) + _, b = bits.Sub64(l2, s2, b) + _, b = bits.Sub64(l3, s3, b) + return b == 0 } // SetBytesWithClamping applies the buffer pruning described in RFC 8032, diff --git a/vendor/filippo.io/edwards25519/tables.go b/vendor/filippo.io/edwards25519/tables.go index 83234bbc0..4a2b54eba 100644 --- a/vendor/filippo.io/edwards25519/tables.go +++ b/vendor/filippo.io/edwards25519/tables.go @@ -4,9 +4,7 @@ package edwards25519 -import ( - "crypto/subtle" -) +import "crypto/subtle" // A dynamic lookup table for variable-base, constant-time scalar muls. type projLookupTable struct { diff --git a/vendor/github.com/Masterminds/semver/.travis.yml b/vendor/github.com/Masterminds/semver/.travis.yml deleted file mode 100644 index 096369d44..000000000 --- a/vendor/github.com/Masterminds/semver/.travis.yml +++ /dev/null @@ -1,29 +0,0 @@ -language: go - -go: - - 1.6.x - - 1.7.x - - 1.8.x - - 1.9.x - - 1.10.x - - 1.11.x - - 1.12.x - - tip - -# Setting sudo access to false will let Travis CI use containers rather than -# VMs to run the tests. For more details see: -# - http://docs.travis-ci.com/user/workers/container-based-infrastructure/ -# - http://docs.travis-ci.com/user/workers/standard-infrastructure/ -sudo: false - -script: - - make setup - - make test - -notifications: - webhooks: - urls: - - https://webhooks.gitter.im/e/06e3328629952dabe3e0 - on_success: change # options: [always|never|change] default: always - on_failure: always # options: [always|never|change] default: always - on_start: never # options: [always|never|change] default: always diff --git a/vendor/github.com/Masterminds/semver/CHANGELOG.md b/vendor/github.com/Masterminds/semver/CHANGELOG.md deleted file mode 100644 index e405c9a84..000000000 --- a/vendor/github.com/Masterminds/semver/CHANGELOG.md +++ /dev/null @@ -1,109 +0,0 @@ -# 1.5.0 (2019-09-11) - -## Added - -- #103: Add basic fuzzing for `NewVersion()` (thanks @jesse-c) - -## Changed - -- #82: Clarify wildcard meaning in range constraints and update tests for it (thanks @greysteil) -- #83: Clarify caret operator range for pre-1.0.0 dependencies (thanks @greysteil) -- #72: Adding docs comment pointing to vert for a cli -- #71: Update the docs on pre-release comparator handling -- #89: Test with new go versions (thanks @thedevsaddam) -- #87: Added $ to ValidPrerelease for better validation (thanks @jeremycarroll) - -## Fixed - -- #78: Fix unchecked error in example code (thanks @ravron) -- #70: Fix the handling of pre-releases and the 0.0.0 release edge case -- #97: Fixed copyright file for proper display on GitHub -- #107: Fix handling prerelease when sorting alphanum and num -- #109: Fixed where Validate sometimes returns wrong message on error - -# 1.4.2 (2018-04-10) - -## Changed -- #72: Updated the docs to point to vert for a console appliaction -- #71: Update the docs on pre-release comparator handling - -## Fixed -- #70: Fix the handling of pre-releases and the 0.0.0 release edge case - -# 1.4.1 (2018-04-02) - -## Fixed -- Fixed #64: Fix pre-release precedence issue (thanks @uudashr) - -# 1.4.0 (2017-10-04) - -## Changed -- #61: Update NewVersion to parse ints with a 64bit int size (thanks @zknill) - -# 1.3.1 (2017-07-10) - -## Fixed -- Fixed #57: number comparisons in prerelease sometimes inaccurate - -# 1.3.0 (2017-05-02) - -## Added -- #45: Added json (un)marshaling support (thanks @mh-cbon) -- Stability marker. See https://masterminds.github.io/stability/ - -## Fixed -- #51: Fix handling of single digit tilde constraint (thanks @dgodd) - -## Changed -- #55: The godoc icon moved from png to svg - -# 1.2.3 (2017-04-03) - -## Fixed -- #46: Fixed 0.x.x and 0.0.x in constraints being treated as * - -# Release 1.2.2 (2016-12-13) - -## Fixed -- #34: Fixed issue where hyphen range was not working with pre-release parsing. - -# Release 1.2.1 (2016-11-28) - -## Fixed -- #24: Fixed edge case issue where constraint "> 0" does not handle "0.0.1-alpha" - properly. - -# Release 1.2.0 (2016-11-04) - -## Added -- #20: Added MustParse function for versions (thanks @adamreese) -- #15: Added increment methods on versions (thanks @mh-cbon) - -## Fixed -- Issue #21: Per the SemVer spec (section 9) a pre-release is unstable and - might not satisfy the intended compatibility. The change here ignores pre-releases - on constraint checks (e.g., ~ or ^) when a pre-release is not part of the - constraint. For example, `^1.2.3` will ignore pre-releases while - `^1.2.3-alpha` will include them. - -# Release 1.1.1 (2016-06-30) - -## Changed -- Issue #9: Speed up version comparison performance (thanks @sdboyer) -- Issue #8: Added benchmarks (thanks @sdboyer) -- Updated Go Report Card URL to new location -- Updated Readme to add code snippet formatting (thanks @mh-cbon) -- Updating tagging to v[SemVer] structure for compatibility with other tools. - -# Release 1.1.0 (2016-03-11) - -- Issue #2: Implemented validation to provide reasons a versions failed a - constraint. - -# Release 1.0.1 (2015-12-31) - -- Fixed #1: * constraint failing on valid versions. - -# Release 1.0.0 (2015-10-20) - -- Initial release diff --git a/vendor/github.com/Masterminds/semver/LICENSE.txt b/vendor/github.com/Masterminds/semver/LICENSE.txt deleted file mode 100644 index 9ff7da9c4..000000000 --- a/vendor/github.com/Masterminds/semver/LICENSE.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (C) 2014-2019, Matt Butcher and Matt Farina - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/vendor/github.com/Masterminds/semver/Makefile b/vendor/github.com/Masterminds/semver/Makefile deleted file mode 100644 index a7a1b4e36..000000000 --- a/vendor/github.com/Masterminds/semver/Makefile +++ /dev/null @@ -1,36 +0,0 @@ -.PHONY: setup -setup: - go get -u gopkg.in/alecthomas/gometalinter.v1 - gometalinter.v1 --install - -.PHONY: test -test: validate lint - @echo "==> Running tests" - go test -v - -.PHONY: validate -validate: - @echo "==> Running static validations" - @gometalinter.v1 \ - --disable-all \ - --enable deadcode \ - --severity deadcode:error \ - --enable gofmt \ - --enable gosimple \ - --enable ineffassign \ - --enable misspell \ - --enable vet \ - --tests \ - --vendor \ - --deadline 60s \ - ./... || exit_code=1 - -.PHONY: lint -lint: - @echo "==> Running linters" - @gometalinter.v1 \ - --disable-all \ - --enable golint \ - --vendor \ - --deadline 60s \ - ./... || : diff --git a/vendor/github.com/Masterminds/semver/README.md b/vendor/github.com/Masterminds/semver/README.md deleted file mode 100644 index 1b52d2f43..000000000 --- a/vendor/github.com/Masterminds/semver/README.md +++ /dev/null @@ -1,194 +0,0 @@ -# SemVer - -The `semver` package provides the ability to work with [Semantic Versions](http://semver.org) in Go. Specifically it provides the ability to: - -* Parse semantic versions -* Sort semantic versions -* Check if a semantic version fits within a set of constraints -* Optionally work with a `v` prefix - -[![Stability: -Active](https://masterminds.github.io/stability/active.svg)](https://masterminds.github.io/stability/active.html) -[![Build Status](https://travis-ci.org/Masterminds/semver.svg)](https://travis-ci.org/Masterminds/semver) [![Build status](https://ci.appveyor.com/api/projects/status/jfk66lib7hb985k8/branch/master?svg=true&passingText=windows%20build%20passing&failingText=windows%20build%20failing)](https://ci.appveyor.com/project/mattfarina/semver/branch/master) [![GoDoc](https://godoc.org/github.com/Masterminds/semver?status.svg)](https://godoc.org/github.com/Masterminds/semver) [![Go Report Card](https://goreportcard.com/badge/github.com/Masterminds/semver)](https://goreportcard.com/report/github.com/Masterminds/semver) - -If you are looking for a command line tool for version comparisons please see -[vert](https://github.com/Masterminds/vert) which uses this library. - -## Parsing Semantic Versions - -To parse a semantic version use the `NewVersion` function. For example, - -```go - v, err := semver.NewVersion("1.2.3-beta.1+build345") -``` - -If there is an error the version wasn't parseable. The version object has methods -to get the parts of the version, compare it to other versions, convert the -version back into a string, and get the original string. For more details -please see the [documentation](https://godoc.org/github.com/Masterminds/semver). - -## Sorting Semantic Versions - -A set of versions can be sorted using the [`sort`](https://golang.org/pkg/sort/) -package from the standard library. For example, - -```go - raw := []string{"1.2.3", "1.0", "1.3", "2", "0.4.2",} - vs := make([]*semver.Version, len(raw)) - for i, r := range raw { - v, err := semver.NewVersion(r) - if err != nil { - t.Errorf("Error parsing version: %s", err) - } - - vs[i] = v - } - - sort.Sort(semver.Collection(vs)) -``` - -## Checking Version Constraints - -Checking a version against version constraints is one of the most featureful -parts of the package. - -```go - c, err := semver.NewConstraint(">= 1.2.3") - if err != nil { - // Handle constraint not being parseable. - } - - v, _ := semver.NewVersion("1.3") - if err != nil { - // Handle version not being parseable. - } - // Check if the version meets the constraints. The a variable will be true. - a := c.Check(v) -``` - -## Basic Comparisons - -There are two elements to the comparisons. First, a comparison string is a list -of comma separated and comparisons. These are then separated by || separated or -comparisons. For example, `">= 1.2, < 3.0.0 || >= 4.2.3"` is looking for a -comparison that's greater than or equal to 1.2 and less than 3.0.0 or is -greater than or equal to 4.2.3. - -The basic comparisons are: - -* `=`: equal (aliased to no operator) -* `!=`: not equal -* `>`: greater than -* `<`: less than -* `>=`: greater than or equal to -* `<=`: less than or equal to - -## Working With Pre-release Versions - -Pre-releases, for those not familiar with them, are used for software releases -prior to stable or generally available releases. Examples of pre-releases include -development, alpha, beta, and release candidate releases. A pre-release may be -a version such as `1.2.3-beta.1` while the stable release would be `1.2.3`. In the -order of precidence, pre-releases come before their associated releases. In this -example `1.2.3-beta.1 < 1.2.3`. - -According to the Semantic Version specification pre-releases may not be -API compliant with their release counterpart. It says, - -> A pre-release version indicates that the version is unstable and might not satisfy the intended compatibility requirements as denoted by its associated normal version. - -SemVer comparisons without a pre-release comparator will skip pre-release versions. -For example, `>=1.2.3` will skip pre-releases when looking at a list of releases -while `>=1.2.3-0` will evaluate and find pre-releases. - -The reason for the `0` as a pre-release version in the example comparison is -because pre-releases can only contain ASCII alphanumerics and hyphens (along with -`.` separators), per the spec. Sorting happens in ASCII sort order, again per the spec. The lowest character is a `0` in ASCII sort order (see an [ASCII Table](http://www.asciitable.com/)) - -Understanding ASCII sort ordering is important because A-Z comes before a-z. That -means `>=1.2.3-BETA` will return `1.2.3-alpha`. What you might expect from case -sensitivity doesn't apply here. This is due to ASCII sort ordering which is what -the spec specifies. - -## Hyphen Range Comparisons - -There are multiple methods to handle ranges and the first is hyphens ranges. -These look like: - -* `1.2 - 1.4.5` which is equivalent to `>= 1.2, <= 1.4.5` -* `2.3.4 - 4.5` which is equivalent to `>= 2.3.4, <= 4.5` - -## Wildcards In Comparisons - -The `x`, `X`, and `*` characters can be used as a wildcard character. This works -for all comparison operators. When used on the `=` operator it falls -back to the pack level comparison (see tilde below). For example, - -* `1.2.x` is equivalent to `>= 1.2.0, < 1.3.0` -* `>= 1.2.x` is equivalent to `>= 1.2.0` -* `<= 2.x` is equivalent to `< 3` -* `*` is equivalent to `>= 0.0.0` - -## Tilde Range Comparisons (Patch) - -The tilde (`~`) comparison operator is for patch level ranges when a minor -version is specified and major level changes when the minor number is missing. -For example, - -* `~1.2.3` is equivalent to `>= 1.2.3, < 1.3.0` -* `~1` is equivalent to `>= 1, < 2` -* `~2.3` is equivalent to `>= 2.3, < 2.4` -* `~1.2.x` is equivalent to `>= 1.2.0, < 1.3.0` -* `~1.x` is equivalent to `>= 1, < 2` - -## Caret Range Comparisons (Major) - -The caret (`^`) comparison operator is for major level changes. This is useful -when comparisons of API versions as a major change is API breaking. For example, - -* `^1.2.3` is equivalent to `>= 1.2.3, < 2.0.0` -* `^0.0.1` is equivalent to `>= 0.0.1, < 1.0.0` -* `^1.2.x` is equivalent to `>= 1.2.0, < 2.0.0` -* `^2.3` is equivalent to `>= 2.3, < 3` -* `^2.x` is equivalent to `>= 2.0.0, < 3` - -# Validation - -In addition to testing a version against a constraint, a version can be validated -against a constraint. When validation fails a slice of errors containing why a -version didn't meet the constraint is returned. For example, - -```go - c, err := semver.NewConstraint("<= 1.2.3, >= 1.4") - if err != nil { - // Handle constraint not being parseable. - } - - v, _ := semver.NewVersion("1.3") - if err != nil { - // Handle version not being parseable. - } - - // Validate a version against a constraint. - a, msgs := c.Validate(v) - // a is false - for _, m := range msgs { - fmt.Println(m) - - // Loops over the errors which would read - // "1.3 is greater than 1.2.3" - // "1.3 is less than 1.4" - } -``` - -# Fuzzing - - [dvyukov/go-fuzz](https://github.com/dvyukov/go-fuzz) is used for fuzzing. - -1. `go-fuzz-build` -2. `go-fuzz -workdir=fuzz` - -# Contribute - -If you find an issue or want to contribute please file an [issue](https://github.com/Masterminds/semver/issues) -or [create a pull request](https://github.com/Masterminds/semver/pulls). diff --git a/vendor/github.com/Masterminds/semver/appveyor.yml b/vendor/github.com/Masterminds/semver/appveyor.yml deleted file mode 100644 index b2778df15..000000000 --- a/vendor/github.com/Masterminds/semver/appveyor.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: build-{build}.{branch} - -clone_folder: C:\gopath\src\github.com\Masterminds\semver -shallow_clone: true - -environment: - GOPATH: C:\gopath - -platform: - - x64 - -install: - - go version - - go env - - go get -u gopkg.in/alecthomas/gometalinter.v1 - - set PATH=%PATH%;%GOPATH%\bin - - gometalinter.v1.exe --install - -build_script: - - go install -v ./... - -test_script: - - "gometalinter.v1 \ - --disable-all \ - --enable deadcode \ - --severity deadcode:error \ - --enable gofmt \ - --enable gosimple \ - --enable ineffassign \ - --enable misspell \ - --enable vet \ - --tests \ - --vendor \ - --deadline 60s \ - ./... || exit_code=1" - - "gometalinter.v1 \ - --disable-all \ - --enable golint \ - --vendor \ - --deadline 60s \ - ./... || :" - - go test -v - -deploy: off diff --git a/vendor/github.com/Masterminds/semver/collection.go b/vendor/github.com/Masterminds/semver/collection.go deleted file mode 100644 index a78235895..000000000 --- a/vendor/github.com/Masterminds/semver/collection.go +++ /dev/null @@ -1,24 +0,0 @@ -package semver - -// Collection is a collection of Version instances and implements the sort -// interface. See the sort package for more details. -// https://golang.org/pkg/sort/ -type Collection []*Version - -// Len returns the length of a collection. The number of Version instances -// on the slice. -func (c Collection) Len() int { - return len(c) -} - -// Less is needed for the sort interface to compare two Version objects on the -// slice. If checks if one is less than the other. -func (c Collection) Less(i, j int) bool { - return c[i].LessThan(c[j]) -} - -// Swap is needed for the sort interface to replace the Version objects -// at two different positions in the slice. -func (c Collection) Swap(i, j int) { - c[i], c[j] = c[j], c[i] -} diff --git a/vendor/github.com/Masterminds/semver/constraints.go b/vendor/github.com/Masterminds/semver/constraints.go deleted file mode 100644 index b94b93413..000000000 --- a/vendor/github.com/Masterminds/semver/constraints.go +++ /dev/null @@ -1,423 +0,0 @@ -package semver - -import ( - "errors" - "fmt" - "regexp" - "strings" -) - -// Constraints is one or more constraint that a semantic version can be -// checked against. -type Constraints struct { - constraints [][]*constraint -} - -// NewConstraint returns a Constraints instance that a Version instance can -// be checked against. If there is a parse error it will be returned. -func NewConstraint(c string) (*Constraints, error) { - - // Rewrite - ranges into a comparison operation. - c = rewriteRange(c) - - ors := strings.Split(c, "||") - or := make([][]*constraint, len(ors)) - for k, v := range ors { - cs := strings.Split(v, ",") - result := make([]*constraint, len(cs)) - for i, s := range cs { - pc, err := parseConstraint(s) - if err != nil { - return nil, err - } - - result[i] = pc - } - or[k] = result - } - - o := &Constraints{constraints: or} - return o, nil -} - -// Check tests if a version satisfies the constraints. -func (cs Constraints) Check(v *Version) bool { - // loop over the ORs and check the inner ANDs - for _, o := range cs.constraints { - joy := true - for _, c := range o { - if !c.check(v) { - joy = false - break - } - } - - if joy { - return true - } - } - - return false -} - -// Validate checks if a version satisfies a constraint. If not a slice of -// reasons for the failure are returned in addition to a bool. -func (cs Constraints) Validate(v *Version) (bool, []error) { - // loop over the ORs and check the inner ANDs - var e []error - - // Capture the prerelease message only once. When it happens the first time - // this var is marked - var prerelesase bool - for _, o := range cs.constraints { - joy := true - for _, c := range o { - // Before running the check handle the case there the version is - // a prerelease and the check is not searching for prereleases. - if c.con.pre == "" && v.pre != "" { - if !prerelesase { - em := fmt.Errorf("%s is a prerelease version and the constraint is only looking for release versions", v) - e = append(e, em) - prerelesase = true - } - joy = false - - } else { - - if !c.check(v) { - em := fmt.Errorf(c.msg, v, c.orig) - e = append(e, em) - joy = false - } - } - } - - if joy { - return true, []error{} - } - } - - return false, e -} - -var constraintOps map[string]cfunc -var constraintMsg map[string]string -var constraintRegex *regexp.Regexp - -func init() { - constraintOps = map[string]cfunc{ - "": constraintTildeOrEqual, - "=": constraintTildeOrEqual, - "!=": constraintNotEqual, - ">": constraintGreaterThan, - "<": constraintLessThan, - ">=": constraintGreaterThanEqual, - "=>": constraintGreaterThanEqual, - "<=": constraintLessThanEqual, - "=<": constraintLessThanEqual, - "~": constraintTilde, - "~>": constraintTilde, - "^": constraintCaret, - } - - constraintMsg = map[string]string{ - "": "%s is not equal to %s", - "=": "%s is not equal to %s", - "!=": "%s is equal to %s", - ">": "%s is less than or equal to %s", - "<": "%s is greater than or equal to %s", - ">=": "%s is less than %s", - "=>": "%s is less than %s", - "<=": "%s is greater than %s", - "=<": "%s is greater than %s", - "~": "%s does not have same major and minor version as %s", - "~>": "%s does not have same major and minor version as %s", - "^": "%s does not have same major version as %s", - } - - ops := make([]string, 0, len(constraintOps)) - for k := range constraintOps { - ops = append(ops, regexp.QuoteMeta(k)) - } - - constraintRegex = regexp.MustCompile(fmt.Sprintf( - `^\s*(%s)\s*(%s)\s*$`, - strings.Join(ops, "|"), - cvRegex)) - - constraintRangeRegex = regexp.MustCompile(fmt.Sprintf( - `\s*(%s)\s+-\s+(%s)\s*`, - cvRegex, cvRegex)) -} - -// An individual constraint -type constraint struct { - // The callback function for the restraint. It performs the logic for - // the constraint. - function cfunc - - msg string - - // The version used in the constraint check. For example, if a constraint - // is '<= 2.0.0' the con a version instance representing 2.0.0. - con *Version - - // The original parsed version (e.g., 4.x from != 4.x) - orig string - - // When an x is used as part of the version (e.g., 1.x) - minorDirty bool - dirty bool - patchDirty bool -} - -// Check if a version meets the constraint -func (c *constraint) check(v *Version) bool { - return c.function(v, c) -} - -type cfunc func(v *Version, c *constraint) bool - -func parseConstraint(c string) (*constraint, error) { - m := constraintRegex.FindStringSubmatch(c) - if m == nil { - return nil, fmt.Errorf("improper constraint: %s", c) - } - - ver := m[2] - orig := ver - minorDirty := false - patchDirty := false - dirty := false - if isX(m[3]) { - ver = "0.0.0" - dirty = true - } else if isX(strings.TrimPrefix(m[4], ".")) || m[4] == "" { - minorDirty = true - dirty = true - ver = fmt.Sprintf("%s.0.0%s", m[3], m[6]) - } else if isX(strings.TrimPrefix(m[5], ".")) { - dirty = true - patchDirty = true - ver = fmt.Sprintf("%s%s.0%s", m[3], m[4], m[6]) - } - - con, err := NewVersion(ver) - if err != nil { - - // The constraintRegex should catch any regex parsing errors. So, - // we should never get here. - return nil, errors.New("constraint Parser Error") - } - - cs := &constraint{ - function: constraintOps[m[1]], - msg: constraintMsg[m[1]], - con: con, - orig: orig, - minorDirty: minorDirty, - patchDirty: patchDirty, - dirty: dirty, - } - return cs, nil -} - -// Constraint functions -func constraintNotEqual(v *Version, c *constraint) bool { - if c.dirty { - - // If there is a pre-release on the version but the constraint isn't looking - // for them assume that pre-releases are not compatible. See issue 21 for - // more details. - if v.Prerelease() != "" && c.con.Prerelease() == "" { - return false - } - - if c.con.Major() != v.Major() { - return true - } - if c.con.Minor() != v.Minor() && !c.minorDirty { - return true - } else if c.minorDirty { - return false - } - - return false - } - - return !v.Equal(c.con) -} - -func constraintGreaterThan(v *Version, c *constraint) bool { - - // If there is a pre-release on the version but the constraint isn't looking - // for them assume that pre-releases are not compatible. See issue 21 for - // more details. - if v.Prerelease() != "" && c.con.Prerelease() == "" { - return false - } - - return v.Compare(c.con) == 1 -} - -func constraintLessThan(v *Version, c *constraint) bool { - // If there is a pre-release on the version but the constraint isn't looking - // for them assume that pre-releases are not compatible. See issue 21 for - // more details. - if v.Prerelease() != "" && c.con.Prerelease() == "" { - return false - } - - if !c.dirty { - return v.Compare(c.con) < 0 - } - - if v.Major() > c.con.Major() { - return false - } else if v.Minor() > c.con.Minor() && !c.minorDirty { - return false - } - - return true -} - -func constraintGreaterThanEqual(v *Version, c *constraint) bool { - - // If there is a pre-release on the version but the constraint isn't looking - // for them assume that pre-releases are not compatible. See issue 21 for - // more details. - if v.Prerelease() != "" && c.con.Prerelease() == "" { - return false - } - - return v.Compare(c.con) >= 0 -} - -func constraintLessThanEqual(v *Version, c *constraint) bool { - // If there is a pre-release on the version but the constraint isn't looking - // for them assume that pre-releases are not compatible. See issue 21 for - // more details. - if v.Prerelease() != "" && c.con.Prerelease() == "" { - return false - } - - if !c.dirty { - return v.Compare(c.con) <= 0 - } - - if v.Major() > c.con.Major() { - return false - } else if v.Minor() > c.con.Minor() && !c.minorDirty { - return false - } - - return true -} - -// ~*, ~>* --> >= 0.0.0 (any) -// ~2, ~2.x, ~2.x.x, ~>2, ~>2.x ~>2.x.x --> >=2.0.0, <3.0.0 -// ~2.0, ~2.0.x, ~>2.0, ~>2.0.x --> >=2.0.0, <2.1.0 -// ~1.2, ~1.2.x, ~>1.2, ~>1.2.x --> >=1.2.0, <1.3.0 -// ~1.2.3, ~>1.2.3 --> >=1.2.3, <1.3.0 -// ~1.2.0, ~>1.2.0 --> >=1.2.0, <1.3.0 -func constraintTilde(v *Version, c *constraint) bool { - // If there is a pre-release on the version but the constraint isn't looking - // for them assume that pre-releases are not compatible. See issue 21 for - // more details. - if v.Prerelease() != "" && c.con.Prerelease() == "" { - return false - } - - if v.LessThan(c.con) { - return false - } - - // ~0.0.0 is a special case where all constraints are accepted. It's - // equivalent to >= 0.0.0. - if c.con.Major() == 0 && c.con.Minor() == 0 && c.con.Patch() == 0 && - !c.minorDirty && !c.patchDirty { - return true - } - - if v.Major() != c.con.Major() { - return false - } - - if v.Minor() != c.con.Minor() && !c.minorDirty { - return false - } - - return true -} - -// When there is a .x (dirty) status it automatically opts in to ~. Otherwise -// it's a straight = -func constraintTildeOrEqual(v *Version, c *constraint) bool { - // If there is a pre-release on the version but the constraint isn't looking - // for them assume that pre-releases are not compatible. See issue 21 for - // more details. - if v.Prerelease() != "" && c.con.Prerelease() == "" { - return false - } - - if c.dirty { - c.msg = constraintMsg["~"] - return constraintTilde(v, c) - } - - return v.Equal(c.con) -} - -// ^* --> (any) -// ^2, ^2.x, ^2.x.x --> >=2.0.0, <3.0.0 -// ^2.0, ^2.0.x --> >=2.0.0, <3.0.0 -// ^1.2, ^1.2.x --> >=1.2.0, <2.0.0 -// ^1.2.3 --> >=1.2.3, <2.0.0 -// ^1.2.0 --> >=1.2.0, <2.0.0 -func constraintCaret(v *Version, c *constraint) bool { - // If there is a pre-release on the version but the constraint isn't looking - // for them assume that pre-releases are not compatible. See issue 21 for - // more details. - if v.Prerelease() != "" && c.con.Prerelease() == "" { - return false - } - - if v.LessThan(c.con) { - return false - } - - if v.Major() != c.con.Major() { - return false - } - - return true -} - -var constraintRangeRegex *regexp.Regexp - -const cvRegex string = `v?([0-9|x|X|\*]+)(\.[0-9|x|X|\*]+)?(\.[0-9|x|X|\*]+)?` + - `(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` + - `(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` - -func isX(x string) bool { - switch x { - case "x", "*", "X": - return true - default: - return false - } -} - -func rewriteRange(i string) string { - m := constraintRangeRegex.FindAllStringSubmatch(i, -1) - if m == nil { - return i - } - o := i - for _, v := range m { - t := fmt.Sprintf(">= %s, <= %s", v[1], v[11]) - o = strings.Replace(o, v[0], t, 1) - } - - return o -} diff --git a/vendor/github.com/Masterminds/semver/doc.go b/vendor/github.com/Masterminds/semver/doc.go deleted file mode 100644 index 6a6c24c6d..000000000 --- a/vendor/github.com/Masterminds/semver/doc.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Package semver provides the ability to work with Semantic Versions (http://semver.org) in Go. - -Specifically it provides the ability to: - - * Parse semantic versions - * Sort semantic versions - * Check if a semantic version fits within a set of constraints - * Optionally work with a `v` prefix - -Parsing Semantic Versions - -To parse a semantic version use the `NewVersion` function. For example, - - v, err := semver.NewVersion("1.2.3-beta.1+build345") - -If there is an error the version wasn't parseable. The version object has methods -to get the parts of the version, compare it to other versions, convert the -version back into a string, and get the original string. For more details -please see the documentation at https://godoc.org/github.com/Masterminds/semver. - -Sorting Semantic Versions - -A set of versions can be sorted using the `sort` package from the standard library. -For example, - - raw := []string{"1.2.3", "1.0", "1.3", "2", "0.4.2",} - vs := make([]*semver.Version, len(raw)) - for i, r := range raw { - v, err := semver.NewVersion(r) - if err != nil { - t.Errorf("Error parsing version: %s", err) - } - - vs[i] = v - } - - sort.Sort(semver.Collection(vs)) - -Checking Version Constraints - -Checking a version against version constraints is one of the most featureful -parts of the package. - - c, err := semver.NewConstraint(">= 1.2.3") - if err != nil { - // Handle constraint not being parseable. - } - - v, err := semver.NewVersion("1.3") - if err != nil { - // Handle version not being parseable. - } - // Check if the version meets the constraints. The a variable will be true. - a := c.Check(v) - -Basic Comparisons - -There are two elements to the comparisons. First, a comparison string is a list -of comma separated and comparisons. These are then separated by || separated or -comparisons. For example, `">= 1.2, < 3.0.0 || >= 4.2.3"` is looking for a -comparison that's greater than or equal to 1.2 and less than 3.0.0 or is -greater than or equal to 4.2.3. - -The basic comparisons are: - - * `=`: equal (aliased to no operator) - * `!=`: not equal - * `>`: greater than - * `<`: less than - * `>=`: greater than or equal to - * `<=`: less than or equal to - -Hyphen Range Comparisons - -There are multiple methods to handle ranges and the first is hyphens ranges. -These look like: - - * `1.2 - 1.4.5` which is equivalent to `>= 1.2, <= 1.4.5` - * `2.3.4 - 4.5` which is equivalent to `>= 2.3.4, <= 4.5` - -Wildcards In Comparisons - -The `x`, `X`, and `*` characters can be used as a wildcard character. This works -for all comparison operators. When used on the `=` operator it falls -back to the pack level comparison (see tilde below). For example, - - * `1.2.x` is equivalent to `>= 1.2.0, < 1.3.0` - * `>= 1.2.x` is equivalent to `>= 1.2.0` - * `<= 2.x` is equivalent to `<= 3` - * `*` is equivalent to `>= 0.0.0` - -Tilde Range Comparisons (Patch) - -The tilde (`~`) comparison operator is for patch level ranges when a minor -version is specified and major level changes when the minor number is missing. -For example, - - * `~1.2.3` is equivalent to `>= 1.2.3, < 1.3.0` - * `~1` is equivalent to `>= 1, < 2` - * `~2.3` is equivalent to `>= 2.3, < 2.4` - * `~1.2.x` is equivalent to `>= 1.2.0, < 1.3.0` - * `~1.x` is equivalent to `>= 1, < 2` - -Caret Range Comparisons (Major) - -The caret (`^`) comparison operator is for major level changes. This is useful -when comparisons of API versions as a major change is API breaking. For example, - - * `^1.2.3` is equivalent to `>= 1.2.3, < 2.0.0` - * `^1.2.x` is equivalent to `>= 1.2.0, < 2.0.0` - * `^2.3` is equivalent to `>= 2.3, < 3` - * `^2.x` is equivalent to `>= 2.0.0, < 3` -*/ -package semver diff --git a/vendor/github.com/Masterminds/semver/version.go b/vendor/github.com/Masterminds/semver/version.go deleted file mode 100644 index 400d4f934..000000000 --- a/vendor/github.com/Masterminds/semver/version.go +++ /dev/null @@ -1,425 +0,0 @@ -package semver - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "regexp" - "strconv" - "strings" -) - -// The compiled version of the regex created at init() is cached here so it -// only needs to be created once. -var versionRegex *regexp.Regexp -var validPrereleaseRegex *regexp.Regexp - -var ( - // ErrInvalidSemVer is returned a version is found to be invalid when - // being parsed. - ErrInvalidSemVer = errors.New("Invalid Semantic Version") - - // ErrInvalidMetadata is returned when the metadata is an invalid format - ErrInvalidMetadata = errors.New("Invalid Metadata string") - - // ErrInvalidPrerelease is returned when the pre-release is an invalid format - ErrInvalidPrerelease = errors.New("Invalid Prerelease string") -) - -// SemVerRegex is the regular expression used to parse a semantic version. -const SemVerRegex string = `v?([0-9]+)(\.[0-9]+)?(\.[0-9]+)?` + - `(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` + - `(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?` - -// ValidPrerelease is the regular expression which validates -// both prerelease and metadata values. -const ValidPrerelease string = `^([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*)$` - -// Version represents a single semantic version. -type Version struct { - major, minor, patch int64 - pre string - metadata string - original string -} - -func init() { - versionRegex = regexp.MustCompile("^" + SemVerRegex + "$") - validPrereleaseRegex = regexp.MustCompile(ValidPrerelease) -} - -// NewVersion parses a given version and returns an instance of Version or -// an error if unable to parse the version. -func NewVersion(v string) (*Version, error) { - m := versionRegex.FindStringSubmatch(v) - if m == nil { - return nil, ErrInvalidSemVer - } - - sv := &Version{ - metadata: m[8], - pre: m[5], - original: v, - } - - var temp int64 - temp, err := strconv.ParseInt(m[1], 10, 64) - if err != nil { - return nil, fmt.Errorf("Error parsing version segment: %s", err) - } - sv.major = temp - - if m[2] != "" { - temp, err = strconv.ParseInt(strings.TrimPrefix(m[2], "."), 10, 64) - if err != nil { - return nil, fmt.Errorf("Error parsing version segment: %s", err) - } - sv.minor = temp - } else { - sv.minor = 0 - } - - if m[3] != "" { - temp, err = strconv.ParseInt(strings.TrimPrefix(m[3], "."), 10, 64) - if err != nil { - return nil, fmt.Errorf("Error parsing version segment: %s", err) - } - sv.patch = temp - } else { - sv.patch = 0 - } - - return sv, nil -} - -// MustParse parses a given version and panics on error. -func MustParse(v string) *Version { - sv, err := NewVersion(v) - if err != nil { - panic(err) - } - return sv -} - -// String converts a Version object to a string. -// Note, if the original version contained a leading v this version will not. -// See the Original() method to retrieve the original value. Semantic Versions -// don't contain a leading v per the spec. Instead it's optional on -// implementation. -func (v *Version) String() string { - var buf bytes.Buffer - - fmt.Fprintf(&buf, "%d.%d.%d", v.major, v.minor, v.patch) - if v.pre != "" { - fmt.Fprintf(&buf, "-%s", v.pre) - } - if v.metadata != "" { - fmt.Fprintf(&buf, "+%s", v.metadata) - } - - return buf.String() -} - -// Original returns the original value passed in to be parsed. -func (v *Version) Original() string { - return v.original -} - -// Major returns the major version. -func (v *Version) Major() int64 { - return v.major -} - -// Minor returns the minor version. -func (v *Version) Minor() int64 { - return v.minor -} - -// Patch returns the patch version. -func (v *Version) Patch() int64 { - return v.patch -} - -// Prerelease returns the pre-release version. -func (v *Version) Prerelease() string { - return v.pre -} - -// Metadata returns the metadata on the version. -func (v *Version) Metadata() string { - return v.metadata -} - -// originalVPrefix returns the original 'v' prefix if any. -func (v *Version) originalVPrefix() string { - - // Note, only lowercase v is supported as a prefix by the parser. - if v.original != "" && v.original[:1] == "v" { - return v.original[:1] - } - return "" -} - -// IncPatch produces the next patch version. -// If the current version does not have prerelease/metadata information, -// it unsets metadata and prerelease values, increments patch number. -// If the current version has any of prerelease or metadata information, -// it unsets both values and keeps curent patch value -func (v Version) IncPatch() Version { - vNext := v - // according to http://semver.org/#spec-item-9 - // Pre-release versions have a lower precedence than the associated normal version. - // according to http://semver.org/#spec-item-10 - // Build metadata SHOULD be ignored when determining version precedence. - if v.pre != "" { - vNext.metadata = "" - vNext.pre = "" - } else { - vNext.metadata = "" - vNext.pre = "" - vNext.patch = v.patch + 1 - } - vNext.original = v.originalVPrefix() + "" + vNext.String() - return vNext -} - -// IncMinor produces the next minor version. -// Sets patch to 0. -// Increments minor number. -// Unsets metadata. -// Unsets prerelease status. -func (v Version) IncMinor() Version { - vNext := v - vNext.metadata = "" - vNext.pre = "" - vNext.patch = 0 - vNext.minor = v.minor + 1 - vNext.original = v.originalVPrefix() + "" + vNext.String() - return vNext -} - -// IncMajor produces the next major version. -// Sets patch to 0. -// Sets minor to 0. -// Increments major number. -// Unsets metadata. -// Unsets prerelease status. -func (v Version) IncMajor() Version { - vNext := v - vNext.metadata = "" - vNext.pre = "" - vNext.patch = 0 - vNext.minor = 0 - vNext.major = v.major + 1 - vNext.original = v.originalVPrefix() + "" + vNext.String() - return vNext -} - -// SetPrerelease defines the prerelease value. -// Value must not include the required 'hypen' prefix. -func (v Version) SetPrerelease(prerelease string) (Version, error) { - vNext := v - if len(prerelease) > 0 && !validPrereleaseRegex.MatchString(prerelease) { - return vNext, ErrInvalidPrerelease - } - vNext.pre = prerelease - vNext.original = v.originalVPrefix() + "" + vNext.String() - return vNext, nil -} - -// SetMetadata defines metadata value. -// Value must not include the required 'plus' prefix. -func (v Version) SetMetadata(metadata string) (Version, error) { - vNext := v - if len(metadata) > 0 && !validPrereleaseRegex.MatchString(metadata) { - return vNext, ErrInvalidMetadata - } - vNext.metadata = metadata - vNext.original = v.originalVPrefix() + "" + vNext.String() - return vNext, nil -} - -// LessThan tests if one version is less than another one. -func (v *Version) LessThan(o *Version) bool { - return v.Compare(o) < 0 -} - -// GreaterThan tests if one version is greater than another one. -func (v *Version) GreaterThan(o *Version) bool { - return v.Compare(o) > 0 -} - -// Equal tests if two versions are equal to each other. -// Note, versions can be equal with different metadata since metadata -// is not considered part of the comparable version. -func (v *Version) Equal(o *Version) bool { - return v.Compare(o) == 0 -} - -// Compare compares this version to another one. It returns -1, 0, or 1 if -// the version smaller, equal, or larger than the other version. -// -// Versions are compared by X.Y.Z. Build metadata is ignored. Prerelease is -// lower than the version without a prerelease. -func (v *Version) Compare(o *Version) int { - // Compare the major, minor, and patch version for differences. If a - // difference is found return the comparison. - if d := compareSegment(v.Major(), o.Major()); d != 0 { - return d - } - if d := compareSegment(v.Minor(), o.Minor()); d != 0 { - return d - } - if d := compareSegment(v.Patch(), o.Patch()); d != 0 { - return d - } - - // At this point the major, minor, and patch versions are the same. - ps := v.pre - po := o.Prerelease() - - if ps == "" && po == "" { - return 0 - } - if ps == "" { - return 1 - } - if po == "" { - return -1 - } - - return comparePrerelease(ps, po) -} - -// UnmarshalJSON implements JSON.Unmarshaler interface. -func (v *Version) UnmarshalJSON(b []byte) error { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return err - } - temp, err := NewVersion(s) - if err != nil { - return err - } - v.major = temp.major - v.minor = temp.minor - v.patch = temp.patch - v.pre = temp.pre - v.metadata = temp.metadata - v.original = temp.original - temp = nil - return nil -} - -// MarshalJSON implements JSON.Marshaler interface. -func (v *Version) MarshalJSON() ([]byte, error) { - return json.Marshal(v.String()) -} - -func compareSegment(v, o int64) int { - if v < o { - return -1 - } - if v > o { - return 1 - } - - return 0 -} - -func comparePrerelease(v, o string) int { - - // split the prelease versions by their part. The separator, per the spec, - // is a . - sparts := strings.Split(v, ".") - oparts := strings.Split(o, ".") - - // Find the longer length of the parts to know how many loop iterations to - // go through. - slen := len(sparts) - olen := len(oparts) - - l := slen - if olen > slen { - l = olen - } - - // Iterate over each part of the prereleases to compare the differences. - for i := 0; i < l; i++ { - // Since the lentgh of the parts can be different we need to create - // a placeholder. This is to avoid out of bounds issues. - stemp := "" - if i < slen { - stemp = sparts[i] - } - - otemp := "" - if i < olen { - otemp = oparts[i] - } - - d := comparePrePart(stemp, otemp) - if d != 0 { - return d - } - } - - // Reaching here means two versions are of equal value but have different - // metadata (the part following a +). They are not identical in string form - // but the version comparison finds them to be equal. - return 0 -} - -func comparePrePart(s, o string) int { - // Fastpath if they are equal - if s == o { - return 0 - } - - // When s or o are empty we can use the other in an attempt to determine - // the response. - if s == "" { - if o != "" { - return -1 - } - return 1 - } - - if o == "" { - if s != "" { - return 1 - } - return -1 - } - - // When comparing strings "99" is greater than "103". To handle - // cases like this we need to detect numbers and compare them. According - // to the semver spec, numbers are always positive. If there is a - at the - // start like -99 this is to be evaluated as an alphanum. numbers always - // have precedence over alphanum. Parsing as Uints because negative numbers - // are ignored. - - oi, n1 := strconv.ParseUint(o, 10, 64) - si, n2 := strconv.ParseUint(s, 10, 64) - - // The case where both are strings compare the strings - if n1 != nil && n2 != nil { - if s > o { - return 1 - } - return -1 - } else if n1 != nil { - // o is a string and s is a number - return -1 - } else if n2 != nil { - // s is a string and o is a number - return 1 - } - // Both are numbers - if si > oi { - return 1 - } - return -1 - -} diff --git a/vendor/github.com/Masterminds/semver/version_fuzz.go b/vendor/github.com/Masterminds/semver/version_fuzz.go deleted file mode 100644 index b42bcd62b..000000000 --- a/vendor/github.com/Masterminds/semver/version_fuzz.go +++ /dev/null @@ -1,10 +0,0 @@ -// +build gofuzz - -package semver - -func Fuzz(data []byte) int { - if _, err := NewVersion(string(data)); err != nil { - return 0 - } - return 1 -} diff --git a/vendor/github.com/coreos/go-semver/LICENSE b/vendor/github.com/coreos/go-semver/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/github.com/coreos/go-semver/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/coreos/go-semver/NOTICE b/vendor/github.com/coreos/go-semver/NOTICE new file mode 100644 index 000000000..23a0ada2f --- /dev/null +++ b/vendor/github.com/coreos/go-semver/NOTICE @@ -0,0 +1,5 @@ +CoreOS Project +Copyright 2018 CoreOS, Inc + +This product includes software developed at CoreOS, Inc. +(http://www.coreos.com/). diff --git a/vendor/github.com/coreos/go-semver/semver/semver.go b/vendor/github.com/coreos/go-semver/semver/semver.go new file mode 100644 index 000000000..eb9fb7ff2 --- /dev/null +++ b/vendor/github.com/coreos/go-semver/semver/semver.go @@ -0,0 +1,296 @@ +// Copyright 2013-2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Semantic Versions http://semver.org +package semver + +import ( + "bytes" + "errors" + "fmt" + "regexp" + "strconv" + "strings" +) + +type Version struct { + Major int64 + Minor int64 + Patch int64 + PreRelease PreRelease + Metadata string +} + +type PreRelease string + +func splitOff(input *string, delim string) (val string) { + parts := strings.SplitN(*input, delim, 2) + + if len(parts) == 2 { + *input = parts[0] + val = parts[1] + } + + return val +} + +func New(version string) *Version { + return Must(NewVersion(version)) +} + +func NewVersion(version string) (*Version, error) { + v := Version{} + + if err := v.Set(version); err != nil { + return nil, err + } + + return &v, nil +} + +// Must is a helper for wrapping NewVersion and will panic if err is not nil. +func Must(v *Version, err error) *Version { + if err != nil { + panic(err) + } + return v +} + +// Set parses and updates v from the given version string. Implements flag.Value +func (v *Version) Set(version string) error { + metadata := splitOff(&version, "+") + preRelease := PreRelease(splitOff(&version, "-")) + dotParts := strings.SplitN(version, ".", 3) + + if len(dotParts) != 3 { + return fmt.Errorf("%s is not in dotted-tri format", version) + } + + if err := validateIdentifier(string(preRelease)); err != nil { + return fmt.Errorf("failed to validate pre-release: %v", err) + } + + if err := validateIdentifier(metadata); err != nil { + return fmt.Errorf("failed to validate metadata: %v", err) + } + + parsed := make([]int64, 3) + + for i, v := range dotParts[:3] { + val, err := strconv.ParseInt(v, 10, 64) + parsed[i] = val + if err != nil { + return err + } + } + + v.Metadata = metadata + v.PreRelease = preRelease + v.Major = parsed[0] + v.Minor = parsed[1] + v.Patch = parsed[2] + return nil +} + +func (v Version) String() string { + var buffer bytes.Buffer + + fmt.Fprintf(&buffer, "%d.%d.%d", v.Major, v.Minor, v.Patch) + + if v.PreRelease != "" { + fmt.Fprintf(&buffer, "-%s", v.PreRelease) + } + + if v.Metadata != "" { + fmt.Fprintf(&buffer, "+%s", v.Metadata) + } + + return buffer.String() +} + +func (v *Version) UnmarshalYAML(unmarshal func(interface{}) error) error { + var data string + if err := unmarshal(&data); err != nil { + return err + } + return v.Set(data) +} + +func (v Version) MarshalJSON() ([]byte, error) { + return []byte(`"` + v.String() + `"`), nil +} + +func (v *Version) UnmarshalJSON(data []byte) error { + l := len(data) + if l == 0 || string(data) == `""` { + return nil + } + if l < 2 || data[0] != '"' || data[l-1] != '"' { + return errors.New("invalid semver string") + } + return v.Set(string(data[1 : l-1])) +} + +// Compare tests if v is less than, equal to, or greater than versionB, +// returning -1, 0, or +1 respectively. +func (v Version) Compare(versionB Version) int { + if cmp := recursiveCompare(v.Slice(), versionB.Slice()); cmp != 0 { + return cmp + } + return preReleaseCompare(v, versionB) +} + +// Equal tests if v is equal to versionB. +func (v Version) Equal(versionB Version) bool { + return v.Compare(versionB) == 0 +} + +// LessThan tests if v is less than versionB. +func (v Version) LessThan(versionB Version) bool { + return v.Compare(versionB) < 0 +} + +// Slice converts the comparable parts of the semver into a slice of integers. +func (v Version) Slice() []int64 { + return []int64{v.Major, v.Minor, v.Patch} +} + +func (p PreRelease) Slice() []string { + preRelease := string(p) + return strings.Split(preRelease, ".") +} + +func preReleaseCompare(versionA Version, versionB Version) int { + a := versionA.PreRelease + b := versionB.PreRelease + + /* Handle the case where if two versions are otherwise equal it is the + * one without a PreRelease that is greater */ + if len(a) == 0 && (len(b) > 0) { + return 1 + } else if len(b) == 0 && (len(a) > 0) { + return -1 + } + + // If there is a prerelease, check and compare each part. + return recursivePreReleaseCompare(a.Slice(), b.Slice()) +} + +func recursiveCompare(versionA []int64, versionB []int64) int { + if len(versionA) == 0 { + return 0 + } + + a := versionA[0] + b := versionB[0] + + if a > b { + return 1 + } else if a < b { + return -1 + } + + return recursiveCompare(versionA[1:], versionB[1:]) +} + +func recursivePreReleaseCompare(versionA []string, versionB []string) int { + // A larger set of pre-release fields has a higher precedence than a smaller set, + // if all of the preceding identifiers are equal. + if len(versionA) == 0 { + if len(versionB) > 0 { + return -1 + } + return 0 + } else if len(versionB) == 0 { + // We're longer than versionB so return 1. + return 1 + } + + a := versionA[0] + b := versionB[0] + + aInt := false + bInt := false + + aI, err := strconv.Atoi(versionA[0]) + if err == nil { + aInt = true + } + + bI, err := strconv.Atoi(versionB[0]) + if err == nil { + bInt = true + } + + // Numeric identifiers always have lower precedence than non-numeric identifiers. + if aInt && !bInt { + return -1 + } else if !aInt && bInt { + return 1 + } + + // Handle Integer Comparison + if aInt && bInt { + if aI > bI { + return 1 + } else if aI < bI { + return -1 + } + } + + // Handle String Comparison + if a > b { + return 1 + } else if a < b { + return -1 + } + + return recursivePreReleaseCompare(versionA[1:], versionB[1:]) +} + +// BumpMajor increments the Major field by 1 and resets all other fields to their default values +func (v *Version) BumpMajor() { + v.Major += 1 + v.Minor = 0 + v.Patch = 0 + v.PreRelease = PreRelease("") + v.Metadata = "" +} + +// BumpMinor increments the Minor field by 1 and resets all other fields to their default values +func (v *Version) BumpMinor() { + v.Minor += 1 + v.Patch = 0 + v.PreRelease = PreRelease("") + v.Metadata = "" +} + +// BumpPatch increments the Patch field by 1 and resets all other fields to their default values +func (v *Version) BumpPatch() { + v.Patch += 1 + v.PreRelease = PreRelease("") + v.Metadata = "" +} + +// validateIdentifier makes sure the provided identifier satisfies semver spec +func validateIdentifier(id string) error { + if id != "" && !reIdentifier.MatchString(id) { + return fmt.Errorf("%s is not a valid semver identifier", id) + } + return nil +} + +// reIdentifier is a regular expression used to check that pre-release and metadata +// identifiers satisfy the spec requirements +var reIdentifier = regexp.MustCompile(`^[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*$`) diff --git a/vendor/github.com/coreos/go-semver/semver/sort.go b/vendor/github.com/coreos/go-semver/semver/sort.go new file mode 100644 index 000000000..e256b41a5 --- /dev/null +++ b/vendor/github.com/coreos/go-semver/semver/sort.go @@ -0,0 +1,38 @@ +// Copyright 2013-2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package semver + +import ( + "sort" +) + +type Versions []*Version + +func (s Versions) Len() int { + return len(s) +} + +func (s Versions) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s Versions) Less(i, j int) bool { + return s[i].LessThan(*s[j]) +} + +// Sort sorts the given slice of Version +func Sort(versions []*Version) { + sort.Sort(Versions(versions)) +} diff --git a/vendor/github.com/go-mysql-org/go-mysql/client/auth.go b/vendor/github.com/go-mysql-org/go-mysql/client/auth.go index 200009609..e163bf533 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/client/auth.go +++ b/vendor/github.com/go-mysql-org/go-mysql/client/auth.go @@ -5,26 +5,35 @@ import ( "crypto/tls" "encoding/binary" "fmt" + "slices" - . "github.com/go-mysql-org/go-mysql/mysql" + "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/packet" "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/parser/charset" ) -const defaultAuthPluginName = AUTH_NATIVE_PASSWORD +const defaultAuthPluginName = mysql.AUTH_NATIVE_PASSWORD + +var optionalCapabilities = []uint32{ + mysql.CLIENT_FOUND_ROWS, + mysql.CLIENT_IGNORE_SPACE, + mysql.CLIENT_MULTI_STATEMENTS, + mysql.CLIENT_MULTI_RESULTS, + mysql.CLIENT_PS_MULTI_RESULTS, + mysql.CLIENT_CONNECT_ATTRS, + mysql.CLIENT_COMPRESS, + mysql.CLIENT_ZSTD_COMPRESSION_ALGORITHM, + mysql.CLIENT_LOCAL_FILES, + mysql.CLIENT_SESSION_TRACK, +} // defines the supported auth plugins -var supportedAuthPlugins = []string{AUTH_NATIVE_PASSWORD, AUTH_SHA256_PASSWORD, AUTH_CACHING_SHA2_PASSWORD} +var supportedAuthPlugins = []string{mysql.AUTH_NATIVE_PASSWORD, mysql.AUTH_SHA256_PASSWORD, mysql.AUTH_CACHING_SHA2_PASSWORD, mysql.AUTH_MARIADB_ED25519} // helper function to determine what auth methods are allowed by this client func authPluginAllowed(pluginName string) bool { - for _, p := range supportedAuthPlugins { - if pluginName == p { - return true - } - } - return false + return slices.Contains(supportedAuthPlugins, pluginName) } // See: @@ -38,12 +47,12 @@ func (c *Conn) readInitialHandshake() error { return errors.Trace(err) } - if data[0] == ERR_HEADER { + if data[0] == mysql.ERR_HEADER { return errors.Annotate(c.handleErrorPacket(data), "read initial handshake error") } - if data[0] != ClassicProtocolVersion { - if data[0] == XProtocolVersion { + if data[0] != mysql.ClassicProtocolVersion { + if data[0] == mysql.XProtocolVersion { return errors.Errorf( "invalid protocol version %d, expected 10. "+ "This might be X Protocol, make sure to connect to the right port", @@ -75,10 +84,10 @@ func (c *Conn) readInitialHandshake() error { // The lower 2 bytes of the Capabilities Flags c.capability = uint32(binary.LittleEndian.Uint16(data[pos : pos+2])) // check protocol - if c.capability&CLIENT_PROTOCOL_41 == 0 { + if c.capability&mysql.CLIENT_PROTOCOL_41 == 0 { return errors.New("the MySQL server can not support protocol 41 and above required by the client") } - if c.capability&CLIENT_SSL == 0 && c.tlsConfig != nil { + if c.capability&mysql.CLIENT_SSL == 0 && c.tlsConfig != nil { return errors.New("the MySQL Server does not support TLS required by the client") } pos += 2 @@ -86,18 +95,18 @@ func (c *Conn) readInitialHandshake() error { if len(data) > pos { // default server a_protocol_character_set, only the lower 8-bits // c.charset = data[pos] - pos += 1 + pos++ c.status = binary.LittleEndian.Uint16(data[pos : pos+2]) pos += 2 // The upper 2 bytes of the Capabilities Flags - c.capability = uint32(binary.LittleEndian.Uint16(data[pos:pos+2]))<<16 | c.capability + c.capability |= uint32(binary.LittleEndian.Uint16(data[pos:pos+2])) << 16 pos += 2 // length of the combined auth_plugin_data (scramble), if auth_plugin_data_len is > 0 authPluginDataLen := data[pos] - if (c.capability&CLIENT_PLUGIN_AUTH == 0) && (authPluginDataLen > 0) { + if (c.capability&mysql.CLIENT_PLUGIN_AUTH == 0) && (authPluginDataLen > 0) { return errors.Errorf("invalid auth plugin data filler %d", authPluginDataLen) } pos++ @@ -105,7 +114,7 @@ func (c *Conn) readInitialHandshake() error { // skip reserved (all [00] ?) pos += 10 - if c.capability&CLIENT_SECURE_CONNECTION != 0 { + if c.capability&mysql.CLIENT_SECURE_CONNECTION != 0 { // Rest of the plugin provided data (scramble) // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_handshake_v10.html @@ -114,10 +123,7 @@ func (c *Conn) readInitialHandshake() error { // https://github.com/mysql/mysql-server/blob/1bfe02bdad6604d54913c62614bde57a055c8332/sql/auth/sql_authentication.cc#L1641-L1642 // the first packet *must* have at least 20 bytes of a scramble. // if a plugin provided less, we pad it to 20 with zeros - rest := int(authPluginDataLen) - 8 - if rest < 13 { - rest = 13 - } + rest := max(int(authPluginDataLen)-8, 13) authPluginDataPart2 := data[pos : pos+rest-1] pos += rest @@ -125,7 +131,7 @@ func (c *Conn) readInitialHandshake() error { c.salt = append(c.salt, authPluginDataPart2...) } - if c.capability&CLIENT_PLUGIN_AUTH != 0 { + if c.capability&mysql.CLIENT_PLUGIN_AUTH != 0 { c.authPluginName = string(data[pos : pos+bytes.IndexByte(data[pos:], 0x00)]) pos += len(c.authPluginName) @@ -153,13 +159,13 @@ func (c *Conn) readInitialHandshake() error { func (c *Conn) genAuthResponse(authData []byte) ([]byte, bool, error) { // password hashing switch c.authPluginName { - case AUTH_NATIVE_PASSWORD: - return CalcPassword(authData[:20], []byte(c.password)), false, nil - case AUTH_CACHING_SHA2_PASSWORD: - return CalcCachingSha2Password(authData, c.password), false, nil - case AUTH_CLEAR_PASSWORD: + case mysql.AUTH_NATIVE_PASSWORD: + return mysql.CalcNativePassword(authData[:20], []byte(c.password)), false, nil + case mysql.AUTH_CACHING_SHA2_PASSWORD: + return mysql.CalcCachingSha2Password(authData, []byte(c.password)), false, nil + case mysql.AUTH_CLEAR_PASSWORD: return []byte(c.password), true, nil - case AUTH_SHA256_PASSWORD: + case mysql.AUTH_SHA256_PASSWORD: if len(c.password) == 0 { return nil, true, nil } @@ -167,11 +173,19 @@ func (c *Conn) genAuthResponse(authData []byte) ([]byte, bool, error) { // write cleartext auth packet // see: https://dev.mysql.com/doc/refman/8.0/en/sha256-pluggable-authentication.html return []byte(c.password), true, nil - } else { - // request public key from server - // see: https://dev.mysql.com/doc/internals/en/public-key-retrieval.html - return []byte{1}, false, nil } + // request public key from server + // see: https://dev.mysql.com/doc/internals/en/public-key-retrieval.html + return []byte{1}, false, nil + case mysql.AUTH_MARIADB_ED25519: + if len(authData) != 32 { + return nil, false, mysql.ErrMalformPacket + } + res, err := mysql.CalcEd25519Password(authData, c.password) + if err != nil { + return nil, false, err + } + return res, false, nil default: // not reachable return nil, false, fmt.Errorf("auth plugin '%s' is not supported", c.authPluginName) @@ -186,35 +200,34 @@ func (c *Conn) genAttributes() []byte { attrData := make([]byte, 0) for k, v := range c.attributes { - attrData = append(attrData, PutLengthEncodedString([]byte(k))...) - attrData = append(attrData, PutLengthEncodedString([]byte(v))...) + attrData = append(attrData, mysql.PutLengthEncodedString([]byte(k))...) + attrData = append(attrData, mysql.PutLengthEncodedString([]byte(v))...) } - return append(PutLengthEncodedInt(uint64(len(attrData))), attrData...) + return append(mysql.PutLengthEncodedInt(uint64(len(attrData))), attrData...) } // See: http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse func (c *Conn) writeAuthHandshake() error { if !authPluginAllowed(c.authPluginName) { - return fmt.Errorf("unknow auth plugin name '%s'", c.authPluginName) + return fmt.Errorf("unknown auth plugin name '%s'", c.authPluginName) } // Set default client capabilities that reflect the abilities of this library - capability := CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | - CLIENT_LONG_PASSWORD | CLIENT_TRANSACTIONS | CLIENT_PLUGIN_AUTH - // Adjust client capability flags based on server support - capability |= c.capability & CLIENT_LONG_FLAG + capability := mysql.CLIENT_PROTOCOL_41 | mysql.CLIENT_SECURE_CONNECTION | + mysql.CLIENT_LONG_PASSWORD | mysql.CLIENT_TRANSACTIONS | mysql.CLIENT_PLUGIN_AUTH | + mysql.CLIENT_LONG_FLAG | mysql.CLIENT_QUERY_ATTRIBUTES | mysql.CLIENT_DEPRECATE_EOF // Adjust client capability flags on specific client requests // Only flags that would make any sense setting and aren't handled elsewhere // in the library are supported here - capability |= c.ccaps&CLIENT_FOUND_ROWS | c.ccaps&CLIENT_IGNORE_SPACE | - c.ccaps&CLIENT_MULTI_STATEMENTS | c.ccaps&CLIENT_MULTI_RESULTS | - c.ccaps&CLIENT_PS_MULTI_RESULTS | c.ccaps&CLIENT_CONNECT_ATTRS | - c.ccaps&CLIENT_COMPRESS | c.ccaps&CLIENT_ZSTD_COMPRESSION_ALGORITHM | - c.ccaps&CLIENT_LOCAL_FILES + for _, optionalCap := range optionalCapabilities { + capability |= c.ccaps & optionalCap + } + + capability &^= c.clientExplicitOffCaps // To enable TLS / SSL if c.tlsConfig != nil { - capability |= CLIENT_SSL + capability |= mysql.CLIENT_SSL } auth, addNull, err := c.genAuthResponse(c.salt) @@ -226,11 +239,11 @@ func (c *Conn) writeAuthHandshake() error { // here we use the Length-Encoded-Integer(LEI) as the data length may not fit into one byte // see: https://dev.mysql.com/doc/internals/en/integer.html#length-encoded-integer var authRespLEIBuf [9]byte - authRespLEI := AppendLengthEncodedInteger(authRespLEIBuf[:0], uint64(len(auth))) + authRespLEI := mysql.AppendLengthEncodedInteger(authRespLEIBuf[:0], uint64(len(auth))) if len(authRespLEI) > 1 { // if the length can not be written in 1 byte, it must be written as a // length encoded integer - capability |= CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA + capability |= mysql.CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA } // packet length @@ -247,22 +260,23 @@ func (c *Conn) writeAuthHandshake() error { } // db name if len(c.db) > 0 { - capability |= CLIENT_CONNECT_WITH_DB + capability |= mysql.CLIENT_CONNECT_WITH_DB length += len(c.db) + 1 } // connection attributes attrData := c.genAttributes() if len(attrData) > 0 { - capability |= CLIENT_CONNECT_ATTRS + capability |= mysql.CLIENT_CONNECT_ATTRS length += len(attrData) } - if c.ccaps&CLIENT_ZSTD_COMPRESSION_ALGORITHM > 0 { + if c.ccaps&mysql.CLIENT_ZSTD_COMPRESSION_ALGORITHM > 0 { length++ } data := make([]byte, length+4) // capability [32 bit] + c.capability &= capability data[4] = byte(capability) data[5] = byte(capability >> 8) data[6] = byte(capability >> 16) @@ -275,10 +289,10 @@ func (c *Conn) writeAuthHandshake() error { data[11] = 0x00 // Charset [1 byte] - // use default collation id 33 here, is `utf8mb3_general_ci` + // use default collation id 255 here, is `utf8mb4_0900_ai_ci` collationName := c.collation if len(collationName) == 0 { - collationName = DEFAULT_COLLATION_NAME + collationName = mysql.DEFAULT_COLLATION_NAME } collation, err := charset.GetCollationByName(collationName) if err != nil { @@ -346,7 +360,7 @@ func (c *Conn) writeAuthHandshake() error { pos += copy(data[pos:], attrData) } - if c.ccaps&CLIENT_ZSTD_COMPRESSION_ALGORITHM > 0 { + if c.ccaps&mysql.CLIENT_ZSTD_COMPRESSION_ALGORITHM > 0 { // zstd_compression_level data[pos] = 0x03 } diff --git a/vendor/github.com/go-mysql-org/go-mysql/client/conn.go b/vendor/github.com/go-mysql-org/go-mysql/client/conn.go index dbf3a3027..8f717ff08 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/client/conn.go +++ b/vendor/github.com/go-mysql-org/go-mysql/client/conn.go @@ -5,15 +5,19 @@ import ( "context" "crypto/tls" "fmt" + "maps" + "math/bits" "net" "runtime" + "runtime/debug" + "slices" "strings" "time" "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/parser/charset" - . "github.com/go-mysql-org/go-mysql/mysql" + "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/packet" "github.com/go-mysql-org/go-mysql/utils" ) @@ -43,6 +47,9 @@ type Conn struct { capability uint32 // client-set capabilities only ccaps uint32 + // Capability flags explicitly disabled by the client via UnsetCapability() + // These flags are removed from the final advertised capability set during handshake. + clientExplicitOffCaps uint32 attributes map[string]string @@ -56,16 +63,21 @@ type Conn struct { authPluginName string connectionID uint32 + + queryAttributes []mysql.QueryAttribute + + // Include the file + line as query attribute. The number set which frame in the stack should be used. + includeLine int } // This function will be called for every row in resultset from ExecuteSelectStreaming. -type SelectPerRowCallback func(row []FieldValue) error +type SelectPerRowCallback func(row []mysql.FieldValue) error // This function will be called once per result from ExecuteSelectStreaming -type SelectPerResultCallback func(result *Result) error +type SelectPerResultCallback func(result *mysql.Result) error // This function will be called once per result from ExecuteMultiple -type ExecPerResultCallback func(result *Result, err error) +type ExecPerResultCallback func(result *mysql.Result, err error) func getNetProto(addr string) string { proto := "tcp" @@ -99,14 +111,22 @@ type Dialer func(ctx context.Context, network, address string) (net.Conn, error) func ConnectWithDialer(ctx context.Context, network, addr, user, password, dbName string, dialer Dialer, options ...Option) (*Conn, error) { c := new(Conn) + c.includeLine = -1 c.BufferSize = defaultBufferSize c.attributes = map[string]string{ - "_client_name": "go-mysql", - // "_client_version": "0.1", + "_client_name": "go-mysql", "_os": runtime.GOOS, "_platform": runtime.GOARCH, "_runtime_version": runtime.Version(), } + if buildInfo, ok := debug.ReadBuildInfo(); ok { + for _, bi := range buildInfo.Deps { + if bi.Path == "github.com/go-mysql-org/go-mysql" { + c.attributes["_client_version"] = bi.Version + break + } + } + } if network == "" { network = getNetProto(addr) @@ -124,7 +144,7 @@ func ConnectWithDialer(ctx context.Context, network, addr, user, password, dbNam c.proto = network // use default charset here, utf-8 - c.charset = DEFAULT_CHARSET + c.charset = mysql.DEFAULT_CHARSET // Apply configuration functions. for _, option := range options { @@ -137,9 +157,9 @@ func ConnectWithDialer(ctx context.Context, network, addr, user, password, dbNam c.Conn = packet.NewConnWithTimeout(conn, c.ReadTimeout, c.WriteTimeout, c.BufferSize) if c.tlsConfig != nil { - seq := c.Conn.Sequence + seq := c.Sequence c.Conn = packet.NewTLSConnWithTimeout(conn, c.ReadTimeout, c.WriteTimeout) - c.Conn.Sequence = seq + c.Sequence = seq } if err = c.handshake(); err != nil { @@ -147,10 +167,10 @@ func ConnectWithDialer(ctx context.Context, network, addr, user, password, dbNam return nil, errors.Trace(err) } - if c.ccaps&CLIENT_COMPRESS > 0 { - c.Conn.Compression = MYSQL_COMPRESS_ZLIB - } else if c.ccaps&CLIENT_ZSTD_COMPRESSION_ALGORITHM > 0 { - c.Conn.Compression = MYSQL_COMPRESS_ZSTD + if c.ccaps&mysql.CLIENT_COMPRESS > 0 { + c.Compression = mysql.MYSQL_COMPRESS_ZLIB + } else if c.ccaps&mysql.CLIENT_ZSTD_COMPRESSION_ALGORITHM > 0 { + c.Compression = mysql.MYSQL_COMPRESS_ZSTD } // if a collation was set with a ID of > 255, then we need to call SET NAMES ... @@ -194,19 +214,21 @@ func (c *Conn) handshake() error { return nil } +// Close directly closes the connection. Use Quit() to first send COM_QUIT to the server and then close the connection. func (c *Conn) Close() error { return c.Conn.Close() } +// Quit sends COM_QUIT to the server and then closes the connection. Use Close() to directly close the connection. func (c *Conn) Quit() error { - if err := c.writeCommand(COM_QUIT); err != nil { + if err := c.writeCommand(mysql.COM_QUIT); err != nil { return err } return c.Close() } func (c *Conn) Ping() error { - if err := c.writeCommand(COM_PING); err != nil { + if err := c.writeCommand(mysql.COM_PING); err != nil { return errors.Trace(err) } @@ -217,19 +239,26 @@ func (c *Conn) Ping() error { return nil } -// SetCapability enables the use of a specific capability -func (c *Conn) SetCapability(cap uint32) { - c.ccaps |= cap +// SetCapability marks the specified flag as explicitly enabled by the client. +func (c *Conn) SetCapability(capability uint32) error { + if !slices.Contains(optionalCapabilities, capability) { + return errors.New("unsupported or unknown capability") + } + c.ccaps |= capability + c.clientExplicitOffCaps &^= capability + return nil } -// UnsetCapability disables the use of a specific capability -func (c *Conn) UnsetCapability(cap uint32) { - c.ccaps &= ^cap +// UnsetCapability marks the specified flag as explicitly disabled by the client. +// This disables the flag even if the server supports it. +func (c *Conn) UnsetCapability(capability uint32) { + c.ccaps &^= capability + c.clientExplicitOffCaps |= capability } // HasCapability returns true if the connection has the specific capability -func (c *Conn) HasCapability(cap uint32) bool { - return c.ccaps&cap > 0 +func (c *Conn) HasCapability(capability uint32) bool { + return c.ccaps&capability != 0 } // UseSSL: use default SSL @@ -245,20 +274,23 @@ func (c *Conn) SetTLSConfig(config *tls.Config) { } func (c *Conn) UseDB(dbName string) error { - if c.db == dbName { - return nil - } + _, err := c.UseDBWithResult(dbName) + return err +} - if err := c.writeCommandStr(COM_INIT_DB, dbName); err != nil { - return errors.Trace(err) +func (c *Conn) UseDBWithResult(dbName string) (*mysql.Result, error) { + if err := c.writeCommandStr(mysql.COM_INIT_DB, dbName); err != nil { + return nil, errors.Trace(err) } - if _, err := c.readOK(); err != nil { - return errors.Trace(err) + var r *mysql.Result + var err error + if r, err = c.readOK(); err != nil { + return r, errors.Trace(err) } c.db = dbName - return nil + return r, nil } func (c *Conn) GetDB() string { @@ -275,22 +307,21 @@ func (c *Conn) GetServerVersion() string { // of the server and returns 0 if they are equal, and 1 if the server version // is higher and -1 if the server version is lower. func (c *Conn) CompareServerVersion(v string) (int, error) { - return CompareServerVersions(c.serverVersion, v) + return mysql.CompareServerVersions(c.serverVersion, v) } -func (c *Conn) Execute(command string, args ...interface{}) (*Result, error) { +func (c *Conn) Execute(command string, args ...any) (*mysql.Result, error) { if len(args) == 0 { return c.exec(command) - } else { - if s, err := c.Prepare(command); err != nil { - return nil, errors.Trace(err) - } else { - var r *Result - r, err = s.Execute(args...) - s.Close() - return r, err - } } + s, err := c.Prepare(command) + if err != nil { + return nil, errors.Trace(err) + } + var r *mysql.Result + r, err = s.Execute(args...) + s.Close() + return r, err } // ExecuteMultiple will call perResultCallback for every result of the multiple queries @@ -299,13 +330,13 @@ func (c *Conn) Execute(command string, args ...interface{}) (*Result, error) { // When ExecuteMultiple is used, the connection should have the SERVER_MORE_RESULTS_EXISTS // flag set to signal the server multiple queries are executed. Handling the responses // is up to the implementation of perResultCallback. -func (c *Conn) ExecuteMultiple(query string, perResultCallback ExecPerResultCallback) (*Result, error) { - if err := c.writeCommandStr(COM_QUERY, query); err != nil { +func (c *Conn) ExecuteMultiple(query string, perResultCallback ExecPerResultCallback) (*mysql.Result, error) { + if err := c.execSend(query); err != nil { return nil, errors.Trace(err) } var err error - var result *Result + var result *mysql.Result bs := utils.ByteSliceGet(16) defer utils.ByteSlicePut(bs) @@ -317,13 +348,13 @@ func (c *Conn) ExecuteMultiple(query string, perResultCallback ExecPerResultCall } switch bs.B[0] { - case OK_HEADER: + case mysql.OK_HEADER: result, err = c.handleOKPacket(bs.B) - case ERR_HEADER: + case mysql.ERR_HEADER: err = c.handleErrorPacket(bytes.Repeat(bs.B, 1)) result = nil - case LocalInFile_HEADER: - err = ErrMalformPacket + case mysql.LocalInFile_HEADER: + err = mysql.ErrMalformPacket result = nil default: result, err = c.readResultset(bs.B, false) @@ -332,7 +363,7 @@ func (c *Conn) ExecuteMultiple(query string, perResultCallback ExecPerResultCall perResultCallback(result, err) // if there was an error of this was the last result, stop looping - if err != nil || result.Status&SERVER_MORE_RESULTS_EXISTS == 0 { + if err != nil || result.Status&mysql.SERVER_MORE_RESULTS_EXISTS == 0 { break } } @@ -341,10 +372,10 @@ func (c *Conn) ExecuteMultiple(query string, perResultCallback ExecPerResultCall // streaming session // if this would end up in WriteValue, it would just be ignored as all // responses should have been handled in perResultCallback - rs := NewResultset(0) - rs.Streaming = StreamingMultiple + rs := mysql.NewResultset(0) + rs.Streaming = mysql.StreamingMultiple rs.StreamingDone = true - return NewResult(rs), nil + return mysql.NewResult(rs), nil } // ExecuteSelectStreaming will call perRowCallback for every row in resultset @@ -352,8 +383,8 @@ func (c *Conn) ExecuteMultiple(query string, perResultCallback ExecPerResultCall // When given, perResultCallback will be called once per result // // ExecuteSelectStreaming should be used only for SELECT queries with a large response resultset for memory preserving. -func (c *Conn) ExecuteSelectStreaming(command string, result *Result, perRowCallback SelectPerRowCallback, perResultCallback SelectPerResultCallback) error { - if err := c.writeCommandStr(COM_QUERY, command); err != nil { +func (c *Conn) ExecuteSelectStreaming(command string, result *mysql.Result, perRowCallback SelectPerRowCallback, perResultCallback SelectPerResultCallback) error { + if err := c.execSend(command); err != nil { return errors.Trace(err) } @@ -365,6 +396,21 @@ func (c *Conn) Begin() error { return errors.Trace(err) } +func (c *Conn) BeginTx(readOnly bool, txIsolation string) error { + if txIsolation != "" { + if _, err := c.exec("SET TRANSACTION ISOLATION LEVEL " + txIsolation); err != nil { + return errors.Trace(err) + } + } + var err error + if readOnly { + _, err = c.exec("START TRANSACTION READ ONLY") + } else { + _, err = c.exec("START TRANSACTION") + } + return errors.Trace(err) +} + func (c *Conn) Commit() error { _, err := c.exec("COMMIT") return errors.Trace(err) @@ -375,10 +421,9 @@ func (c *Conn) Rollback() error { return errors.Trace(err) } +// SetAttributes sets connection attributes func (c *Conn) SetAttributes(attributes map[string]string) { - for k, v := range attributes { - c.attributes[k] = v - } + maps.Copy(c.attributes, attributes) } func (c *Conn) SetCharset(charset string) error { @@ -388,10 +433,9 @@ func (c *Conn) SetCharset(charset string) error { if _, err := c.exec(fmt.Sprintf("SET NAMES %s", charset)); err != nil { return errors.Trace(err) - } else { - c.charset = charset - return nil } + c.charset = charset + return nil } func (c *Conn) SetCollation(collation string) error { @@ -407,13 +451,14 @@ func (c *Conn) GetCollation() string { return c.collation } -func (c *Conn) FieldList(table string, wildcard string) ([]*Field, error) { - if err := c.writeCommandStrStr(COM_FIELD_LIST, table, wildcard); err != nil { +// FieldList uses COM_FIELD_LIST to get a list of fields from a table +func (c *Conn) FieldList(table string, wildcard string) ([]*mysql.Field, error) { + if err := c.writeCommandStrStr(mysql.COM_FIELD_LIST, table, wildcard); err != nil { return nil, errors.Trace(err) } - fs := make([]*Field, 0, 4) - var f *Field + fs := make([]*mysql.Field, 0, 4) + var f *mysql.Field for { data, err := c.ReadPacket() if err != nil { @@ -421,7 +466,7 @@ func (c *Conn) FieldList(table string, wildcard string) ([]*Field, error) { } // ERR Packet - if data[0] == ERR_HEADER { + if data[0] == mysql.ERR_HEADER { return nil, c.handleErrorPacket(data) } @@ -430,7 +475,7 @@ func (c *Conn) FieldList(table string, wildcard string) ([]*Field, error) { return fs, nil } - if f, err = FieldData(data).Parse(); err != nil { + if f, err = mysql.FieldData(data).Parse(); err != nil { return nil, errors.Trace(err) } fs = append(fs, f) @@ -446,12 +491,14 @@ func (c *Conn) SetAutoCommit() error { return nil } +// IsAutoCommit returns true if SERVER_STATUS_AUTOCOMMIT is set func (c *Conn) IsAutoCommit() bool { - return c.status&SERVER_STATUS_AUTOCOMMIT > 0 + return c.status&mysql.SERVER_STATUS_AUTOCOMMIT > 0 } +// IsInTransaction returns true if SERVER_STATUS_IN_TRANS is set func (c *Conn) IsInTransaction() bool { - return c.status&SERVER_STATUS_IN_TRANS > 0 + return c.status&mysql.SERVER_STATUS_IN_TRANS > 0 } func (c *Conn) GetCharset() string { @@ -462,7 +509,7 @@ func (c *Conn) GetConnectionID() uint32 { return c.connectionID } -func (c *Conn) HandleOKPacket(data []byte) *Result { +func (c *Conn) HandleOKPacket(data []byte) *mysql.Result { r, _ := c.handleOKPacket(data) return r } @@ -471,96 +518,81 @@ func (c *Conn) HandleErrorPacket(data []byte) error { return c.handleErrorPacket(data) } -func (c *Conn) ReadOKPacket() (*Result, error) { +func (c *Conn) ReadOKPacket() (*mysql.Result, error) { return c.readOK() } -func (c *Conn) exec(query string) (*Result, error) { - if err := c.writeCommandStr(COM_QUERY, query); err != nil { +// Send COM_QUERY and read the result +func (c *Conn) exec(query string) (*mysql.Result, error) { + err := c.execSend(query) + if err != nil { return nil, errors.Trace(err) } - return c.readResult(false) } +// Sends COM_QUERY +// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query.html +func (c *Conn) execSend(query string) error { + var buf bytes.Buffer + defer clear(c.queryAttributes) + + if c.capability&mysql.CLIENT_QUERY_ATTRIBUTES > 0 { + if c.includeLine >= 0 { + _, file, line, ok := runtime.Caller(c.includeLine) + if ok { + lineAttr := mysql.QueryAttribute{ + Name: "_line", + Value: fmt.Sprintf("%s:%d", file, line), + } + c.queryAttributes = append(c.queryAttributes, lineAttr) + } + } + + numParams := len(c.queryAttributes) + buf.Write(mysql.PutLengthEncodedInt(uint64(numParams))) + buf.WriteByte(0x1) // parameter_set_count, unused + if numParams > 0 { + // null_bitmap, length: (num_params+7)/8 + for i := 0; i < (numParams+7)/8; i++ { + buf.WriteByte(0x0) + } + buf.WriteByte(0x1) // new_params_bind_flag, unused + for _, qa := range c.queryAttributes { + buf.Write(qa.TypeAndFlag()) + buf.Write(mysql.PutLengthEncodedString([]byte(qa.Name))) + } + for _, qa := range c.queryAttributes { + buf.Write(qa.ValueBytes()) + } + } + } + + _, err := buf.Write(utils.StringToByteSlice(query)) + if err != nil { + return err + } + + if err := c.writeCommandBuf(mysql.COM_QUERY, buf.Bytes()); err != nil { + return errors.Trace(err) + } + + return nil +} + // CapabilityString is returning a string with the names of capability flags // separated by "|". Examples of capability names are CLIENT_DEPRECATE_EOF and CLIENT_PROTOCOL_41. +// These are defined as constants in the mysql package. func (c *Conn) CapabilityString() string { - var caps []string capability := c.capability - for i := 0; capability != 0; i++ { - field := uint32(1 << i) - if capability&field == 0 { - continue - } + caps := make([]string, 0, bits.OnesCount32(capability)) + for capability != 0 { + field := uint32(1 << bits.TrailingZeros32(capability)) capability ^= field - switch field { - case CLIENT_LONG_PASSWORD: - caps = append(caps, "CLIENT_LONG_PASSWORD") - case CLIENT_FOUND_ROWS: - caps = append(caps, "CLIENT_FOUND_ROWS") - case CLIENT_LONG_FLAG: - caps = append(caps, "CLIENT_LONG_FLAG") - case CLIENT_CONNECT_WITH_DB: - caps = append(caps, "CLIENT_CONNECT_WITH_DB") - case CLIENT_NO_SCHEMA: - caps = append(caps, "CLIENT_NO_SCHEMA") - case CLIENT_COMPRESS: - caps = append(caps, "CLIENT_COMPRESS") - case CLIENT_ODBC: - caps = append(caps, "CLIENT_ODBC") - case CLIENT_LOCAL_FILES: - caps = append(caps, "CLIENT_LOCAL_FILES") - case CLIENT_IGNORE_SPACE: - caps = append(caps, "CLIENT_IGNORE_SPACE") - case CLIENT_PROTOCOL_41: - caps = append(caps, "CLIENT_PROTOCOL_41") - case CLIENT_INTERACTIVE: - caps = append(caps, "CLIENT_INTERACTIVE") - case CLIENT_SSL: - caps = append(caps, "CLIENT_SSL") - case CLIENT_IGNORE_SIGPIPE: - caps = append(caps, "CLIENT_IGNORE_SIGPIPE") - case CLIENT_TRANSACTIONS: - caps = append(caps, "CLIENT_TRANSACTIONS") - case CLIENT_RESERVED: - caps = append(caps, "CLIENT_RESERVED") - case CLIENT_SECURE_CONNECTION: - caps = append(caps, "CLIENT_SECURE_CONNECTION") - case CLIENT_MULTI_STATEMENTS: - caps = append(caps, "CLIENT_MULTI_STATEMENTS") - case CLIENT_MULTI_RESULTS: - caps = append(caps, "CLIENT_MULTI_RESULTS") - case CLIENT_PS_MULTI_RESULTS: - caps = append(caps, "CLIENT_PS_MULTI_RESULTS") - case CLIENT_PLUGIN_AUTH: - caps = append(caps, "CLIENT_PLUGIN_AUTH") - case CLIENT_CONNECT_ATTRS: - caps = append(caps, "CLIENT_CONNECT_ATTRS") - case CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA: - caps = append(caps, "CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA") - case CLIENT_CAN_HANDLE_EXPIRED_PASSWORDS: - caps = append(caps, "CLIENT_CAN_HANDLE_EXPIRED_PASSWORDS") - case CLIENT_SESSION_TRACK: - caps = append(caps, "CLIENT_SESSION_TRACK") - case CLIENT_DEPRECATE_EOF: - caps = append(caps, "CLIENT_DEPRECATE_EOF") - case CLIENT_OPTIONAL_RESULTSET_METADATA: - caps = append(caps, "CLIENT_OPTIONAL_RESULTSET_METADATA") - case CLIENT_ZSTD_COMPRESSION_ALGORITHM: - caps = append(caps, "CLIENT_ZSTD_COMPRESSION_ALGORITHM") - case CLIENT_QUERY_ATTRIBUTES: - caps = append(caps, "CLIENT_QUERY_ATTRIBUTES") - case MULTI_FACTOR_AUTHENTICATION: - caps = append(caps, "MULTI_FACTOR_AUTHENTICATION") - case CLIENT_CAPABILITY_EXTENSION: - caps = append(caps, "CLIENT_CAPABILITY_EXTENSION") - case CLIENT_SSL_VERIFY_SERVER_CERT: - caps = append(caps, "CLIENT_SSL_VERIFY_SERVER_CERT") - case CLIENT_REMEMBER_OPTIONS: - caps = append(caps, "CLIENT_REMEMBER_OPTIONS") - default: + if capname, ok := mysql.CapNames[field]; ok { + caps = append(caps, capname) + } else { caps = append(caps, fmt.Sprintf("(%d)", field)) } } @@ -568,40 +600,39 @@ func (c *Conn) CapabilityString() string { return strings.Join(caps, "|") } +// StatusString returns a "|" separated list of status fields. Example status values are SERVER_QUERY_WAS_SLOW and SERVER_STATUS_AUTOCOMMIT. +// These are defined as constants in the mysql package. func (c *Conn) StatusString() string { - var stats []string status := c.status - for i := 0; status != 0; i++ { - field := uint16(1 << i) - if status&field == 0 { - continue - } + stats := make([]string, 0, bits.OnesCount16(status)) + for status != 0 { + field := uint16(1 << bits.TrailingZeros16(status)) status ^= field switch field { - case SERVER_STATUS_IN_TRANS: + case mysql.SERVER_STATUS_IN_TRANS: stats = append(stats, "SERVER_STATUS_IN_TRANS") - case SERVER_STATUS_AUTOCOMMIT: + case mysql.SERVER_STATUS_AUTOCOMMIT: stats = append(stats, "SERVER_STATUS_AUTOCOMMIT") - case SERVER_MORE_RESULTS_EXISTS: + case mysql.SERVER_MORE_RESULTS_EXISTS: stats = append(stats, "SERVER_MORE_RESULTS_EXISTS") - case SERVER_STATUS_NO_GOOD_INDEX_USED: + case mysql.SERVER_STATUS_NO_GOOD_INDEX_USED: stats = append(stats, "SERVER_STATUS_NO_GOOD_INDEX_USED") - case SERVER_STATUS_NO_INDEX_USED: + case mysql.SERVER_STATUS_NO_INDEX_USED: stats = append(stats, "SERVER_STATUS_NO_INDEX_USED") - case SERVER_STATUS_CURSOR_EXISTS: + case mysql.SERVER_STATUS_CURSOR_EXISTS: stats = append(stats, "SERVER_STATUS_CURSOR_EXISTS") - case SERVER_STATUS_LAST_ROW_SEND: + case mysql.SERVER_STATUS_LAST_ROW_SEND: stats = append(stats, "SERVER_STATUS_LAST_ROW_SEND") - case SERVER_STATUS_DB_DROPPED: + case mysql.SERVER_STATUS_DB_DROPPED: stats = append(stats, "SERVER_STATUS_DB_DROPPED") - case SERVER_STATUS_NO_BACKSLASH_ESCAPED: + case mysql.SERVER_STATUS_NO_BACKSLASH_ESCAPED: stats = append(stats, "SERVER_STATUS_NO_BACKSLASH_ESCAPED") - case SERVER_STATUS_METADATA_CHANGED: + case mysql.SERVER_STATUS_METADATA_CHANGED: stats = append(stats, "SERVER_STATUS_METADATA_CHANGED") - case SERVER_QUERY_WAS_SLOW: + case mysql.SERVER_QUERY_WAS_SLOW: stats = append(stats, "SERVER_QUERY_WAS_SLOW") - case SERVER_PS_OUT_PARAMS: + case mysql.SERVER_PS_OUT_PARAMS: stats = append(stats, "SERVER_PS_OUT_PARAMS") default: stats = append(stats, fmt.Sprintf("(%d)", field)) @@ -610,3 +641,17 @@ func (c *Conn) StatusString() string { return strings.Join(stats, "|") } + +// SetQueryAttributes sets the query attributes to be send along with the next query +func (c *Conn) SetQueryAttributes(attrs ...mysql.QueryAttribute) error { + c.queryAttributes = attrs + return nil +} + +// IncludeLine can be passed as option when connecting to include the file name and line number +// of the caller as query attribute `_line` when sending queries. +// The argument is used the dept in the stack. The top level is go-mysql and then there are the +// levels of the application. +func (c *Conn) IncludeLine(frame int) { + c.includeLine = frame +} diff --git a/vendor/github.com/go-mysql-org/go-mysql/client/pool.go b/vendor/github.com/go-mysql-org/go-mysql/client/pool.go index a3c306f7e..0c7bcb910 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/client/pool.go +++ b/vendor/github.com/go-mysql-org/go-mysql/client/pool.go @@ -2,9 +2,10 @@ package client import ( "context" - "log" + "log/slog" "math" "math/rand" + "net" "sync" "time" @@ -26,10 +27,10 @@ Usage: type ( Timestamp int64 - LogFunc func(format string, args ...interface{}) + LogFunc func(format string, args ...any) Pool struct { - logFunc LogFunc + logger *slog.Logger minAlive int maxAlive int maxIdle int @@ -109,7 +110,7 @@ func NewPoolWithOptions( } pool := &Pool{ - logFunc: po.logFunc, + logger: po.logger, minAlive: po.minAlive, maxAlive: po.maxAlive, maxIdle: po.maxIdle, @@ -118,7 +119,7 @@ func NewPoolWithOptions( idlePingTimeout: Timestamp(math.Ceil(MaxIdleTimeoutWithoutPing.Seconds())), connect: func() (*Conn, error) { - return Connect(addr, user, password, dbName, po.connOptions...) + return ConnectWithDialer(context.Background(), "", addr, user, password, dbName, po.dialer, po.connOptions...) }, readyConnection: make(chan Connection), @@ -159,7 +160,7 @@ func NewPoolWithOptions( // // Deprecated: use NewPoolWithOptions func NewPool( - logFunc LogFunc, + logger *slog.Logger, minAlive int, maxAlive int, maxIdle int, @@ -174,12 +175,12 @@ func NewPool( user, password, dbName, - WithLogFunc(logFunc), + WithLogger(logger), WithPoolLimits(minAlive, maxAlive, maxIdle), WithConnOptions(options...), ) - if err != nil { - pool.logFunc(`Pool: NewPool: %s`, err.Error()) + if err != nil && logger != nil { + logger.Error("Pool: NewPool", slog.Any("error", err)) } return pool @@ -312,7 +313,9 @@ func (pool *Pool) newConnectionProducer() { pool.synchro.stats.TotalCount-- // Bad luck, should try again pool.synchro.Unlock() - pool.logFunc("Cannot establish new db connection: %s", err.Error()) + if pool.logger != nil { + pool.logger.Error("Pool: cannot establish new db connection", slog.Any("error", err)) + } timer := time.NewTimer( time.Duration(10+rand.Intn(90)) * time.Millisecond, @@ -439,7 +442,7 @@ func (pool *Pool) recheckConnections(connections []Connection) { var wg sync.WaitGroup wg.Add(workerCnt) - for worker := 0; worker < workerCnt; worker++ { + for range workerCnt { go func() { defer wg.Done() for connection := range queue { @@ -460,21 +463,21 @@ func (pool *Pool) spawnConnectionsIfNeeded() bool { pool.synchro.Lock() totalCount := pool.synchro.stats.TotalCount idleCount := len(pool.synchro.idleConnections) - needSpanNew := pool.minAlive - totalCount + needSpawnNew := pool.minAlive - totalCount pool.synchro.Unlock() - if needSpanNew <= 0 { + if needSpawnNew <= 0 { return false } // Не хватает соединений, нужно создать еще - if needSpanNew > MaxNewConnectionAtOnce { - needSpanNew = MaxNewConnectionAtOnce + if needSpawnNew > MaxNewConnectionAtOnce { + needSpawnNew = MaxNewConnectionAtOnce } - pool.logFunc(`Pool: Setup %d new connections (total: %d idle: %d)...`, needSpanNew, totalCount, idleCount) - pool.startNewConnections(needSpanNew) + pool.logger.Info("Pool: Setup new connections", slog.Int("new", needSpawnNew), slog.Int("total", totalCount), slog.Int("idle", idleCount)) + pool.startNewConnections(needSpawnNew) return true } @@ -502,11 +505,9 @@ func (pool *Pool) closeIdleConnectionsIfCan() { return } - closeFromIdx := idleCnt - canCloseCnt - if closeFromIdx < 0 { + closeFromIdx := max(idleCnt-canCloseCnt, // If there are enough requests in the "flight" now, then we can close all unnecessary - closeFromIdx = 0 - } + 0) toClose := append([]Connection{}, pool.synchro.idleConnections[closeFromIdx:]...) @@ -517,7 +518,7 @@ func (pool *Pool) closeIdleConnectionsIfCan() { pool.synchro.Unlock() - pool.logFunc(`Pool: Close %d idle connections (in fly %d)`, len(toClose), inFly) + pool.logger.Info("Pool: close idle connections", slog.Int("closed", len(toClose)), slog.Int("inFly", inFly)) for _, connection := range toClose { pool.closeConn(connection.conn) } @@ -532,17 +533,17 @@ func (pool *Pool) closeConn(conn *Conn) { } func (pool *Pool) startNewConnections(count int) { - pool.logFunc(`Pool: Setup %d new connections (minimal pool size)...`, count) + pool.logger.Info("Pool: Setup new connections (minimal pool size)", slog.Int("count", count)) connections := make([]Connection, 0, count) - for i := 0; i < count; i++ { + for range count { if conn, err := pool.createNewConnection(); err == nil { pool.synchro.Lock() pool.synchro.stats.TotalCount++ pool.synchro.Unlock() connections = append(connections, conn) } else { - pool.logFunc(`Pool: createNewConnection: %s`, err) + pool.logger.Warn("Pool: createNewConnection failed", slog.Any("error", err)) } } @@ -558,7 +559,7 @@ func (pool *Pool) ping(conn *Conn) error { _ = conn.SetDeadline(deadline) err := conn.Ping() if err != nil { - pool.logFunc(`Pool: ping query fail: %s`, err.Error()) + pool.logger.Error("Pool: ping query fail", slog.Any("error", err)) } else { _ = conn.SetDeadline(time.Time{}) } @@ -569,9 +570,9 @@ func (pool *Pool) ping(conn *Conn) error { // So before call Close, Call PutConn to put all connections that in use back to connection pool first. func (pool *Pool) Close() { pool.cancel() - //wait newConnectionProducer exit. + // wait newConnectionProducer exit. pool.wg.Wait() - //close idle connections + // close idle connections pool.synchro.Lock() for _, connection := range pool.synchro.idleConnections { pool.synchro.stats.TotalCount-- @@ -604,10 +605,12 @@ func (pool *Pool) checkConnection(ctx context.Context) error { // getDefaultPoolOptions returns pool config for low load services func getDefaultPoolOptions() poolOptions { + dialer := &net.Dialer{Timeout: 10 * time.Second} return poolOptions{ - logFunc: log.Printf, + logger: slog.Default(), minAlive: 1, maxAlive: 10, maxIdle: 2, + dialer: dialer.DialContext, } } diff --git a/vendor/github.com/go-mysql-org/go-mysql/client/pool_options.go b/vendor/github.com/go-mysql-org/go-mysql/client/pool_options.go index 90bf5bd0d..428ed8c54 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/client/pool_options.go +++ b/vendor/github.com/go-mysql-org/go-mysql/client/pool_options.go @@ -1,12 +1,13 @@ package client import ( + "log/slog" "time" ) type ( poolOptions struct { - logFunc LogFunc + logger *slog.Logger minAlive int maxAlive int @@ -17,6 +18,8 @@ type ( password string dbName string + dialer Dialer + connOptions []Option newPoolPingTimeout time.Duration @@ -40,9 +43,9 @@ func WithPoolLimits(minAlive, maxAlive, maxIdle int) PoolOption { } } -func WithLogFunc(f LogFunc) PoolOption { +func WithLogger(logger *slog.Logger) PoolOption { return func(o *poolOptions) { - o.logFunc = f + o.logger = logger } } @@ -58,3 +61,9 @@ func WithNewPoolPingTimeout(timeout time.Duration) PoolOption { o.newPoolPingTimeout = timeout } } + +func WithDialer(dialer Dialer) PoolOption { + return func(o *poolOptions) { + o.dialer = dialer + } +} diff --git a/vendor/github.com/go-mysql-org/go-mysql/client/req.go b/vendor/github.com/go-mysql-org/go-mysql/client/req.go index 225e6114d..28a7c10b1 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/client/req.go +++ b/vendor/github.com/go-mysql-org/go-mysql/client/req.go @@ -8,10 +8,10 @@ func (c *Conn) writeCommand(command byte) error { c.ResetSequence() return c.WritePacket([]byte{ - 0x01, //1 bytes long + 0x01, // 1 bytes long 0x00, 0x00, - 0x00, //sequence + 0x00, // sequence command, }) } @@ -41,10 +41,10 @@ func (c *Conn) writeCommandUint32(command byte, arg uint32) error { buf := utils.ByteSliceGet(9) - buf.B[0] = 0x05 //5 bytes long + buf.B[0] = 0x05 // 5 bytes long buf.B[1] = 0x00 buf.B[2] = 0x00 - buf.B[3] = 0x00 //sequence + buf.B[3] = 0x00 // sequence buf.B[4] = command diff --git a/vendor/github.com/go-mysql-org/go-mysql/client/resp.go b/vendor/github.com/go-mysql-org/go-mysql/client/resp.go index a82f82408..b5fe09d91 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/client/resp.go +++ b/vendor/github.com/go-mysql-org/go-mysql/client/resp.go @@ -10,71 +10,140 @@ import ( "github.com/pingcap/errors" - . "github.com/go-mysql-org/go-mysql/mysql" + "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/utils" ) -func (c *Conn) readUntilEOF() (err error) { - var data []byte - - for { - data, err = c.ReadPacket() - - if err != nil { - return - } - - // EOF Packet - if c.isEOFPacket(data) { - return - } - } -} - func (c *Conn) isEOFPacket(data []byte) bool { - return data[0] == EOF_HEADER && len(data) <= 5 + // 0xffffff due to https://dev.mysql.com/worklog/task/?id=7766 + // "Server will never send OK packet longer than 16777216 bytes thus limiting + // size of OK packet to be 16777215 bytes" + return data[0] == mysql.EOF_HEADER && len(data) <= 0xffffff } -func (c *Conn) handleOKPacket(data []byte) (*Result, error) { +func (c *Conn) handleOKPacket(data []byte) (*mysql.Result, error) { var n int - var pos = 1 + pos := 1 - r := NewResultReserveResultset(0) + r := mysql.NewResultReserveResultset(0) - r.AffectedRows, _, n = LengthEncodedInt(data[pos:]) + r.AffectedRows, _, n = mysql.LengthEncodedInt(data[pos:]) pos += n - r.InsertId, _, n = LengthEncodedInt(data[pos:]) + r.InsertId, _, n = mysql.LengthEncodedInt(data[pos:]) pos += n - if c.capability&CLIENT_PROTOCOL_41 > 0 { + if c.capability&mysql.CLIENT_PROTOCOL_41 > 0 { r.Status = binary.LittleEndian.Uint16(data[pos:]) c.status = r.Status pos += 2 //todo:strict_mode, check warnings as error r.Warnings = binary.LittleEndian.Uint16(data[pos:]) - // pos += 2 - } else if c.capability&CLIENT_TRANSACTIONS > 0 { + pos += 2 + } else if c.capability&mysql.CLIENT_TRANSACTIONS > 0 { r.Status = binary.LittleEndian.Uint16(data[pos:]) c.status = r.Status - // pos += 2 + pos += 2 } - // new ok package will check CLIENT_SESSION_TRACK too, but I don't support it now. + if (c.capability&mysql.CLIENT_SESSION_TRACK > 0) && + (c.status&mysql.SERVER_SESSION_STATE_CHANGED > 0) { + var err error + + // Example status message: + // "Records: 3 Duplicates: 0 Warnings: 0" + statusMessageLength := int(data[pos]) + pos++ + if statusMessageLength > 0 { + r.StatusMessage = utils.ByteSliceToString(data[pos : pos+statusMessageLength]) + pos += statusMessageLength + } + + sessionTrackingChangeLength := int(data[pos]) + pos++ + dataLength := len(data[pos:]) + if dataLength != sessionTrackingChangeLength { + return nil, fmt.Errorf("incorrect data length for session tracking data: expected %d but got %d", + sessionTrackingChangeLength, dataLength) + } + r.SessionTracking, err = decodeSessionTracking(data[pos:]) + if err != nil { + return nil, err + } + } // skip info return r, nil } +func decodeSessionTracking(data []byte) (s *mysql.SessionTrackingInfo, err error) { + s = &mysql.SessionTrackingInfo{} + pos := 0 + for pos < len(data) { + sessionTrackingChangeType := data[pos] + pos++ // session tracking type + pos++ // length of session tracking data, unused + + switch sessionTrackingChangeType { + case mysql.SESSION_TRACK_SYSTEM_VARIABLES: + if s.Variables == nil { + s.Variables = make(map[string]string, 1) + } + varNameLength := data[pos] + pos++ + varName := utils.ByteSliceToString(data[pos : pos+int(varNameLength)]) + pos += int(varNameLength) + varValueLength := data[pos] + pos++ + s.Variables[varName] = utils.ByteSliceToString(data[pos : pos+int(varValueLength)]) + pos += int(varValueLength) + case mysql.SESSION_TRACK_SCHEMA: + schemaInfoLength := data[pos] + pos++ + s.Schema = utils.ByteSliceToString(data[pos : pos+int(schemaInfoLength)]) + pos += int(schemaInfoLength) + case mysql.SESSION_TRACK_STATE_CHANGE: + s.State = string(data[pos]) + pos++ + case mysql.SESSION_TRACK_GTIDS: + gtidFormat := data[pos] + if gtidFormat != 0 { + return nil, fmt.Errorf("unexpected GTID format %d", gtidFormat) + } + pos++ + gtidLength := data[pos] + pos++ + s.GTID = utils.ByteSliceToString(data[pos : pos+int(gtidLength)]) + pos += int(gtidLength) + case mysql.SESSION_TRACK_TRANSACTION_CHARACTERISTICS: + characteristicsLength := data[pos] + pos++ + if characteristicsLength > 0 { + s.Characteristics = utils.ByteSliceToString(data[pos : pos+int(characteristicsLength)]) + pos += int(characteristicsLength) + } + case mysql.SESSION_TRACK_TRANSACTION_STATE: + transactionStateLength := data[pos] + pos++ + s.TransactionState = utils.ByteSliceToString(data[pos : pos+int(transactionStateLength)]) + pos += int(transactionStateLength) + default: + return nil, fmt.Errorf("got unknown change type %v", sessionTrackingChangeType) + } + } + + return s, nil +} + func (c *Conn) handleErrorPacket(data []byte) error { - e := new(MyError) + e := new(mysql.MyError) - var pos = 1 + pos := 1 e.Code = binary.LittleEndian.Uint16(data[pos:]) pos += 2 - if c.capability&CLIENT_PROTOCOL_41 > 0 { + if c.capability&mysql.CLIENT_PROTOCOL_41 > 0 { // skip '#' pos++ e.State = utils.ByteSliceToString(data[pos : pos+5]) @@ -122,14 +191,16 @@ func (c *Conn) handleAuthResult() error { } // handle caching_sha2_password - if c.authPluginName == AUTH_CACHING_SHA2_PASSWORD { + switch c.authPluginName { + case mysql.AUTH_CACHING_SHA2_PASSWORD: if data == nil { return nil // auth already succeeded } - if data[0] == CACHE_SHA2_FAST_AUTH { + switch data[0] { + case mysql.CACHE_SHA2_FAST_AUTH: _, err = c.readOK() return err - } else if data[0] == CACHE_SHA2_FULL_AUTH { + case mysql.CACHE_SHA2_FULL_AUTH: // need full authentication if c.tlsConfig != nil || c.proto == "unix" { if err = c.WriteClearAuthPacket(c.password); err != nil { @@ -142,10 +213,10 @@ func (c *Conn) handleAuthResult() error { } _, err = c.readOK() return err - } else { + default: return errors.Errorf("invalid packet %x", data[0]) } - } else if c.authPluginName == AUTH_SHA256_PASSWORD { + case mysql.AUTH_SHA256_PASSWORD: if len(data) == 0 { return nil // auth already succeeded } @@ -174,18 +245,18 @@ func (c *Conn) readAuthResult() ([]byte, string, error) { // see: https://insidemysql.com/preparing-your-community-connector-for-mysql-8-part-2-sha256/ // packet indicator switch data[0] { - case OK_HEADER: + case mysql.OK_HEADER: _, err := c.handleOKPacket(data) return nil, "", err - case MORE_DATE_HEADER: + case mysql.MORE_DATE_HEADER: return data[1:], "", err - case EOF_HEADER: + case mysql.EOF_HEADER: // server wants to switch auth if len(data) < 1 { // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest - return nil, AUTH_MYSQL_OLD_PASSWORD, nil + return nil, mysql.AUTH_MYSQL_OLD_PASSWORD, nil } pluginEndIndex := bytes.IndexByte(data, 0x00) if pluginEndIndex < 0 { @@ -200,22 +271,23 @@ func (c *Conn) readAuthResult() ([]byte, string, error) { } } -func (c *Conn) readOK() (*Result, error) { +func (c *Conn) readOK() (*mysql.Result, error) { data, err := c.ReadPacket() if err != nil { return nil, errors.Trace(err) } - if data[0] == OK_HEADER { + switch data[0] { + case mysql.OK_HEADER: return c.handleOKPacket(data) - } else if data[0] == ERR_HEADER { + case mysql.ERR_HEADER: return nil, c.handleErrorPacket(data) - } else { + default: return nil, errors.New("invalid ok packet") } } -func (c *Conn) readResult(binary bool) (*Result, error) { +func (c *Conn) readResult(binary bool) (*mysql.Result, error) { bs := utils.ByteSliceGet(16) defer utils.ByteSlicePut(bs) var err error @@ -225,18 +297,18 @@ func (c *Conn) readResult(binary bool) (*Result, error) { } switch bs.B[0] { - case OK_HEADER: + case mysql.OK_HEADER: return c.handleOKPacket(bs.B) - case ERR_HEADER: + case mysql.ERR_HEADER: return nil, c.handleErrorPacket(bytes.Repeat(bs.B, 1)) - case LocalInFile_HEADER: - return nil, ErrMalformPacket + case mysql.LocalInFile_HEADER: + return nil, mysql.ErrMalformPacket default: return c.readResultset(bs.B, binary) } } -func (c *Conn) readResultStreaming(binary bool, result *Result, perRowCb SelectPerRowCallback, perResCb SelectPerResultCallback) error { +func (c *Conn) readResultStreaming(binary bool, result *mysql.Result, perRowCb SelectPerRowCallback, perResCb SelectPerResultCallback) error { bs := utils.ByteSliceGet(16) defer utils.ByteSlicePut(bs) var err error @@ -246,7 +318,7 @@ func (c *Conn) readResultStreaming(binary bool, result *Result, perRowCb SelectP } switch bs.B[0] { - case OK_HEADER: + case mysql.OK_HEADER: // https://dev.mysql.com/doc/internals/en/com-query-response.html // 14.6.4.1 COM_QUERY Response // If the number of columns in the resultset is 0, this is a OK_Packet. @@ -261,29 +333,29 @@ func (c *Conn) readResultStreaming(binary bool, result *Result, perRowCb SelectP result.InsertId = okResult.InsertId result.Warnings = okResult.Warnings if result.Resultset == nil { - result.Resultset = NewResultset(0) + result.Resultset = mysql.NewResultset(0) } else { result.Reset(0) } return nil - case ERR_HEADER: + case mysql.ERR_HEADER: return c.handleErrorPacket(bytes.Repeat(bs.B, 1)) - case LocalInFile_HEADER: - return ErrMalformPacket + case mysql.LocalInFile_HEADER: + return mysql.ErrMalformPacket default: return c.readResultsetStreaming(bs.B, binary, result, perRowCb, perResCb) } } -func (c *Conn) readResultset(data []byte, binary bool) (*Result, error) { +func (c *Conn) readResultset(data []byte, binary bool) (*mysql.Result, error) { // column count - count, _, n := LengthEncodedInt(data) + count, _, n := mysql.LengthEncodedInt(data) if n-len(data) != 0 { - return nil, ErrMalformPacket + return nil, mysql.ErrMalformPacket } - result := NewResultReserveResultset(int(count)) + result := mysql.NewResultReserveResultset(int(count)) if err := c.readResultColumns(result); err != nil { return nil, errors.Trace(err) @@ -296,22 +368,22 @@ func (c *Conn) readResultset(data []byte, binary bool) (*Result, error) { return result, nil } -func (c *Conn) readResultsetStreaming(data []byte, binary bool, result *Result, perRowCb SelectPerRowCallback, perResCb SelectPerResultCallback) error { - columnCount, _, n := LengthEncodedInt(data) +func (c *Conn) readResultsetStreaming(data []byte, binary bool, result *mysql.Result, perRowCb SelectPerRowCallback, perResCb SelectPerResultCallback) error { + columnCount, _, n := mysql.LengthEncodedInt(data) if n-len(data) != 0 { - return ErrMalformPacket + return mysql.ErrMalformPacket } if result.Resultset == nil { - result.Resultset = NewResultset(int(columnCount)) + result.Resultset = mysql.NewResultset(int(columnCount)) } else { // Reuse memory if can result.Reset(int(columnCount)) } // this is a streaming resultset - result.Resultset.Streaming = StreamingSelect + result.Streaming = mysql.StreamingSelect if err := c.readResultColumns(result); err != nil { return errors.Trace(err) @@ -328,16 +400,15 @@ func (c *Conn) readResultsetStreaming(data []byte, binary bool, result *Result, } // this resultset is done streaming - result.Resultset.StreamingDone = true + result.StreamingDone = true return nil } -func (c *Conn) readResultColumns(result *Result) (err error) { - var i = 0 +func (c *Conn) readResultColumns(result *mysql.Result) (err error) { var data []byte - for { + for i := range result.Fields { rawPkgLen := len(result.RawPkg) result.RawPkg, err = c.ReadPacketReuseMem(result.RawPkg) if err != nil { @@ -345,24 +416,8 @@ func (c *Conn) readResultColumns(result *Result) (err error) { } data = result.RawPkg[rawPkgLen:] - // EOF Packet - if c.isEOFPacket(data) { - if c.capability&CLIENT_PROTOCOL_41 > 0 { - result.Warnings = binary.LittleEndian.Uint16(data[1:]) - // todo add strict_mode, warning will be treat as error - result.Status = binary.LittleEndian.Uint16(data[3:]) - c.status = result.Status - } - - if i != len(result.Fields) { - err = ErrMalformPacket - } - - return err - } - if result.Fields[i] == nil { - result.Fields[i] = &Field{} + result.Fields[i] = &mysql.Field{} } err = result.Fields[i].Parse(data) if err != nil { @@ -370,12 +425,32 @@ func (c *Conn) readResultColumns(result *Result) (err error) { } result.FieldNames[utils.ByteSliceToString(result.Fields[i].Name)] = i + } + + if c.capability&mysql.CLIENT_DEPRECATE_EOF == 0 { + // EOF Packet + rawPkgLen := len(result.RawPkg) + result.RawPkg, err = c.ReadPacketReuseMem(result.RawPkg) + if err != nil { + return err + } + data = result.RawPkg[rawPkgLen:] - i++ + if c.isEOFPacket(data) { + if c.capability&mysql.CLIENT_PROTOCOL_41 > 0 { + result.Warnings = binary.LittleEndian.Uint16(data[1:]) + // todo add strict_mode, warning will be treat as error + result.Status = binary.LittleEndian.Uint16(data[3:]) + c.status = result.Status + } + return nil + } + return mysql.ErrMalformPacket } + return nil } -func (c *Conn) readResultRows(result *Result, isBinary bool) (err error) { +func (c *Conn) readResultRows(result *mysql.Result, isBinary bool) (err error) { var data []byte for { @@ -386,19 +461,25 @@ func (c *Conn) readResultRows(result *Result, isBinary bool) (err error) { } data = result.RawPkg[rawPkgLen:] - // EOF Packet if c.isEOFPacket(data) { - if c.capability&CLIENT_PROTOCOL_41 > 0 { + if c.capability&mysql.CLIENT_DEPRECATE_EOF != 0 { + // Treat like OK + affectedRows, _, n := mysql.LengthEncodedInt(data[1:]) + insertID, _, m := mysql.LengthEncodedInt(data[1+n:]) + result.Status = binary.LittleEndian.Uint16(data[1+n+m:]) + result.AffectedRows = affectedRows + result.InsertId = insertID + c.status = result.Status + } else if c.capability&mysql.CLIENT_PROTOCOL_41 > 0 { result.Warnings = binary.LittleEndian.Uint16(data[1:]) // todo add strict_mode, warning will be treat as error result.Status = binary.LittleEndian.Uint16(data[3:]) c.status = result.Status } - break } - if data[0] == ERR_HEADER { + if data[0] == mysql.ERR_HEADER { return c.handleErrorPacket(data) } @@ -406,14 +487,13 @@ func (c *Conn) readResultRows(result *Result, isBinary bool) (err error) { } if cap(result.Values) < len(result.RowDatas) { - result.Values = make([][]FieldValue, len(result.RowDatas)) + result.Values = make([][]mysql.FieldValue, len(result.RowDatas)) } else { result.Values = result.Values[:len(result.RowDatas)] } for i := range result.Values { result.Values[i], err = result.RowDatas[i].Parse(result.Fields, isBinary, result.Values[i]) - if err != nil { return errors.Trace(err) } @@ -422,10 +502,10 @@ func (c *Conn) readResultRows(result *Result, isBinary bool) (err error) { return nil } -func (c *Conn) readResultRowsStreaming(result *Result, isBinary bool, perRowCb SelectPerRowCallback) (err error) { +func (c *Conn) readResultRowsStreaming(result *mysql.Result, isBinary bool, perRowCb SelectPerRowCallback) (err error) { var ( data []byte - row []FieldValue + row []mysql.FieldValue ) for { @@ -434,9 +514,16 @@ func (c *Conn) readResultRowsStreaming(result *Result, isBinary bool, perRowCb S return err } - // EOF Packet if c.isEOFPacket(data) { - if c.capability&CLIENT_PROTOCOL_41 > 0 { + if c.capability&mysql.CLIENT_DEPRECATE_EOF != 0 { + // Treat like OK + affectedRows, _, n := mysql.LengthEncodedInt(data[1:]) + insertID, _, m := mysql.LengthEncodedInt(data[1+n:]) + result.Status = binary.LittleEndian.Uint16(data[1+n+m:]) + result.AffectedRows = affectedRows + result.InsertId = insertID + c.status = result.Status + } else if c.capability&mysql.CLIENT_PROTOCOL_41 > 0 { result.Warnings = binary.LittleEndian.Uint16(data[1:]) // todo add strict_mode, warning will be treat as error result.Status = binary.LittleEndian.Uint16(data[3:]) @@ -446,12 +533,12 @@ func (c *Conn) readResultRowsStreaming(result *Result, isBinary bool, perRowCb S break } - if data[0] == ERR_HEADER { + if data[0] == mysql.ERR_HEADER { return c.handleErrorPacket(data) } // Parse this row - row, err = RowData(data).Parse(result.Fields, isBinary, row) + row, err = mysql.RowData(data).Parse(result.Fields, isBinary, row) if err != nil { return errors.Trace(err) } diff --git a/vendor/github.com/go-mysql-org/go-mysql/client/stmt.go b/vendor/github.com/go-mysql-org/go-mysql/client/stmt.go index 82c760d72..c7a89981f 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/client/stmt.go +++ b/vendor/github.com/go-mysql-org/go-mysql/client/stmt.go @@ -5,34 +5,35 @@ import ( "encoding/json" "fmt" "math" + "runtime" - . "github.com/go-mysql-org/go-mysql/mysql" + "github.com/go-mysql-org/go-mysql/mysql" + "github.com/go-mysql-org/go-mysql/stmt" "github.com/go-mysql-org/go-mysql/utils" "github.com/pingcap/errors" ) type Stmt struct { - conn *Conn - id uint32 - - params int - columns int + conn *Conn warnings int + + // PreparedStmt contains common fields shared with server.Stmt for proxy passthrough + stmt.PreparedStmt } func (s *Stmt) ParamNum() int { - return s.params + return s.Params } func (s *Stmt) ColumnNum() int { - return s.columns + return s.Columns } func (s *Stmt) WarningsNum() int { return s.warnings } -func (s *Stmt) Execute(args ...interface{}) (*Result, error) { +func (s *Stmt) Execute(args ...any) (*mysql.Result, error) { if err := s.write(args...); err != nil { return nil, errors.Trace(err) } @@ -40,7 +41,7 @@ func (s *Stmt) Execute(args ...interface{}) (*Result, error) { return s.conn.readResult(true) } -func (s *Stmt) ExecuteSelectStreaming(result *Result, perRowCb SelectPerRowCallback, perResCb SelectPerResultCallback, args ...interface{}) error { +func (s *Stmt) ExecuteSelectStreaming(result *mysql.Result, perRowCb SelectPerRowCallback, perResCb SelectPerResultCallback, args ...any) error { if err := s.write(args...); err != nil { return errors.Trace(err) } @@ -49,34 +50,52 @@ func (s *Stmt) ExecuteSelectStreaming(result *Result, perRowCb SelectPerRowCallb } func (s *Stmt) Close() error { - if err := s.conn.writeCommandUint32(COM_STMT_CLOSE, s.id); err != nil { + if err := s.conn.writeCommandUint32(mysql.COM_STMT_CLOSE, s.ID); err != nil { return errors.Trace(err) } return nil } -func (s *Stmt) write(args ...interface{}) error { - paramsNum := s.params +// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_execute.html +func (s *Stmt) write(args ...any) error { + defer clear(s.conn.queryAttributes) + paramsNum := s.Params if len(args) != paramsNum { - return fmt.Errorf("argument mismatch, need %d but got %d", s.params, len(args)) + return fmt.Errorf("argument mismatch, need %d but got %d", s.Params, len(args)) + } + + if (s.conn.capability&mysql.CLIENT_QUERY_ATTRIBUTES > 0) && (s.conn.includeLine >= 0) { + _, file, line, ok := runtime.Caller(s.conn.includeLine) + if ok { + lineAttr := mysql.QueryAttribute{ + Name: "_line", + Value: fmt.Sprintf("%s:%d", file, line), + } + s.conn.queryAttributes = append(s.conn.queryAttributes, lineAttr) + } } - paramTypes := make([]byte, paramsNum<<1) - paramValues := make([][]byte, paramsNum) + qaLen := len(s.conn.queryAttributes) + paramTypes := make([][]byte, paramsNum+qaLen) + paramFlags := make([][]byte, paramsNum+qaLen) + paramValues := make([][]byte, paramsNum+qaLen) + paramNames := make([][]byte, paramsNum+qaLen) - //NULL-bitmap, length: (num-params+7) - nullBitmap := make([]byte, (paramsNum+7)>>3) + // NULL-bitmap, length: (num-params+7) + nullBitmap := make([]byte, (paramsNum+qaLen+7)>>3) length := 1 + 4 + 1 + 4 + ((paramsNum + 7) >> 3) + 1 + (paramsNum << 1) - var newParamBoundFlag byte = 0 + var newParamBoundFlag byte for i := range args { if args[i] == nil { nullBitmap[i/8] |= 1 << (uint(i) % 8) - paramTypes[i<<1] = MYSQL_TYPE_NULL + paramTypes[i] = []byte{mysql.MYSQL_TYPE_NULL} + paramNames[i] = []byte{0} // length encoded, no name + paramFlags[i] = []byte{0} continue } @@ -84,68 +103,82 @@ func (s *Stmt) write(args ...interface{}) error { switch v := args[i].(type) { case int8: - paramTypes[i<<1] = MYSQL_TYPE_TINY + paramTypes[i] = []byte{mysql.MYSQL_TYPE_TINY} paramValues[i] = []byte{byte(v)} case int16: - paramTypes[i<<1] = MYSQL_TYPE_SHORT - paramValues[i] = Uint16ToBytes(uint16(v)) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_SHORT} + paramValues[i] = mysql.Uint16ToBytes(uint16(v)) case int32: - paramTypes[i<<1] = MYSQL_TYPE_LONG - paramValues[i] = Uint32ToBytes(uint32(v)) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_LONG} + paramValues[i] = mysql.Uint32ToBytes(uint32(v)) case int: - paramTypes[i<<1] = MYSQL_TYPE_LONGLONG - paramValues[i] = Uint64ToBytes(uint64(v)) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_LONGLONG} + paramValues[i] = mysql.Uint64ToBytes(uint64(v)) case int64: - paramTypes[i<<1] = MYSQL_TYPE_LONGLONG - paramValues[i] = Uint64ToBytes(uint64(v)) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_LONGLONG} + paramValues[i] = mysql.Uint64ToBytes(uint64(v)) case uint8: - paramTypes[i<<1] = MYSQL_TYPE_TINY - paramTypes[(i<<1)+1] = 0x80 + paramTypes[i] = []byte{mysql.MYSQL_TYPE_TINY} + paramFlags[i] = []byte{mysql.PARAM_UNSIGNED} paramValues[i] = []byte{v} case uint16: - paramTypes[i<<1] = MYSQL_TYPE_SHORT - paramTypes[(i<<1)+1] = 0x80 - paramValues[i] = Uint16ToBytes(v) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_SHORT} + paramFlags[i] = []byte{mysql.PARAM_UNSIGNED} + paramValues[i] = mysql.Uint16ToBytes(v) case uint32: - paramTypes[i<<1] = MYSQL_TYPE_LONG - paramTypes[(i<<1)+1] = 0x80 - paramValues[i] = Uint32ToBytes(v) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_LONG} + paramFlags[i] = []byte{mysql.PARAM_UNSIGNED} + paramValues[i] = mysql.Uint32ToBytes(v) case uint: - paramTypes[i<<1] = MYSQL_TYPE_LONGLONG - paramTypes[(i<<1)+1] = 0x80 - paramValues[i] = Uint64ToBytes(uint64(v)) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_LONGLONG} + paramFlags[i] = []byte{mysql.PARAM_UNSIGNED} + paramValues[i] = mysql.Uint64ToBytes(uint64(v)) case uint64: - paramTypes[i<<1] = MYSQL_TYPE_LONGLONG - paramTypes[(i<<1)+1] = 0x80 - paramValues[i] = Uint64ToBytes(v) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_LONGLONG} + paramFlags[i] = []byte{mysql.PARAM_UNSIGNED} + paramValues[i] = mysql.Uint64ToBytes(v) case bool: - paramTypes[i<<1] = MYSQL_TYPE_TINY + paramTypes[i] = []byte{mysql.MYSQL_TYPE_TINY} if v { paramValues[i] = []byte{1} } else { paramValues[i] = []byte{0} } case float32: - paramTypes[i<<1] = MYSQL_TYPE_FLOAT - paramValues[i] = Uint32ToBytes(math.Float32bits(v)) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_FLOAT} + paramValues[i] = mysql.Uint32ToBytes(math.Float32bits(v)) case float64: - paramTypes[i<<1] = MYSQL_TYPE_DOUBLE - paramValues[i] = Uint64ToBytes(math.Float64bits(v)) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_DOUBLE} + paramValues[i] = mysql.Uint64ToBytes(math.Float64bits(v)) case string: - paramTypes[i<<1] = MYSQL_TYPE_STRING - paramValues[i] = append(PutLengthEncodedInt(uint64(len(v))), v...) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_STRING} + paramValues[i] = append(mysql.PutLengthEncodedInt(uint64(len(v))), v...) case []byte: - paramTypes[i<<1] = MYSQL_TYPE_STRING - paramValues[i] = append(PutLengthEncodedInt(uint64(len(v))), v...) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_STRING} + paramValues[i] = append(mysql.PutLengthEncodedInt(uint64(len(v))), v...) + case mysql.TypedBytes: + paramTypes[i] = []byte{v.Type} + paramValues[i] = append(mysql.PutLengthEncodedInt(uint64(len(v.Bytes))), v.Bytes...) case json.RawMessage: - paramTypes[i<<1] = MYSQL_TYPE_STRING - paramValues[i] = append(PutLengthEncodedInt(uint64(len(v))), v...) + paramTypes[i] = []byte{mysql.MYSQL_TYPE_STRING} + paramValues[i] = append(mysql.PutLengthEncodedInt(uint64(len(v))), v...) default: return fmt.Errorf("invalid argument type %T", args[i]) } + paramNames[i] = []byte{0} // length encoded, no name + if paramFlags[i] == nil { + paramFlags[i] = []byte{0} + } length += len(paramValues[i]) } + for i, qa := range s.conn.queryAttributes { + tf := qa.TypeAndFlag() + paramTypes[(i + paramsNum)] = []byte{tf[0]} + paramFlags[i+paramsNum] = []byte{tf[1]} + paramValues[i+paramsNum] = qa.ValueBytes() + paramNames[i+paramsNum] = mysql.PutLengthEncodedString([]byte(qa.Name)) + } data := utils.BytesBufferGet() defer func() { @@ -156,28 +189,43 @@ func (s *Stmt) write(args ...interface{}) error { } data.Write([]byte{0, 0, 0, 0}) - data.WriteByte(COM_STMT_EXECUTE) - data.Write([]byte{byte(s.id), byte(s.id >> 8), byte(s.id >> 16), byte(s.id >> 24)}) + data.WriteByte(mysql.COM_STMT_EXECUTE) + data.Write([]byte{byte(s.ID), byte(s.ID >> 8), byte(s.ID >> 16), byte(s.ID >> 24)}) - //flag: CURSOR_TYPE_NO_CURSOR - data.WriteByte(0x00) + flags := mysql.CURSOR_TYPE_NO_CURSOR + if paramsNum > 0 { + flags |= mysql.PARAMETER_COUNT_AVAILABLE + } + data.WriteByte(flags) - //iteration-count, always 1 + // iteration-count, always 1 data.Write([]byte{1, 0, 0, 0}) - if s.params > 0 { - data.Write(nullBitmap) - - //new-params-bound-flag - data.WriteByte(newParamBoundFlag) - - if newParamBoundFlag == 1 { - //type of each parameter, length: num-params * 2 - data.Write(paramTypes) - - //value of each parameter - for _, v := range paramValues { - data.Write(v) + if paramsNum > 0 || (s.conn.capability&mysql.CLIENT_QUERY_ATTRIBUTES > 0 && (flags&mysql.PARAMETER_COUNT_AVAILABLE > 0)) { + if s.conn.capability&mysql.CLIENT_QUERY_ATTRIBUTES > 0 { + paramsNum += len(s.conn.queryAttributes) + data.Write(mysql.PutLengthEncodedInt(uint64(paramsNum))) + } + if paramsNum > 0 { + data.Write(nullBitmap) + + // new-params-bound-flag + data.WriteByte(newParamBoundFlag) + + if newParamBoundFlag == 1 { + for i := 0; i < paramsNum; i++ { + data.Write(paramTypes[i]) + data.Write(paramFlags[i]) + + if s.conn.capability&mysql.CLIENT_QUERY_ATTRIBUTES > 0 { + data.Write(paramNames[i]) + } + } + + // value of each parameter + for _, v := range paramValues { + data.Write(v) + } } } } @@ -188,7 +236,7 @@ func (s *Stmt) write(args ...interface{}) error { } func (c *Conn) Prepare(query string) (*Stmt, error) { - if err := c.writeCommandStr(COM_STMT_PREPARE, query); err != nil { + if err := c.writeCommandStr(mysql.COM_STMT_PREPARE, query); err != nil { return nil, errors.Trace(err) } @@ -197,10 +245,10 @@ func (c *Conn) Prepare(query string) (*Stmt, error) { return nil, errors.Trace(err) } - if data[0] == ERR_HEADER { + if data[0] == mysql.ERR_HEADER { return nil, c.handleErrorPacket(data) - } else if data[0] != OK_HEADER { - return nil, ErrMalformPacket + } else if data[0] != mysql.OK_HEADER { + return nil, mysql.ErrMalformPacket } s := new(Stmt) @@ -208,31 +256,60 @@ func (c *Conn) Prepare(query string) (*Stmt, error) { pos := 1 - //for statement id - s.id = binary.LittleEndian.Uint32(data[pos:]) + // for statement id + s.ID = binary.LittleEndian.Uint32(data[pos:]) pos += 4 - //number columns - s.columns = int(binary.LittleEndian.Uint16(data[pos:])) + // number columns + s.Columns = int(binary.LittleEndian.Uint16(data[pos:])) pos += 2 - //number params - s.params = int(binary.LittleEndian.Uint16(data[pos:])) + // number params + s.Params = int(binary.LittleEndian.Uint16(data[pos:])) pos += 2 - //warnings - s.warnings = int(binary.LittleEndian.Uint16(data[pos:])) - // pos += 2 + // reserved + pos++ - if s.params > 0 { - if err := s.conn.readUntilEOF(); err != nil { - return nil, errors.Trace(err) + if len(data) >= 12 { + // warnings + s.warnings = int(binary.LittleEndian.Uint16(data[pos:])) + // pos += 2 + } + + if s.Params > 0 { + s.RawParamFields = make([][]byte, s.Params) + for i := range s.Params { + data, err := s.conn.ReadPacket() + if err != nil { + return nil, errors.Trace(err) + } + s.RawParamFields[i] = data + } + if s.conn.capability&mysql.CLIENT_DEPRECATE_EOF == 0 { + if packet, err := s.conn.ReadPacket(); err != nil { + return nil, errors.Trace(err) + } else if !c.isEOFPacket(packet) { + return nil, mysql.ErrMalformPacket + } } } - if s.columns > 0 { - if err := s.conn.readUntilEOF(); err != nil { - return nil, errors.Trace(err) + if s.Columns > 0 { + s.RawColumnFields = make([][]byte, s.Columns) + for i := range s.Columns { + data, err := s.conn.ReadPacket() + if err != nil { + return nil, errors.Trace(err) + } + s.RawColumnFields[i] = data + } + if s.conn.capability&mysql.CLIENT_DEPRECATE_EOF == 0 { + if packet, err := s.conn.ReadPacket(); err != nil { + return nil, errors.Trace(err) + } else if !c.isEOFPacket(packet) { + return nil, mysql.ErrMalformPacket + } } } diff --git a/vendor/github.com/go-mysql-org/go-mysql/compress/zlib.go b/vendor/github.com/go-mysql-org/go-mysql/compress/zlib.go index 17bad746d..474c02b63 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/compress/zlib.go +++ b/vendor/github.com/go-mysql-org/go-mysql/compress/zlib.go @@ -11,19 +11,13 @@ import ( const DefaultCompressionLevel = 6 var ( - zlibReaderPool *sync.Pool - zlibWriterPool sync.Pool -) - -func init() { - zlibReaderPool = &sync.Pool{ - New: func() interface{} { + zlibReaderPool = sync.Pool{ + New: func() any { return nil }, } - zlibWriterPool = sync.Pool{ - New: func() interface{} { + New: func() any { w, err := zlib.NewWriterLevel(new(bytes.Buffer), DefaultCompressionLevel) if err != nil { panic(err) @@ -31,10 +25,12 @@ func init() { return w }, } -} +) -var _ io.WriteCloser = zlibWriter{} -var _ io.ReadCloser = zlibReader{} +var ( + _ io.WriteCloser = zlibWriter{} + _ io.ReadCloser = zlibReader{} +) type zlibWriter struct { w *zlib.Writer diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/const.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/const.go index a11b05370..a602ab333 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/const.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/const.go @@ -1,3 +1,4 @@ +//nolint:revive // legacy MySQL wire-protocol constants (OK_HEADER, COM_*, CLIENT_*, MYSQL_TYPE_*, AUTH_*, etc.) are kept verbatim from the upstream protocol for backward compatibility package mysql const ( @@ -24,8 +25,12 @@ const ( AUTH_CLEAR_PASSWORD = "mysql_clear_password" AUTH_CACHING_SHA2_PASSWORD = "caching_sha2_password" AUTH_SHA256_PASSWORD = "sha256_password" + AUTH_MARIADB_ED25519 = "client_ed25519" ) +// SERVER_STATUS_flags_enum +// https://dev.mysql.com/doc/dev/mysql-server/latest/mysql__com_8h.html#a1d854e841086925be1883e4d7b4e8cad +// https://github.com/mysql/mysql-server/blob/500c3117e6f638043c4fea8aacf17d63a8d07de6/include/mysql_com.h#L809-L864 const ( SERVER_STATUS_IN_TRANS uint16 = 0x0001 SERVER_STATUS_AUTOCOMMIT uint16 = 0x0002 @@ -39,8 +44,11 @@ const ( SERVER_STATUS_METADATA_CHANGED uint16 = 0x0400 SERVER_QUERY_WAS_SLOW uint16 = 0x0800 SERVER_PS_OUT_PARAMS uint16 = 0x1000 + SERVER_STATUS_IN_TRANS_READONLY uint16 = 0x2000 + SERVER_SESSION_STATE_CHANGED uint16 = 0x4000 ) +// https://github.com/mysql/mysql-server/blob/6b6d3ed3d5c6591b446276184642d7d0504ecc86/include/my_command.h#L48-L103 const ( COM_SLEEP byte = iota COM_QUIT @@ -74,6 +82,8 @@ const ( COM_DAEMON COM_BINLOG_DUMP_GTID COM_RESET_CONNECTION + COM_CLONE + COM_SUBSCRIBE_GROUP_REPLICATION_STREAM ) const ( @@ -113,6 +123,41 @@ const ( CLIENT_REMEMBER_OPTIONS ) +var CapNames = map[uint32]string{ + CLIENT_LONG_PASSWORD: "CLIENT_LONG_PASSWORD", + CLIENT_FOUND_ROWS: "CLIENT_FOUND_ROWS", + CLIENT_LONG_FLAG: "CLIENT_LONG_FLAG", + CLIENT_CONNECT_WITH_DB: "CLIENT_CONNECT_WITH_DB", + CLIENT_NO_SCHEMA: "CLIENT_NO_SCHEMA", + CLIENT_COMPRESS: "CLIENT_COMPRESS", + CLIENT_ODBC: "CLIENT_ODBC", + CLIENT_LOCAL_FILES: "CLIENT_LOCAL_FILES", + CLIENT_IGNORE_SPACE: "CLIENT_IGNORE_SPACE", + CLIENT_PROTOCOL_41: "CLIENT_PROTOCOL_41", + CLIENT_INTERACTIVE: "CLIENT_INTERACTIVE", + CLIENT_SSL: "CLIENT_SSL", + CLIENT_IGNORE_SIGPIPE: "CLIENT_IGNORE_SIGPIPE", + CLIENT_TRANSACTIONS: "CLIENT_TRANSACTIONS", + CLIENT_RESERVED: "CLIENT_RESERVED", + CLIENT_SECURE_CONNECTION: "CLIENT_SECURE_CONNECTION", + CLIENT_MULTI_STATEMENTS: "CLIENT_MULTI_STATEMENTS", + CLIENT_MULTI_RESULTS: "CLIENT_MULTI_RESULTS", + CLIENT_PS_MULTI_RESULTS: "CLIENT_PS_MULTI_RESULTS", + CLIENT_PLUGIN_AUTH: "CLIENT_PLUGIN_AUTH", + CLIENT_CONNECT_ATTRS: "CLIENT_CONNECT_ATTRS", + CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA: "CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA", + CLIENT_CAN_HANDLE_EXPIRED_PASSWORDS: "CLIENT_CAN_HANDLE_EXPIRED_PASSWORDS", + CLIENT_SESSION_TRACK: "CLIENT_SESSION_TRACK", + CLIENT_DEPRECATE_EOF: "CLIENT_DEPRECATE_EOF", + CLIENT_OPTIONAL_RESULTSET_METADATA: "CLIENT_OPTIONAL_RESULTSET_METADATA", + CLIENT_ZSTD_COMPRESSION_ALGORITHM: "CLIENT_ZSTD_COMPRESSION_ALGORITHM", + CLIENT_QUERY_ATTRIBUTES: "CLIENT_QUERY_ATTRIBUTES", + MULTI_FACTOR_AUTHENTICATION: "MULTI_FACTOR_AUTHENTICATION", + CLIENT_CAPABILITY_EXTENSION: "CLIENT_CAPABILITY_EXTENSION", + CLIENT_SSL_VERIFY_SERVER_CERT: "CLIENT_SSL_VERIFY_SERVER_CERT", + CLIENT_REMEMBER_OPTIONS: "CLIENT_REMEMBER_OPTIONS", +} + const ( MYSQL_TYPE_DECIMAL byte = iota MYSQL_TYPE_TINY @@ -138,6 +183,8 @@ const ( MYSQL_TYPE_TIME2 ) +const MYSQL_TYPE_VECTOR = 0xf2 + const ( MYSQL_TYPE_JSON byte = iota + 0xf5 MYSQL_TYPE_NEWDECIMAL @@ -170,15 +217,19 @@ const ( UNIQUE_FLAG = 65536 ) +const ( + PARAM_UNSIGNED = 128 +) + const ( DEFAULT_ADDR = "127.0.0.1:3306" DEFAULT_IPV6_ADDR = "[::1]:3306" DEFAULT_USER = "root" DEFAULT_PASSWORD = "" - DEFAULT_FLAVOR = "mysql" - DEFAULT_CHARSET = "utf8" - DEFAULT_COLLATION_ID uint8 = 33 - DEFAULT_COLLATION_NAME string = "utf8_general_ci" + DEFAULT_FLAVOR = MySQLFlavor + DEFAULT_CHARSET = "utf8mb4" + DEFAULT_COLLATION_ID uint8 = 255 + DEFAULT_COLLATION_NAME string = "utf8mb4_0900_ai_ci" ) const ( @@ -201,3 +252,29 @@ const ( MYSQL_COMPRESS_ZLIB MYSQL_COMPRESS_ZSTD ) + +// See enum_cursor_type in mysql.h +const ( + CURSOR_TYPE_NO_CURSOR byte = 0x0 + CURSOR_TYPE_READ_ONLY byte = 0x1 + CURSOR_TYPE_FOR_UPDATE byte = 0x2 + CURSOR_TYPE_SCROLLABLE byte = 0x4 + PARAMETER_COUNT_AVAILABLE byte = 0x8 +) + +// See enum_session_state_type in mysql_com.h +const ( + SESSION_TRACK_SYSTEM_VARIABLES = iota + SESSION_TRACK_SCHEMA + SESSION_TRACK_STATE_CHANGE + SESSION_TRACK_GTIDS + SESSION_TRACK_TRANSACTION_CHARACTERISTICS + SESSION_TRACK_TRANSACTION_STATE +) + +type GtidFormat int + +const ( + GtidFormatClassic GtidFormat = iota + GtidFormatTagged +) diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/errcode.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/errcode.go index 1be5c44ab..0b4173415 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/errcode.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/errcode.go @@ -1,5 +1,6 @@ package mysql +//nolint:revive // MySQL error code names mirror the upstream server protocol and are intentionally preserved const ( ER_ERROR_FIRST = 1000 ER_HASHCHK = 1000 diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/error.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/error.go index e9915779b..0957e5adc 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/error.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/error.go @@ -24,7 +24,7 @@ func (e *MyError) Error() string { } // NewDefaultError: default mysql error, must adapt errname message format -func NewDefaultError(errCode uint16, args ...interface{}) *MyError { +func NewDefaultError(errCode uint16, args ...any) *MyError { e := new(MyError) e.Code = errCode @@ -62,5 +62,5 @@ func ErrorCode(errMsg string) (code int) { var tmpStr string // golang scanf doesn't support %*,so I used a temporary variable _, _ = fmt.Sscanf(errMsg, "%s%d", &tmpStr, &code) - return + return code } diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/field.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/field.go index 23c32d2e4..e31feb211 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/field.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/field.go @@ -57,78 +57,78 @@ func (f *Field) Parse(p FieldData) (err error) { var n int pos := 0 - //skip catelog, always def + // skip catelog, always def n, err = SkipLengthEncodedString(p) if err != nil { return err } pos += n - //schema + // schema f.Schema, _, n, err = LengthEncodedString(p[pos:]) if err != nil { return err } pos += n - //table + // table f.Table, _, n, err = LengthEncodedString(p[pos:]) if err != nil { return err } pos += n - //org_table + // org_table f.OrgTable, _, n, err = LengthEncodedString(p[pos:]) if err != nil { return err } pos += n - //name + // name f.Name, _, n, err = LengthEncodedString(p[pos:]) if err != nil { return err } pos += n - //org_name + // org_name f.OrgName, _, n, err = LengthEncodedString(p[pos:]) if err != nil { return err } pos += n - //skip oc - pos += 1 + // skip oc + pos++ - //charset + // charset f.Charset = binary.LittleEndian.Uint16(p[pos:]) pos += 2 - //column length + // column length f.ColumnLength = binary.LittleEndian.Uint32(p[pos:]) pos += 4 - //type + // type f.Type = p[pos] pos++ - //flag + // flag f.Flag = binary.LittleEndian.Uint16(p[pos:]) pos += 2 - //decimals 1 + // decimals 1 f.Decimal = p[pos] pos++ - //filter [0x00][0x00] + // filter [0x00][0x00] pos += 2 f.DefaultValue = nil - //if more data, command was field list + // if more data, command was field list if len(p) > pos { - //length of default value lenenc-int + // length of default value lenenc-int f.DefaultValueLength, _, n = LengthEncodedInt(p[pos:]) pos += n @@ -137,7 +137,7 @@ func (f *Field) Parse(p FieldData) (err error) { return err } - //default value string[$len] + // default value string[$len] f.DefaultValue = p[pos:(pos + int(f.DefaultValueLength))] } @@ -207,7 +207,7 @@ func (fv *FieldValue) AsString() []byte { return fv.str } -func (fv *FieldValue) Value() interface{} { +func (fv *FieldValue) Value() any { switch fv.Type { case FieldValueTypeUnsigned: return fv.AsUint64() diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/gtid.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/gtid.go index b73bd7eda..b1e2fa33d 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/gtid.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/gtid.go @@ -17,6 +17,9 @@ type GTIDSet interface { Update(GTIDStr string) error Clone() GTIDSet + + // IsEmpty returns true if the given set is empty and false otherwise. + IsEmpty() bool } func ParseGTIDSet(flavor string, s string) (GTIDSet, error) { diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/mariadb_gtid.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/mariadb_gtid.go index 6e472c657..4b2943654 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/mariadb_gtid.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/mariadb_gtid.go @@ -1,14 +1,13 @@ package mysql import ( - "bytes" "fmt" + "log/slog" "sort" "strconv" "strings" "github.com/pingcap/errors" - "github.com/siddontang/go-log/log" ) // MariadbGTID represent mariadb gtid, [domain ID]-[server-id]-[sequence] @@ -50,7 +49,8 @@ func ParseMariadbGTID(str string) (*MariadbGTID, error) { return &MariadbGTID{ DomainID: uint32(domainID), ServerID: uint32(serverID), - SequenceNumber: sequenceID}, nil + SequenceNumber: sequenceID, + }, nil } func (gtid *MariadbGTID) String() string { @@ -61,7 +61,6 @@ func (gtid *MariadbGTID) String() string { return fmt.Sprintf("%d-%d-%d", gtid.DomainID, gtid.ServerID, gtid.SequenceNumber) } -// Contain return whether one mariadb gtid covers another mariadb gtid func (gtid *MariadbGTID) Contain(other *MariadbGTID) bool { return gtid.DomainID == other.DomainID && gtid.SequenceNumber >= other.SequenceNumber } @@ -75,7 +74,7 @@ func (gtid *MariadbGTID) Clone() *MariadbGTID { func (gtid *MariadbGTID) forward(newer *MariadbGTID) error { if newer.DomainID != gtid.DomainID { - return errors.Errorf("%s is not same with doamin of %s", newer, gtid) + return errors.Errorf("%s is not same with domain of %s", newer, gtid) } /* @@ -93,7 +92,7 @@ func (gtid *MariadbGTID) forward(newer *MariadbGTID) error { | mysqld-bin.000001 | 2215 | Gtid | 111 | 2257 | BEGIN GTID 0-111-6 | */ if newer.SequenceNumber <= gtid.SequenceNumber { - log.Warnf("out of order binlog appears with gtid %s vs current position gtid %s", newer, gtid) + slog.Warn("out of order binlog", slog.Any("new", newer), slog.Any("current", gtid)) } gtid.ServerID = newer.ServerID @@ -101,15 +100,17 @@ func (gtid *MariadbGTID) forward(newer *MariadbGTID) error { return nil } -// MariadbGTIDSet is a set of mariadb gtid +// MariadbGTIDSet represents a MariaDB GTID position (one GTID per domain_id). +// Despite the name, this is a position, not a set — unlike MySQL's executed GTID set. +// See https://github.com/go-mysql-org/go-mysql/issues/1123 type MariadbGTIDSet struct { - Sets map[uint32]map[uint32]*MariadbGTID + Sets map[uint32]*MariadbGTID } // ParseMariadbGTIDSet parses str into mariadb gtid sets func ParseMariadbGTIDSet(str string) (GTIDSet, error) { s := new(MariadbGTIDSet) - s.Sets = make(map[uint32]map[uint32]*MariadbGTID) + s.Sets = make(map[uint32]*MariadbGTID) if str == "" { return s, nil } @@ -120,23 +121,20 @@ func ParseMariadbGTIDSet(str string) (GTIDSet, error) { return s, nil } -// AddSet adds mariadb gtid into mariadb gtid set +// AddSet adds or updates a GTID position for a domain. If the domain already has +// a GTID with a different server_id (e.g. after primary failover), the old entry +// is replaced via forward() to maintain one position per domain_id. func (s *MariadbGTIDSet) AddSet(gtid *MariadbGTID) error { if gtid == nil { return nil } - if serverSets, ok := s.Sets[gtid.DomainID]; !ok { - s.Sets[gtid.DomainID] = map[uint32]*MariadbGTID{ - gtid.ServerID: gtid, - } - } else if o, ok := serverSets[gtid.ServerID]; !ok { - serverSets[gtid.ServerID] = gtid - } else { - err := o.forward(gtid) - if err != nil { + if existing, ok := s.Sets[gtid.DomainID]; ok { + if err := existing.forward(gtid); err != nil { return errors.Trace(err) } + } else { + s.Sets[gtid.DomainID] = gtid.Clone() } return nil @@ -145,8 +143,8 @@ func (s *MariadbGTIDSet) AddSet(gtid *MariadbGTID) error { // Update updates mariadb gtid set func (s *MariadbGTIDSet) Update(GTIDStr string) error { sp := strings.Split(GTIDStr, ",") - //todo, handle redundant same uuid - for i := 0; i < len(sp); i++ { + // todo, handle redundant same uuid + for i := range sp { gtid, err := ParseMariadbGTID(sp[i]) if err != nil { return errors.Trace(err) @@ -161,10 +159,8 @@ func (s *MariadbGTIDSet) Update(GTIDStr string) error { func (s *MariadbGTIDSet) String() string { sets := make([]string, 0, len(s.Sets)) - for _, set := range s.Sets { - for _, gtid := range set { - sets = append(sets, gtid.String()) - } + for _, gtid := range s.Sets { + sets = append(sets, gtid.String()) } sort.Strings(sets) @@ -173,29 +169,16 @@ func (s *MariadbGTIDSet) String() string { // Encode encodes mariadb gtid set func (s *MariadbGTIDSet) Encode() []byte { - var buf bytes.Buffer - sep := "" - for _, set := range s.Sets { - for _, gtid := range set { - buf.WriteString(sep) - buf.WriteString(gtid.String()) - sep = "," - } - } - - return buf.Bytes() + return []byte(s.String()) } // Clone clones a mariadb gtid set func (s *MariadbGTIDSet) Clone() GTIDSet { clone := &MariadbGTIDSet{ - Sets: make(map[uint32]map[uint32]*MariadbGTID), + Sets: make(map[uint32]*MariadbGTID), } - for domainID, set := range s.Sets { - clone.Sets[domainID] = make(map[uint32]*MariadbGTID) - for serverID, gtid := range set { - clone.Sets[domainID][serverID] = gtid.Clone() - } + for domainID, gtid := range s.Sets { + clone.Sets[domainID] = gtid.Clone() } return clone @@ -212,21 +195,14 @@ func (s *MariadbGTIDSet) Equal(o GTIDSet) bool { return false } - for domainID, set := range other.Sets { - serverSet, ok := s.Sets[domainID] + for domainID, gtid := range other.Sets { + ourGTID, ok := s.Sets[domainID] if !ok { return false } - if len(serverSet) != len(set) { + if *gtid != *ourGTID { return false } - for serverID, gtid := range set { - if o, ok := serverSet[serverID]; !ok { - return false - } else if *gtid != *o { - return false - } - } } return true @@ -239,19 +215,19 @@ func (s *MariadbGTIDSet) Contain(o GTIDSet) bool { return false } - for doaminID, set := range other.Sets { - serverSet, ok := s.Sets[doaminID] + for domainID, gtid := range other.Sets { + ourGTID, ok := s.Sets[domainID] if !ok { return false } - for serverID, gtid := range set { - if o, ok := serverSet[serverID]; !ok { - return false - } else if !o.Contain(gtid) { - return false - } + if !ourGTID.Contain(gtid) { + return false } } return true } + +func (s *MariadbGTIDSet) IsEmpty() bool { + return len(s.Sets) == 0 +} diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/mysql_gtid.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/mysql_gtid.go index ecf726c1e..df90539cf 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/mysql_gtid.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/mysql_gtid.go @@ -2,597 +2,437 @@ package mysql import ( "bytes" + "encoding" "encoding/binary" "fmt" - "io" + "log/slog" + "maps" "math" - "sort" - "strconv" + "regexp" + "slices" "strings" - "github.com/go-mysql-org/go-mysql/utils" "github.com/google/uuid" "github.com/pingcap/errors" ) -// Like MySQL GTID Interval struct, [start, stop), left closed and right open -// See MySQL rpl_gtid.h -type Interval struct { - // The first GID of this interval. - Start int64 - // The first GID after this interval. - Stop int64 -} - -// Interval is [start, stop), but the GTID string's format is [n] or [n1-n2], closed interval -func parseInterval(str string) (i Interval, err error) { - p := strings.Split(str, "-") - switch len(p) { - case 1: - i.Start, err = strconv.ParseInt(p[0], 10, 64) - i.Stop = i.Start + 1 - case 2: - i.Start, err = strconv.ParseInt(p[0], 10, 64) - if err == nil { - i.Stop, err = strconv.ParseInt(p[1], 10, 64) - i.Stop++ - } - default: - err = errors.Errorf("invalid interval format, must n[-n]") - } +// Note that MySQL normalized the value set by `SET GTID_NEXT='AUTOMATIC:` +// by: +// - Removing any length of leading and trailing whitespace (tabs, spaces). +// - Lowercasing the tag +var tagRegexp = regexp.MustCompile(`^\s*[a-zA-Z_][a-zA-Z0-9_]{0,31}\s*$`) - if err != nil { - return - } - - if i.Stop <= i.Start { - err = errors.Errorf("invalid interval format, must n[-n] and the end must >= start") - } +// Normalized tags should match: `^[a-z_][a-z0-9_]{0,31}$` - return +// Tag is a GTID Tag +type Tag struct { + normalized string } -func (i Interval) String() string { - if i.Stop == i.Start+1 { - return fmt.Sprintf("%d", i.Start) - } else { - return fmt.Sprintf("%d-%d", i.Start, i.Stop-1) - } -} - -type IntervalSlice []Interval - -func (s IntervalSlice) Len() int { - return len(s) -} +// This ensures that Tag implements the encoding.BinaryMarshaler and Stringer interface +var ( + _ encoding.BinaryMarshaler = Tag{} + _ fmt.Stringer = Tag{} +) -func (s IntervalSlice) Less(i, j int) bool { - if s[i].Start < s[j].Start { - return true - } else if s[i].Start > s[j].Start { - return false - } else { - return s[i].Stop < s[j].Stop +func (t Tag) MarshalBinary() ([]byte, error) { + if len(t.normalized) > 32 { + return nil, errors.New("tag length too long") } + tagLen := uint8(len(t.normalized) << 1) + return append([]byte{tagLen}, []byte(t.normalized)...), nil } -func (s IntervalSlice) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -func (s IntervalSlice) Sort() { - sort.Sort(s) +func (t Tag) String() string { + return t.normalized } -func (s IntervalSlice) Normalize() IntervalSlice { - var n IntervalSlice - if len(s) == 0 { - return n +// NewTag is taking a string and removes leading and trailing whitespace and changes the case to lowercase +func NewTag(str string) Tag { + if str == "" { + return Tag{} } - s.Sort() + return Tag{strings.TrimSpace(strings.ToLower(str))} +} - n = append(n, s[0]) +// MysqlGTIDSet is storing a map of SIDs (UUIDs), each with one or more tags. +// And each tag has one or more Intervals. +// +//nolint:revive // Can't use mysql.GTIDSet instead of mysql.MysqlGTIDSet as the former already exists and this is a specific implementation of that. +type MysqlGTIDSet map[uuid.UUID]map[Tag]IntervalSlice - for i := 1; i < len(s); i++ { - last := n[len(n)-1] - if s[i].Start > last.Stop { - n = append(n, s[i]) - continue - } else { - stop := s[i].Stop - if last.Stop > stop { - stop = last.Stop - } - n[len(n)-1] = Interval{last.Start, stop} - } - } - - return n -} +// This ensures that MysqlGTIDSet implements the GTIDSet interface +var _ GTIDSet = &MysqlGTIDSet{} -func min(a, b int64) int64 { - if a < b { - return a - } - return b +func NewMysqlGTIDSet() MysqlGTIDSet { + return make(map[uuid.UUID]map[Tag]IntervalSlice) } -func max(a, b int64) int64 { - if a > b { - return a +func DecodeMysqlGTIDSet(data []byte) (*MysqlGTIDSet, error) { + s := NewMysqlGTIDSet() + format, n, err := DecodeSid(data) + if err != nil { + return nil, err } - return b -} - -func (s *IntervalSlice) InsertInterval(interval Interval) { - var ( - count int - i int - ) - - *s = append(*s, interval) - total := len(*s) - for i = total - 1; i > 0; i-- { - if (*s)[i].Stop < (*s)[i-1].Start { - (*s)[i], (*s)[i-1] = (*s)[i-1], (*s)[i] - } else if (*s)[i].Start > (*s)[i-1].Stop { - break - } else { - (*s)[i-1].Start = min((*s)[i-1].Start, (*s)[i].Start) - (*s)[i-1].Stop = max((*s)[i-1].Stop, (*s)[i].Stop) - count++ + tag := Tag{} + pos := 8 + for range n { + if len(data) < pos+16 { + return nil, errors.Errorf("invalid gtid set buffer, expected %d or more but got %d", pos+16, len(data)) } - } - if count > 0 { - i++ - if i+count < total { - copy((*s)[i:], (*s)[i+count:]) + sid, err := uuid.FromBytes(data[pos : pos+16]) + if err != nil { + // This can't happen as uuid.FromBytes() only returns an error if the buffer is less than 16 bytes + // and we already check for that. + return nil, err } - *s = (*s)[:total-count] - } -} + pos += 16 -// Contain returns true if sub in s -func (s IntervalSlice) Contain(sub IntervalSlice) bool { - j := 0 - for i := 0; i < len(sub); i++ { - for ; j < len(s); j++ { - if sub[i].Start > s[j].Stop { - continue - } else { - break + if format == GtidFormatTagged { + if pos >= len(data) { + return nil, errors.New("invalid gtid set buffer, tag length expected") } - } - if j == len(s) { - return false - } + tagLen := int(data[pos] >> 1) + pos++ - if sub[i].Start < s[j].Start || sub[i].Stop > s[j].Stop { - return false + if pos+tagLen > len(data) { + return nil, errors.New("invalid gtid set buffer, tag extends beyond data") + } + tag = NewTag(string(data[pos : pos+tagLen])) + pos += tagLen } - } - return true -} - -func (s IntervalSlice) Equal(o IntervalSlice) bool { - if len(s) != len(o) { - return false - } - - for i := 0; i < len(s); i++ { - if s[i].Start != o[i].Start || s[i].Stop != o[i].Stop { - return false + if len(data) < pos+8 { + return nil, errors.Errorf("invalid gtid set buffer, expected %d or more but got %d", pos+8, len(data)) } - } - - return true -} - -func (s IntervalSlice) Compare(o IntervalSlice) int { - if s.Equal(o) { - return 0 - } else if s.Contain(o) { - return 1 - } else { - return -1 - } -} - -// Refer http://dev.mysql.com/doc/refman/5.6/en/replication-gtids-concepts.html -type UUIDSet struct { - SID uuid.UUID - - Intervals IntervalSlice -} - -func ParseUUIDSet(str string) (*UUIDSet, error) { - str = strings.TrimSpace(str) - sep := strings.Split(str, ":") - if len(sep) < 2 { - return nil, errors.Errorf("invalid GTID format, must UUID:interval[:interval]") - } - - var err error - s := new(UUIDSet) - if s.SID, err = uuid.Parse(sep[0]); err != nil { - return nil, errors.Trace(err) - } - - // Handle interval - for i := 1; i < len(sep); i++ { - if in, err := parseInterval(sep[i]); err != nil { - return nil, errors.Trace(err) - } else { - s.Intervals = append(s.Intervals, in) + intervalCount := binary.LittleEndian.Uint64(data[pos : pos+8]) + pos += 8 + if intervalCount == 0 { + return nil, errors.New("invalid gtid set buffer, got zero interval count") } - } - - s.Intervals = s.Intervals.Normalize() - - return s, nil -} - -func NewUUIDSet(sid uuid.UUID, in ...Interval) *UUIDSet { - s := new(UUIDSet) - s.SID = sid - - s.Intervals = in - s.Intervals = s.Intervals.Normalize() - - return s -} - -func (s *UUIDSet) Contain(sub *UUIDSet) bool { - if s.SID != sub.SID { - return false - } - - return s.Intervals.Contain(sub.Intervals) -} - -func (s *UUIDSet) Bytes() []byte { - var buf bytes.Buffer - - buf.WriteString(s.SID.String()) - - for _, i := range s.Intervals { - buf.WriteString(":") - buf.WriteString(i.String()) - } - - return buf.Bytes() -} - -func (s *UUIDSet) AddInterval(in IntervalSlice) { - s.Intervals = append(s.Intervals, in...) - s.Intervals = s.Intervals.Normalize() -} - -func (s *UUIDSet) MinusInterval(in IntervalSlice) { - var n IntervalSlice - in = in.Normalize() - - i, j := 0, 0 - var minuend Interval - var subtrahend Interval - for i < len(s.Intervals) { - if minuend.Stop != s.Intervals[i].Stop { // `i` changed? - minuend = s.Intervals[i] + if intervalCount > math.MaxInt/16 { // 16 = minimum interval size of start+stop (8+8) + return nil, errors.Errorf("invalid gtid set buffer, too many intervals: %d", intervalCount) } - if j < len(in) { - subtrahend = in[j] - } else { - subtrahend = Interval{math.MaxInt64, math.MaxInt64} + if len(data) < pos+(int(intervalCount)*16) { + return nil, errors.Errorf("invalid gtid set buffer, expected %d or more but got %d", pos+(int(intervalCount)*16), len(data)) } - - if minuend.Stop <= subtrahend.Start { - // no overlapping - n = append(n, minuend) - i++ - } else if minuend.Start >= subtrahend.Stop { - // no overlapping - j++ + intervals := make(IntervalSlice, 0, intervalCount) + for range intervalCount { + start := int64(binary.LittleEndian.Uint64(data[pos : pos+8])) + pos += 8 + stop := int64(binary.LittleEndian.Uint64(data[pos : pos+8])) + pos += 8 + + intervals = append(intervals, Interval{Start: start, Stop: stop}) + } + if _, ok := s[sid]; ok { + s[sid][tag] = append(s[sid][tag], intervals...) } else { - if minuend.Start < subtrahend.Start && minuend.Stop <= subtrahend.Stop { - n = append(n, Interval{minuend.Start, subtrahend.Start}) - i++ - } else if minuend.Start >= subtrahend.Start && minuend.Stop > subtrahend.Stop { - minuend = Interval{subtrahend.Stop, minuend.Stop} - j++ - } else if minuend.Start >= subtrahend.Start && minuend.Stop <= subtrahend.Stop { - // minuend is completely removed - i++ - } else if minuend.Start < subtrahend.Start && minuend.Stop > subtrahend.Stop { - n = append(n, Interval{minuend.Start, subtrahend.Start}) - minuend = Interval{subtrahend.Stop, minuend.Stop} - j++ - } else { - panic("should never be here") + s[sid] = map[Tag]IntervalSlice{ + tag: intervals, } } } - s.Intervals = n.Normalize() -} - -func (s *UUIDSet) String() string { - return utils.ByteSliceToString(s.Bytes()) -} - -func (s *UUIDSet) encode(w io.Writer) { - b, _ := s.SID.MarshalBinary() - - _, _ = w.Write(b) - n := int64(len(s.Intervals)) - - _ = binary.Write(w, binary.LittleEndian, n) - - for _, i := range s.Intervals { - _ = binary.Write(w, binary.LittleEndian, i.Start) - _ = binary.Write(w, binary.LittleEndian, i.Stop) - } -} - -func (s *UUIDSet) Encode() []byte { - var buf bytes.Buffer - - s.encode(&buf) - - return buf.Bytes() -} - -func (s *UUIDSet) decode(data []byte) (int, error) { - if len(data) < 24 { - return 0, errors.Errorf("invalid uuid set buffer, less 24") - } - - pos := 0 - var err error - if s.SID, err = uuid.FromBytes(data[0:16]); err != nil { - return 0, err - } - pos += 16 - - n := int64(binary.LittleEndian.Uint64(data[pos : pos+8])) - pos += 8 - if len(data) < int(16*n)+pos { - return 0, errors.Errorf("invalid uuid set buffer, must %d, but %d", pos+int(16*n), len(data)) + for sid := range s { + for tag := range s[sid] { + s[sid][tag] = s[sid][tag].Normalize() + } } - s.Intervals = make([]Interval, 0, n) - - var in Interval - for i := int64(0); i < n; i++ { - in.Start = int64(binary.LittleEndian.Uint64(data[pos : pos+8])) - pos += 8 - in.Stop = int64(binary.LittleEndian.Uint64(data[pos : pos+8])) - pos += 8 - s.Intervals = append(s.Intervals, in) + if pos < len(data) { + return &s, errors.Errorf("invalid gtid set buffer, found %d trailing bytes", len(data)-pos) } - - return pos, nil + return &s, nil } -func (s *UUIDSet) Decode(data []byte) error { - n, err := s.decode(data) - if n != len(data) { - return errors.Errorf("invalid uuid set buffer, must %d, but %d", n, len(data)) - } - return err -} - -func (s *UUIDSet) Clone() *UUIDSet { - clone := new(UUIDSet) - clone.SID = s.SID - clone.Intervals = make([]Interval, len(s.Intervals)) - copy(clone.Intervals, s.Intervals) - return clone -} - -type MysqlGTIDSet struct { - Sets map[string]*UUIDSet -} - -var _ GTIDSet = &MysqlGTIDSet{} - func ParseMysqlGTIDSet(str string) (GTIDSet, error) { - s := new(MysqlGTIDSet) - s.Sets = make(map[string]*UUIDSet) + s := NewMysqlGTIDSet() if str == "" { - return s, nil + return &s, nil } - sp := strings.Split(str, ",") + // Each sp has single UUID/SID, but might have multiple sets, each with a unique tag + sp := strings.SplitSeq(str, ",") - //todo, handle redundant same uuid - for i := 0; i < len(sp); i++ { - if set, err := ParseUUIDSet(sp[i]); err != nil { - return nil, errors.Trace(err) - } else { - s.AddSet(set) + for part := range sp { + // Handle UUID/SID + sep := strings.Split(strings.TrimSpace(part), ":") + if len(sep) < 2 { + return nil, errors.Errorf("invalid GTID format, must UUID[:tag]:interval[[:tag]:interval]") } - } - return s, nil -} - -func DecodeMysqlGTIDSet(data []byte) (*MysqlGTIDSet, error) { - s := new(MysqlGTIDSet) - if len(data) < 8 { - return nil, errors.Errorf("invalid gtid set buffer, less 4") - } - - n := int(binary.LittleEndian.Uint64(data)) - s.Sets = make(map[string]*UUIDSet, n) - - pos := 8 - - for i := 0; i < n; i++ { - set := new(UUIDSet) - if n, err := set.decode(data[pos:]); err != nil { + u, err := uuid.Parse(sep[0]) + if err != nil { return nil, errors.Trace(err) - } else { - pos += n + } - s.AddSet(set) + if _, ok := s[u]; !ok { + s[u] = make(map[Tag]IntervalSlice, 1) + } + // Handle interval(s) and tags + tag := Tag{} + for i := 1; i < len(sep); i++ { + if tagRegexp.MatchString(sep[i]) { + tag = NewTag(sep[i]) + if _, ok := s[u][tag]; !ok { + s[u][tag] = nil + } + } else { + in, err := parseInterval(sep[i]) + if err != nil { + return nil, errors.Trace(err) + } + s[u][tag] = append(s[u][tag], in) + } + } + for tag, val := range s[u] { + if val == nil { + return nil, errors.Errorf("invalid GTID format, missing interval for tag %s", tag) + } + s[u][tag] = val.Normalize() } } - return s, nil + return &s, nil } -func (s *MysqlGTIDSet) AddSet(set *UUIDSet) { - if set == nil { - return - } - sid := set.SID.String() - o, ok := s.Sets[sid] +func (s *MysqlGTIDSet) AddGTID(uuid uuid.UUID, gno int64) { + s.AddGTIDWithTag(uuid, Tag{}, gno) +} + +func (s *MysqlGTIDSet) AddGTIDWithTag(uuid uuid.UUID, tag Tag, gno int64) { + _, ok := (*s)[uuid] if ok { - o.AddInterval(set.Intervals) + (*s)[uuid][tag] = append((*s)[uuid][tag], Interval{gno, gno + 1}).Normalize() } else { - s.Sets[sid] = set + (*s)[uuid] = map[Tag]IntervalSlice{ + tag: { + Interval{gno, gno + 1}, + }, + } } } -func (s *MysqlGTIDSet) MinusSet(set *UUIDSet) { - if set == nil { - return - } - sid := set.SID.String() - uuidSet, ok := s.Sets[sid] - if ok { - uuidSet.MinusInterval(set.Intervals) - if uuidSet.Intervals == nil { - delete(s.Sets, sid) +func (s *MysqlGTIDSet) Clone() GTIDSet { + g := make(MysqlGTIDSet, len(*s)) + for k, v := range *s { + newInnerMap := make(map[Tag]IntervalSlice, len(v)) + for k2, v2 := range v { + newInnerMap[k2] = slices.Clone(v2) } + g[k] = newInnerMap } + return &g } -func (s *MysqlGTIDSet) Update(GTIDStr string) error { - gtidSet, err := ParseMysqlGTIDSet(GTIDStr) - if err != nil { - return err +func (s *MysqlGTIDSet) Contain(o GTIDSet) bool { + om, ok := o.(*MysqlGTIDSet) + if !ok { + return false } - for _, uuidSet := range gtidSet.(*MysqlGTIDSet).Sets { - s.AddSet(uuidSet) + for k := range *om { + if _, ok := (*s)[k]; !ok { + return false + } + for k2 := range (*om)[k] { + i, ok := (*s)[k][k2] + if !ok { + return false + } + if !i.Contain((*om)[k][k2]) { + return false + } + } } - return nil + return true } -func (s *MysqlGTIDSet) AddGTID(uuid uuid.UUID, gno int64) { - sid := uuid.String() - o, ok := s.Sets[sid] - if ok { - o.Intervals.InsertInterval(Interval{gno, gno + 1}) - } else { - s.Sets[sid] = &UUIDSet{uuid, IntervalSlice{Interval{gno, gno + 1}}} - } -} +func (s *MysqlGTIDSet) Encode() []byte { + var buf bytes.Buffer -func (s *MysqlGTIDSet) Add(addend MysqlGTIDSet) error { - for _, uuidSet := range addend.Sets { - s.AddSet(uuidSet) + format := GtidFormatClassic + sidCount := uint64(0) + uuids := make([]uuid.UUID, 0, len(*s)) + for uuid := range *s { + uuids = append(uuids, uuid) + for tag := range (*s)[uuid] { + sidCount++ + if format != GtidFormatTagged && (tag != Tag{}) { + format = GtidFormatTagged + } + } } - return nil -} -func (s *MysqlGTIDSet) Minus(subtrahend MysqlGTIDSet) error { - for _, uuidSet := range subtrahend.Sets { - s.MinusSet(uuidSet) - } - return nil -} + sid := encodeSid(format, sidCount) + buf.Write(sid) -func (s *MysqlGTIDSet) Contain(o GTIDSet) bool { - sub, ok := o.(*MysqlGTIDSet) - if !ok { - return false - } + slices.SortFunc(uuids, func(a, b uuid.UUID) int { + return bytes.Compare(a[:], b[:]) + }) + for _, uuid := range uuids { + tags := slices.Collect(maps.Keys((*s)[uuid])) + slices.SortFunc(tags, func(a, b Tag) int { return strings.Compare(a.normalized, b.normalized) }) - for key, set := range sub.Sets { - o, ok := s.Sets[key] - if !ok { - return false - } + for _, tag := range tags { + ubin, err := uuid.MarshalBinary() + if err != nil { + // should never happen + slog.Warn("encoding uuid failed", "error", err) + } + buf.Write(ubin) + + if format == GtidFormatTagged { + tbin, err := tag.MarshalBinary() + if err != nil { + slog.Warn("encoding tag failed", "error", err) + } + buf.Write(tbin) + } - if !o.Contain(set) { - return false + _ = binary.Write(&buf, binary.LittleEndian, uint64(len((*s)[uuid][tag]))) + for _, interval := range (*s)[uuid][tag] { + _ = binary.Write(&buf, binary.LittleEndian, interval.Start) + _ = binary.Write(&buf, binary.LittleEndian, interval.Stop) + } } } - return true + return buf.Bytes() } func (s *MysqlGTIDSet) Equal(o GTIDSet) bool { - sub, ok := o.(*MysqlGTIDSet) + om, ok := o.(*MysqlGTIDSet) if !ok { return false } - - if len(sub.Sets) != len(s.Sets) { + if len(*s) != len(*om) { return false } - - for key, set := range sub.Sets { - o, ok := s.Sets[key] - if !ok { + for u, sm := range *s { + omm, ok := (*om)[u] + if !ok || len(sm) != len(omm) { return false } - - if !o.Intervals.Equal(set.Intervals) { - return false + for k, i := range sm { + if !i.Equal(omm[k]) { + return false + } } } - return true } +func (s *MysqlGTIDSet) IsEmpty() bool { + return len(*s) == 0 +} + func (s *MysqlGTIDSet) String() string { - // there is only one element in gtid set - if len(s.Sets) == 1 { - for _, set := range s.Sets { - return set.String() + var sb strings.Builder + sep := "" + uuids := make([]uuid.UUID, 0, len(*s)) + for uuid := range *s { + uuids = append(uuids, uuid) + } + slices.SortFunc(uuids, func(a, b uuid.UUID) int { + return bytes.Compare(a[:], b[:]) + }) + for _, uuid := range uuids { + sb.WriteString(sep) + sb.WriteString(uuid.String()) + sep = "," + tags := make([]Tag, 0, len((*s)[uuid])) + for tag := range (*s)[uuid] { + tags = append(tags, tag) + } + // Tags are sorted, empty tag first + slices.SortFunc(tags, func(a, b Tag) int { return strings.Compare(a.normalized, b.normalized) }) + for _, tag := range tags { + if tag != (Tag{}) { + sb.WriteString(":") + sb.WriteString(tag.String()) + } + for _, interval := range (*s)[uuid][tag] { + sb.WriteString(":") + sb.WriteString(interval.String()) + } } } + return sb.String() +} - // sort multi set - var buf bytes.Buffer - sets := make([]string, 0, len(s.Sets)) - for _, set := range s.Sets { - sets = append(sets, set.String()) +func (s *MysqlGTIDSet) Update(GTIDStr string) error { + o, err := ParseMysqlGTIDSet(GTIDStr) + if err != nil { + return err } - sort.Strings(sets) - - sep := "" - for _, set := range sets { - buf.WriteString(sep) - buf.WriteString(set) - sep = "," + om, ok := o.(*MysqlGTIDSet) + if !ok { + // This can't happen as ParseMysqlGTIDSet() always returns a MysqlGTIDSet + return errors.New("incompatible GTID types") + } + for k, v := range *om { + if _, ok := (*s)[k]; ok { + for k2, v2 := range (*om)[k] { + if _, ok := (*s)[k][k2]; ok { + (*s)[k][k2] = append((*s)[k][k2], (*om)[k][k2]...).Normalize() + } else { + (*s)[k][k2] = v2 + } + } + } else { + (*s)[k] = v + } } - - return utils.ByteSliceToString(buf.Bytes()) + return nil } -func (s *MysqlGTIDSet) Encode() []byte { - var buf bytes.Buffer +// DecodeSid the number of sids (source identifiers) and if it is using +// tagged GTIDs or classic (non-tagged) GTIDs. +// +// Note that each gtid tag increases the sidnr here, so a single UUID +// might turn up multiple times if there are multiple tags. +// +// see also: +// decode_nsids_format in mysql/mysql-server +// https://github.com/mysql/mysql-server/blob/61a3a1d8ef15512396b4c2af46e922a19bf2b174/sql/rpl_gtid_set.cc#L1363-L1378 +func DecodeSid(data []byte) (format GtidFormat, sidnr uint64, err error) { + if len(data) < 8 { + return format, 0, errors.New("failed to decode source identifier, input too short") + } + if data[7] == 0x1 { + format = GtidFormatTagged + } - _ = binary.Write(&buf, binary.LittleEndian, uint64(len(s.Sets))) + if format == GtidFormatTagged { + masked := make([]byte, 8) + copy(masked, data[1:7]) + sidnr = binary.LittleEndian.Uint64(masked) + return format, sidnr, nil + } + sidnr = binary.LittleEndian.Uint64(data[:8]) + return format, sidnr, nil +} - for i := range s.Sets { - s.Sets[i].encode(&buf) +func encodeSid(format GtidFormat, sidnr uint64) []byte { + sid := make([]byte, 8) + if format == GtidFormatClassic { + _, _ = binary.Encode(sid, binary.LittleEndian, sidnr) + return sid } + _, _ = binary.Encode(sid, binary.LittleEndian, sidnr<<8) - return buf.Bytes() + sid[0] = 0x01 + sid[7] = 0x01 // Format marker + return sid } -func (gtid *MysqlGTIDSet) Clone() GTIDSet { - clone := &MysqlGTIDSet{ - Sets: make(map[string]*UUIDSet), - } - for sid, uuidSet := range gtid.Sets { - clone.Sets[sid] = uuidSet.Clone() +func (f GtidFormat) String() string { + switch f { + case GtidFormatClassic: + return "GtidFormatClassic" + case GtidFormatTagged: + return "GtidFormatTagged" } - - return clone + return fmt.Sprintf("GtidFormat{%d}", int(f)) } diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/mysql_interval.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/mysql_interval.go new file mode 100644 index 000000000..915ce6048 --- /dev/null +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/mysql_interval.go @@ -0,0 +1,130 @@ +package mysql + +import ( + "fmt" + "slices" + "sort" + "strconv" + "strings" + + "github.com/pingcap/errors" +) + +// Like MySQL GTID Interval struct, [start, stop), left closed and right open +// See MySQL rpl_gtid.h +type Interval struct { + // The start of this interval. + Start int64 + // The stop of this interval. + Stop int64 +} + +// Interval is [start, stop), but the GTID string's format is [n] or [n1-n2], closed interval +func parseInterval(str string) (i Interval, err error) { + p := strings.Split(str, "-") + + switch len(p) { + case 1: + i.Start, err = strconv.ParseInt(p[0], 10, 64) + if err != nil { + return i, errors.Errorf("invalid interval format for '%s', not numeric: %v ", p[0], err) + } + i.Stop = i.Start + 1 + case 2: + i.Start, err = strconv.ParseInt(p[0], 10, 64) + if err == nil { + i.Stop, err = strconv.ParseInt(p[1], 10, 64) + i.Stop++ + } + default: + err = errors.New("invalid interval format, must n[-n]") + } + + if err != nil { + return i, err + } + + if i.Stop <= i.Start { + err = errors.New("invalid interval format, must n[-n] and the end must >= start") + } + + return i, err +} + +func (i Interval) String() (s string) { + if i.Stop == i.Start+1 { + return fmt.Sprintf("%d", i.Start) + } + return fmt.Sprintf("%d-%d", i.Start, i.Stop-1) +} + +type IntervalSlice []Interval + +func (s IntervalSlice) Len() int { + return len(s) +} + +// Sort is sorting intervals. +func (s IntervalSlice) Sort() { + slices.SortFunc(s, func(a, b Interval) int { + if a.Start < b.Start { + return -1 + } else if a.Start > b.Start { + return 1 + } + if a.Stop < b.Stop { + return -1 + } else if a.Stop > b.Stop { + return 1 + } + return 0 + }) +} + +func (s IntervalSlice) Normalize() IntervalSlice { + if s == nil { + return nil + } + + if len(s) == 0 { + return s + } + n := make(IntervalSlice, 0, len(s)) + + s.Sort() + + n = append(n, s[0]) + + for i := 1; i < len(s); i++ { + last := n[len(n)-1] + if s[i].Start > last.Stop { + n = append(n, s[i]) + continue + } + stop := max(last.Stop, s[i].Stop) + n[len(n)-1] = Interval{last.Start, stop} + } + + return n +} + +// Contain returns true if sub in s. s must be sorted and normalized; sub may +// be in any order. +func (s IntervalSlice) Contain(sub IntervalSlice) bool { + for i := range sub { + j := sort.Search(len(s), func(j int) bool { + return sub[i].Start <= s[j].Stop + }) + if j == len(s) { + return false + } + if sub[i].Start < s[j].Start || sub[i].Stop > s[j].Stop { + return false + } + } + return true +} + +func (s IntervalSlice) Equal(o IntervalSlice) bool { + return slices.Equal(s, o) +} diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/parse_binary.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/parse_binary.go index 03b35ca97..1f07932c8 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/parse_binary.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/parse_binary.go @@ -8,6 +8,7 @@ import ( func ParseBinaryInt8(data []byte) int8 { return int8(data[0]) } + func ParseBinaryUint8(data []byte) uint8 { return data[0] } @@ -15,6 +16,7 @@ func ParseBinaryUint8(data []byte) uint8 { func ParseBinaryInt16(data []byte) int16 { return int16(binary.LittleEndian.Uint16(data)) } + func ParseBinaryUint16(data []byte) uint16 { return binary.LittleEndian.Uint16(data) } @@ -26,6 +28,7 @@ func ParseBinaryInt24(data []byte) int32 { } return int32(u32) } + func ParseBinaryUint24(data []byte) uint32 { return uint32(data[0]) | uint32(data[1])<<8 | uint32(data[2])<<16 } @@ -33,6 +36,7 @@ func ParseBinaryUint24(data []byte) uint32 { func ParseBinaryInt32(data []byte) int32 { return int32(binary.LittleEndian.Uint32(data)) } + func ParseBinaryUint32(data []byte) uint32 { return binary.LittleEndian.Uint32(data) } @@ -40,6 +44,7 @@ func ParseBinaryUint32(data []byte) uint32 { func ParseBinaryInt64(data []byte) int64 { return int64(binary.LittleEndian.Uint64(data)) } + func ParseBinaryUint64(data []byte) uint64 { return binary.LittleEndian.Uint64(data) } diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/position.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/position.go index 09bd95f70..eb2f7e868 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/position.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/position.go @@ -25,9 +25,8 @@ func (p Position) Compare(o Position) int { return 1 } else if p.Pos < o.Pos { return -1 - } else { - return 0 } + return 0 } func (p Position) String() string { @@ -81,7 +80,6 @@ func CompareBinlogFileName(a, b string) int { return 1 } else if aSeq < bSeq { return -1 - } else { - return 0 } + return 0 } diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/queryattributes.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/queryattributes.go new file mode 100644 index 000000000..205c01573 --- /dev/null +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/queryattributes.go @@ -0,0 +1,48 @@ +package mysql + +import ( + "encoding/binary" + "fmt" + "log/slog" +) + +// Query Attributes in MySQL are key/value pairs passed along with COM_QUERY or COM_STMT_EXECUTE +// +// Supported Value types: string, uint64 +// +// Resources: +// - https://dev.mysql.com/doc/refman/8.4/en/query-attributes.html +// - https://github.com/mysql/mysql-server/blob/trunk/include/mysql/components/services/mysql_query_attributes.h +// - https://archive.fosdem.org/2021/schedule/event/mysql_protocl/ +type QueryAttribute struct { + Name string + Value any +} + +// TypeAndFlag returns the type MySQL field type of the value and the field flag. +func (qa *QueryAttribute) TypeAndFlag() []byte { + switch v := qa.Value.(type) { + case string: + return []byte{MYSQL_TYPE_STRING, 0x0} + case uint64: + return []byte{MYSQL_TYPE_LONGLONG, PARAM_UNSIGNED} + default: + slog.Warn("query attribute with unsupported type", slog.String("type", fmt.Sprintf("%T", v))) + } + return []byte{0x0, 0x0} // type 0x0, flag 0x0, to not break the protocol +} + +// ValueBytes returns the encoded value +func (qa *QueryAttribute) ValueBytes() []byte { + switch v := qa.Value.(type) { + case string: + return PutLengthEncodedString([]byte(v)) + case uint64: + b := make([]byte, 8) + binary.LittleEndian.PutUint64(b, v) + return b + default: + slog.Warn("query attribute with unsupported type", slog.String("type", fmt.Sprintf("%T", v))) + } + return []byte{0x0} // 0 length value to not break the protocol +} diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/result.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/result.go index 71011c772..06a069991 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/result.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/result.go @@ -6,10 +6,24 @@ type Result struct { Status uint16 Warnings uint16 - InsertId uint64 + InsertId uint64 //nolint:revive // exported field renamed would be a breaking API change AffectedRows uint64 + StatusMessage string + SessionTracking *SessionTrackingInfo + *Resultset + + StreamResult *StreamResult +} + +type SessionTrackingInfo struct { + GTID string + TransactionState string + Variables map[string]string + Schema string + State string + Characteristics string } func NewResult(resultset *Resultset) *Result { @@ -25,12 +39,30 @@ func NewResultReserveResultset(fieldCount int) *Result { } type Executer interface { - Execute(query string, args ...interface{}) (*Result, error) + Execute(query string, args ...any) (*Result, error) } func (r *Result) Close() { if r.Resultset != nil { - r.Resultset.returnToPool() + r.returnToPool() r.Resultset = nil } + if r.StreamResult != nil { + r.StreamResult.Close() + r.StreamResult = nil + } +} + +func (r *Result) HasResultset() bool { + if r == nil { + return false + } + if r.Resultset != nil && len(r.Fields) > 0 { + return true + } + return false +} + +func (r *Result) IsStreaming() bool { + return r != nil && r.StreamResult != nil } diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/resultset.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/resultset.go index 2982dfaed..cfda84d58 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/resultset.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/resultset.go @@ -37,13 +37,11 @@ type Resultset struct { StreamingDone bool } -var ( - resultsetPool = sync.Pool{ - New: func() interface{} { - return &Resultset{} - }, - } -) +var resultsetPool = sync.Pool{ + New: func() any { + return &Resultset{} + }, +} func NewResultset(fieldsCount int) *Resultset { r := resultsetPool.Get().(*Resultset) @@ -89,7 +87,7 @@ func (r *Resultset) ColumnNumber() int { return len(r.Fields) } -func (r *Resultset) GetValue(row, column int) (interface{}, error) { +func (r *Resultset) GetValue(row, column int) (any, error) { if row >= len(r.Values) || row < 0 { return nil, errors.Errorf("invalid row index %d", row) } @@ -104,17 +102,16 @@ func (r *Resultset) GetValue(row, column int) (interface{}, error) { func (r *Resultset) NameIndex(name string) (int, error) { if column, ok := r.FieldNames[name]; ok { return column, nil - } else { - return 0, errors.Errorf("invalid field name %s", name) } + return 0, errors.Errorf("invalid field name %s", name) } -func (r *Resultset) GetValueByName(row int, name string) (interface{}, error) { - if column, err := r.NameIndex(name); err != nil { +func (r *Resultset) GetValueByName(row int, name string) (any, error) { + column, err := r.NameIndex(name) + if err != nil { return nil, errors.Trace(err) - } else { - return r.GetValue(row, column) } + return r.GetValue(row, column) } func (r *Resultset) IsNull(row, column int) (bool, error) { @@ -127,11 +124,11 @@ func (r *Resultset) IsNull(row, column int) (bool, error) { } func (r *Resultset) IsNullByName(row int, name string) (bool, error) { - if column, err := r.NameIndex(name); err != nil { + column, err := r.NameIndex(name) + if err != nil { return false, err - } else { - return r.IsNull(row, column) } + return r.IsNull(row, column) } func (r *Resultset) GetUint(row, column int) (uint64, error) { @@ -177,11 +174,11 @@ func (r *Resultset) GetUint(row, column int) (uint64, error) { } func (r *Resultset) GetUintByName(row int, name string) (uint64, error) { - if column, err := r.NameIndex(name); err != nil { + column, err := r.NameIndex(name) + if err != nil { return 0, err - } else { - return r.GetUint(row, column) } + return r.GetUint(row, column) } func (r *Resultset) GetInt(row, column int) (int64, error) { @@ -278,11 +275,11 @@ func (r *Resultset) GetFloat(row, column int) (float64, error) { } func (r *Resultset) GetFloatByName(row int, name string) (float64, error) { - if column, err := r.NameIndex(name); err != nil { + column, err := r.NameIndex(name) + if err != nil { return 0, err - } else { - return r.GetFloat(row, column) } + return r.GetFloat(row, column) } func (r *Resultset) GetString(row, column int) (string, error) { @@ -311,9 +308,9 @@ func (r *Resultset) GetString(row, column int) (string, error) { } func (r *Resultset) GetStringByName(row int, name string) (string, error) { - if column, err := r.NameIndex(name); err != nil { + column, err := r.NameIndex(name) + if err != nil { return "", err - } else { - return r.GetString(row, column) } + return r.GetString(row, column) } diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/resultset_helper.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/resultset_helper.go index cb94bcad6..b4fc39e0f 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/resultset_helper.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/resultset_helper.go @@ -12,7 +12,7 @@ import ( "github.com/go-mysql-org/go-mysql/utils" ) -func FormatTextValue(value interface{}) ([]byte, error) { +func FormatTextValue(value any) ([]byte, error) { switch v := value.(type) { case int8: return strconv.AppendInt(nil, int64(v), 10), nil @@ -59,7 +59,7 @@ func toBinaryDateTime(t time.Time) ([]byte, error) { } year, month, day := t.Year(), t.Month(), t.Day() - hour, min, sec := t.Hour(), t.Minute(), t.Second() + hour, minute, sec := t.Hour(), t.Minute(), t.Second() nanosec := t.Nanosecond() if nanosec > 0 { @@ -68,16 +68,16 @@ func toBinaryDateTime(t time.Time) ([]byte, error) { buf.WriteByte(byte(month)) buf.WriteByte(byte(day)) buf.WriteByte(byte(hour)) - buf.WriteByte(byte(min)) + buf.WriteByte(byte(minute)) buf.WriteByte(byte(sec)) _ = binary.Write(&buf, binary.LittleEndian, uint32(nanosec/1000)) - } else if hour > 0 || min > 0 || sec > 0 { + } else if hour > 0 || minute > 0 || sec > 0 { buf.WriteByte(byte(7)) _ = binary.Write(&buf, binary.LittleEndian, uint16(year)) buf.WriteByte(byte(month)) buf.WriteByte(byte(day)) buf.WriteByte(byte(hour)) - buf.WriteByte(byte(min)) + buf.WriteByte(byte(minute)) buf.WriteByte(byte(sec)) } else { buf.WriteByte(byte(4)) @@ -89,7 +89,8 @@ func toBinaryDateTime(t time.Time) ([]byte, error) { return buf.Bytes(), nil } -func formatBinaryValue(value interface{}) ([]byte, error) { +// FormatBinaryValue formats a value for binary protocol. +func FormatBinaryValue(value any) ([]byte, error) { switch v := value.(type) { case int8: return Uint64ToBytes(uint64(v)), nil @@ -126,7 +127,7 @@ func formatBinaryValue(value interface{}) ([]byte, error) { } } -func fieldType(value interface{}) (typ uint8, err error) { +func fieldType(value any) (typ uint8, err error) { switch value.(type) { case int8, int16, int32, int64, int: typ = MYSQL_TYPE_LONGLONG @@ -143,10 +144,10 @@ func fieldType(value interface{}) (typ uint8, err error) { default: err = errors.Errorf("unsupport type %T for resultset", value) } - return + return typ, err } -func formatField(field *Field, value interface{}) error { +func formatField(field *Field, value any) error { switch value.(type) { case int8, int16, int32, int64, int: field.Charset = 63 @@ -167,7 +168,7 @@ func formatField(field *Field, value interface{}) error { return nil } -func BuildSimpleTextResultset(names []string, values [][]interface{}) (*Resultset, error) { +func BuildSimpleTextResultset(names []string, values [][]any) (*Resultset, error) { r := NewResultset(len(names)) var b []byte @@ -212,7 +213,6 @@ func BuildSimpleTextResultset(names []string, values [][]interface{}) (*Resultse } } b, err = FormatTextValue(value) - if err != nil { return nil, errors.Trace(err) } @@ -231,7 +231,7 @@ func BuildSimpleTextResultset(names []string, values [][]interface{}) (*Resultse return r, nil } -func BuildSimpleBinaryResultset(names []string, values [][]interface{}) (*Resultset, error) { +func BuildSimpleBinaryResultset(names []string, values [][]any) (*Resultset, error) { r := NewResultset(len(names)) var b []byte @@ -268,8 +268,7 @@ func BuildSimpleBinaryResultset(names []string, values [][]interface{}) (*Result continue } - b, err = formatBinaryValue(value) - + b, err = FormatBinaryValue(value) if err != nil { return nil, errors.Trace(err) } @@ -289,10 +288,9 @@ func BuildSimpleBinaryResultset(names []string, values [][]interface{}) (*Result return r, nil } -func BuildSimpleResultset(names []string, values [][]interface{}, binary bool) (*Resultset, error) { +func BuildSimpleResultset(names []string, values [][]any, binary bool) (*Resultset, error) { if binary { return BuildSimpleBinaryResultset(names, values) - } else { - return BuildSimpleTextResultset(names, values) } + return BuildSimpleTextResultset(names, values) } diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/rowdata.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/rowdata.go index d55178f3b..e4213c136 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/rowdata.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/rowdata.go @@ -12,9 +12,8 @@ type RowData []byte func (p RowData) Parse(f []*Field, binary bool, dst []FieldValue) ([]FieldValue, error) { if binary { return p.ParseBinary(f, dst) - } else { - return p.ParseText(f, dst) } + return p.ParseText(f, dst) } func (p RowData) ParseText(f []*Field, dst []FieldValue) ([]FieldValue, error) { @@ -173,10 +172,10 @@ func (p RowData) ParseBinary(f []*Field, dst []FieldValue) ([]FieldValue, error) pos += 8 continue - case MYSQL_TYPE_DECIMAL, MYSQL_TYPE_NEWDECIMAL, MYSQL_TYPE_VARCHAR, - MYSQL_TYPE_BIT, MYSQL_TYPE_ENUM, MYSQL_TYPE_SET, MYSQL_TYPE_TINY_BLOB, - MYSQL_TYPE_MEDIUM_BLOB, MYSQL_TYPE_LONG_BLOB, MYSQL_TYPE_BLOB, - MYSQL_TYPE_VAR_STRING, MYSQL_TYPE_STRING, MYSQL_TYPE_GEOMETRY, MYSQL_TYPE_JSON: + case MYSQL_TYPE_DECIMAL, MYSQL_TYPE_NEWDECIMAL, MYSQL_TYPE_VARCHAR, MYSQL_TYPE_BIT, + MYSQL_TYPE_ENUM, MYSQL_TYPE_SET, MYSQL_TYPE_TINY_BLOB, MYSQL_TYPE_MEDIUM_BLOB, + MYSQL_TYPE_LONG_BLOB, MYSQL_TYPE_BLOB, MYSQL_TYPE_VAR_STRING, MYSQL_TYPE_STRING, + MYSQL_TYPE_VECTOR, MYSQL_TYPE_GEOMETRY, MYSQL_TYPE_JSON: v, isNull, n, err = LengthEncodedString(p[pos:]) pos += n if err != nil { @@ -187,10 +186,9 @@ func (p RowData) ParseBinary(f []*Field, dst []FieldValue) ([]FieldValue, error) data[i].Type = FieldValueTypeString data[i].str = append(data[i].str[:0], v...) continue - } else { - data[i].Type = FieldValueTypeNull - continue } + data[i].Type = FieldValueTypeNull + continue case MYSQL_TYPE_DATE, MYSQL_TYPE_NEWDATE: var num uint64 diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/state.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/state.go index 568d84b60..4e355c270 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/state.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/state.go @@ -1,5 +1,6 @@ package mysql +//nolint:revive // exported MySQL SQLSTATE constant kept for backward compatibility const ( DEFAULT_MYSQL_STATE = "HY000" ) diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/stream_result.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/stream_result.go new file mode 100644 index 000000000..cb64b4626 --- /dev/null +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/stream_result.go @@ -0,0 +1,135 @@ +package mysql + +import ( + "context" + "sync" +) + +// StreamResult provides a streaming interface for sending query results row by row. +// It is designed for scenarios where the result set is too large to fit in memory +// or when results need to be sent incrementally as they are generated. +// Usage pattern: +// +// sr := mysql.NewStreamResult(fields, bufSize, binary) +// go func() { +// defer sr.Close() // Always close when done +// for row := range rows { +// if !sr.WriteRow(ctx, row) { +// return // Context canceled, exit early +// } +// } +// }() +// err := conn.WriteValue(sr.AsResult()) +// +// IMPORTANT: The producer MUST NOT call WriteRow() after Close() has been called. +// Doing so will result in a panic (send on closed channel). Use "defer sr.Close()" +// to ensure Close() is called after all WriteRow() calls complete. +type StreamResult struct { + // Fields contains the column metadata for the result set. + Fields []*Field + // Binary indicates whether to use binary protocol (true) or text protocol (false). + Binary bool + // rowsChan is the internal channel for streaming rows. + rowsChan chan []any + // done is used to signal that the stream has been closed. + done chan struct{} + + // mu protects the close operation and err field. + mu sync.Mutex + // err stores any error that occurred during streaming. + err error +} + +// NewStreamResult creates a new StreamResult with the specified fields and buffer size. +// The bufSize parameter controls the channel buffer size. A bufSize of 0 creates an +// unbuffered channel for synchronous communication. +func NewStreamResult(fields []*Field, bufSize int, binary bool) *StreamResult { + return &StreamResult{ + Fields: fields, + rowsChan: make(chan []any, bufSize), + done: make(chan struct{}), + Binary: binary, + } +} + +// RowsChan returns a read-only channel for consuming rows. +// The channel is closed when Close() is called on the StreamResult. +// Consumers should range over this channel to receive all rows. +func (sr *StreamResult) RowsChan() <-chan []any { + return sr.rowsChan +} + +// WriteRow sends a row to the stream. It returns true if the row was successfully +// written, or false if the context is canceled. +// This method will block if the channel buffer is full. +// +// IMPORTANT: WriteRow MUST NOT be called after Close() has been called on the same +// StreamResult. Doing so will cause a panic. +func (sr *StreamResult) WriteRow(ctx context.Context, row []any) (ok bool) { + select { + case <-ctx.Done(): + return false + case <-sr.done: + return false + case sr.rowsChan <- row: + return true + } +} + +// SetError records an error that occurred during streaming. +// The consumer can retrieve this error by calling Err() after the channel is closed. +// This is useful for propagating errors from the producer to the consumer. +func (sr *StreamResult) SetError(err error) { + sr.mu.Lock() + sr.err = err + sr.mu.Unlock() +} + +// Err returns any error that was set via SetError(). +// This should typically be called after the RowsChan() channel is closed +// to check if the stream completed successfully. +func (sr *StreamResult) Err() error { + sr.mu.Lock() + defer sr.mu.Unlock() + return sr.err +} + +// Close closes the stream and signals to consumers that no more rows will be sent. +// This method is idempotent and safe to call multiple times. +// +// IMPORTANT: Close MUST only be called after all WriteRow() calls have completed. +// Calling WriteRow() after Close() will cause a panic. The recommended pattern is: +// +// go func() { +// defer sr.Close() +// // ... all WriteRow calls here ... +// }() +func (sr *StreamResult) Close() { + sr.mu.Lock() + defer sr.mu.Unlock() + + if sr.IsClosed() { + return + } + + close(sr.done) + close(sr.rowsChan) +} + +// IsClosed returns true if the stream has been closed. +func (sr *StreamResult) IsClosed() bool { + select { + case <-sr.done: + return true + default: + return false + } +} + +// AsResult wraps the StreamResult in a Result struct for use with the MySQL protocol. +// The returned Result will have IsStreaming() return true. +func (sr *StreamResult) AsResult() *Result { + return &Result{ + StreamResult: sr, + } +} diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/typed_bytes.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/typed_bytes.go new file mode 100644 index 000000000..109d99cbf --- /dev/null +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/typed_bytes.go @@ -0,0 +1,8 @@ +package mysql + +// TypedBytes preserves the original MySQL type alongside the raw bytes +// for binary protocol parameters that are length-encoded. +type TypedBytes struct { + Type byte // Original MySQL type + Bytes []byte // Raw bytes +} diff --git a/vendor/github.com/go-mysql-org/go-mysql/mysql/util.go b/vendor/github.com/go-mysql-org/go-mysql/mysql/util.go index c0c9f5452..ccbdcce88 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/mysql/util.go +++ b/vendor/github.com/go-mysql-org/go-mysql/mysql/util.go @@ -2,20 +2,24 @@ package mysql import ( "bytes" + "cmp" "compress/zlib" "crypto/rand" "crypto/rsa" "crypto/sha1" "crypto/sha256" + "crypto/sha512" + "crypto/subtle" "encoding/binary" + "encoding/hex" "fmt" "io" mrand "math/rand" "runtime" + "strconv" "strings" - "time" - "github.com/Masterminds/semver" + "filippo.io/edwards25519" "github.com/go-mysql-org/go-mysql/utils" "github.com/pingcap/errors" ) @@ -26,7 +30,7 @@ func Pstack() string { return string(buf[0:n]) } -func CalcPassword(scramble, password []byte) []byte { +func CalcNativePassword(scramble, password []byte) []byte { if len(password) == 0 { return nil } @@ -36,27 +40,92 @@ func CalcPassword(scramble, password []byte) []byte { crypt.Write(password) stage1 := crypt.Sum(nil) - // scrambleHash = SHA1(scramble + SHA1(stage1Hash)) - // inner Hash + // stage2Hash = SHA1(stage1Hash) crypt.Reset() crypt.Write(stage1) - hash := crypt.Sum(nil) + stage2 := crypt.Sum(nil) - // outer Hash + // scrambleHash = SHA1(scramble + stage2Hash) crypt.Reset() crypt.Write(scramble) - crypt.Write(hash) - scramble = crypt.Sum(nil) + crypt.Write(stage2) + scrambleHash := crypt.Sum(nil) // token = scrambleHash XOR stage1Hash - for i := range scramble { - scramble[i] ^= stage1[i] + return Xor(scrambleHash, stage1) +} + +// Xor returns a new slice with hash1 XOR hash2, wrapping hash2 if hash1 is longer. +func Xor(hash1 []byte, hash2 []byte) []byte { + result := make([]byte, len(hash1)) + for i := range hash1 { + result[i] = hash1[i] ^ hash2[i%len(hash2)] } - return scramble + return result +} + +// hash_stage1 = xor(reply, sha1(public_seed, hash_stage2)) +func stage1FromReply(scramble []byte, seed []byte, stage2 []byte) []byte { + crypt := sha1.New() + crypt.Write(seed) + crypt.Write(stage2) + seededHash := crypt.Sum(nil) + + return Xor(scramble, seededHash) +} + +// DecodePasswordHex decodes the standard format used by MySQL +// Password hashes in the 4.1 format always begin with a * character +// see https://dev.mysql.com/doc/mysql-security-excerpt/5.7/en/password-hashing.html +// ref vitess.io/vitess/go/mysql/auth_server.go +func DecodePasswordHex(hexEncodedPassword string) ([]byte, error) { + if hexEncodedPassword[0] == '*' { + hexEncodedPassword = hexEncodedPassword[1:] + } + return hex.DecodeString(hexEncodedPassword) +} + +// EncodePasswordHex encodes to the standard format used by MySQL +// adds the optionally leading * to the hashed password +func EncodePasswordHex(passwordHash []byte) string { + hexstr := strings.ToUpper(hex.EncodeToString(passwordHash)) + return "*" + hexstr +} + +// NativePasswordHash = sha1(sha1(password)) +func NativePasswordHash(password []byte) []byte { + if len(password) == 0 { + return nil + } + + // stage1Hash = SHA1(password) + crypt := sha1.New() + crypt.Write(password) + stage1 := crypt.Sum(nil) + + // stage2Hash = SHA1(stage1Hash) + crypt.Reset() + crypt.Write(stage1) + return crypt.Sum(stage1[:0]) +} + +func CompareNativePassword(reply []byte, stored []byte, seed []byte) bool { + if len(stored) == 0 { + return false + } + + // hash_stage1 = xor(reply, sha1(public_seed, hash_stage2)) + stage1 := stage1FromReply(reply, seed, stored) + // andidate_hash2 = sha1(hash_stage1) + stage2 := sha1.Sum(stage1) + + // check(candidate_hash2 == hash_stage2) + // use ConstantTimeCompare to mitigate timing based attacks + return subtle.ConstantTimeCompare(stage2[:], stored) == 1 } // CalcCachingSha2Password: Hash password using MySQL 8+ method (SHA256) -func CalcCachingSha2Password(scramble []byte, password string) []byte { +func CalcCachingSha2Password(scramble []byte, password []byte) []byte { if len(password) == 0 { return nil } @@ -64,7 +133,7 @@ func CalcCachingSha2Password(scramble []byte, password string) []byte { // XOR(SHA256(password), SHA256(SHA256(SHA256(password)), scramble)) crypt := sha256.New() - crypt.Write([]byte(password)) + crypt.Write(password) message1 := crypt.Sum(nil) crypt.Reset() @@ -76,11 +145,45 @@ func CalcCachingSha2Password(scramble []byte, password string) []byte { crypt.Write(scramble) message2 := crypt.Sum(nil) - for i := range message1 { - message1[i] ^= message2[i] + return Xor(message1, message2) +} + +// Taken from https://github.com/go-sql-driver/mysql/pull/1518 +func CalcEd25519Password(scramble []byte, password string) ([]byte, error) { + // Derived from https://github.com/MariaDB/server/blob/d8e6bb00888b1f82c031938f4c8ac5d97f6874c3/plugin/auth_ed25519/ref10/sign.c + // Code style is from https://cs.opensource.google/go/go/+/refs/tags/go1.21.5:src/crypto/ed25519/ed25519.go;l=207 + h := sha512.Sum512([]byte(password)) + + s, err := edwards25519.NewScalar().SetBytesWithClamping(h[:32]) + if err != nil { + return nil, err + } + A := (&edwards25519.Point{}).ScalarBaseMult(s) + + mh := sha512.New() + mh.Write(h[32:]) + mh.Write(scramble) + messageDigest := mh.Sum(nil) + r, err := edwards25519.NewScalar().SetUniformBytes(messageDigest) + if err != nil { + return nil, err } - return message1 + R := (&edwards25519.Point{}).ScalarBaseMult(r) + + kh := sha512.New() + kh.Write(R.Bytes()) + kh.Write(A.Bytes()) + kh.Write(scramble) + hramDigest := kh.Sum(nil) + k, err := edwards25519.NewScalar().SetUniformBytes(hramDigest) + if err != nil { + return nil, err + } + + S := k.MultiplyAdd(k, s, r) + + return append(R.Bytes(), S.Bytes()...), nil } func EncryptPassword(password string, seed []byte, pub *rsa.PublicKey) ([]byte, error) { @@ -94,6 +197,90 @@ func EncryptPassword(password string, seed []byte, pub *rsa.PublicKey) ([]byte, return rsa.EncryptOAEP(sha1v, rand.Reader, pub, plain, nil) } +//nolint:revive // exported auth constants kept for backward compatibility +const ( + SALT_LENGTH = 16 + ITERATION_MULTIPLIER = 1000 + SHA256_PASSWORD_ITERATIONS = 5 +) + +// generateUserSalt generate salt of given length for sha256_password hash +func generateUserSalt(length int) ([]byte, error) { + // Generate a random salt of the given length + // Implement this function for your project + salt := make([]byte, length) + _, err := rand.Read(salt) + if err != nil { + return []byte(""), err + } + + // Restrict to 7-bit to avoid multi-byte UTF-8 + for i := range salt { + salt[i] = salt[i] &^ 128 + for salt[i] == 36 || salt[i] == 0 { // '$' or NUL + newval := make([]byte, 1) + _, err := rand.Read(newval) + if err != nil { + return []byte(""), err + } + salt[i] = newval[0] &^ 128 + } + } + return salt, nil +} + +// hashCrypt256 salt and hash a password the given number of iterations +func hashCrypt256(source, salt string, iterations uint64) (string, error) { + actualIterations := iterations * ITERATION_MULTIPLIER + hashInput := []byte(source + salt) + var hash [32]byte + for range actualIterations { + hash = sha256.Sum256(hashInput) + hashInput = hash[:] + } + + hashHex := hex.EncodeToString(hash[:]) + digest := fmt.Sprintf("$%d$%s$%s", iterations, salt, hashHex) + return digest, nil +} + +// Check256HashingPassword compares a password to a hash for sha256_password +// rather than trying to recreate just the hash we recreate the full hash +// and use that for comparison +func Check256HashingPassword(pwhash []byte, password string) (bool, error) { + pwHashParts := bytes.Split(pwhash, []byte("$")) + if len(pwHashParts) != 4 { + return false, errors.New("failed to decode hash parts") + } + + iterationsPart := pwHashParts[1] + if len(iterationsPart) == 0 { + return false, errors.New("iterations part is empty") + } + + iterations, err := strconv.ParseUint(string(iterationsPart), 10, 64) + if err != nil { + return false, errors.New("failed to decode iterations") + } + salt := pwHashParts[2][:SALT_LENGTH] + + newHash, err := hashCrypt256(password, string(salt), iterations) + if err != nil { + return false, err + } + + return subtle.ConstantTimeCompare(pwhash, []byte(newHash)) == 1, nil +} + +// NewSha256PasswordHash creates a new password hash for sha256_password +func NewSha256PasswordHash(pwd string) (string, error) { + salt, err := generateUserSalt(SALT_LENGTH) + if err != nil { + return "", err + } + return hashCrypt256(pwd, string(salt), SHA256_PASSWORD_ITERATIONS) +} + func DecompressMariadbData(data []byte) ([]byte, error) { // algorithm always 0=zlib // algorithm := (data[pos] & 0x07) >> 4 @@ -130,20 +317,16 @@ func AppendLengthEncodedInteger(b []byte, n uint64) []byte { func RandomBuf(size int) []byte { buf := make([]byte, size) - // When this project supports golang 1.20 as a minimum, then this mrand.New(...) - // line can be eliminated and the random number can be generated by simply - // calling mrand.Intn() - random := mrand.New(mrand.NewSource(time.Now().UTC().UnixNano())) - min, max := 30, 127 - for i := 0; i < size; i++ { - buf[i] = byte(min + random.Intn(max-min)) + minVal, maxVal := 30, 127 + for i := range size { + buf[i] = byte(minVal + mrand.Intn(maxVal-minVal)) } return buf } // FixedLengthInt: little endian func FixedLengthInt(buf []byte) uint64 { - var num uint64 = 0 + var num uint64 for i, b := range buf { num |= uint64(b) << (uint(i) * 8) } @@ -152,7 +335,7 @@ func FixedLengthInt(buf []byte) uint64 { // BFixedLengthInt: big endian func BFixedLengthInt(buf []byte) uint64 { - var num uint64 = 0 + var num uint64 for i, b := range buf { num |= uint64(b) << (uint(len(buf)-i-1) * 8) } @@ -204,8 +387,10 @@ func PutLengthEncodedInt(n uint64) []byte { // handles case n <= 0xffffffffffffffff // using 'default' instead of 'case' to avoid static analysis error // SA4003: every value of type uint64 is <= math.MaxUint64 - return []byte{0xfe, byte(n), byte(n >> 8), byte(n >> 16), byte(n >> 24), - byte(n >> 32), byte(n >> 40), byte(n >> 48), byte(n >> 56)} + return []byte{ + 0xfe, byte(n), byte(n >> 8), byte(n >> 16), byte(n >> 24), + byte(n >> 32), byte(n >> 40), byte(n >> 48), byte(n >> 56), + } } } @@ -285,10 +470,10 @@ func FormatBinaryDate(n int, data []byte) ([]byte, error) { case 0: return []byte("0000-00-00"), nil case 4: - return []byte(fmt.Sprintf("%04d-%02d-%02d", + return fmt.Appendf(nil, "%04d-%02d-%02d", binary.LittleEndian.Uint16(data[:2]), data[2], - data[3])), nil + data[3]), nil default: return nil, errors.Errorf("invalid date packet length %d", n) } @@ -299,21 +484,21 @@ func FormatBinaryDateTime(n int, data []byte) ([]byte, error) { case 0: return []byte("0000-00-00 00:00:00"), nil case 4: - return []byte(fmt.Sprintf("%04d-%02d-%02d 00:00:00", + return fmt.Appendf(nil, "%04d-%02d-%02d 00:00:00", binary.LittleEndian.Uint16(data[:2]), data[2], - data[3])), nil + data[3]), nil case 7: - return []byte(fmt.Sprintf( + return fmt.Appendf(nil, "%04d-%02d-%02d %02d:%02d:%02d", binary.LittleEndian.Uint16(data[:2]), data[2], data[3], data[4], data[5], - data[6])), nil + data[6]), nil case 11: - return []byte(fmt.Sprintf( + return fmt.Appendf(nil, "%04d-%02d-%02d %02d:%02d:%02d.%06d", binary.LittleEndian.Uint16(data[:2]), data[2], @@ -321,7 +506,7 @@ func FormatBinaryDateTime(n int, data []byte) ([]byte, error) { data[4], data[5], data[6], - binary.LittleEndian.Uint32(data[7:11]))), nil + binary.LittleEndian.Uint32(data[7:11])), nil default: return nil, errors.Errorf("invalid datetime packet length %d", n) } @@ -340,22 +525,22 @@ func FormatBinaryTime(n int, data []byte) ([]byte, error) { var bytes []byte switch n { case 8: - bytes = []byte(fmt.Sprintf( + bytes = fmt.Appendf(nil, "%c%02d:%02d:%02d", sign, uint16(data[1])*24+uint16(data[5]), data[6], data[7], - )) + ) case 12: - bytes = []byte(fmt.Sprintf( + bytes = fmt.Appendf(nil, "%c%02d:%02d:%02d.%06d", sign, uint16(data[1])*24+uint16(data[5]), data[6], data[7], binary.LittleEndian.Uint32(data[8:12]), - )) + ) default: return nil, errors.Errorf("invalid time packet length %d", n) } @@ -389,9 +574,8 @@ func Escape(sql string) string { func GetNetProto(addr string) string { if strings.Contains(addr, "/") { return "unix" - } else { - return "tcp" } + return "tcp" } // ErrorEqual returns a boolean indicating whether err1 is equal to err2. @@ -410,21 +594,46 @@ func ErrorEqual(err1, err2 error) bool { return e1.Error() == e2.Error() } +func compareSubVersion(typ, a, b, aFull, bFull string) (int, error) { + if a == "" || b == "" { + return 0, nil + } + + var aNum, bNum int + var err error + + if aNum, err = strconv.Atoi(a); err != nil { + return 0, fmt.Errorf("cannot parse %s version %s of %s", typ, a, aFull) + } + if bNum, err = strconv.Atoi(b); err != nil { + return 0, fmt.Errorf("cannot parse %s version %s of %s", typ, b, bFull) + } + + return cmp.Compare(aNum, bNum), nil +} + +// Compares version triplet strings, ignoring anything past `-` in version. +// A version string like 8.0 will compare as if third triplet were a wildcard. +// A version string like 8 will compare as if second & third triplets were wildcards. func CompareServerVersions(a, b string) (int, error) { - var ( - aVer, bVer *semver.Version - err error - ) + aNumbers, _, _ := strings.Cut(a, "-") + bNumbers, _, _ := strings.Cut(b, "-") + + aMajor, aRest, _ := strings.Cut(aNumbers, ".") + bMajor, bRest, _ := strings.Cut(bNumbers, ".") - if aVer, err = semver.NewVersion(a); err != nil { - return 0, fmt.Errorf("cannot parse %q as semver: %w", a, err) + if majorCompare, err := compareSubVersion("major", aMajor, bMajor, a, b); err != nil || majorCompare != 0 { + return majorCompare, err } - if bVer, err = semver.NewVersion(b); err != nil { - return 0, fmt.Errorf("cannot parse %q as semver: %w", b, err) + aMinor, aPatch, _ := strings.Cut(aRest, ".") + bMinor, bPatch, _ := strings.Cut(bRest, ".") + + if minorCompare, err := compareSubVersion("minor", aMinor, bMinor, a, b); err != nil || minorCompare != 0 { + return minorCompare, err } - return aVer.Compare(bVer), nil + return compareSubVersion("patch", aPatch, bPatch, a, b) } var encodeRef = map[byte]byte{ @@ -443,9 +652,7 @@ func init() { for i := range EncodeMap { EncodeMap[i] = DONTESCAPE } - for i := range EncodeMap { - if to, ok := encodeRef[byte(i)]; ok { - EncodeMap[byte(i)] = to - } + for k, v := range encodeRef { + EncodeMap[k] = v } } diff --git a/vendor/github.com/go-mysql-org/go-mysql/packet/conn.go b/vendor/github.com/go-mysql-org/go-mysql/packet/conn.go index 1d49848f3..8a2eae6d8 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/packet/conn.go +++ b/vendor/github.com/go-mysql-org/go-mysql/packet/conn.go @@ -14,14 +14,16 @@ import ( "time" "github.com/go-mysql-org/go-mysql/compress" - . "github.com/go-mysql-org/go-mysql/mysql" + "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/utils" "github.com/klauspost/compress/zstd" "github.com/pingcap/errors" ) -const MinCompressionLength = 50 -const DefaultBufferSize = 16 * 1024 +const ( + MinCompressionLength = 50 + DefaultBufferSize = 16 * 1024 +) // Conn is the base class to handle MySQL protocol. type Conn struct { @@ -104,7 +106,7 @@ func (c *Conn) ReadPacketReuseMem(dst []byte) ([]byte, error) { utils.BytesBufferPut(buf) }() - if c.Compression != MYSQL_COMPRESS_NONE { + if c.Compression != mysql.MYSQL_COMPRESS_NONE { // it's possible that we're using compression but the server response with a compressed // packet with uncompressed length of 0. In this case we leave compressedReader nil. The // compressedReaderActive flag is important to track the state of the reader, allowing @@ -155,7 +157,7 @@ func (c *Conn) newCompressedPacketReader() (io.Reader, error) { } } if _, err := io.ReadFull(c.reader, c.compressedHeader[:7]); err != nil { - return nil, errors.Wrapf(ErrBadConn, "io.ReadFull(compressedHeader) failed. err %v", err) + return nil, errors.Wrapf(mysql.ErrBadConn, "io.ReadFull(compressedHeader) failed. err %v", err) } compressedSequence := c.compressedHeader[3] @@ -169,9 +171,9 @@ func (c *Conn) newCompressedPacketReader() (io.Reader, error) { if uncompressedLength > 0 { limitedReader := io.LimitReader(c.reader, int64(compressedLength)) switch c.Compression { - case MYSQL_COMPRESS_ZLIB: + case mysql.MYSQL_COMPRESS_ZLIB: return compress.GetPooledZlibReader(limitedReader) - case MYSQL_COMPRESS_ZSTD: + case mysql.MYSQL_COMPRESS_ZSTD: return zstd.NewReader(limitedReader) } } @@ -180,11 +182,10 @@ func (c *Conn) newCompressedPacketReader() (io.Reader, error) { } func (c *Conn) currentPacketReader() io.Reader { - if c.Compression == MYSQL_COMPRESS_NONE || c.compressedReader == nil { + if c.Compression == mysql.MYSQL_COMPRESS_NONE || c.compressedReader == nil { return c.reader - } else { - return c.compressedReader } + return c.compressedReader } func (c *Conn) copyN(dst io.Writer, n int64) (int64, error) { @@ -212,7 +213,7 @@ func (c *Conn) copyN(dst io.Writer, n int64) (int64, error) { // bytes are read. In this case, and when we have compression then advance // the sequence number and reset the compressed reader to continue reading // the remaining bytes in the next compressed packet. - if c.Compression != MYSQL_COMPRESS_NONE && + if c.Compression != mysql.MYSQL_COMPRESS_NONE && (goErrors.Is(err, io.ErrUnexpectedEOF) || goErrors.Is(err, io.EOF)) { // we have read to EOF and read an incomplete uncompressed packet // so advance the compressed sequence number and reset the compressed reader @@ -249,11 +250,10 @@ func (c *Conn) ReadPacketTo(w io.Writer) error { // buffer, since copyN is capable of getting the next compressed // packet and updating the Conn state with a new compressedReader. if _, err := c.copyN(b, 4); err != nil { - return errors.Wrapf(ErrBadConn, "io.ReadFull(header) failed. err %v", err) - } else { - // copy was successful so copy the 4 bytes from the buffer to the header - copy(c.header[:4], b.Bytes()[:4]) + return errors.Wrapf(mysql.ErrBadConn, "io.ReadFull(header) failed. err %v", err) } + // copy was successful so copy the 4 bytes from the buffer to the header + copy(c.header[:4], b.Bytes()[:4]) length := int(uint32(c.header[0]) | uint32(c.header[1])<<8 | uint32(c.header[2])<<16) sequence := c.header[3] @@ -270,17 +270,16 @@ func (c *Conn) ReadPacketTo(w io.Writer) error { } if n, err := c.copyN(w, int64(length)); err != nil { - return errors.Wrapf(ErrBadConn, "io.CopyN failed. err %v, copied %v, expected %v", err, n, length) + return errors.Wrapf(mysql.ErrBadConn, "io.CopyN failed. err %v, copied %v, expected %v", err, n, length) } else if n != int64(length) { - return errors.Wrapf(ErrBadConn, "io.CopyN failed(n != int64(length)). %v bytes copied, while %v expected", n, length) - } else { - if length < MaxPayloadLen { - return nil - } + return errors.Wrapf(mysql.ErrBadConn, "io.CopyN failed(n != int64(length)). %v bytes copied, while %v expected", n, length) + } + if length < mysql.MaxPayloadLen { + return nil + } - if err = c.ReadPacketTo(w); err != nil { - return errors.Wrap(err, "ReadPacketTo failed") - } + if err := c.ReadPacketTo(w); err != nil { + return errors.Wrap(err, "ReadPacketTo failed") } return nil @@ -290,22 +289,23 @@ func (c *Conn) ReadPacketTo(w io.Writer) error { func (c *Conn) WritePacket(data []byte) error { length := len(data) - 4 - for length >= MaxPayloadLen { + for length >= mysql.MaxPayloadLen { data[0] = 0xff data[1] = 0xff data[2] = 0xff data[3] = c.Sequence - if n, err := c.writeWithTimeout(data[:4+MaxPayloadLen]); err != nil { - return errors.Wrapf(ErrBadConn, "Write(payload portion) failed. err %v", err) - } else if n != (4 + MaxPayloadLen) { - return errors.Wrapf(ErrBadConn, "Write(payload portion) failed. only %v bytes written, while %v expected", n, 4+MaxPayloadLen) - } else { - c.Sequence++ - length -= MaxPayloadLen - data = data[MaxPayloadLen:] + if n, err := c.writeWithTimeout(data[:4+mysql.MaxPayloadLen]); err != nil { + return errors.Wrapf(mysql.ErrBadConn, + "Write(payload portion) failed. err %v", err) + } else if n != (4 + mysql.MaxPayloadLen) { + return errors.Wrapf(mysql.ErrBadConn, + "Write(payload portion) failed. only %v bytes written, while %v expected", n, 4+mysql.MaxPayloadLen) } + c.Sequence++ + length -= mysql.MaxPayloadLen + data = data[mysql.MaxPayloadLen:] } data[0] = byte(length) @@ -314,17 +314,17 @@ func (c *Conn) WritePacket(data []byte) error { data[3] = c.Sequence switch c.Compression { - case MYSQL_COMPRESS_NONE: + case mysql.MYSQL_COMPRESS_NONE: if n, err := c.writeWithTimeout(data); err != nil { - return errors.Wrapf(ErrBadConn, "Write failed. err %v", err) + return errors.Wrapf(mysql.ErrBadConn, "Write failed. err %v", err) } else if n != len(data) { - return errors.Wrapf(ErrBadConn, "Write failed. only %v bytes written, while %v expected", n, len(data)) + return errors.Wrapf(mysql.ErrBadConn, "Write failed. only %v bytes written, while %v expected", n, len(data)) } - case MYSQL_COMPRESS_ZLIB, MYSQL_COMPRESS_ZSTD: + case mysql.MYSQL_COMPRESS_ZLIB, mysql.MYSQL_COMPRESS_ZSTD: if n, err := c.writeCompressed(data); err != nil { - return errors.Wrapf(ErrBadConn, "Write failed. err %v", err) + return errors.Wrapf(mysql.ErrBadConn, "Write failed. err %v", err) } else if n != len(data) { - return errors.Wrapf(ErrBadConn, "Write failed. only %v bytes written, while %v expected", n, len(data)) + return errors.Wrapf(mysql.ErrBadConn, "Write failed. only %v bytes written, while %v expected", n, len(data)) } c.compressedReaderActive = false @@ -335,7 +335,7 @@ func (c *Conn) WritePacket(data []byte) error { c.compressedReader = nil } default: - return errors.Wrapf(ErrBadConn, "Write failed. Unsuppored compression algorithm set") + return errors.Wrapf(mysql.ErrBadConn, "Write failed. Unsuppored compression algorithm set") } c.Sequence++ @@ -356,7 +356,7 @@ func (c *Conn) writeCompressed(data []byte) (n int, err error) { var ( compressedLength, uncompressedLength int payload *bytes.Buffer - compressedHeader = make([]byte, 7) + compressedHeader [7]byte ) if len(data) > MinCompressionLength { @@ -365,12 +365,12 @@ func (c *Conn) writeCompressed(data []byte) (n int, err error) { defer utils.BytesBufferPut(payload) switch c.Compression { - case MYSQL_COMPRESS_ZLIB: + case mysql.MYSQL_COMPRESS_ZLIB: w, err = compress.GetPooledZlibWriter(payload) - case MYSQL_COMPRESS_ZSTD: + case mysql.MYSQL_COMPRESS_ZSTD: w, err = zstd.NewWriter(payload) default: - return 0, errors.Wrapf(ErrBadConn, "Write failed. Unsuppored compression algorithm set") + return 0, errors.Wrapf(mysql.ErrBadConn, "Write failed. Unsuppored compression algorithm set") } if err != nil { return 0, err @@ -402,7 +402,7 @@ func (c *Conn) writeCompressed(data []byte) (n int, err error) { compressedHeader[4] = byte(uncompressedLength) compressedHeader[5] = byte(uncompressedLength >> 8) compressedHeader[6] = byte(uncompressedLength >> 16) - if _, err = compressedPacket.Write(compressedHeader); err != nil { + if _, err = compressedPacket.Write(compressedHeader[:]); err != nil { return 0, err } c.CompressedSequence++ @@ -472,7 +472,7 @@ func (c *Conn) WritePublicKeyAuthPacket(password string, cipher []byte) error { } func (c *Conn) WriteEncryptedPassword(password string, seed []byte, pub *rsa.PublicKey) error { - enc, err := EncryptPassword(password, seed, pub) + enc, err := mysql.EncryptPassword(password, seed, pub) if err != nil { return errors.Wrap(err, "EncryptPassword failed") } diff --git a/vendor/github.com/go-mysql-org/go-mysql/replication/backup.go b/vendor/github.com/go-mysql-org/go-mysql/replication/backup.go index f1d7dd28b..2bec2a271 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/replication/backup.go +++ b/vendor/github.com/go-mysql-org/go-mysql/replication/backup.go @@ -8,23 +8,35 @@ import ( "sync" "time" - . "github.com/go-mysql-org/go-mysql/mysql" + "github.com/go-mysql-org/go-mysql/mysql" "github.com/pingcap/errors" ) // StartBackup starts the backup process for the binary log and writes to the backup directory. -func (b *BinlogSyncer) StartBackup(backupDir string, p Position, timeout time.Duration) error { - err := os.MkdirAll(backupDir, 0755) +func (b *BinlogSyncer) StartBackup(backupDir string, p mysql.Position, timeout time.Duration) error { + err := os.MkdirAll(backupDir, 0o755) if err != nil { return errors.Trace(err) } if b.cfg.SynchronousEventHandler == nil { return b.StartBackupWithHandler(p, timeout, func(filename string) (io.WriteCloser, error) { - return os.OpenFile(path.Join(backupDir, filename), os.O_CREATE|os.O_WRONLY, 0644) + return os.OpenFile(path.Join(backupDir, filename), os.O_CREATE|os.O_WRONLY, 0o644) }) - } else { - return b.StartSynchronousBackup(p, timeout) } + return b.StartSynchronousBackup(p, timeout) +} + +func (b *BinlogSyncer) StartBackupGTID(backupDir string, gset mysql.GTIDSet, timeout time.Duration) error { + err := os.MkdirAll(backupDir, 0o755) + if err != nil { + return errors.Trace(err) + } + if b.cfg.SynchronousEventHandler == nil { + return b.StartBackupWithHandlerAndGTID(gset, timeout, func(filename string) (io.WriteCloser, error) { + return os.OpenFile(path.Join(backupDir, filename), os.O_CREATE|os.O_WRONLY, 0o644) + }) + } + return b.StartSynchronousBackupWithGTID(gset, timeout) } // StartBackupWithHandler starts the backup process for the binary log using the specified position and handler. @@ -36,13 +48,15 @@ func (b *BinlogSyncer) StartBackup(backupDir string, p Position, timeout time.Du // - timeout: The maximum duration to wait for new binlog events before stopping the backup process. // If set to 0, a default very long timeout (30 days) is used instead. // - handler: A function that takes a binlog filename and returns an WriteCloser for writing raw events to. -func (b *BinlogSyncer) StartBackupWithHandler(p Position, timeout time.Duration, - handler func(binlogFilename string) (io.WriteCloser, error)) (retErr error) { +func (b *BinlogSyncer) StartBackupWithHandler(p mysql.Position, timeout time.Duration, + handler func(binlogFilename string) (io.WriteCloser, error), +) (retErr error) { if timeout == 0 { // a very long timeout here timeout = 30 * 3600 * 24 * time.Second } if b.cfg.SynchronousEventHandler != nil { + //nolint:revive // leading identifier is a Go function name return errors.New("StartBackupWithHandler cannot be used when SynchronousEventHandler is set. Use StartSynchronousBackup instead.") } @@ -53,52 +67,73 @@ func (b *BinlogSyncer) StartBackupWithHandler(p Position, timeout time.Duration, backupHandler := &BackupEventHandler{ handler: handler, } - s, err := b.StartSync(p) if err != nil { return errors.Trace(err) } + return processWithHandler(b, s, backupHandler, timeout) +} - defer func() { - if backupHandler.w != nil { - closeErr := backupHandler.w.Close() - if retErr == nil { - retErr = closeErr - } - } - }() +// StartBackupWithHandlerAndGTID starts the backup process for the binary log using the specified GTID set and handler. +// - gset: The GTID set from which to begin the backup. +// - timeout: The maximum duration to wait for new binlog events before stopping the backup process. +// If set to 0, a default very long timeout (30 days) is used instead. +// - handler: A function that takes a binlog filename and returns an WriteCloser for writing raw events to. +func (b *BinlogSyncer) StartBackupWithHandlerAndGTID(gset mysql.GTIDSet, timeout time.Duration, + handler func(binlogFilename string) (io.WriteCloser, error), +) (retErr error) { + if timeout == 0 { + // a very long timeout here + timeout = 30 * 3600 * 24 * time.Second + } + if b.cfg.SynchronousEventHandler != nil { + //nolint:revive // leading identifier is a Go function name + return errors.New("StartBackupWithHandlerAndGTID cannot be used when SynchronousEventHandler is set. Use StartSynchronousBackupWithGTID instead.") + } - for { - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() + // Force use raw mode + b.parser.SetRawMode(true) - select { - case <-ctx.Done(): - return nil - case <-b.ctx.Done(): - return nil - case err := <-s.ech: - return errors.Trace(err) - case e := <-s.ch: - err = backupHandler.HandleEvent(e) - if err != nil { - return errors.Trace(err) - } - } + // Set up the backup event handler + backupHandler := &BackupEventHandler{ + handler: handler, } + + s, err := b.StartSyncGTID(gset) + if err != nil { + return errors.Trace(err) + } + return processWithHandler(b, s, backupHandler, timeout) } // StartSynchronousBackup starts the backup process using the SynchronousEventHandler in the BinlogSyncerConfig. -func (b *BinlogSyncer) StartSynchronousBackup(p Position, timeout time.Duration) error { +func (b *BinlogSyncer) StartSynchronousBackup(p mysql.Position, timeout time.Duration) error { if b.cfg.SynchronousEventHandler == nil { return errors.New("SynchronousEventHandler must be set in BinlogSyncerConfig to use StartSynchronousBackup") } - s, err := b.StartSync(p) if err != nil { return errors.Trace(err) } + return process(b, s, timeout) +} + +// StartSynchronousBackupWithGTID starts the backup process using the SynchronousEventHandler in the BinlogSyncerConfig with a specified GTID set. +func (b *BinlogSyncer) StartSynchronousBackupWithGTID(gset mysql.GTIDSet, timeout time.Duration) error { + if b.cfg.SynchronousEventHandler == nil { + return errors.New("SynchronousEventHandler must be set in BinlogSyncerConfig to use StartSynchronousBackupWithGTID") + } + + s, err := b.StartSyncGTID(gset) + if err != nil { + return errors.Trace(err) + } + + return process(b, s, timeout) +} + +func process(b *BinlogSyncer, s *BinlogStreamer, timeout time.Duration) error { var ctx context.Context var cancel context.CancelFunc @@ -122,6 +157,35 @@ func (b *BinlogSyncer) StartSynchronousBackup(p Position, timeout time.Duration) } } +func processWithHandler(b *BinlogSyncer, s *BinlogStreamer, backupHandler *BackupEventHandler, timeout time.Duration) (retErr error) { + defer func() { + if backupHandler.w != nil { + closeErr := backupHandler.w.Close() + if retErr == nil { + retErr = closeErr + } + } + }() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + for { + select { + case <-ctx.Done(): + return nil + case <-b.ctx.Done(): + return nil + case err := <-s.ech: + return errors.Trace(err) + case e := <-s.ch: + err := backupHandler.HandleEvent(e) + if err != nil { + return errors.Trace(err) + } + } + } +} + // BackupEventHandler handles writing events for backup type BackupEventHandler struct { handler func(binlogFilename string) (io.WriteCloser, error) @@ -145,14 +209,15 @@ func (h *BackupEventHandler) HandleEvent(e *BinlogEvent) error { var err error offset := e.Header.LogPos - if e.Header.EventType == ROTATE_EVENT { + switch e.Header.EventType { + case ROTATE_EVENT: rotateEvent := e.Event.(*RotateEvent) h.filename = string(rotateEvent.NextLogName) if e.Header.Timestamp == 0 || offset == 0 { // fake rotate event return nil } - } else if e.Header.EventType == FORMAT_DESCRIPTION_EVENT { + case FORMAT_DESCRIPTION_EVENT: if h.w != nil { if err = h.w.Close(); err != nil { h.w = nil diff --git a/vendor/github.com/go-mysql-org/go-mysql/replication/binlogstreamer.go b/vendor/github.com/go-mysql-org/go-mysql/replication/binlogstreamer.go index 5214e92dd..e7ef127bb 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/replication/binlogstreamer.go +++ b/vendor/github.com/go-mysql-org/go-mysql/replication/binlogstreamer.go @@ -5,7 +5,6 @@ import ( "time" "github.com/pingcap/errors" - "github.com/siddontang/go-log/log" ) var ( @@ -74,8 +73,6 @@ func (s *BinlogStreamer) close() { func (s *BinlogStreamer) closeWithError(err error) { if err == nil { err = ErrSyncClosed - } else { - log.Errorf("close sync with err: %v", err) } select { diff --git a/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go b/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go index 41a02e8d0..6cdf4e586 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go +++ b/vendor/github.com/go-mysql-org/go-mysql/replication/binlogsyncer.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "encoding/binary" "fmt" + "log/slog" "net" "os" "strconv" @@ -14,17 +15,13 @@ import ( "github.com/google/uuid" "github.com/pingcap/errors" - "github.com/siddontang/go-log/log" - "github.com/siddontang/go-log/loggers" "github.com/go-mysql-org/go-mysql/client" - . "github.com/go-mysql-org/go-mysql/mysql" + "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/utils" ) -var ( - errSyncRunning = errors.New("Sync is running, must Close first") -) +var errSyncRunning = errors.New("Sync is running, must Close first") // BinlogSyncerConfig is the configuration for BinlogSyncer. type BinlogSyncerConfig struct { @@ -79,6 +76,9 @@ type BinlogSyncerConfig struct { // Use decimal.Decimal structure for decimals. UseDecimal bool + // FloatWithTrailingZero structure for floats. + UseFloatWithTrailingZero bool + // RecvBufferSize sets the size in bytes of the operating system's receive buffer associated with the connection. RecvBufferSize int @@ -110,12 +110,12 @@ type BinlogSyncerConfig struct { // https://mariadb.com/kb/en/library/annotate_rows_event/ DumpCommandFlag uint16 - //Option function is used to set outside of BinlogSyncerConfig， between mysql connection and COM_REGISTER_SLAVE - //For MariaDB: slave_gtid_ignore_duplicates、skip_replication、slave_until_gtid + // Option function is used to set outside of BinlogSyncerConfig， between mysql connection and COM_REGISTER_SLAVE + // For MariaDB: slave_gtid_ignore_duplicates、skip_replication、slave_until_gtid Option func(*client.Conn) error // Set Logger - Logger loggers.Advanced + Logger *slog.Logger // Set Dialer Dialer client.Dialer @@ -128,6 +128,16 @@ type BinlogSyncerConfig struct { EventCacheCount int + // FillZeroLogPos enables dynamic LogPos calculation for MariaDB. + // When enabled, automatically adds BINLOG_SEND_ANNOTATE_ROWS_EVENT flag + // to ensure correct position calculation in MariaDB 11.4+. + // Only works with MariaDB flavor. + FillZeroLogPos bool + + // PayloadDecoderConcurrency is used to control concurrency for decoding TransactionPayloadEvent. + // Default 0, this will be set to GOMAXPROCS. + PayloadDecoderConcurrency int + // SynchronousEventHandler is used for synchronous event handling. // This should not be used together with StartBackupWithHandler. // If this is not nil, GetEvent does not need to be called. @@ -151,9 +161,9 @@ type BinlogSyncer struct { parser *BinlogParser - nextPos Position + nextPos mysql.Position - prevGset, currGset GTIDSet + prevGset, currGset mysql.GTIDSet // instead of GTIDSet.Clone, use this to speed up calculate prevGset prevMySQLGTIDEvent *GTIDEvent @@ -171,11 +181,11 @@ type BinlogSyncer struct { // NewBinlogSyncer creates the BinlogSyncer with the given configuration. func NewBinlogSyncer(cfg BinlogSyncerConfig) *BinlogSyncer { if cfg.Logger == nil { - streamHandler, _ := log.NewStreamHandler(os.Stdout) - cfg.Logger = log.NewDefault(streamHandler) + cfg.Logger = slog.Default() } if cfg.ServerID == 0 { - cfg.Logger.Fatal("can't use 0 as the server ID") + cfg.Logger.Error("can't use 0 as the server ID, will panic") + panic("can't use 0 as the server ID") } if cfg.Dialer == nil { dialer := &net.Dialer{} @@ -188,7 +198,7 @@ func NewBinlogSyncer(cfg BinlogSyncerConfig) *BinlogSyncer { // Clear the Password to avoid outputting it in logs. pass := cfg.Password cfg.Password = "" - cfg.Logger.Infof("create BinlogSyncer with config %+v", cfg) + cfg.Logger.Info("create BinlogSyncer", slog.Any("config", cfg)) cfg.Password = pass b := new(BinlogSyncer) @@ -200,7 +210,9 @@ func NewBinlogSyncer(cfg BinlogSyncerConfig) *BinlogSyncer { b.parser.SetParseTime(b.cfg.ParseTime) b.parser.SetTimestampStringLocation(b.cfg.TimestampStringLocation) b.parser.SetUseDecimal(b.cfg.UseDecimal) + b.parser.SetUseFloatWithTrailingZero(b.cfg.UseFloatWithTrailingZero) b.parser.SetVerifyChecksum(b.cfg.VerifyChecksum) + b.parser.SetPayloadDecoderConcurrency(cfg.PayloadDecoderConcurrency) b.parser.SetRowsEventDecodeFunc(b.cfg.RowsEventDecodeFunc) b.parser.SetTableMapOptionalMetaDecodeFunc(b.cfg.TableMapOptionalMetaDecodeFunc) b.running = false @@ -228,9 +240,17 @@ func (b *BinlogSyncer) close() { b.cancel() if b.c != nil { - err := b.c.SetReadDeadline(utils.Now().Add(100 * time.Millisecond)) - if err != nil { - b.cfg.Logger.Warnf(`could not set read deadline: %s`, err) + // SetReadDeadline unblocks ReadPacket so onStream notices ctx cancellation. + // Some transports (notably SSH tunnels) refuse deadlines with an error, in + // which case close the underlying connection to force ReadPacket to return. + // Otherwise wg.Wait below parks forever when KILL also fails to reach server + // (e.g. thread already reaped, "ERROR 1094 unknown thread id") + if err := b.c.SetReadDeadline(utils.Now().Add(100 * time.Millisecond)); err != nil { + b.cfg.Logger.Warn("could not set read deadline, closing connection to unblock reader", + slog.Any("error", err)) + if err := b.c.Close(); err != nil { + b.cfg.Logger.Warn("could not close connection", slog.Any("error", err)) + } } } @@ -287,7 +307,7 @@ func (b *BinlogSyncer) registerSlave() error { } } - //set read timeout + // set read timeout if b.cfg.ReadTimeout > 0 { _ = b.c.SetReadDeadline(utils.Now().Add(b.cfg.ReadTimeout)) } @@ -306,32 +326,28 @@ func (b *BinlogSyncer) registerSlave() error { // save last last connection id for kill b.lastConnectionID = b.c.GetConnectionID() - //for mysql 5.6+, binlog has a crc32 checksum - //before mysql 5.6, this will not work, don't matter.:-) - if r, err := b.c.Execute("SHOW GLOBAL VARIABLES LIKE 'BINLOG_CHECKSUM'"); err != nil { + // for mysql 5.6+, binlog has a crc32 checksum + // before mysql 5.6, this will not work, don't matter.:-) + r, err := b.c.Execute("SHOW GLOBAL VARIABLES LIKE 'BINLOG_CHECKSUM'") + if err != nil { return errors.Trace(err) - } else { - s, _ := r.GetString(0, 1) - if s != "" { - // maybe CRC32 or NONE - - // mysqlbinlog.cc use NONE, see its below comments: - // Make a notice to the server that this client - // is checksum-aware. It does not need the first fake Rotate - // necessary checksummed. - // That preference is specified below. + } + s, _ := r.GetString(0, 1) + if s != "" { + // maybe CRC32 or NONE - if _, err = b.c.Execute(`SET @master_binlog_checksum='NONE'`); err != nil { - return errors.Trace(err) - } + // mysqlbinlog.cc use NONE, see its below comments: + // Make a notice to the server that this client + // is checksum-aware. It does not need the first fake Rotate + // necessary checksummed. + // That preference is specified below. - // if _, err = b.c.Execute(`SET @master_binlog_checksum=@@global.binlog_checksum`); err != nil { - // return errors.Trace(err) - // } + if _, err = b.c.Execute(`SET @master_binlog_checksum='NONE', @source_binlog_checksum='NONE'`); err != nil { + return errors.Trace(err) } } - if b.cfg.Flavor == MariaDBFlavor { + if b.cfg.Flavor == mysql.MariaDBFlavor { // Refer https://github.com/alibaba/canal/wiki/BinlogChange(MariaDB5&10) // Tell the server that we understand GTIDs by setting our slave capability // to MARIA_SLAVE_CAPABILITY_GTID = 4 (MariaDB >= 10.0.1). @@ -341,20 +357,23 @@ func (b *BinlogSyncer) registerSlave() error { } if b.cfg.HeartbeatPeriod > 0 { - _, err = b.c.Execute(fmt.Sprintf("SET @master_heartbeat_period=%d;", b.cfg.HeartbeatPeriod)) + _, err = b.c.Execute(fmt.Sprintf("SET @master_heartbeat_period = %d, @source_heartbeat_period = %d", + b.cfg.HeartbeatPeriod, b.cfg.HeartbeatPeriod)) if err != nil { - b.cfg.Logger.Errorf("failed to set @master_heartbeat_period=%d, err: %v", b.cfg.HeartbeatPeriod, err) + b.cfg.Logger.Error( + fmt.Sprintf("failed to set @master_heartbeat_period=%d, @source_heartbeat_period=%d", + b.cfg.HeartbeatPeriod, b.cfg.HeartbeatPeriod), slog.Any("error", err)) return errors.Trace(err) } } serverUUID, err := uuid.NewUUID() if err != nil { - b.cfg.Logger.Errorf("failed to get new uuid %v", err) + b.cfg.Logger.Error("failed to get new uuid", slog.Any("error", err)) return errors.Trace(err) } if _, err = b.c.Execute(fmt.Sprintf("SET @slave_uuid = '%s', @replica_uuid = '%s'", serverUUID, serverUUID)); err != nil { - b.cfg.Logger.Errorf("failed to set @slave_uuid = '%s', err: %v", serverUUID, err) + b.cfg.Logger.Error(fmt.Sprintf("failed to set @slave_uuid = '%s', @replica_uuid = '%s'", serverUUID, serverUUID), slog.Any("error", err)) return errors.Trace(err) } @@ -374,18 +393,18 @@ func (b *BinlogSyncer) enableSemiSync() error { return nil } - if r, err := b.c.Execute("SHOW VARIABLES LIKE 'rpl_semi_sync_master_enabled';"); err != nil { + r, err := b.c.Execute("SHOW VARIABLES LIKE 'rpl_semi_sync_master_enabled';") + if err != nil { return errors.Trace(err) - } else { - s, _ := r.GetString(0, 1) - if s != "ON" { - b.cfg.Logger.Errorf("master does not support semi synchronous replication, use no semi-sync") - b.cfg.SemiSyncEnabled = false - return nil - } + } + s, _ := r.GetString(0, 1) + if s != "ON" { + b.cfg.Logger.Error("master does not support semi synchronous replication, use no semi-sync") + b.cfg.SemiSyncEnabled = false + return nil } - _, err := b.c.Execute(`SET @rpl_semi_sync_slave = 1;`) + _, err = b.c.Execute(`SET @rpl_semi_sync_slave = 1;`) if err != nil { return errors.Trace(err) } @@ -406,7 +425,7 @@ func (b *BinlogSyncer) prepare() error { return errors.Trace(err) } - b.cfg.Logger.Infof("Connected to %s %s server", b.cfg.Flavor, b.c.GetServerVersion()) + b.cfg.Logger.Info("Connected to server", slog.String("flavor", b.cfg.Flavor), slog.String("version", b.c.GetServerVersion())) return nil } @@ -422,25 +441,24 @@ func (b *BinlogSyncer) startDumpStream() *BinlogStreamer { } // GetNextPosition returns the next position of the syncer -func (b *BinlogSyncer) GetNextPosition() Position { +func (b *BinlogSyncer) GetNextPosition() mysql.Position { return b.nextPos } func (b *BinlogSyncer) checkFlavor() { serverVersion := b.c.GetServerVersion() - if b.cfg.Flavor != MariaDBFlavor && - strings.Contains(b.c.GetServerVersion(), "MariaDB") { + if b.cfg.Flavor != mysql.MariaDBFlavor && + strings.Contains(serverVersion, "MariaDB") { // Setting the flavor to `mysql` causes MariaDB to try and behave // in a MySQL compatible way. In this mode MariaDB won't use // MariaDB specific binlog event types, but may used dummy events instead. - b.cfg.Logger.Errorf("misconfigured flavor (%s) for server %s", - b.cfg.Flavor, serverVersion) + b.cfg.Logger.Error("misconfigured flavor for server", slog.String("flavor", b.cfg.Flavor), slog.String("version", serverVersion)) } } // StartSync starts syncing from the `pos` position. -func (b *BinlogSyncer) StartSync(pos Position) (*BinlogStreamer, error) { - b.cfg.Logger.Infof("begin to sync binlog from position %s", pos) +func (b *BinlogSyncer) StartSync(pos mysql.Position) (*BinlogStreamer, error) { + b.cfg.Logger.Info("begin to sync binlog from position", slog.Any("position", pos)) b.m.Lock() defer b.m.Unlock() @@ -459,8 +477,8 @@ func (b *BinlogSyncer) StartSync(pos Position) (*BinlogStreamer, error) { } // StartSyncGTID starts syncing from the `gset` GTIDSet. -func (b *BinlogSyncer) StartSyncGTID(gset GTIDSet) (*BinlogStreamer, error) { - b.cfg.Logger.Infof("begin to sync binlog from GTID set %s", gset) +func (b *BinlogSyncer) StartSyncGTID(gset mysql.GTIDSet) (*BinlogStreamer, error) { + b.cfg.Logger.Info("begin to sync binlog from GTID set", slog.Any("GTID set", gset)) b.prevMySQLGTIDEvent = nil b.prevGset = gset @@ -482,7 +500,7 @@ func (b *BinlogSyncer) StartSyncGTID(gset GTIDSet) (*BinlogStreamer, error) { var err error switch b.cfg.Flavor { - case MariaDBFlavor: + case mysql.MariaDBFlavor: err = b.writeBinlogDumpMariadbGTIDCommand(gset) default: // default use MySQL @@ -498,19 +516,26 @@ func (b *BinlogSyncer) StartSyncGTID(gset GTIDSet) (*BinlogStreamer, error) { return b.startDumpStream(), nil } -func (b *BinlogSyncer) writeBinlogDumpCommand(p Position) error { +func (b *BinlogSyncer) writeBinlogDumpCommand(p mysql.Position) error { b.c.ResetSequence() data := make([]byte, 4+1+4+2+4+len(p.Name)) pos := 4 - data[pos] = COM_BINLOG_DUMP + data[pos] = mysql.COM_BINLOG_DUMP pos++ binary.LittleEndian.PutUint32(data[pos:], p.Pos) pos += 4 - binary.LittleEndian.PutUint16(data[pos:], b.cfg.DumpCommandFlag) + dumpCommandFlag := b.cfg.DumpCommandFlag + if b.cfg.FillZeroLogPos && b.cfg.Flavor == mysql.MariaDBFlavor { + // Add BINLOG_SEND_ANNOTATE_ROWS_EVENT flag when FillZeroLogPos is enabled. + // This ensures the server sends ANNOTATE_ROWS_EVENT events which are needed + // for correct LogPos calculation in MariaDB 11.4+, where some events have LogPos=0. + dumpCommandFlag |= BINLOG_SEND_ANNOTATE_ROWS_EVENT + } + binary.LittleEndian.PutUint16(data[pos:], dumpCommandFlag) pos += 2 binary.LittleEndian.PutUint32(data[pos:], b.cfg.ServerID) @@ -521,15 +546,15 @@ func (b *BinlogSyncer) writeBinlogDumpCommand(p Position) error { return b.c.WritePacket(data) } -func (b *BinlogSyncer) writeBinlogDumpMysqlGTIDCommand(gset GTIDSet) error { - p := Position{Name: "", Pos: 4} +func (b *BinlogSyncer) writeBinlogDumpMysqlGTIDCommand(gset mysql.GTIDSet) error { + p := mysql.Position{Name: "", Pos: 4} gtidData := gset.Encode() b.c.ResetSequence() data := make([]byte, 4+1+2+4+4+len(p.Name)+8+4+len(gtidData)) pos := 4 - data[pos] = COM_BINLOG_DUMP_GTID + data[pos] = mysql.COM_BINLOG_DUMP_GTID pos++ binary.LittleEndian.PutUint16(data[pos:], 0) @@ -557,7 +582,7 @@ func (b *BinlogSyncer) writeBinlogDumpMysqlGTIDCommand(gset GTIDSet) error { return b.c.WritePacket(data) } -func (b *BinlogSyncer) writeBinlogDumpMariadbGTIDCommand(gset GTIDSet) error { +func (b *BinlogSyncer) writeBinlogDumpMariadbGTIDCommand(gset mysql.GTIDSet) error { // Copy from vitess startPos := gset.String() @@ -578,16 +603,20 @@ func (b *BinlogSyncer) writeBinlogDumpMariadbGTIDCommand(gset GTIDSet) error { } // Since we use @slave_connect_state, the file and position here are ignored. - return b.writeBinlogDumpCommand(Position{Name: "", Pos: 0}) + return b.writeBinlogDumpCommand(mysql.Position{Name: "", Pos: 0}) } -// localHostname returns the hostname that register slave would register as. +// localHostname returns the hostname that register replica would register as. +// this gets truncated to 255 bytes. func (b *BinlogSyncer) localHostname() string { - if len(b.cfg.Localhost) == 0 { - h, _ := os.Hostname() + h := b.cfg.Localhost + if len(h) == 0 { + h, _ = os.Hostname() + } + if len(h) <= 255 { return h } - return b.cfg.Localhost + return h[:255] } func (b *BinlogSyncer) writeRegisterSlaveCommand() error { @@ -599,7 +628,7 @@ func (b *BinlogSyncer) writeRegisterSlaveCommand() error { data := make([]byte, 4+1+4+1+len(hostname)+1+len(b.cfg.User)+1+2+4+4) pos := 4 - data[pos] = COM_REGISTER_SLAVE + data[pos] = mysql.COM_REGISTER_SLAVE pos++ binary.LittleEndian.PutUint32(data[pos:], b.cfg.ServerID) @@ -622,7 +651,7 @@ func (b *BinlogSyncer) writeRegisterSlaveCommand() error { binary.LittleEndian.PutUint16(data[pos:], b.cfg.Port) pos += 2 - //replication rank, not used + // replication rank, not used binary.LittleEndian.PutUint32(data[pos:], 0) pos += 4 @@ -632,7 +661,7 @@ func (b *BinlogSyncer) writeRegisterSlaveCommand() error { return b.c.WritePacket(data) } -func (b *BinlogSyncer) replySemiSyncACK(p Position) error { +func (b *BinlogSyncer) replySemiSyncACK(p mysql.Position) error { b.c.ResetSequence() data := make([]byte, 4+1+8+len(p.Name)) @@ -662,17 +691,17 @@ func (b *BinlogSyncer) retrySync() error { b.prevMySQLGTIDEvent = nil if b.prevGset != nil { - msg := fmt.Sprintf("begin to re-sync from %s", b.prevGset.String()) + extra := []any{slog.String("GTID Set", b.prevGset.String())} if b.currGset != nil { - msg = fmt.Sprintf("%v (last read GTID=%v)", msg, b.currGset) + extra = append(extra, slog.String("last read GTID", b.currGset.String())) } - b.cfg.Logger.Infof(msg) + b.cfg.Logger.Info("begin to re-sync", extra...) if err := b.prepareSyncGTID(b.prevGset); err != nil { return errors.Trace(err) } } else { - b.cfg.Logger.Infof("begin to re-sync from %s", b.nextPos) + b.cfg.Logger.Info("begin to re-sync", slog.String("file", b.nextPos.Name), slog.Uint64("position", uint64(b.nextPos.Pos))) if err := b.prepareSyncPos(b.nextPos); err != nil { return errors.Trace(err) } @@ -681,7 +710,7 @@ func (b *BinlogSyncer) retrySync() error { return nil } -func (b *BinlogSyncer) prepareSyncPos(pos Position) error { +func (b *BinlogSyncer) prepareSyncPos(pos mysql.Position) error { // always start from position 4 if pos.Pos < 4 { pos.Pos = 4 @@ -698,7 +727,7 @@ func (b *BinlogSyncer) prepareSyncPos(pos Position) error { return nil } -func (b *BinlogSyncer) prepareSyncGTID(gset GTIDSet) error { +func (b *BinlogSyncer) prepareSyncGTID(gset mysql.GTIDSet) error { var err error // re establishing network connection here and will start getting binlog events from "gset + 1", thus until first @@ -710,7 +739,7 @@ func (b *BinlogSyncer) prepareSyncGTID(gset GTIDSet) error { } switch b.cfg.Flavor { - case MariaDBFlavor: + case mysql.MariaDBFlavor: err = b.writeBinlogDumpMariadbGTIDCommand(gset) default: // default use MySQL @@ -726,7 +755,7 @@ func (b *BinlogSyncer) prepareSyncGTID(gset GTIDSet) error { func (b *BinlogSyncer) onStream(s *BinlogStreamer) { defer func() { if e := recover(); e != nil { - s.closeWithError(fmt.Errorf("Err: %v\n Stack: %s", e, Pstack())) + s.closeWithError(fmt.Errorf("panic %v\nstack: %s", e, mysql.Pstack())) } b.wg.Done() }() @@ -741,7 +770,7 @@ func (b *BinlogSyncer) onStream(s *BinlogStreamer) { } if err != nil { - b.cfg.Logger.Error(err) + b.cfg.Logger.Error(err.Error()) // we meet connection error, should re-connect again with // last nextPos or nextGTID we got. if len(b.nextPos.Name) == 0 && b.prevGset == nil { @@ -765,17 +794,17 @@ func (b *BinlogSyncer) onStream(s *BinlogStreamer) { b.retryCount++ if err = b.retrySync(); err != nil { if b.cfg.MaxReconnectAttempts > 0 && b.retryCount >= b.cfg.MaxReconnectAttempts { - b.cfg.Logger.Errorf( - "retry sync err: %v, exceeded max retries (%d)", - err, b.cfg.MaxReconnectAttempts, + b.cfg.Logger.Error( + "retry sync err, exceeded max retries", + slog.Any("error", err), slog.Int("maxAttempts", b.cfg.MaxReconnectAttempts), ) s.closeWithError(err) return } - b.cfg.Logger.Errorf( - "retry sync err: %v, wait 1s and retry again (retries: %d/%d)", - err, b.retryCount, b.cfg.MaxReconnectAttempts, + b.cfg.Logger.Error( + "retry sync err, wait 1s and retry again", + slog.Any("error", err), slog.Int("retryCount", b.retryCount), slog.Int("maxAttempts", b.cfg.MaxReconnectAttempts), ) continue } @@ -788,7 +817,7 @@ func (b *BinlogSyncer) onStream(s *BinlogStreamer) { continue } - //set read timeout + // set read timeout if b.cfg.ReadTimeout > 0 { _ = b.c.SetReadDeadline(utils.Now().Add(b.cfg.ReadTimeout)) } @@ -797,7 +826,7 @@ func (b *BinlogSyncer) onStream(s *BinlogStreamer) { b.retryCount = 0 switch data[0] { - case OK_HEADER: + case mysql.OK_HEADER: // Parse the event e, needACK, err := b.parseEvent(data) if err != nil { @@ -811,18 +840,18 @@ func (b *BinlogSyncer) onStream(s *BinlogStreamer) { s.closeWithError(err) return } - case ERR_HEADER: + case mysql.ERR_HEADER: err = b.c.HandleErrorPacket(data) s.closeWithError(err) return - case EOF_HEADER: + case mysql.EOF_HEADER: // refer to https://dev.mysql.com/doc/internals/en/com-binlog-dump.html#binlog-dump-non-block // when COM_BINLOG_DUMP command use BINLOG_DUMP_NON_BLOCK flag, // if there is no more event to send an EOF_Packet instead of blocking the connection b.cfg.Logger.Info("receive EOF packet, no more binlog event now.") continue default: - b.cfg.Logger.Errorf("invalid stream header %c", data[0]) + b.cfg.Logger.Error("invalid stream header", slog.Int("header", int(data[0]))) continue } } @@ -858,6 +887,13 @@ func (b *BinlogSyncer) handleEventAndACK(s *BinlogStreamer, e *BinlogEvent, need if e.Header.LogPos > 0 { // Some events like FormatDescriptionEvent return 0, ignore. b.nextPos.Pos = e.Header.LogPos + } else if b.shouldCalculateDynamicLogPos(e) { + calculatedPos := b.nextPos.Pos + e.Header.EventSize + e.Header.LogPos = calculatedPos + b.nextPos.Pos = calculatedPos + b.cfg.Logger.Debug("MariaDB dynamic LogPos calculation", + slog.String("eventType", e.Header.EventType.String()), + slog.Uint64("logPos", uint64(calculatedPos))) } // Handle event types to update positions and GTID sets @@ -865,7 +901,7 @@ func (b *BinlogSyncer) handleEventAndACK(s *BinlogStreamer, e *BinlogEvent, need case *RotateEvent: b.nextPos.Name = string(event.NextLogName) b.nextPos.Pos = uint32(event.Position) - b.cfg.Logger.Infof("rotate to %s", b.nextPos) + b.cfg.Logger.Info("rotate to next binlog", slog.String("file", b.nextPos.Name), slog.Uint64("position", uint64(b.nextPos.Pos))) case *GTIDEvent: if b.prevGset == nil { @@ -878,16 +914,35 @@ func (b *BinlogSyncer) handleEventAndACK(s *BinlogStreamer, e *BinlogEvent, need if err != nil { return errors.Trace(err) } - b.currGset.(*MysqlGTIDSet).AddGTID(u, event.GNO) + b.currGset.(*mysql.MysqlGTIDSet).AddGTID(u, event.GNO) if b.prevMySQLGTIDEvent != nil { u, err = uuid.FromBytes(b.prevMySQLGTIDEvent.SID) if err != nil { return errors.Trace(err) } - b.prevGset.(*MysqlGTIDSet).AddGTID(u, b.prevMySQLGTIDEvent.GNO) + b.prevGset.(*mysql.MysqlGTIDSet).AddGTID(u, b.prevMySQLGTIDEvent.GNO) } b.prevMySQLGTIDEvent = event - + case *GtidTaggedLogEvent: + if b.prevGset == nil { + break + } + if b.currGset == nil { + b.currGset = b.prevGset.Clone() + } + u, err := uuid.FromBytes(event.SID) + if err != nil { + return errors.Trace(err) + } + b.currGset.(*mysql.MysqlGTIDSet).AddGTIDWithTag(u, event.Tag, event.GNO) + if b.prevMySQLGTIDEvent != nil { + u, err = uuid.FromBytes(b.prevMySQLGTIDEvent.SID) + if err != nil { + return errors.Trace(err) + } + b.prevGset.(*mysql.MysqlGTIDSet).AddGTIDWithTag(u, b.prevMySQLGTIDEvent.Tag, b.prevMySQLGTIDEvent.GNO) + } + b.prevMySQLGTIDEvent = &event.GTIDEvent case *MariadbGTIDEvent: if b.prevGset == nil { break @@ -896,7 +951,7 @@ func (b *BinlogSyncer) handleEventAndACK(s *BinlogStreamer, e *BinlogEvent, need b.currGset = b.prevGset.Clone() } prev := b.currGset.Clone() - err := b.currGset.(*MariadbGTIDSet).AddSet(&event.GTID) + err := b.currGset.(*mysql.MariadbGTIDSet).AddSet(&event.GTID) if err != nil { return errors.Trace(err) } @@ -927,7 +982,7 @@ func (b *BinlogSyncer) handleEventAndACK(s *BinlogStreamer, e *BinlogEvent, need select { case s.ch <- e: case <-b.ctx.Done(): - return errors.New("sync is being closed...") + return errors.New("sync is being closed") } } @@ -941,8 +996,21 @@ func (b *BinlogSyncer) handleEventAndACK(s *BinlogStreamer, e *BinlogEvent, need return nil } +// shouldCalculateDynamicLogPos determines if we should calculate LogPos dynamically for MariaDB events. +// This is needed for MariaDB 11.4+ when: +// 1. FillZeroLogPos is enabled +// 2. We're using MariaDB flavor +// 3. The event has LogPos=0 (indicating server didn't set it) +// 4. The event is not artificial (not marked with LOG_EVENT_ARTIFICIAL_F flag) +func (b *BinlogSyncer) shouldCalculateDynamicLogPos(e *BinlogEvent) bool { + return b.cfg.FillZeroLogPos && + b.cfg.Flavor == mysql.MariaDBFlavor && + e.Header.LogPos == 0 && + (e.Header.Flags&LOG_EVENT_ARTIFICIAL_F) == 0 +} + // getCurrentGtidSet returns a clone of the current GTID set. -func (b *BinlogSyncer) getCurrentGtidSet() GTIDSet { +func (b *BinlogSyncer) getCurrentGtidSet() mysql.GTIDSet { if b.currGset != nil { return b.currGset.Clone() } @@ -979,11 +1047,11 @@ func (b *BinlogSyncer) newConnection(ctx context.Context) (*client.Conn, error) func (b *BinlogSyncer) killConnection(conn *client.Conn, id uint32) { cmd := fmt.Sprintf("KILL %d", id) if _, err := conn.Execute(cmd); err != nil { - b.cfg.Logger.Errorf("kill connection %d error %v", id, err) + b.cfg.Logger.Error("kill connection", slog.Any("error", err), slog.Int64("id", int64(id))) // Unknown thread id - if code := ErrorCode(err.Error()); code != ER_NO_SUCH_THREAD { - b.cfg.Logger.Error(errors.Trace(err)) + if code := mysql.ErrorCode(err.Error()); code != mysql.ER_NO_SUCH_THREAD { + b.cfg.Logger.Error(errors.Trace(err).Error()) } } - b.cfg.Logger.Infof("kill last connection id %d", id) + b.cfg.Logger.Info("kill last connection", slog.Int64("id", int64(id))) } diff --git a/vendor/github.com/go-mysql-org/go-mysql/replication/const.go b/vendor/github.com/go-mysql-org/go-mysql/replication/const.go index 555f96b8c..f59a6b544 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/replication/const.go +++ b/vendor/github.com/go-mysql-org/go-mysql/replication/const.go @@ -1,12 +1,13 @@ +//nolint:revive // legacy MySQL/MariaDB binlog protocol constants (LOG_EVENT_*, BINLOG_*, MARIADB_*, COM_*, etc.) are kept verbatim from the upstream protocol for backward compatibility package replication const ( - //we only support MySQL 5.0.0+ binlog format, maybe??? + // we only support MySQL 5.0.0+ binlog format, maybe??? MinBinlogVersion = 4 ) var ( - //binlog header [ fe `bin` ] + // binlog header [ fe `bin` ] BinLogFileHeader = []byte{0xfe, 0x62, 0x69, 0x6e} SemiSyncIndicator byte = 0xef @@ -224,15 +225,28 @@ func (e EventType) String() string { } } +type BinlogChecksum byte + const ( - BINLOG_CHECKSUM_ALG_OFF byte = 0 // Events are without checksum though its generator + BINLOG_CHECKSUM_ALG_OFF BinlogChecksum = 0 // Events are without checksum though its generator // is checksum-capable New Master (NM). - BINLOG_CHECKSUM_ALG_CRC32 byte = 1 // CRC32 of zlib algorithm. + BINLOG_CHECKSUM_ALG_CRC32 BinlogChecksum = 1 // CRC32 of zlib algorithm. // BINLOG_CHECKSUM_ALG_ENUM_END, // the cut line: valid alg range is [1, 0x7f]. - BINLOG_CHECKSUM_ALG_UNDEF byte = 255 // special value to tag undetermined yet checksum + BINLOG_CHECKSUM_ALG_UNDEF BinlogChecksum = 255 // special value to tag undetermined yet checksum // or events from checksum-unaware servers ) +func (e BinlogChecksum) String() string { + switch e { + case BINLOG_CHECKSUM_ALG_OFF: + return "CHECKSUM_OFF" + case BINLOG_CHECKSUM_ALG_CRC32: + return "CHECKSUM_CRC32" + default: + return "CHECKSUM_UNDEF" + } +} + // These are TABLE_MAP_EVENT's optional metadata field type, from: libbinlogevents/include/rows_event.h const ( TABLE_MAP_OPT_META_SIGNEDNESS byte = iota + 1 @@ -261,3 +275,29 @@ const ( ENUM_EXTRA_ROW_INFO_TYPECODE_NDB byte = iota ENUM_EXTRA_ROW_INFO_TYPECODE_PARTITION ) + +// Binlog flags, from include/mysql.h +const ( + USE_HEARTBEAT_EVENT_V2 = 1 << 1 + MYSQL_RPL_SKIP_TAGGED_GTIDS = 1 << 2 + MYSQL_RPL_GTID = 1 << 16 + MYSQL_RPL_SKIP_HEARTBEAT = 1 << 17 +) + +// On-The-Wire HeartBeat fields +// See enum mysql::binlog::event::codecs::binary::Heartbeat::fields in MySQL +const ( + OTW_HB_HEADER_END_MARK = iota + OTW_HB_LOG_FILENAME_FIELD + OTW_HB_LOG_POSITION_FIELD +) + +// Constants for binlog event flags +// Source: https://github.com/mysql/mysql-server/blob/447eb26e094b444a88c532028647e48228c3c04f/libs/mysql/binlog/event/rows_event.h#L891-L910 +const ( + STMT_END_F = 1 << iota + NO_FOREIGN_KEY_CHECKS_F + RELAXED_UNIQUE_CHECKS_F + COMPLETE_ROWS_F + USE_SQL_FOREIGN_KEY_F +) diff --git a/vendor/github.com/go-mysql-org/go-mysql/replication/event.go b/vendor/github.com/go-mysql-org/go-mysql/replication/event.go index c151f32ba..39cc5378d 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/replication/event.go +++ b/vendor/github.com/go-mysql-org/go-mysql/replication/event.go @@ -6,15 +6,16 @@ import ( "encoding/hex" "fmt" "io" + "math" "strconv" "strings" "time" "unicode" + "github.com/go-mysql-org/go-mysql/mysql" + "github.com/go-mysql-org/go-mysql/serialization" "github.com/google/uuid" "github.com/pingcap/errors" - - . "github.com/go-mysql-org/go-mysql/mysql" ) const ( @@ -40,7 +41,7 @@ func (e *BinlogEvent) Dump(w io.Writer) { } type Event interface { - //Dump Event, format like python-mysql-replication + // Dump Event, format like python-mysql-replication Dump(w io.Writer) Decode(data []byte) error @@ -49,10 +50,10 @@ type Event interface { type EventError struct { Header *EventHeader - //Error message + // Error message Err string - //Event data + // Event data Data []byte } @@ -101,11 +102,50 @@ func (h *EventHeader) Decode(data []byte) error { return nil } +// headerFlagsString is returning a pipe separated string with flag names +func headerFlagsString(flags uint16) string { + var flagstr []string + + if (flags & LOG_EVENT_BINLOG_IN_USE_F) != 0 { + flagstr = append(flagstr, "IN_USE") + } + if (flags & LOG_EVENT_FORCED_ROTATE_F) != 0 { + flagstr = append(flagstr, "ROTATE") + } + if (flags & LOG_EVENT_THREAD_SPECIFIC_F) != 0 { + flagstr = append(flagstr, "THREAD_SPECIFIC") + } + if (flags & LOG_EVENT_SUPPRESS_USE_F) != 0 { + flagstr = append(flagstr, "SUPPRESS_USE") + } + if (flags & LOG_EVENT_UPDATE_TABLE_MAP_VERSION_F) != 0 { + flagstr = append(flagstr, "UPDATE_TABLE_MAP_VERSION") + } + if (flags & LOG_EVENT_ARTIFICIAL_F) != 0 { + flagstr = append(flagstr, "ARTIFICIAL") + } + if (flags & LOG_EVENT_RELAY_LOG_F) != 0 { + flagstr = append(flagstr, "RELAY_LOG") + } + if (flags & LOG_EVENT_IGNORABLE_F) != 0 { + flagstr = append(flagstr, "IGNORABLE") + } + if (flags & LOG_EVENT_NO_FILTER_F) != 0 { + flagstr = append(flagstr, "NO_FILTER") + } + if (flags & LOG_EVENT_MTS_ISOLATE_F) != 0 { + flagstr = append(flagstr, "MTS_ISOLATE") + } + + return strings.Join(flagstr, "|") +} + func (h *EventHeader) Dump(w io.Writer) { fmt.Fprintf(w, "=== %s ===\n", h.EventType) - fmt.Fprintf(w, "Date: %s\n", time.Unix(int64(h.Timestamp), 0).Format(TimeFormat)) + fmt.Fprintf(w, "Date: %s\n", time.Unix(int64(h.Timestamp), 0).Format(mysql.TimeFormat)) fmt.Fprintf(w, "Log position: %d\n", h.LogPos) fmt.Fprintf(w, "Event size: %d\n", h.EventSize) + fmt.Fprintf(w, "Header Flags: %s\n", headerFlagsString(h.Flags)) } var ( @@ -153,7 +193,7 @@ type FormatDescriptionEvent struct { EventTypeHeaderLengths []byte // 0 is off, 1 is for CRC32, 255 is undefined - ChecksumAlgorithm byte + ChecksumAlgorithm BinlogChecksum } func (e *FormatDescriptionEvent) Decode(data []byte) error { @@ -188,7 +228,7 @@ func (e *FormatDescriptionEvent) Decode(data []byte) error { if calcVersionProduct(e.ServerVersion) >= checksumProduct { // here, the last 5 bytes is 1 byte check sum alg type and 4 byte checksum if exists - e.ChecksumAlgorithm = data[len(data)-5] + e.ChecksumAlgorithm = BinlogChecksum(data[len(data)-5]) e.EventTypeHeaderLengths = data[pos : len(data)-5] } else { e.ChecksumAlgorithm = BINLOG_CHECKSUM_ALG_UNDEF @@ -201,9 +241,9 @@ func (e *FormatDescriptionEvent) Decode(data []byte) error { func (e *FormatDescriptionEvent) Dump(w io.Writer) { fmt.Fprintf(w, "Version: %d\n", e.Version) fmt.Fprintf(w, "Server version: %s\n", e.ServerVersion) - //fmt.Fprintf(w, "Create date: %s\n", time.Unix(int64(e.CreateTimestamp), 0).Format(TimeFormat)) - fmt.Fprintf(w, "Checksum algorithm: %d\n", e.ChecksumAlgorithm) - //fmt.Fprintf(w, "Event header lengths: \n%s", hex.Dump(e.EventTypeHeaderLengths)) + // fmt.Fprintf(w, "Create date: %s\n", time.Unix(int64(e.CreateTimestamp), 0).Format(TimeFormat)) + fmt.Fprintf(w, "Checksum algorithm: %s\n", e.ChecksumAlgorithm) + // fmt.Fprintf(w, "Event header lengths: \n%s", hex.Dump(e.EventTypeHeaderLengths)) fmt.Fprintln(w) } @@ -229,55 +269,41 @@ type PreviousGTIDsEvent struct { GTIDSets string } -type GtidFormat int - -const ( - GtidFormatClassic = iota - GtidFormatTagged -) - -// Decode the number of sids (source identifiers) and if it is using -// tagged GTIDs or classic (non-tagged) GTIDs. -// -// Note that each gtid tag increases the sidno here, so a single UUID -// might turn up multiple times if there are multipl tags. -// -// see also: -// decode_nsids_format in mysql/mysql-server -// https://github.com/mysql/mysql-server/blob/61a3a1d8ef15512396b4c2af46e922a19bf2b174/sql/rpl_gtid_set.cc#L1363-L1378 -func decodeSid(data []byte) (format GtidFormat, sidnr uint64) { - if data[7] == 1 { - format = GtidFormatTagged - } - - if format == GtidFormatTagged { - masked := make([]byte, 8) - copy(masked, data[1:7]) - sidnr = binary.LittleEndian.Uint64(masked) - return - } - sidnr = binary.LittleEndian.Uint64(data[:8]) - return -} - func (e *PreviousGTIDsEvent) Decode(data []byte) error { pos := 0 - - format, uuidCount := decodeSid(data) + format, uuidCount, err := mysql.DecodeSid(data) + if err != nil { + return err + } + if uuidCount == 0 { + return nil + } + if uuidCount > math.MaxInt32 { + return errors.New("data for PreviousGTIDEvent has an invalid UUID count") + } pos += 8 - previousGTIDSets := make([]string, uuidCount) - currentSetnr := 0 var buf strings.Builder - for range previousGTIDSets { - uuid := e.decodeUuid(data[pos : pos+16]) + for range uuidCount { + if pos+16 > len(data) { + return errors.New("data for PreviousGTIDEvent is truncated: missing UUID") + } + uuid := e.decodeUUID(data[pos : pos+16]) pos += 16 var tag string - if format == GtidFormatTagged { - tagLength := int(data[pos]) / 2 - pos += 1 - if tagLength > 0 { // 0 == no tag, >0 == tag + if format == mysql.GtidFormatTagged { + if pos >= len(data) { + return errors.New("data for PreviousGTIDEvent is truncated: missing tag length") + } + tagLength := int(data[pos] >> 1) + pos++ + if tagLength > 32 { + return errors.New("tag is longer than expected") + } else if tagLength > 0 { // 0 == no tag, >0 == tag + if pos+tagLength > len(data) { + return errors.New("data for PreviousGTIDEvent is truncated: tag extends beyond data") + } tag = string(data[pos : pos+tagLength]) pos += tagLength } @@ -291,11 +317,20 @@ func (e *PreviousGTIDsEvent) Decode(data []byte) error { buf.WriteString(",") } buf.WriteString(uuid) - currentSetnr += 1 + currentSetnr++ } - sliceCount := binary.LittleEndian.Uint16(data[pos : pos+8]) + if pos+8 > len(data) { + return errors.New("data for PreviousGTIDEvent is truncated: missing slice count") + } + sliceCount := binary.LittleEndian.Uint64(data[pos : pos+8]) pos += 8 + + // Avoid integer overflow: divide instead of multiplying sliceCount by 16. + if sliceCount > uint64(len(data)-pos)/16 { + return errors.New("slice count is higher than expected") + } + for range sliceCount { buf.WriteString(":") @@ -310,7 +345,7 @@ func (e *PreviousGTIDsEvent) Decode(data []byte) error { } } if len(tag) == 0 { - currentSetnr += 1 + currentSetnr++ } } e.GTIDSets = buf.String() @@ -322,7 +357,7 @@ func (e *PreviousGTIDsEvent) Dump(w io.Writer) { fmt.Fprintln(w) } -func (e *PreviousGTIDsEvent) decodeUuid(data []byte) string { +func (e *PreviousGTIDsEvent) decodeUUID(data []byte) string { return fmt.Sprintf("%s-%s-%s-%s-%s", hex.EncodeToString(data[0:4]), hex.EncodeToString(data[4:6]), hex.EncodeToString(data[6:8]), hex.EncodeToString(data[8:10]), hex.EncodeToString(data[10:])) } @@ -335,7 +370,7 @@ type XIDEvent struct { XID uint64 // in fact XIDEvent dosen't have the GTIDSet information, just for beneficial to use - GSet GTIDSet + GSet mysql.GTIDSet } func (e *XIDEvent) Decode(data []byte) error { @@ -363,7 +398,7 @@ type QueryEvent struct { compressed bool // in fact QueryEvent dosen't have the GTIDSet information, just for beneficial to use - GSet GTIDSet + GSet mysql.GTIDSet } func (e *QueryEvent) Decode(data []byte) error { @@ -390,11 +425,11 @@ func (e *QueryEvent) Decode(data []byte) error { e.Schema = data[pos : pos+int(schemaLength)] pos += int(schemaLength) - //skip 0x00 + // skip 0x00 pos++ if e.compressed { - decompressedQuery, err := DecompressMariadbData(data[pos:]) + decompressedQuery, err := mysql.DecompressMariadbData(data[pos:]) if err != nil { return err } @@ -409,7 +444,7 @@ func (e *QueryEvent) Dump(w io.Writer) { fmt.Fprintf(w, "Slave proxy ID: %d\n", e.SlaveProxyID) fmt.Fprintf(w, "Execution time: %d\n", e.ExecutionTime) fmt.Fprintf(w, "Error code: %d\n", e.ErrorCode) - //fmt.Fprintf(w, "Status vars: \n%s", hex.Dump(e.StatusVars)) + // fmt.Fprintf(w, "Status vars: \n%s", hex.Dump(e.StatusVars)) fmt.Fprintf(w, "Schema: %s\n", e.Schema) fmt.Fprintf(w, "Query: %s\n", e.Query) if e.GSet != nil { @@ -421,17 +456,18 @@ func (e *QueryEvent) Dump(w io.Writer) { type GTIDEvent struct { CommitFlag uint8 SID []byte + Tag mysql.Tag GNO int64 LastCommitted int64 SequenceNumber int64 // ImmediateCommitTimestamp/OriginalCommitTimestamp are introduced in MySQL-8.0.1, see: - // https://mysqlhighavailability.com/replication-features-in-mysql-8-0-1/ + // https://dev.mysql.com/blog-archive/new-monitoring-replication-features-and-more ImmediateCommitTimestamp uint64 OriginalCommitTimestamp uint64 // Total transaction length (including this GTIDEvent), introduced in MySQL-8.0.2, see: - // https://mysqlhighavailability.com/taking-advantage-of-new-transaction-length-metadata/ + // https://dev.mysql.com/blog-archive/taking-advantage-of-new-transaction-length-metadata TransactionLength uint64 // ImmediateServerVersion/OriginalServerVersion are introduced in MySQL-8.0.14, see @@ -461,12 +497,12 @@ func (e *GTIDEvent) Decode(data []byte) error { if len(data)-pos < 7 { return nil } - e.ImmediateCommitTimestamp = FixedLengthInt(data[pos : pos+7]) + e.ImmediateCommitTimestamp = mysql.FixedLengthInt(data[pos : pos+7]) pos += 7 if (e.ImmediateCommitTimestamp & (uint64(1) << 55)) != 0 { // If the most significant bit set, another 7 byte follows representing OriginalCommitTimestamp e.ImmediateCommitTimestamp &= ^(uint64(1) << 55) - e.OriginalCommitTimestamp = FixedLengthInt(data[pos : pos+7]) + e.OriginalCommitTimestamp = mysql.FixedLengthInt(data[pos : pos+7]) pos += 7 } else { // Otherwise OriginalCommitTimestamp == ImmediateCommitTimestamp @@ -478,7 +514,7 @@ func (e *GTIDEvent) Decode(data []byte) error { return nil } var n int - e.TransactionLength, _, n = LengthEncodedInt(data[pos:]) + e.TransactionLength, _, n = mysql.LengthEncodedInt(data[pos:]) pos += n // IMMEDIATE_SERVER_VERSION_LENGTH = 4 @@ -512,8 +548,8 @@ func (e *GTIDEvent) Dump(w io.Writer) { } fmt.Fprintf(w, "Commit flag: %d\n", e.CommitFlag) - u, _ := uuid.FromBytes(e.SID) - fmt.Fprintf(w, "GTID_NEXT: %s:%d\n", u.String(), e.GNO) + gn, _ := e.GTIDNext() + fmt.Fprintf(w, "GTID_NEXT: %s\n", gn.String()) fmt.Fprintf(w, "LAST_COMMITTED: %d\n", e.LastCommitted) fmt.Fprintf(w, "SEQUENCE_NUMBER: %d\n", e.SequenceNumber) fmt.Fprintf(w, "Immediate commmit timestamp: %d (%s)\n", e.ImmediateCommitTimestamp, fmtTime(e.ImmediateCommitTime())) @@ -524,12 +560,32 @@ func (e *GTIDEvent) Dump(w io.Writer) { fmt.Fprintln(w) } -func (e *GTIDEvent) GTIDNext() (GTIDSet, error) { +func (e *GTIDEvent) GTIDNext() (mysql.GTIDSet, error) { u, err := uuid.FromBytes(e.SID) if err != nil { return nil, err } - return ParseMysqlGTIDSet(strings.Join([]string{u.String(), strconv.FormatInt(e.GNO, 10)}, ":")) + if e.Tag == mysql.NewTag("") { + return mysql.ParseMysqlGTIDSet( + strings.Join( + []string{ + u.String(), + strconv.FormatInt(e.GNO, 10), + }, + ":", + ), + ) + } + return mysql.ParseMysqlGTIDSet( + strings.Join( + []string{ + u.String(), + e.Tag.String(), + strconv.FormatInt(e.GNO, 10), + }, + ":", + ), + ) } // ImmediateCommitTime returns the commit time of this trx on the immediate server @@ -544,6 +600,202 @@ func (e *GTIDEvent) OriginalCommitTime() time.Time { return microSecTimestampToTime(e.OriginalCommitTimestamp) } +// GtidTaggedLogEvent is for a GTID event with a tag. +// This is similar to GTIDEvent, but it has a tag and uses a different serialization format. +type GtidTaggedLogEvent struct { + GTIDEvent +} + +func (e *GtidTaggedLogEvent) Decode(data []byte) error { + msg := serialization.Message{ + Format: serialization.Format{ + Fields: []serialization.Field{ + { + Name: "gtid_flags", + Type: &serialization.FieldIntFixed{ + Length: 1, + }, + }, + { + Name: "uuid", + Type: &serialization.FieldIntFixed{ + Length: 16, + }, + }, + { + Name: "gno", + Type: &serialization.FieldIntVar{}, + }, + { + Name: "tag", + Type: &serialization.FieldString{}, + }, + { + Name: "last_committed", + Type: &serialization.FieldIntVar{}, + }, + { + Name: "sequence_number", + Type: &serialization.FieldIntVar{}, + }, + { + Name: "immediate_commit_timestamp", + Type: &serialization.FieldUintVar{}, + }, + { + Name: "original_commit_timestamp", + Type: &serialization.FieldUintVar{}, + Optional: true, + }, + { + Name: "transaction_length", + Type: &serialization.FieldUintVar{}, + }, + { + Name: "immediate_server_version", + Type: &serialization.FieldUintVar{}, + }, + { + Name: "original_server_version", + Type: &serialization.FieldUintVar{}, + Optional: true, + }, + { + Name: "commit_group_ticket", + Optional: true, + }, + }, + }, + } + + err := serialization.Unmarshal(data, &msg) + if err != nil { + return err + } + + f, err := msg.GetFieldByName("gtid_flags") + if err != nil { + return err + } + if v, ok := f.Type.(*serialization.FieldIntFixed); ok { + e.CommitFlag = v.Value[0] + } else { + return errors.New("failed to get gtid_flags field") + } + + f, err = msg.GetFieldByName("uuid") + if err != nil { + return err + } + if v, ok := f.Type.(*serialization.FieldIntFixed); ok { + e.SID = v.Value + } else { + return errors.New("failed to get uuid field") + } + + f, err = msg.GetFieldByName("gno") + if err != nil { + return err + } + if v, ok := f.Type.(*serialization.FieldIntVar); ok { + e.GNO = v.Value + } else { + return errors.New("failed to get gno field") + } + + f, err = msg.GetFieldByName("tag") + if err != nil { + return err + } + if v, ok := f.Type.(*serialization.FieldString); ok { + e.Tag = mysql.NewTag(v.Value) + } else { + return errors.New("failed to get tag field") + } + + f, err = msg.GetFieldByName("last_committed") + if err != nil { + return err + } + if v, ok := f.Type.(*serialization.FieldIntVar); ok { + e.LastCommitted = v.Value + } else { + return errors.New("failed to get last_committed field") + } + + f, err = msg.GetFieldByName("sequence_number") + if err != nil { + return err + } + if v, ok := f.Type.(*serialization.FieldIntVar); ok { + e.SequenceNumber = v.Value + } else { + return errors.New("failed to get sequence_number field") + } + + f, err = msg.GetFieldByName("immediate_commit_timestamp") + if err != nil { + return err + } + if v, ok := f.Type.(*serialization.FieldUintVar); ok { + e.ImmediateCommitTimestamp = v.Value + } else { + return errors.New("failed to get immediate_commit_timestamp field") + } + + f, err = msg.GetFieldByName("original_commit_timestamp") + if err != nil { + return err + } + if v, ok := f.Type.(*serialization.FieldUintVar); ok { + if f.Skipped { + e.OriginalCommitTimestamp = e.ImmediateCommitTimestamp + } else { + e.OriginalCommitTimestamp = v.Value + } + } else { + return errors.New("failed to get original_commit_timestamp field") + } + + f, err = msg.GetFieldByName("immediate_server_version") + if err != nil { + return err + } + if v, ok := f.Type.(*serialization.FieldUintVar); ok { + e.ImmediateServerVersion = uint32(v.Value) + } else { + return errors.New("failed to get immediate_server_version field") + } + + f, err = msg.GetFieldByName("original_server_version") + if err != nil { + return err + } + if v, ok := f.Type.(*serialization.FieldUintVar); ok { + if f.Skipped { + e.OriginalServerVersion = e.ImmediateServerVersion + } else { + e.OriginalServerVersion = uint32(v.Value) + } + } else { + return errors.New("failed to get original_server_version field") + } + + f, err = msg.GetFieldByName("transaction_length") + if err != nil { + return err + } + if v, ok := f.Type.(*serialization.FieldUintVar); ok { + e.TransactionLength = v.Value + } else { + return errors.New("failed to get transaction_length field") + } + + // TODO: add and test commit_group_ticket + + return nil +} + type BeginLoadQueryEvent struct { FileID uint32 BlockData []byte @@ -655,7 +907,7 @@ func (e *MariadbBinlogCheckPointEvent) Dump(w io.Writer) { } type MariadbGTIDEvent struct { - GTID MariadbGTID + GTID mysql.MariadbGTID Flags byte CommitID uint64 } @@ -679,7 +931,7 @@ func (e *MariadbGTIDEvent) Decode(data []byte) error { e.GTID.DomainID = binary.LittleEndian.Uint32(data[pos:]) pos += 4 e.Flags = data[pos] - pos += 1 + pos++ if (e.Flags & BINLOG_MARIADB_FL_GROUP_COMMIT_ID) > 0 { e.CommitID = binary.LittleEndian.Uint64(data[pos:]) @@ -695,12 +947,12 @@ func (e *MariadbGTIDEvent) Dump(w io.Writer) { fmt.Fprintln(w) } -func (e *MariadbGTIDEvent) GTIDNext() (GTIDSet, error) { - return ParseMariadbGTIDSet(e.GTID.String()) +func (e *MariadbGTIDEvent) GTIDNext() (mysql.GTIDSet, error) { + return mysql.ParseMariadbGTIDSet(e.GTID.String()) } type MariadbGTIDListEvent struct { - GTIDs []MariadbGTID + GTIDs []mysql.MariadbGTID } func (e *MariadbGTIDListEvent) Decode(data []byte) error { @@ -710,9 +962,9 @@ func (e *MariadbGTIDListEvent) Decode(data []byte) error { count := v & uint32((1<<28)-1) - e.GTIDs = make([]MariadbGTID, count) + e.GTIDs = make([]mysql.MariadbGTID, count) - for i := uint32(0); i < count; i++ { + for i := range count { e.GTIDs[i].DomainID = binary.LittleEndian.Uint32(data[pos:]) pos += 4 e.GTIDs[i].ServerID = binary.LittleEndian.Uint32(data[pos:]) @@ -744,3 +996,67 @@ func (i *IntVarEvent) Dump(w io.Writer) { fmt.Fprintf(w, "Type: %d\n", i.Type) fmt.Fprintf(w, "Value: %d\n", i.Value) } + +// HeartbeatEvent is a HEARTBEAT_EVENT or HEARTBEAT_LOG_EVENT_V2 +// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_replication_binlog_event.html#sect_protocol_replication_event_heartbeat +type HeartbeatEvent struct { + // Event version, either 1 for HEARTBEAT_EVENT or 2 for HEARTBEAT_LOG_EVENT_V2 + Version int + + // Filename of the binary log + Filename string + + // Offset is the offset in the binlog file + Offset uint64 +} + +// Decode is decoding a heartbeat event payload (excluding event header and checksum) +func (h *HeartbeatEvent) Decode(data []byte) error { + switch h.Version { + case 1: + // Also known as HEARTBEAT_EVENT + h.Filename = string(data) + case 2: + // Also known as HEARTBEAT_LOG_EVENT_V2 + // + // The server sends this in the binlog stream if the following is set: + // DumpCommandFlag: replication.USE_HEARTBEAT_EVENT_V2 + pos := 0 + for pos < len(data) { + switch data[pos] { + case OTW_HB_LOG_FILENAME_FIELD: + pos++ + nameLength := int(data[pos]) + pos++ + h.Filename = string(data[pos : pos+nameLength]) + pos += nameLength + case OTW_HB_LOG_POSITION_FIELD: + pos++ + offsetLength := int(data[pos]) + pos++ + var n int + h.Offset, _, n = mysql.LengthEncodedInt(data[pos : pos+offsetLength]) + if n != offsetLength { + return errors.New("failed to read binary log offset") + } + pos += offsetLength + case OTW_HB_HEADER_END_MARK: + pos++ + default: + return errors.New("unknown heartbeatv2 field") + } + } + default: + return errors.New("unknown heartbeat version") + } + + return nil +} + +func (h *HeartbeatEvent) Dump(w io.Writer) { + fmt.Fprintf(w, + "Heartbeat Event Version: %d\nBinlog File Name: %s\nBinlog Offset: %d\n", + h.Version, h.Filename, h.Offset, + ) + fmt.Fprintln(w) +} diff --git a/vendor/github.com/go-mysql-org/go-mysql/replication/generic_event.go b/vendor/github.com/go-mysql-org/go-mysql/replication/generic_event.go index ac55703cc..78da9c0d6 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/replication/generic_event.go +++ b/vendor/github.com/go-mysql-org/go-mysql/replication/generic_event.go @@ -22,7 +22,7 @@ func (e *GenericEvent) Decode(data []byte) error { return nil } -//below events are generic events, maybe later I will consider handle some. +// below events are generic events, maybe later I will consider handle some. // type StartEventV3 struct { // Version uint16 @@ -161,6 +161,3 @@ func (e *GenericEvent) Decode(data []byte) error { // MessageLength uint8 // Message []byte // } - -// type HeartbeatEvent struct { -// } diff --git a/vendor/github.com/go-mysql-org/go-mysql/replication/json_binary.go b/vendor/github.com/go-mysql-org/go-mysql/replication/json_binary.go index 34186f09e..1dd3b6f7e 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/replication/json_binary.go +++ b/vendor/github.com/go-mysql-org/go-mysql/replication/json_binary.go @@ -3,14 +3,15 @@ package replication import ( "fmt" "math" + "strconv" + "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/utils" "github.com/goccy/go-json" "github.com/pingcap/errors" - - . "github.com/go-mysql-org/go-mysql/mysql" ) +//nolint:revive // JSONB type tags mirror the upstream MySQL JSON binary format const ( JSONB_SMALL_OBJECT byte = iota // small JSON object JSONB_LARGE_OBJECT // large JSON object @@ -28,6 +29,7 @@ const ( JSONB_OPAQUE byte = 0x0f // custom data (any MySQL data type) ) +//nolint:revive // JSONB literal tags mirror the upstream MySQL JSON binary format const ( JSONB_NULL_LITERAL byte = 0x00 JSONB_TRUE_LITERAL byte = 0x01 @@ -45,16 +47,18 @@ const ( jsonbValueEntrySizeLarge = 1 + jsonbLargeOffsetSize ) -var ( - ErrCorruptedJSONDiff = fmt.Errorf("corrupted JSON diff") // ER_CORRUPTED_JSON_DIFF -) +var ErrCorruptedJSONDiff = fmt.Errorf("corrupted JSON diff") // ER_CORRUPTED_JSON_DIFF +//nolint:revive // exported type renamed would be a breaking API change type ( // JsonDiffOperation is an enum that describes what kind of operation a JsonDiff object represents. // https://github.com/mysql/mysql-server/blob/8.0/sql/json_diff.h JsonDiffOperation byte ) +type FloatWithTrailingZero float64 + +//nolint:revive // exported constants renamed would be a breaking API change const ( // The JSON value in the given path is replaced with a new value. // @@ -74,6 +78,7 @@ const ( JsonDiffOperationRemove ) +//nolint:revive // exported type renamed would be a breaking API change type ( JsonDiff struct { Op JsonDiffOperation @@ -99,6 +104,14 @@ func (jd *JsonDiff) String() string { return fmt.Sprintf("json_diff(op:%s path:%s value:%s)", jd.Op, jd.Path, jd.Value) } +func (f FloatWithTrailingZero) MarshalJSON() ([]byte, error) { + if float64(f) == float64(int(f)) { + return []byte(strconv.FormatFloat(float64(f), 'f', 1, 64)), nil + } + + return []byte(strconv.FormatFloat(float64(f), 'f', -1, 64)), nil +} + func jsonbGetOffsetSize(isSmall bool) int { if isSmall { return jsonbSmallOffsetSize @@ -123,12 +136,13 @@ func jsonbGetValueEntrySize(isSmall bool) int { return jsonbValueEntrySizeLarge } -// decodeJsonBinary decodes the JSON binary encoding data and returns +// decodeJSONBinary decodes the JSON binary encoding data and returns // the common JSON encoding data. -func (e *RowsEvent) decodeJsonBinary(data []byte) ([]byte, error) { +func (e *RowsEvent) decodeJSONBinary(data []byte) ([]byte, error) { d := jsonBinaryDecoder{ - useDecimal: e.useDecimal, - ignoreDecodeErr: e.ignoreJSONDecodeErr, + useDecimal: e.useDecimal, + useFloatWithTrailingZero: e.useFloatWithTrailingZero, + ignoreDecodeErr: e.ignoreJSONDecodeErr, } if d.isDataShort(data, 1) { @@ -144,12 +158,13 @@ func (e *RowsEvent) decodeJsonBinary(data []byte) ([]byte, error) { } type jsonBinaryDecoder struct { - useDecimal bool - ignoreDecodeErr bool - err error + useDecimal bool + useFloatWithTrailingZero bool + ignoreDecodeErr bool + err error } -func (d *jsonBinaryDecoder) decodeValue(tp byte, data []byte) interface{} { +func (d *jsonBinaryDecoder) decodeValue(tp byte, data []byte) any { if d.err != nil { return nil } @@ -178,6 +193,9 @@ func (d *jsonBinaryDecoder) decodeValue(tp byte, data []byte) interface{} { case JSONB_UINT64: return d.decodeUint64(data) case JSONB_DOUBLE: + if d.useFloatWithTrailingZero { + return d.decodeDoubleWithTrailingZero(data) + } return d.decodeDouble(data) case JSONB_STRING: return d.decodeString(data) @@ -190,7 +208,7 @@ func (d *jsonBinaryDecoder) decodeValue(tp byte, data []byte) interface{} { return nil } -func (d *jsonBinaryDecoder) decodeObjectOrArray(data []byte, isSmall bool, isObject bool) interface{} { +func (d *jsonBinaryDecoder) decodeObjectOrArray(data []byte, isSmall bool, isObject bool) any { offsetSize := jsonbGetOffsetSize(isSmall) if d.isDataShort(data, 2*offsetSize) { return nil @@ -227,7 +245,7 @@ func (d *jsonBinaryDecoder) decodeObjectOrArray(data []byte, isSmall bool, isObj var keys []string if isObject { keys = make([]string, count) - for i := 0; i < count; i++ { + for i := range count { // decode key entryOffset := 2*offsetSize + keyEntrySize*i keyOffset := d.decodeCount(data[entryOffset:], isSmall) @@ -251,8 +269,8 @@ func (d *jsonBinaryDecoder) decodeObjectOrArray(data []byte, isSmall bool, isObj return nil } - values := make([]interface{}, count) - for i := 0; i < count; i++ { + values := make([]any, count) + for i := range count { // decode value entryOffset := 2*offsetSize + valueEntrySize*i if isObject { @@ -283,8 +301,8 @@ func (d *jsonBinaryDecoder) decodeObjectOrArray(data []byte, isSmall bool, isObj return values } - m := make(map[string]interface{}, count) - for i := 0; i < count; i++ { + m := make(map[string]any, count) + for i := range count { m[keys[i]] = values[i] } @@ -302,7 +320,7 @@ func isInlineValue(tp byte, isSmall bool) bool { return false } -func (d *jsonBinaryDecoder) decodeLiteral(data []byte) interface{} { +func (d *jsonBinaryDecoder) decodeLiteral(data []byte) any { if d.isDataShort(data, 1) { return nil } @@ -340,7 +358,7 @@ func (d *jsonBinaryDecoder) decodeInt16(data []byte) int16 { return 0 } - v := ParseBinaryInt16(data[0:2]) + v := mysql.ParseBinaryInt16(data[0:2]) return v } @@ -349,7 +367,7 @@ func (d *jsonBinaryDecoder) decodeUint16(data []byte) uint16 { return 0 } - v := ParseBinaryUint16(data[0:2]) + v := mysql.ParseBinaryUint16(data[0:2]) return v } @@ -358,7 +376,7 @@ func (d *jsonBinaryDecoder) decodeInt32(data []byte) int32 { return 0 } - v := ParseBinaryInt32(data[0:4]) + v := mysql.ParseBinaryInt32(data[0:4]) return v } @@ -367,7 +385,7 @@ func (d *jsonBinaryDecoder) decodeUint32(data []byte) uint32 { return 0 } - v := ParseBinaryUint32(data[0:4]) + v := mysql.ParseBinaryUint32(data[0:4]) return v } @@ -376,7 +394,7 @@ func (d *jsonBinaryDecoder) decodeInt64(data []byte) int64 { return 0 } - v := ParseBinaryInt64(data[0:8]) + v := mysql.ParseBinaryInt64(data[0:8]) return v } @@ -385,7 +403,7 @@ func (d *jsonBinaryDecoder) decodeUint64(data []byte) uint64 { return 0 } - v := ParseBinaryUint64(data[0:8]) + v := mysql.ParseBinaryUint64(data[0:8]) return v } @@ -394,10 +412,15 @@ func (d *jsonBinaryDecoder) decodeDouble(data []byte) float64 { return 0 } - v := ParseBinaryFloat64(data[0:8]) + v := mysql.ParseBinaryFloat64(data[0:8]) return v } +func (d *jsonBinaryDecoder) decodeDoubleWithTrailingZero(data []byte) FloatWithTrailingZero { + v := d.decodeDouble(data) + return FloatWithTrailingZero(v) +} + func (d *jsonBinaryDecoder) decodeString(data []byte) string { if d.err != nil { return "" @@ -415,7 +438,7 @@ func (d *jsonBinaryDecoder) decodeString(data []byte) string { return v } -func (d *jsonBinaryDecoder) decodeOpaque(data []byte) interface{} { +func (d *jsonBinaryDecoder) decodeOpaque(data []byte) any { if d.isDataShort(data, 1) { return nil } @@ -432,18 +455,18 @@ func (d *jsonBinaryDecoder) decodeOpaque(data []byte) interface{} { data = data[n : l+n] switch tp { - case MYSQL_TYPE_NEWDECIMAL: + case mysql.MYSQL_TYPE_NEWDECIMAL: return d.decodeDecimal(data) - case MYSQL_TYPE_TIME: + case mysql.MYSQL_TYPE_TIME: return d.decodeTime(data) - case MYSQL_TYPE_DATE, MYSQL_TYPE_DATETIME, MYSQL_TYPE_TIMESTAMP: + case mysql.MYSQL_TYPE_DATE, mysql.MYSQL_TYPE_DATETIME, mysql.MYSQL_TYPE_TIMESTAMP: return d.decodeDateTime(data) default: return utils.ByteSliceToString(data) } } -func (d *jsonBinaryDecoder) decodeDecimal(data []byte) interface{} { +func (d *jsonBinaryDecoder) decodeDecimal(data []byte) any { precision := int(data[0]) scale := int(data[1]) @@ -453,7 +476,7 @@ func (d *jsonBinaryDecoder) decodeDecimal(data []byte) interface{} { return v } -func (d *jsonBinaryDecoder) decodeTime(data []byte) interface{} { +func (d *jsonBinaryDecoder) decodeTime(data []byte) any { v := d.decodeInt64(data) if v == 0 { @@ -468,14 +491,14 @@ func (d *jsonBinaryDecoder) decodeTime(data []byte) interface{} { intPart := v >> 24 hour := (intPart >> 12) % (1 << 10) - min := (intPart >> 6) % (1 << 6) + minute := (intPart >> 6) % (1 << 6) sec := intPart % (1 << 6) frac := v % (1 << 24) - return fmt.Sprintf("%s%02d:%02d:%02d.%06d", sign, hour, min, sec, frac) + return fmt.Sprintf("%s%02d:%02d:%02d.%06d", sign, hour, minute, sec, frac) } -func (d *jsonBinaryDecoder) decodeDateTime(data []byte) interface{} { +func (d *jsonBinaryDecoder) decodeDateTime(data []byte) any { v := d.decodeInt64(data) if v == 0 { return "0000-00-00 00:00:00" @@ -514,10 +537,7 @@ func (d *jsonBinaryDecoder) decodeCount(data []byte, isSmall bool) int { func (d *jsonBinaryDecoder) decodeVariableLength(data []byte) (int, int) { // The max size for variable length is math.MaxUint32, so // here we can use 5 bytes to save it. - maxCount := 5 - if len(data) < maxCount { - maxCount = len(data) - } + maxCount := min(len(data), 5) pos := 0 length := uint64(0) @@ -531,7 +551,7 @@ func (d *jsonBinaryDecoder) decodeVariableLength(data []byte) (int, int) { return 0, 0 } - pos += 1 + pos++ // TODO: should consider length overflow int here. return int(length), pos } @@ -542,7 +562,7 @@ func (d *jsonBinaryDecoder) decodeVariableLength(data []byte) (int, int) { return 0, 0 } -func (e *RowsEvent) decodeJsonPartialBinary(data []byte) (*JsonDiff, error) { +func (e *RowsEvent) decodeJSONPartialBinary(data []byte) (*JsonDiff, error) { // see Json_diff_vector::read_binary() in mysql-server/sql/json_diff.cc operationNumber := JsonDiffOperation(data[0]) switch operationNumber { @@ -554,7 +574,7 @@ func (e *RowsEvent) decodeJsonPartialBinary(data []byte) (*JsonDiff, error) { } data = data[1:] - pathLength, _, n := LengthEncodedInt(data) + pathLength, _, n := mysql.LengthEncodedInt(data) data = data[n:] path := data[:pathLength] @@ -570,10 +590,10 @@ func (e *RowsEvent) decodeJsonPartialBinary(data []byte) (*JsonDiff, error) { return diff, nil } - valueLength, _, n := LengthEncodedInt(data) + valueLength, _, n := mysql.LengthEncodedInt(data) data = data[n:] - d, err := e.decodeJsonBinary(data[:valueLength]) + d, err := e.decodeJSONBinary(data[:valueLength]) if err != nil { return nil, fmt.Errorf("cannot read json diff for field %q: %w", path, err) } diff --git a/vendor/github.com/go-mysql-org/go-mysql/replication/parser.go b/vendor/github.com/go-mysql-org/go-mysql/replication/parser.go index 4caf496c2..804966c74 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/replication/parser.go +++ b/vendor/github.com/go-mysql-org/go-mysql/replication/parser.go @@ -15,10 +15,8 @@ import ( "github.com/go-mysql-org/go-mysql/utils" ) -var ( - // ErrChecksumMismatch indicates binlog checksum mismatch. - ErrChecksumMismatch = errors.New("binlog checksum mismatch, data may be corrupted") -) +// ErrChecksumMismatch indicates binlog checksum mismatch. +var ErrChecksumMismatch = errors.New("binlog checksum mismatch, data may be corrupted") type BinlogParser struct { // "mysql" or "mariadb", if not set, use "mysql" by default @@ -35,11 +33,14 @@ type BinlogParser struct { timestampStringLocation *time.Location // used to start/stop processing - stopProcessing uint32 + stopProcessing atomic.Bool + + useDecimal bool + useFloatWithTrailingZero bool + ignoreJSONDecodeErr bool + verifyChecksum bool - useDecimal bool - ignoreJSONDecodeErr bool - verifyChecksum bool + payloadDecoderConcurrency int rowsEventDecodeFunc func(*RowsEvent, []byte) error @@ -55,11 +56,11 @@ func NewBinlogParser() *BinlogParser { } func (p *BinlogParser) Stop() { - atomic.StoreUint32(&p.stopProcessing, 1) + p.stopProcessing.Store(true) } func (p *BinlogParser) Resume() { - atomic.StoreUint32(&p.stopProcessing, 0) + p.stopProcessing.Store(false) } func (p *BinlogParser) Reset() { @@ -167,11 +168,7 @@ func (p *BinlogParser) parseSingleEvent(r io.Reader, onEvent OnEventFunc) (bool, } func (p *BinlogParser) ParseReader(r io.Reader, onEvent OnEventFunc) error { - for { - if atomic.LoadUint32(&p.stopProcessing) == 1 { - break - } - + for !p.stopProcessing.Load() { done, err := p.parseSingleEvent(r, onEvent) if err != nil { if err == errMissingTableMapEvent { @@ -204,6 +201,10 @@ func (p *BinlogParser) SetUseDecimal(useDecimal bool) { p.useDecimal = useDecimal } +func (p *BinlogParser) SetUseFloatWithTrailingZero(useFloatWithTrailingZero bool) { + p.useFloatWithTrailingZero = useFloatWithTrailingZero +} + func (p *BinlogParser) SetIgnoreJSONDecodeError(ignoreJSONDecodeErr bool) { p.ignoreJSONDecodeErr = ignoreJSONDecodeErr } @@ -216,6 +217,10 @@ func (p *BinlogParser) SetFlavor(flavor string) { p.flavor = flavor } +func (p *BinlogParser) SetPayloadDecoderConcurrency(concurrency int) { + p.payloadDecoderConcurrency = concurrency +} + func (p *BinlogParser) SetRowsEventDecodeFunc(rowsEventDecodeFunc func(*RowsEvent, []byte) error) { p.rowsEventDecodeFunc = rowsEventDecodeFunc } @@ -244,7 +249,7 @@ func (p *BinlogParser) parseEvent(h *EventHeader, data []byte, rawData []byte) ( if p.format != nil && p.format.ChecksumAlgorithm == BINLOG_CHECKSUM_ALG_CRC32 { err := p.verifyCrc32Checksum(rawData) if err != nil { - return nil, err + return nil, fmt.Errorf("failed checksum for %v, log pos %d: %v", h.EventType, h.LogPos, err) } data = data[0 : len(data)-BinlogChecksumLength] } @@ -293,6 +298,8 @@ func (p *BinlogParser) parseEvent(h *EventHeader, data []byte, rawData []byte) ( e = >IDEvent{} case ANONYMOUS_GTID_EVENT: e = >IDEvent{} + case GTID_TAGGED_LOG_EVENT: + e = &GtidTaggedLogEvent{} case BEGIN_LOAD_QUERY_EVENT: e = &BeginLoadQueryEvent{} case EXECUTE_LOAD_QUERY_EVENT: @@ -313,6 +320,10 @@ func (p *BinlogParser) parseEvent(h *EventHeader, data []byte, rawData []byte) ( e = &IntVarEvent{} case TRANSACTION_PAYLOAD_EVENT: e = p.newTransactionPayloadEvent() + case HEARTBEAT_EVENT: + e = &HeartbeatEvent{Version: 1} + case HEARTBEAT_LOG_EVENT_V2: + e = &HeartbeatEvent{Version: 2} default: e = &GenericEvent{} } @@ -327,6 +338,16 @@ func (p *BinlogParser) parseEvent(h *EventHeader, data []byte, rawData []byte) ( } else { err = e.Decode(data) } + + if fde, ok := e.(*FormatDescriptionEvent); ok { + if fde.ChecksumAlgorithm == BINLOG_CHECKSUM_ALG_CRC32 { + err := p.verifyCrc32Checksum(rawData) + if err != nil { + return nil, fmt.Errorf("failed checksum for %v, log pos %d: %v", h.EventType, h.LogPos, err) + } + } + } + if err != nil { return nil, &EventError{h, err.Error(), data} } @@ -355,7 +376,6 @@ func (p *BinlogParser) Parse(data []byte) (*BinlogEvent, error) { rawData := data h, err := p.parseHeader(data) - if err != nil { return nil, err } @@ -411,6 +431,7 @@ func (p *BinlogParser) newRowsEvent(h *EventHeader) *RowsEvent { e.parseTime = p.parseTime e.timestampStringLocation = p.timestampStringLocation e.useDecimal = p.useDecimal + e.useFloatWithTrailingZero = p.useFloatWithTrailingZero e.ignoreJSONDecodeErr = p.ignoreJSONDecodeErr switch h.EventType { @@ -455,6 +476,7 @@ func (p *BinlogParser) newRowsEvent(h *EventHeader) *RowsEvent { func (p *BinlogParser) newTransactionPayloadEvent() *TransactionPayloadEvent { e := &TransactionPayloadEvent{} e.format = *p.format + e.concurrency = p.payloadDecoderConcurrency return e } diff --git a/vendor/github.com/go-mysql-org/go-mysql/replication/row_event.go b/vendor/github.com/go-mysql-org/go-mysql/replication/row_event.go index 0eebe6914..eaef3edd5 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/replication/row_event.go +++ b/vendor/github.com/go-mysql-org/go-mysql/replication/row_event.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "fmt" "io" + "math/bits" "strconv" "strings" "time" @@ -12,7 +13,7 @@ import ( "github.com/pingcap/errors" "github.com/shopspring/decimal" - . "github.com/go-mysql-org/go-mysql/mysql" + "github.com/go-mysql-org/go-mysql/mysql" "github.com/go-mysql-org/go-mysql/utils" ) @@ -90,7 +91,7 @@ type TableMapEvent struct { func (e *TableMapEvent) Decode(data []byte) error { pos := 0 - e.TableID = FixedLengthInt(data[0:e.tableIDSize]) + e.TableID = mysql.FixedLengthInt(data[0:e.tableIDSize]) pos += e.tableIDSize e.Flags = binary.LittleEndian.Uint16(data[pos:]) @@ -115,7 +116,7 @@ func (e *TableMapEvent) Decode(data []byte) error { pos++ var n int - e.ColumnCount, _, n = LengthEncodedInt(data[pos:]) + e.ColumnCount, _, n = mysql.LengthEncodedInt(data[pos:]) pos += n e.ColumnType = data[pos : pos+int(e.ColumnCount)] @@ -123,7 +124,7 @@ func (e *TableMapEvent) Decode(data []byte) error { var err error var metaData []byte - if metaData, _, n, err = LengthEncodedString(data[pos:]); err != nil { + if metaData, _, n, err = mysql.LengthEncodedString(data[pos:]); err != nil { return errors.Trace(err) } @@ -180,6 +181,7 @@ func bitmapByteSize(columnCount int) int { MYSQL_TYPE_DOUBLE MYSQL_TYPE_BLOB MYSQL_TYPE_GEOMETRY + MYSQL_TYPE_VECTOR //maybe MYSQL_TYPE_TIME2 @@ -206,39 +208,40 @@ func (e *TableMapEvent) decodeMeta(data []byte) error { e.ColumnMeta = make([]uint16, e.ColumnCount) for i, t := range e.ColumnType { switch t { - case MYSQL_TYPE_STRING: - var x = uint16(data[pos]) << 8 // real type - x += uint16(data[pos+1]) // pack or field length + case mysql.MYSQL_TYPE_STRING: + x := uint16(data[pos]) << 8 // real type + x += uint16(data[pos+1]) // pack or field length e.ColumnMeta[i] = x pos += 2 - case MYSQL_TYPE_NEWDECIMAL: - var x = uint16(data[pos]) << 8 // precision - x += uint16(data[pos+1]) // decimals + case mysql.MYSQL_TYPE_NEWDECIMAL: + x := uint16(data[pos]) << 8 // precision + x += uint16(data[pos+1]) // decimals e.ColumnMeta[i] = x pos += 2 - case MYSQL_TYPE_VAR_STRING, - MYSQL_TYPE_VARCHAR, - MYSQL_TYPE_BIT: + case mysql.MYSQL_TYPE_VAR_STRING, + mysql.MYSQL_TYPE_VARCHAR, + mysql.MYSQL_TYPE_BIT: e.ColumnMeta[i] = binary.LittleEndian.Uint16(data[pos:]) pos += 2 - case MYSQL_TYPE_BLOB, - MYSQL_TYPE_DOUBLE, - MYSQL_TYPE_FLOAT, - MYSQL_TYPE_GEOMETRY, - MYSQL_TYPE_JSON: + case mysql.MYSQL_TYPE_BLOB, + mysql.MYSQL_TYPE_DOUBLE, + mysql.MYSQL_TYPE_FLOAT, + mysql.MYSQL_TYPE_GEOMETRY, + mysql.MYSQL_TYPE_VECTOR, + mysql.MYSQL_TYPE_JSON: e.ColumnMeta[i] = uint16(data[pos]) pos++ - case MYSQL_TYPE_TIME2, - MYSQL_TYPE_DATETIME2, - MYSQL_TYPE_TIMESTAMP2: + case mysql.MYSQL_TYPE_TIME2, + mysql.MYSQL_TYPE_DATETIME2, + mysql.MYSQL_TYPE_TIMESTAMP2: e.ColumnMeta[i] = uint16(data[pos]) pos++ - case MYSQL_TYPE_NEWDATE, - MYSQL_TYPE_ENUM, - MYSQL_TYPE_SET, - MYSQL_TYPE_TINY_BLOB, - MYSQL_TYPE_MEDIUM_BLOB, - MYSQL_TYPE_LONG_BLOB: + case mysql.MYSQL_TYPE_NEWDATE, + mysql.MYSQL_TYPE_ENUM, + mysql.MYSQL_TYPE_SET, + mysql.MYSQL_TYPE_TINY_BLOB, + mysql.MYSQL_TYPE_MEDIUM_BLOB, + mysql.MYSQL_TYPE_LONG_BLOB: return errors.Errorf("unsupport type in binlog %d", t) default: e.ColumnMeta[i] = 0 @@ -256,7 +259,7 @@ func (e *TableMapEvent) decodeOptionalMeta(data []byte) (err error) { t := data[pos] pos++ - l, _, n := LengthEncodedInt(data[pos:]) + l, _, n := mysql.LengthEncodedInt(data[pos:]) pos += n v := data[pos : pos+int(l)] @@ -337,22 +340,22 @@ func (e *TableMapEvent) decodeOptionalMeta(data []byte) (err error) { func (e *TableMapEvent) decodeIntSeq(v []byte) (ret []uint64, err error) { p := 0 for p < len(v) { - i, _, n := LengthEncodedInt(v[p:]) + i, _, n := mysql.LengthEncodedInt(v[p:]) p += n ret = append(ret, i) } - return + return ret, err } func (e *TableMapEvent) decodeDefaultCharset(v []byte) (ret []uint64, err error) { ret, err = e.decodeIntSeq(v) if err != nil { - return + return ret, err } if len(ret)%2 != 1 { return nil, errors.Errorf("Expect odd item in DefaultCharset but got %d", len(ret)) } - return + return ret, err } func (e *TableMapEvent) decodeColumnNames(v []byte) error { @@ -374,11 +377,11 @@ func (e *TableMapEvent) decodeColumnNames(v []byte) error { func (e *TableMapEvent) decodeStrValue(v []byte) (ret [][][]byte, err error) { p := 0 for p < len(v) { - nVal, _, n := LengthEncodedInt(v[p:]) + nVal, _, n := mysql.LengthEncodedInt(v[p:]) p += n vals := make([][]byte, 0, int(nVal)) for i := 0; i < int(nVal); i++ { - val, _, n, err := LengthEncodedString(v[p:]) + val, _, n, err := mysql.LengthEncodedString(v[p:]) if err != nil { return nil, err } @@ -387,13 +390,13 @@ func (e *TableMapEvent) decodeStrValue(v []byte) (ret [][][]byte, err error) { } ret = append(ret, vals) } - return + return ret, err } func (e *TableMapEvent) decodeSimplePrimaryKey(v []byte) error { p := 0 for p < len(v) { - i, _, n := LengthEncodedInt(v[p:]) + i, _, n := mysql.LengthEncodedInt(v[p:]) e.PrimaryKey = append(e.PrimaryKey, i) e.PrimaryKeyPrefix = append(e.PrimaryKeyPrefix, 0) p += n @@ -404,10 +407,10 @@ func (e *TableMapEvent) decodeSimplePrimaryKey(v []byte) error { func (e *TableMapEvent) decodePrimaryKeyWithPrefix(v []byte) error { p := 0 for p < len(v) { - i, _, n := LengthEncodedInt(v[p:]) + i, _, n := mysql.LengthEncodedInt(v[p:]) e.PrimaryKey = append(e.PrimaryKey, i) p += n - i, _, n = LengthEncodedInt(v[p:]) + i, _, n = mysql.LengthEncodedInt(v[p:]) e.PrimaryKeyPrefix = append(e.PrimaryKeyPrefix, i) p += n } @@ -558,7 +561,7 @@ func (e *TableMapEvent) Dump(w io.Writer) { // i must be in range [0, ColumnCount). func (e *TableMapEvent) Nullable(i int) (available, nullable bool) { if len(e.NullBitmap) == 0 { - return + return available, nullable } return true, e.NullBitmap[i/8]&(1<>= 1 { @@ -776,14 +779,14 @@ func (e *TableMapEvent) realType(i int) byte { typ := e.ColumnType[i] switch typ { - case MYSQL_TYPE_STRING: + case mysql.MYSQL_TYPE_STRING: rtyp := byte(e.ColumnMeta[i] >> 8) - if rtyp == MYSQL_TYPE_ENUM || rtyp == MYSQL_TYPE_SET { + if rtyp == mysql.MYSQL_TYPE_ENUM || rtyp == mysql.MYSQL_TYPE_SET { return rtyp } - case MYSQL_TYPE_DATE: - return MYSQL_TYPE_NEWDATE + case mysql.MYSQL_TYPE_DATE: + return mysql.MYSQL_TYPE_NEWDATE } return typ @@ -791,14 +794,16 @@ func (e *TableMapEvent) realType(i int) byte { func (e *TableMapEvent) IsNumericColumn(i int) bool { switch e.realType(i) { - case MYSQL_TYPE_TINY, - MYSQL_TYPE_SHORT, - MYSQL_TYPE_INT24, - MYSQL_TYPE_LONG, - MYSQL_TYPE_LONGLONG, - MYSQL_TYPE_NEWDECIMAL, - MYSQL_TYPE_FLOAT, - MYSQL_TYPE_DOUBLE: + case mysql.MYSQL_TYPE_TINY, + mysql.MYSQL_TYPE_SHORT, + mysql.MYSQL_TYPE_INT24, + mysql.MYSQL_TYPE_LONG, + mysql.MYSQL_TYPE_LONGLONG, + mysql.MYSQL_TYPE_DECIMAL, + mysql.MYSQL_TYPE_NEWDECIMAL, + mysql.MYSQL_TYPE_FLOAT, + mysql.MYSQL_TYPE_DOUBLE, + mysql.MYSQL_TYPE_YEAR: return true default: @@ -811,13 +816,13 @@ func (e *TableMapEvent) IsNumericColumn(i int) bool { // (JSON is an alias for LONGTEXT in mariadb: https://mariadb.com/kb/en/json-data-type/) func (e *TableMapEvent) IsCharacterColumn(i int) bool { switch e.realType(i) { - case MYSQL_TYPE_STRING, - MYSQL_TYPE_VAR_STRING, - MYSQL_TYPE_VARCHAR, - MYSQL_TYPE_BLOB: + case mysql.MYSQL_TYPE_STRING, + mysql.MYSQL_TYPE_VAR_STRING, + mysql.MYSQL_TYPE_VARCHAR, + mysql.MYSQL_TYPE_BLOB: return true - case MYSQL_TYPE_GEOMETRY: + case mysql.MYSQL_TYPE_GEOMETRY: if e.flavor == "mariadb" { return true } @@ -829,27 +834,29 @@ func (e *TableMapEvent) IsCharacterColumn(i int) bool { } func (e *TableMapEvent) IsEnumColumn(i int) bool { - return e.realType(i) == MYSQL_TYPE_ENUM + return e.realType(i) == mysql.MYSQL_TYPE_ENUM } func (e *TableMapEvent) IsSetColumn(i int) bool { - return e.realType(i) == MYSQL_TYPE_SET + return e.realType(i) == mysql.MYSQL_TYPE_SET } func (e *TableMapEvent) IsGeometryColumn(i int) bool { - return e.realType(i) == MYSQL_TYPE_GEOMETRY + return e.realType(i) == mysql.MYSQL_TYPE_GEOMETRY } func (e *TableMapEvent) IsEnumOrSetColumn(i int) bool { rtyp := e.realType(i) - return rtyp == MYSQL_TYPE_ENUM || rtyp == MYSQL_TYPE_SET + return rtyp == mysql.MYSQL_TYPE_ENUM || rtyp == mysql.MYSQL_TYPE_SET } // JsonColumnCount returns the number of JSON columns in this table +// +//nolint:revive // exported method renamed would be a breaking API change func (e *TableMapEvent) JsonColumnCount() uint64 { count := uint64(0) for _, t := range e.ColumnType { - if t == MYSQL_TYPE_JSON { + if t == mysql.MYSQL_TYPE_JSON { count++ } } @@ -864,32 +871,33 @@ const RowsEventStmtEndFlag = 0x01 // UPDATE_ROWS_EVENT, etc. // RowsEvent.Rows saves the rows data, and the MySQL type to golang type mapping // is -// - MYSQL_TYPE_NULL: nil -// - MYSQL_TYPE_LONG: int32 -// - MYSQL_TYPE_TINY: int8 -// - MYSQL_TYPE_SHORT: int16 -// - MYSQL_TYPE_INT24: int32 -// - MYSQL_TYPE_LONGLONG: int64 -// - MYSQL_TYPE_NEWDECIMAL: string / "github.com/shopspring/decimal".Decimal -// - MYSQL_TYPE_FLOAT: float32 -// - MYSQL_TYPE_DOUBLE: float64 -// - MYSQL_TYPE_BIT: int64 -// - MYSQL_TYPE_TIMESTAMP: string / time.Time -// - MYSQL_TYPE_TIMESTAMP2: string / time.Time -// - MYSQL_TYPE_DATETIME: string / time.Time -// - MYSQL_TYPE_DATETIME2: string / time.Time -// - MYSQL_TYPE_TIME: string -// - MYSQL_TYPE_TIME2: string -// - MYSQL_TYPE_DATE: string -// - MYSQL_TYPE_YEAR: int -// - MYSQL_TYPE_ENUM: int64 -// - MYSQL_TYPE_SET: int64 -// - MYSQL_TYPE_BLOB: []byte -// - MYSQL_TYPE_VARCHAR: string -// - MYSQL_TYPE_VAR_STRING: string -// - MYSQL_TYPE_STRING: string -// - MYSQL_TYPE_JSON: []byte / *replication.JsonDiff -// - MYSQL_TYPE_GEOMETRY: []byte +// - mysql.MYSQL_TYPE_NULL: nil +// - mysql.MYSQL_TYPE_LONG: int32 +// - mysql.MYSQL_TYPE_TINY: int8 +// - mysql.MYSQL_TYPE_SHORT: int16 +// - mysql.MYSQL_TYPE_INT24: int32 +// - mysql.MYSQL_TYPE_LONGLONG: int64 +// - mysql.MYSQL_TYPE_NEWDECIMAL: string / "github.com/shopspring/decimal".Decimal +// - mysql.MYSQL_TYPE_FLOAT: float32 +// - mysql.MYSQL_TYPE_DOUBLE: float64 +// - mysql.MYSQL_TYPE_BIT: int64 +// - mysql.MYSQL_TYPE_TIMESTAMP: string / time.Time +// - mysql.MYSQL_TYPE_TIMESTAMP2: string / time.Time +// - mysql.MYSQL_TYPE_DATETIME: string / time.Time +// - mysql.MYSQL_TYPE_DATETIME2: string / time.Time +// - mysql.MYSQL_TYPE_TIME: string +// - mysql.MYSQL_TYPE_TIME2: string +// - mysql.MYSQL_TYPE_DATE: string +// - mysql.MYSQL_TYPE_YEAR: int +// - mysql.MYSQL_TYPE_ENUM: int64 +// - mysql.MYSQL_TYPE_SET: int64 +// - mysql.MYSQL_TYPE_BLOB: []byte +// - mysql.MYSQL_TYPE_VARCHAR: string +// - mysql.MYSQL_TYPE_VAR_STRING: string +// - mysql.MYSQL_TYPE_STRING: string +// - mysql.MYSQL_TYPE_JSON: []byte / *replication.JsonDiff +// - mysql.MYSQL_TYPE_GEOMETRY: []byte +// - mysql.MYSQL_TYPE_VECTOR: []byte type RowsEvent struct { // 0, 1, 2 Version int @@ -901,6 +909,7 @@ type RowsEvent struct { // for mariadb *_COMPRESSED_EVENT_V1 compressed bool + // raw event type associated with a RowsEvent eventType EventType Table *TableMapEvent @@ -914,8 +923,8 @@ type RowsEvent struct { NdbFormat byte NdbData []byte - PartitionId uint16 - SourcePartitionId uint16 + PartitionId uint16 //nolint:revive // exported field renamed would be a breaking API change + SourcePartitionId uint16 //nolint:revive // exported field renamed would be a breaking API change // lenenc_int ColumnCount uint64 @@ -937,13 +946,37 @@ type RowsEvent struct { ColumnBitmap2 []byte // rows: all return types from RowsEvent.decodeValue() - Rows [][]interface{} + Rows [][]any SkippedColumns [][]int - parseTime bool - timestampStringLocation *time.Location - useDecimal bool - ignoreJSONDecodeErr bool + parseTime bool + timestampStringLocation *time.Location + useDecimal bool + useFloatWithTrailingZero bool + ignoreJSONDecodeErr bool +} + +// EnumRowsEventType is an abridged type describing the operation which triggered the given RowsEvent. +type EnumRowsEventType byte + +const ( + EnumRowsEventTypeUnknown = EnumRowsEventType(iota) + EnumRowsEventTypeInsert + EnumRowsEventTypeUpdate + EnumRowsEventTypeDelete +) + +func (t EnumRowsEventType) String() string { + switch t { + case EnumRowsEventTypeInsert: + return "insert" + case EnumRowsEventTypeUpdate: + return "update" + case EnumRowsEventTypeDelete: + return "delete" + default: + return fmt.Sprintf("unknown (%d)", t) + } } // EnumRowImageType is allowed types for every row in mysql binlog. @@ -976,6 +1009,7 @@ func (t EnumRowImageType) String() string { // Bits for binlog_row_value_options sysvar type EnumBinlogRowValueOptions byte +//nolint:revive // exported constant renamed would be a breaking API change const ( // Store JSON updates in partial form EnumBinlogRowValueOptionsPartialJsonUpdates = EnumBinlogRowValueOptions(iota + 1) @@ -983,7 +1017,7 @@ const ( func (e *RowsEvent) DecodeHeader(data []byte) (int, error) { pos := 0 - e.TableID = FixedLengthInt(data[0:e.tableIDSize]) + e.TableID = mysql.FixedLengthInt(data[0:e.tableIDSize]) pos += e.tableIDSize e.Flags = binary.LittleEndian.Uint16(data[pos:]) @@ -1002,7 +1036,7 @@ func (e *RowsEvent) DecodeHeader(data []byte) (int, error) { } var n int - e.ColumnCount, _, n = LengthEncodedInt(data[pos:]) + e.ColumnCount, _, n = mysql.LengthEncodedInt(data[pos:]) pos += n bitCount := bitmapByteSize(int(e.ColumnCount)) @@ -1019,9 +1053,8 @@ func (e *RowsEvent) DecodeHeader(data []byte) (int, error) { if !ok { if len(e.tables) > 0 { return 0, errors.Errorf("invalid table id %d, no corresponding table map event", e.TableID) - } else { - return 0, errors.Annotatef(errMissingTableMapEvent, "table id %d", e.TableID) } + return 0, errors.Annotatef(errMissingTableMapEvent, "table id %d", e.TableID) } return pos, nil } @@ -1029,13 +1062,13 @@ func (e *RowsEvent) DecodeHeader(data []byte) (int, error) { func (e *RowsEvent) decodeExtraData(data []byte) (err2 error) { pos := 0 extraDataType := data[pos] - pos += 1 + pos++ switch extraDataType { case ENUM_EXTRA_ROW_INFO_TYPECODE_NDB: - var ndbLength int = int(data[pos]) - pos += 1 + ndbLength := int(data[pos]) + pos++ e.NdbFormat = data[pos] - pos += 1 + pos++ e.NdbData = data[pos : pos+ndbLength-2] case ENUM_EXTRA_ROW_INFO_TYPECODE_PARTITION: if e.eventType == UPDATE_ROWS_EVENTv1 || e.eventType == UPDATE_ROWS_EVENTv2 || e.eventType == PARTIAL_UPDATE_ROWS_EVENT { @@ -1051,11 +1084,11 @@ func (e *RowsEvent) decodeExtraData(data []byte) (err2 error) { func (e *RowsEvent) DecodeData(pos int, data []byte) (err2 error) { if e.compressed { - data, err2 = DecompressMariadbData(data[pos:]) + data, err2 = mysql.DecompressMariadbData(data[pos:]) if err2 != nil { - //nolint:nakedret - return + return err2 } + pos = 0 } // Rows_log_event::print_verbose() @@ -1077,7 +1110,7 @@ func (e *RowsEvent) DecodeData(pos int, data []byte) (err2 error) { rowsLen++ } e.SkippedColumns = make([][]int, 0, rowsLen) - e.Rows = make([][]interface{}, 0, rowsLen) + e.Rows = make([][]any, 0, rowsLen) var rowImageType EnumRowImageType switch e.eventType { @@ -1116,6 +1149,19 @@ func (e *RowsEvent) Decode(data []byte) error { return e.DecodeData(pos, data) } +func (e *RowsEvent) Type() EnumRowsEventType { + switch e.eventType { + case WRITE_ROWS_EVENTv0, WRITE_ROWS_EVENTv1, WRITE_ROWS_EVENTv2, MARIADB_WRITE_ROWS_COMPRESSED_EVENT_V1: + return EnumRowsEventTypeInsert + case UPDATE_ROWS_EVENTv0, UPDATE_ROWS_EVENTv1, UPDATE_ROWS_EVENTv2, MARIADB_UPDATE_ROWS_COMPRESSED_EVENT_V1: + return EnumRowsEventTypeUpdate + case DELETE_ROWS_EVENTv0, DELETE_ROWS_EVENTv1, DELETE_ROWS_EVENTv2, MARIADB_DELETE_ROWS_COMPRESSED_EVENT_V1: + return EnumRowsEventTypeDelete + default: + return EnumRowsEventTypeUnknown + } +} + func isBitSet(bitmap []byte, i int) bool { return bitmap[i>>3]&(1<<(uint(i)&7)) > 0 } @@ -1131,30 +1177,33 @@ func (e *RowsEvent) decodeImage(data []byte, bitmap []byte, rowImageType EnumRow pos := 0 - var isPartialJsonUpdate bool + var isPartialJSONUpdate bool var partialBitmap []byte if e.eventType == PARTIAL_UPDATE_ROWS_EVENT && rowImageType == EnumRowImageTypeUpdateAI { - binlogRowValueOptions, _, n := LengthEncodedInt(data[pos:]) // binlog_row_value_options + binlogRowValueOptions, _, n := mysql.LengthEncodedInt(data[pos:]) // binlog_row_value_options pos += n - isPartialJsonUpdate = EnumBinlogRowValueOptions(binlogRowValueOptions)&EnumBinlogRowValueOptionsPartialJsonUpdates != 0 - if isPartialJsonUpdate { + isPartialJSONUpdate = EnumBinlogRowValueOptions(binlogRowValueOptions)&EnumBinlogRowValueOptionsPartialJsonUpdates != 0 + if isPartialJSONUpdate { byteCount := bitmapByteSize(int(e.Table.JsonColumnCount())) partialBitmap = data[pos : pos+byteCount] pos += byteCount } } - row := make([]interface{}, e.ColumnCount) - skips := make([]int, 0) + row := make([]any, e.ColumnCount) + unsignedMap := e.Table.UnsignedMap() // refer: https://github.com/alibaba/canal/blob/c3e38e50e269adafdd38a48c63a1740cde304c67/dbsync/src/main/java/com/taobao/tddl/dbsync/binlog/event/RowsLogBuffer.java#L63 count := 0 - for i := 0; i < int(e.ColumnCount); i++ { - if isBitSet(bitmap, i) { - count++ - } + col := 0 + for ; col+8 <= int(e.ColumnCount); col += 8 { + count += bits.OnesCount8(bitmap[col>>3]) + } + if col < int(e.ColumnCount) { + count += bits.OnesCount8(bitmap[col>>3] & byte((1<<(int(e.ColumnCount)-col))-1)) } + skips := make([]int, 0, int(e.ColumnCount)-count) count = bitmapByteSize(count) nullBitmap := data[pos : pos+count] @@ -1169,9 +1218,9 @@ func (e *RowsEvent) decodeImage(data []byte, bitmap []byte, rowImageType EnumRow the partial_bits bitmap has a bit for every JSON column regardless of whether it is included in the bitmap or not. */ - isPartial := isPartialJsonUpdate && + isPartial := isPartialJSONUpdate && (rowImageType == EnumRowImageTypeUpdateAI) && - (e.Table.ColumnType[i] == MYSQL_TYPE_JSON) && + (e.Table.ColumnType[i] == mysql.MYSQL_TYPE_JSON) && isBitSetIncr(partialBitmap, &partialBitmapIndex) if !isBitSet(bitmap, i) { @@ -1186,8 +1235,7 @@ func (e *RowsEvent) decodeImage(data []byte, bitmap []byte, rowImageType EnumRow var n int var err error - row[i], n, err = e.decodeValue(data[pos:], e.Table.ColumnType[i], e.Table.ColumnMeta[i], isPartial) - + row[i], n, err = e.decodeValue(data[pos:], e.Table.ColumnType[i], e.Table.ColumnMeta[i], isPartial, unsignedMap[i]) if err != nil { return 0, err } @@ -1199,7 +1247,7 @@ func (e *RowsEvent) decodeImage(data []byte, bitmap []byte, rowImageType EnumRow return pos, nil } -func (e *RowsEvent) parseFracTime(t interface{}) interface{} { +func (e *RowsEvent) parseFracTime(t any) any { v, ok := t.(fracTime) if !ok { return t @@ -1215,10 +1263,10 @@ func (e *RowsEvent) parseFracTime(t interface{}) interface{} { } // see mysql sql/log_event.cc log_event_print_value -func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16, isPartial bool) (v interface{}, n int, err error) { - var length = 0 +func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16, isPartial bool, isUnsigned bool) (v any, n int, err error) { + length := 0 - if tp == MYSQL_TYPE_STRING { + if tp == mysql.MYSQL_TYPE_STRING { if meta >= 256 { b0 := uint8(meta >> 8) b1 := uint8(meta & 0xFF) @@ -1236,44 +1284,64 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16, isPartial boo } switch tp { - case MYSQL_TYPE_NULL: + case mysql.MYSQL_TYPE_NULL: return nil, 0, nil - case MYSQL_TYPE_LONG: + case mysql.MYSQL_TYPE_LONG: n = 4 - v = ParseBinaryInt32(data) - case MYSQL_TYPE_TINY: + if isUnsigned { + v = mysql.ParseBinaryUint32(data) + } else { + v = mysql.ParseBinaryInt32(data) + } + case mysql.MYSQL_TYPE_TINY: n = 1 - v = ParseBinaryInt8(data) - case MYSQL_TYPE_SHORT: + if isUnsigned { + v = mysql.ParseBinaryUint8(data) + } else { + v = mysql.ParseBinaryInt8(data) + } + case mysql.MYSQL_TYPE_SHORT: n = 2 - v = ParseBinaryInt16(data) - case MYSQL_TYPE_INT24: + if isUnsigned { + v = mysql.ParseBinaryUint16(data) + } else { + v = mysql.ParseBinaryInt16(data) + } + case mysql.MYSQL_TYPE_INT24: n = 3 - v = ParseBinaryInt24(data) - case MYSQL_TYPE_LONGLONG: + if isUnsigned { + v = mysql.ParseBinaryUint24(data) + } else { + v = mysql.ParseBinaryInt24(data) + } + case mysql.MYSQL_TYPE_LONGLONG: n = 8 - v = ParseBinaryInt64(data) - case MYSQL_TYPE_NEWDECIMAL: + if isUnsigned { + v = mysql.ParseBinaryUint64(data) + } else { + v = mysql.ParseBinaryInt64(data) + } + case mysql.MYSQL_TYPE_NEWDECIMAL: prec := uint8(meta >> 8) scale := uint8(meta & 0xFF) v, n, err = decodeDecimal(data, int(prec), int(scale), e.useDecimal) - case MYSQL_TYPE_FLOAT: + case mysql.MYSQL_TYPE_FLOAT: n = 4 - v = ParseBinaryFloat32(data) - case MYSQL_TYPE_DOUBLE: + v = mysql.ParseBinaryFloat32(data) + case mysql.MYSQL_TYPE_DOUBLE: n = 8 - v = ParseBinaryFloat64(data) - case MYSQL_TYPE_BIT: + v = mysql.ParseBinaryFloat64(data) + case mysql.MYSQL_TYPE_BIT: nbits := ((meta >> 8) * 8) + (meta & 0xFF) n = int(nbits+7) / 8 // use int64 for bit v, err = decodeBit(data, int(nbits), n) - case MYSQL_TYPE_TIMESTAMP: + case mysql.MYSQL_TYPE_TIMESTAMP: n = 4 t := binary.LittleEndian.Uint32(data) if t == 0 { - v = formatZeroTime(0, 0) + v = "0000-00-00 00:00:00" } else { v = e.parseFracTime(fracTime{ Time: time.Unix(int64(t), 0), @@ -1281,54 +1349,65 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16, isPartial boo timestampStringLocation: e.timestampStringLocation, }) } - case MYSQL_TYPE_TIMESTAMP2: + case mysql.MYSQL_TYPE_TIMESTAMP2: v, n, err = decodeTimestamp2(data, meta, e.timestampStringLocation) v = e.parseFracTime(v) - case MYSQL_TYPE_DATETIME: + case mysql.MYSQL_TYPE_DATETIME: n = 8 i64 := binary.LittleEndian.Uint64(data) if i64 == 0 { - v = formatZeroTime(0, 0) + v = "0000-00-00 00:00:00" } else { d := i64 / 1000000 t := i64 % 1000000 - v = e.parseFracTime(fracTime{ - Time: time.Date( - int(d/10000), - time.Month((d%10000)/100), - int(d%100), - int(t/10000), - int((t%10000)/100), - int(t%100), - 0, - time.UTC, - ), - Dec: 0, - }) + years := int(d / 10000) + months := int(d%10000) / 100 + days := int(d % 100) + hours := int(t / 10000) + minutes := int(t%10000) / 100 + seconds := int(t % 100) + if !e.parseTime || months == 0 || days == 0 { + v = fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", + years, months, days, hours, minutes, seconds) + } else { + v = e.parseFracTime(fracTime{ + Time: time.Date( + years, + time.Month(months), + days, + hours, + minutes, + seconds, + 0, + time.UTC, + ), + Dec: 0, + }) + } } - case MYSQL_TYPE_DATETIME2: - v, n, err = decodeDatetime2(data, meta) + case mysql.MYSQL_TYPE_DATETIME2: + v, n, err = decodeDatetime2(data, meta, e.parseTime) v = e.parseFracTime(v) - case MYSQL_TYPE_TIME: + case mysql.MYSQL_TYPE_TIME: n = 3 - i32 := uint32(FixedLengthInt(data[0:3])) + i32 := uint32(mysql.FixedLengthInt(data[0:3])) if i32 == 0 { v = "00:00:00" } else { v = fmt.Sprintf("%02d:%02d:%02d", i32/10000, (i32%10000)/100, i32%100) } - case MYSQL_TYPE_TIME2: + case mysql.MYSQL_TYPE_TIME2: v, n, err = decodeTime2(data, meta) - case MYSQL_TYPE_DATE: + case mysql.MYSQL_TYPE_DATE: n = 3 - i32 := uint32(FixedLengthInt(data[0:3])) + i32 := uint32(mysql.FixedLengthInt(data[0:3])) if i32 == 0 { v = "0000-00-00" } else { v = fmt.Sprintf("%04d-%02d-%02d", i32/(16*32), i32/32%16, i32%32) } - case MYSQL_TYPE_YEAR: + case mysql.MYSQL_TYPE_YEAR: n = 1 year := int(data[0]) if year == 0 { @@ -1336,7 +1415,7 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16, isPartial boo } else { v = year + 1900 } - case MYSQL_TYPE_ENUM: + case mysql.MYSQL_TYPE_ENUM: l := meta & 0xFF switch l { case 1: @@ -1346,24 +1425,24 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16, isPartial boo v = int64(binary.LittleEndian.Uint16(data)) n = 2 default: - err = fmt.Errorf("Unknown ENUM packlen=%d", l) + err = fmt.Errorf("unknown ENUM packlen=%d", l) } - case MYSQL_TYPE_SET: + case mysql.MYSQL_TYPE_SET: n = int(meta & 0xFF) nbits := n * 8 v, err = littleDecodeBit(data, nbits, n) - case MYSQL_TYPE_BLOB: + case mysql.MYSQL_TYPE_BLOB: v, n, err = decodeBlob(data, meta) - case MYSQL_TYPE_VARCHAR, - MYSQL_TYPE_VAR_STRING: + case mysql.MYSQL_TYPE_VARCHAR, + mysql.MYSQL_TYPE_VAR_STRING: length = int(meta) v, n = decodeString(data, length) - case MYSQL_TYPE_STRING: + case mysql.MYSQL_TYPE_STRING: v, n = decodeString(data, length) - case MYSQL_TYPE_JSON: + case mysql.MYSQL_TYPE_JSON: // Refer: https://github.com/shyiko/mysql-binlog-connector-java/blob/master/src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/AbstractRowsEventDataDeserializer.java#L404 - length = int(FixedLengthInt(data[0:meta])) + length = int(mysql.FixedLengthInt(data[0:meta])) n = length + int(meta) /* @@ -1383,21 +1462,21 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16, isPartial boo } else { if isPartial { var diff *JsonDiff - diff, err = e.decodeJsonPartialBinary(data[meta:n]) + diff, err = e.decodeJSONPartialBinary(data[meta:n]) if err == nil { v = diff } else { - fmt.Printf("decodeJsonPartialBinary(%q) fail: %s\n", data[meta:n], err) + fmt.Printf("decodeJSONPartialBinary(%q) fail: %s\n", data[meta:n], err) } } else { var d []byte - d, err = e.decodeJsonBinary(data[meta:n]) + d, err = e.decodeJSONBinary(data[meta:n]) if err == nil { v = utils.ByteSliceToString(d) } } } - case MYSQL_TYPE_GEOMETRY: + case mysql.MYSQL_TYPE_GEOMETRY: // MySQL saves Geometry as Blob in binlog // Seem that the binary format is SRID (4 bytes) + WKB, outer can use // MySQL GeoFromWKB or others to create the geometry data. @@ -1405,6 +1484,8 @@ func (e *RowsEvent) decodeValue(data []byte, tp byte, meta uint16, isPartial boo // I also find some go libs to handle WKB if possible // see https://github.com/twpayne/go-geom or https://github.com/paulmach/go.geo v, n, err = decodeBlob(data, meta) + case mysql.MYSQL_TYPE_VECTOR: + v, n, err = decodeBlob(data, meta) default: err = fmt.Errorf("unsupport type %d in binlog and don't know how to handle", tp) } @@ -1424,7 +1505,7 @@ func decodeString(data []byte, length int) (v string, n int) { v = utils.ByteSliceToString(data[2:n]) } - return + return v, n } // ref: https://github.com/mysql/mysql-server/blob/a9b0c712de3509d8d08d3ba385d41a4df6348775/strings/decimal.c#L137 @@ -1445,12 +1526,12 @@ func decodeDecimalDecompressValue(compIndx int, data []byte, mask uint8) (size i case 4: value = uint32(data[3]^mask) | uint32(data[2]^mask)<<8 | uint32(data[1]^mask)<<16 | uint32(data[0]^mask)<<24 } - return + return size, value } var zeros = [digitsPerInteger]byte{48, 48, 48, 48, 48, 48, 48, 48, 48} -func decodeDecimal(data []byte, precision int, decimals int, useDecimal bool) (interface{}, int, error) { +func decodeDecimal(data []byte, precision int, decimals int, useDecimal bool) (any, int, error) { // see python mysql replication and https://github.com/jeremycole/mysql_binlog integral := precision - decimals uncompIntegral := integral / digitsPerInteger @@ -1473,7 +1554,7 @@ func decodeDecimal(data []byte, precision int, decimals int, useDecimal bool) (i value := uint32(data[0]) var res strings.Builder res.Grow(precision + 2) - var mask uint32 = 0 + var mask uint32 if value&0x80 == 0 { mask = uint32((1 << 32) - 1) res.WriteString("-") @@ -1490,7 +1571,7 @@ func decodeDecimal(data []byte, precision int, decimals int, useDecimal bool) (i res.WriteString(strconv.FormatUint(uint64(value), 10)) } - for i := 0; i < uncompIntegral; i++ { + for range uncompIntegral { value = binary.BigEndian.Uint32(data[pos:]) ^ mask pos += 4 if zeroLeading { @@ -1512,7 +1593,7 @@ func decodeDecimal(data []byte, precision int, decimals int, useDecimal bool) (i if pos < len(data) { res.WriteString(".") - for i := 0; i < uncompFractional; i++ { + for range uncompFractional { value = binary.BigEndian.Uint32(data[pos:]) ^ mask pos += 4 toWrite := strconv.FormatUint(uint64(value), 10) @@ -1547,15 +1628,15 @@ func decodeBit(data []byte, nbits int, length int) (value int64, err error) { case 2: value = int64(binary.BigEndian.Uint16(data)) case 3: - value = int64(BFixedLengthInt(data[0:3])) + value = int64(mysql.BFixedLengthInt(data[0:3])) case 4: value = int64(binary.BigEndian.Uint32(data)) case 5: - value = int64(BFixedLengthInt(data[0:5])) + value = int64(mysql.BFixedLengthInt(data[0:5])) case 6: - value = int64(BFixedLengthInt(data[0:6])) + value = int64(mysql.BFixedLengthInt(data[0:6])) case 7: - value = int64(BFixedLengthInt(data[0:7])) + value = int64(mysql.BFixedLengthInt(data[0:7])) case 8: value = int64(binary.BigEndian.Uint64(data)) default: @@ -1568,7 +1649,7 @@ func decodeBit(data []byte, nbits int, length int) (value int64, err error) { value = int64(data[0]) } } - return + return value, err } func littleDecodeBit(data []byte, nbits int, length int) (value int64, err error) { @@ -1579,15 +1660,15 @@ func littleDecodeBit(data []byte, nbits int, length int) (value int64, err error case 2: value = int64(binary.LittleEndian.Uint16(data)) case 3: - value = int64(FixedLengthInt(data[0:3])) + value = int64(mysql.FixedLengthInt(data[0:3])) case 4: value = int64(binary.LittleEndian.Uint32(data)) case 5: - value = int64(FixedLengthInt(data[0:5])) + value = int64(mysql.FixedLengthInt(data[0:5])) case 6: - value = int64(FixedLengthInt(data[0:6])) + value = int64(mysql.FixedLengthInt(data[0:6])) case 7: - value = int64(FixedLengthInt(data[0:7])) + value = int64(mysql.FixedLengthInt(data[0:7])) case 8: value = int64(binary.LittleEndian.Uint64(data)) default: @@ -1600,10 +1681,10 @@ func littleDecodeBit(data []byte, nbits int, length int) (value int64, err error value = int64(data[0]) } } - return + return value, err } -func decodeTimestamp2(data []byte, dec uint16, timestampStringLocation *time.Location) (interface{}, int, error) { +func decodeTimestamp2(data []byte, dec uint16, timestampStringLocation *time.Location) (any, int, error) { // get timestamp binary length n := int(4 + (dec+1)/2) sec := int64(binary.BigEndian.Uint32(data[0:4])) @@ -1614,7 +1695,7 @@ func decodeTimestamp2(data []byte, dec uint16, timestampStringLocation *time.Loc case 3, 4: usec = int64(binary.BigEndian.Uint16(data[4:])) * 100 case 5, 6: - usec = int64(BFixedLengthInt(data[4:7])) + usec = int64(mysql.BFixedLengthInt(data[4:7])) } if sec == 0 { @@ -1628,14 +1709,15 @@ func decodeTimestamp2(data []byte, dec uint16, timestampStringLocation *time.Loc }, n, nil } +//nolint:revive // DATETIMEF_INT_OFS mirrors the upstream MySQL temporal-encoding constant const DATETIMEF_INT_OFS int64 = 0x8000000000 -func decodeDatetime2(data []byte, dec uint16) (interface{}, int, error) { +func decodeDatetime2(data []byte, dec uint16, parseTime bool) (any, int, error) { // get datetime binary length n := int(5 + (dec+1)/2) - intPart := int64(BFixedLengthInt(data[0:5])) - DATETIMEF_INT_OFS - var frac int64 = 0 + intPart := int64(mysql.BFixedLengthInt(data[0:5])) - DATETIMEF_INT_OFS + var frac int64 switch dec { case 1, 2: @@ -1643,7 +1725,7 @@ func decodeDatetime2(data []byte, dec uint16) (interface{}, int, error) { case 3, 4: frac = int64(binary.BigEndian.Uint16(data[5:7])) * 100 case 5, 6: - frac = int64(BFixedLengthInt(data[5:8])) + frac = int64(mysql.BFixedLengthInt(data[5:8])) } if intPart == 0 { @@ -1680,8 +1762,8 @@ func decodeDatetime2(data []byte, dec uint16) (interface{}, int, error) { // minute = 0 = 0b000000 // second = 0 = 0b000000 // integer value = 0b1100100000010110000100000000000000000 = 107420450816 - if intPart < 107420450816 { - return formatBeforeUnixZeroTime(year, month, day, hour, minute, second, int(frac), int(dec)), n, nil + if !parseTime || intPart < 107420450816 || month == 0 || day == 0 { + return formatDatetime(year, month, day, hour, minute, second, int(frac), int(dec)), n, nil } return fracTime{ @@ -1690,8 +1772,11 @@ func decodeDatetime2(data []byte, dec uint16) (interface{}, int, error) { }, n, nil } -const TIMEF_OFS int64 = 0x800000000000 -const TIMEF_INT_OFS int64 = 0x800000 +//nolint:revive // TIMEF_* mirror the upstream MySQL temporal-encoding constants +const ( + TIMEF_OFS int64 = 0x800000000000 + TIMEF_INT_OFS int64 = 0x800000 +) func decodeTime2(data []byte, dec uint16) (string, int, error) { // time binary length @@ -1702,7 +1787,7 @@ func decodeTime2(data []byte, dec uint16) (string, int, error) { frac := int64(0) switch dec { case 1, 2: - intPart = int64(BFixedLengthInt(data[0:3])) - TIMEF_INT_OFS + intPart = int64(mysql.BFixedLengthInt(data[0:3])) - TIMEF_INT_OFS frac = int64(data[3]) if intPart < 0 && frac != 0 { /* @@ -1727,7 +1812,7 @@ func decodeTime2(data []byte, dec uint16) (string, int, error) { } tmp = intPart<<24 + frac*10000 case 3, 4: - intPart = int64(BFixedLengthInt(data[0:3])) - TIMEF_INT_OFS + intPart = int64(mysql.BFixedLengthInt(data[0:3])) - TIMEF_INT_OFS frac = int64(binary.BigEndian.Uint16(data[3:5])) if intPart < 0 && frac != 0 { /* @@ -1740,10 +1825,10 @@ func decodeTime2(data []byte, dec uint16) (string, int, error) { tmp = intPart<<24 + frac*100 case 5, 6: - tmp = int64(BFixedLengthInt(data[0:6])) - TIMEF_OFS + tmp = int64(mysql.BFixedLengthInt(data[0:6])) - TIMEF_OFS return timeFormat(tmp, dec, n) default: - intPart = int64(BFixedLengthInt(data[0:3])) - TIMEF_INT_OFS + intPart = int64(mysql.BFixedLengthInt(data[0:3])) - TIMEF_INT_OFS tmp = intPart << 24 } @@ -1789,7 +1874,7 @@ func decodeBlob(data []byte, meta uint16) (v []byte, n int, err error) { v = data[2 : 2+length] n = length + 2 case 3: - length = int(FixedLengthInt(data[0:3])) + length = int(mysql.FixedLengthInt(data[0:3])) v = data[3 : 3+length] n = length + 3 case 4: @@ -1800,7 +1885,7 @@ func decodeBlob(data []byte, meta uint16) (v []byte, n int, err error) { err = fmt.Errorf("invalid blob packlen = %d", meta) } - return + return v, n, err } func (e *RowsEvent) Dump(w io.Writer) { @@ -1808,6 +1893,7 @@ func (e *RowsEvent) Dump(w io.Writer) { fmt.Fprintf(w, "Flags: %d\n", e.Flags) fmt.Fprintf(w, "Column count: %d\n", e.ColumnCount) fmt.Fprintf(w, "NDB data: %s\n", e.NdbData) + fmt.Fprintf(w, "Event type: %s (%s)", e.Type(), e.eventType) fmt.Fprintf(w, "Values:\n") for _, rows := range e.Rows { diff --git a/vendor/github.com/go-mysql-org/go-mysql/replication/time.go b/vendor/github.com/go-mysql-org/go-mysql/replication/time.go index 2adc832f4..cfc0a8585 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/replication/time.go +++ b/vendor/github.com/go-mysql-org/go-mysql/replication/time.go @@ -2,13 +2,18 @@ package replication import ( "fmt" - "strings" "time" ) -var ( - fracTimeFormat []string -) +var fracTimeFormat = [7]string{ + "2006-01-02 15:04:05", + "2006-01-02 15:04:05.0", + "2006-01-02 15:04:05.00", + "2006-01-02 15:04:05.000", + "2006-01-02 15:04:05.0000", + "2006-01-02 15:04:05.00000", + "2006-01-02 15:04:05.000000", +} // fracTime is a help structure wrapping Golang Time. type fracTime struct { @@ -39,7 +44,7 @@ func formatZeroTime(frac int, dec int) string { return s[0 : len(s)-(6-dec)] } -func formatBeforeUnixZeroTime(year, month, day, hour, minute, second, frac, dec int) string { +func formatDatetime(year, month, day, hour, minute, second, frac, dec int) string { if dec == 0 { return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second) } @@ -56,12 +61,3 @@ func microSecTimestampToTime(ts uint64) time.Time { } return time.Unix(int64(ts/1000000), int64(ts%1000000)*1000) } - -func init() { - fracTimeFormat = make([]string, 7) - fracTimeFormat[0] = "2006-01-02 15:04:05" - - for i := 1; i <= 6; i++ { - fracTimeFormat[i] = fmt.Sprintf("2006-01-02 15:04:05.%s", strings.Repeat("0", i)) - } -} diff --git a/vendor/github.com/go-mysql-org/go-mysql/replication/transaction_payload_event.go b/vendor/github.com/go-mysql-org/go-mysql/replication/transaction_payload_event.go index 58826143d..aab9d6a0f 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/replication/transaction_payload_event.go +++ b/vendor/github.com/go-mysql-org/go-mysql/replication/transaction_payload_event.go @@ -6,14 +6,15 @@ import ( "fmt" "io" + "github.com/go-mysql-org/go-mysql/mysql" "github.com/klauspost/compress/zstd" - - . "github.com/go-mysql-org/go-mysql/mysql" ) // On The Wire: Field Types // See also binary_log::codecs::binary::Transaction_payload::fields in MySQL // https://dev.mysql.com/doc/dev/mysql-server/latest/classbinary__log_1_1codecs_1_1binary_1_1Transaction__payload.html#a9fff7ac12ba064f40e9216565c53d07b +// +//nolint:revive // OTW_PAYLOAD_* names mirror the upstream MySQL binlog protocol const ( OTW_PAYLOAD_HEADER_END_MARK = iota OTW_PAYLOAD_SIZE_FIELD @@ -29,6 +30,7 @@ const ( type TransactionPayloadEvent struct { format FormatDescriptionEvent + concurrency int Size uint64 UncompressedSize uint64 CompressionType uint64 @@ -72,27 +74,26 @@ func (e *TransactionPayloadEvent) decodeFields(data []byte) error { offset := uint64(0) for { - fieldType := FixedLengthInt(data[offset : offset+1]) + fieldType := mysql.FixedLengthInt(data[offset : offset+1]) offset++ if fieldType == OTW_PAYLOAD_HEADER_END_MARK { e.Payload = data[offset:] break - } else { - fieldLength := FixedLengthInt(data[offset : offset+1]) - offset++ - - switch fieldType { - case OTW_PAYLOAD_SIZE_FIELD: - e.Size = FixedLengthInt(data[offset : offset+fieldLength]) - case OTW_PAYLOAD_COMPRESSION_TYPE_FIELD: - e.CompressionType = FixedLengthInt(data[offset : offset+fieldLength]) - case OTW_PAYLOAD_UNCOMPRESSED_SIZE_FIELD: - e.UncompressedSize = FixedLengthInt(data[offset : offset+fieldLength]) - } - - offset += fieldLength } + fieldLength := mysql.FixedLengthInt(data[offset : offset+1]) + offset++ + + switch fieldType { + case OTW_PAYLOAD_SIZE_FIELD: + e.Size = mysql.FixedLengthInt(data[offset : offset+fieldLength]) + case OTW_PAYLOAD_COMPRESSION_TYPE_FIELD: + e.CompressionType = mysql.FixedLengthInt(data[offset : offset+fieldLength]) + case OTW_PAYLOAD_UNCOMPRESSED_SIZE_FIELD: + e.UncompressedSize = mysql.FixedLengthInt(data[offset : offset+fieldLength]) + } + + offset += fieldLength } return nil @@ -104,7 +105,7 @@ func (e *TransactionPayloadEvent) decodePayload() error { e.CompressionType, e.compressionType()) } - var decoder, err = zstd.NewReader(nil, zstd.WithDecoderConcurrency(0)) + decoder, err := zstd.NewReader(nil, zstd.WithDecoderConcurrency(e.concurrency)) if err != nil { return err } diff --git a/vendor/github.com/go-mysql-org/go-mysql/serialization/serialization.go b/vendor/github.com/go-mysql-org/go-mysql/serialization/serialization.go new file mode 100644 index 000000000..c254fa515 --- /dev/null +++ b/vendor/github.com/go-mysql-org/go-mysql/serialization/serialization.go @@ -0,0 +1,265 @@ +// Package serialization is for working with the mysql::serialization format +// +// mysql::serialization is a serialization format introduced with tagged GTIDs +// +// https://dev.mysql.com/doc/dev/mysql-server/latest/PageLibsMysqlSerialization.html +package serialization + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math/bits" + "strings" +) + +// Message is a mysql::serialization message +type Message struct { + Version uint8 // >= 0 + Format Format + fieldIndex map[string]uint8 +} + +func (m *Message) String() string { + parts := []string{fmt.Sprintf("Message (version: %d)", m.Version)} + parts = append(parts, m.Format.stringParts()...) + return strings.Join(parts, "\n ") +} + +// GetFieldByName returns a field if the name matches and an error if there is no match +func (m *Message) GetFieldByName(name string) (Field, error) { + if idx, ok := m.fieldIndex[name]; ok { + return m.Format.Fields[idx], nil + } + return Field{}, fmt.Errorf("field not found: %s", name) +} + +// Format is describing a `message_format` +type Format struct { + Size uint8 + LastNonIgnorableField uint8 + Fields []Field +} + +func (f *Format) String() (text string) { + return strings.Join(f.stringParts(), "\n") +} + +func (f *Format) stringParts() (parts []string) { + parts = make([]string, 0, len(f.Fields)*2+1) + parts = append(parts, fmt.Sprintf("Format (Size: %d, LastNonIgnorableField: %d)", + f.Size, f.LastNonIgnorableField)) + + for _, f := range f.Fields { + parts = append(parts, fmt.Sprintf("Field %02d (Name: %s, Skipped: %t, Type: %T)", + f.ID, f.Name, f.Skipped, f.Type)) + if f.Type != nil { + parts = append(parts, fmt.Sprintf(" Value: %s", f.Type.String())) + } + } + return parts +} + +// Field represents a `message_field` +type Field struct { + ID uint8 + Type FieldType + Optional bool + Name string + Skipped bool +} + +// FieldType represents a `type_field` +type FieldType interface { + fmt.Stringer + decode(data []byte, pos uint64) (uint64, error) +} + +// FieldIntFixed is for values with a fixed length. +// This is also known as the 'fixlen_integer_format'. +// The encoded value can vary be between 1 and 2 times +// of that of the value before encoding. +type FieldIntFixed struct { + Length int // Length of value before encoding, encoded value can be more + Value []byte +} + +func (f FieldIntFixed) String() string { + if f.Value == nil { + return "" + } + return fmt.Sprintf("0x%x", f.Value) +} + +func (f *FieldIntFixed) decode(data []byte, pos uint64) (uint64, error) { + var b bytes.Buffer + b.Grow(f.Length * 2) // output is between 1 and 2 times that of the input + + for { + if len(data) < int(pos)+1 { + return pos, errors.New("data truncated") + } + if data[pos]%2 == 0 { + b.WriteByte(data[pos] >> 1) + } else { + if len(data) < int(pos)+2 { + return pos, errors.New("data truncated") + } + switch data[pos+1] { + case 0x2: + b.WriteByte((data[pos] >> 2) + 0x80) + case 0x3: + b.WriteByte((data[pos] >> 2) + 0xc0) + default: + return pos, fmt.Errorf("unknown decoding for %v", data[pos]) + } + pos++ + } + pos++ + if b.Len() == f.Length { + break + } + } + f.Value = b.Bytes() + return pos, nil +} + +// FieldIntVar is using the signed integer variant of the 'varlen_integer_format' +// and encodes a value as a byte sequence of 1-9 bytes depending on the value. +type FieldIntVar struct { + Value int64 +} + +func (f FieldIntVar) String() string { + return fmt.Sprintf("%d", f.Value) +} + +func (f *FieldIntVar) decode(data []byte, pos uint64) (uint64, error) { + var val any + val, pos, err := decodeVar(data, pos, false) + if err != nil { + return pos, err + } + if intval, ok := val.(int64); ok { + f.Value = intval + } else { + return pos, errors.New("unexpected type, expecting int64") + } + return pos, nil +} + +// FieldUintVar is using the unsigned integer variant of the 'varlen_integer_format' +// and encodes a value as a byte sequence of 1-9 bytes depending on the value. +type FieldUintVar struct { + Value uint64 +} + +func (f FieldUintVar) String() string { + return fmt.Sprintf("%d", f.Value) +} + +func (f *FieldUintVar) decode(data []byte, pos uint64) (uint64, error) { + var val any + val, pos, err := decodeVar(data, pos, true) + if err != nil { + return pos, err + } + if uintval, ok := val.(uint64); ok { + f.Value = uintval + } else { + return pos, errors.New("unexpected type, expecting uint64") + } + return pos, nil +} + +// FieldString is a 'string_format' field +type FieldString struct { + Value string +} + +func (f *FieldString) decode(data []byte, pos uint64) (uint64, error) { + if len(data) < int(pos)+1 { + return pos, errors.New("string truncated, expected at least one byte") + } + strLen := int(data[pos] >> 1) + pos++ + if len(data) < int(pos)+strLen { + return pos, fmt.Errorf("string truncated, expected length: %d", strLen) + } + f.Value = string(data[pos : pos+uint64(strLen)]) + return pos + uint64(strLen), nil +} + +func (f FieldString) String() string { + return f.Value +} + +func Unmarshal(data []byte, v any) error { + switch m := v.(type) { + case *Message: + m.Version = data[0] >> 1 + err := Unmarshal(data[1:], &m.Format) + if err != nil { + return err + } + if m.fieldIndex == nil { + m.fieldIndex = make(map[string]uint8, len(m.Format.Fields)) + } + for _, field := range m.Format.Fields { + m.fieldIndex[field.Name] = field.ID + } + case *Format: + pos := uint64(0) + m.Size = data[pos] >> 1 + pos++ + m.LastNonIgnorableField = data[pos] >> 1 + pos++ + + for i := 0; i < len(m.Fields); i++ { + if int(pos)+1 > len(data) || int(data[pos]>>1) != i { + // The field number we got doesn't match what we expect, + // so a field was skipped. + m.Fields[i].ID = uint8(i) + m.Fields[i].Skipped = true + continue + } + m.Fields[i].ID = data[pos] >> 1 + pos++ + n, err := m.Fields[i].Type.decode(data, pos) + if err != nil { + return err + } + pos = n + } + + default: + return fmt.Errorf("unsupported type: %T", v) + } + return nil +} + +func decodeVar(data []byte, pos uint64, unsigned bool) (any, uint64, error) { + if len(data) < int(pos)+1 { + return 0, pos, errors.New("data truncated") + } + flen := trailingOneBitCount(data[pos]) + 1 + if len(data) < int(pos)+flen { + return 0, pos, fmt.Errorf("truncated data, expected length: %d", flen) + } + var tNumBytes [8]byte + copy(tNumBytes[:], data[pos:int(pos)+flen]) + tNum := binary.LittleEndian.Uint64(tNumBytes[:]) + pos += uint64(flen) + if unsigned { + return tNum >> flen, pos, nil + } + if positive := (tNum>>flen)&1 == 0; positive { + return int64(tNum >> (flen + 1)), pos, nil + } + return int64(-(1 + (tNum >> (flen + 1)))), pos, nil +} + +func trailingOneBitCount(b byte) int { + return bits.TrailingZeros8(^b) +} diff --git a/vendor/github.com/go-mysql-org/go-mysql/stmt/stmt.go b/vendor/github.com/go-mysql-org/go-mysql/stmt/stmt.go new file mode 100644 index 000000000..0f75724c0 --- /dev/null +++ b/vendor/github.com/go-mysql-org/go-mysql/stmt/stmt.go @@ -0,0 +1,51 @@ +package stmt + +import "github.com/go-mysql-org/go-mysql/mysql" + +type PreparedStmt struct { + ID uint32 + Params int + Columns int + + RawParamFields [][]byte + RawColumnFields [][]byte + + paramFields []*mysql.Field + columnFields []*mysql.Field +} + +func (s *PreparedStmt) GetParamFields() ([]*mysql.Field, error) { + if s.RawParamFields == nil { + return nil, nil + } + if s.paramFields == nil { + fields := make([]*mysql.Field, len(s.RawParamFields)) + for i, raw := range s.RawParamFields { + field := &mysql.Field{} + if err := field.Parse(raw); err != nil { + return nil, err + } + fields[i] = field + } + s.paramFields = fields + } + return s.paramFields, nil +} + +func (s *PreparedStmt) GetColumnFields() ([]*mysql.Field, error) { + if s.RawColumnFields == nil { + return nil, nil + } + if s.columnFields == nil { + fields := make([]*mysql.Field, len(s.RawColumnFields)) + for i, raw := range s.RawColumnFields { + field := &mysql.Field{} + if err := field.Parse(raw); err != nil { + return nil, err + } + fields[i] = field + } + s.columnFields = fields + } + return s.columnFields, nil +} diff --git a/vendor/github.com/go-mysql-org/go-mysql/utils/byte_slice_pool.go b/vendor/github.com/go-mysql-org/go-mysql/utils/byte_slice_pool.go index 8dace1491..c6159b500 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/utils/byte_slice_pool.go +++ b/vendor/github.com/go-mysql-org/go-mysql/utils/byte_slice_pool.go @@ -6,13 +6,11 @@ type ByteSlice struct { B []byte } -var ( - byteSlicePool = sync.Pool{ - New: func() interface{} { - return new(ByteSlice) - }, - } -) +var byteSlicePool = sync.Pool{ + New: func() any { + return new(ByteSlice) + }, +} func ByteSliceGet(length int) *ByteSlice { data := byteSlicePool.Get().(*ByteSlice) diff --git a/vendor/github.com/go-mysql-org/go-mysql/utils/bytes_buffer_pool.go b/vendor/github.com/go-mysql-org/go-mysql/utils/bytes_buffer_pool.go index 60b820f39..0d2596478 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/utils/bytes_buffer_pool.go +++ b/vendor/github.com/go-mysql-org/go-mysql/utils/bytes_buffer_pool.go @@ -9,13 +9,11 @@ const ( TooBigBlockSize = 1024 * 1024 * 4 ) -var ( - bytesBufferPool = sync.Pool{ - New: func() interface{} { - return &bytes.Buffer{} - }, - } -) +var bytesBufferPool = sync.Pool{ + New: func() any { + return &bytes.Buffer{} + }, +} func BytesBufferGet() (data *bytes.Buffer) { data = bytesBufferPool.Get().(*bytes.Buffer) diff --git a/vendor/github.com/go-mysql-org/go-mysql/utils/now_unix.go b/vendor/github.com/go-mysql-org/go-mysql/utils/now_unix.go index c2e6f5dc4..c8890cdad 100644 --- a/vendor/github.com/go-mysql-org/go-mysql/utils/now_unix.go +++ b/vendor/github.com/go-mysql-org/go-mysql/utils/now_unix.go @@ -15,5 +15,6 @@ func Now() time.Time { return time.Now() } - return time.Unix(0, syscall.TimevalToNsec(tv)) + //nolint:unconvert + return time.Unix(int64(tv.Sec), int64(tv.Usec)*1000) } diff --git a/vendor/github.com/goccy/go-json/.golangci.yml b/vendor/github.com/goccy/go-json/.golangci.yml index 57ae5a528..977accaa9 100644 --- a/vendor/github.com/goccy/go-json/.golangci.yml +++ b/vendor/github.com/goccy/go-json/.golangci.yml @@ -56,6 +56,9 @@ linters: - cyclop - containedctx - revive + - nosnakecase + - exhaustruct + - depguard issues: exclude-rules: diff --git a/vendor/github.com/goccy/go-json/Makefile b/vendor/github.com/goccy/go-json/Makefile index 5bbfc4c9a..c030577dc 100644 --- a/vendor/github.com/goccy/go-json/Makefile +++ b/vendor/github.com/goccy/go-json/Makefile @@ -30,7 +30,7 @@ golangci-lint: | $(BIN_DIR) GOLANGCI_LINT_TMP_DIR=$$(mktemp -d); \ cd $$GOLANGCI_LINT_TMP_DIR; \ go mod init tmp; \ - GOBIN=$(BIN_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.48.0; \ + GOBIN=$(BIN_DIR) go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.2; \ rm -rf $$GOLANGCI_LINT_TMP_DIR; \ } diff --git a/vendor/github.com/goccy/go-json/encode.go b/vendor/github.com/goccy/go-json/encode.go index 4bd899f38..c5173825a 100644 --- a/vendor/github.com/goccy/go-json/encode.go +++ b/vendor/github.com/goccy/go-json/encode.go @@ -52,7 +52,7 @@ func (e *Encoder) EncodeContext(ctx context.Context, v interface{}, optFuncs ... rctx.Option.Flag |= encoder.ContextOption rctx.Option.Context = ctx - err := e.encodeWithOption(rctx, v, optFuncs...) + err := e.encodeWithOption(rctx, v, optFuncs...) //nolint: contextcheck encoder.ReleaseRuntimeContext(rctx) return err @@ -120,7 +120,7 @@ func marshalContext(ctx context.Context, v interface{}, optFuncs ...EncodeOption optFunc(rctx.Option) } - buf, err := encode(rctx, v) + buf, err := encode(rctx, v) //nolint: contextcheck if err != nil { encoder.ReleaseRuntimeContext(rctx) return nil, err diff --git a/vendor/github.com/goccy/go-json/internal/decoder/compile.go b/vendor/github.com/goccy/go-json/internal/decoder/compile.go index fab643764..8ad50936c 100644 --- a/vendor/github.com/goccy/go-json/internal/decoder/compile.go +++ b/vendor/github.com/goccy/go-json/internal/decoder/compile.go @@ -5,6 +5,7 @@ import ( "fmt" "reflect" "strings" + "sync" "sync/atomic" "unicode" "unsafe" @@ -17,22 +18,27 @@ var ( typeAddr *runtime.TypeAddr cachedDecoderMap unsafe.Pointer // map[uintptr]decoder cachedDecoder []Decoder + initOnce sync.Once ) -func init() { - typeAddr = runtime.AnalyzeTypeAddr() - if typeAddr == nil { - typeAddr = &runtime.TypeAddr{} - } - cachedDecoder = make([]Decoder, typeAddr.AddrRange>>typeAddr.AddrShift+1) +func initDecoder() { + initOnce.Do(func() { + typeAddr = runtime.AnalyzeTypeAddr() + if typeAddr == nil { + typeAddr = &runtime.TypeAddr{} + } + cachedDecoder = make([]Decoder, typeAddr.AddrRange>>typeAddr.AddrShift+1) + }) } func loadDecoderMap() map[uintptr]Decoder { + initDecoder() p := atomic.LoadPointer(&cachedDecoderMap) return *(*map[uintptr]Decoder)(unsafe.Pointer(&p)) } func storeDecoder(typ uintptr, dec Decoder, m map[uintptr]Decoder) { + initDecoder() newDecoderMap := make(map[uintptr]Decoder, len(m)+1) newDecoderMap[typ] = dec diff --git a/vendor/github.com/goccy/go-json/internal/decoder/compile_norace.go b/vendor/github.com/goccy/go-json/internal/decoder/compile_norace.go index eb7e2b134..025ca85b5 100644 --- a/vendor/github.com/goccy/go-json/internal/decoder/compile_norace.go +++ b/vendor/github.com/goccy/go-json/internal/decoder/compile_norace.go @@ -10,6 +10,7 @@ import ( ) func CompileToGetDecoder(typ *runtime.Type) (Decoder, error) { + initDecoder() typeptr := uintptr(unsafe.Pointer(typ)) if typeptr > typeAddr.MaxTypeAddr { return compileToGetDecoderSlowPath(typeptr, typ) diff --git a/vendor/github.com/goccy/go-json/internal/decoder/compile_race.go b/vendor/github.com/goccy/go-json/internal/decoder/compile_race.go index 49cdda4a1..023b817c3 100644 --- a/vendor/github.com/goccy/go-json/internal/decoder/compile_race.go +++ b/vendor/github.com/goccy/go-json/internal/decoder/compile_race.go @@ -13,6 +13,7 @@ import ( var decMu sync.RWMutex func CompileToGetDecoder(typ *runtime.Type) (Decoder, error) { + initDecoder() typeptr := uintptr(unsafe.Pointer(typ)) if typeptr > typeAddr.MaxTypeAddr { return compileToGetDecoderSlowPath(typeptr, typ) diff --git a/vendor/github.com/goccy/go-json/internal/decoder/ptr.go b/vendor/github.com/goccy/go-json/internal/decoder/ptr.go index de12e105c..ae2299466 100644 --- a/vendor/github.com/goccy/go-json/internal/decoder/ptr.go +++ b/vendor/github.com/goccy/go-json/internal/decoder/ptr.go @@ -85,6 +85,7 @@ func (d *ptrDecoder) Decode(ctx *RuntimeContext, cursor, depth int64, p unsafe.P } c, err := d.dec.Decode(ctx, cursor, depth, newptr) if err != nil { + *(*unsafe.Pointer)(p) = nil return 0, err } cursor = c diff --git a/vendor/github.com/goccy/go-json/internal/decoder/unmarshal_text.go b/vendor/github.com/goccy/go-json/internal/decoder/unmarshal_text.go index 6d37993f0..d711d0f85 100644 --- a/vendor/github.com/goccy/go-json/internal/decoder/unmarshal_text.go +++ b/vendor/github.com/goccy/go-json/internal/decoder/unmarshal_text.go @@ -147,7 +147,7 @@ func (d *unmarshalTextDecoder) DecodePath(ctx *RuntimeContext, cursor, depth int return nil, 0, fmt.Errorf("json: unmarshal text decoder does not support decode path") } -func unquoteBytes(s []byte) (t []byte, ok bool) { +func unquoteBytes(s []byte) (t []byte, ok bool) { //nolint: nonamedreturns length := len(s) if length < 2 || s[0] != '"' || s[length-1] != '"' { return diff --git a/vendor/github.com/goccy/go-json/internal/encoder/code.go b/vendor/github.com/goccy/go-json/internal/encoder/code.go index 5b08faefc..fec45a4b8 100644 --- a/vendor/github.com/goccy/go-json/internal/encoder/code.go +++ b/vendor/github.com/goccy/go-json/internal/encoder/code.go @@ -518,6 +518,7 @@ func (c *StructCode) ToAnonymousOpcode(ctx *compileContext) Opcodes { prevField = firstField codes = codes.Add(fieldCodes...) } + ctx.structTypeToCodes[uintptr(unsafe.Pointer(c.typ))] = codes return codes } diff --git a/vendor/github.com/goccy/go-json/internal/encoder/compact.go b/vendor/github.com/goccy/go-json/internal/encoder/compact.go index 0eb9545d8..e287a6c03 100644 --- a/vendor/github.com/goccy/go-json/internal/encoder/compact.go +++ b/vendor/github.com/goccy/go-json/internal/encoder/compact.go @@ -213,8 +213,8 @@ func compactString(dst, src []byte, cursor int64, escape bool) ([]byte, int64, e dst = append(dst, src[start:cursor]...) dst = append(dst, `\u202`...) dst = append(dst, hex[src[cursor+2]&0xF]) - cursor += 2 start = cursor + 3 + cursor += 2 } } switch c { diff --git a/vendor/github.com/goccy/go-json/internal/encoder/compiler.go b/vendor/github.com/goccy/go-json/internal/encoder/compiler.go index 3ae39ba8c..b10763689 100644 --- a/vendor/github.com/goccy/go-json/internal/encoder/compiler.go +++ b/vendor/github.com/goccy/go-json/internal/encoder/compiler.go @@ -5,6 +5,7 @@ import ( "encoding" "encoding/json" "reflect" + "sync" "sync/atomic" "unsafe" @@ -24,14 +25,17 @@ var ( cachedOpcodeSets []*OpcodeSet cachedOpcodeMap unsafe.Pointer // map[uintptr]*OpcodeSet typeAddr *runtime.TypeAddr + initEncoderOnce sync.Once ) -func init() { - typeAddr = runtime.AnalyzeTypeAddr() - if typeAddr == nil { - typeAddr = &runtime.TypeAddr{} - } - cachedOpcodeSets = make([]*OpcodeSet, typeAddr.AddrRange>>typeAddr.AddrShift+1) +func initEncoder() { + initEncoderOnce.Do(func() { + typeAddr = runtime.AnalyzeTypeAddr() + if typeAddr == nil { + typeAddr = &runtime.TypeAddr{} + } + cachedOpcodeSets = make([]*OpcodeSet, typeAddr.AddrRange>>typeAddr.AddrShift+1) + }) } func loadOpcodeMap() map[uintptr]*OpcodeSet { @@ -480,7 +484,7 @@ func (c *Compiler) mapCode(typ *runtime.Type) (*MapCode, error) { func (c *Compiler) listElemCode(typ *runtime.Type) (Code, error) { switch { - case c.isPtrMarshalJSONType(typ): + case c.implementsMarshalJSONType(typ) || c.implementsMarshalJSONType(runtime.PtrTo(typ)): return c.marshalJSONCode(typ) case !typ.Implements(marshalTextType) && runtime.PtrTo(typ).Implements(marshalTextType): return c.marshalTextCode(typ) diff --git a/vendor/github.com/goccy/go-json/internal/encoder/compiler_norace.go b/vendor/github.com/goccy/go-json/internal/encoder/compiler_norace.go index 20c93cbf7..b6f45a49b 100644 --- a/vendor/github.com/goccy/go-json/internal/encoder/compiler_norace.go +++ b/vendor/github.com/goccy/go-json/internal/encoder/compiler_norace.go @@ -4,6 +4,7 @@ package encoder func CompileToGetCodeSet(ctx *RuntimeContext, typeptr uintptr) (*OpcodeSet, error) { + initEncoder() if typeptr > typeAddr.MaxTypeAddr || typeptr < typeAddr.BaseTypeAddr { codeSet, err := compileToGetCodeSetSlowPath(typeptr) if err != nil { diff --git a/vendor/github.com/goccy/go-json/internal/encoder/compiler_race.go b/vendor/github.com/goccy/go-json/internal/encoder/compiler_race.go index 13ba23fdf..47b482f7f 100644 --- a/vendor/github.com/goccy/go-json/internal/encoder/compiler_race.go +++ b/vendor/github.com/goccy/go-json/internal/encoder/compiler_race.go @@ -10,6 +10,7 @@ import ( var setsMu sync.RWMutex func CompileToGetCodeSet(ctx *RuntimeContext, typeptr uintptr) (*OpcodeSet, error) { + initEncoder() if typeptr > typeAddr.MaxTypeAddr || typeptr < typeAddr.BaseTypeAddr { codeSet, err := compileToGetCodeSetSlowPath(typeptr) if err != nil { diff --git a/vendor/github.com/goccy/go-json/internal/encoder/encoder.go b/vendor/github.com/goccy/go-json/internal/encoder/encoder.go index 14eb6a0d6..b436f5b21 100644 --- a/vendor/github.com/goccy/go-json/internal/encoder/encoder.go +++ b/vendor/github.com/goccy/go-json/internal/encoder/encoder.go @@ -406,6 +406,11 @@ func AppendMarshalJSON(ctx *RuntimeContext, code *Opcode, b []byte, v interface{ rv = newV } } + + if rv.Kind() == reflect.Ptr && rv.IsNil() { + return AppendNull(ctx, b), nil + } + v = rv.Interface() var bb []byte if (code.Flags & MarshalerContextFlags) != 0 { diff --git a/vendor/github.com/goccy/go-json/internal/encoder/int.go b/vendor/github.com/goccy/go-json/internal/encoder/int.go index 85f079609..8b5febeaa 100644 --- a/vendor/github.com/goccy/go-json/internal/encoder/int.go +++ b/vendor/github.com/goccy/go-json/internal/encoder/int.go @@ -1,3 +1,27 @@ +// This files's processing codes are inspired by https://github.com/segmentio/encoding. +// The license notation is as follows. +// +// # MIT License +// +// Copyright (c) 2019 Segment.io, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. package encoder import ( diff --git a/vendor/github.com/goccy/go-json/internal/encoder/string.go b/vendor/github.com/goccy/go-json/internal/encoder/string.go index e4152b27c..4abb84165 100644 --- a/vendor/github.com/goccy/go-json/internal/encoder/string.go +++ b/vendor/github.com/goccy/go-json/internal/encoder/string.go @@ -1,3 +1,27 @@ +// This files's string processing codes are inspired by https://github.com/segmentio/encoding. +// The license notation is as follows. +// +// # MIT License +// +// Copyright (c) 2019 Segment.io, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. package encoder import ( diff --git a/vendor/github.com/goccy/go-json/internal/runtime/rtype.go b/vendor/github.com/goccy/go-json/internal/runtime/rtype.go index 4db10debe..37cfe35a1 100644 --- a/vendor/github.com/goccy/go-json/internal/runtime/rtype.go +++ b/vendor/github.com/goccy/go-json/internal/runtime/rtype.go @@ -252,7 +252,6 @@ func IfaceIndir(*Type) bool //go:noescape func RType2Type(t *Type) reflect.Type -//go:nolint structcheck type emptyInterface struct { _ *Type ptr unsafe.Pointer diff --git a/vendor/github.com/goccy/go-json/internal/runtime/type.go b/vendor/github.com/goccy/go-json/internal/runtime/type.go index 0167cd2c0..4b693cb0b 100644 --- a/vendor/github.com/goccy/go-json/internal/runtime/type.go +++ b/vendor/github.com/goccy/go-json/internal/runtime/type.go @@ -2,6 +2,7 @@ package runtime import ( "reflect" + "sync" "unsafe" ) @@ -23,8 +24,8 @@ type TypeAddr struct { } var ( - typeAddr *TypeAddr - alreadyAnalyzed bool + typeAddr *TypeAddr + once sync.Once ) //go:linkname typelinks reflect.typelinks @@ -34,67 +35,64 @@ func typelinks() ([]unsafe.Pointer, [][]int32) func rtypeOff(unsafe.Pointer, int32) unsafe.Pointer func AnalyzeTypeAddr() *TypeAddr { - defer func() { - alreadyAnalyzed = true - }() - if alreadyAnalyzed { - return typeAddr - } - sections, offsets := typelinks() - if len(sections) != 1 { - return nil - } - if len(offsets) != 1 { - return nil - } - section := sections[0] - offset := offsets[0] - var ( - min uintptr = uintptr(^uint(0)) - max uintptr = 0 - isAligned64 = true - isAligned32 = true - ) - for i := 0; i < len(offset); i++ { - typ := (*Type)(rtypeOff(section, offset[i])) - addr := uintptr(unsafe.Pointer(typ)) - if min > addr { - min = addr + once.Do(func() { + sections, offsets := typelinks() + if len(sections) != 1 { + return } - if max < addr { - max = addr + if len(offsets) != 1 { + return } - if typ.Kind() == reflect.Ptr { - addr = uintptr(unsafe.Pointer(typ.Elem())) + section := sections[0] + offset := offsets[0] + var ( + min uintptr = uintptr(^uint(0)) + max uintptr = 0 + isAligned64 = true + isAligned32 = true + ) + for i := 0; i < len(offset); i++ { + typ := (*Type)(rtypeOff(section, offset[i])) + addr := uintptr(unsafe.Pointer(typ)) if min > addr { min = addr } if max < addr { max = addr } + if typ.Kind() == reflect.Ptr { + addr = uintptr(unsafe.Pointer(typ.Elem())) + if min > addr { + min = addr + } + if max < addr { + max = addr + } + } + isAligned64 = isAligned64 && (addr-min)&63 == 0 + isAligned32 = isAligned32 && (addr-min)&31 == 0 + } + addrRange := max - min + if addrRange == 0 { + return + } + var addrShift uintptr + if isAligned64 { + addrShift = 6 + } else if isAligned32 { + addrShift = 5 } - isAligned64 = isAligned64 && (addr-min)&63 == 0 - isAligned32 = isAligned32 && (addr-min)&31 == 0 - } - addrRange := max - min - if addrRange == 0 { - return nil - } - var addrShift uintptr - if isAligned64 { - addrShift = 6 - } else if isAligned32 { - addrShift = 5 - } - cacheSize := addrRange >> addrShift - if cacheSize > maxAcceptableTypeAddrRange { - return nil - } - typeAddr = &TypeAddr{ - BaseTypeAddr: min, - MaxTypeAddr: max, - AddrRange: addrRange, - AddrShift: addrShift, - } + cacheSize := addrRange >> addrShift + if cacheSize > maxAcceptableTypeAddrRange { + return + } + typeAddr = &TypeAddr{ + BaseTypeAddr: min, + MaxTypeAddr: max, + AddrRange: addrRange, + AddrShift: addrShift, + } + }) + return typeAddr } diff --git a/vendor/github.com/goccy/go-json/json.go b/vendor/github.com/goccy/go-json/json.go index 413cb20bf..fb18065a2 100644 --- a/vendor/github.com/goccy/go-json/json.go +++ b/vendor/github.com/goccy/go-json/json.go @@ -89,31 +89,31 @@ type UnmarshalerContext interface { // // Examples of struct field tags and their meanings: // -// // Field appears in JSON as key "myName". -// Field int `json:"myName"` +// // Field appears in JSON as key "myName". +// Field int `json:"myName"` // -// // Field appears in JSON as key "myName" and -// // the field is omitted from the object if its value is empty, -// // as defined above. -// Field int `json:"myName,omitempty"` +// // Field appears in JSON as key "myName" and +// // the field is omitted from the object if its value is empty, +// // as defined above. +// Field int `json:"myName,omitempty"` // -// // Field appears in JSON as key "Field" (the default), but -// // the field is skipped if empty. -// // Note the leading comma. -// Field int `json:",omitempty"` +// // Field appears in JSON as key "Field" (the default), but +// // the field is skipped if empty. +// // Note the leading comma. +// Field int `json:",omitempty"` // -// // Field is ignored by this package. -// Field int `json:"-"` +// // Field is ignored by this package. +// Field int `json:"-"` // -// // Field appears in JSON as key "-". -// Field int `json:"-,"` +// // Field appears in JSON as key "-". +// Field int `json:"-,"` // // The "string" option signals that a field is stored as JSON inside a // JSON-encoded string. It applies only to fields of string, floating point, // integer, or boolean types. This extra level of encoding is sometimes used // when communicating with JavaScript programs: // -// Int64String int64 `json:",string"` +// Int64String int64 `json:",string"` // // The key name will be used if it's a non-empty string consisting of // only Unicode letters, digits, and ASCII punctuation except quotation @@ -166,7 +166,6 @@ type UnmarshalerContext interface { // JSON cannot represent cyclic data structures and Marshal does not // handle them. Passing cyclic structures to Marshal will result in // an infinite recursion. -// func Marshal(v interface{}) ([]byte, error) { return MarshalWithOption(v) } @@ -264,14 +263,13 @@ func MarshalIndentWithOption(v interface{}, prefix, indent string, optFuncs ...E // // The JSON null value unmarshals into an interface, map, pointer, or slice // by setting that Go value to nil. Because null is often used in JSON to mean -// ``not present,'' unmarshaling a JSON null into any other Go type has no effect +// “not present,” unmarshaling a JSON null into any other Go type has no effect // on the value and produces no error. // // When unmarshaling quoted strings, invalid UTF-8 or // invalid UTF-16 surrogate pairs are not treated as an error. // Instead, they are replaced by the Unicode replacement // character U+FFFD. -// func Unmarshal(data []byte, v interface{}) error { return unmarshal(data, v) } @@ -299,7 +297,6 @@ func UnmarshalNoEscape(data []byte, v interface{}, optFuncs ...DecodeOptionFunc) // Number, for JSON numbers // string, for JSON string literals // nil, for JSON null -// type Token = json.Token // A Number represents a JSON number literal. diff --git a/vendor/github.com/klauspost/compress/.gitattributes b/vendor/github.com/klauspost/compress/.gitattributes index 402433593..57aa6487c 100644 --- a/vendor/github.com/klauspost/compress/.gitattributes +++ b/vendor/github.com/klauspost/compress/.gitattributes @@ -1,2 +1,3 @@ * -text *.bin -text -diff +*.md text eol=lf diff --git a/vendor/github.com/klauspost/compress/.goreleaser.yml b/vendor/github.com/klauspost/compress/.goreleaser.yml index a22953805..804a20181 100644 --- a/vendor/github.com/klauspost/compress/.goreleaser.yml +++ b/vendor/github.com/klauspost/compress/.goreleaser.yml @@ -1,5 +1,5 @@ -# This is an example goreleaser.yaml file with some sane defaults. -# Make sure to check the documentation at http://goreleaser.com +version: 2 + before: hooks: - ./gen.sh @@ -31,6 +31,9 @@ builds: - mips64le goarm: - 7 + ignore: + - goos: windows + goarch: arm - id: "s2d" binary: s2d @@ -57,6 +60,9 @@ builds: - mips64le goarm: - 7 + ignore: + - goos: windows + goarch: arm - id: "s2sx" binary: s2sx @@ -84,6 +90,9 @@ builds: - mips64le goarm: - 7 + ignore: + - goos: windows + goarch: arm archives: - @@ -91,7 +100,7 @@ archives: name_template: "s2-{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" format_overrides: - goos: windows - format: zip + formats: ['zip'] files: - unpack/* - s2/LICENSE @@ -99,7 +108,7 @@ archives: checksum: name_template: 'checksums.txt' snapshot: - name_template: "{{ .Tag }}-next" + version_template: "{{ .Tag }}-next" changelog: sort: asc filters: diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 05c7359e4..fb023f2cf 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -1,700 +1,700 @@ -# compress - -This package provides various compression algorithms. - -* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go. -* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. -* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). -* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. -* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. -* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped requests efficiently. -* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. - -[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) -[![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml) -[![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge) - -# changelog - -* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) - * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 - * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 - -* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5) - * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912 - * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908 - * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913 - * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910 - * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917 -https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918 - -* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4) - * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887 - * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886 - * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892 - * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890 - * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891 - -* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3) - * fse: Fix max header size https://github.com/klauspost/compress/pull/881 - * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877 - * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883 - -* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2) - * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876 - -* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1) - * s2: Fix S2 "best" dictionary wrong encoding by @klauspost in https://github.com/klauspost/compress/pull/871 - * flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869 - * s2: Fix EstimateBlockSize on 6&7 length input by @klauspost in https://github.com/klauspost/compress/pull/867 - -* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0) - * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853 - * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838 - * flate: Add limited window compression https://github.com/klauspost/compress/pull/843 - * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839 - * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837 - * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860 - -
- See changes to v1.16.x - - -* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7) - * zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829 - * s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832 - -* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6) - * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806 - * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824 - * gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815 - * s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663 - -* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5) - * zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802 - * gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804 - -* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4) - * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784 - * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792 - * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785 - * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 - * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 - * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 - * gzhttp: Suppport ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 - -* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) - * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 - * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767 - * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766 - * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773 - * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774 - -* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0) - * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685 - * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752 - * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755 - * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748 - * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 - * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 -
- -
- See changes to v1.15.x - -* Jan 21st, 2023 (v1.15.15) - * deflate: Improve level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/739 - * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 - * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 - * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 - -* Jan 3rd, 2023 (v1.15.14) - - * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 - * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 - * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 - * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 - -* Dec 11, 2022 (v1.15.13) - * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 - * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 - -* Oct 26, 2022 (v1.15.12) - - * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 - * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 - -* Sept 26, 2022 (v1.15.11) - - * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 - * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 - * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 - * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 - -* Sept 16, 2022 (v1.15.10) - - * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 - * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 - * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 - * zstd: Improve "better" compresssion https://github.com/klauspost/compress/pull/657 - * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 - * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 - * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 - * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659 - -* July 21, 2022 (v1.15.9) - - * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 - * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 - * zstd: Allow single segments up to "max decoded size" by @klauspost in https://github.com/klauspost/compress/pull/643 - -* July 13, 2022 (v1.15.8) - - * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 - * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638 - * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636 - * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637 - * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634 - * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640 - * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639 - -* June 29, 2022 (v1.15.7) - - * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633 - * zip: Merge upstream https://github.com/klauspost/compress/pull/631 - * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624 - * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598 - * flate: Faster histograms https://github.com/klauspost/compress/pull/620 - * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622 - -* June 3, 2022 (v1.15.6) - * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613 - * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611 - * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605 - * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606 - * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608 - * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612 - * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609 - * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607 - * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614 - * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610 - -* May 25, 2022 (v1.15.5) - * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602 - * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601 - * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596 - * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588 - * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592 - * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599 - * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593 - * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586 - * flate: Inplace hashing for level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/590 - - -* May 11, 2022 (v1.15.4) - * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577) - * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581) - * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583) - * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580) - -* May 5, 2022 (v1.15.3) - * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572) - * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575) - -* Apr 26, 2022 (v1.15.2) - * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537) - * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539) - * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555) - * Minimum version is Go 1.16, added CI test on 1.18. - -* Mar 11, 2022 (v1.15.1) - * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512) - * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514) - * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520) - * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521) - * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523) - -* Mar 3, 2022 (v1.15.0) - * zstd: Refactor decoder by @klauspost in [#498](https://github.com/klauspost/compress/pull/498) - * zstd: Add stream encoding without goroutines by @klauspost in [#505](https://github.com/klauspost/compress/pull/505) - * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507) - * flate: Inline literal emission by @klauspost in [#509](https://github.com/klauspost/compress/pull/509) - * gzhttp: Add zstd to transport by @klauspost in [#400](https://github.com/klauspost/compress/pull/400) - * gzhttp: Make content-type optional by @klauspost in [#510](https://github.com/klauspost/compress/pull/510) - -Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines. - -Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected. - -While the release has been extensively tested, it is recommended to testing when upgrading. - -
- -
- See changes to v1.14.x - -* Feb 22, 2022 (v1.14.4) - * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503) - * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502) - * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501 - * huff0: Use static decompression buffer up to 30% faster by @klauspost in [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500) - -* Feb 17, 2022 (v1.14.3) - * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478) - * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483) - * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486) - -* Jan 25, 2022 (v1.14.2) - * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476) - * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469) - * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470) - * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472) - * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473) - * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475) - -* Jan 11, 2022 (v1.14.1) - * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) - * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) - * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468) - * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464) - * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445) -
- -
- See changes to v1.13.x - -* Aug 30, 2021 (v1.13.5) - * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) - * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) - * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426) - * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421) - -* Aug 12, 2021 (v1.13.4) - * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy). - * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415) - -* Aug 3, 2021 (v1.13.3) - * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404) - * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411) - * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406) - * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399) - * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401) - * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410) - -* Jun 14, 2021 (v1.13.1) - * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396) - * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394) - * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389) - * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395) - -* Jun 3, 2021 (v1.13.0) - * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors. - * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) - * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) -
- - -
- See changes to v1.12.x - -* May 25, 2021 (v1.12.3) - * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) - * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) - * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) - -* Apr 27, 2021 (v1.12.2) - * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365) - * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363) - * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367) - * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358) - * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362) - * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) - -* Apr 14, 2021 (v1.12.1) - * snappy package removed. Upstream added as dependency. - * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353) - * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352) - * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348) - * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) - * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) - * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) -
- -
- See changes to v1.11.x - -* Mar 26, 2021 (v1.11.13) - * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) - * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336) - * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338) - * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341) - -* Mar 5, 2021 (v1.11.12) - * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives). - * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328) - -* Mar 1, 2021 (v1.11.9) - * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324) - * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325) - * s2: Fix binaries. - -* Feb 25, 2021 (v1.11.8) - * s2: Fixed occational out-of-bounds write on amd64. Upgrade recommended. - * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) - * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) - * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) - * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313) - -* Jan 14, 2021 (v1.11.7) - * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309) - * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310) - * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311) - * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308) - * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312) - -* Jan 7, 2021 (v1.11.6) - * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306) - * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305) - -* Dec 20, 2020 (v1.11.4) - * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304) - * Add header decoder [#299](https://github.com/klauspost/compress/pull/299) - * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297) - * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300) - * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303) - -* Nov 15, 2020 (v1.11.3) - * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293) - * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295) - -* Oct 11, 2020 (v1.11.2) - * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291) - -* Oct 1, 2020 (v1.11.1) - * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286) - -* Sept 8, 2020 (v1.11.0) - * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281) - * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282) - * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274) -
- -
- See changes to v1.10.x - -* July 8, 2020 (v1.10.11) - * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) - * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275) - -* June 23, 2020 (v1.10.10) - * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270) - -* June 16, 2020 (v1.10.9): - * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268) - * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266) - * Fuzzit tests removed. The service has been purchased and is no longer available. - -* June 5, 2020 (v1.10.8): - * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265) - -* June 1, 2020 (v1.10.7): - * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) - * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259) - * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263) - -* May 21, 2020: (v1.10.6) - * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252) - * zstd: Stricter decompression checks. - -* April 12, 2020: (v1.10.5) - * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239) - -* Apr 8, 2020: (v1.10.4) - * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247) -* Mar 11, 2020: (v1.10.3) - * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245) - * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244) - * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240) - * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241) - * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238) - -* Feb 27, 2020: (v1.10.2) - * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232) - * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227) - -* Feb 18, 2020: (v1.10.1) - * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226) - * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224) - * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224) - -* Feb 4, 2020: (v1.10.0) - * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216) - * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218) - * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214) - * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186) - -
- -
- See changes prior to v1.10.0 - -* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206). -* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) -* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed. -* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases. -* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192) -* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder. -* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199) -* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features -* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197) -* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198) -* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit. -* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191) -* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188) -* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187) -* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines. -* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate. -* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184) -* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate. -* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180) -* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB. -* Nov 11, 2019: Reduce inflate memory use by 1KB. -* Nov 10, 2019: Less allocations in deflate bit writer. -* Nov 10, 2019: Fix inconsistent error returned by zstd decoder. -* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174) -* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173) -* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) -* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105) - -
- -
- See changes prior to v1.9.0 - -* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169) -* Oct 3, 2019: Fix inconsistent results on broken zstd streams. -* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) -* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools). -* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip). -* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes). -* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option. -* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables. -* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode. -* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding. -* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. -* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing. -* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing. -* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147) -* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146) -* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144) -* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142) -* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder. -* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder. -* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content. -* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix. -* June 17, 2019: zstd decompression bugfix. -* June 17, 2019: fix 32 bit builds. -* June 17, 2019: Easier use in modules (less dependencies). -* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio. -* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression. -* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels. -* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression! -* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels. -* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added. -* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression). -* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below. -* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0). -* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change. -* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change. -* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function. -* May 28, 2017: Reduce allocations when resetting decoder. -* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7. -* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625). -* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before. -* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update. -* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. -* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression. -* Mar 24, 2016: Small speedup for level 1-3. -* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. -* Feb 19, 2016: Handle small payloads faster in level 1-3. -* Feb 19, 2016: Added faster level 2 + 3 compression modes. -* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progresssion in terms of compression. New default level is 5. -* Feb 14, 2016: Snappy: Merge upstream changes. -* Feb 14, 2016: Snappy: Fix aggressive skipping. -* Feb 14, 2016: Snappy: Update benchmark. -* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression. -* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%. -* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content. -* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup. -* Jan 16, 2016: Optimization on deflate level 1,2,3 compression. -* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives. -* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs. -* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms. -* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update! -* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet). -* Nov 20 2015: Small optimization to bit writer on 64 bit systems. -* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15). -* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate. -* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file -* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x. - -
- -# deflate usage - -The packages are drop-in replacements for standard libraries. Simply replace the import path to use them: - -| old import | new import | Documentation -|--------------------|-----------------------------------------|--------------------| -| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc) -| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc) -| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) -| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) - -* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). - -You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages. - -The packages contains the same as the standard library, so you can use the godoc for that: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/). - -Currently there is only minor speedup on decompression (mostly CRC32 calculation). - -Memory usage is typically 1MB for a Writer. stdlib is in the same range. -If you expect to have a lot of concurrently allocated Writers consider using -the stateless compress described below. - -For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). - -To disable all assembly add `-tags=noasm`. This works across all packages. - -# Stateless compression - -This package offers stateless compression as a special option for gzip/deflate. -It will do compression but without maintaining any state between Write calls. - -This means there will be no memory kept between Write calls, but compression and speed will be suboptimal. - -This is only relevant in cases where you expect to run many thousands of compressors concurrently, -but with very little activity. This is *not* intended for regular web servers serving individual requests. - -Because of this, the size of actual Write calls will affect output size. - -In gzip, specify level `-3` / `gzip.StatelessCompression` to enable. - -For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter) - -A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer: - -```go - // replace 'ioutil.Discard' with your output. - gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression) - if err != nil { - return err - } - defer gzw.Close() - - w := bufio.NewWriterSize(gzw, 4096) - defer w.Flush() - - // Write to 'w' -``` - -This will only use up to 4KB in memory when the writer is idle. - -Compression is almost always worse than the fastest compression level -and each write will allocate (a little) memory. - -# Performance Update 2018 - -It has been a while since we have been looking at the speed of this package compared to the standard library, so I thought I would re-do my tests and give some overall recommendations based on the current state. All benchmarks have been performed with Go 1.10 on my Desktop Intel(R) Core(TM) i7-2600 CPU @3.40GHz. Since I last ran the tests, I have gotten more RAM, which means tests with big files are no longer limited by my SSD. - -The raw results are in my [updated spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). Due to cgo changes and upstream updates i could not get the cgo version of gzip to compile. Instead I included the [zstd](https://github.com/datadog/zstd) cgo implementation. If I get cgo gzip to work again, I might replace the results in the sheet. - -The columns to take note of are: *MB/s* - the throughput. *Reduction* - the data size reduction in percent of the original. *Rel Speed* relative speed compared to the standard library at the same level. *Smaller* - how many percent smaller is the compressed output compared to stdlib. Negative means the output was bigger. *Loss* means the loss (or gain) in compression as a percentage difference of the input. - -The `gzstd` (standard library gzip) and `gzkp` (this package gzip) only uses one CPU core. [`pgzip`](https://github.com/klauspost/pgzip), [`bgzf`](https://github.com/biogo/hts/tree/master/bgzf) uses all 4 cores. [`zstd`](https://github.com/DataDog/zstd) uses one core, and is a beast (but not Go, yet). - - -## Overall differences. - -There appears to be a roughly 5-10% speed advantage over the standard library when comparing at similar compression levels. - -The biggest difference you will see is the result of [re-balancing](https://blog.klauspost.com/rebalancing-deflate-compression-levels/) the compression levels. I wanted by library to give a smoother transition between the compression levels than the standard library. - -This package attempts to provide a more smooth transition, where "1" is taking a lot of shortcuts, "5" is the reasonable trade-off and "9" is the "give me the best compression", and the values in between gives something reasonable in between. The standard library has big differences in levels 1-4, but levels 5-9 having no significant gains - often spending a lot more time than can be justified by the achieved compression. - -There are links to all the test data in the [spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing) in the top left field on each tab. - -## Web Content - -This test set aims to emulate typical use in a web server. The test-set is 4GB data in 53k files, and is a mixture of (mostly) HTML, JS, CSS. - -Since level 1 and 9 are close to being the same code, they are quite close. But looking at the levels in-between the differences are quite big. - -Looking at level 6, this package is 88% faster, but will output about 6% more data. For a web server, this means you can serve 88% more data, but have to pay for 6% more bandwidth. You can draw your own conclusions on what would be the most expensive for your case. - -## Object files - -This test is for typical data files stored on a server. In this case it is a collection of Go precompiled objects. They are very compressible. - -The picture is similar to the web content, but with small differences since this is very compressible. Levels 2-3 offer good speed, but is sacrificing quite a bit of compression. - -The standard library seems suboptimal on level 3 and 4 - offering both worse compression and speed than level 6 & 7 of this package respectively. - -## Highly Compressible File - -This is a JSON file with very high redundancy. The reduction starts at 95% on level 1, so in real life terms we are dealing with something like a highly redundant stream of data, etc. - -It is definitely visible that we are dealing with specialized content here, so the results are very scattered. This package does not do very well at levels 1-4, but picks up significantly at level 5 and levels 7 and 8 offering great speed for the achieved compression. - -So if you know you content is extremely compressible you might want to go slightly higher than the defaults. The standard library has a huge gap between levels 3 and 4 in terms of speed (2.75x slowdown), so it offers little "middle ground". - -## Medium-High Compressible - -This is a pretty common test corpus: [enwik9](http://mattmahoney.net/dc/textdata.html). It contains the first 10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. This is a very good test of typical text based compression and more data heavy streams. - -We see a similar picture here as in "Web Content". On equal levels some compression is sacrificed for more speed. Level 5 seems to be the best trade-off between speed and size, beating stdlib level 3 in both. - -## Medium Compressible - -I will combine two test sets, one [10GB file set](http://mattmahoney.net/dc/10gb.html) and a VM disk image (~8GB). Both contain different data types and represent a typical backup scenario. - -The most notable thing is how quickly the standard library drops to very low compression speeds around level 5-6 without any big gains in compression. Since this type of data is fairly common, this does not seem like good behavior. - - -## Un-compressible Content - -This is mainly a test of how good the algorithms are at detecting un-compressible input. The standard library only offers this feature with very conservative settings at level 1. Obviously there is no reason for the algorithms to try to compress input that cannot be compressed. The only downside is that it might skip some compressible data on false detections. - - -## Huffman only compression - -This compression library adds a special compression level, named `HuffmanOnly`, which allows near linear time compression. This is done by completely disabling matching of previous data, and only reduce the number of bits to represent each character. - -This means that often used characters, like 'e' and ' ' (space) in text use the fewest bits to represent, and rare characters like '¤' takes more bits to represent. For more information see [wikipedia](https://en.wikipedia.org/wiki/Huffman_coding) or this nice [video](https://youtu.be/ZdooBTdW5bM). - -Since this type of compression has much less variance, the compression speed is mostly unaffected by the input data, and is usually more than *180MB/s* for a single core. - -The downside is that the compression ratio is usually considerably worse than even the fastest conventional compression. The compression ratio can never be better than 8:1 (12.5%). - -The linear time compression can be used as a "better than nothing" mode, where you cannot risk the encoder to slow down on some content. For comparison, the size of the "Twain" text is *233460 bytes* (+29% vs. level 1) and encode speed is 144MB/s (4.5x level 1). So in this case you trade a 30% size increase for a 4 times speedup. - -For more information see my blog post on [Fast Linear Time Compression](http://blog.klauspost.com/constant-time-gzipzip-compression/). - -This is implemented on Go 1.7 as "Huffman Only" mode, though not exposed for gzip. - -# Other packages - -Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code): - -* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. -* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. -* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. -* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. -* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. -* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index. -* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor. - -# license - -This code is licensed under the same conditions as the original Go code. See LICENSE file. +# compress + +This package provides various compression algorithms. + +* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go. +* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy. +* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). +* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams. +* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. +* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently. +* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. + +[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) +[![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml) +[![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge) + +# package usage + +Use `go get github.com/klauspost/compress@latest` to add it to your project. + +This package will support the current Go version and 2 versions back. + +* Use the `nounsafe` tag to disable all use of the "unsafe" package. +* Use the `noasm` tag to disable all assembly across packages. + +Use the links above for more information on each. + +# changelog + +* Feb 9th, 2026 [1.18.4](https://github.com/klauspost/compress/releases/tag/v1.18.4) + * gzhttp: Add zstandard to server handler wrapper https://github.com/klauspost/compress/pull/1121 + * zstd: Add ResetWithOptions to encoder/decoder https://github.com/klauspost/compress/pull/1122 + * gzhttp: preserve qvalue when extra parameters follow in Accept-Encoding by @analytically in https://github.com/klauspost/compress/pull/1116 + +* Jan 16th, 2026 [1.18.3](https://github.com/klauspost/compress/releases/tag/v1.18.3) + * Downstream CVE-2025-61728. See [golang/go#77102](https://github.com/golang/go/issues/77102). + +* Dec 1st, 2025 - [1.18.2](https://github.com/klauspost/compress/releases/tag/v1.18.2) + * flate: Fix invalid encoding on level 9 with single value input in https://github.com/klauspost/compress/pull/1115 + * flate: reduce stateless allocations by @RXamzin in https://github.com/klauspost/compress/pull/1106 + +* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1) - RETRACTED + * zstd: Add simple zstd EncodeTo/DecodeTo functions https://github.com/klauspost/compress/pull/1079 + * zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059 + * s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080 + * zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086 + * gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090 + * flate: Faster load+store https://github.com/klauspost/compress/pull/1104 + * flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101 + * flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103 + +* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0) + * Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036 + * fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028 + * flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043 + * flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045 + * s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048 + * flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049 + * flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050 + +
+ See changes to v1.17.x + +* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11) + * zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017 + * s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014 + * gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011 + * gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013 + +* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10) + * gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978 + * gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002 + * s2: Add EncodeBuffer buffer recycling callback https://github.com/klauspost/compress/pull/982 + * zstd: Improve memory usage on small streaming encodes https://github.com/klauspost/compress/pull/1007 + * flate: read data written with partial flush by @vajexal in https://github.com/klauspost/compress/pull/996 + +* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) + * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 + * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 + * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971 + * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951 + +* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8) + * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885 + * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938 + +* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7) + * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927 + * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930 + +* Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) + * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 + * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 + +* Jan 26th, 2024 - [v1.17.5](https://github.com/klauspost/compress/releases/tag/v1.17.5) + * flate: Fix reset with dictionary on custom window encodes https://github.com/klauspost/compress/pull/912 + * zstd: Add Frame header encoding and stripping https://github.com/klauspost/compress/pull/908 + * zstd: Limit better/best default window to 8MB https://github.com/klauspost/compress/pull/913 + * zstd: Speed improvements by @greatroar in https://github.com/klauspost/compress/pull/896 https://github.com/klauspost/compress/pull/910 + * s2: Fix callbacks for skippable blocks and disallow 0xfe (Padding) by @Jille in https://github.com/klauspost/compress/pull/916 https://github.com/klauspost/compress/pull/917 +https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/compress/pull/918 + +* Dec 1st, 2023 - [v1.17.4](https://github.com/klauspost/compress/releases/tag/v1.17.4) + * huff0: Speed up symbol counting by @greatroar in https://github.com/klauspost/compress/pull/887 + * huff0: Remove byteReader by @greatroar in https://github.com/klauspost/compress/pull/886 + * gzhttp: Allow overriding decompression on transport https://github.com/klauspost/compress/pull/892 + * gzhttp: Clamp compression level https://github.com/klauspost/compress/pull/890 + * gzip: Error out if reserved bits are set https://github.com/klauspost/compress/pull/891 + +* Nov 15th, 2023 - [v1.17.3](https://github.com/klauspost/compress/releases/tag/v1.17.3) + * fse: Fix max header size https://github.com/klauspost/compress/pull/881 + * zstd: Improve better/best compression https://github.com/klauspost/compress/pull/877 + * gzhttp: Fix missing content type on Close https://github.com/klauspost/compress/pull/883 + +* Oct 22nd, 2023 - [v1.17.2](https://github.com/klauspost/compress/releases/tag/v1.17.2) + * zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876 + +* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1) + * s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871 + * flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869 + * s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867 + +* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0) + * Add experimental dictionary builder https://github.com/klauspost/compress/pull/853 + * Add xerial snappy read/writer https://github.com/klauspost/compress/pull/838 + * flate: Add limited window compression https://github.com/klauspost/compress/pull/843 + * s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839 + * flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837 + * gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860 + +
+
+ See changes to v1.16.x + + +* July 1st, 2023 - [v1.16.7](https://github.com/klauspost/compress/releases/tag/v1.16.7) + * zstd: Fix default level first dictionary encode https://github.com/klauspost/compress/pull/829 + * s2: add GetBufferCapacity() method by @GiedriusS in https://github.com/klauspost/compress/pull/832 + +* June 13, 2023 - [v1.16.6](https://github.com/klauspost/compress/releases/tag/v1.16.6) + * zstd: correctly ignore WithEncoderPadding(1) by @ianlancetaylor in https://github.com/klauspost/compress/pull/806 + * zstd: Add amd64 match length assembly https://github.com/klauspost/compress/pull/824 + * gzhttp: Handle informational headers by @rtribotte in https://github.com/klauspost/compress/pull/815 + * s2: Improve Better compression slightly https://github.com/klauspost/compress/pull/663 + +* Apr 16, 2023 - [v1.16.5](https://github.com/klauspost/compress/releases/tag/v1.16.5) + * zstd: readByte needs to use io.ReadFull by @jnoxon in https://github.com/klauspost/compress/pull/802 + * gzip: Fix WriterTo after initial read https://github.com/klauspost/compress/pull/804 + +* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4) + * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784 + * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792 + * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785 + * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 + * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 + * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 + * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 + +* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) + * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 + * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767 + * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766 + * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773 + * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774 + +* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0) + * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685 + * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752 + * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755 + * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748 + * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 + * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 +
+ +
+ See changes to v1.15.x + +* Jan 21st, 2023 (v1.15.15) + * deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739 + * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 + * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 + * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 + +* Jan 3rd, 2023 (v1.15.14) + + * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 + * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 + * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 + * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 + +* Dec 11, 2022 (v1.15.13) + * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 + * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 + +* Oct 26, 2022 (v1.15.12) + + * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 + * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 + +* Sept 26, 2022 (v1.15.11) + + * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 + * zstd: Improve "best" compression by @nightwolfz in https://github.com/klauspost/compress/pull/677 + * zstd: Fix+reduce decompression allocations https://github.com/klauspost/compress/pull/668 + * zstd: Fix non-effective noescape tag https://github.com/klauspost/compress/pull/667 + +* Sept 16, 2022 (v1.15.10) + + * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 + * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 + * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 + * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657 + * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 + * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 + * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 + * Use arrays for constant size copies https://github.com/klauspost/compress/pull/659 + +* July 21, 2022 (v1.15.9) + + * zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645 + * zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644 + * zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643 + +* July 13, 2022 (v1.15.8) + + * gzip: fix stack exhaustion bug in Reader.Read https://github.com/klauspost/compress/pull/641 + * s2: Add Index header trim/restore https://github.com/klauspost/compress/pull/638 + * zstd: Optimize seqdeq amd64 asm by @greatroar in https://github.com/klauspost/compress/pull/636 + * zstd: Improve decoder memcopy https://github.com/klauspost/compress/pull/637 + * huff0: Pass a single bitReader pointer to asm by @greatroar in https://github.com/klauspost/compress/pull/634 + * zstd: Branchless getBits for amd64 w/o BMI2 by @greatroar in https://github.com/klauspost/compress/pull/640 + * gzhttp: Remove header before writing https://github.com/klauspost/compress/pull/639 + +* June 29, 2022 (v1.15.7) + + * s2: Fix absolute forward seeks https://github.com/klauspost/compress/pull/633 + * zip: Merge upstream https://github.com/klauspost/compress/pull/631 + * zip: Re-add zip64 fix https://github.com/klauspost/compress/pull/624 + * zstd: translate fseDecoder.buildDtable into asm by @WojciechMula in https://github.com/klauspost/compress/pull/598 + * flate: Faster histograms https://github.com/klauspost/compress/pull/620 + * deflate: Use compound hcode https://github.com/klauspost/compress/pull/622 + +* June 3, 2022 (v1.15.6) + * s2: Improve coding for long, close matches https://github.com/klauspost/compress/pull/613 + * s2c: Add Snappy/S2 stream recompression https://github.com/klauspost/compress/pull/611 + * zstd: Always use configured block size https://github.com/klauspost/compress/pull/605 + * zstd: Fix incorrect hash table placement for dict encoding in default https://github.com/klauspost/compress/pull/606 + * zstd: Apply default config to ZipDecompressor without options https://github.com/klauspost/compress/pull/608 + * gzhttp: Exclude more common archive formats https://github.com/klauspost/compress/pull/612 + * s2: Add ReaderIgnoreCRC https://github.com/klauspost/compress/pull/609 + * s2: Remove sanity load on index creation https://github.com/klauspost/compress/pull/607 + * snappy: Use dedicated function for scoring https://github.com/klauspost/compress/pull/614 + * s2c+s2d: Use official snappy framed extension https://github.com/klauspost/compress/pull/610 + +* May 25, 2022 (v1.15.5) + * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602 + * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601 + * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596 + * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588 + * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592 + * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599 + * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593 + * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586 + * flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590 + + +* May 11, 2022 (v1.15.4) + * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577) + * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581) + * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583) + * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580) + +* May 5, 2022 (v1.15.3) + * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572) + * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575) + +* Apr 26, 2022 (v1.15.2) + * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537) + * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539) + * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555) + * Minimum version is Go 1.16, added CI test on 1.18. + +* Mar 11, 2022 (v1.15.1) + * huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512) + * zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514) + * zstd: Detect extra block data and report as corrupted in [#520](https://github.com/klauspost/compress/pull/520) + * zstd: Handle zero sized frame content size stricter in [#521](https://github.com/klauspost/compress/pull/521) + * zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523) + +* Mar 3, 2022 (v1.15.0) + * zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498) + * zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505) + * huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507) + * flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509) + * gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400) + * gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510) + +Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines. + +Stream decompression is now faster on asynchronous, since the goroutine allocation much more effectively splits the workload. On typical streams this will typically use 2 cores fully for decompression. When a stream has finished decoding no goroutines will be left over, so decoders can now safely be pooled and still be garbage collected. + +While the release has been extensively tested, it is recommended to testing when upgrading. + +
+ +
+ See changes to v1.14.x + +* Feb 22, 2022 (v1.14.4) + * flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503) + * zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502) + * zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501 + * huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500) + +* Feb 17, 2022 (v1.14.3) + * flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478) + * flate: Faster decompression speed, ~5-10%. [#483](https://github.com/klauspost/compress/pull/483) + * s2: Faster compression with Go v1.18 and amd64 microarch level 3+. [#484](https://github.com/klauspost/compress/pull/484) [#486](https://github.com/klauspost/compress/pull/486) + +* Jan 25, 2022 (v1.14.2) + * zstd: improve header decoder by @dsnet [#476](https://github.com/klauspost/compress/pull/476) + * zstd: Add bigger default blocks [#469](https://github.com/klauspost/compress/pull/469) + * zstd: Remove unused decompression buffer [#470](https://github.com/klauspost/compress/pull/470) + * zstd: Fix logically dead code by @ningmingxiao [#472](https://github.com/klauspost/compress/pull/472) + * flate: Improve level 7-9 [#471](https://github.com/klauspost/compress/pull/471) [#473](https://github.com/klauspost/compress/pull/473) + * zstd: Add noasm tag for xxhash [#475](https://github.com/klauspost/compress/pull/475) + +* Jan 11, 2022 (v1.14.1) + * s2: Add stream index in [#462](https://github.com/klauspost/compress/pull/462) + * flate: Speed and efficiency improvements in [#439](https://github.com/klauspost/compress/pull/439) [#461](https://github.com/klauspost/compress/pull/461) [#455](https://github.com/klauspost/compress/pull/455) [#452](https://github.com/klauspost/compress/pull/452) [#458](https://github.com/klauspost/compress/pull/458) + * zstd: Performance improvement in [#420]( https://github.com/klauspost/compress/pull/420) [#456](https://github.com/klauspost/compress/pull/456) [#437](https://github.com/klauspost/compress/pull/437) [#467](https://github.com/klauspost/compress/pull/467) [#468](https://github.com/klauspost/compress/pull/468) + * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464) + * Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445) +
+ +
+ See changes to v1.13.x + +* Aug 30, 2021 (v1.13.5) + * gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425) + * s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413) + * zstd: pooledZipWriter should return Writers to the same pool [#426](https://github.com/klauspost/compress/pull/426) + * Removed golang/snappy as external dependency for tests [#421](https://github.com/klauspost/compress/pull/421) + +* Aug 12, 2021 (v1.13.4) + * Add [snappy replacement package](https://github.com/klauspost/compress/tree/master/snappy). + * zstd: Fix incorrect encoding in "best" mode [#415](https://github.com/klauspost/compress/pull/415) + +* Aug 3, 2021 (v1.13.3) + * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404) + * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411) + * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406) + * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399) + * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401) + * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410) + +* Jun 14, 2021 (v1.13.1) + * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396) + * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394) + * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389) + * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395) + +* Jun 3, 2021 (v1.13.0) + * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors. + * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382) + * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380) +
+ + +
+ See changes to v1.12.x + +* May 25, 2021 (v1.12.3) + * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374) + * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375) + * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373) + +* Apr 27, 2021 (v1.12.2) + * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365) + * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363) + * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367) + * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358) + * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362) + * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368) + +* Apr 14, 2021 (v1.12.1) + * snappy package removed. Upstream added as dependency. + * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353) + * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352) + * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348) + * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352) + * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346) + * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349) +
+ +
+ See changes to v1.11.x + +* Mar 26, 2021 (v1.11.13) + * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345) + * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336) + * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338) + * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341) + +* Mar 5, 2021 (v1.11.12) + * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives). + * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328) + +* Mar 1, 2021 (v1.11.9) + * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324) + * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325) + * s2: Fix binaries. + +* Feb 25, 2021 (v1.11.8) + * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended. + * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) + * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) + * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) + * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313) + +* Jan 14, 2021 (v1.11.7) + * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309) + * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310) + * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311) + * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308) + * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312) + +* Jan 7, 2021 (v1.11.6) + * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306) + * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305) + +* Dec 20, 2020 (v1.11.4) + * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304) + * Add header decoder [#299](https://github.com/klauspost/compress/pull/299) + * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297) + * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300) + * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303) + +* Nov 15, 2020 (v1.11.3) + * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293) + * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295) + +* Oct 11, 2020 (v1.11.2) + * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291) + +* Oct 1, 2020 (v1.11.1) + * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286) + +* Sept 8, 2020 (v1.11.0) + * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281) + * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282) + * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274) +
+ +
+ See changes to v1.10.x + +* July 8, 2020 (v1.10.11) + * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278) + * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275) + +* June 23, 2020 (v1.10.10) + * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270) + +* June 16, 2020 (v1.10.9): + * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268) + * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266) + * Fuzzit tests removed. The service has been purchased and is no longer available. + +* June 5, 2020 (v1.10.8): + * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265) + +* June 1, 2020 (v1.10.7): + * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) + * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259) + * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263) + +* May 21, 2020: (v1.10.6) + * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252) + * zstd: Stricter decompression checks. + +* April 12, 2020: (v1.10.5) + * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239) + +* Apr 8, 2020: (v1.10.4) + * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247) +* Mar 11, 2020: (v1.10.3) + * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245) + * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244) + * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240) + * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241) + * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238) + +* Feb 27, 2020: (v1.10.2) + * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232) + * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227) + +* Feb 18, 2020: (v1.10.1) + * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226) + * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224) + * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224) + +* Feb 4, 2020: (v1.10.0) + * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216) + * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218) + * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214) + * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186) + +
+ +
+ See changes prior to v1.10.0 + +* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206). +* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204) +* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed. +* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases. +* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192) +* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder. +* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199) +* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features +* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197) +* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198) +* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit. +* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191) +* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188) +* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187) +* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines. +* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate. +* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184) +* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate. +* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180) +* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB. +* Nov 11, 2019: Reduce inflate memory use by 1KB. +* Nov 10, 2019: Less allocations in deflate bit writer. +* Nov 10, 2019: Fix inconsistent error returned by zstd decoder. +* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174) +* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173) +* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172) +* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105) + +
+ +
+ See changes prior to v1.9.0 + +* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169) +* Oct 3, 2019: Fix inconsistent results on broken zstd streams. +* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) +* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools). +* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip). +* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes). +* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option. +* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables. +* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode. +* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding. +* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy. +* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing. +* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing. +* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147) +* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146) +* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144) +* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142) +* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder. +* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder. +* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content. +* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix. +* June 17, 2019: zstd decompression bugfix. +* June 17, 2019: fix 32 bit builds. +* June 17, 2019: Easier use in modules (less dependencies). +* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio. +* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression. +* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels. +* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression! +* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels. +* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added. +* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression). +* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below. +* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0). +* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change. +* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change. +* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function. +* May 28, 2017: Reduce allocations when resetting decoder. +* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7. +* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625). +* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before. +* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update. +* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level. +* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression. +* Mar 24, 2016: Small speedup for level 1-3. +* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. +* Feb 19, 2016: Handle small payloads faster in level 1-3. +* Feb 19, 2016: Added faster level 2 + 3 compression modes. +* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5. +* Feb 14, 2016: Snappy: Merge upstream changes. +* Feb 14, 2016: Snappy: Fix aggressive skipping. +* Feb 14, 2016: Snappy: Update benchmark. +* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression. +* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%. +* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content. +* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup. +* Jan 16, 2016: Optimization on deflate level 1,2,3 compression. +* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives. +* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs. +* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms. +* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update! +* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet). +* Nov 20 2015: Small optimization to bit writer on 64 bit systems. +* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15). +* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate. +* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file +* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x. + +
+ +# deflate usage + +The packages are drop-in replacements for standard library [deflate](https://godoc.org/github.com/klauspost/compress/flate), [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip), and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). Simply replace the import path to use them: + +Typical speed is about 2x of the standard library packages. + +| old import | new import | Documentation | +|------------------|---------------------------------------|-------------------------------------------------------------------------| +| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc) | +| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc) | +| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) | +| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) | + +You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop-in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages. + +The packages implement the same API as the standard library, so you can use the original godoc documentation: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/). + +Currently there is only minor speedup on decompression (mostly CRC32 calculation). + +Memory usage is typically 1MB for a Writer. stdlib is in the same range. +If you expect to have a lot of concurrently allocated Writers consider using +the stateless compression described below. + +For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). + +To disable all assembly add `-tags=noasm`. This works across all packages. + +# Stateless compression + +This package offers stateless compression as a special option for gzip/deflate. +It will do compression but without maintaining any state between Write calls. + +This means there will be no memory kept between Write calls, but compression and speed will be suboptimal. + +This is only relevant in cases where you expect to run many thousands of compressors concurrently, +but with very little activity. This is *not* intended for regular web servers serving individual requests. + +Because of this, the size of actual Write calls will affect output size. + +In gzip, specify level `-3` / `gzip.StatelessCompression` to enable. + +For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter) + +A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer: + +```go + // replace 'ioutil.Discard' with your output. + gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression) + if err != nil { + return err + } + defer gzw.Close() + + w := bufio.NewWriterSize(gzw, 4096) + defer w.Flush() + + // Write to 'w' +``` + +This will only use up to 4KB in memory when the writer is idle. + +Compression is almost always worse than the fastest compression level +and each write will allocate (a little) memory. + + +# Other packages + +Here are other packages of good quality and pure Go (no cgo wrappers or autoconverted code): + +* [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. +* [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. +* [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. +* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. +* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. +* [github.com/minio/zipindex](https://github.com/minio/zipindex) - External ZIP directory index. +* [github.com/ybirader/pzip](https://github.com/ybirader/pzip) - Fast concurrent zip archiver and extractor. + +# license + +This code is licensed under the same conditions as the original Go code. See LICENSE file. + + + + + diff --git a/vendor/github.com/klauspost/compress/flate/deflate.go b/vendor/github.com/klauspost/compress/flate/deflate.go index 66d1657d2..57d17eeab 100644 --- a/vendor/github.com/klauspost/compress/flate/deflate.go +++ b/vendor/github.com/klauspost/compress/flate/deflate.go @@ -6,11 +6,12 @@ package flate import ( - "encoding/binary" "errors" "fmt" "io" "math" + + "github.com/klauspost/compress/internal/le" ) const ( @@ -234,12 +235,9 @@ func (d *compressor) fillWindow(b []byte) { // Calculate 256 hashes at the time (more L1 cache hits) loops := (n + 256 - minMatchLength) / 256 - for j := 0; j < loops; j++ { + for j := range loops { startindex := j * 256 - end := startindex + 256 + minMatchLength - 1 - if end > n { - end = n - } + end := min(startindex+256+minMatchLength-1, n) tocheck := d.window[startindex:end] dstSize := len(tocheck) - minMatchLength + 1 @@ -269,18 +267,12 @@ func (d *compressor) fillWindow(b []byte) { // We only look at chainCount possibilities before giving up. // pos = s.index, prevHead = s.chainHead-s.hashOffset, prevLength=minMatchLength-1, lookahead func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, offset int, ok bool) { - minMatchLook := maxMatchLength - if lookahead < minMatchLook { - minMatchLook = lookahead - } + minMatchLook := min(lookahead, maxMatchLength) win := d.window[0 : pos+minMatchLook] // We quit when we get a match that's at least nice long - nice := len(win) - pos - if d.nice < nice { - nice = d.nice - } + nice := min(d.nice, len(win)-pos) // If we've got a match that's good enough, only look in 1/4 the chain. tries := d.chain @@ -288,10 +280,7 @@ func (d *compressor) findMatch(pos int, prevHead int, lookahead int) (length, of wEnd := win[pos+length] wPos := win[pos:] - minIndex := pos - windowSize - if minIndex < 0 { - minIndex = 0 - } + minIndex := max(pos-windowSize, 0) offset = 0 if d.chain < 100 { @@ -374,7 +363,7 @@ func (d *compressor) writeStoredBlock(buf []byte) error { // of the supplied slice. // The caller must ensure that len(b) >= 4. func hash4(b []byte) uint32 { - return hash4u(binary.LittleEndian.Uint32(b), hashBits) + return hash4u(le.Load32(b, 0), hashBits) } // hash4 returns the hash of u to fit in a hash table with h bits. @@ -389,7 +378,7 @@ func bulkHash4(b []byte, dst []uint32) { if len(b) < 4 { return } - hb := binary.LittleEndian.Uint32(b) + hb := le.Load32(b, 0) dst[0] = hash4u(hb, hashBits) end := len(b) - 4 + 1 @@ -432,7 +421,9 @@ func (d *compressor) deflateLazy() { d.h = newHuffmanEncoder(maxFlateBlockTokens) } var tmp [256]uint16 - for _, v := range d.window[s.index:d.windowEnd] { + toIndex := d.window[s.index:d.windowEnd] + toIndex = toIndex[:min(len(toIndex), maxFlateBlockTokens)] + for _, v := range toIndex { tmp[v]++ } d.h.generate(tmp[:], 15) @@ -480,10 +471,7 @@ func (d *compressor) deflateLazy() { prevOffset := s.offset s.length = minMatchLength - 1 s.offset = 0 - minIndex := s.index - windowSize - if minIndex < 0 { - minIndex = 0 - } + minIndex := max(s.index-windowSize, 0) if s.chainHead-s.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy { if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, lookahead); ok { @@ -503,10 +491,7 @@ func (d *compressor) deflateLazy() { if prevLength < maxMatchLength-checkOff { prevIndex := s.index - 1 if prevIndex+prevLength < s.maxInsertIndex { - end := lookahead - if lookahead > maxMatchLength+checkOff { - end = maxMatchLength + checkOff - } + end := min(lookahead, maxMatchLength+checkOff) end += prevIndex // Hash at match end. @@ -603,15 +588,9 @@ func (d *compressor) deflateLazy() { // table. newIndex := s.index + prevLength - 1 // Calculate missing hashes - end := newIndex - if end > s.maxInsertIndex { - end = s.maxInsertIndex - } + end := min(newIndex, s.maxInsertIndex) end += minMatchLength - 1 - startindex := s.index + 1 - if startindex > s.maxInsertIndex { - startindex = s.maxInsertIndex - } + startindex := min(s.index+1, s.maxInsertIndex) tocheck := d.window[startindex:end] dstSize := len(tocheck) - minMatchLength + 1 if dstSize > 0 { @@ -861,7 +840,7 @@ func (d *compressor) reset(w io.Writer) { } switch d.compressionLevel.chain { case 0: - // level was NoCompression or ConstantCompresssion. + // level was NoCompression or ConstantCompression. d.windowEnd = 0 default: s := d.state diff --git a/vendor/github.com/klauspost/compress/flate/dict_decoder.go b/vendor/github.com/klauspost/compress/flate/dict_decoder.go index bb36351a5..cb855abc4 100644 --- a/vendor/github.com/klauspost/compress/flate/dict_decoder.go +++ b/vendor/github.com/klauspost/compress/flate/dict_decoder.go @@ -104,10 +104,7 @@ func (dd *dictDecoder) writeCopy(dist, length int) int { dstBase := dd.wrPos dstPos := dstBase srcPos := dstPos - dist - endPos := dstPos + length - if endPos > len(dd.hist) { - endPos = len(dd.hist) - } + endPos := min(dstPos+length, len(dd.hist)) // Copy non-overlapping section after destination position. // diff --git a/vendor/github.com/klauspost/compress/flate/fast_encoder.go b/vendor/github.com/klauspost/compress/flate/fast_encoder.go index c8124b5c4..791c9dcbf 100644 --- a/vendor/github.com/klauspost/compress/flate/fast_encoder.go +++ b/vendor/github.com/klauspost/compress/flate/fast_encoder.go @@ -6,8 +6,9 @@ package flate import ( - "encoding/binary" "fmt" + + "github.com/klauspost/compress/internal/le" ) type fastEnc interface { @@ -58,11 +59,11 @@ const ( ) func load3232(b []byte, i int32) uint32 { - return binary.LittleEndian.Uint32(b[i:]) + return le.Load32(b, i) } func load6432(b []byte, i int32) uint64 { - return binary.LittleEndian.Uint64(b[i:]) + return le.Load64(b, i) } type tableEntry struct { @@ -134,8 +135,8 @@ func hashLen(u uint64, length, mls uint8) uint32 { // matchlen will return the match length between offsets and t in src. // The maximum length returned is maxMatchLength - 4. // It is assumed that s > t, that t >=0 and s < len(src). -func (e *fastGen) matchlen(s, t int32, src []byte) int32 { - if debugDecode { +func (e *fastGen) matchlen(s, t int, src []byte) int32 { + if debugDeflate { if t >= s { panic(fmt.Sprint("t >=s:", t, s)) } @@ -149,18 +150,14 @@ func (e *fastGen) matchlen(s, t int32, src []byte) int32 { panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) } } - s1 := int(s) + maxMatchLength - 4 - if s1 > len(src) { - s1 = len(src) - } - - // Extend the match to be as long as possible. - return int32(matchLen(src[s:s1], src[t:])) + a := src[s:min(s+maxMatchLength-4, len(src))] + b := src[t:] + return int32(matchLen(a, b)) } // matchlenLong will return the match length between offsets and t in src. // It is assumed that s > t, that t >=0 and s < len(src). -func (e *fastGen) matchlenLong(s, t int32, src []byte) int32 { +func (e *fastGen) matchlenLong(s, t int, src []byte) int32 { if debugDeflate { if t >= s { panic(fmt.Sprint("t >=s:", t, s)) @@ -175,7 +172,6 @@ func (e *fastGen) matchlenLong(s, t int32, src []byte) int32 { panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) } } - // Extend the match to be as long as possible. return int32(matchLen(src[s:], src[t:])) } diff --git a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go index f70594c34..7151140cc 100644 --- a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go +++ b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go @@ -5,10 +5,11 @@ package flate import ( - "encoding/binary" "fmt" "io" "math" + + "github.com/klauspost/compress/internal/le" ) const ( @@ -210,7 +211,9 @@ func (w *huffmanBitWriter) flush() { n++ } w.bits = 0 - w.write(w.bytes[:n]) + if n > 0 { + w.write(w.bytes[:n]) + } w.nbytes = 0 } @@ -302,10 +305,7 @@ func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litE w.codegenFreq[size]++ count-- for count >= 3 { - n := 6 - if n > count { - n = count - } + n := min(6, count) codegen[outIndex] = 16 outIndex++ codegen[outIndex] = uint8(n - 3) @@ -315,10 +315,7 @@ func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litE } } else { for count >= 11 { - n := 138 - if n > count { - n = count - } + n := min(138, count) codegen[outIndex] = 18 outIndex++ codegen[outIndex] = uint8(n - 11) @@ -437,8 +434,8 @@ func (w *huffmanBitWriter) writeOutBits() { w.nbits -= 48 n := w.nbytes - // We over-write, but faster... - binary.LittleEndian.PutUint64(w.bytes[n:], bits) + // We overwrite, but faster... + le.Store64(w.bytes[:], n, bits) n += 6 if n >= bufferFlushSize { @@ -471,7 +468,7 @@ func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, n w.writeBits(int32(numOffsets-1), 5) w.writeBits(int32(numCodegens-4), 4) - for i := 0; i < numCodegens; i++ { + for i := range numCodegens { value := uint(w.codegenEncoding.codes[codegenOrder[i]].len()) w.writeBits(int32(value), 3) } @@ -649,7 +646,7 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b w.lastHeader = 0 } - numLiterals, numOffsets := w.indexTokens(tokens, !sync) + numLiterals, numOffsets := w.indexTokens(tokens, true) extraBits := 0 ssize, storable := w.storedSize(input) @@ -784,7 +781,7 @@ func (w *huffmanBitWriter) fillTokens() { // literalFreq and offsetFreq, and generates literalEncoding // and offsetEncoding. // The number of literal and offset tokens is returned. -func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, numOffsets int) { +func (w *huffmanBitWriter) indexTokens(t *tokens, alwaysEOB bool) (numLiterals, numOffsets int) { //copy(w.literalFreq[:], t.litHist[:]) *(*[256]uint16)(w.literalFreq[:]) = t.litHist //copy(w.literalFreq[256:], t.extraHist[:]) @@ -794,9 +791,10 @@ func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, num if t.n == 0 { return } - if filled { - return maxNumLit, maxNumDist + if alwaysEOB { + w.literalFreq[endBlockMarker] = 1 } + // get the number of literals numLiterals = len(w.literalFreq) for w.literalFreq[numLiterals-1] == 0 { @@ -854,8 +852,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) bits |= c.code64() << (nbits & 63) nbits += c.len() if nbits >= 48 { - binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 @@ -882,8 +879,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) bits |= c.code64() << (nbits & 63) nbits += c.len() if nbits >= 48 { - binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 @@ -905,8 +901,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) bits |= uint64(extraLength) << (nbits & 63) nbits += extraLengthBits if nbits >= 48 { - binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 @@ -931,8 +926,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) bits |= c.code64() << (nbits & 63) nbits += c.len() if nbits >= 48 { - binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 @@ -953,8 +947,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) bits |= uint64((offset-(offsetComb>>8))&matchOffsetOnlyMask) << (nbits & 63) nbits += uint8(offsetComb) if nbits >= 48 { - binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 @@ -1107,7 +1100,7 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { // We must have at least 48 bits free. if nbits >= 8 { n := nbits >> 3 - binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) + le.Store64(w.bytes[:], nbytes, bits) bits >>= (n * 8) & 63 nbits -= n * 8 nbytes += n @@ -1136,8 +1129,7 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { // Remaining... for _, t := range input { if nbits >= 48 { - binary.LittleEndian.PutUint64(w.bytes[nbytes:], bits) - //*(*uint64)(unsafe.Pointer(&w.bytes[nbytes])) = bits + le.Store64(w.bytes[:], nbytes, bits) bits >>= 48 nbits -= 48 nbytes += 6 diff --git a/vendor/github.com/klauspost/compress/flate/huffman_code.go b/vendor/github.com/klauspost/compress/flate/huffman_code.go index be7b58b47..4b312dea3 100644 --- a/vendor/github.com/klauspost/compress/flate/huffman_code.go +++ b/vendor/github.com/klauspost/compress/flate/huffman_code.go @@ -91,7 +91,7 @@ func generateFixedLiteralEncoding() *huffmanEncoder { h := newHuffmanEncoder(literalCount) codes := h.codes var ch uint16 - for ch = 0; ch < literalCount; ch++ { + for ch = range uint16(literalCount) { var bits uint16 var size uint8 switch { @@ -407,8 +407,8 @@ func histogramSplit(b []byte, h []uint16) { for i, t := range x { v0 := &h[t] v1 := &h[y[i]] - v3 := &h[w[i]] v2 := &h[z[i]] + v3 := &h[w[i]] *v0++ *v1++ *v2++ diff --git a/vendor/github.com/klauspost/compress/flate/inflate.go b/vendor/github.com/klauspost/compress/flate/inflate.go index 2f410d64f..6e90126db 100644 --- a/vendor/github.com/klauspost/compress/flate/inflate.go +++ b/vendor/github.com/klauspost/compress/flate/inflate.go @@ -298,6 +298,14 @@ const ( huffmanGenericReader ) +// flushMode tells decompressor when to return data +type flushMode uint8 + +const ( + syncFlush flushMode = iota // return data after sync flush block + partialFlush // return data after each block +) + // Decompress state. type decompressor struct { // Input source. @@ -332,6 +340,8 @@ type decompressor struct { nb uint final bool + + flushMode flushMode } func (f *decompressor) nextBlock() { @@ -475,7 +485,7 @@ func (f *decompressor) readHuffman() error { f.nb -= 5 + 5 + 4 // (HCLEN+4)*3 bits: code lengths in the magic codeOrder order. - for i := 0; i < nclen; i++ { + for i := range nclen { for f.nb < 3 { if err := f.moreBits(); err != nil { return err @@ -618,7 +628,10 @@ func (f *decompressor) dataBlock() { } if n == 0 { - f.toRead = f.dict.readFlush() + if f.flushMode == syncFlush { + f.toRead = f.dict.readFlush() + } + f.finishBlock() return } @@ -657,8 +670,12 @@ func (f *decompressor) finishBlock() { if f.dict.availRead() > 0 { f.toRead = f.dict.readFlush() } + f.err = io.EOF + } else if f.flushMode == partialFlush && f.dict.availRead() > 0 { + f.toRead = f.dict.readFlush() } + f.step = nextBlock } @@ -759,7 +776,7 @@ func fixedHuffmanDecoderInit() { fixedOnce.Do(func() { // These come from the RFC section 3.2.6. var bits [288]int - for i := 0; i < 144; i++ { + for i := range 144 { bits[i] = 8 } for i := 144; i < 256; i++ { @@ -789,15 +806,25 @@ func (f *decompressor) Reset(r io.Reader, dict []byte) error { return nil } -// NewReader returns a new ReadCloser that can be used -// to read the uncompressed version of r. -// If r does not also implement io.ByteReader, -// the decompressor may read more data than necessary from r. -// It is the caller's responsibility to call Close on the ReadCloser -// when finished reading. -// -// The ReadCloser returned by NewReader also implements Resetter. -func NewReader(r io.Reader) io.ReadCloser { +type ReaderOpt func(*decompressor) + +// WithPartialBlock tells decompressor to return after each block, +// so it can read data written with partial flush +func WithPartialBlock() ReaderOpt { + return func(f *decompressor) { + f.flushMode = partialFlush + } +} + +// WithDict initializes the reader with a preset dictionary +func WithDict(dict []byte) ReaderOpt { + return func(f *decompressor) { + f.dict.init(maxMatchOffset, dict) + } +} + +// NewReaderOpts returns new reader with provided options +func NewReaderOpts(r io.Reader, opts ...ReaderOpt) io.ReadCloser { fixedHuffmanDecoderInit() var f decompressor @@ -806,9 +833,26 @@ func NewReader(r io.Reader) io.ReadCloser { f.codebits = new([numCodes]int) f.step = nextBlock f.dict.init(maxMatchOffset, nil) + + for _, opt := range opts { + opt(&f) + } + return &f } +// NewReader returns a new ReadCloser that can be used +// to read the uncompressed version of r. +// If r does not also implement io.ByteReader, +// the decompressor may read more data than necessary from r. +// It is the caller's responsibility to call Close on the ReadCloser +// when finished reading. +// +// The ReadCloser returned by NewReader also implements Resetter. +func NewReader(r io.Reader) io.ReadCloser { + return NewReaderOpts(r) +} + // NewReaderDict is like NewReader but initializes the reader // with a preset dictionary. The returned Reader behaves as if // the uncompressed data stream started with the given dictionary, @@ -817,13 +861,5 @@ func NewReader(r io.Reader) io.ReadCloser { // // The ReadCloser returned by NewReader also implements Resetter. func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser { - fixedHuffmanDecoderInit() - - var f decompressor - f.r = makeReader(r) - f.bits = new([maxNumLit + maxNumDist]int) - f.codebits = new([numCodes]int) - f.step = nextBlock - f.dict.init(maxMatchOffset, dict) - return &f + return NewReaderOpts(r, WithDict(dict)) } diff --git a/vendor/github.com/klauspost/compress/flate/level1.go b/vendor/github.com/klauspost/compress/flate/level1.go index 703b9a89a..c3581a342 100644 --- a/vendor/github.com/klauspost/compress/flate/level1.go +++ b/vendor/github.com/klauspost/compress/flate/level1.go @@ -1,9 +1,9 @@ package flate import ( - "encoding/binary" "fmt" - "math/bits" + + "github.com/klauspost/compress/internal/le" ) // fastGen maintains the table for matches, @@ -77,6 +77,7 @@ func (e *fastEncL1) Encode(dst *tokens, src []byte) { nextS := s var candidate tableEntry + var t int32 for { nextHash := hashLen(cv, tableBits, hashBytes) candidate = e.table[nextHash] @@ -88,9 +89,8 @@ func (e *fastEncL1) Encode(dst *tokens, src []byte) { now := load6432(src, nextS) e.table[nextHash] = tableEntry{offset: s + e.cur} nextHash = hashLen(now, tableBits, hashBytes) - - offset := s - (candidate.offset - e.cur) - if offset < maxMatchOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) { + t = candidate.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { e.table[nextHash] = tableEntry{offset: nextS + e.cur} break } @@ -103,8 +103,8 @@ func (e *fastEncL1) Encode(dst *tokens, src []byte) { now >>= 8 e.table[nextHash] = tableEntry{offset: s + e.cur} - offset = s - (candidate.offset - e.cur) - if offset < maxMatchOffset && uint32(cv) == load3232(src, candidate.offset-e.cur) { + t = candidate.offset - e.cur + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { e.table[nextHash] = tableEntry{offset: nextS + e.cur} break } @@ -120,36 +120,10 @@ func (e *fastEncL1) Encode(dst *tokens, src []byte) { // literal bytes prior to s. // Extend the 4-byte match as long as possible. - t := candidate.offset - e.cur - var l = int32(4) - if false { - l = e.matchlenLong(s+4, t+4, src) + 4 - } else { - // inlined: - a := src[s+4:] - b := src[t+4:] - for len(a) >= 8 { - if diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b); diff != 0 { - l += int32(bits.TrailingZeros64(diff) >> 3) - break - } - l += 8 - a = a[8:] - b = b[8:] - } - if len(a) < 8 { - b = b[:len(a)] - for i := range a { - if a[i] != b[i] { - break - } - l++ - } - } - } + l := e.matchlenLong(int(s+4), int(t+4), src) + 4 // Extend backwards - for t > 0 && s > nextEmit && src[t-1] == src[s-1] { + for t > 0 && s > nextEmit && le.Load8(src, t-1) == le.Load8(src, s-1) { s-- t-- l++ @@ -221,8 +195,8 @@ func (e *fastEncL1) Encode(dst *tokens, src []byte) { candidate = e.table[currHash] e.table[currHash] = tableEntry{offset: o + 2} - offset := s - (candidate.offset - e.cur) - if offset > maxMatchOffset || uint32(x) != load3232(src, candidate.offset-e.cur) { + t = candidate.offset - e.cur + if s-t > maxMatchOffset || uint32(x) != load3232(src, t) { cv = x >> 8 s++ break diff --git a/vendor/github.com/klauspost/compress/flate/level2.go b/vendor/github.com/klauspost/compress/flate/level2.go index 876dfbe30..c8d047f2d 100644 --- a/vendor/github.com/klauspost/compress/flate/level2.go +++ b/vendor/github.com/klauspost/compress/flate/level2.go @@ -126,7 +126,7 @@ func (e *fastEncL2) Encode(dst *tokens, src []byte) { // Extend the 4-byte match as long as possible. t := candidate.offset - e.cur - l := e.matchlenLong(s+4, t+4, src) + 4 + l := e.matchlenLong(int(s+4), int(t+4), src) + 4 // Extend backwards for t > 0 && s > nextEmit && src[t-1] == src[s-1] { diff --git a/vendor/github.com/klauspost/compress/flate/level3.go b/vendor/github.com/klauspost/compress/flate/level3.go index 7aa2b72a1..33f9fb152 100644 --- a/vendor/github.com/klauspost/compress/flate/level3.go +++ b/vendor/github.com/klauspost/compress/flate/level3.go @@ -135,7 +135,7 @@ func (e *fastEncL3) Encode(dst *tokens, src []byte) { // Extend the 4-byte match as long as possible. // t := candidate.offset - e.cur - l := e.matchlenLong(s+4, t+4, src) + 4 + l := e.matchlenLong(int(s+4), int(t+4), src) + 4 // Extend backwards for t > 0 && s > nextEmit && src[t-1] == src[s-1] { diff --git a/vendor/github.com/klauspost/compress/flate/level4.go b/vendor/github.com/klauspost/compress/flate/level4.go index 23c08b325..88509e197 100644 --- a/vendor/github.com/klauspost/compress/flate/level4.go +++ b/vendor/github.com/klauspost/compress/flate/level4.go @@ -98,19 +98,19 @@ func (e *fastEncL4) Encode(dst *tokens, src []byte) { e.bTable[nextHashL] = entry t = lCandidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, lCandidate.offset-e.cur) { + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { // We got a long match. Use that. break } t = sCandidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, sCandidate.offset-e.cur) { + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { // Found a 4 match... lCandidate = e.bTable[hash7(next, tableBits)] // If the next long is a candidate, check if we should use that instead... - lOff := nextS - (lCandidate.offset - e.cur) - if lOff < maxMatchOffset && load3232(src, lCandidate.offset-e.cur) == uint32(next) { + lOff := lCandidate.offset - e.cur + if nextS-lOff < maxMatchOffset && load3232(src, lOff) == uint32(next) { l1, l2 := matchLen(src[s+4:], src[t+4:]), matchLen(src[nextS+4:], src[nextS-lOff+4:]) if l2 > l1 { s = nextS @@ -127,7 +127,7 @@ func (e *fastEncL4) Encode(dst *tokens, src []byte) { // them as literal bytes. // Extend the 4-byte match as long as possible. - l := e.matchlenLong(s+4, t+4, src) + 4 + l := e.matchlenLong(int(s+4), int(t+4), src) + 4 // Extend backwards for t > 0 && s > nextEmit && src[t-1] == src[s-1] { diff --git a/vendor/github.com/klauspost/compress/flate/level5.go b/vendor/github.com/klauspost/compress/flate/level5.go index 1f61ec182..a22ad7d12 100644 --- a/vendor/github.com/klauspost/compress/flate/level5.go +++ b/vendor/github.com/klauspost/compress/flate/level5.go @@ -111,16 +111,16 @@ func (e *fastEncL5) Encode(dst *tokens, src []byte) { t = lCandidate.Cur.offset - e.cur if s-t < maxMatchOffset { - if uint32(cv) == load3232(src, lCandidate.Cur.offset-e.cur) { + if uint32(cv) == load3232(src, t) { // Store the next match e.table[nextHashS] = tableEntry{offset: nextS + e.cur} eLong := &e.bTable[nextHashL] eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur t2 := lCandidate.Prev.offset - e.cur - if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, lCandidate.Prev.offset-e.cur) { - l = e.matchlen(s+4, t+4, src) + 4 - ml1 := e.matchlen(s+4, t2+4, src) + 4 + if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) { + l = e.matchlen(int(s+4), int(t+4), src) + 4 + ml1 := e.matchlen(int(s+4), int(t2+4), src) + 4 if ml1 > l { t = t2 l = ml1 @@ -130,7 +130,7 @@ func (e *fastEncL5) Encode(dst *tokens, src []byte) { break } t = lCandidate.Prev.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, lCandidate.Prev.offset-e.cur) { + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { // Store the next match e.table[nextHashS] = tableEntry{offset: nextS + e.cur} eLong := &e.bTable[nextHashL] @@ -140,9 +140,9 @@ func (e *fastEncL5) Encode(dst *tokens, src []byte) { } t = sCandidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, sCandidate.offset-e.cur) { + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { // Found a 4 match... - l = e.matchlen(s+4, t+4, src) + 4 + l = e.matchlen(int(s+4), int(t+4), src) + 4 lCandidate = e.bTable[nextHashL] // Store the next match @@ -153,8 +153,8 @@ func (e *fastEncL5) Encode(dst *tokens, src []byte) { // If the next long is a candidate, use that... t2 := lCandidate.Cur.offset - e.cur if nextS-t2 < maxMatchOffset { - if load3232(src, lCandidate.Cur.offset-e.cur) == uint32(next) { - ml := e.matchlen(nextS+4, t2+4, src) + 4 + if load3232(src, t2) == uint32(next) { + ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 if ml > l { t = t2 s = nextS @@ -164,8 +164,8 @@ func (e *fastEncL5) Encode(dst *tokens, src []byte) { } // If the previous long is a candidate, use that... t2 = lCandidate.Prev.offset - e.cur - if nextS-t2 < maxMatchOffset && load3232(src, lCandidate.Prev.offset-e.cur) == uint32(next) { - ml := e.matchlen(nextS+4, t2+4, src) + 4 + if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) { + ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 if ml > l { t = t2 s = nextS @@ -185,9 +185,9 @@ func (e *fastEncL5) Encode(dst *tokens, src []byte) { if l == 0 { // Extend the 4-byte match as long as possible. - l = e.matchlenLong(s+4, t+4, src) + 4 + l = e.matchlenLong(int(s+4), int(t+4), src) + 4 } else if l == maxMatchLength { - l += e.matchlenLong(s+l, t+l, src) + l += e.matchlenLong(int(s+l), int(t+l), src) } // Try to locate a better match by checking the end of best match... @@ -203,7 +203,7 @@ func (e *fastEncL5) Encode(dst *tokens, src []byte) { s2 := s + skipBeginning off := s2 - t2 if t2 >= 0 && off < maxMatchOffset && off > 0 { - if l2 := e.matchlenLong(s2, t2, src); l2 > l { + if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l { t = t2 l = l2 s = s2 @@ -423,14 +423,14 @@ func (e *fastEncL5Window) Encode(dst *tokens, src []byte) { t = lCandidate.Cur.offset - e.cur if s-t < maxMatchOffset { - if uint32(cv) == load3232(src, lCandidate.Cur.offset-e.cur) { + if uint32(cv) == load3232(src, t) { // Store the next match e.table[nextHashS] = tableEntry{offset: nextS + e.cur} eLong := &e.bTable[nextHashL] eLong.Cur, eLong.Prev = tableEntry{offset: nextS + e.cur}, eLong.Cur t2 := lCandidate.Prev.offset - e.cur - if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, lCandidate.Prev.offset-e.cur) { + if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) { l = e.matchlen(s+4, t+4, src) + 4 ml1 := e.matchlen(s+4, t2+4, src) + 4 if ml1 > l { @@ -442,7 +442,7 @@ func (e *fastEncL5Window) Encode(dst *tokens, src []byte) { break } t = lCandidate.Prev.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, lCandidate.Prev.offset-e.cur) { + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { // Store the next match e.table[nextHashS] = tableEntry{offset: nextS + e.cur} eLong := &e.bTable[nextHashL] @@ -452,7 +452,7 @@ func (e *fastEncL5Window) Encode(dst *tokens, src []byte) { } t = sCandidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, sCandidate.offset-e.cur) { + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { // Found a 4 match... l = e.matchlen(s+4, t+4, src) + 4 lCandidate = e.bTable[nextHashL] @@ -465,7 +465,7 @@ func (e *fastEncL5Window) Encode(dst *tokens, src []byte) { // If the next long is a candidate, use that... t2 := lCandidate.Cur.offset - e.cur if nextS-t2 < maxMatchOffset { - if load3232(src, lCandidate.Cur.offset-e.cur) == uint32(next) { + if load3232(src, t2) == uint32(next) { ml := e.matchlen(nextS+4, t2+4, src) + 4 if ml > l { t = t2 @@ -476,7 +476,7 @@ func (e *fastEncL5Window) Encode(dst *tokens, src []byte) { } // If the previous long is a candidate, use that... t2 = lCandidate.Prev.offset - e.cur - if nextS-t2 < maxMatchOffset && load3232(src, lCandidate.Prev.offset-e.cur) == uint32(next) { + if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) { ml := e.matchlen(nextS+4, t2+4, src) + 4 if ml > l { t = t2 @@ -677,10 +677,7 @@ func (e *fastEncL5Window) matchlen(s, t int32, src []byte) int32 { panic(fmt.Sprint(s, "-", t, "(", s-t, ") > maxMatchLength (", maxMatchOffset, ")")) } } - s1 := int(s) + maxMatchLength - 4 - if s1 > len(src) { - s1 = len(src) - } + s1 := min(int(s)+maxMatchLength-4, len(src)) // Extend the match to be as long as possible. return int32(matchLen(src[s:s1], src[t:])) diff --git a/vendor/github.com/klauspost/compress/flate/level6.go b/vendor/github.com/klauspost/compress/flate/level6.go index f1e9d98fa..96f5bb430 100644 --- a/vendor/github.com/klauspost/compress/flate/level6.go +++ b/vendor/github.com/klauspost/compress/flate/level6.go @@ -113,7 +113,7 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) { t = lCandidate.Cur.offset - e.cur if s-t < maxMatchOffset { - if uint32(cv) == load3232(src, lCandidate.Cur.offset-e.cur) { + if uint32(cv) == load3232(src, t) { // Long candidate matches at least 4 bytes. // Store the next match @@ -123,9 +123,9 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) { // Check the previous long candidate as well. t2 := lCandidate.Prev.offset - e.cur - if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, lCandidate.Prev.offset-e.cur) { - l = e.matchlen(s+4, t+4, src) + 4 - ml1 := e.matchlen(s+4, t2+4, src) + 4 + if s-t2 < maxMatchOffset && uint32(cv) == load3232(src, t2) { + l = e.matchlen(int(s+4), int(t+4), src) + 4 + ml1 := e.matchlen(int(s+4), int(t2+4), src) + 4 if ml1 > l { t = t2 l = ml1 @@ -136,7 +136,7 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) { } // Current value did not match, but check if previous long value does. t = lCandidate.Prev.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, lCandidate.Prev.offset-e.cur) { + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { // Store the next match e.table[nextHashS] = tableEntry{offset: nextS + e.cur} eLong := &e.bTable[nextHashL] @@ -146,9 +146,9 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) { } t = sCandidate.offset - e.cur - if s-t < maxMatchOffset && uint32(cv) == load3232(src, sCandidate.offset-e.cur) { + if s-t < maxMatchOffset && uint32(cv) == load3232(src, t) { // Found a 4 match... - l = e.matchlen(s+4, t+4, src) + 4 + l = e.matchlen(int(s+4), int(t+4), src) + 4 // Look up next long candidate (at nextS) lCandidate = e.bTable[nextHashL] @@ -162,7 +162,7 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) { const repOff = 1 t2 := s - repeat + repOff if load3232(src, t2) == uint32(cv>>(8*repOff)) { - ml := e.matchlen(s+4+repOff, t2+4, src) + 4 + ml := e.matchlen(int(s+4+repOff), int(t2+4), src) + 4 if ml > l { t = t2 l = ml @@ -175,8 +175,8 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) { // If the next long is a candidate, use that... t2 = lCandidate.Cur.offset - e.cur if nextS-t2 < maxMatchOffset { - if load3232(src, lCandidate.Cur.offset-e.cur) == uint32(next) { - ml := e.matchlen(nextS+4, t2+4, src) + 4 + if load3232(src, t2) == uint32(next) { + ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 if ml > l { t = t2 s = nextS @@ -186,8 +186,8 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) { } // If the previous long is a candidate, use that... t2 = lCandidate.Prev.offset - e.cur - if nextS-t2 < maxMatchOffset && load3232(src, lCandidate.Prev.offset-e.cur) == uint32(next) { - ml := e.matchlen(nextS+4, t2+4, src) + 4 + if nextS-t2 < maxMatchOffset && load3232(src, t2) == uint32(next) { + ml := e.matchlen(int(nextS+4), int(t2+4), src) + 4 if ml > l { t = t2 s = nextS @@ -207,9 +207,9 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) { // Extend the 4-byte match as long as possible. if l == 0 { - l = e.matchlenLong(s+4, t+4, src) + 4 + l = e.matchlenLong(int(s+4), int(t+4), src) + 4 } else if l == maxMatchLength { - l += e.matchlenLong(s+l, t+l, src) + l += e.matchlenLong(int(s+l), int(t+l), src) } // Try to locate a better match by checking the end-of-match... @@ -227,7 +227,7 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) { off := s2 - t2 if off < maxMatchOffset { if off > 0 && t2 >= 0 { - if l2 := e.matchlenLong(s2, t2, src); l2 > l { + if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l { t = t2 l = l2 s = s2 @@ -237,7 +237,7 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) { t2 = eLong.Prev.offset - e.cur - l + skipBeginning off := s2 - t2 if off > 0 && off < maxMatchOffset && t2 >= 0 { - if l2 := e.matchlenLong(s2, t2, src); l2 > l { + if l2 := e.matchlenLong(int(s2), int(t2), src); l2 > l { t = t2 l = l2 s = s2 diff --git a/vendor/github.com/klauspost/compress/flate/matchlen_amd64.go b/vendor/github.com/klauspost/compress/flate/matchlen_amd64.go deleted file mode 100644 index 4bd388584..000000000 --- a/vendor/github.com/klauspost/compress/flate/matchlen_amd64.go +++ /dev/null @@ -1,16 +0,0 @@ -//go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc - -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. - -package flate - -// matchLen returns how many bytes match in a and b -// -// It assumes that: -// -// len(a) <= len(b) and len(a) > 0 -// -//go:noescape -func matchLen(a []byte, b []byte) int diff --git a/vendor/github.com/klauspost/compress/flate/matchlen_amd64.s b/vendor/github.com/klauspost/compress/flate/matchlen_amd64.s deleted file mode 100644 index 9a7655c0f..000000000 --- a/vendor/github.com/klauspost/compress/flate/matchlen_amd64.s +++ /dev/null @@ -1,68 +0,0 @@ -// Copied from S2 implementation. - -//go:build !appengine && !noasm && gc && !noasm - -#include "textflag.h" - -// func matchLen(a []byte, b []byte) int -// Requires: BMI -TEXT ·matchLen(SB), NOSPLIT, $0-56 - MOVQ a_base+0(FP), AX - MOVQ b_base+24(FP), CX - MOVQ a_len+8(FP), DX - - // matchLen - XORL SI, SI - CMPL DX, $0x08 - JB matchlen_match4_standalone - -matchlen_loopback_standalone: - MOVQ (AX)(SI*1), BX - XORQ (CX)(SI*1), BX - TESTQ BX, BX - JZ matchlen_loop_standalone - -#ifdef GOAMD64_v3 - TZCNTQ BX, BX -#else - BSFQ BX, BX -#endif - SARQ $0x03, BX - LEAL (SI)(BX*1), SI - JMP gen_match_len_end - -matchlen_loop_standalone: - LEAL -8(DX), DX - LEAL 8(SI), SI - CMPL DX, $0x08 - JAE matchlen_loopback_standalone - -matchlen_match4_standalone: - CMPL DX, $0x04 - JB matchlen_match2_standalone - MOVL (AX)(SI*1), BX - CMPL (CX)(SI*1), BX - JNE matchlen_match2_standalone - LEAL -4(DX), DX - LEAL 4(SI), SI - -matchlen_match2_standalone: - CMPL DX, $0x02 - JB matchlen_match1_standalone - MOVW (AX)(SI*1), BX - CMPW (CX)(SI*1), BX - JNE matchlen_match1_standalone - LEAL -2(DX), DX - LEAL 2(SI), SI - -matchlen_match1_standalone: - CMPL DX, $0x01 - JB gen_match_len_end - MOVB (AX)(SI*1), BL - CMPB (CX)(SI*1), BL - JNE gen_match_len_end - INCL SI - -gen_match_len_end: - MOVQ SI, ret+48(FP) - RET diff --git a/vendor/github.com/klauspost/compress/flate/matchlen_generic.go b/vendor/github.com/klauspost/compress/flate/matchlen_generic.go index ad5cd814b..6149384aa 100644 --- a/vendor/github.com/klauspost/compress/flate/matchlen_generic.go +++ b/vendor/github.com/klauspost/compress/flate/matchlen_generic.go @@ -1,27 +1,29 @@ -//go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm - // Copyright 2019+ Klaus Post. All rights reserved. // License information can be found in the LICENSE file. package flate import ( - "encoding/binary" "math/bits" + + "github.com/klauspost/compress/internal/le" ) // matchLen returns the maximum common prefix length of a and b. // a must be the shortest of the two. func matchLen(a, b []byte) (n int) { - for ; len(a) >= 8 && len(b) >= 8; a, b = a[8:], b[8:] { - diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b) + left := len(a) + for left >= 8 { + diff := le.Load64(a, n) ^ le.Load64(b, n) if diff != 0 { return n + bits.TrailingZeros64(diff)>>3 } n += 8 + left -= 8 } + a = a[n:] + b = b[n:] for i := range a { if a[i] != b[i] { break @@ -29,5 +31,4 @@ func matchLen(a, b []byte) (n int) { n++ } return n - } diff --git a/vendor/github.com/klauspost/compress/flate/regmask_other.go b/vendor/github.com/klauspost/compress/flate/regmask_other.go index 1b7a2cbd7..e62caf711 100644 --- a/vendor/github.com/klauspost/compress/flate/regmask_other.go +++ b/vendor/github.com/klauspost/compress/flate/regmask_other.go @@ -1,5 +1,4 @@ //go:build !amd64 -// +build !amd64 package flate diff --git a/vendor/github.com/klauspost/compress/flate/stateless.go b/vendor/github.com/klauspost/compress/flate/stateless.go index f3d4139ef..455ed3e2b 100644 --- a/vendor/github.com/klauspost/compress/flate/stateless.go +++ b/vendor/github.com/klauspost/compress/flate/stateless.go @@ -4,6 +4,8 @@ import ( "io" "math" "sync" + + "github.com/klauspost/compress/internal/le" ) const ( @@ -54,18 +56,24 @@ func NewStatelessWriter(dst io.Writer) io.WriteCloser { // bitWriterPool contains bit writers that can be reused. var bitWriterPool = sync.Pool{ - New: func() interface{} { + New: func() any { return newHuffmanBitWriter(nil) }, } +// tokensPool contains tokens struct objects that can be reused +var tokensPool = sync.Pool{ + New: func() any { + return &tokens{} + }, +} + // StatelessDeflate allows compressing directly to a Writer without retaining state. // When returning everything will be flushed. // Up to 8KB of an optional dictionary can be given which is presumed to precede the block. // Longer dictionaries will be truncated and will still produce valid output. // Sending nil dictionary is perfectly fine. func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error { - var dst tokens bw := bitWriterPool.Get().(*huffmanBitWriter) bw.reset(out) defer func() { @@ -89,6 +97,12 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error { // For subsequent loops, keep shallow dict reference to avoid alloc+copy. var inDict []byte + dst := tokensPool.Get().(*tokens) + dst.Reset() + defer func() { + tokensPool.Put(dst) + }() + for len(in) > 0 { todo := in if len(inDict) > 0 { @@ -111,9 +125,9 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error { } // Compress if len(inDict) == 0 { - statelessEnc(&dst, todo, int16(len(dict))) + statelessEnc(dst, todo, int16(len(dict))) } else { - statelessEnc(&dst, inDict[:maxStatelessDict+len(todo)], maxStatelessDict) + statelessEnc(dst, inDict[:maxStatelessDict+len(todo)], maxStatelessDict) } isEof := eof && len(in) == 0 @@ -127,7 +141,7 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error { // If we removed less than 1/16th, huffman compress the block. bw.writeBlockHuff(isEof, uncompressed, len(in) == 0) } else { - bw.writeBlockDynamic(&dst, isEof, uncompressed, len(in) == 0) + bw.writeBlockDynamic(dst, isEof, uncompressed, len(in) == 0) } if len(in) > 0 { // Retain a dict if we have more @@ -152,18 +166,11 @@ func hashSL(u uint32) uint32 { } func load3216(b []byte, i int16) uint32 { - // Help the compiler eliminate bounds checks on the read so it can be done in a single read. - b = b[i:] - b = b[:4] - return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 + return le.Load32(b, i) } func load6416(b []byte, i int16) uint64 { - // Help the compiler eliminate bounds checks on the read so it can be done in a single read. - b = b[i:] - b = b[:8] - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + return le.Load64(b, i) } func statelessEnc(dst *tokens, src []byte, startAt int16) { @@ -189,7 +196,7 @@ func statelessEnc(dst *tokens, src []byte, startAt int16) { // Index until startAt if startAt > 0 { cv := load3232(src, 0) - for i := int16(0); i < startAt; i++ { + for i := range startAt { table[hashSL(cv)] = tableEntry{offset: i} cv = (cv >> 8) | (uint32(src[i+4]) << 24) } diff --git a/vendor/github.com/klauspost/compress/fse/README.md b/vendor/github.com/klauspost/compress/fse/README.md index ea7324da6..27d8ed56f 100644 --- a/vendor/github.com/klauspost/compress/fse/README.md +++ b/vendor/github.com/klauspost/compress/fse/README.md @@ -1,79 +1,79 @@ -# Finite State Entropy - -This package provides Finite State Entropy encoding and decoding. - -Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) -encoding provides a fast near-optimal symbol encoding/decoding -for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). - -This can be used for compressing input with a lot of similar input values to the smallest number of bytes. -This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, -but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. - -* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) - -## News - - * Feb 2018: First implementation released. Consider this beta software for now. - -# Usage - -This package provides a low level interface that allows to compress single independent blocks. - -Each block is separate, and there is no built in integrity checks. -This means that the caller should keep track of block sizes and also do checksums if needed. - -Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. -You must provide input and will receive the output and maybe an error. - -These error values can be returned: - -| Error | Description | -|---------------------|-----------------------------------------------------------------------------| -| `` | Everything ok, output is returned | -| `ErrIncompressible` | Returned when input is judged to be too hard to compress | -| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | -| `(error)` | An internal error occurred. | - -As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. - -To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object -that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same -object can be used for both. - -Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this -you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. - -Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. -You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back -your input was likely corrupted. - -It is important to note that a successful decoding does *not* mean your output matches your original input. -There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. - -For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). - -# Performance - -A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. -All compression functions are currently only running on the calling goroutine so only one core will be used per block. - -The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input -is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be -beneficial to transpose all your input values down by 64. - -With moderate block sizes around 64k speed are typically 200MB/s per core for compression and -around 300MB/s decompression speed. - -The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. - -# Plans - -At one point, more internals will be exposed to facilitate more "expert" usage of the components. - -A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). - -# Contributing - -Contributions are always welcome. Be aware that adding public functions will require good justification and breaking +# Finite State Entropy + +This package provides Finite State Entropy encoding and decoding. + +Finite State Entropy (also referenced as [tANS](https://en.wikipedia.org/wiki/Asymmetric_numeral_systems#tANS)) +encoding provides a fast near-optimal symbol encoding/decoding +for byte blocks as implemented in [zstandard](https://github.com/facebook/zstd). + +This can be used for compressing input with a lot of similar input values to the smallest number of bytes. +This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, +but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. + +* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/fse) + +## News + + * Feb 2018: First implementation released. Consider this beta software for now. + +# Usage + +This package provides a low level interface that allows to compress single independent blocks. + +Each block is separate, and there is no built in integrity checks. +This means that the caller should keep track of block sizes and also do checksums if needed. + +Compressing a block is done via the [`Compress`](https://godoc.org/github.com/klauspost/compress/fse#Compress) function. +You must provide input and will receive the output and maybe an error. + +These error values can be returned: + +| Error | Description | +|---------------------|-----------------------------------------------------------------------------| +| `` | Everything ok, output is returned | +| `ErrIncompressible` | Returned when input is judged to be too hard to compress | +| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | +| `(error)` | An internal error occurred. | + +As can be seen above there are errors that will be returned even under normal operation so it is important to handle these. + +To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/fse#Scratch) object +that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same +object can be used for both. + +Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this +you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. + +Decompressing is done by calling the [`Decompress`](https://godoc.org/github.com/klauspost/compress/fse#Decompress) function. +You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back +your input was likely corrupted. + +It is important to note that a successful decoding does *not* mean your output matches your original input. +There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. + +For more detailed usage, see examples in the [godoc documentation](https://godoc.org/github.com/klauspost/compress/fse#pkg-examples). + +# Performance + +A lot of factors are affecting speed. Block sizes and compressibility of the material are primary factors. +All compression functions are currently only running on the calling goroutine so only one core will be used per block. + +The compressor is significantly faster if symbols are kept as small as possible. The highest byte value of the input +is used to reduce some of the processing, so if all your input is above byte value 64 for instance, it may be +beneficial to transpose all your input values down by 64. + +With moderate block sizes around 64k speed are typically 200MB/s per core for compression and +around 300MB/s decompression speed. + +The same hardware typically does Huffman (deflate) encoding at 125MB/s and decompression at 100MB/s. + +# Plans + +At one point, more internals will be exposed to facilitate more "expert" usage of the components. + +A streaming interface is also likely to be implemented. Likely compatible with [FSE stream format](https://github.com/Cyan4973/FiniteStateEntropy/blob/dev/programs/fileio.c#L261). + +# Contributing + +Contributions are always welcome. Be aware that adding public functions will require good justification and breaking changes will likely not be accepted. If in doubt open an issue before writing the PR. \ No newline at end of file diff --git a/vendor/github.com/klauspost/compress/fse/bitwriter.go b/vendor/github.com/klauspost/compress/fse/bitwriter.go index e82fa3bb7..d58b3fe42 100644 --- a/vendor/github.com/klauspost/compress/fse/bitwriter.go +++ b/vendor/github.com/klauspost/compress/fse/bitwriter.go @@ -143,7 +143,7 @@ func (b *bitWriter) flush32() { // flushAlign will flush remaining full bytes and align to next byte boundary. func (b *bitWriter) flushAlign() { nbBytes := (b.nBits + 7) >> 3 - for i := uint8(0); i < nbBytes; i++ { + for i := range nbBytes { b.out = append(b.out, byte(b.bitContainer>>(i*8))) } b.nBits = 0 diff --git a/vendor/github.com/klauspost/compress/fse/compress.go b/vendor/github.com/klauspost/compress/fse/compress.go index 074018d8f..8c8baa4fc 100644 --- a/vendor/github.com/klauspost/compress/fse/compress.go +++ b/vendor/github.com/klauspost/compress/fse/compress.go @@ -396,7 +396,7 @@ func (s *Scratch) buildCTable() error { if v > largeLimit { s.zeroBits = true } - for nbOccurrences := int16(0); nbOccurrences < v; nbOccurrences++ { + for range v { tableSymbol[position] = symbol position = (position + step) & tableMask for position > highThreshold { diff --git a/vendor/github.com/klauspost/compress/fse/decompress.go b/vendor/github.com/klauspost/compress/fse/decompress.go index cc05d0f7e..0c7dd4ffe 100644 --- a/vendor/github.com/klauspost/compress/fse/decompress.go +++ b/vendor/github.com/klauspost/compress/fse/decompress.go @@ -15,7 +15,7 @@ const ( // It is possible, but by no way guaranteed that corrupt data will // return an error. // It is up to the caller to verify integrity of the returned data. -// Use a predefined Scrach to set maximum acceptable output size. +// Use a predefined Scratch to set maximum acceptable output size. func Decompress(b []byte, s *Scratch) ([]byte, error) { s, err := s.prepare(b) if err != nil { diff --git a/vendor/github.com/klauspost/compress/huff0/README.md b/vendor/github.com/klauspost/compress/huff0/README.md index 8b6e5c663..26d5101b3 100644 --- a/vendor/github.com/klauspost/compress/huff0/README.md +++ b/vendor/github.com/klauspost/compress/huff0/README.md @@ -1,89 +1,89 @@ -# Huff0 entropy compression - -This package provides Huff0 encoding and decoding as used in zstd. - -[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), -a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU -(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. - -This can be used for compressing input with a lot of similar input values to the smallest number of bytes. -This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, -but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. - -* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) - -## News - -This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package. - -This ensures that most functionality is well tested. - -# Usage - -This package provides a low level interface that allows to compress single independent blocks. - -Each block is separate, and there is no built in integrity checks. -This means that the caller should keep track of block sizes and also do checksums if needed. - -Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and -[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. -You must provide input and will receive the output and maybe an error. - -These error values can be returned: - -| Error | Description | -|---------------------|-----------------------------------------------------------------------------| -| `` | Everything ok, output is returned | -| `ErrIncompressible` | Returned when input is judged to be too hard to compress | -| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | -| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | -| `(error)` | An internal error occurred. | - - -As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. - -To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object -that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same -object can be used for both. - -Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this -you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. - -The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. - -## Tables and re-use - -Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. - -The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) -that controls this behaviour. See the documentation for details. This can be altered between each block. - -Do however note that this information is *not* stored in the output block and it is up to the users of the package to -record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, -based on the boolean reported back from the CompressXX call. - -If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the -[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. - -## Decompressing - -The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). -This will initialize the decoding tables. -You can supply the complete block to `ReadTable` and it will return the data part of the block -which can be given to the decompressor. - -Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) -or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. - -For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. - -You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back -your input was likely corrupted. - -It is important to note that a successful decoding does *not* mean your output matches your original input. -There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. - -# Contributing - -Contributions are always welcome. Be aware that adding public functions will require good justification and breaking -changes will likely not be accepted. If in doubt open an issue before writing the PR. +# Huff0 entropy compression + +This package provides Huff0 encoding and decoding as used in zstd. + +[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), +a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU +(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. + +This can be used for compressing input with a lot of similar input values to the smallest number of bytes. +This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, +but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. + +* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) + +## News + +This is used as part of the [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression package. + +This ensures that most functionality is well tested. + +# Usage + +This package provides a low level interface that allows to compress single independent blocks. + +Each block is separate, and there is no built in integrity checks. +This means that the caller should keep track of block sizes and also do checksums if needed. + +Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and +[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. +You must provide input and will receive the output and maybe an error. + +These error values can be returned: + +| Error | Description | +|---------------------|-----------------------------------------------------------------------------| +| `` | Everything ok, output is returned | +| `ErrIncompressible` | Returned when input is judged to be too hard to compress | +| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | +| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | +| `(error)` | An internal error occurred. | + + +As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. + +To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object +that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same +object can be used for both. + +Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this +you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. + +The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. + +## Tables and re-use + +Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. + +The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) +that controls this behaviour. See the documentation for details. This can be altered between each block. + +Do however note that this information is *not* stored in the output block and it is up to the users of the package to +record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, +based on the boolean reported back from the CompressXX call. + +If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the +[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. + +## Decompressing + +The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). +This will initialize the decoding tables. +You can supply the complete block to `ReadTable` and it will return the data part of the block +which can be given to the decompressor. + +Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) +or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. + +For concurrently decompressing content with a fixed table a stateless [`Decoder`](https://godoc.org/github.com/klauspost/compress/huff0#Decoder) can be requested which will remain correct as long as the scratch is unchanged. The capacity of the provided slice indicates the expected output size. + +You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back +your input was likely corrupted. + +It is important to note that a successful decoding does *not* mean your output matches your original input. +There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. + +# Contributing + +Contributions are always welcome. Be aware that adding public functions will require good justification and breaking +changes will likely not be accepted. If in doubt open an issue before writing the PR. diff --git a/vendor/github.com/klauspost/compress/huff0/bitreader.go b/vendor/github.com/klauspost/compress/huff0/bitreader.go index e36d9742f..bfc7a523d 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitreader.go +++ b/vendor/github.com/klauspost/compress/huff0/bitreader.go @@ -6,10 +6,11 @@ package huff0 import ( - "encoding/binary" "errors" "fmt" "io" + + "github.com/klauspost/compress/internal/le" ) // bitReader reads a bitstream in reverse. @@ -46,7 +47,7 @@ func (b *bitReaderBytes) init(in []byte) error { return nil } -// peekBitsFast requires that at least one bit is requested every time. +// peekByteFast requires that at least one byte is requested every time. // There are no checks if the buffer is filled. func (b *bitReaderBytes) peekByteFast() uint8 { got := uint8(b.value >> 56) @@ -66,8 +67,7 @@ func (b *bitReaderBytes) fillFast() { } // 2 bounds checks. - v := b.in[b.off-4 : b.off] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + low := le.Load32(b.in, b.off-4) b.value |= uint64(low) << (b.bitsRead - 32) b.bitsRead -= 32 b.off -= 4 @@ -76,7 +76,7 @@ func (b *bitReaderBytes) fillFast() { // fillFastStart() assumes the bitReaderBytes is empty and there is at least 8 bytes to read. func (b *bitReaderBytes) fillFastStart() { // Do single re-slice to avoid bounds checks. - b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) + b.value = le.Load64(b.in, b.off-8) b.bitsRead = 0 b.off -= 8 } @@ -86,9 +86,8 @@ func (b *bitReaderBytes) fill() { if b.bitsRead < 32 { return } - if b.off > 4 { - v := b.in[b.off-4 : b.off] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + if b.off >= 4 { + low := le.Load32(b.in, b.off-4) b.value |= uint64(low) << (b.bitsRead - 32) b.bitsRead -= 32 b.off -= 4 @@ -175,9 +174,7 @@ func (b *bitReaderShifted) fillFast() { return } - // 2 bounds checks. - v := b.in[b.off-4 : b.off] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + low := le.Load32(b.in, b.off-4) b.value |= uint64(low) << ((b.bitsRead - 32) & 63) b.bitsRead -= 32 b.off -= 4 @@ -185,8 +182,7 @@ func (b *bitReaderShifted) fillFast() { // fillFastStart() assumes the bitReaderShifted is empty and there is at least 8 bytes to read. func (b *bitReaderShifted) fillFastStart() { - // Do single re-slice to avoid bounds checks. - b.value = binary.LittleEndian.Uint64(b.in[b.off-8:]) + b.value = le.Load64(b.in, b.off-8) b.bitsRead = 0 b.off -= 8 } @@ -197,8 +193,7 @@ func (b *bitReaderShifted) fill() { return } if b.off > 4 { - v := b.in[b.off-4 : b.off] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + low := le.Load32(b.in, b.off-4) b.value |= uint64(low) << ((b.bitsRead - 32) & 63) b.bitsRead -= 32 b.off -= 4 diff --git a/vendor/github.com/klauspost/compress/huff0/bitwriter.go b/vendor/github.com/klauspost/compress/huff0/bitwriter.go index 0ebc9aaac..41db94cde 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitwriter.go +++ b/vendor/github.com/klauspost/compress/huff0/bitwriter.go @@ -85,7 +85,7 @@ func (b *bitWriter) flush32() { // flushAlign will flush remaining full bytes and align to next byte boundary. func (b *bitWriter) flushAlign() { nbBytes := (b.nBits + 7) >> 3 - for i := uint8(0); i < nbBytes; i++ { + for i := range nbBytes { b.out = append(b.out, byte(b.bitContainer>>(i*8))) } b.nBits = 0 diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index 84aa3d12f..a97cf1b5d 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -276,7 +276,7 @@ func (s *Scratch) compress4X(src []byte) ([]byte, error) { offsetIdx := len(s.Out) s.Out = append(s.Out, sixZeros[:]...) - for i := 0; i < 4; i++ { + for i := range 4 { toDo := src if len(toDo) > segmentSize { toDo = toDo[:segmentSize] @@ -312,7 +312,7 @@ func (s *Scratch) compress4Xp(src []byte) ([]byte, error) { segmentSize := (len(src) + 3) / 4 var wg sync.WaitGroup wg.Add(4) - for i := 0; i < 4; i++ { + for i := range 4 { toDo := src if len(toDo) > segmentSize { toDo = toDo[:segmentSize] @@ -326,7 +326,7 @@ func (s *Scratch) compress4Xp(src []byte) ([]byte, error) { }(i) } wg.Wait() - for i := 0; i < 4; i++ { + for i := range 4 { o := s.tmpOut[i] if len(o) > math.MaxUint16 { // We cannot store the size in the jump table diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index 54bd08b25..7d0efa881 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -626,7 +626,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { var br [4]bitReaderBytes start := 6 - for i := 0; i < 3; i++ { + for i := range 3 { length := int(src[i*2]) | (int(src[i*2+1]) << 8) if start+length >= len(src) { return nil, errors.New("truncated input (or invalid offset)") @@ -798,10 +798,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { remainBytes := dstEvery - (decoded / 4) for i := range br { offset := dstEvery * i - endsAt := offset + remainBytes - if endsAt > len(out) { - endsAt = len(out) - } + endsAt := min(offset+remainBytes, len(out)) br := &br[i] bitsLeft := br.remaining() for bitsLeft > 0 { @@ -864,7 +861,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) { func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { var br [4]bitReaderBytes start := 6 - for i := 0; i < 3; i++ { + for i := range 3 { length := int(src[i*2]) | (int(src[i*2+1]) << 8) if start+length >= len(src) { return nil, errors.New("truncated input (or invalid offset)") @@ -1035,10 +1032,7 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { remainBytes := dstEvery - (decoded / 4) for i := range br { offset := dstEvery * i - endsAt := offset + remainBytes - if endsAt > len(out) { - endsAt = len(out) - } + endsAt := min(offset+remainBytes, len(out)) br := &br[i] bitsLeft := br.remaining() for bitsLeft > 0 { @@ -1136,7 +1130,7 @@ func (s *Scratch) matches(ct cTable, w io.Writer) { errs++ } if errs > 0 { - fmt.Fprintf(w, "%d errros in base, stopping\n", errs) + fmt.Fprintf(w, "%d errors in base, stopping\n", errs) continue } // Ensure that all combinations are covered. @@ -1152,7 +1146,7 @@ func (s *Scratch) matches(ct cTable, w io.Writer) { errs++ } if errs > 20 { - fmt.Fprintf(w, "%d errros, stopping\n", errs) + fmt.Fprintf(w, "%d errors, stopping\n", errs) break } } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index ba7e8e6b0..2d6ef64be 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc // This file contains the specialisation of Decoder.Decompress4X // and Decoder.Decompress1X that use an asm implementation of thir main loops. @@ -58,7 +57,7 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { var br [4]bitReaderShifted // Decode "jump table" start := 6 - for i := 0; i < 3; i++ { + for i := range 3 { length := int(src[i*2]) | (int(src[i*2+1]) << 8) if start+length >= len(src) { return nil, errors.New("truncated input (or invalid offset)") @@ -109,10 +108,7 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { remainBytes := dstEvery - (decoded / 4) for i := range br { offset := dstEvery * i - endsAt := offset + remainBytes - if endsAt > len(out) { - endsAt = len(out) - } + endsAt := min(offset+remainBytes, len(out)) br := &br[i] bitsLeft := br.remaining() for bitsLeft > 0 { diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go index 908c17de6..610392322 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm // This file contains a generic implementation of Decoder.Decompress4X. package huff0 diff --git a/vendor/github.com/klauspost/compress/huff0/huff0.go b/vendor/github.com/klauspost/compress/huff0/huff0.go index 77ecd68e0..67d9e05b6 100644 --- a/vendor/github.com/klauspost/compress/huff0/huff0.go +++ b/vendor/github.com/klauspost/compress/huff0/huff0.go @@ -201,7 +201,7 @@ func (c cTable) write(s *Scratch) error { for i := range hist[:16] { hist[i] = 0 } - for n := uint8(0); n < maxSymbolValue; n++ { + for n := range maxSymbolValue { v := bitsToWeight[c[n].nBits] & 15 huffWeight[n] = v hist[v]++ @@ -271,7 +271,7 @@ func (c cTable) estTableSize(s *Scratch) (sz int, err error) { for i := range hist[:16] { hist[i] = 0 } - for n := uint8(0); n < maxSymbolValue; n++ { + for n := range maxSymbolValue { v := bitsToWeight[c[n].nBits] & 15 huffWeight[n] = v hist[v]++ diff --git a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go index e802579c4..b97f9056f 100644 --- a/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go +++ b/vendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc package cpuinfo diff --git a/vendor/github.com/klauspost/compress/internal/le/le.go b/vendor/github.com/klauspost/compress/internal/le/le.go new file mode 100644 index 000000000..e54909e16 --- /dev/null +++ b/vendor/github.com/klauspost/compress/internal/le/le.go @@ -0,0 +1,5 @@ +package le + +type Indexer interface { + int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 +} diff --git a/vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go b/vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go new file mode 100644 index 000000000..4f2a0d8c5 --- /dev/null +++ b/vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go @@ -0,0 +1,42 @@ +//go:build !(amd64 || arm64 || ppc64le || riscv64) || nounsafe || purego || appengine + +package le + +import ( + "encoding/binary" +) + +// Load8 will load from b at index i. +func Load8[I Indexer](b []byte, i I) byte { + return b[i] +} + +// Load16 will load from b at index i. +func Load16[I Indexer](b []byte, i I) uint16 { + return binary.LittleEndian.Uint16(b[i:]) +} + +// Load32 will load from b at index i. +func Load32[I Indexer](b []byte, i I) uint32 { + return binary.LittleEndian.Uint32(b[i:]) +} + +// Load64 will load from b at index i. +func Load64[I Indexer](b []byte, i I) uint64 { + return binary.LittleEndian.Uint64(b[i:]) +} + +// Store16 will store v at b. +func Store16(b []byte, v uint16) { + binary.LittleEndian.PutUint16(b, v) +} + +// Store32 will store v at b. +func Store32(b []byte, v uint32) { + binary.LittleEndian.PutUint32(b, v) +} + +// Store64 will store v at b. +func Store64[I Indexer](b []byte, i I, v uint64) { + binary.LittleEndian.PutUint64(b[i:], v) +} diff --git a/vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go b/vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go new file mode 100644 index 000000000..218a38bc4 --- /dev/null +++ b/vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go @@ -0,0 +1,52 @@ +// We enable 64 bit LE platforms: + +//go:build (amd64 || arm64 || ppc64le || riscv64) && !nounsafe && !purego && !appengine + +package le + +import ( + "unsafe" +) + +// Load8 will load from b at index i. +func Load8[I Indexer](b []byte, i I) byte { + //return binary.LittleEndian.Uint16(b[i:]) + //return *(*uint16)(unsafe.Pointer(&b[i])) + return *(*byte)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) +} + +// Load16 will load from b at index i. +func Load16[I Indexer](b []byte, i I) uint16 { + //return binary.LittleEndian.Uint16(b[i:]) + //return *(*uint16)(unsafe.Pointer(&b[i])) + return *(*uint16)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) +} + +// Load32 will load from b at index i. +func Load32[I Indexer](b []byte, i I) uint32 { + //return binary.LittleEndian.Uint32(b[i:]) + //return *(*uint32)(unsafe.Pointer(&b[i])) + return *(*uint32)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) +} + +// Load64 will load from b at index i. +func Load64[I Indexer](b []byte, i I) uint64 { + //return binary.LittleEndian.Uint64(b[i:]) + //return *(*uint64)(unsafe.Pointer(&b[i])) + return *(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) +} + +// Store16 will store v at b. +func Store16(b []byte, v uint16) { + *(*uint16)(unsafe.Pointer(unsafe.SliceData(b))) = v +} + +// Store32 will store v at b. +func Store32(b []byte, v uint32) { + *(*uint32)(unsafe.Pointer(unsafe.SliceData(b))) = v +} + +// Store64 will store v at b[i:]. +func Store64[I Indexer](b []byte, i I, v uint64) { + *(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) = v +} diff --git a/vendor/github.com/klauspost/compress/internal/snapref/decode.go b/vendor/github.com/klauspost/compress/internal/snapref/decode.go index 40796a49d..a2c82fcd2 100644 --- a/vendor/github.com/klauspost/compress/internal/snapref/decode.go +++ b/vendor/github.com/klauspost/compress/internal/snapref/decode.go @@ -209,7 +209,7 @@ func (r *Reader) fill() error { if !r.readFull(r.buf[:len(magicBody)], false) { return r.err } - for i := 0; i < len(magicBody); i++ { + for i := range len(magicBody) { if r.buf[i] != magicBody[i] { r.err = ErrCorrupt return r.err diff --git a/vendor/github.com/klauspost/compress/internal/snapref/encode.go b/vendor/github.com/klauspost/compress/internal/snapref/encode.go index 13c6040a5..860a99416 100644 --- a/vendor/github.com/klauspost/compress/internal/snapref/encode.go +++ b/vendor/github.com/klauspost/compress/internal/snapref/encode.go @@ -20,8 +20,10 @@ import ( func Encode(dst, src []byte) []byte { if n := MaxEncodedLen(len(src)); n < 0 { panic(ErrTooLarge) - } else if len(dst) < n { + } else if cap(dst) < n { dst = make([]byte, n) + } else { + dst = dst[:n] } // The block starts with the varint-encoded length of the decompressed bytes. diff --git a/vendor/github.com/klauspost/compress/s2sx.mod b/vendor/github.com/klauspost/compress/s2sx.mod index 5a4412f90..81bda5e29 100644 --- a/vendor/github.com/klauspost/compress/s2sx.mod +++ b/vendor/github.com/klauspost/compress/s2sx.mod @@ -1,4 +1,3 @@ module github.com/klauspost/compress -go 1.19 - +go 1.22 diff --git a/vendor/github.com/klauspost/compress/zlib/reader.go b/vendor/github.com/klauspost/compress/zlib/reader.go index f127d4776..587e4d47c 100644 --- a/vendor/github.com/klauspost/compress/zlib/reader.go +++ b/vendor/github.com/klauspost/compress/zlib/reader.go @@ -26,6 +26,7 @@ package zlib import ( "bufio" "compress/zlib" + "encoding/binary" "hash" "hash/adler32" "io" @@ -33,7 +34,10 @@ import ( "github.com/klauspost/compress/flate" ) -const zlibDeflate = 8 +const ( + zlibDeflate = 8 + zlibMaxWindow = 7 +) var ( // ErrChecksum is returned when reading ZLIB data that has an invalid checksum. @@ -52,7 +56,7 @@ type reader struct { scratch [4]byte } -// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to +// Resetter resets a ReadCloser returned by [NewReader] or [NewReaderDict] // to switch to a new underlying Reader. This permits reusing a ReadCloser // instead of allocating a new one. type Resetter interface { @@ -63,20 +67,20 @@ type Resetter interface { // NewReader creates a new ReadCloser. // Reads from the returned ReadCloser read and decompress data from r. -// If r does not implement io.ByteReader, the decompressor may read more +// If r does not implement [io.ByteReader], the decompressor may read more // data than necessary from r. // It is the caller's responsibility to call Close on the ReadCloser when done. // -// The ReadCloser returned by NewReader also implements Resetter. +// The [io.ReadCloser] returned by NewReader also implements [Resetter]. func NewReader(r io.Reader) (io.ReadCloser, error) { return NewReaderDict(r, nil) } -// NewReaderDict is like NewReader but uses a preset dictionary. +// NewReaderDict is like [NewReader] but uses a preset dictionary. // NewReaderDict ignores the dictionary if the compressed data does not refer to it. -// If the compressed data refers to a different dictionary, NewReaderDict returns ErrDictionary. +// If the compressed data refers to a different dictionary, NewReaderDict returns [ErrDictionary]. // -// The ReadCloser returned by NewReaderDict also implements Resetter. +// The ReadCloser returned by NewReaderDict also implements [Resetter]. func NewReaderDict(r io.Reader, dict []byte) (io.ReadCloser, error) { z := new(reader) err := z.Reset(r, dict) @@ -108,7 +112,7 @@ func (z *reader) Read(p []byte) (int, error) { return n, z.err } // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). - checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3]) + checksum := binary.BigEndian.Uint32(z.scratch[:4]) if checksum != z.digest.Sum32() { z.err = ErrChecksum return n, z.err @@ -116,9 +120,9 @@ func (z *reader) Read(p []byte) (int, error) { return n, io.EOF } -// Calling Close does not close the wrapped io.Reader originally passed to NewReader. +// Calling Close does not close the wrapped [io.Reader] originally passed to [NewReader]. // In order for the ZLIB checksum to be verified, the reader must be -// fully consumed until the io.EOF. +// fully consumed until the [io.EOF]. func (z *reader) Close() error { if z.err != nil && z.err != io.EOF { return z.err @@ -143,8 +147,8 @@ func (z *reader) Reset(r io.Reader, dict []byte) error { } return z.err } - h := uint(z.scratch[0])<<8 | uint(z.scratch[1]) - if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) { + h := binary.BigEndian.Uint16(z.scratch[:2]) + if (z.scratch[0]&0x0f != zlibDeflate) || (z.scratch[0]>>4 > zlibMaxWindow) || (h%31 != 0) { z.err = ErrHeader return z.err } @@ -157,7 +161,7 @@ func (z *reader) Reset(r io.Reader, dict []byte) error { } return z.err } - checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3]) + checksum := binary.BigEndian.Uint32(z.scratch[:4]) if checksum != adler32.Checksum(dict) { z.err = ErrDictionary return z.err diff --git a/vendor/github.com/klauspost/compress/zlib/writer.go b/vendor/github.com/klauspost/compress/zlib/writer.go index 605816ba4..cab9ef3eb 100644 --- a/vendor/github.com/klauspost/compress/zlib/writer.go +++ b/vendor/github.com/klauspost/compress/zlib/writer.go @@ -5,6 +5,7 @@ package zlib import ( + "encoding/binary" "fmt" "hash" "hash/adler32" @@ -20,7 +21,7 @@ const ( BestSpeed = flate.BestSpeed BestCompression = flate.BestCompression DefaultCompression = flate.DefaultCompression - ConstantCompression = flate.ConstantCompression + ConstantCompression = flate.ConstantCompression // Deprecated: Use HuffmanOnly. HuffmanOnly = flate.HuffmanOnly ) @@ -40,7 +41,7 @@ type Writer struct { // NewWriter creates a new Writer. // Writes to the returned Writer are compressed and written to w. // -// It is the caller's responsibility to call Close on the WriteCloser when done. +// It is the caller's responsibility to call Close on the Writer when done. // Writes may be buffered and not flushed until Close. func NewWriter(w io.Writer) *Writer { z, _ := NewWriterLevelDict(w, DefaultCompression, nil) @@ -116,17 +117,13 @@ func (z *Writer) writeHeader() (err error) { if z.dict != nil { z.scratch[1] |= 1 << 5 } - z.scratch[1] += uint8(31 - (uint16(z.scratch[0])<<8+uint16(z.scratch[1]))%31) + z.scratch[1] += uint8(31 - binary.BigEndian.Uint16(z.scratch[:2])%31) if _, err = z.w.Write(z.scratch[0:2]); err != nil { return err } if z.dict != nil { // The next four bytes are the Adler-32 checksum of the dictionary. - checksum := adler32.Checksum(z.dict) - z.scratch[0] = uint8(checksum >> 24) - z.scratch[1] = uint8(checksum >> 16) - z.scratch[2] = uint8(checksum >> 8) - z.scratch[3] = uint8(checksum >> 0) + binary.BigEndian.PutUint32(z.scratch[:], adler32.Checksum(z.dict)) if _, err = z.w.Write(z.scratch[0:4]); err != nil { return err } @@ -192,10 +189,7 @@ func (z *Writer) Close() error { } checksum := z.digest.Sum32() // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952). - z.scratch[0] = uint8(checksum >> 24) - z.scratch[1] = uint8(checksum >> 16) - z.scratch[2] = uint8(checksum >> 8) - z.scratch[3] = uint8(checksum >> 0) + binary.BigEndian.PutUint32(z.scratch[:], checksum) _, z.err = z.w.Write(z.scratch[0:4]) return z.err } diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md index 92e2347bb..c11d7fa28 100644 --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -6,7 +6,7 @@ A high performance compression algorithm is implemented. For now focused on spee This package provides [compression](#Compressor) to and [decompression](#Decompressor) of Zstandard content. -This package is pure Go and without use of "unsafe". +This package is pure Go. Use `noasm` and `nounsafe` to disable relevant features. The `zstd` package is provided as open source software using a Go standard license. diff --git a/vendor/github.com/klauspost/compress/zstd/bitreader.go b/vendor/github.com/klauspost/compress/zstd/bitreader.go index 25ca98394..d41e3e170 100644 --- a/vendor/github.com/klauspost/compress/zstd/bitreader.go +++ b/vendor/github.com/klauspost/compress/zstd/bitreader.go @@ -5,11 +5,12 @@ package zstd import ( - "encoding/binary" "errors" "fmt" "io" "math/bits" + + "github.com/klauspost/compress/internal/le" ) // bitReader reads a bitstream in reverse. @@ -18,6 +19,7 @@ import ( type bitReader struct { in []byte value uint64 // Maybe use [16]byte, but shifting is awkward. + cursor int // offset where next read should end bitsRead uint8 } @@ -32,6 +34,7 @@ func (b *bitReader) init(in []byte) error { if v == 0 { return errors.New("corrupt stream, did not find end of stream") } + b.cursor = len(in) b.bitsRead = 64 b.value = 0 if len(in) >= 8 { @@ -67,18 +70,15 @@ func (b *bitReader) fillFast() { if b.bitsRead < 32 { return } - v := b.in[len(b.in)-4:] - b.in = b.in[:len(b.in)-4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value = (b.value << 32) | uint64(low) + b.cursor -= 4 + b.value = (b.value << 32) | uint64(le.Load32(b.in, b.cursor)) b.bitsRead -= 32 } // fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read. func (b *bitReader) fillFastStart() { - v := b.in[len(b.in)-8:] - b.in = b.in[:len(b.in)-8] - b.value = binary.LittleEndian.Uint64(v) + b.cursor -= 8 + b.value = le.Load64(b.in, b.cursor) b.bitsRead = 0 } @@ -87,25 +87,23 @@ func (b *bitReader) fill() { if b.bitsRead < 32 { return } - if len(b.in) >= 4 { - v := b.in[len(b.in)-4:] - b.in = b.in[:len(b.in)-4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - b.value = (b.value << 32) | uint64(low) + if b.cursor >= 4 { + b.cursor -= 4 + b.value = (b.value << 32) | uint64(le.Load32(b.in, b.cursor)) b.bitsRead -= 32 return } - b.bitsRead -= uint8(8 * len(b.in)) - for len(b.in) > 0 { - b.value = (b.value << 8) | uint64(b.in[len(b.in)-1]) - b.in = b.in[:len(b.in)-1] + b.bitsRead -= uint8(8 * b.cursor) + for b.cursor > 0 { + b.cursor -= 1 + b.value = (b.value << 8) | uint64(b.in[b.cursor]) } } // finished returns true if all bits have been read from the bit stream. func (b *bitReader) finished() bool { - return len(b.in) == 0 && b.bitsRead >= 64 + return b.cursor == 0 && b.bitsRead >= 64 } // overread returns true if more bits have been requested than is on the stream. @@ -115,13 +113,14 @@ func (b *bitReader) overread() bool { // remain returns the number of bits remaining. func (b *bitReader) remain() uint { - return 8*uint(len(b.in)) + 64 - uint(b.bitsRead) + return 8*uint(b.cursor) + 64 - uint(b.bitsRead) } // close the bitstream and returns an error if out-of-buffer reads occurred. func (b *bitReader) close() error { // Release reference. b.in = nil + b.cursor = 0 if !b.finished() { return fmt.Errorf("%d extra bits on block, should be 0", b.remain()) } diff --git a/vendor/github.com/klauspost/compress/zstd/bitwriter.go b/vendor/github.com/klauspost/compress/zstd/bitwriter.go index 1952f175b..b22b297e6 100644 --- a/vendor/github.com/klauspost/compress/zstd/bitwriter.go +++ b/vendor/github.com/klauspost/compress/zstd/bitwriter.go @@ -88,7 +88,7 @@ func (b *bitWriter) flush32() { // flushAlign will flush remaining full bytes and align to next byte boundary. func (b *bitWriter) flushAlign() { nbBytes := (b.nBits + 7) >> 3 - for i := uint8(0); i < nbBytes; i++ { + for i := range nbBytes { b.out = append(b.out, byte(b.bitContainer>>(i*8))) } b.nBits = 0 diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 03744fbc7..2329e996f 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -5,14 +5,10 @@ package zstd import ( - "bytes" - "encoding/binary" "errors" "fmt" "hash/crc32" "io" - "os" - "path/filepath" "sync" "github.com/klauspost/compress/huff0" @@ -58,11 +54,11 @@ const ( ) var ( - huffDecoderPool = sync.Pool{New: func() interface{} { + huffDecoderPool = sync.Pool{New: func() any { return &huff0.Scratch{} }} - fseDecoderPool = sync.Pool{New: func() interface{} { + fseDecoderPool = sync.Pool{New: func() any { return &fseDecoder{} }} ) @@ -557,7 +553,7 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) { if compMode&3 != 0 { return errors.New("corrupt block: reserved bits not zero") } - for i := uint(0); i < 3; i++ { + for i := range uint(3) { mode := seqCompMode((compMode >> (6 - i*2)) & 3) if debugDecoder { println("Table", tableIndex(i), "is", mode) @@ -598,7 +594,9 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) { printf("RLE set to 0x%x, code: %v", symb, v) } case compModeFSE: - println("Reading table for", tableIndex(i)) + if debugDecoder { + println("Reading table for", tableIndex(i)) + } if seq.fse == nil || seq.fse.preDefined { seq.fse = fseDecoderPool.Get().(*fseDecoder) } @@ -646,21 +644,6 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) { println("initializing sequences:", err) return err } - // Extract blocks... - if false && hist.dict == nil { - fatalErr := func(err error) { - if err != nil { - panic(err) - } - } - fn := fmt.Sprintf("n-%d-lits-%d-prev-%d-%d-%d-win-%d.blk", hist.decoders.nSeqs, len(hist.decoders.literals), hist.recentOffsets[0], hist.recentOffsets[1], hist.recentOffsets[2], hist.windowSize) - var buf bytes.Buffer - fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.litLengths.fse)) - fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.matchLengths.fse)) - fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.offsets.fse)) - buf.Write(in) - os.WriteFile(filepath.Join("testdata", "seqs", fn), buf.Bytes(), os.ModePerm) - } return nil } diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go index 32a7f401d..0e33aea44 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockenc.go +++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go @@ -9,6 +9,7 @@ import ( "fmt" "math" "math/bits" + "slices" "github.com/klauspost/compress/huff0" ) @@ -77,6 +78,7 @@ func (b *blockEnc) initNewEncode() { b.recentOffsets = [3]uint32{1, 4, 8} b.litEnc.Reuse = huff0.ReusePolicyNone b.coders.setPrev(nil, nil, nil) + b.dictLitEnc = nil } // reset will reset the block for a new encode, but in the same stream, @@ -457,16 +459,7 @@ func fuzzFseEncoder(data []byte) int { // All 0 return 0 } - maxCount := func(a []uint32) int { - var max uint32 - for _, v := range a { - if v > max { - max = v - } - } - return int(max) - } - cnt := maxCount(hist[:maxSym]) + cnt := int(slices.Max(hist[:maxSym])) if cnt == len(data) { // RLE return 0 @@ -884,15 +877,6 @@ func (b *blockEnc) genCodes() { } } } - maxCount := func(a []uint32) int { - var max uint32 - for _, v := range a { - if v > max { - max = v - } - } - return int(max) - } if debugAsserts && mlMax > maxMatchLengthSymbol { panic(fmt.Errorf("mlMax > maxMatchLengthSymbol (%d)", mlMax)) } @@ -903,7 +887,7 @@ func (b *blockEnc) genCodes() { panic(fmt.Errorf("llMax > maxLiteralLengthSymbol (%d)", llMax)) } - b.coders.mlEnc.HistogramFinished(mlMax, maxCount(mlH[:mlMax+1])) - b.coders.ofEnc.HistogramFinished(ofMax, maxCount(ofH[:ofMax+1])) - b.coders.llEnc.HistogramFinished(llMax, maxCount(llH[:llMax+1])) + b.coders.mlEnc.HistogramFinished(mlMax, int(slices.Max(mlH[:mlMax+1]))) + b.coders.ofEnc.HistogramFinished(ofMax, int(slices.Max(ofH[:ofMax+1]))) + b.coders.llEnc.HistogramFinished(llMax, int(slices.Max(llH[:llMax+1]))) } diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index bbca17234..c7e500f02 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -39,9 +39,6 @@ type Decoder struct { frame *frameDec - // Custom dictionaries. - dicts map[uint32]*dict - // streamWg is the waitgroup for all streams streamWg sync.WaitGroup } @@ -101,12 +98,10 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { d.current.err = ErrDecoderNilInput } - // Transfer option dicts. - d.dicts = make(map[uint32]*dict, len(d.o.dicts)) - for _, dc := range d.o.dicts { - d.dicts[dc.id] = dc + // Initialize dict map if needed. + if d.o.dicts == nil { + d.o.dicts = make(map[uint32]*dict) } - d.o.dicts = nil // Create decoders d.decoders = make(chan *blockDec, d.o.concurrent) @@ -123,7 +118,7 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { } // Read bytes from the decompressed stream into p. -// Returns the number of bytes written and any error that occurred. +// Returns the number of bytes read and any error that occurred. // When the stream is done, io.EOF will be returned. func (d *Decoder) Read(p []byte) (int, error) { var n int @@ -238,6 +233,21 @@ func (d *Decoder) Reset(r io.Reader) error { return nil } +// ResetWithOptions will reset the decoder and apply the given options +// for the next stream or DecodeAll operation. +// Options are applied on top of the existing options. +// Some options cannot be changed on reset and will return an error. +func (d *Decoder) ResetWithOptions(r io.Reader, opts ...DOption) error { + d.o.resetOpt = true + defer func() { d.o.resetOpt = false }() + for _, o := range opts { + if err := o(&d.o); err != nil { + return err + } + } + return d.Reset(r) +} + // drainOutput will drain the output until errEndOfStream is sent. func (d *Decoder) drainOutput() { if d.current.cancel != nil { @@ -323,6 +333,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { frame.bBuf = nil if frame.history.decoders.br != nil { frame.history.decoders.br.in = nil + frame.history.decoders.br.cursor = 0 } d.decoders <- block }() @@ -372,11 +383,9 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { if cap(dst) == 0 && !d.o.limitToCap { // Allocate len(input) * 2 by default if nothing is provided // and we didn't get frame content size. - size := len(input) * 2 - // Cap to 1 MB. - if size > 1<<20 { - size = 1 << 20 - } + size := min( + // Cap to 1 MB. + len(input)*2, 1<<20) if uint64(size) > d.o.maxDecodedSize { size = int(d.o.maxDecodedSize) } @@ -931,7 +940,7 @@ decodeStream: } func (d *Decoder) setDict(frame *frameDec) (err error) { - dict, ok := d.dicts[frame.DictionaryID] + dict, ok := d.o.dicts[frame.DictionaryID] if ok { if debugDecoder { println("setting dict", frame.DictionaryID) diff --git a/vendor/github.com/klauspost/compress/zstd/decoder_options.go b/vendor/github.com/klauspost/compress/zstd/decoder_options.go index 774c5f00f..537627a07 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder_options.go @@ -20,10 +20,11 @@ type decoderOptions struct { concurrent int maxDecodedSize uint64 maxWindowSize uint64 - dicts []*dict + dicts map[uint32]*dict ignoreChecksum bool limitToCap bool decodeBufsBelow int + resetOpt bool } func (o *decoderOptions) setDefault() { @@ -42,8 +43,15 @@ func (o *decoderOptions) setDefault() { // WithDecoderLowmem will set whether to use a lower amount of memory, // but possibly have to allocate more while running. +// Cannot be changed with ResetWithOptions. func WithDecoderLowmem(b bool) DOption { - return func(o *decoderOptions) error { o.lowMem = b; return nil } + return func(o *decoderOptions) error { + if o.resetOpt && b != o.lowMem { + return errors.New("WithDecoderLowmem cannot be changed on Reset") + } + o.lowMem = b + return nil + } } // WithDecoderConcurrency sets the number of created decoders. @@ -53,18 +61,23 @@ func WithDecoderLowmem(b bool) DOption { // inflight blocks. // When decoding streams and setting maximum to 1, // no async decoding will be done. +// The value supplied must be at least 0. // When a value of 0 is provided GOMAXPROCS will be used. // By default this will be set to 4 or GOMAXPROCS, whatever is lower. +// Cannot be changed with ResetWithOptions. func WithDecoderConcurrency(n int) DOption { return func(o *decoderOptions) error { if n < 0 { - return errors.New("concurrency must be at least 1") + return errors.New("concurrency must be at least 0") } + newVal := n if n == 0 { - o.concurrent = runtime.GOMAXPROCS(0) - } else { - o.concurrent = n + newVal = runtime.GOMAXPROCS(0) } + if o.resetOpt && newVal != o.concurrent { + return errors.New("WithDecoderConcurrency cannot be changed on Reset") + } + o.concurrent = newVal return nil } } @@ -73,6 +86,7 @@ func WithDecoderConcurrency(n int) DOption { // non-streaming operations or maximum window size for streaming operations. // This can be used to control memory usage of potentially hostile content. // Maximum is 1 << 63 bytes. Default is 64GiB. +// Can be changed with ResetWithOptions. func WithDecoderMaxMemory(n uint64) DOption { return func(o *decoderOptions) error { if n == 0 { @@ -92,16 +106,20 @@ func WithDecoderMaxMemory(n uint64) DOption { // "zstd --train" from the Zstandard reference implementation. // // If several dictionaries with the same ID are provided, the last one will be used. +// Can be changed with ResetWithOptions. // // [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format func WithDecoderDicts(dicts ...[]byte) DOption { return func(o *decoderOptions) error { + if o.dicts == nil { + o.dicts = make(map[uint32]*dict) + } for _, b := range dicts { d, err := loadDict(b) if err != nil { return err } - o.dicts = append(o.dicts, d) + o.dicts[d.id] = d } return nil } @@ -109,12 +127,16 @@ func WithDecoderDicts(dicts ...[]byte) DOption { // WithDecoderDictRaw registers a dictionary that may be used by the decoder. // The slice content can be arbitrary data. +// Can be changed with ResetWithOptions. func WithDecoderDictRaw(id uint32, content []byte) DOption { return func(o *decoderOptions) error { if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content)) } - o.dicts = append(o.dicts, &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}) + if o.dicts == nil { + o.dicts = make(map[uint32]*dict) + } + o.dicts[id] = &dict{id: id, content: content, offsets: [3]int{1, 4, 8}} return nil } } @@ -124,6 +146,7 @@ func WithDecoderDictRaw(id uint32, content []byte) DOption { // The Decoder will likely allocate more memory based on the WithDecoderLowmem setting. // If WithDecoderMaxMemory is set to a lower value, that will be used. // Default is 512MB, Maximum is ~3.75 TB as per zstandard spec. +// Can be changed with ResetWithOptions. func WithDecoderMaxWindow(size uint64) DOption { return func(o *decoderOptions) error { if size < MinWindowSize { @@ -141,6 +164,7 @@ func WithDecoderMaxWindow(size uint64) DOption { // or any size set in WithDecoderMaxMemory. // This can be used to limit decoding to a specific maximum output size. // Disabled by default. +// Can be changed with ResetWithOptions. func WithDecodeAllCapLimit(b bool) DOption { return func(o *decoderOptions) error { o.limitToCap = b @@ -153,17 +177,37 @@ func WithDecodeAllCapLimit(b bool) DOption { // This typically uses less allocations but will have the full decompressed object in memory. // Note that DecodeAllCapLimit will disable this, as well as giving a size of 0 or less. // Default is 128KiB. +// Cannot be changed with ResetWithOptions. func WithDecodeBuffersBelow(size int) DOption { return func(o *decoderOptions) error { + if o.resetOpt && size != o.decodeBufsBelow { + return errors.New("WithDecodeBuffersBelow cannot be changed on Reset") + } o.decodeBufsBelow = size return nil } } // IgnoreChecksum allows to forcibly ignore checksum checking. +// Can be changed with ResetWithOptions. func IgnoreChecksum(b bool) DOption { return func(o *decoderOptions) error { o.ignoreChecksum = b return nil } } + +// WithDecoderDictDelete removes dictionaries by ID. +// If no ids are passed, all dictionaries are deleted. +// Should be used with ResetWithOptions. +func WithDecoderDictDelete(ids ...uint32) DOption { + return func(o *decoderOptions) error { + if len(ids) == 0 { + clear(o.dicts) + } + for _, id := range ids { + delete(o.dicts, id) + } + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go index 8d5567fe6..2ffbfdf37 100644 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -194,17 +194,17 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { hist := o.History contents := o.Contents debug := o.DebugOut != nil - println := func(args ...interface{}) { + println := func(args ...any) { if o.DebugOut != nil { fmt.Fprintln(o.DebugOut, args...) } } - printf := func(s string, args ...interface{}) { + printf := func(s string, args ...any) { if o.DebugOut != nil { fmt.Fprintf(o.DebugOut, s, args...) } } - print := func(args ...interface{}) { + print := func(args ...any) { if o.DebugOut != nil { fmt.Fprint(o.DebugOut, args...) } @@ -273,6 +273,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { enc.Encode(&block, b) addValues(&remain, block.literals) litTotal += len(block.literals) + if len(block.sequences) == 0 { + continue + } seqs += len(block.sequences) block.genCodes() addHist(&ll, block.coders.llEnc.Histogram()) @@ -286,6 +289,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if offset == 0 { continue } + if int(offset) >= len(o.History) { + continue + } if offset > 3 { newOffsets[offset-3]++ } else { @@ -336,6 +342,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { if seqs/nUsed < 512 { // Use 512 as minimum. nUsed = seqs / 512 + if nUsed == 0 { + nUsed = 1 + } } copyHist := func(dst *fseEncoder, src *[256]int) ([]byte, error) { hist := dst.Histogram() @@ -358,6 +367,28 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { fakeLength += v hist[i] = uint32(v) } + + // Ensure we aren't trying to represent RLE. + if maxCount == fakeLength { + for i := range hist { + if uint8(i) == maxSym { + fakeLength++ + maxSym++ + hist[i+1] = 1 + if maxSym > 1 { + break + } + } + if hist[0] == 0 { + fakeLength++ + hist[i] = 1 + if maxSym > 1 { + break + } + } + } + } + dst.HistogramFinished(maxSym, maxCount) dst.reUsed = false dst.useRLE = false @@ -393,16 +424,10 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { } // Literal table - avgSize := litTotal - if avgSize > huff0.BlockSizeMax/2 { - avgSize = huff0.BlockSizeMax / 2 - } + avgSize := min(litTotal, huff0.BlockSizeMax/2) huffBuff := make([]byte, 0, avgSize) // Target size - div := litTotal / avgSize - if div < 1 { - div = 1 - } + div := max(litTotal/avgSize, 1) if debug { println("Huffman weights:") } @@ -423,7 +448,7 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { huffBuff = append(huffBuff, 255) } scratch := &huff0.Scratch{TableLog: 11} - for tries := 0; tries < 255; tries++ { + for tries := range 255 { scratch = &huff0.Scratch{TableLog: 11} _, _, err = huff0.Compress1X(huffBuff, scratch) if err == nil { @@ -440,7 +465,7 @@ func BuildDict(o BuildDictOptions) ([]byte, error) { // Bail out.... Just generate something huffBuff = append(huffBuff, bytes.Repeat([]byte{255}, 10000)...) - for i := 0; i < 128; i++ { + for i := range 128 { huffBuff = append(huffBuff, byte(i)) } continue diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go index 5ca46038a..c4de134a7 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -8,7 +8,7 @@ import ( ) const ( - dictShardBits = 6 + dictShardBits = 7 ) type fastBase struct { @@ -21,7 +21,7 @@ type fastBase struct { crc *xxhash.Digest tmp [8]byte blk *blockEnc - lastDictID uint32 + lastDict *dict lowMem bool } @@ -41,11 +41,9 @@ func (e *fastBase) AppendCRC(dst []byte) []byte { // or a window size small enough to contain the input size, if > 0. func (e *fastBase) WindowSize(size int64) int32 { if size > 0 && size < int64(e.maxMatchOff) { - b := int32(1) << uint(bits.Len(uint(size))) - // Keep minimum window. - if b < 1024 { - b = 1024 - } + b := max( + // Keep minimum window. + int32(1)< e.maxMatchOff { diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go index 4613724e9..851799322 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -158,11 +158,9 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) { // Use this to estimate literal cost. // Scaled by 10 bits. - bitsPerByte := int32((compress.ShannonEntropyBits(src) * 1024) / len(src)) - // Huffman can never go < 1 bit/byte - if bitsPerByte < 1024 { - bitsPerByte = 1024 - } + bitsPerByte := max( + // Huffman can never go < 1 bit/byte + int32((compress.ShannonEntropyBits(src)*1024)/len(src)), 1024) // Override src src = e.hist @@ -235,10 +233,7 @@ encodeLoop: // Extend candidate match backwards as far as possible. // Do not extend repeats as we can assume they are optimal // and offsets change if s == nextEmit. - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for offset > tMin && s > nextEmit && src[offset-1] == src[s-1] && l < maxMatchLength { s-- offset-- @@ -382,10 +377,7 @@ encodeLoop: nextEmit = s // Index skipped... - end := s - if s > sLimit+4 { - end = sLimit + 4 - } + end := min(s, sLimit+4) off := index0 + e.cur for index0 < end { cv0 := load6432(src, index0) @@ -444,10 +436,7 @@ encodeLoop: nextEmit = s // Index old s + 1 -> s - 1 or sLimit - end := s - if s > sLimit-4 { - end = sLimit - 4 - } + end := min(s, sLimit-4) off := index0 + e.cur for index0 < end { @@ -490,10 +479,13 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { if d == nil { return } + dictChanged := d != e.lastDict // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) || dictChanged { if len(e.dictTable) != len(e.table) { e.dictTable = make([]prevEntry, len(e.table)) + } else { + clear(e.dictTable) } end := int32(len(d.content)) - 8 + e.maxMatchOff for i := e.maxMatchOff; i < end; i += 4 { @@ -521,13 +513,14 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { offset: i + 3, } } - e.lastDictID = d.id } - // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + // Init or copy dict long table + if len(e.dictLongTable) != len(e.longTable) || dictChanged { if len(e.dictLongTable) != len(e.longTable) { e.dictLongTable = make([]prevEntry, len(e.longTable)) + } else { + clear(e.dictLongTable) } if len(d.content) >= 8 { cv := load6432(d.content, 0) @@ -549,8 +542,8 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { off++ } } - e.lastDictID = d.id } + e.lastDict = d // Reset table to initial state copy(e.longTable[:], e.dictLongTable) diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index a4f5bf91f..3305f0924 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -179,9 +179,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -190,10 +190,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -210,12 +207,12 @@ encodeLoop: // Index match start+1 (long) -> s - 1 index0 := s + repOff - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -241,9 +238,9 @@ encodeLoop: if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) { // Consider history as well. var seq seq - lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) + length := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -252,10 +249,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -270,11 +264,11 @@ encodeLoop: } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff2 + s += length + repOff2 nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -480,10 +474,7 @@ encodeLoop: l := matched // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -708,9 +699,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -719,10 +710,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -738,12 +726,12 @@ encodeLoop: blk.sequences = append(blk.sequences, seq) // Index match start+1 (long) -> s - 1 - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -772,9 +760,9 @@ encodeLoop: if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) { // Consider history as well. var seq seq - lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) + length := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -783,10 +771,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -801,11 +786,11 @@ encodeLoop: } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff2 + s += length + repOff2 nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -1005,10 +990,7 @@ encodeLoop: l := matched // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -1120,10 +1102,13 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { if d == nil { return } + dictChanged := d != e.lastDict // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) || dictChanged { if len(e.dictTable) != len(e.table) { e.dictTable = make([]tableEntry, len(e.table)) + } else { + clear(e.dictTable) } end := int32(len(d.content)) - 8 + e.maxMatchOff for i := e.maxMatchOff; i < end; i += 4 { @@ -1151,14 +1136,15 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { offset: i + 3, } } - e.lastDictID = d.id e.allDirty = true } - // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + // Init or copy dict long table + if len(e.dictLongTable) != len(e.longTable) || dictChanged { if len(e.dictLongTable) != len(e.longTable) { e.dictLongTable = make([]prevEntry, len(e.longTable)) + } else { + clear(e.dictLongTable) } if len(d.content) >= 8 { cv := load6432(d.content, 0) @@ -1180,9 +1166,9 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { off++ } } - e.lastDictID = d.id e.allDirty = true } + e.lastDict = d // Reset table to initial state { diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index a154c18f7..2fb6da112 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -13,7 +13,7 @@ const ( dFastLongLen = 8 // Bytes used for table hash dLongTableShardCnt = 1 << (dFastLongTableBits - dictShardBits) // Number of shards in the table - dLongTableShardSize = dFastLongTableSize / tableShardCnt // Size of an individual shard + dLongTableShardSize = dFastLongTableSize / dLongTableShardCnt // Size of an individual shard dFastShortTableBits = tableBits // Bits used in the short match table dFastShortTableSize = 1 << dFastShortTableBits // Size of the table @@ -138,9 +138,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -149,10 +149,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -166,11 +163,11 @@ encodeLoop: println("repeat sequence", seq, "next s:", s) } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -266,10 +263,7 @@ encodeLoop: l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -462,10 +456,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] { repIndex-- start-- @@ -576,10 +567,7 @@ encodeLoop: l := int32(matchLen(src[s+4:], src[t+4:])) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] { s-- t-- @@ -798,9 +786,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -809,10 +797,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { repIndex-- start-- @@ -826,11 +811,11 @@ encodeLoop: println("repeat sequence", seq, "next s:", s) } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -927,10 +912,7 @@ encodeLoop: l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -1058,15 +1040,18 @@ func (e *doubleFastEncoder) Reset(d *dict, singleBlock bool) { // ResetDict will reset and set a dictionary if not nil func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { allDirty := e.allDirty + dictChanged := d != e.lastDict e.fastEncoderDict.Reset(d, singleBlock) if d == nil { return } // Init or copy dict table - if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID { + if len(e.dictLongTable) != len(e.longTable) || dictChanged { if len(e.dictLongTable) != len(e.longTable) { e.dictLongTable = make([]tableEntry, len(e.longTable)) + } else { + clear(e.dictLongTable) } if len(d.content) >= 8 { cv := load6432(d.content, 0) @@ -1083,7 +1068,6 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { } } } - e.lastDictID = d.id allDirty = true } // Reset table to initial state diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index f45a3da7d..5e104f1a4 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -143,10 +143,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - sMin := s - e.maxMatchOff - if sMin < 0 { - sMin = 0 - } + sMin := max(s-e.maxMatchOff, 0) for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch { repIndex-- start-- @@ -223,10 +220,7 @@ encodeLoop: l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -387,10 +381,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - sMin := s - e.maxMatchOff - if sMin < 0 { - sMin = 0 - } + sMin := max(s-e.maxMatchOff, 0) for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] { repIndex-- start-- @@ -469,10 +460,7 @@ encodeLoop: l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] { s-- t-- @@ -655,10 +643,7 @@ encodeLoop: // and have to do special offset treatment. startLimit := nextEmit + 1 - sMin := s - e.maxMatchOff - if sMin < 0 { - sMin = 0 - } + sMin := max(s-e.maxMatchOff, 0) for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch { repIndex-- start-- @@ -735,10 +720,7 @@ encodeLoop: l := e.matchlen(s+4, t+4, src) + 4 // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } + tMin := max(s-e.maxMatchOff, 0) for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { s-- t-- @@ -823,9 +805,11 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { } // Init or copy dict table - if len(e.dictTable) != len(e.table) || d.id != e.lastDictID { + if len(e.dictTable) != len(e.table) || d != e.lastDict { if len(e.dictTable) != len(e.table) { e.dictTable = make([]tableEntry, len(e.table)) + } else { + clear(e.dictTable) } if true { end := e.maxMatchOff + int32(len(d.content)) - 8 @@ -845,7 +829,7 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { } } } - e.lastDictID = d.id + e.lastDict = d e.allDirty = true } diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 72af7ef0f..0f2a00a00 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -6,6 +6,7 @@ package zstd import ( "crypto/rand" + "errors" "fmt" "io" "math" @@ -130,6 +131,29 @@ func (e *Encoder) Reset(w io.Writer) { s.frameContentSize = 0 } +// ResetWithOptions will re-initialize the writer and apply the given options +// as a new, independent stream. +// Options are applied on top of the existing options. +// Some options cannot be changed on reset and will return an error. +func (e *Encoder) ResetWithOptions(w io.Writer, opts ...EOption) error { + e.o.resetOpt = true + defer func() { e.o.resetOpt = false }() + hadDict := e.o.dict != nil + for _, o := range opts { + if err := o(&e.o); err != nil { + return err + } + } + hasDict := e.o.dict != nil + if hadDict != hasDict { + // Dict presence changed — encoder type must be recreated. + e.state.encoder = nil + e.init = sync.Once{} + } + e.Reset(w) + return nil +} + // ResetContentSize will reset and set a content size for the next stream. // If the bytes written does not match the size given an error will be returned // when calling Close(). @@ -149,6 +173,9 @@ func (e *Encoder) ResetContentSize(w io.Writer, size int64) { // and write CRC if requested. func (e *Encoder) Write(p []byte) (n int, err error) { s := &e.state + if s.eofWritten { + return 0, ErrEncoderClosed + } for len(p) > 0 { if len(p)+len(s.filling) < e.o.blockSize { if e.o.crc { @@ -202,7 +229,7 @@ func (e *Encoder) nextBlock(final bool) error { return nil } if final && len(s.filling) > 0 { - s.current = e.EncodeAll(s.filling, s.current[:0]) + s.current = e.encodeAll(s.encoder, s.filling, s.current[:0]) var n2 int n2, s.err = s.w.Write(s.current) if s.err != nil { @@ -288,6 +315,9 @@ func (e *Encoder) nextBlock(final bool) error { s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current s.nInput += int64(len(s.current)) s.wg.Add(1) + if final { + s.eofWritten = true + } go func(src []byte) { if debugEncoder { println("Adding block,", len(src), "bytes, final:", final) @@ -303,9 +333,6 @@ func (e *Encoder) nextBlock(final bool) error { blk := enc.Block() enc.Encode(blk, src) blk.last = final - if final { - s.eofWritten = true - } // Wait for pending writes. s.wWg.Wait() if s.writeErr != nil { @@ -401,12 +428,20 @@ func (e *Encoder) Flush() error { if len(s.filling) > 0 { err := e.nextBlock(false) if err != nil { + // Ignore Flush after Close. + if errors.Is(s.err, ErrEncoderClosed) { + return nil + } return err } } s.wg.Wait() s.wWg.Wait() if s.err != nil { + // Ignore Flush after Close. + if errors.Is(s.err, ErrEncoderClosed) { + return nil + } return s.err } return s.writeErr @@ -420,8 +455,17 @@ func (e *Encoder) Close() error { if s.encoder == nil { return nil } + if s.w == nil { + if len(s.filling) == 0 && !s.headerWritten && !s.eofWritten && s.nInput == 0 { + return nil + } + return errors.New("zstd: encoder has no writer") + } err := e.nextBlock(true) if err != nil { + if errors.Is(s.err, ErrEncoderClosed) { + return nil + } return err } if s.frameContentSize > 0 { @@ -459,6 +503,11 @@ func (e *Encoder) Close() error { } _, s.err = s.w.Write(frame) } + if s.err == nil { + s.err = ErrEncoderClosed + return nil + } + return s.err } @@ -469,6 +518,15 @@ func (e *Encoder) Close() error { // Data compressed with EncodeAll can be decoded with the Decoder, // using either a stream or DecodeAll. func (e *Encoder) EncodeAll(src, dst []byte) []byte { + e.init.Do(e.initialize) + enc := <-e.encoders + defer func() { + e.encoders <- enc + }() + return e.encodeAll(enc, src, dst) +} + +func (e *Encoder) encodeAll(enc encoder, src, dst []byte) []byte { if len(src) == 0 { if e.o.fullZero { // Add frame header. @@ -491,13 +549,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } return dst } - e.init.Do(e.initialize) - enc := <-e.encoders - defer func() { - // Release encoder reference to last block. - // If a non-single block is needed the encoder will reset again. - e.encoders <- enc - }() + // Use single segments when above minimum window and below window size. single := len(src) <= e.o.windowSize && len(src) > MinWindowSize if e.o.single != nil { diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index 20671dcb9..e217be0a1 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -14,6 +14,7 @@ type EOption func(*encoderOptions) error // options retains accumulated state of multiple options. type encoderOptions struct { + resetOpt bool concurrent int level EncoderLevel single *bool @@ -41,6 +42,7 @@ func (o *encoderOptions) setDefault() { level: SpeedDefault, allLitEntropy: false, lowMem: false, + fullZero: true, } } @@ -71,19 +73,28 @@ func (o encoderOptions) encoder() encoder { // WithEncoderCRC will add CRC value to output. // Output will be 4 bytes larger. +// Can be changed with ResetWithOptions. func WithEncoderCRC(b bool) EOption { return func(o *encoderOptions) error { o.crc = b; return nil } } // WithEncoderConcurrency will set the concurrency, // meaning the maximum number of encoders to run concurrently. -// The value supplied must be at least 1. +// The value supplied must be at least 0. +// When a value of 0 is provided GOMAXPROCS will be used. // For streams, setting a value of 1 will disable async compression. // By default this will be set to GOMAXPROCS. +// Cannot be changed with ResetWithOptions. func WithEncoderConcurrency(n int) EOption { return func(o *encoderOptions) error { - if n <= 0 { - return fmt.Errorf("concurrency must be at least 1") + if n < 0 { + return errors.New("concurrency must at least 0") + } + if n == 0 { + n = runtime.GOMAXPROCS(0) + } + if o.resetOpt && n != o.concurrent { + return errors.New("WithEncoderConcurrency cannot be changed on Reset") } o.concurrent = n return nil @@ -95,6 +106,7 @@ func WithEncoderConcurrency(n int) EOption { // A larger value will enable better compression but allocate more memory and, // for above-default values, take considerably longer. // The default value is determined by the compression level and max 8MB. +// Cannot be changed with ResetWithOptions. func WithWindowSize(n int) EOption { return func(o *encoderOptions) error { switch { @@ -105,6 +117,9 @@ func WithWindowSize(n int) EOption { case (n & (n - 1)) != 0: return errors.New("window size must be a power of 2") } + if o.resetOpt && n != o.windowSize { + return errors.New("WithWindowSize cannot be changed on Reset") + } o.windowSize = n o.customWindow = true @@ -122,6 +137,7 @@ func WithWindowSize(n int) EOption { // n must be > 0 and <= 1GB, 1<<30 bytes. // The padded area will be filled with data from crypto/rand.Reader. // If `EncodeAll` is used with data already in the destination, the total size will be multiple of this. +// Can be changed with ResetWithOptions. func WithEncoderPadding(n int) EOption { return func(o *encoderOptions) error { if n <= 0 { @@ -215,12 +231,16 @@ func (e EncoderLevel) String() string { } // WithEncoderLevel specifies a predefined compression level. +// Cannot be changed with ResetWithOptions. func WithEncoderLevel(l EncoderLevel) EOption { return func(o *encoderOptions) error { switch { case l <= speedNotSet || l >= speedLast: return fmt.Errorf("unknown encoder level") } + if o.resetOpt && l != o.level { + return errors.New("WithEncoderLevel cannot be changed on Reset") + } o.level = l if !o.customWindow { switch o.level { @@ -248,6 +268,7 @@ func WithEncoderLevel(l EncoderLevel) EOption { // WithZeroFrames will encode 0 length input as full frames. // This can be needed for compatibility with zstandard usage, // but is not needed for this package. +// Can be changed with ResetWithOptions. func WithZeroFrames(b bool) EOption { return func(o *encoderOptions) error { o.fullZero = b @@ -259,6 +280,7 @@ func WithZeroFrames(b bool) EOption { // Disabling this will skip incompressible data faster, but in cases with no matches but // skewed character distribution compression is lost. // Default value depends on the compression level selected. +// Can be changed with ResetWithOptions. func WithAllLitEntropyCompression(b bool) EOption { return func(o *encoderOptions) error { o.customALEntropy = true @@ -270,6 +292,7 @@ func WithAllLitEntropyCompression(b bool) EOption { // WithNoEntropyCompression will always skip entropy compression of literals. // This can be useful if content has matches, but unlikely to benefit from entropy // compression. Usually the slight speed improvement is not worth enabling this. +// Can be changed with ResetWithOptions. func WithNoEntropyCompression(b bool) EOption { return func(o *encoderOptions) error { o.noEntropy = b @@ -287,6 +310,7 @@ func WithNoEntropyCompression(b bool) EOption { // This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations. // If this is not specified, block encodes will automatically choose this based on the input size and the window size. // This setting has no effect on streamed encodes. +// Can be changed with ResetWithOptions. func WithSingleSegment(b bool) EOption { return func(o *encoderOptions) error { o.single = &b @@ -298,8 +322,12 @@ func WithSingleSegment(b bool) EOption { // slower encoding speed. // This will not change the window size which is the primary function for reducing // memory usage. See WithWindowSize. +// Cannot be changed with ResetWithOptions. func WithLowerEncoderMem(b bool) EOption { return func(o *encoderOptions) error { + if o.resetOpt && b != o.lowMem { + return errors.New("WithLowerEncoderMem cannot be changed on Reset") + } o.lowMem = b return nil } @@ -311,6 +339,7 @@ func WithLowerEncoderMem(b bool) EOption { // "zstd --train" from the Zstandard reference implementation. // // The encoder *may* choose to use no dictionary instead for certain payloads. +// Can be changed with ResetWithOptions. // // [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format func WithEncoderDict(dict []byte) EOption { @@ -328,6 +357,7 @@ func WithEncoderDict(dict []byte) EOption { // // The slice content may contain arbitrary data. It will be used as an initial // history. +// Can be changed with ResetWithOptions. func WithEncoderDictRaw(id uint32, content []byte) EOption { return func(o *encoderOptions) error { if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { @@ -337,3 +367,12 @@ func WithEncoderDictRaw(id uint32, content []byte) EOption { return nil } } + +// WithEncoderDictDelete clears the dictionary, so no dictionary will be used. +// Should be used with ResetWithOptions. +func WithEncoderDictDelete() EOption { + return func(o *encoderOptions) error { + o.dict = nil + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 53e160f7e..d88f067e5 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -146,7 +146,9 @@ func (d *frameDec) reset(br byteBuffer) error { } return err } - printf("raw: %x, mantissa: %d, exponent: %d\n", wd, wd&7, wd>>3) + if debugDecoder { + printf("raw: %x, mantissa: %d, exponent: %d\n", wd, wd&7, wd>>3) + } windowLog := 10 + (wd >> 3) windowBase := uint64(1) << windowLog windowAdd := (windowBase / 8) * uint64(wd&0x7) @@ -236,10 +238,7 @@ func (d *frameDec) reset(br byteBuffer) error { if d.WindowSize == 0 && d.SingleSegment { // We may not need window in this case. - d.WindowSize = d.FrameContentSize - if d.WindowSize < MinWindowSize { - d.WindowSize = MinWindowSize - } + d.WindowSize = max(d.FrameContentSize, MinWindowSize) if d.WindowSize > d.o.maxDecodedSize { if debugDecoder { printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize) diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go index d04a829b0..b8c8607b5 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc package zstd diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go index 8adfebb02..2138f8091 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm package zstd diff --git a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go index ab26326a8..3a0f4e7fb 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go @@ -149,7 +149,7 @@ func (s *fseEncoder) buildCTable() error { if v > largeLimit { s.zeroBits = true } - for nbOccurrences := int16(0); nbOccurrences < v; nbOccurrences++ { + for range v { tableSymbol[position] = symbol position = (position + step) & tableMask for position > highThreshold { diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s index 17901e080..ae7d4d329 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s @@ -162,12 +162,12 @@ finalize: MOVD h, ret+24(FP) RET -// func writeBlocks(d *Digest, b []byte) int +// func writeBlocks(s *Digest, b []byte) int TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 LDP ·primes+0(SB), (prime1, prime2) // Load state. Assume v[1-4] are stored contiguously. - MOVD d+0(FP), digest + MOVD s+0(FP), digest LDP 0(digest), (v1, v2) LDP 16(digest), (v3, v4) diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go index 0be16cefc..9576426e6 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go @@ -1,5 +1,4 @@ //go:build (!amd64 && !arm64) || appengine || !gc || purego || noasm -// +build !amd64,!arm64 appengine !gc purego noasm package xxhash diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go index f41932b7a..1ed18927f 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc // Copyright 2019+ Klaus Post. All rights reserved. // License information can be found in the LICENSE file. diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s index 9a7655c0f..0782b86e3 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_amd64.s @@ -5,7 +5,6 @@ #include "textflag.h" // func matchLen(a []byte, b []byte) int -// Requires: BMI TEXT ·matchLen(SB), NOSPLIT, $0-56 MOVQ a_base+0(FP), AX MOVQ b_base+24(FP), CX @@ -17,17 +16,16 @@ TEXT ·matchLen(SB), NOSPLIT, $0-56 JB matchlen_match4_standalone matchlen_loopback_standalone: - MOVQ (AX)(SI*1), BX - XORQ (CX)(SI*1), BX - TESTQ BX, BX - JZ matchlen_loop_standalone + MOVQ (AX)(SI*1), BX + XORQ (CX)(SI*1), BX + JZ matchlen_loop_standalone #ifdef GOAMD64_v3 TZCNTQ BX, BX #else BSFQ BX, BX #endif - SARQ $0x03, BX + SHRL $0x03, BX LEAL (SI)(BX*1), SI JMP gen_match_len_end diff --git a/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go b/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go index 57b9c31c0..379746c96 100644 --- a/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/matchlen_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm // Copyright 2019+ Klaus Post. All rights reserved. // License information can be found in the LICENSE file. @@ -7,20 +6,25 @@ package zstd import ( - "encoding/binary" "math/bits" + + "github.com/klauspost/compress/internal/le" ) // matchLen returns the maximum common prefix length of a and b. // a must be the shortest of the two. func matchLen(a, b []byte) (n int) { - for ; len(a) >= 8 && len(b) >= 8; a, b = a[8:], b[8:] { - diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b) + left := len(a) + for left >= 8 { + diff := le.Load64(a, n) ^ le.Load64(b, n) if diff != 0 { return n + bits.TrailingZeros64(diff)>>3 } n += 8 + left -= 8 } + a = a[n:] + b = b[n:] for i := range a { if a[i] != b[i] { diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index d7fe6d82d..0bfb0e43c 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -231,10 +231,7 @@ func (s *sequenceDecs) decodeSync(hist []byte) error { llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize] llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state out := s.out - maxBlockSize := maxCompressedBlockSize - if s.windowSize < maxBlockSize { - maxBlockSize = s.windowSize - } + maxBlockSize := min(s.windowSize, maxCompressedBlockSize) if debugDecoder { println("decodeSync: decoding", seqs, "sequences", br.remain(), "bits remain on stream") @@ -245,7 +242,7 @@ func (s *sequenceDecs) decodeSync(hist []byte) error { return io.ErrUnexpectedEOF } var ll, mo, ml int - if len(br.in) > 4+((maxOffsetBits+16+16)>>3) { + if br.cursor > 4+((maxOffsetBits+16+16)>>3) { // inlined function: // ll, mo, ml = s.nextFast(br, llState, mlState, ofState) diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go index 8adabd828..18c3703dd 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -1,5 +1,4 @@ //go:build amd64 && !appengine && !noasm && gc -// +build amd64,!appengine,!noasm,gc package zstd @@ -79,10 +78,7 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { br := s.br - maxBlockSize := maxCompressedBlockSize - if s.windowSize < maxBlockSize { - maxBlockSize = s.windowSize - } + maxBlockSize := min(s.windowSize, maxCompressedBlockSize) ctx := decodeSyncAsmContext{ llTable: s.litLengths.fse.dt[:maxTablesize], @@ -146,7 +142,7 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) default: - return true, fmt.Errorf("sequenceDecs_decode returned erronous code %d", errCode) + return true, fmt.Errorf("sequenceDecs_decode returned erroneous code %d", errCode) } s.seqSize += ctx.litRemain @@ -237,10 +233,7 @@ func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmC func (s *sequenceDecs) decode(seqs []seqVals) error { br := s.br - maxBlockSize := maxCompressedBlockSize - if s.windowSize < maxBlockSize { - maxBlockSize = s.windowSize - } + maxBlockSize := min(s.windowSize, maxCompressedBlockSize) ctx := decodeAsmContext{ llTable: s.litLengths.fse.dt[:maxTablesize], @@ -292,7 +285,7 @@ func (s *sequenceDecs) decode(seqs []seqVals) error { return io.ErrUnexpectedEOF } - return fmt.Errorf("sequenceDecs_decode_amd64 returned erronous code %d", errCode) + return fmt.Errorf("sequenceDecs_decode_amd64 returned erroneous code %d", errCode) } if ctx.litRemain < 0 { diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 5b06174b8..a708ca6d3 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -7,9 +7,9 @@ TEXT ·sequenceDecs_decode_amd64(SB), $8-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX - MOVBQZX 32(CX), BX + MOVBQZX 40(CX), BX MOVQ (CX), AX - MOVQ 8(CX), SI + MOVQ 32(CX), SI ADDQ SI, AX MOVQ AX, (SP) MOVQ ctx+16(FP), AX @@ -299,8 +299,8 @@ sequenceDecs_decode_amd64_match_len_ofs_ok: MOVQ R13, 160(AX) MOVQ br+8(FP), AX MOVQ DX, 24(AX) - MOVB BL, 32(AX) - MOVQ SI, 8(AX) + MOVB BL, 40(AX) + MOVQ SI, 32(AX) // Return success MOVQ $0x00000000, ret+24(FP) @@ -335,9 +335,9 @@ error_overread: TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX - MOVBQZX 32(CX), BX + MOVBQZX 40(CX), BX MOVQ (CX), AX - MOVQ 8(CX), SI + MOVQ 32(CX), SI ADDQ SI, AX MOVQ AX, (SP) MOVQ ctx+16(FP), AX @@ -598,8 +598,8 @@ sequenceDecs_decode_56_amd64_match_len_ofs_ok: MOVQ R13, 160(AX) MOVQ br+8(FP), AX MOVQ DX, 24(AX) - MOVB BL, 32(AX) - MOVQ SI, 8(AX) + MOVB BL, 40(AX) + MOVQ SI, 32(AX) // Return success MOVQ $0x00000000, ret+24(FP) @@ -634,9 +634,9 @@ error_overread: TEXT ·sequenceDecs_decode_bmi2(SB), $8-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX - MOVBQZX 32(BX), DX + MOVBQZX 40(BX), DX MOVQ (BX), CX - MOVQ 8(BX), BX + MOVQ 32(BX), BX ADDQ BX, CX MOVQ CX, (SP) MOVQ ctx+16(FP), CX @@ -884,8 +884,8 @@ sequenceDecs_decode_bmi2_match_len_ofs_ok: MOVQ R12, 160(CX) MOVQ br+8(FP), CX MOVQ AX, 24(CX) - MOVB DL, 32(CX) - MOVQ BX, 8(CX) + MOVB DL, 40(CX) + MOVQ BX, 32(CX) // Return success MOVQ $0x00000000, ret+24(FP) @@ -920,9 +920,9 @@ error_overread: TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX - MOVBQZX 32(BX), DX + MOVBQZX 40(BX), DX MOVQ (BX), CX - MOVQ 8(BX), BX + MOVQ 32(BX), BX ADDQ BX, CX MOVQ CX, (SP) MOVQ ctx+16(FP), CX @@ -1141,8 +1141,8 @@ sequenceDecs_decode_56_bmi2_match_len_ofs_ok: MOVQ R12, 160(CX) MOVQ br+8(FP), CX MOVQ AX, 24(CX) - MOVB DL, 32(CX) - MOVQ BX, 8(CX) + MOVB DL, 40(CX) + MOVQ BX, 32(CX) // Return success MOVQ $0x00000000, ret+24(FP) @@ -1787,9 +1787,9 @@ empty_seqs: TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX - MOVBQZX 32(CX), BX + MOVBQZX 40(CX), BX MOVQ (CX), AX - MOVQ 8(CX), SI + MOVQ 32(CX), SI ADDQ SI, AX MOVQ AX, (SP) MOVQ ctx+16(FP), AX @@ -1814,7 +1814,7 @@ TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 MOVQ 40(SP), AX ADDQ AX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R10, 32(SP) // outBase += outPosition @@ -2281,8 +2281,8 @@ handle_loop: loop_finished: MOVQ br+8(FP), AX MOVQ DX, 24(AX) - MOVB BL, 32(AX) - MOVQ SI, 8(AX) + MOVB BL, 40(AX) + MOVQ SI, 32(AX) // Update the context MOVQ ctx+16(FP), AX @@ -2349,9 +2349,9 @@ error_not_enough_space: TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX - MOVBQZX 32(BX), DX + MOVBQZX 40(BX), DX MOVQ (BX), CX - MOVQ 8(BX), BX + MOVQ 32(BX), BX ADDQ BX, CX MOVQ CX, (SP) MOVQ ctx+16(FP), CX @@ -2376,7 +2376,7 @@ TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 MOVQ 40(SP), CX ADDQ CX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R9, 32(SP) // outBase += outPosition @@ -2801,8 +2801,8 @@ handle_loop: loop_finished: MOVQ br+8(FP), CX MOVQ AX, 24(CX) - MOVB DL, 32(CX) - MOVQ BX, 8(CX) + MOVB DL, 40(CX) + MOVQ BX, 32(CX) // Update the context MOVQ ctx+16(FP), AX @@ -2869,9 +2869,9 @@ error_not_enough_space: TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 MOVQ br+8(FP), CX MOVQ 24(CX), DX - MOVBQZX 32(CX), BX + MOVBQZX 40(CX), BX MOVQ (CX), AX - MOVQ 8(CX), SI + MOVQ 32(CX), SI ADDQ SI, AX MOVQ AX, (SP) MOVQ ctx+16(FP), AX @@ -2896,7 +2896,7 @@ TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 MOVQ 40(SP), AX ADDQ AX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R10, 32(SP) // outBase += outPosition @@ -3465,8 +3465,8 @@ handle_loop: loop_finished: MOVQ br+8(FP), AX MOVQ DX, 24(AX) - MOVB BL, 32(AX) - MOVQ SI, 8(AX) + MOVB BL, 40(AX) + MOVQ SI, 32(AX) // Update the context MOVQ ctx+16(FP), AX @@ -3533,9 +3533,9 @@ error_not_enough_space: TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 MOVQ br+8(FP), BX MOVQ 24(BX), AX - MOVBQZX 32(BX), DX + MOVBQZX 40(BX), DX MOVQ (BX), CX - MOVQ 8(BX), BX + MOVQ 32(BX), BX ADDQ BX, CX MOVQ CX, (SP) MOVQ ctx+16(FP), CX @@ -3560,7 +3560,7 @@ TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 MOVQ 40(SP), CX ADDQ CX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R9, 32(SP) // outBase += outPosition @@ -4087,8 +4087,8 @@ handle_loop: loop_finished: MOVQ br+8(FP), CX MOVQ AX, 24(CX) - MOVB DL, 32(CX) - MOVQ BX, 8(CX) + MOVB DL, 40(CX) + MOVQ BX, 32(CX) // Update the context MOVQ ctx+16(FP), AX diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go index 2fb35b788..516cd9b07 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_generic.go @@ -1,5 +1,4 @@ //go:build !amd64 || appengine || !gc || noasm -// +build !amd64 appengine !gc noasm package zstd @@ -29,7 +28,7 @@ func (s *sequenceDecs) decode(seqs []seqVals) error { } for i := range seqs { var ll, mo, ml int - if len(br.in) > 4+((maxOffsetBits+16+16)>>3) { + if br.cursor > 4+((maxOffsetBits+16+16)>>3) { // inlined function: // ll, mo, ml = s.nextFast(br, llState, mlState, ofState) diff --git a/vendor/github.com/klauspost/compress/zstd/seqenc.go b/vendor/github.com/klauspost/compress/zstd/seqenc.go index 8014174a7..65045eabd 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqenc.go +++ b/vendor/github.com/klauspost/compress/zstd/seqenc.go @@ -69,7 +69,6 @@ var llBitsTable = [maxLLCode + 1]byte{ func llCode(litLength uint32) uint8 { const llDeltaCode = 19 if litLength <= 63 { - // Compiler insists on bounds check (Go 1.12) return llCodeTable[litLength&63] } return uint8(highBit(litLength)) + llDeltaCode @@ -102,7 +101,6 @@ var mlBitsTable = [maxMLCode + 1]byte{ func mlCode(mlBase uint32) uint8 { const mlDeltaCode = 36 if mlBase <= 127 { - // Compiler insists on bounds check (Go 1.12) return mlCodeTable[mlBase&127] } return uint8(highBit(mlBase)) + mlDeltaCode diff --git a/vendor/github.com/klauspost/compress/zstd/simple_go124.go b/vendor/github.com/klauspost/compress/zstd/simple_go124.go new file mode 100644 index 000000000..2efc0497b --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/simple_go124.go @@ -0,0 +1,56 @@ +// Copyright 2025+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. + +//go:build go1.24 + +package zstd + +import ( + "errors" + "runtime" + "sync" + "weak" +) + +var weakMu sync.Mutex +var simpleEnc weak.Pointer[Encoder] +var simpleDec weak.Pointer[Decoder] + +// EncodeTo appends the encoded data from src to dst. +func EncodeTo(dst []byte, src []byte) []byte { + weakMu.Lock() + enc := simpleEnc.Value() + if enc == nil { + var err error + enc, err = NewWriter(nil, WithEncoderConcurrency(runtime.NumCPU()), WithWindowSize(1<<20), WithLowerEncoderMem(true), WithZeroFrames(true)) + if err != nil { + panic("failed to create simple encoder: " + err.Error()) + } + simpleEnc = weak.Make(enc) + } + weakMu.Unlock() + + return enc.EncodeAll(src, dst) +} + +// DecodeTo appends the decoded data from src to dst. +// The maximum decoded size is 1GiB, +// not including what may already be in dst. +func DecodeTo(dst []byte, src []byte) ([]byte, error) { + weakMu.Lock() + dec := simpleDec.Value() + if dec == nil { + var err error + dec, err = NewReader(nil, WithDecoderConcurrency(runtime.NumCPU()), WithDecoderLowmem(true), WithDecoderMaxMemory(1<<30)) + if err != nil { + weakMu.Unlock() + return nil, errors.New("failed to create simple decoder: " + err.Error()) + } + runtime.SetFinalizer(dec, func(d *Decoder) { + d.Close() + }) + simpleDec = weak.Make(dec) + } + weakMu.Unlock() + return dec.DecodeAll(src, dst) +} diff --git a/vendor/github.com/klauspost/compress/zstd/snappy.go b/vendor/github.com/klauspost/compress/zstd/snappy.go index ec13594e8..336c28893 100644 --- a/vendor/github.com/klauspost/compress/zstd/snappy.go +++ b/vendor/github.com/klauspost/compress/zstd/snappy.go @@ -197,7 +197,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { n, r.err = w.Write(r.block.output) if r.err != nil { - return written, err + return written, r.err } written += int64(n) continue @@ -239,7 +239,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { } n, r.err = w.Write(r.block.output) if r.err != nil { - return written, err + return written, r.err } written += int64(n) continue @@ -257,7 +257,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { if !r.readFull(r.buf[:len(snappyMagicBody)], false) { return written, r.err } - for i := 0; i < len(snappyMagicBody); i++ { + for i := range len(snappyMagicBody) { if r.buf[i] != snappyMagicBody[i] { println("r.buf[i] != snappyMagicBody[i]", r.buf[i], snappyMagicBody[i], i) r.err = ErrSnappyCorrupt diff --git a/vendor/github.com/klauspost/compress/zstd/zip.go b/vendor/github.com/klauspost/compress/zstd/zip.go index 29c15c8c4..3198d7189 100644 --- a/vendor/github.com/klauspost/compress/zstd/zip.go +++ b/vendor/github.com/klauspost/compress/zstd/zip.go @@ -19,7 +19,7 @@ const ZipMethodWinZip = 93 const ZipMethodPKWare = 20 // zipReaderPool is the default reader pool. -var zipReaderPool = sync.Pool{New: func() interface{} { +var zipReaderPool = sync.Pool{New: func() any { z, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderMaxWindow(128<<20), WithDecoderConcurrency(1)) if err != nil { panic(err) diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go index 4be7cc736..1a869710d 100644 --- a/vendor/github.com/klauspost/compress/zstd/zstd.go +++ b/vendor/github.com/klauspost/compress/zstd/zstd.go @@ -5,10 +5,11 @@ package zstd import ( "bytes" - "encoding/binary" "errors" "log" "math" + + "github.com/klauspost/compress/internal/le" ) // enable debug printing @@ -88,29 +89,33 @@ var ( // Close has been called. ErrDecoderClosed = errors.New("decoder used after Close") + // ErrEncoderClosed will be returned if the Encoder was used after + // Close has been called. + ErrEncoderClosed = errors.New("encoder used after Close") + // ErrDecoderNilInput is returned when a nil Reader was provided // and an operation other than Reset/DecodeAll/Close was attempted. ErrDecoderNilInput = errors.New("nil input provided as reader") ) -func println(a ...interface{}) { +func println(a ...any) { if debug || debugDecoder || debugEncoder { log.Println(a...) } } -func printf(format string, a ...interface{}) { +func printf(format string, a ...any) { if debug || debugDecoder || debugEncoder { log.Printf(format, a...) } } func load3232(b []byte, i int32) uint32 { - return binary.LittleEndian.Uint32(b[:len(b):len(b)][i:]) + return le.Load32(b, i) } func load6432(b []byte, i int32) uint64 { - return binary.LittleEndian.Uint64(b[:len(b):len(b)][i:]) + return le.Load64(b, i) } type byter interface { diff --git a/vendor/github.com/pingcap/errors/errors.go b/vendor/github.com/pingcap/errors/errors.go index c30343330..9a2ad45c7 100644 --- a/vendor/github.com/pingcap/errors/errors.go +++ b/vendor/github.com/pingcap/errors/errors.go @@ -2,84 +2,84 @@ // // The traditional error handling idiom in Go is roughly akin to // -// if err != nil { -// return err -// } +// if err != nil { +// return err +// } // // which applied recursively up the call stack results in error reports // without context or debugging information. The errors package allows // programmers to add context to the failure path in their code in a way // that does not destroy the original value of the error. // -// Adding context to an error +// # Adding context to an error // // The errors.Annotate function returns a new error that adds context to the // original error by recording a stack trace at the point Annotate is called, // and the supplied message. For example // -// _, err := ioutil.ReadAll(r) -// if err != nil { -// return errors.Annotate(err, "read failed") -// } +// _, err := ioutil.ReadAll(r) +// if err != nil { +// return errors.Annotate(err, "read failed") +// } // // If additional control is required the errors.AddStack and errors.WithMessage // functions destructure errors.Annotate into its component operations of annotating // an error with a stack trace and an a message, respectively. // -// Retrieving the cause of an error +// # Retrieving the cause of an error // // Using errors.Annotate constructs a stack of errors, adding context to the // preceding error. Depending on the nature of the error it may be necessary // to reverse the operation of errors.Annotate to retrieve the original error // for inspection. Any error value which implements this interface // -// type causer interface { -// Cause() error -// } +// type causer interface { +// Cause() error +// } // // can be inspected by errors.Cause. errors.Cause will recursively retrieve // the topmost error which does not implement causer, which is assumed to be // the original cause. For example: // -// switch err := errors.Cause(err).(type) { -// case *MyError: -// // handle specifically -// default: -// // unknown error -// } +// switch err := errors.Cause(err).(type) { +// case *MyError: +// // handle specifically +// default: +// // unknown error +// } // // causer interface is not exported by this package, but is considered a part // of stable public API. // errors.Unwrap is also available: this will retrieve the next error in the chain. // -// Formatted printing of errors +// # Formatted printing of errors // // All error values returned from this package implement fmt.Formatter and can // be formatted by the fmt package. The following verbs are supported // -// %s print the error. If the error has a Cause it will be -// printed recursively -// %v see %s -// %+v extended format. Each Frame of the error's StackTrace will -// be printed in detail. +// %s print the error. If the error has a Cause it will be +// printed recursively +// %v see %s +// %+v extended format. Each Frame of the error's StackTrace will +// be printed in detail. // -// Retrieving the stack trace of an error or wrapper +// # Retrieving the stack trace of an error or wrapper // // New, Errorf, Annotate, and Annotatef record a stack trace at the point they are invoked. // This information can be retrieved with the StackTracer interface that returns // a StackTrace. Where errors.StackTrace is defined as // -// type StackTrace []Frame +// type StackTrace []Frame // // The Frame type represents a call site in the stack trace. Frame supports // the fmt.Formatter interface that can be used for printing information about // the stack trace of this error. For example: // -// if stacked := errors.GetStackTracer(err); stacked != nil { -// for _, f := range stacked.StackTrace() { -// fmt.Printf("%+s:%d\n", f, f) -// } -// } +// if stacked := errors.GetStackTracer(err); stacked != nil { +// for _, f := range stacked.StackTrace() { +// fmt.Printf("%+s:%d\n", f, f) +// } +// } // // See the documentation for Frame.Format for more details. // @@ -91,6 +91,12 @@ import ( "io" ) +// represent an error carries with message +type messenger interface { + // GetSelfMsg get its own message, the message of its cause error is NOT included. + GetSelfMsg() string +} + // New returns an error with the supplied message. // New also records the stack trace at the point it was called. func New(message string) error { @@ -122,7 +128,11 @@ func HasStack(err error) bool { if errWithStack, ok := err.(StackTraceAware); ok { return errWithStack.HasStack() } - return GetStackTracer(err) != nil + // Error.FastGenXXX or call SuspendStack directly will make an empty stack trace, + // which should be considered as no stack trace, to allow upper layer code to + // add stack trace with Trace. + stackTracer := GetStackTracer(err) + return stackTracer != nil && !stackTracer.Empty() } // fundamental is an error that has a message and a stack, but no caller. @@ -131,8 +141,12 @@ type fundamental struct { *stack } +var _ messenger = (*fundamental)(nil) + func (f *fundamental) Error() string { return f.msg } +func (f *fundamental) GetSelfMsg() string { return f.msg } + func (f *fundamental) Format(s fmt.State, verb rune) { switch verb { case 'v': @@ -168,10 +182,14 @@ func WithStack(err error) error { // AddStack is similar to WithStack. // However, it will first check with HasStack to see if a stack trace already exists in the causer chain before creating another one. func AddStack(err error) error { - if HasStack(err) { + if err == nil || HasStack(err) { return err } - return WithStack(err) + + return &withStack{ + err, + callers(), + } } type withStack struct { @@ -179,8 +197,16 @@ type withStack struct { *stack } +var _ messenger = (*withStack)(nil) + func (w *withStack) Cause() error { return w.error } +func (w *withStack) GetSelfMsg() string { + // it doesn't have its own message, but we still need impl it to avoid calling + // err.Error() for its cause + return "" +} + // Unwrap provides compatibility for Go 1.13 error chains. func (w *withStack) Unwrap() error { return w.error } @@ -263,9 +289,13 @@ type withMessage struct { causeHasStack bool } +var _ messenger = (*withMessage)(nil) + func (w *withMessage) Error() string { return w.msg + ": " + w.cause.Error() } func (w *withMessage) Cause() error { return w.cause } +func (w *withMessage) GetSelfMsg() string { return w.msg } + // Unwrap provides compatibility for Go 1.13 error chains. func (w *withMessage) Unwrap() error { return w.cause } func (w *withMessage) HasStack() bool { return w.causeHasStack } @@ -290,9 +320,9 @@ func (w *withMessage) Format(s fmt.State, verb rune) { // An error value has a cause if it implements the following // interface: // -// type causer interface { -// Cause() error -// } +// type causer interface { +// Cause() error +// } // // If the error does not implement Cause, the original error will // be returned. If the error is nil, nil will be returned without further @@ -330,3 +360,33 @@ func Find(origErr error, test func(error) bool) error { }) return foundErr } + +// GetErrStackMsg get the concat error message the whole error stack. +// it's different from err.Error(), as pingcap/errors.Error will prepend the error +// code in the result of err.Error(), like below: +// +// [types:1292]Truncated incorrect +// +// and when there are multiple errors.Error in the chain, the err.Error() will +// return like this: +// +// [Lightning:Restore:ErrEncodeKV]encode kv error ... : [types:1292]Truncated incorrect DOUBLE value: 'a'" +// +// But sometimes we only want a single error code with pure message part. +func GetErrStackMsg(err error) string { + if err == nil { + return "" + } + m, ok := err.(messenger) + if ok { + msg := m.GetSelfMsg() + causeMsg := GetErrStackMsg(Unwrap(err)) + if msg == "" { + msg = causeMsg + } else if causeMsg != "" { + msg = msg + ": " + causeMsg + } + return msg + } + return err.Error() +} diff --git a/vendor/github.com/pingcap/errors/group.go b/vendor/github.com/pingcap/errors/group.go index e5a969ab7..f07083bce 100644 --- a/vendor/github.com/pingcap/errors/group.go +++ b/vendor/github.com/pingcap/errors/group.go @@ -20,13 +20,20 @@ func Errors(err error) []error { // The visitor function can return true to end the traversal early // In that case, WalkDeep will return true, otherwise false. func WalkDeep(err error, visitor func(err error) bool) bool { + if err == nil { + return false + } + + if visitor(err) { + return true + } + // Go deep - unErr := err - for unErr != nil { - if done := visitor(unErr); done { + unErr := Unwrap(err) + if unErr != nil { + if WalkDeep(unErr, visitor) { return true } - unErr = Unwrap(unErr) } // Go wide diff --git a/vendor/github.com/pingcap/errors/join.go b/vendor/github.com/pingcap/errors/join.go new file mode 100644 index 000000000..af587b1ec --- /dev/null +++ b/vendor/github.com/pingcap/errors/join.go @@ -0,0 +1,62 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package errors + +// Join returns an error that wraps the given errors. +// Any nil error values are discarded. +// Join returns nil if every value in errs is nil. +// The error formats as the concatenation of the strings obtained +// by calling the Error method of each element of errs, with a newline +// between each string. +// +// A non-nil error returned by Join implements the Unwrap() []error method. +func Join(errs ...error) error { + n := 0 + for _, err := range errs { + if err != nil { + n++ + } + } + if n == 0 { + return nil + } + e := &joinError{ + errs: make([]error, 0, n), + } + for _, err := range errs { + if err != nil { + e.errs = append(e.errs, err) + } + } + return e +} + +type joinError struct { + errs []error +} + +func (e *joinError) Error() string { + var b []byte + for i, err := range e.errs { + if i > 0 { + b = append(b, '\n') + } + b = append(b, err.Error()...) + } + return string(b) +} + +func (e *joinError) Unwrap() []error { + return e.errs +} diff --git a/vendor/github.com/pingcap/errors/juju_adaptor.go b/vendor/github.com/pingcap/errors/juju_adaptor.go index 5e000aad4..3c2dd4ece 100644 --- a/vendor/github.com/pingcap/errors/juju_adaptor.go +++ b/vendor/github.com/pingcap/errors/juju_adaptor.go @@ -76,6 +76,11 @@ func NewNoStackErrorf(format string, args ...interface{}) error { } // SuspendStack suspends stack generate for error. +// Deprecated, it's semantic is to clear the stack inside, we still allow upper +// layer to add stack again by using Trace. +// Sometimes we have very deep calling stack, the lower layer calls SuspendStack, +// but the upper layer want to add stack to it, if we disable adding stack permanently +// for an error, it's very hard to diagnose certain issues. func SuspendStack(err error) error { if err == nil { return err diff --git a/vendor/github.com/pingcap/errors/normalize.go b/vendor/github.com/pingcap/errors/normalize.go index 34a474554..af67386ca 100644 --- a/vendor/github.com/pingcap/errors/normalize.go +++ b/vendor/github.com/pingcap/errors/normalize.go @@ -42,6 +42,15 @@ type ErrCodeText string type ErrorID string type RFCErrorCode string +// HackedStr can provide a stable string snapshot for unsafe/ephemeral string sources. +// The method name intentionally uses FreezeStr instead of Clone so normal clone-style +// types are not accidentally treated as error-format arguments. +// During error construction, arguments implementing this interface are replaced +// with FreezeStr() so deferred error formatting does not observe later mutations. +type HackedStr interface { + FreezeStr() string +} + // Error is the 'prototype' of a type of errors. // Use DefineError to make a *Error: // var ErrUnavailable = errors.Normalize("Region %d is unavailable", errors.RFCCodeText("Unavailable")) @@ -82,6 +91,8 @@ type Error struct { line int } +var _ messenger = (*Error)(nil) + // Code returns the numeric code of this error. // ID() will return textual error if there it is, // when you just want to get the purely numeric error @@ -127,10 +138,6 @@ func (e *Error) Error() string { if e == nil { return "" } - describe := e.codeText - if len(describe) == 0 { - describe = ErrCodeText(strconv.Itoa(int(e.code))) - } if e.cause != nil { return fmt.Sprintf("[%s]%s: %s", e.RFCCode(), e.GetMsg(), e.cause.Error()) } @@ -144,6 +151,28 @@ func (e *Error) GetMsg() string { return e.message } +func freezeHackedStringArgs(args []interface{}) []interface{} { + // This helper intentionally mutates the input slice in place. + // It is only used inside error-construction paths where args are internal and should + // not be reused by callers after passing into Gen*/FastGen* APIs. + for i := range args { + hackedArg, ok := args[i].(HackedStr) + if !ok { + continue + } + // TiDB may pass unsafe zero-copy strings (for example from chunk buffers) as error args. + // Error message rendering is deferred until GetMsg/Error, so without freezing here we may + // observe later writes and print a different value than the one used when creating the error. + // Freezing in this central path keeps the copy cost only on error construction. + args[i] = hackedArg.FreezeStr() + } + return args +} + +func (e *Error) GetSelfMsg() string { + return e.GetMsg() +} + func (e *Error) fillLineAndFile(skip int) { // skip this _, file, line, ok := runtime.Caller(skip + 1) @@ -161,7 +190,7 @@ func (e *Error) GenWithStack(format string, args ...interface{}) error { // TODO: RedactErrorArg err := *e err.message = format - err.args = args + err.args = freezeHackedStringArgs(args) err.fillLineAndFile(1) return AddStack(&err) } @@ -170,7 +199,7 @@ func (e *Error) GenWithStack(format string, args ...interface{}) error { func (e *Error) GenWithStackByArgs(args ...interface{}) error { RedactErrorArg(args, e.redactArgsPos) err := *e - err.args = args + err.args = freezeHackedStringArgs(args) err.fillLineAndFile(1) return AddStack(&err) } @@ -181,7 +210,7 @@ func (e *Error) FastGen(format string, args ...interface{}) error { // TODO: RedactErrorArg err := *e err.message = format - err.args = args + err.args = freezeHackedStringArgs(args) return SuspendStack(&err) } @@ -190,7 +219,7 @@ func (e *Error) FastGen(format string, args ...interface{}) error { func (e *Error) FastGenByArgs(args ...interface{}) error { RedactErrorArg(args, e.redactArgsPos) err := *e - err.args = args + err.args = freezeHackedStringArgs(args) return SuspendStack(&err) } @@ -311,7 +340,7 @@ func (e *Error) FastGenWithCause(args ...interface{}) error { if e.cause != nil { err.message = e.cause.Error() } - err.args = args + err.args = freezeHackedStringArgs(args) return SuspendStack(&err) } @@ -320,7 +349,7 @@ func (e *Error) GenWithStackByCause(args ...interface{}) error { if e.cause != nil { err.message = e.cause.Error() } - err.args = args + err.args = freezeHackedStringArgs(args) err.fillLineAndFile(1) return AddStack(&err) } diff --git a/vendor/github.com/pingcap/errors/stack.go b/vendor/github.com/pingcap/errors/stack.go index bb1e6a84f..a981f4913 100644 --- a/vendor/github.com/pingcap/errors/stack.go +++ b/vendor/github.com/pingcap/errors/stack.go @@ -14,6 +14,9 @@ import ( // Generally you would want to use the GetStackTracer function to do that. type StackTracer interface { StackTrace() StackTrace + // Empty returns true if the stack trace is empty, StackTrace might clone the + // stack trace, add this method to avoid unnecessary clone. + Empty() bool } // GetStackTracer will return the first StackTracer in the causer chain. @@ -63,16 +66,16 @@ func (f Frame) line() int { // Format formats the frame according to the fmt.Formatter interface. // -// %s source file -// %d source line -// %n function name -// %v equivalent to %s:%d +// %s source file +// %d source line +// %n function name +// %v equivalent to %s:%d // // Format accepts flags that alter the printing of some verbs, as follows: // -// %+s function name and path of source file relative to the compile time -// GOPATH separated by \n\t (\n\t) -// %+v equivalent to %+s:%d +// %+s function name and path of source file relative to the compile time +// GOPATH separated by \n\t (\n\t) +// %+v equivalent to %+s:%d func (f Frame) Format(s fmt.State, verb rune) { f.format(s, s, verb) } @@ -113,12 +116,12 @@ type StackTrace []Frame // Format formats the stack of Frames according to the fmt.Formatter interface. // -// %s lists source files for each Frame in the stack -// %v lists the source file and line number for each Frame in the stack +// %s lists source files for each Frame in the stack +// %v lists the source file and line number for each Frame in the stack // // Format accepts flags that alter the printing of some verbs, as follows: // -// %+v Prints filename, function, and line number for each Frame in the stack. +// %+v Prints filename, function, and line number for each Frame in the stack. func (st StackTrace) Format(s fmt.State, verb rune) { var b bytes.Buffer switch verb { @@ -192,6 +195,10 @@ func (s *stack) StackTrace() StackTrace { return f } +func (s *stack) Empty() bool { + return len(*s) == 0 +} + func callers() *stack { return callersSkip(4) } diff --git a/vendor/github.com/pingcap/log/config.go b/vendor/github.com/pingcap/log/config.go index 2d581d165..c80229534 100644 --- a/vendor/github.com/pingcap/log/config.go +++ b/vendor/github.com/pingcap/log/config.go @@ -35,6 +35,15 @@ type FileLogConfig struct { MaxDays int `toml:"max-days" json:"max-days"` // Maximum number of old log files to retain. MaxBackups int `toml:"max-backups" json:"max-backups"` + // Compression function for rotated files. + // Currently only `gzip` and empty are supported, empty means compression disabled. + Compression string `toml:"compression" json:"compression"` + // IsBuffered is true means use buffered logger. + IsBuffered bool `toml:"is-buffered" json:"is-buffered"` + // BufferSize is the size of the buffer. + BufferSize int `toml:"buffer-size" json:"buffer-size"` + // BufferFlushInterval is the interval of buffer flush. + BufferFlushInterval time.Duration `toml:"buffer-flush-interval" json:"buffer-flush-interval"` } // Config serializes log related config in toml/json. @@ -77,9 +86,10 @@ type Config struct { // ZapProperties records some information about zap. type ZapProperties struct { - Core zapcore.Core - Syncer zapcore.WriteSyncer - Level zap.AtomicLevel + Core zapcore.Core + Syncer zapcore.WriteSyncer + ErrSyncer zapcore.WriteSyncer + Level zap.AtomicLevel } func (cfg *Config) buildOptions(errSink zapcore.WriteSyncer) []zap.Option { @@ -103,7 +113,7 @@ func (cfg *Config) buildOptions(errSink zapcore.WriteSyncer) []zap.Option { if cfg.Sampling != nil { opts = append(opts, zap.WrapCore(func(core zapcore.Core) zapcore.Core { - return zapcore.NewSampler(core, time.Second, int(cfg.Sampling.Initial), int(cfg.Sampling.Thereafter)) + return zapcore.NewSamplerWithOptions(core, time.Second, int(cfg.Sampling.Initial), int(cfg.Sampling.Thereafter)) })) } return opts diff --git a/vendor/github.com/pingcap/log/log.go b/vendor/github.com/pingcap/log/log.go index 01edeb80d..00db327b7 100644 --- a/vendor/github.com/pingcap/log/log.go +++ b/vendor/github.com/pingcap/log/log.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "sync" "sync/atomic" "time" @@ -48,7 +49,15 @@ func InitLogger(cfg *Config, opts ...zap.Option) (*zap.Logger, *ZapProperties, e if err != nil { return nil, nil, err } - output = zapcore.AddSync(lg) + if cfg.File.IsBuffered { + output = &zapcore.BufferedWriteSyncer{ + WS: zapcore.AddSync(lg), + Size: cfg.File.BufferSize, + FlushInterval: cfg.File.BufferFlushInterval, + } + } else { + output = zapcore.AddSync(lg) + } } else { stdOut, _, err := zap.Open([]string{"stdout"}...) if err != nil { @@ -108,9 +117,10 @@ func InitLoggerWithWriteSyncer(cfg *Config, output, errOutput zapcore.WriteSynce opts = append(cfg.buildOptions(errOutput), opts...) lg := zap.New(core, opts...) r := &ZapProperties{ - Core: core, - Syncer: output, - Level: level, + Core: core, + Syncer: output, + ErrSyncer: errOutput, + Level: level, } return lg, r, nil } @@ -173,15 +183,58 @@ func (s *lockWithTimeoutWrapper) Sync() error { // initFileLog initializes file based logging options. func initFileLog(cfg *FileLogConfig) (*lumberjack.Logger, error) { + // Verify file permissions early to prevent silent failures. + // Since lumberjack creates log files lazily, permission issues wouldn't be detected + // until the first write attempt. This is problematic when tools redirect their + // error output to these log files. + // e.g. dumpling use the log file as ErrorOutputPath, but the log file will not be created + // if there has permission issues, and dumpling will not print any error message. + + // Create the directory if it doesn't exist + dir := filepath.Dir(cfg.Filename) + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, fmt.Errorf("cannot create log directory: %w", err) + } + + // Check if the path is a directory which is invalid if st, err := os.Stat(cfg.Filename); err == nil { if st.IsDir() { return nil, errors.New("can't use directory as log file name") } + + // Check if the file is writable + file, err := os.OpenFile(cfg.Filename, os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + return nil, fmt.Errorf("can't write to log file: %w", err) + } + file.Close() + } else if os.IsNotExist(err) { + // File doesn't exist, verify we can create it + file, err := os.Create(cfg.Filename) + if err != nil { + return nil, fmt.Errorf("can't create log file: %w", err) + } + file.Close() + // Remove the empty file since lumberjack will create it + os.Remove(cfg.Filename) + } else { + return nil, fmt.Errorf("error checking log file: %w", err) } + if cfg.MaxSize == 0 { cfg.MaxSize = defaultLogMaxSize } + compress := false + switch cfg.Compression { + case "": + compress = false + case "gzip": + compress = true + default: + return nil, fmt.Errorf("can't set compression to `%s`", cfg.Compression) + } + // use lumberjack to logrotate return &lumberjack.Logger{ Filename: cfg.Filename, @@ -189,6 +242,7 @@ func initFileLog(cfg *FileLogConfig) (*lumberjack.Logger, error) { MaxBackups: cfg.MaxBackups, MaxAge: cfg.MaxDays, LocalTime: true, + Compress: compress, }, nil } @@ -212,6 +266,8 @@ func S() *zap.SugaredLogger { // ReplaceGlobals replaces the global Logger and SugaredLogger, and returns a // function to restore the original values. It's safe for concurrent use. +// Be careful when using this with buffered logger, the flush goroutine in +// https://pkg.go.dev/go.uber.org/zap/zapcore#BufferedWriteSyncer will not be stopped. func ReplaceGlobals(logger *zap.Logger, props *ZapProperties) func() { // TODO: This globalMu can be replaced by atomic.Swap(), available since go1.17. globalMu.Lock() diff --git a/vendor/github.com/pingcap/log/zap_text_encoder.go b/vendor/github.com/pingcap/log/zap_text_encoder.go index 4c1ef02f9..c6385ec69 100644 --- a/vendor/github.com/pingcap/log/zap_text_encoder.go +++ b/vendor/github.com/pingcap/log/zap_text_encoder.go @@ -96,7 +96,7 @@ func getCallerString(ec zapcore.EntryCaller) string { // For JSON-escaping; see textEncoder.safeAddString below. const _hex = "0123456789abcdef" -var _textPool = sync.Pool{New: func() interface{} { +var _textPool = sync.Pool{New: func() any { return &textEncoder{} }} @@ -223,7 +223,7 @@ func (enc *textEncoder) resetReflectBuf() { } } -func (enc *textEncoder) AddReflected(key string, obj interface{}) error { +func (enc *textEncoder) AddReflected(key string, obj any) error { enc.resetReflectBuf() err := enc.reflectEnc.Encode(obj) if err != nil { @@ -321,7 +321,7 @@ func (enc *textEncoder) AppendInt64(val int64) { enc.buf.AppendInt(val) } -func (enc *textEncoder) AppendReflected(val interface{}) error { +func (enc *textEncoder) AppendReflected(val any) error { enc.resetReflectBuf() err := enc.reflectEnc.Encode(val) if err != nil { diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/charset/BUILD.bazel b/vendor/github.com/pingcap/tidb/pkg/parser/charset/BUILD.bazel index 56718a764..9f8c8b619 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/charset/BUILD.bazel +++ b/vendor/github.com/pingcap/tidb/pkg/parser/charset/BUILD.bazel @@ -8,6 +8,8 @@ go_library( "encoding_ascii.go", "encoding_base.go", "encoding_bin.go", + "encoding_gb18030.go", + "encoding_gb18030_data.go", "encoding_gbk.go", "encoding_latin1.go", "encoding_table.go", @@ -41,7 +43,7 @@ go_test( ], embed = [":charset"], flaky = True, - shard_count = 8, + shard_count = 9, deps = [ "@com_github_stretchr_testify//require", "@org_golang_x_text//transform", diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/charset/charset.go b/vendor/github.com/pingcap/tidb/pkg/parser/charset/charset.go index 69adcae00..6a133c4ad 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/charset/charset.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/charset/charset.go @@ -71,16 +71,18 @@ var CharacterSetInfos = map[string]*Charset{ CharsetLatin1: {CharsetLatin1, CollationLatin1, make(map[string]*Collation), "Latin1", 1}, CharsetBin: {CharsetBin, CollationBin, make(map[string]*Collation), "binary", 1}, CharsetGBK: {CharsetGBK, CollationGBKBin, make(map[string]*Collation), "Chinese Internal Code Specification", 2}, + CharsetGB18030: {CharsetGB18030, CollationGB18030Bin, make(map[string]*Collation), "China National Standard GB18030", 4}, } // All the names supported collations should be in the following table. var supportedCollationNames = map[string]struct{}{ - CollationUTF8: {}, - CollationUTF8MB4: {}, - CollationASCII: {}, - CollationLatin1: {}, - CollationBin: {}, - CollationGBKBin: {}, + CollationUTF8: {}, + CollationUTF8MB4: {}, + CollationASCII: {}, + CollationLatin1: {}, + CollationBin: {}, + CollationGBKBin: {}, + CollationGB18030Bin: {}, } // TiFlashSupportedCharsets is a map which contains TiFlash supports charsets. @@ -238,6 +240,10 @@ const ( CollationGBKBin = "gbk_bin" // CollationGBKChineseCI is the default collation for CharsetGBK when new collation is enabled. CollationGBKChineseCI = "gbk_chinese_ci" + // CollationGB18030Bin is the default collation for CharsetGB18030 when new collation is disabled. + CollationGB18030Bin = "gb18030_bin" + // CollationGB18030ChineseCI is the default collation for CharsetGB18030 when new collation is enabled. + CollationGB18030ChineseCI = "gb18030_chinese_ci" ) const ( @@ -253,6 +259,8 @@ const ( CharsetUTF8MB3 = "utf8mb3" // CharsetUTF8MB4 represents 4 bytes utf8, which works the same way as utf8 in Go. CharsetUTF8MB4 = "utf8mb4" + // CharsetGB18030 represents 4 bytes gb18030. + CharsetGB18030 = "gb18030" //revive:disable:exported CharsetARMSCII8 = "armscii8" CharsetBig5 = "big5" @@ -267,7 +275,6 @@ const ( CharsetDEC8 = "dec8" CharsetEUCJPMS = "eucjpms" CharsetEUCKR = "euckr" - CharsetGB18030 = "gb18030" CharsetGB2312 = "gb2312" CharsetGBK = "gbk" CharsetGEOSTD8 = "geostd8" @@ -298,28 +305,28 @@ var charsets = map[string]*Charset{ CharsetASCII: {Name: CharsetASCII, Maxlen: 1, DefaultCollation: "ascii_general_ci", Desc: "US ASCII", Collations: make(map[string]*Collation)}, CharsetBig5: {Name: CharsetBig5, Maxlen: 2, DefaultCollation: "big5_chinese_ci", Desc: "Big5 Traditional Chinese", Collations: make(map[string]*Collation)}, CharsetBin: {Name: CharsetBin, Maxlen: 1, DefaultCollation: "binary", Desc: "Binary pseudo charset", Collations: make(map[string]*Collation)}, - CharsetLatin1: {Name: CharsetLatin1, Maxlen: 1, DefaultCollation: "cp1250_general_ci", Desc: "Windows Central European", Collations: make(map[string]*Collation)}, - CharsetCP1250: {Name: CharsetCP1250, Maxlen: 1, DefaultCollation: "cp1251_general_ci", Desc: "Windows Cyrillic", Collations: make(map[string]*Collation)}, - CharsetCP1251: {Name: CharsetCP1251, Maxlen: 1, DefaultCollation: "cp1256_general_ci", Desc: "Windows Arabic", Collations: make(map[string]*Collation)}, - CharsetCP1256: {Name: CharsetCP1256, Maxlen: 1, DefaultCollation: "cp1257_general_ci", Desc: "Windows Baltic", Collations: make(map[string]*Collation)}, - CharsetCP1257: {Name: CharsetCP1257, Maxlen: 1, DefaultCollation: "cp850_general_ci", Desc: "DOS West European", Collations: make(map[string]*Collation)}, - CharsetCP850: {Name: CharsetCP850, Maxlen: 1, DefaultCollation: "cp852_general_ci", Desc: "DOS Central European", Collations: make(map[string]*Collation)}, - CharsetCP852: {Name: CharsetCP852, Maxlen: 1, DefaultCollation: "cp866_general_ci", Desc: "DOS Russian", Collations: make(map[string]*Collation)}, - CharsetCP866: {Name: CharsetCP866, Maxlen: 1, DefaultCollation: "cp932_japanese_ci", Desc: "SJIS for Windows Japanese", Collations: make(map[string]*Collation)}, - CharsetCP932: {Name: CharsetCP932, Maxlen: 2, DefaultCollation: "dec8_swedish_ci", Desc: "DEC West European", Collations: make(map[string]*Collation)}, - CharsetDEC8: {Name: CharsetDEC8, Maxlen: 1, DefaultCollation: "eucjpms_japanese_ci", Desc: "UJIS for Windows Japanese", Collations: make(map[string]*Collation)}, - CharsetEUCJPMS: {Name: CharsetEUCJPMS, Maxlen: 3, DefaultCollation: "euckr_korean_ci", Desc: "EUC-KR Korean", Collations: make(map[string]*Collation)}, - CharsetEUCKR: {Name: CharsetEUCKR, Maxlen: 2, DefaultCollation: "gb18030_chinese_ci", Desc: "China National Standard GB18030", Collations: make(map[string]*Collation)}, - CharsetGB18030: {Name: CharsetGB18030, Maxlen: 4, DefaultCollation: "gb2312_chinese_ci", Desc: "GB2312 Simplified Chinese", Collations: make(map[string]*Collation)}, - CharsetGB2312: {Name: CharsetGB2312, Maxlen: 2, DefaultCollation: "gbk_chinese_ci", Desc: "GBK Simplified Chinese", Collations: make(map[string]*Collation)}, - CharsetGBK: {Name: CharsetGBK, Maxlen: 2, DefaultCollation: "geostd8_general_ci", Desc: "GEOSTD8 Georgian", Collations: make(map[string]*Collation)}, - CharsetGEOSTD8: {Name: CharsetGEOSTD8, Maxlen: 1, DefaultCollation: "greek_general_ci", Desc: "ISO 8859-7 Greek", Collations: make(map[string]*Collation)}, - CharsetGreek: {Name: CharsetGreek, Maxlen: 1, DefaultCollation: "hebrew_general_ci", Desc: "ISO 8859-8 Hebrew", Collations: make(map[string]*Collation)}, - CharsetHebrew: {Name: CharsetHebrew, Maxlen: 1, DefaultCollation: "hp8_english_ci", Desc: "HP West European", Collations: make(map[string]*Collation)}, - CharsetHP8: {Name: CharsetHP8, Maxlen: 1, DefaultCollation: "keybcs2_general_ci", Desc: "DOS Kamenicky Czech-Slovak", Collations: make(map[string]*Collation)}, - CharsetKEYBCS2: {Name: CharsetKEYBCS2, Maxlen: 1, DefaultCollation: "koi8r_general_ci", Desc: "KOI8-R Relcom Russian", Collations: make(map[string]*Collation)}, + CharsetCP1250: {Name: CharsetCP1250, Maxlen: 1, DefaultCollation: "cp1250_general_ci", Desc: "Windows Central European", Collations: make(map[string]*Collation)}, + CharsetCP1251: {Name: CharsetCP1251, Maxlen: 1, DefaultCollation: "cp1251_general_ci", Desc: "Windows Cyrillic", Collations: make(map[string]*Collation)}, + CharsetCP1256: {Name: CharsetCP1256, Maxlen: 1, DefaultCollation: "cp1256_general_ci", Desc: "Windows Arabic", Collations: make(map[string]*Collation)}, + CharsetCP1257: {Name: CharsetCP1257, Maxlen: 1, DefaultCollation: "cp1257_general_ci", Desc: "Windows Baltic", Collations: make(map[string]*Collation)}, + CharsetCP850: {Name: CharsetCP850, Maxlen: 1, DefaultCollation: "cp850_general_ci", Desc: "DOS West European", Collations: make(map[string]*Collation)}, + CharsetCP852: {Name: CharsetCP852, Maxlen: 1, DefaultCollation: "cp852_general_ci", Desc: "DOS Central European", Collations: make(map[string]*Collation)}, + CharsetCP866: {Name: CharsetCP866, Maxlen: 1, DefaultCollation: "cp866_general_ci", Desc: "DOS Russian", Collations: make(map[string]*Collation)}, + CharsetCP932: {Name: CharsetCP932, Maxlen: 2, DefaultCollation: "cp932_japanese_ci", Desc: "SJIS for Windows Japanese", Collations: make(map[string]*Collation)}, + CharsetDEC8: {Name: CharsetDEC8, Maxlen: 1, DefaultCollation: "dec8_swedish_ci", Desc: "DEC West European", Collations: make(map[string]*Collation)}, + CharsetEUCJPMS: {Name: CharsetEUCJPMS, Maxlen: 3, DefaultCollation: "eucjpms_japanese_ci", Desc: "UJIS for Windows Japanese", Collations: make(map[string]*Collation)}, + CharsetEUCKR: {Name: CharsetEUCKR, Maxlen: 2, DefaultCollation: "euckr_korean_ci", Desc: "EUC-KR Korean", Collations: make(map[string]*Collation)}, + CharsetGB18030: {Name: CharsetGB18030, Maxlen: 4, DefaultCollation: "gb18030_chinese_ci", Desc: "China National Standard GB18030", Collations: make(map[string]*Collation)}, + CharsetGB2312: {Name: CharsetGB2312, Maxlen: 2, DefaultCollation: "gb2312_chinese_ci", Desc: "GB2312 Simplified Chinese", Collations: make(map[string]*Collation)}, + CharsetGBK: {Name: CharsetGBK, Maxlen: 2, DefaultCollation: "gbk_chinese_ci", Desc: "GBK Simplified Chinese", Collations: make(map[string]*Collation)}, + CharsetGEOSTD8: {Name: CharsetGEOSTD8, Maxlen: 1, DefaultCollation: "geostd8_general_ci", Desc: "GEOSTD8 Georgian", Collations: make(map[string]*Collation)}, + CharsetGreek: {Name: CharsetGreek, Maxlen: 1, DefaultCollation: "greek_general_ci", Desc: "ISO 8859-7 Greek", Collations: make(map[string]*Collation)}, + CharsetHebrew: {Name: CharsetHebrew, Maxlen: 1, DefaultCollation: "hebrew_general_ci", Desc: "ISO 8859-8 Hebrew", Collations: make(map[string]*Collation)}, + CharsetHP8: {Name: CharsetHP8, Maxlen: 1, DefaultCollation: "hp8_english_ci", Desc: "HP West European", Collations: make(map[string]*Collation)}, + CharsetKEYBCS2: {Name: CharsetKEYBCS2, Maxlen: 1, DefaultCollation: "keybcs2_general_ci", Desc: "DOS Kamenicky Czech-Slovak", Collations: make(map[string]*Collation)}, CharsetKOI8R: {Name: CharsetKOI8R, Maxlen: 1, DefaultCollation: "koi8u_general_ci", Desc: "KOI8-U Ukrainian", Collations: make(map[string]*Collation)}, - CharsetKOI8U: {Name: CharsetKOI8U, Maxlen: 1, DefaultCollation: "latin1_swedish_ci", Desc: "cp1252 West European", Collations: make(map[string]*Collation)}, + CharsetKOI8U: {Name: CharsetKOI8U, Maxlen: 1, DefaultCollation: "koi8r_general_ci", Desc: "KOI8-R Relcom Russian", Collations: make(map[string]*Collation)}, + CharsetLatin1: {Name: CharsetLatin1, Maxlen: 1, DefaultCollation: "latin1_swedish_ci", Desc: "cp1252 West European", Collations: make(map[string]*Collation)}, CharsetLatin2: {Name: CharsetLatin2, Maxlen: 1, DefaultCollation: "latin2_general_ci", Desc: "ISO 8859-2 Central European", Collations: make(map[string]*Collation)}, CharsetLatin5: {Name: CharsetLatin5, Maxlen: 1, DefaultCollation: "latin5_turkish_ci", Desc: "ISO 8859-9 Turkish", Collations: make(map[string]*Collation)}, CharsetLatin7: {Name: CharsetLatin7, Maxlen: 1, DefaultCollation: "latin7_general_ci", Desc: "ISO 8859-13 Baltic", Collations: make(map[string]*Collation)}, @@ -625,7 +632,7 @@ func RemoveCharset(c string) { delete(CharacterSetInfos, c) for i := range supportedCollations { if supportedCollations[i].Name == c { - supportedCollations = append(supportedCollations[:i], supportedCollations[i+1:]...) + supportedCollations = slices.Delete(supportedCollations, i, i+1) } } } diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding.go b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding.go index 121a72ebf..5451d74e0 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding.go @@ -23,6 +23,7 @@ var ( _ Encoding = &encodingLatin1{} _ Encoding = &encodingBin{} _ Encoding = &encodingGBK{} + _ Encoding = &encodingGB18030{} ) // IsSupportedEncoding checks if the charset is fully supported. @@ -60,6 +61,7 @@ var encodingMap = map[string]Encoding{ CharsetLatin1: EncodingLatin1Impl, CharsetBin: EncodingBinImpl, CharsetASCII: EncodingASCIIImpl, + CharsetGB18030: EncodingGB18030Impl, } // Encoding provide encode/decode functions for a string with a specific charset. @@ -100,6 +102,7 @@ const ( EncodingTpLatin1 EncodingTpBin EncodingTpGBK + EncodingTpGB18030 ) //revive:enable diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_ascii.go b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_ascii.go index db34f8952..c0554c924 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_ascii.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_ascii.go @@ -53,7 +53,7 @@ func (*encodingASCII) Peek(src []byte) []byte { // IsValid implements Encoding interface. func (*encodingASCII) IsValid(src []byte) bool { srcLen := len(src) - for i := 0; i < srcLen; i++ { + for i := range srcLen { if src[i] > go_unicode.MaxASCII { return false } diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_base.go b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_base.go index ec4f1b260..3ba7b2d46 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_base.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_base.go @@ -16,7 +16,6 @@ package charset import ( "bytes" "fmt" - "reflect" "strings" "unsafe" @@ -86,20 +85,28 @@ func (b encodingBase) Transform(dest *bytes.Buffer, src []byte, op Op) (result [ } func (b encodingBase) Foreach(src []byte, op Op, fn func(from, to []byte, ok bool) bool) { - var tfm transform.Transformer + var ( + tfm transform.Transformer + runeErrorChecker runeErrorMaybeInputTransformer + ok bool + ) var peek func([]byte) []byte if op&opFromUTF8 != 0 { tfm = b.enc.NewEncoder() peek = EncodingUTF8Impl.Peek } else { - tfm = b.enc.NewDecoder() + dec := b.enc.NewDecoder() + tfm = dec + runeErrorChecker, ok = dec.Transformer.(runeErrorMaybeInputTransformer) peek = b.self.Peek } var buf [4]byte for i, w := 0, 0; i < len(src); i += w { w = len(peek(src[i:])) nDst, _, err := tfm.Transform(buf[:], src[i:i+w], false) - meetErr := err != nil || (op&opToUTF8 != 0 && beginWithReplacementChar(buf[:nDst])) + meetErr := err != nil || (op&opToUTF8 != 0 && + beginWithReplacementChar(buf[:nDst]) && + (!ok || !runeErrorChecker.runeErrorIsLastInput())) if !fn(src[i:i+w], buf[:nDst], !meetErr) { return } @@ -123,12 +130,10 @@ func generateEncodingErr(name string, invalidBytes []byte) error { // HackSlice converts string to slice without copy. // Use at your own risk. func HackSlice(s string) (b []byte) { - pBytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - pString := (*reflect.StringHeader)(unsafe.Pointer(&s)) - pBytes.Data = pString.Data - pBytes.Len = pString.Len - pBytes.Cap = pString.Len - return + if len(s) == 0 { + return []byte{} + } + return unsafe.Slice(unsafe.StringData(s), len(s)) } // HackString converts slice to string without copy. @@ -137,9 +142,5 @@ func HackString(b []byte) (s string) { if len(b) == 0 { return "" } - pbytes := (*reflect.SliceHeader)(unsafe.Pointer(&b)) - pstring := (*reflect.StringHeader)(unsafe.Pointer(&s)) - pstring.Data = pbytes.Data - pstring.Len = pbytes.Len - return + return unsafe.String(unsafe.SliceData(b), len(b)) } diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_bin.go b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_bin.go index 8b9043154..c10823d8d 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_bin.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_bin.go @@ -56,7 +56,7 @@ func (*encodingBin) IsValid(_ []byte) bool { // Foreach implements Encoding interface. func (*encodingBin) Foreach(src []byte, _ Op, fn func(from, to []byte, ok bool) bool) { - for i := 0; i < len(src); i++ { + for i := range src { if !fn(src[i:i+1], src[i:i+1], true) { return } diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_gb18030.go b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_gb18030.go new file mode 100644 index 000000000..ca1b69467 --- /dev/null +++ b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_gb18030.go @@ -0,0 +1,277 @@ +// Copyright 2023-2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package charset + +import ( + "encoding/binary" + "strings" + "unicode/utf8" + + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/transform" +) + +// Current implementation meets the requirement of GB18030-2022 + +// EncodingGB18030Impl is the instance of encodingGB18030 +var EncodingGB18030Impl = &encodingGB18030{encodingGBK{encodingBase{enc: customGB18030{}}}} + +func init() { + EncodingGB18030Impl.self = EncodingGB18030Impl +} + +// encodingGB18030 is GB18030 encoding. +type encodingGB18030 struct { + encodingGBK +} + +// Name implements Encoding interface. +func (*encodingGB18030) Name() string { + return CharsetGB18030 +} + +// Tp implements Encoding interface. +func (*encodingGB18030) Tp() EncodingTp { + return EncodingTpGB18030 +} + +// Peek implements Encoding interface. +func (*encodingGB18030) Peek(src []byte) []byte { + return peek(src) +} + +func peek(src []byte) []byte { + length := len(src) + if length == 0 { + return src + } + + // skip the first byte when encountering invalid byte(s) + err := src[:1] + switch { + case src[0] == 0x80 || src[0] == 0xFF: + return err + case src[0] <= 0x7F: + return src[:1] + case 0x81 <= src[0] && src[0] <= 0xFE: + if length < 2 { + return err + } + if 0x40 <= src[1] && src[1] < 0x7F || 0x7F < src[1] && src[1] <= 0xFE { + return src[:2] + } + if length < 4 { + return err + } + if 0x30 <= src[1] && src[1] <= 0x39 && 0x81 <= src[2] && src[2] <= 0xfe && 0x30 <= src[3] && src[3] <= 0x39 { + return src[:4] + } + return err + } + return err +} + +func (*encodingGB18030) MbLen(bs string) int { + if len(bs) < 2 { + return 0 + } + + if 0x81 <= bs[0] && bs[0] <= 0xfe { + if (0x40 <= bs[1] && bs[1] <= 0x7e) || (0x80 <= bs[1] && bs[1] <= 0xfe) { + return 2 + } + if 0x30 <= bs[1] && bs[1] <= 0x39 && 0x81 <= bs[2] && bs[2] <= 0xfe && 0x30 <= bs[3] && bs[3] <= 0x39 { + return 4 + } + } + + return 0 +} + +// ToUpper implements Encoding interface. +func (*encodingGB18030) ToUpper(d string) string { + return strings.ToUpperSpecial(GB18030Case, d) +} + +// ToLower implements Encoding interface. +func (*encodingGB18030) ToLower(d string) string { + return strings.ToLowerSpecial(GB18030Case, d) +} + +// customGB18030 is a simplifiedchinese.GB18030 wrapper. +type customGB18030 struct{} + +// NewCustomGB18030Encoder return a custom GB18030 encoder. +func NewCustomGB18030Encoder() *encoding.Encoder { + return customGB18030{}.NewEncoder() +} + +// NewCustomGB18030Decoder return a custom GB18030 decoder. +func NewCustomGB18030Decoder() *encoding.Decoder { + return customGB18030{}.NewDecoder() +} + +type runeErrorMaybeInputTransformer interface { + runeErrorIsLastInput() bool +} + +// NewDecoder returns simplifiedchinese.GB18030.NewDecoder(). +func (customGB18030) NewDecoder() *encoding.Decoder { + return &encoding.Decoder{ + Transformer: &customGB18030Decoder{ + gb18030Decoder: simplifiedchinese.GB18030.NewDecoder(), + }, + } +} + +// NewEncoder returns simplifiedchinese.gb18030. +func (customGB18030) NewEncoder() *encoding.Encoder { + return &encoding.Encoder{ + Transformer: customGB18030Encoder{ + gb18030Encoder: simplifiedchinese.GB18030.NewEncoder(), + }, + } +} + +type customGB18030Decoder struct { + gb18030Decoder *encoding.Decoder + runeErrorIsLastInputFlag bool +} + +func (c *customGB18030Decoder) runeErrorIsLastInput() bool { + return c.runeErrorIsLastInputFlag +} + +var runeErrorEncodedByGB18030 = convertBytesToUint32([]byte{0x84, 0x31, 0xA4, 0x37}) + +// Transform special treatment for 0x80, +func (c *customGB18030Decoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + c.runeErrorIsLastInputFlag = false + + if len(src) == 0 { + return 0, 0, nil + } + for next := 0; nSrc < len(src); nSrc += next { + if nDst >= len(dst) { + return nDst, nSrc, transform.ErrShortDst + } + next = len(peek(src[nSrc:])) + if nSrc+next > len(src) { + return nDst, nSrc, transform.ErrShortSrc + } + + if src[nSrc] == 0x80 { + nDst += utf8.EncodeRune(dst[nDst:], utf8.RuneError) + continue + } + u32 := convertBytesToUint32(src[nSrc : nSrc+next]) + if r, ok := gb18030ToUnicode[u32]; ok { + nDst += utf8.EncodeRune(dst[nDst:], r) + continue + } + if u32 == runeErrorEncodedByGB18030 { + nDst += utf8.EncodeRune(dst[nDst:], utf8.RuneError) + c.runeErrorIsLastInputFlag = true + continue + } + + d, _, e := c.gb18030Decoder.Transform(dst[nDst:], src[nSrc:nSrc+next], atEOF) + if e != nil { + return nDst, nSrc, e + } + nDst += d + } + return +} + +// Reset is same as simplifiedchinese.GB18030.Reset(). +func (c *customGB18030Decoder) Reset() { + c.gb18030Decoder.Reset() + c.runeErrorIsLastInputFlag = false +} + +type customGB18030Encoder struct { + gb18030Encoder *encoding.Encoder +} + +// Transform special treatment for `€`, +func (c customGB18030Encoder) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + if len(src) == 0 { + return 0, 0, nil + } + for nSrc < len(src) { + if nDst >= len(dst) { + return nDst, nSrc, transform.ErrShortDst + } + r, size := utf8.DecodeRune(src[nSrc:]) + if v, ok := unicodeToGB18030[r]; ok { + bytes := convertUint32ToBytes(v) + if nDst+len(bytes) > len(dst) { + return nDst, nSrc, transform.ErrShortDst + } + copy(dst[nDst:], bytes) + nDst += len(bytes) + nSrc += size + } else { + d, s, e := c.gb18030Encoder.Transform(dst[nDst:], src[nSrc:nSrc+size], atEOF) + if e != nil { + return nDst, nSrc, e + } + nDst += d + nSrc += s + } + } + return +} + +// Reset is same as simplifiedchinese.gb18030. +func (c customGB18030Encoder) Reset() { + c.gb18030Encoder.Reset() +} + +func convertBytesToUint32(b []byte) uint32 { + switch len(b) { + case 4: + return (uint32(b[0]) << 24) | (uint32(b[1]) << 16) | (uint32(b[2]) << 8) | uint32(b[3]) + case 3: + return (uint32(b[0]) << 16) | (uint32(b[1]) << 8) | uint32(b[2]) + case 2: + return (uint32(b[0]) << 8) | uint32(b[1]) + case 1: + return uint32(b[0]) + } + return 0 +} + +func convertUint32ToBytes(v uint32) []byte { + var b []byte + switch { + case v&0xff000000 > 0: + b = make([]byte, 4) + binary.BigEndian.PutUint32(b, v) + case v&0xff0000 > 0: + b = make([]byte, 3) + b[0] = byte(v >> 16) + b[1] = byte(v >> 8) + b[2] = byte(v) + case v&0xff00 > 0: + b = make([]byte, 2) + binary.BigEndian.PutUint16(b, uint16(v)) + default: + b = make([]byte, 1) + b[0] = byte(v) + } + return b +} diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_gb18030_data.go b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_gb18030_data.go new file mode 100644 index 000000000..0fe92ef3b --- /dev/null +++ b/vendor/github.com/pingcap/tidb/pkg/parser/charset/encoding_gb18030_data.go @@ -0,0 +1,792 @@ +// Copyright 2023-2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package charset + +import "unicode" + +func init() { + gb18030EncodingList := []struct { + unicode rune + gb18030 uint32 + }{ + {'\u1e3f', 0xA8BC}, {'\u20ac', 0xA2E3}, {'\u9fb4', 0xFE59}, + {'\u9fb5', 0xFE61}, {'\u9fb6', 0xFE66}, {'\u9fb7', 0xFE67}, + {'\u9fb8', 0xFE6D}, {'\u9fb9', 0xFE7E}, {'\u9fba', 0xFE90}, + {'\u9fbb', 0xFEA0}, {'\ue000', 0xAAA1}, {'\ue001', 0xAAA2}, + {'\ue002', 0xAAA3}, {'\ue003', 0xAAA4}, {'\ue004', 0xAAA5}, + {'\ue005', 0xAAA6}, {'\ue006', 0xAAA7}, {'\ue007', 0xAAA8}, + {'\ue008', 0xAAA9}, {'\ue009', 0xAAAA}, {'\ue00a', 0xAAAB}, + {'\ue00b', 0xAAAC}, {'\ue00c', 0xAAAD}, {'\ue00d', 0xAAAE}, + {'\ue00e', 0xAAAF}, {'\ue00f', 0xAAB0}, {'\ue010', 0xAAB1}, + {'\ue011', 0xAAB2}, {'\ue012', 0xAAB3}, {'\ue013', 0xAAB4}, + {'\ue014', 0xAAB5}, {'\ue015', 0xAAB6}, {'\ue016', 0xAAB7}, + {'\ue017', 0xAAB8}, {'\ue018', 0xAAB9}, {'\ue019', 0xAABA}, + {'\ue01a', 0xAABB}, {'\ue01b', 0xAABC}, {'\ue01c', 0xAABD}, + {'\ue01d', 0xAABE}, {'\ue01e', 0xAABF}, {'\ue01f', 0xAAC0}, + {'\ue020', 0xAAC1}, {'\ue021', 0xAAC2}, {'\ue022', 0xAAC3}, + {'\ue023', 0xAAC4}, {'\ue024', 0xAAC5}, {'\ue025', 0xAAC6}, + {'\ue026', 0xAAC7}, {'\ue027', 0xAAC8}, {'\ue028', 0xAAC9}, + {'\ue029', 0xAACA}, {'\ue02a', 0xAACB}, {'\ue02b', 0xAACC}, + {'\ue02c', 0xAACD}, {'\ue02d', 0xAACE}, {'\ue02e', 0xAACF}, + {'\ue02f', 0xAAD0}, {'\ue030', 0xAAD1}, {'\ue031', 0xAAD2}, + {'\ue032', 0xAAD3}, {'\ue033', 0xAAD4}, {'\ue034', 0xAAD5}, + {'\ue035', 0xAAD6}, {'\ue036', 0xAAD7}, {'\ue037', 0xAAD8}, + {'\ue038', 0xAAD9}, {'\ue039', 0xAADA}, {'\ue03a', 0xAADB}, + {'\ue03b', 0xAADC}, {'\ue03c', 0xAADD}, {'\ue03d', 0xAADE}, + {'\ue03e', 0xAADF}, {'\ue03f', 0xAAE0}, {'\ue040', 0xAAE1}, + {'\ue041', 0xAAE2}, {'\ue042', 0xAAE3}, {'\ue043', 0xAAE4}, + {'\ue044', 0xAAE5}, {'\ue045', 0xAAE6}, {'\ue046', 0xAAE7}, + {'\ue047', 0xAAE8}, {'\ue048', 0xAAE9}, {'\ue049', 0xAAEA}, + {'\ue04a', 0xAAEB}, {'\ue04b', 0xAAEC}, {'\ue04c', 0xAAED}, + {'\ue04d', 0xAAEE}, {'\ue04e', 0xAAEF}, {'\ue04f', 0xAAF0}, + {'\ue050', 0xAAF1}, {'\ue051', 0xAAF2}, {'\ue052', 0xAAF3}, + {'\ue053', 0xAAF4}, {'\ue054', 0xAAF5}, {'\ue055', 0xAAF6}, + {'\ue056', 0xAAF7}, {'\ue057', 0xAAF8}, {'\ue058', 0xAAF9}, + {'\ue059', 0xAAFA}, {'\ue05a', 0xAAFB}, {'\ue05b', 0xAAFC}, + {'\ue05c', 0xAAFD}, {'\ue05d', 0xAAFE}, {'\ue05e', 0xABA1}, + {'\ue05f', 0xABA2}, {'\ue060', 0xABA3}, {'\ue061', 0xABA4}, + {'\ue062', 0xABA5}, {'\ue063', 0xABA6}, {'\ue064', 0xABA7}, + {'\ue065', 0xABA8}, {'\ue066', 0xABA9}, {'\ue067', 0xABAA}, + {'\ue068', 0xABAB}, {'\ue069', 0xABAC}, {'\ue06a', 0xABAD}, + {'\ue06b', 0xABAE}, {'\ue06c', 0xABAF}, {'\ue06d', 0xABB0}, + {'\ue06e', 0xABB1}, {'\ue06f', 0xABB2}, {'\ue070', 0xABB3}, + {'\ue071', 0xABB4}, {'\ue072', 0xABB5}, {'\ue073', 0xABB6}, + {'\ue074', 0xABB7}, {'\ue075', 0xABB8}, {'\ue076', 0xABB9}, + {'\ue077', 0xABBA}, {'\ue078', 0xABBB}, {'\ue079', 0xABBC}, + {'\ue07a', 0xABBD}, {'\ue07b', 0xABBE}, {'\ue07c', 0xABBF}, + {'\ue07d', 0xABC0}, {'\ue07e', 0xABC1}, {'\ue07f', 0xABC2}, + {'\ue080', 0xABC3}, {'\ue081', 0xABC4}, {'\ue082', 0xABC5}, + {'\ue083', 0xABC6}, {'\ue084', 0xABC7}, {'\ue085', 0xABC8}, + {'\ue086', 0xABC9}, {'\ue087', 0xABCA}, {'\ue088', 0xABCB}, + {'\ue089', 0xABCC}, {'\ue08a', 0xABCD}, {'\ue08b', 0xABCE}, + {'\ue08c', 0xABCF}, {'\ue08d', 0xABD0}, {'\ue08e', 0xABD1}, + {'\ue08f', 0xABD2}, {'\ue090', 0xABD3}, {'\ue091', 0xABD4}, + {'\ue092', 0xABD5}, {'\ue093', 0xABD6}, {'\ue094', 0xABD7}, + {'\ue095', 0xABD8}, {'\ue096', 0xABD9}, {'\ue097', 0xABDA}, + {'\ue098', 0xABDB}, {'\ue099', 0xABDC}, {'\ue09a', 0xABDD}, + {'\ue09b', 0xABDE}, {'\ue09c', 0xABDF}, {'\ue09d', 0xABE0}, + {'\ue09e', 0xABE1}, {'\ue09f', 0xABE2}, {'\ue0a0', 0xABE3}, + {'\ue0a1', 0xABE4}, {'\ue0a2', 0xABE5}, {'\ue0a3', 0xABE6}, + {'\ue0a4', 0xABE7}, {'\ue0a5', 0xABE8}, {'\ue0a6', 0xABE9}, + {'\ue0a7', 0xABEA}, {'\ue0a8', 0xABEB}, {'\ue0a9', 0xABEC}, + {'\ue0aa', 0xABED}, {'\ue0ab', 0xABEE}, {'\ue0ac', 0xABEF}, + {'\ue0ad', 0xABF0}, {'\ue0ae', 0xABF1}, {'\ue0af', 0xABF2}, + {'\ue0b0', 0xABF3}, {'\ue0b1', 0xABF4}, {'\ue0b2', 0xABF5}, + {'\ue0b3', 0xABF6}, {'\ue0b4', 0xABF7}, {'\ue0b5', 0xABF8}, + {'\ue0b6', 0xABF9}, {'\ue0b7', 0xABFA}, {'\ue0b8', 0xABFB}, + {'\ue0b9', 0xABFC}, {'\ue0ba', 0xABFD}, {'\ue0bb', 0xABFE}, + {'\ue0bc', 0xACA1}, {'\ue0bd', 0xACA2}, {'\ue0be', 0xACA3}, + {'\ue0bf', 0xACA4}, {'\ue0c0', 0xACA5}, {'\ue0c1', 0xACA6}, + {'\ue0c2', 0xACA7}, {'\ue0c3', 0xACA8}, {'\ue0c4', 0xACA9}, + {'\ue0c5', 0xACAA}, {'\ue0c6', 0xACAB}, {'\ue0c7', 0xACAC}, + {'\ue0c8', 0xACAD}, {'\ue0c9', 0xACAE}, {'\ue0ca', 0xACAF}, + {'\ue0cb', 0xACB0}, {'\ue0cc', 0xACB1}, {'\ue0cd', 0xACB2}, + {'\ue0ce', 0xACB3}, {'\ue0cf', 0xACB4}, {'\ue0d0', 0xACB5}, + {'\ue0d1', 0xACB6}, {'\ue0d2', 0xACB7}, {'\ue0d3', 0xACB8}, + {'\ue0d4', 0xACB9}, {'\ue0d5', 0xACBA}, {'\ue0d6', 0xACBB}, + {'\ue0d7', 0xACBC}, {'\ue0d8', 0xACBD}, {'\ue0d9', 0xACBE}, + {'\ue0da', 0xACBF}, {'\ue0db', 0xACC0}, {'\ue0dc', 0xACC1}, + {'\ue0dd', 0xACC2}, {'\ue0de', 0xACC3}, {'\ue0df', 0xACC4}, + {'\ue0e0', 0xACC5}, {'\ue0e1', 0xACC6}, {'\ue0e2', 0xACC7}, + {'\ue0e3', 0xACC8}, {'\ue0e4', 0xACC9}, {'\ue0e5', 0xACCA}, + {'\ue0e6', 0xACCB}, {'\ue0e7', 0xACCC}, {'\ue0e8', 0xACCD}, + {'\ue0e9', 0xACCE}, {'\ue0ea', 0xACCF}, {'\ue0eb', 0xACD0}, + {'\ue0ec', 0xACD1}, {'\ue0ed', 0xACD2}, {'\ue0ee', 0xACD3}, + {'\ue0ef', 0xACD4}, {'\ue0f0', 0xACD5}, {'\ue0f1', 0xACD6}, + {'\ue0f2', 0xACD7}, {'\ue0f3', 0xACD8}, {'\ue0f4', 0xACD9}, + {'\ue0f5', 0xACDA}, {'\ue0f6', 0xACDB}, {'\ue0f7', 0xACDC}, + {'\ue0f8', 0xACDD}, {'\ue0f9', 0xACDE}, {'\ue0fa', 0xACDF}, + {'\ue0fb', 0xACE0}, {'\ue0fc', 0xACE1}, {'\ue0fd', 0xACE2}, + {'\ue0fe', 0xACE3}, {'\ue0ff', 0xACE4}, {'\ue100', 0xACE5}, + {'\ue101', 0xACE6}, {'\ue102', 0xACE7}, {'\ue103', 0xACE8}, + {'\ue104', 0xACE9}, {'\ue105', 0xACEA}, {'\ue106', 0xACEB}, + {'\ue107', 0xACEC}, {'\ue108', 0xACED}, {'\ue109', 0xACEE}, + {'\ue10a', 0xACEF}, {'\ue10b', 0xACF0}, {'\ue10c', 0xACF1}, + {'\ue10d', 0xACF2}, {'\ue10e', 0xACF3}, {'\ue10f', 0xACF4}, + {'\ue110', 0xACF5}, {'\ue111', 0xACF6}, {'\ue112', 0xACF7}, + {'\ue113', 0xACF8}, {'\ue114', 0xACF9}, {'\ue115', 0xACFA}, + {'\ue116', 0xACFB}, {'\ue117', 0xACFC}, {'\ue118', 0xACFD}, + {'\ue119', 0xACFE}, {'\ue11a', 0xADA1}, {'\ue11b', 0xADA2}, + {'\ue11c', 0xADA3}, {'\ue11d', 0xADA4}, {'\ue11e', 0xADA5}, + {'\ue11f', 0xADA6}, {'\ue120', 0xADA7}, {'\ue121', 0xADA8}, + {'\ue122', 0xADA9}, {'\ue123', 0xADAA}, {'\ue124', 0xADAB}, + {'\ue125', 0xADAC}, {'\ue126', 0xADAD}, {'\ue127', 0xADAE}, + {'\ue128', 0xADAF}, {'\ue129', 0xADB0}, {'\ue12a', 0xADB1}, + {'\ue12b', 0xADB2}, {'\ue12c', 0xADB3}, {'\ue12d', 0xADB4}, + {'\ue12e', 0xADB5}, {'\ue12f', 0xADB6}, {'\ue130', 0xADB7}, + {'\ue131', 0xADB8}, {'\ue132', 0xADB9}, {'\ue133', 0xADBA}, + {'\ue134', 0xADBB}, {'\ue135', 0xADBC}, {'\ue136', 0xADBD}, + {'\ue137', 0xADBE}, {'\ue138', 0xADBF}, {'\ue139', 0xADC0}, + {'\ue13a', 0xADC1}, {'\ue13b', 0xADC2}, {'\ue13c', 0xADC3}, + {'\ue13d', 0xADC4}, {'\ue13e', 0xADC5}, {'\ue13f', 0xADC6}, + {'\ue140', 0xADC7}, {'\ue141', 0xADC8}, {'\ue142', 0xADC9}, + {'\ue143', 0xADCA}, {'\ue144', 0xADCB}, {'\ue145', 0xADCC}, + {'\ue146', 0xADCD}, {'\ue147', 0xADCE}, {'\ue148', 0xADCF}, + {'\ue149', 0xADD0}, {'\ue14a', 0xADD1}, {'\ue14b', 0xADD2}, + {'\ue14c', 0xADD3}, {'\ue14d', 0xADD4}, {'\ue14e', 0xADD5}, + {'\ue14f', 0xADD6}, {'\ue150', 0xADD7}, {'\ue151', 0xADD8}, + {'\ue152', 0xADD9}, {'\ue153', 0xADDA}, {'\ue154', 0xADDB}, + {'\ue155', 0xADDC}, {'\ue156', 0xADDD}, {'\ue157', 0xADDE}, + {'\ue158', 0xADDF}, {'\ue159', 0xADE0}, {'\ue15a', 0xADE1}, + {'\ue15b', 0xADE2}, {'\ue15c', 0xADE3}, {'\ue15d', 0xADE4}, + {'\ue15e', 0xADE5}, {'\ue15f', 0xADE6}, {'\ue160', 0xADE7}, + {'\ue161', 0xADE8}, {'\ue162', 0xADE9}, {'\ue163', 0xADEA}, + {'\ue164', 0xADEB}, {'\ue165', 0xADEC}, {'\ue166', 0xADED}, + {'\ue167', 0xADEE}, {'\ue168', 0xADEF}, {'\ue169', 0xADF0}, + {'\ue16a', 0xADF1}, {'\ue16b', 0xADF2}, {'\ue16c', 0xADF3}, + {'\ue16d', 0xADF4}, {'\ue16e', 0xADF5}, {'\ue16f', 0xADF6}, + {'\ue170', 0xADF7}, {'\ue171', 0xADF8}, {'\ue172', 0xADF9}, + {'\ue173', 0xADFA}, {'\ue174', 0xADFB}, {'\ue175', 0xADFC}, + {'\ue176', 0xADFD}, {'\ue177', 0xADFE}, {'\ue178', 0xAEA1}, + {'\ue179', 0xAEA2}, {'\ue17a', 0xAEA3}, {'\ue17b', 0xAEA4}, + {'\ue17c', 0xAEA5}, {'\ue17d', 0xAEA6}, {'\ue17e', 0xAEA7}, + {'\ue17f', 0xAEA8}, {'\ue180', 0xAEA9}, {'\ue181', 0xAEAA}, + {'\ue182', 0xAEAB}, {'\ue183', 0xAEAC}, {'\ue184', 0xAEAD}, + {'\ue185', 0xAEAE}, {'\ue186', 0xAEAF}, {'\ue187', 0xAEB0}, + {'\ue188', 0xAEB1}, {'\ue189', 0xAEB2}, {'\ue18a', 0xAEB3}, + {'\ue18b', 0xAEB4}, {'\ue18c', 0xAEB5}, {'\ue18d', 0xAEB6}, + {'\ue18e', 0xAEB7}, {'\ue18f', 0xAEB8}, {'\ue190', 0xAEB9}, + {'\ue191', 0xAEBA}, {'\ue192', 0xAEBB}, {'\ue193', 0xAEBC}, + {'\ue194', 0xAEBD}, {'\ue195', 0xAEBE}, {'\ue196', 0xAEBF}, + {'\ue197', 0xAEC0}, {'\ue198', 0xAEC1}, {'\ue199', 0xAEC2}, + {'\ue19a', 0xAEC3}, {'\ue19b', 0xAEC4}, {'\ue19c', 0xAEC5}, + {'\ue19d', 0xAEC6}, {'\ue19e', 0xAEC7}, {'\ue19f', 0xAEC8}, + {'\ue1a0', 0xAEC9}, {'\ue1a1', 0xAECA}, {'\ue1a2', 0xAECB}, + {'\ue1a3', 0xAECC}, {'\ue1a4', 0xAECD}, {'\ue1a5', 0xAECE}, + {'\ue1a6', 0xAECF}, {'\ue1a7', 0xAED0}, {'\ue1a8', 0xAED1}, + {'\ue1a9', 0xAED2}, {'\ue1aa', 0xAED3}, {'\ue1ab', 0xAED4}, + {'\ue1ac', 0xAED5}, {'\ue1ad', 0xAED6}, {'\ue1ae', 0xAED7}, + {'\ue1af', 0xAED8}, {'\ue1b0', 0xAED9}, {'\ue1b1', 0xAEDA}, + {'\ue1b2', 0xAEDB}, {'\ue1b3', 0xAEDC}, {'\ue1b4', 0xAEDD}, + {'\ue1b5', 0xAEDE}, {'\ue1b6', 0xAEDF}, {'\ue1b7', 0xAEE0}, + {'\ue1b8', 0xAEE1}, {'\ue1b9', 0xAEE2}, {'\ue1ba', 0xAEE3}, + {'\ue1bb', 0xAEE4}, {'\ue1bc', 0xAEE5}, {'\ue1bd', 0xAEE6}, + {'\ue1be', 0xAEE7}, {'\ue1bf', 0xAEE8}, {'\ue1c0', 0xAEE9}, + {'\ue1c1', 0xAEEA}, {'\ue1c2', 0xAEEB}, {'\ue1c3', 0xAEEC}, + {'\ue1c4', 0xAEED}, {'\ue1c5', 0xAEEE}, {'\ue1c6', 0xAEEF}, + {'\ue1c7', 0xAEF0}, {'\ue1c8', 0xAEF1}, {'\ue1c9', 0xAEF2}, + {'\ue1ca', 0xAEF3}, {'\ue1cb', 0xAEF4}, {'\ue1cc', 0xAEF5}, + {'\ue1cd', 0xAEF6}, {'\ue1ce', 0xAEF7}, {'\ue1cf', 0xAEF8}, + {'\ue1d0', 0xAEF9}, {'\ue1d1', 0xAEFA}, {'\ue1d2', 0xAEFB}, + {'\ue1d3', 0xAEFC}, {'\ue1d4', 0xAEFD}, {'\ue1d5', 0xAEFE}, + {'\ue1d6', 0xAFA1}, {'\ue1d7', 0xAFA2}, {'\ue1d8', 0xAFA3}, + {'\ue1d9', 0xAFA4}, {'\ue1da', 0xAFA5}, {'\ue1db', 0xAFA6}, + {'\ue1dc', 0xAFA7}, {'\ue1dd', 0xAFA8}, {'\ue1de', 0xAFA9}, + {'\ue1df', 0xAFAA}, {'\ue1e0', 0xAFAB}, {'\ue1e1', 0xAFAC}, + {'\ue1e2', 0xAFAD}, {'\ue1e3', 0xAFAE}, {'\ue1e4', 0xAFAF}, + {'\ue1e5', 0xAFB0}, {'\ue1e6', 0xAFB1}, {'\ue1e7', 0xAFB2}, + {'\ue1e8', 0xAFB3}, {'\ue1e9', 0xAFB4}, {'\ue1ea', 0xAFB5}, + {'\ue1eb', 0xAFB6}, {'\ue1ec', 0xAFB7}, {'\ue1ed', 0xAFB8}, + {'\ue1ee', 0xAFB9}, {'\ue1ef', 0xAFBA}, {'\ue1f0', 0xAFBB}, + {'\ue1f1', 0xAFBC}, {'\ue1f2', 0xAFBD}, {'\ue1f3', 0xAFBE}, + {'\ue1f4', 0xAFBF}, {'\ue1f5', 0xAFC0}, {'\ue1f6', 0xAFC1}, + {'\ue1f7', 0xAFC2}, {'\ue1f8', 0xAFC3}, {'\ue1f9', 0xAFC4}, + {'\ue1fa', 0xAFC5}, {'\ue1fb', 0xAFC6}, {'\ue1fc', 0xAFC7}, + {'\ue1fd', 0xAFC8}, {'\ue1fe', 0xAFC9}, {'\ue1ff', 0xAFCA}, + {'\ue200', 0xAFCB}, {'\ue201', 0xAFCC}, {'\ue202', 0xAFCD}, + {'\ue203', 0xAFCE}, {'\ue204', 0xAFCF}, {'\ue205', 0xAFD0}, + {'\ue206', 0xAFD1}, {'\ue207', 0xAFD2}, {'\ue208', 0xAFD3}, + {'\ue209', 0xAFD4}, {'\ue20a', 0xAFD5}, {'\ue20b', 0xAFD6}, + {'\ue20c', 0xAFD7}, {'\ue20d', 0xAFD8}, {'\ue20e', 0xAFD9}, + {'\ue20f', 0xAFDA}, {'\ue210', 0xAFDB}, {'\ue211', 0xAFDC}, + {'\ue212', 0xAFDD}, {'\ue213', 0xAFDE}, {'\ue214', 0xAFDF}, + {'\ue215', 0xAFE0}, {'\ue216', 0xAFE1}, {'\ue217', 0xAFE2}, + {'\ue218', 0xAFE3}, {'\ue219', 0xAFE4}, {'\ue21a', 0xAFE5}, + {'\ue21b', 0xAFE6}, {'\ue21c', 0xAFE7}, {'\ue21d', 0xAFE8}, + {'\ue21e', 0xAFE9}, {'\ue21f', 0xAFEA}, {'\ue220', 0xAFEB}, + {'\ue221', 0xAFEC}, {'\ue222', 0xAFED}, {'\ue223', 0xAFEE}, + {'\ue224', 0xAFEF}, {'\ue225', 0xAFF0}, {'\ue226', 0xAFF1}, + {'\ue227', 0xAFF2}, {'\ue228', 0xAFF3}, {'\ue229', 0xAFF4}, + {'\ue22a', 0xAFF5}, {'\ue22b', 0xAFF6}, {'\ue22c', 0xAFF7}, + {'\ue22d', 0xAFF8}, {'\ue22e', 0xAFF9}, {'\ue22f', 0xAFFA}, + {'\ue230', 0xAFFB}, {'\ue231', 0xAFFC}, {'\ue232', 0xAFFD}, + {'\ue233', 0xAFFE}, {'\ue234', 0xF8A1}, {'\ue235', 0xF8A2}, + {'\ue236', 0xF8A3}, {'\ue237', 0xF8A4}, {'\ue238', 0xF8A5}, + {'\ue239', 0xF8A6}, {'\ue23a', 0xF8A7}, {'\ue23b', 0xF8A8}, + {'\ue23c', 0xF8A9}, {'\ue23d', 0xF8AA}, {'\ue23e', 0xF8AB}, + {'\ue23f', 0xF8AC}, {'\ue240', 0xF8AD}, {'\ue241', 0xF8AE}, + {'\ue242', 0xF8AF}, {'\ue243', 0xF8B0}, {'\ue244', 0xF8B1}, + {'\ue245', 0xF8B2}, {'\ue246', 0xF8B3}, {'\ue247', 0xF8B4}, + {'\ue248', 0xF8B5}, {'\ue249', 0xF8B6}, {'\ue24a', 0xF8B7}, + {'\ue24b', 0xF8B8}, {'\ue24c', 0xF8B9}, {'\ue24d', 0xF8BA}, + {'\ue24e', 0xF8BB}, {'\ue24f', 0xF8BC}, {'\ue250', 0xF8BD}, + {'\ue251', 0xF8BE}, {'\ue252', 0xF8BF}, {'\ue253', 0xF8C0}, + {'\ue254', 0xF8C1}, {'\ue255', 0xF8C2}, {'\ue256', 0xF8C3}, + {'\ue257', 0xF8C4}, {'\ue258', 0xF8C5}, {'\ue259', 0xF8C6}, + {'\ue25a', 0xF8C7}, {'\ue25b', 0xF8C8}, {'\ue25c', 0xF8C9}, + {'\ue25d', 0xF8CA}, {'\ue25e', 0xF8CB}, {'\ue25f', 0xF8CC}, + {'\ue260', 0xF8CD}, {'\ue261', 0xF8CE}, {'\ue262', 0xF8CF}, + {'\ue263', 0xF8D0}, {'\ue264', 0xF8D1}, {'\ue265', 0xF8D2}, + {'\ue266', 0xF8D3}, {'\ue267', 0xF8D4}, {'\ue268', 0xF8D5}, + {'\ue269', 0xF8D6}, {'\ue26a', 0xF8D7}, {'\ue26b', 0xF8D8}, + {'\ue26c', 0xF8D9}, {'\ue26d', 0xF8DA}, {'\ue26e', 0xF8DB}, + {'\ue26f', 0xF8DC}, {'\ue270', 0xF8DD}, {'\ue271', 0xF8DE}, + {'\ue272', 0xF8DF}, {'\ue273', 0xF8E0}, {'\ue274', 0xF8E1}, + {'\ue275', 0xF8E2}, {'\ue276', 0xF8E3}, {'\ue277', 0xF8E4}, + {'\ue278', 0xF8E5}, {'\ue279', 0xF8E6}, {'\ue27a', 0xF8E7}, + {'\ue27b', 0xF8E8}, {'\ue27c', 0xF8E9}, {'\ue27d', 0xF8EA}, + {'\ue27e', 0xF8EB}, {'\ue27f', 0xF8EC}, {'\ue280', 0xF8ED}, + {'\ue281', 0xF8EE}, {'\ue282', 0xF8EF}, {'\ue283', 0xF8F0}, + {'\ue284', 0xF8F1}, {'\ue285', 0xF8F2}, {'\ue286', 0xF8F3}, + {'\ue287', 0xF8F4}, {'\ue288', 0xF8F5}, {'\ue289', 0xF8F6}, + {'\ue28a', 0xF8F7}, {'\ue28b', 0xF8F8}, {'\ue28c', 0xF8F9}, + {'\ue28d', 0xF8FA}, {'\ue28e', 0xF8FB}, {'\ue28f', 0xF8FC}, + {'\ue290', 0xF8FD}, {'\ue291', 0xF8FE}, {'\ue292', 0xF9A1}, + {'\ue293', 0xF9A2}, {'\ue294', 0xF9A3}, {'\ue295', 0xF9A4}, + {'\ue296', 0xF9A5}, {'\ue297', 0xF9A6}, {'\ue298', 0xF9A7}, + {'\ue299', 0xF9A8}, {'\ue29a', 0xF9A9}, {'\ue29b', 0xF9AA}, + {'\ue29c', 0xF9AB}, {'\ue29d', 0xF9AC}, {'\ue29e', 0xF9AD}, + {'\ue29f', 0xF9AE}, {'\ue2a0', 0xF9AF}, {'\ue2a1', 0xF9B0}, + {'\ue2a2', 0xF9B1}, {'\ue2a3', 0xF9B2}, {'\ue2a4', 0xF9B3}, + {'\ue2a5', 0xF9B4}, {'\ue2a6', 0xF9B5}, {'\ue2a7', 0xF9B6}, + {'\ue2a8', 0xF9B7}, {'\ue2a9', 0xF9B8}, {'\ue2aa', 0xF9B9}, + {'\ue2ab', 0xF9BA}, {'\ue2ac', 0xF9BB}, {'\ue2ad', 0xF9BC}, + {'\ue2ae', 0xF9BD}, {'\ue2af', 0xF9BE}, {'\ue2b0', 0xF9BF}, + {'\ue2b1', 0xF9C0}, {'\ue2b2', 0xF9C1}, {'\ue2b3', 0xF9C2}, + {'\ue2b4', 0xF9C3}, {'\ue2b5', 0xF9C4}, {'\ue2b6', 0xF9C5}, + {'\ue2b7', 0xF9C6}, {'\ue2b8', 0xF9C7}, {'\ue2b9', 0xF9C8}, + {'\ue2ba', 0xF9C9}, {'\ue2bb', 0xF9CA}, {'\ue2bc', 0xF9CB}, + {'\ue2bd', 0xF9CC}, {'\ue2be', 0xF9CD}, {'\ue2bf', 0xF9CE}, + {'\ue2c0', 0xF9CF}, {'\ue2c1', 0xF9D0}, {'\ue2c2', 0xF9D1}, + {'\ue2c3', 0xF9D2}, {'\ue2c4', 0xF9D3}, {'\ue2c5', 0xF9D4}, + {'\ue2c6', 0xF9D5}, {'\ue2c7', 0xF9D6}, {'\ue2c8', 0xF9D7}, + {'\ue2c9', 0xF9D8}, {'\ue2ca', 0xF9D9}, {'\ue2cb', 0xF9DA}, + {'\ue2cc', 0xF9DB}, {'\ue2cd', 0xF9DC}, {'\ue2ce', 0xF9DD}, + {'\ue2cf', 0xF9DE}, {'\ue2d0', 0xF9DF}, {'\ue2d1', 0xF9E0}, + {'\ue2d2', 0xF9E1}, {'\ue2d3', 0xF9E2}, {'\ue2d4', 0xF9E3}, + {'\ue2d5', 0xF9E4}, {'\ue2d6', 0xF9E5}, {'\ue2d7', 0xF9E6}, + {'\ue2d8', 0xF9E7}, {'\ue2d9', 0xF9E8}, {'\ue2da', 0xF9E9}, + {'\ue2db', 0xF9EA}, {'\ue2dc', 0xF9EB}, {'\ue2dd', 0xF9EC}, + {'\ue2de', 0xF9ED}, {'\ue2df', 0xF9EE}, {'\ue2e0', 0xF9EF}, + {'\ue2e1', 0xF9F0}, {'\ue2e2', 0xF9F1}, {'\ue2e3', 0xF9F2}, + {'\ue2e4', 0xF9F3}, {'\ue2e5', 0xF9F4}, {'\ue2e6', 0xF9F5}, + {'\ue2e7', 0xF9F6}, {'\ue2e8', 0xF9F7}, {'\ue2e9', 0xF9F8}, + {'\ue2ea', 0xF9F9}, {'\ue2eb', 0xF9FA}, {'\ue2ec', 0xF9FB}, + {'\ue2ed', 0xF9FC}, {'\ue2ee', 0xF9FD}, {'\ue2ef', 0xF9FE}, + {'\ue2f0', 0xFAA1}, {'\ue2f1', 0xFAA2}, {'\ue2f2', 0xFAA3}, + {'\ue2f3', 0xFAA4}, {'\ue2f4', 0xFAA5}, {'\ue2f5', 0xFAA6}, + {'\ue2f6', 0xFAA7}, {'\ue2f7', 0xFAA8}, {'\ue2f8', 0xFAA9}, + {'\ue2f9', 0xFAAA}, {'\ue2fa', 0xFAAB}, {'\ue2fb', 0xFAAC}, + {'\ue2fc', 0xFAAD}, {'\ue2fd', 0xFAAE}, {'\ue2fe', 0xFAAF}, + {'\ue2ff', 0xFAB0}, {'\ue300', 0xFAB1}, {'\ue301', 0xFAB2}, + {'\ue302', 0xFAB3}, {'\ue303', 0xFAB4}, {'\ue304', 0xFAB5}, + {'\ue305', 0xFAB6}, {'\ue306', 0xFAB7}, {'\ue307', 0xFAB8}, + {'\ue308', 0xFAB9}, {'\ue309', 0xFABA}, {'\ue30a', 0xFABB}, + {'\ue30b', 0xFABC}, {'\ue30c', 0xFABD}, {'\ue30d', 0xFABE}, + {'\ue30e', 0xFABF}, {'\ue30f', 0xFAC0}, {'\ue310', 0xFAC1}, + {'\ue311', 0xFAC2}, {'\ue312', 0xFAC3}, {'\ue313', 0xFAC4}, + {'\ue314', 0xFAC5}, {'\ue315', 0xFAC6}, {'\ue316', 0xFAC7}, + {'\ue317', 0xFAC8}, {'\ue318', 0xFAC9}, {'\ue319', 0xFACA}, + {'\ue31a', 0xFACB}, {'\ue31b', 0xFACC}, {'\ue31c', 0xFACD}, + {'\ue31d', 0xFACE}, {'\ue31e', 0xFACF}, {'\ue31f', 0xFAD0}, + {'\ue320', 0xFAD1}, {'\ue321', 0xFAD2}, {'\ue322', 0xFAD3}, + {'\ue323', 0xFAD4}, {'\ue324', 0xFAD5}, {'\ue325', 0xFAD6}, + {'\ue326', 0xFAD7}, {'\ue327', 0xFAD8}, {'\ue328', 0xFAD9}, + {'\ue329', 0xFADA}, {'\ue32a', 0xFADB}, {'\ue32b', 0xFADC}, + {'\ue32c', 0xFADD}, {'\ue32d', 0xFADE}, {'\ue32e', 0xFADF}, + {'\ue32f', 0xFAE0}, {'\ue330', 0xFAE1}, {'\ue331', 0xFAE2}, + {'\ue332', 0xFAE3}, {'\ue333', 0xFAE4}, {'\ue334', 0xFAE5}, + {'\ue335', 0xFAE6}, {'\ue336', 0xFAE7}, {'\ue337', 0xFAE8}, + {'\ue338', 0xFAE9}, {'\ue339', 0xFAEA}, {'\ue33a', 0xFAEB}, + {'\ue33b', 0xFAEC}, {'\ue33c', 0xFAED}, {'\ue33d', 0xFAEE}, + {'\ue33e', 0xFAEF}, {'\ue33f', 0xFAF0}, {'\ue340', 0xFAF1}, + {'\ue341', 0xFAF2}, {'\ue342', 0xFAF3}, {'\ue343', 0xFAF4}, + {'\ue344', 0xFAF5}, {'\ue345', 0xFAF6}, {'\ue346', 0xFAF7}, + {'\ue347', 0xFAF8}, {'\ue348', 0xFAF9}, {'\ue349', 0xFAFA}, + {'\ue34a', 0xFAFB}, {'\ue34b', 0xFAFC}, {'\ue34c', 0xFAFD}, + {'\ue34d', 0xFAFE}, {'\ue34e', 0xFBA1}, {'\ue34f', 0xFBA2}, + {'\ue350', 0xFBA3}, {'\ue351', 0xFBA4}, {'\ue352', 0xFBA5}, + {'\ue353', 0xFBA6}, {'\ue354', 0xFBA7}, {'\ue355', 0xFBA8}, + {'\ue356', 0xFBA9}, {'\ue357', 0xFBAA}, {'\ue358', 0xFBAB}, + {'\ue359', 0xFBAC}, {'\ue35a', 0xFBAD}, {'\ue35b', 0xFBAE}, + {'\ue35c', 0xFBAF}, {'\ue35d', 0xFBB0}, {'\ue35e', 0xFBB1}, + {'\ue35f', 0xFBB2}, {'\ue360', 0xFBB3}, {'\ue361', 0xFBB4}, + {'\ue362', 0xFBB5}, {'\ue363', 0xFBB6}, {'\ue364', 0xFBB7}, + {'\ue365', 0xFBB8}, {'\ue366', 0xFBB9}, {'\ue367', 0xFBBA}, + {'\ue368', 0xFBBB}, {'\ue369', 0xFBBC}, {'\ue36a', 0xFBBD}, + {'\ue36b', 0xFBBE}, {'\ue36c', 0xFBBF}, {'\ue36d', 0xFBC0}, + {'\ue36e', 0xFBC1}, {'\ue36f', 0xFBC2}, {'\ue370', 0xFBC3}, + {'\ue371', 0xFBC4}, {'\ue372', 0xFBC5}, {'\ue373', 0xFBC6}, + {'\ue374', 0xFBC7}, {'\ue375', 0xFBC8}, {'\ue376', 0xFBC9}, + {'\ue377', 0xFBCA}, {'\ue378', 0xFBCB}, {'\ue379', 0xFBCC}, + {'\ue37a', 0xFBCD}, {'\ue37b', 0xFBCE}, {'\ue37c', 0xFBCF}, + {'\ue37d', 0xFBD0}, {'\ue37e', 0xFBD1}, {'\ue37f', 0xFBD2}, + {'\ue380', 0xFBD3}, {'\ue381', 0xFBD4}, {'\ue382', 0xFBD5}, + {'\ue383', 0xFBD6}, {'\ue384', 0xFBD7}, {'\ue385', 0xFBD8}, + {'\ue386', 0xFBD9}, {'\ue387', 0xFBDA}, {'\ue388', 0xFBDB}, + {'\ue389', 0xFBDC}, {'\ue38a', 0xFBDD}, {'\ue38b', 0xFBDE}, + {'\ue38c', 0xFBDF}, {'\ue38d', 0xFBE0}, {'\ue38e', 0xFBE1}, + {'\ue38f', 0xFBE2}, {'\ue390', 0xFBE3}, {'\ue391', 0xFBE4}, + {'\ue392', 0xFBE5}, {'\ue393', 0xFBE6}, {'\ue394', 0xFBE7}, + {'\ue395', 0xFBE8}, {'\ue396', 0xFBE9}, {'\ue397', 0xFBEA}, + {'\ue398', 0xFBEB}, {'\ue399', 0xFBEC}, {'\ue39a', 0xFBED}, + {'\ue39b', 0xFBEE}, {'\ue39c', 0xFBEF}, {'\ue39d', 0xFBF0}, + {'\ue39e', 0xFBF1}, {'\ue39f', 0xFBF2}, {'\ue3a0', 0xFBF3}, + {'\ue3a1', 0xFBF4}, {'\ue3a2', 0xFBF5}, {'\ue3a3', 0xFBF6}, + {'\ue3a4', 0xFBF7}, {'\ue3a5', 0xFBF8}, {'\ue3a6', 0xFBF9}, + {'\ue3a7', 0xFBFA}, {'\ue3a8', 0xFBFB}, {'\ue3a9', 0xFBFC}, + {'\ue3aa', 0xFBFD}, {'\ue3ab', 0xFBFE}, {'\ue3ac', 0xFCA1}, + {'\ue3ad', 0xFCA2}, {'\ue3ae', 0xFCA3}, {'\ue3af', 0xFCA4}, + {'\ue3b0', 0xFCA5}, {'\ue3b1', 0xFCA6}, {'\ue3b2', 0xFCA7}, + {'\ue3b3', 0xFCA8}, {'\ue3b4', 0xFCA9}, {'\ue3b5', 0xFCAA}, + {'\ue3b6', 0xFCAB}, {'\ue3b7', 0xFCAC}, {'\ue3b8', 0xFCAD}, + {'\ue3b9', 0xFCAE}, {'\ue3ba', 0xFCAF}, {'\ue3bb', 0xFCB0}, + {'\ue3bc', 0xFCB1}, {'\ue3bd', 0xFCB2}, {'\ue3be', 0xFCB3}, + {'\ue3bf', 0xFCB4}, {'\ue3c0', 0xFCB5}, {'\ue3c1', 0xFCB6}, + {'\ue3c2', 0xFCB7}, {'\ue3c3', 0xFCB8}, {'\ue3c4', 0xFCB9}, + {'\ue3c5', 0xFCBA}, {'\ue3c6', 0xFCBB}, {'\ue3c7', 0xFCBC}, + {'\ue3c8', 0xFCBD}, {'\ue3c9', 0xFCBE}, {'\ue3ca', 0xFCBF}, + {'\ue3cb', 0xFCC0}, {'\ue3cc', 0xFCC1}, {'\ue3cd', 0xFCC2}, + {'\ue3ce', 0xFCC3}, {'\ue3cf', 0xFCC4}, {'\ue3d0', 0xFCC5}, + {'\ue3d1', 0xFCC6}, {'\ue3d2', 0xFCC7}, {'\ue3d3', 0xFCC8}, + {'\ue3d4', 0xFCC9}, {'\ue3d5', 0xFCCA}, {'\ue3d6', 0xFCCB}, + {'\ue3d7', 0xFCCC}, {'\ue3d8', 0xFCCD}, {'\ue3d9', 0xFCCE}, + {'\ue3da', 0xFCCF}, {'\ue3db', 0xFCD0}, {'\ue3dc', 0xFCD1}, + {'\ue3dd', 0xFCD2}, {'\ue3de', 0xFCD3}, {'\ue3df', 0xFCD4}, + {'\ue3e0', 0xFCD5}, {'\ue3e1', 0xFCD6}, {'\ue3e2', 0xFCD7}, + {'\ue3e3', 0xFCD8}, {'\ue3e4', 0xFCD9}, {'\ue3e5', 0xFCDA}, + {'\ue3e6', 0xFCDB}, {'\ue3e7', 0xFCDC}, {'\ue3e8', 0xFCDD}, + {'\ue3e9', 0xFCDE}, {'\ue3ea', 0xFCDF}, {'\ue3eb', 0xFCE0}, + {'\ue3ec', 0xFCE1}, {'\ue3ed', 0xFCE2}, {'\ue3ee', 0xFCE3}, + {'\ue3ef', 0xFCE4}, {'\ue3f0', 0xFCE5}, {'\ue3f1', 0xFCE6}, + {'\ue3f2', 0xFCE7}, {'\ue3f3', 0xFCE8}, {'\ue3f4', 0xFCE9}, + {'\ue3f5', 0xFCEA}, {'\ue3f6', 0xFCEB}, {'\ue3f7', 0xFCEC}, + {'\ue3f8', 0xFCED}, {'\ue3f9', 0xFCEE}, {'\ue3fa', 0xFCEF}, + {'\ue3fb', 0xFCF0}, {'\ue3fc', 0xFCF1}, {'\ue3fd', 0xFCF2}, + {'\ue3fe', 0xFCF3}, {'\ue3ff', 0xFCF4}, {'\ue400', 0xFCF5}, + {'\ue401', 0xFCF6}, {'\ue402', 0xFCF7}, {'\ue403', 0xFCF8}, + {'\ue404', 0xFCF9}, {'\ue405', 0xFCFA}, {'\ue406', 0xFCFB}, + {'\ue407', 0xFCFC}, {'\ue408', 0xFCFD}, {'\ue409', 0xFCFE}, + {'\ue40a', 0xFDA1}, {'\ue40b', 0xFDA2}, {'\ue40c', 0xFDA3}, + {'\ue40d', 0xFDA4}, {'\ue40e', 0xFDA5}, {'\ue40f', 0xFDA6}, + {'\ue410', 0xFDA7}, {'\ue411', 0xFDA8}, {'\ue412', 0xFDA9}, + {'\ue413', 0xFDAA}, {'\ue414', 0xFDAB}, {'\ue415', 0xFDAC}, + {'\ue416', 0xFDAD}, {'\ue417', 0xFDAE}, {'\ue418', 0xFDAF}, + {'\ue419', 0xFDB0}, {'\ue41a', 0xFDB1}, {'\ue41b', 0xFDB2}, + {'\ue41c', 0xFDB3}, {'\ue41d', 0xFDB4}, {'\ue41e', 0xFDB5}, + {'\ue41f', 0xFDB6}, {'\ue420', 0xFDB7}, {'\ue421', 0xFDB8}, + {'\ue422', 0xFDB9}, {'\ue423', 0xFDBA}, {'\ue424', 0xFDBB}, + {'\ue425', 0xFDBC}, {'\ue426', 0xFDBD}, {'\ue427', 0xFDBE}, + {'\ue428', 0xFDBF}, {'\ue429', 0xFDC0}, {'\ue42a', 0xFDC1}, + {'\ue42b', 0xFDC2}, {'\ue42c', 0xFDC3}, {'\ue42d', 0xFDC4}, + {'\ue42e', 0xFDC5}, {'\ue42f', 0xFDC6}, {'\ue430', 0xFDC7}, + {'\ue431', 0xFDC8}, {'\ue432', 0xFDC9}, {'\ue433', 0xFDCA}, + {'\ue434', 0xFDCB}, {'\ue435', 0xFDCC}, {'\ue436', 0xFDCD}, + {'\ue437', 0xFDCE}, {'\ue438', 0xFDCF}, {'\ue439', 0xFDD0}, + {'\ue43a', 0xFDD1}, {'\ue43b', 0xFDD2}, {'\ue43c', 0xFDD3}, + {'\ue43d', 0xFDD4}, {'\ue43e', 0xFDD5}, {'\ue43f', 0xFDD6}, + {'\ue440', 0xFDD7}, {'\ue441', 0xFDD8}, {'\ue442', 0xFDD9}, + {'\ue443', 0xFDDA}, {'\ue444', 0xFDDB}, {'\ue445', 0xFDDC}, + {'\ue446', 0xFDDD}, {'\ue447', 0xFDDE}, {'\ue448', 0xFDDF}, + {'\ue449', 0xFDE0}, {'\ue44a', 0xFDE1}, {'\ue44b', 0xFDE2}, + {'\ue44c', 0xFDE3}, {'\ue44d', 0xFDE4}, {'\ue44e', 0xFDE5}, + {'\ue44f', 0xFDE6}, {'\ue450', 0xFDE7}, {'\ue451', 0xFDE8}, + {'\ue452', 0xFDE9}, {'\ue453', 0xFDEA}, {'\ue454', 0xFDEB}, + {'\ue455', 0xFDEC}, {'\ue456', 0xFDED}, {'\ue457', 0xFDEE}, + {'\ue458', 0xFDEF}, {'\ue459', 0xFDF0}, {'\ue45a', 0xFDF1}, + {'\ue45b', 0xFDF2}, {'\ue45c', 0xFDF3}, {'\ue45d', 0xFDF4}, + {'\ue45e', 0xFDF5}, {'\ue45f', 0xFDF6}, {'\ue460', 0xFDF7}, + {'\ue461', 0xFDF8}, {'\ue462', 0xFDF9}, {'\ue463', 0xFDFA}, + {'\ue464', 0xFDFB}, {'\ue465', 0xFDFC}, {'\ue466', 0xFDFD}, + {'\ue467', 0xFDFE}, {'\ue468', 0xFEA1}, {'\ue469', 0xFEA2}, + {'\ue46a', 0xFEA3}, {'\ue46b', 0xFEA4}, {'\ue46c', 0xFEA5}, + {'\ue46d', 0xFEA6}, {'\ue46e', 0xFEA7}, {'\ue46f', 0xFEA8}, + {'\ue470', 0xFEA9}, {'\ue471', 0xFEAA}, {'\ue472', 0xFEAB}, + {'\ue473', 0xFEAC}, {'\ue474', 0xFEAD}, {'\ue475', 0xFEAE}, + {'\ue476', 0xFEAF}, {'\ue477', 0xFEB0}, {'\ue478', 0xFEB1}, + {'\ue479', 0xFEB2}, {'\ue47a', 0xFEB3}, {'\ue47b', 0xFEB4}, + {'\ue47c', 0xFEB5}, {'\ue47d', 0xFEB6}, {'\ue47e', 0xFEB7}, + {'\ue47f', 0xFEB8}, {'\ue480', 0xFEB9}, {'\ue481', 0xFEBA}, + {'\ue482', 0xFEBB}, {'\ue483', 0xFEBC}, {'\ue484', 0xFEBD}, + {'\ue485', 0xFEBE}, {'\ue486', 0xFEBF}, {'\ue487', 0xFEC0}, + {'\ue488', 0xFEC1}, {'\ue489', 0xFEC2}, {'\ue48a', 0xFEC3}, + {'\ue48b', 0xFEC4}, {'\ue48c', 0xFEC5}, {'\ue48d', 0xFEC6}, + {'\ue48e', 0xFEC7}, {'\ue48f', 0xFEC8}, {'\ue490', 0xFEC9}, + {'\ue491', 0xFECA}, {'\ue492', 0xFECB}, {'\ue493', 0xFECC}, + {'\ue494', 0xFECD}, {'\ue495', 0xFECE}, {'\ue496', 0xFECF}, + {'\ue497', 0xFED0}, {'\ue498', 0xFED1}, {'\ue499', 0xFED2}, + {'\ue49a', 0xFED3}, {'\ue49b', 0xFED4}, {'\ue49c', 0xFED5}, + {'\ue49d', 0xFED6}, {'\ue49e', 0xFED7}, {'\ue49f', 0xFED8}, + {'\ue4a0', 0xFED9}, {'\ue4a1', 0xFEDA}, {'\ue4a2', 0xFEDB}, + {'\ue4a3', 0xFEDC}, {'\ue4a4', 0xFEDD}, {'\ue4a5', 0xFEDE}, + {'\ue4a6', 0xFEDF}, {'\ue4a7', 0xFEE0}, {'\ue4a8', 0xFEE1}, + {'\ue4a9', 0xFEE2}, {'\ue4aa', 0xFEE3}, {'\ue4ab', 0xFEE4}, + {'\ue4ac', 0xFEE5}, {'\ue4ad', 0xFEE6}, {'\ue4ae', 0xFEE7}, + {'\ue4af', 0xFEE8}, {'\ue4b0', 0xFEE9}, {'\ue4b1', 0xFEEA}, + {'\ue4b2', 0xFEEB}, {'\ue4b3', 0xFEEC}, {'\ue4b4', 0xFEED}, + {'\ue4b5', 0xFEEE}, {'\ue4b6', 0xFEEF}, {'\ue4b7', 0xFEF0}, + {'\ue4b8', 0xFEF1}, {'\ue4b9', 0xFEF2}, {'\ue4ba', 0xFEF3}, + {'\ue4bb', 0xFEF4}, {'\ue4bc', 0xFEF5}, {'\ue4bd', 0xFEF6}, + {'\ue4be', 0xFEF7}, {'\ue4bf', 0xFEF8}, {'\ue4c0', 0xFEF9}, + {'\ue4c1', 0xFEFA}, {'\ue4c2', 0xFEFB}, {'\ue4c3', 0xFEFC}, + {'\ue4c4', 0xFEFD}, {'\ue4c5', 0xFEFE}, {'\ue4c6', 0xA140}, + {'\ue4c7', 0xA141}, {'\ue4c8', 0xA142}, {'\ue4c9', 0xA143}, + {'\ue4ca', 0xA144}, {'\ue4cb', 0xA145}, {'\ue4cc', 0xA146}, + {'\ue4cd', 0xA147}, {'\ue4ce', 0xA148}, {'\ue4cf', 0xA149}, + {'\ue4d0', 0xA14A}, {'\ue4d1', 0xA14B}, {'\ue4d2', 0xA14C}, + {'\ue4d3', 0xA14D}, {'\ue4d4', 0xA14E}, {'\ue4d5', 0xA14F}, + {'\ue4d6', 0xA150}, {'\ue4d7', 0xA151}, {'\ue4d8', 0xA152}, + {'\ue4d9', 0xA153}, {'\ue4da', 0xA154}, {'\ue4db', 0xA155}, + {'\ue4dc', 0xA156}, {'\ue4dd', 0xA157}, {'\ue4de', 0xA158}, + {'\ue4df', 0xA159}, {'\ue4e0', 0xA15A}, {'\ue4e1', 0xA15B}, + {'\ue4e2', 0xA15C}, {'\ue4e3', 0xA15D}, {'\ue4e4', 0xA15E}, + {'\ue4e5', 0xA15F}, {'\ue4e6', 0xA160}, {'\ue4e7', 0xA161}, + {'\ue4e8', 0xA162}, {'\ue4e9', 0xA163}, {'\ue4ea', 0xA164}, + {'\ue4eb', 0xA165}, {'\ue4ec', 0xA166}, {'\ue4ed', 0xA167}, + {'\ue4ee', 0xA168}, {'\ue4ef', 0xA169}, {'\ue4f0', 0xA16A}, + {'\ue4f1', 0xA16B}, {'\ue4f2', 0xA16C}, {'\ue4f3', 0xA16D}, + {'\ue4f4', 0xA16E}, {'\ue4f5', 0xA16F}, {'\ue4f6', 0xA170}, + {'\ue4f7', 0xA171}, {'\ue4f8', 0xA172}, {'\ue4f9', 0xA173}, + {'\ue4fa', 0xA174}, {'\ue4fb', 0xA175}, {'\ue4fc', 0xA176}, + {'\ue4fd', 0xA177}, {'\ue4fe', 0xA178}, {'\ue4ff', 0xA179}, + {'\ue500', 0xA17A}, {'\ue501', 0xA17B}, {'\ue502', 0xA17C}, + {'\ue503', 0xA17D}, {'\ue504', 0xA17E}, {'\ue505', 0xA180}, + {'\ue506', 0xA181}, {'\ue507', 0xA182}, {'\ue508', 0xA183}, + {'\ue509', 0xA184}, {'\ue50a', 0xA185}, {'\ue50b', 0xA186}, + {'\ue50c', 0xA187}, {'\ue50d', 0xA188}, {'\ue50e', 0xA189}, + {'\ue50f', 0xA18A}, {'\ue510', 0xA18B}, {'\ue511', 0xA18C}, + {'\ue512', 0xA18D}, {'\ue513', 0xA18E}, {'\ue514', 0xA18F}, + {'\ue515', 0xA190}, {'\ue516', 0xA191}, {'\ue517', 0xA192}, + {'\ue518', 0xA193}, {'\ue519', 0xA194}, {'\ue51a', 0xA195}, + {'\ue51b', 0xA196}, {'\ue51c', 0xA197}, {'\ue51d', 0xA198}, + {'\ue51e', 0xA199}, {'\ue51f', 0xA19A}, {'\ue520', 0xA19B}, + {'\ue521', 0xA19C}, {'\ue522', 0xA19D}, {'\ue523', 0xA19E}, + {'\ue524', 0xA19F}, {'\ue525', 0xA1A0}, {'\ue526', 0xA240}, + {'\ue527', 0xA241}, {'\ue528', 0xA242}, {'\ue529', 0xA243}, + {'\ue52a', 0xA244}, {'\ue52b', 0xA245}, {'\ue52c', 0xA246}, + {'\ue52d', 0xA247}, {'\ue52e', 0xA248}, {'\ue52f', 0xA249}, + {'\ue530', 0xA24A}, {'\ue531', 0xA24B}, {'\ue532', 0xA24C}, + {'\ue533', 0xA24D}, {'\ue534', 0xA24E}, {'\ue535', 0xA24F}, + {'\ue536', 0xA250}, {'\ue537', 0xA251}, {'\ue538', 0xA252}, + {'\ue539', 0xA253}, {'\ue53a', 0xA254}, {'\ue53b', 0xA255}, + {'\ue53c', 0xA256}, {'\ue53d', 0xA257}, {'\ue53e', 0xA258}, + {'\ue53f', 0xA259}, {'\ue540', 0xA25A}, {'\ue541', 0xA25B}, + {'\ue542', 0xA25C}, {'\ue543', 0xA25D}, {'\ue544', 0xA25E}, + {'\ue545', 0xA25F}, {'\ue546', 0xA260}, {'\ue547', 0xA261}, + {'\ue548', 0xA262}, {'\ue549', 0xA263}, {'\ue54a', 0xA264}, + {'\ue54b', 0xA265}, {'\ue54c', 0xA266}, {'\ue54d', 0xA267}, + {'\ue54e', 0xA268}, {'\ue54f', 0xA269}, {'\ue550', 0xA26A}, + {'\ue551', 0xA26B}, {'\ue552', 0xA26C}, {'\ue553', 0xA26D}, + {'\ue554', 0xA26E}, {'\ue555', 0xA26F}, {'\ue556', 0xA270}, + {'\ue557', 0xA271}, {'\ue558', 0xA272}, {'\ue559', 0xA273}, + {'\ue55a', 0xA274}, {'\ue55b', 0xA275}, {'\ue55c', 0xA276}, + {'\ue55d', 0xA277}, {'\ue55e', 0xA278}, {'\ue55f', 0xA279}, + {'\ue560', 0xA27A}, {'\ue561', 0xA27B}, {'\ue562', 0xA27C}, + {'\ue563', 0xA27D}, {'\ue564', 0xA27E}, {'\ue565', 0xA280}, + {'\ue566', 0xA281}, {'\ue567', 0xA282}, {'\ue568', 0xA283}, + {'\ue569', 0xA284}, {'\ue56a', 0xA285}, {'\ue56b', 0xA286}, + {'\ue56c', 0xA287}, {'\ue56d', 0xA288}, {'\ue56e', 0xA289}, + {'\ue56f', 0xA28A}, {'\ue570', 0xA28B}, {'\ue571', 0xA28C}, + {'\ue572', 0xA28D}, {'\ue573', 0xA28E}, {'\ue574', 0xA28F}, + {'\ue575', 0xA290}, {'\ue576', 0xA291}, {'\ue577', 0xA292}, + {'\ue578', 0xA293}, {'\ue579', 0xA294}, {'\ue57a', 0xA295}, + {'\ue57b', 0xA296}, {'\ue57c', 0xA297}, {'\ue57d', 0xA298}, + {'\ue57e', 0xA299}, {'\ue57f', 0xA29A}, {'\ue580', 0xA29B}, + {'\ue581', 0xA29C}, {'\ue582', 0xA29D}, {'\ue583', 0xA29E}, + {'\ue584', 0xA29F}, {'\ue585', 0xA2A0}, {'\ue586', 0xA340}, + {'\ue587', 0xA341}, {'\ue588', 0xA342}, {'\ue589', 0xA343}, + {'\ue58a', 0xA344}, {'\ue58b', 0xA345}, {'\ue58c', 0xA346}, + {'\ue58d', 0xA347}, {'\ue58e', 0xA348}, {'\ue58f', 0xA349}, + {'\ue590', 0xA34A}, {'\ue591', 0xA34B}, {'\ue592', 0xA34C}, + {'\ue593', 0xA34D}, {'\ue594', 0xA34E}, {'\ue595', 0xA34F}, + {'\ue596', 0xA350}, {'\ue597', 0xA351}, {'\ue598', 0xA352}, + {'\ue599', 0xA353}, {'\ue59a', 0xA354}, {'\ue59b', 0xA355}, + {'\ue59c', 0xA356}, {'\ue59d', 0xA357}, {'\ue59e', 0xA358}, + {'\ue59f', 0xA359}, {'\ue5a0', 0xA35A}, {'\ue5a1', 0xA35B}, + {'\ue5a2', 0xA35C}, {'\ue5a3', 0xA35D}, {'\ue5a4', 0xA35E}, + {'\ue5a5', 0xA35F}, {'\ue5a6', 0xA360}, {'\ue5a7', 0xA361}, + {'\ue5a8', 0xA362}, {'\ue5a9', 0xA363}, {'\ue5aa', 0xA364}, + {'\ue5ab', 0xA365}, {'\ue5ac', 0xA366}, {'\ue5ad', 0xA367}, + {'\ue5ae', 0xA368}, {'\ue5af', 0xA369}, {'\ue5b0', 0xA36A}, + {'\ue5b1', 0xA36B}, {'\ue5b2', 0xA36C}, {'\ue5b3', 0xA36D}, + {'\ue5b4', 0xA36E}, {'\ue5b5', 0xA36F}, {'\ue5b6', 0xA370}, + {'\ue5b7', 0xA371}, {'\ue5b8', 0xA372}, {'\ue5b9', 0xA373}, + {'\ue5ba', 0xA374}, {'\ue5bb', 0xA375}, {'\ue5bc', 0xA376}, + {'\ue5bd', 0xA377}, {'\ue5be', 0xA378}, {'\ue5bf', 0xA379}, + {'\ue5c0', 0xA37A}, {'\ue5c1', 0xA37B}, {'\ue5c2', 0xA37C}, + {'\ue5c3', 0xA37D}, {'\ue5c4', 0xA37E}, {'\ue5c5', 0xA380}, + {'\ue5c6', 0xA381}, {'\ue5c7', 0xA382}, {'\ue5c8', 0xA383}, + {'\ue5c9', 0xA384}, {'\ue5ca', 0xA385}, {'\ue5cb', 0xA386}, + {'\ue5cc', 0xA387}, {'\ue5cd', 0xA388}, {'\ue5ce', 0xA389}, + {'\ue5cf', 0xA38A}, {'\ue5d0', 0xA38B}, {'\ue5d1', 0xA38C}, + {'\ue5d2', 0xA38D}, {'\ue5d3', 0xA38E}, {'\ue5d4', 0xA38F}, + {'\ue5d5', 0xA390}, {'\ue5d6', 0xA391}, {'\ue5d7', 0xA392}, + {'\ue5d8', 0xA393}, {'\ue5d9', 0xA394}, {'\ue5da', 0xA395}, + {'\ue5db', 0xA396}, {'\ue5dc', 0xA397}, {'\ue5dd', 0xA398}, + {'\ue5de', 0xA399}, {'\ue5df', 0xA39A}, {'\ue5e0', 0xA39B}, + {'\ue5e1', 0xA39C}, {'\ue5e2', 0xA39D}, {'\ue5e3', 0xA39E}, + {'\ue5e4', 0xA39F}, {'\ue5e5', 0xA3A0}, {'\ue5e6', 0xA440}, + {'\ue5e7', 0xA441}, {'\ue5e8', 0xA442}, {'\ue5e9', 0xA443}, + {'\ue5ea', 0xA444}, {'\ue5eb', 0xA445}, {'\ue5ec', 0xA446}, + {'\ue5ed', 0xA447}, {'\ue5ee', 0xA448}, {'\ue5ef', 0xA449}, + {'\ue5f0', 0xA44A}, {'\ue5f1', 0xA44B}, {'\ue5f2', 0xA44C}, + {'\ue5f3', 0xA44D}, {'\ue5f4', 0xA44E}, {'\ue5f5', 0xA44F}, + {'\ue5f6', 0xA450}, {'\ue5f7', 0xA451}, {'\ue5f8', 0xA452}, + {'\ue5f9', 0xA453}, {'\ue5fa', 0xA454}, {'\ue5fb', 0xA455}, + {'\ue5fc', 0xA456}, {'\ue5fd', 0xA457}, {'\ue5fe', 0xA458}, + {'\ue5ff', 0xA459}, {'\ue600', 0xA45A}, {'\ue601', 0xA45B}, + {'\ue602', 0xA45C}, {'\ue603', 0xA45D}, {'\ue604', 0xA45E}, + {'\ue605', 0xA45F}, {'\ue606', 0xA460}, {'\ue607', 0xA461}, + {'\ue608', 0xA462}, {'\ue609', 0xA463}, {'\ue60a', 0xA464}, + {'\ue60b', 0xA465}, {'\ue60c', 0xA466}, {'\ue60d', 0xA467}, + {'\ue60e', 0xA468}, {'\ue60f', 0xA469}, {'\ue610', 0xA46A}, + {'\ue611', 0xA46B}, {'\ue612', 0xA46C}, {'\ue613', 0xA46D}, + {'\ue614', 0xA46E}, {'\ue615', 0xA46F}, {'\ue616', 0xA470}, + {'\ue617', 0xA471}, {'\ue618', 0xA472}, {'\ue619', 0xA473}, + {'\ue61a', 0xA474}, {'\ue61b', 0xA475}, {'\ue61c', 0xA476}, + {'\ue61d', 0xA477}, {'\ue61e', 0xA478}, {'\ue61f', 0xA479}, + {'\ue620', 0xA47A}, {'\ue621', 0xA47B}, {'\ue622', 0xA47C}, + {'\ue623', 0xA47D}, {'\ue624', 0xA47E}, {'\ue625', 0xA480}, + {'\ue626', 0xA481}, {'\ue627', 0xA482}, {'\ue628', 0xA483}, + {'\ue629', 0xA484}, {'\ue62a', 0xA485}, {'\ue62b', 0xA486}, + {'\ue62c', 0xA487}, {'\ue62d', 0xA488}, {'\ue62e', 0xA489}, + {'\ue62f', 0xA48A}, {'\ue630', 0xA48B}, {'\ue631', 0xA48C}, + {'\ue632', 0xA48D}, {'\ue633', 0xA48E}, {'\ue634', 0xA48F}, + {'\ue635', 0xA490}, {'\ue636', 0xA491}, {'\ue637', 0xA492}, + {'\ue638', 0xA493}, {'\ue639', 0xA494}, {'\ue63a', 0xA495}, + {'\ue63b', 0xA496}, {'\ue63c', 0xA497}, {'\ue63d', 0xA498}, + {'\ue63e', 0xA499}, {'\ue63f', 0xA49A}, {'\ue640', 0xA49B}, + {'\ue641', 0xA49C}, {'\ue642', 0xA49D}, {'\ue643', 0xA49E}, + {'\ue644', 0xA49F}, {'\ue645', 0xA4A0}, {'\ue646', 0xA540}, + {'\ue647', 0xA541}, {'\ue648', 0xA542}, {'\ue649', 0xA543}, + {'\ue64a', 0xA544}, {'\ue64b', 0xA545}, {'\ue64c', 0xA546}, + {'\ue64d', 0xA547}, {'\ue64e', 0xA548}, {'\ue64f', 0xA549}, + {'\ue650', 0xA54A}, {'\ue651', 0xA54B}, {'\ue652', 0xA54C}, + {'\ue653', 0xA54D}, {'\ue654', 0xA54E}, {'\ue655', 0xA54F}, + {'\ue656', 0xA550}, {'\ue657', 0xA551}, {'\ue658', 0xA552}, + {'\ue659', 0xA553}, {'\ue65a', 0xA554}, {'\ue65b', 0xA555}, + {'\ue65c', 0xA556}, {'\ue65d', 0xA557}, {'\ue65e', 0xA558}, + {'\ue65f', 0xA559}, {'\ue660', 0xA55A}, {'\ue661', 0xA55B}, + {'\ue662', 0xA55C}, {'\ue663', 0xA55D}, {'\ue664', 0xA55E}, + {'\ue665', 0xA55F}, {'\ue666', 0xA560}, {'\ue667', 0xA561}, + {'\ue668', 0xA562}, {'\ue669', 0xA563}, {'\ue66a', 0xA564}, + {'\ue66b', 0xA565}, {'\ue66c', 0xA566}, {'\ue66d', 0xA567}, + {'\ue66e', 0xA568}, {'\ue66f', 0xA569}, {'\ue670', 0xA56A}, + {'\ue671', 0xA56B}, {'\ue672', 0xA56C}, {'\ue673', 0xA56D}, + {'\ue674', 0xA56E}, {'\ue675', 0xA56F}, {'\ue676', 0xA570}, + {'\ue677', 0xA571}, {'\ue678', 0xA572}, {'\ue679', 0xA573}, + {'\ue67a', 0xA574}, {'\ue67b', 0xA575}, {'\ue67c', 0xA576}, + {'\ue67d', 0xA577}, {'\ue67e', 0xA578}, {'\ue67f', 0xA579}, + {'\ue680', 0xA57A}, {'\ue681', 0xA57B}, {'\ue682', 0xA57C}, + {'\ue683', 0xA57D}, {'\ue684', 0xA57E}, {'\ue685', 0xA580}, + {'\ue686', 0xA581}, {'\ue687', 0xA582}, {'\ue688', 0xA583}, + {'\ue689', 0xA584}, {'\ue68a', 0xA585}, {'\ue68b', 0xA586}, + {'\ue68c', 0xA587}, {'\ue68d', 0xA588}, {'\ue68e', 0xA589}, + {'\ue68f', 0xA58A}, {'\ue690', 0xA58B}, {'\ue691', 0xA58C}, + {'\ue692', 0xA58D}, {'\ue693', 0xA58E}, {'\ue694', 0xA58F}, + {'\ue695', 0xA590}, {'\ue696', 0xA591}, {'\ue697', 0xA592}, + {'\ue698', 0xA593}, {'\ue699', 0xA594}, {'\ue69a', 0xA595}, + {'\ue69b', 0xA596}, {'\ue69c', 0xA597}, {'\ue69d', 0xA598}, + {'\ue69e', 0xA599}, {'\ue69f', 0xA59A}, {'\ue6a0', 0xA59B}, + {'\ue6a1', 0xA59C}, {'\ue6a2', 0xA59D}, {'\ue6a3', 0xA59E}, + {'\ue6a4', 0xA59F}, {'\ue6a5', 0xA5A0}, {'\ue6a6', 0xA640}, + {'\ue6a7', 0xA641}, {'\ue6a8', 0xA642}, {'\ue6a9', 0xA643}, + {'\ue6aa', 0xA644}, {'\ue6ab', 0xA645}, {'\ue6ac', 0xA646}, + {'\ue6ad', 0xA647}, {'\ue6ae', 0xA648}, {'\ue6af', 0xA649}, + {'\ue6b0', 0xA64A}, {'\ue6b1', 0xA64B}, {'\ue6b2', 0xA64C}, + {'\ue6b3', 0xA64D}, {'\ue6b4', 0xA64E}, {'\ue6b5', 0xA64F}, + {'\ue6b6', 0xA650}, {'\ue6b7', 0xA651}, {'\ue6b8', 0xA652}, + {'\ue6b9', 0xA653}, {'\ue6ba', 0xA654}, {'\ue6bb', 0xA655}, + {'\ue6bc', 0xA656}, {'\ue6bd', 0xA657}, {'\ue6be', 0xA658}, + {'\ue6bf', 0xA659}, {'\ue6c0', 0xA65A}, {'\ue6c1', 0xA65B}, + {'\ue6c2', 0xA65C}, {'\ue6c3', 0xA65D}, {'\ue6c4', 0xA65E}, + {'\ue6c5', 0xA65F}, {'\ue6c6', 0xA660}, {'\ue6c7', 0xA661}, + {'\ue6c8', 0xA662}, {'\ue6c9', 0xA663}, {'\ue6ca', 0xA664}, + {'\ue6cb', 0xA665}, {'\ue6cc', 0xA666}, {'\ue6cd', 0xA667}, + {'\ue6ce', 0xA668}, {'\ue6cf', 0xA669}, {'\ue6d0', 0xA66A}, + {'\ue6d1', 0xA66B}, {'\ue6d2', 0xA66C}, {'\ue6d3', 0xA66D}, + {'\ue6d4', 0xA66E}, {'\ue6d5', 0xA66F}, {'\ue6d6', 0xA670}, + {'\ue6d7', 0xA671}, {'\ue6d8', 0xA672}, {'\ue6d9', 0xA673}, + {'\ue6da', 0xA674}, {'\ue6db', 0xA675}, {'\ue6dc', 0xA676}, + {'\ue6dd', 0xA677}, {'\ue6de', 0xA678}, {'\ue6df', 0xA679}, + {'\ue6e0', 0xA67A}, {'\ue6e1', 0xA67B}, {'\ue6e2', 0xA67C}, + {'\ue6e3', 0xA67D}, {'\ue6e4', 0xA67E}, {'\ue6e5', 0xA680}, + {'\ue6e6', 0xA681}, {'\ue6e7', 0xA682}, {'\ue6e8', 0xA683}, + {'\ue6e9', 0xA684}, {'\ue6ea', 0xA685}, {'\ue6eb', 0xA686}, + {'\ue6ec', 0xA687}, {'\ue6ed', 0xA688}, {'\ue6ee', 0xA689}, + {'\ue6ef', 0xA68A}, {'\ue6f0', 0xA68B}, {'\ue6f1', 0xA68C}, + {'\ue6f2', 0xA68D}, {'\ue6f3', 0xA68E}, {'\ue6f4', 0xA68F}, + {'\ue6f5', 0xA690}, {'\ue6f6', 0xA691}, {'\ue6f7', 0xA692}, + {'\ue6f8', 0xA693}, {'\ue6f9', 0xA694}, {'\ue6fa', 0xA695}, + {'\ue6fb', 0xA696}, {'\ue6fc', 0xA697}, {'\ue6fd', 0xA698}, + {'\ue6fe', 0xA699}, {'\ue6ff', 0xA69A}, {'\ue700', 0xA69B}, + {'\ue701', 0xA69C}, {'\ue702', 0xA69D}, {'\ue703', 0xA69E}, + {'\ue704', 0xA69F}, {'\ue705', 0xA6A0}, {'\ue706', 0xA740}, + {'\ue707', 0xA741}, {'\ue708', 0xA742}, {'\ue709', 0xA743}, + {'\ue70a', 0xA744}, {'\ue70b', 0xA745}, {'\ue70c', 0xA746}, + {'\ue70d', 0xA747}, {'\ue70e', 0xA748}, {'\ue70f', 0xA749}, + {'\ue710', 0xA74A}, {'\ue711', 0xA74B}, {'\ue712', 0xA74C}, + {'\ue713', 0xA74D}, {'\ue714', 0xA74E}, {'\ue715', 0xA74F}, + {'\ue716', 0xA750}, {'\ue717', 0xA751}, {'\ue718', 0xA752}, + {'\ue719', 0xA753}, {'\ue71a', 0xA754}, {'\ue71b', 0xA755}, + {'\ue71c', 0xA756}, {'\ue71d', 0xA757}, {'\ue71e', 0xA758}, + {'\ue71f', 0xA759}, {'\ue720', 0xA75A}, {'\ue721', 0xA75B}, + {'\ue722', 0xA75C}, {'\ue723', 0xA75D}, {'\ue724', 0xA75E}, + {'\ue725', 0xA75F}, {'\ue726', 0xA760}, {'\ue727', 0xA761}, + {'\ue728', 0xA762}, {'\ue729', 0xA763}, {'\ue72a', 0xA764}, + {'\ue72b', 0xA765}, {'\ue72c', 0xA766}, {'\ue72d', 0xA767}, + {'\ue72e', 0xA768}, {'\ue72f', 0xA769}, {'\ue730', 0xA76A}, + {'\ue731', 0xA76B}, {'\ue732', 0xA76C}, {'\ue733', 0xA76D}, + {'\ue734', 0xA76E}, {'\ue735', 0xA76F}, {'\ue736', 0xA770}, + {'\ue737', 0xA771}, {'\ue738', 0xA772}, {'\ue739', 0xA773}, + {'\ue73a', 0xA774}, {'\ue73b', 0xA775}, {'\ue73c', 0xA776}, + {'\ue73d', 0xA777}, {'\ue73e', 0xA778}, {'\ue73f', 0xA779}, + {'\ue740', 0xA77A}, {'\ue741', 0xA77B}, {'\ue742', 0xA77C}, + {'\ue743', 0xA77D}, {'\ue744', 0xA77E}, {'\ue745', 0xA780}, + {'\ue746', 0xA781}, {'\ue747', 0xA782}, {'\ue748', 0xA783}, + {'\ue749', 0xA784}, {'\ue74a', 0xA785}, {'\ue74b', 0xA786}, + {'\ue74c', 0xA787}, {'\ue74d', 0xA788}, {'\ue74e', 0xA789}, + {'\ue74f', 0xA78A}, {'\ue750', 0xA78B}, {'\ue751', 0xA78C}, + {'\ue752', 0xA78D}, {'\ue753', 0xA78E}, {'\ue754', 0xA78F}, + {'\ue755', 0xA790}, {'\ue756', 0xA791}, {'\ue757', 0xA792}, + {'\ue758', 0xA793}, {'\ue759', 0xA794}, {'\ue75a', 0xA795}, + {'\ue75b', 0xA796}, {'\ue75c', 0xA797}, {'\ue75d', 0xA798}, + {'\ue75e', 0xA799}, {'\ue75f', 0xA79A}, {'\ue760', 0xA79B}, + {'\ue761', 0xA79C}, {'\ue762', 0xA79D}, {'\ue763', 0xA79E}, + {'\ue764', 0xA79F}, {'\ue765', 0xA7A0}, {'\ue766', 0xA2AB}, + {'\ue767', 0xA2AC}, {'\ue768', 0xA2AD}, {'\ue769', 0xA2AE}, + {'\ue76a', 0xA2AF}, {'\ue76b', 0xA2B0}, {'\ue76d', 0xA2E4}, + {'\ue76e', 0xA2EF}, {'\ue76f', 0xA2F0}, {'\ue770', 0xA2FD}, + {'\ue771', 0xA2FE}, {'\ue772', 0xA4F4}, {'\ue773', 0xA4F5}, + {'\ue774', 0xA4F6}, {'\ue775', 0xA4F7}, {'\ue776', 0xA4F8}, + {'\ue777', 0xA4F9}, {'\ue778', 0xA4FA}, {'\ue779', 0xA4FB}, + {'\ue77a', 0xA4FC}, {'\ue77b', 0xA4FD}, {'\ue77c', 0xA4FE}, + {'\ue77d', 0xA5F7}, {'\ue77e', 0xA5F8}, {'\ue77f', 0xA5F9}, + {'\ue780', 0xA5FA}, {'\ue781', 0xA5FB}, {'\ue782', 0xA5FC}, + {'\ue783', 0xA5FD}, {'\ue784', 0xA5FE}, {'\ue785', 0xA6B9}, + {'\ue786', 0xA6BA}, {'\ue787', 0xA6BB}, {'\ue788', 0xA6BC}, + {'\ue789', 0xA6BD}, {'\ue78a', 0xA6BE}, {'\ue78b', 0xA6BF}, + {'\ue78c', 0xA6C0}, {'\ue78d', 0x84318236}, {'\ue78e', 0x84318238}, + {'\ue78f', 0x84318237}, {'\ue790', 0x84318239}, {'\ue791', 0x84318330}, + {'\ue792', 0x84318331}, {'\ue793', 0x84318332}, {'\ue794', 0x84318333}, + {'\ue795', 0x84318334}, {'\ue796', 0x84318335}, {'\ue797', 0xA6F6}, + {'\ue798', 0xA6F7}, {'\ue799', 0xA6F8}, {'\ue79a', 0xA6F9}, + {'\ue79b', 0xA6FA}, {'\ue79c', 0xA6FB}, {'\ue79d', 0xA6FC}, + {'\ue79e', 0xA6FD}, {'\ue79f', 0xA6FE}, {'\ue7a0', 0xA7C2}, + {'\ue7a1', 0xA7C3}, {'\ue7a2', 0xA7C4}, {'\ue7a3', 0xA7C5}, + {'\ue7a4', 0xA7C6}, {'\ue7a5', 0xA7C7}, {'\ue7a6', 0xA7C8}, + {'\ue7a7', 0xA7C9}, {'\ue7a8', 0xA7CA}, {'\ue7a9', 0xA7CB}, + {'\ue7aa', 0xA7CC}, {'\ue7ab', 0xA7CD}, {'\ue7ac', 0xA7CE}, + {'\ue7ad', 0xA7CF}, {'\ue7ae', 0xA7D0}, {'\ue7af', 0xA7F2}, + {'\ue7b0', 0xA7F3}, {'\ue7b1', 0xA7F4}, {'\ue7b2', 0xA7F5}, + {'\ue7b3', 0xA7F6}, {'\ue7b4', 0xA7F7}, {'\ue7b5', 0xA7F8}, + {'\ue7b6', 0xA7F9}, {'\ue7b7', 0xA7FA}, {'\ue7b8', 0xA7FB}, + {'\ue7b9', 0xA7FC}, {'\ue7ba', 0xA7FD}, {'\ue7bb', 0xA7FE}, + {'\ue7bc', 0xA896}, {'\ue7bd', 0xA897}, {'\ue7be', 0xA898}, + {'\ue7bf', 0xA899}, {'\ue7c0', 0xA89A}, {'\ue7c1', 0xA89B}, + {'\ue7c2', 0xA89C}, {'\ue7c3', 0xA89D}, {'\ue7c4', 0xA89E}, + {'\ue7c5', 0xA89F}, {'\ue7c6', 0xA8A0}, {'\ue7c7', 0x8135F437}, + {'\ue7c9', 0xA8C1}, {'\ue7ca', 0xA8C2}, {'\ue7cb', 0xA8C3}, + {'\ue7cc', 0xA8C4}, {'\ue7cd', 0xA8EA}, {'\ue7ce', 0xA8EB}, + {'\ue7cf', 0xA8EC}, {'\ue7d0', 0xA8ED}, {'\ue7d1', 0xA8EE}, + {'\ue7d2', 0xA8EF}, {'\ue7d3', 0xA8F0}, {'\ue7d4', 0xA8F1}, + {'\ue7d5', 0xA8F2}, {'\ue7d6', 0xA8F3}, {'\ue7d7', 0xA8F4}, + {'\ue7d8', 0xA8F5}, {'\ue7d9', 0xA8F6}, {'\ue7da', 0xA8F7}, + {'\ue7db', 0xA8F8}, {'\ue7dc', 0xA8F9}, {'\ue7dd', 0xA8FA}, + {'\ue7de', 0xA8FB}, {'\ue7df', 0xA8FC}, {'\ue7e0', 0xA8FD}, + {'\ue7e1', 0xA8FE}, {'\ue7e2', 0xA958}, {'\ue7e3', 0xA95B}, + {'\ue7e4', 0xA95D}, {'\ue7e5', 0xA95E}, {'\ue7e6', 0xA95F}, + {'\ue7f4', 0xA997}, {'\ue7f5', 0xA998}, {'\ue7f6', 0xA999}, + {'\ue7f7', 0xA99A}, {'\ue7f8', 0xA99B}, {'\ue7f9', 0xA99C}, + {'\ue7fa', 0xA99D}, {'\ue7fb', 0xA99E}, {'\ue7fc', 0xA99F}, + {'\ue7fd', 0xA9A0}, {'\ue7fe', 0xA9A1}, {'\ue7ff', 0xA9A2}, + {'\ue800', 0xA9A3}, {'\ue801', 0xA9F0}, {'\ue802', 0xA9F1}, + {'\ue803', 0xA9F2}, {'\ue804', 0xA9F3}, {'\ue805', 0xA9F4}, + {'\ue806', 0xA9F5}, {'\ue807', 0xA9F6}, {'\ue808', 0xA9F7}, + {'\ue809', 0xA9F8}, {'\ue80a', 0xA9F9}, {'\ue80b', 0xA9FA}, + {'\ue80c', 0xA9FB}, {'\ue80d', 0xA9FC}, {'\ue80e', 0xA9FD}, + {'\ue80f', 0xA9FE}, {'\ue810', 0xD7FA}, {'\ue811', 0xD7FB}, + {'\ue812', 0xD7FC}, {'\ue813', 0xD7FD}, {'\ue814', 0xD7FE}, + {'\ue816', 0xFE51}, {'\ue817', 0xFE52}, {'\ue818', 0xFE53}, + {'\ue81e', 0x82359037}, {'\ue826', 0x82359038}, {'\ue82b', 0x82359039}, + {'\ue82c', 0x82359130}, {'\ue831', 0xFE6C}, {'\ue832', 0x82359131}, + {'\ue83b', 0xFE76}, {'\ue843', 0x82359132}, {'\ue854', 0x82359133}, + {'\ue855', 0xFE91}, {'\ue864', 0x82359134}, {'\ufe10', 0xA6D9}, + {'\ufe11', 0xA6DB}, {'\ufe12', 0xA6DA}, {'\ufe13', 0xA6DC}, + {'\ufe14', 0xA6DD}, {'\ufe15', 0xA6DE}, {'\ufe16', 0xA6DF}, + {'\ufe17', 0xA6EC}, {'\ufe18', 0xA6ED}, {'\ufe19', 0xA6F3}, + {'\U00020087', 0x95329031}, {'\U00020089', 0x95329033}, {'\U000200cc', 0x95329730}, + {'\U000215d7', 0x9536B937}, {'\U0002298f', 0x9630BA35}, {'\U000241fe', 0x9635B630}, + } + for _, v := range gb18030EncodingList { + unicodeToGB18030[v.unicode] = v.gb18030 + gb18030ToUnicode[v.gb18030] = v.unicode + } +} + +var ( + unicodeToGB18030 = map[rune]uint32{} + gb18030ToUnicode = map[uint32]rune{} + // GB18030Case follows the cases from MySQL + GB18030Case = unicode.SpecialCase{ + unicode.CaseRange{Lo: 0xB5, Hi: 0xB5, Delta: [unicode.MaxCase]rune{0, 775, 0}}, + unicode.CaseRange{Lo: 0x1C5, Hi: 0x1C5, Delta: [unicode.MaxCase]rune{0, 1, 0}}, + unicode.CaseRange{Lo: 0x1C8, Hi: 0x1C8, Delta: [unicode.MaxCase]rune{0, 1, 0}}, + unicode.CaseRange{Lo: 0x1CB, Hi: 0x1CB, Delta: [unicode.MaxCase]rune{0, 1, 0}}, + unicode.CaseRange{Lo: 0x1F2, Hi: 0x1F2, Delta: [unicode.MaxCase]rune{0, 1, 0}}, + unicode.CaseRange{Lo: 0x25C, Hi: 0x25C, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x261, Hi: 0x261, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x265, Hi: 0x266, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x26A, Hi: 0x26A, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x26C, Hi: 0x26C, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x282, Hi: 0x282, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x287, Hi: 0x287, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x29D, Hi: 0x29E, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x37F, Hi: 0x37F, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x3C2, Hi: 0x3C2, Delta: [unicode.MaxCase]rune{0, 1, 0}}, + unicode.CaseRange{Lo: 0x3D0, Hi: 0x3D0, Delta: [unicode.MaxCase]rune{0, -30, 0}}, + unicode.CaseRange{Lo: 0x3D1, Hi: 0x3D1, Delta: [unicode.MaxCase]rune{0, -25, 0}}, + unicode.CaseRange{Lo: 0x3D5, Hi: 0x3D5, Delta: [unicode.MaxCase]rune{0, -15, 0}}, + unicode.CaseRange{Lo: 0x3D6, Hi: 0x3D6, Delta: [unicode.MaxCase]rune{0, -22, 0}}, + unicode.CaseRange{Lo: 0x3F0, Hi: 0x3F0, Delta: [unicode.MaxCase]rune{0, -54, 0}}, + unicode.CaseRange{Lo: 0x3F1, Hi: 0x3F1, Delta: [unicode.MaxCase]rune{0, -48, 0}}, + unicode.CaseRange{Lo: 0x3F3, Hi: 0x3F3, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x3F5, Hi: 0x3F5, Delta: [unicode.MaxCase]rune{0, -64, 0}}, + unicode.CaseRange{Lo: 0x526, Hi: 0x52F, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x10C7, Hi: 0x10C7, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x10CD, Hi: 0x10CD, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x10D0, Hi: 0x10FA, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x10FD, Hi: 0x10FF, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x13A0, Hi: 0x13F5, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x13F8, Hi: 0x13FD, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x1C80, Hi: 0x1C88, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x1C90, Hi: 0x1CBA, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x1CBD, Hi: 0x1CBF, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x1D79, Hi: 0x1D79, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x1D7D, Hi: 0x1D7D, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x1D8E, Hi: 0x1D8E, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x1E9B, Hi: 0x1E9B, Delta: [unicode.MaxCase]rune{0, -58, 0}}, + unicode.CaseRange{Lo: 0x1FBE, Hi: 0x1FBE, Delta: [unicode.MaxCase]rune{0, -7173, 0}}, + unicode.CaseRange{Lo: 0x2CF2, Hi: 0x2CF3, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x2D27, Hi: 0x2D27, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x2D2D, Hi: 0x2D2D, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0xA660, Hi: 0xA661, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0xA698, Hi: 0xA69B, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0xA78D, Hi: 0xA78D, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0xA790, Hi: 0xA794, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0xA796, Hi: 0xA7AE, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0xA7B0, Hi: 0xA7BF, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0xA7C2, Hi: 0xA7CA, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0xA7F5, Hi: 0xA7F6, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0xAB53, Hi: 0xAB53, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0xAB70, Hi: 0xABBF, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x104B0, Hi: 0x104D3, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x104D8, Hi: 0x104FB, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x10C80, Hi: 0x10CB2, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x10CC0, Hi: 0x10CF2, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x118A0, Hi: 0x118DF, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x16E40, Hi: 0x16E7F, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + unicode.CaseRange{Lo: 0x1E900, Hi: 0x1E943, Delta: [unicode.MaxCase]rune{0, 0, 0}}, + } +) diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/format/format.go b/vendor/github.com/pingcap/tidb/pkg/parser/format/format.go index 68a0269e4..a3c054f91 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/format/format.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/format/format.go @@ -21,6 +21,7 @@ import ( "bytes" "fmt" "io" + "slices" "strings" ) @@ -34,7 +35,7 @@ const ( // Formatter is an io.Writer extended formatter by a fmt.Printf like function Format. type Formatter interface { io.Writer - Format(format string, args ...interface{}) (n int, errno error) + Format(format string, args ...any) (n int, errno error) } type indentFormatter struct { @@ -82,9 +83,9 @@ func IndentFormatter(w io.Writer, indent string) Formatter { return &indentFormatter{w, []byte(indent), 0, stBOL} } -func (f *indentFormatter) format(flat bool, format string, args ...interface{}) (n int, errno error) { +func (f *indentFormatter) format(flat bool, format string, args ...any) (n int, errno error) { var buf = make([]byte, 0) - for i := 0; i < len(format); i++ { + for i := range len(format) { c := format[i] switch f.state { case st0: @@ -113,7 +114,7 @@ func (f *indentFormatter) format(flat bool, format string, args ...interface{}) f.state = stBOLPERC default: if !flat { - for i := 0; i < f.indentLevel; i++ { + for range f.indentLevel { buf = append(buf, f.indent...) } } @@ -130,7 +131,7 @@ func (f *indentFormatter) format(flat bool, format string, args ...interface{}) f.state = stBOL default: if !flat { - for i := 0; i < f.indentLevel; i++ { + for range f.indentLevel { buf = append(buf, f.indent...) } } @@ -161,7 +162,7 @@ func (f *indentFormatter) format(flat bool, format string, args ...interface{}) } // Format implements Format interface. -func (f *indentFormatter) Format(format string, args ...interface{}) (n int, errno error) { +func (f *indentFormatter) Format(format string, args ...any) (n int, errno error) { return f.format(false, format, args...) } @@ -187,7 +188,7 @@ func FlatFormatter(w io.Writer) Formatter { } // Format implements Format interface. -func (f *flatFormatter) Format(format string, args ...interface{}) (n int, errno error) { +func (f *flatFormatter) Format(format string, args ...any) (n int, errno error) { return (*indentFormatter)(f).format(true, format, args...) } @@ -239,6 +240,8 @@ const ( RestoreWithoutSchemaName RestoreWithoutTableName RestoreForNonPrepPlanCache + + RestoreBracketAroundBetweenExpr ) const ( @@ -323,6 +326,12 @@ func (rf RestoreFlags) HasStringWithoutDefaultCharset() bool { return rf.has(RestoreStringWithoutDefaultCharset) } +// HasRestoreBracketAroundBetweenExpr returns a boolean indicating +// whether `rf` has `RestoreBracketAroundBetweenExpr` flag. +func (rf RestoreFlags) HasRestoreBracketAroundBetweenExpr() bool { + return rf.has(RestoreBracketAroundBetweenExpr) +} + // HasStringWithoutCharset returns a boolean indicating whether `rf` has `RestoreStringWithoutCharset` flag. func (rf RestoreFlags) HasStringWithoutCharset() bool { return rf.has(RestoreStringWithoutCharset) @@ -397,6 +406,14 @@ func (ctx *RestoreCtx) WriteWithSpecialComments(featureID string, fn func() erro return nil } +// WriteKeyWordWithSpecialComments writes a keyword with a special comment wrapped. +func (ctx *RestoreCtx) WriteKeyWordWithSpecialComments(featureID string, keyWord string) { + _ = ctx.WriteWithSpecialComments(featureID, func() error { + ctx.WriteKeyWord(keyWord) + return nil + }) +} + // WriteString writes the string into writer // `str` may be wrapped in quotes and escaped according to RestoreFlags. func (ctx *RestoreCtx) WriteString(str string) { @@ -448,7 +465,7 @@ func (ctx *RestoreCtx) WritePlain(plainText string) { } // WritePlainf write the plain text into writer without any handling. -func (ctx *RestoreCtx) WritePlainf(format string, a ...interface{}) { +func (ctx *RestoreCtx) WritePlainf(format string, a ...any) { fmt.Fprintf(ctx.In, format, a...) } @@ -459,12 +476,7 @@ type CTERestorer struct { // IsCTETableName returns true if the given tableName comes from CTE. func (c *CTERestorer) IsCTETableName(nameL string) bool { - for _, n := range c.CTENames { - if n == nameL { - return true - } - } - return false + return slices.Contains(c.CTENames, nameL) } // RecordCTEName records the CTE name. diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/BUILD.bazel b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/BUILD.bazel index 7a1a3b80c..d792dc88c 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/BUILD.bazel +++ b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/BUILD.bazel @@ -18,6 +18,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/parser/format", + "@com_github_coreos_go_semver//semver", "@com_github_pingcap_errors//:errors", ], ) @@ -33,6 +34,6 @@ go_test( ], embed = [":mysql"], flaky = True, - shard_count = 9, + shard_count = 11, deps = ["@com_github_stretchr_testify//require"], ) diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/charset.go b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/charset.go index 6e37201e3..1e98f5b9f 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/charset.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/charset.go @@ -75,6 +75,7 @@ var CharsetIDs = map[string]uint8{ "geostd8": 92, "cp932": 95, "eucjpms": 97, + "gb18030": GB18030DefaultCollationID, } // Collations maps MySQL collation ID to its name. @@ -298,6 +299,8 @@ var Collations = map[uint16]string{ 245: "utf8mb4_croatian_ci", 246: "utf8mb4_unicode_520_ci", 247: "utf8mb4_vietnamese_ci", + 248: "gb18030_chinese_ci", + 249: "gb18030_bin", 255: "utf8mb4_0900_ai_ci", 309: "utf8mb4_0900_bin", } @@ -523,6 +526,8 @@ var CollationNames = map[string]uint16{ "utf8mb4_croatian_ci": 245, "utf8mb4_unicode_520_ci": 246, "utf8mb4_vietnamese_ci": 247, + "gb18030_chinese_ci": 248, + "gb18030_bin": 249, "utf8mb4_0900_ai_ci": 255, "utf8mb4_0900_bin": 309, } @@ -539,6 +544,7 @@ const ( UTF8DefaultCollationID = 83 UTF8MB4DefaultCollationID = 46 BinaryDefaultCollationID = 63 + GB18030DefaultCollationID = 248 UTF8MB4DefaultCollation = "utf8mb4_bin" DefaultCollationName = UTF8MB4DefaultCollation UTF8MB4GeneralCICollation = "utf8mb4_general_ci" diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/const.go b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/const.go index 591c08906..e418125c0 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/const.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/const.go @@ -17,6 +17,7 @@ import ( "fmt" "strings" + "github.com/coreos/go-semver/semver" "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/parser/format" ) @@ -32,17 +33,77 @@ const ( // one with MySQL compatibility version, with this fixed then we can parse TiDB // version from ServerVersion. VersionSeparator = "-TiDB-" + + // tidbXReleaseVersionPrefix is used in `select tidb_version()` output of nextgen. + tidbXReleaseVersionPrefix = "CLOUD." + + legacyTiDBReleaseVersionPlaceholder = "v8.4.0-this-is-a-placeholder" + // tidbXPlaceholderReleaseVersion is the default release version for nextgen when no + // release version is injected during build, such as when running in IDE. + tidbXPlaceholderReleaseVersion = "v26.3.0-this-is-a-placeholder" + // TiDBXVerMinYear is set to 2025 just for sanity check. + // our first release of next-gen since 2025 + TiDBXVerMinYear = 2025 + // TiDBXVerMaxYear is set to 2099 just for sanity check, we don't expect the + // year part of release version to be larger than this. + // enough for now. + TiDBXVerMaxYear = 2099 ) // Version information. var ( // TiDBReleaseVersion is initialized by (git describe --tags) in Makefile. - TiDBReleaseVersion = "v8.4.0-this-is-a-placeholder" + TiDBReleaseVersion = legacyTiDBReleaseVersionPlaceholder // ServerVersion is the version information of this tidb-server in MySQL's format. ServerVersion = fmt.Sprintf("%s%s%s", mysqlCompatibilityVersion, VersionSeparator, TiDBReleaseVersion) ) +// NormalizeTiDBReleaseVersionForNextGen rewrites the legacy placeholder into a nextgen +// placeholder that follows `v[2-digit-year].[month].[fix-version]`. +// pkg/parser is Golang project, it cannot use kerneltype pkg to conditionally +// compile different code for next-gen and classic, so we have to rewrite the +// placeholder value in this function. +func NormalizeTiDBReleaseVersionForNextGen(releaseVersion string) string { + // the version is not set if we run next-gen tidb from IDE. + if releaseVersion == legacyTiDBReleaseVersionPlaceholder { + return tidbXPlaceholderReleaseVersion + } + return releaseVersion +} + +// BuildTiDBXReleaseVersion converts mysql.TiDBReleaseVersion into the nextgen visible +// version format `CLOUD.<4-digit-year-2-digit-month>.`. +func BuildTiDBXReleaseVersion(releaseVersion string) (string, error) { + if !strings.HasPrefix(releaseVersion, "v") { + return "", errors.Errorf("invalid TiDB release version %q, should start with 'v'", releaseVersion) + } + rawVer := strings.TrimPrefix(releaseVersion, "v") + ver, err := semver.NewVersion(rawVer) + if err != nil { + return "", errors.Errorf("invalid TiDB release version %q, expect a semantic version", releaseVersion) + } + year := 2000 + ver.Major + if year < TiDBXVerMinYear || year > TiDBXVerMaxYear || ver.Minor < 1 || ver.Minor > 12 { + return "", errors.Errorf("invalid TiDB release version %q, the semantic version part should be in [2-digit-year].[month].[fix-version]-[xxx] format", releaseVersion) + } + preRelease := string(ver.PreRelease) + if preRelease != "" { + preRelease = "-" + preRelease + } + return fmt.Sprintf("%s%d%02d.%d%s", tidbXReleaseVersionPrefix, year, ver.Minor, ver.Patch, preRelease), nil +} + +// BuildTiDBXServerVersion converts mysql.TiDBReleaseVersion into MySQL server version +// format `8.0.11-TiDB-CLOUD.<4-digit-year-2-digit-month>.`. +func BuildTiDBXServerVersion(releaseVersion string) (string, error) { + tidbXReleaseVersion, err := BuildTiDBXReleaseVersion(releaseVersion) + if err != nil { + return "", err + } + return fmt.Sprintf("%s%s%s", mysqlCompatibilityVersion, VersionSeparator, tidbXReleaseVersion), nil +} + // Header information. const ( OKHeader byte = 0x00 @@ -196,7 +257,7 @@ const ( AuthLDAPSASL = "authentication_ldap_sasl" ) -// MySQL database and tables. +// System database and tables that mostly inherited from MySQL. const ( // SystemDB is the name of system database. SystemDB = "mysql" @@ -225,6 +286,8 @@ const ( DefaultRoleTable = "default_roles" // PasswordHistoryTable is the table in system db contains password history. PasswordHistoryTable = "password_history" + // WorkloadSchema is the name of workload repository database. + WorkloadSchema = "workload_schema" ) // MySQL type maximum length. @@ -578,24 +641,6 @@ var CombinationSQLMode = map[string][]string{ "TRADITIONAL": {"STRICT_TRANS_TABLES", "STRICT_ALL_TABLES", "NO_ZERO_IN_DATE", "NO_ZERO_DATE", "ERROR_FOR_DIVISION_BY_ZERO", "NO_AUTO_CREATE_USER", "NO_ENGINE_SUBSTITUTION"}, } -// FormatFunc is the locale format function signature. -type FormatFunc func(string, string) (string, error) - -// GetLocaleFormatFunction get the format function for sepcific locale. -func GetLocaleFormatFunction(loc string) FormatFunc { - locale, exist := locale2FormatFunction[loc] - if !exist { - return formatNotSupport - } - return locale -} - -// locale2FormatFunction is the string represent of locale format function. -var locale2FormatFunction = map[string]FormatFunc{ - "en_US": formatENUS, - "zh_CN": formatZHCN, -} - // PriorityEnum is defined for Priority const values. type PriorityEnum int diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/errcode.go b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/errcode.go index 05d5dc3e6..bfbd19613 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/errcode.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/errcode.go @@ -898,6 +898,7 @@ const ( ErrInvalidJSONText = 3140 ErrInvalidJSONTextInParam = 3141 ErrInvalidJSONPath = 3143 + ErrInvalidJSONCharset = 3144 ErrInvalidTypeForJSON = 3146 ErrInvalidJSONPathWildcard = 3149 ErrInvalidJSONContainsPathType = 3150 diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/errname.go b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/errname.go index f757f825d..845c50a21 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/errname.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/errname.go @@ -907,6 +907,7 @@ var MySQLErrName = map[uint16]*ErrMessage{ ErrInvalidJSONText: Message("Invalid JSON text: %-.192s", nil), ErrInvalidJSONTextInParam: Message("Invalid JSON text in argument %d to function %s: \"%s\" at position %d.", nil), ErrInvalidJSONPath: Message("Invalid JSON path expression %s.", nil), + ErrInvalidJSONCharset: Message("Cannot create a JSON value from a string with CHARACTER SET '%s'.", nil), ErrInvalidTypeForJSON: Message("Invalid data type for JSON data in argument %d to function %s; a JSON string or JSON type is required.", nil), ErrInvalidJSONPathWildcard: Message("In this situation, path expressions may not contain the * and ** tokens or an array range.", nil), ErrInvalidJSONContainsPathType: Message("The second argument can only be either 'one' or 'all'.", nil), diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/error.go b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/error.go index 4d58d9fe2..f3b3a838e 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/error.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/error.go @@ -38,7 +38,7 @@ func (e *SQLError) Error() string { } // NewErr generates a SQL error, with an error code and default format specifier defined in MySQLErrName. -func NewErr(errCode uint16, args ...interface{}) *SQLError { +func NewErr(errCode uint16, args ...any) *SQLError { e := &SQLError{Code: errCode} if s, ok := MySQLState[errCode]; ok { @@ -58,7 +58,7 @@ func NewErr(errCode uint16, args ...interface{}) *SQLError { } // NewErrf creates a SQL error, with an error code and a format specifier. -func NewErrf(errCode uint16, format string, redactArgPos []int, args ...interface{}) *SQLError { +func NewErrf(errCode uint16, format string, redactArgPos []int, args ...any) *SQLError { e := &SQLError{Code: errCode} if s, ok := MySQLState[errCode]; ok { diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/locale_format.go b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/locale_format.go index 9d92c2a46..674b29739 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/locale_format.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/locale_format.go @@ -5,11 +5,195 @@ import ( "strconv" "strings" "unicode" +) + +// LocaleFormatStyle defines the rules for number formatting. +type LocaleFormatStyle struct { + ThousandsSep string // Thousands separator + DecimalPoint string // Decimal point + IsIndianGrouping bool // Special grouping for en_IN, etc. (3,2,2,...) +} - "github.com/pingcap/errors" +// Formatting style IDs (descriptive names) +const ( + styleCommaDot = "CommaDot" // 123,456.78 (en_US) + styleDotComma = "DotComma" // 123.456,78 (de_DE) + styleSpaceComma = "SpaceComma" // 123 456,78 (fr_FR) + styleNoneComma = "NoneComma" // 123456,78 (bg_BG) + styleAposDot = "AposDot" // 123'456.78 (de_CH) + styleAposComma = "AposComma" // 123'456,78 (it_CH) + styleNoneDot = "NoneDot" // 123456.78 (ar_SA) + styleIndian = "Indian" // 1,23,45,67,890.123 (en_IN) ) -func formatENUS(number string, precision string) (string, error) { +// formatStyleMap maps a style ID to its separator definitions. +var formatStyleMap = map[string]LocaleFormatStyle{ + // IsIndianGrouping is false by default + styleCommaDot: {ThousandsSep: ",", DecimalPoint: "."}, + styleDotComma: {ThousandsSep: ".", DecimalPoint: ","}, + styleSpaceComma: {ThousandsSep: " ", DecimalPoint: ","}, + styleNoneComma: {ThousandsSep: "", DecimalPoint: ","}, + styleAposDot: {ThousandsSep: "'", DecimalPoint: "."}, + styleAposComma: {ThousandsSep: "'", DecimalPoint: ","}, + styleNoneDot: {ThousandsSep: "", DecimalPoint: "."}, + styleIndian: {ThousandsSep: ",", DecimalPoint: ".", IsIndianGrouping: true}, +} + +// localeToStyleMap maps locale names (lowercase) to their corresponding format style ID. +var localeToStyleMap = map[string]string{ + // styleCommaDot (123,456.78): Default/fallback format (e.g., en_US) or where MySQL behavior matches. + "aa_et": styleCommaDot, "af_za": styleCommaDot, "ak_gh": styleCommaDot, "am_et": styleCommaDot, "ar_ae": styleCommaDot, "ar_bh": styleCommaDot, + "ar_dz": styleCommaDot, "ar_eg": styleCommaDot, "ar_in": styleCommaDot, "ar_iq": styleCommaDot, "ar_jo": styleCommaDot, "ar_kw": styleCommaDot, + "ar_lb": styleCommaDot, "ar_ly": styleCommaDot, "ar_ma": styleCommaDot, "ar_om": styleCommaDot, "ar_qa": styleCommaDot, "ar_sd": styleCommaDot, + "ar_ss": styleCommaDot, "ar_sy": styleCommaDot, "ar_tn": styleCommaDot, "ar_ye": styleCommaDot, "az_ir": styleCommaDot, "bi_vu": styleCommaDot, + "bo_cn": styleCommaDot, "bo_in": styleCommaDot, "cy_gb": styleCommaDot, "dv_mv": styleCommaDot, "en_ag": styleCommaDot, "en_au": styleCommaDot, + "en_bw": styleCommaDot, "en_ca": styleCommaDot, "en_gb": styleCommaDot, "en_hk": styleCommaDot, "en_ie": styleCommaDot, "en_il": styleCommaDot, + "en_ng": styleCommaDot, "en_nz": styleCommaDot, "en_ph": styleCommaDot, "en_sg": styleCommaDot, "en_us": styleCommaDot, "en_za": styleCommaDot, + "en_zm": styleCommaDot, "en_zw": styleCommaDot, "es_do": styleCommaDot, "es_gt": styleCommaDot, "es_hn": styleCommaDot, "es_ni": styleCommaDot, + "es_pa": styleCommaDot, "es_pr": styleCommaDot, "es_sv": styleCommaDot, "es_us": styleCommaDot, "fa_ir": styleCommaDot, "ga_ie": styleCommaDot, + "gd_gb": styleCommaDot, "gu_in": styleCommaDot, "gv_gb": styleCommaDot, "ha_ng": styleCommaDot, "he_il": styleCommaDot, "hi_in": styleCommaDot, + "hy_am": styleCommaDot, "ig_ng": styleCommaDot, "ik_ca": styleCommaDot, "iu_ca": styleCommaDot, "ja_jp": styleCommaDot, "km_kh": styleCommaDot, + "kn_in": styleCommaDot, "ko_kr": styleCommaDot, "ks_in": styleCommaDot, "kw_gb": styleCommaDot, "lg_ug": styleCommaDot, "lo_la": styleCommaDot, + "mi_nz": styleCommaDot, "mr_in": styleCommaDot, "ms_my": styleCommaDot, "mt_mt": styleCommaDot, "my_mm": styleCommaDot, "ne_np": styleCommaDot, + "nr_za": styleCommaDot, "om_et": styleCommaDot, "om_ke": styleCommaDot, "pa_in": styleCommaDot, "pa_pk": styleCommaDot, "sa_in": styleCommaDot, + "sd_in": styleCommaDot, "si_lk": styleCommaDot, "sm_ws": styleCommaDot, "so_et": styleCommaDot, "so_ke": styleCommaDot, "so_so": styleCommaDot, + "ss_za": styleCommaDot, "st_za": styleCommaDot, "sw_ke": styleCommaDot, "sw_tz": styleCommaDot, "th_th": styleCommaDot, "ti_et": styleCommaDot, + "tk_tm": styleCommaDot, "tl_ph": styleCommaDot, "tn_za": styleCommaDot, "to_to": styleCommaDot, "ts_za": styleCommaDot, "ug_cn": styleCommaDot, + "ur_in": styleCommaDot, "ur_pk": styleCommaDot, "ve_za": styleCommaDot, "xh_za": styleCommaDot, "yi_us": styleCommaDot, "yo_ng": styleCommaDot, + "zh_cn": styleCommaDot, "zh_hk": styleCommaDot, "zh_sg": styleCommaDot, "zh_tw": styleCommaDot, "zu_za": styleCommaDot, + "an_es": styleCommaDot, "az_az": styleCommaDot, "ca_ad": styleCommaDot, "ca_fr": styleCommaDot, "ca_it": styleCommaDot, "de_it": styleCommaDot, + "en_dk": styleCommaDot, "es_pe": styleCommaDot, "ff_sn": styleCommaDot, "fy_de": styleCommaDot, "fy_nl": styleCommaDot, "ka_ge": styleCommaDot, + "kl_gl": styleCommaDot, "ku_tr": styleCommaDot, "lb_lu": styleCommaDot, "li_be": styleCommaDot, "li_nl": styleCommaDot, "nl_aw": styleCommaDot, + "sc_it": styleCommaDot, "se_no": styleCommaDot, "sq_mk": styleCommaDot, "tg_tj": styleCommaDot, "tr_cy": styleCommaDot, "wa_be": styleCommaDot, + "br_fr": styleCommaDot, "kk_kz": styleCommaDot, "nn_no": styleCommaDot, "oc_fr": styleCommaDot, "uz_uz": styleCommaDot, + "bs_ba": styleCommaDot, "el_cy": styleCommaDot, "es_cu": styleCommaDot, "ln_cd": styleCommaDot, "mg_mg": styleCommaDot, "rw_rw": styleCommaDot, "sr_me": styleCommaDot, "wo_sn": styleCommaDot, + "es_mx": styleCommaDot, + "ce_ru": styleCommaDot, "cv_ru": styleCommaDot, "ht_ht": styleCommaDot, "ia_fr": styleCommaDot, "ky_kg": styleCommaDot, "os_ru": styleCommaDot, "tt_ru": styleCommaDot, + "aa_dj": styleCommaDot, "aa_er": styleCommaDot, "so_dj": styleCommaDot, "ti_er": styleCommaDot, + "ps_af": styleCommaDot, + "kv_ru": styleCommaDot, + "su_id": styleCommaDot, + + // styleDotComma (123.456,78): Common in Europe and South America. + "be_by": styleDotComma, "da_dk": styleDotComma, "de_be": styleDotComma, "de_de": styleDotComma, "de_lu": styleDotComma, + "es_ar": styleDotComma, "es_bo": styleDotComma, "es_cl": styleDotComma, "es_co": styleDotComma, + "es_ec": styleDotComma, "es_es": styleDotComma, "es_py": styleDotComma, "es_uy": styleDotComma, "es_ve": styleDotComma, + "fo_fo": styleDotComma, "hu_hu": styleDotComma, "id_id": styleDotComma, "is_is": styleDotComma, + "lt_lt": styleDotComma, "mn_mn": styleDotComma, "ro_ro": styleDotComma, "ru_ua": styleDotComma, "sq_al": styleDotComma, + "tr_tr": styleDotComma, "vi_vn": styleDotComma, + "nb_no": styleDotComma, "uk_ua": styleDotComma, + "no_no": styleDotComma, + + // styleSpaceComma (123 456,78): Uses space as thousands separator. + "cs_cz": styleSpaceComma, "es_cr": styleSpaceComma, "et_ee": styleSpaceComma, "fi_fi": styleSpaceComma, + "lv_lv": styleSpaceComma, "mk_mk": styleSpaceComma, + "ru_ru": styleSpaceComma, "sk_sk": styleSpaceComma, "sv_fi": styleSpaceComma, "sv_se": styleSpaceComma, + + // styleNoneComma (123456,78): No thousands separator. + "el_gr": styleNoneComma, "gl_es": styleNoneComma, "pt_pt": styleNoneComma, "sl_si": styleNoneComma, + "ca_es": styleNoneComma, "de_at": styleNoneComma, "eu_es": styleNoneComma, "fr_be": styleNoneComma, "hr_hr": styleNoneComma, "it_it": styleNoneComma, "nl_be": styleNoneComma, "nl_nl": styleNoneComma, "pt_br": styleNoneComma, + "fr_ca": styleNoneComma, "fr_fr": styleNoneComma, "fr_lu": styleNoneComma, "pl_pl": styleNoneComma, + "fr_ch": styleNoneComma, + "bg_bg": styleNoneComma, + + // styleAposDot (123'456.78): Uses apostrophe as thousands separator. + "de_ch": styleAposDot, + + // styleAposComma (123'456,78): Uses apostrophe separator, comma decimal. + "it_ch": styleAposComma, + + // styleNoneDot (123456.78): No thousands separator, dot decimal. + "ar_sa": styleNoneDot, + "sr_rs": styleNoneDot, + + // styleIndian (1,23,45,67,890.123): Special Indian grouping (3,2,2,...). + "en_in": styleIndian, + "ta_in": styleIndian, + "te_in": styleIndian, +} + +// GetLocaleFormatStyle returns the formatting rules and a bool indicating if the locale was found. +func GetLocaleFormatStyle(locale string) (LocaleFormatStyle, bool) { + styleID, ok := localeToStyleMap[strings.ToLower(locale)] + if !ok { + // Not found. Return default style, but also return 'false'. + return formatStyleMap[styleCommaDot], false + } + // Found. Return style and 'true'. + return formatStyleMap[styleID], true +} + +// FormatByLocale returns (string, bool, error) +// The bool (found) is true if the locale was found in the map, false otherwise. +func FormatByLocale(number, precision, locale string) (string, bool, error) { + // 'locale' is guaranteed to be non-empty by the caller (builtin_string.go). + style, found := GetLocaleFormatStyle(locale) + // if not found, style is set to default (en_US) + formattedString, err := formatWithStyle(number, precision, style) + return formattedString, found, err +} + +// formatWithStandardGrouping applies standard 3-digit grouping (e.g., 1,234,567) +func formatWithStandardGrouping(integerPart string, thousandsSep string) string { + var buffer bytes.Buffer + partLen := len(integerPart) + // Find position of first separator + pos := partLen % 3 + if pos == 0 && partLen > 0 { + pos = 3 + } + + // Write the first group (1-3 digits) + buffer.WriteString(integerPart[:pos]) + + // Write subsequent 3-digit groups + for ; pos < partLen; pos += 3 { + buffer.WriteString(thousandsSep) + buffer.WriteString(integerPart[pos : pos+3]) + } + return buffer.String() +} + +// formatWithIndianGrouping applies Indian grouping (e.g., 1,23,45,67,890) +func formatWithIndianGrouping(integerPart string, thousandsSep string) string { + var buffer bytes.Buffer + s := integerPart + l := len(s) + if l <= 3 { + return s // No grouping needed + } + + // Get the rightmost 3 digits (e.g., 890) + rightmost3 := s[l-3:] + // Get the remaining digits on the left (e.g., 1234567) + remaining := s[:l-3] + remLen := len(remaining) + + // Get the first part (1 or 2 digits) (e.g., 1) + firstPartLen := remLen % 2 + if firstPartLen == 0 && remLen > 0 { + firstPartLen = 2 + } + + if firstPartLen > 0 { + buffer.WriteString(remaining[:firstPartLen]) + } + + // Loop through the remaining 2-digit groups (e.g., 23, 45, 67) + for pos := firstPartLen; pos < remLen; pos += 2 { + buffer.WriteString(thousandsSep) + buffer.WriteString(remaining[pos : pos+2]) + } + + // Add the last separator and the rightmost 3 digits + buffer.WriteString(thousandsSep) + buffer.WriteString(rightmost3) + + return buffer.String() +} + +// formatWithStyle is the generic formatting function. +func formatWithStyle(number string, precision string, style LocaleFormatStyle) (string, error) { var buffer bytes.Buffer if unicode.IsDigit(rune(precision[0])) { for i, v := range precision { @@ -30,15 +214,16 @@ func formatENUS(number string, precision string) (string, error) { if (number[:1] == "-" && !unicode.IsDigit(rune(number[1]))) || (!unicode.IsDigit(rune(number[0])) && number[:1] != "-") { - buffer.Write([]byte{'0'}) + buffer.WriteString("0") position, err := strconv.ParseUint(precision, 10, 64) if err == nil && position > 0 { - buffer.Write([]byte{'.'}) + // Use style-defined decimal point + buffer.WriteString(style.DecimalPoint) buffer.WriteString(strings.Repeat("0", int(position))) } return buffer.String(), nil } else if number[:1] == "-" { - buffer.Write([]byte{'-'}) + buffer.WriteString("-") number = number[1:] } @@ -54,24 +239,27 @@ func formatENUS(number string, precision string) (string, error) { break } - comma := []byte{','} parts := strings.Split(number, ".") - pos := 0 - if len(parts[0])%3 != 0 { - pos += len(parts[0]) % 3 - buffer.WriteString(parts[0][:pos]) - buffer.Write(comma) - } - for ; pos < len(parts[0]); pos += 3 { - buffer.WriteString(parts[0][pos : pos+3]) - buffer.Write(comma) + integerPart := parts[0] + var formattedIntegerPart string + + // Apply grouping logic based on the locale style. + if len(style.ThousandsSep) == 0 { + // No separator (e.g., styleNoneComma), just use the integer part + formattedIntegerPart = integerPart + } else if style.IsIndianGrouping { + // Use 3,2,2... grouping for Indian locales. + formattedIntegerPart = formatWithIndianGrouping(integerPart, style.ThousandsSep) + } else { + // Use standard 3-digit grouping. + formattedIntegerPart = formatWithStandardGrouping(integerPart, style.ThousandsSep) } - buffer.Truncate(buffer.Len() - 1) + buffer.WriteString(formattedIntegerPart) position, err := strconv.ParseUint(precision, 10, 64) if err == nil { if position > 0 { - buffer.Write([]byte{'.'}) + buffer.WriteString(style.DecimalPoint) // Use style-defined decimal point if len(parts) == 2 { if uint64(len(parts[1])) >= position { buffer.WriteString(parts[1][:position]) @@ -87,11 +275,3 @@ func formatENUS(number string, precision string) (string, error) { return buffer.String(), nil } - -func formatZHCN(_ string, _ string) (string, error) { - return "", errors.New("not implemented") -} - -func formatNotSupport(_ string, _ string) (string, error) { - return "", errors.New("not support for the specific locale") -} diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/privs.go b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/privs.go index 060dcc2d0..c384a1ccd 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/privs.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/privs.go @@ -13,6 +13,8 @@ package mysql +import "slices" + // AllPrivilegeLiteral is the string literal for All Privilege. const AllPrivilegeLiteral = "ALL PRIVILEGES" @@ -257,7 +259,7 @@ const ( CreateRolePriv // DropRolePriv is the privilege to drop a role. DropRolePriv - // CreateTMPTablePriv is the privilege to create a temporary table. + // CreateTMPTablePriv is the privilege to create a local temporary table. CreateTMPTablePriv // LockTablesPriv is the privilege to lock tables. LockTablesPriv @@ -305,12 +307,7 @@ type Privileges []PrivilegeType // Has checks whether PrivilegeType has the privilege. func (privs Privileges) Has(p PrivilegeType) bool { - for _, cp := range privs { - if cp == p { - return true - } - } - return false + return slices.Contains(privs, p) } // AllGlobalPrivs is all the privileges in global scope. diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/state.go b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/state.go index 2cbc6f1d2..307965d08 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/state.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/state.go @@ -254,6 +254,7 @@ var MySQLState = map[uint16]string{ ErrInvalidJSONText: "22032", ErrInvalidJSONTextInParam: "22032", ErrInvalidJSONPath: "42000", + ErrInvalidJSONCharset: "22032", ErrInvalidJSONData: "22032", ErrInvalidJSONPathWildcard: "42000", ErrJSONUsedAsKey: "42000", diff --git a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/type.go b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/type.go index 473641770..6f1ace05d 100644 --- a/vendor/github.com/pingcap/tidb/pkg/parser/mysql/type.go +++ b/vendor/github.com/pingcap/tidb/pkg/parser/mysql/type.go @@ -166,3 +166,8 @@ func HasPreventNullInsertFlag(flag uint) bool { func HasEnumSetAsIntFlag(flag uint) bool { return (flag & EnumSetAsIntFlag) > 0 } + +// HasFlag checks if a flag is set. +func HasFlag(flag uint, flagItem uint) bool { + return (flag & flagItem) > 0 +} diff --git a/vendor/github.com/shopspring/decimal/.gitignore b/vendor/github.com/shopspring/decimal/.gitignore index 8a43ce9d7..ff36b987f 100644 --- a/vendor/github.com/shopspring/decimal/.gitignore +++ b/vendor/github.com/shopspring/decimal/.gitignore @@ -4,3 +4,6 @@ # IntelliJ .idea/ *.iml + +# VS code +*.code-workspace diff --git a/vendor/github.com/shopspring/decimal/.travis.yml b/vendor/github.com/shopspring/decimal/.travis.yml deleted file mode 100644 index 55d42b289..000000000 --- a/vendor/github.com/shopspring/decimal/.travis.yml +++ /dev/null @@ -1,13 +0,0 @@ -language: go - -go: - - 1.7.x - - 1.12.x - - 1.13.x - - tip - -install: - - go build . - -script: - - go test -v diff --git a/vendor/github.com/shopspring/decimal/CHANGELOG.md b/vendor/github.com/shopspring/decimal/CHANGELOG.md index 01ba02feb..432d0fd4e 100644 --- a/vendor/github.com/shopspring/decimal/CHANGELOG.md +++ b/vendor/github.com/shopspring/decimal/CHANGELOG.md @@ -1,4 +1,61 @@ -## Decimal v1.2.0 +## Decimal v1.4.0 +#### BREAKING +- Drop support for Go version older than 1.10 [#361](https://github.com/shopspring/decimal/pull/361) + +#### FEATURES +- Add implementation of natural logarithm [#339](https://github.com/shopspring/decimal/pull/339) [#357](https://github.com/shopspring/decimal/pull/357) +- Add improved implementation of power operation [#358](https://github.com/shopspring/decimal/pull/358) +- Add Compare method which forwards calls to Cmp [#346](https://github.com/shopspring/decimal/pull/346) +- Add NewFromBigRat constructor [#288](https://github.com/shopspring/decimal/pull/288) +- Add NewFromUint64 constructor [#352](https://github.com/shopspring/decimal/pull/352) + +#### ENHANCEMENTS +- Migrate to Github Actions [#245](https://github.com/shopspring/decimal/pull/245) [#340](https://github.com/shopspring/decimal/pull/340) +- Fix examples for RoundDown, RoundFloor, RoundUp, and RoundCeil [#285](https://github.com/shopspring/decimal/pull/285) [#328](https://github.com/shopspring/decimal/pull/328) [#341](https://github.com/shopspring/decimal/pull/341) +- Use Godoc standard to mark deprecated Equals and StringScaled methods [#342](https://github.com/shopspring/decimal/pull/342) +- Removed unnecessary min function for RescalePair method [#265](https://github.com/shopspring/decimal/pull/265) +- Avoid reallocation of initial slice in MarshalBinary (GobEncode) [#355](https://github.com/shopspring/decimal/pull/355) +- Optimize NumDigits method [#301](https://github.com/shopspring/decimal/pull/301) [#356](https://github.com/shopspring/decimal/pull/356) +- Optimize BigInt method [#359](https://github.com/shopspring/decimal/pull/359) +- Support scanning uint64 [#131](https://github.com/shopspring/decimal/pull/131) [#364](https://github.com/shopspring/decimal/pull/364) +- Add docs section with alternative libraries [#363](https://github.com/shopspring/decimal/pull/363) + +#### BUGFIXES +- Fix incorrect calculation of decimal modulo [#258](https://github.com/shopspring/decimal/pull/258) [#317](https://github.com/shopspring/decimal/pull/317) +- Allocate new(big.Int) in Copy method to deeply clone it [#278](https://github.com/shopspring/decimal/pull/278) +- Fix overflow edge case in QuoRem method [#322](https://github.com/shopspring/decimal/pull/322) + +## Decimal v1.3.1 + +#### ENHANCEMENTS +- Reduce memory allocation in case of initialization from big.Int [#252](https://github.com/shopspring/decimal/pull/252) + +#### BUGFIXES +- Fix binary marshalling of decimal zero value [#253](https://github.com/shopspring/decimal/pull/253) + +## Decimal v1.3.0 + +#### FEATURES +- Add NewFromFormattedString initializer [#184](https://github.com/shopspring/decimal/pull/184) +- Add NewNullDecimal initializer [#234](https://github.com/shopspring/decimal/pull/234) +- Add implementation of natural exponent function (Taylor, Hull-Abraham) [#229](https://github.com/shopspring/decimal/pull/229) +- Add RoundUp, RoundDown, RoundCeil, RoundFloor methods [#196](https://github.com/shopspring/decimal/pull/196) [#202](https://github.com/shopspring/decimal/pull/202) [#220](https://github.com/shopspring/decimal/pull/220) +- Add XML support for NullDecimal [#192](https://github.com/shopspring/decimal/pull/192) +- Add IsInteger method [#179](https://github.com/shopspring/decimal/pull/179) +- Add Copy helper method [#123](https://github.com/shopspring/decimal/pull/123) +- Add InexactFloat64 helper method [#205](https://github.com/shopspring/decimal/pull/205) +- Add CoefficientInt64 helper method [#244](https://github.com/shopspring/decimal/pull/244) + +#### ENHANCEMENTS +- Performance optimization of NewFromString init method [#198](https://github.com/shopspring/decimal/pull/198) +- Performance optimization of Abs and Round methods [#240](https://github.com/shopspring/decimal/pull/240) +- Additional tests (CI) for ppc64le architecture [#188](https://github.com/shopspring/decimal/pull/188) + +#### BUGFIXES +- Fix rounding in FormatFloat fallback path (roundShortest method, fix taken from Go main repository) [#161](https://github.com/shopspring/decimal/pull/161) +- Add slice range checks to UnmarshalBinary method [#232](https://github.com/shopspring/decimal/pull/232) + +## Decimal v1.2.0 #### BREAKING - Drop support for Go version older than 1.7 [#172](https://github.com/shopspring/decimal/pull/172) diff --git a/vendor/github.com/shopspring/decimal/README.md b/vendor/github.com/shopspring/decimal/README.md index b70f90159..318c9df58 100644 --- a/vendor/github.com/shopspring/decimal/README.md +++ b/vendor/github.com/shopspring/decimal/README.md @@ -1,6 +1,8 @@ # decimal -[![Build Status](https://travis-ci.org/shopspring/decimal.png?branch=master)](https://travis-ci.org/shopspring/decimal) [![GoDoc](https://godoc.org/github.com/shopspring/decimal?status.svg)](https://godoc.org/github.com/shopspring/decimal) [![Go Report Card](https://goreportcard.com/badge/github.com/shopspring/decimal)](https://goreportcard.com/report/github.com/shopspring/decimal) +[![ci](https://github.com/shopspring/decimal/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/shopspring/decimal/actions/workflows/ci.yml) +[![GoDoc](https://godoc.org/github.com/shopspring/decimal?status.svg)](https://godoc.org/github.com/shopspring/decimal) +[![Go Report Card](https://goreportcard.com/badge/github.com/shopspring/decimal)](https://goreportcard.com/report/github.com/shopspring/decimal) Arbitrary-precision fixed-point decimal numbers in go. @@ -20,7 +22,12 @@ Run `go get github.com/shopspring/decimal` ## Requirements -Decimal library requires Go version `>=1.7` +Decimal library requires Go version `>=1.10` + +## Documentation + +http://godoc.org/github.com/shopspring/decimal + ## Usage @@ -57,14 +64,16 @@ func main() { } ``` -## Documentation - -http://godoc.org/github.com/shopspring/decimal +## Alternative libraries -## Production Usage +When working with decimal numbers, you might face problems this library is not perfectly suited for. +Fortunately, thanks to the wonderful community we have a dozen other libraries that you can choose from. +Explore other alternatives to find the one that best fits your needs :) -* [Spring](https://shopspring.com/), since August 14, 2014. -* If you are using this in production, please let us know! +* [cockroachdb/apd](https://github.com/cockroachdb/apd) - arbitrary precision, mutable and rich API similar to `big.Int`, more performant than this library +* [alpacahq/alpacadecimal](https://github.com/alpacahq/alpacadecimal) - high performance, low precision (12 digits), fully compatible API with this library +* [govalues/decimal](https://github.com/govalues/decimal) - high performance, zero-allocation, low precision (19 digits) +* [greatcloak/decimal](https://github.com/greatcloak/decimal) - fork focusing on billing and e-commerce web application related use cases, includes out-of-the-box BSON marshaling support ## FAQ diff --git a/vendor/github.com/shopspring/decimal/const.go b/vendor/github.com/shopspring/decimal/const.go new file mode 100644 index 000000000..e5d6fa87e --- /dev/null +++ b/vendor/github.com/shopspring/decimal/const.go @@ -0,0 +1,63 @@ +package decimal + +import ( + "strings" +) + +const ( + strLn10 = "2.302585092994045684017991454684364207601101488628772976033327900967572609677352480235997205089598298341967784042286248633409525465082806756666287369098781689482907208325554680843799894826233198528393505308965377732628846163366222287698219886746543667474404243274365155048934314939391479619404400222105101714174800368808401264708068556774321622835522011480466371565912137345074785694768346361679210180644507064800027750268491674655058685693567342067058113642922455440575892572420824131469568901675894025677631135691929203337658714166023010570308963457207544037084746994016826928280848118428931484852494864487192780967627127577539702766860595249671667418348570442250719796500471495105049221477656763693866297697952211071826454973477266242570942932258279850258550978526538320760672631716430950599508780752371033310119785754733154142180842754386359177811705430982748238504564801909561029929182431823752535770975053956518769751037497088869218020518933950723853920514463419726528728696511086257149219884997874887377134568620916705849807828059751193854445009978131146915934666241071846692310107598438319191292230792503747298650929009880391941702654416816335727555703151596113564846546190897042819763365836983716328982174407366009162177850541779276367731145041782137660111010731042397832521894898817597921798666394319523936855916447118246753245630912528778330963604262982153040874560927760726641354787576616262926568298704957954913954918049209069438580790032763017941503117866862092408537949861264933479354871737451675809537088281067452440105892444976479686075120275724181874989395971643105518848195288330746699317814634930000321200327765654130472621883970596794457943468343218395304414844803701305753674262153675579814770458031413637793236291560128185336498466942261465206459942072917119370602444929358037007718981097362533224548366988505528285966192805098447175198503666680874970496982273220244823343097169111136813588418696549323714996941979687803008850408979618598756579894836445212043698216415292987811742973332588607915912510967187510929248475023930572665446276200923068791518135803477701295593646298412366497023355174586195564772461857717369368404676577047874319780573853271810933883496338813069945569399346101090745616033312247949360455361849123333063704751724871276379140924398331810164737823379692265637682071706935846394531616949411701841938119405416449466111274712819705817783293841742231409930022911502362192186723337268385688273533371925103412930705632544426611429765388301822384091026198582888433587455960453004548370789052578473166283701953392231047527564998119228742789713715713228319641003422124210082180679525276689858180956119208391760721080919923461516952599099473782780648128058792731993893453415320185969711021407542282796298237068941764740642225757212455392526179373652434440560595336591539160312524480149313234572453879524389036839236450507881731359711238145323701508413491122324390927681724749607955799151363982881058285740538000653371655553014196332241918087621018204919492651483892" +) + +var ( + ln10 = newConstApproximation(strLn10) +) + +type constApproximation struct { + exact Decimal + approximations []Decimal +} + +func newConstApproximation(value string) constApproximation { + parts := strings.Split(value, ".") + coeff, fractional := parts[0], parts[1] + + coeffLen := len(coeff) + maxPrecision := len(fractional) + + var approximations []Decimal + for p := 1; p < maxPrecision; p *= 2 { + r := RequireFromString(value[:coeffLen+p]) + approximations = append(approximations, r) + } + + return constApproximation{ + RequireFromString(value), + approximations, + } +} + +// Returns the smallest approximation available that's at least as precise +// as the passed precision (places after decimal point), i.e. Floor[ log2(precision) ] + 1 +func (c constApproximation) withPrecision(precision int32) Decimal { + i := 0 + + if precision >= 1 { + i++ + } + + for precision >= 16 { + precision /= 16 + i += 4 + } + + for precision >= 2 { + precision /= 2 + i++ + } + + if i >= len(c.approximations) { + return c.exact + } + + return c.approximations[i] +} diff --git a/vendor/github.com/shopspring/decimal/decimal.go b/vendor/github.com/shopspring/decimal/decimal.go index 801c1a045..a37a2301e 100644 --- a/vendor/github.com/shopspring/decimal/decimal.go +++ b/vendor/github.com/shopspring/decimal/decimal.go @@ -4,14 +4,14 @@ // // The best way to create a new Decimal is to use decimal.NewFromString, ex: // -// n, err := decimal.NewFromString("-123.4567") -// n.String() // output: "-123.4567" +// n, err := decimal.NewFromString("-123.4567") +// n.String() // output: "-123.4567" // // To use Decimal as part of a struct: // -// type Struct struct { -// Number Decimal -// } +// type StructName struct { +// Number Decimal +// } // // Note: This can "only" represent numbers with a maximum of 2^31 digits after the decimal point. package decimal @@ -22,6 +22,7 @@ import ( "fmt" "math" "math/big" + "regexp" "strconv" "strings" ) @@ -31,18 +32,31 @@ import ( // // Example: // -// d1 := decimal.NewFromFloat(2).Div(decimal.NewFromFloat(3)) -// d1.String() // output: "0.6666666666666667" -// d2 := decimal.NewFromFloat(2).Div(decimal.NewFromFloat(30000)) -// d2.String() // output: "0.0000666666666667" -// d3 := decimal.NewFromFloat(20000).Div(decimal.NewFromFloat(3)) -// d3.String() // output: "6666.6666666666666667" -// decimal.DivisionPrecision = 3 -// d4 := decimal.NewFromFloat(2).Div(decimal.NewFromFloat(3)) -// d4.String() // output: "0.667" -// +// d1 := decimal.NewFromFloat(2).Div(decimal.NewFromFloat(3)) +// d1.String() // output: "0.6666666666666667" +// d2 := decimal.NewFromFloat(2).Div(decimal.NewFromFloat(30000)) +// d2.String() // output: "0.0000666666666667" +// d3 := decimal.NewFromFloat(20000).Div(decimal.NewFromFloat(3)) +// d3.String() // output: "6666.6666666666666667" +// decimal.DivisionPrecision = 3 +// d4 := decimal.NewFromFloat(2).Div(decimal.NewFromFloat(3)) +// d4.String() // output: "0.667" var DivisionPrecision = 16 +// PowPrecisionNegativeExponent specifies the maximum precision of the result (digits after decimal point) +// when calculating decimal power. Only used for cases where the exponent is a negative number. +// This constant applies to Pow, PowInt32 and PowBigInt methods, PowWithPrecision method is not constrained by it. +// +// Example: +// +// d1, err := decimal.NewFromFloat(15.2).PowInt32(-2) +// d1.String() // output: "0.0043282548476454" +// +// decimal.PowPrecisionNegativeExponent = 24 +// d2, err := decimal.NewFromFloat(15.2).PowInt32(-2) +// d2.String() // output: "0.004328254847645429362881" +var PowPrecisionNegativeExponent = 16 + // MarshalJSONWithoutQuotes should be set to true if you want the decimal to // be JSON marshaled as a number, instead of as a string. // WARNING: this is dangerous for decimals with many digits, since many JSON @@ -51,6 +65,10 @@ var DivisionPrecision = 16 // silently lose precision. var MarshalJSONWithoutQuotes = false +// ExpMaxIterations specifies the maximum number of iterations needed to calculate +// precise natural exponent value using ExpHullAbrham method. +var ExpMaxIterations = 1000 + // Zero constant, to make computations faster. // Zero should never be compared with == or != directly, please use decimal.Equal or decimal.Cmp instead. var Zero = New(0, 1) @@ -63,6 +81,8 @@ var fiveInt = big.NewInt(5) var tenInt = big.NewInt(10) var twentyInt = big.NewInt(20) +var factorials = []Decimal{New(1, 0)} + // Decimal represents a fixed-point decimal. It is immutable. // number = value * 10 ^ exp type Decimal struct { @@ -84,12 +104,12 @@ func New(value int64, exp int32) Decimal { } } -// NewFromInt converts a int64 to Decimal. +// NewFromInt converts an int64 to Decimal. // // Example: // -// NewFromInt(123).String() // output: "123" -// NewFromInt(-10).String() // output: "-10" +// NewFromInt(123).String() // output: "123" +// NewFromInt(-10).String() // output: "-10" func NewFromInt(value int64) Decimal { return Decimal{ value: big.NewInt(value), @@ -97,12 +117,12 @@ func NewFromInt(value int64) Decimal { } } -// NewFromInt32 converts a int32 to Decimal. +// NewFromInt32 converts an int32 to Decimal. // // Example: // -// NewFromInt(123).String() // output: "123" -// NewFromInt(-10).String() // output: "-10" +// NewFromInt(123).String() // output: "123" +// NewFromInt(-10).String() // output: "-10" func NewFromInt32(value int32) Decimal { return Decimal{ value: big.NewInt(int64(value)), @@ -110,23 +130,53 @@ func NewFromInt32(value int32) Decimal { } } +// NewFromUint64 converts an uint64 to Decimal. +// +// Example: +// +// NewFromUint64(123).String() // output: "123" +func NewFromUint64(value uint64) Decimal { + return Decimal{ + value: new(big.Int).SetUint64(value), + exp: 0, + } +} + // NewFromBigInt returns a new Decimal from a big.Int, value * 10 ^ exp func NewFromBigInt(value *big.Int, exp int32) Decimal { return Decimal{ - value: big.NewInt(0).Set(value), + value: new(big.Int).Set(value), exp: exp, } } +// NewFromBigRat returns a new Decimal from a big.Rat. The numerator and +// denominator are divided and rounded to the given precision. +// +// Example: +// +// d1 := NewFromBigRat(big.NewRat(0, 1), 0) // output: "0" +// d2 := NewFromBigRat(big.NewRat(4, 5), 1) // output: "0.8" +// d3 := NewFromBigRat(big.NewRat(1000, 3), 3) // output: "333.333" +// d4 := NewFromBigRat(big.NewRat(2, 7), 4) // output: "0.2857" +func NewFromBigRat(value *big.Rat, precision int32) Decimal { + return Decimal{ + value: new(big.Int).Set(value.Num()), + exp: 0, + }.DivRound(Decimal{ + value: new(big.Int).Set(value.Denom()), + exp: 0, + }, precision) +} + // NewFromString returns a new Decimal from a string representation. // Trailing zeroes are not trimmed. // // Example: // -// d, err := NewFromString("-123.45") -// d2, err := NewFromString(".0001") -// d3, err := NewFromString("1.47000") -// +// d, err := NewFromString("-123.45") +// d2, err := NewFromString(".0001") +// d3, err := NewFromString("1.47000") func NewFromString(value string) (Decimal, error) { originalInput := value var intString string @@ -146,23 +196,45 @@ func NewFromString(value string) (Decimal, error) { exp = expInt } - parts := strings.Split(value, ".") - if len(parts) == 1 { + pIndex := -1 + vLen := len(value) + for i := 0; i < vLen; i++ { + if value[i] == '.' { + if pIndex > -1 { + return Decimal{}, fmt.Errorf("can't convert %s to decimal: too many .s", value) + } + pIndex = i + } + } + + if pIndex == -1 { // There is no decimal point, we can just parse the original string as // an int intString = value - } else if len(parts) == 2 { - intString = parts[0] + parts[1] - expInt := -len(parts[1]) - exp += int64(expInt) } else { - return Decimal{}, fmt.Errorf("can't convert %s to decimal: too many .s", value) + if pIndex+1 < vLen { + intString = value[:pIndex] + value[pIndex+1:] + } else { + intString = value[:pIndex] + } + expInt := -len(value[pIndex+1:]) + exp += int64(expInt) } - dValue := new(big.Int) - _, ok := dValue.SetString(intString, 10) - if !ok { - return Decimal{}, fmt.Errorf("can't convert %s to decimal", value) + var dValue *big.Int + // strconv.ParseInt is faster than new(big.Int).SetString so this is just a shortcut for strings we know won't overflow + if len(intString) <= 18 { + parsed64, err := strconv.ParseInt(intString, 10, 64) + if err != nil { + return Decimal{}, fmt.Errorf("can't convert %s to decimal", value) + } + dValue = big.NewInt(parsed64) + } else { + dValue = new(big.Int) + _, ok := dValue.SetString(intString, 10) + if !ok { + return Decimal{}, fmt.Errorf("can't convert %s to decimal", value) + } } if exp < math.MinInt32 || exp > math.MaxInt32 { @@ -176,14 +248,36 @@ func NewFromString(value string) (Decimal, error) { }, nil } -// RequireFromString returns a new Decimal from a string representation -// or panics if NewFromString would have returned an error. +// NewFromFormattedString returns a new Decimal from a formatted string representation. +// The second argument - replRegexp, is a regular expression that is used to find characters that should be +// removed from given decimal string representation. All matched characters will be replaced with an empty string. // // Example: // -// d := RequireFromString("-123.45") -// d2 := RequireFromString(".0001") +// r := regexp.MustCompile("[$,]") +// d1, err := NewFromFormattedString("$5,125.99", r) +// +// r2 := regexp.MustCompile("[_]") +// d2, err := NewFromFormattedString("1_000_000", r2) +// +// r3 := regexp.MustCompile("[USD\\s]") +// d3, err := NewFromFormattedString("5000 USD", r3) +func NewFromFormattedString(value string, replRegexp *regexp.Regexp) (Decimal, error) { + parsedValue := replRegexp.ReplaceAllString(value, "") + d, err := NewFromString(parsedValue) + if err != nil { + return Decimal{}, err + } + return d, nil +} + +// RequireFromString returns a new Decimal from a string representation +// or panics if NewFromString had returned an error. +// +// Example: // +// d := RequireFromString("-123.45") +// d2 := RequireFromString(".0001") func RequireFromString(value string) Decimal { dec, err := NewFromString(value) if err != nil { @@ -279,8 +373,7 @@ func newFromFloat(val float64, bits uint64, flt *floatInfo) Decimal { // // Example: // -// NewFromFloatWithExponent(123.456, -2).String() // output: "123.46" -// +// NewFromFloatWithExponent(123.456, -2).String() // output: "123.46" func NewFromFloatWithExponent(value float64, exp int32) Decimal { if math.IsNaN(value) || math.IsInf(value, 0) { panic(fmt.Sprintf("Cannot create a Decimal from %v", value)) @@ -361,6 +454,15 @@ func NewFromFloatWithExponent(value float64, exp int32) Decimal { } } +// Copy returns a copy of decimal with the same value and exponent, but a different pointer to value. +func (d Decimal) Copy() Decimal { + d.ensureInitialized() + return Decimal{ + value: new(big.Int).Set(d.value), + exp: d.exp, + } +} + // rescale returns a rescaled version of the decimal. Returned // decimal may be less precise if the given exponent is bigger // than the initial exponent of the Decimal. @@ -368,7 +470,7 @@ func NewFromFloatWithExponent(value float64, exp int32) Decimal { // // Example: // -// d := New(12345, -4) +// d := New(12345, -4) // d2 := d.rescale(-1) // d3 := d2.rescale(-4) // println(d1) @@ -380,7 +482,6 @@ func NewFromFloatWithExponent(value float64, exp int32) Decimal { // 1.2345 // 1.2 // 1.2000 -// func (d Decimal) rescale(exp int32) Decimal { d.ensureInitialized() @@ -410,6 +511,9 @@ func (d Decimal) rescale(exp int32) Decimal { // Abs returns the absolute value of the decimal. func (d Decimal) Abs() Decimal { + if !d.IsNegative() { + return d + } d.ensureInitialized() d2Value := new(big.Int).Abs(d.value) return Decimal{ @@ -487,11 +591,13 @@ func (d Decimal) Div(d2 Decimal) Decimal { return d.DivRound(d2, int32(DivisionPrecision)) } -// QuoRem does divsion with remainder +// QuoRem does division with remainder // d.QuoRem(d2,precision) returns quotient q and remainder r such that -// d = d2 * q + r, q an integer multiple of 10^(-precision) -// 0 <= r < abs(d2) * 10 ^(-precision) if d>=0 -// 0 >= r > -abs(d2) * 10 ^(-precision) if d<0 +// +// d = d2 * q + r, q an integer multiple of 10^(-precision) +// 0 <= r < abs(d2) * 10 ^(-precision) if d>=0 +// 0 >= r > -abs(d2) * 10 ^(-precision) if d<0 +// // Note that precision<0 is allowed as input. func (d Decimal) QuoRem(d2 Decimal, precision int32) (Decimal, Decimal) { d.ensureInitialized() @@ -500,7 +606,7 @@ func (d Decimal) QuoRem(d2 Decimal, precision int32) (Decimal, Decimal) { panic("decimal division by 0") } scale := -precision - e := int64(d.exp - d2.exp - scale) + e := int64(d.exp) - int64(d2.exp) - int64(scale) if e > math.MaxInt32 || e < math.MinInt32 { panic("overflow in decimal QuoRem") } @@ -534,8 +640,10 @@ func (d Decimal) QuoRem(d2 Decimal, precision int32) (Decimal, Decimal) { // DivRound divides and rounds to a given precision // i.e. to an integer multiple of 10^(-precision) -// for a positive quotient digit 5 is rounded up, away from 0 -// if the quotient is negative then digit 5 is rounded down, away from 0 +// +// for a positive quotient digit 5 is rounded up, away from 0 +// if the quotient is negative then digit 5 is rounded down, away from 0 +// // Note that precision<0 is allowed as input. func (d Decimal) DivRound(d2 Decimal, precision int32) Decimal { // QuoRem already checks initialization @@ -563,32 +671,632 @@ func (d Decimal) DivRound(d2 Decimal, precision int32) Decimal { // Mod returns d % d2. func (d Decimal) Mod(d2 Decimal) Decimal { - quo := d.Div(d2).Truncate(0) - return d.Sub(d2.Mul(quo)) + _, r := d.QuoRem(d2, 0) + return r } -// Pow returns d to the power d2 +// Pow returns d to the power of d2. +// When exponent is negative the returned decimal will have maximum precision of PowPrecisionNegativeExponent places after decimal point. +// +// Pow returns 0 (zero-value of Decimal) instead of error for power operation edge cases, to handle those edge cases use PowWithPrecision +// Edge cases not handled by Pow: +// - 0 ** 0 => undefined value +// - 0 ** y, where y < 0 => infinity +// - x ** y, where x < 0 and y is non-integer decimal => imaginary value +// +// Example: +// +// d1 := decimal.NewFromFloat(4.0) +// d2 := decimal.NewFromFloat(4.0) +// res1 := d1.Pow(d2) +// res1.String() // output: "256" +// +// d3 := decimal.NewFromFloat(5.0) +// d4 := decimal.NewFromFloat(5.73) +// res2 := d3.Pow(d4) +// res2.String() // output: "10118.08037125" func (d Decimal) Pow(d2 Decimal) Decimal { - var temp Decimal - if d2.IntPart() == 0 { - return NewFromFloat(1) + baseSign := d.Sign() + expSign := d2.Sign() + + if baseSign == 0 { + if expSign == 0 { + return Decimal{} + } + if expSign == 1 { + return Decimal{zeroInt, 0} + } + if expSign == -1 { + return Decimal{} + } } - temp = d.Pow(d2.Div(NewFromFloat(2))) - if d2.IntPart()%2 == 0 { - return temp.Mul(temp) + + if expSign == 0 { + return Decimal{oneInt, 0} + } + + // TODO: optimize extraction of fractional part + one := Decimal{oneInt, 0} + expIntPart, expFracPart := d2.QuoRem(one, 0) + + if baseSign == -1 && !expFracPart.IsZero() { + return Decimal{} + } + + intPartPow, _ := d.PowBigInt(expIntPart.value) + + // if exponent is an integer we don't need to calculate d1**frac(d2) + if expFracPart.value.Sign() == 0 { + return intPartPow + } + + // TODO: optimize NumDigits for more performant precision adjustment + digitsBase := d.NumDigits() + digitsExponent := d2.NumDigits() + + precision := digitsBase + + if digitsExponent > precision { + precision += digitsExponent + } + + precision += 6 + + // Calculate x ** frac(y), where + // x ** frac(y) = exp(ln(x ** frac(y)) = exp(ln(x) * frac(y)) + fracPartPow, err := d.Abs().Ln(-d.exp + int32(precision)) + if err != nil { + return Decimal{} } - if d2.IntPart() > 0 { - return temp.Mul(temp).Mul(d) + + fracPartPow = fracPartPow.Mul(expFracPart) + + fracPartPow, err = fracPartPow.ExpTaylor(-d.exp + int32(precision)) + if err != nil { + return Decimal{} } - return temp.Mul(temp).Div(d) + + // Join integer and fractional part, + // base ** (expBase + expFrac) = base ** expBase * base ** expFrac + res := intPartPow.Mul(fracPartPow) + + return res } -// Cmp compares the numbers represented by d and d2 and returns: +// PowWithPrecision returns d to the power of d2. +// Precision parameter specifies minimum precision of the result (digits after decimal point). +// Returned decimal is not rounded to 'precision' places after decimal point. +// +// PowWithPrecision returns error when: +// - 0 ** 0 => undefined value +// - 0 ** y, where y < 0 => infinity +// - x ** y, where x < 0 and y is non-integer decimal => imaginary value +// +// Example: +// +// d1 := decimal.NewFromFloat(4.0) +// d2 := decimal.NewFromFloat(4.0) +// res1, err := d1.PowWithPrecision(d2, 2) +// res1.String() // output: "256" +// +// d3 := decimal.NewFromFloat(5.0) +// d4 := decimal.NewFromFloat(5.73) +// res2, err := d3.PowWithPrecision(d4, 5) +// res2.String() // output: "10118.080371595015625" +// +// d5 := decimal.NewFromFloat(-3.0) +// d6 := decimal.NewFromFloat(-6.0) +// res3, err := d5.PowWithPrecision(d6, 10) +// res3.String() // output: "0.0013717421" +func (d Decimal) PowWithPrecision(d2 Decimal, precision int32) (Decimal, error) { + baseSign := d.Sign() + expSign := d2.Sign() + + if baseSign == 0 { + if expSign == 0 { + return Decimal{}, fmt.Errorf("cannot represent undefined value of 0**0") + } + if expSign == 1 { + return Decimal{zeroInt, 0}, nil + } + if expSign == -1 { + return Decimal{}, fmt.Errorf("cannot represent infinity value of 0 ** y, where y < 0") + } + } + + if expSign == 0 { + return Decimal{oneInt, 0}, nil + } + + // TODO: optimize extraction of fractional part + one := Decimal{oneInt, 0} + expIntPart, expFracPart := d2.QuoRem(one, 0) + + if baseSign == -1 && !expFracPart.IsZero() { + return Decimal{}, fmt.Errorf("cannot represent imaginary value of x ** y, where x < 0 and y is non-integer decimal") + } + + intPartPow, _ := d.powBigIntWithPrecision(expIntPart.value, precision) + + // if exponent is an integer we don't need to calculate d1**frac(d2) + if expFracPart.value.Sign() == 0 { + return intPartPow, nil + } + + // TODO: optimize NumDigits for more performant precision adjustment + digitsBase := d.NumDigits() + digitsExponent := d2.NumDigits() + + if int32(digitsBase) > precision { + precision = int32(digitsBase) + } + if int32(digitsExponent) > precision { + precision += int32(digitsExponent) + } + // increase precision by 10 to compensate for errors in further calculations + precision += 10 + + // Calculate x ** frac(y), where + // x ** frac(y) = exp(ln(x ** frac(y)) = exp(ln(x) * frac(y)) + fracPartPow, err := d.Abs().Ln(precision) + if err != nil { + return Decimal{}, err + } + + fracPartPow = fracPartPow.Mul(expFracPart) + + fracPartPow, err = fracPartPow.ExpTaylor(precision) + if err != nil { + return Decimal{}, err + } + + // Join integer and fractional part, + // base ** (expBase + expFrac) = base ** expBase * base ** expFrac + res := intPartPow.Mul(fracPartPow) + + return res, nil +} + +// PowInt32 returns d to the power of exp, where exp is int32. +// Only returns error when d and exp is 0, thus result is undefined. +// +// When exponent is negative the returned decimal will have maximum precision of PowPrecisionNegativeExponent places after decimal point. +// +// Example: +// +// d1, err := decimal.NewFromFloat(4.0).PowInt32(4) +// d1.String() // output: "256" +// +// d2, err := decimal.NewFromFloat(3.13).PowInt32(5) +// d2.String() // output: "300.4150512793" +func (d Decimal) PowInt32(exp int32) (Decimal, error) { + if d.IsZero() && exp == 0 { + return Decimal{}, fmt.Errorf("cannot represent undefined value of 0**0") + } + + isExpNeg := exp < 0 + exp = abs(exp) + + n, result := d, New(1, 0) + + for exp > 0 { + if exp%2 == 1 { + result = result.Mul(n) + } + exp /= 2 + + if exp > 0 { + n = n.Mul(n) + } + } + + if isExpNeg { + return New(1, 0).DivRound(result, int32(PowPrecisionNegativeExponent)), nil + } + + return result, nil +} + +// PowBigInt returns d to the power of exp, where exp is big.Int. +// Only returns error when d and exp is 0, thus result is undefined. +// +// When exponent is negative the returned decimal will have maximum precision of PowPrecisionNegativeExponent places after decimal point. +// +// Example: +// +// d1, err := decimal.NewFromFloat(3.0).PowBigInt(big.NewInt(3)) +// d1.String() // output: "27" +// +// d2, err := decimal.NewFromFloat(629.25).PowBigInt(big.NewInt(5)) +// d2.String() // output: "98654323103449.5673828125" +func (d Decimal) PowBigInt(exp *big.Int) (Decimal, error) { + return d.powBigIntWithPrecision(exp, int32(PowPrecisionNegativeExponent)) +} + +func (d Decimal) powBigIntWithPrecision(exp *big.Int, precision int32) (Decimal, error) { + if d.IsZero() && exp.Sign() == 0 { + return Decimal{}, fmt.Errorf("cannot represent undefined value of 0**0") + } + + tmpExp := new(big.Int).Set(exp) + isExpNeg := exp.Sign() < 0 + + if isExpNeg { + tmpExp.Abs(tmpExp) + } + + n, result := d, New(1, 0) + + for tmpExp.Sign() > 0 { + if tmpExp.Bit(0) == 1 { + result = result.Mul(n) + } + tmpExp.Rsh(tmpExp, 1) + + if tmpExp.Sign() > 0 { + n = n.Mul(n) + } + } + + if isExpNeg { + return New(1, 0).DivRound(result, precision), nil + } + + return result, nil +} + +// ExpHullAbrham calculates the natural exponent of decimal (e to the power of d) using Hull-Abraham algorithm. +// OverallPrecision argument specifies the overall precision of the result (integer part + decimal part). +// +// ExpHullAbrham is faster than ExpTaylor for small precision values, but it is much slower for large precision values. +// +// Example: +// +// NewFromFloat(26.1).ExpHullAbrham(2).String() // output: "220000000000" +// NewFromFloat(26.1).ExpHullAbrham(20).String() // output: "216314672147.05767284" +func (d Decimal) ExpHullAbrham(overallPrecision uint32) (Decimal, error) { + // Algorithm based on Variable precision exponential function. + // ACM Transactions on Mathematical Software by T. E. Hull & A. Abrham. + if d.IsZero() { + return Decimal{oneInt, 0}, nil + } + + currentPrecision := overallPrecision + + // Algorithm does not work if currentPrecision * 23 < |x|. + // Precision is automatically increased in such cases, so the value can be calculated precisely. + // If newly calculated precision is higher than ExpMaxIterations the currentPrecision will not be changed. + f := d.Abs().InexactFloat64() + if ncp := f / 23; ncp > float64(currentPrecision) && ncp < float64(ExpMaxIterations) { + currentPrecision = uint32(math.Ceil(ncp)) + } + + // fail if abs(d) beyond an over/underflow threshold + overflowThreshold := New(23*int64(currentPrecision), 0) + if d.Abs().Cmp(overflowThreshold) > 0 { + return Decimal{}, fmt.Errorf("over/underflow threshold, exp(x) cannot be calculated precisely") + } + + // Return 1 if abs(d) small enough; this also avoids later over/underflow + overflowThreshold2 := New(9, -int32(currentPrecision)-1) + if d.Abs().Cmp(overflowThreshold2) <= 0 { + return Decimal{oneInt, d.exp}, nil + } + + // t is the smallest integer >= 0 such that the corresponding abs(d/k) < 1 + t := d.exp + int32(d.NumDigits()) // Add d.NumDigits because the paper assumes that d.value [0.1, 1) + + if t < 0 { + t = 0 + } + + k := New(1, t) // reduction factor + r := Decimal{new(big.Int).Set(d.value), d.exp - t} // reduced argument + p := int32(currentPrecision) + t + 2 // precision for calculating the sum + + // Determine n, the number of therms for calculating sum + // use first Newton step (1.435p - 1.182) / log10(p/abs(r)) + // for solving appropriate equation, along with directed + // roundings and simple rational bound for log10(p/abs(r)) + rf := r.Abs().InexactFloat64() + pf := float64(p) + nf := math.Ceil((1.453*pf - 1.182) / math.Log10(pf/rf)) + if nf > float64(ExpMaxIterations) || math.IsNaN(nf) { + return Decimal{}, fmt.Errorf("exact value cannot be calculated in <=ExpMaxIterations iterations") + } + n := int64(nf) + + tmp := New(0, 0) + sum := New(1, 0) + one := New(1, 0) + for i := n - 1; i > 0; i-- { + tmp.value.SetInt64(i) + sum = sum.Mul(r.DivRound(tmp, p)) + sum = sum.Add(one) + } + + ki := k.IntPart() + res := New(1, 0) + for i := ki; i > 0; i-- { + res = res.Mul(sum) + } + + resNumDigits := int32(res.NumDigits()) + + var roundDigits int32 + if resNumDigits > abs(res.exp) { + roundDigits = int32(currentPrecision) - resNumDigits - res.exp + } else { + roundDigits = int32(currentPrecision) + } + + res = res.Round(roundDigits) + + return res, nil +} + +// ExpTaylor calculates the natural exponent of decimal (e to the power of d) using Taylor series expansion. +// Precision argument specifies how precise the result must be (number of digits after decimal point). +// Negative precision is allowed. +// +// ExpTaylor is much faster for large precision values than ExpHullAbrham. +// +// Example: // -// -1 if d < d2 -// 0 if d == d2 -// +1 if d > d2 +// d, err := NewFromFloat(26.1).ExpTaylor(2).String() +// d.String() // output: "216314672147.06" +// +// NewFromFloat(26.1).ExpTaylor(20).String() +// d.String() // output: "216314672147.05767284062928674083" +// +// NewFromFloat(26.1).ExpTaylor(-10).String() +// d.String() // output: "220000000000" +func (d Decimal) ExpTaylor(precision int32) (Decimal, error) { + // Note(mwoss): Implementation can be optimized by exclusively using big.Int API only + if d.IsZero() { + return Decimal{oneInt, 0}.Round(precision), nil + } + + var epsilon Decimal + var divPrecision int32 + if precision < 0 { + epsilon = New(1, -1) + divPrecision = 8 + } else { + epsilon = New(1, -precision-1) + divPrecision = precision + 1 + } + + decAbs := d.Abs() + pow := d.Abs() + factorial := New(1, 0) + + result := New(1, 0) + + for i := int64(1); ; { + step := pow.DivRound(factorial, divPrecision) + result = result.Add(step) + + // Stop Taylor series when current step is smaller than epsilon + if step.Cmp(epsilon) < 0 { + break + } + + pow = pow.Mul(decAbs) + + i++ + + // Calculate next factorial number or retrieve cached value + if len(factorials) >= int(i) && !factorials[i-1].IsZero() { + factorial = factorials[i-1] + } else { + // To avoid any race conditions, firstly the zero value is appended to a slice to create + // a spot for newly calculated factorial. After that, the zero value is replaced by calculated + // factorial using the index notation. + factorial = factorials[i-2].Mul(New(i, 0)) + factorials = append(factorials, Zero) + factorials[i-1] = factorial + } + } + + if d.Sign() < 0 { + result = New(1, 0).DivRound(result, precision+1) + } + + result = result.Round(precision) + return result, nil +} + +// Ln calculates natural logarithm of d. +// Precision argument specifies how precise the result must be (number of digits after decimal point). +// Negative precision is allowed. +// +// Example: +// +// d1, err := NewFromFloat(13.3).Ln(2) +// d1.String() // output: "2.59" +// +// d2, err := NewFromFloat(579.161).Ln(10) +// d2.String() // output: "6.3615805046" +func (d Decimal) Ln(precision int32) (Decimal, error) { + // Algorithm based on The Use of Iteration Methods for Approximating the Natural Logarithm, + // James F. Epperson, The American Mathematical Monthly, Vol. 96, No. 9, November 1989, pp. 831-835. + if d.IsNegative() { + return Decimal{}, fmt.Errorf("cannot calculate natural logarithm for negative decimals") + } + + if d.IsZero() { + return Decimal{}, fmt.Errorf("cannot represent natural logarithm of 0, result: -infinity") + } + + calcPrecision := precision + 2 + z := d.Copy() + + var comp1, comp3, comp2, comp4, reduceAdjust Decimal + comp1 = z.Sub(Decimal{oneInt, 0}) + comp3 = Decimal{oneInt, -1} + + // for decimal in range [0.9, 1.1] where ln(d) is close to 0 + usePowerSeries := false + + if comp1.Abs().Cmp(comp3) <= 0 { + usePowerSeries = true + } else { + // reduce input decimal to range [0.1, 1) + expDelta := int32(z.NumDigits()) + z.exp + z.exp -= expDelta + + // Input decimal was reduced by factor of 10^expDelta, thus we will need to add + // ln(10^expDelta) = expDelta * ln(10) + // to the result to compensate that + ln10 := ln10.withPrecision(calcPrecision) + reduceAdjust = NewFromInt32(expDelta) + reduceAdjust = reduceAdjust.Mul(ln10) + + comp1 = z.Sub(Decimal{oneInt, 0}) + + if comp1.Abs().Cmp(comp3) <= 0 { + usePowerSeries = true + } else { + // initial estimate using floats + zFloat := z.InexactFloat64() + comp1 = NewFromFloat(math.Log(zFloat)) + } + } + + epsilon := Decimal{oneInt, -calcPrecision} + + if usePowerSeries { + // Power Series - https://en.wikipedia.org/wiki/Logarithm#Power_series + // Calculating n-th term of formula: ln(z+1) = 2 sum [ 1 / (2n+1) * (z / (z+2))^(2n+1) ] + // until the difference between current and next term is smaller than epsilon. + // Coverage quite fast for decimals close to 1.0 + + // z + 2 + comp2 = comp1.Add(Decimal{twoInt, 0}) + // z / (z + 2) + comp3 = comp1.DivRound(comp2, calcPrecision) + // 2 * (z / (z + 2)) + comp1 = comp3.Add(comp3) + comp2 = comp1.Copy() + + for n := 1; ; n++ { + // 2 * (z / (z+2))^(2n+1) + comp2 = comp2.Mul(comp3).Mul(comp3) + + // 1 / (2n+1) * 2 * (z / (z+2))^(2n+1) + comp4 = NewFromInt(int64(2*n + 1)) + comp4 = comp2.DivRound(comp4, calcPrecision) + + // comp1 = 2 sum [ 1 / (2n+1) * (z / (z+2))^(2n+1) ] + comp1 = comp1.Add(comp4) + + if comp4.Abs().Cmp(epsilon) <= 0 { + break + } + } + } else { + // Halley's Iteration. + // Calculating n-th term of formula: a_(n+1) = a_n - 2 * (exp(a_n) - z) / (exp(a_n) + z), + // until the difference between current and next term is smaller than epsilon + var prevStep Decimal + maxIters := calcPrecision*2 + 10 + + for i := int32(0); i < maxIters; i++ { + // exp(a_n) + comp3, _ = comp1.ExpTaylor(calcPrecision) + // exp(a_n) - z + comp2 = comp3.Sub(z) + // 2 * (exp(a_n) - z) + comp2 = comp2.Add(comp2) + // exp(a_n) + z + comp4 = comp3.Add(z) + // 2 * (exp(a_n) - z) / (exp(a_n) + z) + comp3 = comp2.DivRound(comp4, calcPrecision) + // comp1 = a_(n+1) = a_n - 2 * (exp(a_n) - z) / (exp(a_n) + z) + comp1 = comp1.Sub(comp3) + + if prevStep.Add(comp3).IsZero() { + // If iteration steps oscillate we should return early and prevent an infinity loop + // NOTE(mwoss): This should be quite a rare case, returning error is not necessary + break + } + + if comp3.Abs().Cmp(epsilon) <= 0 { + break + } + + prevStep = comp3 + } + } + + comp1 = comp1.Add(reduceAdjust) + + return comp1.Round(precision), nil +} + +// NumDigits returns the number of digits of the decimal coefficient (d.Value) +func (d Decimal) NumDigits() int { + if d.value == nil { + return 1 + } + + if d.value.IsInt64() { + i64 := d.value.Int64() + // restrict fast path to integers with exact conversion to float64 + if i64 <= (1<<53) && i64 >= -(1<<53) { + if i64 == 0 { + return 1 + } + return int(math.Log10(math.Abs(float64(i64)))) + 1 + } + } + + estimatedNumDigits := int(float64(d.value.BitLen()) / math.Log2(10)) + + // estimatedNumDigits (lg10) may be off by 1, need to verify + digitsBigInt := big.NewInt(int64(estimatedNumDigits)) + errorCorrectionUnit := digitsBigInt.Exp(tenInt, digitsBigInt, nil) + + if d.value.CmpAbs(errorCorrectionUnit) >= 0 { + return estimatedNumDigits + 1 + } + + return estimatedNumDigits +} + +// IsInteger returns true when decimal can be represented as an integer value, otherwise, it returns false. +func (d Decimal) IsInteger() bool { + // The most typical case, all decimal with exponent higher or equal 0 can be represented as integer + if d.exp >= 0 { + return true + } + // When the exponent is negative we have to check every number after the decimal place + // If all of them are zeroes, we are sure that given decimal can be represented as an integer + var r big.Int + q := new(big.Int).Set(d.value) + for z := abs(d.exp); z > 0; z-- { + q.QuoRem(q, tenInt, &r) + if r.Cmp(zeroInt) != 0 { + return false + } + } + return true +} + +// Abs calculates absolute value of any int32. Used for calculating absolute value of decimal's exponent. +func abs(n int32) int32 { + if n < 0 { + return -n + } + return n +} + +// Cmp compares the numbers represented by d and d2 and returns: // +// -1 if d < d2 +// 0 if d == d2 +// +1 if d > d2 func (d Decimal) Cmp(d2 Decimal) int { d.ensureInitialized() d2.ensureInitialized() @@ -602,12 +1310,21 @@ func (d Decimal) Cmp(d2 Decimal) int { return rd.value.Cmp(rd2.value) } +// Compare compares the numbers represented by d and d2 and returns: +// +// -1 if d < d2 +// 0 if d == d2 +// +1 if d > d2 +func (d Decimal) Compare(d2 Decimal) int { + return d.Cmp(d2) +} + // Equal returns whether the numbers represented by d and d2 are equal. func (d Decimal) Equal(d2 Decimal) bool { return d.Cmp(d2) == 0 } -// Equals is deprecated, please use Equal method instead +// Deprecated: Equals is deprecated, please use Equal method instead. func (d Decimal) Equals(d2 Decimal) bool { return d.Equal(d2) } @@ -639,7 +1356,6 @@ func (d Decimal) LessThanOrEqual(d2 Decimal) bool { // -1 if d < 0 // 0 if d == 0 // +1 if d > 0 -// func (d Decimal) Sign() int { if d.value == nil { return 0 @@ -679,12 +1395,18 @@ func (d Decimal) Exponent() int32 { return d.exp } -// Coefficient returns the coefficient of the decimal. It is scaled by 10^Exponent() +// Coefficient returns the coefficient of the decimal. It is scaled by 10^Exponent() func (d Decimal) Coefficient() *big.Int { d.ensureInitialized() - // we copy the coefficient so that mutating the result does not mutate the - // Decimal. - return big.NewInt(0).Set(d.value) + // we copy the coefficient so that mutating the result does not mutate the Decimal. + return new(big.Int).Set(d.value) +} + +// CoefficientInt64 returns the coefficient of the decimal as int64. It is scaled by 10^Exponent() +// If coefficient cannot be represented in an int64, the result will be undefined. +func (d Decimal) CoefficientInt64() int64 { + d.ensureInitialized() + return d.value.Int64() } // IntPart returns the integer component of the decimal. @@ -696,9 +1418,7 @@ func (d Decimal) IntPart() int64 { // BigInt returns integer component of the decimal as a BigInt. func (d Decimal) BigInt() *big.Int { scaledD := d.rescale(0) - i := &big.Int{} - i.SetString(scaledD.String(), 10) - return i + return scaledD.value } // BigFloat returns decimal as BigFloat. @@ -730,18 +1450,24 @@ func (d Decimal) Float64() (f float64, exact bool) { return d.Rat().Float64() } +// InexactFloat64 returns the nearest float64 value for d. +// It doesn't indicate if the returned value represents d exactly. +func (d Decimal) InexactFloat64() float64 { + f, _ := d.Float64() + return f +} + // String returns the string representation of the decimal // with the fixed point. // // Example: // -// d := New(-12345, -3) -// println(d.String()) +// d := New(-12345, -3) +// println(d.String()) // // Output: // -// -12.345 -// +// -12.345 func (d Decimal) String() string { return d.string(true) } @@ -751,14 +1477,13 @@ func (d Decimal) String() string { // // Example: // -// NewFromFloat(0).StringFixed(2) // output: "0.00" -// NewFromFloat(0).StringFixed(0) // output: "0" -// NewFromFloat(5.45).StringFixed(0) // output: "5" -// NewFromFloat(5.45).StringFixed(1) // output: "5.5" -// NewFromFloat(5.45).StringFixed(2) // output: "5.45" -// NewFromFloat(5.45).StringFixed(3) // output: "5.450" -// NewFromFloat(545).StringFixed(-1) // output: "550" -// +// NewFromFloat(0).StringFixed(2) // output: "0.00" +// NewFromFloat(0).StringFixed(0) // output: "0" +// NewFromFloat(5.45).StringFixed(0) // output: "5" +// NewFromFloat(5.45).StringFixed(1) // output: "5.5" +// NewFromFloat(5.45).StringFixed(2) // output: "5.45" +// NewFromFloat(5.45).StringFixed(3) // output: "5.450" +// NewFromFloat(545).StringFixed(-1) // output: "550" func (d Decimal) StringFixed(places int32) string { rounded := d.Round(places) return rounded.string(false) @@ -769,14 +1494,13 @@ func (d Decimal) StringFixed(places int32) string { // // Example: // -// NewFromFloat(0).StringFixedBank(2) // output: "0.00" -// NewFromFloat(0).StringFixedBank(0) // output: "0" -// NewFromFloat(5.45).StringFixedBank(0) // output: "5" -// NewFromFloat(5.45).StringFixedBank(1) // output: "5.4" -// NewFromFloat(5.45).StringFixedBank(2) // output: "5.45" -// NewFromFloat(5.45).StringFixedBank(3) // output: "5.450" -// NewFromFloat(545).StringFixedBank(-1) // output: "540" -// +// NewFromFloat(0).StringFixedBank(2) // output: "0.00" +// NewFromFloat(0).StringFixedBank(0) // output: "0" +// NewFromFloat(5.45).StringFixedBank(0) // output: "5" +// NewFromFloat(5.45).StringFixedBank(1) // output: "5.4" +// NewFromFloat(5.45).StringFixedBank(2) // output: "5.45" +// NewFromFloat(5.45).StringFixedBank(3) // output: "5.450" +// NewFromFloat(545).StringFixedBank(-1) // output: "540" func (d Decimal) StringFixedBank(places int32) string { rounded := d.RoundBank(places) return rounded.string(false) @@ -794,10 +1518,12 @@ func (d Decimal) StringFixedCash(interval uint8) string { // // Example: // -// NewFromFloat(5.45).Round(1).String() // output: "5.5" -// NewFromFloat(545).Round(-1).String() // output: "550" -// +// NewFromFloat(5.45).Round(1).String() // output: "5.5" +// NewFromFloat(545).Round(-1).String() // output: "550" func (d Decimal) Round(places int32) Decimal { + if d.exp == -places { + return d + } // truncate to places + 1 ret := d.rescale(-places - 1) @@ -818,6 +1544,103 @@ func (d Decimal) Round(places int32) Decimal { return ret } +// RoundCeil rounds the decimal towards +infinity. +// +// Example: +// +// NewFromFloat(545).RoundCeil(-2).String() // output: "600" +// NewFromFloat(500).RoundCeil(-2).String() // output: "500" +// NewFromFloat(1.1001).RoundCeil(2).String() // output: "1.11" +// NewFromFloat(-1.454).RoundCeil(1).String() // output: "-1.4" +func (d Decimal) RoundCeil(places int32) Decimal { + if d.exp >= -places { + return d + } + + rescaled := d.rescale(-places) + if d.Equal(rescaled) { + return d + } + + if d.value.Sign() > 0 { + rescaled.value.Add(rescaled.value, oneInt) + } + + return rescaled +} + +// RoundFloor rounds the decimal towards -infinity. +// +// Example: +// +// NewFromFloat(545).RoundFloor(-2).String() // output: "500" +// NewFromFloat(-500).RoundFloor(-2).String() // output: "-500" +// NewFromFloat(1.1001).RoundFloor(2).String() // output: "1.1" +// NewFromFloat(-1.454).RoundFloor(1).String() // output: "-1.5" +func (d Decimal) RoundFloor(places int32) Decimal { + if d.exp >= -places { + return d + } + + rescaled := d.rescale(-places) + if d.Equal(rescaled) { + return d + } + + if d.value.Sign() < 0 { + rescaled.value.Sub(rescaled.value, oneInt) + } + + return rescaled +} + +// RoundUp rounds the decimal away from zero. +// +// Example: +// +// NewFromFloat(545).RoundUp(-2).String() // output: "600" +// NewFromFloat(500).RoundUp(-2).String() // output: "500" +// NewFromFloat(1.1001).RoundUp(2).String() // output: "1.11" +// NewFromFloat(-1.454).RoundUp(1).String() // output: "-1.5" +func (d Decimal) RoundUp(places int32) Decimal { + if d.exp >= -places { + return d + } + + rescaled := d.rescale(-places) + if d.Equal(rescaled) { + return d + } + + if d.value.Sign() > 0 { + rescaled.value.Add(rescaled.value, oneInt) + } else if d.value.Sign() < 0 { + rescaled.value.Sub(rescaled.value, oneInt) + } + + return rescaled +} + +// RoundDown rounds the decimal towards zero. +// +// Example: +// +// NewFromFloat(545).RoundDown(-2).String() // output: "500" +// NewFromFloat(-500).RoundDown(-2).String() // output: "-500" +// NewFromFloat(1.1001).RoundDown(2).String() // output: "1.1" +// NewFromFloat(-1.454).RoundDown(1).String() // output: "-1.4" +func (d Decimal) RoundDown(places int32) Decimal { + if d.exp >= -places { + return d + } + + rescaled := d.rescale(-places) + if d.Equal(rescaled) { + return d + } + return rescaled +} + // RoundBank rounds the decimal to places decimal places. // If the final digit to round is equidistant from the nearest two integers the // rounded value is taken as the even number @@ -826,13 +1649,12 @@ func (d Decimal) Round(places int32) Decimal { // // Examples: // -// NewFromFloat(5.45).Round(1).String() // output: "5.4" -// NewFromFloat(545).Round(-1).String() // output: "540" -// NewFromFloat(5.46).Round(1).String() // output: "5.5" -// NewFromFloat(546).Round(-1).String() // output: "550" -// NewFromFloat(5.55).Round(1).String() // output: "5.6" -// NewFromFloat(555).Round(-1).String() // output: "560" -// +// NewFromFloat(5.45).RoundBank(1).String() // output: "5.4" +// NewFromFloat(545).RoundBank(-1).String() // output: "540" +// NewFromFloat(5.46).RoundBank(1).String() // output: "5.5" +// NewFromFloat(546).RoundBank(-1).String() // output: "550" +// NewFromFloat(5.55).RoundBank(1).String() // output: "5.6" +// NewFromFloat(555).RoundBank(-1).String() // output: "560" func (d Decimal) RoundBank(places int32) Decimal { round := d.Round(places) @@ -854,11 +1676,13 @@ func (d Decimal) RoundBank(places int32) Decimal { // interval. The amount payable for a cash transaction is rounded to the nearest // multiple of the minimum currency unit available. The following intervals are // available: 5, 10, 25, 50 and 100; any other number throws a panic. -// 5: 5 cent rounding 3.43 => 3.45 -// 10: 10 cent rounding 3.45 => 3.50 (5 gets rounded up) -// 25: 25 cent rounding 3.41 => 3.50 -// 50: 50 cent rounding 3.75 => 4.00 -// 100: 100 cent rounding 3.50 => 4.00 +// +// 5: 5 cent rounding 3.43 => 3.45 +// 10: 10 cent rounding 3.45 => 3.50 (5 gets rounded up) +// 25: 25 cent rounding 3.41 => 3.50 +// 50: 50 cent rounding 3.75 => 4.00 +// 100: 100 cent rounding 3.50 => 4.00 +// // For more details: https://en.wikipedia.org/wiki/Cash_rounding func (d Decimal) RoundCash(interval uint8) Decimal { var iVal *big.Int @@ -927,8 +1751,7 @@ func (d Decimal) Ceil() Decimal { // // Example: // -// decimal.NewFromString("123.456").Truncate(2).String() // "123.45" -// +// decimal.NewFromString("123.456").Truncate(2).String() // "123.45" func (d Decimal) Truncate(precision int32) Decimal { d.ensureInitialized() if precision >= 0 && -precision > d.exp { @@ -970,29 +1793,38 @@ func (d Decimal) MarshalJSON() ([]byte, error) { // UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. As a string representation // is already used when encoding to text, this method stores that string as []byte func (d *Decimal) UnmarshalBinary(data []byte) error { + // Verify we have at least 4 bytes for the exponent. The GOB encoded value + // may be empty. + if len(data) < 4 { + return fmt.Errorf("error decoding binary %v: expected at least 4 bytes, got %d", data, len(data)) + } + // Extract the exponent d.exp = int32(binary.BigEndian.Uint32(data[:4])) // Extract the value d.value = new(big.Int) - return d.value.GobDecode(data[4:]) + if err := d.value.GobDecode(data[4:]); err != nil { + return fmt.Errorf("error decoding binary %v: %s", data, err) + } + + return nil } // MarshalBinary implements the encoding.BinaryMarshaler interface. func (d Decimal) MarshalBinary() (data []byte, err error) { - // Write the exponent first since it's a fixed size - v1 := make([]byte, 4) - binary.BigEndian.PutUint32(v1, uint32(d.exp)) - - // Add the value - var v2 []byte - if v2, err = d.value.GobEncode(); err != nil { - return + // exp is written first, but encode value first to know output size + var valueData []byte + if valueData, err = d.value.GobEncode(); err != nil { + return nil, err } + // Write the exponent in front, since it's a fixed size + expData := make([]byte, 4, len(valueData)+4) + binary.BigEndian.PutUint32(expData, uint32(d.exp)) + // Return the byte array - data = append(v1, v2...) - return + return append(expData, valueData...), nil } // Scan implements the sql.Scanner interface for database deserialization. @@ -1015,6 +1847,11 @@ func (d *Decimal) Scan(value interface{}) error { *d = New(v, 0) return nil + case uint64: + // while clickhouse may send 0 in db as uint64 + *d = NewFromUint64(v) + return nil + default: // default is trying to interpret value stored as string str, err := unquoteIfQuoted(v) @@ -1062,7 +1899,8 @@ func (d *Decimal) GobDecode(data []byte) error { } // StringScaled first scales the decimal then calls .String() on it. -// NOTE: buggy, unintuitive, and DEPRECATED! Use StringFixed instead. +// +// Deprecated: buggy and unintuitive. Use StringFixed instead. func (d Decimal) StringScaled(exp int32) string { return d.rescale(exp).String() } @@ -1122,7 +1960,7 @@ func (d *Decimal) ensureInitialized() { // // To call this function with an array, you must do: // -// Min(arr[0], arr[1:]...) +// Min(arr[0], arr[1:]...) // // This makes it harder to accidentally call Min with 0 arguments. func Min(first Decimal, rest ...Decimal) Decimal { @@ -1139,7 +1977,7 @@ func Min(first Decimal, rest ...Decimal) Decimal { // // To call this function with an array, you must do: // -// Max(arr[0], arr[1:]...) +// Max(arr[0], arr[1:]...) // // This makes it harder to accidentally call Max with 0 arguments. func Max(first Decimal, rest ...Decimal) Decimal { @@ -1174,22 +2012,13 @@ func RescalePair(d1 Decimal, d2 Decimal) (Decimal, Decimal) { d1.ensureInitialized() d2.ensureInitialized() - if d1.exp == d2.exp { - return d1, d2 - } - - baseScale := min(d1.exp, d2.exp) - if baseScale != d1.exp { - return d1.rescale(baseScale), d2 + if d1.exp < d2.exp { + return d1, d2.rescale(d1.exp) + } else if d1.exp > d2.exp { + return d1.rescale(d2.exp), d2 } - return d1, d2.rescale(baseScale) -} -func min(x, y int32) int32 { - if x >= y { - return y - } - return x + return d1, d2 } func unquoteIfQuoted(value interface{}) (string, error) { @@ -1201,8 +2030,7 @@ func unquoteIfQuoted(value interface{}) (string, error) { case []byte: bytes = v default: - return "", fmt.Errorf("could not convert value '%+v' to byte array of type '%T'", - value, value) + return "", fmt.Errorf("could not convert value '%+v' to byte array of type '%T'", value, value) } // If the amount is quoted, strip the quotes @@ -1219,6 +2047,13 @@ type NullDecimal struct { Valid bool } +func NewNullDecimal(d Decimal) NullDecimal { + return NullDecimal{ + Decimal: d, + Valid: true, + } +} + // Scan implements the sql.Scanner interface for database deserialization. func (d *NullDecimal) Scan(value interface{}) error { if value == nil { @@ -1255,6 +2090,33 @@ func (d NullDecimal) MarshalJSON() ([]byte, error) { return d.Decimal.MarshalJSON() } +// UnmarshalText implements the encoding.TextUnmarshaler interface for XML +// deserialization +func (d *NullDecimal) UnmarshalText(text []byte) error { + str := string(text) + + // check for empty XML or XML without body e.g., + if str == "" { + d.Valid = false + return nil + } + if err := d.Decimal.UnmarshalText(text); err != nil { + d.Valid = false + return err + } + d.Valid = true + return nil +} + +// MarshalText implements the encoding.TextMarshaler interface for XML +// serialization. +func (d NullDecimal) MarshalText() (text []byte, err error) { + if !d.Valid { + return []byte{}, nil + } + return d.Decimal.MarshalText() +} + // Trig functions // Atan returns the arctangent, in radians, of x. diff --git a/vendor/github.com/shopspring/decimal/rounding.go b/vendor/github.com/shopspring/decimal/rounding.go index 8008f55cb..d4b0cd007 100644 --- a/vendor/github.com/shopspring/decimal/rounding.go +++ b/vendor/github.com/shopspring/decimal/rounding.go @@ -80,39 +80,80 @@ func roundShortest(d *decimal, mant uint64, exp int, flt *floatInfo) { // would round to the original mantissa and not the neighbors. inclusive := mant%2 == 0 + // As we walk the digits we want to know whether rounding up would fall + // within the upper bound. This is tracked by upperdelta: + // + // If upperdelta == 0, the digits of d and upper are the same so far. + // + // If upperdelta == 1, we saw a difference of 1 between d and upper on a + // previous digit and subsequently only 9s for d and 0s for upper. + // (Thus rounding up may fall outside the bound, if it is exclusive.) + // + // If upperdelta == 2, then the difference is greater than 1 + // and we know that rounding up falls within the bound. + var upperdelta uint8 + // Now we can figure out the minimum number of digits required. // Walk along until d has distinguished itself from upper and lower. - for i := 0; i < d.nd; i++ { + for ui := 0; ; ui++ { + // lower, d, and upper may have the decimal points at different + // places. In this case upper is the longest, so we iterate from + // ui==0 and start li and mi at (possibly) -1. + mi := ui - upper.dp + d.dp + if mi >= d.nd { + break + } + li := ui - upper.dp + lower.dp l := byte('0') // lower digit - if i < lower.nd { - l = lower.d[i] + if li >= 0 && li < lower.nd { + l = lower.d[li] + } + m := byte('0') // middle digit + if mi >= 0 { + m = d.d[mi] } - m := d.d[i] // middle digit u := byte('0') // upper digit - if i < upper.nd { - u = upper.d[i] + if ui < upper.nd { + u = upper.d[ui] } // Okay to round down (truncate) if lower has a different digit // or if lower is inclusive and is exactly the result of rounding // down (i.e., and we have reached the final digit of lower). - okdown := l != m || inclusive && i+1 == lower.nd + okdown := l != m || inclusive && li+1 == lower.nd + switch { + case upperdelta == 0 && m+1 < u: + // Example: + // m = 12345xxx + // u = 12347xxx + upperdelta = 2 + case upperdelta == 0 && m != u: + // Example: + // m = 12345xxx + // u = 12346xxx + upperdelta = 1 + case upperdelta == 1 && (m != '9' || u != '0'): + // Example: + // m = 1234598x + // u = 1234600x + upperdelta = 2 + } // Okay to round up if upper has a different digit and either upper // is inclusive or upper is bigger than the result of rounding up. - okup := m != u && (inclusive || m+1 < u || i+1 < upper.nd) + okup := upperdelta > 0 && (inclusive || upperdelta > 1 || ui+1 < upper.nd) // If it's okay to do either, then round to the nearest one. // If it's okay to do only one, do it. switch { case okdown && okup: - d.Round(i + 1) + d.Round(mi + 1) return case okdown: - d.RoundDown(i + 1) + d.RoundDown(mi + 1) return case okup: - d.RoundUp(i + 1) + d.RoundUp(mi + 1) return } } diff --git a/vendor/github.com/siddontang/go-log/LICENSE b/vendor/github.com/siddontang/go-log/LICENSE deleted file mode 100644 index 7ece9fdf5..000000000 --- a/vendor/github.com/siddontang/go-log/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 siddontang - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/siddontang/go-log/log/doc.go b/vendor/github.com/siddontang/go-log/log/doc.go deleted file mode 100644 index 7477f3af4..000000000 --- a/vendor/github.com/siddontang/go-log/log/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -// Package log supplies more advanced features than go orign log package. -// -// It supports log different level: trace, debug, info, warn, error, fatal. -// -// It also supports different log handlers which you can log to stdout, file, socket, etc... -// -// Use -// -// import "github.com/siddontang/go-log/log" -// -// //log with different level -// log.Info("hello world") -// log.Error("hello world") -// -// //create a logger with specified handler -// h := NewStreamHandler(os.Stdout) -// l := log.NewDefault(h) -// l.Info("hello world") -// -package log diff --git a/vendor/github.com/siddontang/go-log/log/filehandler.go b/vendor/github.com/siddontang/go-log/log/filehandler.go deleted file mode 100644 index 454865d81..000000000 --- a/vendor/github.com/siddontang/go-log/log/filehandler.go +++ /dev/null @@ -1,230 +0,0 @@ -package log - -import ( - "fmt" - "os" - "path" - "time" -) - -// FileHandler writes log to a file. -type FileHandler struct { - fd *os.File -} - -// NewFileHandler creates a FileHander -func NewFileHandler(fileName string, flag int) (*FileHandler, error) { - dir := path.Dir(fileName) - os.Mkdir(dir, 0777) - - f, err := os.OpenFile(fileName, flag, 0755) - if err != nil { - return nil, err - } - - h := new(FileHandler) - - h.fd = f - - return h, nil -} - -// Write implements Handler interface -func (h *FileHandler) Write(b []byte) (n int, err error) { - return h.fd.Write(b) -} - -// Close implements Handler interface -func (h *FileHandler) Close() error { - return h.fd.Close() -} - -// RotatingFileHandler writes log a file, if file size exceeds maxBytes, -// it will backup current file and open a new one. -// -// max backup file number is set by backupCount, it will delete oldest if backups too many. -type RotatingFileHandler struct { - fd *os.File - - fileName string - maxBytes int - curBytes int - backupCount int -} - -// NewRotatingFileHandler creates a RotatingFileHandler -func NewRotatingFileHandler(fileName string, maxBytes int, backupCount int) (*RotatingFileHandler, error) { - dir := path.Dir(fileName) - os.MkdirAll(dir, 0777) - - h := new(RotatingFileHandler) - - if maxBytes <= 0 { - return nil, fmt.Errorf("invalid max bytes") - } - - h.fileName = fileName - h.maxBytes = maxBytes - h.backupCount = backupCount - - var err error - h.fd, err = os.OpenFile(fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - return nil, err - } - - f, err := h.fd.Stat() - if err != nil { - return nil, err - } - h.curBytes = int(f.Size()) - - return h, nil -} - -// Write implements Handler interface -func (h *RotatingFileHandler) Write(p []byte) (n int, err error) { - h.doRollover() - n, err = h.fd.Write(p) - h.curBytes += n - return -} - -// Close implements Handler interface -func (h *RotatingFileHandler) Close() error { - if h.fd != nil { - return h.fd.Close() - } - return nil -} - -func (h *RotatingFileHandler) doRollover() { - if h.curBytes < h.maxBytes { - return - } - - f, err := h.fd.Stat() - if err != nil { - return - } - - if h.maxBytes <= 0 { - return - } else if f.Size() < int64(h.maxBytes) { - h.curBytes = int(f.Size()) - return - } - - if h.backupCount > 0 { - h.fd.Close() - - for i := h.backupCount - 1; i > 0; i-- { - sfn := fmt.Sprintf("%s.%d", h.fileName, i) - dfn := fmt.Sprintf("%s.%d", h.fileName, i+1) - - os.Rename(sfn, dfn) - } - - dfn := fmt.Sprintf("%s.1", h.fileName) - os.Rename(h.fileName, dfn) - - h.fd, _ = os.OpenFile(h.fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - h.curBytes = 0 - f, err := h.fd.Stat() - if err != nil { - return - } - h.curBytes = int(f.Size()) - } -} - -// TimeRotatingFileHandler writes log to a file, -// it will backup current and open a new one, with a period time you sepecified. -// -// refer: http://docs.python.org/2/library/logging.handlers.html. -// same like python TimedRotatingFileHandler. -type TimeRotatingFileHandler struct { - fd *os.File - - baseName string - interval int64 - suffix string - rolloverAt int64 -} - -// TimeRotating way -const ( - WhenSecond = iota - WhenMinute - WhenHour - WhenDay -) - -// NewTimeRotatingFileHandler creates a TimeRotatingFileHandler -func NewTimeRotatingFileHandler(baseName string, when int8, interval int) (*TimeRotatingFileHandler, error) { - dir := path.Dir(baseName) - os.MkdirAll(dir, 0777) - - h := new(TimeRotatingFileHandler) - - h.baseName = baseName - - switch when { - case WhenSecond: - h.interval = 1 - h.suffix = "2006-01-02_15-04-05" - case WhenMinute: - h.interval = 60 - h.suffix = "2006-01-02_15-04" - case WhenHour: - h.interval = 3600 - h.suffix = "2006-01-02_15" - case WhenDay: - h.interval = 3600 * 24 - h.suffix = "2006-01-02" - default: - return nil, fmt.Errorf("invalid when_rotate: %d", when) - } - - h.interval = h.interval * int64(interval) - - var err error - h.fd, err = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - if err != nil { - return nil, err - } - - fInfo, _ := h.fd.Stat() - h.rolloverAt = fInfo.ModTime().Unix() + h.interval - - return h, nil -} - -func (h *TimeRotatingFileHandler) doRollover() { - //refer http://hg.python.org/cpython/file/2.7/Lib/logging/handlers.py - now := time.Now() - - if h.rolloverAt <= now.Unix() { - fName := h.baseName + now.Format(h.suffix) - h.fd.Close() - e := os.Rename(h.baseName, fName) - if e != nil { - panic(e) - } - - h.fd, _ = os.OpenFile(h.baseName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) - - h.rolloverAt = time.Now().Unix() + h.interval - } -} - -// Write implements Handler interface -func (h *TimeRotatingFileHandler) Write(b []byte) (n int, err error) { - h.doRollover() - return h.fd.Write(b) -} - -// Close implements Handler interface -func (h *TimeRotatingFileHandler) Close() error { - return h.fd.Close() -} diff --git a/vendor/github.com/siddontang/go-log/log/handler.go b/vendor/github.com/siddontang/go-log/log/handler.go deleted file mode 100644 index 30a6c4f45..000000000 --- a/vendor/github.com/siddontang/go-log/log/handler.go +++ /dev/null @@ -1,54 +0,0 @@ -package log - -import ( - "io" -) - -//Handler writes logs to somewhere -type Handler interface { - Write(p []byte) (n int, err error) - Close() error -} - -// StreamHandler writes logs to a specified io Writer, maybe stdout, stderr, etc... -type StreamHandler struct { - w io.Writer -} - -// NewStreamHandler creates a StreamHandler -func NewStreamHandler(w io.Writer) (*StreamHandler, error) { - h := new(StreamHandler) - - h.w = w - - return h, nil -} - -// Write implements Handler interface -func (h *StreamHandler) Write(b []byte) (n int, err error) { - return h.w.Write(b) -} - -// Close implements Handler interface -func (h *StreamHandler) Close() error { - return nil -} - -// NullHandler does nothing, it discards anything. -type NullHandler struct { -} - -// NewNullHandler creates a NullHandler -func NewNullHandler() (*NullHandler, error) { - return new(NullHandler), nil -} - -// // Write implements Handler interface -func (h *NullHandler) Write(b []byte) (n int, err error) { - return len(b), nil -} - -// Close implements Handler interface -func (h *NullHandler) Close() error { - return nil -} diff --git a/vendor/github.com/siddontang/go-log/log/log.go b/vendor/github.com/siddontang/go-log/log/log.go deleted file mode 100644 index 956186d9c..000000000 --- a/vendor/github.com/siddontang/go-log/log/log.go +++ /dev/null @@ -1,137 +0,0 @@ -package log - -import ( - "fmt" - "os" -) - -var logger = NewDefault(newStdHandler()) - -// SetDefaultLogger changes the global logger -func SetDefaultLogger(l *Logger) { - logger = l -} - -// SetLevel changes the logger level -func SetLevel(level Level) { - logger.SetLevel(level) -} - -// SetLevelByName changes the logger level by name -func SetLevelByName(name string) { - logger.SetLevelByName(name) -} - -// Fatal records the log with fatal level and exits -func Fatal(args ...interface{}) { - logger.Output(2, LevelFatal, fmt.Sprint(args...)) - os.Exit(1) -} - -// Fatalf records the log with fatal level and exits -func Fatalf(format string, args ...interface{}) { - logger.Output(2, LevelFatal, fmt.Sprintf(format, args...)) - os.Exit(1) -} - -// Fatalln records the log with fatal level and exits -func Fatalln(args ...interface{}) { - logger.Output(2, LevelFatal, fmt.Sprintln(args...)) - os.Exit(1) -} - -// Panic records the log with fatal level and panics -func Panic(args ...interface{}) { - msg := fmt.Sprint(args...) - logger.Output(2, LevelError, msg) - panic(msg) -} - -// Panicf records the log with fatal level and panics -func Panicf(format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - logger.Output(2, LevelError, msg) - panic(msg) -} - -// Panicln records the log with fatal level and panics -func Panicln(args ...interface{}) { - msg := fmt.Sprintln(args...) - logger.Output(2, LevelError, msg) - panic(msg) -} - -// Print records the log with trace level -func Print(args ...interface{}) { - logger.Output(2, LevelTrace, fmt.Sprint(args...)) -} - -// Printf records the log with trace level -func Printf(format string, args ...interface{}) { - logger.Output(2, LevelTrace, fmt.Sprintf(format, args...)) -} - -// Println records the log with trace level -func Println(args ...interface{}) { - logger.Output(2, LevelTrace, fmt.Sprintln(args...)) -} - -// Debug records the log with debug level -func Debug(args ...interface{}) { - logger.Output(2, LevelDebug, fmt.Sprint(args...)) -} - -// Debugf records the log with debug level -func Debugf(format string, args ...interface{}) { - logger.Output(2, LevelDebug, fmt.Sprintf(format, args...)) -} - -// Debugln records the log with debug level -func Debugln(args ...interface{}) { - logger.Output(2, LevelDebug, fmt.Sprintln(args...)) -} - -// Error records the log with error level -func Error(args ...interface{}) { - logger.Output(2, LevelError, fmt.Sprint(args...)) -} - -// Errorf records the log with error level -func Errorf(format string, args ...interface{}) { - logger.Output(2, LevelError, fmt.Sprintf(format, args...)) -} - -// Errorln records the log with error level -func Errorln(args ...interface{}) { - logger.Output(2, LevelError, fmt.Sprintln(args...)) -} - -// Info records the log with info level -func Info(args ...interface{}) { - logger.Output(2, LevelInfo, fmt.Sprint(args...)) -} - -// Infof records the log with info level -func Infof(format string, args ...interface{}) { - logger.Output(2, LevelInfo, fmt.Sprintf(format, args...)) -} - -// Infoln records the log with info level -func Infoln(args ...interface{}) { - logger.Output(2, LevelInfo, fmt.Sprintln(args...)) -} - -// Warn records the log with warn level -func Warn(args ...interface{}) { - logger.Output(2, LevelWarn, fmt.Sprint(args...)) -} - -// Warnf records the log with warn level -func Warnf(format string, args ...interface{}) { - logger.Output(2, LevelWarn, fmt.Sprintf(format, args...)) -} - -// Warnln records the log with warn level -func Warnln(args ...interface{}) { - logger.Output(2, LevelWarn, fmt.Sprintln(args...)) -} diff --git a/vendor/github.com/siddontang/go-log/log/logger.go b/vendor/github.com/siddontang/go-log/log/logger.go deleted file mode 100644 index 4f44c3e0a..000000000 --- a/vendor/github.com/siddontang/go-log/log/logger.go +++ /dev/null @@ -1,301 +0,0 @@ -package log - -import ( - "fmt" - "os" - "runtime" - "strconv" - "strings" - "sync" - "time" - - "github.com/siddontang/go-log/loggers" -) - -const ( - timeFormat = "2006/01/02 15:04:05" - maxBufPoolSize = 16 -) - -// Logger flag -const ( - Ltime = 1 << iota // time format "2006/01/02 15:04:05" - Lfile // file.go:123 - Llevel // [Trace|Debug|Info...] -) - -// Level type -type Level int - -// Log level, from low to high, more high means more serious -const ( - LevelTrace Level = iota - LevelDebug - LevelInfo - LevelWarn - LevelError - LevelFatal -) - -// String returns level String -func (l Level) String() string { - switch l { - case LevelTrace: - return "trace" - case LevelDebug: - return "debug" - case LevelInfo: - return "info" - case LevelWarn: - return "warn" - case LevelError: - return "error" - case LevelFatal: - return "fatal" - } - // return default info - return "info" -} - -// Logger is the logger to record log -type Logger struct { - // TODO: support logger.Contextual - loggers.Advanced - - level Level - flag int - - hLock sync.Mutex - handler Handler - - bufs sync.Pool -} - -// New creates a logger with specified handler and flag -func New(handler Handler, flag int) *Logger { - var l = new(Logger) - - l.level = LevelInfo - l.handler = handler - - l.flag = flag - - l.bufs = sync.Pool{ - New: func() interface{} { - return make([]byte, 0, 1024) - }, - } - - return l -} - -// NewDefault creates default logger with specified handler and flag: Ltime|Lfile|Llevel -func NewDefault(handler Handler) *Logger { - return New(handler, Ltime|Lfile|Llevel) -} - -func newStdHandler() *StreamHandler { - h, _ := NewStreamHandler(os.Stdout) - return h -} - -// Close closes the logger -func (l *Logger) Close() { - l.hLock.Lock() - defer l.hLock.Unlock() - l.handler.Close() -} - -// SetLevel sets log level, any log level less than it will not log -func (l *Logger) SetLevel(level Level) { - l.level = level -} - -// SetLevelByName sets log level by name -func (l *Logger) SetLevelByName(name string) { - level := LevelInfo - switch strings.ToLower(name) { - case "trace": - level = LevelTrace - case "debug": - level = LevelDebug - case "warn", "warning": - level = LevelWarn - case "error": - level = LevelError - case "fatal": - level = LevelFatal - default: - level = LevelInfo - } - - l.SetLevel(level) -} - -// Output records the log with special callstack depth and log level. -func (l *Logger) Output(callDepth int, level Level, msg string) { - if l.level > level { - return - } - - buf := l.bufs.Get().([]byte) - buf = buf[0:0] - defer l.bufs.Put(buf) - - if l.flag&Ltime > 0 { - now := time.Now().Format(timeFormat) - buf = append(buf, '[') - buf = append(buf, now...) - buf = append(buf, "] "...) - } - - if l.flag&Llevel > 0 { - buf = append(buf, '[') - buf = append(buf, level.String()...) - buf = append(buf, "] "...) - } - - if l.flag&Lfile > 0 { - _, file, line, ok := runtime.Caller(callDepth) - if !ok { - file = "???" - line = 0 - } else { - for i := len(file) - 1; i > 0; i-- { - if file[i] == '/' { - file = file[i+1:] - break - } - } - } - - buf = append(buf, file...) - buf = append(buf, ':') - - buf = strconv.AppendInt(buf, int64(line), 10) - buf = append(buf, ' ') - } - - buf = append(buf, msg...) - if len(msg) == 0 || msg[len(msg)-1] != '\n' { - buf = append(buf, '\n') - } - - l.hLock.Lock() - l.handler.Write(buf) - l.hLock.Unlock() -} - -// Fatal records the log with fatal level and exits -func (l *Logger) Fatal(args ...interface{}) { - l.Output(2, LevelFatal, fmt.Sprint(args...)) - os.Exit(1) -} - -// Fatalf records the log with fatal level and exits -func (l *Logger) Fatalf(format string, args ...interface{}) { - l.Output(2, LevelFatal, fmt.Sprintf(format, args...)) - os.Exit(1) -} - -// Fatalln records the log with fatal level and exits -func (l *Logger) Fatalln(args ...interface{}) { - l.Output(2, LevelFatal, fmt.Sprintln(args...)) - os.Exit(1) -} - -// Panic records the log with fatal level and panics -func (l *Logger) Panic(args ...interface{}) { - msg := fmt.Sprint(args...) - l.Output(2, LevelError, msg) - panic(msg) -} - -// Panicf records the log with fatal level and panics -func (l *Logger) Panicf(format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - l.Output(2, LevelError, msg) - panic(msg) -} - -// Panicln records the log with fatal level and panics -func (l *Logger) Panicln(args ...interface{}) { - msg := fmt.Sprintln(args...) - l.Output(2, LevelError, msg) - panic(msg) -} - -// Print records the log with trace level -func (l *Logger) Print(args ...interface{}) { - l.Output(2, LevelTrace, fmt.Sprint(args...)) -} - -// Printf records the log with trace level -func (l *Logger) Printf(format string, args ...interface{}) { - l.Output(2, LevelTrace, fmt.Sprintf(format, args...)) -} - -// Println records the log with trace level -func (l *Logger) Println(args ...interface{}) { - l.Output(2, LevelTrace, fmt.Sprintln(args...)) -} - -// Debug records the log with debug level -func (l *Logger) Debug(args ...interface{}) { - l.Output(2, LevelDebug, fmt.Sprint(args...)) -} - -// Debugf records the log with debug level -func (l *Logger) Debugf(format string, args ...interface{}) { - l.Output(2, LevelDebug, fmt.Sprintf(format, args...)) -} - -// Debugln records the log with debug level -func (l *Logger) Debugln(args ...interface{}) { - l.Output(2, LevelDebug, fmt.Sprintln(args...)) -} - -// Error records the log with error level -func (l *Logger) Error(args ...interface{}) { - l.Output(2, LevelError, fmt.Sprint(args...)) -} - -// Errorf records the log with error level -func (l *Logger) Errorf(format string, args ...interface{}) { - l.Output(2, LevelError, fmt.Sprintf(format, args...)) -} - -// Errorln records the log with error level -func (l *Logger) Errorln(args ...interface{}) { - l.Output(2, LevelError, fmt.Sprintln(args...)) -} - -// Info records the log with info level -func (l *Logger) Info(args ...interface{}) { - l.Output(2, LevelInfo, fmt.Sprint(args...)) -} - -// Infof records the log with info level -func (l *Logger) Infof(format string, args ...interface{}) { - l.Output(2, LevelInfo, fmt.Sprintf(format, args...)) -} - -// Infoln records the log with info level -func (l *Logger) Infoln(args ...interface{}) { - l.Output(2, LevelInfo, fmt.Sprintln(args...)) -} - -// Warn records the log with warn level -func (l *Logger) Warn(args ...interface{}) { - l.Output(2, LevelWarn, fmt.Sprint(args...)) -} - -// Warnf records the log with warn level -func (l *Logger) Warnf(format string, args ...interface{}) { - l.Output(2, LevelWarn, fmt.Sprintf(format, args...)) -} - -// Warnln records the log with warn level -func (l *Logger) Warnln(args ...interface{}) { - l.Output(2, LevelWarn, fmt.Sprintln(args...)) -} diff --git a/vendor/github.com/siddontang/go-log/loggers/loggers.go b/vendor/github.com/siddontang/go-log/loggers/loggers.go deleted file mode 100644 index 2723b24a7..000000000 --- a/vendor/github.com/siddontang/go-log/loggers/loggers.go +++ /dev/null @@ -1,68 +0,0 @@ -// MIT License - -// Copyright (c) 2017 Birkir A. Barkarson - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: - -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. - -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package loggers - -// Standard is the interface used by Go's standard library's log package. -type Standard interface { - Fatal(args ...interface{}) - Fatalf(format string, args ...interface{}) - Fatalln(args ...interface{}) - - Panic(args ...interface{}) - Panicf(format string, args ...interface{}) - Panicln(args ...interface{}) - - Print(args ...interface{}) - Printf(format string, args ...interface{}) - Println(args ...interface{}) -} - -// Advanced is an interface with commonly used log level methods. -type Advanced interface { - Standard - - Debug(args ...interface{}) - Debugf(format string, args ...interface{}) - Debugln(args ...interface{}) - - Error(args ...interface{}) - Errorf(format string, args ...interface{}) - Errorln(args ...interface{}) - - Info(args ...interface{}) - Infof(format string, args ...interface{}) - Infoln(args ...interface{}) - - Warn(args ...interface{}) - Warnf(format string, args ...interface{}) - Warnln(args ...interface{}) -} - -// Contextual is an interface that allows context addition to a log statement before -// calling the final print (message/level) method. -type Contextual interface { - Advanced - - WithField(key string, value interface{}) Advanced - WithFields(fields ...interface{}) Advanced -} diff --git a/vendor/github.com/stretchr/testify/assert/assertion_compare.go b/vendor/github.com/stretchr/testify/assert/assertion_compare.go index 7e19eba09..ffb24e8e3 100644 --- a/vendor/github.com/stretchr/testify/assert/assertion_compare.go +++ b/vendor/github.com/stretchr/testify/assert/assertion_compare.go @@ -390,7 +390,8 @@ func Greater(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface if h, ok := t.(tHelper); ok { h.Helper() } - return compareTwoValues(t, e1, e2, []compareResult{compareGreater}, "\"%v\" is not greater than \"%v\"", msgAndArgs...) + failMessage := fmt.Sprintf("\"%v\" is not greater than \"%v\"", e1, e2) + return compareTwoValues(t, e1, e2, []compareResult{compareGreater}, failMessage, msgAndArgs...) } // GreaterOrEqual asserts that the first element is greater than or equal to the second @@ -403,7 +404,8 @@ func GreaterOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...in if h, ok := t.(tHelper); ok { h.Helper() } - return compareTwoValues(t, e1, e2, []compareResult{compareGreater, compareEqual}, "\"%v\" is not greater than or equal to \"%v\"", msgAndArgs...) + failMessage := fmt.Sprintf("\"%v\" is not greater than or equal to \"%v\"", e1, e2) + return compareTwoValues(t, e1, e2, []compareResult{compareGreater, compareEqual}, failMessage, msgAndArgs...) } // Less asserts that the first element is less than the second @@ -415,7 +417,8 @@ func Less(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...interface{}) if h, ok := t.(tHelper); ok { h.Helper() } - return compareTwoValues(t, e1, e2, []compareResult{compareLess}, "\"%v\" is not less than \"%v\"", msgAndArgs...) + failMessage := fmt.Sprintf("\"%v\" is not less than \"%v\"", e1, e2) + return compareTwoValues(t, e1, e2, []compareResult{compareLess}, failMessage, msgAndArgs...) } // LessOrEqual asserts that the first element is less than or equal to the second @@ -428,7 +431,8 @@ func LessOrEqual(t TestingT, e1 interface{}, e2 interface{}, msgAndArgs ...inter if h, ok := t.(tHelper); ok { h.Helper() } - return compareTwoValues(t, e1, e2, []compareResult{compareLess, compareEqual}, "\"%v\" is not less than or equal to \"%v\"", msgAndArgs...) + failMessage := fmt.Sprintf("\"%v\" is not less than or equal to \"%v\"", e1, e2) + return compareTwoValues(t, e1, e2, []compareResult{compareLess, compareEqual}, failMessage, msgAndArgs...) } // Positive asserts that the specified element is positive @@ -440,7 +444,8 @@ func Positive(t TestingT, e interface{}, msgAndArgs ...interface{}) bool { h.Helper() } zero := reflect.Zero(reflect.TypeOf(e)) - return compareTwoValues(t, e, zero.Interface(), []compareResult{compareGreater}, "\"%v\" is not positive", msgAndArgs...) + failMessage := fmt.Sprintf("\"%v\" is not positive", e) + return compareTwoValues(t, e, zero.Interface(), []compareResult{compareGreater}, failMessage, msgAndArgs...) } // Negative asserts that the specified element is negative @@ -452,7 +457,8 @@ func Negative(t TestingT, e interface{}, msgAndArgs ...interface{}) bool { h.Helper() } zero := reflect.Zero(reflect.TypeOf(e)) - return compareTwoValues(t, e, zero.Interface(), []compareResult{compareLess}, "\"%v\" is not negative", msgAndArgs...) + failMessage := fmt.Sprintf("\"%v\" is not negative", e) + return compareTwoValues(t, e, zero.Interface(), []compareResult{compareLess}, failMessage, msgAndArgs...) } func compareTwoValues(t TestingT, e1 interface{}, e2 interface{}, allowedComparesResults []compareResult, failMessage string, msgAndArgs ...interface{}) bool { @@ -468,11 +474,11 @@ func compareTwoValues(t TestingT, e1 interface{}, e2 interface{}, allowedCompare compareResult, isComparable := compare(e1, e2, e1Kind) if !isComparable { - return Fail(t, fmt.Sprintf("Can not compare type \"%s\"", reflect.TypeOf(e1)), msgAndArgs...) + return Fail(t, fmt.Sprintf(`Can not compare type "%T"`, e1), msgAndArgs...) } if !containsValue(allowedComparesResults, compareResult) { - return Fail(t, fmt.Sprintf(failMessage, e1, e2), msgAndArgs...) + return Fail(t, failMessage, msgAndArgs...) } return true diff --git a/vendor/github.com/stretchr/testify/assert/assertion_format.go b/vendor/github.com/stretchr/testify/assert/assertion_format.go index 190634165..c592f6ad5 100644 --- a/vendor/github.com/stretchr/testify/assert/assertion_format.go +++ b/vendor/github.com/stretchr/testify/assert/assertion_format.go @@ -50,10 +50,19 @@ func ElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string return ElementsMatch(t, listA, listB, append([]interface{}{msg}, args...)...) } -// Emptyf asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. +// Emptyf asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". // // assert.Emptyf(t, obj, "error message %s", "formatted") +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { h.Helper() @@ -117,10 +126,8 @@ func EqualValuesf(t TestingT, expected interface{}, actual interface{}, msg stri // Errorf asserts that a function returned an error (i.e. not `nil`). // -// actualObj, err := SomeFunction() -// if assert.Errorf(t, err, "error message %s", "formatted") { -// assert.Equal(t, expectedErrorf, err) -// } +// actualObj, err := SomeFunction() +// assert.Errorf(t, err, "error message %s", "formatted") func Errorf(t TestingT, err error, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { h.Helper() @@ -438,7 +445,19 @@ func IsNonIncreasingf(t TestingT, object interface{}, msg string, args ...interf return IsNonIncreasing(t, object, append([]interface{}{msg}, args...)...) } +// IsNotTypef asserts that the specified objects are not of the same type. +// +// assert.IsNotTypef(t, &NotMyStruct{}, &MyStruct{}, "error message %s", "formatted") +func IsNotTypef(t TestingT, theType interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := t.(tHelper); ok { + h.Helper() + } + return IsNotType(t, theType, object, append([]interface{}{msg}, args...)...) +} + // IsTypef asserts that the specified objects are of the same type. +// +// assert.IsTypef(t, &MyStruct{}, &MyStruct{}, "error message %s", "formatted") func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { h.Helper() @@ -585,8 +604,7 @@ func NotElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg str return NotElementsMatch(t, listA, listB, append([]interface{}{msg}, args...)...) } -// NotEmptyf asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. +// NotEmptyf asserts that the specified object is NOT [Empty]. // // if assert.NotEmptyf(t, obj, "error message %s", "formatted") { // assert.Equal(t, "two", obj[1]) @@ -693,12 +711,15 @@ func NotSamef(t TestingT, expected interface{}, actual interface{}, msg string, return NotSame(t, expected, actual, append([]interface{}{msg}, args...)...) } -// NotSubsetf asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. +// NotSubsetf asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // assert.NotSubsetf(t, [1, 3, 4], [1, 2], "error message %s", "formatted") // assert.NotSubsetf(t, {"x": 1, "y": 2}, {"z": 3}, "error message %s", "formatted") +// assert.NotSubsetf(t, [1, 3, 4], {1: "one", 2: "two"}, "error message %s", "formatted") +// assert.NotSubsetf(t, {"x": 1, "y": 2}, ["z"], "error message %s", "formatted") func NotSubsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { h.Helper() @@ -782,11 +803,15 @@ func Samef(t TestingT, expected interface{}, actual interface{}, msg string, arg return Same(t, expected, actual, append([]interface{}{msg}, args...)...) } -// Subsetf asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. +// Subsetf asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // assert.Subsetf(t, [1, 2, 3], [1, 2], "error message %s", "formatted") // assert.Subsetf(t, {"x": 1, "y": 2}, {"x": 1}, "error message %s", "formatted") +// assert.Subsetf(t, [1, 2, 3], {1: "one", 2: "two"}, "error message %s", "formatted") +// assert.Subsetf(t, {"x": 1, "y": 2}, ["x"], "error message %s", "formatted") func Subsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) bool { if h, ok := t.(tHelper); ok { h.Helper() diff --git a/vendor/github.com/stretchr/testify/assert/assertion_forward.go b/vendor/github.com/stretchr/testify/assert/assertion_forward.go index 21629087b..58db92845 100644 --- a/vendor/github.com/stretchr/testify/assert/assertion_forward.go +++ b/vendor/github.com/stretchr/testify/assert/assertion_forward.go @@ -92,10 +92,19 @@ func (a *Assertions) ElementsMatchf(listA interface{}, listB interface{}, msg st return ElementsMatchf(a.t, listA, listB, msg, args...) } -// Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. +// Empty asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". // // a.Empty(obj) +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -103,10 +112,19 @@ func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) bool { return Empty(a.t, object, msgAndArgs...) } -// Emptyf asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. +// Emptyf asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". // // a.Emptyf(obj, "error message %s", "formatted") +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value func (a *Assertions) Emptyf(object interface{}, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -224,10 +242,8 @@ func (a *Assertions) Equalf(expected interface{}, actual interface{}, msg string // Error asserts that a function returned an error (i.e. not `nil`). // -// actualObj, err := SomeFunction() -// if a.Error(err) { -// assert.Equal(t, expectedError, err) -// } +// actualObj, err := SomeFunction() +// a.Error(err) func (a *Assertions) Error(err error, msgAndArgs ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -297,10 +313,8 @@ func (a *Assertions) ErrorIsf(err error, target error, msg string, args ...inter // Errorf asserts that a function returned an error (i.e. not `nil`). // -// actualObj, err := SomeFunction() -// if a.Errorf(err, "error message %s", "formatted") { -// assert.Equal(t, expectedErrorf, err) -// } +// actualObj, err := SomeFunction() +// a.Errorf(err, "error message %s", "formatted") func (a *Assertions) Errorf(err error, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -868,7 +882,29 @@ func (a *Assertions) IsNonIncreasingf(object interface{}, msg string, args ...in return IsNonIncreasingf(a.t, object, msg, args...) } +// IsNotType asserts that the specified objects are not of the same type. +// +// a.IsNotType(&NotMyStruct{}, &MyStruct{}) +func (a *Assertions) IsNotType(theType interface{}, object interface{}, msgAndArgs ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsNotType(a.t, theType, object, msgAndArgs...) +} + +// IsNotTypef asserts that the specified objects are not of the same type. +// +// a.IsNotTypef(&NotMyStruct{}, &MyStruct{}, "error message %s", "formatted") +func (a *Assertions) IsNotTypef(theType interface{}, object interface{}, msg string, args ...interface{}) bool { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + return IsNotTypef(a.t, theType, object, msg, args...) +} + // IsType asserts that the specified objects are of the same type. +// +// a.IsType(&MyStruct{}, &MyStruct{}) func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAndArgs ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -877,6 +913,8 @@ func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAnd } // IsTypef asserts that the specified objects are of the same type. +// +// a.IsTypef(&MyStruct{}, &MyStruct{}, "error message %s", "formatted") func (a *Assertions) IsTypef(expectedType interface{}, object interface{}, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -1162,8 +1200,7 @@ func (a *Assertions) NotElementsMatchf(listA interface{}, listB interface{}, msg return NotElementsMatchf(a.t, listA, listB, msg, args...) } -// NotEmpty asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. +// NotEmpty asserts that the specified object is NOT [Empty]. // // if a.NotEmpty(obj) { // assert.Equal(t, "two", obj[1]) @@ -1175,8 +1212,7 @@ func (a *Assertions) NotEmpty(object interface{}, msgAndArgs ...interface{}) boo return NotEmpty(a.t, object, msgAndArgs...) } -// NotEmptyf asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. +// NotEmptyf asserts that the specified object is NOT [Empty]. // // if a.NotEmptyf(obj, "error message %s", "formatted") { // assert.Equal(t, "two", obj[1]) @@ -1378,12 +1414,15 @@ func (a *Assertions) NotSamef(expected interface{}, actual interface{}, msg stri return NotSamef(a.t, expected, actual, msg, args...) } -// NotSubset asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. +// NotSubset asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // a.NotSubset([1, 3, 4], [1, 2]) // a.NotSubset({"x": 1, "y": 2}, {"z": 3}) +// a.NotSubset([1, 3, 4], {1: "one", 2: "two"}) +// a.NotSubset({"x": 1, "y": 2}, ["z"]) func (a *Assertions) NotSubset(list interface{}, subset interface{}, msgAndArgs ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -1391,12 +1430,15 @@ func (a *Assertions) NotSubset(list interface{}, subset interface{}, msgAndArgs return NotSubset(a.t, list, subset, msgAndArgs...) } -// NotSubsetf asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. +// NotSubsetf asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // a.NotSubsetf([1, 3, 4], [1, 2], "error message %s", "formatted") // a.NotSubsetf({"x": 1, "y": 2}, {"z": 3}, "error message %s", "formatted") +// a.NotSubsetf([1, 3, 4], {1: "one", 2: "two"}, "error message %s", "formatted") +// a.NotSubsetf({"x": 1, "y": 2}, ["z"], "error message %s", "formatted") func (a *Assertions) NotSubsetf(list interface{}, subset interface{}, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -1556,11 +1598,15 @@ func (a *Assertions) Samef(expected interface{}, actual interface{}, msg string, return Samef(a.t, expected, actual, msg, args...) } -// Subset asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. +// Subset asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // a.Subset([1, 2, 3], [1, 2]) // a.Subset({"x": 1, "y": 2}, {"x": 1}) +// a.Subset([1, 2, 3], {1: "one", 2: "two"}) +// a.Subset({"x": 1, "y": 2}, ["x"]) func (a *Assertions) Subset(list interface{}, subset interface{}, msgAndArgs ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -1568,11 +1614,15 @@ func (a *Assertions) Subset(list interface{}, subset interface{}, msgAndArgs ... return Subset(a.t, list, subset, msgAndArgs...) } -// Subsetf asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. +// Subsetf asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // a.Subsetf([1, 2, 3], [1, 2], "error message %s", "formatted") // a.Subsetf({"x": 1, "y": 2}, {"x": 1}, "error message %s", "formatted") +// a.Subsetf([1, 2, 3], {1: "one", 2: "two"}, "error message %s", "formatted") +// a.Subsetf({"x": 1, "y": 2}, ["x"], "error message %s", "formatted") func (a *Assertions) Subsetf(list interface{}, subset interface{}, msg string, args ...interface{}) bool { if h, ok := a.t.(tHelper); ok { h.Helper() diff --git a/vendor/github.com/stretchr/testify/assert/assertion_order.go b/vendor/github.com/stretchr/testify/assert/assertion_order.go index 1d2f71824..2fdf80fdd 100644 --- a/vendor/github.com/stretchr/testify/assert/assertion_order.go +++ b/vendor/github.com/stretchr/testify/assert/assertion_order.go @@ -33,7 +33,7 @@ func isOrdered(t TestingT, object interface{}, allowedComparesResults []compareR compareResult, isComparable := compare(prevValueInterface, valueInterface, firstValueKind) if !isComparable { - return Fail(t, fmt.Sprintf("Can not compare type \"%s\" and \"%s\"", reflect.TypeOf(value), reflect.TypeOf(prevValue)), msgAndArgs...) + return Fail(t, fmt.Sprintf(`Can not compare type "%T" and "%T"`, value, prevValue), msgAndArgs...) } if !containsValue(allowedComparesResults, compareResult) { diff --git a/vendor/github.com/stretchr/testify/assert/assertions.go b/vendor/github.com/stretchr/testify/assert/assertions.go index 4e91332bb..de8de0cb6 100644 --- a/vendor/github.com/stretchr/testify/assert/assertions.go +++ b/vendor/github.com/stretchr/testify/assert/assertions.go @@ -210,59 +210,77 @@ the problem actually occurred in calling code.*/ // of each stack frame leading from the current test to the assert call that // failed. func CallerInfo() []string { - var pc uintptr - var ok bool var file string var line int var name string + const stackFrameBufferSize = 10 + pcs := make([]uintptr, stackFrameBufferSize) + callers := []string{} - for i := 0; ; i++ { - pc, file, line, ok = runtime.Caller(i) - if !ok { - // The breaks below failed to terminate the loop, and we ran off the - // end of the call stack. - break - } + offset := 1 - // This is a huge edge case, but it will panic if this is the case, see #180 - if file == "" { - break - } + for { + n := runtime.Callers(offset, pcs) - f := runtime.FuncForPC(pc) - if f == nil { - break - } - name = f.Name() - - // testing.tRunner is the standard library function that calls - // tests. Subtests are called directly by tRunner, without going through - // the Test/Benchmark/Example function that contains the t.Run calls, so - // with subtests we should break when we hit tRunner, without adding it - // to the list of callers. - if name == "testing.tRunner" { + if n == 0 { break } - parts := strings.Split(file, "/") - if len(parts) > 1 { - filename := parts[len(parts)-1] - dir := parts[len(parts)-2] - if (dir != "assert" && dir != "mock" && dir != "require") || filename == "mock_test.go" { - callers = append(callers, fmt.Sprintf("%s:%d", file, line)) + frames := runtime.CallersFrames(pcs[:n]) + + for { + frame, more := frames.Next() + pc = frame.PC + file = frame.File + line = frame.Line + + // This is a huge edge case, but it will panic if this is the case, see #180 + if file == "" { + break } - } - // Drop the package - segments := strings.Split(name, ".") - name = segments[len(segments)-1] - if isTest(name, "Test") || - isTest(name, "Benchmark") || - isTest(name, "Example") { - break + f := runtime.FuncForPC(pc) + if f == nil { + break + } + name = f.Name() + + // testing.tRunner is the standard library function that calls + // tests. Subtests are called directly by tRunner, without going through + // the Test/Benchmark/Example function that contains the t.Run calls, so + // with subtests we should break when we hit tRunner, without adding it + // to the list of callers. + if name == "testing.tRunner" { + break + } + + parts := strings.Split(file, "/") + if len(parts) > 1 { + filename := parts[len(parts)-1] + dir := parts[len(parts)-2] + if (dir != "assert" && dir != "mock" && dir != "require") || filename == "mock_test.go" { + callers = append(callers, fmt.Sprintf("%s:%d", file, line)) + } + } + + // Drop the package + dotPos := strings.LastIndexByte(name, '.') + name = name[dotPos+1:] + if isTest(name, "Test") || + isTest(name, "Benchmark") || + isTest(name, "Example") { + break + } + + if !more { + break + } } + + // Next batch + offset += cap(pcs) } return callers @@ -437,17 +455,34 @@ func NotImplements(t TestingT, interfaceObject interface{}, object interface{}, return true } +func isType(expectedType, object interface{}) bool { + return ObjectsAreEqual(reflect.TypeOf(object), reflect.TypeOf(expectedType)) +} + // IsType asserts that the specified objects are of the same type. -func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs ...interface{}) bool { +// +// assert.IsType(t, &MyStruct{}, &MyStruct{}) +func IsType(t TestingT, expectedType, object interface{}, msgAndArgs ...interface{}) bool { + if isType(expectedType, object) { + return true + } if h, ok := t.(tHelper); ok { h.Helper() } + return Fail(t, fmt.Sprintf("Object expected to be of type %T, but was %T", expectedType, object), msgAndArgs...) +} - if !ObjectsAreEqual(reflect.TypeOf(object), reflect.TypeOf(expectedType)) { - return Fail(t, fmt.Sprintf("Object expected to be of type %v, but was %v", reflect.TypeOf(expectedType), reflect.TypeOf(object)), msgAndArgs...) +// IsNotType asserts that the specified objects are not of the same type. +// +// assert.IsNotType(t, &NotMyStruct{}, &MyStruct{}) +func IsNotType(t TestingT, theType, object interface{}, msgAndArgs ...interface{}) bool { + if !isType(theType, object) { + return true } - - return true + if h, ok := t.(tHelper); ok { + h.Helper() + } + return Fail(t, fmt.Sprintf("Object type expected to be different than %T", theType), msgAndArgs...) } // Equal asserts that two objects are equal. @@ -475,7 +510,6 @@ func Equal(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) } return true - } // validateEqualArgs checks whether provided arguments can be safely used in the @@ -510,8 +544,9 @@ func Same(t TestingT, expected, actual interface{}, msgAndArgs ...interface{}) b if !same { // both are pointers but not the same type & pointing to the same address return Fail(t, fmt.Sprintf("Not same: \n"+ - "expected: %p %#v\n"+ - "actual : %p %#v", expected, expected, actual, actual), msgAndArgs...) + "expected: %p %#[1]v\n"+ + "actual : %p %#[2]v", + expected, actual), msgAndArgs...) } return true @@ -530,14 +565,14 @@ func NotSame(t TestingT, expected, actual interface{}, msgAndArgs ...interface{} same, ok := samePointers(expected, actual) if !ok { - //fails when the arguments are not pointers + // fails when the arguments are not pointers return !(Fail(t, "Both arguments must be pointers", msgAndArgs...)) } if same { return Fail(t, fmt.Sprintf( - "Expected and actual point to the same object: %p %#v", - expected, expected), msgAndArgs...) + "Expected and actual point to the same object: %p %#[1]v", + expected), msgAndArgs...) } return true } @@ -549,7 +584,7 @@ func NotSame(t TestingT, expected, actual interface{}, msgAndArgs ...interface{} func samePointers(first, second interface{}) (same bool, ok bool) { firstPtr, secondPtr := reflect.ValueOf(first), reflect.ValueOf(second) if firstPtr.Kind() != reflect.Ptr || secondPtr.Kind() != reflect.Ptr { - return false, false //not both are pointers + return false, false // not both are pointers } firstType, secondType := reflect.TypeOf(first), reflect.TypeOf(second) @@ -610,7 +645,6 @@ func EqualValues(t TestingT, expected, actual interface{}, msgAndArgs ...interfa } return true - } // EqualExportedValues asserts that the types of two objects are equal and their public @@ -665,7 +699,6 @@ func Exactly(t TestingT, expected, actual interface{}, msgAndArgs ...interface{} } return Equal(t, expected, actual, msgAndArgs...) - } // NotNil asserts that the specified object is not nil. @@ -715,37 +748,45 @@ func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { // isEmpty gets whether the specified object is considered empty or not. func isEmpty(object interface{}) bool { - // get nil case out of the way if object == nil { return true } - objValue := reflect.ValueOf(object) + return isEmptyValue(reflect.ValueOf(object)) +} +// isEmptyValue gets whether the specified reflect.Value is considered empty or not. +func isEmptyValue(objValue reflect.Value) bool { + if objValue.IsZero() { + return true + } + // Special cases of non-zero values that we consider empty switch objValue.Kind() { // collection types are empty when they have no element + // Note: array types are empty when they match their zero-initialized state. case reflect.Chan, reflect.Map, reflect.Slice: return objValue.Len() == 0 - // pointers are empty if nil or if the value they point to is empty + // non-nil pointers are empty if the value they point to is empty case reflect.Ptr: - if objValue.IsNil() { - return true - } - deref := objValue.Elem().Interface() - return isEmpty(deref) - // for all other types, compare against the zero value - // array types are empty when they match their zero-initialized state - default: - zero := reflect.Zero(objValue.Type()) - return reflect.DeepEqual(object, zero.Interface()) + return isEmptyValue(objValue.Elem()) } + return false } -// Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. +// Empty asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". // // assert.Empty(t, obj) +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { pass := isEmpty(object) if !pass { @@ -756,11 +797,9 @@ func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { } return pass - } -// NotEmpty asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. +// NotEmpty asserts that the specified object is NOT [Empty]. // // if assert.NotEmpty(t, obj) { // assert.Equal(t, "two", obj[1]) @@ -775,7 +814,6 @@ func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { } return pass - } // getLen tries to get the length of an object. @@ -819,7 +857,6 @@ func True(t TestingT, value bool, msgAndArgs ...interface{}) bool { } return true - } // False asserts that the specified value is false. @@ -834,7 +871,6 @@ func False(t TestingT, value bool, msgAndArgs ...interface{}) bool { } return true - } // NotEqual asserts that the specified values are NOT equal. @@ -857,7 +893,6 @@ func NotEqual(t TestingT, expected, actual interface{}, msgAndArgs ...interface{ } return true - } // NotEqualValues asserts that two objects are not equal even when converted to the same type @@ -880,7 +915,6 @@ func NotEqualValues(t TestingT, expected, actual interface{}, msgAndArgs ...inte // return (true, false) if element was not found. // return (true, true) if element was found. func containsElement(list interface{}, element interface{}) (ok, found bool) { - listValue := reflect.ValueOf(list) listType := reflect.TypeOf(list) if listType == nil { @@ -915,7 +949,6 @@ func containsElement(list interface{}, element interface{}) (ok, found bool) { } } return true, false - } // Contains asserts that the specified string, list(array, slice...) or map contains the @@ -938,7 +971,6 @@ func Contains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) bo } return true - } // NotContains asserts that the specified string, list(array, slice...) or map does NOT contain the @@ -961,14 +993,17 @@ func NotContains(t TestingT, s, contains interface{}, msgAndArgs ...interface{}) } return true - } -// Subset asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. +// Subset asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // assert.Subset(t, [1, 2, 3], [1, 2]) // assert.Subset(t, {"x": 1, "y": 2}, {"x": 1}) +// assert.Subset(t, [1, 2, 3], {1: "one", 2: "two"}) +// assert.Subset(t, {"x": 1, "y": 2}, ["x"]) func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok bool) { if h, ok := t.(tHelper); ok { h.Helper() @@ -983,7 +1018,7 @@ func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok } subsetKind := reflect.TypeOf(subset).Kind() - if subsetKind != reflect.Array && subsetKind != reflect.Slice && listKind != reflect.Map { + if subsetKind != reflect.Array && subsetKind != reflect.Slice && subsetKind != reflect.Map { return Fail(t, fmt.Sprintf("%q has an unsupported type %s", subset, subsetKind), msgAndArgs...) } @@ -1007,6 +1042,13 @@ func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok } subsetList := reflect.ValueOf(subset) + if subsetKind == reflect.Map { + keys := make([]interface{}, subsetList.Len()) + for idx, key := range subsetList.MapKeys() { + keys[idx] = key.Interface() + } + subsetList = reflect.ValueOf(keys) + } for i := 0; i < subsetList.Len(); i++ { element := subsetList.Index(i).Interface() ok, found := containsElement(list, element) @@ -1021,12 +1063,15 @@ func Subset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok return true } -// NotSubset asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. +// NotSubset asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // assert.NotSubset(t, [1, 3, 4], [1, 2]) // assert.NotSubset(t, {"x": 1, "y": 2}, {"z": 3}) +// assert.NotSubset(t, [1, 3, 4], {1: "one", 2: "two"}) +// assert.NotSubset(t, {"x": 1, "y": 2}, ["z"]) func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) (ok bool) { if h, ok := t.(tHelper); ok { h.Helper() @@ -1041,7 +1086,7 @@ func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) } subsetKind := reflect.TypeOf(subset).Kind() - if subsetKind != reflect.Array && subsetKind != reflect.Slice && listKind != reflect.Map { + if subsetKind != reflect.Array && subsetKind != reflect.Slice && subsetKind != reflect.Map { return Fail(t, fmt.Sprintf("%q has an unsupported type %s", subset, subsetKind), msgAndArgs...) } @@ -1065,11 +1110,18 @@ func NotSubset(t TestingT, list, subset interface{}, msgAndArgs ...interface{}) } subsetList := reflect.ValueOf(subset) + if subsetKind == reflect.Map { + keys := make([]interface{}, subsetList.Len()) + for idx, key := range subsetList.MapKeys() { + keys[idx] = key.Interface() + } + subsetList = reflect.ValueOf(keys) + } for i := 0; i < subsetList.Len(); i++ { element := subsetList.Index(i).Interface() ok, found := containsElement(list, element) if !ok { - return Fail(t, fmt.Sprintf("\"%s\" could not be applied builtin len()", list), msgAndArgs...) + return Fail(t, fmt.Sprintf("%q could not be applied builtin len()", list), msgAndArgs...) } if !found { return true @@ -1591,10 +1643,8 @@ func NoError(t TestingT, err error, msgAndArgs ...interface{}) bool { // Error asserts that a function returned an error (i.e. not `nil`). // -// actualObj, err := SomeFunction() -// if assert.Error(t, err) { -// assert.Equal(t, expectedError, err) -// } +// actualObj, err := SomeFunction() +// assert.Error(t, err) func Error(t TestingT, err error, msgAndArgs ...interface{}) bool { if err == nil { if h, ok := t.(tHelper); ok { @@ -1667,7 +1717,6 @@ func matchRegexp(rx interface{}, str interface{}) bool { default: return r.MatchString(fmt.Sprint(v)) } - } // Regexp asserts that a specified regexp matches a string. @@ -1703,7 +1752,6 @@ func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interf } return !match - } // Zero asserts that i is the zero value for its type. @@ -1814,6 +1862,11 @@ func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{ return Fail(t, fmt.Sprintf("Expected value ('%s') is not valid json.\nJSON parsing error: '%s'", expected, err.Error()), msgAndArgs...) } + // Shortcut if same bytes + if actual == expected { + return true + } + if err := json.Unmarshal([]byte(actual), &actualJSONAsInterface); err != nil { return Fail(t, fmt.Sprintf("Input ('%s') needs to be valid json.\nJSON parsing error: '%s'", actual, err.Error()), msgAndArgs...) } @@ -1832,6 +1885,11 @@ func YAMLEq(t TestingT, expected string, actual string, msgAndArgs ...interface{ return Fail(t, fmt.Sprintf("Expected value ('%s') is not valid yaml.\nYAML parsing error: '%s'", expected, err.Error()), msgAndArgs...) } + // Shortcut if same bytes + if actual == expected { + return true + } + if err := yaml.Unmarshal([]byte(actual), &actualYAMLAsInterface); err != nil { return Fail(t, fmt.Sprintf("Input ('%s') needs to be valid yaml.\nYAML error: '%s'", actual, err.Error()), msgAndArgs...) } @@ -1933,6 +1991,7 @@ func Eventually(t TestingT, condition func() bool, waitFor time.Duration, tick t } ch := make(chan bool, 1) + checkCond := func() { ch <- condition() } timer := time.NewTimer(waitFor) defer timer.Stop() @@ -1940,18 +1999,23 @@ func Eventually(t TestingT, condition func() bool, waitFor time.Duration, tick t ticker := time.NewTicker(tick) defer ticker.Stop() - for tick := ticker.C; ; { + var tickC <-chan time.Time + + // Check the condition once first on the initial call. + go checkCond() + + for { select { case <-timer.C: return Fail(t, "Condition never satisfied", msgAndArgs...) - case <-tick: - tick = nil - go func() { ch <- condition() }() + case <-tickC: + tickC = nil + go checkCond() case v := <-ch: if v { return true } - tick = ticker.C + tickC = ticker.C } } } @@ -1964,6 +2028,9 @@ type CollectT struct { errors []error } +// Helper is like [testing.T.Helper] but does nothing. +func (CollectT) Helper() {} + // Errorf collects the error. func (c *CollectT) Errorf(format string, args ...interface{}) { c.errors = append(c.errors, fmt.Errorf(format, args...)) @@ -2021,35 +2088,42 @@ func EventuallyWithT(t TestingT, condition func(collect *CollectT), waitFor time var lastFinishedTickErrs []error ch := make(chan *CollectT, 1) + checkCond := func() { + collect := new(CollectT) + defer func() { + ch <- collect + }() + condition(collect) + } + timer := time.NewTimer(waitFor) defer timer.Stop() ticker := time.NewTicker(tick) defer ticker.Stop() - for tick := ticker.C; ; { + var tickC <-chan time.Time + + // Check the condition once first on the initial call. + go checkCond() + + for { select { case <-timer.C: for _, err := range lastFinishedTickErrs { t.Errorf("%v", err) } return Fail(t, "Condition never satisfied", msgAndArgs...) - case <-tick: - tick = nil - go func() { - collect := new(CollectT) - defer func() { - ch <- collect - }() - condition(collect) - }() + case <-tickC: + tickC = nil + go checkCond() case collect := <-ch: if !collect.failed() { return true } // Keep the errors from the last ended condition, so that they can be copied to t if timeout is reached. lastFinishedTickErrs = collect.errors - tick = ticker.C + tickC = ticker.C } } } @@ -2064,6 +2138,7 @@ func Never(t TestingT, condition func() bool, waitFor time.Duration, tick time.D } ch := make(chan bool, 1) + checkCond := func() { ch <- condition() } timer := time.NewTimer(waitFor) defer timer.Stop() @@ -2071,18 +2146,23 @@ func Never(t TestingT, condition func() bool, waitFor time.Duration, tick time.D ticker := time.NewTicker(tick) defer ticker.Stop() - for tick := ticker.C; ; { + var tickC <-chan time.Time + + // Check the condition once first on the initial call. + go checkCond() + + for { select { case <-timer.C: return true - case <-tick: - tick = nil - go func() { ch <- condition() }() + case <-tickC: + tickC = nil + go checkCond() case v := <-ch: if v { return Fail(t, "Condition satisfied", msgAndArgs...) } - tick = ticker.C + tickC = ticker.C } } } @@ -2100,9 +2180,12 @@ func ErrorIs(t TestingT, err, target error, msgAndArgs ...interface{}) bool { var expectedText string if target != nil { expectedText = target.Error() + if err == nil { + return Fail(t, fmt.Sprintf("Expected error with %q in chain but got nil.", expectedText), msgAndArgs...) + } } - chain := buildErrorChainString(err) + chain := buildErrorChainString(err, false) return Fail(t, fmt.Sprintf("Target error should be in err chain:\n"+ "expected: %q\n"+ @@ -2125,7 +2208,7 @@ func NotErrorIs(t TestingT, err, target error, msgAndArgs ...interface{}) bool { expectedText = target.Error() } - chain := buildErrorChainString(err) + chain := buildErrorChainString(err, false) return Fail(t, fmt.Sprintf("Target error should not be in err chain:\n"+ "found: %q\n"+ @@ -2143,11 +2226,17 @@ func ErrorAs(t TestingT, err error, target interface{}, msgAndArgs ...interface{ return true } - chain := buildErrorChainString(err) + expectedType := reflect.TypeOf(target).Elem().String() + if err == nil { + return Fail(t, fmt.Sprintf("An error is expected but got nil.\n"+ + "expected: %s", expectedType), msgAndArgs...) + } + + chain := buildErrorChainString(err, true) return Fail(t, fmt.Sprintf("Should be in error chain:\n"+ - "expected: %q\n"+ - "in chain: %s", target, chain, + "expected: %s\n"+ + "in chain: %s", expectedType, chain, ), msgAndArgs...) } @@ -2161,24 +2250,46 @@ func NotErrorAs(t TestingT, err error, target interface{}, msgAndArgs ...interfa return true } - chain := buildErrorChainString(err) + chain := buildErrorChainString(err, true) return Fail(t, fmt.Sprintf("Target error should not be in err chain:\n"+ - "found: %q\n"+ - "in chain: %s", target, chain, + "found: %s\n"+ + "in chain: %s", reflect.TypeOf(target).Elem().String(), chain, ), msgAndArgs...) } -func buildErrorChainString(err error) string { +func unwrapAll(err error) (errs []error) { + errs = append(errs, err) + switch x := err.(type) { + case interface{ Unwrap() error }: + err = x.Unwrap() + if err == nil { + return + } + errs = append(errs, unwrapAll(err)...) + case interface{ Unwrap() []error }: + for _, err := range x.Unwrap() { + errs = append(errs, unwrapAll(err)...) + } + } + return +} + +func buildErrorChainString(err error, withType bool) string { if err == nil { return "" } - e := errors.Unwrap(err) - chain := fmt.Sprintf("%q", err.Error()) - for e != nil { - chain += fmt.Sprintf("\n\t%q", e.Error()) - e = errors.Unwrap(e) + var chain string + errs := unwrapAll(err) + for i := range errs { + if i != 0 { + chain += "\n\t" + } + chain += fmt.Sprintf("%q", errs[i].Error()) + if withType { + chain += fmt.Sprintf(" (%T)", errs[i]) + } } return chain } diff --git a/vendor/github.com/stretchr/testify/assert/doc.go b/vendor/github.com/stretchr/testify/assert/doc.go index 4953981d3..a0b953aa5 100644 --- a/vendor/github.com/stretchr/testify/assert/doc.go +++ b/vendor/github.com/stretchr/testify/assert/doc.go @@ -1,5 +1,9 @@ // Package assert provides a set of comprehensive testing tools for use with the normal Go testing system. // +// # Note +// +// All functions in this package return a bool value indicating whether the assertion has passed. +// // # Example Usage // // The following is a complete example using assert in a standard test function: diff --git a/vendor/github.com/stretchr/testify/assert/http_assertions.go b/vendor/github.com/stretchr/testify/assert/http_assertions.go index 861ed4b7c..5a6bb75f2 100644 --- a/vendor/github.com/stretchr/testify/assert/http_assertions.go +++ b/vendor/github.com/stretchr/testify/assert/http_assertions.go @@ -138,7 +138,7 @@ func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method, url string, contains := strings.Contains(body, fmt.Sprint(str)) if !contains { - Fail(t, fmt.Sprintf("Expected response body for \"%s\" to contain \"%s\" but found \"%s\"", url+"?"+values.Encode(), str, body), msgAndArgs...) + Fail(t, fmt.Sprintf("Expected response body for %q to contain %q but found %q", url+"?"+values.Encode(), str, body), msgAndArgs...) } return contains @@ -158,7 +158,7 @@ func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method, url strin contains := strings.Contains(body, fmt.Sprint(str)) if contains { - Fail(t, fmt.Sprintf("Expected response body for \"%s\" to NOT contain \"%s\" but found \"%s\"", url+"?"+values.Encode(), str, body), msgAndArgs...) + Fail(t, fmt.Sprintf("Expected response body for %q to NOT contain %q but found %q", url+"?"+values.Encode(), str, body), msgAndArgs...) } return !contains diff --git a/vendor/github.com/stretchr/testify/assert/yaml/yaml_custom.go b/vendor/github.com/stretchr/testify/assert/yaml/yaml_custom.go index baa0cc7d7..5a74c4f4d 100644 --- a/vendor/github.com/stretchr/testify/assert/yaml/yaml_custom.go +++ b/vendor/github.com/stretchr/testify/assert/yaml/yaml_custom.go @@ -1,5 +1,4 @@ //go:build testify_yaml_custom && !testify_yaml_fail && !testify_yaml_default -// +build testify_yaml_custom,!testify_yaml_fail,!testify_yaml_default // Package yaml is an implementation of YAML functions that calls a pluggable implementation. // diff --git a/vendor/github.com/stretchr/testify/assert/yaml/yaml_default.go b/vendor/github.com/stretchr/testify/assert/yaml/yaml_default.go index b83c6cf64..0bae80e34 100644 --- a/vendor/github.com/stretchr/testify/assert/yaml/yaml_default.go +++ b/vendor/github.com/stretchr/testify/assert/yaml/yaml_default.go @@ -1,5 +1,4 @@ //go:build !testify_yaml_fail && !testify_yaml_custom -// +build !testify_yaml_fail,!testify_yaml_custom // Package yaml is just an indirection to handle YAML deserialization. // diff --git a/vendor/github.com/stretchr/testify/assert/yaml/yaml_fail.go b/vendor/github.com/stretchr/testify/assert/yaml/yaml_fail.go index e78f7dfe6..8041803fd 100644 --- a/vendor/github.com/stretchr/testify/assert/yaml/yaml_fail.go +++ b/vendor/github.com/stretchr/testify/assert/yaml/yaml_fail.go @@ -1,5 +1,4 @@ //go:build testify_yaml_fail && !testify_yaml_custom && !testify_yaml_default -// +build testify_yaml_fail,!testify_yaml_custom,!testify_yaml_default // Package yaml is an implementation of YAML functions that always fail. // diff --git a/vendor/github.com/stretchr/testify/require/doc.go b/vendor/github.com/stretchr/testify/require/doc.go index 968434724..c8e3f94a8 100644 --- a/vendor/github.com/stretchr/testify/require/doc.go +++ b/vendor/github.com/stretchr/testify/require/doc.go @@ -23,6 +23,8 @@ // // The `require` package have same global functions as in the `assert` package, // but instead of returning a boolean result they call `t.FailNow()`. +// A consequence of this is that it must be called from the goroutine running +// the test function, not from other goroutines created during the test. // // Every assertion function also takes an optional string message as the final argument, // allowing custom error messages to be appended to the message the assertion method outputs. diff --git a/vendor/github.com/stretchr/testify/require/require.go b/vendor/github.com/stretchr/testify/require/require.go index d8921950d..2d02f9bce 100644 --- a/vendor/github.com/stretchr/testify/require/require.go +++ b/vendor/github.com/stretchr/testify/require/require.go @@ -117,10 +117,19 @@ func ElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg string t.FailNow() } -// Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. +// Empty asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". // // require.Empty(t, obj) +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -131,10 +140,19 @@ func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) { t.FailNow() } -// Emptyf asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. +// Emptyf asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". // // require.Emptyf(t, obj, "error message %s", "formatted") +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value func Emptyf(t TestingT, object interface{}, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -279,10 +297,8 @@ func Equalf(t TestingT, expected interface{}, actual interface{}, msg string, ar // Error asserts that a function returned an error (i.e. not `nil`). // -// actualObj, err := SomeFunction() -// if require.Error(t, err) { -// require.Equal(t, expectedError, err) -// } +// actualObj, err := SomeFunction() +// require.Error(t, err) func Error(t TestingT, err error, msgAndArgs ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -373,10 +389,8 @@ func ErrorIsf(t TestingT, err error, target error, msg string, args ...interface // Errorf asserts that a function returned an error (i.e. not `nil`). // -// actualObj, err := SomeFunction() -// if require.Errorf(t, err, "error message %s", "formatted") { -// require.Equal(t, expectedErrorf, err) -// } +// actualObj, err := SomeFunction() +// require.Errorf(t, err, "error message %s", "formatted") func Errorf(t TestingT, err error, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -1097,7 +1111,35 @@ func IsNonIncreasingf(t TestingT, object interface{}, msg string, args ...interf t.FailNow() } +// IsNotType asserts that the specified objects are not of the same type. +// +// require.IsNotType(t, &NotMyStruct{}, &MyStruct{}) +func IsNotType(t TestingT, theType interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsNotType(t, theType, object, msgAndArgs...) { + return + } + t.FailNow() +} + +// IsNotTypef asserts that the specified objects are not of the same type. +// +// require.IsNotTypef(t, &NotMyStruct{}, &MyStruct{}, "error message %s", "formatted") +func IsNotTypef(t TestingT, theType interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := t.(tHelper); ok { + h.Helper() + } + if assert.IsNotTypef(t, theType, object, msg, args...) { + return + } + t.FailNow() +} + // IsType asserts that the specified objects are of the same type. +// +// require.IsType(t, &MyStruct{}, &MyStruct{}) func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -1109,6 +1151,8 @@ func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs } // IsTypef asserts that the specified objects are of the same type. +// +// require.IsTypef(t, &MyStruct{}, &MyStruct{}, "error message %s", "formatted") func IsTypef(t TestingT, expectedType interface{}, object interface{}, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -1469,8 +1513,7 @@ func NotElementsMatchf(t TestingT, listA interface{}, listB interface{}, msg str t.FailNow() } -// NotEmpty asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. +// NotEmpty asserts that the specified object is NOT [Empty]. // // if require.NotEmpty(t, obj) { // require.Equal(t, "two", obj[1]) @@ -1485,8 +1528,7 @@ func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) { t.FailNow() } -// NotEmptyf asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. +// NotEmptyf asserts that the specified object is NOT [Empty]. // // if require.NotEmptyf(t, obj, "error message %s", "formatted") { // require.Equal(t, "two", obj[1]) @@ -1745,12 +1787,15 @@ func NotSamef(t TestingT, expected interface{}, actual interface{}, msg string, t.FailNow() } -// NotSubset asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. +// NotSubset asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // require.NotSubset(t, [1, 3, 4], [1, 2]) // require.NotSubset(t, {"x": 1, "y": 2}, {"z": 3}) +// require.NotSubset(t, [1, 3, 4], {1: "one", 2: "two"}) +// require.NotSubset(t, {"x": 1, "y": 2}, ["z"]) func NotSubset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -1761,12 +1806,15 @@ func NotSubset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...i t.FailNow() } -// NotSubsetf asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. +// NotSubsetf asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // require.NotSubsetf(t, [1, 3, 4], [1, 2], "error message %s", "formatted") // require.NotSubsetf(t, {"x": 1, "y": 2}, {"z": 3}, "error message %s", "formatted") +// require.NotSubsetf(t, [1, 3, 4], {1: "one", 2: "two"}, "error message %s", "formatted") +// require.NotSubsetf(t, {"x": 1, "y": 2}, ["z"], "error message %s", "formatted") func NotSubsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -1971,11 +2019,15 @@ func Samef(t TestingT, expected interface{}, actual interface{}, msg string, arg t.FailNow() } -// Subset asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. +// Subset asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // require.Subset(t, [1, 2, 3], [1, 2]) // require.Subset(t, {"x": 1, "y": 2}, {"x": 1}) +// require.Subset(t, [1, 2, 3], {1: "one", 2: "two"}) +// require.Subset(t, {"x": 1, "y": 2}, ["x"]) func Subset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() @@ -1986,11 +2038,15 @@ func Subset(t TestingT, list interface{}, subset interface{}, msgAndArgs ...inte t.FailNow() } -// Subsetf asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. +// Subsetf asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // require.Subsetf(t, [1, 2, 3], [1, 2], "error message %s", "formatted") // require.Subsetf(t, {"x": 1, "y": 2}, {"x": 1}, "error message %s", "formatted") +// require.Subsetf(t, [1, 2, 3], {1: "one", 2: "two"}, "error message %s", "formatted") +// require.Subsetf(t, {"x": 1, "y": 2}, ["x"], "error message %s", "formatted") func Subsetf(t TestingT, list interface{}, subset interface{}, msg string, args ...interface{}) { if h, ok := t.(tHelper); ok { h.Helper() diff --git a/vendor/github.com/stretchr/testify/require/require_forward.go b/vendor/github.com/stretchr/testify/require/require_forward.go index 1bd87304f..e6f7e9446 100644 --- a/vendor/github.com/stretchr/testify/require/require_forward.go +++ b/vendor/github.com/stretchr/testify/require/require_forward.go @@ -93,10 +93,19 @@ func (a *Assertions) ElementsMatchf(listA interface{}, listB interface{}, msg st ElementsMatchf(a.t, listA, listB, msg, args...) } -// Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. +// Empty asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". // // a.Empty(obj) +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -104,10 +113,19 @@ func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) { Empty(a.t, object, msgAndArgs...) } -// Emptyf asserts that the specified object is empty. I.e. nil, "", false, 0 or either -// a slice or a channel with len == 0. +// Emptyf asserts that the given value is "empty". +// +// [Zero values] are "empty". +// +// Arrays are "empty" if every element is the zero value of the type (stricter than "empty"). +// +// Slices, maps and channels with zero length are "empty". +// +// Pointer values are "empty" if the pointer is nil or if the pointed value is "empty". // // a.Emptyf(obj, "error message %s", "formatted") +// +// [Zero values]: https://go.dev/ref/spec#The_zero_value func (a *Assertions) Emptyf(object interface{}, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -225,10 +243,8 @@ func (a *Assertions) Equalf(expected interface{}, actual interface{}, msg string // Error asserts that a function returned an error (i.e. not `nil`). // -// actualObj, err := SomeFunction() -// if a.Error(err) { -// assert.Equal(t, expectedError, err) -// } +// actualObj, err := SomeFunction() +// a.Error(err) func (a *Assertions) Error(err error, msgAndArgs ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -298,10 +314,8 @@ func (a *Assertions) ErrorIsf(err error, target error, msg string, args ...inter // Errorf asserts that a function returned an error (i.e. not `nil`). // -// actualObj, err := SomeFunction() -// if a.Errorf(err, "error message %s", "formatted") { -// assert.Equal(t, expectedErrorf, err) -// } +// actualObj, err := SomeFunction() +// a.Errorf(err, "error message %s", "formatted") func (a *Assertions) Errorf(err error, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -869,7 +883,29 @@ func (a *Assertions) IsNonIncreasingf(object interface{}, msg string, args ...in IsNonIncreasingf(a.t, object, msg, args...) } +// IsNotType asserts that the specified objects are not of the same type. +// +// a.IsNotType(&NotMyStruct{}, &MyStruct{}) +func (a *Assertions) IsNotType(theType interface{}, object interface{}, msgAndArgs ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsNotType(a.t, theType, object, msgAndArgs...) +} + +// IsNotTypef asserts that the specified objects are not of the same type. +// +// a.IsNotTypef(&NotMyStruct{}, &MyStruct{}, "error message %s", "formatted") +func (a *Assertions) IsNotTypef(theType interface{}, object interface{}, msg string, args ...interface{}) { + if h, ok := a.t.(tHelper); ok { + h.Helper() + } + IsNotTypef(a.t, theType, object, msg, args...) +} + // IsType asserts that the specified objects are of the same type. +// +// a.IsType(&MyStruct{}, &MyStruct{}) func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAndArgs ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -878,6 +914,8 @@ func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAnd } // IsTypef asserts that the specified objects are of the same type. +// +// a.IsTypef(&MyStruct{}, &MyStruct{}, "error message %s", "formatted") func (a *Assertions) IsTypef(expectedType interface{}, object interface{}, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -1163,8 +1201,7 @@ func (a *Assertions) NotElementsMatchf(listA interface{}, listB interface{}, msg NotElementsMatchf(a.t, listA, listB, msg, args...) } -// NotEmpty asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. +// NotEmpty asserts that the specified object is NOT [Empty]. // // if a.NotEmpty(obj) { // assert.Equal(t, "two", obj[1]) @@ -1176,8 +1213,7 @@ func (a *Assertions) NotEmpty(object interface{}, msgAndArgs ...interface{}) { NotEmpty(a.t, object, msgAndArgs...) } -// NotEmptyf asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either -// a slice or a channel with len == 0. +// NotEmptyf asserts that the specified object is NOT [Empty]. // // if a.NotEmptyf(obj, "error message %s", "formatted") { // assert.Equal(t, "two", obj[1]) @@ -1379,12 +1415,15 @@ func (a *Assertions) NotSamef(expected interface{}, actual interface{}, msg stri NotSamef(a.t, expected, actual, msg, args...) } -// NotSubset asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. +// NotSubset asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // a.NotSubset([1, 3, 4], [1, 2]) // a.NotSubset({"x": 1, "y": 2}, {"z": 3}) +// a.NotSubset([1, 3, 4], {1: "one", 2: "two"}) +// a.NotSubset({"x": 1, "y": 2}, ["z"]) func (a *Assertions) NotSubset(list interface{}, subset interface{}, msgAndArgs ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -1392,12 +1431,15 @@ func (a *Assertions) NotSubset(list interface{}, subset interface{}, msgAndArgs NotSubset(a.t, list, subset, msgAndArgs...) } -// NotSubsetf asserts that the specified list(array, slice...) or map does NOT -// contain all elements given in the specified subset list(array, slice...) or -// map. +// NotSubsetf asserts that the list (array, slice, or map) does NOT contain all +// elements given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // a.NotSubsetf([1, 3, 4], [1, 2], "error message %s", "formatted") // a.NotSubsetf({"x": 1, "y": 2}, {"z": 3}, "error message %s", "formatted") +// a.NotSubsetf([1, 3, 4], {1: "one", 2: "two"}, "error message %s", "formatted") +// a.NotSubsetf({"x": 1, "y": 2}, ["z"], "error message %s", "formatted") func (a *Assertions) NotSubsetf(list interface{}, subset interface{}, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -1557,11 +1599,15 @@ func (a *Assertions) Samef(expected interface{}, actual interface{}, msg string, Samef(a.t, expected, actual, msg, args...) } -// Subset asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. +// Subset asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // a.Subset([1, 2, 3], [1, 2]) // a.Subset({"x": 1, "y": 2}, {"x": 1}) +// a.Subset([1, 2, 3], {1: "one", 2: "two"}) +// a.Subset({"x": 1, "y": 2}, ["x"]) func (a *Assertions) Subset(list interface{}, subset interface{}, msgAndArgs ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() @@ -1569,11 +1615,15 @@ func (a *Assertions) Subset(list interface{}, subset interface{}, msgAndArgs ... Subset(a.t, list, subset, msgAndArgs...) } -// Subsetf asserts that the specified list(array, slice...) or map contains all -// elements given in the specified subset list(array, slice...) or map. +// Subsetf asserts that the list (array, slice, or map) contains all elements +// given in the subset (array, slice, or map). +// Map elements are key-value pairs unless compared with an array or slice where +// only the map key is evaluated. // // a.Subsetf([1, 2, 3], [1, 2], "error message %s", "formatted") // a.Subsetf({"x": 1, "y": 2}, {"x": 1}, "error message %s", "formatted") +// a.Subsetf([1, 2, 3], {1: "one", 2: "two"}, "error message %s", "formatted") +// a.Subsetf({"x": 1, "y": 2}, ["x"], "error message %s", "formatted") func (a *Assertions) Subsetf(list interface{}, subset interface{}, msg string, args ...interface{}) { if h, ok := a.t.(tHelper); ok { h.Helper() diff --git a/vendor/github.com/stretchr/testify/suite/stats.go b/vendor/github.com/stretchr/testify/suite/stats.go index 261da37f7..be4ccd679 100644 --- a/vendor/github.com/stretchr/testify/suite/stats.go +++ b/vendor/github.com/stretchr/testify/suite/stats.go @@ -16,26 +16,30 @@ type TestInformation struct { } func newSuiteInformation() *SuiteInformation { - testStats := make(map[string]*TestInformation) - return &SuiteInformation{ - TestStats: testStats, + TestStats: make(map[string]*TestInformation), } } -func (s SuiteInformation) start(testName string) { +func (s *SuiteInformation) start(testName string) { + if s == nil { + return + } s.TestStats[testName] = &TestInformation{ TestName: testName, Start: time.Now(), } } -func (s SuiteInformation) end(testName string, passed bool) { +func (s *SuiteInformation) end(testName string, passed bool) { + if s == nil { + return + } s.TestStats[testName].End = time.Now() s.TestStats[testName].Passed = passed } -func (s SuiteInformation) Passed() bool { +func (s *SuiteInformation) Passed() bool { for _, stats := range s.TestStats { if !stats.Passed { return false diff --git a/vendor/github.com/stretchr/testify/suite/suite.go b/vendor/github.com/stretchr/testify/suite/suite.go index 18443a91c..1b19be3bc 100644 --- a/vendor/github.com/stretchr/testify/suite/suite.go +++ b/vendor/github.com/stretchr/testify/suite/suite.go @@ -7,6 +7,7 @@ import ( "reflect" "regexp" "runtime/debug" + "strings" "sync" "testing" "time" @@ -15,7 +16,6 @@ import ( "github.com/stretchr/testify/require" ) -var allTestsFilter = func(_, _ string) (bool, error) { return true, nil } var matchMethod = flag.String("testify.m", "", "regular expression to select tests of the testify suite to run") // Suite is a basic testing suite with methods for storing and @@ -116,6 +116,11 @@ func (suite *Suite) Run(name string, subtest func()) bool { }) } +type test = struct { + name string + run func(t *testing.T) +} + // Run takes a testing suite and runs all of the tests attached // to it. func Run(t *testing.T, suite TestingSuite) { @@ -124,45 +129,39 @@ func Run(t *testing.T, suite TestingSuite) { suite.SetT(t) suite.SetS(suite) - var suiteSetupDone bool - var stats *SuiteInformation if _, ok := suite.(WithStats); ok { stats = newSuiteInformation() } - tests := []testing.InternalTest{} + var tests []test methodFinder := reflect.TypeOf(suite) suiteName := methodFinder.Elem().Name() - for i := 0; i < methodFinder.NumMethod(); i++ { - method := methodFinder.Method(i) - - ok, err := methodFilter(method.Name) + var matchMethodRE *regexp.Regexp + if *matchMethod != "" { + var err error + matchMethodRE, err = regexp.Compile(*matchMethod) if err != nil { fmt.Fprintf(os.Stderr, "testify: invalid regexp for -m: %s\n", err) os.Exit(1) } + } - if !ok { + for i := 0; i < methodFinder.NumMethod(); i++ { + method := methodFinder.Method(i) + + if !strings.HasPrefix(method.Name, "Test") { continue } - - if !suiteSetupDone { - if stats != nil { - stats.Start = time.Now() - } - - if setupAllSuite, ok := suite.(SetupAllSuite); ok { - setupAllSuite.SetupSuite() - } - - suiteSetupDone = true + // Apply -testify.m filter + if matchMethodRE != nil && !matchMethodRE.MatchString(method.Name) { + continue } - test := testing.InternalTest{ - Name: method.Name, - F: func(t *testing.T) { + test := test{ + name: method.Name, + run: func(t *testing.T) { parentT := suite.T() suite.SetT(t) defer recoverAndFailOnPanic(t) @@ -171,10 +170,7 @@ func Run(t *testing.T, suite TestingSuite) { r := recover() - if stats != nil { - passed := !t.Failed() && r == nil - stats.end(method.Name, passed) - } + stats.end(method.Name, !t.Failed() && r == nil) if afterTestSuite, ok := suite.(AfterTest); ok { afterTestSuite.AfterTest(suiteName, method.Name) @@ -195,59 +191,47 @@ func Run(t *testing.T, suite TestingSuite) { beforeTestSuite.BeforeTest(methodFinder.Elem().Name(), method.Name) } - if stats != nil { - stats.start(method.Name) - } + stats.start(method.Name) method.Func.Call([]reflect.Value{reflect.ValueOf(suite)}) }, } tests = append(tests, test) } - if suiteSetupDone { - defer func() { - if tearDownAllSuite, ok := suite.(TearDownAllSuite); ok { - tearDownAllSuite.TearDownSuite() - } - - if suiteWithStats, measureStats := suite.(WithStats); measureStats { - stats.End = time.Now() - suiteWithStats.HandleStats(suiteName, stats) - } - }() + + if len(tests) == 0 { + return } - runTests(t, tests) -} + if stats != nil { + stats.Start = time.Now() + } -// Filtering method according to set regular expression -// specified command-line argument -m -func methodFilter(name string) (bool, error) { - if ok, _ := regexp.MatchString("^Test", name); !ok { - return false, nil + if setupAllSuite, ok := suite.(SetupAllSuite); ok { + setupAllSuite.SetupSuite() } - return regexp.MatchString(*matchMethod, name) + + defer func() { + if tearDownAllSuite, ok := suite.(TearDownAllSuite); ok { + tearDownAllSuite.TearDownSuite() + } + + if suiteWithStats, measureStats := suite.(WithStats); measureStats { + stats.End = time.Now() + suiteWithStats.HandleStats(suiteName, stats) + } + }() + + runTests(t, tests) } -func runTests(t testing.TB, tests []testing.InternalTest) { +func runTests(t *testing.T, tests []test) { if len(tests) == 0 { t.Log("warning: no tests to run") return } - r, ok := t.(runner) - if !ok { // backwards compatibility with Go 1.6 and below - if !testing.RunTests(allTestsFilter, tests) { - t.Fail() - } - return - } - for _, test := range tests { - r.Run(test.Name, test.F) + t.Run(test.name, test.run) } } - -type runner interface { - Run(name string, f func(t *testing.T)) bool -} diff --git a/vendor/go.uber.org/zap/.golangci.yml b/vendor/go.uber.org/zap/.golangci.yml index 2346df135..74faaa71d 100644 --- a/vendor/go.uber.org/zap/.golangci.yml +++ b/vendor/go.uber.org/zap/.golangci.yml @@ -25,7 +25,7 @@ linters-settings: govet: # These govet checks are disabled by default, but they're useful. enable: - - niliness + - nilness - reflectvaluecompare - sortslice - unusedwrite diff --git a/vendor/go.uber.org/zap/CHANGELOG.md b/vendor/go.uber.org/zap/CHANGELOG.md index 6d6cd5f4d..53848733c 100644 --- a/vendor/go.uber.org/zap/CHANGELOG.md +++ b/vendor/go.uber.org/zap/CHANGELOG.md @@ -3,6 +3,20 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 1.28.0 (27 Apr 2026) +Enhancements: +* [#1534][]: Add `zapcore.CheckPreWriteHook` and `CheckedEntry.Before` method for transforming entries before they are written to any Cores. + +## 1.27.1 (19 Nov 2025) +Enhancements: +* [#1501][]: prevent `Object` from panicking on nils +* [#1511][]: Fix a race condition in `WithLazy`. + +Thanks to @rabbbit, @alshopov, @jquirke, @arukiidou for their contributions to this release. + +[#1501]: https://github.com/uber-go/zap/pull/1501 +[#1511]: https://github.com/uber-go/zap/pull/1511 + ## 1.27.0 (20 Feb 2024) Enhancements: * [#1378][]: Add `WithLazy` method for `SugaredLogger`. diff --git a/vendor/go.uber.org/zap/CODE_OF_CONDUCT.md b/vendor/go.uber.org/zap/CODE_OF_CONDUCT.md index e327d9aa5..bc988b72e 100644 --- a/vendor/go.uber.org/zap/CODE_OF_CONDUCT.md +++ b/vendor/go.uber.org/zap/CODE_OF_CONDUCT.md @@ -71,5 +71,5 @@ This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]. -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ +[homepage]: https://contributor-covenant.org +[version]: https://contributor-covenant.org/version/1/4/ diff --git a/vendor/go.uber.org/zap/LICENSE b/vendor/go.uber.org/zap/LICENSE index 6652bed45..3883b9a7e 100644 --- a/vendor/go.uber.org/zap/LICENSE +++ b/vendor/go.uber.org/zap/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2016-2017 Uber Technologies, Inc. +Copyright (c) 2016-2024 Uber Technologies, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/vendor/go.uber.org/zap/Makefile b/vendor/go.uber.org/zap/Makefile index eb1cee53b..f9db385b3 100644 --- a/vendor/go.uber.org/zap/Makefile +++ b/vendor/go.uber.org/zap/Makefile @@ -24,7 +24,7 @@ golangci-lint: @$(foreach mod,$(MODULE_DIRS), \ (cd $(mod) && \ echo "[lint] golangci-lint: $(mod)" && \ - golangci-lint run --path-prefix $(mod)) &&) true + golangci-lint run --path-prefix $(mod) ./...) &&) true .PHONY: tidy tidy: diff --git a/vendor/go.uber.org/zap/field.go b/vendor/go.uber.org/zap/field.go index 6743930b8..1884afabc 100644 --- a/vendor/go.uber.org/zap/field.go +++ b/vendor/go.uber.org/zap/field.go @@ -398,6 +398,9 @@ func Durationp(key string, val *time.Duration) Field { // struct-like user-defined types to the logging context. The struct's // MarshalLogObject method is called lazily. func Object(key string, val zapcore.ObjectMarshaler) Field { + if val == nil { + return nilField(key) + } return Field{Key: key, Type: zapcore.ObjectMarshalerType, Interface: val} } @@ -431,6 +434,13 @@ func (d dictObject) MarshalLogObject(enc zapcore.ObjectEncoder) error { return nil } +// DictObject constructs a [zapcore.ObjectMarshaler] with the given list of fields. +// The resulting object marshaler can be used as input to [Object], [Objects], or +// any other functions that expect an object marshaler. +func DictObject(val ...Field) zapcore.ObjectMarshaler { + return dictObject(val) +} + // We discovered an issue where zap.Any can cause a performance degradation // when used in new goroutines. // diff --git a/vendor/go.uber.org/zap/http_handler.go b/vendor/go.uber.org/zap/http_handler.go index 2be8f6515..1cae2c164 100644 --- a/vendor/go.uber.org/zap/http_handler.go +++ b/vendor/go.uber.org/zap/http_handler.go @@ -71,7 +71,7 @@ import ( func (lvl AtomicLevel) ServeHTTP(w http.ResponseWriter, r *http.Request) { if err := lvl.serveHTTP(w, r); err != nil { w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(w, "internal error: %v", err) + _, _ = fmt.Fprintf(w, "internal error: %v", err) } } diff --git a/vendor/go.uber.org/zap/logger.go b/vendor/go.uber.org/zap/logger.go index c4d300323..2d0ef141b 100644 --- a/vendor/go.uber.org/zap/logger.go +++ b/vendor/go.uber.org/zap/logger.go @@ -381,7 +381,11 @@ func (log *Logger) check(lvl zapcore.Level, msg string) *zapcore.CheckedEntry { if stack.Count() == 0 { if log.addCaller { - fmt.Fprintf(log.errorOutput, "%v Logger.check error: failed to get caller\n", ent.Time.UTC()) + _, _ = fmt.Fprintf( + log.errorOutput, + "%v Logger.check error: failed to get caller\n", + ent.Time.UTC(), + ) _ = log.errorOutput.Sync() } return ce diff --git a/vendor/go.uber.org/zap/options.go b/vendor/go.uber.org/zap/options.go index 43d357ac9..04a3c1e63 100644 --- a/vendor/go.uber.org/zap/options.go +++ b/vendor/go.uber.org/zap/options.go @@ -125,7 +125,11 @@ func IncreaseLevel(lvl zapcore.LevelEnabler) Option { return optionFunc(func(log *Logger) { core, err := zapcore.NewIncreaseLevelCore(log.core, lvl) if err != nil { - fmt.Fprintf(log.errorOutput, "failed to IncreaseLevel: %v\n", err) + _, _ = fmt.Fprintf( + log.errorOutput, + "failed to IncreaseLevel: %v\n", + err, + ) } else { log.core = core } diff --git a/vendor/go.uber.org/zap/sink.go b/vendor/go.uber.org/zap/sink.go index 499772a00..92202280f 100644 --- a/vendor/go.uber.org/zap/sink.go +++ b/vendor/go.uber.org/zap/sink.go @@ -71,7 +71,7 @@ func newSinkRegistry() *sinkRegistry { return sr } -// RegisterScheme registers the given factory for the specific scheme. +// RegisterSink registers the given factory for the specific scheme. func (sr *sinkRegistry) RegisterSink(scheme string, factory func(*url.URL) (Sink, error)) error { sr.mu.Lock() defer sr.mu.Unlock() diff --git a/vendor/go.uber.org/zap/zapcore/buffered_write_syncer.go b/vendor/go.uber.org/zap/zapcore/buffered_write_syncer.go index a40e93b3e..4b426a564 100644 --- a/vendor/go.uber.org/zap/zapcore/buffered_write_syncer.go +++ b/vendor/go.uber.org/zap/zapcore/buffered_write_syncer.go @@ -188,32 +188,33 @@ func (s *BufferedWriteSyncer) flushLoop() { // Stop closes the buffer, cleans up background goroutines, and flushes // remaining unwritten data. func (s *BufferedWriteSyncer) Stop() (err error) { - var stopped bool - // Critical section. - func() { + stopped := func() bool { s.mu.Lock() defer s.mu.Unlock() if !s.initialized { - return + return false } - stopped = s.stopped - if stopped { - return + if s.stopped { + return false } s.stopped = true s.ticker.Stop() close(s.stop) // tell flushLoop to stop - <-s.done // and wait until it has + return true }() - // Don't call Sync on consecutive Stops. + // Not initialized, or already stopped, no need for any cleanup. if !stopped { - err = s.Sync() + return } - return err + // Wait for flushLoop to end outside of the lock, as it may need the lock to complete. + // See https://github.com/uber-go/zap/issues/1428 for details. + <-s.done + + return s.Sync() } diff --git a/vendor/go.uber.org/zap/zapcore/console_encoder.go b/vendor/go.uber.org/zap/zapcore/console_encoder.go index cc2b4e07b..98eea5154 100644 --- a/vendor/go.uber.org/zap/zapcore/console_encoder.go +++ b/vendor/go.uber.org/zap/zapcore/console_encoder.go @@ -105,7 +105,7 @@ func (c consoleEncoder) EncodeEntry(ent Entry, fields []Field) (*buffer.Buffer, if i > 0 { line.AppendString(c.ConsoleSeparator) } - fmt.Fprint(line, arr.elems[i]) + _, _ = fmt.Fprint(line, arr.elems[i]) } putSliceEncoder(arr) diff --git a/vendor/go.uber.org/zap/zapcore/entry.go b/vendor/go.uber.org/zap/zapcore/entry.go index 459a5d7ce..e1fc07a1f 100644 --- a/vendor/go.uber.org/zap/zapcore/entry.go +++ b/vendor/go.uber.org/zap/zapcore/entry.go @@ -201,6 +201,14 @@ func (a CheckWriteAction) OnWrite(ce *CheckedEntry, _ []Field) { var _ CheckWriteHook = CheckWriteAction(0) +// CheckPreWriteHook is a function that transforms an Entry and its Fields +// before they are written to cores. Register one on a CheckedEntry with the +// Before method. +// +// Pre-write hooks run in the order they were added, before any Core's Write +// method is called. They may modify the Entry and Fields freely. +type CheckPreWriteHook func(Entry, []Field) (Entry, []Field) + // CheckedEntry is an Entry together with a collection of Cores that have // already agreed to log it. // @@ -213,6 +221,7 @@ type CheckedEntry struct { dirty bool // best-effort detection of pool misuse after CheckWriteHook cores []Core + before []CheckPreWriteHook } func (ce *CheckedEntry) reset() { @@ -225,6 +234,10 @@ func (ce *CheckedEntry) reset() { ce.cores[i] = nil } ce.cores = ce.cores[:0] + for i := range ce.before { + ce.before[i] = nil + } + ce.before = ce.before[:0] } // Write writes the entry to the stored Cores, returns any errors, and returns @@ -241,19 +254,34 @@ func (ce *CheckedEntry) Write(fields ...Field) { // If the entry is dirty, log an internal error; because the // CheckedEntry is being used after it was returned to the pool, // the message may be an amalgamation from multiple call sites. - fmt.Fprintf(ce.ErrorOutput, "%v Unsafe CheckedEntry re-use near Entry %+v.\n", ce.Time, ce.Entry) + _, _ = fmt.Fprintf( + ce.ErrorOutput, + "%v Unsafe CheckedEntry re-use near Entry %+v.\n", + ce.Time, + ce.Entry, + ) _ = ce.ErrorOutput.Sync() // ignore error } return } ce.dirty = true + ent := ce.Entry + for i := range ce.before { + ent, fields = ce.before[i](ent, fields) + } + var err error for i := range ce.cores { - err = multierr.Append(err, ce.cores[i].Write(ce.Entry, fields)) + err = multierr.Append(err, ce.cores[i].Write(ent, fields)) } if err != nil && ce.ErrorOutput != nil { - fmt.Fprintf(ce.ErrorOutput, "%v write error: %v\n", ce.Time, err) + _, _ = fmt.Fprintf( + ce.ErrorOutput, + "%v write error: %v\n", + ce.Time, + err, + ) _ = ce.ErrorOutput.Sync() // ignore error } @@ -285,6 +313,18 @@ func (ce *CheckedEntry) Should(ent Entry, should CheckWriteAction) *CheckedEntry return ce.After(ent, should) } +// Before adds a pre-write hook that transforms the Entry and Fields before +// they are written to any registered Cores. Multiple hooks run in the order +// they were added. It's safe to call this on nil CheckedEntry references. +func (ce *CheckedEntry) Before(ent Entry, hook CheckPreWriteHook) *CheckedEntry { + if ce == nil { + ce = getCheckedEntry() + ce.Entry = ent + } + ce.before = append(ce.before, hook) + return ce +} + // After sets this CheckEntry's CheckWriteHook, which will be called after this // log entry has been written. It's safe to call this on nil CheckedEntry // references. diff --git a/vendor/go.uber.org/zap/zapcore/lazy_with.go b/vendor/go.uber.org/zap/zapcore/lazy_with.go index 05288d6a8..500809de0 100644 --- a/vendor/go.uber.org/zap/zapcore/lazy_with.go +++ b/vendor/go.uber.org/zap/zapcore/lazy_with.go @@ -23,7 +23,8 @@ package zapcore import "sync" type lazyWithCore struct { - Core + core Core + originalCore Core sync.Once fields []Field } @@ -32,23 +33,45 @@ type lazyWithCore struct { // the logger is written to (or is further chained in a lon-lazy manner). func NewLazyWith(core Core, fields []Field) Core { return &lazyWithCore{ - Core: core, - fields: fields, + core: nil, // core is allocated once `initOnce` is called. + originalCore: core, + fields: fields, } } func (d *lazyWithCore) initOnce() { d.Once.Do(func() { - d.Core = d.Core.With(d.fields) + d.core = d.originalCore.With(d.fields) }) } func (d *lazyWithCore) With(fields []Field) Core { d.initOnce() - return d.Core.With(fields) + return d.core.With(fields) } func (d *lazyWithCore) Check(e Entry, ce *CheckedEntry) *CheckedEntry { + // This is safe because `lazyWithCore` doesn't change the level. + // So we can delagate the level check, any not `initOnce` + // just for the check. + if !d.originalCore.Enabled(e.Level) { + return ce + } + d.initOnce() + return d.core.Check(e, ce) +} + +func (d *lazyWithCore) Enabled(level Level) bool { + // Like above, this is safe because `lazyWithCore` doesn't change the level. + return d.originalCore.Enabled(level) +} + +func (d *lazyWithCore) Write(e Entry, fields []Field) error { + d.initOnce() + return d.core.Write(e, fields) +} + +func (d *lazyWithCore) Sync() error { d.initOnce() - return d.Core.Check(e, ce) + return d.core.Sync() } diff --git a/vendor/go.uber.org/zap/zapcore/level.go b/vendor/go.uber.org/zap/zapcore/level.go index e01a24131..f3e166d67 100644 --- a/vendor/go.uber.org/zap/zapcore/level.go +++ b/vendor/go.uber.org/zap/zapcore/level.go @@ -179,19 +179,19 @@ func (l *Level) UnmarshalText(text []byte) error { func (l *Level) unmarshalText(text []byte) bool { switch string(text) { - case "debug", "DEBUG": + case "debug": *l = DebugLevel - case "info", "INFO", "": // make the zero value useful + case "info", "": // make the zero value useful *l = InfoLevel - case "warn", "WARN": + case "warn", "warning": *l = WarnLevel - case "error", "ERROR": + case "error": *l = ErrorLevel - case "dpanic", "DPANIC": + case "dpanic": *l = DPanicLevel - case "panic", "PANIC": + case "panic": *l = PanicLevel - case "fatal", "FATAL": + case "fatal": *l = FatalLevel default: return false diff --git a/vendor/golang.org/x/sync/errgroup/errgroup.go b/vendor/golang.org/x/sync/errgroup/errgroup.go index 2f45dbc86..f69fd7546 100644 --- a/vendor/golang.org/x/sync/errgroup/errgroup.go +++ b/vendor/golang.org/x/sync/errgroup/errgroup.go @@ -144,8 +144,8 @@ func (g *Group) SetLimit(n int) { g.sem = nil return } - if len(g.sem) != 0 { - panic(fmt.Errorf("errgroup: modify limit while %v goroutines in the group are still active", len(g.sem))) + if active := len(g.sem); active != 0 { + panic(fmt.Errorf("errgroup: modify limit while %v goroutines in the group are still active", active)) } g.sem = make(chan token, n) } diff --git a/vendor/golang.org/x/text/encoding/japanese/eucjp.go b/vendor/golang.org/x/text/encoding/japanese/eucjp.go index 79313fa58..6fce8c5f5 100644 --- a/vendor/golang.org/x/text/encoding/japanese/eucjp.go +++ b/vendor/golang.org/x/text/encoding/japanese/eucjp.go @@ -17,9 +17,9 @@ import ( var EUCJP encoding.Encoding = &eucJP var eucJP = internal.Encoding{ - &internal.SimpleEncoding{eucJPDecoder{}, eucJPEncoder{}}, - "EUC-JP", - identifier.EUCPkdFmtJapanese, + Encoding: &internal.SimpleEncoding{Decoder: eucJPDecoder{}, Encoder: eucJPEncoder{}}, + Name: "EUC-JP", + MIB: identifier.EUCPkdFmtJapanese, } type eucJPDecoder struct{ transform.NopResetter } diff --git a/vendor/golang.org/x/text/encoding/japanese/iso2022jp.go b/vendor/golang.org/x/text/encoding/japanese/iso2022jp.go index 613226df5..6f7bd460a 100644 --- a/vendor/golang.org/x/text/encoding/japanese/iso2022jp.go +++ b/vendor/golang.org/x/text/encoding/japanese/iso2022jp.go @@ -17,9 +17,9 @@ import ( var ISO2022JP encoding.Encoding = &iso2022JP var iso2022JP = internal.Encoding{ - internal.FuncEncoding{iso2022JPNewDecoder, iso2022JPNewEncoder}, - "ISO-2022-JP", - identifier.ISO2022JP, + Encoding: internal.FuncEncoding{Decoder: iso2022JPNewDecoder, Encoder: iso2022JPNewEncoder}, + Name: "ISO-2022-JP", + MIB: identifier.ISO2022JP, } func iso2022JPNewDecoder() transform.Transformer { diff --git a/vendor/golang.org/x/text/encoding/japanese/shiftjis.go b/vendor/golang.org/x/text/encoding/japanese/shiftjis.go index 16fd8a6e3..af65d43d9 100644 --- a/vendor/golang.org/x/text/encoding/japanese/shiftjis.go +++ b/vendor/golang.org/x/text/encoding/japanese/shiftjis.go @@ -18,9 +18,9 @@ import ( var ShiftJIS encoding.Encoding = &shiftJIS var shiftJIS = internal.Encoding{ - &internal.SimpleEncoding{shiftJISDecoder{}, shiftJISEncoder{}}, - "Shift JIS", - identifier.ShiftJIS, + Encoding: &internal.SimpleEncoding{Decoder: shiftJISDecoder{}, Encoder: shiftJISEncoder{}}, + Name: "Shift JIS", + MIB: identifier.ShiftJIS, } type shiftJISDecoder struct{ transform.NopResetter } diff --git a/vendor/golang.org/x/text/encoding/korean/euckr.go b/vendor/golang.org/x/text/encoding/korean/euckr.go index 034337f5d..81c834730 100644 --- a/vendor/golang.org/x/text/encoding/korean/euckr.go +++ b/vendor/golang.org/x/text/encoding/korean/euckr.go @@ -20,9 +20,9 @@ var All = []encoding.Encoding{EUCKR} var EUCKR encoding.Encoding = &eucKR var eucKR = internal.Encoding{ - &internal.SimpleEncoding{eucKRDecoder{}, eucKREncoder{}}, - "EUC-KR", - identifier.EUCKR, + Encoding: &internal.SimpleEncoding{Decoder: eucKRDecoder{}, Encoder: eucKREncoder{}}, + Name: "EUC-KR", + MIB: identifier.EUCKR, } type eucKRDecoder struct{ transform.NopResetter } diff --git a/vendor/golang.org/x/text/encoding/simplifiedchinese/gbk.go b/vendor/golang.org/x/text/encoding/simplifiedchinese/gbk.go index 0e0fabfd6..2f2fd5d44 100644 --- a/vendor/golang.org/x/text/encoding/simplifiedchinese/gbk.go +++ b/vendor/golang.org/x/text/encoding/simplifiedchinese/gbk.go @@ -22,21 +22,21 @@ var ( ) var gbk = internal.Encoding{ - &internal.SimpleEncoding{ - gbkDecoder{gb18030: false}, - gbkEncoder{gb18030: false}, + Encoding: &internal.SimpleEncoding{ + Decoder: gbkDecoder{gb18030: false}, + Encoder: gbkEncoder{gb18030: false}, }, - "GBK", - identifier.GBK, + Name: "GBK", + MIB: identifier.GBK, } var gbk18030 = internal.Encoding{ - &internal.SimpleEncoding{ - gbkDecoder{gb18030: true}, - gbkEncoder{gb18030: true}, + Encoding: &internal.SimpleEncoding{ + Decoder: gbkDecoder{gb18030: true}, + Encoder: gbkEncoder{gb18030: true}, }, - "GB18030", - identifier.GB18030, + Name: "GB18030", + MIB: identifier.GB18030, } type gbkDecoder struct { diff --git a/vendor/golang.org/x/text/encoding/simplifiedchinese/hzgb2312.go b/vendor/golang.org/x/text/encoding/simplifiedchinese/hzgb2312.go index e15b7bf6a..351750e60 100644 --- a/vendor/golang.org/x/text/encoding/simplifiedchinese/hzgb2312.go +++ b/vendor/golang.org/x/text/encoding/simplifiedchinese/hzgb2312.go @@ -17,9 +17,9 @@ import ( var HZGB2312 encoding.Encoding = &hzGB2312 var hzGB2312 = internal.Encoding{ - internal.FuncEncoding{hzGB2312NewDecoder, hzGB2312NewEncoder}, - "HZ-GB2312", - identifier.HZGB2312, + Encoding: internal.FuncEncoding{Decoder: hzGB2312NewDecoder, Encoder: hzGB2312NewEncoder}, + Name: "HZ-GB2312", + MIB: identifier.HZGB2312, } func hzGB2312NewDecoder() transform.Transformer { diff --git a/vendor/golang.org/x/text/encoding/traditionalchinese/big5.go b/vendor/golang.org/x/text/encoding/traditionalchinese/big5.go index 1fcddde08..5046920ee 100644 --- a/vendor/golang.org/x/text/encoding/traditionalchinese/big5.go +++ b/vendor/golang.org/x/text/encoding/traditionalchinese/big5.go @@ -20,9 +20,9 @@ var All = []encoding.Encoding{Big5} var Big5 encoding.Encoding = &big5 var big5 = internal.Encoding{ - &internal.SimpleEncoding{big5Decoder{}, big5Encoder{}}, - "Big5", - identifier.Big5, + Encoding: &internal.SimpleEncoding{Decoder: big5Decoder{}, Encoder: big5Encoder{}}, + Name: "Big5", + MIB: identifier.Big5, } type big5Decoder struct{ transform.NopResetter } diff --git a/vendor/golang.org/x/text/encoding/unicode/unicode.go b/vendor/golang.org/x/text/encoding/unicode/unicode.go index dd99ad14d..ce28c9062 100644 --- a/vendor/golang.org/x/text/encoding/unicode/unicode.go +++ b/vendor/golang.org/x/text/encoding/unicode/unicode.go @@ -60,9 +60,9 @@ func (utf8bomEncoding) NewDecoder() *encoding.Decoder { } var utf8enc = &internal.Encoding{ - &internal.SimpleEncoding{utf8Decoder{}, runes.ReplaceIllFormed()}, - "UTF-8", - identifier.UTF8, + Encoding: &internal.SimpleEncoding{Decoder: utf8Decoder{}, Encoder: runes.ReplaceIllFormed()}, + Name: "UTF-8", + MIB: identifier.UTF8, } type utf8bomDecoder struct { diff --git a/vendor/modules.txt b/vendor/modules.txt index 62ac1e341..6db6be633 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,8 +1,8 @@ # dario.cat/mergo v1.0.1 ## explicit; go 1.13 dario.cat/mergo -# filippo.io/edwards25519 v1.1.0 -## explicit; go 1.20 +# filippo.io/edwards25519 v1.2.0 +## explicit; go 1.24.0 filippo.io/edwards25519 filippo.io/edwards25519/field # github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 @@ -14,7 +14,6 @@ github.com/Azure/go-ansiterm/winterm github.com/DataDog/datadog-go/v5/statsd # github.com/Masterminds/semver v1.5.0 ## explicit -github.com/Masterminds/semver # github.com/Microsoft/go-winio v0.6.2 ## explicit; go 1.21 github.com/Microsoft/go-winio @@ -31,6 +30,9 @@ github.com/containerd/log # github.com/containerd/platforms v0.2.1 ## explicit; go 1.20 github.com/containerd/platforms +# github.com/coreos/go-semver v0.3.1 +## explicit; go 1.8 +github.com/coreos/go-semver/semver # github.com/cpuguy83/dockercfg v0.3.2 ## explicit; go 1.13 github.com/cpuguy83/dockercfg @@ -97,13 +99,15 @@ github.com/go-logr/logr/funcr # github.com/go-logr/stdr v1.2.2 ## explicit; go 1.16 github.com/go-logr/stdr -# github.com/go-mysql-org/go-mysql v1.11.0 -## explicit; go 1.22 +# github.com/go-mysql-org/go-mysql v1.15.0 +## explicit; go 1.25.0 github.com/go-mysql-org/go-mysql/client github.com/go-mysql-org/go-mysql/compress github.com/go-mysql-org/go-mysql/mysql github.com/go-mysql-org/go-mysql/packet github.com/go-mysql-org/go-mysql/replication +github.com/go-mysql-org/go-mysql/serialization +github.com/go-mysql-org/go-mysql/stmt github.com/go-mysql-org/go-mysql/utils # github.com/go-ole/go-ole v1.2.6 ## explicit; go 1.12 @@ -112,8 +116,8 @@ github.com/go-ole/go-ole/oleutil # github.com/go-sql-driver/mysql v1.8.1 ## explicit; go 1.18 github.com/go-sql-driver/mysql -# github.com/goccy/go-json v0.10.2 -## explicit; go 1.12 +# github.com/goccy/go-json v0.10.6 +## explicit; go 1.19 github.com/goccy/go-json github.com/goccy/go-json/internal/decoder github.com/goccy/go-json/internal/encoder @@ -134,13 +138,14 @@ github.com/google/uuid # github.com/hashicorp/go-version v1.7.0 ## explicit github.com/hashicorp/go-version -# github.com/klauspost/compress v1.17.8 -## explicit; go 1.20 +# github.com/klauspost/compress v1.18.6 +## explicit; go 1.24 github.com/klauspost/compress github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 github.com/klauspost/compress/internal/cpuinfo +github.com/klauspost/compress/internal/le github.com/klauspost/compress/internal/snapref github.com/klauspost/compress/zlib github.com/klauspost/compress/zstd @@ -185,14 +190,14 @@ github.com/opencontainers/go-digest ## explicit; go 1.18 github.com/opencontainers/image-spec/specs-go github.com/opencontainers/image-spec/specs-go/v1 -# github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb +# github.com/pingcap/errors v0.11.5-0.20260310054046-9c8b3586e4b2 ## explicit; go 1.14 github.com/pingcap/errors -# github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 -## explicit; go 1.16 +# github.com/pingcap/log v1.1.1-0.20260227082333-572e590d08f1 +## explicit; go 1.25 github.com/pingcap/log -# github.com/pingcap/tidb/pkg/parser v0.0.0-20241118164214-4f047be191be -## explicit; go 1.22 +# github.com/pingcap/tidb/pkg/parser v0.0.0-20260504140133-511dba1dbe17 +## explicit; go 1.25 github.com/pingcap/tidb/pkg/parser/charset github.com/pingcap/tidb/pkg/parser/format github.com/pingcap/tidb/pkg/parser/mysql @@ -214,17 +219,15 @@ github.com/shirou/gopsutil/v4/internal/common github.com/shirou/gopsutil/v4/mem github.com/shirou/gopsutil/v4/net github.com/shirou/gopsutil/v4/process -# github.com/shopspring/decimal v1.2.0 -## explicit; go 1.13 +# github.com/shopspring/decimal v1.4.0 +## explicit; go 1.10 github.com/shopspring/decimal # github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 ## explicit -github.com/siddontang/go-log/log -github.com/siddontang/go-log/loggers # github.com/sirupsen/logrus v1.9.3 ## explicit; go 1.13 github.com/sirupsen/logrus -# github.com/stretchr/testify v1.10.0 +# github.com/stretchr/testify v1.11.1 ## explicit; go 1.17 github.com/stretchr/testify/assert github.com/stretchr/testify/assert/yaml @@ -291,7 +294,7 @@ go.uber.org/atomic # go.uber.org/multierr v1.11.0 ## explicit; go 1.19 go.uber.org/multierr -# go.uber.org/zap v1.27.0 +# go.uber.org/zap v1.28.0 ## explicit; go 1.19 go.uber.org/zap go.uber.org/zap/buffer @@ -313,8 +316,8 @@ golang.org/x/crypto/internal/alias golang.org/x/crypto/internal/poly1305 golang.org/x/crypto/ssh golang.org/x/crypto/ssh/internal/bcrypt_pbkdf -# golang.org/x/sync v0.18.0 -## explicit; go 1.24.0 +# golang.org/x/sync v0.20.0 +## explicit; go 1.25.0 golang.org/x/sync/errgroup # golang.org/x/sys v0.38.0 ## explicit; go 1.24.0 @@ -325,8 +328,8 @@ golang.org/x/sys/windows # golang.org/x/term v0.37.0 ## explicit; go 1.24.0 golang.org/x/term -# golang.org/x/text v0.31.0 -## explicit; go 1.24.0 +# golang.org/x/text v0.36.0 +## explicit; go 1.25.0 golang.org/x/text/encoding golang.org/x/text/encoding/charmap golang.org/x/text/encoding/internal