diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..56adc99 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Generated protobuf code: collapse in diffs/reviews and exclude from language stats. +internal/zetasketch/*.pb.go linguist-generated=true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 0f90870..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Lint -on: - push: - branches: - - main - pull_request: - branches: - - main -jobs: - golangci: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Run lint - uses: golangci/golangci-lint-action@v2 - with: - version: latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6def6da..09658ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,23 +8,4 @@ on: - main jobs: go: - runs-on: ubuntu-latest - strategy: - matrix: - go-version: [1.26.x] - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: ~/go/pkg/mod - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go-version }} - - name: Run tests - run: make test + uses: bsm/misc/.github/workflows/test-go.yml@main diff --git a/go.mod b/go.mod index f85bf91..910fb48 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,4 @@ module github.com/bsm/zetasketch go 1.26.0 -require ( - github.com/bsm/ginkgo v1.16.5 - github.com/bsm/gomega v1.27.10 - google.golang.org/protobuf v1.36.11 -) +require google.golang.org/protobuf v1.36.11 diff --git a/go.sum b/go.sum index 199eaec..296be18 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,3 @@ -github.com/bsm/ginkgo v1.16.5 h1:uTeeWv0Yx1PnDeCk76PFyGrOMVw3D+r9bTNKNcIjDdQ= -github.com/bsm/ginkgo v1.16.5/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk= -github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= -github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/hll.go b/hll.go index 2a627b7..27f0715 100644 --- a/hll.go +++ b/hll.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/bsm/zetasketch/hllplus" - "github.com/bsm/zetasketch/internal/zetasketch" pb "github.com/bsm/zetasketch/internal/zetasketch" "google.golang.org/protobuf/proto" ) @@ -85,7 +84,7 @@ func (h *HLL) proto() *pb.AggregatorStateProto { EncodingVersion: &encodingVersion, NumValues: &numValues, } - proto.SetExtension(msg, zetasketch.E_HyperloglogplusUniqueState, h.h.Proto()) + proto.SetExtension(msg, pb.E_HyperloglogplusUniqueState, h.h.Proto()) return msg } @@ -100,7 +99,7 @@ func (h *HLL) fromProto(msg *pb.AggregatorStateProto) error { return fmt.Errorf("incompatible binary message: no num values") } - ext := proto.GetExtension(msg, zetasketch.E_HyperloglogplusUniqueState) + ext := proto.GetExtension(msg, pb.E_HyperloglogplusUniqueState) hState, ok := ext.(*pb.HyperLogLogPlusUniqueStateProto) if !ok { return fmt.Errorf("incompatible binary message: invalid HyperLogLog++ state") diff --git a/hll_test.go b/hll_test.go index a6411f7..0baa670 100644 --- a/hll_test.go +++ b/hll_test.go @@ -1,57 +1,81 @@ package zetasketch_test import ( - "github.com/bsm/zetasketch" + "testing" - . "github.com/bsm/ginkgo" - . "github.com/bsm/gomega" + "github.com/bsm/zetasketch" ) -var _ = Describe("HLL", func() { - var subject *zetasketch.HLL - var _ zetasketch.Aggregator = subject - - BeforeEach(func() { - subject = zetasketch.NewHLL(nil) - - for i := 0; i < 1_000; i++ { - subject.Add(zetasketch.Uint64Value(uint64(i))) - } - for i := 500; i < 1_000; i++ { - subject.Add(zetasketch.Uint64Value(uint64(i))) - } - }) - - It("should count values", func() { - Expect(subject.NumValues()).To(BeNumerically("==", 1_500)) - }) - - It("should estimate uniques", func() { - Expect(subject.Result()).To(BeNumerically("==", 1_000)) - }) - - It("should merge", func() { - other := zetasketch.NewHLL(nil) - for i := 800; i < 1_200; i++ { - other.Add(zetasketch.Uint64Value(uint64(i))) - } - - Expect(subject.Merge(other)).To(Succeed()) - Expect(subject.NumValues()).To(BeNumerically("==", 1_900)) - Expect(subject.Result()).To(BeNumerically("==", 1_207)) - - // `other` is not modified: - Expect(other.NumValues()).To(BeNumerically("==", 400)) - Expect(other.Result()).To(BeNumerically("==", 400)) - }) - - It("should marshal/unmarshal binary", func() { - data, err := subject.MarshalBinary() - Expect(err).NotTo(HaveOccurred()) - - subject = new(zetasketch.HLL) - Expect(subject.UnmarshalBinary(data)).To(Succeed()) - Expect(subject.NumValues()).To(BeNumerically("==", 1_500)) - Expect(subject.Result()).To(BeNumerically("==", 1_000)) - }) -}) +var _ zetasketch.Aggregator = (*zetasketch.HLL)(nil) + +func newTestHLL() *zetasketch.HLL { + subject := zetasketch.NewHLL(nil) + for i := range 1_000 { + subject.Add(zetasketch.Uint64Value(uint64(i))) + } + for i := 500; i < 1_000; i++ { + subject.Add(zetasketch.Uint64Value(uint64(i))) + } + return subject +} + +func TestHLL_NumValues(t *testing.T) { + subject := newTestHLL() + if got, exp := subject.NumValues(), int64(1_500); got != exp { + t.Errorf("got %d, want %d", got, exp) + } +} + +func TestHLL_Result(t *testing.T) { + subject := newTestHLL() + if got, exp := subject.Result(), int64(1_000); got != exp { + t.Errorf("got %d, want %d", got, exp) + } +} + +func TestHLL_Merge(t *testing.T) { + subject := newTestHLL() + + other := zetasketch.NewHLL(nil) + for i := 800; i < 1_200; i++ { + other.Add(zetasketch.Uint64Value(uint64(i))) + } + + if err := subject.Merge(other); err != nil { + t.Fatal(err) + } + if got, exp := subject.NumValues(), int64(1_900); got != exp { + t.Errorf("NumValues: got %d, want %d", got, exp) + } + if got, exp := subject.Result(), int64(1_207); got != exp { + t.Errorf("Result: got %d, want %d", got, exp) + } + + // `other` is not modified: + if got, exp := other.NumValues(), int64(400); got != exp { + t.Errorf("other.NumValues: got %d, want %d", got, exp) + } + if got, exp := other.Result(), int64(400); got != exp { + t.Errorf("other.Result: got %d, want %d", got, exp) + } +} + +func TestHLL_MarshalBinary(t *testing.T) { + subject := newTestHLL() + + data, err := subject.MarshalBinary() + if err != nil { + t.Fatal(err) + } + + subject = new(zetasketch.HLL) + if err := subject.UnmarshalBinary(data); err != nil { + t.Fatal(err) + } + if got, exp := subject.NumValues(), int64(1_500); got != exp { + t.Errorf("NumValues: got %d, want %d", got, exp) + } + if got, exp := subject.Result(), int64(1_000); got != exp { + t.Errorf("Result: got %d, want %d", got, exp) + } +} diff --git a/hllplus/data.go b/hllplus/data.go index 26c7645..fb8ae5c 100644 --- a/hllplus/data.go +++ b/hllplus/data.go @@ -17,7 +17,6 @@ package hllplus import ( - "math" "math/bits" "sort" ) @@ -748,9 +747,10 @@ func closestBiases(estimate float64, precision uint8) weightedBiases { res := make(weightedBiases, 0, max-min) for i := min; i < max; i++ { + d := means[i] - estimate res = append(res, weightedBias{ Bias: biases[i], - Distance: math.Pow(means[i]-estimate, 2), + Distance: d * d, }) } diff --git a/hllplus/data_test.go b/hllplus/data_test.go index 26dce33..c3e8921 100644 --- a/hllplus/data_test.go +++ b/hllplus/data_test.go @@ -1,24 +1,35 @@ package hllplus_test import ( - "github.com/bsm/zetasketch/hllplus" + "math" + "testing" - . "github.com/bsm/ginkgo" - . "github.com/bsm/gomega" + "github.com/bsm/zetasketch/hllplus" ) -var _ = Describe("Data", func() { - It("should estimate bias", func() { - Expect(hllplus.EstimateBias(0, 15)).To(BeNumerically("==", 0.0)) - Expect(hllplus.EstimateBias(1, 15)).To(BeNumerically("==", 0.0)) - Expect(hllplus.EstimateBias(10_000, 15)).To(BeNumerically("==", 0.0)) - Expect(hllplus.EstimateBias(100_000, 15)).To(BeNumerically("~", 888.1, 0.1)) - Expect(hllplus.EstimateBias(200_000, 15)).To(BeNumerically("==", 0.0)) +func TestEstimateBias(t *testing.T) { + cases := []struct { + e float64 + p uint8 + exp float64 + delta float64 + }{ + {0, 15, 0.0, 0}, + {1, 15, 0.0, 0}, + {10_000, 15, 0.0, 0}, + {100_000, 15, 888.1, 0.1}, + {200_000, 15, 0.0, 0}, - Expect(hllplus.EstimateBias(50_000, 13)).To(BeNumerically("==", 0.0)) - Expect(hllplus.EstimateBias(50_000, 14)).To(BeNumerically("~", 449.7, 0.1)) - Expect(hllplus.EstimateBias(50_000, 15)).To(BeNumerically("~", 7820.2, 0.1)) - Expect(hllplus.EstimateBias(50_000, 16)).To(BeNumerically("~", 44513.2, 0.1)) - Expect(hllplus.EstimateBias(50_000, 17)).To(BeNumerically("==", 0.0)) - }) -}) + {50_000, 13, 0.0, 0}, + {50_000, 14, 449.7, 0.1}, + {50_000, 15, 7820.2, 0.1}, + {50_000, 16, 44513.2, 0.1}, + {50_000, 17, 0.0, 0}, + } + for _, tc := range cases { + got := hllplus.EstimateBias(tc.e, tc.p) + if math.Abs(got-tc.exp) > tc.delta { + t.Errorf("EstimateBias(%v, %d) = %v, want %v (±%v)", tc.e, tc.p, got, tc.exp, tc.delta) + } + } +} diff --git a/hllplus/ext_test.go b/hllplus/ext_test.go index 7e95c17..9490037 100644 --- a/hllplus/ext_test.go +++ b/hllplus/ext_test.go @@ -6,10 +6,7 @@ func EstimateBias(e float64, p uint8) float64 { } func NewNormal(precision uint8) (*HLL, error) { - pp := precision + 5 - if pp > MaxSparsePrecision { - pp = MaxSparsePrecision - } + pp := min(precision+5, MaxSparsePrecision) s, err := New(precision, pp) if err != nil { diff --git a/hllplus/hllplus_test.go b/hllplus/hllplus_test.go index d759e69..7c932b6 100644 --- a/hllplus/hllplus_test.go +++ b/hllplus/hllplus_test.go @@ -1,322 +1,489 @@ package hllplus_test import ( + "fmt" "math/rand" "testing" "github.com/bsm/zetasketch/hllplus" - - . "github.com/bsm/ginkgo" - . "github.com/bsm/ginkgo/extensions/table" - . "github.com/bsm/gomega" ) -var _ = Describe("HLL", func() { - var subject *hllplus.HLL - var rnd *rand.Rand - - BeforeEach(func() { - rnd = rand.New(rand.NewSource(33)) - }) - - DescribeTable("estimate normal (800 unique)", - func(p int, exp int) { - subject, _ = hllplus.NewNormal(uint8(p)) - for i := 0; i < 800; i++ { +func TestHLL_estimateNormal800(t *testing.T) { + cases := []struct { + p int + exp int64 + }{ + {10, 800}, + {11, 794}, + {12, 788}, + {13, 793}, + {14, 791}, + {15, 793}, + {16, 795}, + {17, 797}, + {18, 799}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("p=%d", tc.p), func(t *testing.T) { + rnd := rand.New(rand.NewSource(33)) + subject, _ := hllplus.NewNormal(uint8(tc.p)) + for range 800 { subject.Add(rnd.Uint64()) } - Expect(subject.IsSparse()).To(BeFalse()) - Expect(subject.Estimate()).To(Equal(int64(exp))) - }, - Entry("p=10", 10, 800), - Entry("p=11", 11, 794), - Entry("p=12", 12, 788), - Entry("p=13", 13, 793), - Entry("p=14", 14, 791), - Entry("p=15", 15, 793), - Entry("p=16", 16, 795), - Entry("p=17", 17, 797), - Entry("p=18", 18, 799), - ) - - DescribeTable("estimate sparse (800 unique)", - func(p int, exp int) { - subject, _ = hllplus.New(uint8(p-5), uint8(p)) - for i := 0; i < 800; i++ { + if subject.IsSparse() { + t.Error("expected normal representation") + } + if got := subject.Estimate(); got != tc.exp { + t.Errorf("got %d, want %d", got, tc.exp) + } + }) + } +} + +func TestHLL_estimateSparse800(t *testing.T) { + cases := []struct { + p int + exp int64 + }{ + {16, 795}, + {17, 798}, + {18, 799}, + {19, 799}, + {20, 799}, + {21, 799}, + {22, 800}, + {23, 800}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("p=%d", tc.p), func(t *testing.T) { + rnd := rand.New(rand.NewSource(33)) + subject, _ := hllplus.New(uint8(tc.p-5), uint8(tc.p)) + for range 800 { subject.Add(rnd.Uint64()) } - Expect(subject.IsSparse()).To(BeTrue()) - Expect(subject.Estimate()).To(Equal(int64(exp))) - }, - Entry("p=16", 16, 795), - Entry("p=17", 17, 798), - Entry("p=18", 18, 799), - Entry("p=19", 19, 799), - Entry("p=20", 20, 799), - Entry("p=21", 21, 799), - Entry("p=22", 22, 800), - Entry("p=23", 23, 800), - ) - - DescribeTable("estimate normal (200k unique)", - func(p int, exp int) { - subject, _ = hllplus.NewNormal(uint8(p)) - for i := 0; i < 200_000; i++ { + if !subject.IsSparse() { + t.Error("expected sparse representation") + } + if got := subject.Estimate(); got != tc.exp { + t.Errorf("got %d, want %d", got, tc.exp) + } + }) + } +} + +func TestHLL_estimateNormal200k(t *testing.T) { + cases := []struct { + p int + exp int64 + }{ + {10, 199197}, + {11, 204855}, + {12, 204356}, + {13, 202958}, + {14, 202977}, + {15, 201496}, + {16, 201398}, + {17, 201208}, + {18, 200999}, + {19, 200567}, + {20, 200032}, + {21, 200013}, + {22, 200003}, + {23, 199989}, + {24, 200026}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("p=%d", tc.p), func(t *testing.T) { + rnd := rand.New(rand.NewSource(33)) + subject, _ := hllplus.NewNormal(uint8(tc.p)) + for range 200_000 { subject.Add(rnd.Uint64()) } - Expect(subject.IsSparse()).To(BeFalse()) - Expect(subject.Estimate()).To(Equal(int64(exp))) - }, - Entry("p=10", 10, 199197), - Entry("p=11", 11, 204855), - Entry("p=12", 12, 204356), - Entry("p=13", 13, 202958), - Entry("p=14", 14, 202977), - Entry("p=15", 15, 201496), - Entry("p=16", 16, 201398), - Entry("p=17", 17, 201208), - Entry("p=18", 18, 200999), - Entry("p=19", 19, 200567), - Entry("p=20", 20, 200032), - Entry("p=21", 21, 200013), - Entry("p=22", 22, 200003), - Entry("p=23", 23, 199989), - Entry("p=24", 24, 200026), - ) - - DescribeTable("estimate sparse (200k unique)", - func(p int, exp int) { - subject, _ = hllplus.New(uint8(p-5), uint8(p)) - for i := 0; i < 200_000; i++ { + if subject.IsSparse() { + t.Error("expected normal representation") + } + if got := subject.Estimate(); got != tc.exp { + t.Errorf("got %d, want %d", got, tc.exp) + } + }) + } +} + +func TestHLL_estimateSparse200k(t *testing.T) { + cases := []struct { + p int + exp int64 + }{ + {24, 200039}, + {25, 200048}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("p=%d", tc.p), func(t *testing.T) { + rnd := rand.New(rand.NewSource(33)) + subject, _ := hllplus.New(uint8(tc.p-5), uint8(tc.p)) + for range 200_000 { subject.Add(rnd.Uint64()) } - Expect(subject.IsSparse()).To(BeTrue()) - Expect(subject.Estimate()).To(Equal(int64(exp))) - }, - Entry("p=24", 24, 200039), - Entry("p=25", 25, 200048), - ) - - DescribeTable("estimate normal (100k unique + 2x50k)", - func(p int, exp int) { - subject, _ = hllplus.NewNormal(uint8(p)) - for i := 0; i < 100_000; i++ { + if !subject.IsSparse() { + t.Error("expected sparse representation") + } + if got := subject.Estimate(); got != tc.exp { + t.Errorf("got %d, want %d", got, tc.exp) + } + }) + } +} + +func TestHLL_estimateNormal100k2x50k(t *testing.T) { + cases := []struct { + p int + exp int64 + }{ + {10, 148131}, + {11, 152153}, + {12, 152338}, + {13, 152453}, + {14, 150853}, + {15, 150458}, + {16, 150811}, + {17, 150795}, + {18, 150592}, + {19, 150265}, + {20, 149879}, + {21, 149939}, + {22, 150005}, + {23, 149946}, + {24, 149988}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("p=%d", tc.p), func(t *testing.T) { + rnd := rand.New(rand.NewSource(33)) + subject, _ := hllplus.NewNormal(uint8(tc.p)) + for range 100_000 { subject.Add(rnd.Uint64()) } - for i := 0; i < 50_000; i++ { + for range 50_000 { h := rnd.Uint64() subject.Add(h) subject.Add(h) } - Expect(subject.IsSparse()).To(BeFalse()) - Expect(subject.Estimate()).To(Equal(int64(exp))) - }, - Entry("p=10", 10, 148131), - Entry("p=11", 11, 152153), - Entry("p=12", 12, 152338), - Entry("p=13", 13, 152453), - Entry("p=14", 14, 150853), - Entry("p=15", 15, 150458), - Entry("p=16", 16, 150811), - Entry("p=17", 17, 150795), - Entry("p=18", 18, 150592), - Entry("p=19", 19, 150265), - Entry("p=20", 20, 149879), - Entry("p=21", 21, 149939), - Entry("p=22", 22, 150005), - Entry("p=23", 23, 149946), - Entry("p=24", 24, 149988), - ) - - DescribeTable("estimate sparse (100k unique + 2x50k)", - func(p int, exp int) { - subject, _ = hllplus.New(uint8(p-5), uint8(p)) - for i := 0; i < 100_000; i++ { + if subject.IsSparse() { + t.Error("expected normal representation") + } + if got := subject.Estimate(); got != tc.exp { + t.Errorf("got %d, want %d", got, tc.exp) + } + }) + } +} + +func TestHLL_estimateSparse100k2x50k(t *testing.T) { + cases := []struct { + p int + exp int64 + }{ + {23, 149969}, + {24, 149998}, + {25, 150012}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("p=%d", tc.p), func(t *testing.T) { + rnd := rand.New(rand.NewSource(33)) + subject, _ := hllplus.New(uint8(tc.p-5), uint8(tc.p)) + for range 100_000 { subject.Add(rnd.Uint64()) } - for i := 0; i < 50_000; i++ { + for range 50_000 { h := rnd.Uint64() subject.Add(h) subject.Add(h) } - Expect(subject.IsSparse()).To(BeTrue()) - Expect(subject.Estimate()).To(Equal(int64(exp))) - }, - Entry("p=23", 23, 149969), - Entry("p=24", 24, 149998), - Entry("p=25", 25, 150012), - ) - - It("should estimate sparse (repetitive)", func() { - subject, _ = hllplus.New(12, 17) - - // same element added again and again - Add(N), Estimate() -> Flush(), repeat: - for i := 0; i < 100; i++ { - subject.Add(1) - Expect(subject.Estimate()).To(BeNumerically("==", 1)) - } - - // sanity check that HLL has not been normalized: - Expect(subject.IsSparse()).To(BeTrue()) - }) - - It("should normalize", func() { - subject, _ = hllplus.New(12, 17) - for i := 0; i < 3_084; i++ { - subject.Add(rnd.Uint64()) - } - Expect(subject.IsSparse()).To(BeTrue()) - Expect(subject.Estimate()).To(BeNumerically("==", 3_083)) - - subject.Add(rnd.Uint64()) - Expect(subject.IsSparse()).To(BeFalse()) - Expect(subject.Estimate()).To(BeNumerically("==", 3_072)) - }) - - It("should downgrade", func() { - s1, _ := hllplus.NewNormal(15) - s2, _ := hllplus.NewNormal(12) - - for i := 0; i < 100_000; i++ { - n := rnd.Uint64() - s1.Add(n) - s2.Add(n) - } - - Expect(s1.Estimate()).To(Equal(int64(99879))) - Expect(s2.Estimate()).To(Equal(int64(100680))) - - Expect(s1.Precision()).To(Equal(uint8(15))) - Expect(s1.SparsePrecision()).To(Equal(uint8(20))) - Expect(s1.Downgrade(12, 17)).To(Succeed()) - Expect(s1.Precision()).To(Equal(uint8(12))) - Expect(s1.SparsePrecision()).To(Equal(uint8(17))) - - Expect(s1.Estimate()).To(Equal(int64(100680))) - Expect(s2.Estimate()).To(Equal(int64(100680))) - }) - - Describe("merge", func() { - var s1, s2, s3 *hllplus.HLL - - BeforeEach(func() { - s1, _ = hllplus.NewNormal(15) - s2, _ = hllplus.NewNormal(15) - s3, _ = hllplus.NewNormal(12) - - for i := 0; i < 50_000; i++ { - n := rnd.Uint64() - s1.Add(n) - s2.Add(n) - s3.Add(n) + if !subject.IsSparse() { + t.Error("expected sparse representation") } - for i := 0; i < 50_000; i++ { - s1.Add(rnd.Uint64()) - s2.Add(rnd.Uint64()) - s3.Add(rnd.Uint64()) + if got := subject.Estimate(); got != tc.exp { + t.Errorf("got %d, want %d", got, tc.exp) } - - Expect(s1.Estimate()).To(Equal(int64(100324))) - Expect(s2.Estimate()).To(Equal(int64(100168))) - Expect(s3.Estimate()).To(Equal(int64(100464))) - }) - - It("should support equal precision", func() { - s1.Merge(s2) - Expect(s1.Estimate()).To(Equal(int64(150794))) - }) - - It("should support targets with lower precision", func() { - s1.Merge(s3) - Expect(s1.Estimate()).To(Equal(int64(154744))) - Expect(s1.Precision()).To(Equal(uint8(12))) - Expect(s1.SparsePrecision()).To(Equal(uint8(17))) }) + } +} - It("should support targets with higher precision", func() { - s3.Merge(s1) - Expect(s3.Estimate()).To(Equal(int64(154744))) - Expect(s3.Precision()).To(Equal(uint8(12))) - Expect(s3.SparsePrecision()).To(Equal(uint8(17))) - }) - - It("should succeed if target is empty", func() { - subject, _ = hllplus.NewNormal(15) - Expect(func() { subject.Merge(s1) }).NotTo(Panic()) - - // just a straight copy of s1: - Expect(subject.Estimate()).To(Equal(s1.Estimate())) - Expect(subject.Precision()).To(Equal(s1.Precision())) - Expect(subject.SparsePrecision()).To(Equal(s1.SparsePrecision())) - }) - }) - - Describe("proto", func() { - It("should init normal", func() { - subject, _ = hllplus.New(12, 17) - for i := 0; i < 10_000; i++ { - subject.Add(rnd.Uint64()) - } - Expect(subject.IsSparse()).To(BeFalse()) - Expect(subject.Estimate()).To(BeNumerically("==", 9_912)) - - msg := subject.Proto() - - // both precisions are always stored: - Expect(msg.GetPrecisionOrNumBuckets()).To(BeNumerically("==", 12)) - Expect(msg.GetSparsePrecisionOrNumBuckets()).To(BeNumerically("==", 17)) +func TestHLL_estimateSparseRepetitive(t *testing.T) { + subject, _ := hllplus.New(12, 17) - // expect normal representation: - Expect(msg.GetData()).NotTo(BeEmpty()) + // same element added again and again - Add(N), Estimate() -> Flush(), repeat: + for range 100 { + subject.Add(1) + if got := subject.Estimate(); got != 1 { + t.Fatalf("got %d, want 1", got) + } + } - // expect NO sparse representation: - Expect(msg.SparseSize).To(BeNil()) - Expect(msg.GetSparseData()).To(BeNil()) + // sanity check that HLL has not been normalized: + if !subject.IsSparse() { + t.Error("expected sparse representation") + } +} - // init back from proto: - subject, err := hllplus.NewFromProto(msg) - Expect(err).NotTo(HaveOccurred()) - Expect(subject.IsSparse()).To(BeFalse()) - Expect(subject.Precision()).To(BeNumerically("==", 12)) - Expect(subject.SparsePrecision()).To(BeNumerically("==", 17)) - Expect(subject.Estimate()).To(BeNumerically("==", 9_912)) - }) +func TestHLL_normalize(t *testing.T) { + rnd := rand.New(rand.NewSource(33)) + subject, _ := hllplus.New(12, 17) + for range 3_084 { + subject.Add(rnd.Uint64()) + } + if !subject.IsSparse() { + t.Error("expected sparse representation") + } + if got := subject.Estimate(); got != 3_083 { + t.Errorf("got %d, want 3083", got) + } + + subject.Add(rnd.Uint64()) + if subject.IsSparse() { + t.Error("expected normal representation") + } + if got := subject.Estimate(); got != 3_072 { + t.Errorf("got %d, want 3072", got) + } +} - It("should init sparse", func() { - subject, _ = hllplus.New(12, 17) - for i := 0; i < 800; i++ { - subject.Add(rnd.Uint64()) - } - Expect(subject.IsSparse()).To(BeTrue()) - Expect(subject.Estimate()).To(BeNumerically("==", 798)) +func TestHLL_downgrade(t *testing.T) { + rnd := rand.New(rand.NewSource(33)) + s1, _ := hllplus.NewNormal(15) + s2, _ := hllplus.NewNormal(12) + + for range 100_000 { + n := rnd.Uint64() + s1.Add(n) + s2.Add(n) + } + + if got := s1.Estimate(); got != 99879 { + t.Errorf("s1.Estimate: got %d, want 99879", got) + } + if got := s2.Estimate(); got != 100680 { + t.Errorf("s2.Estimate: got %d, want 100680", got) + } + + if got := s1.Precision(); got != 15 { + t.Errorf("s1.Precision: got %d, want 15", got) + } + if got := s1.SparsePrecision(); got != 20 { + t.Errorf("s1.SparsePrecision: got %d, want 20", got) + } + if err := s1.Downgrade(12, 17); err != nil { + t.Fatal(err) + } + if got := s1.Precision(); got != 12 { + t.Errorf("s1.Precision: got %d, want 12", got) + } + if got := s1.SparsePrecision(); got != 17 { + t.Errorf("s1.SparsePrecision: got %d, want 17", got) + } + + if got := s1.Estimate(); got != 100680 { + t.Errorf("s1.Estimate: got %d, want 100680", got) + } + if got := s2.Estimate(); got != 100680 { + t.Errorf("s2.Estimate: got %d, want 100680", got) + } +} - msg := subject.Proto() +// newMergeFixture builds three sketches sharing 50k values and adds 50k distinct +// values to each, matching the original merge spec's BeforeEach. +func newMergeFixture(t *testing.T) (s1, s2, s3 *hllplus.HLL) { + t.Helper() + + rnd := rand.New(rand.NewSource(33)) + s1, _ = hllplus.NewNormal(15) + s2, _ = hllplus.NewNormal(15) + s3, _ = hllplus.NewNormal(12) + + for range 50_000 { + n := rnd.Uint64() + s1.Add(n) + s2.Add(n) + s3.Add(n) + } + for range 50_000 { + s1.Add(rnd.Uint64()) + s2.Add(rnd.Uint64()) + s3.Add(rnd.Uint64()) + } + + if got := s1.Estimate(); got != 100324 { + t.Fatalf("s1.Estimate: got %d, want 100324", got) + } + if got := s2.Estimate(); got != 100168 { + t.Fatalf("s2.Estimate: got %d, want 100168", got) + } + if got := s3.Estimate(); got != 100464 { + t.Fatalf("s3.Estimate: got %d, want 100464", got) + } + return s1, s2, s3 +} - // both precisions are always stored: - Expect(msg.GetPrecisionOrNumBuckets()).To(BeNumerically("==", 12)) - Expect(msg.GetSparsePrecisionOrNumBuckets()).To(BeNumerically("==", 17)) +func TestHLL_merge_equalPrecision(t *testing.T) { + s1, s2, _ := newMergeFixture(t) + s1.Merge(s2) + if got := s1.Estimate(); got != 150794 { + t.Errorf("got %d, want 150794", got) + } +} - // expect NO normal representation: - Expect(msg.GetData()).To(BeEmpty()) +func TestHLL_merge_lowerPrecisionTarget(t *testing.T) { + s1, _, s3 := newMergeFixture(t) + s1.Merge(s3) + if got := s1.Estimate(); got != 154744 { + t.Errorf("got %d, want 154744", got) + } + if got := s1.Precision(); got != 12 { + t.Errorf("Precision: got %d, want 12", got) + } + if got := s1.SparsePrecision(); got != 17 { + t.Errorf("SparsePrecision: got %d, want 17", got) + } +} - // expect sparse representation: - Expect(msg.GetSparseSize()).To(BeNumerically("==", 796)) // hash/rand collisions are fine, that's why it is != 800 - Expect(msg.GetSparseData()).NotTo(BeEmpty()) +func TestHLL_merge_higherPrecisionTarget(t *testing.T) { + s1, _, s3 := newMergeFixture(t) + s3.Merge(s1) + if got := s3.Estimate(); got != 154744 { + t.Errorf("got %d, want 154744", got) + } + if got := s3.Precision(); got != 12 { + t.Errorf("Precision: got %d, want 12", got) + } + if got := s3.SparsePrecision(); got != 17 { + t.Errorf("SparsePrecision: got %d, want 17", got) + } +} - // init back from proto: - subject, err := hllplus.NewFromProto(msg) - Expect(err).NotTo(HaveOccurred()) - Expect(subject.IsSparse()).To(BeTrue()) - Expect(subject.Precision()).To(BeNumerically("==", 12)) - Expect(subject.SparsePrecision()).To(BeNumerically("==", 17)) - Expect(subject.Estimate()).To(BeNumerically("==", 798)) - }) - }) -}) +func TestHLL_merge_emptyTarget(t *testing.T) { + s1, _, _ := newMergeFixture(t) + + subject, _ := hllplus.NewNormal(15) + subject.Merge(s1) + + // just a straight copy of s1: + if got, exp := subject.Estimate(), s1.Estimate(); got != exp { + t.Errorf("Estimate: got %d, want %d", got, exp) + } + if got, exp := subject.Precision(), s1.Precision(); got != exp { + t.Errorf("Precision: got %d, want %d", got, exp) + } + if got, exp := subject.SparsePrecision(), s1.SparsePrecision(); got != exp { + t.Errorf("SparsePrecision: got %d, want %d", got, exp) + } +} -// -------------------------------------------------------------------- +func TestHLL_proto_initNormal(t *testing.T) { + rnd := rand.New(rand.NewSource(33)) + subject, _ := hllplus.New(12, 17) + for range 10_000 { + subject.Add(rnd.Uint64()) + } + if subject.IsSparse() { + t.Error("expected normal representation") + } + if got := subject.Estimate(); got != 9_912 { + t.Errorf("got %d, want 9912", got) + } + + msg := subject.Proto() + + // both precisions are always stored: + if got := msg.GetPrecisionOrNumBuckets(); got != 12 { + t.Errorf("precision: got %d, want 12", got) + } + if got := msg.GetSparsePrecisionOrNumBuckets(); got != 17 { + t.Errorf("sparse precision: got %d, want 17", got) + } + + // expect normal representation: + if len(msg.GetData()) == 0 { + t.Error("expected non-empty data") + } + + // expect NO sparse representation: + if msg.SparseSize != nil { + t.Error("expected nil sparse size") + } + if msg.GetSparseData() != nil { + t.Error("expected nil sparse data") + } + + // init back from proto: + restored, err := hllplus.NewFromProto(msg) + if err != nil { + t.Fatal(err) + } + if restored.IsSparse() { + t.Error("expected normal representation") + } + if got := restored.Precision(); got != 12 { + t.Errorf("precision: got %d, want 12", got) + } + if got := restored.SparsePrecision(); got != 17 { + t.Errorf("sparse precision: got %d, want 17", got) + } + if got := restored.Estimate(); got != 9_912 { + t.Errorf("got %d, want 9912", got) + } +} -func TestSuite(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "zetasketch/hllplus") +func TestHLL_proto_initSparse(t *testing.T) { + rnd := rand.New(rand.NewSource(33)) + subject, _ := hllplus.New(12, 17) + for range 800 { + subject.Add(rnd.Uint64()) + } + if !subject.IsSparse() { + t.Error("expected sparse representation") + } + if got := subject.Estimate(); got != 798 { + t.Errorf("got %d, want 798", got) + } + + msg := subject.Proto() + + // both precisions are always stored: + if got := msg.GetPrecisionOrNumBuckets(); got != 12 { + t.Errorf("precision: got %d, want 12", got) + } + if got := msg.GetSparsePrecisionOrNumBuckets(); got != 17 { + t.Errorf("sparse precision: got %d, want 17", got) + } + + // expect NO normal representation: + if len(msg.GetData()) != 0 { + t.Error("expected empty data") + } + + // expect sparse representation: + // hash/rand collisions are fine, that's why it is != 800 + if got := msg.GetSparseSize(); got != 796 { + t.Errorf("sparse size: got %d, want 796", got) + } + if len(msg.GetSparseData()) == 0 { + t.Error("expected non-empty sparse data") + } + + // init back from proto: + restored, err := hllplus.NewFromProto(msg) + if err != nil { + t.Fatal(err) + } + if !restored.IsSparse() { + t.Error("expected sparse representation") + } + if got := restored.Precision(); got != 12 { + t.Errorf("precision: got %d, want 12", got) + } + if got := restored.SparsePrecision(); got != 17 { + t.Errorf("sparse precision: got %d, want 17", got) + } + if got := restored.Estimate(); got != 798 { + t.Errorf("got %d, want 798", got) + } } diff --git a/hllplus/lazy_alloc_test.go b/hllplus/lazy_alloc_test.go index dc63340..5f548b2 100644 --- a/hllplus/lazy_alloc_test.go +++ b/hllplus/lazy_alloc_test.go @@ -22,7 +22,7 @@ func buildHLL(tb testing.TB, precision, sparsePrecision uint8, n int, seed int64 } rnd := rand.New(rand.NewSource(seed)) - for i := 0; i < n; i++ { + for range n { h.Add(rnd.Uint64()) } return h @@ -102,12 +102,12 @@ var benchSink *hllplus.HLL func BenchmarkSparseLowCardinality(b *testing.B) { b.ReportAllocs() - for i := 0; i < b.N; i++ { + for b.Loop() { h, err := hllplus.New(12, 17) if err != nil { b.Fatal(err) } - for j := 0; j < 10; j++ { + for j := range 10 { h.Add(uint64(j) * 0x9e3779b97f4a7c15) } benchSink = h diff --git a/internal/hash/fingerprint.go b/internal/hash/fingerprint.go index f4e82b7..ab97355 100644 --- a/internal/hash/fingerprint.go +++ b/internal/hash/fingerprint.go @@ -44,7 +44,7 @@ func mm64(data []byte, seed uint64) uint64 { h := seed ^ uint64(len(data))*c3 nblocks := len(data) / 8 - for b := 0; b < nblocks; b++ { + for b := range nblocks { k := load64(data[b*8:]) * c3 k = shiftMix(k) * c3 @@ -153,7 +153,7 @@ func shiftMix(v uint64) uint64 { return v ^ (v >> 47) } -/// Computes intermediate hash of 32 bytes of byte array from the given offset. +// Computes intermediate hash of 32 bytes of byte array from the given offset. func weakHashLength32WithSeeds(data []byte, seedA, seedB uint64) (uint64, uint64) { p1 := load64(data) p2 := load64(data[8:]) diff --git a/internal/hash/fingerprint_test.go b/internal/hash/fingerprint_test.go index 289c812..54a7bfc 100644 --- a/internal/hash/fingerprint_test.go +++ b/internal/hash/fingerprint_test.go @@ -2,32 +2,38 @@ package hash_test import ( "bytes" + "testing" "github.com/bsm/zetasketch/internal/hash" - - . "github.com/bsm/ginkgo" - . "github.com/bsm/gomega" ) -var _ = Describe("Bytes", func() { - var foobar = []byte("foobar") +func TestBytes(t *testing.T) { + foobar := []byte("foobar") // Partially taken from: // https://github.com/google/zetasketch/blob/master/javatests/com/google/zetasketch/internal/hash/HashTest.java - It("should hash short sequences", func() { - Expect(hash.Bytes(nil)).To(Equal(uint64(0x23ad7c904aa665e3))) - Expect(hash.Bytes(foobar)).To(Equal(uint64(0x36a1e57a138e4467))) - Expect(hash.Bytes([]byte("foo"))).To(Equal(uint64(0xd0bcbfe261b36504))) - Expect(hash.Bytes([]byte("f"))).To(Equal(uint64(16100291902947574842))) - Expect(hash.Bytes([]byte("Z\u00fcrich"))).To(Equal(uint64(0x27efc00f7d2ce548))) - Expect(hash.Bytes([]byte("Zu\u0308rich"))).To(Equal(uint64(0x7dfa3067e55c7e8a))) - Expect(hash.Bytes(bytes.Repeat(foobar, 3))).To(Equal(uint64(0xd7b08d94eeefdccd))) - }) + cases := []struct { + in []byte + exp uint64 + }{ + // short sequences + {nil, 0x23ad7c904aa665e3}, + {foobar, 0x36a1e57a138e4467}, + {[]byte("foo"), 0xd0bcbfe261b36504}, + {[]byte("f"), 16100291902947574842}, + {[]byte("Zürich"), 0x27efc00f7d2ce548}, + {[]byte("Zürich"), 0x7dfa3067e55c7e8a}, + {bytes.Repeat(foobar, 3), 0xd7b08d94eeefdccd}, - It("should hash longer sequences", func() { - Expect(hash.Bytes(bytes.Repeat(foobar, 8))).To(Equal(uint64(0x94386e8403038649))) - Expect(hash.Bytes(bytes.Repeat(foobar, 24))).To(Equal(uint64(0xd019f3291f1d4d37))) - Expect(hash.Bytes(bytes.Repeat(foobar, 45))).To(Equal(uint64(0x549b2e228e80ee1a))) - Expect(hash.Bytes(bytes.Repeat(foobar, 48))).To(Equal(uint64(0x874fce45a1e2a8ae))) - }) -}) + // longer sequences + {bytes.Repeat(foobar, 8), 0x94386e8403038649}, + {bytes.Repeat(foobar, 24), 0xd019f3291f1d4d37}, + {bytes.Repeat(foobar, 45), 0x549b2e228e80ee1a}, + {bytes.Repeat(foobar, 48), 0x874fce45a1e2a8ae}, + } + for _, tc := range cases { + if got := hash.Bytes(tc.in); got != tc.exp { + t.Errorf("Bytes(%q) = %#x, want %#x", tc.in, got, tc.exp) + } + } +} diff --git a/internal/hash/hash_test.go b/internal/hash/hash_test.go index 025450c..553315d 100644 --- a/internal/hash/hash_test.go +++ b/internal/hash/hash_test.go @@ -4,34 +4,54 @@ import ( "testing" "github.com/bsm/zetasketch/internal/hash" - - . "github.com/bsm/ginkgo" - . "github.com/bsm/gomega" ) -var _ = Describe("Hash", func() { - It("should hash uint64s", func() { - Expect(hash.Uint64(0)).To(Equal(uint64(0x853a22bd6e14a48f))) - Expect(hash.Uint64(1)).To(Equal(uint64(0xb91968b83211c978))) - Expect(hash.Uint64(2)).To(Equal(uint64(0x83e2c1afe085d87a))) - Expect(hash.Uint64((1 << 62) - 127)).To(Equal(uint64(0xfd2303b188e412d9))) - }) - - It("should hash uint32s", func() { - Expect(hash.Uint32(0)).To(Equal(uint64(0x905df40cd02611cb))) - Expect(hash.Uint32(1)).To(Equal(uint64(0xba4e724d9d787a26))) - Expect(hash.Uint32(2)).To(Equal(uint64(0x6ef6414ac0c8858e))) - Expect(hash.Uint32((1 << 29) - 127)).To(Equal(uint64(0xbda9eae625a1584))) - }) +func TestUint64(t *testing.T) { + cases := []struct { + in uint64 + exp uint64 + }{ + {0, 0x853a22bd6e14a48f}, + {1, 0xb91968b83211c978}, + {2, 0x83e2c1afe085d87a}, + {(1 << 62) - 127, 0xfd2303b188e412d9}, + } + for _, tc := range cases { + if got := hash.Uint64(tc.in); got != tc.exp { + t.Errorf("Uint64(%d) = %#x, want %#x", tc.in, got, tc.exp) + } + } +} - It("should hash strings", func() { - Expect(hash.String("foo")).To(Equal(uint64(0xd0bcbfe261b36504))) - Expect(hash.String("Z\u00fcrich")).To(Equal(uint64(0x27efc00f7d2ce548))) - Expect(hash.String("Zu\u0308rich")).To(Equal(uint64(0x7dfa3067e55c7e8a))) - }) -}) +func TestUint32(t *testing.T) { + cases := []struct { + in uint32 + exp uint64 + }{ + {0, 0x905df40cd02611cb}, + {1, 0xba4e724d9d787a26}, + {2, 0x6ef6414ac0c8858e}, + {(1 << 29) - 127, 0xbda9eae625a1584}, + } + for _, tc := range cases { + if got := hash.Uint32(tc.in); got != tc.exp { + t.Errorf("Uint32(%d) = %#x, want %#x", tc.in, got, tc.exp) + } + } +} -func TestSuite(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "zetasketch/internal/hash") +func TestString(t *testing.T) { + cases := []struct { + in string + exp uint64 + }{ + {"foo", 0xd0bcbfe261b36504}, + {"Zürich", 0x27efc00f7d2ce548}, + {"Zürich", 0x7dfa3067e55c7e8a}, + } + for _, tc := range cases { + if got := hash.String(tc.in); got != tc.exp { + t.Errorf("String(%q) = %#x, want %#x", tc.in, got, tc.exp) + } + } } diff --git a/internal/zetasketch/aggregator.pb.go b/internal/zetasketch/aggregator.pb.go index 3a89551..cc1e41a 100644 --- a/internal/zetasketch/aggregator.pb.go +++ b/internal/zetasketch/aggregator.pb.go @@ -41,8 +41,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.18.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: aggregator.proto package zetasketch @@ -53,6 +53,7 @@ import ( descriptorpb "google.golang.org/protobuf/types/descriptorpb" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -238,18 +239,16 @@ func (DefaultOpsType_Id) EnumDescriptor() ([]byte, []int) { // Never instantiated, just for scoping an enum and associated options. type DefaultOpsType struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *DefaultOpsType) Reset() { *x = DefaultOpsType{} - if protoimpl.UnsafeEnabled { - mi := &file_aggregator_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_aggregator_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *DefaultOpsType) String() string { @@ -260,7 +259,7 @@ func (*DefaultOpsType) ProtoMessage() {} func (x *DefaultOpsType) ProtoReflect() protoreflect.Message { mi := &file_aggregator_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -279,22 +278,19 @@ func (*DefaultOpsType) Descriptor() ([]byte, []int) { // algorithm. Add additional fields here only if they make sense for // all algorithms. type AggregatorStatsProto struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - extensionFields protoimpl.ExtensionFields - + state protoimpl.MessageState `protogen:"open.v1"` // Total number of values added to this aggregator. - NumValues *int64 `protobuf:"varint,1,req,name=num_values,json=numValues" json:"num_values,omitempty"` + NumValues *int64 `protobuf:"varint,1,req,name=num_values,json=numValues" json:"num_values,omitempty"` + extensionFields protoimpl.ExtensionFields + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *AggregatorStatsProto) Reset() { *x = AggregatorStatsProto{} - if protoimpl.UnsafeEnabled { - mi := &file_aggregator_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_aggregator_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *AggregatorStatsProto) String() string { @@ -305,7 +301,7 @@ func (*AggregatorStatsProto) ProtoMessage() {} func (x *AggregatorStatsProto) ProtoReflect() protoreflect.Message { mi := &file_aggregator_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -331,11 +327,7 @@ func (x *AggregatorStatsProto) GetNumValues() int64 { // if they make sense for all algorithms and if it doesn't make sense to // expose them to the users of the library, e.g. encoding version. type AggregatorStateProto struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - extensionFields protoimpl.ExtensionFields - + state protoimpl.MessageState `protogen:"open.v1"` // The type of the aggregator. Type *AggregatorType `protobuf:"varint,1,req,name=type,enum=zetasketch.AggregatorType" json:"type,omitempty"` NumValues *int64 `protobuf:"varint,2,req,name=num_values,json=numValues" json:"num_values,omitempty"` @@ -356,7 +348,10 @@ type AggregatorStateProto struct { // should consider registering a name for their custom type in // custom-value-type.proto, to facilitate easier discovery and better error // messages when conflicting types are merged. - ValueType *int32 `protobuf:"varint,4,opt,name=value_type,json=valueType" json:"value_type,omitempty"` + ValueType *int32 `protobuf:"varint,4,opt,name=value_type,json=valueType" json:"value_type,omitempty"` + extensionFields protoimpl.ExtensionFields + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } // Default values for AggregatorStateProto fields. @@ -366,11 +361,9 @@ const ( func (x *AggregatorStateProto) Reset() { *x = AggregatorStateProto{} - if protoimpl.UnsafeEnabled { - mi := &file_aggregator_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_aggregator_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *AggregatorStateProto) String() string { @@ -381,7 +374,7 @@ func (*AggregatorStateProto) ProtoMessage() {} func (x *AggregatorStateProto) ProtoReflect() protoreflect.Message { mi := &file_aggregator_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -426,18 +419,16 @@ func (x *AggregatorStateProto) GetValueType() int32 { // Additional metadata for each element in the result iterator. type AggregatorValueStatsProto struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *AggregatorValueStatsProto) Reset() { *x = AggregatorValueStatsProto{} - if protoimpl.UnsafeEnabled { - mi := &file_aggregator_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_aggregator_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *AggregatorValueStatsProto) String() string { @@ -448,7 +439,7 @@ func (*AggregatorValueStatsProto) ProtoMessage() {} func (x *AggregatorValueStatsProto) ProtoReflect() protoreflect.Message { mi := &file_aggregator_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -485,76 +476,61 @@ var ( var File_aggregator_proto protoreflect.FileDescriptor -var file_aggregator_proto_rawDesc = []byte{ - 0x0a, 0x10, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x12, 0x0a, 0x7a, 0x65, 0x74, 0x61, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x1a, 0x20, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, - 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0xdb, 0x02, 0x0a, 0x0e, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x4f, 0x70, 0x73, 0x54, - 0x79, 0x70, 0x65, 0x22, 0xd0, 0x01, 0x0a, 0x02, 0x49, 0x64, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, - 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x04, 0x49, 0x4e, 0x54, 0x38, 0x10, - 0x01, 0x1a, 0x06, 0xa8, 0x97, 0xff, 0xf9, 0x03, 0x05, 0x12, 0x11, 0x0a, 0x05, 0x49, 0x4e, 0x54, - 0x31, 0x36, 0x10, 0x02, 0x1a, 0x06, 0xa8, 0x97, 0xff, 0xf9, 0x03, 0x06, 0x12, 0x11, 0x0a, 0x05, - 0x49, 0x4e, 0x54, 0x33, 0x32, 0x10, 0x03, 0x1a, 0x06, 0xa8, 0x97, 0xff, 0xf9, 0x03, 0x07, 0x12, - 0x11, 0x0a, 0x05, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10, 0x04, 0x1a, 0x06, 0xa8, 0x97, 0xff, 0xf9, - 0x03, 0x08, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x49, 0x4e, 0x54, 0x38, 0x10, 0x05, 0x12, 0x0a, 0x0a, - 0x06, 0x55, 0x49, 0x4e, 0x54, 0x31, 0x36, 0x10, 0x06, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x49, 0x4e, - 0x54, 0x33, 0x32, 0x10, 0x07, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10, - 0x08, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x09, 0x12, 0x0a, 0x0a, 0x06, - 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, 0x0a, 0x12, 0x18, 0x0a, 0x14, 0x42, 0x59, 0x54, 0x45, - 0x53, 0x5f, 0x4f, 0x52, 0x5f, 0x55, 0x54, 0x46, 0x38, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, - 0x10, 0x0b, 0x22, 0x04, 0x08, 0x0c, 0x10, 0x0c, 0x2a, 0x0c, 0x55, 0x54, 0x46, 0x31, 0x36, 0x5f, - 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x32, 0x76, 0x0a, 0x14, 0x75, 0x6e, 0x73, 0x69, 0x67, 0x6e, - 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x61, 0x72, 0x74, 0x12, 0x21, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x18, 0xf5, 0xf2, 0x9f, 0x3f, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x7a, 0x65, 0x74, - 0x61, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x2e, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x4f, - 0x70, 0x73, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x49, 0x64, 0x52, 0x13, 0x75, 0x6e, 0x73, 0x69, 0x67, - 0x6e, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x61, 0x72, 0x74, 0x22, 0x48, - 0x0a, 0x14, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, - 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x02, 0x28, 0x03, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2a, 0x04, 0x08, 0x6c, 0x10, 0x70, 0x2a, 0x05, 0x08, 0x71, 0x10, - 0x8d, 0x01, 0x2a, 0x04, 0x08, 0x70, 0x10, 0x71, 0x22, 0xc5, 0x01, 0x0a, 0x14, 0x41, 0x67, 0x67, - 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x12, 0x2e, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x02, 0x28, 0x0e, 0x32, - 0x1a, 0x2e, 0x7a, 0x65, 0x74, 0x61, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x2e, 0x41, 0x67, 0x67, - 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x75, 0x6d, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, - 0x02, 0x20, 0x02, 0x28, 0x03, 0x52, 0x09, 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, - 0x12, 0x2c, 0x0a, 0x10, 0x65, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x3a, 0x01, 0x31, 0x52, 0x0f, 0x65, - 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1d, - 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, 0x2a, 0x04, 0x08, - 0x64, 0x10, 0x70, 0x2a, 0x05, 0x08, 0x71, 0x10, 0x8d, 0x01, 0x2a, 0x04, 0x08, 0x70, 0x10, 0x71, - 0x22, 0x1b, 0x0a, 0x19, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2a, 0x49, 0x0a, - 0x0e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x07, 0x0a, 0x03, 0x53, 0x55, 0x4d, 0x10, 0x64, 0x12, 0x1b, 0x0a, 0x17, 0x48, 0x59, 0x50, 0x45, - 0x52, 0x4c, 0x4f, 0x47, 0x4c, 0x4f, 0x47, 0x5f, 0x50, 0x4c, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x49, - 0x51, 0x55, 0x45, 0x10, 0x70, 0x22, 0x04, 0x08, 0x00, 0x10, 0x00, 0x22, 0x04, 0x08, 0x65, 0x10, - 0x6f, 0x22, 0x05, 0x08, 0x71, 0x10, 0x8c, 0x01, 0x42, 0x21, 0x0a, 0x1c, 0x63, 0x6f, 0x6d, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2e, 0x7a, 0x65, - 0x74, 0x61, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0xf8, 0x01, 0x01, -} +const file_aggregator_proto_rawDesc = "" + + "\n" + + "\x10aggregator.proto\x12\n" + + "zetasketch\x1a google/protobuf/descriptor.proto\"\xdb\x02\n" + + "\x0eDefaultOpsType\"\xd0\x01\n" + + "\x02Id\x12\v\n" + + "\aUNKNOWN\x10\x00\x12\x10\n" + + "\x04INT8\x10\x01\x1a\x06\xa8\x97\xff\xf9\x03\x05\x12\x11\n" + + "\x05INT16\x10\x02\x1a\x06\xa8\x97\xff\xf9\x03\x06\x12\x11\n" + + "\x05INT32\x10\x03\x1a\x06\xa8\x97\xff\xf9\x03\a\x12\x11\n" + + "\x05INT64\x10\x04\x1a\x06\xa8\x97\xff\xf9\x03\b\x12\t\n" + + "\x05UINT8\x10\x05\x12\n" + + "\n" + + "\x06UINT16\x10\x06\x12\n" + + "\n" + + "\x06UINT32\x10\a\x12\n" + + "\n" + + "\x06UINT64\x10\b\x12\t\n" + + "\x05FLOAT\x10\t\x12\n" + + "\n" + + "\x06DOUBLE\x10\n" + + "\x12\x18\n" + + "\x14BYTES_OR_UTF8_STRING\x10\v\"\x04\b\f\x10\f*\fUTF16_STRING2v\n" + + "\x14unsigned_counterpart\x12!.google.protobuf.EnumValueOptions\x18\xf5\xf2\x9f? \x01(\x0e2\x1d.zetasketch.DefaultOpsType.IdR\x13unsignedCounterpart\"H\n" + + "\x14AggregatorStatsProto\x12\x1d\n" + + "\n" + + "num_values\x18\x01 \x02(\x03R\tnumValues*\x04\bl\x10p*\x05\bq\x10\x8d\x01*\x04\bp\x10q\"\xc5\x01\n" + + "\x14AggregatorStateProto\x12.\n" + + "\x04type\x18\x01 \x02(\x0e2\x1a.zetasketch.AggregatorTypeR\x04type\x12\x1d\n" + + "\n" + + "num_values\x18\x02 \x02(\x03R\tnumValues\x12,\n" + + "\x10encoding_version\x18\x03 \x01(\x05:\x011R\x0fencodingVersion\x12\x1d\n" + + "\n" + + "value_type\x18\x04 \x01(\x05R\tvalueType*\x04\bd\x10p*\x05\bq\x10\x8d\x01*\x04\bp\x10q\"\x1b\n" + + "\x19AggregatorValueStatsProto*I\n" + + "\x0eAggregatorType\x12\a\n" + + "\x03SUM\x10d\x12\x1b\n" + + "\x17HYPERLOGLOG_PLUS_UNIQUE\x10p\"\x04\b\x00\x10\x00\"\x04\be\x10o\"\x05\bq\x10\x8c\x01B!\n" + + "\x1ccom.google.protos.zetasketch\xf8\x01\x01" var ( file_aggregator_proto_rawDescOnce sync.Once - file_aggregator_proto_rawDescData = file_aggregator_proto_rawDesc + file_aggregator_proto_rawDescData []byte ) func file_aggregator_proto_rawDescGZIP() []byte { file_aggregator_proto_rawDescOnce.Do(func() { - file_aggregator_proto_rawDescData = protoimpl.X.CompressGZIP(file_aggregator_proto_rawDescData) + file_aggregator_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_aggregator_proto_rawDesc), len(file_aggregator_proto_rawDesc))) }) return file_aggregator_proto_rawDescData } var file_aggregator_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_aggregator_proto_msgTypes = make([]protoimpl.MessageInfo, 4) -var file_aggregator_proto_goTypes = []interface{}{ +var file_aggregator_proto_goTypes = []any{ (AggregatorType)(0), // 0: zetasketch.AggregatorType (DefaultOpsType_Id)(0), // 1: zetasketch.DefaultOpsType.Id (*DefaultOpsType)(nil), // 2: zetasketch.DefaultOpsType @@ -579,65 +555,11 @@ func file_aggregator_proto_init() { if File_aggregator_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_aggregator_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DefaultOpsType); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_aggregator_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AggregatorStatsProto); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - case 3: - return &v.extensionFields - default: - return nil - } - } - file_aggregator_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AggregatorStateProto); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - case 3: - return &v.extensionFields - default: - return nil - } - } - file_aggregator_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*AggregatorValueStatsProto); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_aggregator_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_aggregator_proto_rawDesc), len(file_aggregator_proto_rawDesc)), NumEnums: 2, NumMessages: 4, NumExtensions: 1, @@ -650,7 +572,6 @@ func file_aggregator_proto_init() { ExtensionInfos: file_aggregator_proto_extTypes, }.Build() File_aggregator_proto = out.File - file_aggregator_proto_rawDesc = nil file_aggregator_proto_goTypes = nil file_aggregator_proto_depIdxs = nil } diff --git a/internal/zetasketch/hllplus-unique.pb.go b/internal/zetasketch/hllplus-unique.pb.go index 7f287be..1f09777 100644 --- a/internal/zetasketch/hllplus-unique.pb.go +++ b/internal/zetasketch/hllplus-unique.pb.go @@ -17,8 +17,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.18.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: hllplus-unique.proto package zetasketch @@ -28,6 +28,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -41,10 +42,7 @@ const ( // For more details on the algorithm, the representations and the concepts // please check the HLL++ paper (https://goo.gl/pc916Z). type HyperLogLogPlusUniqueStateProto struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // Size of sparse list, i.e., how many different indexes are present in // "sparse_data". SparseSize *int32 `protobuf:"varint,2,opt,name=sparse_size,json=sparseSize" json:"sparse_size,omitempty"` @@ -53,10 +51,10 @@ type HyperLogLogPlusUniqueStateProto struct { // This field is used slightly differently across the v1 and v2 versions of // the algorithm (see the encoding_version field in the AggregatorStateProto): // - // * In v1 this field is the total number of buckets 2^p where "p" is the + // - In v1 this field is the total number of buckets 2^p where "p" is the // requested precision. Accepted values are powers of two in the [2^10, // 2^24] interval. - // * In v2 this field is the precision "p" directly. Accepted values are in + // - In v2 this field is the precision "p" directly. Accepted values are in // the range [10, 24]. // // Encoding the precision rather than the number of buckets allows us to save @@ -71,9 +69,9 @@ type HyperLogLogPlusUniqueStateProto struct { // This field is used slightly differently across the v1 and v2 versions of // the algorithm (see the encoding_version field in the AggregatorStateProto): // - // * In v1 this field is 2^sp where "sp" is the sparse precision. Accepted + // - In v1 this field is 2^sp where "sp" is the sparse precision. Accepted // values are powers of two in the [2^p, 2^25] interval. - // * In v2 this field represents the precision "sp" directly. Accepted + // - In v2 this field represents the precision "sp" directly. Accepted // values are in the range [p, 25]. // // Encoding the precision rather than the number of buckets allows us to save @@ -97,7 +95,9 @@ type HyperLogLogPlusUniqueStateProto struct { // For a sorted list of unsigned integers representing sparse data encodings, // this field contains the varint encoding for the differences between // consecutive values in the list: - // list[0], list[1] - list[0], ... , list[n] - list[n - 1] + // + // list[0], list[1] - list[0], ... , list[n] - list[n - 1] + // // Note: if "encoding_version" of the enclosing AggregatorStateProto is 1, the // diffs are encoded as signed varints using ZigZag encoding and the // sparse encodings are the ones defined in the HLL++ paper @@ -105,18 +105,19 @@ type HyperLogLogPlusUniqueStateProto struct { // // In v2, there are two encodings possible for a value in sparse data format: // enc(idx, rhoW) = 1) 1 << (max(sp, p+6)) | (idx >> (sp - p)) | rhoW - // if the last sp - p bits of idx are all 0; - // 2) idx, otherwise. - SparseData []byte `protobuf:"bytes,6,opt,name=sparse_data,json=sparseData" json:"sparse_data,omitempty"` + // + // if the last sp - p bits of idx are all 0; + // 2) idx, otherwise. + SparseData []byte `protobuf:"bytes,6,opt,name=sparse_data,json=sparseData" json:"sparse_data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *HyperLogLogPlusUniqueStateProto) Reset() { *x = HyperLogLogPlusUniqueStateProto{} - if protoimpl.UnsafeEnabled { - mi := &file_hllplus_unique_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_hllplus_unique_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *HyperLogLogPlusUniqueStateProto) String() string { @@ -127,7 +128,7 @@ func (*HyperLogLogPlusUniqueStateProto) ProtoMessage() {} func (x *HyperLogLogPlusUniqueStateProto) ProtoReflect() protoreflect.Message { mi := &file_hllplus_unique_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -214,64 +215,36 @@ var ( var File_hllplus_unique_proto protoreflect.FileDescriptor -var file_hllplus_unique_proto_rawDesc = []byte{ - 0x0a, 0x14, 0x68, 0x6c, 0x6c, 0x70, 0x6c, 0x75, 0x73, 0x2d, 0x75, 0x6e, 0x69, 0x71, 0x75, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x7a, 0x65, 0x74, 0x61, 0x73, 0x6b, 0x65, 0x74, - 0x63, 0x68, 0x1a, 0x10, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x12, 0x75, 0x6e, 0x69, 0x71, 0x75, 0x65, 0x2d, 0x73, 0x74, 0x61, - 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8a, 0x02, 0x0a, 0x1f, 0x48, 0x79, 0x70, - 0x65, 0x72, 0x4c, 0x6f, 0x67, 0x4c, 0x6f, 0x67, 0x50, 0x6c, 0x75, 0x73, 0x55, 0x6e, 0x69, 0x71, - 0x75, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1f, 0x0a, 0x0b, - 0x73, 0x70, 0x61, 0x72, 0x73, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x0a, 0x73, 0x70, 0x61, 0x72, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x37, 0x0a, - 0x18, 0x70, 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x72, 0x5f, 0x6e, 0x75, - 0x6d, 0x5f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x15, 0x70, 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x4f, 0x72, 0x4e, 0x75, 0x6d, 0x42, - 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x44, 0x0a, 0x1f, 0x73, 0x70, 0x61, 0x72, 0x73, 0x65, - 0x5f, 0x70, 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x72, 0x5f, 0x6e, 0x75, - 0x6d, 0x5f, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x1b, 0x73, 0x70, 0x61, 0x72, 0x73, 0x65, 0x50, 0x72, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, - 0x4f, 0x72, 0x4e, 0x75, 0x6d, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x08, 0x01, 0x52, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x12, 0x23, 0x0a, 0x0b, 0x73, 0x70, 0x61, 0x72, 0x73, 0x65, 0x5f, 0x64, - 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x42, 0x02, 0x08, 0x01, 0x52, 0x0a, 0x73, - 0x70, 0x61, 0x72, 0x73, 0x65, 0x44, 0x61, 0x74, 0x61, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x4a, - 0x04, 0x08, 0x07, 0x10, 0x08, 0x3a, 0x81, 0x01, 0x0a, 0x1d, 0x68, 0x79, 0x70, 0x65, 0x72, 0x6c, - 0x6f, 0x67, 0x6c, 0x6f, 0x67, 0x5f, 0x70, 0x6c, 0x75, 0x73, 0x5f, 0x75, 0x6e, 0x69, 0x71, 0x75, - 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x12, 0x20, 0x2e, 0x7a, 0x65, 0x74, 0x61, 0x73, 0x6b, - 0x65, 0x74, 0x63, 0x68, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x6f, 0x72, 0x53, - 0x74, 0x61, 0x74, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x18, 0x70, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x7a, 0x65, 0x74, 0x61, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x2e, 0x55, 0x6e, 0x69, - 0x71, 0x75, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x1a, 0x68, - 0x79, 0x70, 0x65, 0x72, 0x6c, 0x6f, 0x67, 0x6c, 0x6f, 0x67, 0x50, 0x6c, 0x75, 0x73, 0x55, 0x6e, - 0x69, 0x71, 0x75, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x3a, 0x8f, 0x01, 0x0a, 0x1c, 0x68, 0x79, - 0x70, 0x65, 0x72, 0x6c, 0x6f, 0x67, 0x6c, 0x6f, 0x67, 0x70, 0x6c, 0x75, 0x73, 0x5f, 0x75, 0x6e, - 0x69, 0x71, 0x75, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x20, 0x2e, 0x7a, 0x65, 0x74, - 0x61, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x2e, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, - 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x18, 0x70, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x7a, 0x65, 0x74, 0x61, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, 0x2e, - 0x48, 0x79, 0x70, 0x65, 0x72, 0x4c, 0x6f, 0x67, 0x4c, 0x6f, 0x67, 0x50, 0x6c, 0x75, 0x73, 0x55, - 0x6e, 0x69, 0x71, 0x75, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, - 0x1a, 0x68, 0x79, 0x70, 0x65, 0x72, 0x6c, 0x6f, 0x67, 0x6c, 0x6f, 0x67, 0x70, 0x6c, 0x75, 0x73, - 0x55, 0x6e, 0x69, 0x71, 0x75, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x42, 0x1e, 0x0a, 0x1c, 0x63, - 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, - 0x2e, 0x7a, 0x65, 0x74, 0x61, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, -} +const file_hllplus_unique_proto_rawDesc = "" + + "\n" + + "\x14hllplus-unique.proto\x12\n" + + "zetasketch\x1a\x10aggregator.proto\x1a\x12unique-stats.proto\"\x8a\x02\n" + + "\x1fHyperLogLogPlusUniqueStateProto\x12\x1f\n" + + "\vsparse_size\x18\x02 \x01(\x05R\n" + + "sparseSize\x127\n" + + "\x18precision_or_num_buckets\x18\x03 \x01(\x05R\x15precisionOrNumBuckets\x12D\n" + + "\x1fsparse_precision_or_num_buckets\x18\x04 \x01(\x05R\x1bsparsePrecisionOrNumBuckets\x12\x16\n" + + "\x04data\x18\x05 \x01(\fB\x02\b\x01R\x04data\x12#\n" + + "\vsparse_data\x18\x06 \x01(\fB\x02\b\x01R\n" + + "sparseDataJ\x04\b\x01\x10\x02J\x04\b\a\x10\b:\x81\x01\n" + + "\x1dhyperloglog_plus_unique_stats\x12 .zetasketch.AggregatorStatsProto\x18p \x01(\v2\x1c.zetasketch.UniqueStatsProtoR\x1ahyperloglogPlusUniqueStats:\x8f\x01\n" + + "\x1chyperloglogplus_unique_state\x12 .zetasketch.AggregatorStateProto\x18p \x01(\v2+.zetasketch.HyperLogLogPlusUniqueStateProtoR\x1ahyperloglogplusUniqueStateB\x1e\n" + + "\x1ccom.google.protos.zetasketch" var ( file_hllplus_unique_proto_rawDescOnce sync.Once - file_hllplus_unique_proto_rawDescData = file_hllplus_unique_proto_rawDesc + file_hllplus_unique_proto_rawDescData []byte ) func file_hllplus_unique_proto_rawDescGZIP() []byte { file_hllplus_unique_proto_rawDescOnce.Do(func() { - file_hllplus_unique_proto_rawDescData = protoimpl.X.CompressGZIP(file_hllplus_unique_proto_rawDescData) + file_hllplus_unique_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_hllplus_unique_proto_rawDesc), len(file_hllplus_unique_proto_rawDesc))) }) return file_hllplus_unique_proto_rawDescData } var file_hllplus_unique_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_hllplus_unique_proto_goTypes = []interface{}{ +var file_hllplus_unique_proto_goTypes = []any{ (*HyperLogLogPlusUniqueStateProto)(nil), // 0: zetasketch.HyperLogLogPlusUniqueStateProto (*AggregatorStatsProto)(nil), // 1: zetasketch.AggregatorStatsProto (*AggregatorStateProto)(nil), // 2: zetasketch.AggregatorStateProto @@ -296,25 +269,11 @@ func file_hllplus_unique_proto_init() { } file_aggregator_proto_init() file_unique_stats_proto_init() - if !protoimpl.UnsafeEnabled { - file_hllplus_unique_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HyperLogLogPlusUniqueStateProto); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_hllplus_unique_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_hllplus_unique_proto_rawDesc), len(file_hllplus_unique_proto_rawDesc)), NumEnums: 0, NumMessages: 1, NumExtensions: 2, @@ -326,7 +285,6 @@ func file_hllplus_unique_proto_init() { ExtensionInfos: file_hllplus_unique_proto_extTypes, }.Build() File_hllplus_unique_proto = out.File - file_hllplus_unique_proto_rawDesc = nil file_hllplus_unique_proto_goTypes = nil file_hllplus_unique_proto_depIdxs = nil } diff --git a/internal/zetasketch/unique-stats.pb.go b/internal/zetasketch/unique-stats.pb.go index 7a25f89..a407bca 100644 --- a/internal/zetasketch/unique-stats.pb.go +++ b/internal/zetasketch/unique-stats.pb.go @@ -17,8 +17,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.18.0 +// protoc-gen-go v1.36.11 +// protoc v6.33.2 // source: unique-stats.proto package zetasketch @@ -28,6 +28,7 @@ import ( protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -38,23 +39,20 @@ const ( ) type UniqueStatsProto struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - + state protoimpl.MessageState `protogen:"open.v1"` // The estimated number of unique elements in the input set. EstimatedCardinality *int64 `protobuf:"varint,1,opt,name=estimated_cardinality,json=estimatedCardinality" json:"estimated_cardinality,omitempty"` // The expected error of the estimation algorithm. ExpectedError *float64 `protobuf:"fixed64,2,opt,name=expected_error,json=expectedError" json:"expected_error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UniqueStatsProto) Reset() { *x = UniqueStatsProto{} - if protoimpl.UnsafeEnabled { - mi := &file_unique_stats_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_unique_stats_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *UniqueStatsProto) String() string { @@ -65,7 +63,7 @@ func (*UniqueStatsProto) ProtoMessage() {} func (x *UniqueStatsProto) ProtoReflect() protoreflect.Message { mi := &file_unique_stats_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -96,34 +94,29 @@ func (x *UniqueStatsProto) GetExpectedError() float64 { var File_unique_stats_proto protoreflect.FileDescriptor -var file_unique_stats_proto_rawDesc = []byte{ - 0x0a, 0x12, 0x75, 0x6e, 0x69, 0x71, 0x75, 0x65, 0x2d, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x7a, 0x65, 0x74, 0x61, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, - 0x22, 0x6e, 0x0a, 0x10, 0x55, 0x6e, 0x69, 0x71, 0x75, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x50, - 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x33, 0x0a, 0x15, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, - 0x64, 0x5f, 0x63, 0x61, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x14, 0x65, 0x73, 0x74, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x64, 0x43, 0x61, - 0x72, 0x64, 0x69, 0x6e, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x65, 0x78, 0x70, - 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x01, 0x52, 0x0d, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x45, 0x72, 0x72, 0x6f, 0x72, - 0x42, 0x1e, 0x0a, 0x1c, 0x63, 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2e, 0x7a, 0x65, 0x74, 0x61, 0x73, 0x6b, 0x65, 0x74, 0x63, 0x68, -} +const file_unique_stats_proto_rawDesc = "" + + "\n" + + "\x12unique-stats.proto\x12\n" + + "zetasketch\"n\n" + + "\x10UniqueStatsProto\x123\n" + + "\x15estimated_cardinality\x18\x01 \x01(\x03R\x14estimatedCardinality\x12%\n" + + "\x0eexpected_error\x18\x02 \x01(\x01R\rexpectedErrorB\x1e\n" + + "\x1ccom.google.protos.zetasketch" var ( file_unique_stats_proto_rawDescOnce sync.Once - file_unique_stats_proto_rawDescData = file_unique_stats_proto_rawDesc + file_unique_stats_proto_rawDescData []byte ) func file_unique_stats_proto_rawDescGZIP() []byte { file_unique_stats_proto_rawDescOnce.Do(func() { - file_unique_stats_proto_rawDescData = protoimpl.X.CompressGZIP(file_unique_stats_proto_rawDescData) + file_unique_stats_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_unique_stats_proto_rawDesc), len(file_unique_stats_proto_rawDesc))) }) return file_unique_stats_proto_rawDescData } var file_unique_stats_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_unique_stats_proto_goTypes = []interface{}{ +var file_unique_stats_proto_goTypes = []any{ (*UniqueStatsProto)(nil), // 0: zetasketch.UniqueStatsProto } var file_unique_stats_proto_depIdxs = []int32{ @@ -139,25 +132,11 @@ func file_unique_stats_proto_init() { if File_unique_stats_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_unique_stats_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*UniqueStatsProto); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_unique_stats_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_unique_stats_proto_rawDesc), len(file_unique_stats_proto_rawDesc)), NumEnums: 0, NumMessages: 1, NumExtensions: 0, @@ -168,7 +147,6 @@ func file_unique_stats_proto_init() { MessageInfos: file_unique_stats_proto_msgTypes, }.Build() File_unique_stats_proto = out.File - file_unique_stats_proto_rawDesc = nil file_unique_stats_proto_goTypes = nil file_unique_stats_proto_depIdxs = nil } diff --git a/zetasketch_test.go b/zetasketch_test.go deleted file mode 100644 index feeadcd..0000000 --- a/zetasketch_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package zetasketch_test - -import ( - "testing" - - . "github.com/bsm/ginkgo" - . "github.com/bsm/gomega" -) - -func TestSuite(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "zetasketch") -}