@@ -3,13 +3,13 @@ package main
33import (
44 "context"
55 "fmt"
6- "github.com/threatwinds/go-sdk/catcher"
7- "github.com/threatwinds/go-sdk/plugins"
8- "github.com/threatwinds/go-sdk/utils"
96 "runtime"
107 "sync"
118 "time"
129
10+ "github.com/threatwinds/go-sdk/catcher"
11+ "github.com/threatwinds/go-sdk/plugins"
12+
1313 "github.com/threatwinds/go-sdk/opensearch"
1414 "github.com/tidwall/gjson"
1515)
@@ -60,16 +60,6 @@ func startQueue() {
6060 numCPU := runtime .NumCPU () * 2
6161 for i := 0 ; i < numCPU ; i ++ {
6262 go func () {
63- ndM := utils .NewMeter ("NDJson" , utils.MeterOptions {
64- LogSlow : true ,
65- SlowThreshold : 20 * time .Millisecond ,
66- })
67-
68- bulkM := utils .NewMeter ("Bulk" , utils.MeterOptions {
69- LogSlow : true ,
70- SlowThreshold : 1 * time .Second ,
71- })
72-
7363 var ndMutex = & sync.Mutex {}
7464 var nd = make ([]opensearch.BulkItem , 0 , 10 )
7565
@@ -81,7 +71,6 @@ func startQueue() {
8171 }
8272
8373 ndMutex .Lock ()
84- bulkM .Reset ()
8574
8675 err := opensearch .Bulk (context .Background (), nd )
8776 if err != nil {
@@ -91,23 +80,17 @@ func startQueue() {
9180 nd = make ([]opensearch.BulkItem , 0 , 10 )
9281
9382 ndMutex .Unlock ()
94- bulkM .Elapsed ("bulk sent" )
9583 }
9684 }()
9785
9886 for {
9987 l := <- logs
10088
101- ndM .Reset ()
102-
10389 dataType := gjson .Get (l , "dataType" ).String ()
10490 id := gjson .Get (l , "id" ).String ()
10591 index := opensearch .BuildCurrentIndex ("v11" , "log" , dataType )
10692
107- ndM .Elapsed ("get index and id" )
108-
10993 ndMutex .Lock ()
110- ndM .Reset ()
11194
11295 nd = append (nd , opensearch.BulkItem {
11396 Index : index ,
@@ -117,7 +100,6 @@ func startQueue() {
117100 })
118101
119102 ndMutex .Unlock ()
120- ndM .Elapsed ("append to NDJson list" )
121103 }
122104 }()
123105 }
0 commit comments