Skip to content

refactor: refactor checker into cfg #92

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
coanor wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

refactor: refactor checker into cfg #92

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a4201ed
refactor: refactor checker into cfg
coanor Apr 30, 2024
3bf701c
update cfg applied to point
coanor Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 95 additions & 60 deletions point/check.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,69 +10,103 @@ import (
"math"
"reflect"
"strings"
"time"
)

type checker struct {
*cfg
warns []*Warn
}

func (c *checker) reset() {
func (c *cfg) check(pt *Point) *Point {
// reset warns: make the point can check multiple times and the warns(if any)
// not duplicated.
c.warns = c.warns[:0]
}

func (c *checker) check(pt *Point) *Point {
pt.pt.Name = c.checkMeasurement(pt.pt.Name)
pt.pt.Fields = c.checkKVs(pt.pt.Fields)
c.checkTime(pt)

pt.pt.Warns = append(pt.pt.Warns, c.warns...)

// Add more checkings...
pt.SetFlag(Pcheck)

return pt
}

func (c *checker) addWarn(t, msg string) {
func (c *cfg) addWarn(t, msg string) {
c.warns = append(c.warns, &Warn{
Type: t, Msg: msg,
})
}

func (c *checker) checkMeasurement(m string) string {
func (c *cfg) checkTime(pt *Point) {
x := pt.pt.Time

if c.timestamp != -1 && x == 0 { // apply time in cfg
x = c.timestamp
}

if x < 0 && !c.enableNagativeTimestamp { // Set to current time.
c.addWarn(WarnNagativeTimestamp, fmt.Sprintf("got nagative timestamp %d, reset to current time", x))
x = time.Now().Round(0).UnixNano() // trim monotonic clock
}

switch c.precision {
case PrecUS:
x *= int64(time.Microsecond)
case PrecMS:
x *= int64(time.Millisecond)
case PrecS:
x *= int64(time.Second)
case PrecM:
x *= int64(time.Minute)
case PrecH:
x *= int64(time.Hour)
case PrecNS: // pass
case PrecD:
x *= (24 * int64(time.Hour))
case PrecW:
x *= (7 * 24 * int64(time.Hour))
default:
// pass
}

pt.pt.Time = x
}

func (c *cfg) checkMeasurement(m string) string {
if len(m) == 0 {
c.addWarn(WarnInvalidMeasurement,
fmt.Sprintf("empty measurement, use %s", DefaultMeasurementName))
m = DefaultMeasurementName
}

if c.cfg.maxMeasurementLen > 0 && len(m) > c.cfg.maxMeasurementLen {
if c.maxMeasurementLen > 0 && len(m) > c.maxMeasurementLen {
c.addWarn(WarnInvalidMeasurement,
fmt.Sprintf("exceed max measurement length(%d), got length %d, trimmed",
c.cfg.maxMeasurementLen, len(m)))
return m[:c.cfg.maxMeasurementLen]
c.maxMeasurementLen, len(m)))
return m[:c.maxMeasurementLen]
} else {
return m
}
}

func (c *checker) checkKVs(kvs KVs) KVs {
func (c *cfg) checkKVs(kvs KVs) KVs {
tcnt := kvs.TagCount()
fcnt := kvs.FieldCount()

// delete extra fields
if c.cfg.maxFields > 0 && fcnt > c.cfg.maxFields {
if c.maxFields > 0 && fcnt > c.maxFields {
c.addWarn(WarnMaxFields,
fmt.Sprintf("exceed max field count(%d), got %d fields, extra fields deleted",
c.cfg.maxFields, fcnt))
c.maxFields, fcnt))

kvs = kvs.TrimFields(c.cfg.maxFields)
kvs = kvs.TrimFields(c.maxFields)
}

// delete extra tags
if c.cfg.maxTags > 0 && tcnt > c.cfg.maxTags {
if c.maxTags > 0 && tcnt > c.maxTags {
c.addWarn(WarnMaxFields,
fmt.Sprintf("exceed max tag count(%d), got %d tags, extra tags deleted",
c.cfg.maxTags, tcnt))
c.maxTags, tcnt))

kvs = kvs.TrimTags(c.cfg.maxTags)
kvs = kvs.TrimTags(c.maxTags)
}

// check each kv valid
Expand Down Expand Up @@ -113,15 +147,15 @@ func adjustKV(x string) string {
return x
}

func (c *checker) checkKV(f *Field, kvs KVs) (*Field, bool) {
func (c *cfg) checkKV(f *Field, kvs KVs) (*Field, bool) {
if f.IsTag {
return c.checkTag(f, kvs)
} else {
return c.checkField(f, kvs)
}
}

func (c *checker) keyConflict(key string, kvs KVs) bool {
func (c *cfg) keyConflict(key string, kvs KVs) bool {
if kvs.Get(key) != nil { // key exist
c.addWarn(WarnKeyNameConflict,
fmt.Sprintf("same key after rename(%q), kv dropped", key))
Expand All @@ -133,13 +167,13 @@ func (c *checker) keyConflict(key string, kvs KVs) bool {

// checkTag try to auto modify the f. If we need to drop
// f, we return false.
func (c *checker) checkTag(f *Field, kvs KVs) (*Field, bool) {
if c.cfg.maxTagKeyLen > 0 && len(f.Key) > c.cfg.maxTagKeyLen {
func (c *cfg) checkTag(f *Field, kvs KVs) (*Field, bool) {
if c.maxTagKeyLen > 0 && len(f.Key) > c.maxTagKeyLen {
c.addWarn(WarnMaxTagKeyLen,
fmt.Sprintf("exceed max tag key length(%d), got %d, key truncated",
c.cfg.maxTagKeyLen, len(f.Key)))
c.maxTagKeyLen, len(f.Key)))

newKey := f.Key[:c.cfg.maxTagKeyLen]
newKey := f.Key[:c.maxTagKeyLen]
if c.keyConflict(newKey, kvs) {
return f, false
} else {
Expand All @@ -149,12 +183,12 @@ func (c *checker) checkTag(f *Field, kvs KVs) (*Field, bool) {

x := f.Val.(*Field_S)

if c.cfg.maxTagValLen > 0 && len(x.S) > c.cfg.maxTagValLen {
if c.maxTagValLen > 0 && len(x.S) > c.maxTagValLen {
c.addWarn(WarnMaxTagValueLen,
fmt.Sprintf("exceed max tag value length(%d), got %d, value truncated",
c.cfg.maxTagValLen, len(x.S)))
c.maxTagValLen, len(x.S)))

x.S = x.S[:c.cfg.maxTagValLen]
x.S = x.S[:c.maxTagValLen]
f.Val = x
}

Expand All @@ -179,7 +213,7 @@ func (c *checker) checkTag(f *Field, kvs KVs) (*Field, bool) {
}

// replace `.' with `_' in tag keys
if strings.Contains(f.Key, ".") && !c.cfg.enableDotInKey {
if strings.Contains(f.Key, ".") && !c.enableDotInKey {
c.addWarn(WarnInvalidTagKey, fmt.Sprintf("invalid tag key `%s': found `.'", f.Key))

newKey := strings.ReplaceAll(f.Key, ".", "_")
Expand All @@ -200,14 +234,14 @@ func (c *checker) checkTag(f *Field, kvs KVs) (*Field, bool) {

// checkField try to auto modify the f. If we need to drop
// f, we return false.
func (c *checker) checkField(f *Field, kvs KVs) (*Field, bool) {
func (c *cfg) checkField(f *Field, kvs KVs) (*Field, bool) {
// trim key
if c.cfg.maxFieldKeyLen > 0 && len(f.Key) > c.cfg.maxFieldKeyLen {
if c.maxFieldKeyLen > 0 && len(f.Key) > c.maxFieldKeyLen {
c.addWarn(WarnMaxFieldKeyLen,
fmt.Sprintf("exceed max field key length(%d), got %d, key truncated to %s",
c.cfg.maxFieldKeyLen, len(f.Key), f.Key))
c.maxFieldKeyLen, len(f.Key), f.Key))

newKey := f.Key[:c.cfg.maxFieldKeyLen]
newKey := f.Key[:c.maxFieldKeyLen]

if c.keyConflict(newKey, kvs) {
return f, false
Expand All @@ -216,7 +250,7 @@ func (c *checker) checkField(f *Field, kvs KVs) (*Field, bool) {
}
}

if strings.Contains(f.Key, ".") && !c.cfg.enableDotInKey {
if strings.Contains(f.Key, ".") && !c.enableDotInKey {
c.addWarn(WarnDotInkey,
fmt.Sprintf("invalid field key `%s': found `.'", f.Key))

Expand All @@ -236,7 +270,7 @@ func (c *checker) checkField(f *Field, kvs KVs) (*Field, bool) {

switch x := f.Val.(type) {
case *Field_U:
if !c.cfg.enableU64Field {
if !c.enableU64Field {
if x.U > uint64(math.MaxInt64) {
c.addWarn(WarnMaxFieldValueInt,
fmt.Sprintf("too large int field: key=%s, value=%d(> %d)", f.Key, x.U, uint64(math.MaxInt64)))
Expand All @@ -263,35 +297,35 @@ func (c *checker) checkField(f *Field, kvs KVs) (*Field, bool) {

case *Field_D: // same as []uint8

if !c.cfg.enableStrField {
if !c.enableStrField {
c.addWarn(WarnInvalidFieldValueType,
fmt.Sprintf("field(%s) dropped with string value, when [DisableStringField] enabled", f.Key))
return f, false
}

if c.cfg.maxFieldValLen > 0 && len(x.D) > c.cfg.maxFieldValLen {
if c.maxFieldValLen > 0 && len(x.D) > c.maxFieldValLen {
c.addWarn(WarnMaxFieldValueLen,
fmt.Sprintf("field (%s) exceed max field value length(%d), got %d, value truncated",
f.Key, c.cfg.maxFieldValLen, len(x.D)))
f.Key, c.maxFieldValLen, len(x.D)))

x.D = x.D[:c.cfg.maxFieldValLen]
x.D = x.D[:c.maxFieldValLen]
f.Val = x
}

case *Field_S: // same as Field_D

if !c.cfg.enableStrField {
if !c.enableStrField {
c.addWarn(WarnInvalidFieldValueType,
fmt.Sprintf("field(%s) dropped with string value, when [DisableStringField] enabled", f.Key))
return f, false
}

if c.cfg.maxFieldValLen > 0 && len(x.S) > c.cfg.maxFieldValLen {
if c.maxFieldValLen > 0 && len(x.S) > c.maxFieldValLen {
c.addWarn(WarnMaxFieldValueLen,
fmt.Sprintf("field (%s) exceed max field value length(%d), got %d, value truncated",
f.Key, c.cfg.maxFieldValLen, len(x.S)))
f.Key, c.maxFieldValLen, len(x.S)))

x.S = x.S[:c.cfg.maxFieldValLen]
x.S = x.S[:c.maxFieldValLen]
f.Val = x
}

Expand All @@ -317,16 +351,16 @@ func trimSuffixAll(s, sfx string) string {
return x
}

func (c *checker) keyDisabled(k string) bool {
func (c *cfg) keyDisabled(k string) bool {
if k == "" {
return true
}

if c.cfg.disabledKeys == nil {
if c.disabledKeys == nil {
return false
}

for _, item := range c.cfg.disabledKeys {
for _, item := range c.disabledKeys {
if k == item.key {
return true
}
Expand All @@ -335,12 +369,12 @@ func (c *checker) keyDisabled(k string) bool {
return false
}

func (c *checker) keyMiss(kvs KVs) KVs {
if c.cfg.requiredKeys == nil {
func (c *cfg) keyMiss(kvs KVs) KVs {
if c.requiredKeys == nil {
return kvs
}

for _, rk := range c.cfg.requiredKeys {
for _, rk := range c.requiredKeys {
if !kvs.Has(rk.Key()) {
if def := rk.Default(); def != nil {
kvs = kvs.MustAddKV(NewKV(rk.Key(), def))
Expand All @@ -356,23 +390,24 @@ func (c *checker) keyMiss(kvs KVs) KVs {

// CheckPoints used to check pts on various opts.
func CheckPoints(pts []*Point, opts ...Option) (arr []*Point) {
c := GetCfg(opts...)
defer PutCfg(c)

chk := checker{cfg: c}

arr = pts[:0]

for _, pt := range pts {
if pt.pt == nil {
continue
}

pt = chk.check(pt)
pt.SetFlag(Pcheck)
pt.pt.Warns = chk.warns
for _, opt := range opts {
if opt == nil {
continue
}

opt(pt.cfg)
}

pt = pt.cfg.check(pt)

arr = append(arr, pt)
chk.reset()
}

return arr
Expand Down
Loading