Skip to content

Log a warning if the peer timeout for the replica is greater than the repeat interval for a route #417

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
santihernandezc wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Log a warning if the peer timeout for the replica is greater than the repeat interval for a route #417

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions notify/grafana_alertmanager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -711,6 +711,11 @@ func (am *GrafanaAlertmanager) ApplyConfig(cfg NotificationsConfiguration) (err
return err
}

if pos := am.opts.Peer.Position(); pos != 0 {
waitTime := time.Duration(pos) * am.opts.PeerTimeout
am.checkRepeatIntervals(waitTime, cfg.RoutingTree)
}

// Now, let's put together our notification pipeline
routingStage := make(notify.RoutingStage, len(integrationsMap))

Expand Down Expand Up @@ -769,6 +774,17 @@ func (am *GrafanaAlertmanager) ApplyConfig(cfg NotificationsConfiguration) (err
return nil
}

// checkRepeatIntervals logs a warning if pos * peer_timeout > repeat_interval for any route.
// If the time we wait before trying to send a notification is greater than the repeat interval, we risk sending duplicate notifications.
func (am *GrafanaAlertmanager) checkRepeatIntervals(waitTime time.Duration, route *Route) {
if route.RepeatInterval != nil && time.Duration(*route.RepeatInterval) < waitTime {
level.Warn(am.logger).Log("Route's repeat_interval is shorter than the waiting period for the current peer. This can lead to duplicate notifications", "repeat_interval", *route.RepeatInterval, "wait_time_for_peer", waitTime)
}
for _, r := range route.Routes {
am.checkRepeatIntervals(waitTime, r)
}
}

func (am *GrafanaAlertmanager) setInhibitionRulesMetrics(r []InhibitRule) {
am.opts.Metrics.configuredInhibitionRules.WithLabelValues(am.tenantString()).Set(float64(len(r)))
}
Expand Down