Skip to content

#3513: Show muted alerts in the Alert Groups API #3797

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 23, 2024
Merged

#3513: Show muted alerts in the Alert Groups API #3797

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions api/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,16 +45,20 @@ type API struct {
inFlightSem chan struct{}
}

// Options for the creation of an API object. Alerts, Silences, and StatusFunc
// are mandatory to set. The zero value for everything else is a safe default.
// Options for the creation of an API object. Alerts, Silences, AlertStatusFunc
// and GroupMutedFunc are mandatory. The zero value for everything else is a safe
// default.
type Options struct {
// Alerts to be used by the API. Mandatory.
Alerts provider.Alerts
// Silences to be used by the API. Mandatory.
Silences *silence.Silences
// StatusFunc is used be the API to retrieve the AlertStatus of an
// AlertStatusFunc is used be the API to retrieve the AlertStatus of an
// alert. Mandatory.
StatusFunc func(model.Fingerprint) types.AlertStatus
AlertStatusFunc func(model.Fingerprint) types.AlertStatus
// GroupMutedFunc is used be the API to know if an alert is muted.
// Mandatory.
GroupMutedFunc func(routeID, groupKey string) ([]string, bool)
// Peer from the gossip cluster. If nil, no clustering will be used.
Peer cluster.ClusterPeer
// Timeout for all HTTP connections. The zero value (and negative
Expand Down Expand Up @@ -83,8 +87,11 @@ func (o Options) validate() error {
if o.Silences == nil {
return errors.New("mandatory field Silences not set")
}
if o.StatusFunc == nil {
return errors.New("mandatory field StatusFunc not set")
if o.AlertStatusFunc == nil {
return errors.New("mandatory field AlertStatusFunc not set")
}
if o.GroupMutedFunc == nil {
return errors.New("mandatory field GroupMutedFunc not set")
}
if o.GroupFunc == nil {
return errors.New("mandatory field GroupFunc not set")
Expand Down Expand Up @@ -113,7 +120,8 @@ func New(opts Options) (*API, error) {
v2, err := apiv2.NewAPI(
opts.Alerts,
opts.GroupFunc,
opts.StatusFunc,
opts.AlertStatusFunc,
opts.GroupMutedFunc,
opts.Silences,
opts.Peer,
log.With(l, "version", "v2"),
Expand Down
17 changes: 13 additions & 4 deletions api/v2/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ type API struct {
alerts provider.Alerts
alertGroups groupsFn
getAlertStatus getAlertStatusFn
groupMutedFunc groupMutedFunc
uptime time.Time

// mtx protects alertmanagerConfig, setAlertStatus and route.
Expand All @@ -78,6 +79,7 @@ type API struct {

type (
groupsFn func(func(*dispatch.Route) bool, func(*types.Alert, time.Time) bool) (dispatch.AlertGroups, map[prometheus_model.Fingerprint][]string)
groupMutedFunc func(routeID, groupKey string) ([]string, bool)
getAlertStatusFn func(prometheus_model.Fingerprint) types.AlertStatus
setAlertStatusFn func(prometheus_model.LabelSet)
)
Expand All @@ -86,16 +88,18 @@ type (
func NewAPI(
alerts provider.Alerts,
gf groupsFn,
sf getAlertStatusFn,
asf getAlertStatusFn,
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not the best variable name, option to suggestions! 😄

gmf groupMutedFunc,
Comment on lines +91 to +92
Copy link
Member

@SuperQ SuperQ Oct 17, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about this for consistent naming:

Suggested change
asf getAlertStatusFn,
gmf groupMutedFunc,
asf alertStatusFn,
gmf groupMutedFn,

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't mind the types as these two are consistent:

getAlertStatusFn func(prometheus_model.Fingerprint) types.AlertStatus
setAlertStatusFn func(prometheus_model.LabelSet)

It was more the variable names asf and gmf that I didn't like.

silences *silence.Silences,
peer cluster.ClusterPeer,
l log.Logger,
r prometheus.Registerer,
) (*API, error) {
api := API{
alerts: alerts,
getAlertStatus: sf,
getAlertStatus: asf,
alertGroups: gf,
groupMutedFunc: gmf,
peer: peer,
silences: silences,
logger: l,
Expand Down Expand Up @@ -290,7 +294,7 @@ func (api *API) getAlertsHandler(params alert_ops.GetAlertsParams) middleware.Re
continue
}

alert := AlertToOpenAPIAlert(a, api.getAlertStatus(a.Fingerprint()), receivers)
alert := AlertToOpenAPIAlert(a, api.getAlertStatus(a.Fingerprint()), receivers, nil)

res = append(res, alert)
}
Expand Down Expand Up @@ -407,6 +411,11 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams
res := make(open_api_models.AlertGroups, 0, len(alertGroups))

for _, alertGroup := range alertGroups {
mutedBy, isMuted := api.groupMutedFunc(alertGroup.RouteID, alertGroup.GroupKey)
if !*params.Muted && isMuted {
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This skips muted groups if muted is false.

continue
}

ag := &open_api_models.AlertGroup{
Receiver: &open_api_models.Receiver{Name: &alertGroup.Receiver},
Labels: ModelLabelSetToAPILabelSet(alertGroup.Labels),
Expand All @@ -417,7 +426,7 @@ func (api *API) getAlertGroupsHandler(params alertgroup_ops.GetAlertGroupsParams
fp := alert.Fingerprint()
receivers := allReceivers[fp]
status := api.getAlertStatus(fp)
apiAlert := AlertToOpenAPIAlert(alert, status, receivers)
apiAlert := AlertToOpenAPIAlert(alert, status, receivers, mutedBy)
ag.Alerts = append(ag.Alerts, apiAlert)
}
res = append(res, ag)
Expand Down
13 changes: 7 additions & 6 deletions api/v2/api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,17 +406,12 @@ func TestAlertToOpenAPIAlert(t *testing.T) {
UpdatedAt: updated,
}
)
openAPIAlert := AlertToOpenAPIAlert(alert, types.AlertStatus{State: types.AlertStateActive}, receivers)
openAPIAlert := AlertToOpenAPIAlert(alert, types.AlertStatus{State: types.AlertStateActive}, receivers, nil)
require.Equal(t, &open_api_models.GettableAlert{
Annotations: open_api_models.LabelSet{},
Alert: open_api_models.Alert{
Labels: open_api_models.LabelSet{"severity": "critical", "alertname": "alert1"},
},
Status: &open_api_models.AlertStatus{
State: &active,
InhibitedBy: []string{},
SilencedBy: []string{},
},
StartsAt: convertDateTime(start),
EndsAt: convertDateTime(time.Time{}),
UpdatedAt: convertDateTime(updated),
Expand All @@ -425,6 +420,12 @@ func TestAlertToOpenAPIAlert(t *testing.T) {
{Name: &receivers[0]},
{Name: &receivers[1]},
},
Status: &open_api_models.AlertStatus{
State: &active,
InhibitedBy: []string{},
SilencedBy: []string{},
MutedBy: []string{},
},
}, openAPIAlert)
}

Expand Down
39 changes: 39 additions & 0 deletions api/v2/client/alertgroup/get_alert_groups_parameters.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 12 additions & 1 deletion api/v2/compat.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func PostableSilenceToProto(s *open_api_models.PostableSilence) (*silencepb.Sile
}

// AlertToOpenAPIAlert converts internal alerts, alert types, and receivers to *open_api_models.GettableAlert.
func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers []string) *open_api_models.GettableAlert {
func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers, mutedBy []string) *open_api_models.GettableAlert {
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mutedBy is separate from types.AlertStatus because it is related to groups, and does not come from the AlertMarker interface.

startsAt := strfmt.DateTime(alert.StartsAt)
updatedAt := strfmt.DateTime(alert.UpdatedAt)
endsAt := strfmt.DateTime(alert.EndsAt)
Expand All @@ -128,7 +128,13 @@ func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers
}

fp := alert.Fingerprint().String()

state := string(status.State)
if len(mutedBy) > 0 {
// If the alert is muted, change the state to suppressed.
state = open_api_models.AlertStatusStateSuppressed
}

aa := &open_api_models.GettableAlert{
Alert: open_api_models.Alert{
GeneratorURL: strfmt.URI(alert.GeneratorURL),
Expand All @@ -144,6 +150,7 @@ func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers
State: &state,
SilencedBy: status.SilencedBy,
InhibitedBy: status.InhibitedBy,
MutedBy: mutedBy,
},
}

Expand All @@ -155,6 +162,10 @@ func AlertToOpenAPIAlert(alert *types.Alert, status types.AlertStatus, receivers
aa.Status.InhibitedBy = []string{}
}

if aa.Status.MutedBy == nil {
aa.Status.MutedBy = []string{}
}

return aa
}

Expand Down
17 changes: 17 additions & 0 deletions api/v2/models/alert_status.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 10 additions & 0 deletions api/v2/openapi.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,11 @@ paths:
type: boolean
description: Show inhibited alerts
default: true
- in: query
name: muted
type: boolean
description: Show muted alerts
default: true
- name: filter
in: query
description: A list of matchers to filter alerts by
Expand Down Expand Up @@ -501,10 +506,15 @@ definitions:
type: array
items:
type: string
mutedBy:
type: array
items:
type: string
required:
- state
- silencedBy
- inhibitedBy
- mutedBy
receiver:
type: object
properties:
Expand Down
Loading
Loading