Skip to content

Commit d8ad301

Browse files
authored
Enforce 512KB event size limit for Pagerduty events (prometheus#2225)
* Enforce 512kb event size limit for Pagerduty Signed-off-by: Shamil Ishraq <[email protected]> * Add size limit to error message Signed-off-by: Shamil Ishraq <[email protected]> * Replace MaxEventSize setting with a const. Signed-off-by: Shamil Ishraq <[email protected]> * Change to package variable Signed-off-by: Shamil Ishraq <[email protected]> * Removed recursion in encodeMessage() Signed-off-by: Shamil Ishraq <[email protected]> * Unexport maxEventSize Signed-off-by: Shamil Ishraq <[email protected]>
1 parent e7adbea commit d8ad301

File tree

3 files changed

+90
-8
lines changed

3 files changed

+90
-8
lines changed

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
module github.com/prometheus/alertmanager
22

33
require (
4+
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4
45
github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409
56
github.com/cespare/xxhash v1.1.0
67
github.com/go-kit/kit v0.9.0

notify/pagerduty/pagerduty.go

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"net/http"
2323
"strings"
2424

25+
"github.com/alecthomas/units"
2526
"github.com/go-kit/kit/log"
2627
"github.com/go-kit/kit/log/level"
2728
"github.com/pkg/errors"
@@ -34,6 +35,8 @@ import (
3435
"github.com/prometheus/alertmanager/types"
3536
)
3637

38+
const maxEventSize int = 512000
39+
3740
// Notifier implements a Notifier for PagerDuty notifications.
3841
type Notifier struct {
3942
conf *config.PagerdutyConfig
@@ -107,6 +110,33 @@ type pagerDutyPayload struct {
107110
CustomDetails map[string]string `json:"custom_details,omitempty"`
108111
}
109112

113+
func (n *Notifier) encodeMessage(msg *pagerDutyMessage) (bytes.Buffer, error) {
114+
var buf bytes.Buffer
115+
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
116+
return buf, errors.Wrap(err, "failed to encode PagerDuty message")
117+
}
118+
119+
if buf.Len() > maxEventSize {
120+
truncatedMsg := fmt.Sprintf("Custom details have been removed because the original event exceeds the maximum size of %s", units.MetricBytes(maxEventSize).String())
121+
122+
if n.apiV1 != "" {
123+
msg.Details = map[string]string{"error": truncatedMsg}
124+
} else {
125+
msg.Payload.CustomDetails = map[string]string{"error": truncatedMsg}
126+
}
127+
128+
warningMsg := fmt.Sprintf("Truncated Details because message of size %s exceeds limit %s", units.MetricBytes(buf.Len()).String(), units.MetricBytes(maxEventSize).String())
129+
level.Warn(n.logger).Log("msg", warningMsg)
130+
131+
buf.Reset()
132+
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
133+
return buf, errors.Wrap(err, "failed to encode PagerDuty message")
134+
}
135+
}
136+
137+
return buf, nil
138+
}
139+
110140
func (n *Notifier) notifyV1(
111141
ctx context.Context,
112142
eventType string,
@@ -145,12 +175,12 @@ func (n *Notifier) notifyV1(
145175
return false, errors.New("service key cannot be empty")
146176
}
147177

148-
var buf bytes.Buffer
149-
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
150-
return false, errors.Wrap(err, "failed to encode PagerDuty v1 message")
178+
encodedMsg, err := n.encodeMessage(msg)
179+
if err != nil {
180+
return false, err
151181
}
152182

153-
resp, err := notify.PostJSON(ctx, n.client, n.apiV1, &buf)
183+
resp, err := notify.PostJSON(ctx, n.client, n.apiV1, &encodedMsg)
154184
if err != nil {
155185
return true, errors.Wrap(err, "failed to post message to PagerDuty v1")
156186
}
@@ -218,12 +248,12 @@ func (n *Notifier) notifyV2(
218248
return false, errors.New("routing key cannot be empty")
219249
}
220250

221-
var buf bytes.Buffer
222-
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
223-
return false, errors.Wrap(err, "failed to encode PagerDuty v2 message")
251+
encodedMsg, err := n.encodeMessage(msg)
252+
if err != nil {
253+
return false, err
224254
}
225255

226-
resp, err := notify.PostJSON(ctx, n.client, n.conf.URL.String(), &buf)
256+
resp, err := notify.PostJSON(ctx, n.client, n.conf.URL.String(), &encodedMsg)
227257
if err != nil {
228258
return true, errors.Wrap(err, "failed to post message to PagerDuty")
229259
}

notify/pagerduty/pagerduty_test.go

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"net/http"
2323
"net/http/httptest"
2424
"net/url"
25+
"strings"
2526
"testing"
2627
"time"
2728

@@ -271,3 +272,53 @@ func TestErrDetails(t *testing.T) {
271272
})
272273
}
273274
}
275+
276+
func TestEventSizeEnforcement(t *testing.T) {
277+
bigDetails := map[string]string{
278+
"firing": strings.Repeat("a", 513000),
279+
}
280+
281+
// V1 Messages
282+
msgV1 := &pagerDutyMessage{
283+
ServiceKey: "01234567890123456789012345678901",
284+
EventType: "trigger",
285+
Details: bigDetails,
286+
}
287+
288+
notifierV1, err := New(
289+
&config.PagerdutyConfig{
290+
ServiceKey: config.Secret("01234567890123456789012345678901"),
291+
HTTPConfig: &commoncfg.HTTPClientConfig{},
292+
},
293+
test.CreateTmpl(t),
294+
log.NewNopLogger(),
295+
)
296+
require.NoError(t, err)
297+
298+
encodedV1, err := notifierV1.encodeMessage(msgV1)
299+
require.NoError(t, err)
300+
require.Contains(t, encodedV1.String(), `"details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
301+
302+
// V2 Messages
303+
msgV2 := &pagerDutyMessage{
304+
RoutingKey: "01234567890123456789012345678901",
305+
EventAction: "trigger",
306+
Payload: &pagerDutyPayload{
307+
CustomDetails: bigDetails,
308+
},
309+
}
310+
311+
notifierV2, err := New(
312+
&config.PagerdutyConfig{
313+
RoutingKey: config.Secret("01234567890123456789012345678901"),
314+
HTTPConfig: &commoncfg.HTTPClientConfig{},
315+
},
316+
test.CreateTmpl(t),
317+
log.NewNopLogger(),
318+
)
319+
require.NoError(t, err)
320+
321+
encodedV2, err := notifierV2.encodeMessage(msgV2)
322+
require.NoError(t, err)
323+
require.Contains(t, encodedV2.String(), `"custom_details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
324+
}

0 commit comments

Comments
 (0)