Enforce 512KB event size limit for Pagerduty events (prometheus#2225)

* Enforce 512kb event size limit for Pagerduty

Signed-off-by: Shamil Ishraq <[email protected]>

* Add size limit to error message

Signed-off-by: Shamil Ishraq <[email protected]>

* Replace MaxEventSize setting with a const.

Signed-off-by: Shamil Ishraq <[email protected]>

* Change to package variable

Signed-off-by: Shamil Ishraq <[email protected]>

* Removed recursion in encodeMessage()

Signed-off-by: Shamil Ishraq <[email protected]>

* Unexport maxEventSize

Signed-off-by: Shamil Ishraq <[email protected]>

Author: shamilpd

go.mod

@@ -1,6 +1,7 @@
 module github.com/prometheus/alertmanager
 
 require (
+	github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4
 	github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409
 	github.com/cespare/xxhash v1.1.0
 	github.com/go-kit/kit v0.9.0

notify/pagerduty/pagerduty.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/alecthomas/units"
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/pkg/errors"
@@ -34,6 +35,8 @@ import (
 	"github.com/prometheus/alertmanager/types"
 )
 
+const maxEventSize int = 512000
+
 // Notifier implements a Notifier for PagerDuty notifications.
 type Notifier struct {
 	conf    *config.PagerdutyConfig
@@ -107,6 +110,33 @@ type pagerDutyPayload struct {
 	CustomDetails map[string]string `json:"custom_details,omitempty"`
 }
 
+func (n *Notifier) encodeMessage(msg *pagerDutyMessage) (bytes.Buffer, error) {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+		return buf, errors.Wrap(err, "failed to encode PagerDuty message")
+	}
+
+	if buf.Len() > maxEventSize {
+		truncatedMsg := fmt.Sprintf("Custom details have been removed because the original event exceeds the maximum size of %s", units.MetricBytes(maxEventSize).String())
+
+		if n.apiV1 != "" {
+			msg.Details = map[string]string{"error": truncatedMsg}
+		} else {
+			msg.Payload.CustomDetails = map[string]string{"error": truncatedMsg}
+		}
+
+		warningMsg := fmt.Sprintf("Truncated Details because message of size %s exceeds limit %s", units.MetricBytes(buf.Len()).String(), units.MetricBytes(maxEventSize).String())
+		level.Warn(n.logger).Log("msg", warningMsg)
+
+		buf.Reset()
+		if err := json.NewEncoder(&buf).Encode(msg); err != nil {
+			return buf, errors.Wrap(err, "failed to encode PagerDuty message")
+		}
+	}
+
+	return buf, nil
+}
+
 func (n *Notifier) notifyV1(
 	ctx context.Context,
 	eventType string,
@@ -145,12 +175,12 @@ func (n *Notifier) notifyV1(
 		return false, errors.New("service key cannot be empty")
 	}
 
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
-		return false, errors.Wrap(err, "failed to encode PagerDuty v1 message")
+	encodedMsg, err := n.encodeMessage(msg)
+	if err != nil {
+		return false, err
 	}
 
-	resp, err := notify.PostJSON(ctx, n.client, n.apiV1, &buf)
+	resp, err := notify.PostJSON(ctx, n.client, n.apiV1, &encodedMsg)
 	if err != nil {
 		return true, errors.Wrap(err, "failed to post message to PagerDuty v1")
 	}
@@ -218,12 +248,12 @@ func (n *Notifier) notifyV2(
 		return false, errors.New("routing key cannot be empty")
 	}
 
-	var buf bytes.Buffer
-	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
-		return false, errors.Wrap(err, "failed to encode PagerDuty v2 message")
+	encodedMsg, err := n.encodeMessage(msg)
+	if err != nil {
+		return false, err
 	}
 
-	resp, err := notify.PostJSON(ctx, n.client, n.conf.URL.String(), &buf)
+	resp, err := notify.PostJSON(ctx, n.client, n.conf.URL.String(), &encodedMsg)
 	if err != nil {
 		return true, errors.Wrap(err, "failed to post message to PagerDuty")
 	}

notify/pagerduty/pagerduty_test.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"strings"
 	"testing"
 	"time"
 
@@ -271,3 +272,53 @@ func TestErrDetails(t *testing.T) {
 		})
 	}
 }
+
+func TestEventSizeEnforcement(t *testing.T) {
+	bigDetails := map[string]string{
+		"firing": strings.Repeat("a", 513000),
+	}
+
+	// V1 Messages
+	msgV1 := &pagerDutyMessage{
+		ServiceKey: "01234567890123456789012345678901",
+		EventType:  "trigger",
+		Details:    bigDetails,
+	}
+
+	notifierV1, err := New(
+		&config.PagerdutyConfig{
+			ServiceKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		log.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	encodedV1, err := notifierV1.encodeMessage(msgV1)
+	require.NoError(t, err)
+	require.Contains(t, encodedV1.String(), `"details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+
+	// V2 Messages
+	msgV2 := &pagerDutyMessage{
+		RoutingKey:  "01234567890123456789012345678901",
+		EventAction: "trigger",
+		Payload: &pagerDutyPayload{
+			CustomDetails: bigDetails,
+		},
+	}
+
+	notifierV2, err := New(
+		&config.PagerdutyConfig{
+			RoutingKey: config.Secret("01234567890123456789012345678901"),
+			HTTPConfig: &commoncfg.HTTPClientConfig{},
+		},
+		test.CreateTmpl(t),
+		log.NewNopLogger(),
+	)
+	require.NoError(t, err)
+
+	encodedV2, err := notifierV2.encodeMessage(msgV2)
+	require.NoError(t, err)
+	require.Contains(t, encodedV2.String(), `"custom_details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+}