EC2 SD: fix error handling of NewSessionWithOptions (prometheus#8356)

Last change in 4efca5a introduced a problem where NewDiscovery would
just return a nil value, which is not handled well and didn't allow for
fixing configuration issues at runtime without a reload.

Signed-off-by: Alfred Krohmer <[email protected]>

Author: devkid

discovery/ec2/ec2.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
@@ -129,66 +130,71 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // the Discoverer interface.
 type Discovery struct {
 	*refresh.Discovery
-	region   string
-	interval time.Duration
-	port     int
-	filters  []*Filter
-	ec2      *ec2.EC2
+	cfg *SDConfig
+	ec2 *ec2.EC2
 }
 
 // NewDiscovery returns a new EC2Discovery which periodically refreshes its targets.
 func NewDiscovery(conf *SDConfig, logger log.Logger) *Discovery {
-	creds := credentials.NewStaticCredentials(conf.AccessKey, string(conf.SecretKey), "")
-	if conf.AccessKey == "" && conf.SecretKey == "" {
-		creds = nil
-	}
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
+	d := &Discovery{
+		cfg: conf,
+	}
+	d.Discovery = refresh.NewDiscovery(
+		logger,
+		"ec2",
+		time.Duration(d.cfg.RefreshInterval),
+		d.refresh,
+	)
+	return d
+}
+
+func (d *Discovery) ec2Client() (*ec2.EC2, error) {
+	if d.ec2 != nil {
+		return d.ec2, nil
+	}
+
+	creds := credentials.NewStaticCredentials(d.cfg.AccessKey, string(d.cfg.SecretKey), "")
+	if d.cfg.AccessKey == "" && d.cfg.SecretKey == "" {
+		creds = nil
+	}
 
 	sess, err := session.NewSessionWithOptions(session.Options{
 		Config: aws.Config{
-			Endpoint:    &conf.Endpoint,
-			Region:      &conf.Region,
+			Endpoint:    &d.cfg.Endpoint,
+			Region:      &d.cfg.Region,
 			Credentials: creds,
 		},
-		Profile: conf.Profile,
+		Profile: d.cfg.Profile,
 	})
 	if err != nil {
-		return nil
+		return nil, errors.Wrap(err, "could not create aws session")
 	}
 
-	var ec2s *ec2.EC2
-	if conf.RoleARN != "" {
-		creds := stscreds.NewCredentials(sess, conf.RoleARN)
-		ec2s = ec2.New(sess, &aws.Config{Credentials: creds})
+	if d.cfg.RoleARN != "" {
+		creds := stscreds.NewCredentials(sess, d.cfg.RoleARN)
+		d.ec2 = ec2.New(sess, &aws.Config{Credentials: creds})
 	} else {
-		ec2s = ec2.New(sess)
+		d.ec2 = ec2.New(sess)
 	}
 
-	d := &Discovery{
-		region:   conf.Region,
-		filters:  conf.Filters,
-		interval: time.Duration(conf.RefreshInterval),
-		port:     conf.Port,
-		ec2:      ec2s,
-	}
-	d.Discovery = refresh.NewDiscovery(
-		logger,
-		"ec2",
-		time.Duration(conf.RefreshInterval),
-		d.refresh,
-	)
-	return d
+	return d.ec2, nil
 }
 
 func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
+	ec2Client, err := d.ec2Client()
+	if err != nil {
+		return nil, err
+	}
+
 	tg := &targetgroup.Group{
-		Source: d.region,
+		Source: d.cfg.Region,
 	}
 
 	var filters []*ec2.Filter
-	for _, f := range d.filters {
+	for _, f := range d.cfg.Filters {
 		filters = append(filters, &ec2.Filter{
 			Name:   aws.String(f.Name),
 			Values: aws.StringSlice(f.Values),
@@ -197,7 +203,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 
 	input := &ec2.DescribeInstancesInput{Filters: filters}
 
-	if err := d.ec2.DescribeInstancesPagesWithContext(ctx, input, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool {
+	if err := ec2Client.DescribeInstancesPagesWithContext(ctx, input, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool {
 		for _, r := range p.Reservations {
 			for _, inst := range r.Instances {
 				if inst.PrivateIpAddress == nil {
@@ -215,7 +221,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 				if inst.PrivateDnsName != nil {
 					labels[ec2LabelPrivateDNS] = model.LabelValue(*inst.PrivateDnsName)
 				}
-				addr := net.JoinHostPort(*inst.PrivateIpAddress, fmt.Sprintf("%d", d.port))
+				addr := net.JoinHostPort(*inst.PrivateIpAddress, fmt.Sprintf("%d", d.cfg.Port))
 				labels[model.AddressLabel] = model.LabelValue(addr)
 
 				if inst.Platform != nil {
@@ -285,6 +291,9 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 		}
 		return true
 	}); err != nil {
+		if awsErr, ok := err.(awserr.Error); ok && (awsErr.Code() == "AuthFailure" || awsErr.Code() == "UnauthorizedOperation") {
+			d.ec2 = nil
+		}
 		return nil, errors.Wrap(err, "could not describe instances")
 	}
 	return []*targetgroup.Group{tg}, nil