Merge pull request goadapp#112 from cwaltken-edrans/102-break-5min-lambda-barrier

break 5min lambda barrier & other major internal refactoring

Author: zeph

Dockerfile

@@ -1,12 +1,12 @@
-FROM golang:1.5
+FROM golang:1.8
 
 RUN apt-get update
 RUN apt-get install -y zip
-ADD . /go/src/github.com/gophergala2016/goad
-WORKDIR /go/src/github.com/gophergala2016/goad
-RUN go get -u github.com/jteeuwen/go-bindata/... 
-RUN make bindata
-RUN go build -o /go/bin/goad-api webapi/webapi.go
+ADD . /go/src/github.com/goadapp/goad
+WORKDIR /go/src/github.com/goadapp/goad
+RUN go get -u github.com/jteeuwen/go-bindata/...
+RUN make linux
+# RUN go build -o /go/bin/goad-api webapi/webapi.go
 
 CMD ["/go/bin/goad-api", "-addr", ":8080"]
 EXPOSE 8080

Godeps/Godeps.json

@@ -1,5 +1,8 @@
 all: osx linux windows
 
+test:
+	go test ./...
+
 lambda:
 	GOOS=linux GOARCH=amd64 go build -o data/lambda/goad-lambda ./lambda
 	zip -jr data/lambda data/lambda
@@ -21,6 +24,7 @@ windows: bindata
 
 clean:
 	rm -rf data/lambda/goad-lambda
+	rm -rf data/lambda.zip
 	rm -rf build
 
 all-zip: all

Makefile

@@ -56,6 +56,7 @@ var supportedRegions = []string{
 type Test struct {
 	Config *TestConfig
 	infra  *infrastructure.Infrastructure
+	lambdas int
 }
 
 // NewTest returns a configured Test
@@ -85,12 +86,13 @@ func (t *Test) Start() <-chan queue.RegionsAggData {
 	}
 
 	t.infra = infra
+	t.lambdas = numberOfLambdas(t.Config.Concurrency, len(t.Config.Regions))
 	t.invokeLambdas(awsConfig, infra.QueueURL())
 
 	results := make(chan queue.RegionsAggData)
 
 	go func() {
-		for result := range queue.Aggregate(awsConfig, infra.QueueURL(), t.Config.TotalRequests) {
+		for result := range queue.Aggregate(awsConfig, infra.QueueURL(), t.Config.TotalRequests, t.lambdas) {
 			results <- result
 		}
 		close(results)
@@ -100,15 +102,13 @@ func (t *Test) Start() <-chan queue.RegionsAggData {
 }
 
 func (t *Test) invokeLambdas(awsConfig *aws.Config, sqsURL string) {
-	lambdas := numberOfLambdas(t.Config.Concurrency, len(t.Config.Regions))
-
-	for i := 0; i < lambdas; i++ {
+	for i := 0; i < t.lambdas; i++ {
 		region := t.Config.Regions[i%len(t.Config.Regions)]
-		requests, requestsRemainder := divide(t.Config.TotalRequests, lambdas)
-		concurrency, _ := divide(t.Config.Concurrency, lambdas)
+		requests, requestsRemainder := divide(t.Config.TotalRequests, t.lambdas)
+		concurrency, _ := divide(t.Config.Concurrency, t.lambdas)
 		execTimeout := t.Config.ExecTimeout
 
-		if requestsRemainder > 0 && i == lambdas-1 {
+		if requestsRemainder > 0 && i == t.lambdas-1 {
 			requests += requestsRemainder
 		}
 
@@ -129,7 +129,7 @@ func (t *Test) invokeLambdas(awsConfig *aws.Config, sqsURL string) {
 			"-t",
 			fmt.Sprintf("%s", c.RequestTimeout.String()),
 			"-f",
-			fmt.Sprintf("%s", reportingFrequency(lambdas).String()),
+			fmt.Sprintf("%s", reportingFrequency(t.lambdas).String()),
 			"-r",
 			fmt.Sprintf("%s", region),
 			"-m",
@@ -196,8 +196,8 @@ func (c TestConfig) check() error {
 	if (c.TotalRequests < 1 && c.ExecTimeout <= 0) || c.TotalRequests > 2000000 {
 		return errors.New("Invalid total requests (use 1 - 2000000)")
 	}
-	if c.ExecTimeout > 300 {
-		return errors.New("Invalid maximum execution time in seconds (use 0 - 300)")
+	if c.ExecTimeout > 3600 {
+		return errors.New("Invalid maximum execution time in seconds (use 0 - 3600)")
 	}
 	if c.RequestTimeout.Nanoseconds() < nano || c.RequestTimeout.Nanoseconds() > nano*100 {
 		return errors.New("Invalid timeout (1s - 100s)")

goad.go

@@ -202,13 +202,22 @@ func (infra *Infrastructure) createIAMLambdaRolePolicy(roleName string) error {
 		PolicyDocument: aws.String(`{
           "Version": "2012-10-17",
           "Statement": [
-            {
-              "Action": [
-                "sqs:SendMessage"
-              ],
-              "Effect": "Allow",
-              "Resource": "arn:aws:sqs:*:*:goad-*"
-		  	},
+					{
+				 "Action": [
+						 "sqs:SendMessage"
+				 ],
+				 "Effect": "Allow",
+				 "Resource": "arn:aws:sqs:*:*:goad-*"
+		 },
+		 {
+				 "Effect": "Allow",
+				 "Action": [
+						 "lambda:Invoke*"
+				 ],
+				 "Resource": [
+						 "arn:aws:lambda:*:*:goad:*"
+				 ]
+		 },
 			{
               "Action": [
                 "logs:CreateLogGroup",