get rid of docker-compose shell out

cmd/service_deploy.go

@@ -12,7 +12,7 @@ import (
 type ServiceDeployOperation struct {
 	ServiceName    string
 	Image          string
-	ComposeFile    string
+	ComposeFiles   []string
 	Region         string
 	Revision       string
 	WaitForService bool
@@ -21,7 +21,7 @@ type ServiceDeployOperation struct {
 const deployDockerComposeLabel = "aws.ecs.fargate.deploy"
 
 var flagServiceDeployImage string
-var flagServiceDeployDockerComposeFile string
+var flagServiceDeployDockerComposeFile []string
 var flagServiceDeployDockerComposeImageOnly bool
 var flagServiceDeployRevision string
 var flagServiceDeployWaitForService bool
@@ -53,7 +53,7 @@ fargate service deploy -r 37
 			ServiceName:    getServiceName(),
 			Region:         region,
 			Image:          flagServiceDeployImage,
-			ComposeFile:    flagServiceDeployDockerComposeFile,
+			ComposeFiles:   flagServiceDeployDockerComposeFile,
 			Revision:       flagServiceDeployRevision,
 			WaitForService: flagServiceDeployWaitForService,
 		}
@@ -63,6 +63,7 @@ fargate service deploy -r 37
 			return
 		}
 
+		console.Info("here")
 		deployService(operation)
 	},
 }
@@ -72,7 +73,7 @@ func init() {
 
 	serviceDeployCmd.Flags().StringVarP(&flagServiceDeployRevision, "revision", "r", "", "Task definition revision number")
 
-	serviceDeployCmd.Flags().StringVarP(&flagServiceDeployDockerComposeFile, "file", "f", "", "Specify a docker-compose.yml file to deploy. The image and environment variables in the file will be deployed.")
+	serviceDeployCmd.Flags().StringArrayVarP(&flagServiceDeployDockerComposeFile, "file", "f", []string{}, "Specify a docker-compose.yml file to deploy. The image and environment variables in the file will be deployed.")
 
 	serviceDeployCmd.Flags().BoolVar(&flagServiceDeployDockerComposeImageOnly, "image-only", false, "Only deploy the image when a docker-compose.yml file is specified.")
 
@@ -82,9 +83,10 @@ func init() {
 }
 
 func deployService(operation *ServiceDeployOperation) {
+	console.Info("Hello?!")
 	var taskDefinitionArn string
 
-	if operation.ComposeFile != "" {
+	if len(operation.ComposeFiles) > 0 {
 		taskDefinitionArn = deployDockerComposeFile(operation)
 	} else if operation.Revision != "" {
 		taskDefinitionArn = deployRevision(operation)
@@ -112,11 +114,10 @@ func deployService(operation *ServiceDeployOperation) {
 func deployDockerComposeFile(operation *ServiceDeployOperation) string {
 	var taskDefinitionArn string
 
+	dockerService := getDockerServiceFromComposeFile(operation.ComposeFiles)
 	ecs := ECS.New(sess, getClusterName())
 	ecsService := ecs.DescribeService(operation.ServiceName)
 
-	dockerService := getDockerServiceFromComposeFile(operation.ComposeFile)
-
 	envvars := convertDockerComposeEnvVarsToECSEnvVars(dockerService)
 	secrets := convertDockerComposeSecretsToECSSecrets(dockerService)
 
@@ -135,7 +136,7 @@ func deployDockerComposeFile(operation *ServiceDeployOperation) string {
 	if flagServiceDeployDockerComposeImageOnly {
 		console.Info("Deployed %s to service %s", dockerService.Image, operation.ServiceName)
 	} else {
-		console.Info("Deployed %s to service %s as revision %s", operation.ComposeFile, operation.ServiceName, ecs.GetRevisionNumber(taskDefinitionArn))
+		console.Info("Deployed %s to service %s as revision %s", operation.ComposeFiles, operation.ServiceName, ecs.GetRevisionNumber(taskDefinitionArn))
 	}
 
 	return taskDefinitionArn
@@ -177,9 +178,9 @@ func deployImage(operation *ServiceDeployOperation) string {
 	return taskDefinitionArn
 }
 
-func getDockerServiceFromComposeFile(dockerComposeFile string) *dockercompose.Service {
+func getDockerServiceFromComposeFile(dockerComposeFiles []string) *dockercompose.Service {
 	//read the compose file configuration
-	composeFile := dockercompose.Read(dockerComposeFile)
+	composeFile := dockercompose.Read(dockerComposeFiles)
 
 	//determine which docker-compose service/container to deploy
 	_, dockerService := getDockerServiceToDeploy(&composeFile.Data)
@@ -234,7 +235,11 @@ func getDockerServiceToDeploy(dc *dockercompose.DockerCompose) (string, *dockerc
 
 //Check incompatible flag combinations
 func validateFlags(operation *ServiceDeployOperation) bool {
-	strFlags := []string{operation.Image, operation.ComposeFile, operation.Revision}
+	var cf string
+	if len(operation.ComposeFiles) > 0 {
+		cf = operation.ComposeFiles[0]
+	}
+	strFlags := []string{operation.Image, cf, operation.Revision}
 	setFlags := make([]string, 0)
 
 	for _, v := range strFlags {

cmd/task_register.go

@@ -8,7 +8,7 @@ import (
 )
 
 var flagTaskRegisterImage string
-var flagTaskRegisterDockerComposeFile string
+var flagTaskRegisterDockerComposeFile []string
 var flagTaskRegisterEnvVars []string
 var flagTaskRegisterEnvFile string
 var flagTaskRegisterSecretVars []string
@@ -21,7 +21,7 @@ type taskRegisterOperation struct {
 	Image       string
 	EnvVars     []string
 	EnvFile     string
-	ComposeFile string
+	ComposeFile []string
 	SecretVars  []string
 	SecretFile  string
 }
@@ -49,8 +49,8 @@ var taskRegisterCmd = &cobra.Command{
 			len(flagTaskRegisterSecretVars) > 0 ||
 			flagTaskRegisterSecretFile != "")
 
-		if (flagTaskRegisterDockerComposeFile != "" && nonComposeOptions) ||
-			(flagTaskRegisterDockerComposeFile == "" && !nonComposeOptions) {
+		if (len(flagTaskRegisterDockerComposeFile) > 0 && nonComposeOptions) ||
+			(len(flagTaskRegisterDockerComposeFile) == 0 && !nonComposeOptions) {
 			cmd.Help()
 			return
 		}
@@ -74,7 +74,7 @@ func init() {
 
 	taskRegisterCmd.Flags().StringVar(&flagTaskRegisterEnvFile, "env-file", "", "File containing list of environment variables to set, one per line, of the form KEY=value")
 
-	taskRegisterCmd.Flags().StringVarP(&flagTaskRegisterDockerComposeFile, "file", "f", "", "Docker Compose file containing image and environment variables to register.")
+	taskRegisterCmd.Flags().StringArrayVarP(&flagTaskRegisterDockerComposeFile, "file", "f", []string{}, "Docker Compose file containing image and environment variables to register.")
 
 	taskRegisterCmd.Flags().StringArrayVar(&flagTaskRegisterSecretVars, "secret", []string{}, "Secret variables to set [e.g. --secret KEY=valueFrom --secret KEY2=valueFrom]")
 
@@ -91,7 +91,7 @@ func registerTask(op taskRegisterOperation) {
 	var secrets []ECS.Secret
 	replaceVars := false
 
-	if op.ComposeFile != "" {
+	if len(op.ComposeFile) > 0 {
 		dockerService := getDockerServiceFromComposeFile(op.ComposeFile)
 		image = dockerService.Image
 		envvars = convertDockerComposeEnvVarsToECSEnvVars(dockerService)

dockercompose/main.go

@@ -1,23 +1,23 @@
 package dockercompose
 
 import (
-	"bytes"
 	"io/ioutil"
-	"os/exec"
 
+	"github.com/compose-spec/compose-go/loader"
+	compose "github.com/compose-spec/compose-go/types"
 	"github.com/turnerlabs/fargate/console"
 	yaml "gopkg.in/yaml.v2"
 )
 
 //ComposeFile represents a docker-compose.yml file
 //that can be manipulated
 type ComposeFile struct {
-	File string
+	File []string
 	Data DockerCompose
 }
 
 //Read loads a docker-compose.yml file
-func Read(file string) ComposeFile {
+func Read(file []string) ComposeFile {
 	result := ComposeFile{
 		File: file,
 	}
@@ -28,7 +28,7 @@ func Read(file string) ComposeFile {
 //New returns an initialized compose file
 func New(file string) ComposeFile {
 	result := ComposeFile{
-		File: file,
+		File: []string{file},
 		Data: DockerCompose{
 			Version:  "3.7",
 			Services: make(map[string]*Service),
@@ -41,30 +41,62 @@ func New(file string) ComposeFile {
 //note that all variable interpolations are fully rendered
 func (composeFile *ComposeFile) Read() {
 	console.Debug("running docker-compose config [%s]", composeFile.File)
-	cmd := exec.Command("docker-compose", "-f", composeFile.File, "config")
-
-	var outbuf, errbuf bytes.Buffer
-	cmd.Stdout = &outbuf
-	cmd.Stderr = &errbuf
 
-	if err := cmd.Start(); err != nil {
-		console.ErrorExit(err, errbuf.String())
-	}
-
-	if err := cmd.Wait(); err != nil {
-		console.IssueExit(errbuf.String())
+	//Load Docker Compose yaml
+	dcy, err := loadCompose(composeFile.File)
+	if err != nil {
+		console.ErrorExit(err, "error loading docker-compose yaml files")
 	}
 
+	console.Info(string(dcy))
 	//unmarshal the yaml
 	var compose DockerCompose
-	err := yaml.Unmarshal(outbuf.Bytes(), &compose)
+	err = yaml.Unmarshal(dcy, &compose)
 	if err != nil {
 		console.ErrorExit(err, "error unmarshalling docker-compose.yml")
 	}
 
 	composeFile.Data = compose
 }
 
+// Load and merge the docker-compose yaml files into one
+func loadCompose(files []string) ([]byte, error) {
+	var composeConfigFiles []compose.ConfigFile
+	for _, f := range files {
+		file, err := ioutil.ReadFile(f)
+		if err != nil {
+			return nil, err
+		}
+		composeConfig, err := loader.ParseYAML(file)
+		if err != nil {
+			return nil, err
+		}
+		composeConfigFiles = append(composeConfigFiles, compose.ConfigFile{Filename: f, Config: composeConfig})
+
+	}
+	dc, err := loader.Load(compose.ConfigDetails{
+		ConfigFiles: composeConfigFiles,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+	y, err := yaml.Marshal(dc)
+
+	var remap map[string]interface{}
+	err = yaml.Unmarshal(y, &remap)
+
+	remap["version"] = composeConfigFiles[0].Config["version"]
+
+	y, err = yaml.Marshal(remap)
+	if err != nil {
+		return nil, err
+	}
+
+	return y, nil
+
+}
+
 //AddService adds a service to a compose file
 func (composeFile *ComposeFile) AddService(name string) *Service {
 	result := &Service{}
@@ -87,7 +119,7 @@ func (composeFile *ComposeFile) Write() error {
 	if err != nil {
 		return err
 	}
-	err = ioutil.WriteFile(composeFile.File, bits, 0644)
+	err = ioutil.WriteFile(composeFile.File[0], bits, 0644)
 	if err != nil {
 		return err
 	}

dockercompose/main_test.go

@@ -6,7 +6,7 @@ import (
 
 func doTest(t *testing.T, f string) ComposeFile {
 	//round-trip unmarshal and marshal
-	file := Read(f)
+	file := Read([]string{f})
 	b, e := file.Yaml()
 	if e != nil {
 		t.Error(e)
@@ -25,45 +25,6 @@ const (
 	secretValue   = "arn:key:ssm:us-east-1:000000000000:parameter/path/to/my_parameter"
 )
 
-// 2
-func TestComposeV2(t *testing.T) {
-	f := doTest(t, "v2.yml")
-	svc := f.Data.Services["web"]
-	if svc.Image != image {
-		t.Error("expecting image")
-	}
-	if svc.Ports[0].Published != publishedPort {
-		t.Error("expecting published port")
-	}
-	if svc.Ports[0].Target != targetPort {
-		t.Error("expecting published port")
-	}
-	if svc.Labels[labelKey] != labelValue {
-		t.Error("expecting label")
-	}
-}
-
-// 2.4
-func TestComposeV24(t *testing.T) {
-	f := doTest(t, "v2.4.yml")
-	svc := f.Data.Services["web"]
-	if svc.Image != image {
-		t.Error("expecting image")
-	}
-	if svc.Ports[0].Published != publishedPort {
-		t.Error("expecting published port")
-	}
-	if svc.Ports[0].Target != targetPort {
-		t.Error("expecting published port")
-	}
-	if svc.Labels[labelKey] != labelValue {
-		t.Error("expecting label")
-	}
-	if svc.Secrets[secretKey] != secretValue {
-		t.Error("expecting secret")
-	}
-}
-
 // 3.2 short
 func TestComposeV32Short(t *testing.T) {
 	f := doTest(t, "v3.2-short.yml")