check image names (sourcegraph#4014)

* check for images

* remove yaml for use across different file types

* remove commented code

* change import package name

* makes regex more precise, add docstring, exit on first failure

* make variable names more obvious, declare vars in block

* add tests, move tool

* test uses assert.Contains

* move bash script to buildkite dir

* Update tools/check-image-names/check-image-names.go

Co-authored-by: Robert Lin <[email protected]>

* remove comment

Co-authored-by: Robert Lin <[email protected]>

Author: davejrt

.buildkite/check-image-names.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+ROOT="$(pwd)"
+
+pushd tools/check-image-names
+
+echo "--- Check to see if all manifests contain valid image names"
+go run check-image-names.go "${ROOT}"/base

.buildkite/pipeline.yaml

@@ -11,6 +11,8 @@ steps:
     command: .buildkite/shfmt.sh
   - label: ":git: :sleuth_or_spy:"
     command: .buildkite/verify-release/verify-release.sh
+  - label: ":k8s: :sleuth_or_spy:" 
+    command: .buildkite/check-image-names.sh
 
   - wait
 

go.mod

@@ -3,13 +3,19 @@ module github.com/sourcegraph/deploy-sourcegraph
 go 1.16
 
 require (
+	cloud.google.com/go/logging v1.4.2 // indirect
 	github.com/docker/docker v1.13.1 // indirect
+	github.com/fatih/color v1.10.0 // indirect
+	github.com/pkg/errors v0.9.1
 	github.com/pulumi/pulumi v1.12.0
 	github.com/sethgrid/pester v1.1.0
-	github.com/sourcegraph/sourcegraph/enterprise/dev/ci/images v0.0.0-20210928153802-14b7dd055792
+	github.com/sourcegraph/sourcegraph/enterprise/dev/ci/images v0.0.0-20211005203732-b910fedad1f4
 	github.com/sourcegraph/update-docker-tags v0.8.0
-	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect
-	golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 // indirect
+	github.com/spf13/cobra v1.1.3 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gotest.tools/v3 v3.0.3
 )
 
 replace github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.4.3+incompatible

go.sum

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/sourcegraph/sourcegraph/enterprise/dev/ci/images"
+)
+
+var (
+	i       = regexp.MustCompile(`index\.docker\.io\/sourcegraph/(?P<image>[a-z0-9-_.]+):[a-z0-9-_]+@sha256:[[:alnum:]]+`)
+	matches []string
+	data    []byte
+)
+
+func main() {
+
+	path := os.Args[1]
+
+	fmt.Print(CheckImages(path))
+
+}
+
+func CheckImages(path string) error {
+
+	err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Println(err)
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		if strings.HasPrefix(path, ".git") {
+			return nil
+		}
+
+		data, err = ioutil.ReadFile(path)
+		if err != nil {
+			return errors.Wrap(err, "when reading file contents")
+		}
+
+		// matchedImages contains all lines matching our regex. FindAllSubmatch returns
+		// a slice containing the full string, and the capture group `image` for each image in a file.
+		// We then loop over each slice, pull out the capture group and append it to a list of images
+		// to compare with upstream.
+		matchedImages := i.FindAllSubmatch(data, -1)
+		for _, match := range matchedImages {
+			matchd := string(match[1])
+			matches = append(matches, matchd)
+		}
+
+		return nil
+	})
+	matches = Unique(matches)
+	for i, image := range matches {
+		if image != images.DeploySourcegraphDockerImages[i] {
+			return fmt.Errorf("image: %s is not in the upstream list", image)
+		}
+	}
+
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	return nil
+
+}
+
+func Unique(strSlice []string) []string {
+	keys := make(map[string]bool)
+	list := []string{}
+	for _, entry := range strSlice {
+		if _, found := keys[entry]; !found {
+			keys[entry] = true
+			list = append(list, entry)
+		}
+	}
+
+	sort.Strings(list)
+
+	return list
+}

tools/check-image-names/check-image-names.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCheckImages(t *testing.T) {
+
+	errorString := "image: foo is not in the upstream list"
+	err := CheckImages("tests")
+	assert.NotNil(t, err)
+	assert.Contains(t, err.Error(), errorString)
+
+}

tools/check-image-names/check-image-names_test.go

@@ -0,0 +1,2 @@
+foo/test:asdfasdf
+index.docker.io/sourcegraph/foo:2323423@sha256:asdfasdfasdfasdf