adding build numbers listing

Author: krzysied

test-utils/utils/bucket.go

@@ -94,9 +94,28 @@ func (b *Bucket) ExpandPathURL(pathElements ...interface{}) *url.URL {
 // enclosed by the provided path. Note that there's currently no way to get a
 // path that ends in '/'.
 func (b *Bucket) ExpandListURL(pathElements ...interface{}) *url.URL {
+	return buildURL(map[string]string{
+		// GCS api doesn't like preceding '/', so remove it.
+		"prefix": strings.TrimPrefix(joinStringsAndInts(pathElements...), "/"),
+	})
+}
+
+// ExpandListDirURL produces the URL for a list API query which will list directories
+// enclosed by the provided path. Note that there's currently no way to get a
+// path that ends in '/'.
+func (b *Bucket) ExpandListDirURL(pathElements ...interface{}) *url.URL {
+	return buildURL(map[string]string{
+		// GCS api doesn't like preceding '/', so remove it.
+		"prefix":    strings.TrimPrefix(joinStringsAndInts(pathElements...)+"/", "/"),
+		"delimiter": "/",
+	})
+}
+
+func (b *Bucket) buildUrl(parameters map[string]string) *url.URL {
 	q := url.Values{}
-	// GCS api doesn't like preceding '/', so remove it.
-	q.Set("prefix", strings.TrimPrefix(joinStringsAndInts(pathElements...), "/"))
+	for key, value := range parameters {
+		q.Set(key, value)
+	}
 	return &url.URL{
 		Scheme:   b.scheme,
 		Host:     b.listHost,
@@ -109,31 +128,58 @@ func (b *Bucket) ExpandListURL(pathElements ...interface{}) *url.URL {
 // The returned file name included the complete path from bucket root
 func (b *Bucket) List(pathElements ...interface{}) ([]string, error) {
 	listURL := b.ExpandListURL(pathElements...)
-	res, err := getResponseWithRetry(listURL.String())
+	data, err := queryURL(listURL.String())
+	if err != nil {
+		return nil, err
+	}
+	var ret []string
+	if _, ok := data["items"]; !ok {
+		glog.Warningf("No matching files were found (from: %v)", listURL.String())
+		return ret, nil
+	}
+	for _, item := range data["items"].([]interface{}) {
+		ret = append(ret, (item.(map[string]interface{})["name"]).(string))
+	}
+	return ret, nil
+}
+
+// ListDirs returns a list of all directories inside the given path.
+// The returned direcotry name included the complete path from bucket root
+func (b *Bucket) ListDirs(pathElements ...interface{}) ([]string, error) {
+	listURL := b.ExpandListDirURL(pathElements...)
+	data, err := queryURL(listURL.String())
+	if err != nil {
+		return nil, err
+	}
+	var ret []string
+	if _, ok := data["prefixes"]; !ok {
+		glog.Warningf("No matching dirs were found (from: %v)", listURL.String())
+		return ret, nil
+	}
+	for _, item := range data["prefixes"].([]interface{}) {
+		ret = append(ret, item.(string))
+	}
+	return ret, nil
+}
+
+func queryURL(url string) (map[string]interface{}, error) {
+	res, err := getResponseWithRetry(url)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to GET %v: %v", listURL, err)
+		return nil, fmt.Errorf("Failed to GET %v: %v", url, err)
 	}
 	if res.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("Got a non-success response %v while listing %v", res.StatusCode, listURL.String())
+		return nil, fmt.Errorf("Got a non-success response %v while listing %v", res.StatusCode, url)
 	}
 	body, err := ioutil.ReadAll(res.Body)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to read the response for %v: %v", listURL.String(), err)
+		return nil, fmt.Errorf("Failed to read the response for %v: %v", url, err)
 	}
 	var data map[string]interface{}
 	err = json.Unmarshal(body, &data)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to unmarshal %v: %v", string(body), err)
 	}
-	var ret []string
-	if _, ok := data["items"]; !ok {
-		glog.Warningf("No matching files were found (from: %v)", listURL.String())
-		return ret, nil
-	}
-	for _, item := range data["items"].([]interface{}) {
-		ret = append(ret, (item.(map[string]interface{})["name"]).(string))
-	}
-	return ret, nil
+	return data, nil
 }
 
 func joinStringsAndInts(pathElements ...interface{}) string {

test-utils/utils/utils.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -199,15 +200,43 @@ func (u *Utils) GetLastestBuildNumberFromJenkinsGoogleBucket(job string) (int, e
 	return lastBuildNo, nil
 }
 
+// GetBuildNumbersFromJenkinsGoogleBucket returns list of available build numbers
+// of the given job from the Google project's GCS bucket.
+func (u *Utils) GetBuildNumbersFromJenkinsGoogleBucket(job string) ([]int, error) {
+	var results []string
+	var err error
+	if u.needsDeref(job) {
+		results, err = u.bucket.ListDirs(u.pullDirectory, lookUpDirectory, job)
+	} else {
+		results, err = u.bucket.ListDirs(u.directory, job)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	jobPrefix := fmt.Sprintf("%s/%s/", u.directory, job)
+	builds := make([]int, 0)
+	for _, r := range results {
+		build := strings.TrimPrefix(r, jobPrefix)
+		build = strings.TrimSuffix(build, "/")
+		buildNo, err := strconv.Atoi(build)
+		if err != nil {
+			fmt.Printf("jobPrefix: %s", jobPrefix)
+			return nil, fmt.Errorf("unknown build name convention: %s", build)
+		}
+		builds = append(builds, buildNo)
+	}
+	return builds, nil
+}
+
 // StartedFile is a type in which we store test starting information in GCS as started.json
 type StartedFile struct {
 	Version     string `json:"version"`
 	Timestamp   uint64 `json:"timestamp"`
 	JenkinsNode string `json:"jenkins-node"`
 }
 
-// CheckStartedStatus reads the started.json file for a given job and build number.
-// It returns true if the result stored there is success, and false otherwise.
+// CheckStartedStatus returns the started.json file for a given job and build number.
 func (u *Utils) CheckStartedStatus(job string, buildNumber int) (*StartedFile, error) {
 	response, err := u.GetFileFromJenkinsGoogleBucket(job, buildNumber, "started.json")
 	if err != nil {
@@ -273,7 +302,6 @@ func (u *Utils) ListFilesInBuild(job string, buildNumber int, prefix string) ([]
 		}
 		return u.bucket.List(dir, prefix)
 	}
-
 	return u.bucket.List(u.directory, job, buildNumber, prefix)
 }