Implement repo priorities and integrate them into package version com…

…parisons.

client/client.go

@@ -92,19 +92,28 @@ func (ps *PackageState) Match(pi goolib.PackageInfo) bool {
 	return ps.PackageSpec.Name == pi.Name && (ps.PackageSpec.Arch == pi.Arch || pi.Arch == "") && (ps.PackageSpec.Version == pi.Ver || pi.Ver == "")
 }
 
+// Repo represents a single downloaded repo.
+type Repo struct {
+	Priority int
+	Packages []goolib.RepoSpec
+}
+
 // RepoMap describes each repo's packages as seen from a client.
-type RepoMap map[string][]goolib.RepoSpec
+type RepoMap map[string]Repo
 
 // AvailableVersions builds a RepoMap from a list of sources.
-func AvailableVersions(ctx context.Context, srcs []string, cacheDir string, cacheLife time.Duration, proxyServer string) RepoMap {
+func AvailableVersions(ctx context.Context, srcs map[string]int, cacheDir string, cacheLife time.Duration, proxyServer string) RepoMap {
 	rm := make(RepoMap)
-	for _, r := range srcs {
+	for r, pri := range srcs {
 		rf, err := unmarshalRepoPackages(ctx, r, cacheDir, cacheLife, proxyServer)
 		if err != nil {
 			logger.Errorf("error reading repo %q: %v", r, err)
 			continue
 		}
-		rm[r] = rf
+		rm[r] = Repo{
+			Priority: pri,
+			Packages: rf,
+		}
 	}
 	return rm
 }
@@ -157,7 +166,7 @@ func decode(index io.ReadCloser, ct, url, cf string) ([]goolib.RepoSpec, error)
 
 // unmarshalRepoPackages gets and unmarshals a repository URL or uses the cached contents
 // if mtime is less than cacheLife.
-// Sucessfully unmarshalled contents will be written to a cache.
+// Successfully unmarshalled contents will be written to a cache.
 func unmarshalRepoPackages(ctx context.Context, p, cacheDir string, cacheLife time.Duration, proxyServer string) ([]goolib.RepoSpec, error) {
 	pName := strings.TrimPrefix(p, "oauth-")
 
@@ -306,8 +315,8 @@ func unmarshalRepoPackagesGCS(ctx context.Context, bucket, object, url, cf strin
 }
 
 // FindRepoSpec returns the element of pl whose PackageSpec matches pi.
-func FindRepoSpec(pi goolib.PackageInfo, pl []goolib.RepoSpec) (goolib.RepoSpec, error) {
-	for _, p := range pl {
+func FindRepoSpec(pi goolib.PackageInfo, repo Repo) (goolib.RepoSpec, error) {
+	for _, p := range repo.Packages {
 		ps := p.PackageSpec
 		if ps.Name == pi.Name && ps.Arch == pi.Arch && ps.Version == pi.Ver {
 			return p, nil
@@ -316,66 +325,64 @@ func FindRepoSpec(pi goolib.PackageInfo, pl []goolib.RepoSpec) (goolib.RepoSpec,
 	return goolib.RepoSpec{}, fmt.Errorf("no match found for package %s.%s.%s in repo", pi.Name, pi.Arch, pi.Ver)
 }
 
-func latest(psm map[string][]*goolib.PkgSpec) (ver, repo string) {
+func latest(psm map[string][]*goolib.PkgSpec, rm RepoMap) (string, string) {
+	var ver, repo string
+	var pri int
 	for r, pl := range psm {
-		for _, p := range pl {
-			if ver == "" {
+		for _, pkg := range pl {
+			q := rm[r].Priority
+			if ver == "" || q > pri {
 				repo = r
-				ver = p.Version
+				ver = pkg.Version
+				pri = q
 				continue
 			}
-			c, err := goolib.Compare(p.Version, ver)
+			if q < pri {
+				continue
+			}
+			c, err := goolib.Compare(pkg.Version, ver)
 			if err != nil {
-				logger.Errorf("compare of %s to %s failed with error: %v", p.Version, ver, err)
+				logger.Errorf("compare of %s to %s failed with error: %v", pkg.Version, ver, err)
 			}
 			if c == 1 {
 				repo = r
-				ver = p.Version
+				ver = pkg.Version
+				pri = q
 			}
 		}
 	}
-	return
+	return ver, repo
 }
 
 // FindRepoLatest returns the latest version of a package along with its repo and arch.
-func FindRepoLatest(pi goolib.PackageInfo, rm RepoMap, archs []string) (ver, repo, arch string, err error) {
+func FindRepoLatest(pi goolib.PackageInfo, rm RepoMap, archs []string) (string, string, string, error) {
 	psm := make(map[string][]*goolib.PkgSpec)
+	name := pi.Name
 	if pi.Arch != "" {
-		for r, pl := range rm {
-			for _, p := range pl {
-				if p.PackageSpec.Name == pi.Name && p.PackageSpec.Arch == pi.Arch {
-					psm[r] = append(psm[r], p.PackageSpec)
-				}
-			}
-		}
-		if len(psm) != 0 {
-			v, r := latest(psm)
-			return v, r, pi.Arch, nil
-		}
-		return "", "", "", fmt.Errorf("no versions of package %s.%s found in any repo", pi.Name, pi.Arch)
+		archs = []string{pi.Arch}
+		name = fmt.Sprintf("%s.%s", pi.Name, pi.Arch)
 	}
-
 	for _, a := range archs {
-		for r, pl := range rm {
-			for _, p := range pl {
+		for r, repo := range rm {
+			for _, p := range repo.Packages {
 				if p.PackageSpec.Name == pi.Name && p.PackageSpec.Arch == a {
 					psm[r] = append(psm[r], p.PackageSpec)
 				}
 			}
 		}
 		if len(psm) != 0 {
-			v, r := latest(psm)
+			v, r := latest(psm, rm)
 			return v, r, a, nil
 		}
 	}
-	return "", "", "", fmt.Errorf("no versions of package %s found in any repo", pi.Name)
+	return "", "", "", fmt.Errorf("no versions of package %s found in any repo", name)
 }
 
 // WhatRepo returns what repo a package is in.
 // Name, Arch, and Ver fields of PackageInfo must be provided.
 func WhatRepo(pi goolib.PackageInfo, rm RepoMap) (string, error) {
-	for r, pl := range rm {
-		for _, p := range pl {
+	for r, repo := range rm {
+		for _, p := range repo.Packages {
 			if p.PackageSpec.Name == pi.Name && p.PackageSpec.Arch == pi.Arch && p.PackageSpec.Version == pi.Ver {
 				return r, nil
 			}

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/dustin/go-humanize v1.0.0
 	github.com/go-yaml/yaml v2.1.0+incompatible
+	github.com/google/go-cmp v0.6.0
 	github.com/google/logger v1.1.1
 	github.com/google/subcommands v1.2.0
 	github.com/olekukonko/tablewriter v0.0.5

go.sum

@@ -114,8 +114,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/logger v1.1.1 h1:+6Z2geNxc9G+4D4oDO9njjjn2d0wN5d7uOo0vOIW1NQ=
 github.com/google/logger v1.1.1/go.mod h1:BkeJZ+1FhQ+/d087r4dzojEg1u2ZX+ZqG1jTUrLM+zQ=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
@@ -391,7 +392,6 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=

googet.go

@@ -25,6 +25,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -38,13 +39,14 @@ import (
 )
 
 const (
-	stateFile = "googet.state"
-	confFile  = "googet.conf"
-	logFile   = "googet.log"
-	cacheDir  = "cache"
-	repoDir   = "repos"
-	envVar    = "GooGetRoot"
-	logSize   = 10 * 1024 * 1024
+	stateFile       = "googet.state"
+	confFile        = "googet.conf"
+	logFile         = "googet.log"
+	cacheDir        = "cache"
+	repoDir         = "repos"
+	envVar          = "GooGetRoot"
+	logSize         = 10 * 1024 * 1024
+	defaultPriority = 500
 )
 
 var (
@@ -82,6 +84,7 @@ type repoEntry struct {
 	Name     string
 	URL      string
 	UseOAuth bool
+	Priority int
 }
 
 // UnmarshalYAML provides custom unmarshalling for repoEntry objects.
@@ -98,6 +101,12 @@ func (r *repoEntry) UnmarshalYAML(unmarshal func(interface{}) error) error {
 			r.URL = v
 		case "useoauth":
 			r.UseOAuth = strings.ToLower(v) == "true"
+		case "priority":
+			var err error
+			r.Priority, err = strconv.Atoi(v)
+			if err != nil {
+				return fmt.Errorf("invalid priority: %v", v)
+			}
 		}
 	}
 	if r.URL == "" {
@@ -163,44 +172,54 @@ func unmarshalConfFile(p string) (*conf, error) {
 	return &cf, yaml.Unmarshal(b, &cf)
 }
 
-func repoList(dir string) ([]string, error) {
+// validateRepoURL uses the global allowUnsafeURL to determine if u should be checked for https or
+// GCS status.
+func validateRepoURL(u string) bool {
+	if allowUnsafeURL {
+		return true
+	}
+	gcs, _, _ := goolib.SplitGCSUrl(u)
+	parsed, err := url.Parse(u)
+	if err != nil {
+		logger.Errorf("Failed to parse URL '%s', skipping repo", u)
+		return false
+	}
+	if parsed.Scheme != "https" && !gcs {
+		logger.Errorf("%s will not be used as a repository, only https and Google Cloud Storage endpoints will be used unless 'allowunsafeurl' is set to 'true' in googet.conf", u)
+		return false
+	}
+	return true
+}
+
+// repoList returns a deduped set of all repos listed in the repo config files contained in dir.
+// The repos are mapped to priority values. If a repo config does not specify a priority, the repo
+// is assigned the default priority value. If the same repo appears multiple times with different
+// priority values, it is mapped to the highest seen priority value.
+func repoList(dir string) (map[string]int, error) {
 	rfs, err := repos(dir)
 	if err != nil {
 		return nil, err
 	}
-	var rl []string
+	result := make(map[string]int)
 	for _, rf := range rfs {
 		for _, re := range rf.repoEntries {
-			switch {
-			case re.URL != "":
-				if re.UseOAuth {
-					rl = append(rl, "oauth-"+re.URL)
-				} else {
-					rl = append(rl, re.URL)
-				}
-			}
-		}
-	}
-
-	if !allowUnsafeURL {
-		var srl []string
-		for _, r := range rl {
-			rTrimmed := strings.TrimPrefix(r, "oauth-")
-			isGCSURL, _, _ := goolib.SplitGCSUrl(rTrimmed)
-			parsed, err := url.Parse(rTrimmed)
-			if err != nil {
-				logger.Errorf("Failed to parse URL '%s', skipping repo", r)
+			u := re.URL
+			if u == "" || !validateRepoURL(u) {
 				continue
 			}
-			if parsed.Scheme != "https" && !isGCSURL {
-				logger.Errorf("%s will not be used as a repository, only https and Google Cloud Storage endpoints will be used unless 'allowunsafeurl' is set to 'true' in googet.conf", r)
-				continue
+			if re.UseOAuth {
+				u = "oauth-" + u
+			}
+			p := re.Priority
+			if p <= 0 {
+				p = defaultPriority
+			}
+			if q, ok := result[u]; !ok || p > q {
+				result[u] = p
 			}
-			srl = append(srl, r)
 		}
-		return srl, nil
 	}
-	return rl, nil
+	return result, nil
 }
 
 func repos(dir string) ([]repoFile, error) {
@@ -273,12 +292,15 @@ func readStateFromPath(sf string) (*client.GooGetState, error) {
 	return client.UnmarshalState(b)
 }
 
-func buildSources(s string) ([]string, error) {
-	if s != "" {
-		srcs := strings.Split(s, ",")
-		return srcs, nil
+func buildSources(s string) (map[string]int, error) {
+	if s == "" {
+		return repoList(filepath.Join(rootDir, repoDir))
+	}
+	m := make(map[string]int)
+	for _, src := range strings.Split(s, ",") {
+		m[src] = defaultPriority
 	}
-	return repoList(filepath.Join(rootDir, repoDir))
+	return m, nil
 }
 
 func confirmation(msg string) bool {

googet_available.go

@@ -75,8 +75,8 @@ func (cmd *availableCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...inte
 
 	m := make(map[string][]string)
 	rm := client.AvailableVersions(ctx, repos, filepath.Join(rootDir, cacheDir), cacheLife, proxyServer)
-	for r, pl := range rm {
-		for _, p := range pl {
+	for r, repo := range rm {
+		for _, p := range repo.Packages {
 			m[r] = append(m[r], p.PackageSpec.Name+"."+p.PackageSpec.Arch+"."+p.PackageSpec.Version)
 		}
 	}
@@ -111,8 +111,8 @@ func (cmd *availableCmd) Execute(ctx context.Context, f *flag.FlagSet, _ ...inte
 }
 
 func repo(pi goolib.PackageInfo, rm client.RepoMap) {
-	for r, pl := range rm {
-		for _, p := range pl {
+	for r, repo := range rm {
+		for _, p := range repo.Packages {
 			if p.PackageSpec.Name == pi.Name && p.PackageSpec.Arch == pi.Arch && p.PackageSpec.Version == pi.Ver {
 				info(p.PackageSpec, r)
 				return