Skip to content

Commit 0fdcf0d

Browse files
ttaylorrttaylorr
authored
Merge pull request git-lfs#3116 from git-lfs/ttaylorr/multiple-alternates
fs: support multiple object alternates
2 parents d2341b5 + da86172 commit 0fdcf0d

File tree

6 files changed

+184
-41
lines changed

6 files changed

+184
-41
lines changed

‎config/config.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -311,8 +311,8 @@ func (c *Configuration) LocalGitStorageDir() string {
311311
return c.Filesystem().GitStorageDir
312312
}
313313

314-
func (c *Configuration) LocalReferenceDir() string {
315-
return c.Filesystem().ReferenceDir
314+
func (c *Configuration) LocalReferenceDirs() []string {
315+
return c.Filesystem().ReferenceDirs
316316
}
317317

318318
func (c *Configuration) LFSStorageDir() string {

‎fs/fs.go

Lines changed: 58 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package fs
22

33
import (
4+
"bufio"
45
"bytes"
56
"fmt"
67
"io/ioutil"
@@ -12,6 +13,7 @@ import (
1213
"sync"
1314

1415
"github.com/git-lfs/git-lfs/tools"
16+
"github.com/rubyist/tracerx"
1517
)
1618

1719
var oidRE = regexp.MustCompile(`\A[[:alnum:]]{64}`)
@@ -23,9 +25,9 @@ type Object struct {
2325
}
2426

2527
type Filesystem struct {
26-
GitStorageDir string // parent of objects/lfs (may be same as GitDir but may not)
27-
LFSStorageDir string // parent of lfs objects and tmp dirs. Default: ".git/lfs"
28-
ReferenceDir string // alternative local media dir (relative to clone reference repo)
28+
GitStorageDir string // parent of objects/lfs (may be same as GitDir but may not)
29+
LFSStorageDir string // parent of lfs objects and tmp dirs. Default: ".git/lfs"
30+
ReferenceDirs []string // alternative local media dirs (relative to clone reference repo)
2931
lfsobjdir string
3032
tmpdir string
3133
logdir string
@@ -105,12 +107,16 @@ func (f *Filesystem) localObjectDir(oid string) string {
105107
return filepath.Join(f.LFSObjectDir(), oid[0:2], oid[2:4])
106108
}
107109

108-
func (f *Filesystem) ObjectReferencePath(oid string) string {
109-
if len(f.ReferenceDir) == 0 {
110-
return f.ReferenceDir
110+
func (f *Filesystem) ObjectReferencePaths(oid string) []string {
111+
if len(f.ReferenceDirs) == 0 {
112+
return nil
111113
}
112114

113-
return filepath.Join(f.ReferenceDir, oid[0:2], oid[2:4], oid)
115+
var paths []string
116+
for _, ref := range f.ReferenceDirs {
117+
paths = append(paths, filepath.Join(ref, oid[0:2], oid[2:4], oid))
118+
}
119+
return paths
114120
}
115121

116122
func (f *Filesystem) LFSObjectDir() string {
@@ -164,7 +170,7 @@ func New(gitdir, workdir, lfsdir string) *Filesystem {
164170
GitStorageDir: resolveGitStorageDir(gitdir),
165171
}
166172

167-
fs.ReferenceDir = resolveReferenceDir(fs.GitStorageDir)
173+
fs.ReferenceDirs = resolveReferenceDirs(fs.GitStorageDir)
168174

169175
if len(lfsdir) == 0 {
170176
lfsdir = "lfs"
@@ -179,19 +185,55 @@ func New(gitdir, workdir, lfsdir string) *Filesystem {
179185
return fs
180186
}
181187

182-
func resolveReferenceDir(gitStorageDir string) string {
188+
func resolveReferenceDirs(gitStorageDir string) []string {
189+
var references []string
190+
183191
cloneReferencePath := filepath.Join(gitStorageDir, "objects", "info", "alternates")
184192
if tools.FileExists(cloneReferencePath) {
185-
buffer, err := ioutil.ReadFile(cloneReferencePath)
186-
if err == nil {
187-
path := strings.TrimSpace(string(buffer[:]))
188-
referenceLfsStoragePath := filepath.Join(filepath.Dir(path), "lfs", "objects")
189-
if tools.DirExists(referenceLfsStoragePath) {
190-
return referenceLfsStoragePath
193+
f, err := os.Open(cloneReferencePath)
194+
if err != nil {
195+
tracerx.Printf("could not open %s: %s",
196+
cloneReferencePath, err)
197+
return nil
198+
}
199+
defer f.Close()
200+
201+
scanner := bufio.NewScanner(f)
202+
L:
203+
for scanner.Scan() {
204+
text := strings.TrimSpace(scanner.Text())
205+
if len(text) == 0 {
206+
continue
191207
}
208+
209+
var objs string
210+
switch text[0] {
211+
case '#':
212+
continue L
213+
case '"':
214+
unquote := strings.LastIndex(text, "\"")
215+
if unquote == 0 {
216+
continue L
217+
}
218+
219+
objs, _ = strconv.Unquote(text[:unquote+1])
220+
default:
221+
objs = text
222+
}
223+
storage := filepath.Join(filepath.Dir(objs),
224+
"lfs", "objects")
225+
226+
if tools.DirExists(storage) {
227+
references = append(references, storage)
228+
}
229+
}
230+
231+
if err := scanner.Err(); err != nil {
232+
tracerx.Printf("could not scan %s: %s",
233+
cloneReferencePath, err)
192234
}
193235
}
194-
return ""
236+
return references
195237
}
196238

197239
// From a git dir, get the location that objects are to be stored (we will store lfs alongside)

‎lfs/lfs.go

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,14 @@ func Environ(cfg *config.Configuration, manifest *tq.Manifest) []string {
3535

3636
fetchPruneConfig := NewFetchPruneConfig(cfg.Git)
3737

38+
references := strings.Join(cfg.LocalReferenceDirs(), ", ")
39+
3840
env = append(env,
3941
fmt.Sprintf("LocalWorkingDir=%s", cfg.LocalWorkingDir()),
4042
fmt.Sprintf("LocalGitDir=%s", cfg.LocalGitDir()),
4143
fmt.Sprintf("LocalGitStorageDir=%s", cfg.LocalGitStorageDir()),
4244
fmt.Sprintf("LocalMediaDir=%s", cfg.LFSObjectDir()),
43-
fmt.Sprintf("LocalReferenceDir=%s", cfg.LocalReferenceDir()),
45+
fmt.Sprintf("LocalReferenceDirs=%s", references),
4446
fmt.Sprintf("TempDir=%s", cfg.TempDir()),
4547
fmt.Sprintf("ConcurrentTransfers=%d", api.ConcurrentTransfers),
4648
fmt.Sprintf("TusTransfers=%v", cfg.TusTransfersAllowed()),
@@ -98,13 +100,19 @@ func LinkOrCopyFromReference(cfg *config.Configuration, oid string, size int64)
98100
if cfg.LFSObjectExists(oid, size) {
99101
return nil
100102
}
101-
altMediafile := cfg.Filesystem().ObjectReferencePath(oid)
103+
altMediafiles := cfg.Filesystem().ObjectReferencePaths(oid)
102104
mediafile, err := cfg.Filesystem().ObjectPath(oid)
103105
if err != nil {
104106
return err
105107
}
106-
if altMediafile != "" && tools.FileExistsOfSize(altMediafile, size) {
107-
return LinkOrCopy(cfg, altMediafile, mediafile)
108+
for _, altMediafile := range altMediafiles {
109+
tracerx.Printf("altMediafile: %s", altMediafile)
110+
if altMediafile != "" && tools.FileExistsOfSize(altMediafile, size) {
111+
err = LinkOrCopy(cfg, altMediafile, mediafile)
112+
if err == nil {
113+
break
114+
}
115+
}
108116
}
109-
return nil
117+
return err
110118
}

‎test/test-alternates.sh

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/usr/bin/env bash
2+
3+
. "test/testlib.sh"
4+
5+
begin_test "alternates (single)"
6+
(
7+
set -e
8+
9+
reponame="alternates-single-alternate"
10+
setup_remote_repo_with_file "$reponame" "a.txt"
11+
12+
pushd "$TRASHDIR" > /dev/null
13+
clone_repo "$reponame" "${reponame}_alternate"
14+
popd > /dev/null
15+
16+
rm -rf .git/lfs/objects
17+
18+
alternate="$TRASHDIR/${reponame}_alternate/.git/objects"
19+
echo "$alternate" > .git/objects/info/alternates
20+
21+
GIT_TRACE=1 git lfs fetch origin master 2>&1 | tee fetch.log
22+
[ "0" -eq "$(grep -c "sending batch of size 1" fetch.log)" ]
23+
)
24+
end_test
25+
26+
begin_test "alternates (multiple)"
27+
(
28+
set -e
29+
30+
reponame="alternates-multiple-alternates"
31+
setup_remote_repo_with_file "$reponame" "a.txt"
32+
33+
pushd "$TRASHDIR" > /dev/null
34+
clone_repo "$reponame" "${reponame}_alternate_stale"
35+
rm -rf .git/lfs/objects
36+
popd > /dev/null
37+
pushd "$TRASHDIR" > /dev/null
38+
clone_repo "$reponame" "${reponame}_alternate"
39+
popd > /dev/null
40+
41+
rm -rf .git/lfs/objects
42+
43+
alternate_stale="$TRASHDIR/${reponame}_alternate_stale/.git/objects"
44+
alternate="$TRASHDIR/${reponame}_alternate/.git/objects"
45+
echo "$alternate" > .git/objects/info/alternates
46+
echo "$alternate_stale" >> .git/objects/info/alternates
47+
48+
GIT_TRACE=1 git lfs fetch origin master 2>&1 | tee fetch.log
49+
[ "0" -eq "$(grep -c "sending batch of size 1" fetch.log)" ]
50+
)
51+
end_test
52+
53+
begin_test "alternates (commented)"
54+
(
55+
set -e
56+
57+
reponame="alternates-commented-alternate"
58+
setup_remote_repo_with_file "$reponame" "a.txt"
59+
60+
pushd "$TRASHDIR" > /dev/null
61+
clone_repo "$reponame" "${reponame}_alternate"
62+
popd > /dev/null
63+
64+
rm -rf .git/lfs/objects
65+
66+
alternate="$TRASHDIR/${reponame}_alternate/.git/objects"
67+
echo "# $alternate" > .git/objects/info/alternates
68+
69+
GIT_TRACE=1 git lfs fetch origin master 2>&1 | tee fetch.log
70+
[ "1" -eq "$(grep -c "sending batch of size 1" fetch.log)" ]
71+
)
72+
end_test
73+
74+
begin_test "alternates (quoted)"
75+
(
76+
set -e
77+
78+
reponame="alternates-quoted-alternate"
79+
setup_remote_repo_with_file "$reponame" "a.txt"
80+
81+
pushd "$TRASHDIR" > /dev/null
82+
clone_repo "$reponame" "${reponame}_alternate"
83+
popd > /dev/null
84+
85+
rm -rf .git/lfs/objects
86+
87+
alternate="$TRASHDIR/${reponame}_alternate/.git/objects"
88+
echo "\"$alternate\"" > .git/objects/info/alternates
89+
90+
GIT_TRACE=1 git lfs fetch origin master 2>&1 | tee fetch.log
91+
[ "0" -eq "$(grep -c "sending batch of size 1" fetch.log)" ]
92+
)
93+
end_test

0 commit comments

Comments
 (0)