Skip to content

Commit 3321ccc

Browse files
technoweenietechnoweenie
committed
Merge pull request git-lfs#271 from github/dont-clean-pointers
pass pointers straight through 'git lfs clean'
2 parents 8a39372 + 31219e5 commit 3321ccc

File tree

4 files changed

+288
-29
lines changed

4 files changed

+288
-29
lines changed

‎commands/clean_test.go

Lines changed: 257 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,48 +2,280 @@ package commands
22

33
import (
44
"bytes"
5-
"github.com/bmizerany/assert"
5+
"io"
66
"io/ioutil"
7+
"math/rand"
78
"os"
89
"path/filepath"
910
"testing"
1011
)
1112

12-
func TestClean(t *testing.T) {
13+
func TestCleanSmallFile(t *testing.T) {
1314
repo := NewRepository(t, "empty")
1415
defer repo.Test()
1516

16-
content := "HI\n"
17-
oid := "f712374589a4f37f0fd6b941a104c7ccf43f68b1fdecb4d5cd88b80acbf98fc2"
17+
cmd := repo.Command("clean")
18+
cmd.Input = bytes.NewBufferString("whatever\n")
19+
cmd.Output = "version https://git-lfs.github.com/spec/v1\n" +
20+
"oid sha256:cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411\n" +
21+
"size 9"
22+
23+
path := filepath.Join(repo.Path, ".git", "lfs", "objects")
1824
prePushHookFile := filepath.Join(repo.Path, ".git", "hooks", "pre-push")
1925

20-
cmd := repo.Command("clean", "somefile")
21-
cmd.Input = bytes.NewBufferString(content)
22-
cmd.Output = `version https://git-lfs.github.com/spec/v1
23-
oid sha256:` + oid + `
24-
size 3`
26+
cmd.Before(func() {
27+
_, err := os.Open(path)
28+
if _, ok := err.(*os.PathError); !ok {
29+
t.Fatalf("'%s' should not exist", path)
30+
}
31+
})
32+
33+
file := filepath.Join(path, "cd", "29", "cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411")
34+
cmd.After(func() {
35+
by, err := ioutil.ReadFile(file)
36+
if err != nil {
37+
t.Fatal(err)
38+
}
39+
40+
contents := string(by)
41+
if contents != "whatever\n" {
42+
t.Fatalf("wrong contents: '%v'", contents)
43+
}
44+
45+
by, err = ioutil.ReadFile(prePushHookFile)
46+
if err != nil {
47+
t.Fatal(err)
48+
}
49+
50+
pattern := "git lfs push"
51+
if !bytes.Contains(by, []byte(pattern)) {
52+
t.Errorf("hook does not contain %s:\n%s", pattern, string(by))
53+
}
54+
})
55+
}
56+
57+
func TestCleanBigFile(t *testing.T) {
58+
repo := NewRepository(t, "empty")
59+
defer repo.Test()
60+
61+
cmd := repo.Command("clean")
62+
cmd.Input = randomReader(1024)
63+
cmd.Output = "version https://git-lfs.github.com/spec/v1\n" +
64+
"oid sha256:7cd8be1d2cd0dd22cd9d229bb6b5785009a05e8b39d405615d882caac56562b5\n" +
65+
"size 1024"
66+
67+
path := filepath.Join(repo.Path, ".git", "lfs", "objects")
2568

69+
cmd.Before(func() {
70+
_, err := os.Open(path)
71+
if _, ok := err.(*os.PathError); !ok {
72+
t.Fatalf("'%s' should not exist", path)
73+
}
74+
})
75+
76+
file := filepath.Join(path, "7c", "d8", "7cd8be1d2cd0dd22cd9d229bb6b5785009a05e8b39d405615d882caac56562b5")
2677
cmd.After(func() {
27-
// assert hooks
28-
stat, err := os.Stat(prePushHookFile)
29-
assert.Equal(t, nil, err)
30-
assert.Equal(t, false, stat.IsDir())
31-
})
32-
33-
cmd = repo.Command("clean")
34-
cmd.Input = bytes.NewBufferString(content)
35-
cmd.Output = `version https://git-lfs.github.com/spec/v1
36-
oid sha256:` + oid + `
37-
size 3`
38-
customHook := []byte("echo 'yo'")
78+
stat, err := os.Stat(file)
79+
if err != nil {
80+
t.Fatal(err)
81+
}
82+
83+
if fileSize := stat.Size(); fileSize != 1024 {
84+
t.Fatalf("bad size: %d", fileSize)
85+
}
86+
})
87+
}
88+
89+
func TestCleanPointerWithExtra(t *testing.T) {
90+
repo := NewRepository(t, "empty")
91+
defer repo.Test()
92+
93+
cmd := repo.Command("clean")
94+
cmd.Input = bytes.NewBufferString("version https://git-lfs.github.com/spec/v1\n" +
95+
"oid sha256:7cd8be1d2cd0dd22cd9d229bb6b5785009a05e8b39d405615d882caac56562b5\n" +
96+
"size 1024\n\n" +
97+
"This is my test pointer. There are many like it, but this one is mine.")
98+
cmd.Output = "version https://git-lfs.github.com/spec/v1\n" +
99+
"oid sha256:f39333aa7ce293ce27742843a1a1cd6e958eaf6b44cfca9039e6b461088df5ba\n" +
100+
"size 201"
101+
102+
path := filepath.Join(repo.Path, ".git", "lfs", "objects")
103+
104+
cmd.Before(func() {
105+
_, err := os.Open(path)
106+
if _, ok := err.(*os.PathError); !ok {
107+
t.Fatalf("'%s' should not exist", path)
108+
}
109+
})
110+
111+
file := filepath.Join(path, "f3", "93", "f39333aa7ce293ce27742843a1a1cd6e958eaf6b44cfca9039e6b461088df5ba")
112+
cmd.After(func() {
113+
stat, err := os.Stat(file)
114+
if err != nil {
115+
t.Fatal(err)
116+
}
117+
118+
if fileSize := stat.Size(); fileSize != 201 {
119+
t.Fatalf("bad size: %d", fileSize)
120+
}
121+
})
122+
}
123+
124+
func TestCleanPointerWithWhitespaceAndExtra(t *testing.T) {
125+
repo := NewRepository(t, "empty")
126+
defer repo.Test()
127+
128+
head := bytes.NewBufferString("version https://git-lfs.github.com/spec/v1\n" +
129+
"oid sha256:7cd8be1d2cd0dd22cd9d229bb6b5785009a05e8b39d405615d882caac56562b5\n" +
130+
"size 1024\n" +
131+
// extra white space to fill the initial buffer in DecodeFrom()
132+
"\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n")
133+
134+
cmd := repo.Command("clean")
135+
cmd.Input = io.MultiReader(head, randomReader(1024))
136+
cmd.Output = "version https://git-lfs.github.com/spec/v1\n" +
137+
"oid sha256:494f6e07ed15b2e31ca60b31ed8c503bd7553dfd4a76ec73692f766bc67f0bec\n" +
138+
"size 1537"
139+
140+
path := filepath.Join(repo.Path, ".git", "lfs", "objects")
141+
142+
cmd.Before(func() {
143+
_, err := os.Open(path)
144+
if _, ok := err.(*os.PathError); !ok {
145+
t.Fatalf("'%s' should not exist", path)
146+
}
147+
})
148+
149+
file := filepath.Join(path, "49", "4f", "494f6e07ed15b2e31ca60b31ed8c503bd7553dfd4a76ec73692f766bc67f0bec")
150+
cmd.After(func() {
151+
stat, err := os.Stat(file)
152+
if err != nil {
153+
t.Fatal(err)
154+
}
155+
156+
if fileSize := stat.Size(); fileSize != 1537 {
157+
t.Fatalf("bad size: %d", fileSize)
158+
}
159+
})
160+
}
161+
162+
func TestCleanPointer(t *testing.T) {
163+
repo := NewRepository(t, "empty")
164+
defer repo.Test()
165+
166+
versions := []string{
167+
"http://git-media.io/v/2",
168+
"https://hawser.github.com/spec/v1",
169+
"https://git-lfs.github.com/spec/v1",
170+
}
171+
172+
for _, v := range versions {
173+
cmd := repo.Command("clean")
174+
cmd.Input = bytes.NewBufferString("version " + v + "\n" +
175+
"oid sha256:cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411\n" +
176+
"size 9\n")
177+
cmd.Output = "version " + v + "\n" +
178+
"oid sha256:cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411\n" +
179+
"size 9"
180+
181+
path := filepath.Join(repo.Path, ".git", "lfs", "objects")
182+
183+
cmd.Before(func() {
184+
_, err := os.Open(path)
185+
if _, ok := err.(*os.PathError); !ok {
186+
t.Errorf("'%s' should not exist before version %s", path, v)
187+
}
188+
})
189+
190+
cmd.After(func() {
191+
dirs, err := ioutil.ReadDir(path)
192+
if _, ok := err.(*os.PathError); ok {
193+
return // it's ok if this dir does not exist
194+
}
195+
196+
if err != nil {
197+
t.Errorf("Error for version %s: %s", v, err)
198+
return
199+
}
200+
201+
if dirs == nil || len(dirs) == 0 {
202+
return
203+
}
204+
205+
if len(dirs) == 1 && dirs[0].Name() == "logs" {
206+
return
207+
}
208+
209+
for _, dir := range dirs {
210+
t.Logf("DIR: %v", dir.Name())
211+
}
212+
213+
t.Errorf("objects were written to .git/lfs/objects")
214+
})
215+
}
216+
}
217+
218+
func TestCleanWithCustomHook(t *testing.T) {
219+
repo := NewRepository(t, "empty")
220+
defer repo.Test()
221+
222+
cmd := repo.Command("clean")
223+
cmd.Input = bytes.NewBufferString("whatever\n")
224+
cmd.Output = "version https://git-lfs.github.com/spec/v1\n" +
225+
"oid sha256:cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411\n" +
226+
"size 9"
227+
228+
path := filepath.Join(repo.Path, ".git", "lfs", "objects")
229+
prePushHookFile := filepath.Join(repo.Path, ".git", "hooks", "pre-push")
230+
customHook := []byte("test")
231+
39232
cmd.Before(func() {
40233
err := ioutil.WriteFile(prePushHookFile, customHook, 0755)
41-
assert.Equal(t, nil, err)
234+
if err != nil {
235+
t.Fatal(err)
236+
}
237+
238+
_, err = os.Open(path)
239+
if _, ok := err.(*os.PathError); !ok {
240+
t.Fatalf("'%s' should not exist", path)
241+
}
42242
})
43243

244+
file := filepath.Join(path, "cd", "29", "cd293be6cea034bd45a0352775a219ef5dc7825ce55d1f7dae9762d80ce64411")
44245
cmd.After(func() {
45-
by, err := ioutil.ReadFile(prePushHookFile)
46-
assert.Equal(t, nil, err)
47-
assert.Equal(t, string(customHook), string(by))
246+
by, err := ioutil.ReadFile(file)
247+
if err != nil {
248+
t.Fatal(err)
249+
}
250+
251+
contents := string(by)
252+
if contents != "whatever\n" {
253+
t.Fatalf("wrong contents: '%v'", contents)
254+
}
255+
256+
by, err = ioutil.ReadFile(prePushHookFile)
257+
if err != nil {
258+
t.Fatal(err)
259+
}
260+
261+
if string(by) != string(customHook) {
262+
t.Logf(string(by))
263+
t.Errorf("Hook does not contain custom hook")
264+
}
48265
})
49266
}
267+
268+
func randomReader(n int64) io.Reader {
269+
return io.LimitReader(&randomDataMaker{rand.NewSource(42)}, n)
270+
}
271+
272+
type randomDataMaker struct {
273+
src rand.Source
274+
}
275+
276+
func (r *randomDataMaker) Read(p []byte) (n int, err error) {
277+
for i := range p {
278+
p[i] = byte(r.src.Int63() & 0xff)
279+
}
280+
return len(p), nil
281+
}

‎commands/command_clean.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@ func cleanCommand(cmd *cobra.Command, args []string) {
5050
defer cleaned.Teardown()
5151
}
5252

53+
if cpErr, ok := err.(*pointer.CleanedPointerError); ok {
54+
os.Stdout.Write(cpErr.Bytes)
55+
return
56+
}
57+
5358
if err != nil {
5459
Panic(err, "Error cleaning asset.")
5560
}

‎pointer/clean.go

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package pointer
22

33
import (
4+
"bytes"
45
"crypto/sha256"
56
"encoding/hex"
67
"github.com/github/git-lfs/lfs"
@@ -14,6 +15,14 @@ type cleanedAsset struct {
1415
*Pointer
1516
}
1617

18+
type CleanedPointerError struct {
19+
Bytes []byte
20+
}
21+
22+
func (e *CleanedPointerError) Error() string {
23+
return "Cannot clean a Git LFS pointer. Skipping."
24+
}
25+
1726
func Clean(reader io.Reader, size int64, cb lfs.CopyCallback) (*cleanedAsset, error) {
1827
tmp, err := lfs.TempFile("")
1928
if err != nil {
@@ -27,7 +36,13 @@ func Clean(reader io.Reader, size int64, cb lfs.CopyCallback) (*cleanedAsset, er
2736
cb = nil
2837
}
2938

30-
written, err := lfs.CopyWithCallback(writer, reader, size, cb)
39+
by, _, err := DecodeFrom(reader)
40+
if err == nil && len(by) < 512 {
41+
return nil, &CleanedPointerError{by}
42+
}
43+
44+
multi := io.MultiReader(bytes.NewReader(by), reader)
45+
written, err := lfs.CopyWithCallback(writer, multi, size, cb)
3146

3247
pointer := NewPointer(hex.EncodeToString(oidHash.Sum(nil)), written)
3348
return &cleanedAsset{tmp, "", pointer}, err

‎pointer/pointer.go

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,13 +58,20 @@ func Encode(writer io.Writer, pointer *Pointer) (int, error) {
5858
}
5959

6060
func Decode(reader io.Reader) (*Pointer, error) {
61-
buf := make([]byte, 200)
61+
_, p, err := DecodeFrom(reader)
62+
return p, err
63+
}
64+
65+
func DecodeFrom(reader io.Reader) ([]byte, *Pointer, error) {
66+
buf := make([]byte, 512)
6267
written, err := reader.Read(buf)
6368
if err != nil {
64-
return nil, err
69+
return buf, nil, err
6570
}
6671

67-
return decodeKV(bytes.TrimSpace(buf[0:written]))
72+
output := buf[0:written]
73+
p, err := decodeKV(bytes.TrimSpace(output))
74+
return output, p, err
6875
}
6976

7077
func verifyVersion(version string) error {

0 commit comments

Comments
 (0)