Skip to content

Commit 2c88e7f

Browse files
authored
robustio: copy from cmd/go/internal/robustio (rogpeppe#239)
Copied from Go commit b18b05881691861c4279a50010829150f1684fa9.
1 parent fa6a31e commit 2c88e7f

File tree

5 files changed

+220
-0
lines changed

5 files changed

+220
-0
lines changed

robustio/robustio.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Copyright 2019 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
// Package robustio wraps I/O functions that are prone to failure on Windows,
6+
// transparently retrying errors up to an arbitrary timeout.
7+
//
8+
// Errors are classified heuristically and retries are bounded, so the functions
9+
// in this package do not completely eliminate spurious errors. However, they do
10+
// significantly reduce the rate of failure in practice.
11+
//
12+
// If so, the error will likely wrap one of:
13+
// The functions in this package do not completely eliminate spurious errors,
14+
// but substantially reduce their rate of occurrence in practice.
15+
package robustio
16+
17+
// Rename is like os.Rename, but on Windows retries errors that may occur if the
18+
// file is concurrently read or overwritten.
19+
//
20+
// (See golang.org/issue/31247 and golang.org/issue/32188.)
21+
func Rename(oldpath, newpath string) error {
22+
return rename(oldpath, newpath)
23+
}
24+
25+
// ReadFile is like os.ReadFile, but on Windows retries errors that may
26+
// occur if the file is concurrently replaced.
27+
//
28+
// (See golang.org/issue/31247 and golang.org/issue/32188.)
29+
func ReadFile(filename string) ([]byte, error) {
30+
return readFile(filename)
31+
}
32+
33+
// RemoveAll is like os.RemoveAll, but on Windows retries errors that may occur
34+
// if an executable file in the directory has recently been executed.
35+
//
36+
// (See golang.org/issue/19491.)
37+
func RemoveAll(path string) error {
38+
return removeAll(path)
39+
}
40+
41+
// IsEphemeralError reports whether err is one of the errors that the functions
42+
// in this package attempt to mitigate.
43+
//
44+
// Errors considered ephemeral include:
45+
// - syscall.ERROR_ACCESS_DENIED
46+
// - syscall.ERROR_FILE_NOT_FOUND
47+
// - internal/syscall/windows.ERROR_SHARING_VIOLATION
48+
//
49+
// This set may be expanded in the future; programs must not rely on the
50+
// non-ephemerality of any given error.
51+
func IsEphemeralError(err error) bool {
52+
return isEphemeralError(err)
53+
}

robustio/robustio_darwin.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Copyright 2019 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package robustio
6+
7+
import (
8+
"errors"
9+
"syscall"
10+
)
11+
12+
const errFileNotFound = syscall.ENOENT
13+
14+
// isEphemeralError returns true if err may be resolved by waiting.
15+
func isEphemeralError(err error) bool {
16+
var errno syscall.Errno
17+
if errors.As(err, &errno) {
18+
return errno == errFileNotFound
19+
}
20+
return false
21+
}

robustio/robustio_flaky.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
// Copyright 2019 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
//go:build windows || darwin
6+
7+
package robustio
8+
9+
import (
10+
"errors"
11+
"math/rand"
12+
"os"
13+
"syscall"
14+
"time"
15+
)
16+
17+
const arbitraryTimeout = 2000 * time.Millisecond
18+
19+
// retry retries ephemeral errors from f up to an arbitrary timeout
20+
// to work around filesystem flakiness on Windows and Darwin.
21+
func retry(f func() (err error, mayRetry bool)) error {
22+
var (
23+
bestErr error
24+
lowestErrno syscall.Errno
25+
start time.Time
26+
nextSleep time.Duration = 1 * time.Millisecond
27+
)
28+
for {
29+
err, mayRetry := f()
30+
if err == nil || !mayRetry {
31+
return err
32+
}
33+
34+
var errno syscall.Errno
35+
if errors.As(err, &errno) && (lowestErrno == 0 || errno < lowestErrno) {
36+
bestErr = err
37+
lowestErrno = errno
38+
} else if bestErr == nil {
39+
bestErr = err
40+
}
41+
42+
if start.IsZero() {
43+
start = time.Now()
44+
} else if d := time.Since(start) + nextSleep; d >= arbitraryTimeout {
45+
break
46+
}
47+
time.Sleep(nextSleep)
48+
nextSleep += time.Duration(rand.Int63n(int64(nextSleep)))
49+
}
50+
51+
return bestErr
52+
}
53+
54+
// rename is like os.Rename, but retries ephemeral errors.
55+
//
56+
// On Windows it wraps os.Rename, which (as of 2019-06-04) uses MoveFileEx with
57+
// MOVEFILE_REPLACE_EXISTING.
58+
//
59+
// Windows also provides a different system call, ReplaceFile,
60+
// that provides similar semantics, but perhaps preserves more metadata. (The
61+
// documentation on the differences between the two is very sparse.)
62+
//
63+
// Empirical error rates with MoveFileEx are lower under modest concurrency, so
64+
// for now we're sticking with what the os package already provides.
65+
func rename(oldpath, newpath string) (err error) {
66+
return retry(func() (err error, mayRetry bool) {
67+
err = os.Rename(oldpath, newpath)
68+
return err, isEphemeralError(err)
69+
})
70+
}
71+
72+
// readFile is like os.ReadFile, but retries ephemeral errors.
73+
func readFile(filename string) ([]byte, error) {
74+
var b []byte
75+
err := retry(func() (err error, mayRetry bool) {
76+
b, err = os.ReadFile(filename)
77+
78+
// Unlike in rename, we do not retry errFileNotFound here: it can occur
79+
// as a spurious error, but the file may also genuinely not exist, so the
80+
// increase in robustness is probably not worth the extra latency.
81+
return err, isEphemeralError(err) && !errors.Is(err, errFileNotFound)
82+
})
83+
return b, err
84+
}
85+
86+
func removeAll(path string) error {
87+
return retry(func() (err error, mayRetry bool) {
88+
err = os.RemoveAll(path)
89+
return err, isEphemeralError(err)
90+
})
91+
}

robustio/robustio_other.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
// Copyright 2019 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
//go:build !windows && !darwin
6+
7+
package robustio
8+
9+
import (
10+
"os"
11+
)
12+
13+
func rename(oldpath, newpath string) error {
14+
return os.Rename(oldpath, newpath)
15+
}
16+
17+
func readFile(filename string) ([]byte, error) {
18+
return os.ReadFile(filename)
19+
}
20+
21+
func removeAll(path string) error {
22+
return os.RemoveAll(path)
23+
}
24+
25+
func isEphemeralError(err error) bool {
26+
return false
27+
}

robustio/robustio_windows.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Copyright 2019 The Go Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package robustio
6+
7+
import (
8+
"errors"
9+
"syscall"
10+
11+
"github.com/rogpeppe/go-internal/internal/syscall/windows"
12+
)
13+
14+
const errFileNotFound = syscall.ERROR_FILE_NOT_FOUND
15+
16+
// isEphemeralError returns true if err may be resolved by waiting.
17+
func isEphemeralError(err error) bool {
18+
var errno syscall.Errno
19+
if errors.As(err, &errno) {
20+
switch errno {
21+
case syscall.ERROR_ACCESS_DENIED,
22+
syscall.ERROR_FILE_NOT_FOUND,
23+
windows.ERROR_SHARING_VIOLATION:
24+
return true
25+
}
26+
}
27+
return false
28+
}

0 commit comments

Comments
 (0)