Skip to content

Commit e3f535c

Browse files
Thomas Wolfmsohn
Thomas Wolf
authored and
msohn
committed
WorkingTreeIterator: handle different timestamp resolutions
Older JGit stored only milliseconds timestamps in the index. Newer JGit may get finer timestamps from the file system. This leads to slow index diffs when a new JGit runs against an index produced by older JGit because many timestamps will differ and JGit will then do many content checks. See [1]. Handle this migration case by only comparing milliseconds if the index entry has only millisecond precision. The inverse may also occur; also compare only milliseconds if the file timestamp has only millisecond precision. Do the same also for microsecond resolution. On Windows, NTFS may provide 100ns resolution and may be used by external programs writing the index, but Java's WindowsFileAttributes may provide only microseconds. File timestamp precision in Java depends not only on the Java APIs used by different JGit versions but may also change when running the same Java code on different VMs. And of course the resolution may vary among operating and file systems. Moreover, timestamp precision in the index depends on the program that wrote the index. Canonical git may use a different resolution, maybe even different between git versions. [1] https://www.eclipse.org/forums/index.php/t/1100344/ Change-Id: Idfd08606c883cb98787b2138f9baf0cc89a57b56 Signed-off-by: Thomas Wolf <[email protected]> Signed-off-by: Matthias Sohn <[email protected]>
1 parent 84ac86e commit e3f535c

File tree

3 files changed

+307
-18
lines changed

3 files changed

+307
-18
lines changed

‎org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/InstantComparatorTest.java

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
/*
2+
* Copyright (C) 2019, Thomas Wolf <[email protected]>
3+
* and other copyright owners as documented in the project's IP log.
4+
*
5+
* This program and the accompanying materials are made available
6+
* under the terms of the Eclipse Distribution License v1.0 which
7+
* accompanies this distribution, is reproduced below, and is
8+
* available at http://www.eclipse.org/org/documents/edl-v10.php
9+
*
10+
* All rights reserved.
11+
*
12+
* Redistribution and use in source and binary forms, with or
13+
* without modification, are permitted provided that the following
14+
* conditions are met:
15+
*
16+
* - Redistributions of source code must retain the above copyright
17+
* notice, this list of conditions and the following disclaimer.
18+
*
19+
* - Redistributions in binary form must reproduce the above
20+
* copyright notice, this list of conditions and the following
21+
* disclaimer in the documentation and/or other materials provided
22+
* with the distribution.
23+
*
24+
* - Neither the name of the Eclipse Foundation, Inc. nor the
25+
* names of its contributors may be used to endorse or promote
26+
* products derived from this software without specific prior
27+
* written permission.
28+
*
29+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
30+
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
31+
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32+
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
34+
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35+
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
36+
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
37+
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
38+
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
39+
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
41+
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42+
*/
43+
package org.eclipse.jgit.treewalk;
44+
45+
import static org.junit.Assert.assertEquals;
46+
import static org.junit.Assert.assertTrue;
47+
48+
import java.time.Instant;
49+
50+
import org.junit.Test;
51+
52+
public class InstantComparatorTest {
53+
54+
private final InstantComparator cmp = new InstantComparator();
55+
56+
@Test
57+
public void compareNow() {
58+
Instant now = Instant.now();
59+
assertEquals(0, cmp.compare(now, now));
60+
assertEquals(0, cmp.compare(now, now, true));
61+
}
62+
63+
@Test
64+
public void compareSeconds() {
65+
Instant now = Instant.now();
66+
Instant t = Instant.ofEpochSecond(now.getEpochSecond());
67+
Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456789);
68+
assertEquals(0, cmp.compare(t, s));
69+
assertEquals(0, cmp.compare(t, t));
70+
assertEquals(0, cmp.compare(s, t));
71+
}
72+
73+
@Test
74+
public void compareSecondsOnly() {
75+
Instant now = Instant.now();
76+
Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 987654321);
77+
Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456789);
78+
assertEquals(0, cmp.compare(t, s, true));
79+
assertEquals(0, cmp.compare(t, t, true));
80+
assertEquals(0, cmp.compare(s, t, true));
81+
}
82+
83+
@Test
84+
public void compareSecondsUnequal() {
85+
Instant now = Instant.now();
86+
Instant t = Instant.ofEpochSecond(now.getEpochSecond());
87+
Instant s = Instant.ofEpochSecond(now.getEpochSecond() - 1L);
88+
assertTrue(cmp.compare(s, t) < 0);
89+
assertTrue(cmp.compare(t, s) > 0);
90+
}
91+
92+
@Test
93+
public void compareMillisEqual() {
94+
Instant now = Instant.now();
95+
Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123000000);
96+
Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456789);
97+
assertEquals(0, cmp.compare(s, t));
98+
assertEquals(0, cmp.compare(t, t));
99+
assertEquals(0, cmp.compare(t, s));
100+
s = Instant.ofEpochSecond(now.getEpochSecond(), 123456000);
101+
assertEquals(0, cmp.compare(s, t));
102+
assertEquals(0, cmp.compare(t, s));
103+
s = Instant.ofEpochSecond(now.getEpochSecond(), 123400000);
104+
assertEquals(0, cmp.compare(s, t));
105+
assertEquals(0, cmp.compare(t, s));
106+
}
107+
108+
@Test
109+
public void compareMillisUnequal() {
110+
Instant now = Instant.now();
111+
Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123000000);
112+
Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 122000000);
113+
assertTrue(cmp.compare(s, t) < 0);
114+
assertTrue(cmp.compare(t, s) > 0);
115+
t = Instant.ofEpochSecond(now.getEpochSecond(), 130000000);
116+
assertTrue(cmp.compare(s, t) < 0);
117+
assertTrue(cmp.compare(t, s) > 0);
118+
t = Instant.ofEpochSecond(now.getEpochSecond(), 200000000);
119+
assertTrue(cmp.compare(s, t) < 0);
120+
assertTrue(cmp.compare(t, s) > 0);
121+
s = Instant.ofEpochSecond(now.getEpochSecond() - 1L, 123000000);
122+
assertTrue(cmp.compare(s, t) < 0);
123+
assertTrue(cmp.compare(t, s) > 0);
124+
}
125+
126+
@Test
127+
public void compareMicrosEqual() {
128+
Instant now = Instant.now();
129+
Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123456000);
130+
Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456789);
131+
assertEquals(0, cmp.compare(s, t));
132+
assertEquals(0, cmp.compare(t, s));
133+
s = Instant.ofEpochSecond(now.getEpochSecond(), 123456700);
134+
assertEquals(0, cmp.compare(s, t));
135+
assertEquals(0, cmp.compare(t, s));
136+
}
137+
138+
@Test
139+
public void compareMicrosUnequal() {
140+
Instant now = Instant.now();
141+
Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123456000);
142+
Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123455000);
143+
assertTrue(cmp.compare(s, t) < 0);
144+
assertTrue(cmp.compare(t, s) > 0);
145+
t = Instant.ofEpochSecond(now.getEpochSecond(), 123460000);
146+
assertTrue(cmp.compare(s, t) < 0);
147+
assertTrue(cmp.compare(t, s) > 0);
148+
t = Instant.ofEpochSecond(now.getEpochSecond(), 123500000);
149+
assertTrue(cmp.compare(s, t) < 0);
150+
assertTrue(cmp.compare(t, s) > 0);
151+
s = Instant.ofEpochSecond(now.getEpochSecond() - 1L, 123456000);
152+
assertTrue(cmp.compare(s, t) < 0);
153+
assertTrue(cmp.compare(t, s) > 0);
154+
}
155+
156+
@Test
157+
public void compareNanosEqual() {
158+
Instant now = Instant.now();
159+
Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123456789);
160+
Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456789);
161+
assertEquals(0, cmp.compare(s, t));
162+
assertEquals(0, cmp.compare(t, s));
163+
}
164+
165+
@Test
166+
public void compareNanosUnequal() {
167+
Instant now = Instant.now();
168+
Instant t = Instant.ofEpochSecond(now.getEpochSecond(), 123456789);
169+
Instant s = Instant.ofEpochSecond(now.getEpochSecond(), 123456700);
170+
assertTrue(cmp.compare(s, t) < 0);
171+
assertTrue(cmp.compare(t, s) > 0);
172+
t = Instant.ofEpochSecond(now.getEpochSecond(), 123456800);
173+
assertTrue(cmp.compare(s, t) < 0);
174+
assertTrue(cmp.compare(t, s) > 0);
175+
s = Instant.ofEpochSecond(now.getEpochSecond() - 1L, 123456789);
176+
assertTrue(cmp.compare(s, t) < 0);
177+
assertTrue(cmp.compare(t, s) > 0);
178+
s = Instant.ofEpochSecond(now.getEpochSecond(), 123456788);
179+
assertTrue(cmp.compare(s, t) < 0);
180+
assertTrue(cmp.compare(t, s) > 0);
181+
}
182+
}

‎org.eclipse.jgit/src/org/eclipse/jgit/treewalk/InstantComparator.java

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Copyright (C) 2019, Thomas Wolf <[email protected]>
3+
* and other copyright owners as documented in the project's IP log.
4+
*
5+
* This program and the accompanying materials are made available
6+
* under the terms of the Eclipse Distribution License v1.0 which
7+
* accompanies this distribution, is reproduced below, and is
8+
* available at http://www.eclipse.org/org/documents/edl-v10.php
9+
*
10+
* All rights reserved.
11+
*
12+
* Redistribution and use in source and binary forms, with or
13+
* without modification, are permitted provided that the following
14+
* conditions are met:
15+
*
16+
* - Redistributions of source code must retain the above copyright
17+
* notice, this list of conditions and the following disclaimer.
18+
*
19+
* - Redistributions in binary form must reproduce the above
20+
* copyright notice, this list of conditions and the following
21+
* disclaimer in the documentation and/or other materials provided
22+
* with the distribution.
23+
*
24+
* - Neither the name of the Eclipse Foundation, Inc. nor the
25+
* names of its contributors may be used to endorse or promote
26+
* products derived from this software without specific prior
27+
* written permission.
28+
*
29+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
30+
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
31+
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32+
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
34+
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35+
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
36+
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
37+
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
38+
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
39+
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
41+
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42+
*/
43+
package org.eclipse.jgit.treewalk;
44+
45+
import java.time.Instant;
46+
import java.util.Comparator;
47+
48+
/**
49+
* Specialized comparator for {@link Instant}s. If either timestamp has a zero
50+
* fraction, compares only seconds. If either timestamp has no time fraction
51+
* smaller than a millisecond, compares only milliseconds. If either timestamp
52+
* has no fraction smaller than a microsecond, compares only microseconds.
53+
*/
54+
class InstantComparator implements Comparator<Instant> {
55+
56+
@Override
57+
public int compare(Instant a, Instant b) {
58+
return compare(a, b, false);
59+
}
60+
61+
/**
62+
* Compares two {@link Instant}s to the lower resolution of the two
63+
* instants. See {@link InstantComparator}.
64+
*
65+
* @param a
66+
* first {@link Instant} to compare
67+
* @param b
68+
* second {@link Instant} to compare
69+
* @param forceSecondsOnly
70+
* whether to omit all fraction comparison
71+
* @return a value &lt; 0 if a &lt; b, a value &gt; 0 if a &gt; b, and 0 if
72+
* a == b
73+
*/
74+
public int compare(Instant a, Instant b, boolean forceSecondsOnly) {
75+
long aSeconds = a.getEpochSecond();
76+
long bSeconds = b.getEpochSecond();
77+
int result = Long.compare(aSeconds, bSeconds);
78+
if (result != 0) {
79+
return result;
80+
}
81+
int aSubSecond = a.getNano();
82+
int bSubSecond = b.getNano();
83+
if (forceSecondsOnly || (aSubSecond == 0)
84+
|| (bSubSecond == 0)) {
85+
// Don't check the subseconds part.
86+
return 0;
87+
} else if (aSubSecond != bSubSecond) {
88+
// If either has nothing smaller than a millisecond, compare only
89+
// milliseconds.
90+
int aSubMillis = aSubSecond % 1_000_000;
91+
int bSubMillis = bSubSecond % 1_000_000;
92+
if (aSubMillis == 0) {
93+
bSubSecond -= bSubMillis;
94+
} else if (bSubMillis == 0) {
95+
aSubSecond -= aSubMillis;
96+
} else {
97+
// Same again, but for microsecond resolution. NTFS has 100ns
98+
// resolution, but WindowsFileAttributes may provide only
99+
// microseconds (1000ns). Similar for some Unix file systems.
100+
int aSubMicros = aSubSecond % 1000;
101+
int bSubMicros = bSubSecond % 1000;
102+
if (aSubMicros == 0) {
103+
bSubSecond -= bSubMicros;
104+
} else if (bSubMicros == 0) {
105+
aSubSecond -= aSubMicros;
106+
}
107+
}
108+
}
109+
return Integer.compare(aSubSecond, bSubSecond);
110+
}
111+
112+
}

‎org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,9 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator {
174174
/** The offset of the content id in {@link #idBuffer()} */
175175
private int contentIdOffset;
176176

177+
/** A comparator for {@link Instant}s. */
178+
private final InstantComparator timestampComparator = new InstantComparator();
179+
177180
/**
178181
* Create a new iterator with no parent.
179182
*
@@ -935,27 +938,19 @@ public MetadataDiff compareMetadata(DirCacheEntry entry) {
935938
if (!entry.isSmudged() && entry.getLength() != (int) getEntryLength())
936939
return MetadataDiff.DIFFER_BY_METADATA;
937940

938-
// Git under windows only stores seconds so we round the timestamp
939-
// Java gives us if it looks like the timestamp in index is seconds
940-
// only. Otherwise we compare the timestamp at nanosecond precision,
941-
// unless core.checkstat is set to "minimal", in which case we only
942-
// compare the whole second part.
941+
// Cache and file timestamps may differ in resolution. Therefore don't
942+
// compare instants directly but use a comparator that compares only
943+
// up to the lower apparent resolution of either timestamp.
944+
//
945+
// If core.checkstat is set to "minimal", compare only the seconds part.
943946
Instant cacheLastModified = entry.getLastModifiedInstant();
944947
Instant fileLastModified = getEntryLastModifiedInstant();
945-
if ((getOptions().getCheckStat() == CheckStat.MINIMAL)
946-
|| (cacheLastModified.getNano() == 0)
947-
// Some Java version on Linux return whole seconds only even
948-
// when the file systems supports more precision.
949-
|| (fileLastModified.getNano() == 0)) {
950-
if (fileLastModified.getEpochSecond() != cacheLastModified
951-
.getEpochSecond()) {
952-
return MetadataDiff.DIFFER_BY_TIMESTAMP;
953-
} else if (entry.isSmudged()) {
954-
return MetadataDiff.SMUDGED;
955-
}
956-
} else if (!fileLastModified.equals(cacheLastModified)) {
948+
if (timestampComparator.compare(cacheLastModified, fileLastModified,
949+
getOptions().getCheckStat() == CheckStat.MINIMAL) != 0) {
957950
return MetadataDiff.DIFFER_BY_TIMESTAMP;
958-
} else if (entry.isSmudged()) {
951+
}
952+
953+
if (entry.isSmudged()) {
959954
return MetadataDiff.SMUDGED;
960955
}
961956
// The file is clean when when comparing timestamps

0 commit comments

Comments
 (0)