LUCENE-8595: Fix interleaved DV update and reset

lucene/CHANGES.txt

@@ -312,6 +312,10 @@ Bug fixes
 * LUCENE-8586: Intervals.or() could get stuck in an infinite loop on certain indexes
   (Alan Woodward)
 
+* LUCENE-8595: Fix interleaved DV update and reset. Interleaved update and reset value
+  to the same doc in the same updates package looses an update if the reset comes before
+  the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grant)
+
 New Features
 
 * LUCENE-8496: Selective indexing - modify BKDReader/BKDWriter to allow users

lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java

@@ -298,7 +298,7 @@ protected int compare(int i, int j) {
         // increasing docID order:
         // NOTE: we can have ties here, when the same docID was updated in the same segment, in which case we rely on sort being
         // stable and preserving original order so the last update to that docID wins
-        return Long.compare(docs.get(i), docs.get(j));
+        return Long.compare(docs.get(i)>>>1, docs.get(j)>>>1);
       }
     }.sort(0, size);
   }
@@ -392,9 +392,13 @@ public final int nextDoc() {
       }
       long longDoc = docs.get(idx);
       ++idx;
-      while (idx < size && docs.get(idx) == longDoc) {
+      for (; idx < size; idx++) {
         // scan forward to last update to this doc
-        ++idx;
+        final long nextLongDoc = docs.get(idx);
+        if ((longDoc >>> 1) != (nextLongDoc >>> 1)) {
+          break;
+        }
+        longDoc = nextLongDoc;
       }
       hasValue = (longDoc & HAS_VALUE_MASK) >  0;
       if (hasValue) {

lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java

@@ -69,4 +69,67 @@ public void testMergeIterator() {
     assertEquals(24, iterator.longValue());
     assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());
   }
+
+  public void testUpdateAndResetSameDoc() {
+    NumericDocValuesFieldUpdates updates = new NumericDocValuesFieldUpdates(0, "test", 2);
+    updates.add(0, 1);
+    updates.reset(0);
+    updates.finish();
+    NumericDocValuesFieldUpdates.Iterator iterator = updates.iterator();
+    assertEquals(0, iterator.nextDoc());
+    assertFalse(iterator.hasValue());
+    assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());
+  }
+
+  public void testUpdateAndResetUpdateSameDoc() {
+    NumericDocValuesFieldUpdates updates = new NumericDocValuesFieldUpdates(0, "test", 3);
+    updates.add(0, 1);
+    updates.add(0);
+    updates.add(0, 2);
+    updates.finish();
+    NumericDocValuesFieldUpdates.Iterator iterator = updates.iterator();
+    assertEquals(0, iterator.nextDoc());
+    assertTrue(iterator.hasValue());
+    assertEquals(2, iterator.longValue());
+    assertEquals(DocIdSetIterator.NO_MORE_DOCS, iterator.nextDoc());
+  }
+
+  public void testUpdatesAndResetRandom() {
+    NumericDocValuesFieldUpdates updates = new NumericDocValuesFieldUpdates(0, "test", 10);
+    int numUpdates = 10 + random().nextInt(100);
+    Integer[] values = new Integer[5];
+    for (int i = 0; i < 5; i++) {
+      values[i] = random().nextBoolean() ? null : random().nextInt(100);
+      if (values[i] == null) {
+        updates.reset(i);
+      } else {
+        updates.add(i, values[i]);
+      }
+    }
+    for (int i = 0; i < numUpdates; i++) {
+      int docId = random().nextInt(5);
+      values[docId] = random().nextBoolean() ? null : random().nextInt(100);
+      if (values[docId] == null) {
+        updates.reset(docId);
+      } else {
+        updates.add(docId, values[docId]);
+      }
+    }
+
+    updates.finish();
+    NumericDocValuesFieldUpdates.Iterator iterator = updates.iterator();
+    int idx = 0;
+    while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+      assertEquals(idx, iterator.docID());
+      if (values[idx] == null) {
+        assertFalse(iterator.hasValue());
+      } else {
+        assertTrue(iterator.hasValue());
+        assertEquals(values[idx].longValue(), iterator.longValue());
+      }
+      idx++;
+    }
+
+
+  }
 }