Skip to content

Commit 4180ed8

Browse files
nbauernfeindnbauernfeind
committed
Merge remote-tracking branch 'upstream/main' into dh_19045
2 parents 514db5c + fdf7ed0 commit 4180ed8

File tree

78 files changed

+2763
-684
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+2763
-684
lines changed

‎Base/src/main/java/io/deephaven/base/FileUtils.java

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
import java.nio.file.Files;
1313
import java.nio.file.LinkOption;
1414
import java.nio.file.Path;
15+
import java.nio.file.StandardCopyOption;
16+
import java.nio.file.attribute.BasicFileAttributes;
1517
import java.util.ArrayList;
1618
import java.util.regex.Pattern;
1719

@@ -122,9 +124,9 @@ private static void moveRecursivelyInternal(File source, File destination, FileF
122124
/**
123125
* Recursive delete method that copes with .nfs files. Uses the file's parent as the trash directory.
124126
*
125-
* @param file
127+
* @param file File or directory at which to begin recursive deletion.
126128
*/
127-
public static void deleteRecursivelyOnNFS(File file) {
129+
public static void deleteRecursivelyOnNFS(final File file) {
128130
deleteRecursivelyOnNFS(new File(file.getParentFile(), '.' + file.getName() + ".trash"), file);
129131
}
130132

@@ -136,25 +138,43 @@ public static void deleteRecursivelyOnNFS(File file) {
136138
* @param fileToBeDeleted File or directory at which to begin recursive deletion.
137139
*/
138140
public static void deleteRecursivelyOnNFS(final File trashFile, final File fileToBeDeleted) {
139-
if (fileToBeDeleted.isDirectory()) {
140-
File contents[] = fileToBeDeleted.listFiles();
141+
final Path pathToBeDeleted = fileToBeDeleted.toPath();
142+
final BasicFileAttributes attr;
143+
try {
144+
// `NOFOLLOW_LINKS` lets us treat soft-links as basic files. We will delete the link, but not the link's
145+
// target (unless it also happens to be in the path we are recursively deleting ...)
146+
attr = Files.readAttributes(pathToBeDeleted, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
147+
} catch (IOException e) {
148+
return;
149+
}
150+
151+
if (attr.isDirectory()) {
152+
final File[] contents = fileToBeDeleted.listFiles();
141153
if (contents != null) {
142154
for (File childFile : contents) {
143155
deleteRecursivelyOnNFS(trashFile, childFile);
144156
}
145157
}
146-
if (!fileToBeDeleted.delete()) {
147-
throw new RuntimeException(
148-
"Failed to delete expected empty directory " + fileToBeDeleted.getAbsolutePath());
158+
try {
159+
Files.delete(pathToBeDeleted);
160+
} catch (final IOException ioe) {
161+
throw new UncheckedIOException(
162+
"Failed to delete expected empty directory " + fileToBeDeleted.getAbsolutePath(), ioe);
149163
}
150-
} else if (fileToBeDeleted.exists()) {
151-
if (!fileToBeDeleted.renameTo(trashFile)) {
152-
throw new RuntimeException("Failed to move file " + fileToBeDeleted.getAbsolutePath()
153-
+ " to temporary location " + trashFile.getAbsolutePath());
164+
} else {
165+
// if the file does not exist, we would have gotten IOException, which we caught and returned
166+
try {
167+
Files.move(pathToBeDeleted, trashFile.toPath(), StandardCopyOption.ATOMIC_MOVE);
168+
} catch (final IOException ioe) {
169+
throw new UncheckedIOException("Failed to move file " + fileToBeDeleted.getAbsolutePath()
170+
+ " to temporary location " + trashFile.getAbsolutePath(), ioe);
154171
}
155-
if (!trashFile.delete()) {
156-
throw new RuntimeException("Failed to delete temporary location " + trashFile.getAbsolutePath()
157-
+ " for file " + fileToBeDeleted.getAbsolutePath());
172+
173+
try {
174+
Files.delete(trashFile.toPath());
175+
} catch (final IOException ioe) {
176+
throw new UncheckedIOException("Failed to delete temporary location " + trashFile.getAbsolutePath()
177+
+ " for file " + fileToBeDeleted.getAbsolutePath(), ioe);
158178
}
159179
}
160180
}

‎Base/src/test/java/io/deephaven/base/FileUtilsTest.java

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,69 @@
66
import java.io.File;
77
import java.io.IOException;
88
import java.net.URISyntaxException;
9+
import java.nio.file.Files;
10+
import java.nio.file.LinkOption;
911
import java.nio.file.Path;
12+
import java.nio.file.Paths;
13+
import java.nio.file.attribute.DosFileAttributeView;
14+
import java.util.UUID;
1015

1116
import junit.framework.TestCase;
17+
import org.jetbrains.annotations.NotNull;
1218
import org.junit.Assert;
1319

1420
public class FileUtilsTest extends TestCase {
1521

22+
/**
23+
* Create a randomized directory-structure and delete it recursively
24+
*
25+
* @throws IOException
26+
*/
27+
public void testRecursiveDelete() throws IOException {
28+
final Path tmpRoot = Files.createTempDirectory("testRecursiveDelete");
29+
makeRandomDirectoryStructure(tmpRoot);
30+
31+
FileUtils.deleteRecursively(tmpRoot.toFile());
32+
Assert.assertFalse(tmpRoot.toFile().exists());
33+
}
34+
35+
/**
36+
* Create a randomized directory-structure and delete it recursively in a way safe for NFS. Before deletion, each
37+
* File is moved to a temporary-file, which is then deleted. If on NFS, the files may be left, but as temporary
38+
* ".nfs..." files, which will be cleaned up by the NFS server when all handles to them are closed
39+
*
40+
* @throws IOException
41+
*/
42+
public void testRecursiveDeleteNFS() throws IOException {
43+
final Path tmpRoot = Files.createTempDirectory("testRecursiveDeleteNFS");
44+
makeRandomDirectoryStructure(tmpRoot);
45+
46+
FileUtils.deleteRecursivelyOnNFS(tmpRoot.toFile());
47+
Assert.assertFalse(tmpRoot.toFile().exists());
48+
}
49+
50+
private static void makeRandomDirectoryStructure(@NotNull final Path rootPath) throws IOException {
51+
final Path tmpFile = Files.createTempFile(rootPath, "tmp", ".tmp");
52+
final Path tmpDir = Files.createTempDirectory(rootPath, "dir");
53+
Files.createTempFile(tmpDir, "tmp", ".tmp");
54+
55+
final String fileName = String.format("%s.lnk", UUID.randomUUID());
56+
final Path slinkPath = Paths.get(tmpDir.toString(), fileName);
57+
58+
try {
59+
Files.createSymbolicLink(slinkPath, tmpFile);
60+
} catch (final UnsupportedOperationException uoe) {
61+
// no problem; we can't try deleting a soft-link because the concept does not exist on this platform
62+
}
63+
64+
final Path hiddenFile = Files.createTempFile(tmpDir, ".hid", ".tmp");
65+
66+
// the following possibly does nothing (like if we're on *nix system)
67+
final DosFileAttributeView dosView =
68+
Files.getFileAttributeView(hiddenFile, DosFileAttributeView.class, LinkOption.NOFOLLOW_LINKS);
69+
dosView.setHidden(true);
70+
}
71+
1672
public void testConvertToFileURI() throws IOException {
1773
final File currentDir = new File("").getAbsoluteFile();
1874
fileUriTestHelper(currentDir.toString(), true, currentDir.toURI().toString());

‎Util/src/main/java/io/deephaven/util/datastructures/ReleaseTracker.java

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ class StrictReleaseTracker<RESOURCE_TYPE> implements ReleaseTracker<RESOURCE_TYP
5858
private static final StackTraceElement[] ZERO_ELEMENT_STACK_TRACE_ARRAY = new StackTraceElement[0];
5959

6060
private final Map<RESOURCE_TYPE, StackTraceElement[]> lastAcquireMap = new HashMap<>();
61+
private AlreadyReleasedException firstDoubleFree = null;
6162

6263
private static final class LastAcquireAndReleaseInfo {
6364

@@ -96,8 +97,13 @@ public final void reportRelease(@NotNull final RESOURCE_TYPE resource) {
9697
}
9798
final LastAcquireAndReleaseInfo lastAcquireAndRelease = lastAcquireAndReleaseMap.get(resource);
9899
if (lastAcquireAndRelease != null) {
99-
throw new AlreadyReleasedException(stackTrace, lastAcquireAndRelease.lastAcquire,
100-
lastAcquireAndRelease.lastRelease);
100+
final AlreadyReleasedException alreadyReleasedException =
101+
new AlreadyReleasedException(stackTrace, lastAcquireAndRelease.lastAcquire,
102+
lastAcquireAndRelease.lastRelease);
103+
if (firstDoubleFree == null) {
104+
firstDoubleFree = alreadyReleasedException;
105+
}
106+
throw alreadyReleasedException;
101107
}
102108
throw new UnmatchedAcquireException(stackTrace);
103109
}
@@ -116,6 +122,13 @@ public final void check() {
116122
lastAcquireMap.clear();
117123
throw leakedException;
118124
}
125+
// An AlreadyReleasedException can be suppressed when we have an error case that double frees;
126+
// let's be sure to blow up the tests.
127+
final AlreadyReleasedException alreadyReleasedException = firstDoubleFree;
128+
if (alreadyReleasedException != null) {
129+
firstDoubleFree = null;
130+
throw alreadyReleasedException;
131+
}
119132
}
120133
}
121134
}

‎engine/table/src/main/java/io/deephaven/engine/table/impl/BaseTable.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -967,10 +967,9 @@ public static class ListenerImpl extends InstrumentedTableUpdateListener {
967967
private final boolean canReuseModifiedColumnSet;
968968

969969
public ListenerImpl(String description, Table parent, BaseTable<?> dependent) {
970-
super(description, false, () -> {
971-
return (Stream.concat((Stream<Object>) ((BaseTable) parent).parents.stream(), Stream.of(parent)))
972-
.flatMapToLong(BaseTable::getParentPerformanceEntryIds).toArray();
973-
});
970+
super(description, false,
971+
() -> (Stream.concat(((BaseTable<?>) parent).parents.stream(), Stream.of(parent)))
972+
.flatMapToLong(BaseTable::getParentPerformanceEntryIds).toArray());
974973
this.parent = parent;
975974
this.dependent = dependent;
976975
if (parent.isRefreshing()) {

‎engine/table/src/main/java/io/deephaven/engine/table/impl/QueryCompilerRequestProcessor.java

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919
import java.util.ArrayList;
2020
import java.util.List;
2121
import java.util.concurrent.ExecutionException;
22-
import java.util.concurrent.TimeUnit;
23-
import java.util.concurrent.TimeoutException;
2422

2523
public abstract class QueryCompilerRequestProcessor {
2624

@@ -82,11 +80,13 @@ public CompletionStageFuture<Class<?>> submit(@NotNull final QueryCompilerReques
8280
// The earlier we validate the future, the better.
8381
final CompletionStageFuture<Class<?>> future = resolver.getFuture();
8482
try {
85-
future.get(0, TimeUnit.SECONDS);
83+
if (!future.isDone()) {
84+
throw new IllegalStateException("Resolver future is not done, but it should be.");
85+
}
86+
future.get();
8687
} catch (ExecutionException err) {
8788
throw new UncheckedDeephavenException("Compilation failed", err.getCause());
88-
} catch (InterruptedException | TimeoutException err) {
89-
// This should never happen since the future is already completed.
89+
} catch (InterruptedException err) {
9090
throw new UncheckedDeephavenException("Caught unexpected exception", err);
9191
}
9292

@@ -144,8 +144,12 @@ public void compile() {
144144

145145
final List<Throwable> exceptions = new ArrayList<>();
146146
for (CompletionStageFuture.Resolver<Class<?>> resolver : resolvers) {
147+
final CompletionStageFuture<Class<?>> future = resolver.getFuture();
147148
try {
148-
Object ignored2 = resolver.getFuture().get();
149+
if (!future.isDone()) {
150+
throw new IllegalStateException("Resolver future is not done, but it should be.");
151+
}
152+
Object ignored2 = future.get();
149153
} catch (ExecutionException err) {
150154
exceptions.add(err.getCause());
151155
} catch (InterruptedException err) {

‎engine/table/src/main/java/io/deephaven/engine/table/impl/SourceTable.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ protected void destroy() {
363363
@Override
364364
@NotNull
365365
public Stream<PerformanceEntry> sourceEntries() {
366-
if (updateSourceRegistrar != null) {
366+
if (updateSourceRegistrar != null && locationsInitialized) {
367367
final PerformanceEntry entry = locationChangePoller.getEntry();
368368
if (entry != null) {
369369
return Stream.of(entry);

‎engine/table/src/main/java/io/deephaven/engine/table/impl/by/AggregationProcessor.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -533,9 +533,14 @@ final void addMinOrMaxOperator(final boolean isMin, @NotNull final String inputN
533533
}
534534
final IterativeChunkedAggregationOperator operator = operators.get(ii);
535535
if (operator instanceof SsmChunkedMinMaxOperator) {
536-
final SsmChunkedMinMaxOperator minMaxOperator = (SsmChunkedMinMaxOperator) operator;
537-
addOperator(minMaxOperator.makeSecondaryOperator(isMin, resultName), null, inputName);
538-
return;
536+
final Collection<? extends ColumnSource<?>> resultColumns = operator.getResultColumns().values();
537+
Assert.eq(1, "1", resultColumns.size(), "SsmChunkedMinMaxOperator.resultColumns.size()");
538+
final Class<?> existingOperatorType = resultColumns.iterator().next().getType();
539+
if (existingOperatorType == type) {
540+
final SsmChunkedMinMaxOperator minMaxOperator = (SsmChunkedMinMaxOperator) operator;
541+
addOperator(minMaxOperator.makeSecondaryOperator(isMin, resultName), null, inputName);
542+
return;
543+
}
539544
}
540545
}
541546
addOperator(makeMinOrMaxOperator(type, resultName, isMin, isAddOnly || isBlink), inputSource, inputName);

‎engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/RollupTableImpl.java

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import io.deephaven.chunk.attributes.Values;
1717
import io.deephaven.engine.rowset.RowSet;
1818
import io.deephaven.engine.rowset.RowSetFactory;
19+
import io.deephaven.engine.rowset.TrackingRowSet;
1920
import io.deephaven.engine.table.*;
2021
import io.deephaven.engine.table.hierarchical.RollupTable;
2122
import io.deephaven.engine.table.impl.BaseTable.CopyAttributeOperation;
@@ -25,7 +26,6 @@
2526
import io.deephaven.engine.table.impl.SortOperation;
2627
import io.deephaven.engine.table.impl.by.AggregationProcessor;
2728
import io.deephaven.engine.table.impl.by.AggregationRowLookup;
28-
import io.deephaven.engine.table.impl.by.rollup.RollupAggregationOutputs;
2929
import io.deephaven.engine.table.impl.select.SelectColumn;
3030
import io.deephaven.engine.table.impl.select.WhereFilter;
3131
import io.deephaven.engine.table.impl.sources.NullValueColumnSource;
@@ -42,7 +42,6 @@
4242
import java.util.stream.Collectors;
4343
import java.util.stream.Stream;
4444

45-
import static io.deephaven.api.ColumnName.names;
4645
import static io.deephaven.engine.rowset.RowSequence.NULL_ROW_KEY;
4746
import static io.deephaven.engine.table.impl.AbsoluteSortColumnConventions.*;
4847
import static io.deephaven.engine.table.impl.BaseTable.shouldCopyAttribute;
@@ -799,7 +798,20 @@ long findRowKeyInParentUnsorted(
799798
final long childNodeId,
800799
@Nullable final Object childNodeKey,
801800
final boolean usePrev) {
802-
return childNodeId == NULL_NODE_ID ? NULL_ROW_KEY : nodeSlot(childNodeId);
801+
if (childNodeId == NULL_NODE_ID) {
802+
return NULL_ROW_KEY;
803+
}
804+
805+
final int nodeDepth = nodeDepth(childNodeId);
806+
final int nodeSlot = nodeSlot(childNodeId);
807+
808+
final TrackingRowSet rowSet = levelTables[nodeDepth - 1].getRowSet();
809+
if ((usePrev ? rowSet.findPrev(nodeSlot) : rowSet.find(nodeSlot)) == NULL_ROW_KEY) {
810+
// the aggregation knows about this key, but it does not actually exist in the table
811+
return NULL_ROW_KEY;
812+
}
813+
814+
return nodeSlot;
803815
}
804816

805817
@Override

‎engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeSourceRowLookup.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ long get(final Object nodeKey) {
6262
if (idAggregationRow == rowLookup.noEntryValue()) {
6363
return noEntryValue();
6464
}
65-
return sourceRowKeyColumnSource.get(idAggregationRow);
65+
return sourceRowKeyColumnSource.getLong(idAggregationRow);
6666
}
6767

6868
/**
@@ -77,7 +77,7 @@ long getPrev(final Object nodeKey) {
7777
if (idAggregationRow == rowLookup.noEntryValue()) {
7878
return noEntryValue();
7979
}
80-
return sourceRowKeyColumnSource.getPrev(idAggregationRow);
80+
return sourceRowKeyColumnSource.getPrevLong(idAggregationRow);
8181
}
8282

8383
@Override

‎engine/table/src/main/java/io/deephaven/engine/table/impl/hierarchical/TreeTableImpl.java

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -331,13 +331,11 @@ long findRowKeyInParentUnsorted(
331331
if (sourceRowKey == sourceRowLookup.noEntryValue()) {
332332
return NULL_ROW_KEY;
333333
}
334-
if (filtered) {
335-
final long sourceRowPosition = usePrev
336-
? getSource().getRowSet().findPrev(sourceRowKey)
337-
: getSource().getRowSet().find(sourceRowKey);
338-
if (sourceRowPosition == NULL_ROW_KEY) {
339-
return NULL_ROW_KEY;
340-
}
334+
final long sourceRowPosition = usePrev
335+
? getSource().getRowSet().findPrev(sourceRowKey)
336+
: getSource().getRowSet().find(sourceRowKey);
337+
if (sourceRowPosition == NULL_ROW_KEY) {
338+
return NULL_ROW_KEY;
341339
}
342340
return sourceRowKey;
343341
}

‎engine/table/src/main/java/io/deephaven/engine/table/impl/select/AbstractFormulaColumn.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -258,8 +258,10 @@ private Formula getFormula(boolean initLazyMap,
258258
QueryScopeParam<?>... params) {
259259
final FormulaFactory formulaFactory;
260260
try {
261-
// the future must already be completed or else it is an error
262-
formulaFactory = formulaFactoryFuture.get(0, TimeUnit.SECONDS);
261+
// The future's root-parent is definitely complete via the QueryCompilerRequestProcessor. However, we
262+
// might need to wait for follow on thenApply mappers to complete. We expect this to be very fast, so we
263+
// use a 1-minute timeout to avoid blocking indefinitely.
264+
formulaFactory = formulaFactoryFuture.get(1, TimeUnit.MINUTES);
263265
} catch (InterruptedException | TimeoutException e) {
264266
throw new IllegalStateException("Formula factory not already compiled!", e);
265267
} catch (ExecutionException e) {

‎engine/table/src/main/java/io/deephaven/engine/table/impl/select/ConditionFilter.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -375,11 +375,6 @@ public WritableRowSet filter(final RowSet selection, final RowSet fullSet, final
375375
filterKernel.filter(context, currentChunkRowSequence.asRowKeyChunk(), inputChunks);
376376
resultBuilder.appendOrderedRowKeysChunk(matchedIndices);
377377
} catch (Exception e) {
378-
// Clean up the contexts before throwing the exception.
379-
SafeCloseable.closeAll(sourceContexts);
380-
if (sharedContext != null) {
381-
sharedContext.close();
382-
}
383378
throw new FormulaEvaluationException(e.getClass().getName() + " encountered in filter={ "
384379
+ StringEscapeUtils.escapeJava(truncateLongFormula(formula)) + " }", e);
385380
}

‎engine/table/src/main/java/io/deephaven/engine/table/impl/sources/UnionRedirection.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ private static int slotForRowKey(final long rowKey, @NotNull final ThreadLocal<I
197197

198198
private static int slotForRowKey(final long rowKey, int firstSlot,
199199
@NotNull final long[] firstRowKeyForSlot, final int numSlots) {
200-
if (rowKey >= firstRowKeyForSlot[firstSlot]) {
200+
if (firstSlot < numSlots && rowKey >= firstRowKeyForSlot[firstSlot]) {
201201
if (rowKey < firstRowKeyForSlot[firstSlot + 1]) {
202202
return firstSlot;
203203
}

0 commit comments

Comments
 (0)