Skip to content

Commit 6772783

Browse files
sergey-chugunov-1985dgovorukhin
sergey-chugunov-1985
authored and
dgovorukhin
committed
IGNITE-12110 Fixes tests - apache#6815
*.testMassiveServersShutdown2 Fixed isStopped flag was't set during disconnect Don't allow "NodeStoppingException" to fail checkpoint thread on stopping Test improvements: Add page lock tracker and failure handler for all BPlustTree tests Signed-off-by: Dmitriy Govorukhin <[email protected]>
1 parent 4ec489a commit 6772783

File tree

4 files changed

+97
-11
lines changed

4 files changed

+97
-11
lines changed

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -512,7 +512,13 @@ public void onGroupRebalanceFinished(int grpId, AffinityTopologyVersion topVer)
512512
return;
513513

514514
for (Integer grpId0 : session0.disabledGrps) {
515-
cctx.database().walEnabled(grpId0, true, true);
515+
try {
516+
cctx.database().walEnabled(grpId0, true, true);
517+
}
518+
catch (Exception e) {
519+
if (!X.hasCause(e, NodeStoppingException.class))
520+
throw e;
521+
}
516522

517523
CacheGroupContext grp = cctx.cache().cacheGroup(grpId0);
518524

‎modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4178,7 +4178,7 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti
41784178
resTopVer = msg.resultTopologyVersion();
41794179

41804180
if (cctx.exchange().mergeExchanges(this, msg)) {
4181-
assert cctx.kernalContext().isStopping();
4181+
assert cctx.kernalContext().isStopping() || cctx.kernalContext().clientDisconnected();
41824182

41834183
return; // Node is stopping, no need to further process exchange.
41844184
}

‎modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java‎

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import java.util.function.Predicate;
4848
import org.apache.ignite.IgniteCheckedException;
4949
import org.apache.ignite.configuration.DataRegionConfiguration;
50+
import org.apache.ignite.failure.FailureContext;
5051
import org.apache.ignite.internal.IgniteInternalFuture;
5152
import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
5253
import org.apache.ignite.internal.pagemem.FullPageId;
@@ -55,6 +56,7 @@
5556
import org.apache.ignite.internal.pagemem.PageUtils;
5657
import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
5758
import org.apache.ignite.internal.processors.cache.persistence.DataStructure;
59+
import org.apache.ignite.internal.processors.cache.persistence.diagnostic.pagelocktracker.PageLockTrackerManager;
5860
import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
5961
import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
6062
import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
@@ -64,6 +66,7 @@
6466
import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
6567
import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageLockListener;
6668
import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric;
69+
import org.apache.ignite.internal.processors.failure.FailureProcessor;
6770
import org.apache.ignite.internal.util.GridConcurrentHashSet;
6871
import org.apache.ignite.internal.util.GridRandom;
6972
import org.apache.ignite.internal.util.GridStripedLock;
@@ -74,11 +77,14 @@
7477
import org.apache.ignite.internal.util.typedef.internal.SB;
7578
import org.apache.ignite.internal.util.typedef.internal.U;
7679
import org.apache.ignite.testframework.GridTestUtils;
80+
import org.apache.ignite.testframework.junits.GridTestKernalContext;
81+
import org.apache.ignite.testframework.junits.WithSystemProperty;
7782
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
7883
import org.jetbrains.annotations.Nullable;
7984
import org.jsr166.ConcurrentLinkedHashMap;
8085
import org.junit.Test;
8186

87+
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PAGE_LOCK_TRACKER_CHECK_INTERVAL;
8288
import static org.apache.ignite.internal.pagemem.PageIdUtils.effectivePageId;
8389
import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.rnd;
8490
import static org.apache.ignite.internal.processors.database.BPlusTreeSelfTest.TestTree.threadId;
@@ -88,6 +94,7 @@
8894

8995
/**
9096
*/
97+
@WithSystemProperty(key = IGNITE_PAGE_LOCK_TRACKER_CHECK_INTERVAL, value = "20000")
9198
public class BPlusTreeSelfTest extends GridCommonAbstractTest {
9299
/** */
93100
private static final short LONG_INNER_IO = 30000;
@@ -137,6 +144,9 @@ public class BPlusTreeSelfTest extends GridCommonAbstractTest {
137144
/** Future. */
138145
private volatile GridCompoundFuture<?, ?> asyncRunFut;
139146

147+
/** Tracking of locks holding. */
148+
private PageLockTrackerManager lockTrackerManager;
149+
140150
/**
141151
* Check that we do not keep any locks at the moment.
142152
*/
@@ -157,6 +167,10 @@ protected void assertNoLocks() {
157167
pageMem = createPageMemory();
158168

159169
reuseList = createReuseList(CACHE_ID, pageMem, 0, true);
170+
171+
lockTrackerManager = new PageLockTrackerManager(log, "testTreeManager");
172+
173+
lockTrackerManager.start();
160174
}
161175

162176
/**
@@ -208,6 +222,9 @@ protected ReuseList createReuseList(int cacheId, PageMemory pageMem, long rootId
208222
if (pageMem != null)
209223
pageMem.stop(true);
210224

225+
if (lockTrackerManager != null)
226+
lockTrackerManager.stop();
227+
211228
MAX_PER_PAGE = 0;
212229
PUT_INC = 1;
213230
RMV_INC = -1;
@@ -2701,7 +2718,7 @@ public static void checkPageId(long pageId, long pageAddr) {
27012718
*/
27022719
protected TestTree createTestTree(boolean canGetRow) throws IgniteCheckedException {
27032720
TestTree tree = new TestTree(
2704-
reuseList, canGetRow, CACHE_ID, pageMem, allocateMetaPage().pageId(), new TestPageLockListener());
2721+
reuseList, canGetRow, CACHE_ID, pageMem, allocateMetaPage().pageId(), lockTrackerManager);
27052722

27062723
assertEquals(0, tree.size());
27072724
assertEquals(0, tree.rootLevel());
@@ -2738,7 +2755,7 @@ public TestTree(
27382755
int cacheId,
27392756
PageMemory pageMem,
27402757
long metaPageId,
2741-
TestPageLockListener lsnr
2758+
PageLockTrackerManager lockTrackerManager
27422759
) throws IgniteCheckedException {
27432760
super(
27442761
"test",
@@ -2751,8 +2768,14 @@ public TestTree(
27512768
reuseList,
27522769
new IOVersions<>(new LongInnerIO(canGetRow)),
27532770
new IOVersions<>(new LongLeafIO()),
2754-
null,
2755-
lsnr
2771+
new FailureProcessor(new GridTestKernalContext(log)) {
2772+
@Override public boolean process(FailureContext failureCtx) {
2773+
lockTrackerManager.dumpLocksToLog();
2774+
2775+
return true;
2776+
}
2777+
},
2778+
new TestPageLockListener(lockTrackerManager.createPageLockTracker("testTree"))
27562779
);
27572780

27582781
PageIO.registerTest(latestInnerIO(), latestLeafIO());
@@ -3036,19 +3059,32 @@ private static class TestPageLockListener implements PageLockListener {
30363059
/** */
30373060
static ConcurrentMap<Object, Map<Long, Long>> writeLocks = new ConcurrentHashMap<>();
30383061

3062+
/** */
3063+
private final PageLockListener delegate;
3064+
3065+
/**
3066+
* @param delegate Real implementation of page lock listener.
3067+
*/
3068+
private TestPageLockListener(
3069+
PageLockListener delegate) {
3070+
3071+
this.delegate = delegate;
3072+
}
30393073

30403074
/** {@inheritDoc} */
30413075
@Override public void onBeforeReadLock(int cacheId, long pageId, long page) {
3076+
delegate.onBeforeReadLock(cacheId, pageId, page);
3077+
30423078
if (PRINT_LOCKS)
30433079
X.println(" onBeforeReadLock: " + U.hexLong(pageId));
3044-
//
3045-
// U.dumpStack();
30463080

30473081
assertNull(beforeReadLock.put(threadId(), pageId));
30483082
}
30493083

30503084
/** {@inheritDoc} */
30513085
@Override public void onReadLock(int cacheId, long pageId, long page, long pageAddr) {
3086+
delegate.onReadLock(cacheId, pageId, page, pageAddr);
3087+
30523088
if (PRINT_LOCKS)
30533089
X.println(" onReadLock: " + U.hexLong(pageId));
30543090

@@ -3065,6 +3101,8 @@ private static class TestPageLockListener implements PageLockListener {
30653101

30663102
/** {@inheritDoc} */
30673103
@Override public void onReadUnlock(int cacheId, long pageId, long page, long pageAddr) {
3104+
delegate.onReadUnlock(cacheId, pageId, page, pageAddr);
3105+
30683106
if (PRINT_LOCKS)
30693107
X.println(" onReadUnlock: " + U.hexLong(pageId));
30703108

@@ -3077,6 +3115,8 @@ private static class TestPageLockListener implements PageLockListener {
30773115

30783116
/** {@inheritDoc} */
30793117
@Override public void onBeforeWriteLock(int cacheId, long pageId, long page) {
3118+
delegate.onBeforeWriteLock(cacheId, pageId, page);
3119+
30803120
if (PRINT_LOCKS)
30813121
X.println(" onBeforeWriteLock: " + U.hexLong(pageId));
30823122

@@ -3085,10 +3125,10 @@ private static class TestPageLockListener implements PageLockListener {
30853125

30863126
/** {@inheritDoc} */
30873127
@Override public void onWriteLock(int cacheId, long pageId, long page, long pageAddr) {
3128+
delegate.onWriteLock(cacheId, pageId, page, pageAddr);
3129+
30883130
if (PRINT_LOCKS)
30893131
X.println(" onWriteLock: " + U.hexLong(pageId));
3090-
//
3091-
// U.dumpStack();
30923132

30933133
if (pageAddr != 0L) {
30943134
checkPageId(pageId, pageAddr);
@@ -3106,6 +3146,8 @@ private static class TestPageLockListener implements PageLockListener {
31063146

31073147
/** {@inheritDoc} */
31083148
@Override public void onWriteUnlock(int cacheId, long pageId, long page, long pageAddr) {
3149+
delegate.onWriteUnlock(cacheId, pageId, page, pageAddr);
3150+
31093151
if (PRINT_LOCKS)
31103152
X.println(" onWriteUnlock: " + U.hexLong(pageId));
31113153

‎modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/IgniteClientReconnectMassiveShutdownTest.java‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717

1818
package org.apache.ignite.spi.discovery.tcp;
1919

20+
import java.util.ArrayList;
2021
import java.util.HashMap;
22+
import java.util.List;
2123
import java.util.Random;
2224
import java.util.UUID;
2325
import java.util.concurrent.BlockingQueue;
@@ -38,6 +40,7 @@
3840
import org.apache.ignite.internal.IgniteInternalFuture;
3941
import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
4042
import org.apache.ignite.internal.util.typedef.X;
43+
import org.apache.ignite.internal.util.typedef.internal.SB;
4144
import org.apache.ignite.internal.util.typedef.internal.U;
4245
import org.apache.ignite.lang.IgniteFuture;
4346
import org.apache.ignite.testframework.GridTestUtils;
@@ -69,6 +72,7 @@ public class IgniteClientReconnectMassiveShutdownTest extends GridCommonAbstract
6972
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
7073

7174
cfg.setClientMode(clientMode);
75+
cfg.setFailureDetectionTimeout(5_000);
7276

7377
return cfg;
7478
}
@@ -293,15 +297,49 @@ private void massiveServersShutdown(final StopType stopType) throws Exception {
293297
}, 15_000));
294298
}
295299

300+
// Clean up ignite instance from static map in IgnitionEx.grids
301+
if (stopType == StopType.SIMULATE_FAIL){
302+
for (int i = 0; i < srvsToKill; i++) {
303+
grid(i).close();
304+
}
305+
}
306+
296307
awaitPartitionMapExchange();
297308

309+
List<Object> values = new ArrayList<>(GRID_CNT - srvsToKill);
310+
298311
for (int k = 0; k < 10_000; k++) {
299312
String key = String.valueOf(k);
300313

301314
Object val = cache.get(key);
302315

303316
for (int i = srvsToKill; i < GRID_CNT; i++)
304-
assertEquals(val, ignite(i).cache(DEFAULT_CACHE_NAME).get(key));
317+
values.add(ignite(i).cache(DEFAULT_CACHE_NAME).get(key));
318+
319+
for (Object val0 : values) {
320+
if (val == null && val0 == null)
321+
continue;
322+
323+
if (!val.equals(val0)) {
324+
SB sb = new SB();
325+
326+
sb.a("\nExp:").a(val);
327+
328+
sb.a("\nActual:");
329+
330+
for (int i = 0; i < values.size(); i++) {
331+
sb.a("[grid=")
332+
.a(grid(srvsToKill + i).name())
333+
.a(" val=")
334+
.a(values.get(i))
335+
.a("]\n");
336+
}
337+
338+
fail(sb.toString());
339+
}
340+
}
341+
342+
values.clear();
305343
}
306344
}
307345
finally {

0 commit comments

Comments
 (0)