Skip to content

Commit c544ea3

Browse files
poorbarcodepoorbarcode
authored
[fix][test] Fix flaky test deleteNamespaceGracefully (apache#18220)
1 parent 15be2da commit c544ea3

18 files changed

+85
-334
lines changed

‎pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ public void resetClusters() throws Exception {
164164
pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true);
165165
for (String tenant : admin.tenants().getTenants()) {
166166
for (String namespace : admin.namespaces().getNamespaces(tenant)) {
167-
deleteNamespaceGraceFullyByMultiPulsars(namespace, true, admin, pulsar,
167+
deleteNamespaceWithRetry(namespace, true, admin, pulsar,
168168
mockPulsarSetup.getPulsar());
169169
}
170170
admin.tenants().deleteTenant(tenant, true);
@@ -1515,7 +1515,7 @@ public void testDeleteTenant() throws Exception {
15151515
assertFalse(admin.topics().getList(namespace).isEmpty());
15161516

15171517
try {
1518-
deleteNamespaceGraceFully(namespace, false);
1518+
admin.namespaces().deleteNamespace(namespace, false);
15191519
fail("should have failed due to namespace not empty");
15201520
} catch (PulsarAdminException e) {
15211521
// Expected: cannot delete non-empty tenant
@@ -1526,7 +1526,7 @@ public void testDeleteTenant() throws Exception {
15261526
assertTrue(admin.topics().getList(namespace).isEmpty());
15271527

15281528
// delete namespace
1529-
deleteNamespaceGraceFully(namespace, false);
1529+
deleteNamespaceWithRetry(namespace, false);
15301530
assertFalse(admin.namespaces().getNamespaces(tenant).contains(namespace));
15311531
assertTrue(admin.namespaces().getNamespaces(tenant).isEmpty());
15321532

@@ -1607,7 +1607,7 @@ public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception {
16071607
assertFalse(admin.topics().getList(namespace).isEmpty());
16081608

16091609
try {
1610-
deleteNamespaceGraceFully(namespace, false);
1610+
admin.namespaces().deleteNamespace(namespace, false);
16111611
fail("should have failed due to namespace not empty");
16121612
} catch (PulsarAdminException e) {
16131613
// Expected: cannot delete non-empty tenant
@@ -1618,7 +1618,7 @@ public void testDeleteNamespace(NamespaceAttr namespaceAttr) throws Exception {
16181618
assertTrue(admin.topics().getList(namespace).isEmpty());
16191619

16201620
// delete namespace
1621-
deleteNamespaceGraceFully(namespace, false);
1621+
deleteNamespaceWithRetry(namespace, false);
16221622
assertFalse(admin.namespaces().getNamespaces(tenant).contains(namespace));
16231623
assertTrue(admin.namespaces().getNamespaces(tenant).isEmpty());
16241624

@@ -1667,7 +1667,7 @@ public void testDeleteNamespaceWithTopicPolicies() throws Exception {
16671667
});
16681668
producer.close();
16691669
admin.topics().delete(topic);
1670-
deleteNamespaceGraceFully(namespace, false);
1670+
deleteNamespaceWithRetry(namespace, false);
16711671
Awaitility.await().untilAsserted(() -> {
16721672
assertTrue(admin.namespaces().getNamespaces(tenant).isEmpty());
16731673
});
@@ -1880,7 +1880,7 @@ public void testForceDeleteNamespace() throws Exception {
18801880
final String topic = "persistent://" + namespaceName + "/test" + UUID.randomUUID();
18811881
pulsarClient.newProducer(Schema.DOUBLE).topic(topic).create().close();
18821882
Awaitility.await().untilAsserted(() -> assertNotNull(admin.schemas().getSchemaInfo(topic)));
1883-
deleteNamespaceGraceFully(namespaceName, true);
1883+
deleteNamespaceWithRetry(namespaceName, true);
18841884
try {
18851885
admin.schemas().getSchemaInfo(topic);
18861886
Assert.fail("fail");

‎pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ public void resetClusters() throws Exception {
202202
pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true);
203203
for (String tenant : admin.tenants().getTenants()) {
204204
for (String namespace : admin.namespaces().getNamespaces(tenant)) {
205-
deleteNamespaceGraceFully(namespace, true);
205+
deleteNamespaceWithRetry(namespace, true);
206206
}
207207
admin.tenants().deleteTenant(tenant, true);
208208
}
@@ -297,7 +297,7 @@ public void clusters() throws Exception {
297297
Awaitility.await()
298298
.untilAsserted(() -> assertEquals(admin.clusters().getClusters(), List.of("test")));
299299

300-
deleteNamespaceGraceFully("prop-xyz/ns1", false);
300+
deleteNamespaceWithRetry("prop-xyz/ns1", false);
301301
admin.clusters().deleteCluster("test");
302302
assertEquals(admin.clusters().getClusters(), new ArrayList<>());
303303

@@ -526,7 +526,7 @@ public void brokers() throws Exception {
526526
String.format("%s:%d", parts[0], pulsar.getListenPortHTTPS().get()));
527527
Assert.assertEquals(nsMap2.size(), 2);
528528

529-
deleteNamespaceGraceFully("prop-xyz/ns1", false);
529+
deleteNamespaceWithRetry("prop-xyz/ns1", false);
530530
admin.clusters().deleteCluster("test");
531531
assertEquals(admin.clusters().getClusters(), new ArrayList<>());
532532
}
@@ -757,7 +757,7 @@ public void properties() throws Exception {
757757
assertEquals(e.getStatusCode(), 409);
758758
assertEquals(e.getMessage(), "The tenant still has active namespaces");
759759
}
760-
deleteNamespaceGraceFully("prop-xyz/ns1", false);
760+
deleteNamespaceWithRetry("prop-xyz/ns1", false);
761761
admin.tenants().deleteTenant("prop-xyz");
762762
assertEquals(admin.tenants().getTenants(), new ArrayList<>());
763763

@@ -786,7 +786,7 @@ public void namespaces() throws Exception {
786786
assertEquals(admin.namespaces().getPolicies("prop-xyz/ns3").bundles.getNumBundles(), 4);
787787
assertEquals(admin.namespaces().getPolicies("prop-xyz/ns3").bundles.getBoundaries().size(), 5);
788788

789-
deleteNamespaceGraceFully("prop-xyz/ns3", false);
789+
deleteNamespaceWithRetry("prop-xyz/ns3", false);
790790

791791
try {
792792
admin.namespaces().createNamespace("non-existing/ns1");
@@ -860,7 +860,7 @@ public void namespaces() throws Exception {
860860
}
861861
assertTrue(i < 10);
862862

863-
deleteNamespaceGraceFully("prop-xyz/ns1", false);
863+
deleteNamespaceWithRetry("prop-xyz/ns1", false);
864864
assertEquals(admin.namespaces().getNamespaces("prop-xyz"), List.of("prop-xyz/ns2"));
865865

866866
try {
@@ -1289,7 +1289,7 @@ public void testGetPartitionedStatsInternal() throws Exception {
12891289

12901290
@Test(dataProvider = "numBundles")
12911291
public void testDeleteNamespaceBundle(Integer numBundles) throws Exception {
1292-
deleteNamespaceGraceFully("prop-xyz/ns1", false);
1292+
deleteNamespaceWithRetry("prop-xyz/ns1", false);
12931293
admin.namespaces().createNamespace("prop-xyz/ns1-bundles", numBundles);
12941294
admin.namespaces().setNamespaceReplicationClusters("prop-xyz/ns1-bundles", Set.of("test"));
12951295

@@ -1301,7 +1301,7 @@ public void testDeleteNamespaceBundle(Integer numBundles) throws Exception {
13011301

13021302
assertEquals(admin.namespaces().getTopics("prop-xyz/ns1-bundles"), new ArrayList<>());
13031303

1304-
deleteNamespaceGraceFully("prop-xyz/ns1-bundles", false);
1304+
deleteNamespaceWithRetry("prop-xyz/ns1-bundles", false);
13051305
assertEquals(admin.namespaces().getNamespaces("prop-xyz", "test"), new ArrayList<>());
13061306
}
13071307

@@ -1388,14 +1388,14 @@ public void testDeleteNamespaceForcefully() throws Exception {
13881388
assertFalse(admin.topics().getList(namespace).isEmpty());
13891389

13901390
try {
1391-
deleteNamespaceGraceFully(namespace, false);
1391+
admin.namespaces().deleteNamespace(namespace, false);
13921392
fail("should have failed due to namespace not empty");
13931393
} catch (PulsarAdminException e) {
13941394
// Expected: cannot delete non-empty tenant
13951395
}
13961396

13971397
// delete namespace forcefully
1398-
deleteNamespaceGraceFully(namespace, true);
1398+
deleteNamespaceWithRetry(namespace, true);
13991399
assertFalse(admin.namespaces().getNamespaces(tenant).contains(namespace));
14001400
assertTrue(admin.namespaces().getNamespaces(tenant).isEmpty());
14011401

@@ -2287,7 +2287,7 @@ public void testBackwardCompatibility() throws Exception {
22872287
assertEquals(result.someNewIntField, 0);
22882288
assertNull(result.someNewString);
22892289

2290-
deleteNamespaceGraceFully("prop-xyz/ns1", false);
2290+
deleteNamespaceWithRetry("prop-xyz/ns1", false);
22912291
admin.tenants().deleteTenant("prop-xyz");
22922292
assertEquals(admin.tenants().getTenants(), new ArrayList<>());
22932293
}
@@ -3202,9 +3202,9 @@ public void testSubscriptionExpiry() throws Exception {
32023202
admin.topics().delete(topic1);
32033203
admin.topics().delete(topic2);
32043204
admin.topics().delete(topic3);
3205-
deleteNamespaceGraceFully(namespace1, false);
3206-
deleteNamespaceGraceFully(namespace2, false);
3207-
deleteNamespaceGraceFully(namespace3, false);
3205+
deleteNamespaceWithRetry(namespace1, false);
3206+
deleteNamespaceWithRetry(namespace2, false);
3207+
deleteNamespaceWithRetry(namespace3, false);
32083208
}
32093209

32103210
@Test
@@ -3217,11 +3217,11 @@ public void testCreateAndDeleteNamespaceWithBundles() throws Exception {
32173217
String ns = BrokerTestUtil.newUniqueName("prop-xyz/ns");
32183218

32193219
admin.namespaces().createNamespace(ns, 24);
3220-
deleteNamespaceGraceFully(ns, false);
3220+
deleteNamespaceWithRetry(ns, false);
32213221

32223222
// Re-create and re-delete
32233223
admin.namespaces().createNamespace(ns, 32);
3224-
deleteNamespaceGraceFully(ns, false);
3224+
deleteNamespaceWithRetry(ns, false);
32253225
}
32263226

32273227
@Test

‎pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -459,7 +459,7 @@ public void testDeleteNamespace() throws Exception {
459459
admin.topics().delete("tenant1/ns1/foobar", true);
460460

461461
log.info("Deleting namespace");
462-
deleteNamespaceGraceFully("tenant1/ns1", false, admin);
462+
deleteNamespaceWithRetry("tenant1/ns1", false, admin);
463463
}
464464
}
465465

‎pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminRestTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ protected void setup() throws Exception {
117117
protected void cleanup() throws Exception {
118118
// cleanup.
119119
admin.topics().delete(topicName);
120-
deleteNamespaceGraceFully(namespaceName, false);
120+
deleteNamespaceWithRetry(namespaceName, false);
121121
admin.tenants().deleteTenant(tenantName);
122122
admin.clusters().deleteCluster(clusterName);
123123
// super cleanup.

‎pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2017,7 +2017,7 @@ private void cleanupNamespaceByNsCollection(Collection<String> namespaces)
20172017
if (StringUtils.isEmpty(ns)){
20182018
continue;
20192019
}
2020-
deleteNamespaceGraceFully(ns, true);
2020+
deleteNamespaceWithRetry(ns, true);
20212021
}
20222022
pulsar.getConfiguration().setForceDeleteNamespaceAllowed(forceDeleteNamespaceAllowedOriginalValue);
20232023
}

‎pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ public void reset() throws Exception {
183183
pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true);
184184
for (String tenant : admin.tenants().getTenants()) {
185185
for (String namespace : admin.namespaces().getNamespaces(tenant)) {
186-
deleteNamespaceGraceFullyByMultiPulsars(namespace, true, admin, pulsar,
186+
deleteNamespaceWithRetry(namespace, true, admin, pulsar,
187187
mockPulsarSetup.getPulsar());
188188
}
189189
}

‎pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import java.net.URL;
3636
import java.nio.charset.StandardCharsets;
3737
import java.util.ArrayList;
38+
import java.util.Arrays;
3839
import java.util.Collection;
3940
import java.util.Date;
4041
import java.util.HashSet;
@@ -62,7 +63,6 @@
6263
import org.apache.pulsar.broker.intercept.CounterBrokerInterceptor;
6364
import org.apache.pulsar.broker.namespace.NamespaceService;
6465
import org.apache.pulsar.broker.service.BrokerTestBase;
65-
import org.apache.pulsar.broker.service.CanPausedNamespaceService;
6666
import org.apache.pulsar.broker.service.PulsarMetadataEventSynchronizer;
6767
import org.apache.pulsar.client.admin.PulsarAdmin;
6868
import org.apache.pulsar.client.admin.PulsarAdminBuilder;
@@ -87,6 +87,7 @@
8787
import org.mockito.internal.util.MockUtil;
8888
import org.slf4j.Logger;
8989
import org.slf4j.LoggerFactory;
90+
import org.testcontainers.shaded.org.awaitility.Awaitility;
9091

9192
/**
9293
* Base class for all tests that need a Pulsar instance without a ZK and BK cluster.
@@ -384,7 +385,7 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception {
384385
: createConfigurationMetadataStore()).when(pulsar).createConfigurationMetadataStore(any());
385386

386387
Supplier<NamespaceService> namespaceServiceSupplier =
387-
() -> spyWithClassAndConstructorArgs(CanPausedNamespaceService.class, pulsar);
388+
() -> spyWithClassAndConstructorArgs(NamespaceService.class, pulsar);
388389
doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider();
389390

390391
doReturn(sameThreadOrderedSafeExecutor).when(pulsar).getOrderedExecutor();
@@ -669,27 +670,44 @@ public Map<Class<?>, Collection<Class<?>>> register(Object callback, Object... c
669670
}
670671

671672
/**
672-
* see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)}
673+
* see {@link BrokerTestBase#deleteNamespaceWithRetry(String, boolean, PulsarAdmin, Collection)}
673674
*/
674-
protected void deleteNamespaceGraceFully(String ns, boolean force)
675+
protected void deleteNamespaceWithRetry(String ns, boolean force)
675676
throws Exception {
676-
BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsar);
677+
BrokerTestBase.deleteNamespaceWithRetry(ns, force, admin, pulsar);
677678
}
678679

679680
/**
680-
* see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)}
681+
* see {@link BrokerTestBase#deleteNamespaceWithRetry(String, boolean, PulsarAdmin, Collection)}
681682
*/
682-
protected void deleteNamespaceGraceFully(String ns, boolean force, PulsarAdmin admin)
683+
protected void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmin admin)
683684
throws Exception {
684-
BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsar);
685+
BrokerTestBase.deleteNamespaceWithRetry(ns, force, admin, pulsar);
685686
}
686687

687688
/**
688-
* see see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)}
689+
* see {@link MockedPulsarServiceBaseTest#deleteNamespaceWithRetry(String, boolean, PulsarAdmin, Collection)}
689690
*/
690-
protected void deleteNamespaceGraceFullyByMultiPulsars(String ns, boolean force, PulsarAdmin admin,
691-
PulsarService...pulsars) throws Exception {
692-
BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsars);
691+
public static void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmin admin, PulsarService...pulsars)
692+
throws Exception {
693+
deleteNamespaceWithRetry(ns, force, admin, Arrays.asList(pulsars));
694+
}
695+
696+
/**
697+
* 1. Pause system "__change_event" topic creates.
698+
* 2. Do delete namespace with retry because maybe fail by race-condition with create topics.
699+
*/
700+
public static void deleteNamespaceWithRetry(String ns, boolean force, PulsarAdmin admin,
701+
Collection<PulsarService> pulsars) throws Exception {
702+
Awaitility.await().atMost(5, TimeUnit.SECONDS).until(() -> {
703+
try {
704+
// Maybe fail by race-condition with create topics, just retry.
705+
admin.namespaces().deleteNamespace(ns, force);
706+
return true;
707+
} catch (Exception ex) {
708+
return false;
709+
}
710+
});
693711
}
694712

695713
private static final Logger log = LoggerFactory.getLogger(MockedPulsarServiceBaseTest.class);

‎pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/NamespaceOwnershipListenerTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ public void unLoad(NamespaceBundle bundle) {
9999
Assert.assertTrue(onLoad.get());
100100
Assert.assertTrue(unLoad.get());
101101
admin.topics().delete(topic);
102-
deleteNamespaceGraceFully(namespace, false);
102+
deleteNamespaceWithRetry(namespace, false);
103103
}
104104

105105
@Test
@@ -120,7 +120,7 @@ public void testGetAllPartitions() throws Exception {
120120
}
121121

122122
admin.topics().deletePartitionedTopic(topicName);
123-
deleteNamespaceGraceFully(namespace, false);
123+
deleteNamespaceWithRetry(namespace, false);
124124
}
125125

126126
@Test
@@ -170,6 +170,6 @@ public boolean test(NamespaceBundle namespaceBundle) {
170170
Assert.assertEquals(onLoad.get(), 1);
171171
Assert.assertEquals(unLoad.get(), 1);
172172
admin.topics().delete(topic);
173-
deleteNamespaceGraceFully(namespace, false);
173+
deleteNamespaceWithRetry(namespace, false);
174174
}
175175
}

‎pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@
4545
import org.apache.pulsar.broker.ServiceConfiguration;
4646
import org.apache.pulsar.broker.auth.SameThreadOrderedSafeExecutor;
4747
import org.apache.pulsar.broker.intercept.CounterBrokerInterceptor;
48-
import org.apache.pulsar.broker.service.CanPausedNamespaceService;
4948
import org.apache.pulsar.client.admin.PulsarAdmin;
5049
import org.apache.pulsar.client.api.PulsarClient;
5150
import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended;
@@ -133,7 +132,7 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception {
133132
doReturn(new ZKMetadataStore(mockZooKeeperSession)).when(pulsar).createLocalMetadataStore(null);
134133
doReturn(new ZKMetadataStore(mockZooKeeperSession)).when(pulsar).createConfigurationMetadataStore(null);
135134
Supplier<NamespaceService> namespaceServiceSupplier = () -> spyWithClassAndConstructorArgs(
136-
CanPausedNamespaceService.class, pulsar);
135+
NamespaceService.class, pulsar);
137136
doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider();
138137

139138
SameThreadOrderedSafeExecutor executor = new SameThreadOrderedSafeExecutor();

‎pulsar-broker/src/test/java/org/apache/pulsar/broker/resourcegroup/ResourceGroupConfigListenerTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ public void testResourceGroupAttachToNamespace() throws Exception {
197197
assertNull(pulsar.getResourceGroupServiceManager()
198198
.getNamespaceResourceGroup(NamespaceName.get(namespaceName))));
199199

200-
deleteNamespaceGraceFully(namespaceName, false);
200+
deleteNamespaceWithRetry(namespaceName, false);
201201
deleteResourceGroup(rgName);
202202
}
203203

0 commit comments

Comments
 (0)