Skip to content

HDDS-10307. Speed up TestOzoneManagerHAWithStoppedNodes #10658

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 4 commits into from
Jul 4, 2026
+41 −20
Merged

HDDS-10307. Speed up TestOzoneManagerHAWithStoppedNodes #10658

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
17e8810
HDDS-10307. Replace Thread.sleep with deterministic waits in OM HA tests
hevinhsu Jul 3, 2026
a0f4012
HDDS-10307. Speed up OM HA stopped-nodes tests via config hook and re…
hevinhsu Jul 3, 2026
79cfee1
address comment
hevinhsu Jul 4, 2026
c3fc0f3
Merge remote-tracking branch 'origin/master' into HDDS-10307
hevinhsu Jul 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions ...integration-test/src/test/java/org/apache/hadoop/ozone/om/AbstractOzoneManagerHATest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import java.util.Iterator;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
import java.util.function.Consumer;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.client.ReplicationFactor;
Expand Down Expand Up @@ -86,6 +87,16 @@ public abstract class AbstractOzoneManagerHATest {
private static final Duration RETRY_CACHE_DURATION = Duration.ofSeconds(30);
private static OzoneClient client;

/**
* Hook for subclasses to apply extra configuration before the cluster is built.
* Call {@link #setExtraClusterConfig} in a {@code static {}} block to ensure it runs before {@code @BeforeAll}.
*/
private static Consumer<OzoneConfiguration> extraClusterConfig = c -> { };

protected static void setExtraClusterConfig(Consumer<OzoneConfiguration> config) {
extraClusterConfig = config;
}

public MiniOzoneHAClusterImpl getCluster() {
return cluster;
}
Expand Down Expand Up @@ -127,6 +138,11 @@ public static Duration getRetryCacheDuration() {
}

protected static void initCluster(boolean followerReadEnabled) throws Exception {
initCluster(followerReadEnabled, extraClusterConfig);
}

protected static void initCluster(boolean followerReadEnabled,
Consumer<OzoneConfiguration> extraConfig) throws Exception {
conf = new OzoneConfiguration();
omServiceId = "om-service-test1";
conf.setBoolean(OZONE_ACL_ENABLED, true);
Expand Down Expand Up @@ -175,6 +191,8 @@ protected static void initCluster(boolean followerReadEnabled) throws Exception
conf.set(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, "10s");
conf.set(OZONE_KEY_DELETING_LIMIT_PER_TASK, "2");

extraConfig.accept(conf);

MiniOzoneHAClusterImpl.Builder clusterBuilder = MiniOzoneCluster.newHABuilder(conf)
.setOMServiceId(omServiceId)
.setNumOfOzoneManagers(numOfOMs);
Expand Down
15 changes: 7 additions & 8 deletions .../test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAFollowerReadWithStoppedNodes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.om;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.ozone.MiniOzoneHAClusterImpl.NODE_FAILURE_TIMEOUT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
Expand Down Expand Up @@ -52,6 +51,7 @@
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
import org.apache.log4j.Logger;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.MethodOrderer;
Expand Down Expand Up @@ -94,7 +94,7 @@ void oneOMDown() throws Exception {
changeFollowerReadInitialProxy(1);

getCluster().stopOzoneManager(1);
Thread.sleep(NODE_FAILURE_TIMEOUT * 4);
waitForLeaderToBeReady();

createVolumeTest(true);
createKeyTest(true);
Expand All @@ -109,7 +109,6 @@ void twoOMDown() throws Exception {

getCluster().stopOzoneManager(1);
getCluster().stopOzoneManager(2);
Thread.sleep(NODE_FAILURE_TIMEOUT * 4);

// Write requests will fail with OMNotLeaderException
createVolumeTest(false);
Expand Down Expand Up @@ -157,7 +156,7 @@ private void testMultipartUploadWithOneOmNodeDown() throws Exception {
// Stop one of the ozone manager, to see when the OM leader changes
// multipart upload is happening successfully or not.
getCluster().stopOzoneManager(leaderOMNodeId);
Thread.sleep(NODE_FAILURE_TIMEOUT * 4);
waitForLeaderToBeReady();

createMultipartKeyAndReadKey(ozoneBucket, keyName, uploadID);

Expand Down Expand Up @@ -220,11 +219,12 @@ void testLeaderOmProxyProviderFailoverOnConnectionFailure() throws Exception {
// On stopping the current OM Proxy, the next connection attempt should
// failover to a another OM proxy.
getCluster().stopOzoneManager(firstProxyNodeId);
Thread.sleep(OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT * 4);

// Next request to the proxy provider should result in a failover
createVolumeTest(true);
Thread.sleep(OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT);
GenericTestUtils.waitFor(
() -> !firstProxyNodeId.equals(omFailoverProxyProvider.getCurrentProxyOMNodeId()),
100, (int) (OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT * 5));

// Get the new OM Proxy NodeId
String newProxyNodeId = omFailoverProxyProvider.getCurrentProxyOMNodeId();
Expand Down Expand Up @@ -276,7 +276,6 @@ void testFollowerReadSkipsStoppedFollower() throws Exception {

String stoppedFollowerNodeId = followerOmNodeIds.get(0);
getCluster().stopOzoneManager(stoppedFollowerNodeId);
Thread.sleep(NODE_FAILURE_TIMEOUT * 4);

followerReadFailoverProxyProvider.changeInitialProxyForTest(stoppedFollowerNodeId);
objectStore.getClientProxy().listVolumes(null, null, 10);
Expand All @@ -300,7 +299,7 @@ void testIncrementalWaitTimeWithSameNodeFailover() throws Exception {
String leaderOMNodeId = omFailoverProxyProvider.getCurrentProxyOMNodeId();

getCluster().stopOzoneManager(leaderOMNodeId);
Thread.sleep(NODE_FAILURE_TIMEOUT * 4);
waitForLeaderToBeReady();
createKeyTest(true); // failover should happen to new node

long numTimesTriedToSameNode = omFailoverProxyProvider.getWaitTime()
Expand Down
26 changes: 15 additions & 11 deletions ...ion-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAWithStoppedNodes.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.om;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.ozone.MiniOzoneHAClusterImpl.NODE_FAILURE_TIMEOUT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
Expand Down Expand Up @@ -96,6 +96,10 @@ public class TestOzoneManagerHAWithStoppedNodes extends TestOzoneManagerHA {
private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(
TestOzoneManagerHAWithStoppedNodes.class);

static {
setExtraClusterConfig(c -> c.set(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, "2s"));
}

/**
* After restarting OMs we need to wait
* for a leader to be elected and ready.
Expand All @@ -122,7 +126,7 @@ void resetCluster() throws Exception {
@Test
void oneOMDown() throws Exception {
getCluster().stopOzoneManager(1);
Thread.sleep(NODE_FAILURE_TIMEOUT * 4);
waitForLeaderToBeReady();

createVolumeTest(true);
createKeyTest(true);
Expand All @@ -135,7 +139,6 @@ void oneOMDown() throws Exception {
void twoOMDown() throws Exception {
getCluster().stopOzoneManager(1);
getCluster().stopOzoneManager(2);
Thread.sleep(NODE_FAILURE_TIMEOUT * 4);

createVolumeTest(false);
createKeyTest(false);
Expand Down Expand Up @@ -175,7 +178,7 @@ private void testMultipartUploadWithOneOmNodeDown() throws Exception {
// Stop one of the ozone manager, to see when the OM leader changes
// multipart upload is happening successfully or not.
getCluster().stopOzoneManager(leaderOMNodeId);
Thread.sleep(NODE_FAILURE_TIMEOUT * 4);
waitForLeaderToBeReady();

createMultipartKeyAndReadKey(ozoneBucket, keyName, uploadID);

Expand Down Expand Up @@ -242,11 +245,12 @@ public void testOMProxyProviderFailoverOnConnectionFailure()
// On stopping the current OM Proxy, the next connection attempt should
// failover to a another OM proxy.
getCluster().stopOzoneManager(firstProxyNodeId);
Thread.sleep(OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT * 4);

// Next request to the proxy provider should result in a failover
createVolumeTest(true);
Thread.sleep(OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT);
GenericTestUtils.waitFor(
() -> !firstProxyNodeId.equals(omFailoverProxyProvider.getCurrentProxyOMNodeId()),
100, (int) (OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT * 5));

// Get the new OM Proxy NodeId
String newProxyNodeId = omFailoverProxyProvider.getCurrentProxyOMNodeId();
Expand Down Expand Up @@ -354,7 +358,7 @@ void testListParts() throws Exception {

// Stop leader OM, and then validate list parts.
stopLeaderOM();
Thread.sleep(NODE_FAILURE_TIMEOUT * 4);
waitForLeaderToBeReady();

validateListParts(ozoneBucket, keyName, uploadID, partsMap);

Expand Down Expand Up @@ -438,9 +442,9 @@ public void testKeyDeletion() throws Exception {

// Check on leader OM Count.
GenericTestUtils.waitFor(() ->
keyDeletingService.getRunCount().get() >= 2, 10000, 120000);
keyDeletingService.getRunCount().get() >= 2, 1000, 120000);
GenericTestUtils.waitFor(() ->
keyDeletingService.getDeletedKeyCount().get() == 4, 10000, 120000);
keyDeletingService.getDeletedKeyCount().get() == 4, 1000, 120000);

// Check delete table is empty or not on all OMs.
getCluster().getOzoneManagersList().forEach((om) -> {
Expand All @@ -454,7 +458,7 @@ public void testKeyDeletion() throws Exception {
return false;
}
},
10000, 120000);
1000, 120000);
} catch (Exception ex) {
fail("TestOzoneManagerHAKeyDeletion failed");
}
Expand Down Expand Up @@ -599,7 +603,7 @@ void testListVolumes() throws Exception {

// Stop leader OM, and then validate list volumes for user.
stopLeaderOM();
Thread.sleep(NODE_FAILURE_TIMEOUT * 2);
waitForLeaderToBeReady();

validateVolumesList(expectedVolumes,
objectStore.listVolumesByUser(userName, prefix, ""));
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
private int waitForClusterToBeReadyTimeout = 120000; // 2 min

private static final int RATIS_RPC_TIMEOUT = 1000; // 1 second
public static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds
private static final int NODE_FAILURE_TIMEOUT = 2000; // 2 seconds

public MiniOzoneHAClusterImpl(
OzoneConfiguration conf,
Expand Down
Loading