From 16f38a4a2fb3f5ad5aa057542ae54794c01b1ced Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Fri, 31 Jul 2026 21:30:06 +0530 Subject: [PATCH 1/5] HDDS-16057. OM bootstrap fails when checkpoint install is rejected during BOOTSTRAPPING. --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 114 ++++++++++++++++++ .../apache/hadoop/ozone/om/OzoneManager.java | 6 +- 2 files changed, 119 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 1925b6a204a..2c007aeb601 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -28,6 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.io.File; import java.io.IOException; @@ -51,12 +52,14 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.FaultInjector; +import org.apache.hadoop.hdds.utils.RDBSnapshotProvider; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.hdds.utils.db.DBCheckpoint; import org.apache.hadoop.hdds.utils.db.RDBCheckpointUtils; import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -72,6 +75,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServerConfig; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.GenericTestUtils.LogCapturer; @@ -93,8 +97,13 @@ */ public class TestOMRatisSnapshots { private static final String OM_SERVICE_ID = "om-service-test1"; + private static final String BOOTSTRAP_OM_SERVICE_ID = "om-service-bootstrap"; private static final int NUM_OF_OMS = 3; + private static final int BOOTSTRAP_LOG_PURGE_GAP = 5; + private static final long BOOTSTRAP_TARGET_LOG_INDEX = 200; + private static final int BOOTSTRAP_INSTALL_START_DEADLINE_MS = 30_000; + private MiniOzoneHAClusterImpl cluster = null; private ObjectStore objectStore; private OzoneConfiguration conf; @@ -597,6 +606,111 @@ public void testInstallSnapshotFromLeaderFailedDownloadCleanupSucceeds() followerOM.getOmSnapshotProvider().setInjector(null); } + /** + * Regression test for bootstrap when leader logs are purged: checkpoint install + * must proceed during {@code BOOTSTRAPPING} with the default v2 checkpoint API. + * Stops once download starts so the test targets the BOOTSTRAPPING guard only. + */ + @Test + public void testBootstrapInstallSnapshotDuringBootstrapping() throws Exception { + IOUtils.closeQuietly(client); + if (cluster != null) { + cluster.shutdown(); + } + + OzoneConfiguration bootstrapConf = new OzoneConfiguration(); + bootstrapConf.setInt(OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, 5); + bootstrapConf.setInt(OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_GAP, BOOTSTRAP_LOG_PURGE_GAP); + bootstrapConf.setLong(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY, + SNAPSHOT_THRESHOLD); + bootstrapConf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_SIZE_KEY, 16, + StorageUnit.KB); + bootstrapConf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY, + 16, StorageUnit.KB); + + OzoneManagerRatisServerConfig omRatisConf = + bootstrapConf.getObject(OzoneManagerRatisServerConfig.class); + omRatisConf.setLogAppenderWaitTimeMin(10); + bootstrapConf.setFromObject(omRatisConf); + + cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(bootstrapConf) + .setOMServiceId(BOOTSTRAP_OM_SERVICE_ID) + .setNumOfOzoneManagers(2) + .setNumDatanodes(1) + .build(); + cluster.waitForClusterToBeReady(); + + client = OzoneClientFactory.getRpcClient(BOOTSTRAP_OM_SERVICE_ID, bootstrapConf); + objectStore = client.getObjectStore(); + String bootstrapVolume = uniqueObjectName("volume"); + String bootstrapBucket = uniqueObjectName("bucket"); + objectStore.createVolume(bootstrapVolume); + OzoneVolume volume = objectStore.getVolume(bootstrapVolume); + volume.createBucket(bootstrapBucket, + BucketArgs.newBuilder().setBucketLayout(TEST_BUCKET_LAYOUT).build()); + ozoneBucket = volume.getBucket(bootstrapBucket); + + OzoneManager leader = cluster.getOMLeader(); + writeKeysToIncreaseLogIndex(leader.getOmRatisServer(), BOOTSTRAP_TARGET_LOG_INDEX); + assertThat(leader.getRatisSnapshotIndex()) + .as("leader should have purged early logs") + .isGreaterThan((long) BOOTSTRAP_LOG_PURGE_GAP); + + LogCapturer omLog = LogCapturer.captureLogs(OzoneManager.class); + LogCapturer stateMachineLog = + LogCapturer.captureLogs(OzoneManagerStateMachine.class); + LogCapturer snapshotProviderLog = + LogCapturer.captureLogs(RDBSnapshotProvider.class); + String newNodeId = "omNode-bootstrap-ratis-snapshots"; + ExecutorService executor = Executors.newSingleThreadExecutor(); + Future bootstrapFuture = executor.submit(() -> { + try { + cluster.bootstrapOzoneManager(newNodeId); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + try { + waitForBootstrapCheckpointInstallToStart(omLog, snapshotProviderLog); + } finally { + bootstrapFuture.cancel(true); + executor.shutdownNow(); + if (cluster != null) { + cluster.shutdown(); + } + } + + assertThat(stateMachineLog.getOutput()) + .as("Ratis should notify the bootstrapping OM to install a checkpoint") + .contains("Received install snapshot notification from OM leader"); + assertThat(omLog.getOutput()) + .as("checkpoint install must not be aborted during BOOTSTRAPPING") + .doesNotContain("Abort install snapshot from Leader"); + assertThat(snapshotProviderLog.getOutput()) + .as("checkpoint download should start after install is accepted") + .contains("Prepare to download the snapshot from leader OM"); + } + + private void waitForBootstrapCheckpointInstallToStart( + LogCapturer omLog, + LogCapturer snapshotProviderLog) + throws InterruptedException, TimeoutException { + try { + GenericTestUtils.waitFor(() -> { + if (omLog.getOutput().contains("Abort install snapshot from Leader")) { + fail("Checkpoint install was aborted during BOOTSTRAPPING."); + } + return snapshotProviderLog.getOutput() + .contains("Prepare to download the snapshot from leader OM"); + }, 200, BOOTSTRAP_INSTALL_START_DEADLINE_MS); + } catch (TimeoutException e) { + fail("Checkpoint download did not start within " + BOOTSTRAP_INSTALL_START_DEADLINE_MS + + "ms. OzoneManager log: " + omLog.getOutput() + + ", RDBSnapshotProvider log: " + snapshotProviderLog.getOutput()); + } + } + /** * Moves all contents from the checkpoint location into the omDbDir. * This reorganizes the checkpoint structure so that all checkpoint files diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 2db20c59f31..198b54ba86a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -4201,7 +4201,7 @@ public List getAcl(OzoneObj obj) throws IOException { * @throws IOException if download or cleanup fails */ public synchronized TermIndex installSnapshotFromLeader(String leaderId) throws IOException { - if (!isRunning() || testInstallSnapshot) { + if (!isRunningOrBootstrapping() || testInstallSnapshot) { LOG.warn("OzoneManager is not in running state, state {}. Abort install snapshot from Leader.", omState); return null; @@ -4249,6 +4249,10 @@ public synchronized TermIndex installSnapshotFromLeader(String leaderId) throws return termIndex; } + private boolean isRunningOrBootstrapping() { + return omState == State.RUNNING || omState == State.BOOTSTRAPPING; + } + private void cleanupCheckpoint(DBCheckpoint omDBCheckpoint) throws IOException { if (omDBCheckpoint != null) { try { From 22f07bcc60861ac77c7cc4109ed37661f7a0057f Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Sun, 2 Aug 2026 14:44:48 +0800 Subject: [PATCH 2/5] HDDS-16057. Address review: complete bootstrap test and clarify log message Wait for bootstrapFuture to finish and assert the new OM joined the Ratis peer list. Update installSnapshotFromLeader abort log per review. Co-authored-by: Cursor Change-Id: I9581d000c2868cfb843263d775a6d2910290ca6d --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 19 +++++++++++++++++-- .../apache/hadoop/ozone/om/OzoneManager.java | 4 ++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 2c007aeb601..0696f53d170 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -103,6 +103,7 @@ public class TestOMRatisSnapshots { private static final int BOOTSTRAP_LOG_PURGE_GAP = 5; private static final long BOOTSTRAP_TARGET_LOG_INDEX = 200; private static final int BOOTSTRAP_INSTALL_START_DEADLINE_MS = 30_000; + private static final int BOOTSTRAP_COMPLETION_DEADLINE_MS = 60_000; private MiniOzoneHAClusterImpl cluster = null; private ObjectStore objectStore; @@ -608,8 +609,8 @@ public void testInstallSnapshotFromLeaderFailedDownloadCleanupSucceeds() /** * Regression test for bootstrap when leader logs are purged: checkpoint install - * must proceed during {@code BOOTSTRAPPING} with the default v2 checkpoint API. - * Stops once download starts so the test targets the BOOTSTRAPPING guard only. + * must proceed during {@code BOOTSTRAPPING} with the default v2 checkpoint API, + * and bootstrap must complete so the new OM joins the Ratis group. */ @Test public void testBootstrapInstallSnapshotDuringBootstrapping() throws Exception { @@ -673,6 +674,8 @@ public void testBootstrapInstallSnapshotDuringBootstrapping() throws Exception { try { waitForBootstrapCheckpointInstallToStart(omLog, snapshotProviderLog); + bootstrapFuture.get(BOOTSTRAP_COMPLETION_DEADLINE_MS, TimeUnit.MILLISECONDS); + assertBootstrapOmJoinedRatisGroup(newNodeId); } finally { bootstrapFuture.cancel(true); executor.shutdownNow(); @@ -692,6 +695,18 @@ public void testBootstrapInstallSnapshotDuringBootstrapping() throws Exception { .contains("Prepare to download the snapshot from leader OM"); } + private void assertBootstrapOmJoinedRatisGroup(String newNodeId) { + OzoneManager newOm = cluster.getOzoneManager(newNodeId); + assertNotNull(newOm, "Bootstrapped OM should be registered on the cluster"); + for (OzoneManager om : cluster.getOzoneManagersList()) { + assertTrue(om.doesPeerExist(newNodeId), + "New OM node " + newNodeId + " not present in peer list of OM " + om.getOMNodeId()); + assertTrue(om.getOmRatisServer().doesPeerExist(newNodeId), + "New OM node " + newNodeId + " not present in Ratis peer list of OM " + + om.getOMNodeId()); + } + } + private void waitForBootstrapCheckpointInstallToStart( LogCapturer omLog, LogCapturer snapshotProviderLog) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 198b54ba86a..e15231dcfc2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -4202,8 +4202,8 @@ public List getAcl(OzoneObj obj) throws IOException { */ public synchronized TermIndex installSnapshotFromLeader(String leaderId) throws IOException { if (!isRunningOrBootstrapping() || testInstallSnapshot) { - LOG.warn("OzoneManager is not in running state, state {}. Abort install snapshot from Leader.", - omState); + LOG.warn("OzoneManager is not in running state nor bootstrapping, state {}. " + + "Abort install snapshot from Leader.", omState); return null; } From e2f7d3dcc3385aa0ee3bb10579cf659c392303de Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Sun, 2 Aug 2026 14:46:01 +0800 Subject: [PATCH 3/5] HDDS-16057. Assert checkpoint install finished in bootstrap test Co-authored-by: Cursor Change-Id: I1dcf1ba282663bf6c1e1806fa49de066d4809f70 --- .../java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 0696f53d170..5ea51b1bf35 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -690,6 +690,9 @@ public void testBootstrapInstallSnapshotDuringBootstrapping() throws Exception { assertThat(omLog.getOutput()) .as("checkpoint install must not be aborted during BOOTSTRAPPING") .doesNotContain("Abort install snapshot from Leader"); + assertThat(omLog.getOutput()) + .as("checkpoint installation should finish") + .contains("Install Checkpoint is finished"); assertThat(snapshotProviderLog.getOutput()) .as("checkpoint download should start after install is accepted") .contains("Prepare to download the snapshot from leader OM"); From f47d0597831866a18cf0da566b066cbb4de659cf Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Mon, 3 Aug 2026 01:18:35 +0530 Subject: [PATCH 4/5] fix test --- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 7 +++++-- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 16 +++++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 5ea51b1bf35..66f8d0659b3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -609,8 +609,8 @@ public void testInstallSnapshotFromLeaderFailedDownloadCleanupSucceeds() /** * Regression test for bootstrap when leader logs are purged: checkpoint install - * must proceed during {@code BOOTSTRAPPING} with the default v2 checkpoint API, - * and bootstrap must complete so the new OM joins the Ratis group. + * must proceed during {@code BOOTSTRAPPING} with the default v2 checkpoint API + * and complete successfully. */ @Test public void testBootstrapInstallSnapshotDuringBootstrapping() throws Exception { @@ -696,6 +696,9 @@ public void testBootstrapInstallSnapshotDuringBootstrapping() throws Exception { assertThat(snapshotProviderLog.getOutput()) .as("checkpoint download should start after install is accepted") .contains("Prepare to download the snapshot from leader OM"); + assertThat(snapshotProviderLog.getOutput()) + .as("checkpoint tarball should be assembled on the bootstrapping OM") + .contains("DB snapshot transfer is complete."); } private void assertBootstrapOmJoinedRatisGroup(String newNodeId) { diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java index 507777d8bfe..8df0f587c60 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java @@ -881,10 +881,24 @@ private OzoneConfiguration addNewOMToConfig(String omServiceId, /** * Update the configurations of the given list of OMs. + * Merges {@code newConf} with each OM's existing node-local storage paths so + * bootstrap peer updates do not clobber per-node {@code ozone.metadata.dirs}. */ private void updateOMConfigs(OzoneConfiguration newConf) { for (OzoneManager om : omhaService.getActiveServices()) { - om.setConfiguration(newConf); + OzoneConfiguration merged = new OzoneConfiguration(newConf); + OzoneConfiguration current = om.getConfiguration(); + copyConfigIfSet(current, merged, OZONE_METADATA_DIRS); + copyConfigIfSet(current, merged, OMConfigKeys.OZONE_OM_DB_DIRS); + om.setConfiguration(merged); + } + } + + private static void copyConfigIfSet(OzoneConfiguration from, + OzoneConfiguration to, String key) { + String value = from.get(key); + if (StringUtils.isNotEmpty(value)) { + to.set(key, value); } } From 00e4cba0f9f7046951a397fa0d71b233770f71a9 Mon Sep 17 00:00:00 2001 From: sadanand48 Date: Mon, 3 Aug 2026 01:21:00 +0530 Subject: [PATCH 5/5] address copilot comment --- .../java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index 66f8d0659b3..7de95f58bc2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -678,6 +678,9 @@ public void testBootstrapInstallSnapshotDuringBootstrapping() throws Exception { assertBootstrapOmJoinedRatisGroup(newNodeId); } finally { bootstrapFuture.cancel(true); + omLog.stopCapturing(); + stateMachineLog.stopCapturing(); + snapshotProviderLog.stopCapturing(); executor.shutdownNow(); if (cluster != null) { cluster.shutdown();