diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index 9a3e988e27ad..4c7e304d357f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -79,6 +79,7 @@
import org.apache.hadoop.ozone.container.keyvalue.impl.KeyValueStreamDataChannel;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.util.Time;
+import org.apache.ratis.io.MD5Hash;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.proto.RaftProtos.LogEntryProto;
import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
@@ -96,6 +97,7 @@
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.RaftLog;
+import org.apache.ratis.server.storage.FileInfo;
import org.apache.ratis.server.storage.RaftStorage;
import org.apache.ratis.statemachine.StateMachine;
import org.apache.ratis.statemachine.StateMachineStorage;
@@ -109,6 +111,7 @@
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.LifeCycle;
+import org.apache.ratis.util.MD5FileUtil;
import org.apache.ratis.util.TaskQueue;
import org.apache.ratis.util.function.CheckedConsumer;
import org.apache.ratis.util.function.CheckedSupplier;
@@ -434,6 +437,9 @@ public long takeSnapshot() throws IOException {
snapshotFile);
throw ioe;
}
+ final MD5Hash md5 = MD5FileUtil.computeAndSaveMd5ForFile(snapshotFile);
+ final FileInfo fileInfo = new FileInfo(snapshotFile.toPath(), md5);
+ storage.updateLatestSnapshot(new SingleFileSnapshotInfo(fileInfo, ti));
LOG.info("{}: Finished taking a snapshot at:{} file:{} took: {} ms",
getGroupId(), ti, snapshotFile, (Time.monotonicNow() - startTime));
return ti.getIndex();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
index 9a045178a8c0..cff52d969119 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
@@ -23,13 +23,11 @@
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
-import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import java.io.File;
import java.io.IOException;
-import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -58,6 +56,7 @@
import org.apache.hadoop.ozone.container.common.transport.server.ratis.RatisServerConfiguration;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.ozone.test.GenericTestUtils;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.statemachine.impl.StatemachineImplTestUtil;
import org.junit.jupiter.api.AfterEach;
@@ -100,6 +99,11 @@ public void setup() throws Exception {
clientConfig.setStreamBufferFlushDelay(false);
conf.setFromObject(clientConfig);
+ RatisServerConfiguration ratisServerConfiguration =
+ conf.getObject(RatisServerConfiguration.class);
+ ratisServerConfiguration.setNumSnapshotsRetained(5);
+ conf.setFromObject(ratisServerConfiguration);
+
// conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.toString());
cluster =
MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1)
@@ -191,13 +195,7 @@ public void testRatisSnapshotRetention() throws Exception {
RatisServerConfiguration ratisServerConfiguration =
conf.getObject(RatisServerConfiguration.class);
- Path parentPath = getSnapshotPath(storage).getParent();
- assertThat(parentPath).isNotNull();
- File[] files = parentPath.toFile().listFiles();
- assertThat(files).isNotNull();
- int numSnapshots = files.length;
- assertThat(Math.abs(ratisServerConfiguration.getNumSnapshotsRetained() - numSnapshots))
- .isLessThanOrEqualTo(1);
+ assertSnapshotsWithinRetention(storage, ratisServerConfiguration);
// Write 10 more keys. Num Snapshots should remain the same.
for (int i = 11; i <= 20; i++) {
@@ -212,16 +210,20 @@ public void testRatisSnapshotRetention() throws Exception {
key.write(("ratis" + i).getBytes(UTF_8));
}
}
- files = parentPath.toFile().listFiles();
- assertThat(files).isNotNull();
- numSnapshots = files.length;
- assertThat(Math.abs(ratisServerConfiguration.getNumSnapshotsRetained() - numSnapshots))
- .isLessThanOrEqualTo(1);
+ assertSnapshotsWithinRetention(storage, ratisServerConfiguration);
}
- static Path getSnapshotPath(SimpleStateMachineStorage storage)
- throws IOException {
- return StatemachineImplTestUtil.findLatestSnapshot(storage)
- .getFile().getPath();
+ private static void assertSnapshotsWithinRetention(
+ SimpleStateMachineStorage storage,
+ RatisServerConfiguration ratisServerConfiguration) throws Exception {
+ final int numSnapshotsRetained = ratisServerConfiguration.getNumSnapshotsRetained();
+ GenericTestUtils.waitFor(() -> {
+ try {
+ int numSnapshots = StatemachineImplTestUtil.countSnapshots(storage);
+ return Math.abs(numSnapshotsRetained - numSnapshots) <= 1;
+ } catch (IOException e) {
+ return false;
+ }
+ }, 1000, 30000);
}
}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/ratis/statemachine/impl/StatemachineImplTestUtil.java b/hadoop-ozone/integration-test/src/test/java/org/apache/ratis/statemachine/impl/StatemachineImplTestUtil.java
index abd866608b98..66d4bf33e5c8 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/ratis/statemachine/impl/StatemachineImplTestUtil.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/ratis/statemachine/impl/StatemachineImplTestUtil.java
@@ -29,4 +29,10 @@ static SingleFileSnapshotInfo findLatestSnapshot(
final File dir = storage.getStateMachineDir();
return SimpleStateMachineStorage.findLatestSnapshot(dir.toPath());
}
+
+ static int countSnapshots(SimpleStateMachineStorage storage)
+ throws IOException {
+ return SimpleStateMachineStorage.getSingleFileSnapshotInfos(
+ storage.getStateMachineDir().toPath()).size();
+ }
}
diff --git a/pom.xml b/pom.xml
index d736d5347e46..834d05582d08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -189,7 +189,7 @@
4.1.130.Final
3.25.8
1.0.11
- 3.2.1
+ 3.3.0
1.7
0.10.2
1.2.26