Skip to content

HDDS-16196. Bump Apache Ratis to 3.3.0 #11032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
amaliujia wants to merge 5 commits into
base: master
Choose a base branch
from
Open

HDDS-16196. Bump Apache Ratis to 3.3.0 #11032

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
106ce59
bump Ratis to 3.3.0
amaliujia Aug 17, 2026
08b0554
Fix TestContainerStateMachine because of RATIS-244
amaliujia Aug 17, 2026
4d094c8
Fix TestOzoneManagerHAWithStoppedNodes
amaliujia Aug 17, 2026
379de1b
fix checkstyle
amaliujia Aug 17, 2026
a2774fc
Revert TestOzoneManagerHAWithStoppedNodes.java
amaliujia Aug 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions ...rg/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
import org.apache.hadoop.ozone.container.keyvalue.impl.KeyValueStreamDataChannel;
import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.util.Time;
import org.apache.ratis.io.MD5Hash;
import org.apache.ratis.proto.RaftProtos;
import org.apache.ratis.proto.RaftProtos.LogEntryProto;
import org.apache.ratis.proto.RaftProtos.RaftPeerRole;
Expand All @@ -96,6 +97,7 @@
import org.apache.ratis.server.RaftServer;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.ratis.server.raftlog.RaftLog;
import org.apache.ratis.server.storage.FileInfo;
import org.apache.ratis.server.storage.RaftStorage;
import org.apache.ratis.statemachine.StateMachine;
import org.apache.ratis.statemachine.StateMachineStorage;
Expand All @@ -109,6 +111,7 @@
import org.apache.ratis.util.FileUtils;
import org.apache.ratis.util.JavaUtils;
import org.apache.ratis.util.LifeCycle;
import org.apache.ratis.util.MD5FileUtil;
import org.apache.ratis.util.TaskQueue;
import org.apache.ratis.util.function.CheckedConsumer;
import org.apache.ratis.util.function.CheckedSupplier;
Expand Down Expand Up @@ -434,6 +437,9 @@ public long takeSnapshot() throws IOException {
snapshotFile);
throw ioe;
}
final MD5Hash md5 = MD5FileUtil.computeAndSaveMd5ForFile(snapshotFile);
final FileInfo fileInfo = new FileInfo(snapshotFile.toPath(), md5);
storage.updateLatestSnapshot(new SingleFileSnapshotInfo(fileInfo, ti));

@amaliujia amaliujia Aug 17, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we should not do this but just update the test case? Maybe related to https://issues.apache.org/jira/browse/RATIS-244.

Without these a few lines of code I saw a mis-match snapshot count in TestContainerStateMachine.

LOG.info("{}: Finished taking a snapshot at:{} file:{} took: {} ms",
getGroupId(), ti, snapshotFile, (Time.monotonicNow() - startTime));
return ti.getIndex();
Expand Down
38 changes: 20 additions & 18 deletions ...tion-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,11 @@
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.TimeUnit;
Expand Down Expand Up @@ -58,6 +56,7 @@
import org.apache.hadoop.ozone.container.common.transport.server.ratis.RatisServerConfiguration;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import org.apache.ratis.statemachine.impl.StatemachineImplTestUtil;
import org.junit.jupiter.api.AfterEach;
Expand Down Expand Up @@ -100,6 +99,11 @@ public void setup() throws Exception {
clientConfig.setStreamBufferFlushDelay(false);
conf.setFromObject(clientConfig);

RatisServerConfiguration ratisServerConfiguration =
conf.getObject(RatisServerConfiguration.class);
ratisServerConfiguration.setNumSnapshotsRetained(5);
conf.setFromObject(ratisServerConfiguration);

// conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.toString());
cluster =
MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1)
Expand Down Expand Up @@ -191,13 +195,7 @@ public void testRatisSnapshotRetention() throws Exception {
RatisServerConfiguration ratisServerConfiguration =
conf.getObject(RatisServerConfiguration.class);

Path parentPath = getSnapshotPath(storage).getParent();
assertThat(parentPath).isNotNull();
File[] files = parentPath.toFile().listFiles();
assertThat(files).isNotNull();
int numSnapshots = files.length;
assertThat(Math.abs(ratisServerConfiguration.getNumSnapshotsRetained() - numSnapshots))
.isLessThanOrEqualTo(1);
assertSnapshotsWithinRetention(storage, ratisServerConfiguration);

// Write 10 more keys. Num Snapshots should remain the same.
for (int i = 11; i <= 20; i++) {
Expand All @@ -212,16 +210,20 @@ public void testRatisSnapshotRetention() throws Exception {
key.write(("ratis" + i).getBytes(UTF_8));
}
}
files = parentPath.toFile().listFiles();
assertThat(files).isNotNull();
numSnapshots = files.length;
assertThat(Math.abs(ratisServerConfiguration.getNumSnapshotsRetained() - numSnapshots))
.isLessThanOrEqualTo(1);
assertSnapshotsWithinRetention(storage, ratisServerConfiguration);
}

static Path getSnapshotPath(SimpleStateMachineStorage storage)
throws IOException {
return StatemachineImplTestUtil.findLatestSnapshot(storage)
.getFile().getPath();
private static void assertSnapshotsWithinRetention(
SimpleStateMachineStorage storage,
RatisServerConfiguration ratisServerConfiguration) throws Exception {
final int numSnapshotsRetained = ratisServerConfiguration.getNumSnapshotsRetained();
GenericTestUtils.waitFor(() -> {
try {
int numSnapshots = StatemachineImplTestUtil.countSnapshots(storage);
return Math.abs(numSnapshotsRetained - numSnapshots) <= 1;
} catch (IOException e) {
return false;
}
}, 1000, 30000);
}
}
6 changes: 6 additions & 0 deletions ...ation-test/src/test/java/org/apache/ratis/statemachine/impl/StatemachineImplTestUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,10 @@ static SingleFileSnapshotInfo findLatestSnapshot(
final File dir = storage.getStateMachineDir();
return SimpleStateMachineStorage.findLatestSnapshot(dir.toPath());
}

static int countSnapshots(SimpleStateMachineStorage storage)
throws IOException {
return SimpleStateMachineStorage.getSingleFileSnapshotInfos(
storage.getStateMachineDir().toPath()).size();
}
}
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@
<ratis-thirdparty.netty.version>4.1.130.Final</ratis-thirdparty.netty.version>
<ratis-thirdparty.protobuf.version>3.25.8</ratis-thirdparty.protobuf.version>
<ratis.thirdparty.version>1.0.11</ratis.thirdparty.version>
<ratis.version>3.2.1</ratis.version>
<ratis.version>3.3.0</ratis.version>
<re2j.version>1.7</re2j.version>
<reflections.version>0.10.2</reflections.version>
<reload4j.version>1.2.26</reload4j.version>
Expand Down
Loading