Skip to content

Commit 1b23023

Browse files
zoewanggzoewangg
committed
Add additional checksum algorithms
1 parent b7c7933 commit 1b23023

25 files changed

Lines changed: 785 additions & 118 deletions

File tree

‎codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/traits/HttpChecksumTrait.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,19 @@
3939
*/
4040
public class HttpChecksumTrait {
4141

42+
// TODO: run perf tests
4243
private static final Map<String, Integer> CHECKSUM_ALGORITHM_PRIORITY =
4344
ImmutableMap.<String, Integer>builder()
4445
.put(DefaultChecksumAlgorithm.CRC32C.algorithmId(), 1)
4546
.put(DefaultChecksumAlgorithm.CRC32.algorithmId(), 2)
4647
.put(DefaultChecksumAlgorithm.CRC64NVME.algorithmId(), 3)
47-
.put(DefaultChecksumAlgorithm.SHA1.algorithmId(), 4)
48-
.put(DefaultChecksumAlgorithm.SHA256.algorithmId(), 5)
48+
.put(DefaultChecksumAlgorithm.XXHASH128.algorithmId(), 4)
49+
.put(DefaultChecksumAlgorithm.XXHASH3.algorithmId(), 5)
50+
.put(DefaultChecksumAlgorithm.XXHASH64.algorithmId(), 6)
51+
.put(DefaultChecksumAlgorithm.SHA1.algorithmId(), 7)
52+
.put(DefaultChecksumAlgorithm.SHA256.algorithmId(), 8)
53+
.put(DefaultChecksumAlgorithm.SHA512.algorithmId(), 9)
54+
.put(DefaultChecksumAlgorithm.MD5.algorithmId(), 10)
4955
.build();
5056

5157
private HttpChecksumTrait() {

‎core/checksums/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@
5555
<groupId>software.amazon.awssdk.crt</groupId>
5656
<artifactId>aws-crt</artifactId>
5757
<version>${awscrt.version}</version>
58-
<scope>test</scope>
58+
<optional>true</optional>
5959
</dependency>
6060

6161
<dependency>

‎core/checksums/src/main/java/software/amazon/awssdk/checksums/DefaultChecksumAlgorithm.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,11 @@ public final class DefaultChecksumAlgorithm {
3131
public static final ChecksumAlgorithm MD5 = of("MD5");
3232
public static final ChecksumAlgorithm SHA256 = of("SHA256");
3333
public static final ChecksumAlgorithm SHA1 = of("SHA1");
34+
public static final ChecksumAlgorithm SHA512 = of("SHA512");
3435
public static final ChecksumAlgorithm CRC64NVME = of("CRC64NVME");
36+
public static final ChecksumAlgorithm XXHASH64 = of("XXHASH64");
37+
public static final ChecksumAlgorithm XXHASH3 = of("XXHASH3");
38+
public static final ChecksumAlgorithm XXHASH128 = of("XXHASH128");
3539

3640
private DefaultChecksumAlgorithm() {
3741
}

‎core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818
import java.nio.ByteBuffer;
1919
import java.util.zip.Checksum;
2020
import software.amazon.awssdk.annotations.SdkProtectedApi;
21+
import software.amazon.awssdk.checksums.internal.ChecksumProvider;
2122
import software.amazon.awssdk.checksums.internal.Crc32Checksum;
2223
import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum;
23-
import software.amazon.awssdk.checksums.internal.CrcChecksumProvider;
2424
import software.amazon.awssdk.checksums.internal.DigestAlgorithm;
2525
import software.amazon.awssdk.checksums.internal.DigestAlgorithmChecksum;
2626
import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
@@ -38,17 +38,25 @@ public interface SdkChecksum extends Checksum {
3838
static SdkChecksum forAlgorithm(ChecksumAlgorithm algorithm) {
3939
switch (algorithm.algorithmId()) {
4040
case "CRC32C":
41-
return CrcChecksumProvider.crc32cImplementation();
41+
return ChecksumProvider.crc32cImplementation();
4242
case "CRC32":
4343
return new Crc32Checksum();
4444
case "SHA1":
4545
return new DigestAlgorithmChecksum(DigestAlgorithm.SHA1);
4646
case "SHA256":
4747
return new DigestAlgorithmChecksum(DigestAlgorithm.SHA256);
48+
case "SHA512":
49+
return new DigestAlgorithmChecksum(DigestAlgorithm.SHA512);
4850
case "MD5":
4951
return new DigestAlgorithmChecksum(DigestAlgorithm.MD5);
5052
case "CRC64NVME":
5153
return new Crc64NvmeChecksum();
54+
case "XXHASH64":
55+
return ChecksumProvider.xxHash64CrtImplementation();
56+
case "XXHASH3":
57+
return ChecksumProvider.xxHash3CrtImplementation();
58+
case "XXHASH128":
59+
return ChecksumProvider.xxHash128CrtImplementation();
5260
default:
5361
throw new UnsupportedOperationException("Unsupported checksum algorithm: " + algorithm);
5462
}

core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/CrcChecksumProvider.java renamed to core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ChecksumProvider.java

Lines changed: 83 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,16 @@
1717

1818
import java.util.zip.Checksum;
1919
import software.amazon.awssdk.annotations.SdkInternalApi;
20+
import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm;
2021
import software.amazon.awssdk.checksums.SdkChecksum;
22+
import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
23+
import software.amazon.awssdk.crt.checksums.CRC32C;
24+
import software.amazon.awssdk.crt.checksums.CRC64NVME;
25+
import software.amazon.awssdk.utils.ClassLoaderHelper;
26+
import software.amazon.awssdk.utils.Lazy;
2127

2228
/**
23-
* Utility class providing implementations of CRC checksums, specifically CRC32C and CRC64NVME.
29+
* Utility class providing implementations of checksums.
2430
*
2531
* <p>Supports the following implementations for CRC32C:</p>
2632
* <ul>
@@ -29,23 +35,38 @@
2935
* <li>SDK-based CRC32C (fallback)</li>
3036
* </ul>
3137
*
32-
* <p>Only supports CRT-based implementation for CRC64NVME (using AWS CRT library).</p>
38+
* <p>Supports CRT-based implementations for CRC64NVME and XXHASH algorithms (using AWS CRT library).</p>
3339
*
3440
* <p>For internal use only ({@link SdkInternalApi}).</p>
3541
*/
3642
@SdkInternalApi
37-
public final class CrcChecksumProvider {
38-
43+
public final class ChecksumProvider {
3944

4045
// Class paths for different CRC32C implementations
4146
private static final String CRT_CRC32C_CLASS_PATH = "software.amazon.awssdk.crt.checksums.CRC32C";
4247
private static final String JAVA_CRC32C_CLASS_PATH = "java.util.zip.CRC32C";
4348
private static final ConstructorCache CONSTRUCTOR_CACHE = new ConstructorCache();
4449
private static final String CRT_CRC64NVME_PATH = "software.amazon.awssdk.crt.checksums.CRC64NVME";
50+
private static final String CRT_XXHASH_PATH = "software.amazon.awssdk.crt.checksums.XXHash";
4551
private static final String CRT_MODULE = "software.amazon.awssdk.crt:aws-crt";
4652

53+
private static Lazy<Boolean> isXxHashAvailable = checkCrtAvailability(CRT_XXHASH_PATH);
54+
private static Lazy<Boolean> isCrc64NvmeAvailable = checkCrtAvailability(CRT_CRC64NVME_PATH);
55+
private static Lazy<Boolean> isCrc32CAvailable = checkCrtAvailability(CRT_CRC32C_CLASS_PATH);
56+
4757
// Private constructor to prevent instantiation
48-
private CrcChecksumProvider() {
58+
private ChecksumProvider() {
59+
}
60+
61+
private static Lazy<Boolean> checkCrtAvailability(String fqcn) {
62+
return new Lazy<>(() -> {
63+
try {
64+
ClassLoaderHelper.loadClass(fqcn, false);
65+
} catch (ClassNotFoundException e) {
66+
return false;
67+
}
68+
return true;
69+
});
4970
}
5071

5172
/**
@@ -74,14 +95,12 @@ public static SdkChecksum crc32cImplementation() {
7495
}
7596

7697
static SdkChecksum createCrtCrc32C() {
77-
return CONSTRUCTOR_CACHE.getConstructor(CRT_CRC32C_CLASS_PATH).map(constructor -> {
78-
try {
79-
Checksum checksumInstance = (Checksum) constructor.newInstance();
80-
return new CrcCloneOnMarkChecksum(checksumInstance);
81-
} catch (ReflectiveOperationException e) {
82-
throw new IllegalStateException("Failed to instantiate " + CRT_CRC32C_CLASS_PATH, e);
83-
}
84-
}).orElse(null);
98+
if (!isCrc32CAvailable.getValue()) {
99+
return null;
100+
}
101+
102+
Checksum checksumInstance = new CRC32C();
103+
return new CrcCloneOnMarkChecksum(checksumInstance);
85104
}
86105

87106
/**
@@ -96,16 +115,57 @@ static SdkChecksum createCrtCrc32C() {
96115
* @throws RuntimeException if the `CRC64NVME` implementation is not available.
97116
*/
98117
static SdkChecksum crc64NvmeCrtImplementation() {
99-
return CONSTRUCTOR_CACHE.getConstructor(CRT_CRC64NVME_PATH).map(constructor -> {
100-
try {
101-
Checksum checksumInstance = (Checksum) constructor.newInstance();
102-
return new CrcCloneOnMarkChecksum(checksumInstance);
103-
} catch (ReflectiveOperationException e) {
104-
throw new IllegalStateException("Failed to instantiate " + CRT_CRC32C_CLASS_PATH, e);
105-
}
106-
}).orElseThrow(() -> new RuntimeException(
107-
"Could not load " + CRT_CRC64NVME_PATH + ". Add dependency on '" + CRT_MODULE
108-
+ "' module to enable CRC64NVME feature."));
118+
119+
if (!isCrc64NvmeAvailable.getValue()) {
120+
throw new RuntimeException(
121+
"Could not load " + CRT_CRC64NVME_PATH + ". Add dependency on '" + CRT_MODULE
122+
+ "' module to enable CRC64NVME feature.");
123+
}
124+
125+
return new CrcCloneOnMarkChecksum(new CRC64NVME());
126+
}
127+
128+
/**
129+
* Creates an instance of the CRT-based XXHASH64 checksum using AWS's CRT library.
130+
*
131+
* @return An {@link SdkChecksum} instance for XXHASH64.
132+
* @throws IllegalStateException if instantiation fails.
133+
* @throws RuntimeException if the CRT implementation is not available.
134+
*/
135+
public static SdkChecksum xxHash64CrtImplementation() {
136+
return crtXxHash(DefaultChecksumAlgorithm.XXHASH64);
137+
}
138+
139+
/**
140+
* Creates an instance of the CRT-based XXHASH3 checksum using AWS's CRT library.
141+
*
142+
* @return An {@link SdkChecksum} instance for XXHASH3.
143+
* @throws IllegalStateException if instantiation fails.
144+
* @throws RuntimeException if the CRT implementation is not available.
145+
*/
146+
public static SdkChecksum xxHash3CrtImplementation() {
147+
return crtXxHash(DefaultChecksumAlgorithm.XXHASH3);
148+
}
149+
150+
/**
151+
* Creates an instance of the CRT-based XXHASH128 checksum using AWS's CRT library.
152+
*
153+
* @return An {@link SdkChecksum} instance for XXHASH128.
154+
* @throws IllegalStateException if instantiation fails.
155+
* @throws RuntimeException if the CRT implementation is not available.
156+
*/
157+
public static SdkChecksum xxHash128CrtImplementation() {
158+
return crtXxHash(DefaultChecksumAlgorithm.XXHASH128);
159+
}
160+
161+
static SdkChecksum crtXxHash(ChecksumAlgorithm algorithm) {
162+
if (!isXxHashAvailable.getValue()) {
163+
throw new RuntimeException(
164+
"Could not load " + CRT_XXHASH_PATH + ". Add dependency on '" + CRT_MODULE
165+
+ "' module.");
166+
}
167+
168+
return new XxHashChecksum(algorithm);
109169
}
110170

111171
static SdkChecksum createJavaCrc32C() {
@@ -117,5 +177,4 @@ static SdkChecksum createJavaCrc32C() {
117177
}
118178
}).orElse(null);
119179
}
120-
121180
}

‎core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc64NvmeChecksum.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public final class Crc64NvmeChecksum implements SdkChecksum {
2929
private final SdkChecksum sdkChecksum;
3030

3131
public Crc64NvmeChecksum() {
32-
this.sdkChecksum = CrcChecksumProvider.crc64NvmeCrtImplementation();
32+
this.sdkChecksum = ChecksumProvider.crc64NvmeCrtImplementation();
3333
}
3434

3535
@Override

‎core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/DigestAlgorithm.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ public enum DigestAlgorithm {
2929
SHA1("SHA-1"),
3030

3131
MD5("MD5"),
32-
SHA256("SHA-256")
32+
SHA256("SHA-256"),
33+
SHA512("SHA-512")
3334
;
3435

3536
private static final Supplier<MessageDigest> CLOSED_DIGEST = () -> {

‎core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/XxHashChecksum.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.checksums.internal;
17+
18+
import software.amazon.awssdk.annotations.SdkInternalApi;
19+
import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm;
20+
import software.amazon.awssdk.checksums.SdkChecksum;
21+
import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
22+
import software.amazon.awssdk.crt.checksums.XXHash;
23+
24+
@SdkInternalApi
25+
public final class XxHashChecksum implements SdkChecksum {
26+
27+
private final XXHash xxHash;
28+
29+
public XxHashChecksum(ChecksumAlgorithm algorithm) {
30+
if (algorithm == DefaultChecksumAlgorithm.XXHASH64) {
31+
xxHash = XXHash.newXXHash64();
32+
} else if (algorithm == DefaultChecksumAlgorithm.XXHASH3) {
33+
xxHash = XXHash.newXXHash3_64();
34+
} else if (algorithm == DefaultChecksumAlgorithm.XXHASH128) {
35+
xxHash = XXHash.newXXHash3_128();
36+
} else {
37+
throw new UnsupportedOperationException("Unsupported algorithm: " + algorithm.algorithmId());
38+
}
39+
}
40+
41+
@Override
42+
public void update(byte[] b) {
43+
xxHash.update(b);
44+
}
45+
46+
@Override
47+
public byte[] getChecksumBytes() {
48+
return xxHash.digest();
49+
}
50+
51+
52+
@Override
53+
public void update(int b) {
54+
xxHash.update(b);
55+
}
56+
57+
@Override
58+
public void update(byte[] b, int off, int len) {
59+
xxHash.update(b, off, len);
60+
}
61+
62+
@Override
63+
public void reset() {
64+
throw new UnsupportedOperationException("mark and reset is not supported");
65+
}
66+
67+
@Override
68+
public void mark(int readLimit) {
69+
throw new UnsupportedOperationException("mark and reset is not supported");
70+
}
71+
72+
@Override
73+
public long getValue() {
74+
throw new UnsupportedOperationException("Use getChecksumBytes() instead.");
75+
}
76+
}

‎core/checksums/src/test/java/software/amazon/awssdk/checksums/internal/DefaultCRC32CChecksumTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public class DefaultCRC32CChecksumTest extends Crc32CChecksumTest {
2222

2323
@BeforeEach
2424
public void setUp() {
25-
sdkChecksum = CrcChecksumProvider.crc32cImplementation();
25+
sdkChecksum = ChecksumProvider.crc32cImplementation();
2626
}
2727

2828

0 commit comments

Comments
 (0)