Skip to content

Refactor benchmark setup, use protocol test harness, fix rpcV2 binary tests #1181

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adwsingh merged 1 commit into from
May 18, 2026
Merged

Refactor benchmark setup, use protocol test harness, fix rpcV2 binary tests #1181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 54 additions & 42 deletions benchmarks/serde-benchmarks/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
plugins {
id("smithy-java.java-conventions")
alias(libs.plugins.jmh)
alias(libs.plugins.shadow)
alias(libs.plugins.jmh)
id("software.amazon.smithy.gradle.smithy-base")
}

Expand All @@ -24,20 +24,15 @@ tasks.withType<JavaCompile>().configureEach {
}

dependencies {
// Smithy traits needed at build time (smithy-build) AND at runtime (when
// BenchmarkContext loads the model).
// Smithy traits needed at build time (smithy-build) and at runtime (when
// BenchmarkTestCases loads the model). The jmh configuration extends
// implementation, so these are available to both.
implementation(libs.smithy.model)
implementation(libs.smithy.aws.traits)
implementation(libs.smithy.protocol.traits)
implementation(libs.smithy.protocol.test.traits)
implementation(libs.smithy.utils)

jmh(libs.smithy.model)
jmh(libs.smithy.aws.traits)
jmh(libs.smithy.protocol.traits)
jmh(libs.smithy.protocol.test.traits)
jmh(libs.smithy.utils)

// The Smithy Java codegen plugin produces typed shape classes plus
// ApiOperation classes per service (see `smithy-build.json`). The
// client-core dep is required because the generated client classes
Expand All @@ -64,6 +59,9 @@ dependencies {
jmh(project(":aws:client:aws-client-awsquery"))
jmh(project(":aws:client:aws-client-restjson"))
jmh(project(":aws:client:aws-client-restxml"))

// Protocol test document for converting test case params into typed shapes.
jmh(project(":protocol-test-harness"))
}

// Smithy benchmark model files (tagged @httpRequestTests / @httpResponseTests
Expand All @@ -73,11 +71,11 @@ dependencies {
// `Model.assembler().discoverModels()`, which walks `META-INF/smithy/manifest`
// resources on the classpath.
abstract class GenerateSmithyManifest : DefaultTask() {
@get:org.gradle.api.tasks.InputDirectory
abstract val sourceDir: org.gradle.api.file.DirectoryProperty
@get:InputDirectory
abstract val sourceDir: DirectoryProperty

@get:org.gradle.api.tasks.OutputDirectory
abstract val outputDir: org.gradle.api.file.DirectoryProperty
@get:OutputDirectory
abstract val outputDir: DirectoryProperty

@org.gradle.api.tasks.TaskAction
fun run() {
Expand Down Expand Up @@ -152,47 +150,66 @@ tasks.named("compileJmhJava") {
dependsOn("smithyBuild")
}

// All JMH parameters are configured here (single source of truth).
// Per-class annotations (@Warmup, @Measurement, @Fork, etc.) are not used.
//
// Fast mode: -Pjmh.fast (1 warmup, 3 measurement, 1 fork, 5s each)
// Profilers: -Pjmh.profilers=gc,stack (comma-separated JMH profiler names)
// Filter: -Pjmh.includes=RpcV2CborSerializeBenchmark.serialize
// Test case: -Pjmh.testCaseId=rpcv2Cbor_PutItemRequest_BinaryData_S
val fast = providers.gradleProperty("jmh.fast").isPresent
jmh {
// Per-benchmark @Warmup / @Measurement / @Fork annotations on each class
// are authoritative. These extension defaults apply only when annotations
// are absent.
warmupIterations = 5
iterations = 10
benchmarkMode.addAll("sample")
timeUnit = "ns"
warmupIterations = if (fast) 1 else 5
warmup = if (fast) "5s" else "2s"
iterations = if (fast) 3 else 10
timeOnIteration = if (fast) "5s" else "5s"
fork = 1
// Select the native smithy-java JSON provider (rather than the default
// Jackson-backed provider, which has higher ServiceLoader priority).
// The system property is read once during static initialization of
// `JsonSettings`, so it must be set before the codec class loads.
jvmArgs.addAll("-Dsmithy-java.json-provider=smithy")
jvmArgs.addAll(
"-Xms1g",
"-Xmx1g",
"-XX:+UseG1GC",
"-XX:+AlwaysPreTouch",
"-Dsmithy-java.json-provider=smithy",
)
includes.addAll(
providers
.gradleProperty("jmh.includes")
.map { listOf(it) }
.orElse(emptyList()),
)
// Emit JSON output so it can be picked up by the `convertJmhResults` task.
profilers.addAll(
providers
.gradleProperty("jmh.profilers")
.map { it.split(",") }
.orElse(emptyList()),
)
providers.gradleProperty("jmh.testCaseId").orNull?.let { id ->
val prop = objects.listProperty(String::class.java)
prop.add(id)
benchmarkParameters.put("testCaseId", prop)
}
resultFormat = "json"
resultsFile = layout.buildDirectory.file("results/jmh/results.json")
}

// The me.champeau.jmh plugin auto-integrates with com.gradleup.shadow when
// both plugins are present: the `jmh` task uses a shadow-style classpath so
// duplicate META-INF/services/ entries (one per upstream module) are merged
// rather than overwritten. Without this, multiple TraitService SPI files
// collide and only one wins.
tasks.named<com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar>("shadowJar") {
// With shadow applied before jmh, jmhJar is a ShadowJar. Configure
// mergeServiceFiles() so duplicate META-INF/services/ entries from
// multiple codegen projections are concatenated rather than overwritten.
tasks.jmhJar {
mergeServiceFiles()
append("META-INF/smithy/manifest")
}

// Run the cross-language result converter. Reads the JMH JSON written by the
// `jmh` task and writes a JSON + Markdown pair conforming to the shared
// benchmark output schema.
// benchmark output schema. OS, instance type (via EC2 IMDS), and smithy-java
// version are detected at runtime.
//
// ./gradlew :benchmarks:serde-benchmarks:convertJmhResults
//
// Optional properties:
// -Pinstance=m7i.xlarge instance label written to metadata.instance
// -Pos="<descriptor>" OS label written to metadata.os
// -PoutputPrefix=<path> prefix for the output files (default: build/results/jmh/output)
tasks.register<JavaExec>("convertJmhResults") {
group = "benchmarks"
Expand All @@ -204,9 +221,6 @@ tasks.register<JavaExec>("convertJmhResults") {
val inputFile = layout.buildDirectory.file("results/jmh/results.json")
val defaultPrefix = layout.buildDirectory.file("results/jmh/output").map { it.asFile.absolutePath }
val outputPrefix = providers.gradleProperty("outputPrefix").orElse(defaultPrefix)
val instance = providers.gradleProperty("instance").orElse("unknown")
val os = providers.gradleProperty("os").orElse(System.getProperty("os.name") + " " + System.getProperty("os.version"))
val smithyJavaVersion = project.file("${project.rootDir}/VERSION").readText().trim()

inputs.file(inputFile)
outputs.files(
Expand All @@ -219,11 +233,9 @@ tasks.register<JavaExec>("convertJmhResults") {
inputFile.get().asFile.absolutePath,
"--output-prefix",
outputPrefix.get(),
"--instance",
instance.get(),
"--os",
os.get(),
"--software-version",
smithyJavaVersion,
)
}

tasks.named("jmh") {
finalizedBy("convertJmhResults")
}
45 changes: 0 additions & 45 deletions ...chmarks/src/jmh/java/software/amazon/smithy/java/benchmarks/serde/ApiOperationLookup.java
View file
Open in desktop

This file was deleted.

12 changes: 0 additions & 12 deletions ...jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0DeserializeBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,11 @@
package software.amazon.smithy.java.benchmarks.serde;

import java.nio.charset.StandardCharsets;
import java.util.concurrent.TimeUnit;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.smithy.java.aws.client.awsjson.AwsJson1Protocol;
import software.amazon.smithy.java.core.schema.ApiOperation;
Expand All @@ -30,11 +23,6 @@
* response and produce a typed output.
*/
@State(Scope.Benchmark)
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 10, time = 5, timeUnit = TimeUnit.SECONDS)
@Fork(1)
public class AwsJson1_0DeserializeBenchmark {

private static final String GENERATED_PACKAGE =
Expand Down
12 changes: 0 additions & 12 deletions ...c/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsJson1_0SerializeBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,11 @@

package software.amazon.smithy.java.benchmarks.serde;

import java.util.concurrent.TimeUnit;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.smithy.java.aws.client.awsjson.AwsJson1Protocol;
import software.amazon.smithy.java.core.schema.ApiOperation;
Expand All @@ -31,11 +24,6 @@
* generated client uses internally.
*/
@State(Scope.Benchmark)
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 10, time = 5, timeUnit = TimeUnit.SECONDS)
@Fork(1)
public class AwsJson1_0SerializeBenchmark {

private static final String GENERATED_PACKAGE =
Expand Down
12 changes: 0 additions & 12 deletions ...c/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQueryDeserializeBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,11 @@

package software.amazon.smithy.java.benchmarks.serde;

import java.util.concurrent.TimeUnit;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.smithy.java.aws.client.awsquery.AwsQueryClientProtocol;
import software.amazon.smithy.java.core.schema.ApiOperation;
Expand All @@ -29,11 +22,6 @@
* protocol-specific result wrapper element.
*/
@State(Scope.Benchmark)
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 10, time = 5, timeUnit = TimeUnit.SECONDS)
@Fork(1)
public class AwsQueryDeserializeBenchmark {

private static final String GENERATED_PACKAGE =
Expand Down
12 changes: 0 additions & 12 deletions ...src/jmh/java/software/amazon/smithy/java/benchmarks/serde/AwsQuerySerializeBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,11 @@

package software.amazon.smithy.java.benchmarks.serde;

import java.util.concurrent.TimeUnit;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import org.openjdk.jmh.infra.Blackhole;
import software.amazon.smithy.java.aws.client.awsquery.AwsQueryClientProtocol;
import software.amazon.smithy.java.core.schema.ApiOperation;
Expand All @@ -29,11 +22,6 @@
* {@link AwsQueryClientProtocol#createRequest}.
*/
@State(Scope.Benchmark)
@BenchmarkMode(Mode.SampleTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@Warmup(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 10, time = 5, timeUnit = TimeUnit.SECONDS)
@Fork(1)
public class AwsQuerySerializeBenchmark {

private static final String GENERATED_PACKAGE =
Expand Down
Loading