diff --git a/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/ServiceGenerator.java b/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/ServiceGenerator.java index 4f00bb16..9d7dc462 100644 --- a/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/ServiceGenerator.java +++ b/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/ServiceGenerator.java @@ -209,7 +209,7 @@ String formatCaseStatement() { kind = "bidiStreaming"; methodLambda = "typedReplies -> " + name + "(typedReplies, options)"; } - + // spotless:off return """ case $methodName -> Pipelines.<$requestType, $replyType>$kind() .mapRequest(bytes -> parse$simpleRequestType(bytes, options)) @@ -225,9 +225,11 @@ String formatCaseStatement() { .replace("$replyType", replyType) .replace("$simpleReplyType", replyType.replace(".", "")) .replace("$kind", kind); + // spotless:on } private String formatUnaryMethodImplementation() { + // spotless:off return """ @Override $methodSignatureWithoutOptions { @@ -298,9 +300,11 @@ public void onComplete() { .replace("$replyType", replyType) .replace("$simpleReplyType", replyType.replace(".", "")) .replace("$methodName", name); + // spotless:on } private String formatClientStreamingMethodImplementation() { + // spotless:off return """ @Override $methodSignatureWithoutOptions { @@ -369,9 +373,11 @@ public void onComplete() { .replace("$replyType", replyType) .replace("$simpleReplyType", replyType.replace(".", "")) .replace("$methodName", name); + // spotless:on } private String formatServerStreamingMethodImplementation() { + // spotless:off return """ @Override $methodSignatureWithoutOptions { @@ -429,9 +435,11 @@ public void onComplete() { .replace("$replyType", replyType) .replace("$simpleReplyType", replyType.replace(".", "")) .replace("$methodName", name); + // spotless:on } private String formatBidiStreamingMethodImplementation() { + // spotless:off return """ @Override $methodSignatureWithoutOptions { @@ -496,6 +504,7 @@ public void onComplete() { .replace("$replyType", replyType) .replace("$simpleReplyType", replyType.replace(".", "")) .replace("$methodName", name); + // spotless:on } String formatMethodImplementation() { @@ -771,7 +780,7 @@ private static String formatParseRequestMethod(final String requestType) { Objects.requireNonNull(options); // not strict, no unknown fields, hard-code maxDepth for now, and use custom maxSize: - return get$simpleRequestTypeCodec(options).parse(message.toReadableSequentialData(), false, false, 16, options.maxMessageSizeBytes()); + return get$simpleRequestTypeCodec(options).parse(message, false, false, 16, options.maxMessageSizeBytes()); } """ .replace("$requestType", requestType) diff --git a/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecFastEqualsMethodGenerator.java b/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecFastEqualsMethodGenerator.java index 60c579f0..a9671b97 100644 --- a/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecFastEqualsMethodGenerator.java +++ b/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecFastEqualsMethodGenerator.java @@ -15,6 +15,7 @@ class CodecFastEqualsMethodGenerator { static String generateFastEqualsMethod(final String modelClassName, final List fields) { // Placeholder implementation, replace faster implementation than full parse if there is one + // spotless:off return """ /** * Compares the given item with the bytes in the input, and returns false if it determines that @@ -28,11 +29,12 @@ static String generateFastEqualsMethod(final String modelClassName, final List fields) { // Placeholder implementation, replace faster implementation than full parse if there is one + // spotless:off return """ /** * Reads from this data input the length of the data within the input. The implementation may @@ -24,7 +25,7 @@ static String generateMeasureMethod(final String modelClassName, final List * The {@code maxSize} specifies a custom value for the default `Codec.DEFAULT_MAX_SIZE` limit. IMPORTANT: * specifying a value larger than the default one can put the application at risk because a maliciously-crafted @@ -82,36 +82,31 @@ static String generateParseMethod( * @return Parsed $modelClassName model object or null if data input was null or empty * @throws ParseException If parsing fails */ - public @NonNull $modelClassName parse( - @NonNull final ReadableSequentialData input, + public $modelClassName parse( + @NonNull final PbjReader input, final boolean strictMode, final boolean parseUnknownFields, final int maxDepth, final int maxSize) throws ParseException { if (maxDepth < 0) { - throw new ParseException("Reached maximum allowed depth of nested messages"); + input.setError(PbjReader.MaxDepthReached); + return null; } - try { - // -- TEMP STATE FIELDS -------------------------------------- - $fieldDefs - List $unknownFields = null; + // -- TEMP STATE FIELDS -------------------------------------- + $fieldDefs + List $unknownFields = null; - $parseLoop - $listFieldsWriteProtection - if ($unknownFields != null) { - Collections.sort($unknownFields); - $initialSizeOfUnknownFieldsArray = Math.max($initialSizeOfUnknownFieldsArray, $unknownFields.size()); - } - $cacheableSupport - } catch (final Exception anyException) { - if (anyException instanceof ParseException parseException) { - throw parseException; - } - throw new ParseException(anyException); + $parseLoop + $listFieldsWriteProtection + if ($unknownFields != null) { + Collections.sort($unknownFields); + $initialSizeOfUnknownFieldsArray = Math.max($initialSizeOfUnknownFieldsArray, $unknownFields.size()); } + if (input.error() > 0) return null; + $cacheableSupport } - private List defaultCase(int tag, int field, FieldDefinition f, boolean strictMode, boolean parseUnknownFields, List $unknownFields, ReadableSequentialData input, int maxSize) throws ParseException, IOException { + private List defaultCase(int tag, int field, FieldDefinition f, boolean strictMode, boolean parseUnknownFields, List $unknownFields, PbjReader input, int maxSize) { $defaultCaseBody return $unknownFields; } @@ -182,20 +177,10 @@ static ParseAndDefaultBody generateParseLoop( // -- PARSE LOOP --------------------------------------------- // Continue to parse bytes out of the input stream until we get to the end. while (input.hasRemaining()) { - // Note: ReadableStreamingData.hasRemaining() won't flip to false - // until the end of stream is actually hit with a read operation. - // So we catch this exception here and **only** here, because an EOFException - // anywhere else suggests that we're processing malformed data and so - // we must re-throw the exception then. - final int $prefixtag; - try { - // Read the "tag" byte which gives us the field number for the next field to read - // and the wire type (way it is encoded on the wire). - $prefixtag = input.readVarInt(false); - } catch (EOFException e) { - // There's no more fields. Stop the parsing loop. - break; - } + + // Read the "tag" byte which gives us the field number for the next field to read + // and the wire type (way it is encoded on the wire). + final int $prefixtag = input.readVarIntNoZZ(); // The field is the top 5 bits of the byte. Read this off final int $prefixfield = $prefixtag >>> TAG_FIELD_OFFSET; @@ -216,20 +201,22 @@ static ParseAndDefaultBody generateParseLoop( // handle error cases here, so we do not do if statements in normal loop // Validate the field number is valid (must be > 0) if ($prefixfield == 0) { - throw new IOException("Bad protobuf encoding. We read a field value of " + input.setError(PbjReader.IOError, "Bad protobuf encoding. We read a field value of " + $prefixfield); + return $unknownFields; } // Validate the wire type is valid (must be >=0 && <= 5). // Otherwise we cannot parse this. // Note: it is always >= 0 at this point (see code above where it is defined). if (wireType > 5) { - throw new IOException("Cannot understand wire_type of " + wireType); + input.setError(PbjReader.Parse, "Cannot understand wire_type of " + wireType); + return $unknownFields; } // It may be that the parser subclass doesn't know about this field if ($prefixf == null) { if (strictMode) { - // Since we are parsing is strict mode, this is an exceptional condition. - throw new UnknownFieldException($prefixfield); + input.setError(PbjReader.UnknownField); + return $unknownFields; } else if (parseUnknownFields) { if ($unknownFields == null) { $unknownFields = new ArrayList<>($initialSizeOfUnknownFieldsArray); @@ -245,8 +232,9 @@ static ParseAndDefaultBody generateParseLoop( skipField(input, ProtoConstants.get(wireType), $skipMaxSize); } } else { - throw new IOException("Bad tag [" + $prefixtag + "], field [" + $prefixfield + input.setError(PbjReader.IOError, "Bad tag [" + $prefixtag + "], field [" + $prefixfield + "] wireType [" + wireType + "]"); + return $unknownFields; }"""); for (int i = 0; i < list.size(); i++) { list.set(i, list.get(i) @@ -299,11 +287,18 @@ private static void generateFieldCaseStatementPacked( sbCase.append("case %d /* type=%d [%s] packed-repeated field=%d [%s] */ -> {%n" .formatted(tag, wireType, field.type(), fieldNum, field.name())); sbCase.append("%s = case%d(input, maxSize, %s);%n".formatted(tempFieldName, tag, tempFieldName)); - sbFunc.append(""" -%s case%d(ReadableSequentialData input, int maxSize, %s %s) throws ParseException, IOException {""".formatted(fieldType, tag, fieldType, tempFieldName)); + sbFunc.append("%s case%d(PbjReader input, int maxSize, %s %s) {%n".formatted(fieldType, tag, fieldType, tempFieldName)); final String preRead; + int divideAmount = fieldType.equals("List") ? 2 + : fieldType.equals("List") ? 4 + : fieldType.equals("List") ? 2 + : fieldType.equals("List") ? 4 + : fieldType.equals("List") ? 1 + : 0; + if (field.type() == Field.FieldType.ENUM) { // spotless:off + divideAmount = 1; preRead = """ final int enumOrdinal = readEnum(input); Object value = $enumName.fromProtobufOrdinal(enumOrdinal); @@ -319,29 +314,37 @@ private static void generateFieldCaseStatementPacked( } else { preRead = ""; } + + if (divideAmount == 0) { + throw new RuntimeException("Need to implement"); + } + sbFunc.append(""" // Read the length of packed repeated field data - final long length = input.readVarInt(false); + final int length = input.readVarIntNoZZ(); if (length > $maxSize) { - throw new ParseException("$fieldName size " + length + " is greater than max " + $maxSize); - } - if (input.remaining() < length) { - throw new BufferUnderflowException(); + input.setError(PbjReader.Parse, "$fieldName size " + length + " is greater than max " + $maxSize); + return $tempFieldName; } - final var beforeLimit = input.limit(); - final long beforePosition = input.position(); + final var startLimit = input.limit(); + final long startPosition = input.position(); input.limit(input.position() + length); + var list = new UnmodifiableArray$fieldType(); + list.ensureCapacity(length$divideString); while (input.hasRemaining()) { - $preRead$tempFieldName = addToList($tempFieldName,$readMethod); + $preReadlist.add($readMethod); } - input.limit(beforeLimit); - if (input.position() != beforePosition + length) { - throw new BufferUnderflowException(); + $tempFieldName = list; + input.limit(startLimit); + if (input.position() != startPosition + length) { + input.setError(PbjReader.BufferUnderflow); }""".replace("$tempFieldName", tempFieldName) .replace("$preRead", preRead) + .replace("$fieldType", fieldType) .replace("$readMethod", field.type() == Field.FieldType.ENUM ? "value" : readMethod(field)) .replace("$maxSize", field.maxSize() >= 0 ? String.valueOf(field.maxSize()) : "maxSize") .replace("$fieldName", field.name()) + .replace("$divideString", divideAmount == 1 ? "" : "/%d".formatted(divideAmount)) .indent(DEFAULT_INDENT)); sbCase.append("\n}\n"); sbFunc.append(" return %s;\n }\n".formatted(tempFieldName)); @@ -368,19 +371,20 @@ private static void generateFieldCaseStatement( if (field.optionalValueType()) { sbCase.append(""" // Read the message size, it is not needed - final var valueTypeMessageSize = input.readVarInt(false); + final var valueTypeMessageSize = input.readVarIntNoZZ(); final $fieldType value; if (valueTypeMessageSize > 0) { - final var beforeLimit = input.limit(); + final var startLimit = input.limit(); input.limit(input.position() + valueTypeMessageSize); // read inner tag - final int valueFieldTag = input.readVarInt(false); + final int valueFieldTag = input.readVarIntNoZZ(); + assert input.throwOnErrorOrTrue(); // assert tag is as expected assert (valueFieldTag >>> TAG_FIELD_OFFSET) == 1; assert (valueFieldTag & TAG_WIRE_TYPE_MASK) == $valueTypeWireType; // read value value = $readMethod; - input.limit(beforeLimit); + input.limit(startLimit); } else { // means optional is default value value = $defaultValue; @@ -413,32 +417,32 @@ private static void generateFieldCaseStatement( } else if (field.type() == Field.FieldType.MESSAGE) { // spotless:off sbCase.append(""" - final var messageLength = input.readVarInt(false); + final var messageLength = input.readVarIntNoZZ(); final $fieldType value; if (messageLength == 0) { value = $fieldType.DEFAULT; } else { if (messageLength > $maxSize) { - throw new ParseException("$fieldName size " + messageLength + " is greater than max " + $maxSize); + input.setError(PbjReader.Parse, "$fieldName size " + messageLength + " is greater than max " + $maxSize); + return null; } - final var limitBefore = input.limit(); + final var startLimit = input.limit(); // Make sure that we have enough bytes in the message // to read the subObject. // If the buffer is truncated on the boundary of a subObject, // we will not throw. - final var startPos = input.position(); - try { - if ((startPos + messageLength) > limitBefore) { - throw new BufferUnderflowException(); - } - input.limit(startPos + messageLength); - value = $readMethod; - // Make sure we read the full number of bytes. for the types - if ((startPos + messageLength) != input.position()) { - throw new BufferOverflowException(); - } - } finally { - input.limit(limitBefore); + final var startPosition = input.position(); + if ((startPosition + messageLength) > startLimit) { + input.setError(PbjReader.BufferUnderflow); + return null; + } + input.limit(startPosition + messageLength); + value = $readMethod; + input.limit(startLimit); + // Make sure we read the full number of bytes. for the types + if ((startPosition + messageLength) != input.position()) { + input.setError(PbjReader.BufferOverflow); + return null; } } """ @@ -458,12 +462,13 @@ private static void generateFieldCaseStatement( generateCaseStatements(sbFunc, mapEntryFields, schemaClassName), "map_entry_", schemaClassName); // spotless:off sbCase.append(""" - final var __map_messageLength = input.readVarInt(false); + final var __map_messageLength = input.readVarIntNoZZ(); $fieldDefs if (__map_messageLength != 0) { if (__map_messageLength > $maxSize) { - throw new ParseException("$fieldName size " + __map_messageLength + " is greater than max " + $maxSize); + input.setError(PbjReader.Parse, "$fieldName size " + __map_messageLength + " is greater than max " + $maxSize); + return null; } final var __map_limitBefore = input.limit(); // Make sure that we have enough bytes in the message @@ -473,13 +478,15 @@ private static void generateFieldCaseStatement( final var __map_startPos = input.position(); try { if ((__map_startPos + __map_messageLength) > __map_limitBefore) { - throw new BufferUnderflowException(); + input.setError(PbjReader.BufferUnderflow); + return null; } input.limit(__map_startPos + __map_messageLength); $mapParseLoop // Make sure we read the full number of bytes. for the types if ((__map_startPos + __map_messageLength) != input.position()) { - throw new BufferOverflowException(); + input.setError(PbjReader.BufferOverflow); + return null; } } finally { input.limit(__map_limitBefore); @@ -523,7 +530,8 @@ private static void generateFieldCaseStatement( sbCase.append( """ if (temp_%s.size() >= %s) { - throw new ParseException("%1$s size %%d is greater than max %2$s".formatted(temp_%1$s.size())); + input.setError(PbjReader.Parse, "%1$s size %%d is greater than max %2$s".formatted(temp_%1$s.size())); + return null; } temp_%1$s = addToList(temp_%1$s,value); """.formatted(field.name(), field.maxSize() >= 0 ? String.valueOf(field.maxSize()) : "maxSize")); @@ -533,7 +541,8 @@ private static void generateFieldCaseStatement( """ if (__map_messageLength != 0) { if (temp_%s.size() >= %s) { - throw new ParseException("%1$s size %%d is greater than max %2$s".formatted(temp_%1$s.size())); + input.setError(PbjReader.Parse, "%1$s size %%d is greater than max %2$s".formatted(temp_%1$s.size())); + return null; } temp_%1$s = addToMap(temp_%1$s, temp_%s, temp_%s); } diff --git a/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecWriteMethodGenerator.java b/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecWriteMethodGenerator.java index fc83e5f1..33a0d6bd 100644 --- a/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecWriteMethodGenerator.java +++ b/pbj-core/pbj-compiler/src/main/java/com/hedera/pbj/compiler/impl/generators/protobuf/CodecWriteMethodGenerator.java @@ -44,15 +44,14 @@ static String generateWriteMethod( * * @param data The input model data to write * @param out The output stream to write to - * @throws IOException If there is a problem writing */ - public void write(@NonNull $modelClass data, @NonNull final WritableSequentialData out) throws IOException { + public void write(@NonNull $modelClass data, @NonNull final PbjWriter out) { $fieldWriteLines // Check if not-empty to avoid creating a lambda if there's nothing to write. if (!data.getUnknownFields().isEmpty()) { data.getUnknownFields().forEach(uf -> { final int tag = (uf.field() << TAG_FIELD_OFFSET) | uf.wireType().ordinal(); - out.writeVarInt(tag, false); + out.writeVarIntNoZZ(tag); uf.bytes().writeTo(out); }); } @@ -178,7 +177,7 @@ private static String generateFieldWriteLines( $V v = pbjMap.get(k); int size = 0; $fieldSizeOfLines - out.writeVarInt(size, false); + out.writeVarIntNoZZ(size); $fieldWriteLines } } diff --git a/pbj-core/pbj-grpc-client-helidon/src/main/java/com/hedera/pbj/grpc/client/helidon/PbjGrpcCall.java b/pbj-core/pbj-grpc-client-helidon/src/main/java/com/hedera/pbj/grpc/client/helidon/PbjGrpcCall.java index 7f75cdae..978cee1e 100644 --- a/pbj-core/pbj-grpc-client-helidon/src/main/java/com/hedera/pbj/grpc/client/helidon/PbjGrpcCall.java +++ b/pbj-core/pbj-grpc-client-helidon/src/main/java/com/hedera/pbj/grpc/client/helidon/PbjGrpcCall.java @@ -154,8 +154,8 @@ public static void setNetworkBytesInspector(PbjGrpcNetworkBytesInspector network */ @Override public void sendRequest(final RequestT request, final boolean endOfStream) { - final Bytes requestBytes = requestCodec.toBytes(request); - final Bytes bytes = GrpcCompression.getCompressor(grpcOutgoingEncoding).compress(requestBytes); + final Bytes wrappedInternalBytes = requestCodec.toBytesUnsafeWrapped(request); + final Bytes bytes = GrpcCompression.getCompressor(grpcOutgoingEncoding).compress(wrappedInternalBytes); PbjGrpcCall.networkBytesInspector.sent(bytes); final BufferData bufferData = BufferData.create(PbjGrpcDatagramReader.PREFIX_LENGTH + Math.toIntExact(bytes.length())); @@ -242,7 +242,7 @@ private void receiveRepliesLoop() { try { final ReplyT reply = replyCodec.parse( - replyBytes.toReadableSequentialData(), + replyBytes, false, false, Codec.DEFAULT_MAX_DEPTH, diff --git a/pbj-core/pbj-grpc-client-helidon/src/test/java/com/hedera/pbj/grpc/client/helidon/PbjGrpcCallTest.java b/pbj-core/pbj-grpc-client-helidon/src/test/java/com/hedera/pbj/grpc/client/helidon/PbjGrpcCallTest.java index c2162b49..b6106d57 100644 --- a/pbj-core/pbj-grpc-client-helidon/src/test/java/com/hedera/pbj/grpc/client/helidon/PbjGrpcCallTest.java +++ b/pbj-core/pbj-grpc-client-helidon/src/test/java/com/hedera/pbj/grpc/client/helidon/PbjGrpcCallTest.java @@ -21,7 +21,6 @@ import com.hedera.pbj.runtime.grpc.GrpcStatus; import com.hedera.pbj.runtime.grpc.Pipeline; import com.hedera.pbj.runtime.grpc.ServiceInterface; -import com.hedera.pbj.runtime.io.ReadableSequentialData; import com.hedera.pbj.runtime.io.buffer.Bytes; import io.helidon.common.buffers.BufferData; import io.helidon.common.buffers.DataWriter; @@ -158,7 +157,7 @@ public void testSendRequest(final boolean endOfStream) { final Object request = mock(Object.class); final Bytes bytes = Bytes.wrap("test bytes string"); - doReturn(bytes).when(requestCodec).toBytes(request); + doReturn(bytes).when(requestCodec).toBytesUnsafeWrapped(request); call.sendRequest(request, endOfStream); @@ -279,12 +278,7 @@ public void testReceiveRepliesLoopSingleReply(final boolean isTimeout) throws Ex final Object reply = mock(Object.class); doReturn(reply) .when(replyCodec) - .parse( - any(ReadableSequentialData.class), - eq(false), - eq(false), - eq(Codec.DEFAULT_MAX_DEPTH), - eq(Codec.DEFAULT_MAX_SIZE)); + .parse(any(Bytes.class), eq(false), eq(false), eq(Codec.DEFAULT_MAX_DEPTH), eq(Codec.DEFAULT_MAX_SIZE)); runnable.run(); @@ -342,12 +336,7 @@ public void testReceiveRepliesLoopParseException() throws Exception { final ParseException exception = new ParseException("test"); doThrow(exception) .when(replyCodec) - .parse( - any(ReadableSequentialData.class), - eq(false), - eq(false), - eq(Codec.DEFAULT_MAX_DEPTH), - eq(Codec.DEFAULT_MAX_SIZE)); + .parse(any(Bytes.class), eq(false), eq(false), eq(Codec.DEFAULT_MAX_DEPTH), eq(Codec.DEFAULT_MAX_SIZE)); runnable.run(); diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Codec.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Codec.java index 848d3bde..18e9578a 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Codec.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Codec.java @@ -1,13 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.pbj.runtime; +import com.hedera.pbj.runtime.io.PbjReader; +import com.hedera.pbj.runtime.io.PbjWriter; import com.hedera.pbj.runtime.io.ReadableSequentialData; import com.hedera.pbj.runtime.io.WritableSequentialData; -import com.hedera.pbj.runtime.io.buffer.BufferedData; import com.hedera.pbj.runtime.io.buffer.Bytes; import com.hedera.pbj.runtime.io.stream.WritableStreamingData; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; +import java.io.InputStream; import java.io.UncheckedIOException; /** @@ -16,6 +18,20 @@ * @param The type of object to serialize and deserialize */ public interface Codec { + class WriteCache { + PbjWriter writer = new PbjWriter(); + /** For recursive situations if the caller ever requires it */ + boolean inUse = false; + } + + class ReadCache { + PbjReader reader = new PbjReader(Bytes.EMPTY); + /** For recursive situations if the caller ever requires it */ + boolean inUse = false; + } + + ThreadLocal tlsWriter = ThreadLocal.withInitial(WriteCache::new); + ThreadLocal tlsReader = ThreadLocal.withInitial(ReadCache::new); /** * The default maximum size of a repeated or length-encoded field (Bytes, String, Message, etc.). @@ -38,7 +54,7 @@ public interface Codec { * If {@code strictMode} is {@code true}, then throws an exception if fields * have been defined on the encoded object that are not supported by the parser. This * breaks forwards compatibility (an older parser cannot parse a newer encoded object), - * which is sometimes requires to avoid parsing an object that is newer than the code + * which is sometimes required to avoid parsing an object that is newer than the code * parsing it is prepared to handle. *

* The {@code maxDepth} specifies the maximum allowed depth of nested messages. The parsing @@ -62,13 +78,44 @@ public interface Codec { * @return The parsed object. It must not return null. * @throws ParseException If parsing fails */ - @NonNull - T parse( + default @NonNull T parse( @NonNull ReadableSequentialData input, boolean strictMode, boolean parseUnknownFields, int maxDepth, int maxSize) + throws ParseException { + return wrapParse(input, strictMode, parseUnknownFields, maxDepth, maxSize); + } + + default @NonNull T wrapParse( + @NonNull ReadableSequentialData input, + boolean strictMode, + boolean parseUnknownFields, + int maxDepth, + int maxSize) + throws ParseException { + ReadCache cache = tlsReader.get(); + if (cache.inUse) { + PbjReader reader = new PbjReader(input); + T res = parse(reader, strictMode, parseUnknownFields, maxDepth, maxSize); + reader.throwOnError(); + return res; + } + cache.inUse = true; + try { + PbjReader reader = cache.reader; + reader.resetWith(input); + T res = parse(reader, strictMode, parseUnknownFields, maxDepth, maxSize); + reader.throwOnError(); + return res; + } finally { + cache.inUse = false; + } + } + + @NonNull + T parse(@NonNull PbjReader input, boolean strictMode, boolean parseUnknownFields, int maxDepth, int maxSize) throws ParseException; /** @@ -77,7 +124,7 @@ T parse( * If {@code strictMode} is {@code true}, then throws an exception if fields * have been defined on the encoded object that are not supported by the parser. This * breaks forwards compatibility (an older parser cannot parse a newer encoded object), - * which is sometimes requires to avoid parsing an object that is newer than the code + * which is sometimes required to avoid parsing an object that is newer than the code * parsing it is prepared to handle. *

* The {@code maxDepth} specifies the maximum allowed depth of nested messages. The parsing @@ -96,6 +143,23 @@ T parse( @NonNull default T parse(@NonNull ReadableSequentialData input, boolean strictMode, boolean parseUnknownFields, int maxDepth) throws ParseException { + return wrapParse(input, strictMode, parseUnknownFields, maxDepth, DEFAULT_MAX_SIZE); + } + + default T parse(@NonNull PbjReader input, boolean strictMode, boolean parseUnknownFields, int maxDepth) + throws ParseException { + return parse(input, strictMode, parseUnknownFields, maxDepth, DEFAULT_MAX_SIZE); + } + + @NonNull + default T parse(@NonNull Bytes input, boolean strictMode, boolean parseUnknownFields, int maxDepth) + throws ParseException { + return parse(input, strictMode, parseUnknownFields, maxDepth, DEFAULT_MAX_SIZE); + } + + @NonNull + default T parse(@NonNull byte[] input, boolean strictMode, boolean parseUnknownFields, int maxDepth) + throws ParseException { return parse(input, strictMode, parseUnknownFields, maxDepth, DEFAULT_MAX_SIZE); } /** @@ -104,7 +168,7 @@ default T parse(@NonNull ReadableSequentialData input, boolean strictMode, boole * If {@code strictMode} is {@code true}, then throws an exception if fields * have been defined on the encoded object that are not supported by the parser. This * breaks forwards compatibility (an older parser cannot parse a newer encoded object), - * which is sometimes requires to avoid parsing an object that is newer than the code + * which is sometimes required to avoid parsing an object that is newer than the code * parsing it is prepared to handle. *

* The {@code maxDepth} specifies the maximum allowed depth of nested messages. The parsing @@ -119,30 +183,20 @@ default T parse(@NonNull ReadableSequentialData input, boolean strictMode, boole @NonNull default T parse(@NonNull ReadableSequentialData input, final boolean strictMode, final int maxDepth) throws ParseException { + return wrapParse(input, strictMode, false, maxDepth, DEFAULT_MAX_SIZE); + } + + @NonNull + default T parse(@NonNull PbjReader input, final boolean strictMode, final int maxDepth) throws ParseException { return parse(input, strictMode, false, maxDepth); } - /** - * Parses an object from the {@link Bytes} and returns it. - *

- * If {@code strictMode} is {@code true}, then throws an exception if fields - * have been defined on the encoded object that are not supported by the parser. This - * breaks forwards compatibility (an older parser cannot parse a newer encoded object), - * which is sometimes requires to avoid parsing an object that is newer than the code - * parsing it is prepared to handle. - *

- * The {@code maxDepth} specifies the maximum allowed depth of nested messages. The parsing - * will fail with a ParseException if the maximum depth is reached. - * - * @param bytes The {@link Bytes} from which to read the data to construct an object - * @param strictMode when {@code true}, the parser errors out on unknown fields; otherwise they'll be simply skipped. - * @param maxDepth a ParseException will be thrown if the depth of nested messages exceeds the maxDepth value. - * @return The parsed object. It must not return null. - * @throws ParseException If parsing fails - */ @NonNull - default T parse(@NonNull Bytes bytes, final boolean strictMode, final int maxDepth) throws ParseException { - return parse(bytes.toReadableSequentialData(), strictMode, maxDepth); + default T parseAndThrow(@NonNull PbjReader input, final boolean strictMode, final int maxDepth) + throws ParseException { + T res = parse(input, strictMode, false, maxDepth); + input.throwOnError(); + return res; } /** @@ -154,9 +208,52 @@ default T parse(@NonNull Bytes bytes, final boolean strictMode, final int maxDep */ @NonNull default T parse(@NonNull ReadableSequentialData input) throws ParseException { + return wrapParse(input, false, false, DEFAULT_MAX_DEPTH, DEFAULT_MAX_SIZE); + } + + @NonNull + default T parse(@NonNull PbjReader input) throws ParseException { return parse(input, false, DEFAULT_MAX_DEPTH); } + @NonNull + default T parse(@NonNull InputStream in) throws ParseException { + return parse(in, false, DEFAULT_MAX_DEPTH); + } + + @NonNull + default T parse(@NonNull InputStream in, boolean strictMode, int maxDepth) throws ParseException { + return parse(in, strictMode, false, maxDepth); + } + + @NonNull + default T parse(@NonNull InputStream in, boolean strictMode, boolean parseUnknownFields, int maxDepth) + throws ParseException { + return parse(in, strictMode, false, maxDepth, DEFAULT_MAX_SIZE); + } + + @NonNull + default T parse(@NonNull InputStream in, boolean strictMode, boolean parseUnknownFields, int maxDepth, int maxSize) + throws ParseException { + ReadCache cache = tlsReader.get(); + if (cache.inUse) { + PbjReader reader = new PbjReader(in); + T res = parse(reader, strictMode, parseUnknownFields, maxDepth, maxSize); + reader.throwOnError(); + return res; + } + cache.inUse = true; + try { + PbjReader reader = cache.reader; + reader.resetWith(in); + T res = parse(reader, strictMode, parseUnknownFields, maxDepth, maxSize); + reader.throwOnError(); + return res; + } finally { + cache.inUse = false; + } + } + /** * Parses an object from the {@link Bytes} and returns it. * @@ -166,14 +263,90 @@ default T parse(@NonNull ReadableSequentialData input) throws ParseException { */ @NonNull default T parse(@NonNull Bytes bytes) throws ParseException { - return parse(bytes.toReadableSequentialData()); + return parse(bytes, false, DEFAULT_MAX_DEPTH); + } + + default T parse(@NonNull Bytes bytes, final boolean strictMode, final int maxDepth) throws ParseException { + return parse(bytes, strictMode, false, maxDepth, DEFAULT_MAX_SIZE); + } + + @NonNull + default T parse(@NonNull byte[] bytes) throws ParseException { + return parse(bytes, false, DEFAULT_MAX_DEPTH); + } + + default T parse(@NonNull byte[] bytes, final boolean strictMode, final int maxDepth) throws ParseException { + return parse(bytes, strictMode, false, maxDepth, DEFAULT_MAX_SIZE); + } + + default T parse(@NonNull byte[] bytes, boolean strictMode, boolean parseUnknownFields, int maxDepth, int maxSize) + throws ParseException { + ReadCache cache = tlsReader.get(); + if (cache.inUse) { + PbjReader reader = new PbjReader(bytes); + T res = parse(reader, strictMode, parseUnknownFields, maxDepth, maxSize); + reader.throwOnError(); + return res; + } + cache.inUse = true; + try { + PbjReader reader = cache.reader; + reader.resetWith(bytes); + T res = parse(reader, strictMode, parseUnknownFields, maxDepth, maxSize); + reader.throwOnError(); + return res; + } finally { + cache.inUse = false; + } + } + + /** + * Parses an object from the {@link Bytes} and returns it. + *

+ * If {@code strictMode} is {@code true}, then throws an exception if fields + * have been defined on the encoded object that are not supported by the parser. This + * breaks forwards compatibility (an older parser cannot parse a newer encoded object), + * which is sometimes required to avoid parsing an object that is newer than the code + * parsing it is prepared to handle. + *

+ * The {@code maxDepth} specifies the maximum allowed depth of nested messages. The parsing + * will fail with a ParseException if the maximum depth is reached. + * + * @param bytes The {@link Bytes} from which to read the data to construct an object + * @param strictMode when {@code true}, the parser errors out on unknown fields; otherwise they'll be simply skipped. + * @param parseUnknownFields when {@code true} and strictMode is {@code false}, the parser will collect unknown + * fields in the unknownFields list in the model; otherwise they'll be simply skipped. + * @param maxDepth a ParseException will be thrown if the depth of nested messages exceeds the maxDepth value. + * @param maxSize a ParseException will be thrown if the size of a delimited field exceeds the limit + * @return The parsed object. It must not return null. + * @throws ParseException If parsing fails + */ + default T parse(@NonNull Bytes bytes, boolean strictMode, boolean parseUnknownFields, int maxDepth, int maxSize) + throws ParseException { + ReadCache cache = tlsReader.get(); + if (cache.inUse) { + PbjReader reader = new PbjReader(bytes); + T res = parse(reader, strictMode, parseUnknownFields, maxDepth, maxSize); + reader.throwOnError(); + return res; + } + cache.inUse = true; + try { + PbjReader reader = cache.reader; + reader.resetWith(bytes); + T res = parse(reader, strictMode, parseUnknownFields, maxDepth, maxSize); + reader.throwOnError(); + return res; + } finally { + cache.inUse = false; + } } /** * Parses an object from the {@link ReadableSequentialData} and returns it. Throws an exception if fields * have been defined on the encoded object that are not supported by the parser. This * breaks forwards compatibility (an older parser cannot parse a newer encoded object), - * which is sometimes requires to avoid parsing an object that is newer than the code + * which is sometimes required to avoid parsing an object that is newer than the code * parsing it is prepared to handle. * * @param input The {@link ReadableSequentialData} from which to read the data to construct an object @@ -182,14 +355,18 @@ default T parse(@NonNull Bytes bytes) throws ParseException { */ @NonNull default T parseStrict(@NonNull ReadableSequentialData input) throws ParseException { - return parse(input, true, DEFAULT_MAX_DEPTH); + return wrapParse(input, true, false, DEFAULT_MAX_DEPTH, DEFAULT_MAX_SIZE); } + @NonNull + default T parseStrict(@NonNull PbjReader input) throws ParseException { + return parse(input, true, DEFAULT_MAX_DEPTH); + } /** * Parses an object from the {@link Bytes} and returns it. Throws an exception if fields * have been defined on the encoded object that are not supported by the parser. This * breaks forwards compatibility (an older parser cannot parse a newer encoded object), - * which is sometimes requires to avoid parsing an object that is newer than the code + * which is sometimes required to avoid parsing an object that is newer than the code * parsing it is prepared to handle. * * @param bytes The {@link Bytes} from which to read the data to construct an object @@ -198,17 +375,39 @@ default T parseStrict(@NonNull ReadableSequentialData input) throws ParseExcepti */ @NonNull default T parseStrict(@NonNull Bytes bytes) throws ParseException { - return parseStrict(bytes.toReadableSequentialData()); + return parse(bytes, true, DEFAULT_MAX_DEPTH); + } + + @NonNull + default T parseStrict(@NonNull byte[] bytes) throws ParseException { + return parseStrict(Bytes.wrap(bytes)); } /** - * Writes an item to the given {@link WritableSequentialData}. + * Writes an item to the given {@link PbjWriter}. * * @param item The item to write. Must not be null. - * @param output The {@link WritableSequentialData} to write to. - * @throws IOException If the {@link WritableSequentialData} cannot be written to. + * @param output The {@link PbjWriter} to write to. */ - void write(@NonNull T item, @NonNull WritableSequentialData output) throws IOException; + void write(@NonNull T item, @NonNull PbjWriter output); + + default void write(@NonNull T item, @NonNull WritableSequentialData output) throws IOException { + WriteCache cache = tlsWriter.get(); + if (cache.inUse) { + PbjWriter writer = new PbjWriter(output); + write(item, writer); + writer.flush(); + return; + } + cache.inUse = true; + try { + cache.writer.resetWith(output); + write(item, cache.writer); + cache.writer.flush(); + } finally { + cache.inUse = false; + } + } /** * Writes an item to the given byte array, this is a performance focused method. In non-performance centric use @@ -218,17 +417,13 @@ default T parseStrict(@NonNull Bytes bytes) throws ParseException { * @param output The byte array to write to, this must be large enough to hold the entire item. * @param startOffset The offset in the output array to start writing at. * @return The number of bytes written to the output array. - * @throws UncheckedIOException If the there is a problem writing to the output array. + * @throws UncheckedIOException If there is a problem writing to the output array. * @throws IndexOutOfBoundsException If the output array is not large enough to hold the entire item. */ default int write(@NonNull T item, @NonNull byte[] output, final int startOffset) { - final BufferedData bufferedData = BufferedData.wrap(output, startOffset, output.length - startOffset); - try { - write(item, bufferedData); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return (int) bufferedData.position(); + PbjWriter writer = new PbjWriter(output, startOffset); + write(item, writer); + return writer.position(); } /** @@ -240,8 +435,15 @@ default int write(@NonNull T item, @NonNull byte[] output, final int startOffset * @return The length of the data item in the input * @throws ParseException If parsing fails */ - int measure(@NonNull ReadableSequentialData input) throws ParseException; + default int measure(@NonNull PbjReader input) throws ParseException { + final long startPosition = input.position(); + parse(input); + return (int) (input.position() - startPosition); + } + default int measure(@NonNull ReadableSequentialData input) throws ParseException { + return measure(new PbjReader(input)); + } /** * Compute number of bytes that would be written when calling {@code write()} method. * @@ -262,26 +464,94 @@ default int write(@NonNull T item, @NonNull byte[] output, final int startOffset * @return true if the bytes represent the item, false otherwise. * @throws ParseException If parsing fails */ - boolean fastEquals(@NonNull T item, @NonNull ReadableSequentialData input) throws ParseException; + boolean fastEquals(@NonNull T item, @NonNull PbjReader input) throws ParseException; + + default boolean fastEquals(@NonNull T item, @NonNull ReadableSequentialData input) throws ParseException { + ReadCache cache = tlsReader.get(); + if (cache.inUse) { + PbjReader reader = new PbjReader(input); + boolean res = fastEquals(item, reader); + reader.throwOnError(); + return res; + } + cache.inUse = true; + try { + PbjReader reader = cache.reader; + reader.resetWith(input); + boolean res = fastEquals(item, reader); + reader.throwOnError(); + return res; + } finally { + cache.inUse = false; + } + } + + default byte[] toByteArray(@NonNull T item) { + WriteCache cache = tlsWriter.get(); + if (cache.inUse) { + int len = measureRecord(item); + PbjWriter writer = new PbjWriter(len, false); + write(item, writer); + return writer.toByteArray(); + } + cache.inUse = true; + try { + cache.writer.resetWithNull(); + write(item, cache.writer); + return cache.writer.toByteArray(); + } finally { + cache.inUse = false; + } + } + + default Bytes toBytes(@NonNull T item) { + WriteCache cache = tlsWriter.get(); + if (cache.inUse) { + int len = measureRecord(item); + PbjWriter writer = new PbjWriter(len, false); + return toBytes(item, writer); + } + cache.inUse = true; + try { + cache.writer.resetWithNull(); + return toBytes(item, cache.writer); + } finally { + cache.inUse = false; + } + } + + default Bytes toBytes(@NonNull T item, PbjWriter writer) { + write(item, writer); + return writer.toByteArrayWrapped(); + } /** - * Converts a Record into a Bytes object + * Serializes an item to {@link Bytes} that wrap the codec's internal byte array directly, without copying. + * This avoids an allocation compared to {@link #toBytes(Object)}, but the returned {@link Bytes} shares + * the backing buffer with the thread-local write cache, so it must not be retained beyond the current call + * frame. The intended use case is when the caller wants to immediately consume the bytes — for example, + * to compress them — and does not need to hold a reference afterward. * - * @param item The input model data to convert into a Bytes object. - * @return The new Bytes object. - * @throws RuntimeException wrapping an IOException If it is impossible - * to write to the {@link WritableStreamingData} + * @param item The item to serialize. Must not be null. + * @return A {@link Bytes} view backed by the internal write buffer. Must not be stored or used after + * the current call completes. */ - default Bytes toBytes(@NonNull T item) { - // it is cheaper performance wise to measure the size of the object first than grow a buffer as needed - final byte[] bytes = new byte[measureRecord(item)]; - final BufferedData bufferedData = BufferedData.wrap(bytes); + default Bytes toBytesUnsafeWrapped(@NonNull T item) { + WriteCache cache = tlsWriter.get(); + if (cache.inUse) { + int len = measureRecord(item); + PbjWriter writer = new PbjWriter(len, false); + write(item, writer); + return writer.internalArrayWrapped(); + } + cache.inUse = true; try { - write(item, bufferedData); - } catch (IOException e) { - throw new UncheckedIOException(e); + cache.writer.resetWithNull(); + write(item, cache.writer); + return cache.writer.internalArrayWrapped(); + } finally { + cache.inUse = false; } - return Bytes.wrap(bytes); } /** diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonCodec.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonCodec.java index b403be3e..1ec95223 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonCodec.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonCodec.java @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.pbj.runtime; -import com.hedera.pbj.runtime.io.ReadableSequentialData; +import com.hedera.pbj.runtime.io.PbjReader; +import com.hedera.pbj.runtime.io.PbjWriter; import com.hedera.pbj.runtime.io.WritableSequentialData; import com.hedera.pbj.runtime.io.stream.WritableStreamingData; import com.hedera.pbj.runtime.jsonparser.JSONParser; @@ -20,7 +21,7 @@ public interface JsonCodec extends Codec { /** {@inheritDoc} */ default @NonNull T parse( - @NonNull ReadableSequentialData input, + @NonNull PbjReader input, final boolean strictMode, final boolean parseUnknownFields, final int maxDepth, @@ -65,6 +66,10 @@ default void write(@NonNull T item, @NonNull WritableSequentialData output) thro output.writeUTF8(toJSON(item)); } + default void write(@NonNull T item, @NonNull PbjWriter output) { + output.writeStringNoTag(toJSON(item)); + } + /** * Returns JSON string representing an item. * @@ -95,7 +100,7 @@ default String toJSON(@NonNull T item) { * @return The length of the data item in the input * @throws ParseException If parsing fails */ - default int measure(@NonNull ReadableSequentialData input) throws ParseException { + default int measure(@NonNull PbjReader input) throws ParseException { final long startPosition = input.position(); parse(input); return (int) (input.position() - startPosition); @@ -134,7 +139,7 @@ default int measureRecord(T item) { * @return true if the bytes represent the item, false otherwise. * @throws ParseException If parsing fails */ - default boolean fastEquals(@NonNull T item, @NonNull ReadableSequentialData input) throws ParseException { + default boolean fastEquals(@NonNull T item, @NonNull PbjReader input) throws ParseException { return Objects.equals(item, parse(input)); } diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonTools.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonTools.java index 196124d3..e2338488 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonTools.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/JsonTools.java @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.pbj.runtime; -import com.hedera.pbj.runtime.io.ReadableSequentialData; +import com.hedera.pbj.runtime.io.PbjReader; import com.hedera.pbj.runtime.io.buffer.Bytes; import com.hedera.pbj.runtime.jsonparser.JSONLexer; import com.hedera.pbj.runtime.jsonparser.JSONParser; @@ -94,7 +94,7 @@ public static String escape(@Nullable String string) { * @return the Antlr JSON context object * @throws IOException if there was a problem parsing the JSON */ - public static JSONParser.ObjContext parseJson(@NonNull final ReadableSequentialData input) throws IOException { + public static JSONParser.ObjContext parseJson(@NonNull final PbjReader input) throws IOException { final JSONLexer lexer = new JSONLexer(CharStreams.fromStream(input.asInputStream())); final JSONParser parser = new JSONParser(new CommonTokenStream(lexer)); final JSONParser.JsonContext jsonContext = parser.json(); diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ParseException.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ParseException.java index 0f9bea01..9bb715b1 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ParseException.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ParseException.java @@ -16,4 +16,12 @@ public ParseException(Throwable cause) { public ParseException(String message) { super(message); } + + public ParseException(String message, Throwable cause) { + super(message, cause); + } + + public ParseException(Throwable cause, boolean skipStackTag) { + super(null, cause, true, false); + } } diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/PbjProtoWriter.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/PbjProtoWriter.java new file mode 100644 index 00000000..1f4e55f3 --- /dev/null +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/PbjProtoWriter.java @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.pbj.runtime; + +import com.hedera.pbj.runtime.io.PbjWriter; +import java.io.IOException; + +/** + * Interface for referencing the static write method from generated writer classes, using {@link PbjWriter}. + * + * @param The model object that is being written + */ +public interface PbjProtoWriter { + + /** + * Write out a {@code T} model to output stream in protobuf format. + * + * @param data The input model data to write + * @param out The output stream to write to + * @throws IOException If there is a problem writing + */ + void write(T data, PbjWriter out); +} diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoParserTools.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoParserTools.java index 9caa91a6..2e60be2d 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoParserTools.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoParserTools.java @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.pbj.runtime; +import com.hedera.pbj.runtime.io.PbjReader; import com.hedera.pbj.runtime.io.ReadableSequentialData; import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; @@ -81,6 +82,10 @@ public static int readInt32(final ReadableSequentialData input) { return input.readVarInt(false); } + public static int readInt32(PbjReader input) { + return input.readVarIntNoZZ(); + } + /** * Read a protobuf int64(long) from input * @@ -91,6 +96,10 @@ public static long readInt64(final ReadableSequentialData input) { return input.readVarLong(false); } + public static long readInt64(PbjReader input) { + return input.readVarLongNoZZ(); + } + /** * Read a protobuf uint32 from input * @@ -101,6 +110,10 @@ public static int readUint32(final ReadableSequentialData input) { return input.readVarInt(false); } + public static int readUint32(PbjReader input) { + return input.readVarIntNoZZ(); + } + /** * Read a protobuf uint64 from input * @@ -111,6 +124,9 @@ public static long readUint64(final ReadableSequentialData input) { return input.readVarLong(false); } + public static long readUint64(PbjReader input) { + return input.readVarLongNoZZ(); + } /** * Read a protobuf bool from input * @@ -126,6 +142,14 @@ public static boolean readBool(final ReadableSequentialData input) throws IOExce return i == 1; } + public static boolean readBool(PbjReader input) { + final var i = input.readVarIntNoZZ(); + if (i != 1 && i != 0) { + input.setError(PbjReader.DataEncoding); + } + return i == 1; + } + /** * Read a protobuf enum from input * @@ -136,6 +160,9 @@ public static int readEnum(final ReadableSequentialData input) { return input.readVarInt(false); } + public static int readEnum(PbjReader input) { + return input.readVarIntNoZZ(); + } /** * Read a protobuf sint32 from input * @@ -146,6 +173,10 @@ public static int readSignedInt32(final ReadableSequentialData input) { return input.readVarInt(true); } + public static int readSignedInt32(PbjReader input) { + return input.readVarIntZZ(); + } + /** * Read a protobuf uint64(long) from input * @@ -156,6 +187,10 @@ public static long readSignedInt64(final ReadableSequentialData input) { return input.readVarLong(true); } + public static long readSignedInt64(PbjReader input) { + return input.readVarLongZZ(); + } + /** * Read a protobuf sfixed32 from input * @@ -166,6 +201,10 @@ public static int readSignedFixed32(final ReadableSequentialData input) { return input.readInt(ByteOrder.LITTLE_ENDIAN); } + public static int readSignedFixed32(PbjReader input) { + return input.readIntLE(); + } + /** * Read a protobuf fixed32 from input * @@ -176,6 +215,10 @@ public static int readFixed32(final ReadableSequentialData input) { return input.readInt(ByteOrder.LITTLE_ENDIAN); } + public static int readFixed32(PbjReader input) { + return input.readIntLE(); + } + /** * Read a protobuf float from input * @@ -186,6 +229,10 @@ public static float readFloat(final ReadableSequentialData input) { return input.readFloat(ByteOrder.LITTLE_ENDIAN); } + public static float readFloat(PbjReader input) { + return input.readFloatLE(); + } + /** * Read a protobuf sfixed64 from input * @@ -196,6 +243,10 @@ public static long readSignedFixed64(final ReadableSequentialData input) { return input.readLong(ByteOrder.LITTLE_ENDIAN); } + public static long readSignedFixed64(final PbjReader input) { + return input.readLongLE(); + } + /** * Read a fixed 64, which is a fixed size encoded long * @@ -206,6 +257,10 @@ public static long readFixed64(final ReadableSequentialData input) { return input.readLong(ByteOrder.LITTLE_ENDIAN); } + public static long readFixed64(PbjReader input) { + return input.readLongLE(); + } + /** * Read a double from input data * @@ -216,6 +271,10 @@ public static double readDouble(final ReadableSequentialData input) { return input.readDouble(ByteOrder.LITTLE_ENDIAN); } + public static double readDouble(PbjReader input) { + return input.readDoubleLE(); + } + /** * Read a String field from data input * @@ -230,6 +289,10 @@ public static String readString(final ReadableSequentialData input) throws IOExc } } + public static String readString(final PbjReader input) { + return readString(input, Long.MAX_VALUE); + } + /** * Read a String field from data input * @@ -267,6 +330,130 @@ public static String readString(final ReadableSequentialData input, final long m } } + private static int fromUTF8Tail(char[] dst, int di, byte[] src, int i, int endPos) { + while (i < endPos) { + int a = src[i]; + if ((a & 0x80) == 0) { + dst[di++] = (char) a; + i++; + continue; + } + if (i + 1 >= endPos) return -1; + + int b = src[i + 1]; + if ((a & 0xE0) == 0xC0) { + if ((b & 0xC0) == 0x80) { + dst[di++] = (char) (((a & 0x1F) << 6) | (b & 0x3F)); + i += 2; + continue; + } else { + return -1; // Bad encoding + } + } + + if (i + 2 >= endPos) return -1; + + int c = src[i + 2]; + int codepoint = -1; + if ((a & 0xF0) == 0xE0) { + if ((b & 0xC0) == 0x80 && (c & 0xC0) == 0x80) { + codepoint = ((a & 0xF) << 12) | ((b & 0x3F) << 6) | (c & 0x3F); + i += 3; + } else { + return -1; // Bad encoding + } + } else { + if (i + 3 >= endPos) return -1; + int d = src[i + 3]; + if ((a & 0xF8) == 0xF0 && (b & 0xC0) == 0x80 && (c & 0xC0) == 0x80 && (d & 0xC0) == 0x80) { + codepoint = ((a & 7) << 18) | ((b & 0x3F) << 12) | ((c & 0x3F) << 6) | (d & 0x3F); + i += 4; + } else { + return -1; // Bad encoding + } + } + + if (codepoint <= 0xFFFF) { + if (codepoint < 0 || (codepoint >= 0xD800 && codepoint < 0xE000)) return -1; // [D800, E000) is illegal + dst[di++] = (char) codepoint; + continue; + } + + if (codepoint > 0x10FFFF) return -1; // Illegal range + int v = codepoint - 0x10000; + dst[di + 0] = (char) (0xD800 + ((v >> 10) & 0x3FF)); + dst[di + 1] = (char) (0xDC00 + (v & 0x3FF)); + di += 2; + } + return di; + } + + /** + * Decodes UTF-8 bytes from {@code src} into the {@code char[]} destination, supporting all + * four UTF-8 byte widths (1–4 bytes). Codepoints above U+FFFF are written as surrogate pairs. + * Prefer using the simpler {@link #readString} function. + * + *

Decoding starts at {@code src[offset + pos]} and continues until {@code src[offset + length]}. + * The main loop keeps a 4-byte lookahead (exits when fewer than 5 bytes remain), delegating + * the final bytes to {@link #fromUTF8Tail}. Any unrecognised byte sequence returns {@code -1}. + * + * @param dst the destination char array, written starting at index {@code pos} + * @param src the source byte array containing UTF-8 data + * @param offset the base index within {@code src} where the UTF-8 region begins + * @param pos bytes already decoded by the ASCII fast path; doubles as the read offset + * (relative to {@code offset}) and the initial write index in {@code dst} + * @param length the total byte length of the UTF-8 region in {@code src} starting at {@code offset} + * @return the total number of {@code char}s written to {@code dst}, or {@code -1} if the + * input contains an illegal byte sequence (surrogate range or out-of-range codepoint) + */ + public static int fromUTF8(char[] dst, byte[] src, int offset, int pos, int length) { + int i = offset + pos; + int di = pos; + while (i + 4 < offset + length) { + int a = src[i]; + if ((a & 0x80) == 0) { + dst[di++] = (char) a; + i++; + continue; + } + int b = src[i + 1]; + if ((a & 0xE0) == 0xC0 && (b & 0xC0) == 0x80) { + dst[di++] = (char) (((a & 0x1F) << 6) | (b & 0x3F)); + i += 2; + continue; + } + int c = src[i + 2]; + int codepoint = -1; + if ((a & 0xF0) == 0xE0 && (b & 0xC0) == 0x80 && (c & 0xC0) == 0x80) { + codepoint = ((a & 0xF) << 12) | ((b & 0x3F) << 6) | (c & 0x3F); + i += 3; + } else { + int d = src[i + 3]; + if ((a & 0xF8) == 0xF0 && (b & 0xC0) == 0x80 && (c & 0xC0) == 0x80 && (d & 0xC0) == 0x80) { + codepoint = ((a & 7) << 18) | ((b & 0x3F) << 12) | ((c & 0x3F) << 6) | (d & 0x3F); + i += 4; + } + } + + if (codepoint <= 0xFFFF) { + if (codepoint < 0 || (codepoint >= 0xD800 && codepoint < 0xE000)) return -1; // [D800, E000) is illegal + dst[di++] = (char) codepoint; + continue; + } + + if (codepoint > 0x10FFFF) return -1; // illegal range + int v = codepoint - 0x10000; + dst[di + 0] = (char) (0xD800 + ((v >> 10) & 0x3FF)); + dst[di + 1] = (char) (0xDC00 + (v & 0x3FF)); + di += 2; + } + return i == offset + length ? di : fromUTF8Tail(dst, di, src, i, offset + length); + } + + public static String readString(PbjReader input, final long maxSize) { + return input.readString(maxSize); + } + /** * Read a Bytes field from data input * @@ -307,6 +494,15 @@ public static Bytes readBytes(final ReadableSequentialData input, final long max return bytes; } + public static Bytes readBytes(PbjReader input, final long maxSize) { + final int length = input.readVarIntNoZZ(); + if (length > maxSize || length < 0) { + input.setError(PbjReader.Parse); + return Bytes.EMPTY; + } + return input.readBytes(length); + } + /** * Reads a requested length-delimited protobuf field from the input and returns it as a * {@link Bytes} object. If the requested field is repeated or not length-delimited, this @@ -365,6 +561,34 @@ public static Bytes extractFieldBytes( return null; } + @Nullable + public static Bytes extractFieldBytes(@NonNull PbjReader input, @NonNull final FieldDefinition field) + throws IOException, ParseException { + Objects.requireNonNull(input); + Objects.requireNonNull(field); + if (field.repeated()) { + throw new IllegalArgumentException("Cannot extract field bytes for a repeated field: " + field); + } + if (ProtoWriterTools.wireType(field) != ProtoConstants.WIRE_TYPE_DELIMITED) { + throw new IllegalArgumentException("Cannot extract field bytes for a non-length-delimited field: " + field); + } + while (input.hasRemaining()) { + final int tag = input.readVarIntNoZZ(); + final int fieldNum = tag >> TAG_FIELD_OFFSET; + final ProtoConstants wireType = ProtoConstants.get(tag & ProtoConstants.TAG_WIRE_TYPE_MASK); + if (fieldNum == field.number()) { + if (wireType != ProtoConstants.WIRE_TYPE_DELIMITED) { + input.setError(PbjReader.Parse); + } + final int length = input.readVarIntNoZZ(); + return input.readBytes(length); + } else { + skipField(input, wireType); + } + } + return null; + } + /** * Extract the bytes in a stream for a given wire type. Assumes you have already read tag. * @@ -402,6 +626,42 @@ public static Bytes extractField( }; } + public static Bytes extractField(PbjReader input, final ProtoConstants wireType, final long maxSize) { + return switch (wireType) { + case WIRE_TYPE_FIXED_64_BIT -> input.readBytes(8); + case WIRE_TYPE_FIXED_32_BIT -> input.readBytes(4); + // The value for "zigZag" when calling varint doesn't matter because we are just reading past + // the varint, we don't care how to interpret it (zigzag is only used for interpretation of + // the bytes, not how many of them there are) + case WIRE_TYPE_VARINT_OR_ZIGZAG -> input.readVarLongBytes(); + case WIRE_TYPE_DELIMITED -> { + final Bytes lenBytes = input.readVarLongBytes(); + final int length = lenBytes.getVarInt(0, false); + if (length < 0) { + input.setError(PbjReader.IOError); + yield Bytes.EMPTY; + } + if (length > maxSize) { + input.setError(PbjReader.Parse); + yield Bytes.EMPTY; + } + yield Bytes.merge(lenBytes, input.readBytes(length)); + } + case WIRE_TYPE_GROUP_START -> { + input.setError(PbjReader.Unsupported); + yield Bytes.EMPTY; + } + case WIRE_TYPE_GROUP_END -> { + input.setError(PbjReader.Unsupported); + yield Bytes.EMPTY; + } + default -> { + input.setError(PbjReader.IOError); + yield Bytes.EMPTY; + } + }; + } + /** * Skip over the bytes in a stream for a given wire type. Assumes you have already read tag. * @@ -417,6 +677,10 @@ public static void skipField(final ReadableSequentialData input, final ProtoCons } } + public static void skipField(PbjReader input, final ProtoConstants wireType) { + skipField(input, wireType, Long.MAX_VALUE); + } + /** * Skip over the bytes in a stream for a given wire type. Assumes you have already read tag. * @@ -451,6 +715,30 @@ public static void skipField(final ReadableSequentialData input, final ProtoCons } } + public static void skipField(PbjReader input, final ProtoConstants wireType, final long maxSize) { + switch (wireType) { + case WIRE_TYPE_FIXED_64_BIT -> input.skip(8); + case WIRE_TYPE_FIXED_32_BIT -> input.skip(4); + // The value for "zigZag" when calling varint doesn't matter because we are just reading past + // the varint, we don't care how to interpret it (zigzag is only used for interpretation of + // the bytes, not how many of them there are) + case WIRE_TYPE_VARINT_OR_ZIGZAG -> input.readVarLongNoZZ(); + case WIRE_TYPE_DELIMITED -> { + final int length = input.readVarIntNoZZ(); + if (length < 0) { + input.setError(PbjReader.IOError); + } + if (length > maxSize) { + input.setError(PbjReader.Parse); + } + input.skip(length); + } + case WIRE_TYPE_GROUP_START -> input.setError(PbjReader.Unsupported); + case WIRE_TYPE_GROUP_END -> input.setError(PbjReader.Unsupported); + default -> input.setError(PbjReader.IOError); + } + } + /** * Read the next field number from the input * @@ -461,4 +749,9 @@ public static int readNextFieldNumber(final ReadableSequentialData input) { final int tag = input.readVarInt(false); return tag >> TAG_FIELD_OFFSET; } + + public static int readNextFieldNumber(PbjReader input) { + final int tag = input.readVarIntNoZZ(); + return tag >> TAG_FIELD_OFFSET; + } } diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoWriterTools.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoWriterTools.java index 0f6360b1..057cc17a 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoWriterTools.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/ProtoWriterTools.java @@ -3,6 +3,7 @@ import static com.hedera.pbj.runtime.ProtoConstants.*; +import com.hedera.pbj.runtime.io.PbjWriter; import com.hedera.pbj.runtime.io.WritableSequentialData; import com.hedera.pbj.runtime.io.buffer.Bytes; import com.hedera.pbj.runtime.io.buffer.RandomAccessData; @@ -10,7 +11,6 @@ import edu.umd.cs.findbugs.annotations.Nullable; import java.io.IOException; import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.function.Consumer; import java.util.function.ToIntFunction; @@ -59,6 +59,10 @@ public static void writeTag(final WritableSequentialData out, final FieldDefinit writeTag(out, field, wireType(field)); } + public static void writeTag(PbjWriter out, final FieldDefinition field) { + writeTag(out, field, wireType(field)); + } + /** * Write a protobuf tag to the output. * @@ -71,6 +75,10 @@ public static void writeTag( out.writeVarInt((field.number() << TAG_TYPE_BITS) | wireType.ordinal(), false); } + public static void writeTag(PbjWriter out, final FieldDefinition field, final ProtoConstants wireType) { + out.writeVarIntNoZZ((field.number() << TAG_TYPE_BITS) | wireType.ordinal()); + } + /** Create an unsupported field type exception */ private static RuntimeException unsupported() { return new RuntimeException("Unsupported field type. Bug in ProtoOutputStream, shouldn't happen."); @@ -90,6 +98,10 @@ public static void writeInteger(WritableSequentialData out, FieldDefinition fiel writeInteger(out, field, value, true); } + public static void writeInteger(PbjWriter out, FieldDefinition field, int value) { + writeInteger(out, field, value, true); + } + /** * Write a integer to data output * @@ -132,6 +144,39 @@ assert switch (field.type()) { } } + public static void writeInteger(PbjWriter out, FieldDefinition field, int value, boolean skipDefault) { + assert switch (field.type()) { + case INT32, UINT32, SINT32, FIXED32, SFIXED32 -> true; + default -> false; + } + : "Not an integer type " + field; + assert !field.repeated() : "Use writeIntegerList with repeated types"; + + if (skipDefault && !field.oneOf() && value == 0) { + return; + } + switch (field.type()) { + case INT32 -> { + writeTag(out, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + out.writeVarIntNoZZ(value); + } + case UINT32 -> { + writeTag(out, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + out.writeVarLongNoZZ(Integer.toUnsignedLong(value)); + } + case SINT32 -> { + writeTag(out, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + out.writeVarIntZZ(value); + } + case SFIXED32, FIXED32 -> { + // The bytes in protobuf are in little-endian order -- backwards for Java. + // Smallest byte first. + writeTag(out, field, WIRE_TYPE_FIXED_32_BIT); + out.writeIntLE(value); + } + default -> throw unsupported(); + } + } /** * Write a long to data output * @@ -143,6 +188,10 @@ public static void writeLong(WritableSequentialData out, FieldDefinition field, writeLong(out, field, value, true); } + public static void writeLong(PbjWriter out, FieldDefinition field, long value) { + writeLong(out, field, value, true); + } + /** * Write a long to data output * @@ -180,6 +229,35 @@ assert switch (field.type()) { } } + public static void writeLong(PbjWriter out, FieldDefinition field, long value, boolean skipDefault) { + assert switch (field.type()) { + case INT64, UINT64, SINT64, FIXED64, SFIXED64 -> true; + default -> false; + } + : "Not a long type " + field; + assert !field.repeated() : "Use writeLongList with repeated types"; + if (skipDefault && !field.oneOf() && value == 0) { + return; + } + switch (field.type()) { + case INT64, UINT64 -> { + writeTag(out, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + out.writeVarLongNoZZ(value); + } + case SINT64 -> { + writeTag(out, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + out.writeVarLongZZ(value); + } + case SFIXED64, FIXED64 -> { + // The bytes in protobuf are in little-endian order -- backwards for Java. + // Smallest byte first. + writeTag(out, field, WIRE_TYPE_FIXED_64_BIT); + out.writeLongLE(value); + } + default -> throw unsupported(); + } + } + /** * Write a float to data output * @@ -198,6 +276,17 @@ public static void writeFloat(WritableSequentialData out, FieldDefinition field, out.writeFloat(value, ByteOrder.LITTLE_ENDIAN); } + public static void writeFloat(PbjWriter out, FieldDefinition field, float value) { + assert field.type() == FieldType.FLOAT : "Not a float type " + field; + assert !field.repeated() : "Use writeFloatList with repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && value == 0) { + return; + } + writeTag(out, field, WIRE_TYPE_FIXED_32_BIT); + out.writeFloatLE(value); + } + /** * Write a double to data output * @@ -216,6 +305,17 @@ public static void writeDouble(WritableSequentialData out, FieldDefinition field out.writeDouble(value, ByteOrder.LITTLE_ENDIAN); } + public static void writeDouble(PbjWriter out, FieldDefinition field, double value) { + assert field.type() == FieldType.DOUBLE : "Not a double type " + field; + assert !field.repeated() : "Use writeDoubleList with repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && value == 0) { + return; + } + writeTag(out, field, WIRE_TYPE_FIXED_64_BIT); + out.writeDoubleLE(value); + } + /** * Write a boolean to data output * @@ -227,6 +327,10 @@ public static void writeBoolean(WritableSequentialData out, FieldDefinition fiel writeBoolean(out, field, value, true); } + public static void writeBoolean(PbjWriter out, FieldDefinition field, boolean value) { + writeBoolean(out, field, value, true); + } + /** * Write a boolean to data output * @@ -246,6 +350,16 @@ public static void writeBoolean( } } + public static void writeBoolean(PbjWriter out, FieldDefinition field, boolean value, boolean skipDefault) { + assert field.type() == FieldType.BOOL : "Not a boolean type " + field; + assert !field.repeated() : "Use writeBooleanList with repeated types"; + // In the case of oneOf we write the value even if it is default value of false + if (value || field.oneOf() || !skipDefault) { + writeTag(out, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + out.writeByte(value ? (byte) 1 : 0); + } + } + /** * Write a enum to data output * @@ -264,6 +378,17 @@ public static void writeEnum(WritableSequentialData out, FieldDefinition field, out.writeVarInt(enumValue.protoOrdinal(), false); } + public static void writeEnum(PbjWriter out, FieldDefinition field, EnumWithProtoMetadata enumValue) { + assert field.type() == FieldType.ENUM : "Not an enum type " + field; + assert !field.repeated() : "Use writeEnumList with repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && (enumValue == null || enumValue.protoOrdinal() == 0)) { + return; + } + writeTag(out, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + out.writeVarIntNoZZ(enumValue.protoOrdinal()); + } + /** * Write a enum protoOrdinal to data output. * @@ -282,6 +407,17 @@ public static void writeEnumProtoOrdinal(WritableSequentialData out, FieldDefini out.writeVarInt(protoOrdinal, false); } + public static void writeEnumProtoOrdinal(PbjWriter out, FieldDefinition field, int protoOrdinal) { + assert field.type() == FieldType.ENUM : "Not an enum type " + field; + assert !field.repeated() : "Use writeEnumList with repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && protoOrdinal == 0) { + return; + } + writeTag(out, field, WIRE_TYPE_VARINT_OR_ZIGZAG); + out.writeVarIntNoZZ(protoOrdinal); + } + /** * Write a string to data output, assuming the field is non-repeated. * @@ -295,6 +431,10 @@ public static void writeString(final WritableSequentialData out, final FieldDefi writeString(out, field, value, true); } + public static void writeString(PbjWriter out, final FieldDefinition field, final String value) { + writeString(out, field, value, true); + } + /** * Write a string to data output, assuming the field is non-repeated. * @@ -312,6 +452,13 @@ public static void writeString( writeStringNoChecks(out, field, value, skipDefault); } + public static void writeString( + PbjWriter out, final FieldDefinition field, final String value, boolean skipDefault) { + assert field.type() == FieldType.STRING : "Not a string type " + field; + assert !field.repeated() : "Use writeStringList with repeated types"; + writeStringNoChecks(out, field, value, skipDefault); + } + /** * Write a string to data output, assuming the field is repeated. Usually this method is called multiple * times, one for every repeated value. If all values are available immediately, {@link #writeStringList( @@ -329,6 +476,13 @@ public static void writeOneRepeatedString( writeStringNoChecks(out, field, value); } + public static void writeOneRepeatedString(PbjWriter out, final FieldDefinition field, final String value) + throws IOException { + assert field.type() == FieldType.STRING : "Not a string type " + field; + assert field.repeated() : "writeOneRepeatedString can only be used with repeated fields"; + writeStringNoChecks(out, field, value); + } + /** * Write a integer to data output - no validation checks. * @@ -342,6 +496,11 @@ private static void writeStringNoChecks( writeStringNoChecks(out, field, value, true); } + private static void writeStringNoChecks(PbjWriter out, final FieldDefinition field, final String value) + throws IOException { + writeStringNoChecks(out, field, value, true); + } + /** * Write a integer to data output - no validation checks. * @@ -363,6 +522,17 @@ private static void writeStringNoChecks( Utf8Tools.encodeUtf8(value, out); } + private static void writeStringNoChecks( + PbjWriter out, final FieldDefinition field, final String value, boolean skipDefault) { + // When not a oneOf don't write default value + if (skipDefault && !field.oneOf() && (value == null || value.isEmpty())) { + return; + } + writeTag(out, field, WIRE_TYPE_DELIMITED); + out.writeVarIntNoZZ(sizeOfStringNoTag(value)); + out.writeStringNoTag(value); + } + /** * Write a bytes to data output, assuming the corresponding field is non-repeated, and field type * is any delimited: bytes, string, or message. @@ -378,6 +548,10 @@ public static void writeBytes( writeBytes(out, field, value, true); } + public static void writeBytes(PbjWriter out, final FieldDefinition field, final RandomAccessData value) { + writeBytes(out, field, value, true); + } + /** * Write a bytes to data output, assuming the corresponding field is non-repeated, and field type * is any delimited: bytes, string, or message. @@ -399,6 +573,13 @@ public static void writeBytes( writeBytesNoChecks(out, field, value, skipDefault); } + public static void writeBytes( + PbjWriter out, final FieldDefinition field, final RandomAccessData value, boolean skipDefault) { + assert field.type() == FieldType.BYTES : "Not a byte[] type " + field; + assert !field.repeated() : "Use writeBytesList with repeated types"; + writeBytesNoChecks(out, field, value, skipDefault); + } + /** * Write a bytes to data output, assuming the corresponding field is repeated, and field type * is any delimited: bytes, string, or message. Usually this method is called multiple times, one @@ -418,6 +599,13 @@ public static void writeOneRepeatedBytes( writeBytesNoChecks(out, field, value, true); } + public static void writeOneRepeatedBytes(PbjWriter out, final FieldDefinition field, final RandomAccessData value) + throws IOException { + assert field.type() == FieldType.BYTES : "Not a byte[] type " + field; + assert field.repeated() : "writeOneRepeatedBytes can only be used with repeated fields"; + writeBytesNoChecks(out, field, value, true); + } + /** * Write a bytes to data output - no validation checks. * @@ -447,6 +635,27 @@ private static void writeBytesNoChecks( } } + private static void writeBytesNoChecks( + final PbjWriter out, + final FieldDefinition field, + final RandomAccessData value, + final boolean skipZeroLength) { + // When not a oneOf don't write default value + if (!field.oneOf() && (skipZeroLength && (value.length() == 0))) { + return; + } + writeTag(out, field, WIRE_TYPE_DELIMITED); + out.writeVarIntNoZZ(Math.toIntExact(value.length())); + final long posBefore = out.position(); + out.writeBytes(value); + final long bytesWritten = out.position() - posBefore; + if (bytesWritten != value.length()) { + out.setError( + PbjWriter.IOError, + "Wrote less bytes [" + bytesWritten + "] than expected [" + value.length() + "]"); + } + } + /** * Write a message to data output, assuming the corresponding field is non-repeated. * @@ -465,6 +674,13 @@ public static void writeMessage( writeMessageNoChecks(out, field, message, codec); } + public static void writeMessage( + PbjWriter out, final FieldDefinition field, final T message, final Codec codec) { + assert field.type() == FieldType.MESSAGE : "Not a message type " + field; + assert !field.repeated() : "Use writeMessageList with repeated types"; + writeMessageNoChecks(out, field, message, codec); + } + /** * Write a message to data output, assuming the corresponding field is repeated. Usually this method is * called multiple times, one for every repeated value. If all values are available immediately, {@link @@ -486,6 +702,13 @@ public static void writeOneRepeatedMessage( writeMessageNoChecks(out, field, message, codec); } + public static void writeOneRepeatedMessage( + PbjWriter out, final FieldDefinition field, final T message, final Codec codec) throws IOException { + assert field.type() == FieldType.MESSAGE : "Not a message type " + field; + assert field.repeated() : "writeOneRepeatedMessage can only be used with repeated fields"; + writeMessageNoChecks(out, field, message, codec); + } + /** * Write a message to data output - no validation checks. * @@ -513,6 +736,22 @@ private static void writeMessageNoChecks( } } + private static void writeMessageNoChecks( + PbjWriter out, final FieldDefinition field, final T message, final Codec codec) { + // When not a oneOf don't write default value + if (field.oneOf() && message == null) { + writeTag(out, field, WIRE_TYPE_DELIMITED); + out.writeVarIntNoZZ(0); + } else if (message != null) { + writeTag(out, field, WIRE_TYPE_DELIMITED); + final int size = codec.measureRecord(message); + out.writeVarIntNoZZ(size); + if (size > 0) { + codec.write(message, out); + } + } + } + public static void writeMap( final WritableSequentialData out, final FieldDefinition field, @@ -545,6 +784,38 @@ public static void writeMap( } } + public static void writeMap( + final PbjWriter out, + final FieldDefinition field, + @NonNull final PbjMap map, + final PbjProtoWriter kWriter, + final PbjProtoWriter vWriter, + final ToIntFunction sizeOfK, + final ToIntFunction sizeOfV) + throws IOException { + // https://protobuf.dev/programming-guides/proto3/#maps + // On the wire, a map is equivalent to: + // message MapFieldEntry { + // key_type key = 1; + // value_type value = 2; + // } + // repeated MapFieldEntry map_field = N; + if (map.isEmpty()) { + return; + } + final int size = map.size(); + for (int i = 0; i < size; i++) { + K k = map.getSortedKeys().get(i); + V v = map.get(k); + writeTag(out, field, WIRE_TYPE_DELIMITED); + final int sizeK = sizeOfK.applyAsInt(k); + final int sizeV = sizeOfV.applyAsInt(v); + out.writeVarIntNoZZ(sizeK + sizeV); + kWriter.write(k, out); + vWriter.write(v, out); + } + } + // ================================================================================================================ // OPTIONAL VERSIONS OF WRITE METHODS @@ -565,6 +836,15 @@ public static void writeOptionalInteger( } } + public static void writeOptionalInteger(PbjWriter out, FieldDefinition field, @Nullable Integer value) { + if (value != null) { + writeTag(out, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + out.writeVarIntNoZZ(sizeOfInteger(newField, value)); + writeInteger(out, newField, value); + } + } + /** * Write an optional long to data output * @@ -581,6 +861,15 @@ public static void writeOptionalLong(WritableSequentialData out, FieldDefinition } } + public static void writeOptionalLong(PbjWriter out, FieldDefinition field, @Nullable Long value) { + if (value != null) { + writeTag(out, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + out.writeVarIntNoZZ(sizeOfLong(newField, value)); + writeLong(out, newField, value); + } + } + /** * Write an optional float to data output * @@ -597,6 +886,15 @@ public static void writeOptionalFloat(WritableSequentialData out, FieldDefinitio } } + public static void writeOptionalFloat(PbjWriter out, FieldDefinition field, @Nullable Float value) { + if (value != null) { + writeTag(out, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + out.writeVarIntNoZZ(sizeOfFloat(newField, value)); + writeFloat(out, newField, value); + } + } + /** * Write an optional double to data output * @@ -613,6 +911,15 @@ public static void writeOptionalDouble(WritableSequentialData out, FieldDefiniti } } + public static void writeOptionalDouble(PbjWriter out, FieldDefinition field, @Nullable Double value) { + if (value != null) { + writeTag(out, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + out.writeVarIntNoZZ(sizeOfDouble(newField, value)); + writeDouble(out, newField, value); + } + } + /** * Write an optional boolean to data output * @@ -630,6 +937,15 @@ public static void writeOptionalBoolean( } } + public static void writeOptionalBoolean(PbjWriter out, FieldDefinition field, @Nullable Boolean value) { + if (value != null) { + writeTag(out, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + out.writeVarIntNoZZ(sizeOfBoolean(newField, value)); + writeBoolean(out, newField, value); + } + } + /** * Write an optional string to data output * @@ -648,6 +964,15 @@ public static void writeOptionalString(WritableSequentialData out, FieldDefiniti } } + public static void writeOptionalString(PbjWriter out, FieldDefinition field, @Nullable String value) { + if (value != null) { + writeTag(out, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + out.writeVarIntNoZZ(sizeOfString(newField, value)); + writeString(out, newField, value); + } + } + /** * Write an optional bytes to data output * @@ -669,6 +994,18 @@ public static void writeOptionalBytes(WritableSequentialData out, FieldDefinitio } } + public static void writeOptionalBytes(PbjWriter out, FieldDefinition field, @Nullable Bytes value) { + if (value != null) { + writeTag(out, field, WIRE_TYPE_DELIMITED); + final var newField = field.type().optionalFieldDefinition; + final int size = sizeOfBytes(newField, value); + out.writeVarIntNoZZ(size); + if (size > 0) { + writeBytes(out, newField, value); + } + } + } + // ================================================================================================================ // LIST VERSIONS OF WRITE METHODS @@ -747,6 +1084,189 @@ assert switch (field.type()) { } } + public static void writeIntegerList(PbjWriter out, FieldDefinition field, List list) { + assert switch (field.type()) { + case INT32, UINT32, SINT32, FIXED32, SFIXED32 -> true; + default -> false; + } + : "Not an integer type " + field; + assert field.repeated() : "Use writeInteger with non-repeated types"; + + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return; + } + + writeTag(out, field, WIRE_TYPE_DELIMITED); + switch (field.type()) { + case INT32 -> writeInt32List(out, list); + case UINT32 -> writeUInt32List(out, list); + case SINT32 -> writeSInt32List(out, list); + case SFIXED32, FIXED32 -> writeFixed32List(out, list); + default -> throw unsupported(); + } + } + + private static void writeInt32List(PbjWriter out, List list) { + int listSize = list.size(); + if (listSize > 0x7F) { + writeInt32ListLarge(out, list); + return; + } + out.reserveRel(0x7F * 10 + 2); // worst case + int pos = out.position(); + out.placehold(1); + for (int i = 0; i < listSize; i++) { + out.writeVarIntNoZZ(list.get(i)); + } + int size = out.position() - pos - 1; + if (size <= 0x7F) { + out.writeAtUnsafe(pos, (byte) size); + } else { + out.reinsertVarInt(pos); + } + } + + private static void writeInt32ListTwoPass(PbjWriter out, List list) { + int listSize = list.size(); + int size = 0; + for (int i = 0; i < listSize; i++) { + int val = list.get(i); + size += ((val & ~0x7FL) == 0) ? 1 : sizeOfVarInt32(val); + } + out.writeVarIntNoZZ(size); + for (int i = 0; i < listSize; i++) { + out.writeVarIntNoZZ(list.get(i)); + } + } + + private static void writeInt32ListLarge(PbjWriter out, List list) { + int listSize = list.size(); + if (listSize > 1638) { + // 1639+ elements * 10 bytes worst case exceeds 16k buffer; fall back to two-pass + writeInt32ListTwoPass(out, list); + return; + } + out.reserveRel(listSize * 10 + 2); + int pos = out.position(); + out.placehold(2); + for (int i = 0; i < listSize; i++) { + out.writeVarIntNoZZ(list.get(i)); + } + int size = out.position() - pos - 2; + out.writeAtUnsafe(pos, (byte) ((size & 0x7F) | 0x80)); + out.writeAtUnsafe(pos + 1, (byte) (size >>> 7)); + } + + private static void writeUInt32List(PbjWriter out, List list) { + int listSize = list.size(); + if (listSize > 0x7F) { + writeUInt32ListLarge(out, list); + return; + } + out.reserveRel(0x7F * 5 + 2); // worst case + int pos = out.position(); + out.placehold(1); + for (int i = 0; i < listSize; i++) { + out.writeVarLongNoZZ(Integer.toUnsignedLong(list.get(i))); + } + int size = out.position() - pos - 1; + if (size <= 0x7F) { + out.writeAtUnsafe(pos, (byte) size); + } else { + out.reinsertVarInt(pos); + } + } + + private static void writeUInt32ListTwoPass(PbjWriter out, List list) { + int listSize = list.size(); + int size = 0; + for (int i = 0; i < listSize; i++) { + size += sizeOfUnsignedVarInt64(Integer.toUnsignedLong(list.get(i))); + } + out.writeVarIntNoZZ(size); + for (int i = 0; i < listSize; i++) { + out.writeVarLongNoZZ(Integer.toUnsignedLong(list.get(i))); + } + } + + private static void writeUInt32ListLarge(PbjWriter out, List list) { + int listSize = list.size(); + if (listSize > 3276) { + // 3277+ elements * 5 bytes worst case exceeds 16k buffer + writeUInt32ListTwoPass(out, list); + return; + } + out.reserveRel(listSize * 5 + 2); + int pos = out.position(); + out.placehold(2); + for (int i = 0; i < listSize; i++) { + out.writeVarLongNoZZ(Integer.toUnsignedLong(list.get(i))); + } + int size = out.position() - pos - 2; + out.writeAtUnsafe(pos, (byte) ((size & 0x7F) | 0x80)); + out.writeAtUnsafe(pos + 1, (byte) (size >>> 7)); + } + + private static void writeSInt32List(PbjWriter out, List list) { + int listSize = list.size(); + if (listSize > 0x7F) { + writeSInt32ListLarge(out, list); + return; + } + out.reserveRel(0x7F * 5 + 2); // worst case + int pos = out.position(); + out.placehold(1); + for (int i = 0; i < listSize; i++) { + out.writeVarIntZZ(list.get(i)); + } + int size = out.position() - pos - 1; + if (size <= 0x7F) { + out.writeAtUnsafe(pos, (byte) size); + } else { + out.reinsertVarInt(pos); + } + } + + private static void writeSInt32ListTwoPass(PbjWriter out, List list) { + int listSize = list.size(); + int size = 0; + for (int i = 0; i < listSize; i++) { + final int val = list.get(i); + size += sizeOfUnsignedVarInt64(((long) val << 1) ^ ((long) val >> 63)); + } + out.writeVarIntNoZZ(size); + for (int i = 0; i < listSize; i++) { + out.writeVarIntZZ(list.get(i)); + } + } + + private static void writeSInt32ListLarge(PbjWriter out, List list) { + int listSize = list.size(); + if (listSize > 3276) { + // 3277+ elements * 5 bytes worst case exceeds 16k buffer + writeSInt32ListTwoPass(out, list); + return; + } + out.reserveRel(listSize * 5 + 2); + int pos = out.position(); + out.placehold(2); + for (int i = 0; i < listSize; i++) { + out.writeVarIntZZ(list.get(i)); + } + int size = out.position() - pos - 2; + out.writeAtUnsafe(pos, (byte) ((size & 0x7F) | 0x80)); + out.writeAtUnsafe(pos + 1, (byte) (size >>> 7)); + } + + private static void writeFixed32List(PbjWriter out, List list) { + // The bytes in protobuf are in little-endian order -- backwards for Java. + out.writeVarLongNoZZ((long) list.size() * FIXED32_SIZE); + for (int i = 0; i < list.size(); i++) { + out.writeIntLE(list.get(i)); + } + } + /** * Write a list of longs to data output * @@ -809,6 +1329,61 @@ assert switch (field.type()) { } } + public static void writeLongList(PbjWriter out, FieldDefinition field, List list) { + assert switch (field.type()) { + case INT64, UINT64, SINT64, FIXED64, SFIXED64 -> true; + default -> false; + } + : "Not a long type " + field; + assert field.repeated() : "Use writeLong with non-repeated types"; + + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return; + } + + final int listSize = list.size(); + switch (field.type()) { + case INT64, UINT64 -> { + int size = 0; + for (int i = 0; i < listSize; i++) { + final long val = list.get(i); + size += sizeOfUnsignedVarInt64(val); + } + writeTag(out, field, WIRE_TYPE_DELIMITED); + out.writeVarIntNoZZ(size); + for (int i = 0; i < listSize; i++) { + final long val = list.get(i); + out.writeVarLongNoZZ(val); + } + } + case SINT64 -> { + int size = 0; + for (int i = 0; i < listSize; i++) { + final long val = list.get(i); + size += sizeOfUnsignedVarInt64((val << 1) ^ (val >> 63)); + } + writeTag(out, field, WIRE_TYPE_DELIMITED); + out.writeVarIntNoZZ(size); + for (int i = 0; i < listSize; i++) { + final long val = list.get(i); + out.writeVarLongZZ(val); + } + } + case SFIXED64, FIXED64 -> { + // The bytes in protobuf are in little-endian order -- backwards for Java. + // Smallest byte first. + writeTag(out, field, WIRE_TYPE_DELIMITED); + out.writeVarLongNoZZ((long) list.size() * FIXED64_SIZE); + for (int i = 0; i < listSize; i++) { + final long val = list.get(i); + out.writeLongLE(val); + } + } + default -> throw unsupported(); + } + } + /** * Write a list of floats to data output * @@ -832,6 +1407,22 @@ public static void writeFloatList(WritableSequentialData out, FieldDefinition fi } } + public static void writeFloatList(PbjWriter out, FieldDefinition field, List list) { + assert field.type() == FieldType.FLOAT : "Not a float type " + field; + assert field.repeated() : "Use writeFloat with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return; + } + final int size = list.size() * FIXED32_SIZE; + writeTag(out, field, WIRE_TYPE_DELIMITED); + out.writeVarIntNoZZ(size); + final int listSize = list.size(); + for (int i = 0; i < listSize; i++) { + out.writeFloatLE(list.get(i)); + } + } + /** * Write a list of doubles to data output * @@ -855,6 +1446,22 @@ public static void writeDoubleList(WritableSequentialData out, FieldDefinition f } } + public static void writeDoubleList(PbjWriter out, FieldDefinition field, List list) { + assert field.type() == FieldType.DOUBLE : "Not a double type " + field; + assert field.repeated() : "Use writeDouble with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return; + } + final int size = list.size() * FIXED64_SIZE; + writeTag(out, field, WIRE_TYPE_DELIMITED); + out.writeVarIntNoZZ(size); + final int listSize = list.size(); + for (int i = 0; i < listSize; i++) { + out.writeDoubleLE(list.get(i)); + } + } + /** * Write a list of booleans to data output * @@ -879,6 +1486,23 @@ public static void writeBooleanList(WritableSequentialData out, FieldDefinition } } + public static void writeBooleanList(PbjWriter out, FieldDefinition field, List list) { + assert field.type() == FieldType.BOOL : "Not a boolean type " + field; + assert field.repeated() : "Use writeBoolean with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return; + } + // write + writeTag(out, field, WIRE_TYPE_DELIMITED); + out.writeVarIntNoZZ(list.size()); + final int listSize = list.size(); + for (int i = 0; i < listSize; i++) { + final boolean b = list.get(i); + out.writeVarIntNoZZ(b ? 1 : 0); + } + } + /** * Write a list of enums to data output * @@ -906,6 +1530,25 @@ public static void writeEnumListProtoOrdinals( } } + public static void writeEnumListProtoOrdinals(PbjWriter out, FieldDefinition field, List list) { + assert field.type() == FieldType.ENUM : "Not an enum type " + field; + assert field.repeated() : "Use writeEnum with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return; + } + final int listSize = list.size(); + int size = 0; + for (int i = 0; i < listSize; i++) { + size += sizeOfUnsignedVarInt32(list.get(i)); + } + writeTag(out, field, WIRE_TYPE_DELIMITED); + out.writeVarIntNoZZ(size); + for (int i = 0; i < listSize; i++) { + out.writeVarIntNoZZ(list.get(i)); + } + } + /** * Write a list of strings to data output * @@ -931,6 +1574,21 @@ public static void writeStringList(WritableSequentialData out, FieldDefinition f } } + public static void writeStringList(PbjWriter out, FieldDefinition field, List list) { + assert field.type() == FieldType.STRING : "Not a string type " + field; + assert field.repeated() : "Use writeString with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return; + } + final int listSize = list.size(); + for (int i = 0; i < listSize; i++) { + final String value = list.get(i); + writeTag(out, field, WIRE_TYPE_DELIMITED); + out.writeStringWithTag(value); + } + } + /** * Write a list of messages to data output * @@ -955,6 +1613,19 @@ public static void writeMessageList( } } + public static void writeMessageList(PbjWriter out, FieldDefinition field, List list, Codec codec) { + assert field.type() == FieldType.MESSAGE : "Not a message type " + field; + assert field.repeated() : "Use writeMessage with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return; + } + final int listSize = list.size(); + for (int i = 0; i < listSize; i++) { + writeMessageNoChecks(out, field, list.get(i), codec); + } + } + /** * Write a list of bytes objects to data output * @@ -978,6 +1649,19 @@ public static void writeBytesList( } } + public static void writeBytesList(PbjWriter out, FieldDefinition field, List list) { + assert field.type() == FieldType.BYTES : "Not a message type " + field; + assert field.repeated() : "Use writeBytes with non-repeated types"; + // When not a oneOf don't write default value + if (!field.oneOf() && list.isEmpty()) { + return; + } + final int listSize = list.size(); + for (int i = 0; i < listSize; i++) { + writeBytesNoChecks(out, field, list.get(i), false); + } + } + /** * Write a generic delimited field by delegating to a supplied `writer` to write the actual elements. * @@ -985,7 +1669,6 @@ public static void writeBytesList( * @param field the descriptor for the field we are writing * @param size the size of all the elements together, in bytes * @param writer the Consumer that accepts the `out` and writes the actual elements - * @param the type of the data output that extends WritableSequentialData */ public static void writeDelimited( final T out, final FieldDefinition field, final int size, final Consumer writer) { @@ -994,6 +1677,13 @@ public static void writeDelimited( writer.accept(out); } + public static void writeDelimited( + PbjWriter out, final FieldDefinition field, final int size, final Consumer writer) { + writeTag(out, field); + out.writeVarInt(size, false); + writer.accept(out); + } + // ================================================================================================================ // SIZE OF METHODS @@ -1340,16 +2030,12 @@ public static int sizeOfString(FieldDefinition field, String value, boolean skip * @param value string value to get encoded size for * @return the number of bytes for encoded value */ - static int sizeOfStringNoTag(String value) { + public static int sizeOfStringNoTag(String value) { // When not a oneOf don't write default value if ((value == null || value.isEmpty())) { return 0; } - try { - return Utf8Tools.encodedLength(value); - } catch (IOException e) { // fall back to JDK - return value.getBytes(StandardCharsets.UTF_8).length; - } + return Utf8Tools.encodedLength(value); } /** diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Utf8Tools.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Utf8Tools.java index c757c892..0102c6e0 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Utf8Tools.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/Utf8Tools.java @@ -3,6 +3,7 @@ import static java.lang.Character.*; +import com.hedera.pbj.runtime.io.PbjWriter; import com.hedera.pbj.runtime.io.WritableSequentialData; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; @@ -12,6 +13,35 @@ */ public final class Utf8Tools { + // return length or 0 on error + static int encodedLength(String sz) { + if (sz == null) return 0; + int count = 0; + for (int i = 0; i < sz.length(); i++) { + char c = sz.charAt(i); + if (c < 0x80) { + count += 1; + } else if (c < 0x800) { + count += 2; + } else if (c < 0xD800 || c >= 0xE000) { + count += 3; + } else if (c <= 0xDBFF) { // high surrogate: D800–DBFF + if (i + 1 >= sz.length()) { + throw new MalformedUtf8Exception("Illegal Encoding at %d".formatted(i)); + } + char low = sz.charAt(i + 1); + if (low < 0xDC00 || low > 0xDFFF) { + throw new MalformedUtf8Exception("Illegal Encoding at %d".formatted(i)); + } + i++; + count += 4; + } else { + return 0; + } + } + return count; + } + /** * Returns the number of bytes in the UTF-8-encoded form of {@code sequence}. For a string, this * method is equivalent to {@code string.getBytes(UTF_8).length}, but is more efficient in both @@ -115,6 +145,10 @@ static void encodeUtf8(final CharSequence in, final WritableSequentialData out) } } + static void writeUTF8(String str, PbjWriter out) { + out.writeStringWithTag(str); + } + /** * Encodes the input character sequence to a byte array using the same algorithm as protoc, so we are byte for * byte the same. Returns the number of bytes written. diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/ByteArraySequentialData.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/ByteArraySequentialData.java new file mode 100644 index 00000000..6738167e --- /dev/null +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/ByteArraySequentialData.java @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.pbj.runtime.io; + +/** + * Implemented by {@link ReadableSequentialData} types that are backed directly by a heap byte array. + * Callers (e.g. {@link PbjReader}) can detect this and use the array directly instead of copying + * data through an intermediate buffer. + */ +public interface ByteArraySequentialData { + /** The raw backing byte array. */ + byte[] byteArrayUnsafe(); + + /** Absolute index within {@link #byteArrayUnsafe()} where currently readable data begins. */ + int byteArrayUnsafeOffset(); + + /** Absolute exclusive index within {@link #byteArrayUnsafe()} where currently readable data ends. */ + int byteArrayUnsafeEnd(); +} diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/PbjReader.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/PbjReader.java new file mode 100644 index 00000000..87caf910 --- /dev/null +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/PbjReader.java @@ -0,0 +1,1022 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.pbj.runtime.io; + +import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.ProtoParserTools; +import com.hedera.pbj.runtime.UnknownFieldException; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.BufferOverflowException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; + +/** + * A buffer-backed reader for decoding data. Not thread safe. + * + *

PbjReader maintains an internal byte buffer and reads from a {@link ReadableSequentialData}, + * {@link InputStream}, or a byte array. When backed by a stream the internal buffer is refilled + * on demand as data is consumed. A stream will never be read past its limit. + * + *

Errors are tracked internally rather than thrown immediately. Call {@link #error()} to check + * for a pending error code, or {@link #throwOnError()} to surface it as a {@link ParseException}. + * Once an error is set it is sticky — subsequent reads return default values (zero or empty). + * EOF is not an error and will return 0 when error() is called. + * ByteBuffer using a direct buffer is not supported + */ +public class PbjReader { + private byte[] buf; + private int pos, end; + private int relLimit, err; + private long absoluteLimit = Long.MAX_VALUE, offset; + private ReadableSequentialData rsd; + private InputStream stream; + private Exception cause; + private boolean seenEOF, includeCause; + private byte[] ownedBuf; + + private char[] charArray; + private static final boolean useStacktrace = + !"false".equalsIgnoreCase(System.getProperty("pbj.ReaderWriter.useStackTrace")); + + public static final int EOF = -1, + DataEncoding = 1, + BufferUnderflow = 2, + Parse = 3, + IllegalArgument = 4, + IOError = 5, + Unsupported = 6, // used with WIRE_TYPE_GROUP_START, WIRE_TYPE_GROUP_END + UsageError = 9, + UnknownField = 10, + BufferOverflow = 11, + MaxDepthReached = 12, + // For PbjWriter + Closed = -2, + MalformString = 13; + + private static final UnknownFieldException premadeUnknown; + private static final BufferUnderflowException premadeUnderflow; + private static final BufferOverflowException premadeOverflow; + private static final RuntimeException premadeRuntime, premadeUnsupported; + private static final DataEncodingException premadeDataEncoding; + private static final IllegalArgumentException premadeIllegal; + private static final ParseException premadeParseEmpty, premadeParseUnknown, premadeMaxDepth; + + static { + premadeUnknown = new UnknownFieldException(""); + premadeUnderflow = new BufferUnderflowException(); + premadeOverflow = new BufferOverflowException(); + premadeRuntime = new RuntimeException(); + premadeUnsupported = new RuntimeException("Hit an unsupported feature"); + premadeDataEncoding = new DataEncodingException(""); + premadeIllegal = new IllegalArgumentException(""); + + premadeParseEmpty = new ParseException("parse error"); + premadeParseUnknown = new ParseException("parse error", premadeUnknown); + premadeMaxDepth = new ParseException("Reached maximum allowed depth"); + } + + private void construct(byte[] buffer, int position, int endPosition) { + buf = buffer; + pos = position; + end = endPosition; + relLimit = end; + absoluteLimit = end; + seenEOF = true; + err = EOF; + } + + /** + * Resets this reader to read from the given byte array slice, discarding any previous state. + * + * @param buffer the backing byte array + * @param position the inclusive start index within {@code buffer} + * @param endPosition the exclusive end index within {@code buffer} + */ + public void resetWith(byte[] buffer, int position, int endPosition) { + err = 0; + offset = 0; + cause = null; + includeCause = false; + rsd = null; + stream = null; + construct(buffer, position, endPosition); + } + + private void construct(ReadableSequentialData seq, InputStream inputStream) { + if (seq instanceof ByteArraySequentialData ba && ba.byteArrayUnsafe() != null) { + construct(ba.byteArrayUnsafe(), ba.byteArrayUnsafeOffset(), ba.byteArrayUnsafeEnd()); + return; + } else if (seq != null) { + rsd = seq; + absoluteLimit = seq.limit(); + offset = (int) seq.position(); + } else if (inputStream != null) { + stream = inputStream; + } + ownedBuf = buf = new byte[16 << 10]; // 16k is friendly to x86-64 L1 cache + } + + private void resetWith(ReadableSequentialData seq, InputStream inputStream) { + err = 0; + cause = null; + includeCause = false; + if ((seq == null && inputStream == null) || (seq != null && inputStream != null)) { + setError(UsageError); + return; + } + + if (seq instanceof ByteArraySequentialData ba && ba.byteArrayUnsafe() != null) { + resetWith(ba.byteArrayUnsafe(), ba.byteArrayUnsafeOffset(), ba.byteArrayUnsafeEnd()); + return; + } + if (seq != null) { + rsd = seq; + stream = null; + absoluteLimit = seq.limit(); + offset = (int) seq.position(); + } else if (inputStream != null) { + rsd = null; + stream = inputStream; + absoluteLimit = Long.MAX_VALUE; + offset = 0; + } + if (ownedBuf == null) { + ownedBuf = new byte[16 << 10]; // 16k is friendly to x86-64 L1 cache + } + buf = ownedBuf; + pos = 0; + end = 0; + relLimit = 0; + seenEOF = false; + } + + /** Creates a reader backed by the given {@link ReadableSequentialData}. */ + public PbjReader(ReadableSequentialData seq) { + construct(seq, null); + } + /** Creates a reader backed by the given {@link InputStream}. */ + public PbjReader(InputStream inputStream) { + construct(null, inputStream); + } + /** Creates a reader backed by the given byte array. */ + public PbjReader(byte[] data) { + construct(data, 0, data.length); + } + /** + * Creates a reader backed by the given byte array slice. + * + * @param data the backing byte array + * @param offset the inclusive start index + * @param end the exclusive end index + */ + public PbjReader(byte[] data, int offset, int end) { + construct(data, offset, end); + } + /** Creates a reader backed by the given {@link ByteBuffer}. */ + public PbjReader(ByteBuffer bb) { + int position = bb.arrayOffset() + bb.position(); + construct(bb.array(), position, position + bb.remaining()); + } + + /** Creates a reader backed by the given {@link Bytes}. */ + public PbjReader(Bytes bytes) { + construct(bytes.arrayUnsafe(), bytes.arrayUnsafeOffset(), bytes.arrayUnsafeOffset() + (int) bytes.length()); + } + + /** Resets this reader to read from the given {@link ReadableSequentialData}. */ + public void resetWith(ReadableSequentialData seq) { + resetWith(seq, null); + } + /** Resets this reader to read from the given {@link InputStream}. */ + public void resetWith(InputStream inputStream) { + resetWith(null, inputStream); + } + /** Resets this reader to read from the given byte array. */ + public void resetWith(byte[] data) { + resetWith(data, 0, data.length); + } + /** Resets this reader to read from the given {@link ByteBuffer}. */ + public void resetWith(ByteBuffer bb) { + int position = bb.arrayOffset() + bb.position(); + resetWith(bb.array(), position, position + bb.remaining()); + } + /** Resets this reader to read from the given {@link Bytes}. */ + public void resetWith(Bytes bytes) { + resetWith(bytes.arrayUnsafe(), bytes.arrayUnsafeOffset(), bytes.arrayUnsafeOffset() + (int) bytes.length()); + } + + private void bufferMore(int relAmount) { + if (err != 0) return; + offset += pos; + if (pos < end && pos != 0) { + int moveLen = end - pos; + System.arraycopy(buf, pos, buf, 0, end - pos); + end = moveLen; + } else if (pos == end) { + end = 0; + } + pos = 0; + int rdlen = readFromInput(buf, end, buf.length - end); + end += rdlen; + relLimit = (int) Math.min(absoluteLimit - offset, end); + if (rdlen == 0) { + seenEOF = true; + err = EOF; + } + } + + /** + * Returns {@code true} if there are bytes remaining to be read. + * For streaming readers, triggers a buffer refill if the local buffer is exhausted. + * + * @return {@code true} if at least one byte can be read + */ + public boolean hasRemaining() { + // small and likely to inline + if (pos < relLimit) return true; + if (offset + pos == absoluteLimit) return false; + return hasRemainingInternal(); + } + // still small, but less likely to hit this case in steaming, and only once when not streaming + private boolean hasRemainingInternal() { + if (seenEOF) return false; + bufferMore(1); + return pos < relLimit; + } + + /** + * Returns the absolute byte limit beyond which reading is not permitted. + * + * @return the absolute limit position + */ + public long limit() { + return absoluteLimit; + } + + /** + * Sets the absolute byte limit beyond which reading is not permitted. + * If using a stream, it will not read past that limit + * + * @param limit the new limit position + */ + public void limit(long limit) { + absoluteLimit = limit; + if (rsd != null) { + rsd.limit(limit); + } + if (err > 0) return; // keep relLimit -1 in error state + relLimit = (int) Math.min(absoluteLimit - offset, end); + } + + /** + * Returns the current absolute read position, accounting for any bytes already consumed + * from the internal buffer plus any previously buffered data. + * + * @return the current read position + */ + public long position() { + return pos + offset; + } + + /** + * Skips over {@code count} bytes, advancing the read position without returning the data. + * Sets {@link #BufferUnderflow} if there are fewer than {@code count} bytes remaining. + * + * @param count the number of bytes to skip + */ + public void skip(int count) { + if (count >= 0 && pos + count <= relLimit) { + pos += count; + return; + } + skipInternal(count); + } + + private void skipInternal(int count) { + if (seenEOF) { + setError(BufferUnderflow); + return; + } + + int skippedInBuffer = relLimit - pos; + count -= skippedInBuffer; + pos = relLimit; + offset += count; + if (stream != null) { + try { + long remaining = count; + while (remaining > 0) { + long skipped = stream.skip(remaining); + if (skipped > 0) { + remaining -= skipped; + } else { + // Docs suggest skup can return 0 w/o reaching EOF + int b = stream.read(); + if (b == -1) break; + remaining--; + } + } + if (remaining != 0) { + setError(BufferUnderflow); + } + } catch (IOException e) { + setError(IOError); + } + } else { + rsd.skip(count); // may throw + } + } + + /** + * Reads a base-128 varint and returns its value as an {@code int} (no zigzag decoding). + * On a malformed varint, sets the error flag and returns {@code -1}; however, + * {@code -1} is also a valid decoded value, so callers must check {@link #error()} to + * distinguish an error from a legitimate result. + * + * @return the decoded integer value + */ + public int readVarIntNoZZ() { + return (int) readVarLongNoZZ(); + } + + /** + * Reads a base-128 varint and returns its value as an {@code int}, with optional zigzag decoding. + * On a malformed varint, sets the error flag and returns {@code -1}; however, + * {@code -1} is also a valid decoded value, so callers must check {@link #error()} to + * distinguish an error from a legitimate result. + * + * @param zigZag if {@code true}, decodes using zigzag: {@code (n >>> 1) ^ -(n & 1)} + * @return the decoded integer value + */ + public int readVarInt(boolean zigZag) { + return (int) readVarLong(zigZag); + } + + /** + * Reads a base-128 varint and returns its zigzag-decoded value as an {@code int}. + * Zigzag decoding maps the raw unsigned value {@code n} to {@code (n >>> 1) ^ -(n & 1)}. + * On a malformed varint, sets the error flag and returns {@code -1}; however, + * {@code -1} is also a valid decoded value, so callers must check {@link #error()} to + * distinguish an error from a legitimate result. + * + * @return the decoded integer value + */ + public int readVarIntZZ() { + return (int) readVarLongZZ(); + } + + /** + * Reads a base-128 varint and returns its value as a {@code long}. + * On a malformed varint (more than 10 bytes), sets {@link #DataEncoding} and returns + * {@code -1}; however, {@code -1} is also a valid decoded value, so callers must check + * {@link #error()} to distinguish an error from a legitimate result. + * + * @param zigZag if {@code true}, decodes using zigzag: {@code (n >>> 1) ^ -(n & 1)} + * @return the decoded long value + */ + public long readVarLong(boolean zigZag) { + long value = readVarLongNoZZ(); + return zigZag ? (value >>> 1) ^ -(value & 1) : value; + } + + /** + * Reads a base-128 varint and returns its zigzag-decoded value as a {@code long}. + * Zigzag decoding maps the raw unsigned value {@code n} to {@code (n >>> 1) ^ -(n & 1)}. + * On a malformed varint, sets the error flag and returns {@code -1}; however, + * {@code -1} is also a valid decoded value, so callers must check {@link #error()} to + * distinguish an error from a legitimate result. + * + * @return the decoded long value + */ + public long readVarLongZZ() { + long value = readVarLongNoZZ(); + return (value >>> 1) ^ -(value & 1); + } + + /** + * Reads a base-128 varint and returns its value as a {@code long} (no zigzag decoding). + * On a malformed varint (more than 10 bytes), sets {@link #DataEncoding} and returns + * {@code -1}; however, {@code -1} is also a valid decoded value, so callers must check + * {@link #error()} to distinguish an error from a legitimate result. + * + * @return the decoded long value + */ + public long readVarLongNoZZ() { + if (pos + 10 <= relLimit) { + long value = 0; + for (int i = 0; i < 10; i++) { + byte b = buf[pos++]; + value |= (long) (b & 0x7F) << (i * 7); + if (b >= 0) { + return value; + } + } + setError(DataEncoding); + return -1; + } + return readVarLongNoZZInternal(); + } + + private long readVarLongNoZZInternal() { + long value = 0; + for (int i = 0; i < 10; i++) { + byte b = readByte(); + value |= (long) (b & 0x7F) << (i * 7); + if (b >= 0) { + return value; + } + } + setError(DataEncoding); + return -1; + } + + /** + * Reads a base-128 varint and returns the raw encoded bytes without decoding them. + * Sets {@link #DataEncoding} if the varint is malformed. + * + * @return the raw varint bytes, or {@link Bytes#EMPTY} on error + */ + public Bytes readVarLongBytes() { + byte[] bytes = new byte[10]; + if (pos + 10 <= relLimit) { + for (int i = 0; i < 10; i++) { + bytes[i] = readByte(); + if (bytes[i] >= 0) { + return Bytes.wrap(bytes, 0, i + 1); + } + } + setError(DataEncoding); + return Bytes.EMPTY; + } + return readVarLongBytesInternal(bytes); + } + + private Bytes readVarLongBytesInternal(byte[] bytes) { + for (int i = 0; i < 10; i++) { + bytes[i] = readByte(); + if (bytes[i] >= 0) { + return Bytes.wrap(bytes, 0, i + 1); + } + } + setError(DataEncoding); + return Bytes.EMPTY; + } + + /** + * Records an error on this reader with no message. Once an error is recorded + * subsequent reads return default values (zero or empty). + * + * @param errorKind one of the error-code constants ({@link #DataEncoding}, {@link #BufferUnderflow}, etc.) + */ + public void setError(int errorKind) { + setError(errorKind, ""); + } + + /** + * Records an error on this reader if no previous error is set. Once an error is recorded + * subsequent reads return default values (zero or empty). + * + * @param errorKind one of the error-code constants ({@link #DataEncoding}, {@link #BufferUnderflow}, etc.) + * @param message a detail message associated with the error + */ + public void setError(int errorKind, String message) { + if (err > 0) return; // if an error exists, don't overwrite + err = errorKind; + relLimit = -1; + seenEOF = true; + // TODO simplify when exceptions are not required + includeCause = true; + if (useStacktrace) { + if (errorKind == UnknownField) { + cause = new UnknownFieldException(message); + } else if (errorKind == BufferUnderflow) { + cause = new BufferUnderflowException(); + } else if (errorKind == BufferOverflow) { + cause = new BufferOverflowException(); + } else if (errorKind == Parse) { + cause = new RuntimeException(message); + includeCause = false; + } else { + cause = new RuntimeException(message); + } + } else { + if (errorKind == UnknownField) { + cause = premadeUnknown; + } else if (errorKind == BufferUnderflow) { + cause = premadeUnderflow; + } else if (errorKind == BufferOverflow) { + cause = premadeOverflow; + } else if (errorKind == Parse) { + cause = premadeParseEmpty; + includeCause = false; + } else { + cause = premadeRuntime; + } + } + } + + /** + * Returns {@code true} if no error is set; otherwise throws a {@link ParseException}. + * + * @return {@code true} if no error has occurred + * @throws ParseException if an error is set + */ + public boolean throwOnErrorOrTrue() throws ParseException { + if (err <= 0) return true; + throw throwOnErrorImpl(); + } + + /** + * Throws a {@link ParseException} if an error is set; otherwise does nothing. + * + * @throws ParseException if {@link #error()} is non-zero + */ + public void throwOnError() throws ParseException { + if (err <= 0) return; + throw throwOnErrorImpl(); + } + + private ParseException throwOnErrorImpl() throws ParseException { + if (useStacktrace) { + switch (err) { + case DataEncoding: + throw new DataEncodingException("throwOnError", cause); + case IllegalArgument: + throw new IllegalArgumentException("throwOnError", cause); + case Unsupported: + throw new RuntimeException("Hit an unsupported feature", cause); + case MaxDepthReached: + throw new ParseException("Reached maximum allowed depth"); + case Parse: + default: + if (includeCause) throw new ParseException(cause); + ParseException ex = new ParseException(""); + ex.setStackTrace(cause.getStackTrace()); + throw ex; + } + } else { + switch (err) { + case DataEncoding: + throw premadeDataEncoding; + case IllegalArgument: + throw premadeIllegal; + case Unsupported: + throw premadeUnsupported; + case MaxDepthReached: + throw premadeMaxDepth; + case Parse: + default: + if (includeCause) { + throw new ParseException(cause, true); + } + throw premadeParseEmpty; + } + } + } + + /** + * Like {@link #throwOnError()}, but throws an {@link IOException} for {@link #Unsupported} + * errors instead of a {@link ParseException}. Intended for tests that expect checked I/O + * exceptions. + * + * @throws ParseException if a parse error is set + * @throws IOException if an unsupported-feature error is set + */ + public void throwOnError2() throws ParseException, IOException { + if (err == Unsupported) { + throw new IOException("Hit an unsupported feature", cause); + } + throwOnError(); + } + + /** + * Returns the current error code, or {@code 0} if no error has occurred. + * + * @return a positive error-code constant, or {@code 0} for no error + */ + public int error() { + return err > 0 ? err : 0; + } + + private int bufferedInternal(int count) { + if (count <= buf.length) { + bufferMore(count); + if (pos + count <= relLimit) { + int origPos = pos; + pos += count; + return origPos; + } + } + return -1; + } + + private int buffered(int count) { + if (pos + count <= relLimit) { + int origPos = pos; + pos += count; + return origPos; + } + return bufferedInternal(count); + } + + private int readFromInput(@NonNull byte[] dst, int off, int len) { + if (stream != null) { + int total = 0; + try { + while (total < len) { + int n = stream.read(dst, off + total, len - total); + if (n < 0) break; + total += n; + } + } catch (IOException e) { + setError(IOError); + } + return total; + } + long remaining = rsd.remaining(); + if (remaining <= 0) return 0; + len = (int) Math.min(len, remaining); // a test requires not buffering past limit + return (int) rsd.readBytes(dst, off, len); + } + + private int readBytesInternalCopy(@NonNull byte[] dst, int dstOffset, int count) { + if (err > 0) return -1; + int copiedLen = Math.min(count, relLimit - pos); + System.arraycopy(buf, pos, dst, dstOffset, copiedLen); + pos += copiedLen; + if (copiedLen == count || err != 0) return copiedLen; + + offset += pos; + relLimit = pos = end = 0; + int rdlen = readFromInput(dst, dstOffset + copiedLen, count - copiedLen); + if (rdlen == 0) { + seenEOF = true; + err = EOF; + } + offset += rdlen; + return copiedLen + rdlen; + } + + /** + * Reads up to {@code dst.length} bytes into the given array. If fewer bytes are available + * the array is partially filled. The number of bytes copied is returned. + * Returns {@code -1} on an error + * + * @param dst the destination array + * @return the number of bytes read, or {@code -1} if a pre-existing error is set + */ + public long readBytes(@NonNull final byte[] dst) { + return readBytesInternalCopy(dst, 0, dst.length); + } + + /** + * Reads up to {@code length} bytes into the given array starting at {@code offset}. If fewer + * bytes are available the array is partially filled. The number of bytes copied is returned. + * Returns {@code -1} on an error + * + * @param dst the destination array + * @param offset the start index within {@code dst} + * @param length the number of bytes to read + * @return the number of bytes read, or {@code -1} if a pre-existing error is set + */ + public long readBytes(@NonNull final byte[] dst, int offset, int length) { + return readBytesInternalCopy(dst, offset, length); + } + + /** + * Reads bytes into the given {@link ByteBuffer}, advancing its position by the number of + * bytes read. If fewer bytes are available than the buffer's remaining capacity, the buffer + * is partially filled, and the number of bytes copied is returned. + * Returns {@code -1} on an error + * + * @param dst the destination buffer; bytes are written starting at its current position + * @return the number of bytes actually read, or {@code -1} if a pre-existing error is set + */ + public long readBytes(@NonNull ByteBuffer dst) { + int len = readBytesInternalCopy(dst.array(), dst.arrayOffset() + dst.position(), dst.remaining()); + if (len > 0) { // handles error case (which sets len == -1) + dst.position(dst.position() + len); + } + return len; + } + + /** + * Reads exactly {@code length} bytes and returns them as a {@link Bytes} instance. + * Sets {@link #BufferUnderflow} and returns {@link Bytes#EMPTY} if fewer bytes are available. + * + * @param length the number of bytes to read + * @return the read bytes, or {@link Bytes#EMPTY} on error + */ + public @NonNull Bytes readBytes(int length) { + if (length <= relLimit - pos && err <= 0) { + byte[] dst = new byte[length]; + System.arraycopy(buf, pos, dst, 0, length); + pos += length; + return Bytes.wrap(dst); + } + return readBytesInternal(length); + } + + @NonNull + private Bytes readBytesInternal(int length) { + if (length == 0 || err > 0) { + return Bytes.EMPTY; + } else if (length < 0) { + setError(IllegalArgument); + return Bytes.EMPTY; + } + byte[] dst = new byte[length]; + int copiedLen = readBytesInternalCopy(dst, 0, length); + if (copiedLen < length) { + setError(BufferUnderflow); + return Bytes.EMPTY; + } + return Bytes.wrap(dst); + } + + /** + * Reads a 32-bit integer in big-endian byte order. Alias for {@link #readIntBE()}. + * + * @return the integer value + */ + public int readInt() { + return readIntBE(); + } + + /** + * Reads a 32-bit integer in big-endian byte order. + * Sets {@link #BufferUnderflow} and returns {@code 0} if fewer than 4 bytes are available. + * + * @return the integer value + */ + public int readIntBE() { + if (pos + 4 <= relLimit) { + int v = 0; + for (int i = 0; i < 4; i++) { + v |= (buf[pos + 3 - i] & 255) << (i * 8); + } + pos += 4; + return v; + } + return readIntBEInternal(); + } + + /** + * Reads a 32-bit integer in little-endian byte order. + * Sets {@link #BufferUnderflow} and returns {@code 0} if fewer than 4 bytes are available. + * + * @return the integer value + */ + public int readIntLE() { + if (pos + 4 <= relLimit) { + int v = 0; + for (int i = 0; i < 4; i++) { + v |= (buf[pos + i] & 255) << (i * 8); + } + pos += 4; + return v; + } + return readIntLEInternal(); + } + + private int readIntBEInternal() { + bufferMore(4); + if (pos + 4 > relLimit) { + setError(BufferUnderflow); + return 0; + } + int v = 0; + for (int i = 0; i < 4; i++) { + v |= (buf[pos + 3 - i] & 255) << (i * 8); + } + pos += 4; + return v; + } + + private int readIntLEInternal() { + bufferMore(4); + if (pos + 4 > relLimit) { + setError(BufferUnderflow); + return 0; + } + int v = 0; + for (int i = 0; i < 4; i++) { + v |= (buf[pos + i] & 255) << (i * 8); + } + pos += 4; + return v; + } + + /** + * Reads a 64-bit integer in big-endian byte order. Alias for {@link #readLongBE()}. + * + * @return the long value + */ + public long readLong() { + return readLongBE(); + } + + /** + * Reads a 64-bit integer in big-endian byte order. + * Sets {@link #BufferUnderflow} and returns {@code 0} if fewer than 8 bytes are available. + * + * @return the long value + */ + public long readLongBE() { + if (pos + 8 <= relLimit) { + long v = 0; + for (int i = 0; i < 8; i++) { + v |= (long) (buf[pos + 7 - i] & 255) << (i * 8); + } + pos += 8; + return v; + } + return readLongBEInternal(); + } + + private long readLongBEInternal() { + bufferMore(8); + if (pos + 8 > relLimit) { + setError(BufferUnderflow); + return 0; + } + long v = 0; + for (int i = 0; i < 8; i++) { + v |= (long) (buf[pos + 7 - i] & 255) << (i * 8); + } + pos += 8; + return v; + } + + /** + * Reads a 64-bit integer in little-endian byte order. + * Sets {@link #BufferUnderflow} and returns {@code 0} if fewer than 8 bytes are available. + * + * @return the long value + */ + public long readLongLE() { + if (pos + 8 <= relLimit) { + long v = 0; + for (int i = 0; i < 8; i++) { + v |= (long) (buf[pos + i] & 255) << (i * 8); + } + pos += 8; + return v; + } + return readLongLEInternal(); + } + + private long readLongLEInternal() { + bufferMore(8); + if (pos + 8 > relLimit) { + setError(BufferUnderflow); + return 0; + } + long v = 0; + for (int i = 0; i < 8; i++) { + v |= (long) (buf[pos + i] & 255) << (i * 8); + } + pos += 8; + return v; + } + + /** + * Reads a 32-bit float in big-endian byte order. + * + * @return the float value + */ + public float readFloat() { + return Float.intBitsToFloat(readInt()); + } + + /** + * Reads a 32-bit float in little-endian byte order. + * + * @return the float value + */ + public float readFloatLE() { + return Float.intBitsToFloat(readIntLE()); + } + + /** + * Reads a 64-bit double in big-endian byte order. + * + * @return the double value + */ + public double readDouble() { + return Double.longBitsToDouble(readLong()); + } + + /** + * Reads a 64-bit double in little-endian byte order. + * + * @return the double value + */ + public double readDoubleLE() { + return Double.longBitsToDouble(readLongLE()); + } + + /** + * Returns an {@link InputStream} view of the remaining data in this reader. + * For streaming readers, returns the underlying stream directly. For byte-array readers, + * returns a {@link ByteArrayInputStream} over the buffered data. + * + * @return an {@code InputStream} over the unread bytes + * @throws UnsupportedOperationException if the reader is in a partially-buffered streaming state + */ + public InputStream asInputStream() { + if (end == 0) return stream != null ? stream : rsd.asInputStream(); + if (seenEOF && offset == 0) { + return new ByteArrayInputStream(buf, pos, end - pos); + } + throw new UnsupportedOperationException(); + } + + /** + * Reads a single byte and returns {@code true} if it is non-zero, {@code false} otherwise. + * + * @return the boolean value + */ + public boolean readBoolean() { + return readByte() != 0; + } + + /** + * Reads and returns a single signed byte, advancing the position by 1. + * Sets {@link #BufferUnderflow} and returns {@code 0} if no bytes remain. + * + * @return the byte value + */ + public byte readByte() { + if (pos + 1 <= relLimit) return buf[pos++]; + return readByteInternal(); + } + + private byte readByteInternal() { + if (pos + 1 > relLimit) { + bufferMore(1); + if (pos + 1 > relLimit) { + setError(BufferUnderflow); + return 0; + } + } + return buf[pos++]; + } + + /** + * Reads a length-prefixed UTF-8 string. The length is read as a base-128 varint followed + * by that many UTF-8 bytes. Sets {@link #Parse} and returns {@code ""} if the length exceeds + * {@code maxSize}, is negative, or if the UTF-8 bytes are malformed. + * + * @param maxSize the maximum allowed string byte length + * @return the decoded string, or {@code ""} on error + */ + public String readString(final long maxSize) { + final int length = readVarIntNoZZ(); + if (length > maxSize || length < 0) { + setError(PbjReader.Parse); + return ""; + } + + int bufPos = buffered(length); + byte[] data = null; + if (bufPos >= 0) { + data = buf; + } else { + data = new byte[length]; + int copiedLen = readBytesInternalCopy(data, 0, length); + if (copiedLen < length) { + setError(BufferUnderflow); + return ""; + } + bufPos = 0; + } + + if (charArray == null || length > charArray.length) { + int power2Capacity = 2 << (63 - Long.numberOfLeadingZeros(Math.max(2048, length))); + charArray = new char[power2Capacity]; + } + + int i = 0; + // Ascii fast path + { + for (; i < length; i++) { + byte b = data[bufPos + i]; + if ((b & 0x80) != 0) break; + charArray[i] = (char) b; + } + if (i == length) { + return new String(charArray, 0, length); + } + } + int utf16Len = ProtoParserTools.fromUTF8(charArray, data, bufPos, i, length); + if (utf16Len >= 0) { + return new String(charArray, 0, utf16Len); + } + setError(PbjReader.Parse); + return ""; + } +} diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/PbjWriter.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/PbjWriter.java new file mode 100644 index 00000000..f48ea834 --- /dev/null +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/PbjWriter.java @@ -0,0 +1,1067 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.pbj.runtime.io; + +import static java.lang.Character.MAX_SURROGATE; +import static java.lang.Character.MIN_SURROGATE; +import static java.lang.Character.isSurrogatePair; +import static java.lang.Character.toCodePoint; + +import com.hedera.pbj.runtime.ProtoWriterTools; +import com.hedera.pbj.runtime.io.buffer.BufferedData; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import com.hedera.pbj.runtime.io.buffer.RandomAccessData; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; + +/** + * A buffer-backed writer for encoding protobuf data. Not thread safe. + * + *

PbjWriter maintains an internal byte buffer and writes to an {@link OutputStream}, + * {@link WritableSequentialData}, or a fixed byte array. When backed by an output stream the + * buffer is flushed automatically when full. When not backed by a stream the buffer grows as + * needed (unless constructed with {@code mayGrow = false}). + * + *

Errors are tracked internally rather than thrown immediately. Call {@link #error()} to check + * for a pending error code, or {@link #throwOnError()} to surface it as an exception. Once an + * error is set it is sticky — subsequent write calls become no-ops. + * + *

Implements {@link AutoCloseable}: closing flushes pending bytes to the underlying stream. + */ +public class PbjWriter implements AutoCloseable { + private byte[] buf; + private int pos, cap; + private int offset, err; + private RuntimeException cause; + private OutputStream output; + private boolean reuseable; + private boolean mayGrow = true; + + private static final boolean useStacktrace = + !"false".equalsIgnoreCase(System.getProperty("pbj.ReaderWriter.useStackTrace")); + public static final int EOF = PbjReader.EOF, + DataEncoding = PbjReader.DataEncoding, + BufferUnderflow = PbjReader.BufferUnderflow, + Parse = PbjReader.Parse, + IllegalArgument = PbjReader.IllegalArgument, + IOError = PbjReader.IOError, + Unsupported = PbjReader.Unsupported, + UsageError = PbjReader.UsageError, + UnknownField = PbjReader.UnknownField, + BufferOverflow = PbjReader.BufferOverflow, + MaxDepthReached = PbjReader.MaxDepthReached, + // For PbjWriter + Closed = PbjReader.Closed, + MalformString = PbjReader.MalformString; + + private static final RuntimeException premadeRuntimeException; + + static { + premadeRuntimeException = new RuntimeException("Stacktrace not enabled in PbjWriter"); + } + + /** + * Creates a writer that streams output to the given {@link OutputStream}. + * An internal 16 KB buffer is used; the buffer is flushed to the stream when full. + * + * @param output the output stream to write to + */ + public PbjWriter(@NonNull OutputStream output) { + this.output = output; + buf = new byte[16 << 10]; // 16k is friendly to x86-64 L1 cache + cap = buf.length; + reuseable = true; + } + + /** + * Creates a writer backed by a {@link ByteBuffer}. + * + *

If the buffer has a backing array it is used directly. Otherwise an internal 16 KB + * streaming buffer is used and bytes are forwarded to the {@link ByteBuffer} on flush. + * + * @param buffer the target byte buffer + */ + public PbjWriter(ByteBuffer buffer) { + if (buffer.hasArray()) { + buf = buffer.array(); + pos = buffer.arrayOffset() + buffer.position(); + cap = buffer.arrayOffset() + buffer.limit(); + } else { + this.output = new OutputStream() { + @Override + public void write(int b) { + buffer.put((byte) b); + } + + @Override + public void write(@NonNull byte[] b, int off, int len) { + buffer.put(b, off, len); + } + }; + buf = new byte[16 << 10]; + cap = buf.length; + reuseable = true; + } + } + + /** + * Creates a writer that writes directly into the given byte array starting at {@code pos}, + * writing up to the end of the array. No flushing or growing occurs. + * + * @param buffer the backing byte array + * @param pos the starting write position within the array + */ + public PbjWriter(byte[] buffer, int pos) { + this.buf = buffer; + this.pos = pos; + this.cap = buffer.length; + } + + /** + * Creates a writer that streams output to the given {@link WritableSequentialData}. + * An internal 16 KB buffer is used; the buffer is flushed to the target when full. + * + * @param output the writable sequential data target + */ + public PbjWriter(@NonNull WritableSequentialData output) { + this(new OutputStream() { + @Override + public void write(int b) { + output.writeByte((byte) b); + } + + @Override + public void write(@NonNull byte[] b, int off, int len) { + output.writeBytes(b, off, len); + } + }); + } + + /** + * Creates a standalone, growable writer with an initial 16 KB internal buffer. + * No backing output stream is attached; use {@link #toByteArray()} to retrieve the written bytes. + */ + public PbjWriter() { + buf = new byte[16 << 10]; // 16k is friendly to x86-64 L1 cache + cap = buf.length; + reuseable = true; + } + + /** + * Creates a growable (or fixed-size) standalone writer with the specified initial capacity. + * + * @param reserveSize the initial buffer capacity in bytes; if {@code mayGrow} is {@code true} + * the capacity is at least 16 KB regardless of this value + * @param mayGrow {@code true} to allow the internal buffer to grow automatically; + * {@code false} to keep the buffer fixed at {@code reserveSize} bytes + */ + public PbjWriter(int reserveSize, boolean mayGrow) { + if (mayGrow) buf = new byte[Math.max(reserveSize, 16 << 10)]; // 16k is friendly to x86-64 L1 cache + else { + buf = new byte[reserveSize]; + } + cap = buf.length; + reuseable = true; + this.mayGrow = mayGrow; + } + + /** + * Ensures that at least {@code len} bytes of space are available starting at the current + * position, flushing or growing the internal buffer if necessary. + * + * @param len the number of bytes to reserve + */ + public void reserveRel(int len) { + if (pos + len <= cap) return; + flushOrGrow(len); + } + + /** + * Advances the write position by {@code len} bytes without writing any data. + * Used to reserve placeholder space that will be filled in later via {@link #writeAtUnsafe}. + * + * @param len the number of bytes to skip over + */ + public void placehold(int len) { + pos += len; + } + + /** + * Returns the current absolute write position, accounting for any bytes already flushed + * to the underlying output stream. + * + * @return the current write position as a non-negative integer + */ + public int position() { + return offset + pos; + } + + /** + * Overwrites a single byte at the given absolute position without advancing the write cursor. + * Used to patch in placeholder bytes reserved earlier via {@link #placehold}. + * + * @param pos the absolute write position to patch + * @param value the byte value to write + */ + public void writeAtUnsafe(int pos, byte value) { + buf[pos - offset] = value; + } + + /** + * Expands a 1-byte varint placeholder at the given absolute position into a 2-byte varint + * and shifts all subsequent bytes forward by one. The placeholder must have been reserved + * via {@link #placehold(int)}. + * + * @param position the absolute position of the 1-byte placeholder + */ + public void reinsertVarInt(int position) { + int relPos = position - offset; + int len = this.pos - relPos - 1; + System.arraycopy(buf, relPos + 1, buf, relPos + 2, len); + buf[relPos] = (byte) ((len & 0x7F) | 0x80); + buf[relPos + 1] = (byte) (len >>> 7); + this.pos++; + } + + /** + * Writes a boolean as a single byte ({@code 1} for {@code true}, {@code 0} for {@code false}). + * + * @param b the boolean value to write + */ + public void writeBoolean(boolean b) { + writeByte((byte) (b ? 1 : 0)); + } + + /** + * Writes a single signed byte. + * + * @param b the byte to write + */ + public void writeByte(byte b) { + if (pos < cap) { + buf[pos++] = b; + return; + } + writeByteInternal(b); + } + + private void writeByteInternal(byte b) { + flushOrGrow(1); + buf[pos++] = b; + } + + /** + * Writes two bytes in sequence. + * + * @param b1 the first byte + * @param b2 the second byte + */ + public void writeByte2(byte b1, byte b2) { + if (pos + 2 <= cap) { + buf[pos] = b1; + buf[pos + 1] = b2; + pos += 2; + return; + } + writeByte2Internal(b1, b2); + } + + private void writeByte2Internal(byte b1, byte b2) { + flushOrGrow(2); + buf[pos] = b1; + buf[pos + 1] = b2; + pos += 2; + } + + /** + * Writes three bytes in sequence. + * + * @param b1 the first byte + * @param b2 the second byte + * @param b3 the third byte + */ + public void writeByte3(byte b1, byte b2, byte b3) { + if (pos + 3 <= cap) { + buf[pos] = b1; + buf[pos + 1] = b2; + buf[pos + 2] = b3; + pos += 3; + return; + } + writeByte3Internal(b1, b2, b3); + } + + private void writeByte3Internal(byte b1, byte b2, byte b3) { + flushOrGrow(3); + buf[pos] = b1; + buf[pos + 1] = b2; + buf[pos + 2] = b3; + pos += 3; + } + + /** + * Writes four bytes in sequence. + * + * @param b1 the first byte + * @param b2 the second byte + * @param b3 the third byte + * @param b4 the fourth byte + */ + public void writeByte4(byte b1, byte b2, byte b3, byte b4) { + if (pos + 4 <= cap) { + buf[pos] = b1; + buf[pos + 1] = b2; + buf[pos + 2] = b3; + buf[pos + 3] = b4; + pos += 4; + return; + } + writeByte4Internal(b1, b2, b3, b4); + } + + private void writeByte4Internal(byte b1, byte b2, byte b3, byte b4) { + flushOrGrow(4); + buf[pos] = b1; + buf[pos + 1] = b2; + buf[pos + 2] = b3; + buf[pos + 3] = b4; + pos += 4; + } + + /** + * Writes all remaining bytes from the given {@link BufferedData}, advancing its position. + * + * @param src the source buffer; all {@link BufferedData#remaining()} bytes are written + */ + public void writeBytes(@NonNull final BufferedData src) { + int len = (int) src.remaining(); + if (len <= 0) return; + long srcPos = src.position(); + if (pos + len <= cap) { + src.getBytes(srcPos, buf, pos, len); + src.skip(len); + pos += len; + return; + } + writeBytesBDInternal(src, len, srcPos); + } + + private void writeBytesBDInternal(BufferedData src, int len, long srcPos) { + if (output == null) { + flushOrGrow(len); // to grow at least to pos + len + src.getBytes(srcPos, buf, pos, len); + src.skip(len); + pos += len; + } else { + int remaining = len; + while (remaining > 0) { + if (pos == cap) { + try { + output.write(buf, 0, pos); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + offset += pos; + pos = 0; + } + int chunk = Math.min(remaining, cap - pos); + src.getBytes(srcPos, buf, pos, chunk); + pos += chunk; + srcPos += chunk; + remaining -= chunk; + } + src.skip(len); + } + } + + /** + * Writes all bytes from the given array. + * + * @param src the source byte array + */ + public void writeBytes(@NonNull byte[] src) { + writeBytes(src, 0, src.length); + } + + /** + * Writes {@code length} bytes from the given array starting at {@code offset}. + * + * @param src the source byte array + * @param offset the start index within {@code src} + * @param length the number of bytes to write + */ + public void writeBytes(@NonNull byte[] src, int offset, int length) { + if (length <= 0) return; + if (pos + length <= cap && (output == null || length < 2048)) { + System.arraycopy(src, offset, buf, pos, length); + pos += length; + return; + } + writeBytesInternal(src, offset, length); + } + + private void writeBytesInternal(byte[] src, int srcOffset, int length) { + if (output != null && length >= 2048) { + if (pos > 0) { + try { + output.write(buf, 0, pos); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + offset += pos; + pos = 0; + } + try { + output.write(src, srcOffset, length); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + offset += length; + return; + } + flushOrGrow(length); + System.arraycopy(src, srcOffset, buf, pos, length); + pos += length; + } + + /** + * Writes all bytes from the given {@link RandomAccessData}. + * + * @param src the source data + */ + public void writeBytes(@NonNull RandomAccessData src) { + int len = (int) src.length(); + if (len <= 0) return; + if (pos + len <= cap) { + src.getBytes(0, buf, pos, len); + pos += len; + return; + } + writeBytesRAInternal(src, len); + } + + private void writeBytesRAInternal(RandomAccessData src, int len) { + if (output == null) { + flushOrGrow(len); + src.getBytes(0, buf, pos, len); + pos += len; + } else { + // Maybe the below can be improved. This path seems rare + int srcOffset = 0; + int remaining = len; + while (remaining > 0) { + if (pos == cap) { + try { + output.write(buf, 0, pos); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + offset += pos; + pos = 0; + } + int chunk = Math.min(remaining, cap - pos); + src.getBytes(srcOffset, buf, pos, chunk); + pos += chunk; + srcOffset += chunk; + remaining -= chunk; + } + } + } + + /** + * Writes a 32-bit integer in big-endian byte order. Alias for {@link #writeIntBE(int)}. + * + * @param value the integer to write + */ + public void writeInt(int value) { + writeIntBE(value); + } + + /** + * Writes a 32-bit integer in big-endian byte order (most significant byte first). + * + * @param value the integer to write + */ + public void writeIntBE(int value) { + if (pos + 4 <= cap) { + buf[pos] = (byte) (value >>> 24); + buf[pos + 1] = (byte) (value >>> 16); + buf[pos + 2] = (byte) (value >>> 8); + buf[pos + 3] = (byte) value; + pos += 4; + return; + } + writeIntBEInternal(value); + } + + private void writeIntBEInternal(int value) { + flushOrGrow(4); + buf[pos] = (byte) (value >>> 24); + buf[pos + 1] = (byte) (value >>> 16); + buf[pos + 2] = (byte) (value >>> 8); + buf[pos + 3] = (byte) value; + pos += 4; + } + + /** + * Writes a 32-bit integer in little-endian byte order (least significant byte first). + * + * @param value the integer to write + */ + public void writeIntLE(int value) { + if (pos + 4 <= cap) { + buf[pos] = (byte) value; + buf[pos + 1] = (byte) (value >>> 8); + buf[pos + 2] = (byte) (value >>> 16); + buf[pos + 3] = (byte) (value >>> 24); + pos += 4; + return; + } + writeIntLEInternal(value); + } + + private void writeIntLEInternal(int value) { + flushOrGrow(4); + buf[pos] = (byte) value; + buf[pos + 1] = (byte) (value >>> 8); + buf[pos + 2] = (byte) (value >>> 16); + buf[pos + 3] = (byte) (value >>> 24); + pos += 4; + } + + /** + * Writes a 64-bit integer in little-endian byte order (least significant byte first). + * + * @param value the long to write + */ + public void writeLongLE(long value) { + if (pos + 8 <= cap) { + buf[pos] = (byte) value; + buf[pos + 1] = (byte) (value >>> 8); + buf[pos + 2] = (byte) (value >>> 16); + buf[pos + 3] = (byte) (value >>> 24); + buf[pos + 4] = (byte) (value >>> 32); + buf[pos + 5] = (byte) (value >>> 40); + buf[pos + 6] = (byte) (value >>> 48); + buf[pos + 7] = (byte) (value >>> 56); + pos += 8; + return; + } + writeLongLEInternal(value); + } + + private void writeLongLEInternal(long value) { + flushOrGrow(8); + buf[pos] = (byte) value; + buf[pos + 1] = (byte) (value >>> 8); + buf[pos + 2] = (byte) (value >>> 16); + buf[pos + 3] = (byte) (value >>> 24); + buf[pos + 4] = (byte) (value >>> 32); + buf[pos + 5] = (byte) (value >>> 40); + buf[pos + 6] = (byte) (value >>> 48); + buf[pos + 7] = (byte) (value >>> 56); + pos += 8; + } + + /** + * Writes a 32-bit float in big-endian byte order. Alias for {@link #writeFloatBE(float)}. + * + * @param value the float to write + */ + public void writeFloat(float value) { + writeFloatBE(value); + } + + /** + * Writes a 32-bit float in big-endian byte order using {@link Float#floatToRawIntBits}. + * + * @param value the float to write + */ + public void writeFloatBE(float value) { + writeIntBE(Float.floatToRawIntBits(value)); + } + + /** + * Writes a 32-bit float in little-endian byte order using {@link Float#floatToRawIntBits}. + * + * @param value the float to write + */ + public void writeFloatLE(float value) { + writeIntLE(Float.floatToRawIntBits(value)); + } + + /** + * Writes a 64-bit double in big-endian byte order. Alias for {@link #writeDoubleBE(double)}. + * + * @param value the double to write + */ + public void writeDouble(double value) { + writeDoubleBE(value); + } + + /** + * Writes a 64-bit double in big-endian byte order using {@link Double#doubleToRawLongBits}. + * + * @param value the double to write + */ + public void writeDoubleBE(double value) { + writeLongBE(Double.doubleToRawLongBits(value)); + } + + /** + * Writes a 64-bit double in little-endian byte order using {@link Double#doubleToRawLongBits}. + * + * @param value the double to write + */ + public void writeDoubleLE(double value) { + writeLongLE(Double.doubleToRawLongBits(value)); + } + + /** + * Writes a 64-bit integer in big-endian byte order. Alias for {@link #writeLongBE(long)}. + * + * @param value the long to write + */ + public void writeLong(long value) { + writeLongBE(value); + } + + /** + * Writes a 64-bit integer in big-endian byte order (most significant byte first). + * + * @param value the long to write + */ + public void writeLongBE(long value) { + if (pos + 8 <= cap) { + buf[pos] = (byte) (value >>> 56); + buf[pos + 1] = (byte) (value >>> 48); + buf[pos + 2] = (byte) (value >>> 40); + buf[pos + 3] = (byte) (value >>> 32); + buf[pos + 4] = (byte) (value >>> 24); + buf[pos + 5] = (byte) (value >>> 16); + buf[pos + 6] = (byte) (value >>> 8); + buf[pos + 7] = (byte) value; + pos += 8; + return; + } + writeLongBEInternal(value); + } + + private void writeLongBEInternal(long value) { + flushOrGrow(8); + buf[pos] = (byte) (value >>> 56); + buf[pos + 1] = (byte) (value >>> 48); + buf[pos + 2] = (byte) (value >>> 40); + buf[pos + 3] = (byte) (value >>> 32); + buf[pos + 4] = (byte) (value >>> 24); + buf[pos + 5] = (byte) (value >>> 16); + buf[pos + 6] = (byte) (value >>> 8); + buf[pos + 7] = (byte) value; + pos += 8; + } + + /** + * Writes an {@code int} as a zigzag-encoded varint. Negative values are sign-extended to + * 64 bits before encoding, producing the 10-byte wire format required by the protobuf spec. + * + * @param value the signed int to encode + */ + public void writeVarIntZZ(int value) { + writeVarLongZZ(value); + } + + /** + * Writes a {@code long} as a zigzag-encoded varint. The value is mapped via + * {@code (value << 1) ^ (value >> 63)} before varint encoding so that small negative + * numbers require few bytes. + * + * @param value the signed long to encode + */ + public void writeVarLongZZ(long value) { + writeVarLongNoZZ((value << 1) ^ (value >> 63)); + } + + /** + * Writes an {@code int} as a base-128 varint with optional zigzag encoding. + * + * @param value the value to encode + * @param zigZag if {@code true}, applies zigzag encoding before varint encoding + */ + public void writeVarInt(int value, boolean zigZag) { + long v = zigZag ? ((long) value << 1) ^ ((long) value >> 63) : value; + writeVarLongNoZZ(v); + } + + /** + * Writes a {@code long} as a base-128 varint with optional zigzag encoding. + * + * @param value the value to encode + * @param zigZag if {@code true}, applies zigzag encoding before varint encoding + */ + public void writeVarLong(long value, boolean zigZag) { + long v = zigZag ? (value << 1) ^ (value >> 63) : value; + writeVarLongNoZZ(v); + } + + private void writeVarLongInternal(long v) { + flushOrGrow(10); + while ((v & ~0x7FL) != 0) { + buf[pos++] = (byte) (((int) v & 0x7F) | 0x80); + v >>>= 7; + } + buf[pos++] = (byte) v; + } + + /** + * Writes an {@code int} as a base-128 varint without zigzag encoding. + * The value is zero-extended to 64 bits before encoding. + * + * @param value the value to encode + */ + public void writeVarIntNoZZ(int value) { + writeVarLongNoZZ(value); + } + + /** + * Writes a {@code long} as a base-128 varint without zigzag encoding. + * Each 7-bit group is written as a byte with the high bit set if more bytes follow. + * + * @param v the value to encode + */ + public void writeVarLongNoZZ(long v) { + if (pos + 10 <= cap) { + while ((v & ~0x7FL) != 0) { + buf[pos++] = (byte) (((int) v & 0x7F) | 0x80); + v >>>= 7; + } + buf[pos++] = (byte) v; + return; + } + writeVarLongInternal(v); + } + + /** + * Flushes all buffered bytes to the underlying output stream and resets the internal buffer. + * + *

After the flush the internal write position is reset to zero, and the absolute byte + * offset is advanced by the number of bytes written. This is a no-op when no output stream + * is attached (standalone writers). Any pending error is surfaced before writing. + * + * @throws RuntimeException if a prior error was recorded on this writer + * @throws UncheckedIOException if the underlying stream throws an {@link java.io.IOException} + */ + public void flush() { + throwOnError(); + if (output == null) return; + try { + output.write(buf, 0, pos); + output.flush(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + offset += pos; + pos = 0; + } + + /** + * Flushes any remaining bytes to the underlying output stream and closes it. + * If no output stream is attached this method does nothing. + */ + @Override + public void close() { + if (output == null) return; + flush(); + try { + output.close(); + } catch (IOException ex) { + setError(IOError, ex.getMessage()); + } + err = Closed; + } + + private void flushOrGrow(int minLength) { + if (output != null) { + if (minLength > cap) { + setError(IOError, "minLength is greater than capacity"); + return; + } + try { + output.write(buf, 0, pos); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + offset += pos; + pos = 0; + } else if (reuseable && mayGrow) { + int power2Capacity = (int) 2L << (63 - Long.numberOfLeadingZeros(Math.max(buf.length, pos + minLength))); + byte[] newBuf = new byte[power2Capacity]; + System.arraycopy(buf, 0, newBuf, 0, pos); + buf = newBuf; + cap = buf.length; + } + // A possible else case is using a byte array and trying to reserve (or grow) past the length of it + // reserving shouldn't cause a throw so this else is ignored + } + + /** + * Flushes any buffered output and resets this writer's position, offset, and error state, + * leaving it ready for reuse while keeping the same output destination. + */ + public void reset() { + flush(); + pos = 0; + offset = 0; + err = 0; + cause = null; + } + + /** + * Resets this writer and detaches it from any output stream, allowing subsequent use as + * a standalone in-memory writer. + */ + public void resetWithNull() { + resetWith((OutputStream) null); + } + + /** + * Resets this writer and redirects output to a new {@link OutputStream}. + * Only valid on writers that were originally created with an output stream. + * Sets the error code to {@link #UsageError} if called on a non-reuseable writer. + * + * @param out the new output stream + */ + public void resetWith(OutputStream out) { + reset(); + if (!reuseable) { + setError(UsageError, "resetWith on non-reuseable PbjWriter"); + return; + } + output = out; + } + + /** + * Resets this writer and redirects output to a new {@link WritableSequentialData}. + * Only valid on writers that were originally created with an output stream. + * + * @param out the new writable sequential data target + */ + public void resetWith(@NonNull WritableSequentialData out) { + resetWith(new OutputStream() { + @Override + public void write(int b) { + out.writeByte((byte) b); + } + + @Override + public void write(@NonNull byte[] b, int off, int len) { + out.writeBytes(b, off, len); + } + }); + } + + /** + * Returns the raw internal byte array. The valid data occupies indices {@code [0, position())}. + * Intended for low-level inspection; prefer {@link #toByteArray()} for a correctly sized copy + * + * @return the internal buffer array + */ + public byte[] internalArray() { + return buf; + } + + /** + * Returns a zero-copy {@link Bytes} view wrapping the internal buffer from index 0 up to + * the current position. The backing array is shared, so the returned {@code Bytes} must + * not be retained beyond the next write operation. + * + * @return a {@code Bytes} view of the current contents + */ + public Bytes internalArrayWrapped() { + return Bytes.wrap(buf, 0, pos); + } + + /** + * Returns a {@link Bytes} wrapping a fresh copy of the written bytes. + * Only valid on standalone (non-streaming) writers; sets {@link #UsageError} and returns + * {@link Bytes#EMPTY} if called on a streaming writer. + * + * @return the written bytes wrapped in a {@code Bytes} instance + */ + public Bytes toByteArrayWrapped() { + if (output != null) { + setError(UsageError, "toByteArrayWrapped used on a streaming object"); + return Bytes.EMPTY; + } + return Bytes.wrap(toByteArray()); + } + + /** + * Returns a newly allocated byte array containing exactly the bytes written so far. + * Only valid on standalone (non-streaming) writers; sets {@link #UsageError} and returns + * {@code null} if called on a streaming writer. + * + * @return a copy of the written bytes, or {@code null} on error + */ + public byte[] toByteArray() { + if (output != null) { + setError(UsageError, "toByteArray used on a streaming object"); + return null; + } + byte[] bytes = new byte[pos]; + System.arraycopy(buf, 0, bytes, 0, pos); + return bytes; + } + + /** + * Creates a {@link PbjReader} backed by the current internal buffer contents. + * Useful for reading back data written to a standalone writer without copying. + * Only valid on standalone (non-streaming) writers; sets {@link #UsageError} and returns + * {@code null} if called on a streaming writer. + * + * @return a new {@code PbjReader} positioned at the start of the written bytes, or {@code null} on error + */ + public PbjReader toPbjReader() { + if (output != null) { + setError(UsageError, "toPbjReader on a streaming object"); + return null; + } + return new PbjReader(buf, 0, pos); + } + + /** + * Records an error on this writer if no previous error is set. Once an error is recorded + * subsequent writes become no-ops. + * + * @param errorKind one of the error-code constants ({@link #IOError}, {@link #UsageError}, etc.) + * @param message a message returned with an exception + */ + public void setError(int errorKind, String message) { + if (err > 0) return; + err = errorKind; + if (useStacktrace) { + cause = new RuntimeException(message); + } else { + cause = premadeRuntimeException; + } + } + + /** + * Returns the current error code, or {@code 0} if no error has occurred. + * + * @return a positive error-code constant, or {@code 0} for no error + */ + public int error() { + return err > 0 ? err : 0; + } + + /** + * Throws the recorded error as a {@link RuntimeException} if an error is set. + * + * @throws RuntimeException if {@link #error()} is non-zero + */ + public void throwOnError() { + if (err > 0) { + throw cause; + } + } + + /** + * Returns the exception that was recorded when the current error was set, or {@code null} + * if no error has occurred. + * + * @return the recorded exception, or {@code null} + */ + public RuntimeException getCause() { + return cause; + } + + /** + * Writes a UTF-8 encoded string without a preceding length varint. + * Surrogate pairs are encoded as a 4-byte UTF-8 sequence. An unpaired surrogate sets + * the error code to {@link #MalformString}. + * + * @param str the string to encode + */ + public void writeStringNoTag(String str) { + int inLength = str.length(); + for (int i = 0; i < inLength; ++i) { + char c = str.charAt(i); + if (c < 0x80) { + writeByte((byte) c); + } else if (c < 0x800) { + writeByte2((byte) (0xC0 | (c >>> 6)), (byte) (0x80 | (0x3F & c))); + } else if (c < MIN_SURROGATE || MAX_SURROGATE < c) { + writeByte3((byte) (0xE0 | (c >>> 12)), (byte) (0x80 | (0x3F & (c >>> 6))), (byte) (0x80 | (0x3F & c))); + } else { + char low; + if (i + 1 == inLength || !isSurrogatePair(c, (low = str.charAt(++i)))) { + setError(MalformString, "Unpaired surrogate at index " + i + " of " + inLength); + return; + } + int codePoint = toCodePoint(c, low); + writeByte4( + (byte) ((0xF << 4) | (codePoint >>> 18)), + (byte) (0x80 | (0x3F & (codePoint >>> 12))), + (byte) (0x80 | (0x3F & (codePoint >>> 6))), + (byte) (0x80 | (0x3F & codePoint))); + } + } + } + + /** + * Writes a UTF-8 encoded string preceded by its length as a base-128 varint, as required + * by the protobuf wire format for {@code TYPE_STRING} fields. + * + * @param str the string to encode + */ + public void writeStringWithTag(String str) { + int inLength = str.length(); + if (inLength > 0x7F) { + writeUTF8_2byte(str); + return; + } + // fast path 1 byte tag case + reserveRel(0x7F * 4 + 2); // worse case size + int pos = position(); + placehold(1); + writeStringNoTag(str); + int endPos = position(); + int utf8Len = endPos - pos - 1; + if (utf8Len <= 0x7F) { + writeAtUnsafe(pos, (byte) utf8Len); + } else { + reinsertVarInt(pos); + } + } + + private void writeUTF8_2byte(String str) { + // buffer is 16k, string is UTF16, so worst case is len*3. + // 5460 was picked bc its (16k - 2byte tag) / 3 byte worse case + if (str.length() > 5460) { + // Can't fit in buffer, todo check if we'll grow anyways + // I don't think anything hits this case? + // These two lines counts the length then write the length, making this 2 pass + writeVarIntNoZZ(ProtoWriterTools.sizeOfStringNoTag(str)); + writeStringNoTag(str); + return; + } + reserveRel(str.length() * 3 + 2); + int pos = position(); + placehold(2); + writeStringNoTag(str); + int utf8Len = position() - pos - 2; + writeAtUnsafe(pos, (byte) ((utf8Len & 0x7F) | 0x80)); + writeAtUnsafe(pos + 1, (byte) (utf8Len >>> 7)); + } + + /** + * Advances the write position by {@code count} bytes without writing any data. + * + * @param count the number of bytes to skip + */ + public void skip(int count) { + pos += count; + } +} diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java index dbbdbc4a..db6024c6 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/Bytes.java @@ -5,6 +5,8 @@ import com.hedera.pbj.runtime.hashing.XXH3_64; import com.hedera.pbj.runtime.io.DataEncodingException; +import com.hedera.pbj.runtime.io.PbjReader; +import com.hedera.pbj.runtime.io.PbjWriter; import com.hedera.pbj.runtime.io.ReadableSequentialData; import com.hedera.pbj.runtime.io.UnsafeUtils; import com.hedera.pbj.runtime.io.WritableSequentialData; @@ -372,6 +374,9 @@ public void writeTo(@NonNull final WritableSequentialData wsd) { wsd.writeBytes(buffer, start, length); } + public void writeTo(@NonNull PbjWriter wsd) { + wsd.writeBytes(buffer, start, length); + } /** * A helper method for efficient copy of our data into an WritableSequentialData without creating a defensive copy * of the data. The implementation relies on a well-behaved WritableSequentialData that doesn't modify the buffer data. @@ -485,6 +490,11 @@ public ReadableSequentialData toReadableSequentialData() { return new RandomAccessSequenceAdapter(this); } + @NonNull + public PbjReader toPbjReader() { + return new PbjReader(this); + } + /** * Exposes this {@link Bytes} as an {@link InputStream}. This is a zero-copy operation. * @@ -759,6 +769,26 @@ public byte[] toByteArray(final int offset, final int len) { return ret; } + /** + * Returns the raw backing byte array. The logical content starts at {@link #arrayUnsafeOffset()} and spans + * {@link #length()} bytes. Mutating the returned array breaks the immutability contract of this class. + * + * @return the internal backing byte array + */ + @NonNull + public byte[] arrayUnsafe() { + return buffer; + } + + /** + * Returns the offset within {@link #arrayUnsafe()} where the logical content of this {@link Bytes} begins. + * + * @return start offset into the backing array + */ + public int arrayUnsafeOffset() { + return start; + } + private void validateOffset(final long offset) { if ((offset < 0) || (offset >= this.length)) { throw new IndexOutOfBoundsException("offset=" + offset + ", length=" + this.length); diff --git a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/RandomAccessSequenceAdapter.java b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/RandomAccessSequenceAdapter.java index 8c550f17..f72248df 100644 --- a/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/RandomAccessSequenceAdapter.java +++ b/pbj-core/pbj-runtime/src/main/java/com/hedera/pbj/runtime/io/buffer/RandomAccessSequenceAdapter.java @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.pbj.runtime.io.buffer; +import com.hedera.pbj.runtime.io.ByteArraySequentialData; import com.hedera.pbj.runtime.io.ReadableSequentialData; import edu.umd.cs.findbugs.annotations.NonNull; import java.nio.BufferUnderflowException; @@ -12,7 +13,7 @@ * {@link RandomAccessData} instance. Since {@link RandomAccessData} has no position or limit, this class adds those, * and otherwise delegates to the underlying {@link RandomAccessData} instance. */ -final class RandomAccessSequenceAdapter implements ReadableSequentialData { +final class RandomAccessSequenceAdapter implements ReadableSequentialData, ByteArraySequentialData { /** The delegate {@link RandomAccessData} instance */ private final RandomAccessData delegate; @@ -56,6 +57,24 @@ final class RandomAccessSequenceAdapter implements ReadableSequentialData { } } + // ================================================================================================================ + // ByteArraySequentialData Methods + + @Override + public byte[] byteArrayUnsafe() { + return delegate instanceof Bytes b ? b.arrayUnsafe() : null; + } + + @Override + public int byteArrayUnsafeOffset() { + return delegate instanceof Bytes b ? b.arrayUnsafeOffset() + (int) (start + position) : 0; + } + + @Override + public int byteArrayUnsafeEnd() { + return delegate instanceof Bytes b ? b.arrayUnsafeOffset() + (int) (start + limit) : 0; + } + // ================================================================================================================ // SequentialData Methods diff --git a/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/CodecWrapper.java b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/CodecWrapper.java index 9bfb3b33..6b20e95c 100644 --- a/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/CodecWrapper.java +++ b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/CodecWrapper.java @@ -1,10 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.pbj.runtime; -import com.hedera.pbj.runtime.io.ReadableSequentialData; -import com.hedera.pbj.runtime.io.WritableSequentialData; +import com.hedera.pbj.runtime.io.PbjReader; +import com.hedera.pbj.runtime.io.PbjWriter; import edu.umd.cs.findbugs.annotations.NonNull; -import java.io.IOException; import java.util.function.ToIntFunction; /** @@ -14,33 +13,28 @@ * @param The type of the object to be encoded/decoded */ class CodecWrapper implements Codec { - private final ProtoWriter writer; + private final PbjProtoWriter writer; private final ToIntFunction sizeOf; - CodecWrapper(ProtoWriter writer, ToIntFunction sizeOf) { + CodecWrapper(PbjProtoWriter writer, ToIntFunction sizeOf) { this.writer = writer; this.sizeOf = sizeOf; } @NonNull @Override - public T parse( - @NonNull ReadableSequentialData input, - boolean strictMode, - boolean parseUnknownFields, - int maxDepth, - int maxSize) + public T parse(@NonNull PbjReader input, boolean strictMode, boolean parseUnknownFields, int maxDepth, int maxSize) throws ParseException { throw new UnsupportedOperationException(); } @Override - public void write(@NonNull T item, @NonNull WritableSequentialData output) throws IOException { + public void write(@NonNull T item, @NonNull PbjWriter output) { writer.write(item, output); } @Override - public int measure(@NonNull ReadableSequentialData input) throws ParseException { + public int measure(@NonNull PbjReader input) throws ParseException { throw new UnsupportedOperationException(); } @@ -50,7 +44,7 @@ public int measureRecord(T item) { } @Override - public boolean fastEquals(@NonNull T item, @NonNull ReadableSequentialData input) throws ParseException { + public boolean fastEquals(@NonNull T item, @NonNull PbjReader input) throws ParseException { throw new UnsupportedOperationException(); } diff --git a/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoParserToolsTest.java b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoParserToolsTest.java index c9602d2b..dcdcc8fa 100644 --- a/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoParserToolsTest.java +++ b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoParserToolsTest.java @@ -29,16 +29,15 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import com.hedera.pbj.runtime.io.PbjReader; +import com.hedera.pbj.runtime.io.PbjWriter; import com.hedera.pbj.runtime.io.ReadableSequentialData; -import com.hedera.pbj.runtime.io.WritableSequentialData; import com.hedera.pbj.runtime.io.buffer.BufferedData; import com.hedera.pbj.runtime.io.buffer.Bytes; import com.hedera.pbj.runtime.io.stream.ReadableStreamingData; -import com.hedera.pbj.runtime.io.stream.WritableStreamingData; import com.hedera.pbj.runtime.test.UncheckedThrowingFunction; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.BufferUnderflowException; import java.nio.ByteOrder; @@ -215,6 +214,94 @@ void testReadString() { length + 1); } + @ParameterizedTest + @ValueSource( + strings = { + "Test Ascii", + "UTF16 ☃", + "Hangul Syllable Hwen 휀", + "Private Use E000 \uE000", + "Linear B Syllable \uD800\uDC00", + "4 byte char \uDB40\uDDEF", + "☃ UTF16", + "휀Hangul Syllable Hwen", + "\uE000Private Use E000", + "\uD800\uDC00Linear B Syllable", + "\uDB40\uDDEF4 byte char", + "\u007F", + "\u013F" + }) + void testReadString_readString_unicode(final String expected) { + byte[] utf8 = expected.getBytes(StandardCharsets.UTF_8); + BufferedData data = BufferedData.allocate(utf8.length + 5); + data.writeVarInt(utf8.length, false); + data.writeBytes(utf8); + data.flip(); + PbjReader reader = new PbjReader(data.toInputStream()); + assertEquals(expected, readString(reader)); + } + + @Test + void testReadString_readString_malformed_utf8() { + BufferedData data = BufferedData.allocate(128); + byte[][] manyBadEncoding = { + {(byte) 0xED, (byte) 0xA0, (byte) 0x80}, + {(byte) 0xED, (byte) 0x9F, (byte) 0xC0}, + {(byte) 0xE2, (byte) 0x98, (byte) 0x03}, + {(byte) 0xE2, (byte) 0x18, (byte) 0x83}, + {(byte) 0x82, (byte) 0x98, (byte) 0x83}, + {(byte) 0xC4, (byte) 0x3F}, + {(byte) 0x84, (byte) 0x3F}, + {(byte) 0x3F, (byte) 0x84, 32}, + {(byte) 0xF0, (byte) 0x90, (byte) 0x84, (byte) 0x3F}, + {(byte) 0xF0, (byte) 0x90, (byte) 0x04, (byte) 0xBF}, + {(byte) 0xF0, (byte) 0x10, (byte) 0x84, (byte) 0xBF}, + {(byte) 0x70, (byte) 0x90, (byte) 0x84, (byte) 0xBF}, + + // Largest Legal Value is 10FFFF (F4 8F BF BF) + {(byte) 0xF4, (byte) 0x90, (byte) 0x80, (byte) 0x80}, + {(byte) 0xF4, (byte) 0x8F, (byte) 0xBF, (byte) 0xC0}, + {(byte) 0xF5, (byte) 0x80, (byte) 0x80, (byte) 0x80}, + {(byte) 0xF5, (byte) 0xBF, (byte) 0x80, (byte) 0x80}, + {(byte) 0xF6, (byte) 0xBF, (byte) 0x80, (byte) 0x80}, + {(byte) 0xF7, (byte) 0xBF, (byte) 0x80, (byte) 0x80}, + {(byte) 0xF8, (byte) 0xBF, (byte) 0x80, (byte) 0x80}, + + // Check if the non tail codepath checks for these + {(byte) 0xF4, (byte) 0x90, (byte) 0x80, (byte) 0x80, 65}, + {(byte) 0xF4, (byte) 0x8F, (byte) 0xBF, (byte) 0xC0, 65}, + {(byte) 0xF5, (byte) 0x80, (byte) 0x80, (byte) 0x80, 65}, + {(byte) 0xF5, (byte) 0xBF, (byte) 0x80, (byte) 0x80, 65}, + {(byte) 0xF6, (byte) 0xBF, (byte) 0x80, (byte) 0x80, 65}, + {(byte) 0xF7, (byte) 0xBF, (byte) 0x80, (byte) 0x80, 65}, + {(byte) 0xF8, (byte) 0xBF, (byte) 0x80, (byte) 0x80, 65}, + }; + + // First check the code is encoding correctly + { + byte[] ok = {(byte) 0xED, (byte) 0x9F, (byte) 0xBF}; + data.reset(); + data.writeVarInt(ok.length, false); + data.writeBytes(ok); + data.flip(); + PbjReader reader = new PbjReader(data.toInputStream()); + String oksz = readString(reader); + assertEquals("\uD7FF", oksz); + assert (reader.error() <= 0); + } + + for (var bad : manyBadEncoding) { + data.reset(); + data.writeVarInt(bad.length, false); + data.writeBytes(bad); + data.flip(); + PbjReader reader = new PbjReader(data.toInputStream()); + String badsz = readString(reader); + assertEquals("", badsz); + assertEquals(reader.error(), PbjReader.Parse); + } + } + @Test void testReadString_maxSize() throws IOException { final int length = 1; @@ -273,6 +360,18 @@ void testReadBytes_maxSize() throws IOException { assertThrows(ParseException.class, () -> ProtoParserTools.readBytes(streamingData, maxSize)); } + @Test + void testReadBytes_pbjreader_maxSize_throwsParseExceptionWithNoCause() { + int maxSize = 1024; + BufferedData data = BufferedData.allocate(16); + data.writeVarInt(maxSize + 1, false); + data.flip(); + PbjReader reader = new PbjReader(data.toInputStream()); + ProtoParserTools.readBytes(reader, maxSize); + ParseException ex = assertThrows(ParseException.class, reader::throwOnError); + assertNull(ex.getCause(), "CN Expects no cause"); + } + @Test void testReadBytes_incomplete() throws IOException { final int length = rng.nextInt(0, 100); @@ -358,18 +457,18 @@ void testSkipUnsupported(ProtoConstants unsupportedType) { @Test void testExtractBytesNullInput() { final FieldDefinition field = createFieldDefinition(BYTES); - assertThrows(NullPointerException.class, () -> ProtoParserTools.extractFieldBytes(null, field)); + assertThrows(NullPointerException.class, () -> ProtoParserTools.extractFieldBytes((PbjReader) null, field)); } @Test void testExtractBytesNullField() { - final ReadableSequentialData input = Bytes.EMPTY.toReadableSequentialData(); + final PbjReader input = Bytes.EMPTY.toPbjReader(); assertThrows(NullPointerException.class, () -> ProtoParserTools.extractFieldBytes(input, null)); } @Test void testExtractBytesRepeatedField() { - final ReadableSequentialData input = Bytes.EMPTY.toReadableSequentialData(); + final PbjReader input = Bytes.EMPTY.toPbjReader(); final FieldDefinition field = new FieldDefinition("field", FieldType.BYTES, true, true, false, 1); assertThrows(IllegalArgumentException.class, () -> ProtoParserTools.extractFieldBytes(input, field)); } @@ -404,22 +503,20 @@ void testExtractBytesRepeatedField() { private static final FieldDefinition BOOL_F = new FieldDefinition("boolfield", BOOL, false, true, false, 11); private static final boolean BOOL_V = true; - private static Bytes prepareExtractBytesTestInput() throws IOException { - try (final ByteArrayOutputStream bout = new ByteArrayOutputStream(); - final WritableStreamingData out = new WritableStreamingData(bout)) { - ProtoWriterTools.writeInteger(out, INT32_F, INT32_V); - ProtoWriterTools.writeInteger(out, FIXED_F, FIXED32_V); - ProtoWriterTools.writeString(out, STRING_F, STRING_V); - ProtoWriterTools.writeBytes(out, BYTES_F, BYTES_V); - ProtoWriterTools.writeMessage(out, MESSAGE_F, MESSAGE_V, TestMessageCodec.INSTANCE); - ProtoWriterTools.writeDouble(out, DOUBLE_F, DOUBLE32_V); - return Bytes.wrap(bout.toByteArray()); - } + private static PbjReader prepareExtractBytesTestInput() throws IOException { + PbjWriter out = new PbjWriter(); + ProtoWriterTools.writeInteger(out, INT32_F, INT32_V); + ProtoWriterTools.writeInteger(out, FIXED_F, FIXED32_V); + ProtoWriterTools.writeString(out, STRING_F, STRING_V); + ProtoWriterTools.writeBytes(out, BYTES_F, BYTES_V); + ProtoWriterTools.writeMessage(out, MESSAGE_F, MESSAGE_V, TestMessageCodec.INSTANCE); + ProtoWriterTools.writeDouble(out, DOUBLE_F, DOUBLE32_V); + return out.toPbjReader(); } @Test void testExtractBytesStringField() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + PbjReader input = prepareExtractBytesTestInput(); final Bytes bytes = ProtoParserTools.extractFieldBytes(input, STRING_F); assertNotNull(bytes); assertEquals(STRING_V, new String(bytes.toByteArray(), StandardCharsets.UTF_8)); @@ -427,14 +524,14 @@ void testExtractBytesStringField() throws IOException, ParseException { @Test void testExtractFieldBytesInvalidType() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + PbjReader input = prepareExtractBytesTestInput(); // should throw because INT32 is not a delimited type assertThrows(IllegalArgumentException.class, () -> ProtoParserTools.extractFieldBytes(input, INT32_F)); } @Test void testExtractBytesBytesField() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + PbjReader input = prepareExtractBytesTestInput(); final Bytes bytes = ProtoParserTools.extractFieldBytes(input, BYTES_F); assertNotNull(bytes); assertEquals(BYTES_V, bytes); @@ -442,52 +539,58 @@ void testExtractBytesBytesField() throws IOException, ParseException { @Test void testExtractBytesMessageField() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + PbjReader input = prepareExtractBytesTestInput(); final Bytes bytes = ProtoParserTools.extractFieldBytes(input, MESSAGE_F); assertNotNull(bytes); - final TestMessage value = TestMessageCodec.INSTANCE.parse(bytes.toReadableSequentialData()); + final TestMessage value = TestMessageCodec.INSTANCE.parse(bytes); assertNotNull(value); assertEquals(MESSAGE_V, value); } @Test void testExtractBytesUnknownField() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + PbjReader input = prepareExtractBytesTestInput(); final Bytes bytes = ProtoParserTools.extractFieldBytes(input, UNKNOWN_F); assertNull(bytes); } @Test void testExtractField32Bit() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + PbjReader input = prepareExtractBytesTestInput(); final var res = ProtoParserTools.extractField(input, WIRE_TYPE_FIXED_32_BIT, 32); assertNotNull(res); } @Test void testExtractField64Bit() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + PbjReader input = prepareExtractBytesTestInput(); final var res = ProtoParserTools.extractField(input, WIRE_TYPE_FIXED_64_BIT, 32); assertNotNull(res); } @Test void testExtractFieldVarInt() throws IOException, ParseException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); + PbjReader input = prepareExtractBytesTestInput(); final var res = ProtoParserTools.extractField(input, WIRE_TYPE_VARINT_OR_ZIGZAG, 32); assertNotNull(res); } @Test void testExtractFieldGroupStartUnsupported() throws IOException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); - assertThrows(IOException.class, () -> ProtoParserTools.extractField(input, WIRE_TYPE_GROUP_START, 32)); + PbjReader input = prepareExtractBytesTestInput(); + assertThrows(IOException.class, () -> { + ProtoParserTools.extractField(input, WIRE_TYPE_GROUP_START, 32); + input.throwOnError2(); + }); } @Test void testExtractFieldGroupEndUnsupported() throws IOException { - final ReadableSequentialData input = prepareExtractBytesTestInput().toReadableSequentialData(); - assertThrows(IOException.class, () -> ProtoParserTools.extractField(input, WIRE_TYPE_GROUP_END, 32)); + PbjReader input = prepareExtractBytesTestInput(); + assertThrows(IOException.class, () -> { + ProtoParserTools.extractField(input, WIRE_TYPE_GROUP_END, 32); + input.throwOnError2(); + }); } private static void skipTag(BufferedData data) { @@ -539,10 +642,8 @@ private static final class TestMessageCodec implements Codec { public static final FieldDefinition VALUE_FIELD = new FieldDefinition("value", FieldType.STRING, false, true, false, 1); - @NonNull - @Override public TestMessage parse( - @NonNull final ReadableSequentialData in, + @NonNull final PbjReader in, final boolean strictMode, final boolean parseUnknownFields, final int maxDepth, @@ -550,17 +651,12 @@ public TestMessage parse( throws ParseException { String value = null; while (in.hasRemaining()) { - final int tag = in.readVarInt(false); + final int tag = in.readVarIntNoZZ(); final int fieldNum = tag >> ProtoParserTools.TAG_FIELD_OFFSET; final int wireType = tag & TAG_WIRE_TYPE_MASK; if ((fieldNum == VALUE_FIELD.number()) && (wireType == ProtoWriterTools.wireType(VALUE_FIELD).ordinal())) { - final int length = in.readVarInt(false); - final byte[] valueBytes = new byte[length]; - if (in.readBytes(valueBytes) != length) { - throw new ParseException("Failed to read value bytes"); - } - value = new String(valueBytes, StandardCharsets.UTF_8); + value = readString(in); } else { throw new ParseException("Unknown field: " + tag); } @@ -569,8 +665,7 @@ public TestMessage parse( } @Override - public void write(@NonNull final TestMessage item, @NonNull final WritableSequentialData out) - throws IOException { + public void write(@NonNull final TestMessage item, @NonNull PbjWriter out) { final String value = item.getValue(); if (value != null) { ProtoWriterTools.writeString(out, VALUE_FIELD, value); @@ -592,8 +687,7 @@ public int measureRecord(@NonNull final TestMessage item) { } @Override - public boolean fastEquals(@NonNull TestMessage item, @NonNull ReadableSequentialData input) - throws ParseException { + public boolean fastEquals(@NonNull TestMessage item, @NonNull PbjReader input) throws ParseException { throw new UnsupportedOperationException(); } diff --git a/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoWriterToolsTest.java b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoWriterToolsTest.java index 20e0d8f1..d948f952 100644 --- a/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoWriterToolsTest.java +++ b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/ProtoWriterToolsTest.java @@ -512,7 +512,7 @@ void testWriteOneRepeatedMessage() throws IOException { final String appleStr2 = RANDOM_STRING.nextString(); final Apple apple2 = Apple.newBuilder().setVariety(appleStr2).build(); final BufferedData buf1 = BufferedData.allocate(256); - final ProtoWriter writer = (data, out) -> out.writeBytes(data.toByteArray()); + final PbjProtoWriter writer = (data, out) -> out.writeBytes(data.toByteArray()); ProtoWriterTools.writeMessageList( buf1, definition, List.of(apple1, apple2), new CodecWrapper<>(writer, Apple::getSerializedSize)); final Bytes writtenBytes1 = buf1.getBytes(0, buf1.position()); diff --git a/pbj-core/version.txt b/pbj-core/version.txt index 4f3da74f..ff1dac96 100644 --- a/pbj-core/version.txt +++ b/pbj-core/version.txt @@ -1 +1 @@ -0.15.0-SNAPSHOT +0.pbj.3 diff --git a/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/GenericParserQuickBench.java b/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/GenericParserQuickBench.java deleted file mode 100644 index d4c72fc2..00000000 --- a/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/GenericParserQuickBench.java +++ /dev/null @@ -1,111 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -package com.hedera.pbj.integration.jmh; - -import com.hedera.pbj.runtime.Codec; -import com.hedera.pbj.runtime.ParseException; -import com.hedera.pbj.runtime.io.buffer.BufferedData; -import com.hedera.pbj.runtime.io.buffer.Bytes; -import com.hedera.pbj.test.proto.pbj.NotCacheableAccountID; -import java.io.IOException; -import java.util.Random; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OperationsPerInvocation; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; -import org.openjdk.jmh.runner.Runner; -import org.openjdk.jmh.runner.options.Options; -import org.openjdk.jmh.runner.options.OptionsBuilder; - -@SuppressWarnings("unused") -@State(Scope.Benchmark) -@Fork(3) -@Warmup(iterations = 3) -@Measurement(iterations = 5) -@OutputTimeUnit(TimeUnit.MICROSECONDS) -@BenchmarkMode(Mode.Throughput) -public class GenericParserQuickBench { - private static final int INVOCATIONS = 1 * 1024; - - @State(Scope.Thread) - public static class BenchState { - record Model(int maxSize, Function factory, Codec codec) {} - - public enum Type { - NotCacheableAccountIDType(new Model( - 256, - random -> { - final NotCacheableAccountID.Builder builder = NotCacheableAccountID.newBuilder() - .shardNum(random.nextLong()) - .realmNum(random.nextLong()); - if (random.nextBoolean()) { - builder.accountNum(random.nextLong()); - } else { - byte[] arr = new byte[32]; - random.nextBytes(arr); - builder.alias(Bytes.wrap(arr)); - } - return builder.build(); - }, - NotCacheableAccountID.PROTOBUF)); - - private final Model model; - - Type(Model model) { - this.model = model; - } - } - - @Param - Type type; - - Model model; - byte[] array; - BufferedData bd; - - @Setup(Level.Trial) - public void setup() throws IOException { - model = type.model; - - array = new byte[INVOCATIONS * model.maxSize]; - // For determinism: - final Random random = new Random(723049435); - bd = BufferedData.wrap(array); - for (int i = 0, j = 0; i < INVOCATIONS; i++) { - model.codec.write(model.factory.apply(random), bd); - } - bd.flip(); - } - - @TearDown(Level.Trial) - public void tearDown() {} - } - - @Benchmark - @OperationsPerInvocation(INVOCATIONS) - public void bench(final BenchState state, final Blackhole blackhole) throws ParseException { - for (int invocation = 0; invocation < INVOCATIONS; invocation++) { - blackhole.consume(state.model.codec.parse(state.bd)); - } - } - - public static void main(String[] args) throws Exception { - Options opt = new OptionsBuilder() - .include(GenericParserQuickBench.class.getSimpleName()) - .build(); - - new Runner(opt).run(); - } -} diff --git a/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/JsonBench.java b/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/JsonBench.java index 397a27e4..8af892e5 100644 --- a/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/JsonBench.java +++ b/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/JsonBench.java @@ -11,10 +11,12 @@ import com.hedera.pbj.runtime.Codec; import com.hedera.pbj.runtime.JsonCodec; import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.io.PbjReader; import com.hedera.pbj.runtime.io.buffer.BufferedData; import com.hedera.pbj.test.proto.pbj.Everything; import com.hederahashgraph.api.proto.java.GetAccountDetailsResponse; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import org.openjdk.jmh.annotations.Benchmark; @@ -49,6 +51,8 @@ public static class JsonBenchmarkState { // input bytes private BufferedData jsonDataBuffer; private String jsonString; + private byte[] jsonStringUTF8; + private PbjReader jsonPbjDataReader; // output buffers private BufferedData outDataBuffer; @@ -69,6 +73,8 @@ public void configure( jsonDataBuffer.flip(); // get as string for parse tests jsonString = jsonDataBuffer.asUtf8String(); + jsonStringUTF8 = jsonString.getBytes(StandardCharsets.UTF_8); + jsonPbjDataReader = new PbjReader(jsonStringUTF8); // write to temp data buffer and then read into byte array BufferedData tempDataBuffer = BufferedData.allocate(5 * 1024 * 1024); @@ -90,13 +96,19 @@ public void configure( } } - /** Same as parsePbjByteBuffer because DataBuffer.wrap(byte[]) uses ByteBuffer today, added this because makes result plotting easier */ @Benchmark - public void parsePbj(JsonBenchmarkState benchmarkState, Blackhole blackhole) throws ParseException { + public void parsePbjBufferedData(JsonBenchmarkState benchmarkState, Blackhole blackhole) + throws ParseException { benchmarkState.jsonDataBuffer.position(0); blackhole.consume(benchmarkState.pbjJsonCodec.parse(benchmarkState.jsonDataBuffer)); } + @Benchmark + public void parsePbjReader(JsonBenchmarkState benchmarkState, Blackhole blackhole) throws ParseException { + benchmarkState.jsonPbjDataReader.resetWith(benchmarkState.jsonStringUTF8); + blackhole.consume(benchmarkState.pbjJsonCodec.parse(benchmarkState.jsonPbjDataReader)); + } + @Benchmark public void parseProtoC(JsonBenchmarkState benchmarkState, Blackhole blackhole) throws IOException { var builder = benchmarkState.builderSupplier.get(); diff --git a/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/ProtobufObjectBench.java b/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/ProtobufObjectBench.java index 640c988e..ad6bfc0d 100644 --- a/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/ProtobufObjectBench.java +++ b/pbj-integration-tests/src/jmh/java/com/hedera/pbj/integration/jmh/ProtobufObjectBench.java @@ -11,6 +11,8 @@ import com.hedera.pbj.integration.NonSynchronizedByteArrayOutputStream; import com.hedera.pbj.runtime.Codec; import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.io.PbjReader; +import com.hedera.pbj.runtime.io.PbjWriter; import com.hedera.pbj.runtime.io.buffer.BufferedData; import com.hedera.pbj.runtime.io.stream.ReadableStreamingData; import com.hedera.pbj.runtime.io.stream.WritableStreamingData; @@ -63,6 +65,7 @@ public static class BenchmarkState { private ByteBuffer protobufByteBufferDirect; private BufferedData protobufDataBufferDirect; private NonSynchronizedByteArrayInputStream bin; + private PbjReader pbjProtobufDataBuffer; // output buffers private NonSynchronizedByteArrayOutputStream bout; @@ -71,6 +74,7 @@ public static class BenchmarkState { private ByteBuffer bbout; private ByteBuffer bboutDirect; private byte[] outArray; + private PbjWriter outPbjWriter; public void configure( P pbjModelObject, @@ -85,11 +89,9 @@ public void configure( this.googleByteBufferParseMethod = googleByteBufferParseMethod; this.googleInputStreamParseMethod = googleInputStreamParseMethod; // write to temp data buffer and then read into byte array - BufferedData tempDataBuffer = BufferedData.allocate(5 * 1024 * 1024); + PbjWriter tempDataBuffer = new PbjWriter(5 << 20, false); pbjCodec.write(pbjModelObject, tempDataBuffer); - tempDataBuffer.flip(); - this.protobuf = new byte[(int) tempDataBuffer.remaining()]; - tempDataBuffer.readBytes(this.protobuf); + this.protobuf = tempDataBuffer.toByteArray(); // start by parsing using protoc this.googleModelObject = googleByteArrayParseMethod.parse(this.protobuf); @@ -101,12 +103,15 @@ public void configure( this.protobufDataBufferDirect = BufferedData.wrap(this.protobufByteBufferDirect); this.bin = new NonSynchronizedByteArrayInputStream(this.protobuf); ReadableStreamingData din = new ReadableStreamingData(this.bin); + this.pbjProtobufDataBuffer = new PbjReader(this.protobuf); + // output buffers this.bout = new NonSynchronizedByteArrayOutputStream(); WritableStreamingData dout = new WritableStreamingData(this.bout); this.outArray = new byte[this.protobuf.length * 2]; // make sure big enough this.outDataBuffer = BufferedData.allocate(this.protobuf.length); this.outDataBufferDirect = BufferedData.allocateOffHeap(this.protobuf.length); + this.outPbjWriter = new PbjWriter(this.protobuf.length, false); this.bbout = ByteBuffer.allocate(this.protobuf.length); this.bboutDirect = ByteBuffer.allocateDirect(this.protobuf.length); } catch (IOException e) { @@ -120,38 +125,10 @@ public void configure( /** Same as parsePbjByteBuffer because DataBuffer.wrap(byte[]) uses ByteBuffer today, added this because makes result plotting easier */ @Benchmark @OperationsPerInvocation(OPERATION_COUNT) - public void parsePbjByteArray(BenchmarkState benchmarkState, Blackhole blackhole) throws ParseException { - for (int i = 0; i < OPERATION_COUNT; i++) { - benchmarkState.protobufDataBuffer.resetPosition(); - blackhole.consume(benchmarkState.pbjCodec.parse(benchmarkState.protobufDataBuffer)); - } - } - - @Benchmark - @OperationsPerInvocation(OPERATION_COUNT) - public void parsePbjByteBuffer(BenchmarkState benchmarkState, Blackhole blackhole) throws ParseException { - for (int i = 0; i < OPERATION_COUNT; i++) { - benchmarkState.protobufDataBuffer.resetPosition(); - blackhole.consume(benchmarkState.pbjCodec.parse(benchmarkState.protobufDataBuffer)); - } - } - - @Benchmark - @OperationsPerInvocation(OPERATION_COUNT) - public void parsePbjByteBufferDirect(BenchmarkState benchmarkState, Blackhole blackhole) - throws ParseException { + public void parsePbjReaderArray(BenchmarkState benchmarkState, Blackhole blackhole) throws ParseException { for (int i = 0; i < OPERATION_COUNT; i++) { - benchmarkState.protobufDataBufferDirect.resetPosition(); - blackhole.consume(benchmarkState.pbjCodec.parse(benchmarkState.protobufDataBufferDirect)); - } - } - - @Benchmark - @OperationsPerInvocation(OPERATION_COUNT) - public void parsePbjInputStream(BenchmarkState benchmarkState, Blackhole blackhole) throws ParseException { - for (int i = 0; i < OPERATION_COUNT; i++) { - benchmarkState.bin.resetPosition(); - blackhole.consume(benchmarkState.pbjCodec.parse(new ReadableStreamingData(benchmarkState.bin))); + benchmarkState.pbjProtobufDataBuffer.resetWith(benchmarkState.protobuf); + blackhole.consume(benchmarkState.pbjCodec.parse(benchmarkState.pbjProtobufDataBuffer)); } } @@ -221,6 +198,16 @@ public void writePbjByteDirect(BenchmarkState benchmarkState, Blackhole bl } } + @Benchmark + @OperationsPerInvocation(OPERATION_COUNT) + public void writePbjWriter(BenchmarkState benchmarkState, Blackhole blackhole) throws IOException { + for (int i = 0; i < OPERATION_COUNT; i++) { + benchmarkState.outPbjWriter.reset(); + benchmarkState.pbjCodec.write(benchmarkState.pbjModelObject, benchmarkState.outPbjWriter); + blackhole.consume(benchmarkState.outPbjWriter); + } + } + @Benchmark @OperationsPerInvocation(OPERATION_COUNT) public void writePbjOutputStream(BenchmarkState benchmarkState, Blackhole blackhole) throws IOException { diff --git a/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/KeyDepthLimitTest.java b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/KeyDepthLimitTest.java new file mode 100644 index 00000000..20cc1530 --- /dev/null +++ b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/KeyDepthLimitTest.java @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.pbj.integration.test; + +import static com.hedera.pbj.runtime.Codec.DEFAULT_MAX_DEPTH; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import com.hedera.hapi.node.base.Key; +import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import java.io.ByteArrayOutputStream; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +/** + * Reproduces a bug where deeply nested Key/KeyList structures do not produce a ParseException + * with the message "Reached maximum allowed depth" when the depth limit is exceeded. + * + *

Original failing test: + * com.hedera.hapi.node.base.codec.KeyProtoCodecTest.deeplyNestedSerializedKeyUnderSixKiBIsRejectedByDefaultDepthLimit + * in hiero-consensus-node. + */ +public class KeyDepthLimitTest { + private static final int MAX_TRANSACTION_BYTES = 6 * 1024; + private static final int PBJ_MESSAGE_FRAMES_PER_KEY_LIST_LEVEL = 2; + private static final int DEEPEST_DEFAULT_ALLOWED_KEY_LIST_LEVELS = + DEFAULT_MAX_DEPTH / PBJ_MESSAGE_FRAMES_PER_KEY_LIST_LEVEL; + private static final int FIRST_KEY_LIST_LEVEL_REJECTED_BY_DEFAULT_DEPTH = + DEEPEST_DEFAULT_ALLOWED_KEY_LIST_LEVELS + 1; + private static final byte KEY_LIST_TAG = 50; + private static final byte KEY_LIST_KEYS_TAG = 10; + private static final byte ED25519_TAG = 18; + private static final byte[] ED25519_KEY = lengthDelimited(ED25519_TAG, new byte[32]); + + /** + * A deeply nested Key/KeyList structure under 6KiB must be rejected by the default depth limit, + * and the ParseException message must contain "Reached maximum allowed depth". + * + *

This test currently FAILS because the ParseException is thrown with an empty message + * instead of the expected "Reached maximum allowed depth" message. + */ + @Test + void deeplyNestedSerializedKeyUnderSixKiBIsRejectedByDefaultDepthLimit() { + final var serializedKey = deepestKeyListNestUnder(MAX_TRANSACTION_BYTES); + + assertTrue(serializedKey.bytes().length <= MAX_TRANSACTION_BYTES); + assertTrue(serializedKey.nestingLevels() > FIRST_KEY_LIST_LEVEL_REJECTED_BY_DEFAULT_DEPTH); + + final ParseException thrown = assertThrows( + ParseException.class, + () -> Key.PROTOBUF.parse( + serializedKey.bytes(), false, false, DEFAULT_MAX_DEPTH, MAX_TRANSACTION_BYTES)); + assertTrue( + thrown.getMessage() != null && thrown.getMessage().contains("Reached maximum allowed depth"), + "Expected ParseException message to contain 'Reached maximum allowed depth', but got: " + + thrown.getMessage()); + } + + @Test + void pbjParserRejectsFirstKeyListLevelBeyondDefaultDepthWithParseException() { + final var serializedKey = keyListNest(FIRST_KEY_LIST_LEVEL_REJECTED_BY_DEFAULT_DEPTH); + + assertTrue(serializedKey.bytes().length <= MAX_TRANSACTION_BYTES); + final ParseException thrown = + assertThrows(ParseException.class, () -> Key.PROTOBUF.parse(Bytes.wrap(serializedKey.bytes()))); + assertTrue( + thrown.getMessage() != null && thrown.getMessage().contains("Reached maximum allowed depth"), + "Expected ParseException message to contain 'Reached maximum allowed depth', but got: " + + thrown.getMessage()); + } + + @Test + void deeplyNestedSerializedKeyUnderSixKiBCanOverflowUnboundedParserStack() throws InterruptedException { + final var serializedKey = deepestKeyListNestUnder(MAX_TRANSACTION_BYTES); + + assertTrue(serializedKey.bytes().length <= MAX_TRANSACTION_BYTES); + assertTrue(serializedKey.nestingLevels() > FIRST_KEY_LIST_LEVEL_REJECTED_BY_DEFAULT_DEPTH); + + final Throwable result = parseWithUnboundedDepthOnConstrainedStack(serializedKey.bytes()); + assertInstanceOf(StackOverflowError.class, result); + } + + private static Throwable parseWithUnboundedDepthOnConstrainedStack(final byte[] serializedKey) + throws InterruptedException { + return parseOnStack(serializedKey, Integer.MAX_VALUE, 64 * 1024); + } + + private static Throwable parseOnStack(final byte[] serializedKey, final int maxDepth, final int stackSize) + throws InterruptedException { + final var thrown = new AtomicReference(); + final var thread = new Thread( + null, + () -> { + try { + Key.PROTOBUF.parse(serializedKey, false, false, maxDepth, MAX_TRANSACTION_BYTES); + } catch (final Throwable t) { + thrown.set(t); + } + }, + "pbj-key-stack-overflow-test", + stackSize); + thread.setUncaughtExceptionHandler((ignored, t) -> thrown.set(t)); + + thread.start(); + thread.join(Duration.ofSeconds(10).toMillis()); + if (thread.isAlive()) { + thread.interrupt(); + fail("Timed out while parsing deeply nested key"); + } + return thrown.get(); + } + + private static SerializedKey deepestKeyListNestUnder(final int maxBytes) { + byte[] bytes = ED25519_KEY; + int nestingLevels = 0; + while (true) { + final var next = keyWithSingleNestedKeyList(bytes); + if (next.length > maxBytes) { + return new SerializedKey(bytes, nestingLevels); + } + bytes = next; + nestingLevels++; + } + } + + private static SerializedKey keyListNest(final int nestingLevels) { + byte[] bytes = ED25519_KEY; + for (int i = 0; i < nestingLevels; i++) { + bytes = keyWithSingleNestedKeyList(bytes); + } + return new SerializedKey(bytes, nestingLevels); + } + + private static byte[] keyWithSingleNestedKeyList(final byte[] nestedKey) { + final var keyList = lengthDelimited(KEY_LIST_KEYS_TAG, nestedKey); + return lengthDelimited(KEY_LIST_TAG, keyList); + } + + private static byte[] lengthDelimited(final byte tag, final byte[] contents) { + final var out = new ByteArrayOutputStream(1 + varIntSize(contents.length) + contents.length); + out.write(tag); + writeVarInt(out, contents.length); + out.writeBytes(contents); + return out.toByteArray(); + } + + private static void writeVarInt(final ByteArrayOutputStream out, int value) { + while (true) { + if ((value & ~0x7F) == 0) { + out.write(value); + return; + } + out.write((value & 0x7F) | 0x80); + value >>>= 7; + } + } + + private static int varIntSize(int value) { + int size = 1; + while ((value & ~0x7F) != 0) { + size++; + value >>>= 7; + } + return size; + } + + private record SerializedKey(byte[] bytes, int nestingLevels) {} +} diff --git a/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxDepthTest.java b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxDepthTest.java index f9b6c8ef..c91172cb 100644 --- a/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxDepthTest.java +++ b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxDepthTest.java @@ -1,10 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 package com.hedera.pbj.integration.test; -import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertEquals; -import com.hedera.pbj.runtime.ParseException; -import com.hedera.pbj.runtime.io.buffer.BufferedData; +import com.hedera.pbj.runtime.io.PbjReader; +import com.hedera.pbj.runtime.io.PbjWriter; import com.hedera.pbj.test.proto.pbj.MessageWithMessage; import org.junit.jupiter.api.Test; @@ -14,13 +14,13 @@ void testMaxDepth_depth0() throws Exception { MessageWithMessage msg; msg = MessageWithMessage.newBuilder().build(); - BufferedData bd = BufferedData.allocate(MessageWithMessage.PROTOBUF.measureRecord(msg)); - MessageWithMessage.PROTOBUF.write(msg, bd); + PbjWriter writer = new PbjWriter(MessageWithMessage.PROTOBUF.measureRecord(msg), false); + MessageWithMessage.PROTOBUF.write(msg, writer); // None should throw - MessageWithMessage.PROTOBUF.parse(bd, false, 0); - MessageWithMessage.PROTOBUF.parse(bd, false, 1); - MessageWithMessage.PROTOBUF.parse(bd, false, 2); + MessageWithMessage.PROTOBUF.parse(writer.toPbjReader(), false, 0); + MessageWithMessage.PROTOBUF.parse(writer.toPbjReader(), false, 1); + MessageWithMessage.PROTOBUF.parse(writer.toPbjReader(), false, 2); } @Test @@ -32,16 +32,11 @@ void testMaxDepth_depth1_actually0() throws Exception { // so parse() wouldn't be called to read it, and hence the actual depth is still 0 .message(MessageWithMessage.newBuilder().build()) .build(); - BufferedData bd = BufferedData.allocate(MessageWithMessage.PROTOBUF.measureRecord(msg)); - MessageWithMessage.PROTOBUF.write(msg, bd); - - // None should throw - bd.reset(); - MessageWithMessage.PROTOBUF.parse(bd, false, 0); - bd.reset(); - MessageWithMessage.PROTOBUF.parse(bd, false, 1); - bd.reset(); - MessageWithMessage.PROTOBUF.parse(bd, false, 2); + PbjWriter writer = new PbjWriter(MessageWithMessage.PROTOBUF.measureRecord(msg), false); + MessageWithMessage.PROTOBUF.write(msg, writer); + MessageWithMessage.PROTOBUF.parse(writer.toPbjReader(), false, 0); + MessageWithMessage.PROTOBUF.parse(writer.toPbjReader(), false, 1); + MessageWithMessage.PROTOBUF.parse(writer.toPbjReader(), false, 2); } @Test @@ -55,16 +50,21 @@ void testMaxDepth_depth2_actually1() throws Exception { .message(MessageWithMessage.newBuilder().build()) .build()) .build(); - BufferedData bd = BufferedData.allocate(MessageWithMessage.PROTOBUF.measureRecord(msg)); - MessageWithMessage.PROTOBUF.write(msg, bd); - - // 0 should throw - bd.reset(); - assertThrows(ParseException.class, () -> MessageWithMessage.PROTOBUF.parse(bd, false, 0)); - bd.reset(); - MessageWithMessage.PROTOBUF.parse(bd, false, 1); - bd.reset(); - MessageWithMessage.PROTOBUF.parse(bd, false, 2); + PbjWriter writer = new PbjWriter(MessageWithMessage.PROTOBUF.measureRecord(msg), false); + MessageWithMessage.PROTOBUF.write(msg, writer); + + // 0 should error + PbjReader reader = writer.toPbjReader(); + MessageWithMessage.PROTOBUF.parse(reader, false, 0); + assertEquals(PbjReader.MaxDepthReached, reader.error()); + + reader = writer.toPbjReader(); + MessageWithMessage.PROTOBUF.parse(reader, false, 1); + assertEquals(0, reader.error()); + + reader = writer.toPbjReader(); + MessageWithMessage.PROTOBUF.parse(reader, false, 2); + assertEquals(0, reader.error()); } @Test @@ -80,15 +80,19 @@ void testMaxDepth_depth3_actually2() throws Exception { .build()) .build()) .build(); - BufferedData bd = BufferedData.allocate(MessageWithMessage.PROTOBUF.measureRecord(msg)); - MessageWithMessage.PROTOBUF.write(msg, bd); - - // 0 and 1 should throw - bd.reset(); - assertThrows(ParseException.class, () -> MessageWithMessage.PROTOBUF.parse(bd, false, 0)); - bd.reset(); - assertThrows(ParseException.class, () -> MessageWithMessage.PROTOBUF.parse(bd, false, 1)); - bd.reset(); - MessageWithMessage.PROTOBUF.parse(bd, false, 2); + PbjWriter writer = new PbjWriter(MessageWithMessage.PROTOBUF.measureRecord(msg), false); + MessageWithMessage.PROTOBUF.write(msg, writer); + + PbjReader reader = writer.toPbjReader(); + MessageWithMessage.PROTOBUF.parse(reader, false, 0); + assertEquals(PbjReader.MaxDepthReached, reader.error()); + + reader = writer.toPbjReader(); + MessageWithMessage.PROTOBUF.parse(reader, false, 1); + assertEquals(PbjReader.MaxDepthReached, reader.error()); + + reader = writer.toPbjReader(); + MessageWithMessage.PROTOBUF.parse(reader, false, 2); + assertEquals(0, reader.error()); } } diff --git a/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxSizeTest.java b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxSizeTest.java index 5ca4f5ed..90b302c3 100644 --- a/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxSizeTest.java +++ b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MaxSizeTest.java @@ -88,32 +88,22 @@ void testNestedMaxSize() throws Exception { final Bytes bytes = Everything.PROTOBUF.toBytes(everything); // Try negative cases first: - assertThrows(ParseException.class, () -> Everything.PROTOBUF.parse(bytes.toReadableSequentialData())); + assertThrows(ParseException.class, () -> Everything.PROTOBUF.parse(bytes)); assertThrows( ParseException.class, - () -> Everything.PROTOBUF.parse( - bytes.toReadableSequentialData(), false, false, Codec.DEFAULT_MAX_DEPTH, 256)); + () -> Everything.PROTOBUF.parse(bytes, false, false, Codec.DEFAULT_MAX_DEPTH, 256)); assertThrows( ParseException.class, - () -> Everything.PROTOBUF.parse( - bytes.toReadableSequentialData(), - false, - false, - Codec.DEFAULT_MAX_DEPTH, - Codec.DEFAULT_MAX_SIZE)); + () -> Everything.PROTOBUF.parse(bytes, false, false, Codec.DEFAULT_MAX_DEPTH, Codec.DEFAULT_MAX_SIZE)); // +1 still shouldn't work because the outer and the inner objects are still larger: assertThrows( ParseException.class, () -> Everything.PROTOBUF.parse( - bytes.toReadableSequentialData(), - false, - false, - Codec.DEFAULT_MAX_DEPTH, - Codec.DEFAULT_MAX_SIZE + 1)); + bytes, false, false, Codec.DEFAULT_MAX_DEPTH, Codec.DEFAULT_MAX_SIZE + 1)); // Now try supplying a large enough maxSize to parse it: - final Everything parsedEverything = Everything.PROTOBUF.parse( - bytes.toReadableSequentialData(), false, false, Codec.DEFAULT_MAX_DEPTH, Codec.DEFAULT_MAX_SIZE * 2); + final Everything parsedEverything = + Everything.PROTOBUF.parse(bytes, false, false, Codec.DEFAULT_MAX_DEPTH, Codec.DEFAULT_MAX_SIZE * 2); assertEquals(everything, parsedEverything); } diff --git a/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MixedByteBufferWriteTest.java b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MixedByteBufferWriteTest.java new file mode 100644 index 00000000..e31d800c --- /dev/null +++ b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/MixedByteBufferWriteTest.java @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.pbj.integration.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.io.buffer.BufferedData; +import com.hedera.pbj.test.proto.pbj.TimestampTest; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.junit.jupiter.api.Test; + +/** + * Verifies that direct {@link ByteBuffer} writes and writes via a {@link BufferedData} wrapping + * the same buffer share the underlying position — mirroring the pattern used in + * {@code PcesFileChannelWriter}, which writes an int header directly to a {@link ByteBuffer} and + * then writes a protobuf message via the {@link com.hedera.pbj.runtime.io.WritableSequentialData} + * that wraps it. + */ +class MixedByteBufferWriteTest { + + /** + * Replicates the PcesFileChannelWriter write sequence: + * 1. measure serialized size + * 2. write size as int directly to ByteBuffer + * 3. write protobuf message via the BufferedData that wraps the same ByteBuffer + * 4. flip and read back both fields, asserting round-trip correctness. + */ + @Test + void writeIntHeaderThenProtobufMessageRoundTrips() throws IOException, ParseException { + final TimestampTest message = new TimestampTest(5155135L, 44513); + final int size = TimestampTest.PROTOBUF.measureRecord(message); + + final ByteBuffer buffer = ByteBuffer.allocateDirect(Integer.BYTES + size); + final BufferedData writableSequentialData = BufferedData.wrap(buffer); + + // Direct write to ByteBuffer — advances buffer.position() by 4 + buffer.putInt(size); + + // Write protobuf message via the wrapping BufferedData — continues from position 4 + TimestampTest.PROTOBUF.write(message, writableSequentialData); + + buffer.flip(); + + final int readSize = buffer.getInt(); + assertEquals(size, readSize, "size header mismatch"); + + final byte[] messageBytes = new byte[readSize]; + buffer.get(messageBytes); + final TimestampTest parsed = TimestampTest.PROTOBUF.parse(BufferedData.wrap(messageBytes)); + + assertEquals(message, parsed, "round-tripped message mismatch"); + } + + /** + * Explicitly verifies that position advances made directly on the ByteBuffer are immediately + * visible through the wrapping BufferedData and vice versa. + */ + @Test + void positionIsSharedBetweenByteBufferAndWrappedBufferedData() throws IOException { + final TimestampTest message = new TimestampTest(999L, 1); + final int size = TimestampTest.PROTOBUF.measureRecord(message); + + final ByteBuffer buffer = ByteBuffer.allocateDirect(Integer.BYTES + size); + final BufferedData wsd = BufferedData.wrap(buffer); + + assertEquals(0, buffer.position(), "initial ByteBuffer position"); + assertEquals(0, wsd.position(), "initial BufferedData position"); + + // Direct write advances both views + buffer.putInt(size); + assertEquals(Integer.BYTES, buffer.position(), "ByteBuffer position after putInt"); + assertEquals(Integer.BYTES, wsd.position(), "BufferedData position after putInt"); + + // Write via BufferedData advances both views + TimestampTest.PROTOBUF.write(message, wsd); + assertEquals(Integer.BYTES + size, buffer.position(), "ByteBuffer position after protobuf write"); + assertEquals(Integer.BYTES + size, wsd.position(), "BufferedData position after protobuf write"); + } + + /** + * Verifies the buffer-expansion path: when the serialized size exceeds the current buffer + * capacity a new ByteBuffer is allocated and re-wrapped, and the data still round-trips. + * This mirrors the expandBuffer branch in PcesFileChannelWriter.writeEvent(). + */ + @Test + void bufferExpansionAndRewrapRoundTrips() throws IOException, ParseException { + final TimestampTest message = new TimestampTest(123456789L, 999999999); + final int size = TimestampTest.PROTOBUF.measureRecord(message); + + // Start with a buffer that is intentionally too small + ByteBuffer buffer = ByteBuffer.allocateDirect(1); + + final boolean needsExpansion = (size + Integer.BYTES) > buffer.capacity(); + if (needsExpansion) { + buffer = ByteBuffer.allocateDirect(size + Integer.BYTES); + } + + final BufferedData writableSequentialData = BufferedData.wrap(buffer); + + buffer.putInt(size); + TimestampTest.PROTOBUF.write(message, writableSequentialData); + + buffer.flip(); + + final int readSize = buffer.getInt(); + assertEquals(size, readSize, "size header mismatch after expansion"); + + final byte[] messageBytes = new byte[readSize]; + buffer.get(messageBytes); + final TimestampTest parsed = TimestampTest.PROTOBUF.parse(BufferedData.wrap(messageBytes)); + + assertEquals(message, parsed, "round-tripped message mismatch after expansion"); + } +} diff --git a/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/PbjReaderWriterTest.java b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/PbjReaderWriterTest.java new file mode 100644 index 00000000..85256b2b --- /dev/null +++ b/pbj-integration-tests/src/test/java/com/hedera/pbj/integration/test/PbjReaderWriterTest.java @@ -0,0 +1,1494 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.pbj.integration.test; + +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.io.PbjReader; +import com.hedera.pbj.runtime.io.PbjWriter; +import com.hedera.pbj.runtime.io.ReadableSequentialData; +import com.hedera.pbj.runtime.io.WritableSequentialData; +import com.hedera.pbj.runtime.io.buffer.BufferedData; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import com.hedera.pbj.runtime.io.stream.ReadableStreamingData; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.junit.jupiter.api.Test; + +public class PbjReaderWriterTest { + + @Test + void WriteConstructorBufferSizeConsistent() { + final int expectedSize = new PbjWriter().internalArray().length; + assertEquals(expectedSize, new PbjWriter((OutputStream) null).internalArray().length); + assertEquals(expectedSize, new PbjWriter((WritableSequentialData) null).internalArray().length); + assertEquals(expectedSize, new PbjWriter(128, true).internalArray().length); + // Does not apply to ByteBuffer, byte[], or reserve when large, or the below + assertEquals(128, new PbjWriter(128, false).internalArray().length); + } + + @Test + void writerConstructorCanReserveLarge() { + PbjWriter writer = new PbjWriter(2 << 20, true); + assertEquals(2 << 20, writer.internalArray().length); + } + + @Test + void writeByteBufferConstructorHeap() { + ByteBuffer bb = ByteBuffer.allocate(32); + PbjWriter writer = new PbjWriter(bb); + writer.writeByte3((byte) 11, (byte) 22, (byte) 33); + assertEquals(11, bb.array()[bb.arrayOffset()]); + assertEquals(22, bb.array()[bb.arrayOffset() + 1]); + assertEquals(33, bb.array()[bb.arrayOffset() + 2]); + assertEquals(3, writer.position()); + } + + @Test + void writeByteBufferConstructorDirect() { + ByteBuffer bb = ByteBuffer.allocateDirect(32); + PbjWriter writer = new PbjWriter(bb); + writer.writeByte3((byte) 11, (byte) 22, (byte) 33); + writer.flush(); + bb.flip(); + assertEquals(11, bb.get()); + assertEquals(22, bb.get()); + assertEquals(33, bb.get()); + } + + @Test + void writeBytesFromRandomAccessData() { + Bytes src = Bytes.wrap(new byte[] {10, 20, 30, 40}); + PbjWriter writer = new PbjWriter(); + writer.writeBytes(src); + assertArrayEquals(new byte[] {10, 20, 30, 40}, writer.toByteArray()); + } + + @Test + void writeBytesFromBufferedData() { + BufferedData src = BufferedData.wrap(new byte[] {10, 20, 30, 40}); + PbjWriter writer = new PbjWriter(); + writer.writeBytes(src); + assertArrayEquals(new byte[] {10, 20, 30, 40}, writer.toByteArray()); + } + + @Test + void closeFlushesDataToOutputStream() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + writer.writeByte2((byte) 55, (byte) 66); + assertArrayEquals(new byte[] {}, baos.toByteArray()); + writer.close(); + assertArrayEquals(new byte[] {55, 66}, baos.toByteArray()); + } + + @Test + void closeIsNoopWithoutOutputStream() { + PbjWriter writer = new PbjWriter(); + writer.writeByte((byte) 1); + writer.close(); + assertEquals(1, writer.position()); + } + + @Test + void toByteArrayWrappedErrorsOnStreamingWriter() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + assertEquals(Bytes.EMPTY, writer.internalArrayWrapped()); + assertEquals(Bytes.EMPTY, writer.toByteArrayWrapped()); + assertEquals(PbjWriter.UsageError, writer.error()); + assertThrows(RuntimeException.class, () -> writer.throwOnError()); + } + + @Test + void toByteArrayErrorsOnStreamingWriter() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + assertEquals(null, writer.toByteArray()); + assertEquals(PbjWriter.UsageError, writer.error()); + assertThrows(RuntimeException.class, () -> writer.throwOnError()); + } + + @Test + void writerRelativeReserve() { + PbjWriter writer = new PbjWriter(); + byte[] origArray = writer.internalArray(); + writer.reserveRel(origArray.length); + byte[] arr2 = writer.internalArray(); + assertEquals(origArray, arr2); // same object + + writer.reserveRel(origArray.length + 1); + byte[] arr3 = writer.internalArray(); + assertNotEquals(origArray, arr3); // diff object + + writer.skip(arr3.length - 1); + writer.reserveRel(1); + byte[] arr4 = writer.internalArray(); + assertEquals(arr3, arr4); + + writer.skip(1); + writer.reserveRel(0); + byte[] arr5 = writer.internalArray(); + assertEquals(arr4, arr5); + + writer.reserveRel(1); + byte[] arr6 = writer.internalArray(); + assertNotEquals(arr4, arr6); + } + + @Test + void writerUsesByteArray() { + byte[] bytes = new byte[128]; + PbjWriter writer = new PbjWriter(bytes, 0); + writer.writeByte3((byte) 10, (byte) 20, (byte) 30); + assertEquals(bytes[0], 10); + assertEquals(bytes[1], 20); + assertEquals(bytes[2], 30); + } + + @Test + void manyStringRoundtrip() { + byte[] bytes = new byte[128]; + PbjWriter writer = new PbjWriter(bytes, 0); + String strings[] = { + "a", + "\u0100", + "\u2603", + "\uD800\uDC00", + "\uE000", + "a\u0100\u2603\uE000\uD800\uDC00", + "\uD800\uDC00\uE000\u2603\u0100a", + "\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uE000\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100a" + }; + for (String str : strings) { + writer.reset(); + writer.writeStringWithTag(str); + String res = writer.toPbjReader().readString(128); + assertEquals(str, res); + } + } + + @Test + void toByteArrayDoesAClone() { + byte[] bytes = new byte[128]; + PbjWriter writer = new PbjWriter(bytes, 0); + for (int i = 0; i < 128; i++) { + writer.writeVarIntNoZZ(i); + } + assertEquals(128, writer.position()); + assertEquals(bytes, writer.internalArray()); + assertEquals(bytes, writer.internalArrayWrapped().arrayUnsafe()); + byte[] arr1 = writer.toByteArray(); + assertNotEquals(bytes, arr1); + Bytes arr2 = writer.toByteArrayWrapped(); + assertNotEquals(bytes, arr1); + assertNotEquals(bytes, arr2.arrayUnsafe()); + for (int i = 0; i < 128; i++) { + assertEquals(i, arr1[i]); + assertEquals(i, arr2.arrayUnsafe()[i]); + } + } + + @Test + void writeResetCheck() { + PbjWriter writer = new PbjWriter(); + writer.writeByte((byte) 10); + assertEquals(1, writer.position()); + writer.reset(); + assertEquals(0, writer.position()); + writer.writeByte2((byte) 99, (byte) 88); + assertEquals(2, writer.position()); + assertEquals(99, writer.internalArray()[0]); + assertEquals(88, writer.internalArray()[1]); + assertArrayEquals(new byte[] {99, 88}, writer.toByteArray()); + } + + @Test + void throwOnErrorThrows_NoPrevOverwrite() { + PbjWriter writer = new PbjWriter(); + writer.writeStringWithTag("\uD800"); // lone surrogate sets MalformString + assertEquals(PbjWriter.MalformString, writer.error()); + assertThrows(RuntimeException.class, writer::throwOnError); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer2 = new PbjWriter(baos); + writer2.writeStringWithTag("\uD800"); // lone surrogate sets MalformString + assertEquals(PbjWriter.MalformString, writer2.error()); + assertThrows(RuntimeException.class, writer2::throwOnError); + // Confirm error didn't change + assertEquals(Bytes.EMPTY, writer2.toByteArrayWrapped()); // if no error, this would set usage error + assertEquals(PbjWriter.MalformString, writer2.error()); + + baos = new ByteArrayOutputStream(); + PbjWriter writer3 = new PbjWriter(baos); + assertEquals(Bytes.EMPTY, writer3.toByteArrayWrapped()); + assertEquals(PbjWriter.UsageError, writer3.error()); + } + + @Test + void encodeUtf8HighSurrogateFollowedByHighSurrogate() { + PbjWriter writer = new PbjWriter(); + writer.writeStringNoTag("\uD800\uD801"); + assertEquals(0, writer.position()); + } + + @Test + void doesntThrowOnNoError() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4}); + assertEquals(0x04030201, reader.readIntLE()); + assertEquals(0, reader.error()); + assertDoesNotThrow(() -> reader.throwOnError()); // must not throw + + PbjWriter writer = new PbjWriter(); + writer.writeByte((byte) 1); + assertEquals(0, writer.error()); + writer.throwOnError(); // must not throw + } + + @Test + void manyRoundtrip() { + PbjWriter w = new PbjWriter(); + + w.writeInt(0x01020304); + w.writeIntBE(0x05060708); + w.writeIntLE(0x090A0B0C); + + w.writeLong(0x0102030405060708L); + w.writeLongBE(0x090A0B0C0D0E0F10L); + w.writeLongLE(0x1112131415161718L); + + w.writeFloat(1.5f); + w.writeFloatBE(2.5f); + w.writeFloatLE(3.5f); + + w.writeDouble(1.25); + w.writeDoubleBE(2.25); + w.writeDoubleLE(3.25); + + w.writeBoolean(true); + w.writeBoolean(false); + w.writeByte((byte) 127); + + byte[] arr1 = {10, 20, 30, 40, 50}; + w.writeBytes(arr1); + w.writeBytes(arr1, 1, 3); + w.writeBytes(Bytes.wrap(new byte[] {60, 70})); + BufferedData bd = BufferedData.allocate(2); + bd.writeByte((byte) 80); + bd.writeByte((byte) 90); + bd.flip(); + w.writeBytes(bd); + + w.writeStringWithTag("hello"); + w.writeStringWithTag("Ā☃"); + + PbjReader reader = w.toPbjReader(); + + assertEquals(0x01020304, reader.readInt()); + assertEquals(0x05060708, reader.readIntBE()); + assertEquals(0x090A0B0C, reader.readIntLE()); + + assertEquals(0x0102030405060708L, reader.readLong()); + assertEquals(0x090A0B0C0D0E0F10L, reader.readLongBE()); + assertEquals(0x1112131415161718L, reader.readLongLE()); + + assertEquals(1.5f, reader.readFloat(), 0f); + assertEquals(2.5f, reader.readFloat(), 0f); + assertEquals(3.5f, reader.readFloatLE(), 0f); + + assertEquals(1.25, reader.readDouble(), 0.0); + assertEquals(2.25, reader.readDouble(), 0.0); + assertEquals(3.25, reader.readDoubleLE(), 0.0); + + assertEquals(true, reader.readBoolean()); + assertEquals(false, reader.readBoolean()); + assertEquals(127, reader.readByte()); + + byte[] dst1 = new byte[5]; + reader.readBytes(dst1); + assertArrayEquals(arr1, dst1); + + byte[] dst2 = new byte[3]; + reader.readBytes(dst2); + assertArrayEquals(new byte[] {20, 30, 40}, dst2); + + byte[] dst3 = new byte[2]; + reader.readBytes(dst3); + assertArrayEquals(new byte[] {60, 70}, dst3); + + byte[] dst4 = new byte[2]; + reader.readBytes(dst4); + assertArrayEquals(new byte[] {80, 90}, dst4); + + assertEquals("hello", reader.readString(100)); + assertEquals("Ā☃", reader.readString(100)); + + assertFalse(reader.hasRemaining()); + assertEquals(0, w.error()); + } + + @Test + void writeVarIntZigZagRoundtrip() { + int[] values = {0, 1, -1, Integer.MAX_VALUE, Integer.MIN_VALUE, 100, -100}; + PbjWriter writer = new PbjWriter(); + for (int v : values) { + writer.reset(); + writer.writeVarInt(v, true); + assertEquals(v, writer.toPbjReader().readVarIntZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarInt(v, false); + assertEquals(v, writer.toPbjReader().readVarIntNoZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarIntZZ(v); + assertEquals(v, writer.toPbjReader().readVarIntZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarIntNoZZ(v); + assertEquals(v, writer.toPbjReader().readVarIntNoZZ(), "Failed for value " + v); + } + } + + @Test + void writeVarLongZigZagRoundtrip() { + long[] values = {0L, 1L, -1L, Long.MAX_VALUE, Long.MIN_VALUE, 1_000_000_000L, -1_000_000_000L}; + PbjWriter writer = new PbjWriter(); + for (long v : values) { + writer.reset(); + writer.writeVarLong(v, true); + assertEquals(v, writer.toPbjReader().readVarLongZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarLong(v, false); + assertEquals(v, writer.toPbjReader().readVarLongNoZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarLongZZ(v); + assertEquals(v, writer.toPbjReader().readVarLongZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarLongNoZZ(v); + int noZZLen = writer.position(); + assertEquals(v, writer.toPbjReader().readVarLongNoZZ(), "Failed for value " + v); + + // edge test + writer.reset(); + int len = writer.internalArray().length; + writer.skip(len); + writer.writeVarLongNoZZ(v); + byte[] buf = writer.internalArray(); + for (int i = 0; i < noZZLen; i++) { + assertEquals(buf[i], buf[i + len]); + } + } + } + + @Test + void testWriteByteAtEdge() { + PbjWriter writer = new PbjWriter(); + int defaultLen = writer.internalArray().length; + writer.skip(defaultLen - 1); + writer.writeByte((byte) 1); + byte[] internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 1]); + writer.writeByte((byte) 1); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 1]); + assertEquals((byte) 1, internalArray[defaultLen]); + + writer = new PbjWriter(); + writer.skip(defaultLen - 2); + writer.writeByte2((byte) 1, (byte) 2); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 2]); + assertEquals((byte) 2, internalArray[defaultLen - 1]); + writer.skip(-1); + writer.writeByte2((byte) 1, (byte) 2); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 1]); + assertEquals((byte) 2, internalArray[defaultLen]); + + writer = new PbjWriter(); + writer.skip(defaultLen - 3); + writer.writeByte3((byte) 1, (byte) 2, (byte) 3); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 3]); + assertEquals((byte) 2, internalArray[defaultLen - 2]); + assertEquals((byte) 3, internalArray[defaultLen - 1]); + writer.skip(-2); + writer.writeByte3((byte) 1, (byte) 2, (byte) 3); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 2]); + assertEquals((byte) 2, internalArray[defaultLen - 1]); + assertEquals((byte) 3, internalArray[defaultLen]); + + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeByte4((byte) 1, (byte) 2, (byte) 3, (byte) 4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 4]); + assertEquals((byte) 2, internalArray[defaultLen - 3]); + assertEquals((byte) 3, internalArray[defaultLen - 2]); + assertEquals((byte) 4, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeByte4((byte) 1, (byte) 2, (byte) 3, (byte) 4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 3]); + assertEquals((byte) 2, internalArray[defaultLen - 2]); + assertEquals((byte) 3, internalArray[defaultLen - 1]); + assertEquals((byte) 4, internalArray[defaultLen]); + + // writeInt writes big-endian: 4 = {0, 0, 0, 4} + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeInt(4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 0, internalArray[defaultLen - 4]); + assertEquals((byte) 0, internalArray[defaultLen - 3]); + assertEquals((byte) 0, internalArray[defaultLen - 2]); + assertEquals((byte) 4, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeInt(4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 0, internalArray[defaultLen - 3]); + assertEquals((byte) 0, internalArray[defaultLen - 2]); + assertEquals((byte) 0, internalArray[defaultLen - 1]); + assertEquals((byte) 4, internalArray[defaultLen]); + + // writeIntLE writes little-endian: 4 = {4, 0, 0, 0} + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeIntLE(4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 4, internalArray[defaultLen - 4]); + assertEquals((byte) 0, internalArray[defaultLen - 3]); + assertEquals((byte) 0, internalArray[defaultLen - 2]); + assertEquals((byte) 0, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeIntLE(4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 4, internalArray[defaultLen - 3]); + assertEquals((byte) 0, internalArray[defaultLen - 2]); + assertEquals((byte) 0, internalArray[defaultLen - 1]); + assertEquals((byte) 0, internalArray[defaultLen]); + + // writeLong writes big-endian: 8 = {0, 0, 0, 0, 0, 0, 0, 8} + writer = new PbjWriter(); + writer.skip(defaultLen - 8); + writer.writeLong(8); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 8, internalArray[defaultLen - 1]); + writer.skip(-7); + writer.writeLong(8); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 0, internalArray[defaultLen - 1]); + assertEquals((byte) 8, internalArray[defaultLen]); + + // writeFloatLE writes little-endian: 4.0f = 0x40800000 = {0x00, 0x00, 0x80, 0x40} + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeFloatLE(4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 0x00, internalArray[defaultLen - 4]); + assertEquals((byte) 0x00, internalArray[defaultLen - 3]); + assertEquals((byte) 0x80, internalArray[defaultLen - 2]); + assertEquals((byte) 0x40, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeFloatLE(4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 0x00, internalArray[defaultLen - 3]); + assertEquals((byte) 0x00, internalArray[defaultLen - 2]); + assertEquals((byte) 0x80, internalArray[defaultLen - 1]); + assertEquals((byte) 0x40, internalArray[defaultLen]); + + // writeDoubleLE writes little-endian: 8.0 = 0x4020000000000000 = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // 0x40} + writer = new PbjWriter(); + writer.skip(defaultLen - 8); + writer.writeDoubleLE(8); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 0x20, internalArray[defaultLen - 2]); + assertEquals((byte) 0x40, internalArray[defaultLen - 1]); + writer.skip(-7); + writer.writeDoubleLE(8); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 0x20, internalArray[defaultLen - 1]); + assertEquals((byte) 0x40, internalArray[defaultLen]); + + byte[] array4 = new byte[] {10, 20, 30, 40}; + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeBytes(array4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 4]); + assertEquals((byte) 20, internalArray[defaultLen - 3]); + assertEquals((byte) 30, internalArray[defaultLen - 2]); + assertEquals((byte) 40, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeBytes(array4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 3]); + assertEquals((byte) 20, internalArray[defaultLen - 2]); + assertEquals((byte) 30, internalArray[defaultLen - 1]); + assertEquals((byte) 40, internalArray[defaultLen]); + + Bytes bytes4 = Bytes.wrap(new byte[] {10, 20, 30, 40}); + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeBytes(bytes4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 4]); + assertEquals((byte) 20, internalArray[defaultLen - 3]); + assertEquals((byte) 30, internalArray[defaultLen - 2]); + assertEquals((byte) 40, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeBytes(bytes4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 3]); + assertEquals((byte) 20, internalArray[defaultLen - 2]); + assertEquals((byte) 30, internalArray[defaultLen - 1]); + assertEquals((byte) 40, internalArray[defaultLen]); + + BufferedData bb4 = BufferedData.wrap(new byte[] {10, 20, 30, 40}); + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeBytes(bb4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 4]); + assertEquals((byte) 20, internalArray[defaultLen - 3]); + assertEquals((byte) 30, internalArray[defaultLen - 2]); + assertEquals((byte) 40, internalArray[defaultLen - 1]); + writer.skip(-3); + bb4.resetPosition(); + writer.writeBytes(bb4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 3]); + assertEquals((byte) 20, internalArray[defaultLen - 2]); + assertEquals((byte) 30, internalArray[defaultLen - 1]); + assertEquals((byte) 40, internalArray[defaultLen]); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + writer = new PbjWriter(baos); + writer.skip(defaultLen - 4); + bb4.resetPosition(); + writer.writeBytes(bb4); + assertEquals(0, baos.size()); + + writer.skip(-3); + bb4.resetPosition(); + writer.writeBytes(bb4); + assertEquals(defaultLen + 1, writer.position()); + + assertEquals(defaultLen, baos.size()); + byte[] data = baos.toByteArray(); + assertEquals(defaultLen, data.length); + assertEquals((byte) 10, data[defaultLen - 4]); + + assertEquals((byte) 10, data[defaultLen - 3]); + assertEquals((byte) 20, data[defaultLen - 2]); + assertEquals((byte) 30, data[defaultLen - 1]); + // last byte inside internal buffer + + // Again but the Bytes path + baos = new ByteArrayOutputStream(); + writer = new PbjWriter(baos); + writer.skip(defaultLen - 4); + writer.writeBytes(bytes4); + assertEquals(0, baos.size()); + + writer.skip(-3); + writer.writeBytes(bytes4); + assertEquals(defaultLen + 1, writer.position()); + + assertEquals(defaultLen, baos.size()); + data = baos.toByteArray(); + assertEquals(defaultLen, data.length); + assertEquals((byte) 10, data[defaultLen - 4]); + + assertEquals((byte) 10, data[defaultLen - 3]); + assertEquals((byte) 20, data[defaultLen - 2]); + assertEquals((byte) 30, data[defaultLen - 1]); + // last byte inside internal buffer + } + + @Test + void utf8EncodingTest() { + PbjWriter writer = new PbjWriter(); + char arr[] = new char[6 << 10]; + for (int i = 0; i < 127; i++) { + arr[i] = 'a'; + } + + writer.writeStringWithTag(new String(arr, 0, 127)); + assertEquals(128, writer.position()); + + arr[50] = 'Ā'; + writer.reset(); + writer.writeStringWithTag(new String(arr, 0, 127)); + assertEquals(130, writer.position()); + + for (int i = 0; i < 127; i++) { + arr[i] = 'Ā'; + } + writer.reset(); + writer.writeStringWithTag(new String(arr, 0, 127)); + assertEquals(127 * 2 + 2, writer.position()); + + for (int i = 0; i < 5460; i++) { + arr[i] = 'b'; + } + writer.reset(); + writer.writeStringWithTag(new String(arr, 0, 5460)); + assertEquals(5460 + 2, writer.position()); + + for (int i = 0; i < 5460; i++) { + arr[i] = 'ā'; + } + + writer.reset(); + writer.writeStringWithTag(new String(arr, 0, 5460)); + assertEquals(5460 * 2 + 2, writer.position()); + + for (int i = 0; i < 6 << 10; i++) { + arr[i] = (char) (32 + i); + } + + writer.reset(); + String str = new String(arr); + writer.writeStringWithTag(str); + String res = writer.toPbjReader().readString(1 << 20); + assertEquals(str, res); + } + + @Test + void readerLimit() { + byte[] data = new byte[] {1, 2, 3, 4, 5, 6, 7, 8}; + PbjReader reader = new PbjReader(data); + + assertTrue(reader.hasRemaining()); + assertEquals(0, reader.position()); + assertEquals(data.length, reader.limit()); + reader.limit(0); + assertTrue(!reader.hasRemaining()); + assertEquals(0, reader.position()); + + reader.limit(4); + assertTrue(reader.hasRemaining()); + assertEquals(0x01020304, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + assertEquals(4, reader.position()); + assertEquals(0, reader.readIntBE()); + assertEquals(PbjReader.BufferUnderflow, reader.error()); + } + + @Test + void readerByteBufferConstructor() { + PbjReader reader = new PbjReader(ByteBuffer.wrap(new byte[] {1, 2, 3, 4})); + assertEquals(0x01020304, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + } + + @Test + void readerResetWithByteBuffer() { + PbjReader reader = new PbjReader(ByteBuffer.wrap(new byte[] {1, 2, 3, 4})); + assertEquals(0x01020304, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + reader.resetWith(ByteBuffer.wrap(new byte[] {5, 6, 7, 8})); + assertEquals(0, reader.position()); + assertEquals(0x05060708, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + assertEquals(0, reader.error()); + } + + @Test + void readerBufferBytesConstructor() { + PbjReader reader = new PbjReader(Bytes.wrap(new byte[] {0x0A, 0x0B, 0x0C, 0x0D})); + assertEquals(0x0A0B0C0D, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + } + + @Test + void readerBufferInputStreamConstructor() { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2, 3, 4})); + assertEquals(0x01020304, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + } + + @Test + void readerSkipAdvancesPosition() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4, 5}); + reader.skip(3); + assertEquals(3, reader.position()); + assertTrue(reader.hasRemaining()); + } + + @Test + void readerSkipBeyondDataSetsBufferUnderflow() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4}); + reader.skip(5); + assertEquals(PbjReader.BufferUnderflow, reader.error()); + } + + @Test + void readerResetAllowsReRead() { + byte[] data = {1, 2, 3, 4}; + PbjReader reader = new PbjReader(data); + assertEquals(0x01020304, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + reader.resetWith(data); + assertEquals(0, reader.position()); + assertTrue(reader.hasRemaining()); + assertEquals(0x01020304, reader.readIntBE()); + } + + @Test + void readerResetWithReplacesBuffer() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4}); + assertEquals(0x01020304, reader.readIntBE()); + reader.resetWith(Bytes.wrap(new byte[] {5, 6, 7, 8})); + assertEquals(0x05060708, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + } + + @Test + void writeLargeBytesBypassInternalBuffer() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + writer.writeByte((byte) 7); + byte[] large = new byte[4096]; + large[0] = 11; + large[4095] = 22; + writer.writeBytes(large); + + byte[] result = baos.toByteArray(); + assertEquals(4097, result.length); + assertEquals(7, result[0]); + assertEquals(11, result[1]); + assertEquals(22, result[4096]); + + byte[] internalArray = writer.internalArray(); + assertEquals(7, internalArray[0]); + assertEquals(0, internalArray[1]); + } + + @Test + void writeBytesRandomAccessDataZeroLengthWritesNothing() { + PbjWriter writer = new PbjWriter(); + writer.writeBytes(Bytes.wrap(new byte[0])); + assertEquals(0, writer.position()); + writer.writeBytes(Bytes.EMPTY); + assertEquals(0, writer.position()); + } + + @Test + void writeBytesBufferedDataZeroRemainingWritesNothing() { + PbjWriter writer = new PbjWriter(); + writer.writeBytes(BufferedData.wrap(new byte[0])); + assertEquals(0, writer.position()); + + BufferedData bd = BufferedData.wrap(new byte[] {1, 2, 3}); + bd.skip(3); + writer.writeBytes(bd); + assertEquals(0, writer.position()); + } + + @Test + void writeBytesArrayZeroAndNegativeLengthWritesNothing() { + byte[] src = new byte[] {1, 2, 3, 4}; + PbjWriter writer = new PbjWriter(); + writer.writeBytes(src, 0, 0); + assertEquals(0, writer.position()); + writer.writeBytes(src, 0, -2); + assertEquals(0, writer.position()); + } + + @Test + void quickTest() { + byte[] a = {2, 3}; + var by = Bytes.wrap(a); + var adap = by.toReadableSequentialData(); + var r = new PbjReader(adap); + var w = new PbjWriter(); + w.setError(4, ""); + int q = 0; + } + + @Test + void largeWriteBypassCorrectness() { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(out); + + byte[] fewBytes = {1, 2, 3, 4, 5}; + byte[] manyBytes = new byte[8_000]; + Arrays.fill(manyBytes, (byte) 0x7F); + + writer.writeBytes(fewBytes); + writer.writeBytes(manyBytes); + writer.flush(); + + byte[] expected = new byte[fewBytes.length + manyBytes.length]; + System.arraycopy(fewBytes, 0, expected, 0, fewBytes.length); + System.arraycopy(manyBytes, 0, expected, fewBytes.length, manyBytes.length); + + assertArrayEquals(expected, out.toByteArray(), + "Buffered prefix bytes must not be dropped when a large payload bypasses the buffer"); + } + + @Test + void writeBytesArrayFastPathBoundaryWithOutputStream() { + + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + byte[] src = new byte[2047]; + src[0] = 11; + src[2046] = 22; + writer.writeBytes(src, 0, 2047); + assertEquals(11, writer.internalArray()[0]); + assertEquals(0, baos.toByteArray().length); + writer.reset(); + baos.reset(); + int internalLen = writer.internalArray().length; + writer.skip(internalLen - 1); + writer.writeBytes(src, 0, 2047); + assertEquals(internalLen - 1, baos.toByteArray().length); + } + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + byte[] src = new byte[2048]; + src[0] = 11; + src[2047] = 22; + writer.writeBytes(src, 0, 2048); + writer.flush(); + assertEquals(0, writer.internalArray()[0]); // this length bypasses buffer + byte[] result = baos.toByteArray(); + assertEquals(2048, result.length); + assertEquals((byte) 11, result[0]); + assertEquals((byte) 22, result[2047]); + byte[] internalArray = writer.internalArray(); + assertEquals(0, internalArray[0]); + assertEquals(0, internalArray[2047]); + } + } + + @Test + void writeBytesBufferedDataFlushThrowsPropagated() { + OutputStream failing = new OutputStream() { + @Override + public void write(int b) {} + + @Override + public void write(byte[] b, int off, int len) throws IOException { + throw new IOException("Called write"); + } + }; + PbjWriter writer = new PbjWriter(failing); + byte[] chunk = new byte[1024]; // 2k is a fastpath + for (int i = 0; i < 16; i++) writer.writeBytes(chunk); + UncheckedIOException ex = + assertThrows(UncheckedIOException.class, () -> writer.writeBytes(BufferedData.wrap(new byte[] {1}))); + assertEquals("java.io.IOException: Called write", ex.getMessage()); + } + + @Test + void writeLargeByteArrayFlushThrowsPropagated() { + OutputStream failOnLarge = new OutputStream() { + @Override + public void write(int b) {} + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (len == 1) return; + if (len >= 2048) throw new IOException("2k write alt path"); + throw new RuntimeException("Called write"); + } + }; + PbjWriter writer = new PbjWriter(failOnLarge); + writer.writeBoolean(true); + UncheckedIOException ex = assertThrows(UncheckedIOException.class, () -> writer.writeBytes(new byte[2048])); + assertEquals("java.io.IOException: 2k write alt path", ex.getMessage()); + } + + @Test + void writeBytesRandomAccessDataFlushThrowsPropagated() { + OutputStream failing = new OutputStream() { + @Override + public void write(int b) {} + + @Override + public void write(byte[] b, int off, int len) throws IOException { + throw new IOException("writeBytesRAInternal write"); + } + }; + PbjWriter writer = new PbjWriter(failing); + byte[] chunk = new byte[1024]; + for (int i = 0; i < 16; i++) writer.writeBytes(chunk); // fill 16k internal buffer + UncheckedIOException ex = + assertThrows(UncheckedIOException.class, () -> writer.writeBytes(Bytes.wrap(new byte[] {1}))); + assertEquals("java.io.IOException: writeBytesRAInternal write", ex.getMessage()); + } + + @Test + void closeSwallowsOutputCloseException() { + OutputStream failOnClose = new OutputStream() { + @Override + public void write(int b) {} + + @Override + public void write(byte[] b, int off, int len) {} + + @Override + public void close() throws IOException { + throw new IOException("close failed"); + } + }; + PbjWriter writer = new PbjWriter(failOnClose); + assertDoesNotThrow(writer::close); + assertEquals(0, writer.error()); + } + + @Test + void flushOrGrowFlushThrowsPropagated() { + OutputStream failing = new OutputStream() { + @Override + public void write(int b) {} + + @Override + public void write(byte[] b, int off, int len) throws IOException { + throw new IOException("Called write"); + } + }; + PbjWriter writer = new PbjWriter(failing); + byte[] chunk = new byte[1024]; + for (int i = 0; i < 16; i++) writer.writeBytes(chunk); // fill 16k internal buffer + UncheckedIOException ex = assertThrows(UncheckedIOException.class, () -> writer.writeByte((byte) 1)); + assertEquals("java.io.IOException: Called write", ex.getMessage()); + } + + @Test + void flushPropagatesIOExceptionAsUnchecked() { + OutputStream failing = new OutputStream() { + @Override + public void write(int b) {} + + @Override + public void write(byte[] b, int off, int len) throws IOException { + throw new IOException("fake disk full"); + } + }; + PbjWriter writer = new PbjWriter(failing); + writer.writeByte((byte) 1); + UncheckedIOException ex = assertThrows(UncheckedIOException.class, writer::flush); + assertEquals("java.io.IOException: fake disk full", ex.getMessage()); + } + + @Test + void constructorWithWritableSequentialDataFlushesThrough() { + BufferedData bd = BufferedData.allocate(16); + PbjWriter writer = new PbjWriter(bd); + writer.writeByte2((byte) 10, (byte) 20); + writer.flush(); + assertEquals(10, bd.getByte(0)); + assertEquals(20, bd.getByte(1)); + assertEquals(2, bd.position()); + } + + @Test + void resetWithWritableSequentialDataSwitchesOutput() { + ByteArrayOutputStream out1 = new ByteArrayOutputStream(); + BufferedData bd = BufferedData.allocate(16); + PbjWriter writer = new PbjWriter(out1); + writer.writeByte((byte) 5); + writer.resetWith(bd); + writer.writeByte((byte) 7); + writer.flush(); + assertArrayEquals(new byte[] {5}, out1.toByteArray()); // flushed to out1 during reset + assertEquals(7, bd.getByte(0)); + assertEquals(1, bd.position()); + } + + @Test + void resetWithFlushesAndSwitchesOutput() { + ByteArrayOutputStream out1 = new ByteArrayOutputStream(); + ByteArrayOutputStream out2 = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(out1); + writer.writeByte((byte) 10); + assertEquals(0, out1.size()); + writer.resetWith(out2); // flushes before reset + assertArrayEquals(new byte[] {10}, out1.toByteArray()); + + assertEquals(0, writer.position()); + writer.writeByte((byte) 20); + writer.flush(); + assertArrayEquals(new byte[] {20}, out2.toByteArray()); + writer.writeByte((byte) 5); + writer.resetWithNull(); + assertArrayEquals(new byte[] {20, 5}, out2.toByteArray()); + writer.writeByte((byte) 7); + assertArrayEquals(new byte[] {7}, writer.toByteArray()); + assertArrayEquals(new byte[] {20, 5}, out2.toByteArray()); + } + + @Test + void resetWithOnNonReuseableWriterSetsError() { + byte[] buf = new byte[64]; + PbjWriter writer = new PbjWriter(buf, 0); + writer.resetWith(new ByteArrayOutputStream()); + assertEquals(PbjWriter.UsageError, writer.error()); + } + + @Test + void bufferMoreBeyondAbsoluteLimitSetsBufferUnderflow() { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2, 3, 4, 5})); + reader.limit(3); + assertEquals(1, reader.readByte()); + assertEquals(2, reader.readByte()); + assertEquals(3, reader.readByte()); + assertEquals(false, reader.hasRemaining()); + reader.readByte(); + assertEquals(PbjReader.BufferUnderflow, reader.error()); + } + + @Test + void limitPropagatedToUnderlyingReadableSequentialData() { + BufferedData bd = BufferedData.allocate(8); + for (byte b = 0; b < 8; b++) bd.writeByte(b); + bd.flip(); + PbjReader reader = new PbjReader(bd); + reader.limit(4); + reader.readByte(); + assertEquals(4, bd.position()); + assertEquals(1, reader.readByte()); + assertEquals(2, reader.readByte()); + assertEquals(3, reader.readByte()); + assertFalse(reader.hasRemaining()); + } + + @Test + void skipTriggersUnderflow() { + byte[] data = {1, 2, 3, 4, 5}; + PbjReader reader = new PbjReader(data); + assertEquals(1, reader.readByte()); + reader.skip(5); + assertEquals(PbjReader.BufferUnderflow, reader.error()); + } + + @Test + void skipInternalAccountsForBytesRemainingInBuffer() { + byte[] data = {1, 2, 3, 4, 5}; + PbjReader reader = new PbjReader(new ByteArrayInputStream(data)); + assertEquals(1, reader.readByte()); + reader.skip(5); + assertEquals(PbjReader.BufferUnderflow, reader.error()); + } + + @Test + void readVarLongWithMoreThanTenBytesSetsDataEncoding() { + byte[] malformed = new byte[16]; + for (int i = 0; i < 16; i++) malformed[i] = (byte) 0xFF; + PbjReader reader = new PbjReader(malformed); + reader.readVarLongNoZZ(); + assertEquals(PbjReader.DataEncoding, reader.error()); + reader.resetWith(malformed); + reader.readVarLongBytes(); + assertEquals(PbjReader.DataEncoding, reader.error()); + } + + @Test + void skipInternalSuccessfullySkipsFromInputStream() { + InputStream in = new ByteArrayInputStream(new byte[] {1, 2, 3, 4, 5}); + PbjReader reader = new PbjReader(in); + reader.skip(3); + assertEquals(3, reader.position()); + assertEquals(4, reader.readByte()); + } + + @Test + void skipInternalSuccessfullySkipsFromReadableSequentialData() { + BufferedData bd = BufferedData.allocate(8); + for (int i = 0; i < 5; i++) { + bd.writeByte((byte) i); + } + bd.flip(); + PbjReader reader = new PbjReader(bd); + reader.skip(3); + assertEquals(3, reader.position()); + assertEquals(3, reader.readByte()); + } + + @Test + void skipInternalSetsIOErrorWhenInputStreamSkipThrows() { + InputStream in = new InputStream() { + @Override + public int read() { + return -1; + } + + @Override + public long skip(long n) throws IOException { + throw new IOException("skip failed"); + } + }; + PbjReader reader = new PbjReader(in); + reader.skip(5); + assertEquals(PbjReader.IOError, reader.error()); + } + + @Test + void skipInternalCallsUnderlyingInputSkipForReadableSequentialData() { + BufferedData bd = BufferedData.allocate(8); + for (byte b = 0; b < 8; b++) bd.writeByte(b); + bd.flip(); + PbjReader reader = new PbjReader(bd); + reader.skip(3); + assertEquals(3, reader.position()); + assertEquals(3, bd.position()); + assertEquals(3, reader.readByte()); + } + + @Test + void readVarLongBytesReturnsWrappedBytesForValidVarint() { + PbjReader reader = new PbjReader(new byte[] {(byte) 0xAC, 0x02}); + Bytes result = reader.readVarLongBytes(); + assertEquals(2, result.length()); + assertEquals((byte) 0xAC, result.getByte(0)); + assertEquals((byte) 0x02, result.getByte(1)); + assertEquals(0, reader.error()); + } + + @Test + void readLargeStringCantBeBuffered() { + int len = (16 << 10) + 1; + String str = "a".repeat(len); + PbjWriter writer = new PbjWriter(); + writer.writeStringWithTag(str); + PbjReader reader = writer.toPbjReader(); + assertEquals(len, reader.readVarIntNoZZ()); + assertEquals(0, reader.error()); + } + + @Test + void readStringBufferedInternalSuccessPath() { + String str = "a".repeat(16384); + PbjWriter writer = new PbjWriter(); + writer.writeStringWithTag(str); + PbjReader reader = new PbjReader(new ByteArrayInputStream(writer.toByteArray())); + assertEquals(str, reader.readString(16384 + 10)); + assertEquals(0, reader.error()); + } + + @Test + void readStringBufferedInternalSuccessPathLarger() { + String str = "a".repeat(16385); + PbjWriter writer = new PbjWriter(); + writer.writeStringWithTag(str); + PbjReader reader = new PbjReader(new ByteArrayInputStream(writer.toByteArray())); + assertEquals(str, reader.readString(16385 + 10)); + assertEquals(0, reader.error()); + } + + @Test + void readFromInputSetsIOErrorWhenInputStreamReadThrows() { + InputStream failingOnSecondRead = new InputStream() { + private boolean doThrow = false; + + @Override + public int read() { + return 1; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (doThrow) throw new IOException("io error"); + doThrow = true; + b[off] = 25; + return 1; + } + }; + PbjReader reader = new PbjReader(failingOnSecondRead); + reader.readByte(); + assertEquals(PbjReader.IOError, reader.error()); + } + + @Test + void readStringLengthExceedsMaxSize() { + var reader = new PbjReader(new byte[] {5, 'h', 'e', 'l', 'l', 'o'}); + assertEquals("", reader.readString(2)); + assertEquals(PbjReader.Parse, reader.error()); + } + + @Test + void readStringNegativeLengthSetsParseError() { + var reader = new PbjReader(new byte[] {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, 0x0F}); + assertEquals("", reader.readString(Long.MAX_VALUE)); + assertEquals(PbjReader.Parse, reader.error()); + } + + @Test + void readStringInvalidUtf8SetParseError() { + var reader = new PbjReader(new byte[] {1, (byte) 0x80}); + assertEquals("", reader.readString(Long.MAX_VALUE)); + assertEquals(PbjReader.Parse, reader.error()); + } + + @Test + void readStringReturnsEmptyOnInsufficientData() { + var reader = new PbjReader(new byte[] {5, 'a', 'b'}); // length=5 but only 2 bytes follow + assertEquals("", reader.readString(Long.MAX_VALUE)); + assertEquals(PbjReader.BufferUnderflow, reader.error()); + } + + ///// Reset ///// + + private static final byte[] DATA = {1, 2, 3, 4, 5}; + + @Test + void resetWithByteArraySequentialData_readsCorrectBytes() throws ParseException { + ReadableSequentialData byteArraySeq = Bytes.wrap(DATA).toReadableSequentialData(); + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[0])); + + reader.resetWith(byteArraySeq); + + for (byte expected : DATA) { + assertEquals(expected, reader.readByte()); + } + reader.throwOnError(); + } + + @Test + void resetWithNonByteArraySequentialData_readsCorrectBytes() throws ParseException { + ReadableSequentialData buffered = BufferedData.wrap(DATA); + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[0])); + + reader.resetWith(buffered); + + for (byte expected : DATA) { + assertEquals(expected, reader.readByte()); + } + reader.throwOnError(); + } + + @Test + void resetWithInputStream_readsCorrectBytes() throws ParseException { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[0])); + + reader.resetWith(new ByteArrayInputStream(DATA)); + + for (byte expected : DATA) { + assertEquals(expected, reader.readByte()); + } + reader.throwOnError(); + } + + @Test + void resetWithNull_throwsOnError() { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[0])); + reader.resetWith((ReadableSequentialData) null); + assertThrows(ParseException.class, reader::throwOnError); + + PbjReader reader2 = new PbjReader(new ByteArrayInputStream(new byte[0])); + reader2.resetWith((ReadableSequentialData) null); + assertThrows(ParseException.class, reader2::throwOnError); + } + + @Test + void resetFromBytesToStream() { + var reader = new PbjReader(new byte[] {1}); + assertEquals((byte) 1, reader.readByte()); + reader.resetWith(new ByteArrayInputStream(new byte[] {7, 8})); + assertEquals((byte) 7, reader.readByte()); + assertEquals((byte) 8, reader.readByte()); + assertEquals(0, reader.error()); + } + + // + + @Test + void asInputStreamReturnsUnderlyingStreamIfNeverRead() { + var stream = new ByteArrayInputStream(new byte[] {1, 2, 3}); + var reader = new PbjReader(stream); + assertSame(stream, reader.asInputStream()); + } + + @Test + void asInputStreamDelegatesToInputIfNeverRead() throws Exception { + var innerStream = new ByteArrayInputStream(new byte[] {21}); + var reader = new PbjReader(new ReadableStreamingData(innerStream)); + var is = reader.asInputStream(); + assertNotNull(is); + assertEquals(21, is.read()); + } + + @Test + void asInputStreamForByteArrayReader() throws Exception { + var reader = new PbjReader(new byte[] {1, 2, 3}); + var is = reader.asInputStream(); + assertEquals(1, is.read()); + assertEquals(2, is.read()); + assertEquals(3, is.read()); + } + + @Test + void readIntLESlowPathSuccess() { + var reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2, 3, 4})); + assertEquals(0x04030201, reader.readIntLE()); + assertEquals(0, reader.error()); + } + + @Test + void readLongLESlowPathSuccess() { + var reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2, 3, 4, 5, 6, 7, 8})); + assertEquals(0x0807060504030201L, reader.readLongLE()); + assertEquals(0, reader.error()); + } + + @Test + void readIntLEUnderflow() { + var reader = new PbjReader(new byte[] {1, 2, 3}); // only 3 bytes, need 4 + assertEquals(0, reader.readIntLE()); + assertEquals(PbjReader.BufferUnderflow, reader.error()); + } + + @Test + void readLongLEUnderflow() { + var reader = new PbjReader(new byte[] {1, 2, 3, 4, 5, 6, 7}); // only 7 bytes, need 8 + assertEquals(0L, reader.readLongLE()); + assertEquals(PbjReader.BufferUnderflow, reader.error()); + } + + ///// readBytes ///// + + @Test + void readBytesArrayOffsetLen_writesIntoCorrectSlice() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4, 5}); + byte[] dst = new byte[7]; + long n = reader.readBytes(dst, 2, 3); + assertEquals(3, n); + assertArrayEquals(new byte[] {0, 0, 1, 2, 3, 0, 0}, dst); + } + + @Test + void readBytesIntoByteBuffer_dataReadAndPositionAdvances() { + PbjReader reader = new PbjReader(new byte[] {10, 20, 30}); + ByteBuffer bb = ByteBuffer.allocate(5); + long n = reader.readBytes(bb); + assertEquals(3, n); + assertEquals(3, bb.position()); + assertArrayEquals(new byte[] {10, 20, 30, 0, 0}, bb.array()); + } + + @Test + void readBytesIntoByteBuffer_positionUnchangedWhenError() { + PbjReader reader = new PbjReader(new byte[] {1, 2}); + reader.skip(10); + assertTrue(reader.error() > 0); + ByteBuffer bb = ByteBuffer.allocate(5); + long n = reader.readBytes(bb); + assertEquals(-1, n); + assertEquals(0, bb.position()); + } + + @Test + void readBytesInt_fastPath_bytesBuffered() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4, 5}); + Bytes result = reader.readBytes(3); + assertEquals(3, result.length()); + assertArrayEquals(new byte[] {1, 2, 3}, result.toByteArray()); + assertEquals(0, reader.error()); + } + + @Test + void readBytesInt_slowPath_triggersReadBytesInternal() { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[] {7, 8, 9})); + Bytes result = reader.readBytes(3); + assertEquals(3, result.length()); + assertArrayEquals(new byte[] {7, 8, 9}, result.toByteArray()); + assertEquals(0, reader.error()); + } + + @Test + void readBytesInternal_zeroLength_returnsEmpty() { + PbjReader reader = new PbjReader(new byte[] {1, 2}); + reader.skip(10); + assertEquals(PbjReader.BufferUnderflow, reader.error()); + assertSame(Bytes.EMPTY, reader.readBytes(2)); + } + + @Test + void readBytesInternal_notEnoughData_setsBufferUnderflow() { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2})); + Bytes result = reader.readBytes(5); + assertSame(Bytes.EMPTY, result); + assertEquals(PbjReader.BufferUnderflow, reader.error()); + } + + @Test + void readLongBEInternal_streamingReaderSucceeds() { + byte[] bytes = {0, 0, 0, 0, 0, 0, 0, 7}; + PbjReader reader = new PbjReader(new ByteArrayInputStream(bytes)); + assertEquals(7L, reader.readLongBE()); + assertEquals(0, reader.error()); + } + + @Test + void readLongBEInternal_notEnoughData_setsBufferUnderflow() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3}); + reader.readLongBE(); + assertEquals(PbjReader.BufferUnderflow, reader.error()); + } + + @Test + void readBytesNegativeLengthSetsIllegalArgument() { + var reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2, 3, 4, 5})); + reader.readByte(); + reader.readByte(); + reader.limit(0); + var result = reader.readBytes(-1); + assertEquals(Bytes.EMPTY, result); + assertEquals(PbjReader.IllegalArgument, reader.error()); + } +}