diff --git a/server/docker/docker-compose.yml b/server/docker/docker-compose.yml index d182b4e3b..f3a945597 100644 --- a/server/docker/docker-compose.yml +++ b/server/docker/docker-compose.yml @@ -8,6 +8,7 @@ services: - "8080:8080" - "5005:5005" - "9999:9999" + - "8849:8849" cadvisor: image: "gcr.io/cadvisor/cadvisor:v0.47.0" diff --git a/server/docker/logging.properties b/server/docker/logging.properties index 1ed70c240..60bb91bd6 100644 --- a/server/docker/logging.properties +++ b/server/docker/logging.properties @@ -21,11 +21,15 @@ io.helidon.common.level=INFO #com.hedera.block.level=FINE #com.hedera.block.server.level=FINE -# Configure specific loggers +# Configure specific Block Node loggers #com.hedera.block.server.producer.ProducerBlockItemObserver.level=FINE #com.hedera.block.server.mediator.LiveStreamMediatorImpl.level=FINE #com.hedera.block.server.persistence.storage.write.BlockAsDirWriter.level=FINE #com.hedera.block.server.consumer.ConsumerStreamResponseObserver.level=FINE +#com.hedera.block.server.pbj.PbjBlockStreamServiceProxy.level=FINE + +# Helidon PBJ Plugin loggers +#com.hedera.pbj.grpc.helidon.PbjProtocolHandler.level=FINE # Console handler configuration handlers = java.util.logging.ConsoleHandler diff --git a/server/docker/update-env.sh b/server/docker/update-env.sh index 9594fcf8d..29a593980 100755 --- a/server/docker/update-env.sh +++ b/server/docker/update-env.sh @@ -21,8 +21,9 @@ echo "REGISTRY_PREFIX=" >> .env echo "BLOCKNODE_STORAGE_ROOT_PATH=/app/storage" >> .env if [ true = "$is_debug" ]; then - # wait for debugger to attach - echo "SERVER_OPTS='-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'" >> .env + # The server will wait for the debugger to attach on port 5005 + # JProfiler can attach on port 8849 + echo "SERVER_OPTS='-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005' -agentpath:/path/to/libjprofilerti.so=port=8849 " >> .env fi if [ true = "$is_smoke_test" ]; then diff --git a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java index f0c388694..b9a4db929 100644 --- a/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java +++ b/server/src/main/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserver.java @@ -27,8 +27,8 @@ import com.hedera.block.server.mediator.SubscriptionHandler; import com.hedera.block.server.metrics.BlockNodeMetricTypes; import com.hedera.block.server.metrics.MetricsService; -import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.hedera.pbj.runtime.OneOf; import edu.umd.cs.findbugs.annotations.NonNull; import java.time.InstantSource; @@ -43,13 +43,14 @@ * by Helidon). The ConsumerBlockItemObserver implements the BlockNodeEventHandler interface so the * Disruptor can invoke the onEvent() method when a new SubscribeStreamResponse is available. */ -public class ConsumerStreamResponseObserver implements BlockNodeEventHandler> { +public class ConsumerStreamResponseObserver + implements BlockNodeEventHandler> { private final Logger LOGGER = System.getLogger(getClass().getName()); private final MetricsService metricsService; - private final Flow.Subscriber subscribeStreamResponseObserver; - private final SubscriptionHandler subscriptionHandler; + private final Flow.Subscriber subscribeStreamResponseObserver; + private final SubscriptionHandler subscriptionHandler; private final AtomicBoolean isResponsePermitted = new AtomicBoolean(true); private final ResponseSender statusResponseSender = new StatusResponseSender(); @@ -74,8 +75,8 @@ public class ConsumerStreamResponseObserver implements BlockNodeEventHandler subscriptionHandler, - @NonNull final Flow.Subscriber subscribeStreamResponseObserver, + @NonNull final SubscriptionHandler subscriptionHandler, + @NonNull final Flow.Subscriber subscribeStreamResponseObserver, @NonNull final BlockNodeContext blockNodeContext) { this.livenessCalculator = new LivenessCalculator( @@ -102,7 +103,8 @@ public ConsumerStreamResponseObserver( * @param b true if the event is the last in the sequence */ @Override - public void onEvent(@NonNull final ObjectEvent event, final long l, final boolean b) { + public void onEvent( + @NonNull final ObjectEvent event, final long l, final boolean b) { // Only send the response if the consumer has not cancelled // or closed the stream. @@ -114,7 +116,7 @@ public void onEvent(@NonNull final ObjectEvent event, f // Refresh the producer liveness and pass the BlockItem to the downstream observer. livenessCalculator.refresh(); - final SubscribeStreamResponse subscribeStreamResponse = event.get(); + final SubscribeStreamResponseUnparsed subscribeStreamResponse = event.get(); final ResponseSender responseSender = getResponseSender(subscribeStreamResponse); responseSender.send(subscribeStreamResponse); } @@ -127,9 +129,10 @@ public boolean isTimeoutExpired() { } @NonNull - private ResponseSender getResponseSender(@NonNull final SubscribeStreamResponse subscribeStreamResponse) { + private ResponseSender getResponseSender(@NonNull final SubscribeStreamResponseUnparsed subscribeStreamResponse) { - final OneOf responseType = subscribeStreamResponse.response(); + final OneOf responseType = + subscribeStreamResponse.response(); return switch (responseType.kind()) { case STATUS -> { isResponsePermitted.set(false); @@ -142,13 +145,13 @@ private ResponseSender getResponseSender(@NonNull final SubscribeStreamResponse } private interface ResponseSender { - void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse); + void send(@NonNull final SubscribeStreamResponseUnparsed subscribeStreamResponse); } private final class BlockItemsResponseSender implements ResponseSender { private boolean streamStarted = false; - public void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse) { + public void send(@NonNull final SubscribeStreamResponseUnparsed subscribeStreamResponse) { if (subscribeStreamResponse.blockItems() == null) { final String message = PROTOCOL_VIOLATION_MESSAGE.formatted( @@ -157,21 +160,19 @@ public void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse) throw new IllegalArgumentException(message); } - final List blockItems = + final List blockItems = Objects.requireNonNull(subscribeStreamResponse.blockItems()).blockItems(); + // Only start sending BlockItems after we've reached // the beginning of a block. - if (!streamStarted && blockItems.getFirst().hasBlockHeader()) { - LOGGER.log( - DEBUG, - "Sending BlockItem Batch downstream for block: " - + blockItems.getFirst().blockHeader().number()); + final BlockItemUnparsed firstBlockItem = blockItems.getFirst(); + if (!streamStarted && firstBlockItem.hasBlockHeader()) { streamStarted = true; } if (streamStarted) { metricsService - .get(BlockNodeMetricTypes.Counter.LiveBlockItemsReceived) + .get(BlockNodeMetricTypes.Counter.LiveBlockItemsConsumed) .add(blockItems.size()); subscribeStreamResponseObserver.onNext(subscribeStreamResponse); } @@ -181,7 +182,7 @@ public void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse) // TODO: Implement another StatusResponseSender that will unsubscribe the observer once the // status code is fixed. private final class StatusResponseSender implements ResponseSender { - public void send(@NonNull final SubscribeStreamResponse subscribeStreamResponse) { + public void send(@NonNull final SubscribeStreamResponseUnparsed subscribeStreamResponse) { LOGGER.log(DEBUG, "Sending SubscribeStreamResponse downstream: " + subscribeStreamResponse); subscribeStreamResponseObserver.onNext(subscribeStreamResponse); subscribeStreamResponseObserver.onComplete(); diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediator.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediator.java index a738a58a7..25d2d354e 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediator.java +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediator.java @@ -17,8 +17,8 @@ package com.hedera.block.server.mediator; import com.hedera.block.server.notifier.Notifiable; -import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import java.util.List; /** @@ -26,4 +26,4 @@ * Hedera network with the contract to be notified of critical system events. */ public interface LiveStreamMediator - extends StreamMediator, SubscribeStreamResponse>, Notifiable {} + extends StreamMediator, SubscribeStreamResponseUnparsed>, Notifiable {} diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorBuilder.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorBuilder.java index 2816f1a77..a547f2b60 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorBuilder.java +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorBuilder.java @@ -21,7 +21,7 @@ import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; -import com.hedera.hapi.block.SubscribeStreamResponse; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.lmax.disruptor.BatchEventProcessor; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.Map; @@ -41,16 +41,15 @@ public class LiveStreamMediatorBuilder { private final ServiceStatus serviceStatus; private Map< - BlockNodeEventHandler>, - BatchEventProcessor>> + BlockNodeEventHandler>, + BatchEventProcessor>> subscribers; /** The initial capacity of the subscriber map. */ private static final int SUBSCRIBER_INIT_CAPACITY = 32; private LiveStreamMediatorBuilder( - @NonNull final BlockNodeContext blockNodeContext, - @NonNull final ServiceStatus serviceStatus) { + @NonNull final BlockNodeContext blockNodeContext, @NonNull final ServiceStatus serviceStatus) { this.subscribers = new ConcurrentHashMap<>(SUBSCRIBER_INIT_CAPACITY); this.blockNodeContext = blockNodeContext; this.serviceStatus = serviceStatus; @@ -67,8 +66,7 @@ private LiveStreamMediatorBuilder( */ @NonNull public static LiveStreamMediatorBuilder newBuilder( - @NonNull final BlockNodeContext blockNodeContext, - @NonNull final ServiceStatus serviceStatus) { + @NonNull final BlockNodeContext blockNodeContext, @NonNull final ServiceStatus serviceStatus) { return new LiveStreamMediatorBuilder(blockNodeContext, serviceStatus); } @@ -85,8 +83,8 @@ public static LiveStreamMediatorBuilder newBuilder( public LiveStreamMediatorBuilder subscribers( @NonNull final Map< - BlockNodeEventHandler>, - BatchEventProcessor>> + BlockNodeEventHandler>, + BatchEventProcessor>> subscribers) { this.subscribers = subscribers; return this; diff --git a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java index a9e6551e9..a7c30304d 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java +++ b/server/src/main/java/com/hedera/block/server/mediator/LiveStreamMediatorImpl.java @@ -29,10 +29,10 @@ import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.service.ServiceStatus; -import com.hedera.hapi.block.BlockItemSet; -import com.hedera.hapi.block.SubscribeStreamResponse; +import com.hedera.hapi.block.BlockItemSetUnparsed; +import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.SubscribeStreamResponseCode; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.lmax.disruptor.BatchEventProcessor; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.List; @@ -46,7 +46,8 @@ * subscribers as they arrive via a RingBuffer maintained in the base class and persists the block * items to a store. */ -class LiveStreamMediatorImpl extends SubscriptionHandlerBase implements LiveStreamMediator { +class LiveStreamMediatorImpl extends SubscriptionHandlerBase + implements LiveStreamMediator { private final Logger LOGGER = System.getLogger(getClass().getName()); @@ -69,8 +70,8 @@ class LiveStreamMediatorImpl extends SubscriptionHandlerBase>, - BatchEventProcessor>> + BlockNodeEventHandler>, + BatchEventProcessor>> subscribers, @NonNull final ServiceStatus serviceStatus, @NonNull final BlockNodeContext blockNodeContext) { @@ -96,17 +97,19 @@ class LiveStreamMediatorImpl extends SubscriptionHandlerBase blockItems) { + public void publish(@NonNull final List blockItems) { if (serviceStatus.isRunning()) { // Publish the block for all subscribers to receive - LOGGER.log(DEBUG, "Publishing BlockItem"); - final BlockItemSet blockItemsSet = - BlockItemSet.newBuilder().blockItems(blockItems).build(); - final var subscribeStreamResponse = SubscribeStreamResponse.newBuilder() + final BlockItemSetUnparsed blockItemsSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItems).build(); + + final var subscribeStreamResponse = SubscribeStreamResponseUnparsed.newBuilder() .blockItems(blockItemsSet) .build(); + + LOGGER.log(DEBUG, "Publishing BlockItems: " + blockItems.size()); ringBuffer.publishEvent((event, sequence) -> event.set(subscribeStreamResponse)); long remainingCapacity = ringBuffer.remainingCapacity(); @@ -114,7 +117,6 @@ public void publish(@NonNull final List blockItems) { // Increment the block item counter by all block items published metricsService.get(LiveBlockItems).add(blockItems.size()); - } else { LOGGER.log(ERROR, "StreamMediator is not accepting BlockItems"); } @@ -133,16 +135,16 @@ public void notifyUnrecoverableError() { LOGGER.log(ERROR, "Sending an error response to end the stream for all consumers."); // Publish an end of stream response to all downstream consumers - final SubscribeStreamResponse endStreamResponse = buildEndStreamResponse(); + final SubscribeStreamResponseUnparsed endStreamResponse = buildEndStreamResponse(); ringBuffer.publishEvent((event, sequence) -> event.set(endStreamResponse)); } @NonNull - private static SubscribeStreamResponse buildEndStreamResponse() { + private static SubscribeStreamResponseUnparsed buildEndStreamResponse() { // The current spec does not contain a generic error code for // SubscribeStreamResponseCode. // TODO: Replace READ_STREAM_SUCCESS (2) with a generic error code? - return SubscribeStreamResponse.newBuilder() + return SubscribeStreamResponseUnparsed.newBuilder() .status(SubscribeStreamResponseCode.READ_STREAM_SUCCESS) .build(); } diff --git a/server/src/main/java/com/hedera/block/server/mediator/MediatorInjectionModule.java b/server/src/main/java/com/hedera/block/server/mediator/MediatorInjectionModule.java index 95644ce5e..fc4be38af 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/MediatorInjectionModule.java +++ b/server/src/main/java/com/hedera/block/server/mediator/MediatorInjectionModule.java @@ -19,7 +19,7 @@ import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.notifier.Notifiable; import com.hedera.block.server.service.ServiceStatus; -import com.hedera.hapi.block.SubscribeStreamResponse; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import dagger.Binds; import dagger.Module; import dagger.Provides; @@ -61,7 +61,7 @@ static LiveStreamMediator providesLiveStreamMediator( */ @Binds @Singleton - SubscriptionHandler bindSubscriptionHandler( + SubscriptionHandler bindSubscriptionHandler( @NonNull final LiveStreamMediator liveStreamMediator); /** diff --git a/server/src/main/java/com/hedera/block/server/mediator/NoOpLiveStreamMediator.java b/server/src/main/java/com/hedera/block/server/mediator/NoOpLiveStreamMediator.java index 2cf443d59..5a2ea6573 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/NoOpLiveStreamMediator.java +++ b/server/src/main/java/com/hedera/block/server/mediator/NoOpLiveStreamMediator.java @@ -23,8 +23,8 @@ import com.hedera.block.server.events.BlockNodeEventHandler; import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.metrics.MetricsService; -import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import edu.umd.cs.findbugs.annotations.NonNull; import java.util.List; @@ -51,7 +51,7 @@ public NoOpLiveStreamMediator(@NonNull final BlockNodeContext blockNodeContext) * {@inheritDoc} */ @Override - public void publish(@NonNull List blockItems) { + public void publish(@NonNull List blockItems) { metricsService.get(LiveBlockItems).add(blockItems.size()); } @@ -59,19 +59,19 @@ public void publish(@NonNull List blockItems) { * {@inheritDoc} */ @Override - public void subscribe(@NonNull BlockNodeEventHandler> handler) {} + public void subscribe(@NonNull BlockNodeEventHandler> handler) {} /** * {@inheritDoc} */ @Override - public void unsubscribe(@NonNull BlockNodeEventHandler> handler) {} + public void unsubscribe(@NonNull BlockNodeEventHandler> handler) {} /** * {@inheritDoc} */ @Override - public boolean isSubscribed(@NonNull BlockNodeEventHandler> handler) { + public boolean isSubscribed(@NonNull BlockNodeEventHandler> handler) { return false; } diff --git a/server/src/main/java/com/hedera/block/server/mediator/SubscriptionHandlerBase.java b/server/src/main/java/com/hedera/block/server/mediator/SubscriptionHandlerBase.java index bd058f28c..6cf2330b0 100644 --- a/server/src/main/java/com/hedera/block/server/mediator/SubscriptionHandlerBase.java +++ b/server/src/main/java/com/hedera/block/server/mediator/SubscriptionHandlerBase.java @@ -39,8 +39,7 @@ */ public abstract class SubscriptionHandlerBase implements SubscriptionHandler { - private final Map>, BatchEventProcessor>> - subscribers; + private final Map>, BatchEventProcessor>> subscribers; /** The ring buffer to publish events to the subscribers. */ protected final RingBuffer> ringBuffer; @@ -60,11 +59,7 @@ public abstract class SubscriptionHandlerBase implements SubscriptionHandler< * @param ringBufferSize the size of the ring buffer */ protected SubscriptionHandlerBase( - @NonNull - final Map< - BlockNodeEventHandler>, - BatchEventProcessor>> - subscribers, + @NonNull final Map>, BatchEventProcessor>> subscribers, @NonNull final LongGauge subscriptionGauge, final int ringBufferSize) { @@ -89,8 +84,7 @@ public void subscribe(@NonNull final BlockNodeEventHandler> handl if (!subscribers.containsKey(handler)) { // Initialize the batch event processor and set it on the ring buffer final var batchEventProcessor = - new BatchEventProcessorBuilder() - .build(ringBuffer, ringBuffer.newBarrier(), handler); + new BatchEventProcessorBuilder().build(ringBuffer, ringBuffer.newBarrier(), handler); ringBuffer.addGatingSequences(batchEventProcessor.getSequence()); executor.execute(batchEventProcessor); diff --git a/server/src/main/java/com/hedera/block/server/notifier/Notifier.java b/server/src/main/java/com/hedera/block/server/notifier/Notifier.java index 137ae3679..3e81e0869 100644 --- a/server/src/main/java/com/hedera/block/server/notifier/Notifier.java +++ b/server/src/main/java/com/hedera/block/server/notifier/Notifier.java @@ -17,13 +17,12 @@ package com.hedera.block.server.notifier; import com.hedera.block.server.mediator.StreamMediator; +import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.PublishStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; import java.util.List; /** * Use this interface to combine the contract for streaming block items with the contract to be * notified of critical system events. */ -public interface Notifier - extends StreamMediator, PublishStreamResponse>, Notifiable {} +public interface Notifier extends StreamMediator, PublishStreamResponse>, Notifiable {} diff --git a/server/src/main/java/com/hedera/block/server/notifier/NotifierImpl.java b/server/src/main/java/com/hedera/block/server/notifier/NotifierImpl.java index 3537913e3..98f494967 100644 --- a/server/src/main/java/com/hedera/block/server/notifier/NotifierImpl.java +++ b/server/src/main/java/com/hedera/block/server/notifier/NotifierImpl.java @@ -27,11 +27,11 @@ import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.service.ServiceStatus; import com.hedera.hapi.block.Acknowledgement; +import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.EndOfStream; import com.hedera.hapi.block.ItemAcknowledgement; import com.hedera.hapi.block.PublishStreamResponse; import com.hedera.hapi.block.PublishStreamResponseCode; -import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; import java.security.NoSuchAlgorithmException; @@ -109,7 +109,7 @@ public void notifyUnrecoverableError() { * upstream producers */ @Override - public void publish(@NonNull List blockItems) { + public void publish(@NonNull List blockItems) { try { if (serviceStatus.isRunning()) { @@ -155,7 +155,7 @@ static PublishStreamResponse buildErrorStreamResponse() { * @throws NoSuchAlgorithmException if the hash algorithm is not supported */ @NonNull - Acknowledgement buildAck(@NonNull final List blockItems) throws NoSuchAlgorithmException { + Acknowledgement buildAck(@NonNull final List blockItems) throws NoSuchAlgorithmException { final ItemAcknowledgement itemAck = ItemAcknowledgement.newBuilder() // TODO: Replace this with a real hash generator .itemsHash(Bytes.wrap(getFakeHash(blockItems))) diff --git a/server/src/main/java/com/hedera/block/server/pbj/PbjBlockAccessServiceProxy.java b/server/src/main/java/com/hedera/block/server/pbj/PbjBlockAccessServiceProxy.java index 2f9aecb00..64b2bfee5 100644 --- a/server/src/main/java/com/hedera/block/server/pbj/PbjBlockAccessServiceProxy.java +++ b/server/src/main/java/com/hedera/block/server/pbj/PbjBlockAccessServiceProxy.java @@ -25,10 +25,10 @@ import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.service.ServiceStatus; +import com.hedera.hapi.block.BlockUnparsed; import com.hedera.hapi.block.SingleBlockRequest; -import com.hedera.hapi.block.SingleBlockResponse; import com.hedera.hapi.block.SingleBlockResponseCode; -import com.hedera.hapi.block.stream.Block; +import com.hedera.hapi.block.SingleBlockResponseUnparsed; import com.hedera.pbj.runtime.ParseException; import com.hedera.pbj.runtime.grpc.Pipeline; import com.hedera.pbj.runtime.grpc.Pipelines; @@ -50,7 +50,7 @@ public class PbjBlockAccessServiceProxy implements PbjBlockAccessService { private final System.Logger LOGGER = System.getLogger(getClass().getName()); private final ServiceStatus serviceStatus; - private final BlockReader blockReader; + private final BlockReader blockReader; private final MetricsService metricsService; /** @@ -63,7 +63,7 @@ public class PbjBlockAccessServiceProxy implements PbjBlockAccessService { @Inject public PbjBlockAccessServiceProxy( @NonNull final ServiceStatus serviceStatus, - @NonNull final BlockReader blockReader, + @NonNull final BlockReader blockReader, @NonNull final BlockNodeContext blockNodeContext) { this.serviceStatus = serviceStatus; this.blockReader = blockReader; @@ -83,8 +83,8 @@ public Pipeline open( try { final var m = (BlockAccessMethod) method; return switch (m) { - case singleBlock -> Pipelines.unary() - .mapRequest(bytes -> parseSingleBlockRequest(bytes, options)) + case singleBlock -> Pipelines.unary() + .mapRequest(bytes -> parseSingleBlockRequest(bytes)) .method(this::singleBlock) .mapResponse(reply -> createSingleBlockResponse(reply, options)) .respondTo(replies) @@ -96,19 +96,19 @@ public Pipeline open( } } - SingleBlockResponse singleBlock(SingleBlockRequest singleBlockRequest) { + SingleBlockResponseUnparsed singleBlock(SingleBlockRequest singleBlockRequest) { LOGGER.log(DEBUG, "Executing Unary singleBlock gRPC method"); if (serviceStatus.isRunning()) { final long blockNumber = singleBlockRequest.blockNumber(); try { - final Optional blockOpt = blockReader.read(blockNumber); + final Optional blockOpt = blockReader.read(blockNumber); if (blockOpt.isPresent()) { LOGGER.log(DEBUG, "Successfully returning block number: {0}", blockNumber); metricsService.get(SingleBlocksRetrieved).increment(); - return SingleBlockResponse.newBuilder() + return SingleBlockResponseUnparsed.newBuilder() .status(SingleBlockResponseCode.READ_BLOCK_SUCCESS) .block(blockOpt.get()) .build(); @@ -116,42 +116,40 @@ SingleBlockResponse singleBlock(SingleBlockRequest singleBlockRequest) { LOGGER.log(DEBUG, "Block number {0} not found", blockNumber); metricsService.get(SingleBlocksNotFound).increment(); - return SingleBlockResponse.newBuilder() + return SingleBlockResponseUnparsed.newBuilder() .status(SingleBlockResponseCode.READ_BLOCK_NOT_FOUND) .build(); } } catch (IOException e) { LOGGER.log(ERROR, "Error reading block number: {0}", blockNumber); - return SingleBlockResponse.newBuilder() + return SingleBlockResponseUnparsed.newBuilder() .status(SingleBlockResponseCode.READ_BLOCK_NOT_AVAILABLE) .build(); } catch (ParseException e) { LOGGER.log(ERROR, "Error parsing block number: {0}", blockNumber); - return SingleBlockResponse.newBuilder() + return SingleBlockResponseUnparsed.newBuilder() .status(SingleBlockResponseCode.READ_BLOCK_NOT_AVAILABLE) .build(); } } else { LOGGER.log(ERROR, "Unary singleBlock gRPC method is not currently running"); - return SingleBlockResponse.newBuilder() + return SingleBlockResponseUnparsed.newBuilder() .status(SingleBlockResponseCode.READ_BLOCK_NOT_AVAILABLE) .build(); } } @NonNull - private SingleBlockRequest parseSingleBlockRequest( - @NonNull final Bytes message, @NonNull final RequestOptions options) throws ParseException { - // TODO: Copying bytes to avoid using references passed from Helidon. Investigate if this is necessary. - return SingleBlockRequest.PROTOBUF.parse(Bytes.wrap(message.toByteArray())); + private SingleBlockRequest parseSingleBlockRequest(@NonNull final Bytes message) throws ParseException { + return SingleBlockRequest.PROTOBUF.parse(message); } @NonNull private Bytes createSingleBlockResponse( - @NonNull final SingleBlockResponse reply, @NonNull final RequestOptions options) { - return SingleBlockResponse.PROTOBUF.toBytes(reply); + @NonNull final SingleBlockResponseUnparsed reply, @NonNull final RequestOptions options) { + return SingleBlockResponseUnparsed.PROTOBUF.toBytes(reply); } } diff --git a/server/src/main/java/com/hedera/block/server/pbj/PbjBlockStreamServiceProxy.java b/server/src/main/java/com/hedera/block/server/pbj/PbjBlockStreamServiceProxy.java index ddae0f5eb..353e36270 100644 --- a/server/src/main/java/com/hedera/block/server/pbj/PbjBlockStreamServiceProxy.java +++ b/server/src/main/java/com/hedera/block/server/pbj/PbjBlockStreamServiceProxy.java @@ -29,17 +29,19 @@ import com.hedera.block.server.producer.ProducerBlockItemObserver; import com.hedera.block.server.producer.ProducerConfig; import com.hedera.block.server.service.ServiceStatus; -import com.hedera.hapi.block.PublishStreamRequest; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.PublishStreamRequestUnparsed; import com.hedera.hapi.block.PublishStreamResponse; import com.hedera.hapi.block.SubscribeStreamRequest; -import com.hedera.hapi.block.SubscribeStreamResponse; import com.hedera.hapi.block.SubscribeStreamResponseCode; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.hedera.pbj.runtime.ParseException; import com.hedera.pbj.runtime.grpc.Pipeline; import com.hedera.pbj.runtime.grpc.Pipelines; import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; import java.time.Clock; +import java.util.List; import java.util.concurrent.Flow; import javax.inject.Inject; @@ -71,7 +73,7 @@ public class PbjBlockStreamServiceProxy implements PbjBlockStreamService { public PbjBlockStreamServiceProxy( @NonNull final LiveStreamMediator streamMediator, @NonNull final ServiceStatus serviceStatus, - @NonNull final BlockNodeEventHandler> streamPersistenceHandler, + @NonNull final BlockNodeEventHandler> streamPersistenceHandler, @NonNull final Notifier notifier, @NonNull final BlockNodeContext blockNodeContext) { this.serviceStatus = serviceStatus; @@ -97,7 +99,7 @@ public Pipeline open( return switch (m) { case publishBlockStream -> { notifier.unsubscribeAllExpired(); - yield Pipelines.bidiStreaming() + yield Pipelines., PublishStreamResponse>bidiStreaming() .mapRequest(bytes -> parsePublishStreamRequest(bytes, options)) .method(this::publishBlockStream) .mapResponse(bytes -> createPublishStreamResponse(bytes, options)) @@ -105,7 +107,7 @@ public Pipeline open( .build(); } case subscribeBlockStream -> Pipelines - .serverStreaming() + .serverStreaming() .mapRequest(bytes -> parseSubscribeStreamRequest(bytes, options)) .method(this::subscribeBlockStream) .mapResponse(reply -> createSubscribeStreamResponse(reply, options)) @@ -118,7 +120,7 @@ public Pipeline open( } } - Flow.Subscriber publishBlockStream( + Flow.Subscriber> publishBlockStream( Flow.Subscriber helidonProducerObserver) { LOGGER.log(DEBUG, "Executing bidirectional publishBlockStream gRPC method"); @@ -154,7 +156,7 @@ Flow.Subscriber publishBlockStream( void subscribeBlockStream( SubscribeStreamRequest subscribeStreamRequest, - Flow.Subscriber subscribeStreamResponseObserver) { + Flow.Subscriber subscribeStreamResponseObserver) { LOGGER.log(DEBUG, "Executing Server Streaming subscribeBlockStream gRPC method"); @@ -169,7 +171,7 @@ void subscribeBlockStream( } else { LOGGER.log(ERROR, "Server Streaming subscribeBlockStream gRPC Service is not currently running"); - subscribeStreamResponseObserver.onNext(SubscribeStreamResponse.newBuilder() + subscribeStreamResponseObserver.onNext(SubscribeStreamResponseUnparsed.newBuilder() .status(SubscribeStreamResponseCode.READ_STREAM_SUCCESS) .build()); } @@ -178,21 +180,21 @@ void subscribeBlockStream( @NonNull private SubscribeStreamRequest parseSubscribeStreamRequest( @NonNull final Bytes message, @NonNull final RequestOptions options) throws ParseException { - // TODO: Copying bytes to avoid using references passed from Helidon. Investigate if this is necessary. - return SubscribeStreamRequest.PROTOBUF.parse(Bytes.wrap(message.toByteArray())); + return SubscribeStreamRequest.PROTOBUF.parse(message); } @NonNull private Bytes createSubscribeStreamResponse( - @NonNull final SubscribeStreamResponse subscribeStreamResponse, @NonNull final RequestOptions options) { - return SubscribeStreamResponse.PROTOBUF.toBytes(subscribeStreamResponse); + @NonNull final SubscribeStreamResponseUnparsed subscribeStreamResponse, + @NonNull final RequestOptions options) { + return SubscribeStreamResponseUnparsed.PROTOBUF.toBytes(subscribeStreamResponse); } @NonNull - private PublishStreamRequest parsePublishStreamRequest( + private List parsePublishStreamRequest( @NonNull final Bytes message, @NonNull final RequestOptions options) throws ParseException { - // TODO: Copying bytes to avoid using references passed from Helidon. Investigate if this is necessary. - return PublishStreamRequest.PROTOBUF.parse(Bytes.wrap(message.toByteArray())); + final PublishStreamRequestUnparsed request = PublishStreamRequestUnparsed.PROTOBUF.parse(message); + return request.blockItems().blockItems(); } @NonNull diff --git a/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java b/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java index 28c0f7426..518033e8c 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java +++ b/server/src/main/java/com/hedera/block/server/persistence/PersistenceInjectionModule.java @@ -25,9 +25,9 @@ import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.persistence.storage.write.NoOpBlockWriter; -import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import dagger.Binds; import dagger.Module; import dagger.Provides; @@ -47,7 +47,7 @@ public interface PersistenceInjectionModule { */ @Provides @Singleton - static BlockWriter> providesBlockWriter(BlockNodeContext blockNodeContext) { + static BlockWriter> providesBlockWriter(BlockNodeContext blockNodeContext) { final String persistenceType = blockNodeContext .configuration() .getConfigData(PersistenceStorageConfig.class) @@ -70,7 +70,7 @@ static BlockWriter> providesBlockWriter(BlockNodeContext blockNo */ @Provides @Singleton - static BlockReader providesBlockReader(PersistenceStorageConfig config) { + static BlockReader providesBlockReader(PersistenceStorageConfig config) { return BlockAsDirReaderBuilder.newBuilder(config).build(); } @@ -82,6 +82,6 @@ static BlockReader providesBlockReader(PersistenceStorageConfig config) { */ @Binds @Singleton - BlockNodeEventHandler> bindBlockNodeEventHandler( + BlockNodeEventHandler> bindBlockNodeEventHandler( StreamPersistenceHandlerImpl streamPersistenceHandler); } diff --git a/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java b/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java index 90eee841f..76d3e96ac 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java +++ b/server/src/main/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImpl.java @@ -29,9 +29,10 @@ import com.hedera.block.server.notifier.Notifier; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; -import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.hedera.pbj.runtime.OneOf; +import com.hedera.pbj.runtime.ParseException; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.util.List; @@ -49,12 +50,12 @@ */ @Singleton public class StreamPersistenceHandlerImpl - implements BlockNodeEventHandler> { + implements BlockNodeEventHandler> { private final System.Logger LOGGER = System.getLogger(getClass().getName()); - private final SubscriptionHandler subscriptionHandler; - private final BlockWriter> blockWriter; + private final SubscriptionHandler subscriptionHandler; + private final BlockWriter> blockWriter; private final Notifier notifier; private final MetricsService metricsService; private final ServiceStatus serviceStatus; @@ -77,9 +78,9 @@ public class StreamPersistenceHandlerImpl */ @Inject public StreamPersistenceHandlerImpl( - @NonNull final SubscriptionHandler subscriptionHandler, + @NonNull final SubscriptionHandler subscriptionHandler, @NonNull final Notifier notifier, - @NonNull final BlockWriter> blockWriter, + @NonNull final BlockWriter> blockWriter, @NonNull final BlockNodeContext blockNodeContext, @NonNull final ServiceStatus serviceStatus) { this.subscriptionHandler = subscriptionHandler; @@ -98,30 +99,26 @@ public StreamPersistenceHandlerImpl( * @param b true if the event is the last in the sequence */ @Override - public void onEvent(ObjectEvent event, long l, boolean b) { + public void onEvent(ObjectEvent event, long l, boolean b) { try { if (serviceStatus.isRunning()) { - final SubscribeStreamResponse subscribeStreamResponse = event.get(); - final OneOf oneOfTypeOneOf = + final SubscribeStreamResponseUnparsed subscribeStreamResponse = event.get(); + final OneOf oneOfTypeOneOf = subscribeStreamResponse.response(); switch (oneOfTypeOneOf.kind()) { case BLOCK_ITEMS -> { if (subscribeStreamResponse.blockItems() == null) { - final String message = - PROTOCOL_VIOLATION_MESSAGE.formatted( - "SubscribeStreamResponse", - "BLOCK_ITEM", - "block_item", - subscribeStreamResponse); + final String message = PROTOCOL_VIOLATION_MESSAGE.formatted( + "SubscribeStreamResponse", "BLOCK_ITEM", "block_item", subscribeStreamResponse); LOGGER.log(ERROR, message); metricsService.get(StreamPersistenceHandlerError).increment(); throw new BlockStreamProtocolException(message); } else { // Persist the BlockItems - List blockItems = + List blockItems = subscribeStreamResponse.blockItems().blockItems(); - Optional> result = blockWriter.write(blockItems); + Optional> result = blockWriter.write(blockItems); if (result.isPresent()) { // Publish the block item back upstream to the notifier // to send responses to producers. @@ -129,8 +126,7 @@ public void onEvent(ObjectEvent event, long l, boolean } } } - case STATUS -> LOGGER.log( - DEBUG, "Unexpected received a status message rather than a block item"); + case STATUS -> LOGGER.log(DEBUG, "Unexpected received a status message rather than a block item"); default -> { final String message = "Unknown response type: " + oneOfTypeOneOf.kind(); LOGGER.log(ERROR, message); @@ -139,11 +135,12 @@ public void onEvent(ObjectEvent event, long l, boolean } } } else { - LOGGER.log( - ERROR, "Service is not running. Block item will not be processed further."); + LOGGER.log(ERROR, "Service is not running. Block item will not be processed further."); } - } catch (BlockStreamProtocolException | IOException e) { + } catch (BlockStreamProtocolException | IOException | ParseException e) { + + LOGGER.log(ERROR, "Failed to persist BlockItems: ", e); metricsService.get(StreamPersistenceHandlerError).increment(); diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java index 908776b81..3a082c4b0 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReader.java @@ -23,8 +23,8 @@ import static java.lang.System.Logger.Level.INFO; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; import com.hedera.pbj.runtime.ParseException; import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; @@ -47,7 +47,7 @@ * The BlockAsDirReader class reads a block from the file system. The block is stored as a directory * containing block items. The block items are stored as files within the block directory. */ -class BlockAsDirReader implements BlockReader { +class BlockAsDirReader implements BlockReader { private final Logger LOGGER = System.getLogger(getClass().getName()); private final Path blockNodeRootPath; private final FileAttribute> folderPermissions; @@ -96,7 +96,7 @@ class BlockAsDirReader implements BlockReader { */ @NonNull @Override - public Optional read(final long blockNumber) throws IOException, ParseException { + public Optional read(final long blockNumber) throws IOException, ParseException { // Verify path attributes of the block node root path if (isPathDisqualified(blockNodeRootPath)) { @@ -121,11 +121,11 @@ public Optional read(final long blockNumber) throws IOException, ParseExc // 10.blk), the loop will directly fetch the BlockItems in order based on // their file names. The loop will exit when it attempts to read a // BlockItem file that does not exist (e.g., 11.blk). - final Block.Builder builder = Block.newBuilder(); - final List blockItems = new ArrayList<>(); + final BlockUnparsed.Builder builder = BlockUnparsed.newBuilder(); + final List blockItems = new ArrayList<>(); for (int i = 1; ; i++) { final Path blockItemPath = blockPath.resolve(i + BLOCK_FILE_EXTENSION); - final Optional blockItemOpt = readBlockItem(blockItemPath.toString()); + final Optional blockItemOpt = readBlockItem(blockItemPath.toString()); if (blockItemOpt.isPresent()) { blockItems.add(blockItemOpt.get()); continue; @@ -134,7 +134,7 @@ public Optional read(final long blockNumber) throws IOException, ParseExc break; } - builder.items(blockItems); + builder.blockItems(blockItems); // Return the Block return Optional.of(builder.build()); @@ -145,12 +145,11 @@ public Optional read(final long blockNumber) throws IOException, ParseExc } @NonNull - private Optional readBlockItem(@NonNull final String blockItemPath) throws IOException, ParseException { + private Optional readBlockItem(@NonNull final String blockItemPath) + throws IOException, ParseException { try (final FileInputStream fis = new FileInputStream(blockItemPath)) { - - BlockItem blockItem = BlockItem.PROTOBUF.parse(Bytes.wrap(fis.readAllBytes())); - return Optional.of(blockItem); + return Optional.of(BlockItemUnparsed.PROTOBUF.parse(Bytes.wrap(fis.readAllBytes()))); } catch (FileNotFoundException io) { final File f = new File(blockItemPath); if (!f.exists()) { diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java index 6bb4f7da2..8cb90cbee 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderBuilder.java @@ -17,7 +17,7 @@ package com.hedera.block.server.persistence.storage.read; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.hapi.block.stream.Block; +import com.hedera.hapi.block.BlockUnparsed; import edu.umd.cs.findbugs.annotations.NonNull; import java.nio.file.attribute.FileAttribute; import java.nio.file.attribute.PosixFilePermission; @@ -69,7 +69,7 @@ public BlockAsDirReaderBuilder folderPermissions( * @return a new block reader configured with the parameters provided to the builder. */ @NonNull - public BlockReader build() { + public BlockReader build() { return new BlockAsDirReader(config, folderPermissions); } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java index f6457ae60..461c74911 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriter.java @@ -29,7 +29,10 @@ import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.remove.BlockRemover; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.stream.output.BlockHeader; +import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.io.buffer.Bytes; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.FileOutputStream; import java.io.IOException; @@ -52,7 +55,7 @@ * to remove the current, incomplete block (directory) before re-throwing the exception to the * caller. */ -class BlockAsDirWriter implements BlockWriter> { +class BlockAsDirWriter implements BlockWriter> { private final Logger LOGGER = System.getLogger(getClass().getName()); private final Path blockNodeRootPath; private final FileAttribute> folderPermissions; @@ -112,17 +115,19 @@ class BlockAsDirWriter implements BlockWriter> { * @throws IOException if an error occurs while writing the block item */ @Override - public Optional> write(@NonNull final List blockItems) throws IOException { + public Optional> write(@NonNull final List blockItems) + throws IOException, ParseException { - if (blockItems.getFirst().hasBlockHeader()) { - resetState(blockItems.getFirst()); + final Bytes unparsedBlockHeader = blockItems.getFirst().blockHeader(); + if (unparsedBlockHeader != null) { + resetState(BlockHeader.PROTOBUF.parse(unparsedBlockHeader)); } - for (BlockItem blockItem : blockItems) { + for (BlockItemUnparsed blockItemUnparsed : blockItems) { final Path blockItemFilePath = calculateBlockItemPath(); for (int retries = 0; ; retries++) { try { - write(blockItemFilePath, blockItem); + write(blockItemFilePath, blockItemUnparsed); break; } catch (IOException e) { @@ -159,10 +164,11 @@ public Optional> write(@NonNull final List blockItems * @param blockItem the block item to write * @throws IOException if an error occurs while writing the block item */ - protected void write(@NonNull final Path blockItemFilePath, @NonNull final BlockItem blockItem) throws IOException { + protected void write(@NonNull final Path blockItemFilePath, @NonNull final BlockItemUnparsed blockItem) + throws IOException { try (final FileOutputStream fos = new FileOutputStream(blockItemFilePath.toString())) { - - BlockItem.PROTOBUF.toBytes(blockItem).writeTo(fos); + // Write the Bytes directly + BlockItemUnparsed.PROTOBUF.toBytes(blockItem).writeTo(fos); LOGGER.log(DEBUG, "Successfully wrote the block item file: {0}", blockItemFilePath); } catch (IOException e) { LOGGER.log(ERROR, "Error writing the BlockItem protobuf to a file: ", e); @@ -170,10 +176,10 @@ protected void write(@NonNull final Path blockItemFilePath, @NonNull final Block } } - private void resetState(@NonNull final BlockItem blockItem) throws IOException { + private void resetState(@NonNull final BlockHeader blockHeader) throws IOException { // Here a "block" is represented as a directory of BlockItems. // Create the "block" directory based on the block_number - currentBlockDir = Path.of(String.valueOf(blockItem.blockHeader().number())); + currentBlockDir = Path.of(String.valueOf(blockHeader.number())); // Check the blockNodeRootPath permissions and // attempt to repair them if possible diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java index c5378dcc9..b6e826807 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterBuilder.java @@ -20,7 +20,7 @@ import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.remove.BlockAsDirRemover; import com.hedera.block.server.persistence.storage.remove.BlockRemover; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.nio.file.Path; @@ -95,7 +95,7 @@ public BlockAsDirWriterBuilder blockRemover(@NonNull final BlockRemover blockRem * @throws IOException when an error occurs while persisting block items to storage. */ @NonNull - public BlockWriter> build() throws IOException { + public BlockWriter> build() throws IOException { return new BlockAsDirWriter(blockRemover, folderPermissions, blockNodeContext); } } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java index af6630337..eccbe4291 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/BlockWriter.java @@ -16,6 +16,7 @@ package com.hedera.block.server.persistence.storage.write; +import com.hedera.pbj.runtime.ParseException; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.util.Optional; @@ -34,6 +35,7 @@ public interface BlockWriter { * @return an optional containing the block item written to storage if the block item was a * block proof signaling the end of the block, an empty optional otherwise. * @throws IOException when failing to write the block item to storage. + * @throws ParseException when failing to parse a block item. */ - Optional write(@NonNull final V blockItem) throws IOException; + Optional write(@NonNull final V blockItem) throws IOException, ParseException; } diff --git a/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java b/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java index bacf8f323..a805c1fd6 100644 --- a/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java +++ b/server/src/main/java/com/hedera/block/server/persistence/storage/write/NoOpBlockWriter.java @@ -16,12 +16,10 @@ package com.hedera.block.server.persistence.storage.write; -import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.BlocksPersisted; import static java.lang.System.Logger.Level.INFO; import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.metrics.MetricsService; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.util.List; @@ -32,9 +30,7 @@ * designed to isolate the Producer and Mediator components from storage implementation during testing while still * providing metrics and logging for troubleshooting. */ -public class NoOpBlockWriter implements BlockWriter> { - - private final MetricsService metricsService; +public class NoOpBlockWriter implements BlockWriter> { /** * Creates a new NoOpBlockWriter instance for testing and troubleshooting only. @@ -42,7 +38,6 @@ public class NoOpBlockWriter implements BlockWriter> { * @param blockNodeContext the block node context */ public NoOpBlockWriter(BlockNodeContext blockNodeContext) { - this.metricsService = blockNodeContext.metricsService(); System.getLogger(getClass().getName()).log(INFO, "Using " + getClass().getSimpleName()); } @@ -50,9 +45,12 @@ public NoOpBlockWriter(BlockNodeContext blockNodeContext) { * {@inheritDoc} */ @Override - public Optional> write(@NonNull List blockItems) throws IOException { + public Optional> write(@NonNull List blockItems) throws IOException { if (blockItems.getLast().hasBlockProof()) { - metricsService.get(BlocksPersisted).increment(); + // Returning the BlockItems triggers a + // PublishStreamResponse to be sent to the + // upstream producer. + return Optional.of(blockItems); } return Optional.empty(); diff --git a/server/src/main/java/com/hedera/block/server/producer/NoOpProducerObserver.java b/server/src/main/java/com/hedera/block/server/producer/NoOpProducerObserver.java index dcdd61b1d..ab695e6b4 100644 --- a/server/src/main/java/com/hedera/block/server/producer/NoOpProducerObserver.java +++ b/server/src/main/java/com/hedera/block/server/producer/NoOpProducerObserver.java @@ -24,9 +24,10 @@ import com.hedera.block.server.events.BlockNodeEventHandler; import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.metrics.MetricsService; -import com.hedera.hapi.block.PublishStreamRequest; +import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.PublishStreamResponse; import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.List; import java.util.concurrent.Flow; /** @@ -35,7 +36,7 @@ * still providing metrics and logging for troubleshooting. */ public class NoOpProducerObserver - implements Flow.Subscriber, BlockNodeEventHandler> { + implements Flow.Subscriber>, BlockNodeEventHandler> { private final System.Logger LOGGER = System.getLogger(getClass().getName()); private final MetricsService metricsService; @@ -57,10 +58,9 @@ public NoOpProducerObserver( * {@inheritDoc} */ @Override - public void onNext(PublishStreamRequest publishStreamRequest) { - metricsService - .get(LiveBlockItemsReceived) - .add(publishStreamRequest.blockItems().blockItems().size()); + public void onNext(List blockItems) { + + metricsService.get(LiveBlockItemsReceived).add(blockItems.size()); } /** diff --git a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java index 922e9be4f..191d485c3 100644 --- a/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java +++ b/server/src/main/java/com/hedera/block/server/producer/ProducerBlockItemObserver.java @@ -31,12 +31,10 @@ import com.hedera.block.server.mediator.SubscriptionHandler; import com.hedera.block.server.metrics.MetricsService; import com.hedera.block.server.service.ServiceStatus; -import com.hedera.hapi.block.BlockItemSet; +import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.EndOfStream; -import com.hedera.hapi.block.PublishStreamRequest; import com.hedera.hapi.block.PublishStreamResponse; import com.hedera.hapi.block.PublishStreamResponseCode; -import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.grpc.Pipeline; import edu.umd.cs.findbugs.annotations.NonNull; import java.time.InstantSource; @@ -51,12 +49,12 @@ * server). */ public class ProducerBlockItemObserver - implements Pipeline, BlockNodeEventHandler> { + implements Pipeline>, BlockNodeEventHandler> { private final Logger LOGGER = System.getLogger(getClass().getName()); private final SubscriptionHandler subscriptionHandler; - private final Publisher> publisher; + private final Publisher> publisher; private final ServiceStatus serviceStatus; private final MetricsService metricsService; private final Flow.Subscriber publishStreamResponseObserver; @@ -83,7 +81,7 @@ public class ProducerBlockItemObserver */ public ProducerBlockItemObserver( @NonNull final InstantSource producerLivenessClock, - @NonNull final Publisher> publisher, + @NonNull final Publisher> publisher, @NonNull final SubscriptionHandler subscriptionHandler, @NonNull final Flow.Subscriber publishStreamResponseObserver, @NonNull final BlockNodeContext blockNodeContext, @@ -113,17 +111,16 @@ public void onSubscribe(Flow.Subscription subscription) { * producer. The method publish the block item data to all subscribers via the Publisher and * sends a response back to the upstream producer. * - * @param publishStreamRequest the PublishStreamRequest received from the upstream producer */ @Override - public void onNext(@NonNull final PublishStreamRequest publishStreamRequest) { + public void onNext(@NonNull final List blockItems) { - final BlockItemSet blockItemSet = publishStreamRequest.blockItems(); - LOGGER.log( - DEBUG, - "Received PublishStreamRequest from producer with " - + blockItemSet.blockItems().size() + " BlockItems."); - metricsService.get(LiveBlockItemsReceived).add(blockItemSet.blockItems().size()); + LOGGER.log(DEBUG, "Received PublishStreamRequest from producer with " + blockItems.size() + " BlockItems."); + if (blockItems.isEmpty()) { + return; + } + + metricsService.get(LiveBlockItemsReceived).add(blockItems.size()); // Publish the block to all the subscribers unless // there's an issue with the StreamMediator. @@ -132,7 +129,7 @@ public void onNext(@NonNull final PublishStreamRequest publishStreamRequest) { livenessCalculator.refresh(); // Publish the block to the mediator - publisher.publish(blockItemSet.blockItems()); + publisher.publish(blockItems); } else { LOGGER.log(ERROR, getClass().getName() + " is not accepting BlockItems"); @@ -152,6 +149,7 @@ public void onEvent(ObjectEvent event, long sequence, boo if (isResponsePermitted.get()) { if (isTimeoutExpired()) { + isResponsePermitted.set(false); subscriptionHandler.unsubscribe(this); LOGGER.log(DEBUG, "Producer liveness timeout. Unsubscribed ProducerBlockItemObserver."); } else { @@ -195,8 +193,6 @@ public void onComplete() { isResponsePermitted.set(false); subscriptionHandler.unsubscribe(this); LOGGER.log(DEBUG, "Producer completed the stream. Observer unsubscribed."); - - publishStreamResponseObserver.onComplete(); } @Override diff --git a/server/src/main/java/com/hedera/block/server/producer/Util.java b/server/src/main/java/com/hedera/block/server/producer/Util.java index d6ad74488..08f323c9f 100644 --- a/server/src/main/java/com/hedera/block/server/producer/Util.java +++ b/server/src/main/java/com/hedera/block/server/producer/Util.java @@ -16,7 +16,7 @@ package com.hedera.block.server.producer; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; import edu.umd.cs.findbugs.annotations.NonNull; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -34,10 +34,10 @@ private Util() {} * @return the fake hash for the given block item * @throws NoSuchAlgorithmException thrown if the SHA-384 algorithm is not available */ - public static byte[] getFakeHash(@NonNull final List blockItems) + public static byte[] getFakeHash(@NonNull final List blockItems) throws NoSuchAlgorithmException { // Calculate the SHA-384 hash MessageDigest digest = MessageDigest.getInstance("SHA-384"); - return digest.digest(BlockItem.PROTOBUF.toBytes(blockItems.getLast()).toByteArray()); + return digest.digest(blockItems.getLast().blockProof().toByteArray()); } } diff --git a/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java b/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java index 81ea98e50..40e067b50 100644 --- a/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java +++ b/server/src/test/java/com/hedera/block/server/BlockNodeAppTest.java @@ -30,8 +30,8 @@ import com.hedera.block.server.pbj.PbjBlockStreamServiceProxy; import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.service.ServiceStatus; -import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.Block; +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.hedera.pbj.grpc.helidon.PbjRouting; import com.hedera.pbj.grpc.helidon.config.PbjConfig; import io.helidon.webserver.WebServer; @@ -63,10 +63,10 @@ class BlockNodeAppTest { private LiveStreamMediator liveStreamMediator; @Mock - private BlockReader blockReader; + private BlockReader blockReader; @Mock - private BlockNodeEventHandler> blockNodeEventHandler; + private BlockNodeEventHandler> blockNodeEventHandler; @Mock private Notifier notifier; diff --git a/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java b/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java index d70c18cf8..a8d0e4beb 100644 --- a/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java +++ b/server/src/test/java/com/hedera/block/server/consumer/ConsumerStreamResponseObserverTest.java @@ -26,14 +26,14 @@ import com.hedera.block.server.events.ObjectEvent; import com.hedera.block.server.mediator.StreamMediator; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.hapi.block.BlockItemSet; -import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemSetUnparsed; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.hedera.hapi.block.stream.BlockProof; import com.hedera.hapi.block.stream.input.EventHeader; import com.hedera.hapi.block.stream.output.BlockHeader; -import com.hedera.hapi.platform.event.EventCore; import com.hedera.pbj.runtime.grpc.Pipeline; +import com.hedera.pbj.runtime.io.buffer.Bytes; import java.io.IOException; import java.time.InstantSource; import java.util.Map; @@ -51,13 +51,13 @@ public class ConsumerStreamResponseObserverTest { private static final int testTimeout = 1000; @Mock - private StreamMediator streamMediator; + private StreamMediator streamMediator; @Mock - private Pipeline responseStreamObserver; + private Pipeline responseStreamObserver; @Mock - private ObjectEvent objectEvent; + private ObjectEvent objectEvent; @Mock private InstantSource testClock; @@ -78,12 +78,14 @@ public void testProducerTimeoutWithinWindow() { new ConsumerStreamResponseObserver(testClock, streamMediator, responseStreamObserver, testContext); final BlockHeader blockHeader = BlockHeader.newBuilder().number(1).build(); - final BlockItem blockItem = - BlockItem.newBuilder().blockHeader(blockHeader).build(); - final BlockItemSet blockItemSet = - BlockItemSet.newBuilder().blockItems(blockItem).build(); - final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItems(blockItemSet).build(); + final BlockItemUnparsed blockItem = BlockItemUnparsed.newBuilder() + .blockHeader(BlockHeader.PROTOBUF.toBytes(blockHeader)) + .build(); + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItem).build(); + final SubscribeStreamResponseUnparsed subscribeStreamResponse = SubscribeStreamResponseUnparsed.newBuilder() + .blockItems(blockItemSet) + .build(); when(objectEvent.get()).thenReturn(subscribeStreamResponse); @@ -124,37 +126,40 @@ public void testConsumerNotToSendBeforeBlockHeader() { for (int i = 1; i <= 10; i++) { if (i % 2 == 0) { - final EventHeader eventHeader = EventHeader.newBuilder() - .eventCore(EventCore.newBuilder().build()) - .build(); - final BlockItem blockItem = - BlockItem.newBuilder().eventHeader(eventHeader).build(); - final BlockItemSet blockItemSet = - BlockItemSet.newBuilder().blockItems(blockItem).build(); - final SubscribeStreamResponse subscribeStreamResponse = SubscribeStreamResponse.newBuilder() - .blockItems(blockItemSet) - .build(); + final Bytes eventHeader = + EventHeader.PROTOBUF.toBytes(EventHeader.newBuilder().build()); + final BlockItemUnparsed blockItem = + BlockItemUnparsed.newBuilder().eventHeader(eventHeader).build(); + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItem).build(); + final SubscribeStreamResponseUnparsed subscribeStreamResponse = + SubscribeStreamResponseUnparsed.newBuilder() + .blockItems(blockItemSet) + .build(); when(objectEvent.get()).thenReturn(subscribeStreamResponse); } else { - final BlockProof blockProof = BlockProof.newBuilder().block(i).build(); - final BlockItem blockItem = - BlockItem.newBuilder().blockProof(blockProof).build(); - final BlockItemSet blockItemSet = - BlockItemSet.newBuilder().blockItems(blockItem).build(); - final SubscribeStreamResponse subscribeStreamResponse = SubscribeStreamResponse.newBuilder() - .blockItems(blockItemSet) - .build(); + final Bytes blockProof = BlockProof.PROTOBUF.toBytes( + BlockProof.newBuilder().block(i).build()); + final BlockItemUnparsed blockItem = + BlockItemUnparsed.newBuilder().blockProof(blockProof).build(); + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItem).build(); + final SubscribeStreamResponseUnparsed subscribeStreamResponse = + SubscribeStreamResponseUnparsed.newBuilder() + .blockItems(blockItemSet) + .build(); when(objectEvent.get()).thenReturn(subscribeStreamResponse); } consumerBlockItemObserver.onEvent(objectEvent, 0, true); } - final BlockItem blockItem = BlockItem.newBuilder().build(); - final BlockItemSet blockItemSet = - BlockItemSet.newBuilder().blockItems(blockItem).build(); - final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItems(blockItemSet).build(); + final BlockItemUnparsed blockItem = BlockItemUnparsed.newBuilder().build(); + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItem).build(); + final SubscribeStreamResponseUnparsed subscribeStreamResponse = SubscribeStreamResponseUnparsed.newBuilder() + .blockItems(blockItemSet) + .build(); // Confirm that the observer was called with the next BlockItem // since we never send a BlockItem with a Header to start the stream. @@ -167,11 +172,12 @@ public void testSubscriberStreamResponseIsBlockItemWhenBlockItemIsNull() { // The generated objects contain safeguards to prevent a SubscribeStreamResponse // being created with a null BlockItem. Here, I have to used a spy() to even // manufacture this scenario. This should not happen in production. - final BlockItem blockItem = BlockItem.newBuilder().build(); - final BlockItemSet blockItemSet = - BlockItemSet.newBuilder().blockItems(blockItem).build(); - final SubscribeStreamResponse subscribeStreamResponse = spy( - SubscribeStreamResponse.newBuilder().blockItems(blockItemSet).build()); + final BlockItemUnparsed blockItem = BlockItemUnparsed.newBuilder().build(); + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItem).build(); + final SubscribeStreamResponseUnparsed subscribeStreamResponse = spy(SubscribeStreamResponseUnparsed.newBuilder() + .blockItems(blockItemSet) + .build()); when(subscribeStreamResponse.blockItems()).thenReturn(null); when(objectEvent.get()).thenReturn(subscribeStreamResponse); @@ -184,8 +190,8 @@ public void testSubscriberStreamResponseIsBlockItemWhenBlockItemIsNull() { @Test public void testSubscribeStreamResponseTypeNotSupported() { - final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().build(); + final SubscribeStreamResponseUnparsed subscribeStreamResponse = + SubscribeStreamResponseUnparsed.newBuilder().build(); when(objectEvent.get()).thenReturn(subscribeStreamResponse); final var consumerBlockItemObserver = diff --git a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java index d038ce71f..895c5dccb 100644 --- a/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java +++ b/server/src/test/java/com/hedera/block/server/grpc/BlockAccessServiceTest.java @@ -18,23 +18,36 @@ import static com.hedera.block.server.Constants.FULL_SERVICE_NAME_BLOCK_ACCESS; import static com.hedera.block.server.Constants.SERVICE_NAME_BLOCK_ACCESS; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.pbj.PbjBlockAccessService; import com.hedera.block.server.pbj.PbjBlockAccessServiceProxy; import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; +import com.hedera.block.server.persistence.storage.read.BlockAsDirReaderBuilder; import com.hedera.block.server.persistence.storage.read.BlockReader; +import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.hapi.block.protoc.SingleBlockResponse; -import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.hapi.block.SingleBlockRequest; +import com.hedera.hapi.block.SingleBlockResponseCode; +import com.hedera.hapi.block.SingleBlockResponseUnparsed; +import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.grpc.Pipeline; +import com.hedera.pbj.runtime.io.buffer.Bytes; import java.io.IOException; import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.Flow; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -47,13 +60,13 @@ class BlockAccessServiceTest { @Mock - private Flow.Subscriber responseObserver; + private Flow.Subscriber responseObserver; @Mock - private BlockReader blockReader; + private BlockReader blockReader; @Mock - private BlockWriter> blockWriter; + private BlockWriter> blockWriter; @Mock private ServiceStatus serviceStatus; @@ -92,181 +105,149 @@ public void testMethods() { assertEquals(1, blockAccessService.methods().size()); } - // @Test - // void testProto() { - // BlockAccessService blockAccessService = - // new BlockAccessService( - // serviceStatus, blockReader, blockNodeContext.metricsService()); - // Descriptors.FileDescriptor fileDescriptor = blockAccessService.proto(); - // // Verify the current rpc methods on - // Descriptors.ServiceDescriptor blockAccessServiceDescriptor = - // fileDescriptor.getServices().stream() - // .filter( - // service -> - // service.getName() - // .equals(Constants.SERVICE_NAME_BLOCK_ACCESS)) - // .findFirst() - // .orElse(null); - // - // Assertions.assertNotNull( - // blockAccessServiceDescriptor, - // "Service descriptor not found for: " + Constants.SERVICE_NAME_BLOCK_ACCESS); - // assertEquals(1, blockAccessServiceDescriptor.getMethods().size()); - // - // assertNotNull(blockAccessServiceDescriptor.getName(), blockAccessService.serviceName()); - // - // // Verify the current rpc methods on the service - // Descriptors.MethodDescriptor singleBlockMethod = - // blockAccessServiceDescriptor.getMethods().stream() - // .filter(method -> method.getName().equals(SINGLE_BLOCK_METHOD_NAME)) - // .findFirst() - // .orElse(null); - // - // assertEquals(SINGLE_BLOCK_METHOD_NAME, singleBlockMethod.getName()); - // } - // - // @Test - // void testSingleBlockHappyPath() throws IOException, ParseException { - // - // final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); - // - // final BlockAccessService blockAccessService = - // new BlockAccessService( - // serviceStatus, blockReader, blockNodeContext.metricsService()); - // - // // Enable the serviceStatus - // when(serviceStatus.isRunning()).thenReturn(true); - // - // // Generate and persist a block - // final BlockWriter> blockWriter = - // BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - // final List blockItems = generateBlockItems(1); - // blockWriter.write(blockItems); - // - // // Get the block so we can verify the response payload - // final Optional blockOpt = blockReader.read(1); - // if (blockOpt.isEmpty()) { - // fail("Block 1 should be present"); - // return; - // } - // - // // Build a response to verify what's passed to the response observer - // final com.hedera.hapi.block.protoc.SingleBlockResponse expectedSingleBlockResponse = - // fromPbjSingleBlockSuccessResponse(blockOpt.get()); - // - // // Build a request to invoke the service - // final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = - // com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() - // .setBlockNumber(1) - // .build(); - // - // // Call the service - // blockAccessService.protocSingleBlock(singleBlockRequest, responseObserver); - // verify(responseObserver, times(1)).onNext(expectedSingleBlockResponse); - // } - // - // @Test - // void testSingleBlockNotFoundPath() throws IOException, ParseException { - // - // // Get the block so we can verify the response payload - // when(blockReader.read(1)).thenReturn(Optional.empty()); - // - // // Build a response to verify what's passed to the response observer - // final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotFound = - // buildSingleBlockNotFoundResponse(); - // - // // Build a request to invoke the service - // final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = - // com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() - // .setBlockNumber(1) - // .build(); - // - // final BlockAccessService blockAccessService = - // new BlockAccessService( - // serviceStatus, blockReader, blockNodeContext.metricsService()); - // - // // Enable the serviceStatus - // when(serviceStatus.isRunning()).thenReturn(true); - // - // blockAccessService.protocSingleBlock(singleBlockRequest, responseObserver); - // verify(responseObserver, times(1)).onNext(expectedNotFound); - // } - // - // @Test - // void testSingleBlockServiceNotAvailable() { - // - // final BlockAccessService blockAccessService = - // new BlockAccessService( - // serviceStatus, blockReader, blockNodeContext.metricsService()); - // - // // Set the service status to not running - // when(serviceStatus.isRunning()).thenReturn(false); - // - // final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotAvailable = - // buildSingleBlockNotAvailableResponse(); - // - // // Build a request to invoke the service - // final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = - // com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() - // .setBlockNumber(1) - // .build(); - // blockAccessService.protocSingleBlock(singleBlockRequest, responseObserver); - // verify(responseObserver, times(1)).onNext(expectedNotAvailable); - // } - // - // @Test - // public void testSingleBlockIOExceptionPath() throws IOException, ParseException { - // final BlockAccessService blockAccessService = - // new BlockAccessService( - // serviceStatus, blockReader, blockNodeContext.metricsService()); - // - // when(serviceStatus.isRunning()).thenReturn(true); - // when(blockReader.read(1)).thenThrow(new IOException("Test exception")); - // - // final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotAvailable = - // buildSingleBlockNotAvailableResponse(); - // - // // Build a request to invoke the service - // final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = - // com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() - // .setBlockNumber(1) - // .build(); - // blockAccessService.protocSingleBlock(singleBlockRequest, responseObserver); - // verify(responseObserver, times(1)).onNext(expectedNotAvailable); - // } - // - // @Test - // public void testSingleBlockParseExceptionPath() throws IOException, ParseException { - // final BlockAccessService blockAccessService = - // new BlockAccessService( - // serviceStatus, blockReader, blockNodeContext.metricsService()); - // - // when(serviceStatus.isRunning()).thenReturn(true); - // when(blockReader.read(1)).thenThrow(new ParseException("Test exception")); - // - // final com.hedera.hapi.block.protoc.SingleBlockResponse expectedNotAvailable = - // buildSingleBlockNotAvailableResponse(); - // - // // Build a request to invoke the service - // final com.hedera.hapi.block.protoc.SingleBlockRequest singleBlockRequest = - // com.hedera.hapi.block.protoc.SingleBlockRequest.newBuilder() - // .setBlockNumber(1) - // .build(); - // blockAccessService.protocSingleBlock(singleBlockRequest, responseObserver); - // verify(responseObserver, times(1)).onNext(expectedNotAvailable); - // } - // - // @Test - // public void testUpdateInvokesRoutingWithLambdas() { - // - // final BlockAccessService blockAccessService = - // new BlockAccessService( - // serviceStatus, blockReader, blockNodeContext.metricsService()); - // - // GrpcService.Routing routing = mock(GrpcService.Routing.class); - // blockAccessService.update(routing); - // - // verify(routing, timeout(testTimeout).times(1)) - // .unary(eq(SINGLE_BLOCK_METHOD_NAME), any(ServerCalls.UnaryMethod.class)); - // } + @Test + void testSingleBlockHappyPath() throws IOException, ParseException { + + final BlockReader blockReader = + BlockAsDirReaderBuilder.newBuilder(config).build(); + + final PbjBlockAccessService blockAccessService = + new PbjBlockAccessServiceProxy(serviceStatus, blockReader, blockNodeContext); + + // Enable the serviceStatus + when(serviceStatus.isRunning()).thenReturn(true); + + // Generate and persist a block + final BlockWriter> blockWriter = + BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); + final List blockItems = generateBlockItemsUnparsed(1); + blockWriter.write(blockItems); + + // Get the block so we can verify the response payload + final Optional blockOpt = blockReader.read(1); + if (blockOpt.isEmpty()) { + fail("Block 1 should be present"); + return; + } + + // Build a response to verify what's passed to the response observer + final SingleBlockResponseUnparsed expectedSingleBlockResponse = SingleBlockResponseUnparsed.newBuilder() + .block(blockOpt.get()) + .status(SingleBlockResponseCode.READ_BLOCK_SUCCESS) + .build(); + + // Build a request to invoke the service + final SingleBlockRequest singleBlockRequest = + SingleBlockRequest.newBuilder().blockNumber(1).build(); + + final Pipeline pipeline = + blockAccessService.open(PbjBlockAccessService.BlockAccessMethod.singleBlock, null, responseObserver); + + // Call the service + pipeline.onNext(SingleBlockRequest.PROTOBUF.toBytes(singleBlockRequest)); + verify(responseObserver, times(1)) + .onNext(SingleBlockResponseUnparsed.PROTOBUF.toBytes(expectedSingleBlockResponse)); + } + + @Test + void testSingleBlockNotFoundPath() throws IOException, ParseException { + + // Get the block so we can verify the response payload + when(blockReader.read(1)).thenReturn(Optional.empty()); + + // Build a response to verify what's passed to the response observer + final SingleBlockResponseUnparsed expectedNotFound = SingleBlockResponseUnparsed.newBuilder() + .status(SingleBlockResponseCode.READ_BLOCK_NOT_FOUND) + .build(); + + // Build a request to invoke the service + final SingleBlockRequest singleBlockRequest = + SingleBlockRequest.newBuilder().blockNumber(1).build(); + + final PbjBlockAccessService blockAccessService = + new PbjBlockAccessServiceProxy(serviceStatus, blockReader, blockNodeContext); + + // Enable the serviceStatus + when(serviceStatus.isRunning()).thenReturn(true); + + final Pipeline pipeline = + blockAccessService.open(PbjBlockAccessService.BlockAccessMethod.singleBlock, null, responseObserver); + + // Call the service + pipeline.onNext(SingleBlockRequest.PROTOBUF.toBytes(singleBlockRequest)); + verify(responseObserver, times(1)).onNext(SingleBlockResponseUnparsed.PROTOBUF.toBytes(expectedNotFound)); + } + + @Test + void testSingleBlockServiceNotAvailable() { + + final PbjBlockAccessService blockAccessService = + new PbjBlockAccessServiceProxy(serviceStatus, blockReader, blockNodeContext); + + // Set the service status to not running + when(serviceStatus.isRunning()).thenReturn(false); + + final SingleBlockResponseUnparsed expectedNotAvailable = SingleBlockResponseUnparsed.newBuilder() + .status(SingleBlockResponseCode.READ_BLOCK_NOT_AVAILABLE) + .build(); + + // Build a request to invoke the service + final SingleBlockRequest singleBlockRequest = + SingleBlockRequest.newBuilder().blockNumber(1).build(); + + final Pipeline pipeline = + blockAccessService.open(PbjBlockAccessService.BlockAccessMethod.singleBlock, null, responseObserver); + + // Call the service + pipeline.onNext(SingleBlockRequest.PROTOBUF.toBytes(singleBlockRequest)); + verify(responseObserver, times(1)).onNext(SingleBlockResponseUnparsed.PROTOBUF.toBytes(expectedNotAvailable)); + } + + @Test + public void testSingleBlockIOExceptionPath() throws IOException, ParseException { + final PbjBlockAccessService blockAccessService = + new PbjBlockAccessServiceProxy(serviceStatus, blockReader, blockNodeContext); + + when(serviceStatus.isRunning()).thenReturn(true); + when(blockReader.read(1)).thenThrow(new IOException("Test exception")); + + final SingleBlockResponseUnparsed expectedNotAvailable = SingleBlockResponseUnparsed.newBuilder() + .status(SingleBlockResponseCode.READ_BLOCK_NOT_AVAILABLE) + .build(); + + // Build a request to invoke the service + final SingleBlockRequest singleBlockRequest = + SingleBlockRequest.newBuilder().blockNumber(1).build(); + + final Pipeline pipeline = + blockAccessService.open(PbjBlockAccessService.BlockAccessMethod.singleBlock, null, responseObserver); + + // Call the service + pipeline.onNext(SingleBlockRequest.PROTOBUF.toBytes(singleBlockRequest)); + verify(responseObserver, times(1)).onNext(SingleBlockResponseUnparsed.PROTOBUF.toBytes(expectedNotAvailable)); + } + + @Test + public void testSingleBlockParseExceptionPath() throws IOException, ParseException { + final PbjBlockAccessService blockAccessService = + new PbjBlockAccessServiceProxy(serviceStatus, blockReader, blockNodeContext); + + when(serviceStatus.isRunning()).thenReturn(true); + when(blockReader.read(1)).thenThrow(new ParseException("Test exception")); + + final SingleBlockResponseUnparsed expectedNotAvailable = SingleBlockResponseUnparsed.newBuilder() + .status(SingleBlockResponseCode.READ_BLOCK_NOT_AVAILABLE) + .build(); + + // Build a request to invoke the service + final SingleBlockRequest singleBlockRequest = + SingleBlockRequest.newBuilder().blockNumber(1).build(); + + final Pipeline pipeline = + blockAccessService.open(PbjBlockAccessService.BlockAccessMethod.singleBlock, null, responseObserver); + + // Call the service + pipeline.onNext(SingleBlockRequest.PROTOBUF.toBytes(singleBlockRequest)); + verify(responseObserver, times(1)).onNext(SingleBlockResponseUnparsed.PROTOBUF.toBytes(expectedNotAvailable)); + } } diff --git a/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java b/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java deleted file mode 100644 index 8f722f7eb..000000000 --- a/server/src/test/java/com/hedera/block/server/grpc/BlockStreamServiceTest.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2024 Hedera Hashgraph, LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.hedera.block.server.grpc; - -import static com.hedera.block.server.Constants.FULL_SERVICE_NAME_BLOCK_STREAM; -import static com.hedera.block.server.Constants.SERVICE_NAME_BLOCK_STREAM; -import static java.lang.System.Logger; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.hedera.block.server.config.BlockNodeContext; -import com.hedera.block.server.mediator.LiveStreamMediator; -import com.hedera.block.server.notifier.Notifier; -import com.hedera.block.server.pbj.PbjBlockStreamService; -import com.hedera.block.server.pbj.PbjBlockStreamServiceProxy; -import com.hedera.block.server.persistence.StreamPersistenceHandlerImpl; -import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; -import com.hedera.block.server.persistence.storage.read.BlockReader; -import com.hedera.block.server.persistence.storage.write.BlockWriter; -import com.hedera.block.server.service.ServiceStatus; -import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.hapi.block.SingleBlockResponse; -import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Flow; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -public class BlockStreamServiceTest { - - @Mock - private Notifier notifier; - - @Mock - private Flow.Subscriber responseObserver; - - @Mock - private LiveStreamMediator streamMediator; - - @Mock - private BlockReader blockReader; - - @Mock - private BlockWriter> blockWriter; - - @Mock - private ServiceStatus serviceStatus; - - private final Logger LOGGER = System.getLogger(getClass().getName()); - - private static final int testTimeout = 1000; - - private PbjBlockStreamService blockStreamService; - - @TempDir - private Path testPath; - - private BlockNodeContext blockNodeContext; - private PersistenceStorageConfig config; - - @BeforeEach - public void setUp() throws IOException { - blockNodeContext = - TestConfigUtil.getTestBlockNodeContext(Map.of("persistence.storage.rootPath", testPath.toString())); - config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); - - final var blockNodeEventHandler = new StreamPersistenceHandlerImpl( - streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - blockStreamService = new PbjBlockStreamServiceProxy( - streamMediator, serviceStatus, blockNodeEventHandler, notifier, blockNodeContext); - } - - @Test - public void testServiceName() { - assertEquals(SERVICE_NAME_BLOCK_STREAM, blockStreamService.serviceName()); - } - - @Test - public void testFullName() { - assertEquals(FULL_SERVICE_NAME_BLOCK_STREAM, blockStreamService.fullName()); - } - - @Test - public void testMethods() { - assertEquals(2, blockStreamService.methods().size()); - } -} diff --git a/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java b/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java index a2f86653a..940513fe1 100644 --- a/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java +++ b/server/src/test/java/com/hedera/block/server/mediator/LiveStreamMediatorImplTest.java @@ -18,7 +18,7 @@ import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockStreamMediatorError; -import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -39,11 +39,12 @@ import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.service.ServiceStatusImpl; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.hapi.block.BlockItemSet; -import com.hedera.hapi.block.SubscribeStreamResponse; +import com.hedera.hapi.block.BlockItemSetUnparsed; +import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.SubscribeStreamResponseCode; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.hedera.hapi.block.stream.output.BlockHeader; +import com.hedera.pbj.runtime.ParseException; import com.hedera.pbj.runtime.grpc.Pipeline; import com.swirlds.metrics.api.LongGauge; import java.io.IOException; @@ -61,28 +62,28 @@ public class LiveStreamMediatorImplTest { @Mock - private BlockNodeEventHandler> observer1; + private BlockNodeEventHandler> observer1; @Mock - private BlockNodeEventHandler> observer2; + private BlockNodeEventHandler> observer2; @Mock - private BlockNodeEventHandler> observer3; + private BlockNodeEventHandler> observer3; @Mock - private BlockWriter> blockWriter; + private BlockWriter> blockWriter; @Mock private Notifier notifier; @Mock - private Pipeline streamObserver1; + private Pipeline helidonSubscribeStreamObserver1; @Mock - private Pipeline streamObserver2; + private Pipeline helidonSubscribeStreamObserver2; @Mock - private Pipeline streamObserver3; + private Pipeline helidonSubscribeStreamObserver3; @Mock private InstantSource testClock; @@ -135,19 +136,20 @@ public void testUnsubscribeEach() throws InterruptedException, IOException { } @Test - public void testMediatorPersistenceWithoutSubscribers() throws IOException { + public void testMediatorPersistenceWithoutSubscribers() throws IOException, ParseException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); final var streamMediator = LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus) .build(); - final BlockItem blockItem = BlockItem.newBuilder().build(); + + final BlockItemUnparsed blockItem = BlockItemUnparsed.newBuilder().build(); // register the stream validator when(blockWriter.write(List.of(blockItem))).thenReturn(Optional.empty()); - final var streamValidator = new StreamPersistenceHandlerImpl( + final var handler = new StreamPersistenceHandlerImpl( streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - streamMediator.subscribe(streamValidator); + streamMediator.subscribe(handler); // Acting as a producer, notify the mediator of a new block streamMediator.publish(List.of(blockItem)); @@ -161,7 +163,7 @@ public void testMediatorPersistenceWithoutSubscribers() throws IOException { } @Test - public void testMediatorPublishEventToSubscribers() throws IOException { + public void testMediatorPublishEventToSubscribers() throws IOException, ParseException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); @@ -170,14 +172,14 @@ public void testMediatorPublishEventToSubscribers() throws IOException { when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); - final var concreteObserver1 = - new ConsumerStreamResponseObserver(testClock, streamMediator, streamObserver1, testContext); + final var concreteObserver1 = new ConsumerStreamResponseObserver( + testClock, streamMediator, helidonSubscribeStreamObserver1, testContext); - final var concreteObserver2 = - new ConsumerStreamResponseObserver(testClock, streamMediator, streamObserver2, testContext); + final var concreteObserver2 = new ConsumerStreamResponseObserver( + testClock, streamMediator, helidonSubscribeStreamObserver2, testContext); - final var concreteObserver3 = - new ConsumerStreamResponseObserver(testClock, streamMediator, streamObserver3, testContext); + final var concreteObserver3 = new ConsumerStreamResponseObserver( + testClock, streamMediator, helidonSubscribeStreamObserver3, testContext); // Set up the subscribers streamMediator.subscribe(concreteObserver1); @@ -191,19 +193,20 @@ public void testMediatorPublishEventToSubscribers() throws IOException { assertTrue( streamMediator.isSubscribed(concreteObserver3), "Expected the mediator to have observer3 subscribed"); - final BlockHeader blockHeader = BlockHeader.newBuilder().number(1).build(); - final BlockItem blockItem = - BlockItem.newBuilder().blockHeader(blockHeader).build(); - final BlockItemSet blockItemSet = - BlockItemSet.newBuilder().blockItems(blockItem).build(); - final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItems(blockItemSet).build(); + final BlockItemUnparsed blockItem = BlockItemUnparsed.newBuilder() + .blockHeader(BlockHeader.PROTOBUF.toBytes( + BlockHeader.newBuilder().number(1).build())) + .build(); + final SubscribeStreamResponseUnparsed subscribeStreamResponse = SubscribeStreamResponseUnparsed.newBuilder() + .blockItems( + BlockItemSetUnparsed.newBuilder().blockItems(blockItem).build()) + .build(); // register the stream validator when(blockWriter.write(List.of(blockItem))).thenReturn(Optional.empty()); - final var streamValidator = new StreamPersistenceHandlerImpl( + final var handler = new StreamPersistenceHandlerImpl( streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - streamMediator.subscribe(streamValidator); + streamMediator.subscribe(handler); // Acting as a producer, notify the mediator of a new block streamMediator.publish(List.of(blockItem)); @@ -211,9 +214,9 @@ public void testMediatorPublishEventToSubscribers() throws IOException { assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); // Confirm each subscriber was notified of the new block - verify(streamObserver1, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); - verify(streamObserver2, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); - verify(streamObserver3, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); + verify(helidonSubscribeStreamObserver1, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); + verify(helidonSubscribeStreamObserver2, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); + verify(helidonSubscribeStreamObserver3, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); // Confirm the BlockStorage write method was called verify(blockWriter, timeout(testTimeout).times(1)).write(List.of(blockItem)); @@ -229,14 +232,14 @@ public void testSubAndUnsubHandling() throws IOException { when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); - final var concreteObserver1 = - new ConsumerStreamResponseObserver(testClock, streamMediator, streamObserver1, testContext); + final var concreteObserver1 = new ConsumerStreamResponseObserver( + testClock, streamMediator, helidonSubscribeStreamObserver1, testContext); - final var concreteObserver2 = - new ConsumerStreamResponseObserver(testClock, streamMediator, streamObserver2, testContext); + final var concreteObserver2 = new ConsumerStreamResponseObserver( + testClock, streamMediator, helidonSubscribeStreamObserver2, testContext); - final var concreteObserver3 = - new ConsumerStreamResponseObserver(testClock, streamMediator, streamObserver3, testContext); + final var concreteObserver3 = new ConsumerStreamResponseObserver( + testClock, streamMediator, helidonSubscribeStreamObserver3, testContext); // Set up the subscribers streamMediator.subscribe(concreteObserver1); @@ -259,8 +262,8 @@ public void testSubscribeWhenHandlerAlreadySubscribed() throws IOException { final var streamMediator = LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus) .build(); - final var concreteObserver1 = - new ConsumerStreamResponseObserver(testClock, streamMediator, streamObserver1, testContext); + final var concreteObserver1 = new ConsumerStreamResponseObserver( + testClock, streamMediator, helidonSubscribeStreamObserver1, testContext); streamMediator.subscribe(concreteObserver1); assertTrue(streamMediator.isSubscribed(concreteObserver1)); @@ -279,57 +282,56 @@ public void testSubscribeWhenHandlerAlreadySubscribed() throws IOException { assertEquals(0L, consumersGauge.get()); } - // @Test - // public void testOnCancelSubscriptionHandling() throws IOException { - // - // final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); - // final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); - // final var streamMediator = - // LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); + // @Test + // public void testOnCancelSubscriptionHandling() throws IOException { // - // when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); + // final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + // final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); + // final var streamMediator = + // LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus).build(); // - // final List blockItems = generateBlockItems(1); + // when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); // - // // register the stream validator - // when(blockWriter.write(blockItems.getFirst())).thenReturn(Optional.empty()); - // final var streamValidator = - // new StreamPersistenceHandlerImpl( - // streamMediator, notifier, blockWriter, blockNodeContext, - // serviceStatus); - // streamMediator.subscribe(streamValidator); + // final List blockItems = generateBlockItemsUnparsed(1); // - // // register the test observer - // final var testConsumerBlockItemObserver = - // new TestConsumerStreamResponseObserver( - // testClock, streamMediator, serverCallStreamObserver, testContext); + // // register the stream validator + // when(blockWriter.write(List.of(blockItems.getFirst()))).thenReturn(Optional.empty()); + // final var streamValidator = + // new StreamPersistenceHandlerImpl( + // streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + // streamMediator.subscribe(streamValidator); // - // streamMediator.subscribe(testConsumerBlockItemObserver); - // assertTrue(streamMediator.isSubscribed(testConsumerBlockItemObserver)); + // // register the test observer + // final var testConsumerBlockItemObserver = + // new TestConsumerStreamResponseObserver( + // testClock, streamMediator, serverCallStreamObserver, testContext); // - // // Simulate the producer notifying the mediator of a new block - // streamMediator.publish(blockItems.getFirst()); + // streamMediator.subscribe(testConsumerBlockItemObserver); + // assertTrue(streamMediator.isSubscribed(testConsumerBlockItemObserver)); // - // // Simulate the consumer cancelling the stream - // testConsumerBlockItemObserver.getOnCancel().run(); + // // Simulate the producer notifying the mediator of a new block + // streamMediator.publish(blockItems.getFirst()); // - // // Verify the block item incremented the counter - // assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); + // // Simulate the consumer cancelling the stream + // testConsumerBlockItemObserver.getOnCancel().run(); // - // // Verify the event made it to the consumer - // verify(serverCallStreamObserver, - // timeout(testTimeout).times(1)).setOnCancelHandler(any()); + // // Verify the block item incremented the counter + // assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); // - // // Confirm the mediator unsubscribed the consumer - // assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); + // // Verify the event made it to the consumer + // verify(serverCallStreamObserver, + // timeout(testTimeout).times(1)).setOnCancelHandler(any()); // - // // Confirm the BlockStorage write method was called - // verify(blockWriter, timeout(testTimeout).times(1)).write(blockItems.getFirst()); + // // Confirm the mediator unsubscribed the consumer + // assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); // - // // Confirm the stream validator is still subscribed - // assertTrue(streamMediator.isSubscribed(streamValidator)); - // } + // // Confirm the BlockStorage write method was called + // verify(blockWriter, timeout(testTimeout).times(1)).write(blockItems.getFirst()); // + // // Confirm the stream validator is still subscribed + // assertTrue(streamMediator.isSubscribed(streamValidator)); + // } + // @Test // public void testOnCloseSubscriptionHandling() throws IOException { // @@ -381,23 +383,23 @@ public void testSubscribeWhenHandlerAlreadySubscribed() throws IOException { // // Confirm the stream validator is still subscribed // assertTrue(streamMediator.isSubscribed(streamValidator)); // } - // + @Test - public void testMediatorBlocksPublishAfterException() throws IOException, InterruptedException { + public void testMediatorBlocksPublishAfterException() throws IOException, InterruptedException, ParseException { final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); final var streamMediator = LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus) .build(); - final var concreteObserver1 = - new ConsumerStreamResponseObserver(testClock, streamMediator, streamObserver1, testContext); + final var concreteObserver1 = new ConsumerStreamResponseObserver( + testClock, streamMediator, helidonSubscribeStreamObserver1, testContext); - final var concreteObserver2 = - new ConsumerStreamResponseObserver(testClock, streamMediator, streamObserver2, testContext); + final var concreteObserver2 = new ConsumerStreamResponseObserver( + testClock, streamMediator, helidonSubscribeStreamObserver2, testContext); - final var concreteObserver3 = - new ConsumerStreamResponseObserver(testClock, streamMediator, streamObserver3, testContext); + final var concreteObserver3 = new ConsumerStreamResponseObserver( + testClock, streamMediator, helidonSubscribeStreamObserver3, testContext); // Set up the subscribers streamMediator.subscribe(concreteObserver1); @@ -405,14 +407,14 @@ public void testMediatorBlocksPublishAfterException() throws IOException, Interr streamMediator.subscribe(concreteObserver3); final Notifier notifier = new NotifierImpl(streamMediator, blockNodeContext, serviceStatus); - final var streamValidator = new StreamPersistenceHandlerImpl( + final var handler = new StreamPersistenceHandlerImpl( streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); // Set up the stream verifier - streamMediator.subscribe(streamValidator); + streamMediator.subscribe(handler); - final List blockItems = generateBlockItems(1); - final BlockItem firstBlockItem = blockItems.getFirst(); + final List blockItems = generateBlockItemsUnparsed(1); + final BlockItemUnparsed firstBlockItem = blockItems.getFirst(); // Right now, only a single producer calls publishEvent. In // that case, they will get an IOException bubbled up to them. @@ -438,72 +440,64 @@ public void testMediatorBlocksPublishAfterException() throws IOException, Interr // Send another block item after the exception streamMediator.publish(List.of(blockItems.get(1))); - final BlockItemSet blockItemSet = - BlockItemSet.newBuilder().blockItems(firstBlockItem).build(); - final var subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItems(blockItemSet).build(); - verify(streamObserver1, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); - verify(streamObserver2, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); - verify(streamObserver3, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(firstBlockItem).build(); + final var subscribeStreamResponse = SubscribeStreamResponseUnparsed.newBuilder() + .blockItems(blockItemSet) + .build(); + verify(helidonSubscribeStreamObserver1, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); + verify(helidonSubscribeStreamObserver2, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); + verify(helidonSubscribeStreamObserver3, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); // TODO: Replace READ_STREAM_SUCCESS (2) with a generic error code? - final SubscribeStreamResponse endOfStreamResponse = SubscribeStreamResponse.newBuilder() + final var endOfStreamResponse = SubscribeStreamResponseUnparsed.newBuilder() .status(SubscribeStreamResponseCode.READ_STREAM_SUCCESS) .build(); - verify(streamObserver1, timeout(testTimeout).times(1)).onNext(endOfStreamResponse); - verify(streamObserver2, timeout(testTimeout).times(1)).onNext(endOfStreamResponse); - verify(streamObserver3, timeout(testTimeout).times(1)).onNext(endOfStreamResponse); + verify(helidonSubscribeStreamObserver1, timeout(testTimeout).times(1)).onNext(endOfStreamResponse); + verify(helidonSubscribeStreamObserver2, timeout(testTimeout).times(1)).onNext(endOfStreamResponse); + verify(helidonSubscribeStreamObserver3, timeout(testTimeout).times(1)).onNext(endOfStreamResponse); // verify write method only called once despite the second block being published. verify(blockWriter, timeout(testTimeout).times(1)).write(List.of(firstBlockItem)); } - // @Test - // public void testUnsubscribeWhenNotSubscribed() throws IOException { - // - // final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); - // final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); - // final var streamMediator = LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus) - // .build(); - // - // // register the stream validator - // final var streamValidator = new StreamPersistenceHandlerImpl( - // streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - // streamMediator.subscribe(streamValidator); - // - // final var testConsumerBlockItemObserver = - // new TestConsumerStreamResponseObserver(testClock, streamMediator, streamObserver1, testContext); - // - // // Confirm the observer is not subscribed - // assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); - // - // // Attempt to unsubscribe the observer - // streamMediator.unsubscribe(testConsumerBlockItemObserver); - // - // // Confirm the observer is still not subscribed - // assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); - // - // // Confirm the stream validator is still subscribed - // assertTrue(streamMediator.isSubscribed(streamValidator)); - // } + @Test + public void testUnsubscribeWhenNotSubscribed() throws IOException { - // private static class TestConsumerStreamResponseObserver extends ConsumerStreamResponseObserver { - // public TestConsumerStreamResponseObserver( - // @NonNull final InstantSource producerLivenessClock, - // @NonNull final StreamMediator, SubscribeStreamResponse> streamMediator, - // @NonNull final Pipeline responseStreamObserver, - // @NonNull final BlockNodeContext blockNodeContext) { - // super(producerLivenessClock, streamMediator, responseStreamObserver, blockNodeContext); - // } - // - // @NonNull - // public Runnable getOnCancel() { - // return onCancel; - // } - // - // @NonNull - // public Runnable getOnClose() { - // return onClose; + final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); + final var streamMediator = LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus) + .build(); + + // register the stream validator + final var handler = new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + streamMediator.subscribe(handler); + + final var testConsumerBlockItemObserver = new ConsumerStreamResponseObserver( + testClock, streamMediator, helidonSubscribeStreamObserver1, testContext); + + // Confirm the observer is not subscribed + assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); + + // Attempt to unsubscribe the observer + streamMediator.unsubscribe(testConsumerBlockItemObserver); + + // Confirm the observer is still not subscribed + assertFalse(streamMediator.isSubscribed(testConsumerBlockItemObserver)); + + // Confirm the stream validator is still subscribed + assertTrue(streamMediator.isSubscribed(handler)); + } + + // private static class TestConsumerStreamResponseObserver extends ConsumerStreamResponseObserver { + // public TestConsumerStreamResponseObserver( + // @NonNull final InstantSource producerLivenessClock, + // @NonNull final StreamMediator, SubscribeStreamResponseUnparsed> + // streamMediator, + // @NonNull final Pipeline responseStreamObserver, + // @NonNull final BlockNodeContext blockNodeContext) { + // super(producerLivenessClock, streamMediator, responseStreamObserver, blockNodeContext); + // } // } - // } } diff --git a/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java b/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java index 65baf1b8a..9178cb590 100644 --- a/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java +++ b/server/src/test/java/com/hedera/block/server/mediator/MediatorInjectionModuleTest.java @@ -16,15 +16,17 @@ package com.hedera.block.server.mediator; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import java.io.IOException; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -34,7 +36,8 @@ @ExtendWith(MockitoExtension.class) class MediatorInjectionModuleTest { - @Mock private ServiceStatus serviceStatus; + @Mock + private ServiceStatus serviceStatus; @BeforeEach void setup() { @@ -47,10 +50,26 @@ void testProvidesStreamMediator() throws IOException { BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); // Call the method under test - StreamMediator, SubscribeStreamResponse> streamMediator = + StreamMediator, SubscribeStreamResponseUnparsed> streamMediator = MediatorInjectionModule.providesLiveStreamMediator(blockNodeContext, serviceStatus); // Verify that the streamMediator is correctly instantiated assertNotNull(streamMediator); + assertInstanceOf(LiveStreamMediatorImpl.class, streamMediator); + } + + @Test + void testNoOpProvidesStreamMediator() throws IOException { + + Map properties = Map.of("mediator.type", "NOOP"); + BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(properties); + + // Call the method under test + StreamMediator, SubscribeStreamResponseUnparsed> streamMediator = + MediatorInjectionModule.providesLiveStreamMediator(blockNodeContext, serviceStatus); + + // Verify that the streamMediator is correctly instantiated + assertNotNull(streamMediator); + assertInstanceOf(NoOpLiveStreamMediator.class, streamMediator); } } diff --git a/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java b/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java index f57bf01d2..c8b324cfb 100644 --- a/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java +++ b/server/src/test/java/com/hedera/block/server/notifier/NotifierImplTest.java @@ -16,34 +16,54 @@ package com.hedera.block.server.notifier; +import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Gauge.Producers; import static com.hedera.block.server.notifier.NotifierImpl.buildErrorStreamResponse; -import static com.hedera.block.server.pbj.PbjBlockStreamServiceIntegrationTest.buildAck; -import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; -import static org.junit.jupiter.api.Assertions.assertFalse; +import static com.hedera.block.server.util.PbjProtoTestUtils.buildAck; +import static com.hedera.block.server.util.PbjProtoTestUtils.buildEmptyPublishStreamRequest; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; +import static java.lang.System.Logger.Level.INFO; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.hedera.block.server.config.BlockNodeContext; +import com.hedera.block.server.events.BlockNodeEventHandler; +import com.hedera.block.server.events.ObjectEvent; +import com.hedera.block.server.mediator.LiveStreamMediator; +import com.hedera.block.server.mediator.LiveStreamMediatorBuilder; import com.hedera.block.server.mediator.Publisher; import com.hedera.block.server.mediator.SubscriptionHandler; +import com.hedera.block.server.pbj.PbjBlockStreamService; +import com.hedera.block.server.pbj.PbjBlockStreamServiceProxy; +import com.hedera.block.server.persistence.StreamPersistenceHandlerImpl; +import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.producer.ProducerBlockItemObserver; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.service.ServiceStatusImpl; import com.hedera.block.server.util.TestConfigUtil; +import com.hedera.block.server.util.TestUtils; import com.hedera.hapi.block.Acknowledgement; +import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.PublishStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.hedera.pbj.runtime.grpc.Pipeline; +import com.hedera.pbj.runtime.grpc.ServiceInterface; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import com.lmax.disruptor.BatchEventProcessor; import edu.umd.cs.findbugs.annotations.NonNull; +import io.helidon.webserver.WebServer; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.NoSuchAlgorithmException; import java.time.InstantSource; -import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -52,11 +72,13 @@ @ExtendWith(MockitoExtension.class) public class NotifierImplTest { + private final System.Logger LOGGER = System.getLogger(getClass().getName()); + @Mock private Notifiable mediator; @Mock - private Publisher> publisher; + private Publisher> publisher; @Mock private ServiceStatus serviceStatus; @@ -65,134 +87,117 @@ public class NotifierImplTest { private SubscriptionHandler subscriptionHandler; @Mock - private Pipeline streamObserver1; + private Pipeline helidonPublishStreamObserver1; @Mock - private Pipeline streamObserver2; + private Pipeline helidonPublishStreamObserver2; @Mock - private Pipeline streamObserver3; + private Pipeline helidonPublishStreamObserver3; @Mock - private InstantSource testClock; - - private final long TIMEOUT_THRESHOLD_MILLIS = 100L; - private final long TEST_TIME = 1_719_427_664_950L; + private Pipeline publishStreamObserver1; - private static final int testTimeout = 1000; - - private final BlockNodeContext testContext; + @Mock + private Pipeline publishStreamObserver2; - public NotifierImplTest() throws IOException { - Map properties = new HashMap<>(); - properties.put(TestConfigUtil.MEDIATOR_RING_BUFFER_SIZE_KEY, String.valueOf(1024)); - this.testContext = TestConfigUtil.getTestBlockNodeContext(properties); - } + @Mock + private Pipeline publishStreamObserver3; - @Test - public void testRegistration() throws NoSuchAlgorithmException { + @Mock + private InstantSource testClock; - final ServiceStatus serviceStatus = new ServiceStatusImpl(testContext); - final var notifier = new NotifierImpl(mediator, testContext, serviceStatus); + @Mock + private WebServer webServer; - when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); + @Mock + private BlockWriter> blockWriter; - final var concreteObserver1 = new ProducerBlockItemObserver( - testClock, publisher, subscriptionHandler, streamObserver1, testContext, serviceStatus); + @Mock + private ServiceInterface.RequestOptions options; - final var concreteObserver2 = new ProducerBlockItemObserver( - testClock, publisher, subscriptionHandler, streamObserver2, testContext, serviceStatus); + private final long TIMEOUT_THRESHOLD_MILLIS = 100L; + private final long TEST_TIME = 1_719_427_664_950L; - final var concreteObserver3 = new ProducerBlockItemObserver( - testClock, publisher, subscriptionHandler, streamObserver3, testContext, serviceStatus); + private static final int testTimeout = 1000; - notifier.subscribe(concreteObserver1); - notifier.subscribe(concreteObserver2); - notifier.subscribe(concreteObserver3); + private Path testPath; + private BlockNodeContext blockNodeContext; - assertTrue(notifier.isSubscribed(concreteObserver1), "Expected the notifier to have observer1 subscribed"); - assertTrue(notifier.isSubscribed(concreteObserver2), "Expected the notifier to have observer2 subscribed"); - assertTrue(notifier.isSubscribed(concreteObserver3), "Expected the notifier to have observer3 subscribed"); + private static final String TEMP_DIR = "notifier-unit-test-dir"; - List blockItems = generateBlockItems(1); - notifier.publish(blockItems); + @BeforeEach + public void setUp() throws IOException { + testPath = Files.createTempDirectory(TEMP_DIR); + LOGGER.log(INFO, "Created temp directory: " + testPath.toString()); - // Verify the response was received by all observers - final var publishStreamResponse = PublishStreamResponse.newBuilder() - .acknowledgement(buildAck(blockItems)) - .build(); - verify(streamObserver1, timeout(testTimeout).times(1)).onNext(publishStreamResponse); - verify(streamObserver2, timeout(testTimeout).times(1)).onNext(publishStreamResponse); - verify(streamObserver3, timeout(testTimeout).times(1)).onNext(publishStreamResponse); - - // Unsubscribe the observers - notifier.unsubscribe(concreteObserver1); - assertFalse(notifier.isSubscribed(concreteObserver1), "Expected the notifier to have unsubscribed observer1"); - - notifier.unsubscribe(concreteObserver2); - assertFalse(notifier.isSubscribed(concreteObserver2), "Expected the notifier to have unsubscribed observer2"); + blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); + } - notifier.unsubscribe(concreteObserver3); - assertFalse(notifier.isSubscribed(concreteObserver3), "Expected the notifier to have unsubscribed observer3"); + @AfterEach + public void tearDown() { + TestUtils.deleteDirectory(testPath.toFile()); } @Test - public void testTimeoutExpiredHandling() throws InterruptedException { + public void testRegistration() throws NoSuchAlgorithmException { + // when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS); when(serviceStatus.isRunning()).thenReturn(true); - final var notifier = new NotifierImpl(mediator, testContext, serviceStatus); - - // Set the clocks to be expired - final InstantSource testClock1 = mock(InstantSource.class); - when(testClock1.millis()).thenReturn(TEST_TIME, TEST_TIME + 1501L); - - final InstantSource testClock2 = mock(InstantSource.class); - when(testClock2.millis()).thenReturn(TEST_TIME, TEST_TIME + 1501L); + final var notifier = new NotifierImpl(mediator, blockNodeContext, serviceStatus); + final PbjBlockStreamServiceProxy pbjBlockStreamServiceProxy = buildBlockStreamService(notifier); - final InstantSource testClock3 = mock(InstantSource.class); - when(testClock3.millis()).thenReturn(TEST_TIME, TEST_TIME + 1501L); + // Register 3 producers - Opening a pipeline is not enough to register a producer. + // pipeline.onNext() must be invoked to register the producer at the Helidon PBJ layer. + final Pipeline producerPipeline1 = pbjBlockStreamServiceProxy.open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, options, helidonPublishStreamObserver1); + producerPipeline1.onNext(buildEmptyPublishStreamRequest()); - final var concreteObserver1 = new ProducerBlockItemObserver( - testClock1, publisher, notifier, streamObserver1, testContext, serviceStatus); - - final var concreteObserver2 = new ProducerBlockItemObserver( - testClock2, publisher, notifier, streamObserver2, testContext, serviceStatus); - - final var concreteObserver3 = new ProducerBlockItemObserver( - testClock3, publisher, notifier, streamObserver3, testContext, serviceStatus); + final Pipeline producerPipeline2 = pbjBlockStreamServiceProxy.open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, options, helidonPublishStreamObserver2); + producerPipeline2.onNext(buildEmptyPublishStreamRequest()); - notifier.subscribe(concreteObserver1); - notifier.subscribe(concreteObserver2); - notifier.subscribe(concreteObserver3); + final Pipeline producerPipeline3 = pbjBlockStreamServiceProxy.open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, options, helidonPublishStreamObserver3); + producerPipeline3.onNext(buildEmptyPublishStreamRequest()); - assertTrue(notifier.isSubscribed(concreteObserver1), "Expected the notifier to have observer1 subscribed"); - assertTrue(notifier.isSubscribed(concreteObserver2), "Expected the notifier to have observer2 subscribed"); - assertTrue(notifier.isSubscribed(concreteObserver3), "Expected the notifier to have observer3 subscribed"); + long producers = blockNodeContext.metricsService().get(Producers).get(); + assertEquals(3, producers, "Expected 3 producers to be registered"); - List blockItems = generateBlockItems(1); + List blockItems = generateBlockItemsUnparsed(1); notifier.publish(blockItems); - Thread.sleep(testTimeout); + // Verify the response was received by all observers + final Bytes publishStreamResponse = PublishStreamResponse.PROTOBUF.toBytes(PublishStreamResponse.newBuilder() + .acknowledgement(buildAck(blockItems)) + .build()); + verify(helidonPublishStreamObserver1, timeout(testTimeout).times(1)).onNext(publishStreamResponse); + verify(helidonPublishStreamObserver2, timeout(testTimeout).times(1)).onNext(publishStreamResponse); + verify(helidonPublishStreamObserver3, timeout(testTimeout).times(1)).onNext(publishStreamResponse); + + // Unsubscribe the observers + producerPipeline1.onComplete(); + producerPipeline2.onComplete(); + producerPipeline3.onComplete(); - assertFalse(notifier.isSubscribed(concreteObserver1), "Expected the notifier to have observer1 unsubscribed"); - assertFalse(notifier.isSubscribed(concreteObserver2), "Expected the notifier to have observer2 unsubscribed"); - assertFalse(notifier.isSubscribed(concreteObserver3), "Expected the notifier to have observer3 unsubscribed"); + producers = blockNodeContext.metricsService().get(Producers).get(); + assertEquals(0, producers, "Expected 0 producers to be registered"); } @Test public void testPublishThrowsNoSuchAlgorithmException() { when(serviceStatus.isRunning()).thenReturn(true); - final var notifier = new TestNotifier(mediator, testContext, serviceStatus); + final var notifier = new TestNotifier(mediator, blockNodeContext, serviceStatus); final var concreteObserver1 = new ProducerBlockItemObserver( - testClock, publisher, subscriptionHandler, streamObserver1, testContext, serviceStatus); + testClock, publisher, subscriptionHandler, publishStreamObserver1, blockNodeContext, serviceStatus); final var concreteObserver2 = new ProducerBlockItemObserver( - testClock, publisher, subscriptionHandler, streamObserver2, testContext, serviceStatus); + testClock, publisher, subscriptionHandler, publishStreamObserver2, blockNodeContext, serviceStatus); final var concreteObserver3 = new ProducerBlockItemObserver( - testClock, publisher, subscriptionHandler, streamObserver3, testContext, serviceStatus); + testClock, publisher, subscriptionHandler, publishStreamObserver3, blockNodeContext, serviceStatus); notifier.subscribe(concreteObserver1); notifier.subscribe(concreteObserver2); @@ -202,13 +207,13 @@ public void testPublishThrowsNoSuchAlgorithmException() { assertTrue(notifier.isSubscribed(concreteObserver2), "Expected the notifier to have observer2 subscribed"); assertTrue(notifier.isSubscribed(concreteObserver3), "Expected the notifier to have observer3 subscribed"); - List blockItems = generateBlockItems(1); + List blockItems = generateBlockItemsUnparsed(1); notifier.publish(blockItems); final PublishStreamResponse errorResponse = buildErrorStreamResponse(); - verify(streamObserver1, timeout(testTimeout).times(1)).onNext(errorResponse); - verify(streamObserver2, timeout(testTimeout).times(1)).onNext(errorResponse); - verify(streamObserver3, timeout(testTimeout).times(1)).onNext(errorResponse); + verify(publishStreamObserver1, timeout(testTimeout).times(1)).onNext(errorResponse); + verify(publishStreamObserver2, timeout(testTimeout).times(1)).onNext(errorResponse); + verify(publishStreamObserver3, timeout(testTimeout).times(1)).onNext(errorResponse); } @Test @@ -216,15 +221,15 @@ public void testServiceStatusNotRunning() throws NoSuchAlgorithmException { // Set the serviceStatus to not running when(serviceStatus.isRunning()).thenReturn(false); - final var notifier = new TestNotifier(mediator, testContext, serviceStatus); + final var notifier = new TestNotifier(mediator, blockNodeContext, serviceStatus); final var concreteObserver1 = new ProducerBlockItemObserver( - testClock, publisher, subscriptionHandler, streamObserver1, testContext, serviceStatus); + testClock, publisher, subscriptionHandler, publishStreamObserver1, blockNodeContext, serviceStatus); final var concreteObserver2 = new ProducerBlockItemObserver( - testClock, publisher, subscriptionHandler, streamObserver2, testContext, serviceStatus); + testClock, publisher, subscriptionHandler, publishStreamObserver2, blockNodeContext, serviceStatus); final var concreteObserver3 = new ProducerBlockItemObserver( - testClock, publisher, subscriptionHandler, streamObserver3, testContext, serviceStatus); + testClock, publisher, subscriptionHandler, publishStreamObserver3, blockNodeContext, serviceStatus); notifier.subscribe(concreteObserver1); notifier.subscribe(concreteObserver2); @@ -234,16 +239,16 @@ public void testServiceStatusNotRunning() throws NoSuchAlgorithmException { assertTrue(notifier.isSubscribed(concreteObserver2), "Expected the notifier to have observer2 subscribed"); assertTrue(notifier.isSubscribed(concreteObserver3), "Expected the notifier to have observer3 subscribed"); - final List blockItems = generateBlockItems(1); + final List blockItems = generateBlockItemsUnparsed(1); notifier.publish(blockItems); // Verify once the serviceStatus is not running that we do not publish the responses final var publishStreamResponse = PublishStreamResponse.newBuilder() .acknowledgement(buildAck(blockItems)) .build(); - verify(streamObserver1, timeout(testTimeout).times(0)).onNext(publishStreamResponse); - verify(streamObserver2, timeout(testTimeout).times(0)).onNext(publishStreamResponse); - verify(streamObserver3, timeout(testTimeout).times(0)).onNext(publishStreamResponse); + verify(publishStreamObserver1, timeout(testTimeout).times(0)).onNext(publishStreamResponse); + verify(publishStreamObserver2, timeout(testTimeout).times(0)).onNext(publishStreamResponse); + verify(publishStreamObserver3, timeout(testTimeout).times(0)).onNext(publishStreamResponse); } private static final class TestNotifier extends NotifierImpl { @@ -256,8 +261,33 @@ public TestNotifier( @Override @NonNull - Acknowledgement buildAck(@NonNull final List blockItems) throws NoSuchAlgorithmException { + Acknowledgement buildAck(@NonNull final List blockItems) throws NoSuchAlgorithmException { throw new NoSuchAlgorithmException("Test exception"); } } + + private PbjBlockStreamServiceProxy buildBlockStreamService(final Notifier notifier) { + + final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); + final var streamMediator = buildStreamMediator(new ConcurrentHashMap<>(32), serviceStatus); + final var blockNodeEventHandler = new StreamPersistenceHandlerImpl( + streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); + + return new PbjBlockStreamServiceProxy( + streamMediator, serviceStatus, blockNodeEventHandler, notifier, blockNodeContext); + } + + private LiveStreamMediator buildStreamMediator( + final Map< + BlockNodeEventHandler>, + BatchEventProcessor>> + subscribers, + final ServiceStatus serviceStatus) { + + serviceStatus.setWebServer(webServer); + + return LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus) + .subscribers(subscribers) + .build(); + } } diff --git a/server/src/test/java/com/hedera/block/server/pbj/PbjBlockAccessServiceProxyTest.java b/server/src/test/java/com/hedera/block/server/pbj/PbjBlockAccessServiceProxyTest.java index 9ede231a2..344e86b1a 100644 --- a/server/src/test/java/com/hedera/block/server/pbj/PbjBlockAccessServiceProxyTest.java +++ b/server/src/test/java/com/hedera/block/server/pbj/PbjBlockAccessServiceProxyTest.java @@ -26,11 +26,12 @@ import com.hedera.block.server.persistence.storage.read.BlockReader; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; import com.hedera.hapi.block.SingleBlockRequest; import com.hedera.hapi.block.SingleBlockResponse; import com.hedera.hapi.block.SingleBlockResponseCode; -import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.SingleBlockResponseUnparsed; import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.pbj.runtime.ParseException; import com.hedera.pbj.runtime.grpc.Pipeline; @@ -54,7 +55,7 @@ public class PbjBlockAccessServiceProxyTest { private ServiceStatus serviceStatus; @Mock - private BlockReader blockReader; + private BlockReader blockReader; @Mock private ServiceInterface.RequestOptions options; @@ -94,24 +95,25 @@ public void testSingleBlock() throws IOException, ParseException { when(serviceStatus.isRunning()).thenReturn(true); - final Block block = Block.newBuilder() - .items(BlockItem.newBuilder() - .blockHeader(BlockHeader.newBuilder().number(1).build()) - .build()) + final var blockItems = BlockItemUnparsed.newBuilder() + .blockHeader(BlockHeader.PROTOBUF.toBytes( + BlockHeader.newBuilder().number(1).build())) .build(); + final BlockUnparsed block = + BlockUnparsed.newBuilder().blockItems(blockItems).build(); when(blockReader.read(1)).thenReturn(Optional.of(block)); final SingleBlockRequest singleBlockRequest = SingleBlockRequest.newBuilder().blockNumber(1).build(); pipeline.onNext(SingleBlockRequest.PROTOBUF.toBytes(singleBlockRequest)); - final var readSuccessResponse = SingleBlockResponse.newBuilder() + final var readSuccessResponse = SingleBlockResponseUnparsed.newBuilder() .status(SingleBlockResponseCode.READ_BLOCK_SUCCESS) .block(block) .build(); verify(replies, timeout(testTimeout).times(1)).onSubscribe(any()); verify(replies, timeout(testTimeout).times(1)) - .onNext(SingleBlockResponse.PROTOBUF.toBytes(readSuccessResponse)); + .onNext(SingleBlockResponseUnparsed.PROTOBUF.toBytes(readSuccessResponse)); verify(replies, timeout(testTimeout).times(1)).onComplete(); } diff --git a/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java b/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java index 56c4eb2b5..13feee431 100644 --- a/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java +++ b/server/src/test/java/com/hedera/block/server/pbj/PbjBlockStreamServiceIntegrationTest.java @@ -16,9 +16,10 @@ package com.hedera.block.server.pbj; -import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.LiveBlockItems; -import static com.hedera.block.server.producer.Util.getFakeHash; -import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; +import static com.hedera.block.server.util.PbjProtoTestUtils.buildAck; +import static com.hedera.block.server.util.PbjProtoTestUtils.buildEmptyPublishStreamRequest; +import static com.hedera.block.server.util.PbjProtoTestUtils.buildEmptySubscribeStreamRequest; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.INFO; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -46,24 +47,23 @@ import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; import com.hedera.hapi.block.Acknowledgement; -import com.hedera.hapi.block.BlockItemSet; +import com.hedera.hapi.block.BlockItemSetUnparsed; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; import com.hedera.hapi.block.EndOfStream; -import com.hedera.hapi.block.ItemAcknowledgement; -import com.hedera.hapi.block.PublishStreamRequest; +import com.hedera.hapi.block.PublishStreamRequestUnparsed; import com.hedera.hapi.block.PublishStreamResponse; import com.hedera.hapi.block.PublishStreamResponseCode; import com.hedera.hapi.block.SingleBlockRequest; -import com.hedera.hapi.block.SingleBlockResponse; import com.hedera.hapi.block.SingleBlockResponseCode; -import com.hedera.hapi.block.SubscribeStreamRequest; -import com.hedera.hapi.block.SubscribeStreamResponse; +import com.hedera.hapi.block.SingleBlockResponseUnparsed; import com.hedera.hapi.block.SubscribeStreamResponseCode; -import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; +import com.hedera.pbj.runtime.ParseException; import com.hedera.pbj.runtime.grpc.Pipeline; +import com.hedera.pbj.runtime.grpc.ServiceInterface; import com.hedera.pbj.runtime.io.buffer.Bytes; import com.lmax.disruptor.BatchEventProcessor; -import edu.umd.cs.findbugs.annotations.NonNull; import io.helidon.webserver.WebServer; import java.io.IOException; import java.nio.file.Files; @@ -75,7 +75,6 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Flow; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -92,43 +91,43 @@ public class PbjBlockStreamServiceIntegrationTest { private Notifier notifier; @Mock - private Pipeline helidonPublishStreamObserver1; + private Pipeline helidonPublishStreamObserver1; @Mock - private Pipeline helidonPublishStreamObserver2; + private Pipeline helidonPublishStreamObserver2; @Mock - private Pipeline helidonPublishStreamObserver3; + private Pipeline helidonPublishStreamObserver3; @Mock - private SubscribeStreamRequest subscribeStreamRequest; + private Pipeline subscribeStreamObserver1; @Mock - private Pipeline subscribeStreamObserver1; + private Pipeline subscribeStreamObserver2; @Mock - private Pipeline subscribeStreamObserver2; + private Pipeline subscribeStreamObserver3; @Mock - private Pipeline subscribeStreamObserver3; + private Pipeline subscribeStreamObserver4; @Mock - private Pipeline subscribeStreamObserver4; + private Pipeline subscribeStreamObserver5; @Mock - private Pipeline subscribeStreamObserver5; + private Pipeline subscribeStreamObserver6; @Mock - private Pipeline subscribeStreamObserver6; + private WebServer webServer; @Mock - private WebServer webServer; + private BlockWriter> blockWriter; @Mock - private BlockWriter> blockWriter; + private BlockReader blockReader; @Mock - private BlockReader blockReader; + private ServiceInterface.RequestOptions options; private static final String TEMP_DIR = "block-node-unit-test-dir"; @@ -154,31 +153,41 @@ public void tearDown() { } @Test - public void testPublishBlockStreamRegistrationAndExecution() throws IOException, NoSuchAlgorithmException { + public void testPublishBlockStreamRegistrationAndExecution() + throws IOException, NoSuchAlgorithmException, ParseException { - final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); - final var streamMediator = LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus) - .build(); - final var notifier = new NotifierImpl(streamMediator, blockNodeContext, serviceStatus); - final var blockNodeEventHandler = new StreamPersistenceHandlerImpl( - streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - - final PbjBlockStreamServiceProxy pbjBlockStreamServiceProxy = new PbjBlockStreamServiceProxy( - streamMediator, serviceStatus, blockNodeEventHandler, notifier, blockNodeContext); - - // Register 3 producers - final Flow.Subscriber publishStreamObserver = - pbjBlockStreamServiceProxy.publishBlockStream(helidonPublishStreamObserver1); - - pbjBlockStreamServiceProxy.publishBlockStream(helidonPublishStreamObserver2); - pbjBlockStreamServiceProxy.publishBlockStream(helidonPublishStreamObserver3); - - // Register 3 consumers - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver1); - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver2); - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver3); + final PbjBlockStreamServiceProxy pbjBlockStreamServiceProxy = buildBlockStreamService(blockWriter); - List blockItems = generateBlockItems(1); + // Register 3 producers - Opening a pipeline is not enough to register a producer. + // pipeline.onNext() must be invoked to register the producer at the Helidon PBJ layer. + final Pipeline producerPipeline = pbjBlockStreamServiceProxy.open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, options, helidonPublishStreamObserver1); + pbjBlockStreamServiceProxy + .open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, + options, + helidonPublishStreamObserver2) + .onNext(buildEmptyPublishStreamRequest()); + pbjBlockStreamServiceProxy + .open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, + options, + helidonPublishStreamObserver3) + .onNext(buildEmptyPublishStreamRequest()); + + // Register 3 consumers - Opening a pipeline is not enough to register a consumer. + // pipeline.onNext() must be invoked to register the consumer at the Helidon PBJ layer. + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver1) + .onNext(buildEmptySubscribeStreamRequest()); + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver2) + .onNext(buildEmptySubscribeStreamRequest()); + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver3) + .onNext(buildEmptySubscribeStreamRequest()); + + final List blockItems = generateBlockItemsUnparsed(1); for (int i = 0; i < blockItems.size(); i++) { if (i == 9) { when(blockWriter.write(List.of(blockItems.get(i)))).thenReturn(Optional.of(List.of(blockItems.get(i)))); @@ -187,14 +196,14 @@ public void testPublishBlockStreamRegistrationAndExecution() throws IOException, } } - for (BlockItem blockItem : blockItems) { - final PublishStreamRequest publishStreamRequest = PublishStreamRequest.newBuilder() - .blockItems(new BlockItemSet(List.of(blockItem))) + for (BlockItemUnparsed blockItem : blockItems) { + // Calling onNext() as Helidon does + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItem).build(); + final PublishStreamRequestUnparsed publishStreamRequest = PublishStreamRequestUnparsed.newBuilder() + .blockItems(blockItemSet) .build(); - - // Calling onNext() as Helidon does with each block item for - // the first producer. - publishStreamObserver.onNext(publishStreamRequest); + producerPipeline.onNext(PublishStreamRequestUnparsed.PROTOBUF.toBytes(publishStreamRequest)); } // Verify all 10 BlockItems were sent to each of the 3 consumers @@ -225,139 +234,137 @@ public void testPublishBlockStreamRegistrationAndExecution() throws IOException, PublishStreamResponse.newBuilder().acknowledgement(itemAck).build(); // Verify all 3 producers received the response - verify(helidonPublishStreamObserver1, timeout(testTimeout).times(1)).onNext(publishStreamResponse); - - verify(helidonPublishStreamObserver2, timeout(testTimeout).times(1)).onNext(publishStreamResponse); - - verify(helidonPublishStreamObserver3, timeout(testTimeout).times(1)).onNext(publishStreamResponse); + final Bytes responseBytes = PublishStreamResponse.PROTOBUF.toBytes(publishStreamResponse); + verify(helidonPublishStreamObserver1, timeout(testTimeout).times(1)).onNext(responseBytes); + verify(helidonPublishStreamObserver2, timeout(testTimeout).times(1)).onNext(responseBytes); + verify(helidonPublishStreamObserver3, timeout(testTimeout).times(1)).onNext(responseBytes); // Close the stream as Helidon does - publishStreamObserver.onComplete(); + helidonPublishStreamObserver1.onComplete(); // verify the onCompleted() method is invoked on the wrapped StreamObserver verify(helidonPublishStreamObserver1, timeout(testTimeout).times(1)).onComplete(); } - @Test - public void testSubscribeBlockStream() throws IOException { - - final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); - serviceStatus.setWebServer(webServer); - - final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); - - final var streamMediator = LiveStreamMediatorBuilder.newBuilder(blockNodeContext, serviceStatus) - .build(); - - // Build the BlockStreamService - final var blockNodeEventHandler = new StreamPersistenceHandlerImpl( - streamMediator, notifier, blockWriter, blockNodeContext, serviceStatus); - - final PbjBlockStreamServiceProxy pbjBlockStreamServiceProxy = new PbjBlockStreamServiceProxy( - streamMediator, serviceStatus, blockNodeEventHandler, notifier, blockNodeContext); - - // Subscribe the consumers - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver1); - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver2); - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver3); - - // Subscribe the producer - final Flow.Subscriber producerBlockItemsObserver = - pbjBlockStreamServiceProxy.publishBlockStream(helidonPublishStreamObserver1); - - // Build the BlockItem - final List blockItems = generateBlockItems(1); - final PublishStreamRequest publishStreamRequest = PublishStreamRequest.newBuilder() - .blockItems(new BlockItemSet(List.of(blockItems.getFirst()))) - .build(); - - // Calling onNext() with a BlockItem - producerBlockItemsObserver.onNext(publishStreamRequest); - - // Verify the counter was incremented - assertEquals(1, blockNodeContext.metricsService().get(LiveBlockItems).get()); - - verify(blockWriter, timeout(testTimeout).times(1)).write(List.of(blockItems.getFirst())); - - final BlockItemSet blockItemSet = BlockItemSet.newBuilder() - .blockItems(List.of(blockItems.getFirst())) - .build(); - final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItems(blockItemSet).build(); - - verify(subscribeStreamObserver1, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); - verify(subscribeStreamObserver2, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); - verify(subscribeStreamObserver3, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); - } - @Test public void testFullProducerConsumerHappyPath() throws IOException { int numberOfBlocks = 100; - final BlockWriter> blockWriter = + // Use a real BlockWriter to test the full integration + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); final PbjBlockStreamServiceProxy pbjBlockStreamServiceProxy = buildBlockStreamService(blockWriter); - final Flow.Subscriber producerBlockItemObserver = - pbjBlockStreamServiceProxy.publishBlockStream(helidonPublishStreamObserver1); - - // Subscribe the consumers - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver1); - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver2); - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver3); - - final List blockItems = generateBlockItems(numberOfBlocks); - for (BlockItem blockItem : blockItems) { - final PublishStreamRequest publishStreamRequest = PublishStreamRequest.newBuilder() - .blockItems(new BlockItemSet(List.of(blockItem))) + // Register 3 producers - Opening a pipeline is not enough to register a producer. + // pipeline.onNext() must be invoked to register the producer at the Helidon PBJ layer. + final Pipeline producerPipeline = pbjBlockStreamServiceProxy.open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, options, helidonPublishStreamObserver1); + pbjBlockStreamServiceProxy + .open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, + options, + helidonPublishStreamObserver2) + .onNext(buildEmptyPublishStreamRequest()); + pbjBlockStreamServiceProxy + .open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, + options, + helidonPublishStreamObserver3) + .onNext(buildEmptyPublishStreamRequest()); + + // Register 3 consumers - Opening a pipeline is not enough to register a consumer. + // pipeline.onNext() must be invoked to register the consumer at the Helidon PBJ layer. + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver1) + .onNext(buildEmptySubscribeStreamRequest()); + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver2) + .onNext(buildEmptySubscribeStreamRequest()); + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver3) + .onNext(buildEmptySubscribeStreamRequest()); + + final List blockItems = generateBlockItemsUnparsed(numberOfBlocks); + for (BlockItemUnparsed blockItem : blockItems) { + // Calling onNext() as Helidon does + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItem).build(); + final PublishStreamRequestUnparsed publishStreamRequest = PublishStreamRequestUnparsed.newBuilder() + .blockItems(blockItemSet) .build(); - producerBlockItemObserver.onNext(publishStreamRequest); + producerPipeline.onNext(PublishStreamRequestUnparsed.PROTOBUF.toBytes(publishStreamRequest)); } + // Verify the subscribers received the data verifySubscribeStreamResponse(numberOfBlocks, 0, numberOfBlocks, subscribeStreamObserver1, blockItems); verifySubscribeStreamResponse(numberOfBlocks, 0, numberOfBlocks, subscribeStreamObserver2, blockItems); verifySubscribeStreamResponse(numberOfBlocks, 0, numberOfBlocks, subscribeStreamObserver3, blockItems); + // Verify the producers received all the responses verify(helidonPublishStreamObserver1, timeout(testTimeout).times(100)).onNext(any()); + verify(helidonPublishStreamObserver2, timeout(testTimeout).times(100)).onNext(any()); + verify(helidonPublishStreamObserver3, timeout(testTimeout).times(100)).onNext(any()); } @Test public void testFullWithSubscribersAddedDynamically() { - int numberOfBlocks = 100; - final PbjBlockStreamServiceProxy blockStreamServiceProxy = buildBlockStreamService(blockWriter); - final Flow.Subscriber streamObserver = - blockStreamServiceProxy.publishBlockStream(helidonPublishStreamObserver1); - - final List blockItems = generateBlockItems(numberOfBlocks); - - // Subscribe the initial consumers - blockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver1); - blockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver2); - blockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver3); + final PbjBlockStreamServiceProxy pbjBlockStreamServiceProxy = buildBlockStreamService(blockWriter); + // Register a producer + final Pipeline producerPipeline = pbjBlockStreamServiceProxy.open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, options, helidonPublishStreamObserver1); + + // Register 3 consumers - Opening a pipeline is not enough to register a consumer. + // pipeline.onNext() must be invoked to register the consumer at the Helidon PBJ layer. + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver1) + .onNext(buildEmptySubscribeStreamRequest()); + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver2) + .onNext(buildEmptySubscribeStreamRequest()); + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver3) + .onNext(buildEmptySubscribeStreamRequest()); + + final List blockItems = generateBlockItemsUnparsed(numberOfBlocks); for (int i = 0; i < blockItems.size(); i++) { - final PublishStreamRequest publishStreamRequest = PublishStreamRequest.newBuilder() - .blockItems(new BlockItemSet(List.of(blockItems.get(i)))) + final PublishStreamRequestUnparsed publishStreamRequest = PublishStreamRequestUnparsed.newBuilder() + .blockItems(new BlockItemSetUnparsed(List.of(blockItems.get(i)))) .build(); // Add a new subscriber if (i == 51) { - blockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver4); + pbjBlockStreamServiceProxy + .open( + PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, + options, + subscribeStreamObserver4) + .onNext(buildEmptySubscribeStreamRequest()); } // Transmit the BlockItem - streamObserver.onNext(publishStreamRequest); + producerPipeline.onNext(PublishStreamRequestUnparsed.PROTOBUF.toBytes(publishStreamRequest)); // Add a new subscriber if (i == 76) { - blockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver5); + pbjBlockStreamServiceProxy + .open( + PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, + options, + subscribeStreamObserver5) + .onNext(buildEmptySubscribeStreamRequest()); } // Add a new subscriber if (i == 88) { - blockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver6); + pbjBlockStreamServiceProxy + .open( + PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, + options, + subscribeStreamObserver6) + .onNext(buildEmptySubscribeStreamRequest()); } } @@ -378,12 +385,10 @@ public void testFullWithSubscribersAddedDynamically() { @Test public void testSubAndUnsubWhileStreaming() throws InterruptedException { - int numberOfBlocks = 100; - final LinkedHashMap< - BlockNodeEventHandler>, - BatchEventProcessor>> + BlockNodeEventHandler>, + BatchEventProcessor>> consumers = new LinkedHashMap<>(); final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); @@ -393,22 +398,29 @@ public void testSubAndUnsubWhileStreaming() throws InterruptedException { final PbjBlockStreamServiceProxy pbjBlockStreamServiceProxy = new PbjBlockStreamServiceProxy( streamMediator, serviceStatus, blockNodeEventHandler, notifier, blockNodeContext); - // Pass a StreamObserver to the producer as Helidon does - final Flow.Subscriber streamObserver = - pbjBlockStreamServiceProxy.publishBlockStream(helidonPublishStreamObserver1); - - final List blockItems = generateBlockItems(numberOfBlocks); - - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver1); - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver2); - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver3); - + final Pipeline producerPipeline = pbjBlockStreamServiceProxy.open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, options, helidonPublishStreamObserver1); + + // Register 3 consumers - Opening a pipeline is not enough to register a consumer. + // pipeline.onNext() must be invoked to register the consumer at the Helidon PBJ layer. + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver1) + .onNext(buildEmptySubscribeStreamRequest()); + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver2) + .onNext(buildEmptySubscribeStreamRequest()); + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver3) + .onNext(buildEmptySubscribeStreamRequest()); + + final List blockItems = generateBlockItemsUnparsed(numberOfBlocks); for (int i = 0; i < blockItems.size(); i++) { // Transmit the BlockItem - streamObserver.onNext(PublishStreamRequest.newBuilder() - .blockItems(new BlockItemSet(List.of(blockItems.get(i)))) - .build()); + final PublishStreamRequestUnparsed publishStreamRequest = PublishStreamRequestUnparsed.newBuilder() + .blockItems(new BlockItemSetUnparsed(List.of(blockItems.get(i)))) + .build(); + producerPipeline.onNext(PublishStreamRequestUnparsed.PROTOBUF.toBytes(publishStreamRequest)); // Remove 1st subscriber if (i == 10) { @@ -430,7 +442,12 @@ public void testSubAndUnsubWhileStreaming() throws InterruptedException { // Add a new subscriber if (i == 51) { - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver4); + pbjBlockStreamServiceProxy + .open( + PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, + options, + subscribeStreamObserver4) + .onNext(buildEmptySubscribeStreamRequest()); } // Remove 3rd subscriber @@ -444,12 +461,22 @@ public void testSubAndUnsubWhileStreaming() throws InterruptedException { // Add a new subscriber if (i == 76) { - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver5); + pbjBlockStreamServiceProxy + .open( + PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, + options, + subscribeStreamObserver5) + .onNext(buildEmptySubscribeStreamRequest()); } // Add a new subscriber if (i == 88) { - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver6); + pbjBlockStreamServiceProxy + .open( + PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, + options, + subscribeStreamObserver6) + .onNext(buildEmptySubscribeStreamRequest()); } } @@ -467,14 +494,14 @@ public void testSubAndUnsubWhileStreaming() throws InterruptedException { verifySubscribeStreamResponse(numberOfBlocks, 79, numberOfBlocks, subscribeStreamObserver5, blockItems); verifySubscribeStreamResponse(numberOfBlocks, 89, numberOfBlocks, subscribeStreamObserver6, blockItems); - streamObserver.onComplete(); + producerPipeline.onComplete(); } @Test - public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOException { + public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOException, ParseException { final ConcurrentHashMap< - BlockNodeEventHandler>, - BatchEventProcessor>> + BlockNodeEventHandler>, + BatchEventProcessor>> consumers = new ConcurrentHashMap<>(); // Use a spy to use the real object but also verify the behavior. @@ -484,10 +511,10 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep doCallRealMethod().when(serviceStatus).stopWebServer(any()); serviceStatus.setWebServer(webServer); - final List blockItems = generateBlockItems(1); + final List blockItems = generateBlockItemsUnparsed(1); // Use a spy to make sure the write() method throws an IOException - final BlockWriter> blockWriter = + final BlockWriter> blockWriter = spy(BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build()); doThrow(IOException.class).when(blockWriter).write(blockItems); @@ -498,23 +525,31 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep final PbjBlockStreamServiceProxy pbjBlockStreamServiceProxy = new PbjBlockStreamServiceProxy( streamMediator, serviceStatus, blockNodeEventHandler, notifier, blockNodeContext); - // Subscribe the consumers - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver1); - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver2); - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver3); + // Register a producer + final Pipeline producerPipeline = pbjBlockStreamServiceProxy.open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, options, helidonPublishStreamObserver1); + + // Register 3 consumers - Opening a pipeline is not enough to register a consumer. + // pipeline.onNext() must be invoked to register the consumer at the Helidon PBJ layer. + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver1) + .onNext(buildEmptySubscribeStreamRequest()); + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver2) + .onNext(buildEmptySubscribeStreamRequest()); + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver3) + .onNext(buildEmptySubscribeStreamRequest()); // 3 subscribers + 1 streamPersistenceHandler assertEquals(4, consumers.size()); - // Initialize the producer - final Flow.Subscriber producerBlockItemObserver = - pbjBlockStreamServiceProxy.publishBlockStream(helidonPublishStreamObserver1); - // Transmit a BlockItem - final PublishStreamRequest publishStreamRequest = PublishStreamRequest.newBuilder() - .blockItems(new BlockItemSet(blockItems)) - .build(); - producerBlockItemObserver.onNext(publishStreamRequest); + final Bytes publishStreamRequest = + PublishStreamRequestUnparsed.PROTOBUF.toBytes(PublishStreamRequestUnparsed.newBuilder() + .blockItems(new BlockItemSetUnparsed(blockItems)) + .build()); + producerPipeline.onNext(publishStreamRequest); // Use verify to make sure the serviceStatus.stopRunning() method is called // before the next block is transmitted. @@ -522,9 +557,10 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep // Simulate another producer attempting to connect to the Block Node after the exception. // Later, verify they received a response indicating the stream is closed. - final Flow.Subscriber expectedNoOpStreamObserver = - pbjBlockStreamServiceProxy.publishBlockStream(helidonPublishStreamObserver2); - expectedNoOpStreamObserver.onNext(publishStreamRequest); + final Pipeline expectedNoOpProducerPipeline = pbjBlockStreamServiceProxy.open( + PbjBlockStreamService.BlockStreamMethod.publishBlockStream, options, helidonPublishStreamObserver2); + + expectedNoOpProducerPipeline.onNext(publishStreamRequest); verify(helidonPublishStreamObserver2, timeout(testTimeout).times(1)).onNext(buildEndOfStreamResponse()); @@ -536,30 +572,34 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep new PbjBlockAccessServiceProxy(serviceStatus, blockReader, blockNodeContext); // Simulate a consumer attempting to connect to the Block Node after the exception. - final SingleBlockResponse singleBlockResponse = pbjBlockAccessServiceProxy.singleBlock(singleBlockRequest); + final SingleBlockResponseUnparsed singleBlockResponse = + pbjBlockAccessServiceProxy.singleBlock(singleBlockRequest); // Build a request to invoke the subscribeBlockStream service - final SubscribeStreamRequest subscribeStreamRequest = - SubscribeStreamRequest.newBuilder().startBlockNumber(1).build(); // Simulate a consumer attempting to connect to the Block Node after the exception. - pbjBlockStreamServiceProxy.subscribeBlockStream(subscribeStreamRequest, subscribeStreamObserver4); + pbjBlockStreamServiceProxy + .open(PbjBlockStreamService.BlockStreamMethod.subscribeBlockStream, options, subscribeStreamObserver4) + .onNext(buildEmptySubscribeStreamRequest()); // The BlockItem expected to pass through since it was published // before the IOException was thrown. - final BlockItemSet blockItemSet = - BlockItemSet.newBuilder().blockItems(blockItems).build(); + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItems).build(); - final SubscribeStreamResponse subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItems(blockItemSet).build(); + final Bytes subscribeStreamResponse = + SubscribeStreamResponseUnparsed.PROTOBUF.toBytes(SubscribeStreamResponseUnparsed.newBuilder() + .blockItems(blockItemSet) + .build()); verify(subscribeStreamObserver1, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); verify(subscribeStreamObserver2, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); verify(subscribeStreamObserver3, timeout(testTimeout).times(1)).onNext(subscribeStreamResponse); // Verify all the consumers received the end of stream response // TODO: Fix the response code when it's available - final SubscribeStreamResponse endStreamResponse = SubscribeStreamResponse.newBuilder() - .status(SubscribeStreamResponseCode.READ_STREAM_SUCCESS) - .build(); + final Bytes endStreamResponse = + SubscribeStreamResponseUnparsed.PROTOBUF.toBytes(SubscribeStreamResponseUnparsed.newBuilder() + .status(SubscribeStreamResponseCode.READ_STREAM_SUCCESS) + .build()); verify(subscribeStreamObserver1, timeout(testTimeout).times(1)).onNext(endStreamResponse); verify(subscribeStreamObserver2, timeout(testTimeout).times(1)).onNext(endStreamResponse); verify(subscribeStreamObserver3, timeout(testTimeout).times(1)).onNext(endStreamResponse); @@ -575,9 +615,11 @@ public void testMediatorExceptionHandlingWhenPersistenceFailure() throws IOExcep assertEquals(SingleBlockResponseCode.READ_BLOCK_NOT_AVAILABLE, singleBlockResponse.status()); // TODO: Fix the response code when it's available - final SubscribeStreamResponse expectedSubscriberStreamNotAvailable = SubscribeStreamResponse.newBuilder() - .status(SubscribeStreamResponseCode.READ_STREAM_SUCCESS) - .build(); + final Bytes expectedSubscriberStreamNotAvailable = + SubscribeStreamResponseUnparsed.PROTOBUF.toBytes(SubscribeStreamResponseUnparsed.newBuilder() + .status(SubscribeStreamResponseCode.READ_STREAM_SUCCESS) + .build()); + verify(subscribeStreamObserver4, timeout(testTimeout).times(1)).onNext(expectedSubscriberStreamNotAvailable); } @@ -585,8 +627,8 @@ private static void verifySubscribeStreamResponse( int numberOfBlocks, int blockItemsToWait, int blockItemsToSkip, - Pipeline streamObserver, - List blockItems) { + Pipeline pipeline, + List blockItems) { // Each block has 10 BlockItems. Verify all the BlockItems // in a given block per iteration. @@ -596,37 +638,37 @@ private static void verifySubscribeStreamResponse( continue; } - final BlockItem headerBlockItem = blockItems.get(block); - final SubscribeStreamResponse headerSubStreamResponse = buildSubscribeStreamResponse(headerBlockItem); + final BlockItemUnparsed headerBlockItem = blockItems.get(block); + final Bytes headerSubStreamResponse = buildSubscribeStreamResponse(headerBlockItem); - final BlockItem bodyBlockItem = blockItems.get(block + 1); - final SubscribeStreamResponse bodySubStreamResponse = buildSubscribeStreamResponse(bodyBlockItem); + final BlockItemUnparsed bodyBlockItem = blockItems.get(block + 1); + final Bytes bodySubStreamResponse = buildSubscribeStreamResponse(bodyBlockItem); - final BlockItem stateProofBlockItem = blockItems.get(block + 9); - final SubscribeStreamResponse stateProofStreamResponse = buildSubscribeStreamResponse(stateProofBlockItem); + final BlockItemUnparsed stateProofBlockItem = blockItems.get(block + 9); + final Bytes stateProofStreamResponse = buildSubscribeStreamResponse(stateProofBlockItem); - verify(streamObserver, timeout(testTimeout).times(1)).onNext(headerSubStreamResponse); - verify(streamObserver, timeout(testTimeout).times(8)).onNext(bodySubStreamResponse); - verify(streamObserver, timeout(testTimeout).times(1)).onNext(stateProofStreamResponse); + verify(pipeline, timeout(testTimeout).times(1)).onNext(headerSubStreamResponse); + verify(pipeline, timeout(testTimeout).times(8)).onNext(bodySubStreamResponse); + verify(pipeline, timeout(testTimeout).times(1)).onNext(stateProofStreamResponse); } } - private static SubscribeStreamResponse buildSubscribeStreamResponse(BlockItem blockItem) { - final BlockItemSet subscribeStreamResponseSet = - BlockItemSet.newBuilder().blockItems(blockItem).build(); - return SubscribeStreamResponse.newBuilder() - .blockItems(subscribeStreamResponseSet) - .build(); + private static Bytes buildSubscribeStreamResponse(BlockItemUnparsed blockItem) { + return SubscribeStreamResponseUnparsed.PROTOBUF.toBytes(SubscribeStreamResponseUnparsed.newBuilder() + .blockItems( + BlockItemSetUnparsed.newBuilder().blockItems(blockItem).build()) + .build()); } - private static PublishStreamResponse buildEndOfStreamResponse() { + private static Bytes buildEndOfStreamResponse() { final EndOfStream endOfStream = EndOfStream.newBuilder() .status(PublishStreamResponseCode.STREAM_ITEMS_UNKNOWN) .build(); - return PublishStreamResponse.newBuilder().status(endOfStream).build(); + return PublishStreamResponse.PROTOBUF.toBytes( + PublishStreamResponse.newBuilder().status(endOfStream).build()); } - private PbjBlockStreamServiceProxy buildBlockStreamService(final BlockWriter> blockWriter) { + private PbjBlockStreamServiceProxy buildBlockStreamService(final BlockWriter> blockWriter) { final ServiceStatus serviceStatus = new ServiceStatusImpl(blockNodeContext); final var streamMediator = buildStreamMediator(new ConcurrentHashMap<>(32), serviceStatus); @@ -640,8 +682,8 @@ private PbjBlockStreamServiceProxy buildBlockStreamService(final BlockWriter>, - BatchEventProcessor>> + BlockNodeEventHandler>, + BatchEventProcessor>> subscribers, final ServiceStatus serviceStatus) { @@ -651,12 +693,4 @@ private LiveStreamMediator buildStreamMediator( .subscribers(subscribers) .build(); } - - public static Acknowledgement buildAck(@NonNull final List blockItems) throws NoSuchAlgorithmException { - ItemAcknowledgement itemAck = ItemAcknowledgement.newBuilder() - .itemsHash(Bytes.wrap(getFakeHash(blockItems))) - .build(); - - return Acknowledgement.newBuilder().itemAck(itemAck).build(); - } } diff --git a/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java b/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java index d4d9f4753..0f32b98df 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/PersistenceInjectionModuleTest.java @@ -16,7 +16,9 @@ package com.hedera.block.server.persistence; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -30,9 +32,9 @@ import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.swirlds.config.api.Configuration; import java.io.IOException; import java.util.List; @@ -52,13 +54,13 @@ class PersistenceInjectionModuleTest { private PersistenceStorageConfig persistenceStorageConfig; @Mock - private SubscriptionHandler subscriptionHandler; + private SubscriptionHandler subscriptionHandler; @Mock private Notifier notifier; @Mock - private BlockWriter> blockWriter; + private BlockWriter> blockWriter; @Mock private ServiceStatus serviceStatus; @@ -73,7 +75,8 @@ void setup() throws IOException { @Test void testProvidesBlockWriter() { - BlockWriter> blockWriter = PersistenceInjectionModule.providesBlockWriter(blockNodeContext); + BlockWriter> blockWriter = + PersistenceInjectionModule.providesBlockWriter(blockNodeContext); assertNotNull(blockWriter); } @@ -98,8 +101,8 @@ void testProvidesBlockWriter_IOException() { @Test void testProvidesBlockReader() { - BlockReader blockReader = PersistenceInjectionModule.providesBlockReader(persistenceStorageConfig); - + BlockReader blockReader = + PersistenceInjectionModule.providesBlockReader(persistenceStorageConfig); assertNotNull(blockReader); } @@ -109,8 +112,9 @@ void testProvidesStreamValidatorBuilder() throws IOException { BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); // Call the method under test - BlockNodeEventHandler> streamVerifier = new StreamPersistenceHandlerImpl( - subscriptionHandler, notifier, blockWriter, blockNodeContext, serviceStatus); + BlockNodeEventHandler> streamVerifier = + new StreamPersistenceHandlerImpl( + subscriptionHandler, notifier, blockWriter, blockNodeContext, serviceStatus); assertNotNull(streamVerifier); } diff --git a/server/src/test/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImplTest.java b/server/src/test/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImplTest.java index 97c8cee9b..6081e8bdb 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImplTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/StreamPersistenceHandlerImplTest.java @@ -17,7 +17,7 @@ package com.hedera.block.server.persistence; import static com.hedera.block.server.metrics.BlockNodeMetricTypes.Counter.StreamPersistenceHandlerError; -import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; import static com.hedera.hapi.block.SubscribeStreamResponseCode.READ_STREAM_SUCCESS; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.never; @@ -34,9 +34,9 @@ import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.hapi.block.BlockItemSet; -import com.hedera.hapi.block.SubscribeStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemSetUnparsed; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.SubscribeStreamResponseUnparsed; import com.hedera.pbj.runtime.OneOf; import java.io.IOException; import java.util.List; @@ -49,10 +49,10 @@ public class StreamPersistenceHandlerImplTest { @Mock - private SubscriptionHandler subscriptionHandler; + private SubscriptionHandler subscriptionHandler; @Mock - private BlockWriter> blockWriter; + private BlockWriter> blockWriter; @Mock private Notifier notifier; @@ -66,7 +66,7 @@ public class StreamPersistenceHandlerImplTest { @Mock private MetricsService metricsService; - private static final int testTimeout = 0; + private static final int testTimeout = 50; @Test public void testOnEventWhenServiceIsNotRunning() { @@ -77,12 +77,13 @@ public void testOnEventWhenServiceIsNotRunning() { final var streamPersistenceHandler = new StreamPersistenceHandlerImpl( subscriptionHandler, notifier, blockWriter, blockNodeContext, serviceStatus); - final List blockItems = generateBlockItems(1); - final BlockItemSet blockItemSet = - BlockItemSet.newBuilder().blockItems(blockItems).build(); - final var subscribeStreamResponse = - SubscribeStreamResponse.newBuilder().blockItems(blockItemSet).build(); - final ObjectEvent event = new ObjectEvent<>(); + final List blockItems = generateBlockItemsUnparsed(1); + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItems).build(); + final var subscribeStreamResponse = SubscribeStreamResponseUnparsed.newBuilder() + .blockItems(blockItemSet) + .build(); + final ObjectEvent event = new ObjectEvent<>(); event.set(subscribeStreamResponse); streamPersistenceHandler.onEvent(event, 0, false); @@ -102,15 +103,16 @@ public void testBlockItemIsNull() throws IOException { final var streamPersistenceHandler = new StreamPersistenceHandlerImpl( subscriptionHandler, notifier, blockWriter, blockNodeContext, serviceStatus); - final List blockItems = generateBlockItems(1); - final BlockItemSet blockItemSet = - BlockItemSet.newBuilder().blockItems(blockItems).build(); - final var subscribeStreamResponse = spy( - SubscribeStreamResponse.newBuilder().blockItems(blockItemSet).build()); + final List blockItems = generateBlockItemsUnparsed(1); + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItems).build(); + final var subscribeStreamResponse = spy(SubscribeStreamResponseUnparsed.newBuilder() + .blockItems(blockItemSet) + .build()); // Force the block item to be null when(subscribeStreamResponse.blockItems()).thenReturn(null); - final ObjectEvent event = new ObjectEvent<>(); + final ObjectEvent event = new ObjectEvent<>(); event.set(subscribeStreamResponse); streamPersistenceHandler.onEvent(event, 0, false); @@ -128,18 +130,19 @@ public void testSubscribeStreamResponseTypeUnknown() throws IOException { final var streamPersistenceHandler = new StreamPersistenceHandlerImpl( subscriptionHandler, notifier, blockWriter, blockNodeContext, serviceStatus); - final List blockItems = generateBlockItems(1); - final BlockItemSet blockItemSet = - BlockItemSet.newBuilder().blockItems(blockItems).build(); - final var subscribeStreamResponse = spy( - SubscribeStreamResponse.newBuilder().blockItems(blockItemSet).build()); + final List blockItems = generateBlockItemsUnparsed(1); + final BlockItemSetUnparsed blockItemSet = + BlockItemSetUnparsed.newBuilder().blockItems(blockItems).build(); + final var subscribeStreamResponse = spy(SubscribeStreamResponseUnparsed.newBuilder() + .blockItems(blockItemSet) + .build()); // Force the block item to be UNSET - final OneOf illegalOneOf = - new OneOf<>(SubscribeStreamResponse.ResponseOneOfType.UNSET, null); + final OneOf illegalOneOf = + new OneOf<>(SubscribeStreamResponseUnparsed.ResponseOneOfType.UNSET, null); when(subscribeStreamResponse.response()).thenReturn(illegalOneOf); - final ObjectEvent event = new ObjectEvent<>(); + final ObjectEvent event = new ObjectEvent<>(); event.set(subscribeStreamResponse); streamPersistenceHandler.onEvent(event, 0, false); @@ -157,9 +160,10 @@ public void testSubscribeStreamResponseTypeStatus() { final var streamPersistenceHandler = new StreamPersistenceHandlerImpl( subscriptionHandler, notifier, blockWriter, blockNodeContext, serviceStatus); - final SubscribeStreamResponse subscribeStreamResponse = spy( - SubscribeStreamResponse.newBuilder().status(READ_STREAM_SUCCESS).build()); - final ObjectEvent event = new ObjectEvent<>(); + final SubscribeStreamResponseUnparsed subscribeStreamResponse = spy(SubscribeStreamResponseUnparsed.newBuilder() + .status(READ_STREAM_SUCCESS) + .build()); + final ObjectEvent event = new ObjectEvent<>(); event.set(subscribeStreamResponse); streamPersistenceHandler.onEvent(event, 0, false); diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java index 5c24babcb..6a86cc045 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/read/BlockAsDirReaderTest.java @@ -17,10 +17,13 @@ package com.hedera.block.server.persistence.storage.read; import static com.hedera.block.server.Constants.BLOCK_FILE_EXTENSION; -import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; import static com.hedera.block.server.util.PersistTestUtils.reverseByteArray; -import static java.lang.System.Logger; -import static org.junit.jupiter.api.Assertions.*; +import static com.hedera.block.server.util.PersistTestUtils.writeBlockItemToPath; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.spy; @@ -29,11 +32,10 @@ import com.hedera.block.server.persistence.storage.PersistenceStorageConfig; import com.hedera.block.server.persistence.storage.write.BlockAsDirWriterBuilder; import com.hedera.block.server.persistence.storage.write.BlockWriter; -import com.hedera.block.server.util.PersistTestUtils; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; -import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; import com.hedera.pbj.runtime.ParseException; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.FileInputStream; @@ -52,36 +54,34 @@ public class BlockAsDirReaderTest { - private final Logger LOGGER = System.getLogger(getClass().getName()); - @TempDir private Path testPath; private BlockNodeContext blockNodeContext; private PersistenceStorageConfig config; + private List blockItems; @BeforeEach public void setUp() throws IOException { blockNodeContext = TestConfigUtil.getTestBlockNodeContext(Map.of("persistence.storage.rootPath", testPath.toString())); config = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + blockItems = generateBlockItemsUnparsed(1); } @Test public void testReadBlockDoesNotExist() throws IOException, ParseException { - final BlockReader blockReader = + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); - final Optional blockOpt = blockReader.read(10000); + final Optional blockOpt = blockReader.read(10000); assertTrue(blockOpt.isEmpty()); } @Test public void testReadPermsRepairSucceeded() throws IOException, ParseException { - final List blockItems = generateBlockItems(1); - - final BlockWriter> blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (BlockItem blockItem : blockItems) { + for (BlockItemUnparsed blockItem : blockItems) { blockWriter.write(List.of(blockItem)); } @@ -89,18 +89,16 @@ public void testReadPermsRepairSucceeded() throws IOException, ParseException { removeBlockReadPerms(1, config); // The default BlockReader will attempt to repair the permissions and should succeed - final BlockReader blockReader = + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); - final Optional blockOpt = blockReader.read(1); + final Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); - assertEquals(10, blockOpt.get().items().size()); + assertEquals(10, blockOpt.get().blockItems().size()); } @Test public void testRemoveBlockReadPermsRepairFailed() throws IOException, ParseException { - final List blockItems = generateBlockItems(1); - - final BlockWriter> blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); blockWriter.write(blockItems); @@ -109,49 +107,44 @@ public void testRemoveBlockReadPermsRepairFailed() throws IOException, ParseExce // For this test, build the Reader with ineffective repair permissions to // simulate a failed repair (root changed the perms, etc.) - final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config) + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config) .folderPermissions(TestUtils.getNoPerms()) .build(); - final Optional blockOpt = blockReader.read(1); + final Optional blockOpt = blockReader.read(1); assertTrue(blockOpt.isEmpty()); } @Test - public void testRemoveBlockItemReadPerms() throws IOException { - final List blockItems = generateBlockItems(1); - - final BlockWriter> blockWriter = + public void testRemoveBlockItemReadPerms() throws IOException, ParseException { + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); blockWriter.write(blockItems); removeBlockItemReadPerms(1, 1, config); - final BlockReader blockReader = + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); assertThrows(IOException.class, () -> blockReader.read(1)); } @Test public void testPathIsNotDirectory() throws IOException, ParseException { - final List blockItems = generateBlockItems(1); + final Path blockNodeRootPath = Path.of(config.rootPath()); // Write a file named "1" where a directory should be - PersistTestUtils.writeBlockItemToPath(blockNodeRootPath.resolve(Path.of("1")), blockItems.getFirst()); + writeBlockItemToPath(blockNodeRootPath.resolve(Path.of("1")), blockItems.getFirst()); // Should return empty because the path is not a directory - final BlockReader blockReader = + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); - final Optional blockOpt = blockReader.read(1); + final Optional blockOpt = blockReader.read(1); assertTrue(blockOpt.isEmpty()); } @Test public void testRepairReadPermsFails() throws IOException, ParseException { - - final List blockItems = generateBlockItems(1); - - final BlockWriter> blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); blockWriter.write(blockItems); @@ -163,7 +156,7 @@ public void testRepairReadPermsFails() throws IOException, ParseException { final TestBlockAsDirReader blockReader = spy(new TestBlockAsDirReader(config)); doThrow(IOException.class).when(blockReader).setPerm(any(), any()); - final Optional blockOpt = blockReader.read(1); + final Optional blockOpt = blockReader.read(1); assertTrue(blockOpt.isEmpty()); } @@ -179,22 +172,20 @@ public void testBlockNodePathReadFails() throws IOException, ParseException { final TestBlockAsDirReader blockReader = spy(new TestBlockAsDirReader(config)); doThrow(IOException.class).when(blockReader).setPerm(any(), any()); - final Optional blockOpt = blockReader.read(1); + final Optional blockOpt = blockReader.read(1); assertTrue(blockOpt.isEmpty()); } @Test public void testParseExceptionHandling() throws IOException, ParseException { - final List blockItems = generateBlockItems(1); - - final BlockWriter> blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); blockWriter.write(blockItems); // Read the block back and confirm it's read successfully - final BlockReader blockReader = + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(config).build(); - final Optional blockOpt = blockReader.read(1); + final Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); final PersistenceStorageConfig persistenceStorageConfig = diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java index f954a6f43..fe7921a53 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/remove/BlockAsDirRemoverTest.java @@ -28,8 +28,8 @@ import com.hedera.block.server.persistence.storage.write.BlockWriter; import com.hedera.block.server.util.PersistTestUtils; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; import com.hedera.pbj.runtime.ParseException; import java.io.IOException; import java.nio.file.Files; @@ -64,11 +64,11 @@ public void setUp() throws IOException { public void testRemoveNonExistentBlock() throws IOException, ParseException { // Write a block - final var blockItems = PersistTestUtils.generateBlockItems(1); + final var blockItems = PersistTestUtils.generateBlockItemsUnparsed(1); - final BlockWriter> blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); - for (final BlockItem blockItem : blockItems) { + for (BlockItemUnparsed blockItem : blockItems) { blockWriter.write(List.of(blockItem)); } @@ -77,13 +77,13 @@ public void testRemoveNonExistentBlock() throws IOException, ParseException { blockRemover.remove(2); // Verify the block was not removed - final BlockReader blockReader = + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(testConfig).build(); - Optional blockOpt = blockReader.read(1); + Optional blockOpt = blockReader.read(1); assert (blockOpt.isPresent()); assertEquals( blockItems.getFirst().blockHeader(), - blockOpt.get().items().getFirst().blockHeader()); + blockOpt.get().blockItems().getFirst().blockHeader()); // Now remove the block blockRemover.remove(1); diff --git a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java index eb6dd5801..e5583f9d4 100644 --- a/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java +++ b/server/src/test/java/com/hedera/block/server/persistence/storage/write/BlockAsDirWriterTest.java @@ -17,7 +17,7 @@ package com.hedera.block.server.persistence.storage.write; import static com.hedera.block.server.persistence.storage.read.BlockAsDirReaderTest.removeBlockReadPerms; -import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; import static com.hedera.block.server.util.TestConfigUtil.DEFAULT_TEST_FOLDER_PERMISSIONS; import static java.lang.System.Logger; import static java.lang.System.Logger.Level.ERROR; @@ -27,10 +27,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Mockito.any; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.same; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.same; import static org.mockito.Mockito.spy; import com.hedera.block.server.config.BlockNodeContext; @@ -41,8 +41,9 @@ import com.hedera.block.server.persistence.storage.remove.BlockRemover; import com.hedera.block.server.util.TestConfigUtil; import com.hedera.block.server.util.TestUtils; -import com.hedera.hapi.block.stream.Block; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.BlockUnparsed; +import com.hedera.hapi.block.stream.output.BlockHeader; import com.hedera.pbj.runtime.ParseException; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; @@ -69,6 +70,8 @@ public class BlockAsDirWriterTest { private BlockNodeContext blockNodeContext; private PersistenceStorageConfig testConfig; + private List blockItems; + @BeforeEach public void setUp() throws IOException { testPath = Files.createTempDirectory(TEMP_DIR); @@ -77,6 +80,7 @@ public void setUp() throws IOException { blockNodeContext = TestConfigUtil.getTestBlockNodeContext(Map.of(PERSISTENCE_STORAGE_ROOT_PATH_KEY, testPath.toString())); testConfig = blockNodeContext.configuration().getConfigData(PersistenceStorageConfig.class); + blockItems = generateBlockItemsUnparsed(1); } @AfterEach @@ -89,38 +93,35 @@ public void tearDown() { @Test public void testWriterAndReaderHappyPath() throws IOException, ParseException { - // Write a block - final List blockItems = generateBlockItems(1); - - final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext) + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext) .folderPermissions(DEFAULT_TEST_FOLDER_PERMISSIONS) .build(); for (int i = 0; i < 10; i++) { if (i == 9) { - Optional> result = blockWriter.write(List.of(blockItems.get(i))); + Optional> result = blockWriter.write(List.of(blockItems.get(i))); if (result.isPresent()) { assertEquals(blockItems.get(i), result.get().get(0)); } else { fail("The optional should contain the last block proof block item"); } } else { - Optional> result = blockWriter.write(List.of(blockItems.get(i))); + Optional> result = blockWriter.write(List.of(blockItems.get(i))); assertTrue(result.isEmpty()); } } // Confirm the block - BlockReader blockReader = + BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(testConfig).build(); - Optional blockOpt = blockReader.read(1); + Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); boolean hasHeader = false; boolean hasBlockProof = false; boolean hasStartEvent = false; - Block block = blockOpt.get(); - for (BlockItem blockItem : block.items()) { + BlockUnparsed block = blockOpt.get(); + for (BlockItemUnparsed blockItem : block.blockItems()) { if (blockItem.hasBlockHeader()) { hasHeader = true; } else if (blockItem.hasBlockProof()) { @@ -140,9 +141,7 @@ public void testWriterAndReaderHappyPath() throws IOException, ParseException { @Test public void testRemoveBlockWritePerms() throws IOException, ParseException { - final List blockItems = generateBlockItems(1); - - final BlockWriter> blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); // Change the permissions on the block node root directory @@ -150,16 +149,16 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { // The first BlockItem contains a header which will create a new block directory. // The BlockWriter will attempt to repair the permissions and should succeed. - Optional> result = blockWriter.write(List.of(blockItems.getFirst())); + Optional> result = blockWriter.write(List.of(blockItems.getFirst())); assertFalse(result.isPresent()); // Confirm we're able to read 1 block item - BlockReader blockReader = + BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(testConfig).build(); - Optional blockOpt = blockReader.read(1); + Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); - assertEquals(1, blockOpt.get().items().size()); - assertTrue(blockOpt.get().items().get(0).hasBlockHeader()); + assertEquals(1, blockOpt.get().blockItems().size()); + assertTrue(blockOpt.get().blockItems().getFirst().hasBlockHeader()); // Remove all permissions on the block directory and // attempt to write the next block item @@ -170,8 +169,8 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { // There should now be 2 blockItems in the block blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); - assertEquals(2, blockOpt.get().items().size()); - assertFalse(blockOpt.get().items().get(1).hasBlockHeader()); + assertEquals(2, blockOpt.get().blockItems().size()); + assertFalse(blockOpt.get().blockItems().get(1).hasBlockHeader()); // Remove read permission on the block directory removeBlockReadPerms(1, testConfig); @@ -181,34 +180,33 @@ public void testRemoveBlockWritePerms() throws IOException, ParseException { // There should now be 3 blockItems in the block blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); - assertEquals(3, blockOpt.get().items().size()); - assertFalse(blockOpt.get().items().get(1).hasBlockHeader()); + assertEquals(3, blockOpt.get().blockItems().size()); + assertFalse(blockOpt.get().blockItems().get(1).hasBlockHeader()); } @Test - public void testUnrecoverableIOExceptionOnWrite() throws IOException { + public void testUnrecoverableIOExceptionOnWrite() throws IOException, ParseException { - final List blockItems = generateBlockItems(1); final BlockRemover blockRemover = new BlockAsDirRemover(Path.of(testConfig.rootPath())); // Use a spy to simulate an IOException when the first block item is written - final BlockWriter> blockWriter = spy(BlockAsDirWriterBuilder.newBuilder(blockNodeContext) - .blockRemover(blockRemover) - .build()); + final BlockWriter> blockWriter = + spy(BlockAsDirWriterBuilder.newBuilder(blockNodeContext) + .blockRemover(blockRemover) + .build()); doThrow(IOException.class).when(blockWriter).write(blockItems); assertThrows(IOException.class, () -> blockWriter.write(blockItems)); } @Test public void testRemoveRootDirReadPerm() throws IOException, ParseException { - final List blockItems = generateBlockItems(1); - final BlockWriter> blockWriter = + final BlockWriter> blockWriter = BlockAsDirWriterBuilder.newBuilder(blockNodeContext).build(); // Write the first block item to create the block // directory - Optional> result = blockWriter.write(List.of(blockItems.getFirst())); + Optional> result = blockWriter.write(List.of(blockItems.getFirst())); assertFalse(result.isPresent()); // Remove root dir read permissions and @@ -232,16 +230,16 @@ public void testRemoveRootDirReadPerm() throws IOException, ParseException { } } - BlockReader blockReader = + BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(testConfig).build(); - Optional blockOpt = blockReader.read(1); + Optional blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); - assertEquals(10, blockOpt.get().items().size()); + assertEquals(10, blockOpt.get().blockItems().size()); } @Test public void testPartialBlockRemoval() throws IOException, ParseException { - final List blockItems = generateBlockItems(3); + final List blockItems = generateBlockItemsUnparsed(3); final BlockRemover blockRemover = new BlockAsDirRemover(Path.of(testConfig.rootPath())); // Use a spy of TestBlockAsDirWriter to proxy block items to the real writer @@ -262,7 +260,7 @@ public void testPartialBlockRemoval() throws IOException, ParseException { // Now make the calls for (int i = 0; i < 23; i++) { - Optional> result = blockWriter.write(List.of(blockItems.get(i))); + Optional> result = blockWriter.write(List.of(blockItems.get(i))); if (i == 9 || i == 19) { // The last block item in each block is the block proof // and should be returned by the write method @@ -278,21 +276,29 @@ public void testPartialBlockRemoval() throws IOException, ParseException { assertThrows(IOException.class, () -> blockWriter.write(List.of(blockItems.get(23)))); // Verify the partially written block was removed - final BlockReader blockReader = + final BlockReader blockReader = BlockAsDirReaderBuilder.newBuilder(testConfig).build(); - Optional blockOpt = blockReader.read(3); + Optional blockOpt = blockReader.read(3); assertTrue(blockOpt.isEmpty()); // Confirm blocks 1 and 2 still exist blockOpt = blockReader.read(1); assertFalse(blockOpt.isEmpty()); - assertEquals(10, blockOpt.get().items().size()); - assertEquals(1, blockOpt.get().items().getFirst().blockHeader().number()); + assertEquals(10, blockOpt.get().blockItems().size()); + assertEquals( + 1, + BlockHeader.PROTOBUF + .parse(blockOpt.get().blockItems().getFirst().blockHeader()) + .number()); blockOpt = blockReader.read(2); assertFalse(blockOpt.isEmpty()); - assertEquals(10, blockOpt.get().items().size()); - assertEquals(2, blockOpt.get().items().getFirst().blockHeader().number()); + assertEquals(10, blockOpt.get().blockItems().size()); + assertEquals( + 2, + BlockHeader.PROTOBUF + .parse(blockOpt.get().blockItems().getFirst().blockHeader()) + .number()); } private void removeRootWritePerms(final PersistenceStorageConfig config) throws IOException { @@ -311,8 +317,8 @@ private void removeBlockAllPerms(final int blockNumber, final PersistenceStorage Files.setPosixFilePermissions(blockPath, TestUtils.getNoPerms().value()); } - // TestBlockAsDirWriter overrides the write() method to allow a test spy to simulate an - // IOException while allowing the real write() method to remain protected. + // TestBlockAsDirWriter overrides the write() method to allow a test spy to simulate an + // IOException while allowing the real write() method to remain protected. private static final class TestBlockAsDirWriter extends BlockAsDirWriter { public TestBlockAsDirWriter( final BlockRemover blockRemover, @@ -323,7 +329,7 @@ public TestBlockAsDirWriter( } @Override - public void write(@NonNull final Path blockItemFilePath, @NonNull final BlockItem blockItem) + public void write(@NonNull final Path blockItemFilePath, @NonNull final BlockItemUnparsed blockItem) throws IOException { super.write(blockItemFilePath, blockItem); } diff --git a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java index a0ce5428a..d27889e8e 100644 --- a/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java +++ b/server/src/test/java/com/hedera/block/server/producer/ProducerBlockItemObserverTest.java @@ -16,10 +16,12 @@ package com.hedera.block.server.producer; -import static com.hedera.block.server.util.PersistTestUtils.generateBlockItems; +import static com.hedera.block.server.util.PersistTestUtils.generateBlockItemsUnparsed; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import com.hedera.block.server.config.BlockNodeContext; import com.hedera.block.server.events.ObjectEvent; @@ -28,17 +30,14 @@ import com.hedera.block.server.service.ServiceStatus; import com.hedera.block.server.service.ServiceStatusImpl; import com.hedera.block.server.util.TestConfigUtil; -import com.hedera.hapi.block.BlockItemSet; -import com.hedera.hapi.block.PublishStreamRequest; +import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.PublishStreamResponse; -import com.hedera.hapi.block.stream.BlockItem; import com.hedera.pbj.runtime.grpc.Pipeline; import java.io.IOException; import java.time.InstantSource; import java.util.List; import java.util.Map; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -51,13 +50,13 @@ public class ProducerBlockItemObserverTest { private InstantSource testClock; @Mock - private Publisher> publisher; + private Publisher> publisher; @Mock private SubscriptionHandler subscriptionHandler; @Mock - private Pipeline publishStreamResponseObserver; + private Pipeline helidonPublishPipeline; @Mock private ServiceStatus serviceStatus; @@ -65,6 +64,7 @@ public class ProducerBlockItemObserverTest { @Mock private ObjectEvent objectEvent; + private final long TEST_TIME = 1_719_427_664_950L; private final long TIMEOUT_THRESHOLD_MILLIS = 50L; private static final int testTimeout = 1000; @@ -77,21 +77,55 @@ public void setUp() throws IOException { } @Test - @Disabled - public void testOnError() throws IOException { + public void testConfirmOnErrorNotCalled() { - final BlockNodeContext blockNodeContext = TestConfigUtil.getTestBlockNodeContext(); final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - testClock, - publisher, - subscriptionHandler, - publishStreamResponseObserver, - blockNodeContext, - serviceStatus); - + testClock, publisher, subscriptionHandler, helidonPublishPipeline, testContext, serviceStatus); + + // Confirm that onError will call the handler + // to unsubscribe but make sure onError is never + // called on the helidonPublishPipeline. + // Calling onError() on the helidonPublishPipeline + // passed by the Helidon PBJ plugin may cause + // a loop of calls. final Throwable t = new Throwable("Test error"); producerBlockItemObserver.onError(t); - verify(publishStreamResponseObserver).onError(t); + verify(subscriptionHandler, timeout(testTimeout).times(1)).unsubscribe(any()); + verify(helidonPublishPipeline, never()).onError(t); + } + + @Test + public void testOnEventCallsUnsubscribeOnExpiration() { + + when(testClock.millis()).thenReturn(TEST_TIME, TEST_TIME + TIMEOUT_THRESHOLD_MILLIS + 1); + final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( + testClock, publisher, subscriptionHandler, helidonPublishPipeline, testContext, serviceStatus); + + producerBlockItemObserver.onEvent(objectEvent, 0, true); + producerBlockItemObserver.onEvent(objectEvent, 0, true); + + verify(subscriptionHandler, timeout(testTimeout).times(1)).unsubscribe(producerBlockItemObserver); + } + + @Test + public void testOnSubscribe() { + + final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( + testClock, publisher, subscriptionHandler, helidonPublishPipeline, testContext, serviceStatus); + + // Currently, our implementation of onSubscribe() is a + // no-op. + producerBlockItemObserver.onSubscribe(null); + } + + @Test + public void testEmptyBlockItems() { + + final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( + testClock, publisher, subscriptionHandler, helidonPublishPipeline, testContext, serviceStatus); + + producerBlockItemObserver.onNext(List.of()); + verify(publisher, never()).publish(any()); } @Test @@ -100,23 +134,32 @@ public void testOnlyErrorStreamResponseAllowedAfterStatusChange() { final ServiceStatus serviceStatus = new ServiceStatusImpl(testContext); final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( - testClock, publisher, subscriptionHandler, publishStreamResponseObserver, testContext, serviceStatus); + testClock, publisher, subscriptionHandler, helidonPublishPipeline, testContext, serviceStatus); - final List blockItems = generateBlockItems(1); - final PublishStreamRequest publishStreamRequest = PublishStreamRequest.newBuilder() - .blockItems(new BlockItemSet(blockItems)) - .build(); + final List blockItems = generateBlockItemsUnparsed(1); - // Confirm that the observer is called with the first BlockItem - producerBlockItemObserver.onNext(publishStreamRequest); + // Send a request + producerBlockItemObserver.onNext(blockItems); // Change the status of the service serviceStatus.stopRunning(getClass().getName()); - // Confirm that the observer is called with the first BlockItem - producerBlockItemObserver.onNext(publishStreamRequest); + // Send another request + producerBlockItemObserver.onNext(blockItems); // Confirm that closing the observer allowed only 1 response to be sent. - verify(publishStreamResponseObserver, timeout(testTimeout).times(1)).onNext(any()); + verify(helidonPublishPipeline, timeout(testTimeout).times(1)).onNext(any()); + } + + @Test + public void testClientEndStreamReceived() { + + final ProducerBlockItemObserver producerBlockItemObserver = new ProducerBlockItemObserver( + testClock, publisher, subscriptionHandler, helidonPublishPipeline, testContext, serviceStatus); + + producerBlockItemObserver.clientEndStreamReceived(); + + // Confirm that the observer was unsubscribed + verify(subscriptionHandler, timeout(testTimeout).times(1)).unsubscribe(producerBlockItemObserver); } } diff --git a/server/src/test/java/com/hedera/block/server/util/PbjProtoTestUtils.java b/server/src/test/java/com/hedera/block/server/util/PbjProtoTestUtils.java new file mode 100644 index 000000000..33278ce9d --- /dev/null +++ b/server/src/test/java/com/hedera/block/server/util/PbjProtoTestUtils.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.server.util; + +import static com.hedera.block.server.producer.Util.getFakeHash; + +import com.hedera.hapi.block.Acknowledgement; +import com.hedera.hapi.block.BlockItemSetUnparsed; +import com.hedera.hapi.block.BlockItemUnparsed; +import com.hedera.hapi.block.ItemAcknowledgement; +import com.hedera.hapi.block.PublishStreamRequestUnparsed; +import com.hedera.hapi.block.SubscribeStreamRequest; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.security.NoSuchAlgorithmException; +import java.util.List; + +public final class PbjProtoTestUtils { + private PbjProtoTestUtils() {} + + public static Acknowledgement buildAck(@NonNull final List blockItems) + throws NoSuchAlgorithmException { + ItemAcknowledgement itemAck = ItemAcknowledgement.newBuilder() + .itemsHash(Bytes.wrap(getFakeHash(blockItems))) + .build(); + + return Acknowledgement.newBuilder().itemAck(itemAck).build(); + } + + public static Bytes buildEmptyPublishStreamRequest() { + return PublishStreamRequestUnparsed.PROTOBUF.toBytes(PublishStreamRequestUnparsed.newBuilder() + .blockItems(BlockItemSetUnparsed.newBuilder().build()) + .build()); + } + + public static Bytes buildEmptySubscribeStreamRequest() { + return SubscribeStreamRequest.PROTOBUF.toBytes( + SubscribeStreamRequest.newBuilder().build()); + } +} diff --git a/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java b/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java index 219d52aea..eac41ae0f 100644 --- a/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java +++ b/server/src/test/java/com/hedera/block/server/util/PersistTestUtils.java @@ -19,7 +19,7 @@ import static java.lang.System.Logger; import static java.lang.System.Logger.Level.INFO; -import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.block.BlockItemUnparsed; import com.hedera.hapi.block.stream.BlockProof; import com.hedera.hapi.block.stream.input.EventHeader; import com.hedera.hapi.block.stream.output.BlockHeader; @@ -38,10 +38,9 @@ public final class PersistTestUtils { private PersistTestUtils() {} - public static void writeBlockItemToPath(final Path path, final BlockItem blockItem) - throws IOException { + public static void writeBlockItemToPath(final Path path, final BlockItemUnparsed blockItem) throws IOException { - Bytes bytes = BlockItem.PROTOBUF.toBytes(blockItem); + Bytes bytes = BlockItemUnparsed.PROTOBUF.toBytes(blockItem); writeBytesToPath(path, bytes.toByteArray()); } @@ -52,46 +51,40 @@ public static void writeBytesToPath(final Path path, final byte[] bytes) throws } } - public static List generateBlockItems(int numOfBlocks) { + public static List generateBlockItemsUnparsed(int numOfBlocks) { - List blockItems = new ArrayList<>(); + List blockItems = new ArrayList<>(); for (int i = 1; i <= numOfBlocks; i++) { for (int j = 1; j <= 10; j++) { switch (j) { case 1: // First block is always the header - blockItems.add( - BlockItem.newBuilder() - .blockHeader( - BlockHeader.newBuilder() - .number(i) - .softwareVersion( - SemanticVersion.newBuilder() - .major(1) - .minor(0) - .build()) - .build()) - .build()); + blockItems.add(BlockItemUnparsed.newBuilder() + .blockHeader(BlockHeader.PROTOBUF.toBytes(BlockHeader.newBuilder() + .number(i) + .softwareVersion(SemanticVersion.newBuilder() + .major(1) + .minor(0) + .build()) + .build())) + .build()); break; case 10: // Last block is always the state proof - blockItems.add( - BlockItem.newBuilder() - .blockProof(BlockProof.newBuilder().block(i).build()) - .build()); + blockItems.add(BlockItemUnparsed.newBuilder() + .blockProof(BlockProof.PROTOBUF.toBytes( + BlockProof.newBuilder().block(i).build())) + .build()); break; default: // Middle blocks are events - blockItems.add( - BlockItem.newBuilder() - .eventHeader( - EventHeader.newBuilder() - .eventCore( - EventCore.newBuilder() - .creatorNodeId(i) - .build()) - .build()) - .build()); + blockItems.add(BlockItemUnparsed.newBuilder() + .eventHeader(EventHeader.PROTOBUF.toBytes(EventHeader.newBuilder() + .eventCore(EventCore.newBuilder() + .creatorNodeId(i) + .build()) + .build())) + .build()); break; } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 6d46a758c..39d8cc392 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -47,8 +47,9 @@ dependencyResolutionManagement { // Define a constant for protobuf version. val protobufVersion = "4.28.2" - val helidonVersion = "4.1.1" + val grpcIoVersion = "1.65.1" + var pbjVersion = "0.9.10" // Compile time dependencies version("io.helidon.webserver.http2", helidonVersion) @@ -69,22 +70,21 @@ dependencyResolutionManagement { version("info.picocli", "4.7.6") // gRPC dependencies for the stream subproject - version("io.grpc", "1.65.1") - version("io.grpc.protobuf", "1.65.1") - version("io.grpc.stub", "1.65.1") + version("io.grpc", grpcIoVersion) + version("io.grpc.protobuf", grpcIoVersion) + version("io.grpc.stub", grpcIoVersion) // netty dependency for the simulator subproject - version("io.grpc.netty.shaded", "1.65.1") + version("io.grpc.netty.shaded", grpcIoVersion) // Reference from the protobuf plugin version("google.proto", protobufVersion) - version("grpc.protobuf.grpc", "1.65.1") + version("grpc.protobuf.grpc", grpcIoVersion) // Google protobuf dependencies version("com.google.protobuf", protobufVersion) version("com.google.protobuf.util", protobufVersion) - var pbjVersion = "0.9.10" // PBJ dependencies plugin("pbj", "com.hedera.pbj.pbj-compiler").version(pbjVersion) diff --git a/stream/src/main/proto/com/hedera/hapi/block/unparsed.proto b/stream/src/main/proto/com/hedera/hapi/block/unparsed.proto new file mode 100644 index 000000000..66b69286f --- /dev/null +++ b/stream/src/main/proto/com/hedera/hapi/block/unparsed.proto @@ -0,0 +1,60 @@ +syntax = "proto3"; + +package com.hedera.hapi.block; + +option java_package = "com.hedera.hapi.block.protoc"; +// <<>> This comment is special code for setting PBJ Compiler java package +option java_multiple_files = true; + +import "block_service.proto"; + +message PublishStreamRequestUnparsed { + BlockItemSetUnparsed block_items = 1; +} + +message SubscribeStreamResponseUnparsed { + oneof response { + /** + * A final response item describing the terminal status of this stream. + *

+ * The block node server SHALL end the stream following this message. + */ + com.hedera.hapi.block.SubscribeStreamResponseCode status = 1; + + /** + * A stream response item containing one or more `BlockItem`s. + *

+ * The full stream SHALL consist of many `block_items` messages + * followed by a single `status` message. + */ + BlockItemSetUnparsed block_items = 2; + } +} + +message SingleBlockResponseUnparsed { + com.hedera.hapi.block.SingleBlockResponseCode status = 1; + BlockUnparsed block = 2; +} + +message BlockUnparsed { + repeated BlockItemUnparsed block_items = 1; +} + +message BlockItemSetUnparsed { + repeated BlockItemUnparsed block_items = 1; +} + +message BlockItemUnparsed { + oneof item { + bytes block_header = 1; + bytes event_header = 2; + bytes round_header = 3; + bytes event_transaction = 4; + bytes transaction_result = 5; + bytes transaction_output = 6; + bytes state_changes = 7; + bytes filtered_item_hash = 8; + bytes block_proof = 9; + bytes record_file = 10; + } +}