From 3042fd2e7ba5797382961d99ededd51d87c970b6 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 10 May 2022 10:11:16 -0400 Subject: [PATCH 1/9] feat: add built in metrics measure and views --- .../stats/BuiltinMetricsRecorder.java | 170 +++++++ .../stats/BuiltinMetricsRecorderTest.java | 433 ++++++++++++++++++ 2 files changed, 603 insertions(+) create mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java create mode 100644 google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java new file mode 100644 index 0000000000..51a0feb5cb --- /dev/null +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java @@ -0,0 +1,170 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import com.google.api.core.InternalApi; +import com.google.api.gax.tracing.ApiTracerFactory.OperationType; +import com.google.api.gax.tracing.SpanName; +import io.opencensus.stats.MeasureMap; +import io.opencensus.stats.StatsRecorder; +import io.opencensus.tags.TagContext; +import io.opencensus.tags.TagContextBuilder; +import io.opencensus.tags.TagKey; +import io.opencensus.tags.TagValue; +import io.opencensus.tags.Tagger; +import io.opencensus.tags.Tags; +import java.util.Map; + +/** Add built-in metrics to the measure map * */ +@InternalApi("For internal use only") +public class BuiltinMetricsRecorder { + + private final OperationType operationType; + + private final Tagger tagger; + private final StatsRecorder statsRecorder; + private final TagContext parentContext; + private final SpanName spanName; + private final Map statsAttributes; + + private MeasureMap attemptLevelNoStreaming; + private MeasureMap attemptLevelWithStreaming; + private MeasureMap operationLevelNoStreaming; + private MeasureMap operationLevelWithStreaming; + + public BuiltinMetricsRecorder( + OperationType operationType, + SpanName spanName, + Map statsAttributes, + StatsWrapper builtinMetricsWrapper) { + this.operationType = operationType; + this.tagger = Tags.getTagger(); + this.statsRecorder = builtinMetricsWrapper.getStatsRecorder(); + this.spanName = spanName; + this.parentContext = tagger.getCurrentTagContext(); + this.statsAttributes = statsAttributes; + + this.attemptLevelNoStreaming = statsRecorder.newMeasureMap(); + this.attemptLevelWithStreaming = statsRecorder.newMeasureMap(); + this.operationLevelNoStreaming = statsRecorder.newMeasureMap(); + this.operationLevelWithStreaming = statsRecorder.newMeasureMap(); + } + + public void recordAttemptLevelWithoutStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + attemptLevelNoStreaming.record(tagCtx.build()); + } + + public void recordAttemptLevelWithStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + if (operationType == OperationType.ServerStreaming + && spanName.getMethodName().equals("ReadRows")) { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); + } else { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); + } + + attemptLevelWithStreaming.record(tagCtx.build()); + } + + public void recordOperationLevelWithoutStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + operationLevelNoStreaming.record(tagCtx.build()); + } + + public void recordOperationLevelWithStreaming( + String status, String tableId, String zone, String cluster) { + TagContextBuilder tagCtx = + newTagContextBuilder(tableId, zone, cluster) + .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); + + if (operationType == OperationType.ServerStreaming + && spanName.getMethodName().equals("ReadRows")) { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); + } else { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); + } + + operationLevelWithStreaming.record(tagCtx.build()); + } + + public void recordOperationLatencies(long operationLatency) { + operationLevelWithStreaming.put(BuiltinMeasureConstants.OPERATION_LATENCIES, operationLatency); + } + + public void recordAttemptLatency(long attemptLatency) { + attemptLevelWithStreaming.put(BuiltinMeasureConstants.ATTEMPT_LATENCIES, attemptLatency); + } + + public void recordRetryCount(int attemptCount) { + operationLevelNoStreaming.put(BuiltinMeasureConstants.RETRY_COUNT, attemptCount); + } + + public void recordApplicationLatency(long applicationLatency) { + operationLevelWithStreaming.put( + BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency); + } + + public void recordFirstResponseLatency(long firstResponseLatency) { + operationLevelNoStreaming.put( + BuiltinMeasureConstants.FIRST_RESPONSE_LATENCIES, firstResponseLatency); + } + + public void recordGfeLatencies(long serverLatency) { + attemptLevelWithStreaming.put(BuiltinMeasureConstants.SERVER_LATENCIES, serverLatency); + } + + public void recordGfeMissingHeaders(long connectivityErrors) { + attemptLevelNoStreaming.put( + BuiltinMeasureConstants.CONNECTIVITY_ERROR_COUNT, connectivityErrors); + } + + public void recordBatchRequestThrottled( + long throttledTimeMs, String tableId, String zone, String cluster) { + MeasureMap measures = + statsRecorder + .newMeasureMap() + .put(BuiltinMeasureConstants.THROTTLING_LATENCIES, throttledTimeMs); + measures.record(newTagContextBuilder(tableId, zone, cluster).build()); + } + + private TagContextBuilder newTagContextBuilder(String tableId, String zone, String cluster) { + TagContextBuilder tagContextBuilder = + tagger + .toBuilder(parentContext) + .putLocal(BuiltinMeasureConstants.CLIENT_NAME, TagValue.create("bigtable-java")) + .putLocal(BuiltinMeasureConstants.METHOD, TagValue.create(spanName.toString())) + .putLocal(BuiltinMeasureConstants.TABLE, TagValue.create(tableId)) + .putLocal(BuiltinMeasureConstants.ZONE, TagValue.create(zone)) + .putLocal(BuiltinMeasureConstants.CLUSTER, TagValue.create(cluster)); + for (Map.Entry entry : statsAttributes.entrySet()) { + tagContextBuilder.putLocal(TagKey.create(entry.getKey()), TagValue.create(entry.getValue())); + } + return tagContextBuilder; + } +} diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java new file mode 100644 index 0000000000..8f0c2688bf --- /dev/null +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java @@ -0,0 +1,433 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.bigtable.stats; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.tracing.ApiTracerFactory; +import com.google.api.gax.tracing.SpanName; +import com.google.common.collect.ImmutableMap; +import io.grpc.Status; +import org.junit.Before; +import org.junit.Test; + +public class BuiltinMetricsRecorderTest { + + private final String PROJECT_ID = "fake-project"; + private final String INSTANCE_ID = "fake-instance"; + private final String APP_PROFILE_ID = "fake-app-profile"; + + private final String TABLE_ID = "fake-table-id"; + private final String ZONE = "fake-zone"; + private final String CLUSTER = "fake-cluster"; + + private StatsWrapper wrapper; + + @Before + public void setup() { + this.wrapper = new StatsWrapper(true); + BuiltinViews views = new BuiltinViews(wrapper); + views.registerBigtableBuiltinViews(); + } + + @Test + public void testStreamingOperation() throws InterruptedException { + BuiltinMetricsRecorder tracer = + new BuiltinMetricsRecorder( + ApiTracerFactory.OperationType.ServerStreaming, + SpanName.of("Bigtable", "ReadRows"), + ImmutableMap.of( + BuiltinMeasureConstants.PROJECT_ID.getName(), PROJECT_ID, + BuiltinMeasureConstants.INSTANCE_ID.getName(), INSTANCE_ID, + BuiltinMeasureConstants.APP_PROFILE.getName(), APP_PROFILE_ID), + wrapper); + + long operationLatency = 1234; + int attemptCount = 2; + long attemptLatency = 56; + long serverLatency = 78; + long applicationLatency = 901; + long connectivityErrorCount = 15; + long throttlingLatency = 50; + long firstResponseLatency = 90; + + tracer.recordOperationLatencies(operationLatency); + tracer.recordRetryCount(attemptCount); + tracer.recordAttemptLatency(attemptLatency); + tracer.recordApplicationLatency(applicationLatency); + tracer.recordGfeLatencies(serverLatency); + tracer.recordGfeMissingHeaders(connectivityErrorCount); + tracer.recordFirstResponseLatency(firstResponseLatency); + tracer.recordBatchRequestThrottled(throttlingLatency, TABLE_ID, ZONE, CLUSTER); + + tracer.recordAttemptLevelWithoutStreaming( + Status.UNAVAILABLE.toString(), TABLE_ID, ZONE, CLUSTER); + tracer.recordAttemptLevelWithStreaming(Status.ABORTED.toString(), TABLE_ID, ZONE, CLUSTER); + tracer.recordOperationLevelWithoutStreaming("OK", TABLE_ID, ZONE, CLUSTER); + tracer.recordOperationLevelWithStreaming("OK", TABLE_ID, ZONE, CLUSTER); + + Thread.sleep(100); + + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.OPERATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, "OK", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java", + BuiltinMeasureConstants.STREAMING, "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(operationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.RETRY_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.SERVER_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(serverLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "true"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(applicationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.STATUS, + Status.UNAVAILABLE.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(connectivityErrorCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(throttlingLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.ReadRows", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(firstResponseLatency); + } + + @Test + public void testUnaryOperations() throws InterruptedException { + BuiltinMetricsRecorder tracer = + new BuiltinMetricsRecorder( + ApiTracerFactory.OperationType.ServerStreaming, + SpanName.of("Bigtable", "MutateRow"), + ImmutableMap.of( + BuiltinMeasureConstants.PROJECT_ID.getName(), PROJECT_ID, + BuiltinMeasureConstants.INSTANCE_ID.getName(), INSTANCE_ID, + BuiltinMeasureConstants.APP_PROFILE.getName(), APP_PROFILE_ID), + wrapper); + + long operationLatency = 1234; + int attemptCount = 2; + long attemptLatency = 56; + long serverLatency = 78; + long applicationLatency = 901; + long connectivityErrorCount = 15; + long throttlingLatency = 50; + long firstResponseLatency = 90; + + tracer.recordOperationLatencies(operationLatency); + tracer.recordRetryCount(attemptCount); + tracer.recordAttemptLatency(attemptLatency); + tracer.recordApplicationLatency(applicationLatency); + tracer.recordGfeLatencies(serverLatency); + tracer.recordGfeMissingHeaders(connectivityErrorCount); + tracer.recordFirstResponseLatency(firstResponseLatency); + tracer.recordBatchRequestThrottled(throttlingLatency, TABLE_ID, ZONE, CLUSTER); + + tracer.recordOperationLevelWithStreaming("OK", TABLE_ID, ZONE, CLUSTER); + tracer.recordOperationLevelWithoutStreaming("OK", TABLE_ID, ZONE, CLUSTER); + tracer.recordAttemptLevelWithoutStreaming( + Status.UNAVAILABLE.toString(), TABLE_ID, ZONE, CLUSTER); + tracer.recordAttemptLevelWithStreaming(Status.ABORTED.toString(), TABLE_ID, ZONE, CLUSTER); + + Thread.sleep(100); + + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.OPERATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, "OK", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java", + BuiltinMeasureConstants.STREAMING, "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(operationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.RETRY_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(attemptCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.SERVER_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + Status.ABORTED.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(serverLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.STREAMING, + "false"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(applicationLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.STATUS, + Status.UNAVAILABLE.toString(), + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(connectivityErrorCount); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, "Bigtable.MutateRow", + BuiltinMeasureConstants.TABLE, TABLE_ID, + BuiltinMeasureConstants.ZONE, ZONE, + BuiltinMeasureConstants.CLUSTER, CLUSTER, + BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(throttlingLatency); + assertThat( + wrapper.getAggregationValueAsLong( + BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, + ImmutableMap.of( + BuiltinMeasureConstants.METHOD, + "Bigtable.MutateRow", + BuiltinMeasureConstants.TABLE, + TABLE_ID, + BuiltinMeasureConstants.ZONE, + ZONE, + BuiltinMeasureConstants.CLUSTER, + CLUSTER, + BuiltinMeasureConstants.STATUS, + "OK", + BuiltinMeasureConstants.CLIENT_NAME, + "bigtable-java"), + PROJECT_ID, + INSTANCE_ID, + APP_PROFILE_ID)) + .isEqualTo(firstResponseLatency); + } +} From 278b2a6458313fbaad2c4355dfb63a73fd6c3a83 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 11 May 2022 14:20:34 -0400 Subject: [PATCH 2/9] remove status from application latency --- .../stats/BuiltinMetricsRecorder.java | 19 ++++++++++++++++--- .../stats/BuiltinMetricsRecorderTest.java | 4 ++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java index 51a0feb5cb..a66da11d0b 100644 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java +++ b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java @@ -125,9 +125,22 @@ public void recordRetryCount(int attemptCount) { operationLevelNoStreaming.put(BuiltinMeasureConstants.RETRY_COUNT, attemptCount); } - public void recordApplicationLatency(long applicationLatency) { - operationLevelWithStreaming.put( - BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency); + public void recordApplicationLatency( + long applicationLatency, String tableId, String zone, String cluster) { + MeasureMap measures = + statsRecorder + .newMeasureMap() + .put(BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency); + + TagContextBuilder tagCtx = newTagContextBuilder(tableId, zone, cluster); + if (operationType == OperationType.ServerStreaming + && spanName.getMethodName().equals("ReadRows")) { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); + } else { + tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); + } + + measures.record(tagCtx.build()); } public void recordFirstResponseLatency(long firstResponseLatency) { diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java index 8f0c2688bf..ce0cb0e011 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java @@ -67,7 +67,7 @@ public void testStreamingOperation() throws InterruptedException { tracer.recordOperationLatencies(operationLatency); tracer.recordRetryCount(attemptCount); tracer.recordAttemptLatency(attemptLatency); - tracer.recordApplicationLatency(applicationLatency); + tracer.recordApplicationLatency(applicationLatency, TABLE_ID, ZONE, CLUSTER); tracer.recordGfeLatencies(serverLatency); tracer.recordGfeMissingHeaders(connectivityErrorCount); tracer.recordFirstResponseLatency(firstResponseLatency); @@ -261,7 +261,7 @@ public void testUnaryOperations() throws InterruptedException { tracer.recordOperationLatencies(operationLatency); tracer.recordRetryCount(attemptCount); tracer.recordAttemptLatency(attemptLatency); - tracer.recordApplicationLatency(applicationLatency); + tracer.recordApplicationLatency(applicationLatency, TABLE_ID, ZONE, CLUSTER); tracer.recordGfeLatencies(serverLatency); tracer.recordGfeMissingHeaders(connectivityErrorCount); tracer.recordFirstResponseLatency(firstResponseLatency); From e7886cb19e5ce983533a4002a85bb98ba5ad2ff9 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 7 Jun 2022 15:56:56 -0400 Subject: [PATCH 3/9] Rename methods and add comments --- .../stats/BuiltinMetricsRecorder.java | 183 -------- .../stats/BuiltinMetricsRecorderTest.java | 433 ------------------ .../stats/StatsRecorderWrapperTest.java | 1 + 3 files changed, 1 insertion(+), 616 deletions(-) delete mode 100644 google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java delete mode 100644 google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java diff --git a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java b/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java deleted file mode 100644 index a66da11d0b..0000000000 --- a/google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.stats; - -import com.google.api.core.InternalApi; -import com.google.api.gax.tracing.ApiTracerFactory.OperationType; -import com.google.api.gax.tracing.SpanName; -import io.opencensus.stats.MeasureMap; -import io.opencensus.stats.StatsRecorder; -import io.opencensus.tags.TagContext; -import io.opencensus.tags.TagContextBuilder; -import io.opencensus.tags.TagKey; -import io.opencensus.tags.TagValue; -import io.opencensus.tags.Tagger; -import io.opencensus.tags.Tags; -import java.util.Map; - -/** Add built-in metrics to the measure map * */ -@InternalApi("For internal use only") -public class BuiltinMetricsRecorder { - - private final OperationType operationType; - - private final Tagger tagger; - private final StatsRecorder statsRecorder; - private final TagContext parentContext; - private final SpanName spanName; - private final Map statsAttributes; - - private MeasureMap attemptLevelNoStreaming; - private MeasureMap attemptLevelWithStreaming; - private MeasureMap operationLevelNoStreaming; - private MeasureMap operationLevelWithStreaming; - - public BuiltinMetricsRecorder( - OperationType operationType, - SpanName spanName, - Map statsAttributes, - StatsWrapper builtinMetricsWrapper) { - this.operationType = operationType; - this.tagger = Tags.getTagger(); - this.statsRecorder = builtinMetricsWrapper.getStatsRecorder(); - this.spanName = spanName; - this.parentContext = tagger.getCurrentTagContext(); - this.statsAttributes = statsAttributes; - - this.attemptLevelNoStreaming = statsRecorder.newMeasureMap(); - this.attemptLevelWithStreaming = statsRecorder.newMeasureMap(); - this.operationLevelNoStreaming = statsRecorder.newMeasureMap(); - this.operationLevelWithStreaming = statsRecorder.newMeasureMap(); - } - - public void recordAttemptLevelWithoutStreaming( - String status, String tableId, String zone, String cluster) { - TagContextBuilder tagCtx = - newTagContextBuilder(tableId, zone, cluster) - .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); - - attemptLevelNoStreaming.record(tagCtx.build()); - } - - public void recordAttemptLevelWithStreaming( - String status, String tableId, String zone, String cluster) { - TagContextBuilder tagCtx = - newTagContextBuilder(tableId, zone, cluster) - .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); - - if (operationType == OperationType.ServerStreaming - && spanName.getMethodName().equals("ReadRows")) { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); - } else { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); - } - - attemptLevelWithStreaming.record(tagCtx.build()); - } - - public void recordOperationLevelWithoutStreaming( - String status, String tableId, String zone, String cluster) { - TagContextBuilder tagCtx = - newTagContextBuilder(tableId, zone, cluster) - .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); - - operationLevelNoStreaming.record(tagCtx.build()); - } - - public void recordOperationLevelWithStreaming( - String status, String tableId, String zone, String cluster) { - TagContextBuilder tagCtx = - newTagContextBuilder(tableId, zone, cluster) - .putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status)); - - if (operationType == OperationType.ServerStreaming - && spanName.getMethodName().equals("ReadRows")) { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); - } else { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); - } - - operationLevelWithStreaming.record(tagCtx.build()); - } - - public void recordOperationLatencies(long operationLatency) { - operationLevelWithStreaming.put(BuiltinMeasureConstants.OPERATION_LATENCIES, operationLatency); - } - - public void recordAttemptLatency(long attemptLatency) { - attemptLevelWithStreaming.put(BuiltinMeasureConstants.ATTEMPT_LATENCIES, attemptLatency); - } - - public void recordRetryCount(int attemptCount) { - operationLevelNoStreaming.put(BuiltinMeasureConstants.RETRY_COUNT, attemptCount); - } - - public void recordApplicationLatency( - long applicationLatency, String tableId, String zone, String cluster) { - MeasureMap measures = - statsRecorder - .newMeasureMap() - .put(BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency); - - TagContextBuilder tagCtx = newTagContextBuilder(tableId, zone, cluster); - if (operationType == OperationType.ServerStreaming - && spanName.getMethodName().equals("ReadRows")) { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true")); - } else { - tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false")); - } - - measures.record(tagCtx.build()); - } - - public void recordFirstResponseLatency(long firstResponseLatency) { - operationLevelNoStreaming.put( - BuiltinMeasureConstants.FIRST_RESPONSE_LATENCIES, firstResponseLatency); - } - - public void recordGfeLatencies(long serverLatency) { - attemptLevelWithStreaming.put(BuiltinMeasureConstants.SERVER_LATENCIES, serverLatency); - } - - public void recordGfeMissingHeaders(long connectivityErrors) { - attemptLevelNoStreaming.put( - BuiltinMeasureConstants.CONNECTIVITY_ERROR_COUNT, connectivityErrors); - } - - public void recordBatchRequestThrottled( - long throttledTimeMs, String tableId, String zone, String cluster) { - MeasureMap measures = - statsRecorder - .newMeasureMap() - .put(BuiltinMeasureConstants.THROTTLING_LATENCIES, throttledTimeMs); - measures.record(newTagContextBuilder(tableId, zone, cluster).build()); - } - - private TagContextBuilder newTagContextBuilder(String tableId, String zone, String cluster) { - TagContextBuilder tagContextBuilder = - tagger - .toBuilder(parentContext) - .putLocal(BuiltinMeasureConstants.CLIENT_NAME, TagValue.create("bigtable-java")) - .putLocal(BuiltinMeasureConstants.METHOD, TagValue.create(spanName.toString())) - .putLocal(BuiltinMeasureConstants.TABLE, TagValue.create(tableId)) - .putLocal(BuiltinMeasureConstants.ZONE, TagValue.create(zone)) - .putLocal(BuiltinMeasureConstants.CLUSTER, TagValue.create(cluster)); - for (Map.Entry entry : statsAttributes.entrySet()) { - tagContextBuilder.putLocal(TagKey.create(entry.getKey()), TagValue.create(entry.getValue())); - } - return tagContextBuilder; - } -} diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java deleted file mode 100644 index ce0cb0e011..0000000000 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorderTest.java +++ /dev/null @@ -1,433 +0,0 @@ -/* - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.cloud.bigtable.stats; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.api.gax.tracing.ApiTracerFactory; -import com.google.api.gax.tracing.SpanName; -import com.google.common.collect.ImmutableMap; -import io.grpc.Status; -import org.junit.Before; -import org.junit.Test; - -public class BuiltinMetricsRecorderTest { - - private final String PROJECT_ID = "fake-project"; - private final String INSTANCE_ID = "fake-instance"; - private final String APP_PROFILE_ID = "fake-app-profile"; - - private final String TABLE_ID = "fake-table-id"; - private final String ZONE = "fake-zone"; - private final String CLUSTER = "fake-cluster"; - - private StatsWrapper wrapper; - - @Before - public void setup() { - this.wrapper = new StatsWrapper(true); - BuiltinViews views = new BuiltinViews(wrapper); - views.registerBigtableBuiltinViews(); - } - - @Test - public void testStreamingOperation() throws InterruptedException { - BuiltinMetricsRecorder tracer = - new BuiltinMetricsRecorder( - ApiTracerFactory.OperationType.ServerStreaming, - SpanName.of("Bigtable", "ReadRows"), - ImmutableMap.of( - BuiltinMeasureConstants.PROJECT_ID.getName(), PROJECT_ID, - BuiltinMeasureConstants.INSTANCE_ID.getName(), INSTANCE_ID, - BuiltinMeasureConstants.APP_PROFILE.getName(), APP_PROFILE_ID), - wrapper); - - long operationLatency = 1234; - int attemptCount = 2; - long attemptLatency = 56; - long serverLatency = 78; - long applicationLatency = 901; - long connectivityErrorCount = 15; - long throttlingLatency = 50; - long firstResponseLatency = 90; - - tracer.recordOperationLatencies(operationLatency); - tracer.recordRetryCount(attemptCount); - tracer.recordAttemptLatency(attemptLatency); - tracer.recordApplicationLatency(applicationLatency, TABLE_ID, ZONE, CLUSTER); - tracer.recordGfeLatencies(serverLatency); - tracer.recordGfeMissingHeaders(connectivityErrorCount); - tracer.recordFirstResponseLatency(firstResponseLatency); - tracer.recordBatchRequestThrottled(throttlingLatency, TABLE_ID, ZONE, CLUSTER); - - tracer.recordAttemptLevelWithoutStreaming( - Status.UNAVAILABLE.toString(), TABLE_ID, ZONE, CLUSTER); - tracer.recordAttemptLevelWithStreaming(Status.ABORTED.toString(), TABLE_ID, ZONE, CLUSTER); - tracer.recordOperationLevelWithoutStreaming("OK", TABLE_ID, ZONE, CLUSTER); - tracer.recordOperationLevelWithStreaming("OK", TABLE_ID, ZONE, CLUSTER); - - Thread.sleep(100); - - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.OPERATION_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, "OK", - BuiltinMeasureConstants.TABLE, TABLE_ID, - BuiltinMeasureConstants.ZONE, ZONE, - BuiltinMeasureConstants.CLUSTER, CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java", - BuiltinMeasureConstants.STREAMING, "true"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(operationLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, - Status.ABORTED.toString(), - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "true"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(attemptLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.RETRY_COUNT_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(attemptCount); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.SERVER_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, - Status.ABORTED.toString(), - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "true", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(serverLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "true"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(applicationLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.STATUS, - Status.UNAVAILABLE.toString(), - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(connectivityErrorCount); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, "Bigtable.ReadRows", - BuiltinMeasureConstants.TABLE, TABLE_ID, - BuiltinMeasureConstants.ZONE, ZONE, - BuiltinMeasureConstants.CLUSTER, CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(throttlingLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.ReadRows", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(firstResponseLatency); - } - - @Test - public void testUnaryOperations() throws InterruptedException { - BuiltinMetricsRecorder tracer = - new BuiltinMetricsRecorder( - ApiTracerFactory.OperationType.ServerStreaming, - SpanName.of("Bigtable", "MutateRow"), - ImmutableMap.of( - BuiltinMeasureConstants.PROJECT_ID.getName(), PROJECT_ID, - BuiltinMeasureConstants.INSTANCE_ID.getName(), INSTANCE_ID, - BuiltinMeasureConstants.APP_PROFILE.getName(), APP_PROFILE_ID), - wrapper); - - long operationLatency = 1234; - int attemptCount = 2; - long attemptLatency = 56; - long serverLatency = 78; - long applicationLatency = 901; - long connectivityErrorCount = 15; - long throttlingLatency = 50; - long firstResponseLatency = 90; - - tracer.recordOperationLatencies(operationLatency); - tracer.recordRetryCount(attemptCount); - tracer.recordAttemptLatency(attemptLatency); - tracer.recordApplicationLatency(applicationLatency, TABLE_ID, ZONE, CLUSTER); - tracer.recordGfeLatencies(serverLatency); - tracer.recordGfeMissingHeaders(connectivityErrorCount); - tracer.recordFirstResponseLatency(firstResponseLatency); - tracer.recordBatchRequestThrottled(throttlingLatency, TABLE_ID, ZONE, CLUSTER); - - tracer.recordOperationLevelWithStreaming("OK", TABLE_ID, ZONE, CLUSTER); - tracer.recordOperationLevelWithoutStreaming("OK", TABLE_ID, ZONE, CLUSTER); - tracer.recordAttemptLevelWithoutStreaming( - Status.UNAVAILABLE.toString(), TABLE_ID, ZONE, CLUSTER); - tracer.recordAttemptLevelWithStreaming(Status.ABORTED.toString(), TABLE_ID, ZONE, CLUSTER); - - Thread.sleep(100); - - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.OPERATION_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, "OK", - BuiltinMeasureConstants.TABLE, TABLE_ID, - BuiltinMeasureConstants.ZONE, ZONE, - BuiltinMeasureConstants.CLUSTER, CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java", - BuiltinMeasureConstants.STREAMING, "false"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(operationLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.ATTEMPT_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, - Status.ABORTED.toString(), - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "false"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(attemptLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.RETRY_COUNT_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(attemptCount); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.SERVER_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, - Status.ABORTED.toString(), - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "false", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(serverLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.APPLICATION_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.STREAMING, - "false"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(applicationLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.CONNECTIVITY_ERROR_COUNT_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.STATUS, - Status.UNAVAILABLE.toString(), - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(connectivityErrorCount); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.THROTTLING_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, "Bigtable.MutateRow", - BuiltinMeasureConstants.TABLE, TABLE_ID, - BuiltinMeasureConstants.ZONE, ZONE, - BuiltinMeasureConstants.CLUSTER, CLUSTER, - BuiltinMeasureConstants.CLIENT_NAME, "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(throttlingLatency); - assertThat( - wrapper.getAggregationValueAsLong( - BuiltinViewConstants.FIRST_RESPONSE_LATENCIES_VIEW, - ImmutableMap.of( - BuiltinMeasureConstants.METHOD, - "Bigtable.MutateRow", - BuiltinMeasureConstants.TABLE, - TABLE_ID, - BuiltinMeasureConstants.ZONE, - ZONE, - BuiltinMeasureConstants.CLUSTER, - CLUSTER, - BuiltinMeasureConstants.STATUS, - "OK", - BuiltinMeasureConstants.CLIENT_NAME, - "bigtable-java"), - PROJECT_ID, - INSTANCE_ID, - APP_PROFILE_ID)) - .isEqualTo(firstResponseLatency); - } -} diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java index ed67472623..af21842188 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java @@ -149,6 +149,7 @@ public void testStreamingOperation() throws InterruptedException { INSTANCE_ID, APP_PROFILE_ID, statsComponent.getViewManager())) + .isEqualTo(attemptCount); assertThat( getAggregationValueAsLong( From acfb71c5593d67db36fc2bd6bd445382404db278 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 8 Jun 2022 15:09:45 -0400 Subject: [PATCH 4/9] update based on comments --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 74f02cdeb4..d786fd2741 100644 --- a/pom.xml +++ b/pom.xml @@ -336,6 +336,7 @@ google-cloud-bigtable + google-cloud-bigtable-stats grpc-google-cloud-bigtable-admin-v2 grpc-google-cloud-bigtable-v2 proto-google-cloud-bigtable-admin-v2 From 5e7b6f32b34d75faf0c2c572f63b92b55bca102f Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Tue, 10 May 2022 11:49:36 -0400 Subject: [PATCH 5/9] feat: update tracers to use built in metrics --- .../cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java | 1 + .../data/v2/stub/metrics/BuiltinMetricsTracerTest.java | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index b6d17baadf..a693d080d5 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -90,6 +90,7 @@ import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsUserCallable; import com.google.cloud.bigtable.data.v2.stub.readrows.RowMergingCallable; import com.google.cloud.bigtable.gaxx.retrying.ApiResultRetryAlgorithm; +import com.google.cloud.bigtable.stats.StatsWrapper; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index a48df92254..a7ec72b7ff 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -1,5 +1,9 @@ /* +<<<<<<< HEAD * Copyright 2022 Google LLC +======= + * Copyright 2021 Google LLC +>>>>>>> 15af3181 (feat: update tracers to use built in metrics) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 1b186f1b9603b824b2016bb90d9ba8671230da0b Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 8 Jun 2022 20:50:02 -0400 Subject: [PATCH 6/9] update on comments --- .../data/v2/stub/metrics/BuiltinMetricsTracerTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index a7ec72b7ff..d06db6800b 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -1,9 +1,13 @@ /* +<<<<<<< HEAD <<<<<<< HEAD * Copyright 2022 Google LLC ======= * Copyright 2021 Google LLC >>>>>>> 15af3181 (feat: update tracers to use built in metrics) +======= + * Copyright 2022 Google LLC +>>>>>>> 82e63f02 (update on comments) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From f6100d215d540daa7849200ed93848de7f535f6e Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 8 Jun 2022 21:08:16 -0400 Subject: [PATCH 7/9] fix: rename HeaderTracer callables to BigtableTracer callables --- .../clirr-ignored-differences.xml | 10 ++++ .../data/v2/stub/EnhancedBigtableStub.java | 49 ++++++++++--------- ...a => BigtableTracerStreamingCallable.java} | 20 ++++---- ....java => BigtableTracerUnaryCallable.java} | 8 +-- ...t.java => BigtableTracerCallableTest.java} | 2 +- 5 files changed, 48 insertions(+), 41 deletions(-) rename google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/{HeaderTracerStreamingCallable.java => BigtableTracerStreamingCallable.java} (85%) rename google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/{HeaderTracerUnaryCallable.java => BigtableTracerUnaryCallable.java} (89%) rename google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/{HeaderTracerCallableTest.java => BigtableTracerCallableTest.java} (99%) diff --git a/google-cloud-bigtable/clirr-ignored-differences.xml b/google-cloud-bigtable/clirr-ignored-differences.xml index 767e12e548..588327d0de 100644 --- a/google-cloud-bigtable/clirr-ignored-differences.xml +++ b/google-cloud-bigtable/clirr-ignored-differences.xml @@ -61,4 +61,14 @@ com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerUnaryCallable * + + + 8001 + com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerStreamingCallable + + + + 8001 + com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerUnaryCallable + diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index a693d080d5..d01e9186a4 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -70,10 +70,10 @@ import com.google.cloud.bigtable.data.v2.models.RowAdapter; import com.google.cloud.bigtable.data.v2.models.RowMutation; import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; +import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerStreamingCallable; +import com.google.cloud.bigtable.data.v2.stub.metrics.BigtableTracerUnaryCallable; import com.google.cloud.bigtable.data.v2.stub.metrics.BuiltinMetricsTracerFactory; import com.google.cloud.bigtable.data.v2.stub.metrics.CompositeTracerFactory; -import com.google.cloud.bigtable.data.v2.stub.metrics.HeaderTracerStreamingCallable; -import com.google.cloud.bigtable.data.v2.stub.metrics.HeaderTracerUnaryCallable; import com.google.cloud.bigtable.data.v2.stub.metrics.MetricsTracerFactory; import com.google.cloud.bigtable.data.v2.stub.metrics.RpcMeasureConstants; import com.google.cloud.bigtable.data.v2.stub.metrics.StatsHeadersServerStreamingCallable; @@ -378,7 +378,7 @@ public UnaryCallable createReadRowCallable(RowAdapter *
  • Upon receiving the response stream, it will merge the {@link * com.google.bigtable.v2.ReadRowsResponse.CellChunk}s in logical rows. The actual row * implementation can be configured by the {@code rowAdapter} parameter. - *
  • Add header tracer for tracking GFE metrics. + *
  • Add bigtable tracer for tracking bigtable specific metrics. *
  • Retry/resume on failure. *
  • Filter out marker rows. * @@ -429,13 +429,13 @@ public Map extract(ReadRowsRequest readRowsRequest) { ServerStreamingCallable watched = Callables.watched(merging, innerSettings, clientContext); - ServerStreamingCallable withHeaderTracer = - new HeaderTracerStreamingCallable<>(watched); + ServerStreamingCallable withBigtableTracer = + new BigtableTracerStreamingCallable<>(watched); // Retry logic is split into 2 parts to workaround a rare edge case described in // ReadRowsRetryCompletedCallable ServerStreamingCallable retrying1 = - new ReadRowsRetryCompletedCallable<>(withHeaderTracer); + new ReadRowsRetryCompletedCallable<>(withBigtableTracer); ServerStreamingCallable retrying2 = Callables.retrying(retrying1, innerSettings, clientContext); @@ -474,11 +474,11 @@ private UnaryCallable> createBulkReadRowsCallable( UnaryCallable> tracedBatcher = new TracedBatcherUnaryCallable<>(readRowsUserCallable.all()); - UnaryCallable> withHeaderTracer = - new HeaderTracerUnaryCallable<>(tracedBatcher); + UnaryCallable> withBigtableTracer = + new BigtableTracerUnaryCallable<>(tracedBatcher); UnaryCallable> traced = - new TracedUnaryCallable<>(withHeaderTracer, clientContext.getTracerFactory(), span); + new TracedUnaryCallable<>(withBigtableTracer, clientContext.getTracerFactory(), span); return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } @@ -520,11 +520,11 @@ public Map extract( UnaryCallable> withStatsHeaders = new StatsHeadersUnaryCallable<>(spoolable); - UnaryCallable> withHeaderTracer = - new HeaderTracerUnaryCallable<>(withStatsHeaders); + UnaryCallable> withBigtableTracer = + new BigtableTracerUnaryCallable<>(withStatsHeaders); UnaryCallable> retryable = - Callables.retrying(withHeaderTracer, settings.sampleRowKeysSettings(), clientContext); + Callables.retrying(withBigtableTracer, settings.sampleRowKeysSettings(), clientContext); return createUserFacingUnaryCallable( methodName, new SampleRowKeysCallable(retryable, requestContext)); @@ -559,11 +559,11 @@ public Map extract(MutateRowRequest mutateRowRequest) { UnaryCallable withStatsHeaders = new StatsHeadersUnaryCallable<>(base); - UnaryCallable withHeaderTracer = - new HeaderTracerUnaryCallable<>(withStatsHeaders); + UnaryCallable withBigtableTracer = + new BigtableTracerUnaryCallable<>(withStatsHeaders); UnaryCallable retrying = - Callables.retrying(withHeaderTracer, settings.mutateRowSettings(), clientContext); + Callables.retrying(withBigtableTracer, settings.mutateRowSettings(), clientContext); return createUserFacingUnaryCallable( methodName, new MutateRowCallable(retrying, requestContext)); @@ -606,10 +606,10 @@ private UnaryCallable createBulkMutateRowsCallable() { UnaryCallable tracedBatcherUnaryCallable = new TracedBatcherUnaryCallable<>(userFacing); - UnaryCallable withHeaderTracer = - new HeaderTracerUnaryCallable<>(tracedBatcherUnaryCallable); + UnaryCallable withBigtableTracer = + new BigtableTracerUnaryCallable<>(tracedBatcherUnaryCallable); UnaryCallable traced = - new TracedUnaryCallable<>(withHeaderTracer, clientContext.getTracerFactory(), spanName); + new TracedUnaryCallable<>(withBigtableTracer, clientContext.getTracerFactory(), spanName); return traced.withDefaultCallContext(clientContext.getDefaultCallContext()); } @@ -747,11 +747,11 @@ public Map extract( UnaryCallable withStatsHeaders = new StatsHeadersUnaryCallable<>(base); - UnaryCallable withHeaderTracer = - new HeaderTracerUnaryCallable<>(withStatsHeaders); + UnaryCallable withBigtableTracer = + new BigtableTracerUnaryCallable<>(withStatsHeaders); UnaryCallable retrying = - Callables.retrying(withHeaderTracer, settings.checkAndMutateRowSettings(), clientContext); + Callables.retrying(withBigtableTracer, settings.checkAndMutateRowSettings(), clientContext); return createUserFacingUnaryCallable( methodName, new CheckAndMutateRowCallable(retrying, requestContext)); @@ -788,11 +788,12 @@ public Map extract(ReadModifyWriteRowRequest request) { new StatsHeadersUnaryCallable<>(base); String methodName = "ReadModifyWriteRow"; - UnaryCallable withHeaderTracer = - new HeaderTracerUnaryCallable<>(withStatsHeaders); + UnaryCallable withBigtableTracer = + new BigtableTracerUnaryCallable<>(withStatsHeaders); UnaryCallable retrying = - Callables.retrying(withHeaderTracer, settings.readModifyWriteRowSettings(), clientContext); + Callables.retrying( + withBigtableTracer, settings.readModifyWriteRowSettings(), clientContext); return createUserFacingUnaryCallable( methodName, new ReadModifyWriteRowCallable(retrying, requestContext)); diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerStreamingCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java similarity index 85% rename from google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerStreamingCallable.java rename to google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java index f73511dc4c..09bb906a89 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerStreamingCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerStreamingCallable.java @@ -28,26 +28,26 @@ import javax.annotation.Nonnull; /** - * This callable will inject a {@link GrpcResponseMetadata} to access the headers and trailers - * returned by gRPC methods upon completion. The {@link BigtableTracer} will process metrics that - * were injected in the header/trailer and publish them to OpenCensus. If {@link - * GrpcResponseMetadata#getMetadata()} returned null, it probably means that the request has never - * reached GFE, and it'll increment the gfe_header_missing_counter in this case. + * This callable will * - *

    If GFE metrics are not registered in {@link RpcViews}, skip injecting GrpcResponseMetadata. - * This is for the case where direct path is enabled, all the requests won't go through GFE and - * therefore won't have the server-timing header. + *

    -inject a {@link GrpcResponseMetadata} to access the headers and trailers returned by gRPC + * methods upon completion. The {@link BigtableTracer} will process metrics that were injected in + * the header/trailer and publish them to OpenCensus. If {@link GrpcResponseMetadata#getMetadata()} + * returned null, it probably means that the request has never reached GFE, and it'll increment the + * gfe_header_missing_counter in this case. + * + *

    -Call {@link BigtableTracer#onRequest()} to record the request events in a stream. * *

    This class is considered an internal implementation detail and not meant to be used by * applications. */ @InternalApi -public class HeaderTracerStreamingCallable +public class BigtableTracerStreamingCallable extends ServerStreamingCallable { private final ServerStreamingCallable innerCallable; - public HeaderTracerStreamingCallable( + public BigtableTracerStreamingCallable( @Nonnull ServerStreamingCallable callable) { this.innerCallable = Preconditions.checkNotNull(callable, "Inner callable must be set"); } diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerUnaryCallable.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java similarity index 89% rename from google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerUnaryCallable.java rename to google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java index adbb6c84a9..fd2eb60f27 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerUnaryCallable.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerUnaryCallable.java @@ -34,20 +34,16 @@ * GrpcResponseMetadata#getMetadata()} returned null, it probably means that the request has never * reached GFE, and it'll increment the gfe_header_missing_counter in this case. * - *

    If GFE metrics are not registered in {@link RpcViews}, skip injecting GrpcResponseMetadata. - * This is for the case where direct path is enabled, all the requests won't go through GFE and - * therefore won't have the server-timing header. - * *

    This class is considered an internal implementation detail and not meant to be used by * applications. */ @InternalApi -public class HeaderTracerUnaryCallable +public class BigtableTracerUnaryCallable extends UnaryCallable { private final UnaryCallable innerCallable; - public HeaderTracerUnaryCallable(@Nonnull UnaryCallable innerCallable) { + public BigtableTracerUnaryCallable(@Nonnull UnaryCallable innerCallable) { this.innerCallable = Preconditions.checkNotNull(innerCallable, "Inner callable must be set"); } diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerCallableTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java similarity index 99% rename from google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerCallableTest.java rename to google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java index d93859bbad..1b833f5c06 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/HeaderTracerCallableTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableTracerCallableTest.java @@ -68,7 +68,7 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) -public class HeaderTracerCallableTest { +public class BigtableTracerCallableTest { private Server server; private Server serverNoHeader; From d72d7a1e4af8b31ce0299cd89e04ad1c517591be Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 29 Jun 2022 11:12:08 -0400 Subject: [PATCH 8/9] deflake test --- .../data/v2/stub/metrics/BuiltinMetricsTracerTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index d06db6800b..81fecad3d3 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -315,7 +315,10 @@ public void testRetryCount() { stub.mutateRowCallable() .call(RowMutation.create(TABLE_ID, "random-row").setCell("cf", "q", "value")); - verify(statsRecorderWrapper).putRetryCount(retryCount.capture()); + // onOperationComplete() is called in TracerFinisher which will be called after the mutateRow + // call is returned. So there's a race between when the call returns and when the putRetryCount + // is called in onOperationCompletion(). + verify(statsRecorderWrapper, timeout(20)).putRetryCount(retryCount.capture()); assertThat(retryCount.getValue()).isEqualTo(fakeService.getAttemptCounter().get() - 1); } @@ -336,7 +339,7 @@ public void testMutateRowAttempts() { // calls releaseWaiters(). onOperationComplete() is called in TracerFinisher which will be // called after the mutateRow call is returned. So there's a race between when the call returns // and when the record() is called in onOperationCompletion(). - verify(statsRecorderWrapper, timeout(10).times(fakeService.getAttemptCounter().get() + 1)) + verify(statsRecorderWrapper, timeout(20).times(fakeService.getAttemptCounter().get() + 1)) .record(status.capture(), tableId.capture(), zone.capture(), cluster.capture()); assertThat(zone.getAllValues()).containsExactly(UNDEFINED, UNDEFINED, UNDEFINED, UNDEFINED); assertThat(cluster.getAllValues()).containsExactly(UNDEFINED, UNDEFINED, UNDEFINED, UNDEFINED); From dca390110f50fc86b06ef40dc3722954173ea614 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Wed, 29 Jun 2022 11:21:23 -0400 Subject: [PATCH 9/9] fix broken test --- .../cloud/bigtable/stats/StatsRecorderWrapperTest.java | 1 - .../cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java | 1 - .../data/v2/stub/metrics/BuiltinMetricsTracerTest.java | 8 -------- pom.xml | 1 - 4 files changed, 11 deletions(-) diff --git a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java index af21842188..ed67472623 100644 --- a/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java +++ b/google-cloud-bigtable-stats/src/test/java/com/google/cloud/bigtable/stats/StatsRecorderWrapperTest.java @@ -149,7 +149,6 @@ public void testStreamingOperation() throws InterruptedException { INSTANCE_ID, APP_PROFILE_ID, statsComponent.getViewManager())) - .isEqualTo(attemptCount); assertThat( getAggregationValueAsLong( diff --git a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java index d01e9186a4..ec237aabf7 100644 --- a/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java +++ b/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/EnhancedBigtableStub.java @@ -90,7 +90,6 @@ import com.google.cloud.bigtable.data.v2.stub.readrows.ReadRowsUserCallable; import com.google.cloud.bigtable.data.v2.stub.readrows.RowMergingCallable; import com.google.cloud.bigtable.gaxx.retrying.ApiResultRetryAlgorithm; -import com.google.cloud.bigtable.stats.StatsWrapper; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java index 81fecad3d3..9294033f26 100644 --- a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java +++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/metrics/BuiltinMetricsTracerTest.java @@ -1,13 +1,5 @@ /* -<<<<<<< HEAD -<<<<<<< HEAD * Copyright 2022 Google LLC -======= - * Copyright 2021 Google LLC ->>>>>>> 15af3181 (feat: update tracers to use built in metrics) -======= - * Copyright 2022 Google LLC ->>>>>>> 82e63f02 (update on comments) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pom.xml b/pom.xml index d786fd2741..74f02cdeb4 100644 --- a/pom.xml +++ b/pom.xml @@ -336,7 +336,6 @@ google-cloud-bigtable - google-cloud-bigtable-stats grpc-google-cloud-bigtable-admin-v2 grpc-google-cloud-bigtable-v2 proto-google-cloud-bigtable-admin-v2