From 5a08e6185020fe2c18ffeb64075b0ce33cc74778 Mon Sep 17 00:00:00 2001 From: jack-berg <34418638+jack-berg@users.noreply.github.com> Date: Wed, 25 Jan 2023 19:49:56 -0600 Subject: [PATCH] Add random errors in AdService (#694) --- CHANGELOG.md | 2 + docker-compose.yml | 1 + docs/current_architecture.md | 2 + docs/feature_flags.md | 7 +- src/adservice/Dockerfile | 2 +- src/adservice/README.md | 1 + src/adservice/settings.gradle | 2 +- .../src/main/java/hipstershop/AdService.java | 78 +++++++++++++++---- .../20220524172636_create_featureflags.exs | 6 ++ 9 files changed, 81 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7355c8bf9c..2607e6a3b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -200,6 +200,8 @@ significant modifications will be credited to OpenTelemetry Authors. ([#693](https://github.com/open-telemetry/opentelemetry-demo/pull/693)) * Update recommendationservice python base image and dependencies ([#700](https://github.com/open-telemetry/opentelemetry-demo/pull/700)) +* Add adServiceFailure feature flag triggering Ad Service errors +([#694](https://github.com/open-telemetry/opentelemetry-demo/pull/694)) * Reduce spans generated from quote service ([#702](https://github.com/open-telemetry/opentelemetry-demo/pull/702)) * Update emailservice Dockerfile to use alpine and multistage build diff --git a/docker-compose.yml b/docker-compose.yml index 5707d19e4f..4a75e1396f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,7 @@ services: - "${AD_SERVICE_PORT}" environment: - AD_SERVICE_PORT + - FEATURE_FLAG_GRPC_SERVICE_ADDR - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE diff --git a/docs/current_architecture.md b/docs/current_architecture.md index 2535af4e19..aa9a764207 100644 --- a/docs/current_architecture.md +++ b/docs/current_architecture.md @@ -52,6 +52,8 @@ frontend -->|gRPC| shippingservice -->|HTTP| quoteservice frauddetectionservice -->|TCP| queue +adservice -->|gRPC| featureflagservice + productcatalogservice -->|gRPC| featureflagservice recommendationservice -->|gRPC| featureflagservice diff --git a/docs/feature_flags.md b/docs/feature_flags.md index 0346027e49..e250f9b63c 100644 --- a/docs/feature_flags.md +++ b/docs/feature_flags.md @@ -5,7 +5,8 @@ in specific services. By default the flags are disabled. Using the Feature Flags UI you will be able to control the status of these feature flags. -| Feature Flag | Service(s) | Description | -|-------------------------|-----------------|---------------------------------------------------------------------------------------------------------| -| `productCatalogFailure` | Product Catalog | Generate an error for `GetProduct` requests with product id: `OLJCESPC7Z` | +| Feature Flag | Service(s) | Description | +|-------------------------|-----------------|----------------------------------------------------------------------------------------------------------| +| `adServiceFailure` | Ad Service | Generate an error for `GetAds` 1/10th of the time | +| `productCatalogFailure` | Product Catalog | Generate an error for `GetProduct` requests with product id: `OLJCESPC7Z` | | `recommendationCache` | Recommendation | Create a memory leak due to an exponentially growing cache. 1.4x growth, 50% of requests trigger growth. | diff --git a/src/adservice/Dockerfile b/src/adservice/Dockerfile index 318694ce05..22c6a634ca 100644 --- a/src/adservice/Dockerfile +++ b/src/adservice/Dockerfile @@ -39,4 +39,4 @@ RUN chmod 644 /usr/src/app/opentelemetry-javaagent.jar ENV JAVA_TOOL_OPTIONS=-javaagent:/usr/src/app/opentelemetry-javaagent.jar EXPOSE ${AD_SERVICE_PORT} -ENTRYPOINT [ "./build/install/hipstershop/bin/AdService" ] +ENTRYPOINT [ "./build/install/opentelemetry-demo-ad-service/bin/AdService" ] diff --git a/src/adservice/README.md b/src/adservice/README.md index 04ac27647c..bc63335729 100644 --- a/src/adservice/README.md +++ b/src/adservice/README.md @@ -20,6 +20,7 @@ To run the Ad Service: ```sh export AD_SERVICE_PORT=8080 +export FEATURE_FLAG_GRPC_SERVICE_ADDR=featureflagservice:50053 ./build/install/hipstershop/bin/AdService ``` diff --git a/src/adservice/settings.gradle b/src/adservice/settings.gradle index 0bbe0117c8..9dcbccda17 100644 --- a/src/adservice/settings.gradle +++ b/src/adservice/settings.gradle @@ -1 +1 @@ -rootProject.name = 'hipstershop' +rootProject.name = 'opentelemetry-demo-ad-service' diff --git a/src/adservice/src/main/java/hipstershop/AdService.java b/src/adservice/src/main/java/hipstershop/AdService.java index 390451e1c7..b08720b55a 100644 --- a/src/adservice/src/main/java/hipstershop/AdService.java +++ b/src/adservice/src/main/java/hipstershop/AdService.java @@ -21,9 +21,9 @@ import hipstershop.Demo.Ad; import hipstershop.Demo.AdRequest; import hipstershop.Demo.AdResponse; -import io.grpc.Server; -import io.grpc.ServerBuilder; -import io.grpc.StatusRuntimeException; +import hipstershop.Demo.GetFlagResponse; +import hipstershop.FeatureFlagServiceGrpc.FeatureFlagServiceBlockingStub; +import io.grpc.*; import io.grpc.health.v1.HealthCheckResponse.ServingStatus; import io.grpc.protobuf.services.*; import io.grpc.stub.StreamObserver; @@ -62,24 +62,40 @@ public final class AdService { private static final Tracer tracer = GlobalOpenTelemetry.getTracer("adservice"); private static final Meter meter = GlobalOpenTelemetry.getMeter("adservice"); - private static final LongCounter adRequestsCounter = meter + private static final LongCounter adRequestsCounter = + meter .counterBuilder("app.ads.ad_requests") .setDescription("Counts ad requests by request and response type") .build(); - private static final AttributeKey adRequestTypeKey = AttributeKey.stringKey("app.ads.ad_request_type"); - private static final AttributeKey adResponseTypeKey = AttributeKey.stringKey("app.ads.ad_response_type"); + private static final AttributeKey adRequestTypeKey = + AttributeKey.stringKey("app.ads.ad_request_type"); + private static final AttributeKey adResponseTypeKey = + AttributeKey.stringKey("app.ads.ad_response_type"); private void start() throws IOException { - int port = Integer.parseInt(Optional.ofNullable(System.getenv("AD_SERVICE_PORT")).orElseThrow( - () -> new IOException( - "environment vars: AD_SERVICE_PORT must not be null") - )); + int port = + Integer.parseInt( + Optional.ofNullable(System.getenv("AD_SERVICE_PORT")) + .orElseThrow( + () -> + new IllegalStateException( + "environment vars: AD_SERVICE_PORT must not be null"))); healthMgr = new HealthStatusManager(); + String featureFlagServiceAddr = + Optional.ofNullable(System.getenv("FEATURE_FLAG_GRPC_SERVICE_ADDR")) + .orElseThrow( + () -> + new IllegalStateException( + "environment vars: FEATURE_FLAG_GRPC_SERVICE_ADDR must not be null")); + FeatureFlagServiceBlockingStub featureFlagServiceStub = + FeatureFlagServiceGrpc.newBlockingStub( + ManagedChannelBuilder.forTarget(featureFlagServiceAddr).usePlaintext().build()); + server = ServerBuilder.forPort(port) - .addService(new AdServiceImpl()) + .addService(new AdServiceImpl(featureFlagServiceStub)) .addService(healthMgr.getHealthService()) .build() .start(); @@ -105,15 +121,25 @@ private void stop() { } private enum AdRequestType { - TARGETED, NOT_TARGETED + TARGETED, + NOT_TARGETED } private enum AdResponseType { - TARGETED, RANDOM + TARGETED, + RANDOM } private static class AdServiceImpl extends hipstershop.AdServiceGrpc.AdServiceImplBase { + private static final String ADSERVICE_FAIL_FEATURE_FLAG = "adServiceFailure"; + + private final FeatureFlagServiceBlockingStub featureFlagServiceStub; + + private AdServiceImpl(FeatureFlagServiceBlockingStub featureFlagServiceStub) { + this.featureFlagServiceStub = featureFlagServiceStub; + } + /** * Retrieves ads based on context provided in the request {@code AdRequest}. * @@ -156,7 +182,15 @@ public void getAds(AdRequest req, StreamObserver responseObserver) { span.setAttribute("app.ads.ad_request_type", adRequestType.name()); span.setAttribute("app.ads.ad_response_type", adResponseType.name()); - adRequestsCounter.add(1, Attributes.of(adRequestTypeKey, adRequestType.name(), adResponseTypeKey, adResponseType.name())); + adRequestsCounter.add( + 1, + Attributes.of( + adRequestTypeKey, adRequestType.name(), adResponseTypeKey, adResponseType.name())); + + if (checkAdFailure()) { + logger.warn(ADSERVICE_FAIL_FEATURE_FLAG + " fail feature flag enabled"); + throw new StatusRuntimeException(Status.RESOURCE_EXHAUSTED); + } AdResponse reply = AdResponse.newBuilder().addAllAds(allAds).build(); responseObserver.onNext(reply); @@ -169,6 +203,20 @@ public void getAds(AdRequest req, StreamObserver responseObserver) { responseObserver.onError(e); } } + + boolean checkAdFailure() { + // Flip a coin and fail 1/10th of the time if feature flag is enabled + if (random.nextInt(10) != 1) { + return false; + } + + GetFlagResponse response = + featureFlagServiceStub.getFlag( + hipstershop.Demo.GetFlagRequest.newBuilder() + .setName(ADSERVICE_FAIL_FEATURE_FLAG) + .build()); + return response.getFlag().getEnabled(); + } } private static final ImmutableListMultimap adsMap = createAdsMap(); @@ -259,7 +307,7 @@ private static ImmutableListMultimap createAdsMap() { .putAll("accessories", colorImager, solarFilter, cleaningKit) .putAll("assembly", opticalTube) .putAll("travel", travelTelescope) - // Keep the books category free of ads to ensure the random code branch is tested + // Keep the books category free of ads to ensure the random code branch is tested .build(); } diff --git a/src/featureflagservice/priv/repo/migrations/20220524172636_create_featureflags.exs b/src/featureflagservice/priv/repo/migrations/20220524172636_create_featureflags.exs index a3e7387655..4961199376 100644 --- a/src/featureflagservice/priv/repo/migrations/20220524172636_create_featureflags.exs +++ b/src/featureflagservice/priv/repo/migrations/20220524172636_create_featureflags.exs @@ -25,10 +25,16 @@ defmodule Featureflagservice.Repo.Migrations.CreateFeatureflags do name: "recommendationCache", description: "Cache recommendations", enabled: false}) + + repo().insert(%Featureflagservice.FeatureFlags.FeatureFlag{ + name: "adServiceFailure", + description: "Fail ad service requests sporadically", + enabled: false}) end defp execute_down do repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "productCatalogFailure"}) repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "recommendationCache"}) + repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "adServiceFailure"}) end end