Add random errors in AdService (#694)

CHANGELOG.md

@@ -200,6 +200,8 @@ significant modifications will be credited to OpenTelemetry Authors.
 ([#693](https://github.com/open-telemetry/opentelemetry-demo/pull/693))
 * Update recommendationservice python base image and dependencies
 ([#700](https://github.com/open-telemetry/opentelemetry-demo/pull/700))
+* Add adServiceFailure feature flag triggering Ad Service errors
+([#694](https://github.com/open-telemetry/opentelemetry-demo/pull/694))
 * Reduce spans generated from quote service
 ([#702](https://github.com/open-telemetry/opentelemetry-demo/pull/702))
 * Update emailservice Dockerfile to use alpine and multistage build

docker-compose.yml

@@ -74,6 +74,7 @@ services:
       - "${AD_SERVICE_PORT}"
     environment:
       - AD_SERVICE_PORT
+      - FEATURE_FLAG_GRPC_SERVICE_ADDR
       - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
       - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
       - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE

docs/current_architecture.md

@@ -52,6 +52,8 @@ frontend -->|gRPC| shippingservice -->|HTTP| quoteservice
 
 frauddetectionservice -->|TCP| queue
 
+adservice -->|gRPC| featureflagservice
+
 productcatalogservice -->|gRPC| featureflagservice
 
 recommendationservice -->|gRPC| featureflagservice

docs/feature_flags.md

@@ -5,7 +5,8 @@ in specific services. By default the flags are disabled. Using the Feature Flags
 UI <http://localhost:8080/feature> you will be able to control the status of these
 feature flags.
 
-| Feature Flag            | Service(s)      | Description |
-|-------------------------|-----------------|---------------------------------------------------------------------------------------------------------|
-| `productCatalogFailure` | Product Catalog | Generate an error for `GetProduct` requests with product id: `OLJCESPC7Z`                               |
+| Feature Flag            | Service(s)      | Description                                                                                              |
+|-------------------------|-----------------|----------------------------------------------------------------------------------------------------------|
+| `adServiceFailure`      | Ad Service      | Generate an error for `GetAds` 1/10th of the time                                                        |
+| `productCatalogFailure` | Product Catalog | Generate an error for `GetProduct` requests with product id: `OLJCESPC7Z`                                |
 | `recommendationCache`   | Recommendation  | Create a memory leak due to an exponentially growing cache. 1.4x growth, 50% of requests trigger growth. |

src/adservice/Dockerfile

@@ -39,4 +39,4 @@ RUN chmod 644 /usr/src/app/opentelemetry-javaagent.jar
 ENV JAVA_TOOL_OPTIONS=-javaagent:/usr/src/app/opentelemetry-javaagent.jar
 
 EXPOSE ${AD_SERVICE_PORT}
-ENTRYPOINT [ "./build/install/hipstershop/bin/AdService" ]
+ENTRYPOINT [ "./build/install/opentelemetry-demo-ad-service/bin/AdService" ]

src/adservice/README.md

@@ -20,6 +20,7 @@ To run the Ad Service:
 
 ```sh
 export AD_SERVICE_PORT=8080
+export FEATURE_FLAG_GRPC_SERVICE_ADDR=featureflagservice:50053
 ./build/install/hipstershop/bin/AdService
 ```
 

src/adservice/settings.gradle

@@ -1 +1 @@
-rootProject.name = 'hipstershop'
+rootProject.name = 'opentelemetry-demo-ad-service'

src/adservice/src/main/java/hipstershop/AdService.java

@@ -21,9 +21,9 @@
 import hipstershop.Demo.Ad;
 import hipstershop.Demo.AdRequest;
 import hipstershop.Demo.AdResponse;
-import io.grpc.Server;
-import io.grpc.ServerBuilder;
-import io.grpc.StatusRuntimeException;
+import hipstershop.Demo.GetFlagResponse;
+import hipstershop.FeatureFlagServiceGrpc.FeatureFlagServiceBlockingStub;
+import io.grpc.*;
 import io.grpc.health.v1.HealthCheckResponse.ServingStatus;
 import io.grpc.protobuf.services.*;
 import io.grpc.stub.StreamObserver;
@@ -62,24 +62,40 @@ public final class AdService {
   private static final Tracer tracer = GlobalOpenTelemetry.getTracer("adservice");
   private static final Meter meter = GlobalOpenTelemetry.getMeter("adservice");
 
-  private static final LongCounter adRequestsCounter = meter
+  private static final LongCounter adRequestsCounter =
+      meter
           .counterBuilder("app.ads.ad_requests")
           .setDescription("Counts ad requests by request and response type")
           .build();
 
-  private static final AttributeKey<String> adRequestTypeKey = AttributeKey.stringKey("app.ads.ad_request_type");
-  private static final AttributeKey<String> adResponseTypeKey = AttributeKey.stringKey("app.ads.ad_response_type");
+  private static final AttributeKey<String> adRequestTypeKey =
+      AttributeKey.stringKey("app.ads.ad_request_type");
+  private static final AttributeKey<String> adResponseTypeKey =
+      AttributeKey.stringKey("app.ads.ad_response_type");
 
   private void start() throws IOException {
-    int port = Integer.parseInt(Optional.ofNullable(System.getenv("AD_SERVICE_PORT")).orElseThrow(
-        () -> new IOException(
-            "environment vars: AD_SERVICE_PORT must not be null")
-    ));
+    int port =
+        Integer.parseInt(
+            Optional.ofNullable(System.getenv("AD_SERVICE_PORT"))
+                .orElseThrow(
+                    () ->
+                        new IllegalStateException(
+                            "environment vars: AD_SERVICE_PORT must not be null")));
     healthMgr = new HealthStatusManager();
 
+    String featureFlagServiceAddr =
+        Optional.ofNullable(System.getenv("FEATURE_FLAG_GRPC_SERVICE_ADDR"))
+            .orElseThrow(
+                () ->
+                    new IllegalStateException(
+                        "environment vars: FEATURE_FLAG_GRPC_SERVICE_ADDR must not be null"));
+    FeatureFlagServiceBlockingStub featureFlagServiceStub =
+        FeatureFlagServiceGrpc.newBlockingStub(
+            ManagedChannelBuilder.forTarget(featureFlagServiceAddr).usePlaintext().build());
+
     server =
         ServerBuilder.forPort(port)
-            .addService(new AdServiceImpl())
+            .addService(new AdServiceImpl(featureFlagServiceStub))
             .addService(healthMgr.getHealthService())
             .build()
             .start();
@@ -105,15 +121,25 @@ private void stop() {
   }
 
   private enum AdRequestType {
-    TARGETED, NOT_TARGETED
+    TARGETED,
+    NOT_TARGETED
   }
 
   private enum AdResponseType {
-    TARGETED, RANDOM
+    TARGETED,
+    RANDOM
   }
 
   private static class AdServiceImpl extends hipstershop.AdServiceGrpc.AdServiceImplBase {
 
+    private static final String ADSERVICE_FAIL_FEATURE_FLAG = "adServiceFailure";
+
+    private final FeatureFlagServiceBlockingStub featureFlagServiceStub;
+
+    private AdServiceImpl(FeatureFlagServiceBlockingStub featureFlagServiceStub) {
+      this.featureFlagServiceStub = featureFlagServiceStub;
+    }
+
     /**
      * Retrieves ads based on context provided in the request {@code AdRequest}.
      *
@@ -156,7 +182,15 @@ public void getAds(AdRequest req, StreamObserver<AdResponse> responseObserver) {
         span.setAttribute("app.ads.ad_request_type", adRequestType.name());
         span.setAttribute("app.ads.ad_response_type", adResponseType.name());
 
-        adRequestsCounter.add(1, Attributes.of(adRequestTypeKey, adRequestType.name(), adResponseTypeKey, adResponseType.name()));
+        adRequestsCounter.add(
+            1,
+            Attributes.of(
+                adRequestTypeKey, adRequestType.name(), adResponseTypeKey, adResponseType.name()));
+
+        if (checkAdFailure()) {
+          logger.warn(ADSERVICE_FAIL_FEATURE_FLAG + " fail feature flag enabled");
+          throw new StatusRuntimeException(Status.RESOURCE_EXHAUSTED);
+        }
 
         AdResponse reply = AdResponse.newBuilder().addAllAds(allAds).build();
         responseObserver.onNext(reply);
@@ -169,6 +203,20 @@ public void getAds(AdRequest req, StreamObserver<AdResponse> responseObserver) {
         responseObserver.onError(e);
       }
     }
+
+    boolean checkAdFailure() {
+      // Flip a coin and fail 1/10th of the time if feature flag is enabled
+      if (random.nextInt(10) != 1) {
+        return false;
+      }
+
+      GetFlagResponse response =
+          featureFlagServiceStub.getFlag(
+              hipstershop.Demo.GetFlagRequest.newBuilder()
+                  .setName(ADSERVICE_FAIL_FEATURE_FLAG)
+                  .build());
+      return response.getFlag().getEnabled();
+    }
   }
 
   private static final ImmutableListMultimap<String, Ad> adsMap = createAdsMap();
@@ -259,7 +307,7 @@ private static ImmutableListMultimap<String, Ad> createAdsMap() {
         .putAll("accessories", colorImager, solarFilter, cleaningKit)
         .putAll("assembly", opticalTube)
         .putAll("travel", travelTelescope)
-            // Keep the books category free of ads to ensure the random code branch is tested
+        // Keep the books category free of ads to ensure the random code branch is tested
         .build();
   }
 

src/featureflagservice/priv/repo/migrations/20220524172636_create_featureflags.exs

@@ -25,10 +25,16 @@ defmodule Featureflagservice.Repo.Migrations.CreateFeatureflags do
       name: "recommendationCache",
       description: "Cache recommendations",
       enabled: false})
+
+    repo().insert(%Featureflagservice.FeatureFlags.FeatureFlag{
+      name: "adServiceFailure",
+      description: "Fail ad service requests sporadically",
+      enabled: false})
   end
 
   defp execute_down do
     repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "productCatalogFailure"})
     repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "recommendationCache"})
+    repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "adServiceFailure"})
   end
 end