Add random errors in AdService (#694)

CHANGELOG.md

@@ -200,6 +200,8 @@ significant modifications will be credited to OpenTelemetry Authors.
 ([#693](https:/open-telemetry/opentelemetry-demo/pull/693))
 * Update recommendationservice python base image and dependencies
 ([#700](https:/open-telemetry/opentelemetry-demo/pull/700))
+* Add adServiceFailure feature flag triggering Ad Service errors
+([#694](https:/open-telemetry/opentelemetry-demo/pull/694))
 * Reduce spans generated from quote service
 ([#702](https:/open-telemetry/opentelemetry-demo/pull/702))
 * Update emailservice Dockerfile to use alpine and multistage build

docker-compose.yml

@@ -74,6 +74,7 @@ services:
  - "${AD_SERVICE_PORT}"
  environment:
  - AD_SERVICE_PORT
+ - FEATURE_FLAG_GRPC_SERVICE_ADDR
  - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
  - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
  - OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE

docs/current_architecture.md

@@ -52,6 +52,8 @@ frontend -->|gRPC| shippingservice -->|HTTP| quoteservice
 
 frauddetectionservice -->|TCP| queue
 
+adservice -->|gRPC| featureflagservice
+
 productcatalogservice -->|gRPC| featureflagservice
 
 recommendationservice -->|gRPC| featureflagservice

docs/feature_flags.md

@@ -5,7 +5,8 @@ in specific services. By default the flags are disabled. Using the Feature Flags
 UI <http://localhost:8080/feature> you will be able to control the status of these
 feature flags.
 
-| Feature Flag | Service(s) | Description |
-|-------------------------|-----------------|---------------------------------------------------------------------------------------------------------|
-| `productCatalogFailure` | Product Catalog | Generate an error for `GetProduct` requests with product id: `OLJCESPC7Z` |
+| Feature Flag | Service(s) | Description |
+|-------------------------|-----------------|----------------------------------------------------------------------------------------------------------|
+| `adServiceFailure` | Ad Service | Generate an error for `GetAds` 1/10th of the time |
+| `productCatalogFailure` | Product Catalog | Generate an error for `GetProduct` requests with product id: `OLJCESPC7Z` |
 | `recommendationCache` | Recommendation | Create a memory leak due to an exponentially growing cache. 1.4x growth, 50% of requests trigger growth. |

src/adservice/Dockerfile

@@ -39,4 +39,4 @@ RUN chmod 644 /usr/src/app/opentelemetry-javaagent.jar
 ENV JAVA_TOOL_OPTIONS=-javaagent:/usr/src/app/opentelemetry-javaagent.jar
 
 EXPOSE ${AD_SERVICE_PORT}
-ENTRYPOINT [ "./build/install/hipstershop/bin/AdService" ]
+ENTRYPOINT [ "./build/install/opentelemetry-demo-ad-service/bin/AdService" ]

src/adservice/README.md

@@ -20,6 +20,7 @@ To run the Ad Service:
 
 ```sh
 export AD_SERVICE_PORT=8080
+export FEATURE_FLAG_GRPC_SERVICE_ADDR=featureflagservice:50053
 ./build/install/hipstershop/bin/AdService
 ```
 

src/adservice/settings.gradle

@@ -1 +1 @@
-rootProject.name = 'hipstershop'
+rootProject.name = 'opentelemetry-demo-ad-service'

src/adservice/src/main/java/hipstershop/AdService.java

@@ -21,9 +21,9 @@
 import hipstershop.Demo.Ad;
 import hipstershop.Demo.AdRequest;
 import hipstershop.Demo.AdResponse;
-import io.grpc.Server;
-import io.grpc.ServerBuilder;
-import io.grpc.StatusRuntimeException;
+import hipstershop.Demo.GetFlagResponse;
+import hipstershop.FeatureFlagServiceGrpc.FeatureFlagServiceBlockingStub;
+import io.grpc.*;
 import io.grpc.health.v1.HealthCheckResponse.ServingStatus;
 import io.grpc.protobuf.services.*;
 import io.grpc.stub.StreamObserver;
@@ -62,24 +62,40 @@ public final class AdService {
  private static final Tracer tracer = GlobalOpenTelemetry.getTracer("adservice");
  private static final Meter meter = GlobalOpenTelemetry.getMeter("adservice");
 
- private static final LongCounter adRequestsCounter = meter
+ private static final LongCounter adRequestsCounter =
+ meter
  .counterBuilder("app.ads.ad_requests")
  .setDescription("Counts ad requests by request and response type")
  .build();
 
- private static final AttributeKey<String> adRequestTypeKey = AttributeKey.stringKey("app.ads.ad_request_type");
- private static final AttributeKey<String> adResponseTypeKey = AttributeKey.stringKey("app.ads.ad_response_type");
+ private static final AttributeKey<String> adRequestTypeKey =
+ AttributeKey.stringKey("app.ads.ad_request_type");
+ private static final AttributeKey<String> adResponseTypeKey =
+ AttributeKey.stringKey("app.ads.ad_response_type");
 
  private void start() throws IOException {
- int port = Integer.parseInt(Optional.ofNullable(System.getenv("AD_SERVICE_PORT")).orElseThrow(
- () -> new IOException(
- "environment vars: AD_SERVICE_PORT must not be null")
- ));
+ int port =
+ Integer.parseInt(
+ Optional.ofNullable(System.getenv("AD_SERVICE_PORT"))
+ .orElseThrow(
+ () ->
+ new IllegalStateException(
+ "environment vars: AD_SERVICE_PORT must not be null")));
  healthMgr = new HealthStatusManager();
 
+ String featureFlagServiceAddr =
+ Optional.ofNullable(System.getenv("FEATURE_FLAG_GRPC_SERVICE_ADDR"))
+ .orElseThrow(
+ () ->
+ new IllegalStateException(
+ "environment vars: FEATURE_FLAG_GRPC_SERVICE_ADDR must not be null"));
+ FeatureFlagServiceBlockingStub featureFlagServiceStub =
+ FeatureFlagServiceGrpc.newBlockingStub(
+ ManagedChannelBuilder.forTarget(featureFlagServiceAddr).usePlaintext().build());
+
  server =
  ServerBuilder.forPort(port)
- .addService(new AdServiceImpl())
+ .addService(new AdServiceImpl(featureFlagServiceStub))
  .addService(healthMgr.getHealthService())
  .build()
  .start();
@@ -105,15 +121,25 @@ private void stop() {
  }
 
  private enum AdRequestType {
- TARGETED, NOT_TARGETED
+ TARGETED,
+ NOT_TARGETED
  }
 
  private enum AdResponseType {
- TARGETED, RANDOM
+ TARGETED,
+ RANDOM
  }
 
  private static class AdServiceImpl extends hipstershop.AdServiceGrpc.AdServiceImplBase {
 
+ private static final String ADSERVICE_FAIL_FEATURE_FLAG = "adServiceFailure";
+
+ private final FeatureFlagServiceBlockingStub featureFlagServiceStub;
+
+ private AdServiceImpl(FeatureFlagServiceBlockingStub featureFlagServiceStub) {
+ this.featureFlagServiceStub = featureFlagServiceStub;
+ }
+
  /**
  * Retrieves ads based on context provided in the request {@code AdRequest}.
  *
@@ -156,7 +182,15 @@ public void getAds(AdRequest req, StreamObserver<AdResponse> responseObserver) {
  span.setAttribute("app.ads.ad_request_type", adRequestType.name());
  span.setAttribute("app.ads.ad_response_type", adResponseType.name());
 
- adRequestsCounter.add(1, Attributes.of(adRequestTypeKey, adRequestType.name(), adResponseTypeKey, adResponseType.name()));
+ adRequestsCounter.add(
+ 1,
+ Attributes.of(
+ adRequestTypeKey, adRequestType.name(), adResponseTypeKey, adResponseType.name()));
+
+ if (checkAdFailure()) {
+ logger.warn(ADSERVICE_FAIL_FEATURE_FLAG + " fail feature flag enabled");
+ throw new StatusRuntimeException(Status.RESOURCE_EXHAUSTED);
+ }
 
  AdResponse reply = AdResponse.newBuilder().addAllAds(allAds).build();
  responseObserver.onNext(reply);
@@ -169,6 +203,20 @@ public void getAds(AdRequest req, StreamObserver<AdResponse> responseObserver) {
  responseObserver.onError(e);
  }
  }
+
+ boolean checkAdFailure() {
+ // Flip a coin and fail 1/10th of the time if feature flag is enabled
+ if (random.nextInt(10) != 1) {
+ return false;
+ }
+
+ GetFlagResponse response =
+ featureFlagServiceStub.getFlag(
+ hipstershop.Demo.GetFlagRequest.newBuilder()
+ .setName(ADSERVICE_FAIL_FEATURE_FLAG)
+ .build());
+ return response.getFlag().getEnabled();
+ }
  }
 
  private static final ImmutableListMultimap<String, Ad> adsMap = createAdsMap();
@@ -259,7 +307,7 @@ private static ImmutableListMultimap<String, Ad> createAdsMap() {
  .putAll("accessories", colorImager, solarFilter, cleaningKit)
  .putAll("assembly", opticalTube)
  .putAll("travel", travelTelescope)
-  // Keep the books category free of ads to ensure the random code branch is tested
+ // Keep the books category free of ads to ensure the random code branch is tested
  .build();
  }
 

src/featureflagservice/priv/repo/migrations/20220524172636_create_featureflags.exs

@@ -25,10 +25,16 @@ defmodule Featureflagservice.Repo.Migrations.CreateFeatureflags do
  name: "recommendationCache",
  description: "Cache recommendations",
  enabled: false})
+
+ repo().insert(%Featureflagservice.FeatureFlags.FeatureFlag{
+ name: "adServiceFailure",
+ description: "Fail ad service requests sporadically",
+ enabled: false})
  end
 
  defp execute_down do
  repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "productCatalogFailure"})
  repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "recommendationCache"})
+ repo().delete(%Featureflagservice.FeatureFlags.FeatureFlag{name: "adServiceFailure"})
  end
 end