diff --git a/.circleci/config.yml b/.circleci/config.yml
index df5bd409767..b95fcdd13eb 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -6,6 +6,9 @@ init_task: &init_task
 build_task: &build_task
   name: Build
   command: make test
+compile_benchmark_task: &compile_benchmark_task
+  name: Compile JMH
+  command: make benchmark
 verify_task: &verify_task
   name: Verify Google Java Format
   command: make verify-format
@@ -32,6 +35,8 @@ jobs:
           <<: *verify_task
       - run:
           <<: *build_task
+      - run:
+          <<: *compile_benchmark_task
       - run:
           name: Cover
           command: ./gradlew check :opentelemetry-all:jacocoTestReport
@@ -65,6 +70,8 @@ jobs:
           <<: *verify_task
       - run:
           <<: *build_task
+      - run:
+          <<: *compile_benchmark_task
       - save_cache:
           paths:
             - ~/.gradle
diff --git a/Makefile b/Makefile
index 72e444fdc87..0491a13ac2f 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,10 @@
 test:
 	./gradlew clean assemble check --stacktrace
 
+.PHONY: benchmark
+benchmark:
+	./gradlew compileJmhJava
+
 .PHONY: init-git-submodules
 init-git-submodules:
 	git submodule init
diff --git a/build.gradle b/build.gradle
index d5757716d94..42e75c8acae 100644
--- a/build.gradle
+++ b/build.gradle
@@ -81,7 +81,7 @@ subprojects {
         it.options.errorprone.excludedPaths = ".*generated.*" // "-XepExcludedPaths:.*/build/generated/source/proto/.*"
 
         // Enforce errorprone warnings to be errors.
-        if (!JavaVersion.current().isJava9() && !JavaVersion.current().isJava10() && !JavaVersion.current().isJava11()) {
+        if (!JavaVersion.current().isJava9() && !JavaVersion.current().isJava10() && !JavaVersion.current().isJava11() && it != compileJmhJava) {
             // TODO: Enable -Werror for Java 9+
             it.options.compilerArgs += ["-Werror"]
         }