diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000..1712df25da --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,72 @@ +name: OpenTelemetry-cpp benchmarks +on: + push: + branches: + - main + +permissions: + contents: write + deployments: write + +jobs: + benchmark: + name: Run OpenTelemetry-cpp benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + - name: Mount Bazel Cache + uses: actions/cache@v2 + env: + cache-name: bazel_cache + with: + path: /home/runner/.cache/bazel + key: bazel_benchmark + - name: setup + run: | + sudo ./ci/setup_cmake.sh + sudo ./ci/setup_ci_environment.sh + - name: Run benchmark + id: run_benchmarks + run: | + ci/do_ci.sh bazel.benchmark + mkdir -p benchmarks + mv api-benchmark_result.json benchmarks + mv sdk-benchmark_result.json benchmarks + mv exporters-benchmark_result.json benchmarks + - uses: actions/upload-artifact@master + with: + name: benchmark_results + path: benchmarks + store_benchmark: + needs: benchmark + strategy: + matrix: + components: ["api", "sdk", "exporters"] + name: Store benchmark result + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@master + with: + name: benchmark_results + path: benchmarks + - name: Print json files + id: print_json + run: | + cat benchmarks/* + - name: Push benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: OpenTelemetry-cpp ${{ matrix.components }} Benchmark + tool: 'googlecpp' + output-file-path: benchmarks/${{ matrix.components }}-benchmark_result.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + gh-pages-branch: gh-pages + benchmark-data-dir-path: benchmarks diff --git a/bazel/otel_cc_benchmark.bzl b/bazel/otel_cc_benchmark.bzl index 78da697f48..c40681917f 100644 --- a/bazel/otel_cc_benchmark.bzl +++ b/bazel/otel_cc_benchmark.bzl @@ -24,16 +24,17 @@ def otel_cc_benchmark(name, srcs, deps, tags = [""]): srcs = srcs, deps = deps + ["@com_github_google_benchmark//:benchmark"], tags = tags + ["manual"], + defines = ["BAZEL_BUILD"], ) # The result of running the benchmark, captured into a text file. native.genrule( name = name + "_result", - outs = [name + "_result.txt"], + outs = [name + "_result.json"], tools = [":" + name], tags = tags + ["benchmark_result", "manual"], testonly = True, - cmd = "$(location :" + name + (") --benchmark_color=false --benchmark_min_time=.1 &> $@"), + cmd = "$(location :" + name + (") --benchmark_format=json --benchmark_color=false --benchmark_min_time=.1 &> $@"), ) # This is run as part of "bazel test ..." to smoke-test benchmarks. It's @@ -44,4 +45,5 @@ def otel_cc_benchmark(name, srcs, deps, tags = [""]): deps = deps + ["@com_github_google_benchmark//:benchmark"], args = ["--benchmark_min_time=0"], tags = tags + ["benchmark"], + defines = ["BAZEL_BUILD"], ) diff --git a/ci/do_ci.sh b/ci/do_ci.sh index e005549bda..368190b6ec 100755 --- a/ci/do_ci.sh +++ b/ci/do_ci.sh @@ -17,6 +17,42 @@ function install_prometheus_cpp_client popd } +function run_benchmarks +{ + docker run -d --rm -it -p 4317:4317 -p 4318:4318 -v \ + $(pwd)/examples/otlp:/cfg otel/opentelemetry-collector:0.38.0 \ + --config=/cfg/opentelemetry-collector-config/config.dev.yaml + + [ -z "${BENCHMARK_DIR}" ] && export BENCHMARK_DIR=$HOME/benchmark + mkdir -p $BENCHMARK_DIR + bazel $BAZEL_STARTUP_OPTIONS build $BAZEL_OPTIONS -c opt -- \ + $(bazel query 'attr("tags", "benchmark_result", ...)') + echo "" + echo "Benchmark results in $BENCHMARK_DIR:" + ( + cd bazel-bin + find . -name \*_result.json -exec bash -c \ + 'echo "$@" && mkdir -p "$BENCHMARK_DIR/$(dirname "$@")" && \ + cp "$@" "$BENCHMARK_DIR/$@" && chmod +w "$BENCHMARK_DIR/$@"' _ {} \; + ) + + # collect benchmark results into one array + pushd $BENCHMARK_DIR + components=(api sdk exporters) + for component in "${components[@]}" + do + out=$component-benchmark_result.json + find ./$component -type f -name "*_result.json" -exec cat {} \; > $component_tmp_bench.json + cat $component_tmp_bench.json | docker run -i --rm itchyny/gojq:0.12.6 -s \ + '.[0].benchmarks = ([.[].benchmarks] | add) | + if .[0].benchmarks == null then null else .[0] end' > $BENCHMARK_DIR/$out + done + + mv *benchmark_result.json ${SRC_DIR} + popd + docker kill $(docker ps -q) +} + [ -z "${SRC_DIR}" ] && export SRC_DIR="`pwd`" [ -z "${BUILD_DIR}" ] && export BUILD_DIR=$HOME/build mkdir -p "${BUILD_DIR}" @@ -124,6 +160,10 @@ elif [[ "$1" == "cmake.exporter.otprotocol.test" ]]; then make -j $(nproc) cd exporters/otlp && make test exit 0 +elif [[ "$1" == "bazel.with_abseil" ]]; then + bazel $BAZEL_STARTUP_OPTIONS build $BAZEL_OPTIONS --//api:with_abseil=true //... + bazel $BAZEL_STARTUP_OPTIONS test $BAZEL_TEST_OPTIONS --//api:with_abseil=true //... + exit 0 elif [[ "$1" == "cmake.test_example_plugin" ]]; then # Build the plugin cd "${BUILD_DIR}" @@ -162,9 +202,8 @@ elif [[ "$1" == "bazel.test" ]]; then bazel $BAZEL_STARTUP_OPTIONS build $BAZEL_OPTIONS //... bazel $BAZEL_STARTUP_OPTIONS test $BAZEL_TEST_OPTIONS //... exit 0 -elif [[ "$1" == "bazel.with_abseil" ]]; then - bazel $BAZEL_STARTUP_OPTIONS build $BAZEL_OPTIONS --//api:with_abseil=true //... - bazel $BAZEL_STARTUP_OPTIONS test $BAZEL_TEST_OPTIONS --//api:with_abseil=true //... +elif [[ "$1" == "bazel.benchmark" ]]; then + run_benchmarks exit 0 elif [[ "$1" == "bazel.macos.test" ]]; then bazel $BAZEL_STARTUP_OPTIONS build $BAZEL_MACOS_OPTIONS //... @@ -200,7 +239,7 @@ elif [[ "$1" == "benchmark" ]]; then echo "Benchmark results in $BENCHMARK_DIR:" ( cd bazel-bin - find . -name \*_result.txt -exec bash -c \ + find . -name \*_result.json -exec bash -c \ 'echo "$@" && mkdir -p "$BENCHMARK_DIR/$(dirname "$@")" && \ cp "$@" "$BENCHMARK_DIR/$@" && chmod +w "$BENCHMARK_DIR/$@"' _ {} \; ) diff --git a/exporters/otlp/BUILD b/exporters/otlp/BUILD index 829d834ea6..1a39a82baf 100644 --- a/exporters/otlp/BUILD +++ b/exporters/otlp/BUILD @@ -279,5 +279,6 @@ otel_cc_benchmark( ], deps = [ ":otlp_grpc_exporter", + "//examples/common/foo_library:common_foo_library", ], ) diff --git a/exporters/otlp/test/otlp_grpc_exporter_benchmark.cc b/exporters/otlp/test/otlp_grpc_exporter_benchmark.cc index ec90556e88..c86f5e53ba 100644 --- a/exporters/otlp/test/otlp_grpc_exporter_benchmark.cc +++ b/exporters/otlp/test/otlp_grpc_exporter_benchmark.cc @@ -4,6 +4,22 @@ #include "opentelemetry/exporters/otlp/otlp_grpc_exporter.h" #include "opentelemetry/exporters/otlp/otlp_recordable.h" +#include +#include "opentelemetry/sdk/trace/simple_processor.h" +#include "opentelemetry/sdk/trace/tracer_provider.h" +#include "opentelemetry/trace/provider.h" + +#ifdef BAZEL_BUILD +# include "examples/common/foo_library/foo_library.h" +#else +# include "foo_library/foo_library.h" +#endif + +namespace trace = opentelemetry::trace; +namespace nostd = opentelemetry::nostd; +namespace trace_sdk = opentelemetry::sdk::trace; +namespace otlp = opentelemetry::exporter::otlp; + #include OPENTELEMETRY_BEGIN_NAMESPACE @@ -171,4 +187,30 @@ BENCHMARK(BM_OtlpExporterDenseSpans); } // namespace exporter OPENTELEMETRY_END_NAMESPACE +namespace +{ +opentelemetry::exporter::otlp::OtlpGrpcExporterOptions opts; +void InitTracer() +{ + // Create OTLP exporter instance + auto exporter = std::unique_ptr(new otlp::OtlpGrpcExporter(opts)); + auto processor = std::unique_ptr( + new trace_sdk::SimpleSpanProcessor(std::move(exporter))); + auto provider = + nostd::shared_ptr(new trace_sdk::TracerProvider(std::move(processor))); + // Set the global trace provider + trace::Provider::SetTracerProvider(provider); +} + +void BM_otlp_grpc_with_collector(benchmark::State &state) +{ + InitTracer(); + while (state.KeepRunning()) + { + foo_library(); + } +} +BENCHMARK(BM_otlp_grpc_with_collector); +} // namespace + BENCHMARK_MAIN();