Add Graceful Shutdown Support

packages/opentelemetry-core/src/platform/browser/ShutdownNotifier.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Adds an event listener to trigger a callback when an unload event in the window is detected
+ */
+export function notifyOnGlobalShutdown(cb: () => void): () => void {
+ window.addEventListener('unload', cb, { once: true });
+ return function removeCallbackFromGlobalShutdown() {
+ window.removeEventListener('unload', cb, false);
+ };
+}
+
+/**
+ * Warning: meant for internal use only! Closes the current window, triggering the unload event
+ */
+export function _invokeGlobalShutdown() {
+ window.close();
+}

packages/opentelemetry-core/src/platform/browser/index.ts

@@ -21,3 +21,4 @@ export * from './RandomIdGenerator';
 export * from './performance';
 export * from './sdk-info';
 export * from './timer-util';
+export * from './ShutdownNotifier';

packages/opentelemetry-core/src/platform/node/ShutdownNotifier.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Adds an event listener to trigger a callback when a SIGTERM is detected in the process
+ */
+export function notifyOnGlobalShutdown(cb: () => void): () => void {
+ process.once('SIGTERM', cb);
+ return function removeCallbackFromGlobalShutdown() {
+ process.removeListener('SIGTERM', cb);
+ };
+}
+
+/**
+ * Warning: meant for internal use only! Sends a SIGTERM to the current process
+ */
+export function _invokeGlobalShutdown() {
+ process.kill(process.pid, 'SIGTERM');
+}

packages/opentelemetry-core/src/platform/node/index.ts

@@ -21,3 +21,4 @@ export * from './RandomIdGenerator';
 export * from './performance';
 export * from './sdk-info';
 export * from './timer-util';
+export * from './ShutdownNotifier';

packages/opentelemetry-exporter-prometheus/test/prometheus.test.ts

@@ -15,6 +15,10 @@
  */
 
 import { HrTime, ObserverResult } from '@opentelemetry/api';
+import {
+ notifyOnGlobalShutdown,
+ _invokeGlobalShutdown,
+} from '@opentelemetry/core';
 import {
  CounterMetric,
  SumAggregator,
@@ -32,6 +36,7 @@ const mockedTimeMS = 1586347902211000;
 
 describe('PrometheusExporter', () => {
  let toPoint: () => Point<Sum>;
+ let removeEvent: Function | undefined;
  before(() => {
  toPoint = SumAggregator.prototype.toPoint;
  SumAggregator.prototype.toPoint = function (): Point<Sum> {
@@ -185,16 +190,27 @@ describe('PrometheusExporter', () => {
 
  describe('export', () => {
  let exporter: PrometheusExporter;
+ let meterProvider: MeterProvider;
  let meter: Meter;
 
  beforeEach(done => {
  exporter = new PrometheusExporter();
- meter = new MeterProvider().getMeter('test-prometheus');
+ meterProvider = new MeterProvider({
+ interval: Math.pow(2, 31) - 1,
+ gracefulShutdown: true,
+ });
+ meter = meterProvider.getMeter('test-prometheus', '1', {
+ exporter: exporter,
+ });
  exporter.startServer(done);
  });
 
  afterEach(done => {
  exporter.shutdown(done);
+ if (removeEvent) {
+ removeEvent();
+ removeEvent = undefined;
+ }
  });
 
  it('should export a count aggregation', done => {
@@ -320,6 +336,70 @@ describe('PrometheusExporter', () => {
  });
  });
 
+ it('should export multiple labels on graceful shutdown', done => {
+ const counter = meter.createCounter('counter', {
+ description: 'a test description',
+ }) as CounterMetric;
+
+ counter.bind({ counterKey1: 'labelValue1' }).add(10);
+ counter.bind({ counterKey1: 'labelValue2' }).add(20);
+ counter.bind({ counterKey1: 'labelValue3' }).add(30);
+
+ removeEvent = notifyOnGlobalShutdown(() => {
+ http
+ .get('http://localhost:9464/metrics', res => {
+ res.on('data', chunk => {
+ const body = chunk.toString();
+ const lines = body.split('\n');
+
+ assert.deepStrictEqual(lines, [
+ '# HELP counter a test description',
+ '# TYPE counter counter',
+ `counter{counterKey1="labelValue1"} 10 ${mockedTimeMS}`,
+ `counter{counterKey1="labelValue2"} 20 ${mockedTimeMS}`,
+ `counter{counterKey1="labelValue3"} 30 ${mockedTimeMS}`,
+ '',
+ ]);
+
+ done();
+ });
+ })
+ .on('error', errorHandler(done));
+ });
+ _invokeGlobalShutdown();
+ });
+
+ it('should export multiple labels on manual shutdown', done => {
+ const counter = meter.createCounter('counter', {
+ description: 'a test description',
+ }) as CounterMetric;
+
+ counter.bind({ counterKey1: 'labelValue1' }).add(10);
+ counter.bind({ counterKey1: 'labelValue2' }).add(20);
+ counter.bind({ counterKey1: 'labelValue3' }).add(30);
+ meterProvider.shutdown(() => {
+ http
+ .get('http://localhost:9464/metrics', res => {
+ res.on('data', chunk => {
+ const body = chunk.toString();
+ const lines = body.split('\n');
+
+ assert.deepStrictEqual(lines, [
+ '# HELP counter a test description',
+ '# TYPE counter counter',
+ `counter{counterKey1="labelValue1"} 10 ${mockedTimeMS}`,
+ `counter{counterKey1="labelValue2"} 20 ${mockedTimeMS}`,
+ `counter{counterKey1="labelValue3"} 30 ${mockedTimeMS}`,
+ '',
+ ]);
+
+ done();
+ });
+ })
+ .on('error', errorHandler(done));
+ });
+ });
+
  it('should export a comment if no metrics are registered', done => {
  exporter.export([], () => {
  http

packages/opentelemetry-metrics/src/Meter.ts

@@ -40,6 +40,7 @@ export class Meter implements api.Meter {
  private readonly _batcher: Batcher;
  private readonly _resource: Resource;
  private readonly _instrumentationLibrary: InstrumentationLibrary;
+ private readonly _controller: PushController;
 
  /**
  * Constructs a new Meter instance.
@@ -55,7 +56,7 @@ export class Meter implements api.Meter {
  // start the push controller
  const exporter = config.exporter || new NoopExporter();
  const interval = config.interval;
- new PushController(this, exporter, interval);
+ this._controller = new PushController(this, exporter, interval);
  }
 
  /**
@@ -309,6 +310,10 @@ export class Meter implements api.Meter {
  return this._batcher;
  }
 
+ async shutdown(): Promise<void> {
+ await this._controller.shutdown();
+ }
+
  /**
  * Registers metric to register.
  * @param name The name of the metric.

packages/opentelemetry-metrics/src/MeterProvider.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { ConsoleLogger } from '@opentelemetry/core';
+import { ConsoleLogger, notifyOnGlobalShutdown } from '@opentelemetry/core';
 import * as api from '@opentelemetry/api';
 import { Resource } from '@opentelemetry/resources';
 import { Meter } from '.';
@@ -26,6 +26,7 @@ import { DEFAULT_CONFIG, MeterConfig } from './types';
 export class MeterProvider implements api.MeterProvider {
  private readonly _config: MeterConfig;
  private readonly _meters: Map<string, Meter> = new Map();
+ private _cleanNotifyOnGlobalShutdown: Function | undefined;
  readonly resource: Resource;
  readonly logger: api.Logger;
 
@@ -36,6 +37,11 @@ export class MeterProvider implements api.MeterProvider {
  logger: this.logger,
  resource: this.resource,
  });
+ if (this._config.gracefulShutdown) {
+ this._cleanNotifyOnGlobalShutdown = notifyOnGlobalShutdown(
+ this._shutdownAllMeters.bind(this)
+ );
+ }
  }
 
  /**
@@ -54,4 +60,23 @@ export class MeterProvider implements api.MeterProvider {
 
  return this._meters.get(key)!;
  }
+
+ shutdown(cb: () => void = () => {}): void {
+ this._shutdownAllMeters().then(() => {
+ setTimeout(cb, 0);
+ });
+ if (this._cleanNotifyOnGlobalShutdown) {
+ this._cleanNotifyOnGlobalShutdown();
+ this._cleanNotifyOnGlobalShutdown = undefined;
+ }
+ }
+
+ private _shutdownAllMeters() {
+ if (this._config.exporter) {
+ this._config.exporter.shutdown();
+ }
+ return Promise.all(
+ Array.from(this._meters, ([_, meter]) => meter.shutdown())
+ );
+ }
 }

packages/opentelemetry-metrics/src/export/Controller.ts

@@ -38,12 +38,24 @@ export class PushController extends Controller {
  unrefTimer(this._timer);
  }
 
- private _collect() {
- this._meter.collect();
- this._exporter.export(this._meter.getBatcher().checkPointSet(), result => {
- if (result !== ExportResult.SUCCESS) {
- // @todo: log error
- }
+ async shutdown(): Promise<void> {
+ await this._collect();
+ }
+
+ private async _collect(): Promise<void> {
+ await this._meter.collect();
+ return new Promise((resolve, reject) => {
+ this._exporter.export(
+ this._meter.getBatcher().checkPointSet(),
+ result => {
+ if (result === ExportResult.SUCCESS) {
+ resolve();
+ } else {
+ // @todo log error
+ reject();
+ }
+ }
+ );
  });
  }
 }

packages/opentelemetry-metrics/src/types.ts

@@ -39,11 +39,15 @@ export interface MeterConfig {
 
  /** Metric batcher. */
  batcher?: Batcher;
+
+ /** Bool for whether or not graceful shutdown is enabled. If disabled metrics will not be exported when SIGTERM is recieved */
+ gracefulShutdown?: boolean;
 }
 
 /** Default Meter configuration. */
 export const DEFAULT_CONFIG = {
  logLevel: getEnv().OTEL_LOG_LEVEL,
+ gracefulShutdown: true,
 };
 
 /** The default metric creation options value. */