OpenTelemetry Runtime Metrics

Most OpenTelemetry SDKs, such as Go, Java, and Python, support automatic collection and export of runtime metrics like CPU, memory, and threads as OpenTelemetry metrics. If used in conjunction with APM, Kloudfuse displays the runtime metrics of each APM service in the Runtime tab of the corresponding service detail interface.

Example of OpenTelemetry runtime metrics

Node.js

The OpenTelemetry Node.js SDK does not natively support collecting and exposing runtime metrics. You can use the Node.js prom-client library to integrate runtime metrics with the Kloudfuse APM Service detail page. See the prom-client documentation of default metrics to enable default metrics.

TypeScript Example

This example demonstrates how to use the prom-client in conjunction with Node.js OpenTelemetry SDK using TypeScript.

We define a RunTimeInstrumentation class. It instantiates and collects the runtime metrics using prom-client. You can add the RunTimeInstrumentation class to the OpenTelemetry NodeSDK list of instrumentations.

Collecting runtime metrics with prom-client
...
import {
	MetricReader,
	PeriodicExportingMetricReader,
	PushMetricExporter,
	AggregationTemporality,
} from '@opentelemetry/sdk-metrics'
import { RunTimeInstrumentation } from './opentelemetry/runTimeMetric.class'
...

const metricsExporter: PushMetricExporter = new OTLPMetricExporter({
	url: process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT,
	headers: { service: process.env.OTEL_SERVICE_NAME },
	keepAlive: true,
	temporalityPreference: AggregationTemporality.DELTA,
})

const metricReader: MetricReader = new PeriodicExportingMetricReader({
	exporter: metricsExporter,
	exportIntervalMillis: 5000,
})

const sdk = new NodeSDK({
	...
	instrumentations: [
		...
		new RunTimeInstrumentation(),
	],
	metricReader: metricReader,
	...
})
sdk.start()
runTimeMetric.class.ts
import {
	BatchObservableResult,
	Histogram,
	Observable,
	ObservableCounter,
	ObservableGauge,
} from '@opentelemetry/api'
import * as Prometheus from 'prom-client'
import { PerformanceEntry, PerformanceObserver, constants } from 'perf_hooks'
import { InstrumentationBase } from '@opentelemetry/instrumentation';
import type { InstrumentationConfig } from '@opentelemetry/instrumentation';

const NODEJS_GC_DURATION_SECONDS = 'nodejs_gc_duration_seconds'

export class RunTimeInstrumentation extends InstrumentationBase {
	static instance: RunTimeInstrumentation

	registry: Prometheus.Registry

	private metricMap: Map<string, Observable>

	private enabled: boolean

	constructor(config: InstrumentationConfig = {}) {
		super('@opentelemetry/instrumentation-node-run-time', '1.0', config);
	}

	init() {
		// Not instrumenting or patching a Node.js module
	}

	override _updateMetricInstruments() {
		this.metricMap = new Map<string, Observable>()
		this.registry = new Prometheus.Registry()
		this.registry.setContentType(
			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
			// @ts-ignore
			Prometheus.openMetricsContentType,
		)
		Prometheus.collectDefaultMetrics({ register: this.registry })
		this.registry.removeSingleMetric(NODEJS_GC_DURATION_SECONDS)
		this.createOtelObservers()
	}

	override enable() {
		this.enabled = true
	}

	override disable() {
		this.enabled = false
	}

	private createOtelObservers() {
		const metrics: Prometheus.MetricObject[] = this.registry.getMetricsAsArray()
		for (const metric of metrics) {
			switch (metric?.type?.toString()) {
				case 'counter':
					this.handleCounter(metric)
					break
				case 'gauge':
					this.handleGuage(metric)
					break
				default:
					// eslint-disable-next-line no-console
					console.log(`Not supported name: ${metric.name} type: ${metric?.type?.toString()}`)
			}
		}
		this.collectGC()
		this.meter.addBatchObservableCallback(
			async (observableResult: BatchObservableResult) => {
				await this.batchObservableCallback(observableResult)
			},
			[...this.metricMap.values()],
		)
	}

	async batchObservableCallback(observableResult: BatchObservableResult) {
		if (!this.enabled) {
			return
		}
		const metrics: Prometheus.MetricObjectWithValues<Prometheus.MetricValue<string>>[] =
			await this.registry.getMetricsAsJSON()
		this.registry.resetMetrics()
		for (const [metricName, observableMetric] of this.metricMap.entries()) {
			const metric: Prometheus.MetricObjectWithValues<Prometheus.MetricValue<string>> = metrics.find(
				(metric) => metric.name === metricName,
			)
			for (const metricValue of metric.values || []) {
				const { value, labels = {} } = metricValue
				observableResult.observe(observableMetric, value, labels)
			}
		}
	}

	handleCounter(metric: Prometheus.MetricObject) {
		const counter: ObservableCounter = this.meter.createObservableCounter(this.getMetricName(metric.name), {
			description: metric.help,
		})
		this.metricMap.set(metric.name, counter)
	}

	handleGuage(metric: Prometheus.MetricObject) {
		const gauge: ObservableGauge = this.meter.createObservableGauge(this.getMetricName(metric.name), {
			description: metric.help,
		})
		this.metricMap.set(metric.name, gauge)
	}

	collectGC() {
		const histogram: Histogram = this.meter.createHistogram(NODEJS_GC_DURATION_SECONDS, {
			description: 'Garbage collection duration by kind, one of major, minor, incremental or weakcb.',
		})
		const labels = {}
		const kinds = {
			[constants.NODE_PERFORMANCE_GC_MAJOR]: { ...labels, kind: 'major' },
			[constants.NODE_PERFORMANCE_GC_MINOR]: { ...labels, kind: 'minor' },
			[constants.NODE_PERFORMANCE_GC_INCREMENTAL]: { ...labels, kind: 'incremental' },
			[constants.NODE_PERFORMANCE_GC_WEAKCB]: { ...labels, kind: 'weakcb' },
		}
		const obs = new PerformanceObserver((list) => {
			if (!this.enabled) {
				return
			}
			const entry: PerformanceEntry = list.getEntries()[0]
			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
			// @ts-ignore
			const kind: number = entry.detail ? entry.detail.kind : entry.kind
			// Convert duration from milliseconds to seconds
			histogram.record(entry.duration / 1000, kinds[kind])
		})
		obs.observe({ entryTypes: ['gc'] })
	}

	private getMetricName(metricName: string) {
		if (metricName.startsWith('nodejs_')) {
			return metricName
		}
		return `nodejs_${metricName}`
	}
}

JavaScript Example

This example demonstrates how to use the prom-client in conjunction with Node.js OpenTelemetry SDK using JavaScript.

We define a RunTimeInstrumentation class. It instantiates and collects the runtime metrics using prom-client. You can add the RunTimeInstrumentation class to the OpenTelemetry NodeSDK list of instrumentations.

Collecting runtime metrics with prom-client
const { RunTimeInstrumentation } = require('./runTimeMetric.class');
const {PeriodicExportingMetricReader} = require('@opentelemetry/sdk-metrics');

const sdk = new opentelemetry.NodeSDK({
  ...
  instrumentations: [
  ...
    new RunTimeInstrumentation()
  ],
  metricReader: new PeriodicExportingMetricReader({
    exporter: new OTLPMetricExporter(),
  }),
  ...
});
runTimeMetric.class.js
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.RunTimeInstrumentation = void 0;
const Prometheus = require("prom-client");
const perf_hooks_1 = require("perf_hooks");
const instrumentation_1 = require("@opentelemetry/instrumentation");
const NODEJS_GC_DURATION_SECONDS = 'nodejs_gc_duration_seconds';
class RunTimeInstrumentation extends instrumentation_1.InstrumentationBase {
    constructor(config = {}) {
        super('@opentelemetry/instrumentation-node-run-time', '1.0', config);
    }
    init() {
        // Not instrumenting or patching a Node.js module
    }
    _updateMetricInstruments() {
        this.metricMap = new Map();
        this.registry = new Prometheus.Registry();
        this.registry.setContentType(
        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
        // @ts-ignore
        Prometheus.openMetricsContentType);
        Prometheus.collectDefaultMetrics({ register: this.registry });
        this.registry.removeSingleMetric(NODEJS_GC_DURATION_SECONDS);
        this.createOtelObservers();
    }
    enable() {
        this.enabled = true;
    }
    disable() {
        this.enabled = false;
    }
    createOtelObservers() {
        var _a, _b;
        const metrics = this.registry.getMetricsAsArray();
        for (const metric of metrics) {
            switch ((_a = metric === null || metric === void 0 ? void 0 : metric.type) === null || _a === void 0 ? void 0 : _a.toString()) {
                case 'counter':
                    this.handleCounter(metric);
                    break;
                case 'gauge':
                    this.handleGuage(metric);
                    break;
                default:
                    // eslint-disable-next-line no-console
                    console.log(`Not supported name: ${metric.name} type: ${(_b = metric === null || metric === void 0 ? void 0 : metric.type) === null || _b === void 0 ? void 0 : _b.toString()}`);
            }
        }
        this.collectGC();
        this.meter.addBatchObservableCallback(async (observableResult) => {
            await this.batchObservableCallback(observableResult);
        }, [...this.metricMap.values()]);
    }
    async batchObservableCallback(observableResult) {
        if (!this.enabled) {
            return;
        }
        const metrics = await this.registry.getMetricsAsJSON();
        this.registry.resetMetrics();
        for (const [metricName, observableMetric] of this.metricMap.entries()) {
            const metric = metrics.find((metric) => metric.name === metricName);
            for (const metricValue of metric.values || []) {
                const { value, labels = {} } = metricValue;
                observableResult.observe(observableMetric, value, labels);
            }
        }
    }
    handleCounter(metric) {
        const counter = this.meter.createObservableCounter(this.getMetricName(metric.name), {
            description: metric.help,
        });
        this.metricMap.set(metric.name, counter);
    }
    handleGuage(metric) {
        const gauge = this.meter.createObservableGauge(this.getMetricName(metric.name), {
            description: metric.help,
        });
        this.metricMap.set(metric.name, gauge);
    }
    collectGC() {
        const histogram = this.meter.createHistogram(NODEJS_GC_DURATION_SECONDS, {
            description: 'Garbage collection duration by kind, one of major, minor, incremental or weakcb.',
        });
        const labels = {};
        const kinds = {
            [perf_hooks_1.constants.NODE_PERFORMANCE_GC_MAJOR]: Object.assign(Object.assign({}, labels), { kind: 'major' }),
            [perf_hooks_1.constants.NODE_PERFORMANCE_GC_MINOR]: Object.assign(Object.assign({}, labels), { kind: 'minor' }),
            [perf_hooks_1.constants.NODE_PERFORMANCE_GC_INCREMENTAL]: Object.assign(Object.assign({}, labels), { kind: 'incremental' }),
            [perf_hooks_1.constants.NODE_PERFORMANCE_GC_WEAKCB]: Object.assign(Object.assign({}, labels), { kind: 'weakcb' }),
        };
        const obs = new perf_hooks_1.PerformanceObserver((list) => {
            if (!this.enabled) {
                return;
            }
            const entry = list.getEntries()[0];
            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
            // @ts-ignore
            const kind = entry.detail ? entry.detail.kind : entry.kind;
            // Convert duration from milliseconds to seconds
            histogram.record(entry.duration / 1000, kinds[kind]);
        });
        obs.observe({ entryTypes: ['gc'] });
    }
    getMetricName(metricName) {
        if (metricName.startsWith('nodejs_')) {
            return metricName;
        }
        return `nodejs_${metricName}`;
    }
}
exports.RunTimeInstrumentation = RunTimeInstrumentation;
//# sourceMappingURL=runTimeMetric.class.js.map