APM Instrumentation Best Practices

# otel-config.properties
otel.service.name=checkout-service
otel.traces.exporter=otlp
otel.exporter.otlp.endpoint=http://kf-agent:4317
otel.exporter.otlp.compression=gzip
otel.traces.sampler=parentbased_traceidratio
otel.traces.sampler.arg=0.05
otel.propagators=tracecontext,baggage
otel.resource.providers.aws.enabled=true
otel.javaagent.logging=application
otel.bsp.schedule.delay=2000
otel.bsp.max.queue.size=4096

propertiesCopied!

java -javaagent:/path/to/opentelemetry-javaagent.jar \
     -Dotel.javaagent.configuration-file=/etc/otel/otel-config.properties \
     -jar app.jar

bashCopied!

# Suppress a specific library
-Dotel.instrumentation.log4j-appender.enabled=false

# Start from zero and enable only what you need
-Dotel.instrumentation.common.default-enabled=false
-Dotel.instrumentation.opentelemetry-api.enabled=true
-Dotel.instrumentation.spring-webmvc.enabled=true
-Dotel.instrumentation.jdbc.enabled=true

bashCopied!

@WithSpan("process-payment")
public PaymentResult processPayment(
    @SpanAttribute("payment.method") String method,
    @SpanAttribute("payment.amount") double amount
) {
    // Span is created automatically; parameters become span attributes
}

javaCopied!

-Dotel.resource.providers.aws.enabled=true    # EC2, ECS, EKS, Lambda
-Dotel.resource.providers.gcp.enabled=true    # GCE, GKE, Cloud Run
-Dotel.resource.providers.azure.enabled=true  # Azure VMs, AKS

bashCopied!

pip install opentelemetry-distro[otlp] opentelemetry-instrumentation
opentelemetry-bootstrap -a install    # detects and installs instrumentation packages

bashCopied!

OTEL_SERVICE_NAME=my-service \
OTEL_EXPORTER_OTLP_ENDPOINT=http://kf-agent:4317 \
OTEL_TRACES_SAMPLER=parentbased_traceidratio \
OTEL_TRACES_SAMPLER_ARG=0.1 \
opentelemetry-instrument python app.py

bashCopied!

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource, SERVICE_NAME

resource = Resource.create({
    SERVICE_NAME: "checkout-service",
    "service.version": "1.4.2",
    "deployment.environment.name": "production",
})

provider = TracerProvider(resource=resource)
exporter = OTLPSpanExporter(endpoint="http://kf-agent:4317")
provider.add_span_processor(BatchSpanProcessor(exporter))
trace.set_tracer_provider(provider)

pythonCopied!

# gunicorn.conf.py
def post_fork(server, worker):
    # Initialize OTel here — after fork, not at module import time
    setup_opentelemetry()

pythonCopied!

func initTracer(ctx context.Context) (func(context.Context) error, error) {
    exp, err := otlptracegrpc.New(ctx,
        otlptracegrpc.WithEndpoint("kf-agent:4317"),
        otlptracegrpc.WithInsecure(),
    )
    if err != nil {
        return nil, err
    }

    res, err := resource.Merge(
        resource.Default(),
        resource.NewWithAttributes(
            semconv.SchemaURL,
            semconv.ServiceName("my-service"),
            semconv.ServiceVersion("1.0.0"),
            attribute.String("deployment.environment.name", "production"),
        ),
    )
    if err != nil {
        return nil, err
    }

    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exp),
        sdktrace.WithResource(res),
        sdktrace.WithSampler(sdktrace.ParentBased(
            sdktrace.TraceIDRatioBased(0.05),
        )),
    )

    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    return tp.Shutdown, nil
}

func main() {
    ctx := context.Background()
    shutdown, err := initTracer(ctx)
    if err != nil {
        log.Fatal(err)
    }
    defer func() {
        shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
        defer cancel()
        if err := shutdown(shutdownCtx); err != nil {
            log.Printf("tracer shutdown error: %v", err)
        }
    }()

    // ... application code
}

goCopied!

func handleRequest(ctx context.Context) {
    ctx, span := tracer.Start(ctx, "handle-request")
    defer span.End()

    // Pass ctx to all child operations
    result, err := fetchData(ctx)

    // Pass ctx explicitly to goroutines — do not capture from outer scope
    go func(ctx context.Context) {
        _, childSpan := tracer.Start(ctx, "async-task")
        defer childSpan.End()
        // ...
    }(ctx)
}

goCopied!

import "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"

// Server: wraps the handler and extracts incoming trace context
http.Handle("/api/orders", otelhttp.NewHandler(ordersHandler, "handle-orders"))

// Client: injects trace context into outgoing requests
client := &http.Client{
    Transport: otelhttp.NewTransport(http.DefaultTransport),
}

goCopied!

// Inject into outgoing request
otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(req.Header))

// Extract from incoming request
ctx = otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header))

goCopied!