OpenTelemetry (OTel)

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024
  memory_limiter:
    limit_mib: 512
    check_interval: 1s

exporters:
  otlp:
    endpoint: http://tempo:4317
    tls:
      insecure: true

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch, memory_limiter]
      exporters: [otlp]
    metrics:
      receivers: [otlp]
      processors: [batch, memory_limiter]
      exporters: [otlp]
    logs:
      receivers: [otlp]
      processors: [batch, memory_limiter]
      exporters: [otlp]

processors:
  tail_sampling:
    decision_wait: 10s
    num_traces: 100000
    expected_new_traces_per_sec: 100
    policies:
      - name: errors-policy
        type: status_code
        status_code: {status_codes: [ERROR]}
      - name: slow-traces-policy
        type: latency
        latency: {threshold_ms: 1000}
      - name: probabilistic-policy
        type: probabilistic
        probabilistic: {sampling_percentage: 10}
      - name: latency-slo-policy
        type: and
        and: {and_policy_requirements:
          - policy: latency
            latency: {threshold_ms: 100}
          - policy: status_code
            status_code: {status_codes: [OK]}
        }

connectors:
  spanmetrics:
    metrics_exporter: prometheus

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [otlp, spanmetrics]
    metrics:
      receivers: [spanmetrics]
      exporters: [prometheus]

traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01
             │  │                                │              │
             │  trace_id (32 hex chars)         │   │    │
             │                                   │   │    └── flags (2 hex chars)
             │                                   │   └─────────── span_id (16 hex chars)
             │                                   └────────────── version (2 hex chars)
             └────────────────────────────────────────────────── version prefix

tracestate: congo=t61rcWkgMzE,rojo=00f067aa0ba902b7

import "go.opentelemetry.io/otel/propagation"

// Register a composite propagator (trace context + baggage)
otel.SetTextMapPropagator(propagation.NewCompositePropagator(
    propagation.TraceContext{},   // W3C Trace Context
    propagation.Baggage{},         // W3C Baggage
))

from opentelemetry import propagate
from opentelemetry.propagate import set_global_textmap
from opentelemetry.sdk.trace.propagation.tracecontext import TraceContextPropagator

set_global_textmap(TraceContextPropagator())

// Inject: extract context from span and inject into HTTP headers
func makeHTTPRequest(ctx context.Context, url string) (*http.Response, error) {
    req, _ := http.NewRequest("GET", url, nil)
    propagator := propagation.TraceContext{}
    propagator.Inject(ctx, req.Header, propagation.HeaderCarrier(req.Header))
    return http.DefaultClient.Do(req)
}

// Extract: extract trace context from incoming HTTP headers
func handleHTTPRequest(w http.ResponseWriter, r *http.Request) {
    propagator := propagation.TraceContext{}
    ctx := propagator.Extract(r.Context(), propagation.HeaderCarrier(r.Header))
    ctx, span := tracer.Start(ctx, "handler")
    defer span.End()
}

// Via tracestate (preferred — forwarded by more proxies)
tracestate: otel.baggagegage="key1=value1,key2=value2"

// Or via dedicated header (less common)
baggage: key1=value1, key2=value2

import "go.opentelemetry.io/otel/baggage"

// Add baggage
b, _ := baggage.NewMember("tenant.id", "acme-corp")
m, _ := baggage.NewMember("user.role", "admin")
bag, _ := baggage.New(b, m)
ctx := baggage.ContextWithBaggage(ctx, bag)

// Read baggage anywhere in the trace
baggage := baggage.FromContext(ctx)
if val, ok := baggage.Member("tenant.id"); ok {
    span.SetAttributes(attribute.String("tenant.id", val))
}

# Publishing: inject trace context into message headers
from opentelemetry.propagate import inject
headers = {}
inject(headers)  # injects traceparent + baggage into headers
producer.send("my-topic", value=data, headers=headers)

# Consuming: extract context from message and create linked span
from opentelemetry.propagate import extract
ctx = extract(message.headers)
with tracer.start_as_current_span("process-message", context=ctx) as span:
    # span is linked to the producer span
    pass

// 1. Create TracerProvider (once at startup)
tp := trace.NewTracerProvider(
    trace.WithResource(resource.New(ctx,
        resource.WithAttributes(
            attribute.String("service.name", "order-service"),
        ),
    )),
    trace.WithSampler(trace.AlwaysSample()),
)

// 2. Register globally
otel.SetTracerProvider(tp)

// 3. Get a Tracer
tracer := tp.Tracer("order-service")

// 4. Start a span
ctx, span := tracer.Start(ctx, "handleOrders")
defer span.End()

// 5. Add attributes (metadata)
span.SetAttributes(
    attribute.String("order.id", orderID),
    attribute.Float64("order.amount", amount),
    attribute.String("http.method", "POST"),
)

// 6. Add an event (a log point in time)
span.AddEvent("order validated")
span.AddEvent("invoice response received", trace.WithAttributes(
    attribute.Int("http.status_code", 201),
))

// 7. Mark error if needed
span.SetStatus(codes.Error, "failed to call invoice service")

from opentelemetry import trace
from opentelemetry.trace import Status, StatusCode

# 1. Get the global tracer
tracer = trace.get_tracer("invoice-service")

# 2. Start a span (context manager auto-ends)
with tracer.start_as_current_span("generate_invoice") as span:
    span.set_attribute("invoice.order_id", str(order_id))
    span.set_attribute("invoice.amount", amount)
    span.add_event("invoice generation started")

    invoice = generate_invoice(order_id, amount)

    span.set_status(Status(StatusCode.OK))
    # or: span.set_status(Status(StatusCode.ERROR, "reason"))

// 10% of traces; if parent was sampled → sample everything
sampler := trace.ParentBased(
    trace.TraceIDRatioBased(0.1),
)

// Create at startup
counter, err := meter.Int64Counter(
    "http_requests_total",
    metric.WithDescription("Total HTTP requests received"),
    metric.WithUnit("requests"),
)

// Record — always Add() with a positive value for counters
counter.Add(ctx, 1,
    metric.WithAttributes(
        attribute.String("method", "GET"),
        attribute.String("path", "/orders"),
        attribute.String("status", "200"),
    ),
)

// Create at startup
histogram, err := meter.Float64Histogram(
    "order_processing_duration_ms",
    metric.WithDescription("Order processing time in milliseconds"),
    metric.WithUnit("ms"),
    metric.WithExplicitBucketBoundaries(
        5.0, 10.0, 25.0, 50.0, 100.0, 250.0,
        500.0, 1000.0, 2500.0, 5000.0, 10000.0,
    ),
)

// Record a measurement
histogram.Record(ctx, 127.5,
    metric.WithAttributes(
        attribute.String("method", "POST"),
        attribute.String("path", "/orders"),
    ),
)

var currentQueueSize int64

_, err := meter.Int64ObservableGauge(
    "queue_size",
    metric.WithDescription("Current number of items in queue"),
    metric.WithCallback(func(_ context.Context, o metric.Int64Observer) error {
        o.Observe(currentQueueSize)
        return nil
    }),
)

# Create at startup
duration_histogram = meter.create_histogram(
    name="order_processing_duration_ms",
    description="Order processing time in milliseconds",
    unit="ms",
)

# Record
duration_histogram.record(127.5, {"method": "POST", "path": "/orders"})

# Good — low cardinality
counter.add(1, {"customer_tier": "premium"})

# Bad — high cardinality (one time series per unique value)
counter.add(1, {"request_id": "req-abc-123"})

func initMeter(ctx context.Context) (func(), error) {
    exporter, err := otlpmetricgrpc.New(ctx)
    if err != nil {
        return nil, err
    }

    res, err := resource.New(ctx,
        resource.WithAttributes(
            attribute.String("service.name", "order-service"),
        ),
    )

    reader := metric.NewPeriodicBatchReader(exporter,
        metric.WithInterval(10 * time.Second),
    )

    mp := metric.NewMeterProvider(
        metric.WithResource(res),
        metric.WithReader(reader),
    )

    otel.SetMeterProvider(mp)
    return func() { mp.Shutdown(ctx) }, nil
}

from opentelemetry import trace

tracer = trace.get_tracer(__name__)

with tracer.start_as_current_span("process-order") as span:
    # Log within this span — trace_id/span_id auto-attached
    span.add_event("log", attributes={
        "message": "order processing started",
        "order_id": order_id,
    })
    # Span events are logs within a trace context
    # Use span.add_event() for in-trace logging

import logging
from opentelemetry.sdk._logs import LoggingHandler

# Bridge Python stdlib logging → OTel SDK
otel_handler = LoggingHandler()
logging.root.addHandler(otel_handler)

# All logging.info(), logging.error() calls now flow through OTel
logging.info("payment processed", extra={"order_id": "12345"})
# These carry trace_id/span_id when called within a traced context

import "go.opentelemetry.io/otel/log"

// Create a logger (once at startup)
logger := mp.Logger("order-service")

// Emit a log record within a trace context
logger.Info(ctx, "order processed",
    log.WithAttributes(
        attribute.String("order.id", orderID),
        attribute.String("status", "completed"),
    ),
)
// ctx carries trace context — log record gets trace_id + span_id auto

helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
helm install otel-collector open-telemetry/opentelemetry-collector \
  --set mode=daemonset \
  --set config.receivers.otlp.protocols.grpc.endpoint=0.0.0.0:4317 \
  --set config.receivers.otlp.protocols.http.endpoint=0.0.0.0:4318

helm install otel-collector-gateway open-telemetry/opentelemetry-collector \
  --set mode=deployment \
  --set config.receivers.otlp.protocols.grpc.endpoint=0.0.0.0:4317 \
  --set config.exporters.otlp.endpoint=http://tempo:4317

# Install OpenTelemetry Operator
kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml

# Create Instrumentation resource (per namespace)
apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
  name: my-instrumentation
  namespace: default
spec:
  exporter:
    endpoint: http://otel-collector:4317
  propagators:
    - tracecontext
    - baggage
  resource:
    addK8sUID: true
    addAttributes: true

# Annotate pod to enable auto-instrumentation
apiVersion: v1
kind: Pod
metadata:
  annotations:
    instrumentation.opentelemetry.io/inject-sdk: "true"
    instrumentation.opentelemetry.io/inject-tracer: "opentelemetry-auto"
spec:
  containers:
  - name: my-app
    image: my-app:latest

processors:
  k8sattributes:
    extract:
      metadata:
        - k8s.namespace.name
        - k8s.deployment.name
        - k8s.pod.name
        - k8s.pod.uid
        - k8s.pod.start_time
        - k8s.container.name
        - k8s.container.restart_count
    filter:
      node: ".*worker.*"   # Only pods on worker nodes