A Simple Analogy

OpenTelemetry is like X-ray and thermometer for your application. It sees inside (traces), measures temperature (metrics), and records findings (logs) all without modifying business code.

---

What Is OpenTelemetry?

OpenTelemetry is a vendor-neutral standard for collecting traces, metrics, and logs from applications. One instrumentation, multiple backends.

---

Setup

builder.Services.AddOpenTelemetry()
    .WithTracing(tracing =>
    {
        tracing.SetResourceBuilder(ResourceBuilder
            .CreateDefault()
            .AddService("OrderApi", serviceVersion: "1.0"));
        
        tracing.AddAspNetCoreInstrumentation()
            .AddHttpClientInstrumentation()
            .AddSqlClientInstrumentation()
            .AddOtlpExporter(options =>
            {
                options.Endpoint = new Uri("http://localhost:4317");
            });
    })
    .WithMetrics(metrics =>
    {
        metrics.SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("OrderApi"));
        
        metrics.AddAspNetCoreInstrumentation()
            .AddHttpClientInstrumentation()
            .AddRuntimeInstrumentation()
            .AddOtlpExporter(options =>
            {
                options.Endpoint = new Uri("http://localhost:4317");
            });
    });

---

Custom Instrumentation

public class OrderService
{
    private static readonly ActivitySource ActivitySource = 
        new ActivitySource("OrderService", "1.0");
    
    public async Task<Order> CreateOrderAsync(Order order)
    {
        using var activity = ActivitySource.StartActivity("CreateOrder");
        activity?.SetTag("order.id", order.Id);
        activity?.SetTag("customer.id", order.CustomerId);
        
        try
        {
            // Simulate work
            await Task.Delay(100);
            
            activity?.SetTag("order.total", order.Total);
            activity?.AddEvent(new ActivityEvent("Order created successfully"));
            
            return order;
        }
        catch (Exception ex)
        {
            activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
            activity?.RecordException(ex);
            throw;
        }
    }
}

---

Metrics

public class OrderMetrics
{
    private readonly Counter<int> _ordersCreated;
    private readonly Counter<int> _ordersCancelled;
    private readonly Histogram<double> _orderAmount;
    
    public OrderMetrics(IMeterFactory meterFactory)
    {
        var meter = meterFactory.Create("OrderService");
        
        _ordersCreated = meter.CreateCounter<int>(
            "orders.created",
            description: "Number of orders created");
        
        _ordersCancelled = meter.CreateCounter<int>(
            "orders.cancelled",
            description: "Number of orders cancelled");
        
        _orderAmount = meter.CreateHistogram<double>(
            "order.amount",
            unit: "USD",
            description: "Order amounts");
    }
    
    public void RecordOrderCreated(Order order)
    {
        _ordersCreated.Add(1);
        _orderAmount.Record(Convert.ToDouble(order.Total));
    }
    
    public void RecordOrderCancelled()
    {
        _ordersCancelled.Add(1);
    }
}

// In service
public class OrderService
{
    private readonly OrderMetrics _metrics;
    
    public async Task<Order> CreateOrderAsync(Order order)
    {
        await SaveAsync(order);
        _metrics.RecordOrderCreated(order);
        return order;
    }
}

---

Structured Logging

var logger = app.Services.GetRequiredService<ILogger<Program>>();

app.MapPost("/api/orders", async (CreateOrderRequest request, IOrderService service) =>
{
    logger.LogInformation(
        "Creating order for customer {CustomerId} with {ItemCount} items",
        request.CustomerId,
        request.Items.Count);
    
    try
    {
        var order = await service.CreateAsync(request);
        
        logger.LogInformation(
            "Order {OrderId} created successfully with total {Total}",
            order.Id,
            order.Total);
        
        return order;
    }
    catch (Exception ex)
    {
        logger.LogError(
            ex,
            "Failed to create order for customer {CustomerId}",
            request.CustomerId);
        
        throw;
    }
});

---

Distributed Tracing

// Service A calls Service B
public class OrderService
{
    private readonly HttpClient _httpClient;
    
    public async Task ProcessOrderAsync(Order order)
    {
        // HttpClient instrumentation automatically propagates context
        var paymentResult = await _httpClient.PostAsJsonAsync(
            "https://payment-service/api/process",
            new PaymentRequest { OrderId = order.Id });
        
        // Trace automatically connects: OrderService -> PaymentService
    }
}

---

Observability Backend

# Jaeger for traces
docker run -d --name jaeger \
  -p 16686:16686 \
  jaegertracing/all-in-one

# Prometheus for metrics
docker run -d --name prometheus \
  -p 9090:9090 \
  -v prometheus.yml:/etc/prometheus/prometheus.yml \
  prom/prometheus

# Grafana for visualization
docker run -d --name grafana \
  -p 3000:3000 \
  grafana/grafana

---

Best Practices

1. Use semantic conventions: Standard tag names
2. Sample appropriately: Not all traces needed
3. Include context: Request IDs, user IDs
4. Monitor observability: Ensure it's not slow
5. Alert on baselines: Detect anomalies

---

Related Concepts

• Prometheus metrics format
• Jaeger distributed tracing
• ELK stack for logs
• Service mesh observability

---

Summary

OpenTelemetry provides unified observability across traces, metrics, and logs. Instrument code once, export to multiple backends for comprehensive visibility into distributed systems.