OpenTelemetry Distributed Observability

// SPDX-License-Identifier: AGPL-3.0-or-later

use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt};

/// Initialize structured JSON logging with the given default level.
///
/// Respects `RUST_LOG` for fine-grained control. Falls back to `default_level`
/// if `RUST_LOG` is not set.
pub fn init(default_level: &str) {
    let filter =
        EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default_level));

    tracing_subscriber::registry()
        .with(filter)
        .with(fmt::layer().json().flatten_event(true))
        .init();
}

# Add to [workspace.dependencies] section:
opentelemetry = "0.29"
opentelemetry_sdk = { version = "0.29", features = ["rt-tokio"] }
opentelemetry-otlp = { version = "0.29", features = ["tonic"] }
opentelemetry-semantic-conventions = "0.29"
tracing-opentelemetry = "0.30"
opentelemetry-prometheus = "0.29"
prometheus = "0.13"

# Add to [dependencies]:
opentelemetry = { workspace = true, optional = true }
opentelemetry_sdk = { workspace = true, optional = true }
opentelemetry-otlp = { workspace = true, optional = true }
tracing-opentelemetry = { workspace = true, optional = true }
opentelemetry-prometheus = { workspace = true, optional = true }
prometheus = { workspace = true, optional = true }

# Add new section:
[features]
default = ["otel"]
otel = [
    "dep:opentelemetry",
    "dep:opentelemetry_sdk",
    "dep:opentelemetry-otlp",
    "dep:tracing-opentelemetry",
    "dep:opentelemetry-prometheus",
    "dep:prometheus",
]

// SPDX-License-Identifier: AGPL-3.0-or-later

use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt};

/// Guard that flushes OTEL providers on drop. Hold this in `main()`.
///
/// When OTEL is not configured, the fields are `None` and drop is a no-op.
pub struct TelemetryGuard {
    #[cfg(feature = "otel")]
    _trace_provider: Option<opentelemetry_sdk::trace::SdkTracerProvider>,
    #[cfg(feature = "otel")]
    _meter_provider: Option<opentelemetry_sdk::metrics::SdkMeterProvider>,
}

#[cfg(feature = "otel")]
impl Drop for TelemetryGuard {
    fn drop(&mut self) {
        if let Some(tp) = self._trace_provider.take() {
            if let Err(e) = tp.shutdown() {
                eprintln!("error shutting down trace provider: {e}");
            }
        }
        if let Some(mp) = self._meter_provider.take() {
            if let Err(e) = mp.shutdown() {
                eprintln!("error shutting down meter provider: {e}");
            }
        }
    }
}

#[cfg(feature = "otel")]
static METRICS_REGISTRY: std::sync::OnceLock<prometheus::Registry> = std::sync::OnceLock::new();

/// Returns the Prometheus metrics registry, if OTEL was initialized.
#[cfg(feature = "otel")]
pub fn metrics_registry() -> Option<&'static prometheus::Registry> {
    METRICS_REGISTRY.get()
}

/// Returns `None` when compiled without the `otel` feature.
#[cfg(not(feature = "otel"))]
pub fn metrics_registry() -> Option<&'static ()> {
    None
}

/// Initialize telemetry. If the `otel` feature is enabled AND
/// `OTEL_EXPORTER_OTLP_ENDPOINT` is set, enables OpenTelemetry trace export
/// and Prometheus metrics. Otherwise, JSON logging only (same as before).
///
/// The `service_name` parameter is used as the OTEL resource service name
/// and is ignored when OTEL is not active.
///
/// Returns a `TelemetryGuard` that MUST be held alive for the duration of
/// `main()`. Dropping it flushes pending spans and metrics.
pub fn init(default_level: &str, service_name: &str) -> TelemetryGuard {
    let filter =
        EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default_level));

    #[cfg(feature = "otel")]
    {
        let otel_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok();

        if let Some(endpoint) = otel_endpoint {
            // -- OTLP trace exporter --
            let exporter = opentelemetry_otlp::SpanExporter::builder()
                .with_tonic()
                .with_endpoint(&endpoint)
                .build()
                .expect("failed to build OTLP span exporter");

            let trace_provider = opentelemetry_sdk::trace::SdkTracerProvider::builder()
                .with_batch_exporter(exporter)
                .with_resource(
                    opentelemetry_sdk::Resource::builder()
                        .with_service_name(service_name.to_owned())
                        .build(),
                )
                .build();
            opentelemetry::global::set_tracer_provider(trace_provider.clone());

            // -- Prometheus metrics exporter --
            let registry = prometheus::Registry::new();
            let prometheus_exporter = opentelemetry_prometheus::exporter()
                .with_registry(registry.clone())
                .build()
                .expect("failed to build Prometheus exporter");

            let meter_provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder()
                .with_reader(prometheus_exporter)
                .build();
            opentelemetry::global::set_meter_provider(meter_provider.clone());
            METRICS_REGISTRY.set(registry).ok();

            // -- W3C TraceContext propagator --
            opentelemetry::global::set_text_map_propagator(
                opentelemetry_sdk::propagation::TraceContextPropagator::new(),
            );

            // -- Compose subscriber: JSON fmt + OpenTelemetry layer --
            let otel_layer = tracing_opentelemetry::OpenTelemetryLayer::new(
                trace_provider.tracer(service_name.to_owned()),
            );

            tracing_subscriber::registry()
                .with(filter)
                .with(fmt::layer().json().flatten_event(true))
                .with(otel_layer)
                .init();

            return TelemetryGuard {
                _trace_provider: Some(trace_provider),
                _meter_provider: Some(meter_provider),
            };
        }
    }

    // No OTEL — behave exactly as today (JSON structured logging)
    tracing_subscriber::registry()
        .with(filter)
        .with(fmt::layer().json().flatten_event(true))
        .init();

    TelemetryGuard {
        #[cfg(feature = "otel")]
        _trace_provider: None,
        #[cfg(feature = "otel")]
        _meter_provider: None,
    }
}

/// Inject the current span's W3C trace context headers into an outbound
/// `reqwest::RequestBuilder`. No-op when OTEL is not active.
///
/// Usage: `let req = inject_trace_headers(client.get(url).bearer_auth(token));`
pub fn inject_trace_headers(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
    #[cfg(feature = "otel")]
    {
        use tracing_opentelemetry::OpenTelemetrySpanExt;
        let cx = tracing::Span::current().context();
        let mut carrier = std::collections::HashMap::new();
        opentelemetry::global::get_text_map_propagator(|p| {
            p.inject_context(&cx, &mut carrier);
        });
        let mut builder = builder;
        for (key, value) in carrier {
            if let Ok(name) = reqwest::header::HeaderName::from_bytes(key.as_bytes()) {
                builder = builder.header(name, value);
            }
        }
        builder
    }
    #[cfg(not(feature = "otel"))]
    {
        builder
    }
}

# Add to [dependencies]:
reqwest = { workspace = true }

# SPDX-License-Identifier: AGPL-3.0-or-later

[package]
name = "craig-common"
version.workspace = true
edition.workspace = true
license.workspace = true

[dependencies]
anyhow = { workspace = true }
axum = { workspace = true }
chrono = { workspace = true }
config = { workspace = true }
reqwest = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
utoipa = { workspace = true }
uuid = { workspace = true }
# OTEL (optional, default = on)
opentelemetry = { workspace = true, optional = true }
opentelemetry_sdk = { workspace = true, optional = true }
opentelemetry-otlp = { workspace = true, optional = true }
tracing-opentelemetry = { workspace = true, optional = true }
opentelemetry-prometheus = { workspace = true, optional = true }
prometheus = { workspace = true, optional = true }

[dev-dependencies]
tokio = { workspace = true }

[features]
default = ["otel"]
otel = [
    "dep:opentelemetry",
    "dep:opentelemetry_sdk",
    "dep:opentelemetry-otlp",
    "dep:tracing-opentelemetry",
    "dep:opentelemetry-prometheus",
    "dep:prometheus",
]

// SPDX-License-Identifier: AGPL-3.0-or-later

//! OpenTelemetry middleware for W3C TraceContext propagation on inbound HTTP
//! requests. When OTEL is not configured, the global propagator is a no-op
//! and this middleware adds zero overhead.

use axum::extract::Request;
use axum::middleware::Next;
use axum::response::Response;

/// Axum middleware: extracts W3C `traceparent`/`tracestate` from inbound
/// request headers and sets as parent context for the current tracing span.
///
/// Layer this BELOW `TraceLayer` in the `.layer()` chain (Axum layers execute
/// bottom-to-top) so that OTEL context extraction runs first, then TraceLayer
/// creates its HTTP span as a child of the extracted parent.
pub async fn otel_propagation(request: Request, next: Next) -> Response {
    #[cfg(feature = "otel")]
    {
        use tracing::Instrument;
        use tracing_opentelemetry::OpenTelemetrySpanExt;

        let headers = request.headers();
        let parent_cx = opentelemetry::global::get_text_map_propagator(|propagator| {
            propagator.extract(&opentelemetry_http::HeaderExtractor(headers))
        });

        let span = tracing::info_span!(
            "http_request",
            otel.kind = "server",
            http.method = %request.method(),
            http.route = %request.uri().path(),
        );
        span.set_parent(parent_cx);

        next.run(request).instrument(span).await
    }

    #[cfg(not(feature = "otel"))]
    {
        next.run(request).await
    }
}

mod bootstrap;
pub mod idempotency;
mod otel;

    pub fn router(
        state: AppState,
        service_routes: Router<AppState>,
        opts: ServerOptions,
        api_doc: Option<utoipa::openapi::OpenApi>,
    ) -> Router {
        let cors = build_cors(&opts.cors_origins);

        let idempotency_cache = IdempotencyCache::new();
        let auth_layer = axum::middleware::from_fn_with_state(state.auth.clone(), auth_middleware);
        let idempotency_layer =
            axum::middleware::from_fn_with_state(idempotency_cache, idempotency_middleware);

        // Layers execute bottom-to-top: auth runs first (inserts Claims),
        // then idempotency checks for cached responses.
        let protected = service_routes.layer(idempotency_layer).layer(auth_layer);

        let mut router = Router::new()
            .nest("/v1", protected)
            .route("/healthz", get(health_check));

        if let Some(doc) = api_doc {
            router = router.merge(SwaggerUi::new("/swagger-ui").url("/api-doc/openapi.json", doc));
        }

        router
            .layer(DefaultBodyLimit::max(opts.body_limit))
            .layer(TraceLayer::new_for_http())
            .layer(CompressionLayer::new())
            .layer(cors)
            .with_state(state)
    }

        let mut router = Router::new()
            .nest("/v1", protected)
            .route("/healthz", get(health_check))
            .route("/metrics", get(metrics_handler));

        if let Some(doc) = api_doc {
            router = router.merge(SwaggerUi::new("/swagger-ui").url("/api-doc/openapi.json", doc));
        }

        router
            .layer(DefaultBodyLimit::max(opts.body_limit))
            .layer(TraceLayer::new_for_http())
            .layer(axum::middleware::from_fn(otel::otel_propagation))
            .layer(CompressionLayer::new())
            .layer(cors)
            .with_state(state)

async fn metrics_handler() -> axum::response::Response {
    use axum::http::header;
    use axum::response::IntoResponse;

    match craig_common::telemetry::metrics_registry() {
        Some(registry) => {
            use prometheus::Encoder;
            let encoder = prometheus::TextEncoder::new();
            let mut buffer = Vec::new();
            let _ = encoder.encode(&registry.gather(), &mut buffer);
            (
                [(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
                buffer,
            )
                .into_response()
        }
        None => (
            [(header::CONTENT_TYPE, "text/plain; charset=utf-8")],
            Vec::<u8>::new(),
        )
            .into_response(),
    }
}

# Add to [dependencies]:
prometheus = { workspace = true }
opentelemetry = { workspace = true }
opentelemetry-http = { workspace = true }
tracing-opentelemetry = { workspace = true }

# Add to [workspace.dependencies]:
opentelemetry-http = "0.29"

// SPDX-License-Identifier: AGPL-3.0-or-later

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;

/// Standard message envelope for all events on the message bus.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EventEnvelope {
    /// Unique message ID.
    pub id: Uuid,
    /// Timestamp when the event was produced.
    pub timestamp: DateTime<Utc>,
    /// Service that produced the event.
    pub source_service: String,
    /// Event type used as routing key (e.g. "case.created").
    pub event_type: String,
    /// Event-specific payload.
    pub payload: serde_json::Value,
}

impl EventEnvelope {
    pub fn new(
        source_service: impl Into<String>,
        event_type: impl Into<String>,
        payload: serde_json::Value,
    ) -> Self {
        Self {
            id: Uuid::now_v7(),
            timestamp: Utc::now(),
            source_service: source_service.into(),
            event_type: event_type.into(),
            payload,
        }
    }
}

// SPDX-License-Identifier: AGPL-3.0-or-later

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;

/// W3C TraceContext carried inside event envelopes for cross-service
/// trace correlation through RabbitMQ.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TraceContext {
    pub traceparent: String,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub tracestate: Option<String>,
}

/// Standard message envelope for all events on the message bus.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EventEnvelope {
    /// Unique message ID.
    pub id: Uuid,
    /// Timestamp when the event was produced.
    pub timestamp: DateTime<Utc>,
    /// Service that produced the event.
    pub source_service: String,
    /// Event type used as routing key (e.g. "case.created").
    pub event_type: String,
    /// Event-specific payload.
    pub payload: serde_json::Value,
    /// Optional W3C trace context for distributed tracing across MQ boundaries.
    /// Backward compatible: old messages without this field deserialize as None.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub trace_context: Option<TraceContext>,
}

impl EventEnvelope {
    pub fn new(
        source_service: impl Into<String>,
        event_type: impl Into<String>,
        payload: serde_json::Value,
    ) -> Self {
        Self {
            id: Uuid::now_v7(),
            timestamp: Utc::now(),
            source_service: source_service.into(),
            event_type: event_type.into(),
            payload,
            trace_context: None,
        }
    }
}

    /// Publish an event envelope to the topic exchange.
    ///
    /// The `event_type` field of the envelope is used as the routing key.
    pub async fn publish(&self, envelope: &EventEnvelope) -> Result<(), lapin::Error> {
        let payload =
            serde_json::to_vec(envelope).expect("EventEnvelope serialization should not fail");

        let routing_key = &envelope.event_type;

        self.channel
            .basic_publish(
                EVENTS_EXCHANGE.into(),
                routing_key.as_str().into(),
                BasicPublishOptions::default(),
                &payload,
                BasicProperties::default()
                    .with_content_type("application/json".into())
                    .with_delivery_mode(2), // persistent
            )
            .await?
            .await?;

        debug!(routing_key, id = %envelope.id, "event published");
        Ok(())
    }

    /// Publish an event envelope to the topic exchange.
    ///
    /// The `event_type` field of the envelope is used as the routing key.
    /// If OTEL is active, injects the current span's W3C trace context into
    /// the envelope so consumers can link their spans to the publisher's trace.
    pub async fn publish(&self, envelope: &EventEnvelope) -> Result<(), lapin::Error> {
        let mut envelope = envelope.clone();
        envelope.trace_context = Self::inject_trace_context();

        let payload =
            serde_json::to_vec(&envelope).expect("EventEnvelope serialization should not fail");

        let routing_key = &envelope.event_type;

        self.channel
            .basic_publish(
                EVENTS_EXCHANGE.into(),
                routing_key.as_str().into(),
                BasicPublishOptions::default(),
                &payload,
                BasicProperties::default()
                    .with_content_type("application/json".into())
                    .with_delivery_mode(2), // persistent
            )
            .await?
            .await?;

        debug!(routing_key, id = %envelope.id, "event published");
        Ok(())
    }

    /// Extract the current span's W3C trace context for injection into the
    /// event envelope. Returns `None` when OTEL is not configured or the
    /// current span has no valid trace context.
    #[cfg(feature = "otel")]
    fn inject_trace_context() -> Option<crate::envelope::TraceContext> {
        use tracing_opentelemetry::OpenTelemetrySpanExt;
        let cx = tracing::Span::current().context();
        let mut carrier = std::collections::HashMap::new();
        opentelemetry::global::get_text_map_propagator(|p| {
            p.inject_context(&cx, &mut carrier);
        });
        carrier.get("traceparent").map(|tp| crate::envelope::TraceContext {
            traceparent: tp.clone(),
            tracestate: carrier.get("tracestate").cloned(),
        })
    }

    #[cfg(not(feature = "otel"))]
    fn inject_trace_context() -> Option<crate::envelope::TraceContext> {
        None
    }

                        match serde_json::from_slice::<EventEnvelope>(&delivery.data) {
                            Ok(envelope) => {
                                if let Err(e) = handler(envelope).await {

                        match serde_json::from_slice::<EventEnvelope>(&delivery.data) {
                            Ok(envelope) => {
                                let result = {
                                    let _span_guard = create_consumer_span(&envelope).entered();
                                    handler(envelope).await
                                };
                                if let Err(e) = result {

/// Create a tracing span for event processing, optionally linked to the
/// publisher's trace context for distributed trace correlation.
fn create_consumer_span(envelope: &EventEnvelope) -> tracing::Span {
    let span = tracing::info_span!(
        "event_process",
        messaging.system = "rabbitmq",
        messaging.operation = "process",
        messaging.destination.name = %envelope.event_type,
        craig.event_id = %envelope.id,
        craig.source_service = %envelope.source_service,
    );

    #[cfg(feature = "otel")]
    if let Some(ref tc) = envelope.trace_context {
        use tracing_opentelemetry::OpenTelemetrySpanExt;
        let mut carrier = std::collections::HashMap::new();
        carrier.insert("traceparent".to_string(), tc.traceparent.clone());
        if let Some(ref ts) = tc.tracestate {
            carrier.insert("tracestate".to_string(), ts.clone());
        }
        let parent_cx = opentelemetry::global::get_text_map_propagator(|p| p.extract(&carrier));
        span.set_parent(parent_cx);
    }

    span
}

# Add to [dependencies]:
opentelemetry = { workspace = true, optional = true }
tracing-opentelemetry = { workspace = true, optional = true }

# Add new section:
[features]
default = ["otel"]
otel = ["dep:opentelemetry", "dep:tracing-opentelemetry"]

    let mq_conn = craig_mq::connect(&settings.rabbitmq_url)
        .await
        .context("failed to connect to RabbitMQ")?;

    let mq_health = MqHealth(std::sync::Arc::new(mq_conn));

    let mq_conn = craig_mq::connect(&settings.rabbitmq_url)
        .await
        .context("failed to connect to RabbitMQ")?;
    let mq_conn = std::sync::Arc::new(mq_conn);
    let mq_health = crate::MqHealth(mq_conn.clone());

    let pub_channel = mq_conn
        .create_channel()
        .await
        .context("failed to create publisher channel")?;
    let publisher = Publisher::new(pub_channel);

    let sub_channel = mq_conn
        .create_channel()
        .await
        .context("failed to create subscriber channel")?;
    let subscriber = Subscriber::new(sub_channel);

pub struct BootstrapResult {
    pub db: DbPool,
    pub auth: AuthLayer,
    pub publisher: Publisher,
    pub subscriber: Subscriber,
    pub mq_health: crate::MqHealth,
    pub _telemetry: craig_common::telemetry::TelemetryGuard,
}

    // After `ApiServer::router(...)`:
    let router = router.layer(Extension(mq_health));

      OTEL_EXPORTER_OTLP_ENDPOINT: http://jaeger:4317

      jaeger:
        condition: service_healthy

  jaeger:
    image: jaegertracing/jaeger:2
    ports:
      - "16686:16686"   # Jaeger UI
      - "4317:4317"     # OTLP gRPC receiver
    environment:
      COLLECTOR_OTLP_ENABLED: "true"
    healthcheck:
      test: ["CMD-SHELL", "wget --spider -q http://localhost:16686 || exit 1"]
      interval: 5s
      timeout: 3s
      retries: 5

global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'craig-rules'
    metrics_path: '/metrics'
    static_configs:
      - targets: ['craig-rules:8001']
  - job_name: 'craig-cases'
    metrics_path: '/metrics'
    static_configs:
      - targets: ['craig-cases:8002']
  - job_name: 'craig-placement'
    metrics_path: '/metrics'
    static_configs:
      - targets: ['craig-placement:8003']
  - job_name: 'craig-exchange'
    metrics_path: '/metrics'
    static_configs:
      - targets: ['craig-exchange:8004']
  - job_name: 'craig-financial'
    metrics_path: '/metrics'
    static_configs:
      - targets: ['craig-financial:8005']
  - job_name: 'craig-reporting'
    metrics_path: '/metrics'
    static_configs:
      - targets: ['craig-reporting:8006']
  - job_name: 'craig-security'
    metrics_path: '/metrics'
    static_configs:
      - targets: ['craig-security:8007']
  - job_name: 'craig-intake'
    metrics_path: '/metrics'
    static_configs:
      - targets: ['craig-intake:8008']
  - job_name: 'craig-web'
    metrics_path: '/metrics'
    static_configs:
      - targets: ['craig-web:8080']

FROM prom/prometheus:latest
COPY prometheus.yml /etc/prometheus/prometheus.yml

  prometheus:
    build: ./devstack/prometheus
    ports:
      - "9090:9090"
    depends_on:
      craig-rules:
        condition: service_healthy
      craig-cases:
        condition: service_healthy
      craig-placement:
        condition: service_healthy
      craig-exchange:
        condition: service_healthy
      craig-financial:
        condition: service_healthy
      craig-reporting:
        condition: service_healthy
      craig-security:
        condition: service_healthy
      craig-intake:
        condition: service_healthy
      craig-web:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy"]
      interval: 10s
      timeout: 3s
      retries: 5
    profiles:
      - observability

FROM grafana/grafana:latest
COPY provisioning/ /etc/grafana/provisioning/

apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
  - name: Jaeger
    type: jaeger
    access: proxy
    url: http://jaeger:16686

apiVersion: 1
providers:
  - name: 'default'
    orgId: 1
    folder: ''
    type: file
    disableDeletion: false
    editable: true
    options:
      path: /etc/grafana/provisioning/dashboards/json
      foldersFromFilesStructure: false

{
  "annotations": { "list": [] },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "title": "Request Rate by Service",
      "type": "timeseries",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
      "targets": [
        {
          "expr": "rate(http_server_request_duration_seconds_count[5m])",
          "legendFormat": "{{job}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10 },
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": { "legend": { "displayMode": "table", "placement": "bottom" } }
    },
    {
      "title": "p95 Latency by Service",
      "type": "timeseries",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
      "targets": [
        {
          "expr": "histogram_quantile(0.95, rate(http_server_request_duration_seconds_bucket[5m]))",
          "legendFormat": "{{job}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10 },
          "unit": "s"
        },
        "overrides": []
      },
      "options": { "legend": { "displayMode": "table", "placement": "bottom" } }
    },
    {
      "title": "Error Rate (5xx) by Service",
      "type": "timeseries",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
      "targets": [
        {
          "expr": "rate(http_server_request_duration_seconds_count{http_status_code=~\"5..\"}[5m])",
          "legendFormat": "{{job}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "custom": { "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10 },
          "color": { "mode": "fixed", "fixedColor": "red" },
          "unit": "reqps"
        },
        "overrides": []
      },
      "options": { "legend": { "displayMode": "table", "placement": "bottom" } }
    },
    {
      "title": "Active Spans (Gauge)",
      "type": "stat",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
      "targets": [
        {
          "expr": "sum by (job) (http_server_active_requests)",
          "legendFormat": "{{job}}",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": { "unit": "short" },
        "overrides": []
      },
      "options": {
        "reduceOptions": { "calcs": ["lastNotNull"] },
        "textMode": "auto",
        "colorMode": "value",
        "graphMode": "area"
      }
    }
  ],
  "refresh": "10s",
  "schemaVersion": 39,
  "tags": ["craig", "overview"],
  "templating": { "list": [] },
  "time": { "from": "now-1h", "to": "now" },
  "timepicker": {},
  "timezone": "",
  "title": "CRAIG Overview",
  "uid": "craig-overview",
  "version": 1
}

  grafana:
    build: ./devstack/grafana
    ports:
      - "3001:3000"
    environment:
      GF_SECURITY_ADMIN_PASSWORD: admin
      GF_AUTH_ANONYMOUS_ENABLED: "true"
      GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
    depends_on:
      prometheus:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"]
      interval: 10s
      timeout: 3s
      retries: 5
    profiles:
      - observability