{
  "title": "Fuzzball Fleet Health",
  "uid": "fuzzball-node-reliability",
  "description": "Nodes by status and by hold, hold churn, and the node-health alerts firing across the fleet.",
  "tags": ["fuzzball", "node-health"],
  "timezone": "browser",
  "schemaVersion": 39,
  "refresh": "1m",
  "time": { "from": "now-24h", "to": "now" },
  "templating": {
    "list": [
      {
        "name": "datasource",
        "label": "Prometheus",
        "type": "datasource",
        "query": "prometheus"
      }
    ]
  },
  "links": [
    {
      "title": "Node view",
      "type": "link",
      "url": "/d/fuzzball-node-health"
    }
  ],
  "panels": [
    {
      "type": "timeseries",
      "title": "Nodes by status",
      "description": "What Fuzzball has observed about each node. A cordon is a hold, not a status: see the hold panels.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
      "fieldConfig": { "defaults": { "custom": { "stacking": { "mode": "normal" } } } },
      "targets": [
        {
          "refId": "A",
          "expr": "count by (status) (max by (node, node_id, status) (fuzzball_node_status) == 1)",
          "legendFormat": "{{status}}"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Held nodes by effect",
      "description": "Nodes holding at least one hold of each effect. A node with a cordon and a drain counts under both.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
      "targets": [
        {
          "refId": "A",
          "expr": "count by (effect) (max by (node, node_id, effect) (fuzzball_node_hold) > 0)",
          "legendFormat": "{{effect}}"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Hold churn",
      "description": "Hold placements and releases per hour, by source. A monitoring line that climbs while the fleet is stable is a rule flapping.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
      "targets": [
        {
          "refId": "A",
          "expr": "sum by (source) (changes((max by (node, node_id, effect, source) (fuzzball_node_hold))[1h:1m]))",
          "legendFormat": "{{source}}"
        }
      ]
    },
    {
      "type": "bargauge",
      "title": "Held nodes by source",
      "description": "Who placed the holds in force now: an operator, monitoring, or the system.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
      "targets": [
        {
          "refId": "A",
          "expr": "count by (source) (max by (node, node_id, source) (fuzzball_node_hold) > 0)",
          "legendFormat": "{{source}}",
          "instant": true
        }
      ]
    },
    {
      "type": "table",
      "title": "Firing node-health alerts",
      "description": "One row per firing alert. fb_action is what the alert asks Fuzzball to do to the node; empty means advisory.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (alertname, node, node_id, fb_action) (ALERTS{alertstate=\"firing\", alertname=~\"FuzzballNode.*\"})",
          "format": "table",
          "instant": true
        }
      ]
    },
    {
      "type": "table",
      "title": "Silent nodes",
      "description": "Nodes Fuzzball has not heard from for more than ten minutes, with the seconds since their last report.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
      "targets": [
        {
          "refId": "A",
          "expr": "time() - max by (node, node_id) (fuzzball_node_last_health_report_timestamp_seconds) > 600",
          "format": "table",
          "instant": true
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Job failures attributed to nodes (top 10)",
      "description": "Cumulative jobs that stopped responding on a node. A gauge restored on control-plane restart, so read the level, not its rate.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
      "targets": [
        {
          "refId": "A",
          "expr": "topk(10, max by (node, node_id) (fuzzball_node_attributed_job_failures))",
          "legendFormat": "{{node}}"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Nodes with a failed device plugin",
      "description": "A failed plugin keeps the node out of every job requesting its device kind while the node stays ready for the rest.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
      "targets": [
        {
          "refId": "A",
          "expr": "count by (plugin) (max by (node, node_id, plugin) (fuzzball_node_device_plugin{state=\"failed\"}) == 1)",
          "legendFormat": "{{plugin}}"
        }
      ]
    }
  ]
}
