{
  "title": "Fuzzball Node Health",
  "uid": "fuzzball-node-health",
  "description": "One node: its status, its holds, the alerts firing on it, and the exporter series behind them.",
  "tags": ["fuzzball", "node-health"],
  "timezone": "browser",
  "schemaVersion": 39,
  "refresh": "1m",
  "time": { "from": "now-24h", "to": "now" },
  "templating": {
    "list": [
      {
        "name": "datasource",
        "label": "Prometheus",
        "type": "datasource",
        "query": "prometheus"
      },
      {
        "name": "node",
        "label": "Node",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "${datasource}" },
        "query": { "query": "label_values(fuzzball_node_info, node)", "refId": "node" },
        "refresh": 2,
        "sort": 1
      }
    ]
  },
  "links": [
    {
      "title": "Fleet view",
      "type": "link",
      "url": "/d/fuzzball-node-reliability"
    },
    {
      "title": "Node resources",
      "type": "link",
      "url": "/d/fuzzball-node-resources",
      "includeVars": true,
      "keepTime": true
    }
  ],
  "panels": [
    {
      "type": "stat",
      "title": "Status",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
      "options": { "textMode": "name", "reduceOptions": { "calcs": ["lastNotNull"] } },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node_id, status) (fuzzball_node_status{node=\"$node\"}) == 1",
          "legendFormat": "{{status}}",
          "instant": true
        }
      ]
    },
    {
      "type": "stat",
      "title": "Since last report",
      "description": "A node silent for ten minutes raises FuzzballNodeUnknown.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
      "fieldConfig": { "defaults": { "unit": "s" } },
      "targets": [
        {
          "refId": "A",
          "expr": "time() - max by (node_id) (fuzzball_node_last_health_report_timestamp_seconds{node=\"$node\"})",
          "legendFormat": "{{node_id}}",
          "instant": true
        }
      ]
    },
    {
      "type": "stat",
      "title": "Running allocations",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node_id) (fuzzball_node_running_allocations{node=\"$node\"})",
          "legendFormat": "{{node_id}}",
          "instant": true
        }
      ]
    },
    {
      "type": "stat",
      "title": "Attributed job failures",
      "description": "Cumulative. FuzzballNodeAttributedJobFailures cordons the node at five in a day.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node_id) (fuzzball_node_attributed_job_failures{node=\"$node\"})",
          "legendFormat": "{{node_id}}",
          "instant": true
        }
      ]
    },
    {
      "type": "table",
      "title": "Holds",
      "description": "Holds in force, by effect and source. The hold's id and reason are in fuzzball node hold list and fuzzball node events.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 4 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node_id, effect, source) (fuzzball_node_hold{node=\"$node\"}) > 0",
          "format": "table",
          "instant": true
        }
      ]
    },
    {
      "type": "table",
      "title": "Firing alerts",
      "description": "Node-health alerts firing on this node. fb_action is what the alert asks Fuzzball to do; empty means advisory.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 4 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (alertname, node_id, fb_action) (ALERTS{alertstate=\"firing\", alertname=~\"FuzzballNode.*\", node=\"$node\"})",
          "format": "table",
          "instant": true
        }
      ]
    },
    {
      "type": "state-timeline",
      "title": "Holds over time",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 10 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (effect, source) (fuzzball_node_hold{node=\"$node\"})",
          "legendFormat": "{{effect}} ({{source}})"
        }
      ]
    },
    {
      "type": "state-timeline",
      "title": "Device plugins",
      "description": "1 for the state each plugin is in. A failed plugin keeps the node out of jobs requesting its device kind.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 10 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (plugin, state) (fuzzball_node_device_plugin{node=\"$node\"}) == 1",
          "legendFormat": "{{plugin}} {{state}}"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Memory errors (per hour)",
      "description": "Corrected and uncorrected memory errors from Fuzzball's node exporter, and EDAC corrected errors where node_exporter's EDAC collector is installed -- the series FuzzballNodeMemoryErrors reads.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node) (sum by (node, node_id) (increase(fuzzball_node_memory_correctable_errors_total{node=\"$node\"}[1h])))",
          "legendFormat": "correctable"
        },
        {
          "refId": "B",
          "expr": "max by (node) (sum by (node, node_id) (increase(fuzzball_node_memory_uncorrectable_errors_total{node=\"$node\"}[1h])))",
          "legendFormat": "uncorrectable"
        },
        {
          "refId": "C",
          "expr": "max by (node) (sum by (node, node_id) (increase(node_edac_correctable_errors_total{node=\"$node\"}[1h])))",
          "legendFormat": "EDAC correctable"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "GPU errors (per hour)",
      "description": "Xid errors from Fuzzball's node exporter, and uncorrectable ECC from DCGM where it is installed -- the series FuzzballNodeGPUUncorrectableECC reads.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node) (sum by (node, node_id) (increase(fuzzball_node_gpu_xid_errors_node_total{node=\"$node\"}[1h])))",
          "legendFormat": "Xid"
        },
        {
          "refId": "B",
          "expr": "max by (node) (sum by (node, node_id) (increase(DCGM_FI_DEV_ECC_DBE_VOL_TOTAL{node=\"$node\"}[1h])))",
          "legendFormat": "DCGM uncorrectable ECC"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Temperature",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
      "fieldConfig": { "defaults": { "unit": "celsius" } },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (zone) (fuzzball_node_thermal_celsius{node=\"$node\"})",
          "legendFormat": "{{zone}}"
        }
      ]
    },
    {
      "type": "state-timeline",
      "title": "Exporters reachable",
      "description": "Whether Prometheus could scrape each of the node's exporters. A failed scrape says a target could not be read, not that the node is faulty.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (job) (up{node=\"$node\"})",
          "legendFormat": "{{job}}"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Machine checks (per hour)",
      "description": "Machine-check records read from the kernel log by Fuzzball's node exporter, all and fatal -- no other exporter sees them. Counted since the extension started, so a floor.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node) (sum by (node, node_id) (increase(fuzzball_node_machine_check_errors_total{node=\"$node\"}[1h])))",
          "legendFormat": "all"
        },
        {
          "refId": "B",
          "expr": "max by (node) (sum by (node, node_id) (increase(fuzzball_node_machine_check_fatal_errors_total{node=\"$node\"}[1h])))",
          "legendFormat": "fatal"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Disk I/O errors (per hour)",
      "description": "I/O errors the kernel logged against each disk, from Fuzzball's node exporter. A disk that dropped off the bus keeps its line.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (device) (increase(fuzzball_node_disk_io_errors_total{node=\"$node\"}[1h]))",
          "legendFormat": "{{device}}"
        }
      ]
    },
    {
      "type": "state-timeline",
      "title": "Disk faults",
      "description": "1 while a disk is mounted read-only or SMART reports it failing, from Fuzzball's node exporter. A disk SMART could not answer for has no SMART line: see Signal coverage.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (device) (fuzzball_node_disk_read_only{node=\"$node\"})",
          "legendFormat": "{{device}} read-only"
        },
        {
          "refId": "B",
          "expr": "max by (device) (fuzzball_node_disk_smart_failing{node=\"$node\"})",
          "legendFormat": "{{device}} SMART failing"
        }
      ]
    },
    {
      "type": "state-timeline",
      "title": "Signal coverage",
      "description": "Whether each of Fuzzball's node collectors, and SMART on each disk, produced a reading. A zero on an error panel only means healthy where its signal reads checked; worst substrate on the host wins.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 },
      "fieldConfig": {
        "defaults": {
          "mappings": [
            {
              "type": "value",
              "options": {
                "1": { "text": "checked", "color": "green", "index": 0 },
                "0": { "text": "not available", "color": "text", "index": 1 },
                "-1": { "text": "error", "color": "red", "index": 2 }
              }
            }
          ]
        }
      },
      "targets": [
        {
          "refId": "A",
          "expr": "min by (signal) (fuzzball_node_signal_status{node=\"$node\"})",
          "legendFormat": "{{signal}}"
        },
        {
          "refId": "B",
          "expr": "min by (device) (fuzzball_node_disk_smart_status{node=\"$node\"})",
          "legendFormat": "SMART {{device}}"
        }
      ]
    }
  ]
}
