{
  "title": "Fuzzball Node Resources",
  "uid": "fuzzball-node-resources",
  "description": "How busy a node is: LSF lsload's load indices from node_exporter and Fuzzball's node exporter, beside Fuzzball's status and holds as an LSF-style host state.",
  "tags": ["fuzzball", "node-resources"],
  "timezone": "browser",
  "schemaVersion": 39,
  "refresh": "1m",
  "time": { "from": "now-6h", "to": "now" },
  "templating": {
    "list": [
      {
        "name": "datasource",
        "label": "Prometheus",
        "type": "datasource",
        "query": "prometheus"
      },
      {
        "name": "node",
        "label": "Node",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "${datasource}" },
        "query": { "query": "label_values(fuzzball_node_info, node)", "refId": "node" },
        "refresh": 2,
        "sort": 1
      }
    ]
  },
  "links": [
    {
      "title": "Fleet view",
      "type": "link",
      "url": "/d/fuzzball-node-reliability"
    },
    {
      "title": "Node view",
      "type": "link",
      "url": "/d/fuzzball-node-health",
      "includeVars": true,
      "keepTime": true
    }
  ],
  "panels": [
    {
      "type": "row",
      "title": "Fleet",
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
      "panels": []
    },
    {
      "type": "bargauge",
      "title": "Nodes by state",
      "description": "Nodes in each LSF-style host state, from fuzzball_node_status and fuzzball_node_hold -- the mapping is in the node row's State panel. States with no nodes are not drawn.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1 },
      "targets": [
        {
          "refId": "A",
          "expr": "label_replace(label_replace(label_replace(label_replace(label_replace(count_values(\"code\", (max by (node, node_id) (fuzzball_node_status{status=\"not_ready\"} * 2 or fuzzball_node_status{status=\"unknown\"} * 3 or fuzzball_node_status{status=\"offline\"} * 4) > clamp_max(max by (node, node_id) (fuzzball_node_hold), 1)) or clamp_max(max by (node, node_id) (fuzzball_node_hold), 1)), \"state\", \"OK\", \"code\", \"0\"), \"state\", \"Locked\", \"code\", \"1\"), \"state\", \"Not ready\", \"code\", \"2\"), \"state\", \"Unknown\", \"code\", \"3\"), \"state\", \"Down\", \"code\", \"4\")",
          "legendFormat": "{{state}}",
          "instant": true
        }
      ]
    },
    {
      "type": "table",
      "title": "Nodes not OK",
      "description": "Every node whose LSF-style state is not OK, from fuzzball_node_status and fuzzball_node_hold. A held node that is also not ready, unknown or offline shows the more severe state.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 6, "x": 6, "y": 1 },
      "fieldConfig": {
        "defaults": {},
        "overrides": [
          {
            "matcher": { "id": "byName", "options": "Value" },
            "properties": [
              { "id": "displayName", "value": "State" },
              {
                "id": "mappings",
                "value": [
                  {
                    "type": "value",
                    "options": {
                      "1": { "text": "Locked", "color": "orange", "index": 0 },
                      "2": { "text": "Not ready", "color": "yellow", "index": 1 },
                      "3": { "text": "Unknown", "color": "purple", "index": 2 },
                      "4": { "text": "Down", "color": "red", "index": 3 }
                    }
                  }
                ]
              }
            ]
          }
        ]
      },
      "targets": [
        {
          "refId": "A",
          "expr": "((max by (node, node_id) (fuzzball_node_status{status=\"not_ready\"} * 2 or fuzzball_node_status{status=\"unknown\"} * 3 or fuzzball_node_status{status=\"offline\"} * 4) > clamp_max(max by (node, node_id) (fuzzball_node_hold), 1)) or clamp_max(max by (node, node_id) (fuzzball_node_hold), 1)) > 0",
          "format": "table",
          "instant": true
        }
      ]
    },
    {
      "type": "bargauge",
      "title": "Busiest nodes by CPU (top 10)",
      "description": "Share of CPU time not idle over the last 5 minutes (LSF ut), from node_exporter's node_cpu_seconds_total.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 6, "x": 12, "y": 1 },
      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 } },
      "targets": [
        {
          "refId": "A",
          "expr": "topk(10, 100 * max by (node) (1 - avg by (node, node_id) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))))",
          "legendFormat": "{{node}}",
          "instant": true
        }
      ]
    },
    {
      "type": "bargauge",
      "title": "Longest run queues (top 10)",
      "description": "Runnable tasks averaged over 15 seconds (LSF r15s), from Fuzzball's node exporter.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 6, "x": 18, "y": 1 },
      "targets": [
        {
          "refId": "A",
          "expr": "topk(10, max by (node) (fuzzball_node_run_queue_15s))",
          "legendFormat": "{{node}}",
          "instant": true
        }
      ]
    },
    {
      "type": "row",
      "title": "Node $node",
      "collapsed": false,
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
      "panels": []
    },
    {
      "type": "stat",
      "title": "State",
      "description": "Fuzzball's status and holds as one LSF-style host state, from fuzzball_node_status and fuzzball_node_hold. Most severe wins: offline -> Down (LSF unavail), unknown -> Unknown (unreach), not_ready -> Not ready (closed), any cordon, evict or deprovision hold -> Locked (lockU), otherwise OK (ok). A held node that is also not ready, unknown or offline shows the more severe state; Holds still lists the hold.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 10 },
      "fieldConfig": {
        "defaults": {
          "mappings": [
            {
              "type": "value",
              "options": {
                "0": { "text": "OK", "color": "green", "index": 0 },
                "1": { "text": "Locked", "color": "orange", "index": 1 },
                "2": { "text": "Not ready", "color": "yellow", "index": 2 },
                "3": { "text": "Unknown", "color": "purple", "index": 3 },
                "4": { "text": "Down", "color": "red", "index": 4 }
              }
            }
          ]
        }
      },
      "options": { "colorMode": "background", "textMode": "value", "reduceOptions": { "calcs": ["lastNotNull"] } },
      "targets": [
        {
          "refId": "A",
          "expr": "(max by (node, node_id) (fuzzball_node_status{node=\"$node\", status=\"not_ready\"} * 2 or fuzzball_node_status{node=\"$node\", status=\"unknown\"} * 3 or fuzzball_node_status{node=\"$node\", status=\"offline\"} * 4) > clamp_max(max by (node, node_id) (fuzzball_node_hold{node=\"$node\"}), 1)) or clamp_max(max by (node, node_id) (fuzzball_node_hold{node=\"$node\"}), 1)",
          "legendFormat": "{{node_id}}",
          "instant": true
        }
      ]
    },
    {
      "type": "stat",
      "title": "Fuzzball status",
      "description": "The status Fuzzball has observed for each substrate on the host, from fuzzball_node_status. A cordon is a hold, not a status.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 10 },
      "options": { "textMode": "name", "reduceOptions": { "calcs": ["lastNotNull"] } },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node_id, status) (fuzzball_node_status{node=\"$node\"}) == 1",
          "legendFormat": "{{status}}",
          "instant": true
        }
      ]
    },
    {
      "type": "stat",
      "title": "Holds",
      "description": "Holds in force by effect, from fuzzball_node_hold. Who placed them and why is on Node Health and in fuzzball node hold list.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 10 },
      "options": { "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } },
      "targets": [
        {
          "refId": "A",
          "expr": "sum by (effect) (max by (node_id, effect, source) (fuzzball_node_hold{node=\"$node\"}))",
          "legendFormat": "{{effect}}",
          "instant": true
        }
      ]
    },
    {
      "type": "stat",
      "title": "Running allocations",
      "description": "Allocations Fuzzball is running on each substrate on the host, from fuzzball_node_running_allocations.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 10 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node_id) (fuzzball_node_running_allocations{node=\"$node\"})",
          "legendFormat": "{{node_id}}",
          "instant": true
        }
      ]
    },
    {
      "type": "stat",
      "title": "GPUs",
      "description": "GPUs Fuzzball's node exporter sees, one fuzzball_node_gpu_hardware_xid series each. 0 means the GPU collector ran and found none; no value means it did not report, or failed (fuzzball_node_signal_status{signal=\"gpu\"} is -1).",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 8, "x": 0, "y": 14 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node) (count by (node, node_id) (fuzzball_node_gpu_hardware_xid{node=\"$node\"})) or 0 * count by (node) (fuzzball_node_signal_status{node=\"$node\", signal=\"gpu\"} >= 0)",
          "legendFormat": "GPUs",
          "instant": true
        }
      ]
    },
    {
      "type": "stat",
      "title": "Idle time",
      "description": "Time since the last input on any logged-in terminal (LSF it), from Fuzzball's node exporter's fuzzball_node_idle_seconds.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 8, "x": 8, "y": 14 },
      "fieldConfig": { "defaults": { "unit": "dtdurations" } },
      "targets": [
        {
          "refId": "A",
          "expr": "min by (node) (fuzzball_node_idle_seconds{node=\"$node\"})",
          "legendFormat": "idle",
          "instant": true
        }
      ]
    },
    {
      "type": "stat",
      "title": "Login sessions",
      "description": "User sessions recorded in utmp (LSF ls), from Fuzzball's node exporter's fuzzball_node_login_sessions.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 4, "w": 8, "x": 16, "y": 14 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node) (fuzzball_node_login_sessions{node=\"$node\"})",
          "legendFormat": "sessions",
          "instant": true
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "CPU utilization",
      "description": "Share of CPU time not idle across every core over 5 minutes (LSF ut), from node_exporter's node_cpu_seconds_total. iowait counts as busy.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 18 },
      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 } },
      "targets": [
        {
          "refId": "A",
          "expr": "100 * max by (node) (1 - avg by (node, node_id) (rate(node_cpu_seconds_total{node=\"$node\", mode=\"idle\"}[5m])))",
          "legendFormat": "busy"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Run queue (15 s)",
      "description": "Runnable tasks averaged over 15 seconds (LSF r15s), from Fuzzball's node exporter; the CPUs line is node_exporter's core count, for scale.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 18 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node) (fuzzball_node_run_queue_15s{node=\"$node\"})",
          "legendFormat": "r15s"
        },
        {
          "refId": "B",
          "expr": "max by (node) (count by (node, node_id) (node_cpu_seconds_total{node=\"$node\", mode=\"idle\"}))",
          "legendFormat": "CPUs"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Load average",
      "description": "The kernel's 1, 5 and 15 minute load averages (LSF r1m, r15m), from node_exporter. Unlike the run queue, they count tasks in uninterruptible sleep.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 18 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node) (node_load1{node=\"$node\"})",
          "legendFormat": "1m"
        },
        {
          "refId": "B",
          "expr": "max by (node) (node_load5{node=\"$node\"})",
          "legendFormat": "5m"
        },
        {
          "refId": "C",
          "expr": "max by (node) (node_load15{node=\"$node\"})",
          "legendFormat": "15m"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Paging",
      "description": "Per second, from node_exporter's /proc/vmstat counters: block I/O paged in and out (KiB, as the kernel counts pgpgin/pgpgout), swap pages in and out, and major faults (LSF pg).",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node) (sum by (node, node_id) (rate(node_vmstat_pgpgin{node=\"$node\"}[5m])))",
          "legendFormat": "paged in (KiB/s)"
        },
        {
          "refId": "B",
          "expr": "max by (node) (sum by (node, node_id) (rate(node_vmstat_pgpgout{node=\"$node\"}[5m])))",
          "legendFormat": "paged out (KiB/s)"
        },
        {
          "refId": "C",
          "expr": "max by (node) (sum by (node, node_id) (rate(node_vmstat_pswpin{node=\"$node\"}[5m])))",
          "legendFormat": "swapped in (pages/s)"
        },
        {
          "refId": "D",
          "expr": "max by (node) (sum by (node, node_id) (rate(node_vmstat_pswpout{node=\"$node\"}[5m])))",
          "legendFormat": "swapped out (pages/s)"
        },
        {
          "refId": "E",
          "expr": "max by (node) (sum by (node, node_id) (rate(node_vmstat_pgmajfault{node=\"$node\"}[5m])))",
          "legendFormat": "major faults/s"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Memory",
      "description": "Available (LSF mem), free and total memory, from node_exporter's /proc/meminfo series.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 },
      "fieldConfig": { "defaults": { "unit": "bytes", "min": 0 } },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node) (node_memory_MemAvailable_bytes{node=\"$node\"})",
          "legendFormat": "available"
        },
        {
          "refId": "B",
          "expr": "max by (node) (node_memory_MemFree_bytes{node=\"$node\"})",
          "legendFormat": "free"
        },
        {
          "refId": "C",
          "expr": "max by (node) (node_memory_MemTotal_bytes{node=\"$node\"})",
          "legendFormat": "total"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Filesystem utilization",
      "description": "Share of each filesystem in use, from node_exporter. /tmp is always shown, whatever its type, because it is what LSF tmp means. Memory-backed, image and container filesystems are left out -- including the FUSE mounts Fuzzball and Apptainer serve images from -- as are the trees Fuzzball and container runtimes mount workloads under. A site's FUSE shared filesystems are kept.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 },
      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 } },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (mountpoint) (100 * (1 - node_filesystem_avail_bytes{node=\"$node\", fstype!~\"tmpfs|devtmpfs|ramfs|overlay|squashfs|erofs|fuse[.](erofsfuse|squashfuse(_ll)?|fuse-overlayfs|lxcfs)|nsfs|autofs\", mountpoint!~\"/(dev|proc|sys|run|var/run)(/.*)?|/var/lib/(docker|containers|kubelet|fuzzball)/.+|/tmp/(fuzzball/|erofsmount-).+\"} / node_filesystem_size_bytes{node=\"$node\", fstype!~\"tmpfs|devtmpfs|ramfs|overlay|squashfs|erofs|fuse[.](erofsfuse|squashfuse(_ll)?|fuse-overlayfs|lxcfs)|nsfs|autofs\", mountpoint!~\"/(dev|proc|sys|run|var/run)(/.*)?|/var/lib/(docker|containers|kubelet|fuzzball)/.+|/tmp/(fuzzball/|erofsmount-).+\"}) or 100 * (1 - node_filesystem_avail_bytes{node=\"$node\", mountpoint=\"/tmp\"} / node_filesystem_size_bytes{node=\"$node\", mountpoint=\"/tmp\"}))",
          "legendFormat": "{{mountpoint}}"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Swap utilization",
      "description": "Share of swap in use (LSF swp is the free amount), from node_exporter's SwapFree and SwapTotal. No line: no swap configured, or node_exporter is not reachable -- see Node Health.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 },
      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 } },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (node) (100 * (1 - node_memory_SwapFree_bytes{node=\"$node\"} / (node_memory_SwapTotal_bytes{node=\"$node\"} > 0)))",
          "legendFormat": "swap used"
        }
      ]
    },
    {
      "type": "timeseries",
      "title": "Network I/O",
      "description": "Bytes per second received and sent on each interface, from node_exporter. Loopback and container virtual interfaces are left out.",
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 42 },
      "fieldConfig": { "defaults": { "unit": "Bps", "min": 0 } },
      "targets": [
        {
          "refId": "A",
          "expr": "max by (device) (rate(node_network_receive_bytes_total{node=\"$node\", device!~\"lo|veth.*|cni.*|docker.*|br-.*|virbr.*|flannel.*|cali.*|cilium.*|lxc.*|vxlan.*|genev.*|tun.*|tap.*|kube.*\"}[5m]))",
          "legendFormat": "{{device}} rx"
        },
        {
          "refId": "B",
          "expr": "max by (device) (rate(node_network_transmit_bytes_total{node=\"$node\", device!~\"lo|veth.*|cni.*|docker.*|br-.*|virbr.*|flannel.*|cali.*|cilium.*|lxc.*|vxlan.*|genev.*|tun.*|tap.*|kube.*\"}[5m]))",
          "legendFormat": "{{device}} tx"
        }
      ]
    }
  ]
}
