⚖️ Load Average Explained

# View load average
uptime
cat /proc/loadavg

# How many CPUs does this server have?
nproc                          # logical CPU count
lscpu | grep "^CPU(s):"        # detailed CPU info
grep -c processor /proc/cpuinfo

# See load average + CPU count together
echo "CPUs: $(nproc)   Load: $(uptime | awk -F'load average:' '{print $2}')"

The three load average numbers are exponentially weighted moving averages over 1, 5, and 15 minutes. Reading them together tells you whether the situation is improving or getting worse:

# Read the trend, not just the current value
uptime

This is the scenario that confuses most people. High load average but top shows CPUs are mostly idle:

# High load average but top shows this:
# %Cpu(s):  2.1 us,  0.8 sy,  0.0 ni, 72.3 id, 24.5 wa
#                                      ^^^^^^^^  ^^^^^^^^
#                                      72% idle  24% waiting for I/O

# This is a DISK bottleneck, not a CPU bottleneck
# Processes are stuck waiting for disk I/O (D state)
# They count toward load average even though CPUs are idle

# Verify -- count processes in D state
ps aux | awk '$8 == "D" {print $0}'
ps -eo pid,stat,comm | grep "^.*D"

# How many D-state processes right now?
ps aux | awk '$8 ~ /D/ {count++} END {print count " processes in D state"}'

# Confirm with iostat
iostat -x 1 5   # look for high %util and await on a device

# Find what is doing the I/O
sudo iotop -o

# Calculate load per CPU right now
CPUS=$(nproc)
LOAD=$(awk '{print $1}' /proc/loadavg)
PER_CPU=$(echo "scale=2; $LOAD / $CPUS" | bc)
echo "Load per CPU: $PER_CPU (load=$LOAD, cpus=$CPUS)"

# Thresholds -- these are guidelines not absolute rules
# Per-CPU load:
#  < 0.7  -- comfortable, plenty of headroom
#  0.7-1.0 -- getting busy, monitor closely
#  1.0-2.0 -- overloaded, processes queueing
#  > 2.0  -- seriously overloaded, investigate immediately
#  > 5.0  -- severe, system may become unresponsive

# Also check the run queue directly
vmstat 1 5 | awk 'NR>2 {print "Run queue:", $1, "  Blocked:", $2}'

# r column = processes waiting for CPU
# b column = processes in uninterruptible sleep (disk wait)
# If r > nproc consistently = CPU bottleneck
# If b > 0 consistently = I/O bottleneck

# View load average history with sar
sar -q

# Load average for a specific time range
sar -q -s 14:00:00 -e 15:00:00

# Load average from a previous day
sar -q -f /var/log/sa/sa25

Zombie processes appear in the load average discussion but work differently — they don't actually affect load average but they indicate a problem:

# Find zombie processes
ps aux | awk '$8 == "Z" {print $0}'
ps -eo pid,ppid,stat,comm | grep " Z"

# How many zombies?
ps aux | awk '$8 == "Z"' | wc -l

# top shows zombies in the Tasks line:
# Tasks: 185 total,   1 running, 183 sleeping,   0 stopped,   1 zombie

# Find the parent of a zombie (the parent needs to reap it)
ZOMBIE_PID=12345
ps -o ppid= -p $ZOMBIE_PID | xargs ps -p

# Zombies cannot be killed -- they are already dead
# Kill the parent process to clean them up (parent will reap on exit)
# Or send SIGCHLD to the parent to trigger reaping
kill -CHLD $PARENT_PID

# Find processes with very high thread counts
ps -eo pid,nlwp,comm --sort=-nlwp | head -10
# nlwp = number of lightweight processes (threads)

# Count total threads on the system
ps -eo nlwp | tail -n +2 | awk '{total += $1} END {print total " total threads"}'

# Find context switch rate (high cs in vmstat = too many threads)
vmstat 1 5 | awk 'NR>2 {print "Context switches/sec:", $12}'

# View thread details for a specific process
ps -L -p 12345    # list all threads of PID 12345
ls /proc/12345/task/ | wc -l   # count threads

# Java applications are common culprits
ps -eo pid,nlwp,comm | grep java | sort -k2 -rn | head -5

#!/bin/bash
# load-monitor.sh -- alert when load per CPU exceeds threshold

THRESHOLD=2.0    # alert when load per CPU exceeds this
CPUS=$(nproc)
LOAD=$(awk '{print $1}' /proc/loadavg)
LOAD5=$(awk '{print $2}' /proc/loadavg)
LOAD15=$(awk '{print $3}' /proc/loadavg)
PER_CPU=$(echo "scale=2; $LOAD / $CPUS" | bc)

echo "=== Load Average Check: $(hostname) ==="
echo "CPUs:          $CPUS"
echo "Load (1/5/15): $LOAD / $LOAD5 / $LOAD15"
echo "Per CPU:       $PER_CPU"

# Determine status
STATUS=$(echo "$PER_CPU $THRESHOLD" | awk '{if ($1 > $2) print "ALERT"; else print "OK"}')
echo "Status:        $STATUS"

if [[ "$STATUS" == "ALERT" ]]; then
    echo ""
    echo "--- TOP PROCESSES BY CPU ---"
    ps aux --sort=-%cpu | head -6

    echo ""
    echo "--- D-STATE PROCESSES (I/O wait) ---"
    DCOUNT=$(ps aux | awk '$8 ~ /D/ {count++} END {print count+0}')
    echo "D-state count: $DCOUNT"
    ps aux | awk '$8 ~ /D/ {print $0}' | head -5

    echo ""
    echo "--- VMSTAT SNAPSHOT ---"
    vmstat 1 3

    # Send alert email
    echo "High load on $(hostname): $LOAD (per-CPU: $PER_CPU, threshold: $THRESHOLD)" | \
        mail -s "LOAD ALERT: $(hostname)" admin@example.com 2>/dev/null
fi