📋 Logging Basics

# View the /var/log directory
ls -lh /var/log/

# The most important log files on RHEL/Rocky:
sudo tail -f /var/log/messages      # general system messages
sudo tail -f /var/log/secure        # auth, SSH, sudo
sudo tail -f /var/log/cron          # cron job execution
sudo tail -f /var/log/maillog       # mail system
sudo tail -f /var/log/boot.log      # boot messages
sudo tail -f /var/log/audit/audit.log  # SELinux, auditd

# On Debian/Ubuntu:
sudo tail -f /var/log/syslog        # general system (equiv to messages)
sudo tail -f /var/log/auth.log      # auth (equiv to secure)
sudo tail -f /var/log/kern.log      # kernel messages
sudo tail -f /var/log/dpkg.log      # package install/remove

# Application-specific logs
sudo tail -f /var/log/httpd/access_log    # Apache access
sudo tail -f /var/log/httpd/error_log     # Apache errors
sudo tail -f /var/log/nginx/access.log    # Nginx access
sudo tail -f /var/log/nginx/error.log     # Nginx errors
sudo tail -f /var/log/postgresql/*.log    # PostgreSQL

# View last 50 lines
sudo tail -50 /var/log/messages
sudo tail -50 /var/log/secure

# View first lines (useful for log file header/start)
sudo head -20 /var/log/messages

# Search for a pattern
sudo grep "Failed password" /var/log/secure
sudo grep "error" /var/log/messages
sudo grep -i "error\|fail\|warn" /var/log/messages

# Search with context (3 lines before and after match)
sudo grep -B3 -A3 "segfault" /var/log/messages

# Filter by date (logs use month day format)
sudo grep "^Apr 25" /var/log/messages
sudo grep "^Apr 25 10:" /var/log/messages   # specific hour

# Count occurrences
sudo grep -c "Failed password" /var/log/secure

# Find most common SSH failure sources
sudo grep "Failed password" /var/log/secure | \
    awk '{print $11}' | sort | uniq -c | sort -rn | head -10

# Search compressed rotated logs too
sudo zgrep "error" /var/log/messages-20260420.gz
sudo zcat /var/log/messages-20260420.gz | grep "error"

rsyslog is configured in /etc/rsyslog.conf and files in /etc/rsyslog.d/. Rules follow the format facility.severity destination:

# View rsyslog configuration
cat /etc/rsyslog.conf
ls /etc/rsyslog.d/

# Check rsyslog status
sudo systemctl status rsyslog

# Create a custom log rule -- drop in /etc/rsyslog.d/
sudo tee /etc/rsyslog.d/myapp.conf << 'EOF'
# Log myapp messages to its own file
if $programname == 'myapp' then /var/log/myapp.log
& stop
EOF

# Restart rsyslog to apply
sudo systemctl restart rsyslog

# Test: send a test message
logger -t myapp "Test log message"
sudo tail /var/log/myapp.log

logger sends messages directly to syslog from the command line or scripts — the right way to log from shell scripts:

# Basic usage
logger "Backup started"
logger "Backup completed successfully"

# Specify tag (program name in log)
logger -t mybackup "Starting nightly backup"
logger -t mybackup "Backup completed: 42 files, 1.2GB"

# Specify facility and severity
logger -p local0.info    "Informational message"
logger -p local0.warning "Warning: disk at 85%"
logger -p local0.err     "Error: backup failed"
logger -p auth.notice    "Admin login from 192.168.1.5"

# Log both to syslog AND stderr
logger -s -t myapp "Error occurred"

# Use in scripts
#!/bin/bash
log() {
    logger -t "$(basename $0)" "$*"
    echo "$(date '+%Y-%m-%d %H:%M:%S') $*"
}

log "Script started"
/usr/local/bin/do-work.sh && log "Work completed" || log "Work FAILED"

Traditional syslog format follows a consistent structure:

# Format: TIMESTAMP HOSTNAME PROGRAM[PID]: MESSAGE
sudo head -5 /var/log/messages

# The missing year is a known syslog limitation
# Use journalctl for full ISO timestamps with year:
sudo journalctl --since "2026-04-25" | head -5

# Convert syslog timestamps to something more useful
sudo grep "Apr 25" /var/log/messages | \
    awk '{print 2026, $1, $2, $3, $5}' | head -5

# Follow multiple log files simultaneously with multitail
sudo dnf install -y multitail 2>/dev/null || sudo apt install -y multitail
multitail /var/log/messages /var/log/secure

# Watch for specific patterns across all logs
sudo tail -f /var/log/messages /var/log/secure /var/log/cron | \
    grep --line-buffered -i "error\|fail\|warn"

# Alert on critical events
sudo tail -f /var/log/messages | while read line; do
    if echo "$line" | grep -qi "critical\|emergency\|kernel panic"; then
        echo "$line" | mail -s "CRITICAL: $(hostname)" admin@example.com
    fi
done

# Check disk space used by logs
sudo du -sh /var/log/*  | sort -rh | head -20

# Find the largest log files
sudo find /var/log -type f -name "*.log" -exec du -sh {} \; | \
    sort -rh | head -10

# What rsyslog rules are active?
sudo rsyslogd -N1    # validate config
sudo rsyslogd -dn 2>&1 | head -30   # debug mode output

# List all log files being written to
sudo lsof | grep /var/log | awk '{print $9}' | sort -u

# What services are NOT sending to syslog (writing their own logs)?
sudo find /var/log -name "*.log" -newer /var/log/messages | head -20

# Check if auditd is running (security audit log)
sudo systemctl is-active auditd
sudo auditctl -l    # list active audit rules

# Verify messages are actually flowing
logger -t test "Test message $(date)"
sudo grep "Test message" /var/log/messages
sudo journalctl -t test -n 5

#!/bin/bash
# log-daily-summary.sh -- morning report on yesterday's log activity

YESTERDAY=$(date -d "yesterday" "+%b %e" | sed 's/  / /')
LOG=/var/log/messages
SECURE=/var/log/secure

echo "========================================"
echo " Daily Log Summary: $(hostname)"
echo " Reporting on: $YESTERDAY"
echo " Generated: $(date)"
echo "========================================"

echo ""
echo "--- AUTHENTICATION FAILURES ---"
count=$(grep "$YESTERDAY" $SECURE 2>/dev/null | grep -c "Failed password")
echo "  SSH failed passwords:     $count"
count=$(grep "$YESTERDAY" $SECURE 2>/dev/null | grep -c "Invalid user")
echo "  Invalid usernames tried:  $count"
count=$(grep "$YESTERDAY" $SECURE 2>/dev/null | grep -c "sudo:")
echo "  sudo commands run:        $count"

echo ""
echo "--- TOP SSH FAILURE SOURCES ---"
grep "$YESTERDAY" $SECURE 2>/dev/null | grep "Failed password" | \
    awk '{print $11}' | sort | uniq -c | sort -rn | head -5 | \
    awk '{printf "  %5d attempts from %s\n", $1, $2}'

echo ""
echo "--- SYSTEM ERRORS ---"
count=$(grep "$YESTERDAY" $LOG 2>/dev/null | grep -c " error\| Error\| ERROR")
echo "  Error messages:    $count"
count=$(grep "$YESTERDAY" $LOG 2>/dev/null | grep -c " warn\| Warn\| WARN")
echo "  Warning messages:  $count"
count=$(grep "$YESTERDAY" $LOG 2>/dev/null | grep -c "kernel:")
echo "  Kernel messages:   $count"

echo ""
echo "--- DISK USAGE ON /var/log ---"
du -sh /var/log/ 2>/dev/null | awk '{print "  Total: " $1}'
df -h /var/log 2>/dev/null | tail -1 | awk '{print "  Available: " $4}'

echo "========================================"