📡 Centralized Logging

Set up the central log server to accept incoming syslog messages:

# On the LOG SERVER

# Create /etc/rsyslog.d/00-server.conf
sudo tee /etc/rsyslog.d/00-server.conf << 'EOF'
# Enable TCP syslog reception on port 514
module(load="imtcp")
input(type="imtcp" port="514")

# Enable UDP syslog reception (for legacy clients)
module(load="imudp")
input(type="imudp" port="514")

# Template: store logs in per-host directories
template(name="PerHostLog" type="string"
    string="/var/log/remote/%HOSTNAME%/%PROGRAMNAME%.log")

# Apply template to all remote messages
if $fromhost-ip != "127.0.0.1" then {
    action(type="omfile" dynaFile="PerHostLog")
    stop
}
EOF

# Create the remote log directory
sudo mkdir -p /var/log/remote
sudo chmod 755 /var/log/remote

# Open firewall for syslog
sudo firewall-cmd --permanent --add-port=514/tcp
sudo firewall-cmd --permanent --add-port=514/udp
sudo firewall-cmd --reload

# Restart rsyslog
sudo systemctl restart rsyslog

# Verify it is listening
sudo ss -tulpn | grep 514

# On each CLIENT SERVER

# Create /etc/rsyslog.d/99-forward.conf
sudo tee /etc/rsyslog.d/99-forward.conf << 'EOF'
# Forward all logs to central log server via TCP
*.* action(type="omfwd"
           target="logserver.example.com"
           port="514"
           protocol="tcp"
           action.resumeRetryCount="100"
           queue.type="linkedList"
           queue.size="10000"
           queue.filename="fwd-queue"
           queue.saveOnShutdown="on")
EOF

# Restart rsyslog on the client
sudo systemctl restart rsyslog

# Test -- send a test message and check the server
logger -t test "Forwarding test from $(hostname)"

# On the log server, verify receipt:
sudo tail /var/log/remote/web01/test.log

Plain TCP syslog is unencrypted — anyone on the network can read it. Use TLS for sensitive environments:

# Install TLS support
sudo dnf install -y rsyslog-gnutls    # RHEL
sudo apt install -y rsyslog-gnutls    # Debian

# Generate certificates (on the log server)
sudo openssl req -x509 -newkey rsa:4096 \
    -keyout /etc/rsyslog-keys/server.key \
    -out /etc/rsyslog-keys/server.crt \
    -days 3650 -nodes \
    -subj "/CN=logserver.example.com"

# Log SERVER TLS config (/etc/rsyslog.d/00-tls-server.conf)
sudo tee /etc/rsyslog.d/00-tls-server.conf << 'EOF'
module(load="imtcp"
       StreamDriver.Name="gtls"
       StreamDriver.Mode="1"
       StreamDriver.Authmode="anon")

global(
    DefaultNetstreamDriver="gtls"
    DefaultNetstreamDriverCAFile="/etc/rsyslog-keys/server.crt"
    DefaultNetstreamDriverCertFile="/etc/rsyslog-keys/server.crt"
    DefaultNetstreamDriverKeyFile="/etc/rsyslog-keys/server.key"
)

input(type="imtcp" port="6514")
EOF

# Log CLIENT TLS config (/etc/rsyslog.d/99-tls-forward.conf)
sudo tee /etc/rsyslog.d/99-tls-forward.conf << 'EOF'
global(
    DefaultNetstreamDriver="gtls"
    DefaultNetstreamDriverCAFile="/etc/rsyslog-keys/server.crt"
)

*.* action(type="omfwd"
           target="logserver.example.com"
           port="6514"
           protocol="tcp"
           StreamDriver="gtls"
           StreamDriverMode="1"
           StreamDriverAuthMode="anon")
EOF

sudo systemctl restart rsyslog

# Advanced template -- organize by hostname, date, and program
sudo tee /etc/rsyslog.d/00-templates.conf << 'EOF'
# Store per host, per program
template(name="PerHostProgram" type="string"
    string="/var/log/remote/%HOSTNAME%/%PROGRAMNAME%.log")

# Store per host, per day (date-based files)
template(name="PerHostDaily" type="string"
    string="/var/log/remote/%HOSTNAME%/%$YEAR%-%$MONTH%-%$DAY%.log")

# Combined format with hostname in message
template(name="FullMsg" type="string"
    string="%TIMESTAMP% %HOSTNAME% %syslogtag%%msg%\n")

# Route by facility
if $syslogfacility-text == 'authpriv' then {
    action(type="omfile"
           dynaFile="PerHostProgram"
           template="FullMsg")
    stop
}

# Everything else by host and program
*.* action(type="omfile" dynaFile="PerHostProgram")
EOF

sudo systemctl restart rsyslog

# Set up logrotate for remote logs
sudo tee /etc/logrotate.d/remote-logs << 'EOF'
/var/log/remote/*/*.log {
    daily
    rotate 90
    compress
    delaycompress
    missingok
    notifempty
    dateext
    sharedscripts
    postrotate
        systemctl kill -s HUP rsyslog.service 2>/dev/null || true
    endscript
}
EOF

Rather than forwarding everything, selectively forward the logs that matter most for security and compliance:

# Forward only auth/security messages (bandwidth efficient)
sudo tee /etc/rsyslog.d/99-security-forward.conf << 'EOF'
# Forward auth messages
authpriv.* action(type="omfwd"
                  target="logserver.example.com"
                  port="514"
                  protocol="tcp")

# Forward kernel messages
kern.* action(type="omfwd"
              target="logserver.example.com"
              port="514"
              protocol="tcp")

# Forward errors and above from everything
*.err action(type="omfwd"
             target="logserver.example.com"
             port="514"
             protocol="tcp")

# Forward cron (for audit trail)
cron.* action(type="omfwd"
              target="logserver.example.com"
              port="514"
              protocol="tcp")
EOF

sudo systemctl restart rsyslog

With logs centralized by hostname, searching across your entire infrastructure is straightforward:

# Search for failed SSH logins across ALL servers
sudo grep -r "Failed password" /var/log/remote/

# Which servers had the most failures?
sudo grep -r "Failed password" /var/log/remote/ | \
    awk -F/ '{print $5}' | sort | uniq -c | sort -rn

# Find all sudo commands run today across all servers
sudo grep -r "$(date +%b\ %e)" /var/log/remote/*/sudo.log 2>/dev/null

# Find all errors in the last hour across all servers
sudo find /var/log/remote -name "*.log" -newer /tmp/1hour -exec \
    grep -l "error\|Error\|ERROR" {} \;

# Which server generated a specific event?
sudo grep -r "Out of memory" /var/log/remote/*/kern.log

# Create a 1-hour marker file for the search above
touch -t $(date -d "1 hour ago" +%Y%m%d%H%M) /tmp/1hour

On systemd systems, make journald feed rsyslog so journal entries are also forwarded centrally:

# Verify journald is forwarding to syslog
grep ForwardToSyslog /etc/systemd/journald.conf

# Enable if not set
sudo vi /etc/systemd/journald.conf
# Set: ForwardToSyslog=yes
sudo systemctl restart systemd-journald

# Now rsyslog receives journal entries and forwards them
# Verify the chain is working:
logger -t chaintest "Testing journal to rsyslog to central"

# Check locally
journalctl -t chaintest -n 3

# Check on log server
sudo tail /var/log/remote/web01/chaintest.log

# Alternative: forward journal directly without rsyslog
# systemd-journal-remote on the server side
sudo dnf install -y systemd-journal-remote
sudo systemctl enable --now systemd-journal-remote.socket

rsyslog centralization works well up to a point. Here is how to know when you need a dedicated log management platform:

# Check how much log data you are generating daily
sudo du -sh /var/log/remote/
sudo find /var/log/remote -name "*.log" -mtime -1 -exec du -sh {} \; | \
    awk '{total += $1} END {print "Daily volume: " total "K"}'

# Count log events per second across all servers
sudo tail -f /var/log/remote/*/*.log | pv -l -i 5 > /dev/null