Skip to content

Linux System Logs Guide

System logs are essential for troubleshooting, security auditing, and system monitoring. This guide covers log locations, viewing tools, and log management in Linux.

Overview: Log System

Log Locations

Location Purpose Contents
/var/log/syslog System log General system messages (Debian/Ubuntu)
/var/log/messages System messages General system messages (RHEL/CentOS)
/var/log/auth.log Authentication Login attempts, sudo usage
/var/log/kern.log Kernel Kernel messages
/var/log/apache2/ Apache web server Web server logs
/var/log/nginx/ Nginx web server Web server logs
journalctl systemd journal All systemd logs

systemd Journal with journalctl

Basic journalctl Usage

# View all logs
journalctl

# Follow new messages (like tail -f)
journalctl -f

# Show only recent logs
journalctl -n 50

# Show since specific time
journalctl --since "2025-02-07 10:30:00"
journalctl --since "1 hour ago"
journalctl --since yesterday
journalctl --since today

# Show time range
journalctl --since "2025-02-07" --until "2025-02-08"

# Reverse order (newest first)
journalctl -r

Filtering Logs

# Specific unit/service
journalctl -u nginx
journalctl -u sshd

# Multiple units
journalctl -u nginx -u mysql

# Kernel messages
journalctl -k
journalctl --dmesg

# Specific priority
journalctl -p err          # Errors only
journalctl -p warning      # Warnings and above

# Priority levels:
# emerg (0), alert (1), crit (2), err (3)
# warning (4), notice (5), info (6), debug (7)

# Specific boot
journalctl -b              # Current boot
journalctl -b -1           # Previous boot
journalctl --list-boots    # List all boots

# Specific user
journalctl _UID=1000

Output Formats

# JSON output
journalctl -o json

# Pretty JSON
journalctl -o json-pretty

# Short format
journalctl -o short

# Verbose format
journalctl -o verbose

# Export format
journalctl -o export

# Cat format (no metadata)
journalctl -o cat

Advanced journalctl

# Show disk usage
journalctl --disk-usage

# Vacuum logs (clean old logs)
sudo journalctl --vacuum-time=7d   # Keep last 7 days
sudo journalctl --vacuum-size=1G   # Keep 1GB max

# Verify journal integrity
sudo journalctl --verify

# Follow logs for specific process
journalctl _PID=1234 -f

# Show logs with full output (no truncation)
journalctl --no-pager

# Grep in logs
journalctl | grep error

# Combine filters
journalctl -u nginx --since today -p err

Traditional Log Files

Viewing Logs

# View log file
cat /var/log/syslog

# Tail logs (last 10 lines)
tail /var/log/syslog

# Tail with specific number
tail -n 50 /var/log/syslog

# Follow log in real-time
tail -f /var/log/syslog

# Head (first lines)
head -n 20 /var/log/syslog

# Less (scrollable)
less /var/log/syslog

# Search in logs
grep "error" /var/log/syslog
grep -i "failed" /var/log/auth.log

# Count occurrences
grep -c "error" /var/log/syslog

Common Log Files

# Authentication logs
sudo tail -f /var/log/auth.log          # Debian/Ubuntu
sudo tail -f /var/log/secure            # RHEL/CentOS

# System logs
sudo tail -f /var/log/syslog            # Debian/Ubuntu
sudo tail -f /var/log/messages          # RHEL/CentOS

# Kernel logs
sudo tail -f /var/log/kern.log
dmesg                                    # Kernel ring buffer

# Boot logs
less /var/log/boot.log

# Cron logs
sudo tail -f /var/log/cron

# Mail logs
sudo tail -f /var/log/mail.log

# Apache logs
sudo tail -f /var/log/apache2/access.log
sudo tail -f /var/log/apache2/error.log

# Nginx logs
sudo tail -f /var/log/nginx/access.log
sudo tail -f /var/log/nginx/error.log

# MySQL logs
sudo tail -f /var/log/mysql/error.log

dmesg - Kernel Ring Buffer

Basic dmesg Usage

# View kernel messages
dmesg

# Follow new messages
dmesg -w

# Human-readable timestamps
dmesg -T

# Clear dmesg buffer
sudo dmesg -C

# Show specific level
dmesg -l err,warn

# Color output
dmesg --color=always

# Recent messages
dmesg | tail -50

Filtering dmesg

# Hardware messages
dmesg | grep -i usb
dmesg | grep -i disk
dmesg | grep -i eth

# Error messages
dmesg | grep -i error
dmesg | grep -i fail

# Memory issues
dmesg | grep -i memory
dmesg | grep -i oom

# Specific facility
dmesg -f kern
dmesg -f user

Log Analysis

Finding Errors

# Find all errors today
journalctl --since today -p err

# Find failed services
systemctl --failed

# Search for specific error
journalctl | grep -i "failed"
grep -i "error" /var/log/syslog

# Count errors by type
grep "error" /var/log/syslog | cut -d: -f4 | sort | uniq -c | sort -rn

Authentication Analysis

# Failed login attempts
sudo grep "Failed password" /var/log/auth.log

# Successful logins
sudo grep "Accepted" /var/log/auth.log

# sudo usage
sudo grep "sudo" /var/log/auth.log

# SSH connections
sudo grep "sshd" /var/log/auth.log

# Count failed logins by IP
sudo grep "Failed password" /var/log/auth.log | \
    awk '{print $(NF-3)}' | sort | uniq -c | sort -rn

Web Server Logs

# Apache access log analysis
# Top 10 IP addresses
awk '{print $1}' /var/log/apache2/access.log | \
    sort | uniq -c | sort -rn | head -10

# Top requested pages
awk '{print $7}' /var/log/apache2/access.log | \
    sort | uniq -c | sort -rn | head -10

# HTTP status codes
awk '{print $9}' /var/log/apache2/access.log | \
    sort | uniq -c | sort -rn

# 404 errors
grep " 404 " /var/log/apache2/access.log

# Bandwidth usage by IP
awk '{bytes[$1]+=$10} END {for (ip in bytes) print ip, bytes[ip]}' \
    /var/log/apache2/access.log | sort -k2 -rn | head -10

Log Rotation with logrotate

logrotate Configuration

# View main config
cat /etc/logrotate.conf

# Service-specific configs
ls /etc/logrotate.d/

# Example nginx config
cat /etc/logrotate.d/nginx

Custom logrotate Configuration

# Create custom config
sudo nano /etc/logrotate.d/myapp

# Configuration example:
/var/log/myapp/*.log {
    daily
    rotate 7
    compress
    delaycompress
    missingok
    notifempty
    create 0640 www-data www-data
    sharedscripts
    postrotate
        systemctl reload myapp
    endscript
}

logrotate Options

# Rotation frequency
daily           # Rotate daily
weekly          # Rotate weekly
monthly         # Rotate monthly
yearly          # Rotate yearly

# Retention
rotate 7        # Keep 7 rotated logs

# Compression
compress        # Compress rotated logs
delaycompress   # Compress on next rotation
nocompress      # Don't compress

# Permissions
create 0640 user group  # Permissions for new log

# Error handling
missingok       # Don't error if log missing
notifempty      # Don't rotate if empty

# Scripts
prerotate       # Run before rotation
postrotate      # Run after rotation

Manual logrotate

# Test configuration
sudo logrotate -d /etc/logrotate.d/nginx

# Force rotation
sudo logrotate -f /etc/logrotate.d/nginx

# Verbose output
sudo logrotate -v /etc/logrotate.conf

Centralized Logging

Using rsyslog

# rsyslog configuration
sudo nano /etc/rsyslog.conf

# Forward logs to remote server
*.* @@remote-server:514        # TCP
*.* @remote-server:514          # UDP

# Restart rsyslog
sudo systemctl restart rsyslog

Remote Logging Server

# On server, edit rsyslog.conf
sudo nano /etc/rsyslog.conf

# Enable TCP reception
module(load="imtcp")
input(type="imtcp" port="514")

# Enable UDP reception
module(load="imudp")
input(type="imudp" port="514")

# Restart service
sudo systemctl restart rsyslog

Log Monitoring Tools

Real-Time Monitoring

# Multi-file tail
tail -f /var/log/syslog /var/log/auth.log

# With labels
tail -f /var/log/{syslog,auth.log}

# Using multitail (if installed)
multitail /var/log/syslog /var/log/auth.log

# Watch for specific pattern
tail -f /var/log/syslog | grep --color error

# lnav - log file navigator
lnav /var/log/syslog

Log Analysis Scripts

#!/bin/bash
# Daily log summary

echo "=== System Log Summary ==="
echo "Date: $(date)"
echo ""

echo "Errors today:"
journalctl --since today -p err --no-pager | wc -l

echo ""
echo "Failed login attempts:"
sudo grep "Failed password" /var/log/auth.log | wc -l

echo ""
echo "Top error messages:"
journalctl --since today -p err --no-pager | tail -20

Troubleshooting with Logs

System Boot Issues

# Check boot messages
journalctl -b
journalctl -b -1  # Previous boot

# Kernel messages during boot
dmesg | less

# Failed services
systemctl --failed

# Service status
systemctl status servicename
journalctl -u servicename

Service Issues

# Check service logs
journalctl -u nginx -n 100

# Follow service logs
journalctl -u nginx -f

# Service errors only
journalctl -u nginx -p err

# Service logs since last start
journalctl -u nginx --since "$(systemctl show -p ActiveEnterTimestamp nginx | cut -d= -f2)"

Performance Issues

# High memory usage events
journalctl | grep -i "out of memory"

# Disk issues
journalctl | grep -i "disk"
dmesg | grep -i "I/O error"

# Network issues
journalctl | grep -i "network"
journalctl -u NetworkManager

Best Practices

Log Management

# Regular log review
sudo journalctl -p err --since today

# Monitor log size
du -sh /var/log/*
journalctl --disk-usage

# Clean old logs
sudo journalctl --vacuum-time=30d
sudo find /var/log -name "*.gz" -mtime +30 -delete

# Set persistent journal
sudo mkdir -p /var/log/journal
sudo systemd-tmpfiles --create --prefix /var/log/journal

Security Monitoring

# Monitor authentication
sudo tail -f /var/log/auth.log | grep --color Failed

# Watch for suspicious sudo usage
journalctl -u sudo -f

# Monitor file access (with auditd)
sudo ausearch -k file_access

# Check for common attacks
sudo grep -i "attack" /var/log/syslog

Quick Reference

# journalctl
journalctl                   # All logs
journalctl -f                # Follow logs
journalctl -u service        # Service logs
journalctl --since today     # Today's logs
journalctl -p err            # Errors only
journalctl -b                # Current boot

# Traditional logs
tail -f /var/log/syslog      # Follow syslog
less /var/log/auth.log       # View auth log
grep error /var/log/syslog   # Search logs

# dmesg
dmesg                        # Kernel messages
dmesg -w                     # Follow kernel logs
dmesg -T                     # Human timestamps

# Log rotation
sudo logrotate -f /etc/logrotate.d/service  # Force rotate
sudo journalctl --vacuum-time=7d             # Clean journal