Linux Text Processing: Mastering grep, sed, and awk for Log Analysis

# Basic grep
grep "error" logfile.txt           # Find lines with "error"
grep -i "error" logfile.txt         # Case insensitive
grep -r "error" /var/log/           # Recursive search
grep -v "error" logfile.txt         # Invert (show lines WITHOUT error)
grep -n "error" logfile.txt         # Show line numbers
grep -c "error" logfile.txt         # Count matches
grep -l "error" *.log               # List filenames only
grep -A 5 "error" logfile.txt       # Show 5 lines after
grep -B 5 "error" logfile.txt       # Show 5 lines before
grep -C 3 "error" logfile.txt       # Show 3 lines before and after

# Basic regex patterns
grep "^ERROR" log.txt           # Lines starting with ERROR
grep "ERROR$" log.txt           # Lines ending with ERROR
grep "error.*fail" log.txt      # error followed by anything then fail
grep "[0-9]" log.txt            # Any digit
grep "[A-Za-z]" log.txt         # Any letter
grep "error\|warning" log.txt   # error OR warning
grep -E "error|warning" log.txt # Extended regex (same as above)

# Extended regex (-E or egrep)
grep -E "[0-9]{3}-[0-9]{3}-[0-9]{4}" log.txt  # Phone numbers
grep -E "[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" log.txt  # IP addresses

# Perl-compatible regex (-P) for advanced
grep -P "(?<=error: ).*" log.txt  # Text after "error: "

# Substitute (find and replace)
sed 's/old/new/' file.txt           # Replace first occurrence per line
sed 's/old/new/g' file.txt          # Replace all occurrences per line
sed 's/old/new/2' file.txt          # Replace second occurrence
sed 's/old/new/gI' file.txt         # Case insensitive (GNU sed)

# In-place editing (modify file directly)
sed -i 's/old/new/g' file.txt       # Edit file in place
sed -i.bak 's/old/new/g' file.txt   # Create backup first

# Delete lines
sed '/pattern/d' file.txt           # Delete lines matching pattern
sed '3d' file.txt                   # Delete line 3
sed '5,10d' file.txt                # Delete lines 5-10

# Print specific lines
sed -n '5p' file.txt                # Print line 5 only
sed -n '10,20p' file.txt            # Print lines 10-20

# Multiple commands
sed -e 's/old/new/g' -e 's/bad/good/g' file.txt
sed 's/old/new/g; s/bad/good/g' file.txt

# Use different delimiter (when path contains /)
sed 's|/old/path|/new/path|g' file.txt

# Basic awk structure: pattern { action }
awk '{print $1}' file.txt        # Print first field (space-separated)
awk '{print $1, $3}' file.txt    # Print fields 1 and 3
awk -F: '{print $1}' /etc/passwd # Use colon as field separator

# Built-in variables
awk '{print NR, $0}' file.txt    # Line number and entire line
awk '{print NF, $0}' file.txt    # Number of fields and line
awk '{print $NF}' file.txt       # Last field

# Pattern matching
awk '/error/ {print $0}' log.txt # Print lines with error
awk '$3 > 100 {print $1, $3}' data.txt  # Where field3 > 100

# Calculations
awk '{sum += $1} END {print "Total:", sum}' numbers.txt
awk '{count++} END {print "Lines:", count}' file.txt
awk '{sum+=$3; count++} END {print "Average:", sum/count}' data.txt

# Formatting output
awk '{printf "%-10s %5d\n", $1, $2}' file.txt

# BEGIN and END blocks
awk 'BEGIN {print "Start Processing"} {print $0} END {print "Done"}' file.txt

# Create sample log file for practice
cat > ~/sample.log << 'EOF'
192.168.1.1 - - [17/Jan/2025:10:00:01] "GET /index.html HTTP/1.1" 200 1234
192.168.1.2 - - [17/Jan/2025:10:00:05] "POST /api/login HTTP/1.1" 401 234
192.168.1.1 - - [17/Jan/2025:10:00:10] "GET /dashboard HTTP/1.1" 200 5678
192.168.1.3 - - [17/Jan/2025:10:00:15] "GET /index.html HTTP/1.1" 404 0
192.168.1.1 - - [17/Jan/2025:10:00:20] "POST /api/data HTTP/1.1" 500 123
192.168.1.2 - - [17/Jan/2025:10:00:25] "GET /index.html HTTP/1.1" 200 456
192.168.1.4 - - [17/Jan/2025:10:00:30] "GET /admin HTTP/1.1" 403 0
192.168.1.1 - - [17/Jan/2025:10:00:35] "GET /dashboard HTTP/1.1" 200 2345
192.168.1.3 - - [17/Jan/2025:10:00:40] "GET /index.html HTTP/1.1" 200 789
192.168.1.2 - - [17/Jan/2025:10:00:45] "POST /api/login HTTP/1.1" 200 456
EOF

# Find all error responses (400-599)
grep -E "HTTP/1.1\" [4-5][0-9]{2}" ~/sample.log

# Find 404 errors (page not found)
grep "404" ~/sample.log

# Find all requests from IP 192.168.1.1
grep "192.168.1.1" ~/sample.log

# Count unique IP addresses
grep -oE "[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" ~/sample.log | sort | uniq -c

# Find POST requests
grep "POST" ~/sample.log

# Find errors with context (2 lines before and after)
grep -B2 -A2 "500" ~/sample.log

# Replace IP address 192.168.1.1 with 10.0.0.1
sed 's/192.168.1.1/10.0.0.1/g' ~/sample.log

# Delete all lines with 404 errors
sed '/404/d' ~/sample.log

# Add a timestamp to every line
sed 's/^/[2025-01-17] /' ~/sample.log

# Convert all HTTP methods to lowercase
sed 's/GET/get/g; s/POST/post/g' ~/sample.log

# Extract only the first 10 lines
sed -n '1,10p' ~/sample.log

# Extract IP addresses (field 1)
awk '{print $1}' ~/sample.log

# Count requests per IP
awk '{print $1}' ~/sample.log | sort | uniq -c

# Extract status codes (field 9)
awk '{print $9}' ~/sample.log

# Count responses by status code
awk '{print $9}' ~/sample.log | sort | uniq -c

# Sum of response sizes (field 10) for 200 responses
awk '$9==200 {sum+=$10} END {print "Total 200 response size:", sum}' ~/sample.log

# Average response size for each endpoint
awk '{print $7, $10}' ~/sample.log | awk '{sum[$1]+=$2; count[$1]++} END {for(i in sum) print i, sum[i]/count[i]}' | sort -k2 -rn

# Print lines where response time > 1000 (field 10 > 1000)
awk '$10 > 1000 {print $0}' ~/sample.log

#!/bin/bash
# analyze_logs.sh - Complete log analysis script

LOG_FILE="$1"

if [ -z "$LOG_FILE" ]; then
    echo "Usage: $0 <logfile>"
    exit 1
fi

if [ ! -f "$LOG_FILE" ]; then
    echo "Error: File $LOG_FILE not found"
    exit 1
fi

echo "=== Web Server Log Analysis ==="
echo "Analyzing: $LOG_FILE"
echo ""

# 1. Top IP addresses
echo "Top 10 IP addresses:"
awk '{print $1}' "$LOG_FILE" | sort | uniq -c | sort -rn | head -10 | awk '{printf "%15s: %d requests\n", $2, $1}'
echo ""

# 2. Status code distribution
echo "HTTP Status Codes:"
awk '{print $9}' "$LOG_FILE" | sort | uniq -c | sort -rn | while read count code; do
    case $code in
        2??) color="\033[0;32m";;  # Green
        3??) color="\033[1;33m";;  # Yellow
        4??) color="\033[0;33m";;  # Orange
        5??) color="\033[0;31m";;  # Red
        *)   color="\033[0m";;
    esac
    echo -e "  $color$code\033[0m: $count requests"
done
echo ""

# 3. Error rate
TOTAL=$(wc -l < "$LOG_FILE")
ERRORS=$(awk '$9 >= 400 {print $9}' "$LOG_FILE" | wc -l)
ERROR_RATE=$(echo "scale=2; $ERRORS * 100 / $TOTAL" | bc)
echo "Error Rate: $ERROR_RATE% ($ERRORS of $TOTAL)"
echo ""

# 4. Most popular endpoints
echo "Most requested endpoints:"
awk '{print $7}' "$LOG_FILE" | sort | uniq -c | sort -rn | head -10 | awk '{printf "  %-30s: %d requests\n", $2, $1}'
echo ""

# 5. Slowest requests (by response size as proxy)
echo "Largest responses:"
awk '{print $7, $10}' "$LOG_FILE" | sort -k2 -rn | head -10 | awk '{printf "  %-30s: %s bytes\n", $1, $2}'
echo ""

# 6. Error endpoints
echo "Endpoints with errors (4xx/5xx):"
awk '$9 >= 400 {print $7, $9}' "$LOG_FILE" | sort | uniq -c | sort -rn | head -10 | awk '{printf "  %-30s: %d times (status %s)\n", $2, $1, $3}'
echo ""

# 7. Traffic by hour
echo "Traffic by hour:"
awk '{print $4}' "$LOG_FILE" | cut -d: -f2 | sort | uniq -c | while read count hour; do
    printf "  %02d:00 - %d requests\n" "$hour" "$count"
done
echo ""

# 8. Failed login attempts
echo "Failed login attempts (401):"
grep "401" "$LOG_FILE" | awk '{print $1, $7}' | sort | uniq -c | sort -rn | head -5
echo ""

echo "=== Analysis Complete ==="

# Power of pipes: combine grep, sed, awk, sort, uniq

# Find unique error messages
cat log.txt | grep "ERROR" | sed 's/.*ERROR: //' | sort | uniq -c | sort -rn

# Top 10 IPs with failed logins
cat auth.log | grep "Failed password" | awk '{print $(NF-3)}' | sort | uniq -c | sort -rn | head -10

# Extract and format slow queries
cat slow.log | grep "Query_time:" | awk '{print $3, $NF}' | sed 's/ms//' | awk '$1 > 1 {print $1, $2}' | sort -rn

# Count requests by user-agent
grep "GET" access.log | awk -F'"' '{print $6}' | sort | uniq -c | sort -rn | head -10