Bash File I/O and Data Processing: Reading, Writing, and Manipulating Files

# Write output to a file (overwrite)
echo "Backup started: $(date)" > /tmp/backup.log
cat /tmp/backup.log

# Append output to a file
echo "Processing file 1 of 10" >> /tmp/backup.log
echo "Processing file 2 of 10" >> /tmp/backup.log
cat /tmp/backup.log

# Redirect stderr to a file (separate from stdout)
ls /nonexistent 2> /tmp/errors.log
cat /tmp/errors.log

# Redirect both stdout and stderr to the same file
./deploy.sh > /tmp/deploy.log 2>&1

# Modern syntax (preferred in newer scripts)
./deploy.sh &> /tmp/deploy.log

# Send both to a file AND see output on screen
./deploy.sh 2>&1 | tee /tmp/deploy.log

#!/bin/bash
# Method 1: Read entire file into a variable
CONFIG=$(cat /etc/hostname)
echo "Hostname: $CONFIG"

# Method 2: Read line by line with while loop
while IFS= read -r line; do
  echo "Line: $line"
done < /etc/passwd

# Method 3: Read specific fields by delimiter
while IFS=: read -r username password uid gid name home shell; do
  echo "User $username has UID $uid and home $home"
done < /etc/passwd

# Method 4: Read file into an array
mapfile -t lines < /etc/hosts
echo "Hosts file has ${#lines[@]} lines"
for i in "${!lines[@]}"; do
  if [[ ${lines[$i]} != \#* ]]; then
    echo "Active entry $((i+1)): ${lines[$i]}"
  fi
done

#!/bin/bash
LOG_DIR="/var/log/myapp"
LOG_FILE="${LOG_DIR}/pipeline.log"

mkdir -p "$LOG_DIR"

# Function to log with timestamp
log() {
  local level="$1"
  local message="$2"
  echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $message" >> "$LOG_FILE"
}

log "INFO" "Pipeline started"
log "INFO" "Downloading data..."

# Simulate work
sleep 1

log "INFO" "Processing records..."
sleep 1

log "INFO" "Pipeline completed successfully"

# Show the log
cat "$LOG_FILE"

#!/bin/bash
# A pipeline: count unique IPs in an Apache log
# Step 1: Extract IP addresses (first field, space-separated)
# Step 2: Sort them (necessary before uniq)
# Step 3: Count unique occurrences
# Step 4: Sort by most frequent
# Step 5: Show top 10

grep "GET /api" /var/log/apache2/access.log | \
  awk '{print $1}' | \
  sort | \
  uniq -c | \
  sort -rn | \
  head -10

# Alternative: chain without line continuations
cat /var/log/apache2/access.log | grep "HTTP/1.1"\" 5 | awk '{print $1}' | sort -u | wc -l
echo "Unique clients with 5xx errors above"

#!/bin/bash

# Basic heredoc: writes to a file
cat > /tmp/deploy_config.ini << EOF
[server]
host=prod-01.example.com
port=443
user=deploy

[database]
host=db.internal
name=app_prod
pool_size=10
EOF

echo "Config file created"

# Heredoc with variable expansion
date_today=$(date +%Y-%m-%d)
cat << EOF
Report for: $date_today
Server: $(hostname)
----
All systems nominal
EOF

# Quoted delimiter prevents variable expansion
# Note: use 'EOF' (single-quoted) to prevent expansion
cat << 'EOF'
Literal text: $HOME is NOT expanded
$(whoami) is NOT executed
`date` is NOT interpreted
EOF

#!/bin/bash
# Sample CSV data (inline for demo)
CSV_DATA='name,role,department,start_date
John Smith,Engineer,DevOps,2024-01-15
Jane Doe,SRE,Infra,2024-03-01
Bob Wilson,Security Lead,Security,2024-06-10
Alice Brown,Engineer,DevOps,2024-09-01'

echo "$CSV_DATA" > /tmp/team.csv

# Parse with cut (good for simple columns)
echo "=== All engineers (cut) ==="
grep "Engineer" /tmp/team.csv | cut -d',' -f1,3

echo ""
echo "=== Full table with awk ==="
awk -F',' 'NR>1 {printf "%-20s %-15s %s\n", $1, $2, $3}' /tmp/team.csv

echo ""
echo "=== Department report with awk ==="
awk -F',' 'NR>1 {depts[$3]++} END {for (d in depts) printf "%s: %d employees\n", d, depts[d]}' /tmp/team.csv

echo ""
echo "=== Parse in pure Bash ==="
while IFS=',' read -r name role dept date; do
  [ "$name" = "name" ] && continue  # skip header
  echo "$name ($role) - $dept since $date"
done < /tmp/team.csv

# Cleanup
rm /tmp/team.csv

#!/bin/bash
# log-analyzer.sh — Analyze NGINX access log and generate report
# Usage: ./log-analyzer.sh /var/log/nginx/access.log

LOG_FILE="${1:-/var/log/nginx/access.log}"
REPORT_DIR="/tmp/reports"
REPORT_FILE="${REPORT_DIR}/access_report_$(date +%Y%m%d).txt"

mkdir -p "$REPORT_DIR"

# Check input file
if [ ! -f "$LOG_FILE" ]; then
  echo "ERROR: Log file $LOG_FILE not found" >&2
  exit 1
fi

# Start the report
{
  echo "========================================"
  echo "NGINX Access Log Report"
  echo "Generated: $(date)"
  echo "Source: $LOG_FILE"
  echo "========================================"
  echo ""
  
  # Top 10 IP addresses
  echo "Top 10 IP Addresses:"
  awk '{print $1}' "$LOG_FILE" | sort | uniq -c | sort -rn | head -10
  echo ""
  
  # Top 10 requested URLs
  echo "Top 10 Requested URLs:"
  awk '{print $7}' "$LOG_FILE" | sort | uniq -c | sort -rn | head -10
  echo ""
  
  # HTTP status code distribution
  echo "HTTP Status Code Distribution:"
  awk '{print $9}' "$LOG_FILE" | sort | uniq -c | sort -rn
  echo ""
  
  # 404 errors
  echo "404 Errors (not found):"
  grep ' "404 ' "$LOG_FILE" | awk '{print $7}' | sort | uniq -c | sort -rn | head -5
  echo ""
  
  # Unique user agents
  echo "Unique User Agents: $(awk -F'"' '{print $6}' "$LOG_FILE" | sort -u | wc -l)"
  echo ""
  
  # Total requests
  echo "Total Requests: $(wc -l < "$LOG_FILE")"
  
} > "$REPORT_FILE" 2>&1

echo "Report saved to: $REPORT_FILE"
cat "$REPORT_FILE"