Centralized Logging: Loki and ELK Stack for Log Management

# docker-compose-loki.yml
version: '3.8'

services:
  # Loki - log storage and query engine
  loki:
    image: grafana/loki:latest
    ports:
      - "3100:3100"
    command: -config.file=/etc/loki/local-config.yaml
    volumes:
      - ./loki-config.yaml:/etc/loki/local-config.yaml
      - loki-data:/loki
    restart: unless-stopped

  # Promtail - log collector
  promtail:
    image: grafana/promtail:latest
    volumes:
      - /var/log:/var/log
      - /var/lib/docker/containers:/var/lib/docker/containers
      - ./promtail-config.yaml:/etc/promtail/config.yml
    command: -config.file=/etc/promtail/config.yml
    depends_on:
      - loki
    restart: unless-stopped

  # Grafana - visualization
  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana-data:/var/lib/grafana
    depends_on:
      - loki
    restart: unless-stopped

volumes:
  loki-data:
  grafana-data:

# loki-config.yaml
 auth_enabled: false

 server:
   http_listen_port: 3100

 ingester:
   lifecycler:
     address: 127.0.0.1
     ring:
       kvstore:
         store: inmemory
       replication_factor: 1
   chunk_idle_period: 5m
   chunk_retain_period: 30s

 schema_config:
   configs:
     - from: 2020-10-24
       store: boltdb-shipper
       object_store: filesystem
       schema: v11
       index:
         prefix: index_
         period: 24h

 storage_config:
   boltdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
     shared_store: filesystem
   filesystem:
     directory: /loki/chunks

 compactor:
   working_directory: /loki/compactor
   shared_store: filesystem

 limits_config:
   retention_period: 720h  # 30 days

 # promtail-config.yaml
 scrape_configs:
   - job_name: system
     static_configs:
       - targets:
           - localhost
         labels:
           job: syslog
           host: localhost
           __path__: /var/log/*.log

   - job_name: docker
     docker_sd_configs:
       - host: unix:///var/run/docker.sock
         refresh_interval: 10s
     relabel_configs:
       - source_labels: ['__meta_docker_container_name']
         regex: '/(.*)'
         target_label: 'container'
       - source_labels: ['__meta_docker_container_label_com_docker_compose_service']
         target_label: 'service'

   - job_name: kubernetes-pods
     kubernetes_sd_configs:
       - role: pod
     relabel_configs:
       - source_labels: [__meta_kubernetes_pod_label_app]
         target_label: app
       - source_labels: [__meta_kubernetes_pod_node_name]
         target_label: node
       - source_labels: [__meta_kubernetes_namespace]
         target_label: namespace
       - source_labels: [__meta_kubernetes_pod_container_name]
         target_label: container

# Basic LogQL Queries

# Select all logs from a job
{job="syslog"}

# Filter by container
{container="nginx"} |= "error"

# Regex filtering
{app="myapp"} |~ "(?i)error|timeout"

# Negative filter
{job="syslog"} != "INFO"

# JSON parsing
{container="api"} | json | line_format "{{.level}} {{.message}}"

# Logfmt parsing
{app="web"} | logfmt | level="error"

# Regular expression extraction
{app="myapp"} | regexp "(?P<ip>\d+\.\d+\.\d+\.\d+) - -"

# Rate of error logs
rate({app="myapp"} |= "error" [5m])

# Count errors over time
count_over_time({app="myapp"} |= "error" [1h])

# Top error messages
topk(10, count by (message) ({app="myapp"} |= "error"))

# Aggregate by label
sum by (container) (count_over_time({job="docker"} [5m]))

# Pattern extraction
{container="nginx"} | pattern `<_> - - [<_>] "<method> <path> <_>" <status> <size>`

# Time range
{app="myapp"} |= "error" | line_format "{{.timestamp}} - {{.message}}"

# Complex queries
{namespace="production"} 
  | json 
  | level="error" 
  | duration > 1s 
  | line_format "{{.timestamp}} [{{.level}}] {{.message}}"

# structured_logging.py - JSON structured logging
import logging
import json
import sys
from datetime import datetime

class JSONFormatter(logging.Formatter):
    """Custom JSON formatter for structured logging"""
    
    def format(self, record):
        log_entry = {
            "timestamp": datetime.utcnow().isoformat() + "Z",
            "level": record.levelname,
            "logger": record.name,
            "message": record.getMessage(),
            "module": record.module,
            "function": record.funcName,
            "line": record.lineno,
        }
        
        # Add exception info if present
        if record.exc_info:
            log_entry["exception"] = self.formatException(record.exc_info)
        
        # Add extra fields
        if hasattr(record, 'extra_data'):
            log_entry.update(record.extra_data)
        
        return json.dumps(log_entry)

# Configure logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JSONFormatter())
logger.addHandler(handler)

# Usage with extra context
logger.info("User logged in", extra={
    'extra_data': {
        'user_id': 12345,
        'ip_address': '192.168.1.100',
        'user_agent': 'Mozilla/5.0'
    }
})

logger.warning("High memory usage", extra={
    'extra_data': {
        'memory_usage_mb': 2048,
        'threshold_mb': 1024
    }
})

try:
    # Some operation
    result = 1 / 0
except Exception as e:
    logger.error("Operation failed", extra={
        'extra_data': {
            'operation': 'divide',
            'error_type': type(e).__name__
        }
    }, exc_info=True)

# docker-compose-elk.yml
version: '3.8'

services:
  # Elasticsearch - storage and search
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ports:
      - "9200:9200"
    volumes:
      - elasticsearch-data:/usr/share/elasticsearch/data
    restart: unless-stopped

  # Logstash - log processing pipeline
  logstash:
    image: docker.elastic.co/logstash/logstash:8.11.0
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
    ports:
      - "5000:5000"
      - "5044:5044"
    depends_on:
      - elasticsearch
    restart: unless-stopped

  # Kibana - visualization
  kibana:
    image: docker.elastic.co/kibana/kibana:8.11.0
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    depends_on:
      - elasticsearch
    restart: unless-stopped

  # Filebeat - log collector
  filebeat:
    image: docker.elastic.co/beats/filebeat:8.11.0
    volumes:
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
      - /var/log:/var/log:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
    depends_on:
      - logstash
    restart: unless-stopped

volumes:
  elasticsearch-data:

# logstash.conf
input {
  beats {
    port => 5044
  }
  
  tcp {
    port => 5000
    codec => json_lines
  }
  
  file {
    path => "/var/log/*.log"
    type => "syslog"
  }
}

filter {
  # Parse JSON logs
  if [message] =~ /^\{.*\}$/ {
    json {
      source => "message"
      target => "parsed"
    }
    mutate {
      add_field => { "log_type" => "json" }
    }
  }
  
  # Parse nginx access logs
  if [type] == "nginx-access" {
    grok {
      match => { "message" => "%{IP:client_ip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:bytes_sent} \"%{DATA:referrer}\" \"%{DATA:user_agent}\"" }
    }
  }
  
  # Parse syslog
  if [type] == "syslog" {
    grok {
      match => { "message" => "<%{NUMBER:syslog_pri}>%{SYSLOGTIMESTAMP:timestamp} %{SYSLOGHOST:hostname} %{DATA:program}(?:\[%{NUMBER:pid}\])?: %{GREEDYDATA:message}" }
    }
  }
  
  # Add timestamp
  date {
    match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z", "yyyy-MM-dd HH:mm:ss", "ISO8601" ]
    target => "@timestamp"
  }
  
  # Remove unnecessary fields
  mutate {
    remove_field => [ "host", "agent", "ecs" ]
  }
}

output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    index => "logs-%{+YYYY.MM.dd}"
    user => "elastic"
    password => "changeme"
  }
  
  stdout {
    codec => rubydebug
  }
}

# filebeat.yml
filebeat.inputs:
  - type: log
    enabled: true
    paths:
      - /var/log/*.log
    fields:
      source: system
    fields_under_root: true

  - type: container
    enabled: true
    paths:
      - '/var/lib/docker/containers/*/*.log'
    fields:
      source: docker
    fields_under_root: true

  - type: filestream
    enabled: true
    paths:
      - /var/log/nginx/*.log
    fields:
      type: nginx-access
      source: nginx
    fields_under_root: true

processors:
  - add_host_metadata: {}
  - add_docker_metadata:
      host: "unix:///var/run/docker.sock"
  - add_kubernetes_metadata:
      host: ${NODE_NAME}
      matchers:
        - logs_path:
            logs_path: "/var/log/containers/"

output.logstash:
  hosts: ["logstash:5044"]

logging:
  level: info

# docker-compose-full-logging.yml
version: '3.8'

services:
  # Loki Stack
  loki:
    image: grafana/loki:latest
    ports:
      - "3100:3100"
    volumes:
      - ./loki-config.yaml:/etc/loki/local-config.yaml
      - loki-data:/loki

  promtail:
    image: grafana/promtail:latest
    volumes:
      - /var/log:/var/log
      - /var/lib/docker/containers:/var/lib/docker/containers
      - ./promtail-config.yaml:/etc/promtail/config.yml

  # ELK Stack
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
    ports:
      - "9200:9200"
    volumes:
      - elasticsearch-data:/usr/share/elasticsearch/data

  logstash:
    image: docker.elastic.co/logstash/logstash:8.11.0
    ports:
      - "5000:5000"
      - "5044:5044"
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf

  kibana:
    image: docker.elastic.co/kibana/kibana:8.11.0
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200

  filebeat:
    image: docker.elastic.co/beats/filebeat:8.11.0
    volumes:
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
      - /var/log:/var/log:ro
      - /var/lib/docker/containers:/var/lib/docker/containers:ro

  # Grafana (unified visualization)
  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana-data:/var/lib/grafana

volumes:
  loki-data:
  elasticsearch-data:
  grafana-data:

networks:
  default:
    name: logging-network

# Loki retention (in loki-config.yaml)
limits_config:
  retention_period: 720h  # 30 days

# Elasticsearch ILM (Index Lifecycle Management)
# In Kibana Dev Tools:
PUT _ilm/policy/logs_policy
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_size": "50GB",
            "max_age": "30d"
          }
        }
      },
      "warm": {
        "min_age": "30d",
        "actions": {
          "shrink": {"number_of_shards": 1},
          "forcemerge": {"max_num_segments": 1}
        }
      },
      "cold": {
        "min_age": "60d",
        "actions": {
          "freeze": {},
          "set_priority": {"priority": 0}
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

# Apply to index template
PUT _index_template/logs_template
{
  "index_patterns": ["logs-*"],
  "template": {
    "settings": {
      "number_of_shards": 2,
      "number_of_replicas": 1,
      "index.lifecycle.name": "logs_policy",
      "index.lifecycle.rollover_alias": "logs"
    }
  }
}

# Log rotation for application logs
# /etc/logrotate.d/myapp
/var/log/myapp/*.log {
    daily
    missingok
    rotate 30
    compress
    delaycompress
    notifempty
    create 0640 appuser appgroup
    sharedscripts
    postrotate
        docker kill --signal=HUP $(docker ps -q -f name=myapp)
    endscript
}

# Docker container logs rotation
# /etc/docker/daemon.json
{
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  }
}