Prometheus: Metrics Collection, Storage, and Querying

# Download Prometheus
wget https://github.com/prometheus/prometheus/releases/download/v2.48.0/prometheus-2.48.0.linux-amd64.tar.gz
tar xvf prometheus-2.48.0.linux-amd64.tar.gz
cd prometheus-2.48.0.linux-amd64

# Create prometheus.yml
cat > prometheus.yml << 'EOF'
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  
  - job_name: 'node'
    static_configs:
      - targets: ['localhost:9100']
  
  - job_name: 'application'
    static_configs:
      - targets: ['localhost:8000']
EOF

# Run Prometheus
./prometheus --config.file=prometheus.yml

# Docker Compose
cat > docker-compose.yml << 'EOF'
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
    restart: unless-stopped

volumes:
  prometheus-data:
EOF

docker-compose up -d

from prometheus_client import Counter, Gauge, Histogram, Summary, generate_latest
import random
import time

# Counter - only increases (requests, errors, tasks processed)
requests_total = Counter('http_requests_total', 'Total HTTP requests', ['method', 'endpoint'])
errors_total = Counter('http_errors_total', 'Total HTTP errors', ['error_type'])

# Increment counter
requests_total.labels(method='GET', endpoint='/api/users').inc()
requests_total.labels(method='POST', endpoint='/api/users').inc(5)
errors_total.labels(error_type='database').inc()

# Gauge - can go up and down (CPU, memory, connections)
cpu_usage = Gauge('cpu_usage_percent', 'CPU usage percentage')
memory_usage = Gauge('memory_usage_bytes', 'Memory usage in bytes')
active_connections = Gauge('active_connections', 'Active connections')

# Set gauge values
cpu_usage.set(45.5)
memory_usage.set(1024 * 1024 * 500)  # 500MB
active_connections.inc()  # +1
active_connections.dec()  # -1

# Histogram - measures distributions (request duration, response sizes)
request_duration = Histogram('http_request_duration_seconds', 'Request duration in seconds', buckets=[0.1, 0.5, 1, 2, 5])

# Observe values
for _ in range(100):
    duration = random.uniform(0.05, 1.5)
    request_duration.observe(duration)

# Summary - similar to histogram but client-side quantiles
request_latency = Summary('http_request_latency_seconds', 'Request latency', ['method'])

request_latency.labels(method='GET').observe(0.2)
request_latency.labels(method='GET').observe(0.3)
request_latency.labels(method='POST').observe(0.5)

# Flask application with metrics
from flask import Flask
app = Flask(__name__)

@app.route('/metrics')
def metrics():
    return generate_latest()

@app.route('/')
def home():
    start = time.time()
    requests_total.labels(method='GET', endpoint='/').inc()
    request_duration.observe(time.time() - start)
    return "Hello, World!"

if __name__ == '__main__':
    app.run(port=8000)

# Basic Queries

# Get current value
http_requests_total

# Filter by labels
http_requests_total{method="GET"}
http_requests_total{method="POST", endpoint="/api/users"}

# Regular expression matching
http_requests_total{method=~"GET|POST"}
http_requests_total{endpoint=~"/api/.*"}

# Rate calculations
rate(http_requests_total[5m])  # Requests per second over 5m
irate(http_requests_total[5m]) # Instant rate (for spiky data)

# Increase over time
increase(http_requests_total[1h])  # Total increase over 1 hour

# Histogram quantiles
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))

# Aggregation
sum(http_requests_total) by (method)           # Sum by method
avg(cpu_usage) by (instance)                   # Average by instance
topk(5, http_requests_total)                   # Top 5 highest
count(http_requests_total)                     # Count time series

# Mathematical operations
http_requests_total / 1000                     # Divide by 1000
cpu_usage > 80                                 # Filter CPU > 80%

# Time functions
http_requests_total offset 1d                  # Values from 1 day ago
http_requests_total[30d]                       # Last 30 days of data

# Practical examples
# Error rate
sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m]))

# 95th percentile latency
histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))

# Predicted disk full in hours
(predict_linear(node_filesystem_free_bytes[1h], 3600) <= 0)

# Memory usage percentage
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100

# docker-compose-exporters.yml
version: '3.8'

services:
  # Node Exporter (system metrics)
  node-exporter:
    image: prom/node-exporter:latest
    ports:
      - "9100:9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--path.rootfs=/rootfs'

  # cAdvisor (container metrics)
  cadvisor:
    image: gcr.io/cadvisor/cadvisor:latest
    ports:
      - "8080:8080"
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro

  # Blackbox Exporter (endpoint monitoring)
  blackbox:
    image: prom/blackbox-exporter:latest
    ports:
      - "9115:9115"
    volumes:
      - ./blackbox.yml:/etc/blackbox_exporter/config.yml

  # MySQL Exporter
  mysqld-exporter:
    image: prom/mysqld-exporter:latest
    ports:
      - "9104:9104"
    environment:
      DATA_SOURCE_NAME: "user:password@(mysql:3306)/"

  # Redis Exporter
  redis-exporter:
    image: oliver006/redis_exporter:latest
    ports:
      - "9121:9121"
    environment:
      REDIS_ADDR: "redis://redis:6379"

  # Postgres Exporter
  postgres-exporter:
    image: prometheuscommunity/postgres-exporter:latest
    ports:
      - "9187:9187"
    environment:
      DATA_SOURCE_NAME: "postgresql://user:password@postgres:5432/db?sslmode=disable"

  # Nginx Exporter
  nginx-exporter:
    image: nginx/nginx-prometheus-exporter:latest
    ports:
      - "9113:9113"
    command:
      - '-nginx.scrape-uri=http://nginx:80/stub_status'

# prometheus.yml additions:
# scrape_configs:
#   - job_name: 'node'
#     static_configs:
#       - targets: ['node-exporter:9100']
#   
#   - job_name: 'cadvisor'
#     static_configs:
#       - targets: ['cadvisor:8080']
#   
#   - job_name: 'blackbox'
#     static_configs:
#       - targets: ['blackbox:9115']
#   
#   - job_name: 'mysql'
#     static_configs:
#       - targets: ['mysqld-exporter:9104']

# prometheus.yml with service discovery

# Kubernetes service discovery
scrape_configs:
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__

  - job_name: 'kubernetes-services'
    kubernetes_sd_configs:
      - role: service
    relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)
        replacement: $1:__meta_kubernetes_service_annotation_prometheus_io_port

# AWS EC2 discovery
  - job_name: 'aws-ec2'
    ec2_sd_configs:
      - region: us-east-1
        access_key: YOUR_KEY
        secret_key: YOUR_SECRET
        port: 9100
    relabel_configs:
      - source_labels: [__meta_ec2_tag_Name]
        target_label: instance_name
      - source_labels: [__meta_ec2_instance_id]
        target_label: instance_id

# File-based discovery
  - job_name: 'file-sd'
    file_sd_configs:
      - files:
        - /etc/prometheus/targets/*.json
        refresh_interval: 5m

# targets.json
[
  {
    "targets": ["10.0.1.10:9100", "10.0.1.11:9100"],
    "labels": {
      "environment": "production",
      "service": "web"
    }
  }
]

#!/usr/bin/env python3
"""
monitored_app.py - Complete application with Prometheus metrics
"""

from flask import Flask, request, jsonify
from prometheus_client import Counter, Histogram, Gauge, generate_latest, REGISTRY
from prometheus_client import Summary
import time
import random
import threading

app = Flask(__name__)

# Business metrics
REQUEST_COUNT = Counter('app_requests_total', 'Total requests', ['method', 'endpoint', 'status'])
REQUEST_DURATION = Histogram('app_request_duration_seconds', 'Request duration', ['method', 'endpoint'])
ERROR_COUNT = Counter('app_errors_total', 'Total errors', ['error_type'])
ACTIVE_REQUESTS = Gauge('app_active_requests', 'Active requests')

# Business metrics
ORDERS_CREATED = Counter('app_orders_created_total', 'Total orders created', ['product_type'])
DB_QUERY_DURATION = Histogram('app_db_query_duration_seconds', 'DB query duration', ['query_type'])
CACHE_HIT_RATIO = Gauge('app_cache_hit_ratio', 'Cache hit ratio')

# Simulated metrics generator
def simulate_metrics():
    """Background task to generate synthetic metrics"""
    while True:
        # Simulate cache hit ratio between 0.7 and 0.95
        hit_ratio = random.uniform(0.7, 0.95)
        CACHE_HIT_RATIO.set(hit_ratio)
        
        # Simulate orders
        if random.random() > 0.7:
            product = random.choice(['electronics', 'clothing', 'books'])
            ORDERS_CREATED.labels(product_type=product).inc()
        
        time.sleep(5)

# Start background metrics generator
threading.Thread(target=simulate_metrics, daemon=True).start()

@app.before_request
def before_request():
    request.start_time = time.time()
    ACTIVE_REQUESTS.inc()

@app.after_request
def after_request(response):
    duration = time.time() - request.start_time
    REQUEST_DURATION.labels(
        method=request.method,
        endpoint=request.path
    ).observe(duration)
    
    REQUEST_COUNT.labels(
        method=request.method,
        endpoint=request.path,
        status=response.status_code
    ).inc()
    
    ACTIVE_REQUESTS.dec()
    return response

@app.route('/metrics')
def metrics():
    return generate_latest(REGISTRY), 200, {'Content-Type': 'text/plain'}

@app.route('/')
def home():
    return jsonify({
        'service': 'monitored-app',
        'version': '1.0.0',
        'status': 'healthy'
    })

@app.route('/api/users', methods=['GET'])
def get_users():
    # Simulate database query
    start = time.time()
    time.sleep(random.uniform(0.01, 0.1))
    DB_QUERY_DURATION.labels(query_type='select_users').observe(time.time() - start)
    
    return jsonify({'users': ['alice', 'bob', 'charlie']})

@app.route('/api/users', methods=['POST'])
def create_user():
    data = request.json
    
    # Simulate database insert
    start = time.time()
    time.sleep(random.uniform(0.05, 0.2))
    DB_QUERY_DURATION.labels(query_type='insert_user').observe(time.time() - start)
    
    # Simulate occasional errors
    if random.random() < 0.05:
        ERROR_COUNT.labels(error_type='database_timeout').inc()
        return jsonify({'error': 'Database timeout'}), 500
    
    ORDERS_CREATED.labels(product_type='user_creation').inc()
    return jsonify({'user': data.get('name'), 'id': random.randint(1, 1000)}), 201

@app.route('/api/checkout', methods=['POST'])
def checkout():
    start = time.time()
    
    # Simulate payment processing
    time.sleep(random.uniform(0.1, 0.5))
    
    # Simulate errors
    if random.random() < 0.1:
        ERROR_COUNT.labels(error_type='payment_failed').inc()
        return jsonify({'error': 'Payment failed'}), 400
    
    DB_QUERY_DURATION.labels(query_type='create_order').observe(time.time() - start)
    return jsonify({'order_id': random.randint(1000, 9999), 'status': 'completed'})

@app.route('/health')
def health():
    return jsonify({'status': 'healthy'}), 200

@app.route('/ready')
def ready():
    # Check dependencies
    checks = {'database': True, 'cache': True}
    all_ready = all(checks.values())
    status_code = 200 if all_ready else 503
    return jsonify({'ready': all_ready, 'checks': checks}), status_code

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8000)

# alerts.yml
 groups:
   - name: application_alerts
     rules:
       - alert: HighErrorRate
         expr: |
           sum(rate(app_errors_total[5m])) / sum(rate(app_requests_total[5m])) > 0.05
         for: 5m
         labels:
           severity: critical
           team: backend
         annotations:
           summary: "High error rate detected"
           description: "Error rate is {{ $value | humanizePercentage }} for last 5 minutes"
       
       - alert: SlowResponses
         expr: |
           histogram_quantile(0.95, sum(rate(app_request_duration_seconds_bucket[5m])) by (le, endpoint)) > 2
         for: 10m
         labels:
           severity: warning
           team: backend
         annotations:
           summary: "Slow responses on {{ $labels.endpoint }}"
           description: "95th percentile latency is {{ $value }}s"
       
       - alert: HighCPUUsage
         expr: |
           100 - (avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
         for: 15m
         labels:
           severity: warning
           team: infrastructure
         annotations:
           summary: "High CPU usage on {{ $labels.instance }}"
           description: "CPU usage is {{ $value }}%"
       
       - alert: LowCacheHitRatio
         expr: app_cache_hit_ratio < 0.7
         for: 10m
         labels:
           severity: warning
           team: backend
         annotations:
           summary: "Low cache hit ratio"
           description: "Cache hit ratio is {{ $value | humanizePercentage }}"
       
       - alert: DatabaseQuerySlow
         expr: |
           histogram_quantile(0.95, sum(rate(app_db_query_duration_seconds_bucket[5m])) by (le, query_type)) > 1
         for: 5m
         labels:
           severity: warning
           team: database
         annotations:
           summary: "Slow database queries"
           description: "95th percentile for {{ $labels.query_type }} is {{ $value }}s"
       
       - alert: NoMetrics
         expr: absent(up{job="application"} == 1)
         for: 5m
         labels:
           severity: critical
         annotations:
           summary: "Application not reporting metrics"
           description: "No metrics received from {{ $labels.job }} for 5 minutes"