Kubernetes Probes & Health Checks: Keeping Applications Healthy

Three Probe Types:

1. Liveness Probe: Is the container alive?
   - If fails: Container is restarted
   - Purpose: Recover from deadlocks, crashes

2. Readiness Probe: Is the container ready to serve traffic?
   - If fails: Traffic is NOT sent to this pod
   - Purpose: Ensure only healthy pods receive requests

3. Startup Probe: Has the application started successfully?
   - If fails: Container is restarted
   - Purpose: Slow-starting apps (overrides liveness during startup)

# deployment-with-http-probes.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: webapp
spec:
  replicas: 3
  selector:
    matchLabels:
      app: webapp
  template:
    metadata:
      labels:
        app: webapp
    spec:
      containers:
      - name: app
        image: myapp:latest
        ports:
        - containerPort: 8080
        
        # Liveness Probe
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
            httpHeaders:
            - name: Custom-Header
              value: liveness-check
          initialDelaySeconds: 30  # Wait before first probe
          periodSeconds: 10        # Check every 10 seconds
          timeoutSeconds: 5        # Wait 5 seconds for response
          successThreshold: 1      # Consecutive successes needed
          failureThreshold: 3      # Consecutive failures to restart
        
        # Readiness Probe
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5
          timeoutSeconds: 3
          successThreshold: 1
          failureThreshold: 3
        
        # Startup Probe (for slow-starting apps)
        startupProbe:
          httpGet:
            path: /startup
            port: 8080
          initialDelaySeconds: 0
          periodSeconds: 5
          failureThreshold: 30  # Allow 30 * 5 = 150 seconds to start

# Application endpoints needed:
# /health - returns 200 if app is alive, 5xx if dead
# /ready - returns 200 if app ready for traffic
# /startup - returns 200 when app fully started

# tcp-probe.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: database
spec:
  template:
    spec:
      containers:
      - name: postgres
        image: postgres:15-alpine
        ports:
        - containerPort: 5432
        
        livenessProbe:
          tcpSocket:
            port: 5432
          initialDelaySeconds: 30
          periodSeconds: 10
          failureThreshold: 3
        
        readinessProbe:
          tcpSocket:
            port: 5432
          initialDelaySeconds: 5
          periodSeconds: 5

# TCP probe checks if the port is open
# No need for custom health endpoints
# Works for databases, Redis, etc.

# exec-probe.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-with-exec-probe
spec:
  template:
    spec:
      containers:
      - name: app
        image: myapp:latest
        
        livenessProbe:
          exec:
            command:
            - /bin/sh
            - -c
            - |
              # Check if process is responding
              pgrep myapp && \
              curl -f http://localhost:8080/internal-health
          initialDelaySeconds: 30
          periodSeconds: 10
        
        readinessProbe:
          exec:
            command:
            - test
            - -f
            - /tmp/ready
          initialDelaySeconds: 5
          periodSeconds: 5

# Exec probe runs command inside container
# Exit code 0 = success, non-zero = failure
# Good for custom health logic

# Python Flask example with health endpoints
from flask import Flask, jsonify
import redis
import psycopg2
import os

app = Flask(__name__)

# Get dependencies from environment
redis_client = redis.Redis(host=os.environ.get('REDIS_HOST', 'localhost'))

def get_db_connection():
    return psycopg2.connect(
        host=os.environ.get('DB_HOST', 'localhost'),
        database=os.environ.get('DB_NAME', 'app'),
        user=os.environ.get('DB_USER', 'postgres'),
        password=os.environ.get('DB_PASSWORD', '')
    )

@app.route('/health')
def health():
    """Liveness probe - just check if process is alive"""
    return jsonify({'status': 'alive'}), 200

@app.route('/ready')
def ready():
    """Readiness probe - check if ready to serve traffic"""
    checks = {
        'database': False,
        'redis': False,
        'disk_space': False
    }
    
    # Check database
    try:
        conn = get_db_connection()
        conn.close()
        checks['database'] = True
    except Exception:
        pass
    
    # Check Redis
    try:
        redis_client.ping()
        checks['redis'] = True
    except Exception:
        pass
    
    # Check disk space
    import shutil
    disk = shutil.disk_usage('/')
    if disk.free / disk.total > 0.1:  # At least 10% free
        checks['disk_space'] = True
    
    if all(checks.values()):
        return jsonify({'status': 'ready', 'checks': checks}), 200
    else:
        return jsonify({'status': 'not ready', 'checks': checks}), 503

@app.route('/startup')
def startup():
    """Startup probe - check if app is fully initialized"""
    # Check if migrations are complete
    # Check if cache is warmed
    # Check if all services are connected
    return jsonify({'status': 'started'}), 200

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080)

// Node.js/Express health endpoints
const express = require('express');
const mongoose = require('mongoose');
const redis = require('redis');

const app = express();
const redisClient = redis.createClient();

// Liveness probe
app.get('/health', (req, res) => {
  res.status(200).json({ status: 'alive', timestamp: Date.now() });
});

// Readiness probe
app.get('/ready', async (req, res) => {
  const checks = { database: false, redis: false };
  let ready = true;
  
  // Check MongoDB connection
  if (mongoose.connection.readyState !== 1) {
    checks.database = false;
    ready = false;
  } else {
    checks.database = true;
  }
  
  // Check Redis connection
  try {
    await redisClient.ping();
    checks.redis = true;
  } catch (err) {
    checks.redis = false;
    ready = false;
  }
  
  if (ready) {
    res.status(200).json({ status: 'ready', checks });
  } else {
    res.status(503).json({ status: 'not ready', checks });
  }
});

// Startup probe
let appStarted = false;

app.get('/startup', (req, res) => {
  if (appStarted) {
    res.status(200).json({ status: 'started' });
  } else {
    res.status(503).json({ status: 'starting' });
  }
});

// Mark as started after initialization
setTimeout(() => {
  appStarted = true;
  console.log('Application started successfully');
}, 30000);  // 30 second startup time

app.listen(8080);

# complete-health-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: production-app
  namespace: production
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels:
      app: production-app
  template:
    metadata:
      labels:
        app: production-app
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
        prometheus.io/path: "/metrics"
    spec:
      containers:
      - name: app
        image: myapp:latest
        ports:
        - containerPort: 8080
          name: http
        - containerPort: 9090
          name: metrics
        
        # Environment variables
        env:
        - name: DATABASE_URL
          valueFrom:
            secretKeyRef:
              name: db-secret
              key: url
        - name: REDIS_URL
          valueFrom:
            configMapKeyRef:
              name: app-config
              key: redis_url
        
        # Resource limits
        resources:
          requests:
            memory: "256Mi"
            cpu: "250m"
          limits:
            memory: "512Mi"
            cpu: "500m"
        
        # Startup probe (for slow-starting app)
        startupProbe:
          httpGet:
            path: /startup
            port: 8080
          initialDelaySeconds: 0
          periodSeconds: 5
          failureThreshold: 60  # 5 minutes total
          timeoutSeconds: 5
        
        # Liveness probe (after startup)
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 0  # Startup probe handles initial delay
          periodSeconds: 30
          timeoutSeconds: 5
          failureThreshold: 3
          successThreshold: 1
        
        # Readiness probe (traffic only when ready)
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 0
          periodSeconds: 10
          timeoutSeconds: 3
          failureThreshold: 3
          successThreshold: 1
        
        # Lifecycle hooks
        lifecycle:
          preStop:
            exec:
              command:
              - /bin/sh
              - -c
              - |
                echo "Shutting down..."
                sleep 15
                echo "Goodbye"
      
      # Graceful termination period
      terminationGracePeriodSeconds: 30

---
# Service
apiVersion: v1
kind: Service
metadata:
  name: production-app-service
spec:
  selector:
    app: production-app
  ports:
  - port: 80
    targetPort: 8080
  type: ClusterIP

---
# PodDisruptionBudget (ensure availability during voluntary disruptions)
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: production-app-pdb
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app: production-app

# Test probes
kubectl get pods -w
kubectl describe pod -l app=production-app
kubectl logs -l app=production-app --tail=20

# Simulate failure
kubectl exec -it <pod-name> -- kill 1
# Watch Kubernetes restart the pod
kubectl get pods -w

# Check probe status in pod events
kubectl describe pod <pod-name>
# Look for:
#   Liveness: ...
#   Readiness: ...

# Check if pod is ready
kubectl get pods
# READY column shows: 1/1 = ready, 0/1 = not ready

# Check pod conditions
kubectl get pod <pod-name> -o jsonpath='{.status.conditions}' | jq .

# View probe failures in logs
kubectl logs <pod-name> --previous  # See why it restarted

# Temporarily disable probes for debugging (edit deployment)
kubectl edit deployment <deployment-name>
# Remove or comment out probe sections

# Common probe failure reasons:
# 1. Health endpoint returns 5xx
# 2. Health endpoint times out
# 3. Wrong port or path
# 4. initialDelaySeconds too short
# 5. failureThreshold too low