CI/CD Fundamentals: Automating Build, Test, and Deployment Pipelines

# .github/workflows/ci.yml
name: CI/CD Pipeline

# When to run
on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]
  workflow_dispatch:  # Manual trigger

# Environment variables for entire workflow
env:
  DOCKER_REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

# Jobs run in parallel by default
jobs:
  # JOB 1: Test
  test:
    name: Run Tests
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Setup Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    
    - name: Install dependencies
      run: |
        pip install -r requirements.txt
        pip install pytest pytest-cov flake8 black
    
    - name: Lint with flake8
      run: flake8 . --count --max-complexity=10 --statistics
    
    - name: Check formatting with black
      run: black --check .
    
    - name: Test with pytest
      run: pytest tests/ --cov=myapp --cov-report=xml
    
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v3
      with:
        file: ./coverage.xml
  
  # JOB 2: Build Docker Image
  build:
    name: Build and Push Docker Image
    runs-on: ubuntu-latest
    needs: test  # Only runs if test job succeeds
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3
    
    - name: Log in to GitHub Container Registry
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    
    - name: Extract metadata for Docker
      id: meta
      uses: docker/metadata-action@v5
      with:
        images: ghcr.io/${{ github.repository }}
        tags: |
          type=ref,event=branch
          type=sha,format=short
          type=raw,value=latest,enable={{is_default_branch}}
    
    - name: Build and push Docker image
      uses: docker/build-push-action@v5
      with:
        context: .
        push: true
        tags: ${{ steps.meta.outputs.tags }}
        labels: ${{ steps.meta.outputs.labels }}
        cache-from: type=gha
        cache-to: type=gha,mode=max
  
  # JOB 3: Deploy to Staging
  deploy-staging:
    name: Deploy to Staging
    runs-on: ubuntu-latest
    needs: build
    environment:
      name: staging
      url: https://staging.example.com
    
    steps:
    - name: Deploy to Kubernetes
      run: |
        kubectl set image deployment/myapp \
          myapp=ghcr.io/${{ github.repository }}:${{ github.sha }} \
          --namespace=staging
        kubectl rollout status deployment/myapp --namespace=staging
  
  # JOB 4: Integration Tests on Staging
  integration-tests:
    name: Run Integration Tests
    runs-on: ubuntu-latest
    needs: deploy-staging
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Run integration tests
      run: |
        pip install -r requirements-test.txt
        pytest tests/integration/ --url=https://staging.example.com
  
  # JOB 5: Deploy to Production (manual approval)
  deploy-production:
    name: Deploy to Production
    runs-on: ubuntu-latest
    needs: integration-tests
    environment:
      name: production
      url: https://example.com
    if: github.ref == 'refs/heads/main'
    
    steps:
    - name: Deploy to Kubernetes
      run: |
        kubectl set image deployment/myapp \
          myapp=ghcr.io/${{ github.repository }}:${{ github.sha }} \
          --namespace=production
        kubectl rollout status deployment/myapp --namespace=production

# app.py - Sample Flask application for CI/CD demo
from flask import Flask, jsonify
import os
import datetime

app = Flask(__name__)

@app.route('/')
def hello():
    return jsonify({
        'message': 'Hello from CI/CD Pipeline!',
        'version': os.environ.get('APP_VERSION', '1.0.0'),
        'environment': os.environ.get('ENVIRONMENT', 'development'),
        'timestamp': datetime.datetime.now().isoformat()
    })

@app.route('/health')
def health():
    return jsonify({'status': 'healthy'}), 200

@app.route('/ready')
def ready():
    # Could check database connections here
    return jsonify({'ready': True}), 200

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080)

# tests/test_app.py
import pytest
from app import app

@pytest.fixture
def client():
    app.config['TESTING'] = True
    with app.test_client() as client:
        yield client

def test_hello_endpoint(client):
    response = client.get('/')
    assert response.status_code == 200
    data = response.get_json()
    assert 'message' in data
    assert 'version' in data

def test_health_endpoint(client):
    response = client.get('/health')
    assert response.status_code == 200
    data = response.get_json()
    assert data['status'] == 'healthy'

def test_ready_endpoint(client):
    response = client.get('/ready')
    assert response.status_code == 200
    data = response.get_json()
    assert data['ready'] is True

# Test environment variables
def test_environment_variable(client):
    response = client.get('/')
    data = response.get_json()
    # Should show development by default in tests
    assert data['environment'] == 'development'

# Dockerfile - Multi-stage build
FROM python:3.11-slim AS builder

WORKDIR /app
COPY requirements.txt .
RUN pip install --user --no-cache-dir -r requirements.txt

FROM python:3.11-slim

RUN useradd --create-home appuser
WORKDIR /app

COPY --from=builder /root/.local /home/appuser/.local
COPY --chown=appuser:appuser . .

USER appuser
ENV PATH=/home/appuser/.local/bin:$PATH

EXPOSE 8080

HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
  CMD curl -f http://localhost:8080/health || exit 1

CMD ["python", "app.py"]

# requirements.txt
flask==2.3.0
pytest==7.4.0
pytest-cov==4.1.0

# .github/workflows/python-package.yml
name: Python Package CI/CD

on:
  push:
    branches: [main]
    tags: ['v*']
  pull_request:
    branches: [main]

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.9', '3.10', '3.11', '3.12']
    
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install flake8 pytest black mypy
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
    - name: Lint with flake8
      run: flake8 . --count --show-source --statistics
    - name: Type check with mypy
      run: mypy src/
    - name: Format with black
      run: black --check .
    - name: Test with pytest
      run: pytest tests/ -v --cov=src --cov-report=xml
    - name: Upload coverage
      uses: codecov/codecov-action@v3

  publish:
    needs: test
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/')
    
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    - name: Build package
      run: |
        pip install build
        python -m build
    - name: Publish to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        password: ${{ secrets.PYPI_API_TOKEN }}

# .github/workflows/react-app.yml
name: React App CI/CD

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

jobs:
  test-and-build:
    runs-on: ubuntu-latest
    
    steps:
    - uses: actions/checkout@v4
    - name: Setup Node.js
      uses: actions/setup-node@v4
      with:
        node-version: '20'
        cache: 'npm'
    - name: Install dependencies
      run: npm ci
    - name: Lint
      run: npm run lint
    - name: Run tests
      run: npm test -- --coverage
    - name: Build application
      run: npm run build
    - name: Upload build artifact
      uses: actions/upload-artifact@v4
      with:
        name: build
        path: dist/

  deploy-to-s3:
    needs: test-and-build
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    
    steps:
    - name: Download build artifact
      uses: actions/download-artifact@v4
      with:
        name: build
        path: dist/
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: us-east-1
    - name: Sync to S3
      run: |
        aws s3 sync dist/ s3://my-react-app-bucket/ --delete
    - name: Invalidate CloudFront
      run: |
        aws cloudfront create-invalidation \
          --distribution-id ${{ secrets.CLOUDFRONT_DIST_ID }} \
          --paths "/*"

# .github/workflows/database-migrations.yml
name: Database Migration Pipeline

on:
  pull_request:
    paths:
      - 'migrations/*.sql'
  push:
    branches: [main]
    paths:
      - 'migrations/*.sql'

jobs:
  validate-migrations:
    runs-on: ubuntu-latest
    
    services:
      postgres:
        image: postgres:15
        env:
          POSTGRES_PASSWORD: testpassword
          POSTGRES_DB: testdb
        ports:
          - 5432:5432
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    
    steps:
    - uses: actions/checkout@v4
    - name: Install goose (migration tool)
      run: |
        curl -fsSL https://github.com/pressly/goose/releases/download/v3.15.0/goose_linux_x86_64.tar.gz | tar xz
        sudo mv goose /usr/local/bin/
    - name: Test migrations
      env:
        DATABASE_URL: postgres://postgres:testpassword@localhost:5432/testdb?sslmode=disable
      run: |
        goose -dir migrations postgres "$DATABASE_URL" up
        goose -dir migrations postgres "$DATABASE_URL" down
        goose -dir migrations postgres "$DATABASE_URL" up
    - name: Verify migration integrity
      run: |
        goose -dir migrations postgres "$DATABASE_URL" status

  apply-to-staging:
    needs: validate-migrations
    runs-on: ubuntu-latest
    environment: staging
    if: github.ref == 'refs/heads/main'
    
    steps:
    - uses: actions/checkout@v4
    - name: Install goose
      run: |
        curl -fsSL https://github.com/pressly/goose/releases/download/v3.15.0/goose_linux_x86_64.tar.gz | tar xz
        sudo mv goose /usr/local/bin/
    - name: Apply to staging
      env:
        DATABASE_URL: ${{ secrets.STAGING_DATABASE_URL }}
      run: goose -dir migrations postgres "$DATABASE_URL" up

  apply-to-production:
    needs: apply-to-staging
    runs-on: ubuntu-latest
    environment:
      name: production
      require_approval: true  # Manual approval required
    if: github.ref == 'refs/heads/main'
    
    steps:
    - uses: actions/checkout@v4
    - name: Install goose
      run: |
        curl -fsSL https://github.com/pressly/goose/releases/download/v3.15.0/goose_linux_x86_64.tar.gz | tar xz
        sudo mv goose /usr/local/bin/
    - name: Backup database
      env:
        DATABASE_URL: ${{ secrets.PRODUCTION_DATABASE_URL }}
      run: |
        pg_dump "$DATABASE_URL" > backup_$(date +%Y%m%d_%H%M%S).sql
    - name: Apply to production
      env:
        DATABASE_URL: ${{ secrets.PRODUCTION_DATABASE_URL }}
      run: goose -dir migrations postgres "$DATABASE_URL" up

# Security scanning in CI/CD

# Add this as a separate job
security-scan:
  runs-on: ubuntu-latest
  steps:
  - uses: actions/checkout@v4
  
  # SAST (Static Application Security Testing)
  - name: Run CodeQL Analysis
    uses: github/codeql-action/init@v2
    with:
      languages: python, javascript
  - name: Perform CodeQL Analysis
    uses: github/codeql-action/analyze@v2
  
  # Dependency scanning
  - name: Check Python dependencies
    run: |
      pip install safety
      safety check -r requirements.txt
  
  # Container scanning
  - name: Scan Docker image
    uses: aquasecurity/trivy-action@master
    with:
      image-ref: ${{ env.IMAGE_NAME }}:latest
      format: 'sarif'
      output: 'trivy-results.sarif'
  
  # Secret scanning
  - name: Check for secrets
    uses: gitleaks/gitleaks-action@v2
    env:
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

# Popular GitHub Actions you'll use daily

# Cache dependencies
- name: Cache pip packages
  uses: actions/cache@v3
  with:
    path: ~/.cache/pip
    key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
    restore-keys: ${{ runner.os }}-pip-

# Upload test results
- name: Upload test results
  uses: actions/upload-artifact@v4
  with:
    name: test-results
    path: test-results/
    retention-days: 30

# Slack notifications
- name: Notify Slack
  uses: slackapi/slack-github-action@v1
  with:
    payload: |
      {
        "text": "✅ Deployment successful!",
        "blocks": [{"type": "section", "text": {"type": "mrkdwn", "text": "*${{ github.repository }}*\nEnvironment: production\nStatus: ✅ Success"}}]
      }
  env:
    SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

# Send to Discord
- name: Discord Webhook
  uses: tsickert/discord-webhook@v5
  with:
    webhook-url: ${{ secrets.DISCORD_WEBHOOK }}
    content: "🚀 Deployment to production completed!"

# Wait for deployment
- name: Wait for deployment
  uses: nev7n/wait_for_response@v1
  with:
    url: https://staging.example.com/health
    response: 200
    timeout: 300000
    interval: 10000