Advanced CI/CD Patterns: Matrix Builds, Caching, and Self-Hosted Runners

# matrix-build.yaml
name: Matrix Build

on: [push, pull_request]

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        node-version: [18.x, 20.x, 22.x]
        os: [ubuntu-latest, windows-latest, macos-latest]
        include:
          # Additional combinations
          - node-version: 20.x
            os: ubuntu-latest
            experimental: true
        exclude:
          # Exclude specific combinations
          - node-version: 18.x
            os: windows-latest
      fail-fast: false  # Don't cancel all jobs if one fails
      max-parallel: 4   # Run at most 4 jobs at once
    
    steps:
      - uses: actions/checkout@v4
      
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      
      - name: Install dependencies
        run: npm ci
      
      - name: Run tests
        run: npm test

  # Python matrix example
  python-test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ['3.10', '3.11', '3.12']
        django-version: ['4.2', '5.0']
    
    steps:
      - uses: actions/checkout@v4
      
      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      
      - name: Install dependencies
        run: |
          pip install django==${{ matrix.django-version }}
          pip install -r requirements.txt
      
      - name: Run tests
        run: pytest

# caching.yaml
name: Optimized with Caching

on: [push]

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      # NPM cache
      - name: Cache npm dependencies
        uses: actions/cache@v3
        with:
          path: ~/.npm
          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node-
      
      # pip cache
      - name: Cache pip dependencies
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      
      # Docker layers
      - name: Cache Docker layers
        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-
      
      # Go modules
      - name: Cache Go modules
        uses: actions/cache@v3
        with:
          path: ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-
      
      # Rust cargo
      - name: Cache cargo
        uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
      
      # Build
      - name: Build
        run: |
          npm ci
          npm run build

# Set up self-hosted runner

# 1. Create EC2 instance (t3.medium recommended)
# 2. Install Docker, Python, Node.js, etc.
# 3. Add runner

mkdir actions-runner && cd actions-runner
curl -o actions-runner-linux-x64-2.308.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.308.0/actions-runner-linux-x64-2.308.0.tar.gz
tar xzf ./actions-runner-linux-x64-2.308.0.tar.gz

# Configure (get token from GitHub org/repo settings)
./config.sh --url https://github.com/owner/repo --token TOKEN --labels production,linux,x64

# Run as service
sudo ./svc.sh install
sudo ./svc.sh start

# Check status
sudo ./svc.sh status

# self-hosted-runner.yaml
name: Self-Hosted Runner Jobs

on: [push]

jobs:
  # Use self-hosted runner for large builds
  large-build:
    runs-on: [self-hosted, linux, x64]
    steps:
      - uses: actions/checkout@v4
      - name: Build large application
        run: make build
      - name: Run heavy tests
        run: make test-heavy
  
  # Mix of self-hosted and GitHub-hosted
  hybrid:
    runs-on: ubuntu-latest
    steps:
      - name: Quick test
        run: npm test
      
      - name: Deploy
        runs-on: [self-hosted, production]
        run: ./deploy.sh

# optimization.yaml
name: Optimized Workflow

on: [push]

# Concurrency - cancel in-progress runs on same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

# Conditional job execution
jobs:
  changes:
    runs-on: ubuntu-latest
    outputs:
      backend: ${{ steps.filter.outputs.backend }}
      frontend: ${{ steps.filter.outputs.frontend }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            backend:
              - 'backend/**'
              - 'shared/**'
            frontend:
              - 'frontend/**'
  
  backend-test:
    needs: changes
    if: needs.changes.outputs.backend == 'true'
    runs-on: ubuntu-latest
    steps:
      - run: echo "Running backend tests"
  
  frontend-test:
    needs: changes
    if: needs.changes.outputs.frontend == 'true'
    runs-on: ubuntu-latest
    steps:
      - run: echo "Running frontend tests"

  # Job timeout
  long-job:
    timeout-minutes: 30
    runs-on: ubuntu-latest
    steps:
      - run: ./long-script.sh

  # Retry on failure
  flaky-test:
    runs-on: ubuntu-latest
    steps:
      - name: Flaky test
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 10
          max_attempts: 3
          command: npm run test:flaky

# .github/workflows/reusable-test.yml
name: Reusable Test Workflow

on:
  workflow_call:
    inputs:
      node-version:
        required: false
        type: string
        default: '20.x'
      test-command:
        required: true
        type: string
    secrets:
      API_KEY:
        required: false

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ inputs.node-version }}
      - run: npm ci
      - run: ${{ inputs.test-command }}
        env:
          API_KEY: ${{ secrets.API_KEY }}

# Using the reusable workflow
# .github/workflows/ci.yml
name: CI

on: [push]

jobs:
  test-node-18:
    uses: ./.github/workflows/reusable-test.yml
    with:
      node-version: '18.x'
      test-command: 'npm run test:unit'
  
  test-node-20:
    uses: ./.github/workflows/reusable-test.yml
    with:
      node-version: '20.x'
      test-command: 'npm run test:integration'
    secrets:
      API_KEY: ${{ secrets.API_KEY }}

  test-node-22:
    uses: ./.github/workflows/reusable-test.yml
    with:
      node-version: '22.x'
      test-command: 'npm run test:e2e'

# .github/workflows/production-ready.yml
name: Production-Ready Pipeline

on:
  push:
    branches: [main, develop]
    tags: ['v*']
  pull_request:
    branches: [main]
  workflow_dispatch:

env:
  NODE_VERSION: 20.x
  CACHE_KEY_PREFIX: v1

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  changes:
    runs-on: ubuntu-latest
    outputs:
      app: ${{ steps.filter.outputs.app }}
      api: ${{ steps.filter.outputs.api }}
      infra: ${{ steps.filter.outputs.infra }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            app:
              - 'app/**'
            api:
              - 'api/**'
            infra:
              - 'terraform/**'

  lint-and-test:
    runs-on: ubuntu-latest
    if: needs.changes.outputs.app == 'true' || needs.changes.outputs.api == 'true'
    needs: changes
    steps:
      - uses: actions/checkout@v4
      
      - name: Cache dependencies
        uses: actions/cache@v3
        with:
          path: ~/.npm
          key: ${{ env.CACHE_KEY_PREFIX }}-${{ runner.os }}-npm-${{ hashFiles('package-lock.json') }}
          restore-keys: |
            ${{ env.CACHE_KEY_PREFIX }}-${{ runner.os }}-npm-
      
      - name: Install dependencies
        run: npm ci
      
      - name: Lint
        run: npm run lint
      
      - name: Type check
        run: npm run type-check
      
      - name: Unit tests
        run: npm run test:unit -- --coverage
      
      - name: Upload coverage
        uses: codecov/codecov-action@v3

  integration-tests:
    runs-on: ubuntu-latest
    needs: lint-and-test
    if: github.event_name != 'pull_request'
    steps:
      - uses: actions/checkout@v4
      
      - name: Start services
        run: docker-compose -f docker-compose.test.yml up -d
      
      - name: Wait for services
        run: sleep 10
      
      - name: Run integration tests
        run: npm run test:integration
      
      - name: Stop services
        run: docker-compose -f docker-compose.test.yml down

  e2e-tests:
    runs-on: ubuntu-latest
    needs: integration-tests
    if: github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v4
      
      - name: Build and run
        run: |
          docker build -t myapp:e2e .
          docker run -d -p 3000:3000 --name e2e-app myapp:e2e
      
      - name: Run Playwright tests
        uses: microsoft/playwright-github-action@v1
      
      - name: Run E2E tests
        run: npm run test:e2e
      
      - name: Upload screenshots
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: playwright-screenshots
          path: test-results/

  performance-tests:
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v4
      
      - name: Run k6 load tests
        uses: grafana/k6-action@v0.3.0
        with:
          filename: tests/performance/load-test.js
          flags: --out json=results.json
      
      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: performance-results
          path: results.json

  security-scan:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Run Trivy
        uses: aquasecurity/trivy-action@master
        with:
          scan-type: 'fs'
          scan-ref: '.'
          format: 'sarif'
          output: 'trivy-results.sarif'
      
      - name: Run CodeQL
        uses: github/codeql-action/init@v2
        with:
          languages: javascript, typescript
      
      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v2

  build:
    runs-on: ubuntu-latest
    needs: [lint-and-test, security-scan]
    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v4
      
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      
      - name: Log in to registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      
      - name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: |
            ghcr.io/${{ github.repository }}:${{ github.sha }}
            ghcr.io/${{ github.repository }}:latest
          cache-from: type=gha
          cache-to: type=gha,mode=max

  deploy-staging:
    runs-on: ubuntu-latest
    needs: build
    environment: staging
    if: github.ref == 'refs/heads/main'
    steps:
      - name: Deploy to staging
        run: |
          kubectl set image deployment/myapp myapp=ghcr.io/${{ github.repository }}:${{ github.sha }}
          kubectl rollout status deployment/myapp

  deploy-production:
    runs-on: ubuntu-latest
    needs: [deploy-staging, performance-tests, e2e-tests]
    environment:
      name: production
      url: https://myapp.example.com
    if: github.ref == 'refs/heads/main'
    steps:
      - name: Approve deployment
        uses: trstringer/manual-approval@v1
        with:
          secret: ${{ github.TOKEN }}
          approvers: admin,lead-engineer
      
      - name: Deploy to production
        run: |
          kubectl set image deployment/myapp myapp=ghcr.io/${{ github.repository }}:${{ github.sha }}
          kubectl rollout status deployment/myapp

  notify:
    runs-on: ubuntu-latest
    needs: [deploy-staging, deploy-production]
    if: always()
    steps:
      - name: Slack notification
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "Pipeline ${{ job.status }}: ${{ github.repository }}"
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}