# ❌ Minimal CI
name: CI
on: [push]
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: npm install
      - run: npm test

# ✅ Comprehensive CI
name: CI Pipeline
on:
  pull_request:
    branches: [main, develop]
  push:
    branches: [main]
 
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
 
jobs:
  # Fast feedback first
  quick-checks:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v4
      
      - uses: actions/setup-node@v4
        with:
          node-version: '18'
          cache: 'npm'
      
      - run: npm ci
      
      - name: Lint
        run: npm run lint
      
      - name: Type Check
        run: npm run type-check
      
      - name: Format Check
        run: npm run format:check
 
  # Unit tests with coverage
  test-unit:
    needs: quick-checks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: '18'
          cache: 'npm'
      
      - run: npm ci
      
      - name: Run Unit Tests
        run: npm run test:unit -- --coverage
      
      - name: Upload Coverage
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_coverage_decreases: true
 
  # Integration tests
  test-integration:
    needs: quick-checks
    runs-on: ubuntu-latest
    services:
      postgres:
        image: postgres:15
        env:
          POSTGRES_PASSWORD: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
      redis:
        image: redis:7
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 6379:6379
    
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
      
      - run: npm ci
      
      - name: Run Migrations
        run: npm run db:migrate
      
      - name: Run Integration Tests
        run: npm run test:integration
        env:
          DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test
          REDIS_URL: redis://localhost:6379
 
  # E2E tests
  test-e2e:
    needs: [test-unit, test-integration]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
      
      - run: npm ci
      
      - name: Install Playwright
        run: npx playwright install --with-deps
      
      - name: Build Application
        run: npm run build
      
      - name: Start Server
        run: npm run start &
      
      - name: Wait for Server
        run: npx wait-on http://localhost:3000
      
      - name: Run E2E Tests
        run: npm run test:e2e
      
      - name: Upload Test Results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: playwright-report
          path: playwright-report/
 
  # Security scanning
  security-scan:
    needs: quick-checks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
      
      - run: npm ci
      
      - name: Run Snyk
        uses: snyk/actions/node@master
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        continue-on-error: true
      
      - name: Run npm audit
        run: npm audit --audit-level=high
      
      - name: Check for Secrets
        uses: trufflesecurity/trufflehog@main
        with:
          extra_args: --only-verified
 
  # Build artifacts
  build:
    needs: [test-unit, test-integration, security-scan]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
      
      - run: npm ci
      - run: npm run build
      
      - name: Upload Build Artifact
        uses: actions/upload-artifact@v4
        with:
          name: build
          path: dist/
 
  # Quality gate
  quality-gate:
    needs: [quick-checks, test-unit, test-integration, test-e2e, security-scan, build]
    runs-on: ubuntu-latest
    steps:
      - name: All Checks Passed
        run: echo "✅ All quality gates passed"

# .github/workflows/deploy.yml
 
name: Deploy Pipeline
 
on:
  push:
    branches: [main]
  workflow_dispatch:
    inputs:
      environment:
        description: 'Environment to deploy to'
        required: true
        default: 'staging'
        type: choice
        options:
          - staging
          - production
 
env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}
 
jobs:
  # Build and push Docker image
  build-and-push:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
    
    steps:
      - uses: actions/checkout@v4
      
      - name: Log in to Container Registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      
      - name: Extract Metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            type=sha,prefix=
            type=ref,event=branch
            type=semver,pattern={{version}}
      
      - name: Build and Push
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
 
  # Deploy to Staging (automatic)
  deploy-staging:
    needs: build-and-push
    runs-on: ubuntu-latest
    environment: staging
    
    steps:
      - uses: actions/checkout@v4
      
      - name: Deploy to Staging
        uses: azure/k8s-deploy@v4
        with:
          namespace: staging
          manifests: |
            k8s/staging/deployment.yaml
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
      
      - name: Run Smoke Tests
        run: |
          npm run test:smoke
        env:
          BASE_URL: https://staging.example.com
      
      - name: Notify Slack
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "✅ Staging deployment successful",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "*Staging Deployment Successful*\nCommit: ${{ github.sha }}\nDeployed by: ${{ github.actor }}"
                  }
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_STAGING_WEBHOOK }}
 
  # Deploy to Production (with approval)
  deploy-production:
    needs: deploy-staging
    runs-on: ubuntu-latest
    environment:
      name: production
      url: https://example.com
    if: github.ref == 'refs/heads/main'
    
    steps:
      - uses: actions/checkout@v4
      
      - name: Wait for Staging Verification
        run: |
          echo "⏳ Waiting 15 minutes for staging verification..."
          sleep 900
      
      - name: Deploy to Production
        uses: azure/k8s-deploy@v4
        with:
          namespace: production
          manifests: |
            k8s/production/deployment.yaml
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
      
      - name: Run Production Smoke Tests
        run: |
          npm run test:smoke
        env:
          BASE_URL: https://api.example.com
      
      - name: Health Check
        run: |
          for i in {1..10}; do
            if curl -f https://api.example.com/health; then
              echo "✅ Health check passed"
              exit 0
            fi
            echo "⏳ Waiting for health check... (attempt $i)"
            sleep 30
          done
          echo "❌ Health check failed after 10 attempts"
          exit 1
      
      - name: Notify Slack
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "🚀 Production deployment successful",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "*Production Deployment Successful*\nCommit: ${{ github.sha }}\nDeployed by: ${{ github.actor }}\n<https://github.com/${{ github.repository }}/commit/${{ github.sha }}|View Commit>"
                  }
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PROD_WEBHOOK }}
 
  # Rollback on failure
  rollback:
    needs: deploy-production
    if: failure()
    runs-on: ubuntu-latest
    
    steps:
      - name: Get Previous Stable Version
        id: previous
        run: |
          PREVIOUS_SHA=$(git rev-parse HEAD~1)
          echo "previous_sha=$PREVIOUS_SHA" >> $GITHUB_OUTPUT
      
      - name: Rollback Deployment
        uses: azure/k8s-deploy@v4
        with:
          namespace: production
          manifests: |
            k8s/production/deployment.yaml
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.previous.outputs.previous_sha }}
      
      - name: Notify Rollback
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "⚠️ Production rollback triggered",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "*Production Rollback*\nDeployment failed. Rolled back to: ${{ steps.previous.outputs.previous_sha }}\n@channel"
                  }
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PROD_WEBHOOK }}

# k8s/blue-green-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app-blue
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
      version: blue
  template:
    metadata:
      labels:
        app: myapp
        version: blue
    spec:
      containers:
        - name: app
          image: myapp:latest
---
apiVersion: v1
kind: Service
metadata:
  name: app-active
spec:
  selector:
    app: myapp
    version: blue  # Switch to 'green' to activate new version
  ports:
    - port: 80
      targetPort: 3000

#!/bin/bash
# scripts/blue-green-deploy
 
set -e
 
CURRENT_VERSION=$(kubectl get service app-active -o jsonpath='{.spec.selector.version}')
NEW_VERSION=$([ "$CURRENT_VERSION" = "blue" ] && echo "green" || echo "blue")
 
echo "🔄 Current version: $CURRENT_VERSION"
echo "🔄 Deploying new version: $NEW_VERSION"
 
# Deploy new version
kubectl set image deployment/app-$NEW_VERSION app=myapp:$IMAGE_TAG
 
# Wait for rollout
kubectl rollout status deployment/app-$NEW_VERSION
 
# Run smoke tests
./scripts/smoke-tests --version $NEW_VERSION
 
# Switch traffic
kubectl patch service app-active -p "{\"spec\":{\"selector\":{\"version\":\"$NEW_VERSION\"}}}"
 
echo "✅ Traffic switched to $NEW_VERSION"
 
# Keep old version running for 1 hour, then scale down
sleep 3600
kubectl scale deployment/app-$CURRENT_VERSION --replicas=0

# Using Istio for traffic splitting
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  name: myapp-canary
spec:
  hosts:
    - myapp.example.com
  http:
    - route:
        - destination:
            host: myapp
            subset: stable
          weight: 90
        - destination:
            host: myapp
            subset: canary
          weight: 10

# .github/workflows/canary-deploy.yml
name: Canary Deployment
 
on:
  push:
    branches: [main]
 
jobs:
  canary:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Deploy Canary (5% traffic)
        run: |
          kubectl set image deployment/myapp myapp=myapp:$IMAGE_TAG
          istio-ctl traffic-set myapp --canary 5
      
      - name: Monitor Metrics (5 minutes)
        run: |
          ./scripts/monitor-canary.sh --duration 300
      
      - name: Analyze Results
        id: analysis
        run: |
          ERROR_RATE=$(curl -s grafana/api/error-rate)
          LATENCY=$(curl -s grafana/api/p95-latency)
          
          if (( $(echo "$ERROR_RATE > 1" | bc -l) )); then
            echo "success=false" >> $GITHUB_OUTPUT
          elif (( $(echo "$LATENCY > 500" | bc -l) )); then
            echo "success=false" >> $GITHUB_OUTPUT
          else
            echo "success=true" >> $GITHUB_OUTPUT
          fi
      
      - name: Expand Canary (25% traffic)
        if: steps.analysis.outputs.success == 'true'
        run: istio-ctl traffic-set myapp --canary 25
      
      - name: Monitor Metrics (10 minutes)
        if: steps.analysis.outputs.success == 'true'
        run: ./scripts/monitor-canary.sh --duration 600
      
      - name: Full Rollout (100% traffic)
        if: success()
        run: istio-ctl traffic-set myapp --canary 100
      
      - name: Rollback
        if: failure()
        run: |
          istio-ctl traffic-set myapp --canary 0
          kubectl rollout undo deployment/myapp

// Feature flag configuration
interface FeatureFlags {
  newCheckout: {
    enabled: boolean;
    rolloutPercentage: number;
    allowedUserIds?: string[];
  };
  darkMode: {
    enabled: boolean;
    rolloutPercentage: number;
  };
}
 
// Usage in code
import { flags } from './feature-flags';
 
if (flags.newCheckout.enabled && Math.random() < flags.newCheckout.rolloutPercentage) {
  return <NewCheckout />;
}
return <LegacyCheckout />;

# .github/workflows/feature-flag-check.yml
name: Feature Flag Check
 
on:
  pull_request:
 
jobs:
  check-flags:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Check for Long-Running Flags
        run: |
          OLD_FLAGS=$(cat .github/feature-flags.json)
          NEW_FLAGS=$(cat feature-flags.json)
          
          # Warn if flag is older than 30 days
          node scripts/check-flag-age.js "$OLD_FLAGS" "$NEW_FLAGS"
      
      - name: Validate Flag Cleanup Plan
        run: |
          # Ensure new flags have cleanup tickets
          node scripts/validate-flag-tickets.js

# .github/workflows/db-migrations.yml
name: Database Migrations
 
on:
  push:
    branches: [main]
    paths:
      - 'migrations/**'
 
jobs:
  migrate:
    runs-on: ubuntu-latest
    environment: staging
    
    steps:
      - uses: actions/checkout@v4
      
      - name: Run Migrations (Dry Run)
        run: npm run db:migrate -- --dry-run
      
      - name: Backup Database
        run: |
          pg_dump $DATABASE_URL > backup-$(date +%Y%m%d-%H%M%S).sql
      
      - name: Run Migrations
        run: npm run db:migrate
      
      - name: Verify Schema
        run: npm run db:verify
      
      - name: Run Post-Migration Tests
        run: npm run test:migrations
      
      - name: Rollback Plan
        if: failure()
        run: |
          npm run db:rollback
          # Notify team
          ./scripts/notify-rollback.sh

-- ✅ Good: Expand and contract pattern
-- Step 1: Add new column (deploy this first)
ALTER TABLE users ADD COLUMN new_email VARCHAR(255);
 
-- Step 2: Backfill data (run as separate job)
UPDATE users SET new_email = email WHERE new_email IS NULL;
 
-- Step 3: Update application to write to both columns
-- (deploy application change)
 
-- Step 4: Update application to read from new column
-- (deploy application change)
 
-- Step 5: Remove old column (deploy this last)
ALTER TABLE users DROP COLUMN email;
 
-- ❌ Bad: Breaking change in single migration
ALTER TABLE users RENAME COLUMN email TO new_email;
-- This breaks any running application instance

// scripts/pipeline-metrics.ts
interface PipelineMetrics {
  averageBuildTime: number;
  successRate: number;
  deploymentFrequency: number;
  meanTimeToRecovery: number;
  changeFailureRate: number;
}
 
async function collectMetrics(): Promise<PipelineMetrics> {
  const builds = await github.actions.listWorkflowRuns();
  
  return {
    averageBuildTime: calculateAverage(builds.map(b => b.duration)),
    successRate: builds.filter(b => b.conclusion === 'success').length / builds.length,
    deploymentFrequency: countDeploymentsLastWeek(),
    meanTimeToRecovery: calculateMTTR(),
    changeFailureRate: calculateFailureRate(),
  };
}
 
// DORA Metrics
const DORA = {
  // Deployment Frequency: How often does the organization release to production?
  deploymentFrequency: 'Elite: On-demand, Multiple/day',
  
  // Lead Time for Changes: How long does it take for commits to get to production?
  leadTime: 'Elite: <1 hour',
  
  // Mean Time to Recovery (MTTR): How long to restore service after failure?
  MTTR: 'Elite: <1 hour',
  
  // Change Failure Rate: What percentage of changes cause failures?
  changeFailureRate: 'Elite: 0-15%',
};

# .github/workflows/pipeline-alerts.yml
name: Pipeline Health Alerts
 
on:
  workflow_run:
    workflows: ["Deploy Pipeline"]
    types: [completed]
 
jobs:
  alert-on-failure:
    runs-on: ubuntu-latest
    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
    
    steps:
      - name: Send PagerDuty Alert
        run: |
          curl -X POST https://events.pagerduty.com/v2/enqueue \
            -H "Content-Type: application/json" \
            -d '{
              "routing_key": "${{ secrets.PAGERDUTY_KEY }}",
              "event_action": "trigger",
              "payload": {
                "summary": "Production deployment failed",
                "severity": "critical",
                "source": "github-actions"
              }
            }'
      
      - name: Create Incident Ticket
        run: |
          curl -X POST https://api.linear.app/graphql \
            -H "Authorization: ${{ secrets.LINEAR_API_KEY }}" \
            -d '{
              "query": "mutation { issueCreate(input: { title: \"Production deployment failed\", teamId: \"team_123\" }) { success } }"
            }'
 
  alert-on-slow-build:
    runs-on: ubuntu-latest
    if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.run_duration_ms > 1800000 }}
    
    steps:
      - name: Notify Slow Build
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "⚠️ Build taking longer than 30 minutes",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "*Slow Build Alert*\nBuild time: ${{ github.event.workflow_run.run_duration_ms / 60000 }} minutes\nWorkflow: ${{ github.event.workflow_run.name }}"
                  }
                }
              ]
            }

# On-Call Runbook
 
## When Deployment Fails
 
1. Check the deployment logs: [Link to logs]
2. Review the diff: [Link to comparison]
3. Run smoke tests manually: `./scripts/smoke-tests`
4. If critical, rollback: `./scripts/rollback`
5. Create incident ticket
6. Notify team in #incidents
 
## Common Issues
 
### Database Migration Failed
- Check migration logs
- Verify database connection
- Run migration manually with verbose output
- If stuck, rollback and notify DBA
 
### Health Check Failed
- Check application logs
- Verify external dependencies
- Check resource utilization (CPU, memory)
- Restart pods if needed
 
### Smoke Test Failed
- Review failing tests
- Check if test is flaky
- Verify external API dependencies
- Consider rollback if production impact

# Quarantine flaky tests
- name: Run Stable Tests
  run: npm run test:stable
 
- name: Run Flaky Tests (allowed to fail)
  run: npm run test:flaky
  continue-on-error: true
 
# In package.json
{
  "scripts": {
    "test:stable": "vitest --exclude='**/*.flaky.test.ts'",
    "test:flaky": "vitest --only='**/*.flaky.test.ts'"
  }
}

# Parallel test execution
test-matrix:
  strategy:
    matrix:
      node: [16, 18, 20]
      os: [ubuntu-latest, macos-latest]
  
  runs-on: ${{ matrix.os }}
  
  steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-node@v4
      with:
        node-version: ${{ matrix.node }}
    - run: npm test

#!/bin/bash
# scripts/drift-check
 
echo "🔍 Checking for environment drift..."
 
STAGING_CONFIG=$(kubectl get configmap app-config -n staging -o json)
PROD_CONFIG=$(kubectl get configmap app-config -n production -o json)
 
DIFF=$(diff <(echo "$STAGING_CONFIG") <(echo "$PROD_CONFIG"))
 
if [ -n "$DIFF" ]; then
  echo "⚠️ Configuration drift detected:"
  echo "$DIFF"
  exit 1
else
  echo "✅ No drift detected"
fi