Scaling Microservices with Kubernetes and Docker

# Modular monolith structure
my_app/
├── modules/
│   ├── authentication/
│   │   ├── __init__.py
│   │   ├── service.py
│   │   └── models.py
│   ├── payments/
│   │   ├── __init__.py
│   │   ├── service.py
│   │   └── models.py
│   └── inventory/
│       ├── __init__.py
│       ├── service.py
│       └── models.py

# Install Minikube
brew install minikube

# Start cluster
minikube start --cpus=4 --memory=8192

# Enable ingress
minikube addons enable ingress

# Verify
kubectl cluster-info

# deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api-service
  labels:
    app: api
    version: v1
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0  # Zero downtime
  selector:
    matchLabels:
      app: api
  template:
    metadata:
      labels:
        app: api
        version: v1
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "3000"
    spec:
      containers:
      - name: api
        image: your-registry/api:v1.2.3
        ports:
        - containerPort: 3000
          name: http
        env:
        - name: NODE_ENV
          value: "production"
        - name: DB_PASSWORD
          valueFrom:
            secretKeyRef:
              name: db-credentials
              key: password
        resources:
          requests:
            memory: "256Mi"
            cpu: "250m"
          limits:
            memory: "512Mi"
            cpu: "500m"
        livenessProbe:
          httpGet:
            path: /health
            port: 3000
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /ready
            port: 3000
          initialDelaySeconds: 5
          periodSeconds: 5
---
apiVersion: v1
kind: Service
metadata:
  name: api-service
spec:
  selector:
    app: api
  ports:
  - protocol: TCP
    port: 80
    targetPort: 3000
  type: ClusterIP

# Set up monitoring and observability
- Deploy Prometheus and Grafana
- Implement distributed tracing with Jaeger
- Establish service mesh (Istio/Linkerd)

# Identify boundaries
- Map domain models
- Analyze dependencies
- Create service boundaries

# Extract a non-critical service first
Selected: Notification Service
Reason: 
  - Low dependency on other components
  - Well-defined boundary
  - Non-critical to core business
  - High independent scalability needs

# Service A publishes events when data changes
@app.route('/orders', methods=['POST'])
def create_order():
    order = Order.create(request.json)
    
    # Emit event for other services
    event_bus.publish('order.created', {
        'order_id': order.id,
        'customer_id': order.customer_id,
        'total': order.total
    })
    
    return jsonify(order)

# Service B listens and maintains its own copy
@event_bus.subscribe('order.created')
def handle_order_created(event):
    # Update local read model
    local_db.orders.insert({
        'id': event['order_id'],
        'customer_id': event['customer_id'],
        # ... only fields needed by this service
    })

# Build stage
FROM node:18-alpine AS builder
WORKDIR /app

# Copy dependency files first (better caching)
COPY package*.json ./
RUN npm ci --only=production

# Copy source code
COPY . .

# Build if needed (e.g., TypeScript)
RUN npm run build

# Production stage
FROM node:18-alpine
WORKDIR /app

# Security: Run as non-root user
RUN addgroup -g 1001 -S nodejs
RUN adduser -S nodejs -u 1001

# Copy only necessary files from builder
COPY --from=builder --chown=nodejs:nodejs /app/node_modules ./node_modules
COPY --from=builder --chown=nodejs:nodejs /app/dist ./dist
COPY --from=builder --chown=nodejs:nodejs /app/package.json ./

# Switch to non-root user
USER nodejs

# Health check
HEALTHCHECK --interval=30s --timeout=3s --start-period=40s \
  CMD node healthcheck.js || exit 1

EXPOSE 3000
CMD ["node", "dist/server.js"]

# docker-compose.yml
version: '3.8'

services:
  api-gateway:
    build: ./gateway
    ports:
      - "8080:8080"
    environment:
      - AUTH_SERVICE_URL=http://auth:3001
      - ORDER_SERVICE_URL=http://orders:3002
    depends_on:
      - auth
      - orders

  auth:
    build: ./services/auth
    environment:
      - DB_HOST=postgres
      - REDIS_HOST=redis
    depends_on:
      - postgres
      - redis

  orders:
    build: ./services/orders
    environment:
      - DB_HOST=postgres
      - KAFKA_BROKERS=kafka:9092
    depends_on:
      - postgres
      - kafka

  postgres:
    image: postgres:15-alpine
    environment:
      - POSTGRES_PASSWORD=secret
    volumes:
      - postgres_data:/var/lib/postgresql/data

  redis:
    image: redis:7-alpine
    volumes:
      - redis_data:/data

  kafka:
    image: confluentinc/cp-kafka:latest
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092

volumes:
  postgres_data:
  redis_data:

apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: api-service
spec:
  hosts:
  - api-service
  http:
  - match:
    - headers:
        user-type:
          exact: beta-tester
    route:
    - destination:
        host: api-service
        subset: v2
  - route:
    - destination:
        host: api-service
        subset: v1
      weight: 90
    - destination:
        host: api-service
        subset: v2
      weight: 10  # 10% traffic to new version
---
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: api-service
spec:
  host: api-service
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 100
      http:
        http1MaxPendingRequests: 50
        http2MaxRequests: 100
    outlierDetection:
      consecutive5xxErrors: 5
      interval: 30s
      baseEjectionTime: 30s
  subsets:
  - name: v1
    labels:
      version: v1
  - name: v2
    labels:
      version: v2

apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: order-service-circuit-breaker
spec:
  host: order-service
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 1
      http:
        http1MaxPendingRequests: 1
        maxRequestsPerConnection: 1
    outlierDetection:
      consecutiveErrors: 1
      interval: 1s
      baseEjectionTime: 3m
      maxEjectionPercent: 100

# prometheus-config.yaml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

# Alerting configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      - alertmanager:9093

rule_files:
  - /etc/prometheus/alerts.yml

scrape_configs:
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
    - role: pod
    relabel_configs:
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
      action: keep
      regex: true
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
      action: replace
      target_label: __metrics_path__
      regex: (.+)
    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
      action: replace
      regex: ([^:]+)(?::\d+)?;(\d+)
      replacement: $1:$2
      target_label: __address__

// Node.js service with Prometheus metrics
const express = require('express');
const promClient = require('prom-client');

const app = express();

// Create metrics
const httpRequestDuration = new promClient.Histogram({
  name: 'http_request_duration_seconds',
  help: 'Duration of HTTP requests in seconds',
  labelNames: ['method', 'route', 'status_code']
});

const activeConnections = new promClient.Gauge({
  name: 'active_connections',
  help: 'Number of active connections'
});

// Middleware to track requests
app.use((req, res, next) => {
  const start = Date.now();
  activeConnections.inc();
  
  res.on('finish', () => {
    const duration = (Date.now() - start) / 1000;
    httpRequestDuration
      .labels(req.method, req.route?.path || req.path, res.statusCode)
      .observe(duration);
    activeConnections.dec();
  });
  
  next();
});

// Metrics endpoint
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', promClient.register.contentType);
  res.end(await promClient.register.metrics());
});

name: Deploy Microservice
on:
  push:
    branches: [main]
    paths:
      - 'services/api/**'

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      
      - name: Run Unit Tests
        run: |
          cd services/api
          npm ci
          npm test
      
      - name: Run Integration Tests
        run: |
          docker-compose -f docker-compose.test.yml up -d
          npm run test:integration
          docker-compose -f docker-compose.test.yml down

  build:
    needs: test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      
      - name: Build and Push Docker Image
        run: |
          echo ${{ secrets.REGISTRY_PASSWORD }} | docker login -u ${{ secrets.REGISTRY_USER }} --password-stdin
          docker build -t ${{ secrets.REGISTRY }}/api:${{ github.sha }} ./services/api
          docker build -t ${{ secrets.REGISTRY }}/api:latest ./services/api
          docker push ${{ secrets.REGISTRY }}/api:${{ github.sha }}
          docker push ${{ secrets.REGISTRY }}/api:latest
      
      - name: Scan Image for Vulnerabilities
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ secrets.REGISTRY }}/api:${{ github.sha }}
          format: 'sarif'
          output: 'trivy-results.sarif'

  deploy:
    needs: build
    runs-on: ubuntu-latest
    steps:
      - name: Deploy to Kubernetes
        run: |
          echo ${{ secrets.KUBECONFIG }} | base64 -d > kubeconfig
          export KUBECONFIG=./kubeconfig
          
          kubectl set image deployment/api-service \
            api=${{ secrets.REGISTRY }}/api:${{ github.sha }} \
            --record
          
          kubectl rollout status deployment/api-service
          
          # Rollback if health checks fail
          if ! kubectl rollout status deployment/api-service; then
            kubectl rollout undo deployment/api-service
            exit 1
          fi

# Each service with its own database connection
class OrderService:
    def __init__(self):
        self.db = PostgreSQLConnection(
            host=os.getenv('ORDER_DB_HOST'),
            database='orders'
        )

class InventoryService:
    def __init__(self):
        # Different database type based on needs
        self.db = MongoDBConnection(
            host=os.getenv('INVENTORY_DB_HOST'),
            database='inventory'
        )

// Saga orchestrator example
class OrderSaga {
  async execute(orderData) {
    const saga = new Saga();
    
    // Step 1: Reserve inventory
    saga.addStep({
      action: async () => {
        const result = await inventoryService.reserve(orderData.items);
        return { reservationId: result.id };
      },
      compensate: async (context) => {
        await inventoryService.release(context.reservationId);
      }
    });
    
    // Step 2: Process payment
    saga.addStep({
      action: async () => {
        const result = await paymentService.charge({
          amount: orderData.total,
          customerId: orderData.customerId
        });
        return { paymentId: result.id };
      },
      compensate: async (context) => {
        await paymentService.refund(context.paymentId);
      }
    });
    
    // Step 3: Create order
    saga.addStep({
      action: async (context) => {
        const order = await orderService.create({
          ...orderData,
          reservationId: context.reservationId,
          paymentId: context.paymentId
        });
        return { orderId: order.id };
      },
      compensate: async (context) => {
        await orderService.cancel(context.orderId);
      }
    });
    
    try {
      const result = await saga.execute();
      await this.notifyOrderCreated(result.orderId);
      return result;
    } catch (error) {
      // Saga automatically runs compensating transactions
      logger.error('Order saga failed', error);
      throw error;
    }
  }
}

// Consumer test (Order Service)
const { Pact } = require('@pact-foundation/pact');

describe('Order Service - Inventory Contract', () => {
  const provider = new Pact({
    consumer: 'OrderService',
    provider: 'InventoryService'
  });

  it('can check inventory availability', async () => {
    await provider.addInteraction({
      state: 'product 123 has stock',
      uponReceiving: 'a request to check availability',
      withRequest: {
        method: 'GET',
        path: '/inventory/123',
        headers: { 'Accept': 'application/json' }
      },
      willRespondWith: {
        status: 200,
        body: {
          productId: 123,
          available: 50
        }
      }
    });

    const response = await orderService.checkInventory(123);
    expect(response.available).toBe(50);
  });
});

// Provider verification (Inventory Service)
const { Verifier } = require('@pact-foundation/pact');

describe('Inventory Service Pact Verification', () => {
  it('validates consumer expectations', async () => {
    await new Verifier({
      provider: 'InventoryService',
      providerBaseUrl: 'http://localhost:3000',
      pactUrls: ['./pacts/orderservice-inventoryservice.json']
    }).verifyProvider();
  });
});

# Network delay experiment
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: network-delay
spec:
  action: delay
  mode: one
  selector:
    namespaces:
      - production
    labelSelectors:
      app: api-service
  delay:
    latency: "10s"
    correlation: "100"
    jitter: "0ms"
  duration: "30s"
---
# Pod failure experiment
apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: pod-failure
spec:
  action: pod-failure
  mode: fixed
  value: "1"
  selector:
    namespaces:
      - production
    labelSelectors:
      app: api-service
  duration: "60s"

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: api-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: api-service
  minReplicas: 2
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80
  - type: Pods
    pods:
      metric:
        name: http_requests_per_second
      target:
        type: AverageValue
        averageValue: "1000"
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300  # Wait 5 min before scaling down
      policies:
      - type: Percent
        value: 50
        periodSeconds: 60  # Max 50% reduction per minute
    scaleUp:
      stabilizationWindowSeconds: 0
      policies:
      - type: Percent
        value: 100
        periodSeconds: 15  # Aggressive scale up

# Based on 95th percentile usage over 30 days
import numpy as np

cpu_usage_samples = get_metrics('cpu_usage', days=30)
memory_usage_samples = get_metrics('memory_usage', days=30)

cpu_request = np.percentile(cpu_usage_samples, 50)  # P50
cpu_limit = np.percentile(cpu_usage_samples, 95)     # P95

memory_request = np.percentile(memory_usage_samples, 50)
memory_limit = np.percentile(memory_usage_samples, 95)

print(f"""
Recommended resources:
  requests:
    cpu: {cpu_request}m
    memory: {memory_request}Mi
  limits:
    cpu: {cpu_limit}m
    memory: {memory_limit}Mi
""")

# Analyze actual usage
kubectl top pods --containers=true

# Adjust resources to match usage
# Before: 1000m CPU requested, 200m used → 80% waste
# After: 300m CPU requested, 200m used → 33% headroom

apiVersion: v1
kind: ConfigMap
metadata:
  name: cluster-autoscaler
data:
  config.yaml: |
    minNodes: 3
    maxNodes: 20
    scaleDownDelayAfterAdd: 10m
    scaleDownUnneededTime: 10m
    scaleDownUtilizationThreshold: 0.5

apiVersion: v1
kind: Node
metadata:
  labels:
    node.kubernetes.io/instance-type: spot
spec:
  taints:
  - effect: NoSchedule
    key: spot
    value: "true"
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: batch-processor
spec:
  template:
    spec:
      tolerations:
      - key: spot
        operator: Equal
        value: "true"
        effect: NoSchedule
      nodeSelector:
        node.kubernetes.io/instance-type: spot

# Diagnose
kubectl describe pod <pod-name>

# Common causes:
# 1. Wrong image name/tag
# 2. Missing registry credentials
# 3. Network issues

# Fix: Create image pull secret
kubectl create secret docker-registry regcred \
  --docker-server=myregistry.io \
  --docker-username=user \
  --docker-password=pass

# Check logs
kubectl logs <pod-name> --previous

# Common causes:
# 1. Application error
# 2. Missing environment variables
# 3. Failed health checks

# Debug with shell
kubectl run debug --rm -it --image=busybox -- /bin/sh

# Test service connectivity
kubectl run curl --image=curlimages/curl -it --rm -- \
  curl http://api-service.default.svc.cluster.local

# Check DNS resolution
kubectl run dnsutils --image=tutum/dnsutils -it --rm -- \
  nslookup api-service

# Network policy debugging
kubectl get networkpolicies
kubectl describe networkpolicy <policy-name>

# CPU/Memory usage
kubectl top nodes
kubectl top pods

# Detailed resource analysis
kubectl describe node <node-name>

# Application profiling
kubectl port-forward pod/<pod-name> 9229:9229
# Then connect profiler to localhost:9229

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: api-network-policy
spec:
  podSelector:
    matchLabels:
      app: api
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: gateway
    ports:
    - protocol: TCP
      port: 3000
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: database
    ports:
    - protocol: TCP
      port: 5432

apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: service-deployer
rules:
- apiGroups: ["apps"]
  resources: ["deployments"]
  verbs: ["get", "list", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: deploy-role-binding
subjects:
- kind: ServiceAccount
  name: ci-cd-bot
roleRef:
  kind: Role
  name: service-deployer
  apiGroup: rbac.authorization.k8s.io

# Using External Secrets Operator
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: app-secrets
spec:
  refreshInterval: 1h
  secretStoreRef:
    name: vault-backend
    kind: SecretStore
  target:
    name: app-secrets
    creationPolicy: Owner
  data:
  - secretKey: database-password
    remoteRef:
      key: secret/data/database
      property: password

# Trivy scan in CI/CD
trivy image --severity HIGH,CRITICAL myapp:latest

# Fail pipeline if vulnerabilities found
trivy image --exit-code 1 --severity CRITICAL myapp:latest

apiVersion: v1
kind: Namespace
metadata:
  name: production
  labels:
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/warn: restricted