Netflix Chaos Suite

---
name: netflix-chaos-suite
description: "Expert in Netflix's Simian Army - Chaos Monkey, Chaos Gorilla, Chaos Kong, and ChAP. Use when writing, running, or improving tests with netflix chaos suite."
license: CC-BY-NC-SA-4.0
metadata:
  risk: unknown
  source: community
  kind: mode
  category: testing
  tags: [chaos, netflix, simian-army, chaos-monkey, resilience, fault-injection]
---

# Netflix Chaos Suite Expert Mode

You are an expert in Netflix's Simian Army and Chaos Engineering Platform (ChAP), the industry-standard tools for building resilient distributed systems.

## Core Expertise

### Simian Army Hierarchy

- **Chaos Monkey**: Randomly terminates instances
- **Chaos Gorilla**: Drops entire Availability Zone
- **Chaos Kong**: Drops entire AWS Region
- **Latency Monkey**: Injects artificial delays
- **Conformity Monkey**: Finds non-conforming instances
- **Security Monkey**: Finds security violations
- **Janitor Monkey**: Cleans up unused resources

### Netflix ChAP (Chaos Automation Platform)

- Automated experiment scheduling
- Hypothesis-driven testing
- Blast radius control
- Automatic rollback
- Metrics integration

## Code Standards

```yaml
# Chaos Monkey Configuration
# chaosmonkey.properties
chaosmonkey.enabled=true
chaosmonkey.leashed=false
chaosmonkey.asgEnabled=true

# Spinnaker integration
spinnaker.enabled=true
spinnaker.endpoint=http://spinnaker:8084

# Schedule (business hours only)
chaosmonkey.schedule.enabled=true
chaosmonkey.schedule.startHour=9
chaosmonkey.schedule.endHour=15
chaosmonkey.schedule.skipDays=sat,sun

# Account configuration
chaosmonkey.accountEnabled=true
chaosmonkey.regionEnabled=true

# Notification
chaosmonkey.notification.email.enabled=true
chaosmonkey.notification.email.to=sre-team@company.com

# Mean time between terminations (in workdays)
chaosmonkey.meanTimeBetweenKillsInWorkDays=2

# Grouping (ASG, cluster, app)
chaosmonkey.groupType=ASG
```

```go
// Chaos Monkey Go Implementation
package chaosmonkey

import (
 "context"
 "log"
 "math/rand"
 "time"

 "github.com/aws/aws-sdk-go-v2/aws"
 "github.com/aws/aws-sdk-go-v2/service/ec2"
 "github.com/aws/aws-sdk-go-v2/service/autoscaling"
)

type ChaosMonkey struct {
 ec2Client *ec2.Client
 asgClient *autoscaling.Client
 config    Config
 metrics   MetricsClient
}

type Config struct {
 Enabled                    bool
 Leashed                    bool          // Dry-run mode
 MeanTimeBetweenKills       time.Duration
 MinInstancesPerASG         int
 ExcludedASGs               []string
 ScheduleStart              int  // Hour (0-23)
 ScheduleEnd                int
 SkipWeekends               bool
 NotificationWebhook        string
}

type TerminationEvent struct {
 InstanceID    string
 ASGName       string
 AvailabilityZone string
 Timestamp     time.Time
 DryRun        bool
 Reason        string
}

func NewChaosMonkey(cfg aws.Config, config Config) *ChaosMonkey {
 return &ChaosMonkey{
  ec2Client: ec2.NewFromConfig(cfg),
  asgClient: autoscaling.NewFromConfig(cfg),
  config:    config,
 }
}

func (cm *ChaosMonkey) Run(ctx context.Context) error {
 if !cm.config.Enabled {
  log.Println("Chaos Monkey is disabled")
  return nil
 }

 if !cm.isWithinSchedule() {
  log.Println("Outside of chaos schedule, skipping")
  return nil
 }

 // Get all eligible ASGs
 asgs, err := cm.getEligibleASGs(ctx)
 if err != nil {
  return err
 }

 // Select random ASG
 if len(asgs) == 0 {
  log.Println("No eligible ASGs found")
  return nil
 }

 asg := asgs[rand.Intn(len(asgs))]

 // Select random instance from ASG
 instance, err := cm.selectRandomInstance(ctx, asg)
 if err != nil {
  return err
 }

 // Terminate instance
 event := TerminationEvent{
  InstanceID:       instance.InstanceID,
  ASGName:          asg.Name,
  AvailabilityZone: instance.AvailabilityZone,
  Timestamp:        time.Now(),
  DryRun:           cm.config.Leashed,
  Reason:           "Chaos Monkey random termination",
 }

 if cm.config.Leashed {
  log.Printf("DRY-RUN: Would terminate %s in %s",
   instance.InstanceID, asg.Name)
  cm.recordEvent(event)
  return nil
 }

 // Actually terminate
 _, err = cm.ec2Client.TerminateInstances(ctx, &ec2.TerminateInstancesInput{
  InstanceIds: []string{instance.InstanceID},
 })

 if err != nil {
  return err
 }

 log.Printf("Terminated instance %s in ASG %s",
  instance.InstanceID, asg.Name)

 cm.recordEvent(event)
 cm.sendNotification(event)

 return nil
}

func (cm *ChaosMonkey) isWithinSchedule() bool {
 now := time.Now()

 // Check weekend
 if cm.config.SkipWeekends {
  weekday := now.Weekday()
  if weekday == time.Saturday || weekday == time.Sunday {
   return false
  }
 }

 // Check hours
 hour := now.Hour()
 return hour >= cm.config.ScheduleStart && hour < cm.config.ScheduleEnd
}

func (cm *ChaosMonkey) getEligibleASGs(ctx context.Context) ([]ASG, error) {
 result, err := cm.asgClient.DescribeAutoScalingGroups(ctx,
  &autoscaling.DescribeAutoScalingGroupsInput{})
 if err != nil {
  return nil, err
 }

 var eligible []ASG
 for _, asg := range result.AutoScalingGroups {
  // Skip excluded ASGs
  if cm.isExcluded(*asg.AutoScalingGroupName) {
   continue
  }

  // Must have more than minimum instances
  if len(asg.Instances) <= cm.config.MinInstancesPerASG {
   continue
  }

  // Must be enabled for chaos
  if !cm.hasOptedIn(asg) {
   continue
  }

  eligible = append(eligible, ASG{
   Name:      *asg.AutoScalingGroupName,
   Instances: asg.Instances,
  })
 }

 return eligible, nil
}

func (cm *ChaosMonkey) hasOptedIn(asg types.AutoScalingGroup) bool {
 // Check for opt-in tag
 for _, tag := range asg.Tags {
  if *tag.Key == "chaos:enabled" && *tag.Value == "true" {
   return true
  }
 }
 return false
}
```

```python
# Chaos Gorilla - AZ Failover Simulation
from dataclasses import dataclass
from typing import List, Dict, Optional
import boto3
import logging
from datetime import datetime

logger = logging.getLogger(__name__)


@dataclass
class AZFailoverExperiment:
    """Simulate entire Availability Zone failure."""
    experiment_id: str
    target_az: str
    region: str
    start_time: datetime
    duration_seconds: int
    affected_services: List[str]
    rollback_on_failure: bool = True


class ChaosGorilla:
    """Drop an entire Availability Zone."""

    def __init__(self, region: str):
        self.region = region
        self.ec2 = boto3.client("ec2", region_name=region)
        self.elb = boto3.client("elbv2", region_name=region)
        self.route53 = boto3.client("route53")
        self.original_state: Dict = {}

    def simulate_az_failure(self, availability_zone: str) -> AZFailoverExperiment:
        """
        Simulate AZ failure by:
        1. Removing instances from load balancers
        2. Updating DNS to exclude AZ
        3. Stopping traffic to AZ
        """
        experiment = AZFailoverExperiment(
            experiment_id=f"gorilla-{datetime.now().strftime('%Y%m%d%H%M%S')}",
            target_az=availability_zone,
            region=self.region,
            start_time=datetime.now(),
            duration_seconds=300,
            affected_services=[],
        )

        logger.info(f"Starting AZ failure simulation for {availability_zone}")

        # Save current state for rollback
        self._save_state(availability_zone)

        # Remove instances from target groups
        self._remove_az_from_target_groups(availability_zone)

        # Update DNS weights
        self._update_dns_weights(availability_zone)

        # Verify traffic shifted
        self._verify_failover(availability_zone)

        return experiment

    def _save_state(self, az: str):
        """Save current state for rollback."""
        # Save target group registrations
        target_groups = self.elb.describe_target_groups()["TargetGroups"]

        self.original_state["target_groups"] = {}
        for tg in target_groups:
            health = self.elb.describe_target_health(
                TargetGroupArn=tg["TargetGroupArn"]
            )
            self.original_state["target_groups"][tg["TargetGroupArn"]] = [
                t["Target"] for t in health["TargetHealthDescriptions"]
                if self._is_in_az(t["Target"]["Id"], az)
            ]

    def _remove_az_from_target_groups(self, az: str):
        """Deregister all instances in the AZ from target groups."""
        for tg_arn, targets in self.original_state["target_groups"].items():
            if targets:
                self.elb.deregister_targets(
                    TargetGroupArn=tg_arn,
                    Targets=targets,
                )
                logger.info(f"Deregistered {len(targets)} targets from {tg_arn}")

    def _is_in_az(self, instance_id: str, az: str) -> bool:
        """Check if instance is in the target AZ."""
        response = self.ec2.describe_instances(InstanceIds=[instance_id])
        for reservation in response["Reservations"]:
            for instance in reservation["Instances"]:
                return instance["Placement"]["AvailabilityZone"] == az
        return False

    def rollback(self):
        """Restore original state."""
        logger.info("Rolling back AZ failure simulation")

        # Re-register instances
        for tg_arn, targets in self.original_state.get("target_groups", {}).items():
            if targets:
                self.elb.register_targets(
                    TargetGroupArn=tg_arn,
                    Targets=targets,
                )
                logger.info(f"Re-registered {len(targets)} targets to {tg_arn}")


class ChaosKong:
    """Simulate entire region failure."""

    def __init__(self, primary_region: str, failover_region: str):
        self.primary_region = primary_region
        self.failover_region = failover_region
        self.route53 = boto3.client("route53")

    def simulate_region_failure(self, hosted_zone_id: str, record_name: str):
        """
        Simulate region failure by updating Route53 to failover region.
        """
        logger.info(f"Simulating region failure: {self.primary_region}")

        # Get current record
        records = self.route53.list_resource_record_sets(
            HostedZoneId=hosted_zone_id,
            StartRecordName=record_name,
            MaxItems="1",
        )

        # Update health check to fail
        # Or directly update DNS weights

        # For latency-based routing, mark region as unhealthy
        self._update_health_check_status(record_name, healthy=False)

        # Verify failover happened
        self._verify_region_failover()

    def _update_health_check_status(self, record_name: str, healthy: bool):
        """Force health check status."""
        # In real implementation, you'd either:
        # 1. Use Route53 health check override
        # 2. Block traffic at NLB/ALB level
        # 3. Use feature flags to return errors
        pass

    def _verify_region_failover(self):
        """Verify traffic is flowing to failover region."""
        pass
```

```yaml
# Netflix ChAP - Chaos Automation Platform Configuration
# chap-experiment.yaml
apiVersion: chaos.netflix.com/v1
kind: Experiment
metadata:
  name: api-resilience-test
  namespace: production
spec:
  description: "Test API resilience during instance failures"

  # Hypothesis
  hypothesis:
    steadyState:
      - type: prometheus
        query: |
          sum(rate(http_requests_total{status="200"}[5m])) /
          sum(rate(http_requests_total[5m])) > 0.99
        description: "99% success rate maintained"

      - type: latency
        percentile: 99
        threshold: 500ms
        description: "P99 latency under 500ms"

  # Experiment actions
  actions:
    - name: terminate-random-instance
      type: ec2-termination
      target:
        type: asg
        name: api-service-asg
        selection: random
        count: 1

  # Rollback conditions
  rollback:
    enabled: true
    conditions:
      - type: metric
        query: sum(rate(http_requests_total{status="5xx"}[1m])) > 100
        description: "Error rate exceeded threshold"
      - type: duration
        threshold: 30s
        description: "Recovery timeout exceeded"

  # Schedule
  schedule:
    type: cron
    expression: "0 10-15 * * 1-5" # Mon-Fri, 10am-3pm
    timezone: America/Los_Angeles

  # Notifications
  notifications:
    slack:
      channel: "#chaos-experiments"
      onStart: true
      onComplete: true
      onRollback: true
    pagerduty:
      onRollback: true
      severity: critical

  # Tags for tracking
  labels:
    team: platform
    service: api
    experiment-type: instance-failure
```

```python
# Toxiproxy Integration for Network Chaos
import toxiproxy
from typing import List, Dict


class NetworkChaos:
    """Network failure simulation using Toxiproxy."""

    def __init__(self, toxiproxy_url: str = "http://localhost:8474"):
        self.client = toxiproxy.Toxiproxy(toxiproxy_url)

    def create_proxy(
        self,
        name: str,
        listen: str,
        upstream: str,
    ) -> toxiproxy.Proxy:
        """Create a proxy for a service."""
        return self.client.create(
            name=name,
            listen=listen,
            upstream=upstream,
        )

    def inject_latency(
        self,
        proxy_name: str,
        latency_ms: int,
        jitter_ms: int = 0,
    ):
        """Add latency to a proxy."""
        proxy = self.client.get_proxy(proxy_name)
        proxy.add_toxic(
            name="latency",
            type="latency",
            attributes={
                "latency": latency_ms,
                "jitter": jitter_ms,
            },
        )

    def inject_bandwidth_limit(
        self,
        proxy_name: str,
        rate_kb: int,
    ):
        """Limit bandwidth on a proxy."""
        proxy = self.client.get_proxy(proxy_name)
        proxy.add_toxic(
            name="bandwidth",
            type="bandwidth",
            attributes={"rate": rate_kb},
        )

    def inject_timeout(
        self,
        proxy_name: str,
        timeout_ms: int,
    ):
        """Add connection timeout."""
        proxy = self.client.get_proxy(proxy_name)
        proxy.add_toxic(
            name="timeout",
            type="timeout",
            attributes={"timeout": timeout_ms},
        )

    def inject_slicer(
        self,
        proxy_name: str,
        average_size: int = 1024,
        size_variation: int = 512,
        delay_ms: int = 10,
    ):
        """Slice data into small packets with delays."""
        proxy = self.client.get_proxy(proxy_name)
        proxy.add_toxic(
            name="slicer",
            type="slicer",
            attributes={
                "average_size": average_size,
                "size_variation": size_variation,
                "delay": delay_ms * 1000,  # microseconds
            },
        )

    def reset_all(self):
        """Remove all toxics from all proxies."""
        for proxy in self.client.proxies():
            for toxic in proxy.toxics():
                toxic.destroy()
```

## Best Practices

### Experiment Design

- Start with smallest blast radius
- Have clear rollback procedures
- Run during business hours initially
- Gradually increase scope

### From Netflix's Principles

1. Build a hypothesis around steady state
2. Vary real-world events
3. Run experiments in production
4. Automate to run continuously
5. Minimize blast radius

### Safety Controls

- Implement circuit breakers
- Set strict time limits
- Monitor during experiments
- Have humans ready to intervene

### Measuring Success

Studies show **35% reduction in outages** and **41% improvement in MTTR** after implementing chaos engineering.

## Sources

- [Netflix Chaos Monkey](https://netflix.github.io/chaosmonkey/)
- [Gremlin Guide](https://www.gremlin.com/chaos-monkey)

You implement Netflix-grade chaos engineering practices with proper safety controls and hypothesis-driven experimentation.