Configuration Drift Detection & Incident Response

# Terraform declares: SSH restricted to bastion host
resource "aws_security_group_rule" "ssh" {
  security_group_id = aws_security_group.app.id
  type              = "ingress"
  from_port         = 22
  to_port           = 22
  protocol          = "tcp"
  cidr_blocks       = ["10.0.1.0/24"]  # Bastion subnet only
}

# Actual AWS state: SSH open to entire internet
# Someone added 0.0.0.0/0 via console during incident troubleshooting

# Terraform declares: Database encryption enabled
resource "aws_db_instance" "main" {
  identifier              = "prod-database"
  storage_encrypted       = true
  kms_key_id             = aws_kms_key.db.arn
  backup_retention_period = 7
}

# Actual AWS state: Encryption disabled after restore from unencrypted snapshot
# Emergency database recovery bypassed encryption requirement

# Basic drift detection
terraform plan -refresh-only

# This shows what Terraform would need to change to match actual state
# Output includes:
# - Resources that exist in state but not in cloud (deleted outside Terraform)
# - Resources modified outside Terraform (configuration drift)
# - Resources created outside Terraform (shadow IT)

# Exit code-based automation
terraform plan -detailed-exitcode -refresh-only
# Exit code 0: No drift detected
# Exit code 1: Error occurred
# Exit code 2: Drift detected (successful plan with changes)

# .github/workflows/drift-detection.yml
name: Drift Detection
on:
  schedule:
    - cron: '0 */4 * * *'  # Every 4 hours
  workflow_dispatch:

jobs:
  detect-drift:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v3

      - name: Terraform Init
        run: terraform init

      - name: Detect Drift
        id: drift
        run: |
          terraform plan -refresh-only -detailed-exitcode -out=drift.tfplan
          echo "exitcode=$?" >> $GITHUB_OUTPUT
        continue-on-error: true

      - name: Alert on Drift
        if: steps.drift.outputs.exitcode == '2'
        uses: slackapi/slack-github-action@v1
        with:
          payload: |
            {
              "text": "⚠️ Configuration Drift Detected",
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "*Configuration drift detected in production infrastructure*\n\nReview the drift: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                  }
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.DRIFT_ALERT_WEBHOOK }}

      - name: Generate Drift Report
        if: steps.drift.outputs.exitcode == '2'
        run: |
          terraform show -json drift.tfplan > drift-report.json

      - name: Upload Drift Report
        if: steps.drift.outputs.exitcode == '2'
        uses: actions/upload-artifact@v4
        with:
          name: drift-report
          path: drift-report.json

{
  "ConfigRuleName": "s3-bucket-public-read-prohibited",
  "Description": "Checks that S3 buckets do not allow public read access",
  "Source": {
    "Owner": "AWS",
    "SourceIdentifier": "S3_BUCKET_PUBLIC_READ_PROHIBITED"
  },
  "Scope": {
    "ComplianceResourceTypes": ["AWS::S3::Bucket"]
  }
}

{
  "properties": {
    "displayName": "Require encryption on storage accounts",
    "policyType": "BuiltIn",
    "mode": "All",
    "description": "This policy ensures encryption is enabled on storage accounts",
    "policyRule": {
      "if": {
        "allOf": [
          {
            "field": "type",
            "equals": "Microsoft.Storage/storageAccounts"
          },
          {
            "field": "Microsoft.Storage/storageAccounts/encryption.services.blob.enabled",
            "notEquals": "true"
          }
        ]
      },
      "then": {
        "effect": "deny"
      }
    }
  }
}

# Refresh Terraform state to match reality
terraform refresh

# Generate JSON representation of current state
terraform show -json > current-state.json

# Generate JSON representation of planned changes
terraform plan -out=tfplan
terraform show -json tfplan > planned-changes.json

# Compare states to identify drift
# Use jq to extract specific resources
jq '.values.root_module.resources[] | select(.type == "aws_security_group")' current-state.json

# Compare declared manifest with running deployment
kubectl diff -f deployment.yaml

# Get live resource configuration
kubectl get deployment myapp -o yaml > live-config.yaml

# Compare with version-controlled manifest
diff deployment.yaml live-config.yaml

# Use specialized tools
# kubediff - Shows differences between Kubernetes manifests and cluster state
kubediff --context=production --namespace=default

# kubectl-neat - Removes system-generated fields for cleaner comparison
kubectl get deployment myapp -o yaml | kubectl neat > clean-config.yaml

# Document expected environment variables
cat > expected-env.txt <<EOF
DATABASE_URL=postgresql://prod-db:5432/app
REDIS_URL=redis://prod-cache:6379
LOG_LEVEL=info
ENCRYPTION_ENABLED=true
EOF

# Extract actual environment variables from running containers
kubectl exec -it myapp-pod -- env | sort > actual-env.txt

# Compare expected vs. actual
diff expected-env.txt actual-env.txt

# baseline-checker.py
import boto3
import json
from datetime import datetime

def check_security_group_baseline(sg_id, baseline_file):
    """Compare security group against approved baseline"""
    ec2 = boto3.client('ec2')

    # Load baseline configuration
    with open(baseline_file) as f:
        baseline = json.load(f)

    # Get current configuration
    response = ec2.describe_security_groups(GroupIds=[sg_id])
    current = response['SecurityGroups'][0]

    # Compare ingress rules
    baseline_ingress = set(
        (r['FromPort'], r['ToPort'], r['IpProtocol'],
         tuple(r.get('IpRanges', [])))
        for r in baseline.get('IpPermissions', [])
    )

    current_ingress = set(
        (r.get('FromPort'), r.get('ToPort'), r['IpProtocol'],
         tuple(ip['CidrIp'] for ip in r.get('IpRanges', [])))
        for r in current.get('IpPermissions', [])
    )

    drift = current_ingress - baseline_ingress

    if drift:
        print(f"⚠️ Drift detected in {sg_id}")
        print(f"Unauthorized rules: {drift}")
        return False
    else:
        print(f"✅ {sg_id} matches baseline")
        return True

# Scenario: Someone manually created an S3 bucket that should be managed by Terraform

# Step 1: Import the resource into Terraform state
terraform import aws_s3_bucket.manual_bucket prod-manual-bucket

# Step 2: Generate configuration for the imported resource
terraform show -json | jq '.values.root_module.resources[] | select(.address == "aws_s3_bucket.manual_bucket")'

# Step 3: Add resource definition to Terraform code
# main.tf
resource "aws_s3_bucket" "manual_bucket" {
  bucket = "prod-manual-bucket"
  # ... copy other attributes from terraform show output
}

# Step 4: Verify no drift remains
terraform plan  # Should show "No changes"

# Scenario: Security group rules were loosened during troubleshooting

# Step 1: Review what will change
terraform plan

# Output shows:
# ~ resource "aws_security_group_rule" "ssh" {
#     ~ cidr_blocks = ["0.0.0.0/0"] -> ["10.0.1.0/24"]
# }

# Step 2: Apply IaC state to revert unauthorized changes
terraform apply

# Step 3: Document the drift in incident report
echo "$(date): Reverted unauthorized SSH access to 0.0.0.0/0" >> drift-log.md

# Use Terraform lifecycle blocks for selective drift tolerance

resource "aws_autoscaling_group" "app" {
  name                 = "app-asg"
  desired_capacity     = 3
  min_size            = 2
  max_size            = 10

  # Ignore capacity changes made by auto-scaling policies
  lifecycle {
    ignore_changes = [
      desired_capacity,  # Allow auto-scaling to modify this
    ]
  }
}

resource "aws_db_instance" "main" {
  identifier        = "prod-db"
  storage_encrypted = true

  # NEVER ignore security-critical attributes
  lifecycle {
    prevent_destroy = true  # Block accidental deletion
    # Do NOT ignore: storage_encrypted, publicly_accessible, backup_retention_period
  }
}

# policy/drift-tolerance.rego
package terraform.drift

# Deny drift that disables encryption
deny[msg] {
  resource := input.resource_changes[_]
  resource.type == "aws_db_instance"

  # Detect change from encrypted to unencrypted
  resource.change.before.storage_encrypted == true
  resource.change.after.storage_encrypted == false

  msg := sprintf("Drift detected: Encryption disabled on %s", [resource.address])
}

# Deny drift that opens security groups to internet
deny[msg] {
  resource := input.resource_changes[_]
  resource.type == "aws_security_group_rule"

  # Check for 0.0.0.0/0 in cidr_blocks
  cidr := resource.change.after.cidr_blocks[_]
  cidr == "0.0.0.0/0"

  msg := sprintf("Drift detected: Security group %s opened to internet", [resource.address])
}

# Allow drift for auto-scaling managed attributes
allow[msg] {
  resource := input.resource_changes[_]
  resource.type == "aws_autoscaling_group"

  # Only capacity changes are allowed
  changed_attrs := {attr | resource.change.before[attr] != resource.change.after[attr]}
  allowed_changes := {"desired_capacity", "min_size", "max_size"}

  changed_attrs - allowed_changes == set()

  msg := "Auto-scaling capacity drift is acceptable"
}

# Ignore AWS-managed attributes that update automatically
resource "aws_eks_cluster" "main" {
  name     = "prod-cluster"
  version  = "1.28"

  lifecycle {
    ignore_changes = [
      # AWS updates these automatically during maintenance windows
      platform_version,
      certificate_authority,
    ]
  }
}

# Ignore tags added by AWS cost allocation
resource "aws_instance" "app" {
  ami           = "ami-12345678"
  instance_type = "t3.medium"

  lifecycle {
    ignore_changes = [
      tags["aws:autoscaling:groupName"],
      tags["aws:cloudformation:stack-name"],
    ]
  }
}

# Terraform prevent_destroy lifecycle policy
resource "aws_s3_bucket" "critical_data" {
  bucket = "critical-customer-data"

  lifecycle {
    prevent_destroy = true  # Terraform will refuse to destroy this resource
  }
}

# AWS Service Control Policy (SCP) to block manual changes
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "DenyManualSecurityGroupChanges",
      "Effect": "Deny",
      "Action": [
        "ec2:AuthorizeSecurityGroupIngress",
        "ec2:RevokeSecurityGroupIngress",
        "ec2:AuthorizeSecurityGroupEgress",
        "ec2:RevokeSecurityGroupEgress"
      ],
      "Resource": "arn:aws:ec2:*:*:security-group/*",
      "Condition": {
        "StringNotEquals": {
          "aws:PrincipalArn": "arn:aws:iam::123456789012:role/TerraformRole"
        }
      }
    }
  ]
}

# AWS IAM Policy: Read-only console access for developers
{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Effect": "Allow",
      "Action": [
        "ec2:Describe*",
        "s3:List*",
        "s3:Get*",
        "rds:Describe*",
        "cloudwatch:Get*",
        "cloudwatch:List*"
      ],
      "Resource": "*"
    },
    {
      "Effect": "Deny",
      "Action": [
        "ec2:*",
        "s3:Put*",
        "s3:Delete*",
        "rds:Modify*",
        "rds:Create*",
        "rds:Delete*"
      ],
      "Resource": "*",
      "Condition": {
        "StringNotEquals": {
          "aws:RequestedRegion": "us-east-1"
        }
      }
    }
  ]
}

# AWS Config continuous compliance
aws configservice put-config-rule \
  --config-rule file://s3-encryption-rule.json

# s3-encryption-rule.json
{
  "ConfigRuleName": "s3-bucket-encryption-enabled",
  "Description": "Checks that S3 buckets have encryption enabled",
  "Source": {
    "Owner": "AWS",
    "SourceIdentifier": "S3_BUCKET_SERVER_SIDE_ENCRYPTION_ENABLED"
  },
  "Scope": {
    "ComplianceResourceTypes": ["AWS::S3::Bucket"]
  }
}

# AWS Config remediation action
RemediationConfiguration:
  ConfigRuleName: s3-bucket-encryption-enabled
  TargetType: SSM_DOCUMENT
  TargetIdentifier: AWS-EnableS3BucketEncryption
  Parameters:
    AutomationAssumeRole:
      StaticValue:
        Values:
          - arn:aws:iam::123456789012:role/ConfigRemediationRole
    BucketName:
      ResourceValue:
        Value: RESOURCE_ID
  Automatic: true
  MaximumAutomaticAttempts: 3
  RetryAttemptSeconds: 60

# Cron-based drift detection script
# /etc/cron.d/terraform-drift-check

# Run drift detection every 4 hours
0 */4 * * * terraform-user cd /opt/terraform/production && ./detect-drift.sh

# detect-drift.sh
#!/bin/bash
set -euo pipefail

cd "$(dirname "$0")"

terraform init -backend=true

if ! terraform plan -refresh-only -detailed-exitcode > /dev/null 2>&1; then
  EXITCODE=$?

  if [ $EXITCODE -eq 2 ]; then
    # Drift detected
    terraform plan -refresh-only -no-color > drift-report.txt

    # Send alert to Slack
    curl -X POST "$SLACK_WEBHOOK_URL" \
      -H 'Content-Type: application/json' \
      -d '{
        "text": "⚠️ Configuration Drift Detected",
        "attachments": [{
          "color": "warning",
          "title": "Production Infrastructure Drift",
          "text": "Configuration drift detected in production. Review required.",
          "fields": [
            {
              "title": "Environment",
              "value": "production",
              "short": true
            },
            {
              "title": "Timestamp",
              "value": "'"$(date -Iseconds)"'",
              "short": true
            }
          ]
        }]
      }'

    # Email report to ops team
    cat drift-report.txt | mail -s "Terraform Drift Detected" ops@company.com

    exit 1
  else
    # Error occurred
    echo "Error running terraform plan: exit code $EXITCODE"
    exit $EXITCODE
  fi
else
  # No drift
  echo "No configuration drift detected"
  exit 0
fi

# Sentinel policy: Require encryption on all storage
import "tfplan/v2" as tfplan

# Get all S3 bucket resources
s3_buckets = filter tfplan.resource_changes as _, rc {
  rc.type is "aws_s3_bucket" and
  rc.mode is "managed" and
  (rc.change.actions contains "create" or rc.change.actions contains "update")
}

# Encryption must be enabled
main = rule {
  all s3_buckets as _, bucket {
    bucket.change.after.server_side_encryption_configuration is not null
  }
}

# Alert sources trigger investigation
# - SIEM alert: "Unusual network traffic from database server"
# - Cloud monitoring: "Security group rule modified to allow 0.0.0.0/0"
# - AWS GuardDuty: "UnauthorizedAccess:EC2/SSHBruteForce"
# - Customer report: "Cannot access application"

# Initial triage steps
# 1. Extract alert metadata
ALERT_TIME="2025-01-07T14:32:18Z"
AFFECTED_RESOURCE="aws_security_group.prod-db-sg"
ALERT_SEVERITY="P1-High"

# 2. Convert timestamps to standardized format
# Use Unix Timestamp Converter: /tools/developer/unix-timestamp-converter
# Input: 1736259138 (Unix epoch)
# Output: 2025-01-07 14:32:18 UTC

# 3. Classify incident severity
# P0/Critical: Production down, revenue loss, active data breach
# P1/High: Major degradation, security control bypassed, potential breach
# P2/Medium: Minor degradation, suspicious activity, compliance drift
# P3/Low: No immediate impact, informational alerts

# 4. Determine initial scope
aws ec2 describe-security-groups --group-ids sg-0a1b2c3d4e5f6g7h8 > sg-current-state.json

# 5. Test affected service endpoints
# Use HTTP Request Builder: /tools/developer/http-request-builder
curl -v https://api.example.com/health

# Preserve evidence before remediation
# 1. Create forensic snapshots
aws ec2 create-snapshot \
  --volume-id vol-0a1b2c3d4e5f6g7h8 \
  --description "Forensic snapshot - Incident #INC-2025-0107-001" \
  --tag-specifications 'ResourceType=snapshot,Tags=[{Key=incident-id,Value=INC-2025-0107-001},{Key=chain-of-custody,Value=true}]'

# 2. Export configuration state
aws ec2 describe-instances --instance-ids i-0a1b2c3d4e5f6g7h8 > evidence/instance-config-$(date +%s).json
aws ec2 describe-security-groups --group-ids sg-0a1b2c3d4e5f6g7h8 > evidence/security-group-config-$(date +%s).json

# 3. Capture logs before rotation
aws logs create-export-task \
  --log-group-name /aws/lambda/production-api \
  --from $(date -d '2 hours ago' +%s)000 \
  --to $(date +%s)000 \
  --destination s3-forensics-bucket \
  --destination-prefix incident-INC-2025-0107-001/

# 4. Document chain of custody
cat > evidence/chain-of-custody.txt <<EOF
Incident ID: INC-2025-0107-001
Evidence Collected By: John Doe (john.doe@company.com)
Collection Timestamp: $(date -Iseconds)
Collection Method: AWS CLI automated export
Hash Verification: $(sha256sum evidence/*.json)
Storage Location: s3://forensics-bucket/incident-INC-2025-0107-001/
Access Log: CloudTrail logging enabled on forensics bucket
EOF

# Reconstruct attack timeline
# Use Unix Timestamp Converter to build chronological event sequence

# 1. Query CloudTrail for unauthorized actions
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=ResourceName,AttributeValue=sg-0a1b2c3d4e5f6g7h8 \
  --start-time 2025-01-07T12:00:00Z \
  --end-time 2025-01-07T15:00:00Z > cloudtrail-events.json

# 2. Extract IOCs (Indicators of Compromise)
jq -r '.Events[] | select(.EventName == "AuthorizeSecurityGroupIngress") |
  {
    time: .EventTime,
    user: .Username,
    source_ip: .SourceIPAddress,
    user_agent: .UserAgent
  }' cloudtrail-events.json

# Example output:
# {
#   "time": "2025-01-07T14:32:18Z",
#   "user": "compromised-user",
#   "source_ip": "203.0.113.42",
#   "user_agent": "aws-cli/2.9.0"
# }

# 3. Pivot on IOCs
# Search for other actions from same source IP
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=Username,AttributeValue=compromised-user \
  --start-time 2025-01-06T00:00:00Z > user-activity.json

# 4. Identify lateral movement
# Check for assumed roles, privilege escalation, data access
jq -r '.Events[] | select(.EventName | contains("Assume") or contains("Get") or contains("List")) |
  {time: .EventTime, action: .EventName, resource: .Resources}' user-activity.json

# Short-term containment (immediate)
# 1. Revert unauthorized security group changes
terraform apply -auto-approve -target=aws_security_group_rule.ssh

# 2. Rotate compromised credentials
aws iam delete-access-key --user-name compromised-user --access-key-id AKIAIOSFODNN7EXAMPLE
aws iam create-access-key --user-name compromised-user > new-credentials.json

# 3. Isolate affected instances (quarantine)
aws ec2 modify-instance-attribute \
  --instance-id i-0a1b2c3d4e5f6g7h8 \
  --groups sg-quarantine

# Long-term containment (prevent recurrence)
# 4. Enable MFA enforcement
aws iam create-virtual-mfa-device --virtual-mfa-device-name compromised-user-mfa
aws iam enable-mfa-device --user-name compromised-user --serial-number arn:aws:iam::123456789012:mfa/compromised-user-mfa

# 5. Restrict console access
aws iam attach-user-policy \
  --user-name compromised-user \
  --policy-arn arn:aws:iam::aws:policy/ReadOnlyAccess

# Eradication (remove threat completely)
# 6. Scan for backdoors and persistence mechanisms
# Check for unauthorized IAM roles, Lambda functions, CloudFormation stacks
aws iam list-roles --query 'Roles[?CreateDate >= `2025-01-07`]'
aws lambda list-functions --query 'Functions[?LastModified >= `2025-01-07`]'

# 7. Remove malicious resources
aws iam delete-role --role-name suspicious-admin-role

# 1. Validate infrastructure state matches IaC
terraform plan -refresh-only  # Should show "No changes"

# 2. Run health checks
for endpoint in api.example.com admin.example.com db.example.com; do
  curl -sf https://$endpoint/health || echo "$endpoint FAILED health check"
done

# 3. Restore from forensic snapshots if needed
aws ec2 create-volume \
  --snapshot-id snap-forensic-clean \
  --availability-zone us-east-1a

# 4. Gradual service restoration (canary approach)
# Restore 10% traffic first, monitor for issues
kubectl scale deployment myapp --replicas=1  # Start with 1 pod
# Monitor metrics for 15 minutes
# If stable, scale to full capacity
kubectl scale deployment myapp --replicas=10

# 5. Enable enhanced monitoring post-recovery
aws rds modify-db-instance \
  --db-instance-identifier prod-db \
  --enable-cloudwatch-logs-exports '["error","general","slowquery"]' \
  --monitoring-interval 1

# Internal War Room Message (Slack #incident-response)

🚨 **INCIDENT DECLARED: INC-2025-0107-001**

**Severity**: P1-High
**Status**: Investigating
**Incident Commander**: @john-doe
**Affected Services**: Production API, Customer Database
**Impact**: Database security group modified - potential unauthorized access
**Timeline**: Alert received 14:32 UTC, investigation started 14:35 UTC

**Current Actions**:
- Reviewing CloudTrail logs for unauthorized changes
- Assessing data access logs
- Preparing containment plan

**Next Update**: 15:00 UTC (in 25 minutes)

📊 **Live Dashboard**: https://status.company.com/incident/INC-2025-0107-001

# Executive Stakeholder Update (Email to CTO, VP Engineering, Security Lead)

Subject: [P1-HIGH] Security Incident - Unauthorized Infrastructure Change

Dear Leadership Team,

We are responding to a high-severity security incident involving unauthorized changes to our production database infrastructure.

**Summary**:
- **Incident ID**: INC-2025-0107-001
- **Detection Time**: 2025-01-07 14:32 UTC
- **Severity**: P1-High (security control bypassed)
- **Status**: Active investigation and containment underway

**Details**:
An automated alert detected unauthorized modification of a production database security group rule, potentially exposing the database to broader network access than intended. The incident response team is actively investigating the scope and implementing containment measures.

**Current Impact**:
- Production API remains operational
- No confirmed data breach at this time
- Database access logs under review
- Affected security group has been reverted to secure configuration

**Next Steps**:
1. Complete forensic analysis of all actions by the involved credentials
2. Rotate affected credentials and enforce MFA
3. Assess data access logs for unauthorized queries
4. Implement additional preventive controls

**Communication Schedule**:
- Updates every 2 hours until contained
- Post-incident report within 48 hours
- Lessons learned review within 1 week

**Incident Commander**: John Doe (john.doe@company.com, +1-555-0123)

Please direct all questions through the incident commander to avoid disrupting the response effort.

Best regards,
Incident Response Team

# Status Page Update (status.company.com)

🟡 **Investigating** - Security Review in Progress
Posted: 2025-01-07 15:00 UTC

We are conducting a security review of our infrastructure following an automated alert. Our services remain operational, and we have implemented additional monitoring and controls as a precautionary measure.

**Affected Services**: Production API (no service interruption)
**Impact**: None at this time
**Next Update**: 17:00 UTC or when new information is available

We take security very seriously and will provide updates as our investigation progresses.

# Internal All-Clear Message

✅ **INCIDENT RESOLVED: INC-2025-0107-001**

**Resolution Time**: 2025-01-07 18:45 UTC
**Total Duration**: 4 hours 13 minutes
**Final Status**: Contained and remediated

**Summary**:
Unauthorized security group modification was traced to a compromised access key. The key has been rotated, MFA enforced, and all infrastructure validated against IaC baseline. No evidence of data exfiltration found.

**Actions Completed**:
- ✅ Reverted security group to secure configuration
- ✅ Rotated compromised credentials
- ✅ Enforced MFA on affected user account
- ✅ Reviewed all CloudTrail events from affected credentials
- ✅ Validated infrastructure matches Terraform state
- ✅ Enhanced monitoring enabled

**Next Steps**:
- Post-incident report due: 2025-01-09
- Lessons learned meeting: 2025-01-10 10:00 UTC
- Follow-up action items tracked in JIRA

Thank you to the incident response team for quick and effective response.

# Incident Post-Mortem: INC-2025-0107-001

## Executive Summary

On January 7, 2025, an automated security alert detected unauthorized modification of a production database security group, potentially exposing the database to broader network access. The incident was contained within 4 hours with no confirmed data breach or service disruption. Root cause analysis identified gaps in our infrastructure change management process that allowed manual console changes to bypass code review and drift detection.

**Impact**: No service interruption, no data breach confirmed
**Duration**: 4 hours 13 minutes (detection to full remediation)
**Root Cause**: Compromised AWS access key used to modify security group outside Terraform workflow
**Prevention**: MFA enforcement, automated drift alerts, policy-as-code gates implemented

## Detailed Timeline (All times UTC)

| Time | Event | Source | Actor |
|------|-------|--------|-------|
| 14:32:18 | Security group rule modified to allow 0.0.0.0/0 SSH access | CloudTrail | compromised-user (access key AKIA...7EXA) |
| 14:32:45 | AWS Config compliance rule triggered: "s3-bucket-public-read-prohibited" | AWS Config | Automated |
| 14:33:12 | Security alert sent to #security-alerts Slack channel | SIEM | Automated |
| 14:35:03 | On-call engineer acknowledges alert, begins investigation | PagerDuty | John Doe |
| 14:37:29 | CloudTrail query reveals unauthorized access key usage | CloudTrail Insights | John Doe |
| 14:40:15 | Incident declared P1-High, war room created | Incident Commander | John Doe |
| 14:45:00 | Security group rule reverted via Terraform apply | Terraform | Automated (approved by IC) |
| 14:52:33 | Compromised access key rotated and deleted | IAM | Security Team |
| 15:00:00 | First stakeholder update sent to executive team | Email | Communications Lead |
| 15:15:22 | MFA enforcement enabled on affected user account | IAM | Security Team |
| 15:30:00 | CloudTrail analysis complete - no evidence of data access | CloudTrail | Security Analyst |
| 16:00:00 | Forensic snapshots created, evidence preserved | EC2 | Lead Investigator |
| 17:15:30 | Enhanced drift detection deployed (hourly scans) | GitHub Actions | DevOps Team |
| 18:45:00 | Infrastructure validated against IaC baseline, incident resolved | Terraform | DevOps Team |

## Root Cause Analysis

**Immediate Cause**: Compromised AWS access key (AKIA...7EXA) used to modify security group outside Terraform workflow.

**Contributing Factors**:
1. **Weak Credential Management**: Access key was long-lived (created 18 months ago), never rotated
2. **No MFA Enforcement**: Affected user account did not require MFA for console or API access
3. **Overly Permissive IAM Policies**: User had `ec2:*` permissions instead of scoped, least-privilege access
4. **Slow Drift Detection**: Drift scans ran daily, allowing 24-hour window for undetected changes
5. **No Policy-as-Code Gates**: Security group changes not blocked by OPA/Sentinel policies
6. **Console Access Enabled**: Developers had write access to AWS console for troubleshooting convenience

**Root Cause (5 Whys)**:
1. Why was the security group modified? → Compromised access key was used
2. Why was the access key compromised? → Long-lived key stored in developer laptop
3. Why wasn't the key rotated? → No automated key rotation policy
4. Why did the key have such broad permissions? → IAM policies not scoped to least privilege
5. Why weren't least-privilege policies enforced? → No policy-as-code review in IAM management workflow

**Systemic Gap**: Infrastructure access management lacked defense-in-depth controls (credential rotation, MFA, least privilege, policy gates).

## Impact Assessment

**Technical Impact**:
- 1 security group rule modified (reverted within 13 minutes)
- 0 unauthorized database queries detected in access logs
- 0 services disrupted
- 4.25 hours of engineering time (3 engineers × ~1.5 hours each)

**Business Impact**:
- **Revenue**: $0 (no service interruption)
- **Customer Impact**: 0 customers affected directly
- **Reputational Impact**: Low (incident contained before customer visibility)
- **Compliance Impact**: Medium (GDPR 72-hour notification not required due to no data breach, but internal compliance review triggered)

**Cost Breakdown**:
- Engineering response time: $1,200 (4.25 hours × $280/hour blended rate)
- Enhanced monitoring infrastructure: $150/month ongoing
- Forensic storage (S3): $45 one-time
- **Total**: $1,395 + $150/month ongoing

**Opportunity Cost**:
- Feature development delayed by 1 sprint (2 weeks) while implementing preventive controls
- Estimated revenue impact of delayed features: $15,000 (based on projected customer adoption)

## IOCs & Evidence

**Compromised Credentials**:
- Access Key ID: AKIA...7EXA (rotated and deleted)
- User Account: compromised-user (MFA enforced, permissions scoped)

**Suspicious Activity**:
- Source IP: 203.0.113.42 (reverse DNS: attacker.example.net)
- User-Agent: `aws-cli/2.9.0 Python/3.11.1 Linux/5.15.0`
- API Calls: 7 unauthorized calls between 14:32-14:35 UTC

**Modified Resources**:
- Security Group: sg-0a1b2c3d4e5f6g7h8 (reverted)
- Rule Added: SSH (22/tcp) from 0.0.0.0/0

**Evidence Preserved**:
- CloudTrail logs: s3://forensics-bucket/incident-INC-2025-0107-001/cloudtrail/
- Forensic snapshots: snap-0abc1234def5678 (database volume)
- Configuration exports: s3://forensics-bucket/incident-INC-2025-0107-001/configs/
- Chain of custody: Maintained by Lead Investigator, logged in evidence/chain-of-custody.txt

## Lessons Learned

### What Went Well ✅
- **Fast Detection**: Automated alert triggered within 27 seconds of unauthorized change
- **Clear Escalation**: Incident Commander role immediately assumed, no confusion about authority
- **Effective Communication**: Stakeholder updates maintained every 2 hours, no information gaps
- **Quick Containment**: Security group reverted within 13 minutes of investigation start
- **Thorough Investigation**: CloudTrail analysis complete, no lateral movement detected

### What Went Wrong ❌
- **Credential Lifespan**: Access key was 18 months old, never rotated (policy: rotate every 90 days)
- **No MFA Enforcement**: User account allowed API access without MFA
- **Slow Drift Detection**: Daily scans missed 24-hour window for faster detection
- **Overly Permissive IAM**: User had `ec2:*` instead of scoped permissions
- **No Policy Gates**: Security group changes not blocked by policy-as-code

### Unexpected Positive Findings 🔍
- Forensic snapshot automation worked perfectly (previously untested in real incident)
- Cross-team collaboration exceeded expectations (security + DevOps + leadership)
- Status page integration provided real-time visibility without manual updates

### Knowledge Gaps Identified 📚
- Team unfamiliar with AWS IAM Access Analyzer (could have detected overly permissive policies)
- Runbook didn't cover credential compromise scenario (only focused on misconfigurations)
- No training on GDPR notification timelines (fortunately not needed, but gap identified)

## Action Items

### Immediate (Complete within 24 hours)
- [x] Rotate all access keys older than 90 days (Completed: 2025-01-08, Owner: Security Team)
- [x] Enforce MFA on all user accounts (Completed: 2025-01-08, Owner: IAM Admin)
- [x] Deploy hourly drift detection scans (Completed: 2025-01-08, Owner: DevOps)
- [x] Create runbook for credential compromise scenarios (Completed: 2025-01-08, Owner: Incident Commander)

### Short-Term (Complete within 1-2 weeks)
- [ ] Implement policy-as-code gates for security group changes (Due: 2025-01-21, Owner: Platform Team)
- [ ] Scope IAM policies to least privilege using IAM Access Analyzer (Due: 2025-01-21, Owner: Security Team)
- [ ] Deploy automated access key rotation (90-day lifecycle) (Due: 2025-01-21, Owner: Security Automation)
- [ ] Add AWS Config rules for MFA enforcement compliance (Due: 2025-01-21, Owner: Compliance Team)
- [ ] Conduct incident response training on credential compromise (Due: 2025-01-21, Owner: Security Lead)

### Long-Term (Complete within 1-3 months)
- [ ] Migrate to temporary credentials (IAM roles, AWS SSO) instead of long-lived access keys (Due: 2025-03-15, Owner: Platform Team)
- [ ] Implement Service Control Policies (SCPs) to restrict console access for developers (Due: 2025-03-15, Owner: Cloud Governance)
- [ ] Deploy continuous compliance monitoring (AWS Security Hub, Prowler) (Due: 2025-03-15, Owner: Security Team)
- [ ] Quarterly tabletop exercises for incident response scenarios (Due: 2025-04-01, Owner: Security Lead)

# Post-Mortem Review Meeting Agenda

**Meeting**: INC-2025-0107-001 Post-Mortem Review
**Date**: 2025-01-10 10:00 UTC
**Duration**: 60 minutes
**Attendees**: Engineering team, Security team, Management (optional)

## Agenda

**1. Incident Overview (5 min)** - Incident Commander
   - Severity, duration, impact summary
   - High-level timeline

**2. Technical Deep Dive (15 min)** - Lead Investigator
   - Root cause analysis walkthrough
   - IOCs and evidence review
   - 5 Whys analysis

**3. What Went Well (10 min)** - Team Discussion
   - Celebrate effective responses
   - Identify strengths to maintain

**4. What Went Wrong (10 min)** - Team Discussion
   - Blameless discussion of gaps
   - Systemic issues, not individual blame

**5. Lessons Learned (10 min)** - Team Discussion
   - Knowledge gaps identified
   - Training opportunities
   - Documentation improvements

**6. Action Items Review (10 min)** - Incident Commander
   - Present immediate, short-term, long-term actions
   - Assign owners and deadlines
   - Track in JIRA/Linear for visibility

**7. Q&A & Open Discussion (10 min)** - All
   - Address team questions
   - Solicit additional insights

## Follow-Up
- Post-mortem report published to internal wiki
- Action items tracked in JIRA with weekly review
- Quarterly review of action item completion rates

# policy/terraform-security.rego
package terraform.security

import future.keywords.in

# Deny unencrypted S3 buckets
deny[msg] {
  resource := input.resource_changes[_]
  resource.type == "aws_s3_bucket"
  resource.mode == "managed"

  # Check if encryption is explicitly disabled or missing
  encryption := resource.change.after.server_side_encryption_configuration
  encryption == null

  msg := sprintf("S3 bucket '%s' must have encryption enabled (PCI-DSS, HIPAA requirement)", [resource.address])
}

# Deny security groups open to internet
deny[msg] {
  resource := input.resource_changes[_]
  resource.type == "aws_security_group_rule"
  resource.change.after.type == "ingress"

  # Check for 0.0.0.0/0 in cidr_blocks
  cidr := resource.change.after.cidr_blocks[_]
  cidr == "0.0.0.0/0"

  # Allow HTTPS (443) and HTTP (80) from internet (common for public-facing apps)
  not resource.change.after.from_port in [80, 443]

  msg := sprintf("Security group rule '%s' opens port %d to internet (0.0.0.0/0) - DENIED", [resource.address, resource.change.after.from_port])
}

# Deny databases without backup retention
deny[msg] {
  resource := input.resource_changes[_]
  resource.type == "aws_db_instance"

  retention := resource.change.after.backup_retention_period
  retention < 7

  msg := sprintf("RDS instance '%s' must have backup_retention_period >= 7 days (current: %d days)", [resource.address, retention])
}

# Deny public RDS instances
deny[msg] {
  resource := input.resource_changes[_]
  resource.type == "aws_db_instance"

  resource.change.after.publicly_accessible == true

  msg := sprintf("RDS instance '%s' cannot be publicly accessible - SECURITY VIOLATION", [resource.address])
}

# Warn on oversized instances (cost optimization)
warn[msg] {
  resource := input.resource_changes[_]
  resource.type == "aws_instance"

  # Flag instances larger than m5.2xlarge
  large_types := ["m5.4xlarge", "m5.8xlarge", "m5.16xlarge", "c5.4xlarge", "r5.4xlarge"]
  resource.change.after.instance_type in large_types

  msg := sprintf("Instance '%s' uses large instance type %s - verify sizing is necessary", [resource.address, resource.change.after.instance_type])
}

# .github/workflows/terraform-validate.yml
name: Terraform Policy Validation

on:
  pull_request:
    paths:
      - '**.tf'

jobs:
  policy-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v3

      - name: Terraform Init & Plan
        run: |
          terraform init
          terraform plan -out=tfplan
          terraform show -json tfplan > tfplan.json

      - name: Install OPA
        run: |
          curl -L -o opa https://openpolicyagent.org/downloads/latest/opa_linux_amd64
          chmod +x opa

      - name: Run OPA Policy Checks
        id: opa
        run: |
          ./opa eval --data policy/ --input tfplan.json --format pretty 'data.terraform.security.deny' > opa-violations.txt

          if [ -s opa-violations.txt ]; then
            echo "❌ Policy violations detected"
            cat opa-violations.txt
            exit 1
          else
            echo "✅ All policy checks passed"
            exit 0
          fi

      - name: Comment PR with Policy Results
        if: always()
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const violations = fs.readFileSync('opa-violations.txt', 'utf8');

            const body = violations.length > 0
              ? `## ❌ Policy Violations Detected\n\n\`\`\`\n${violations}\n\`\`\`\n\nPlease fix these violations before merging.`
              : `## ✅ All Policy Checks Passed\n\nYour Terraform changes comply with security policies.`;

            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: body
            });

# sentinel.hcl
policy "require-encryption" {
  source = "./policies/require-encryption.sentinel"
  enforcement_level = "hard-mandatory"  # Cannot override
}

policy "restrict-public-access" {
  source = "./policies/restrict-public-access.sentinel"
  enforcement_level = "hard-mandatory"
}

policy "cost-optimization" {
  source = "./policies/cost-optimization.sentinel"
  enforcement_level = "soft-mandatory"  # Can override with reason
}

# policies/require-encryption.sentinel
import "tfplan/v2" as tfplan

# Find all S3 buckets
s3_buckets = filter tfplan.resource_changes as _, rc {
  rc.type is "aws_s3_bucket" and
  rc.mode is "managed" and
  (rc.change.actions contains "create" or rc.change.actions contains "update")
}

# Require encryption
main = rule {
  all s3_buckets as _, bucket {
    bucket.change.after.server_side_encryption_configuration is not null
  }
}

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "ReadOnlyAccess",
      "Effect": "Allow",
      "Action": [
        "ec2:Describe*",
        "s3:List*",
        "s3:Get*",
        "rds:Describe*",
        "iam:Get*",
        "iam:List*"
      ],
      "Resource": "*"
    },
    {
      "Sid": "DenyDestructiveActions",
      "Effect": "Deny",
      "Action": [
        "ec2:Terminate*",
        "ec2:Delete*",
        "s3:Delete*",
        "rds:Delete*",
        "iam:Delete*"
      ],
      "Resource": "*"
    },
    {
      "Sid": "AllowTerraformRole",
      "Effect": "Allow",
      "Action": "*",
      "Resource": "*",
      "Condition": {
        "StringEquals": {
          "aws:PrincipalArn": "arn:aws:iam::123456789012:role/TerraformAutomationRole"
        }
      }
    }
  ]
}

{
  "Version": "2012-10-17",
  "Statement": [
    {
      "Sid": "DenyConsoleChangesToProduction",
      "Effect": "Deny",
      "Action": [
        "ec2:*",
        "s3:Put*",
        "s3:Delete*",
        "rds:Modify*",
        "rds:Create*",
        "rds:Delete*"
      ],
      "Resource": "*",
      "Condition": {
        "StringNotEquals": {
          "aws:PrincipalArn": [
            "arn:aws:iam::*:role/TerraformAutomationRole",
            "arn:aws:iam::*:role/BreakGlassEmergencyRole"
          ]
        },
        "StringEquals": {
          "aws:RequestedRegion": "us-east-1"
        }
      }
    },
    {
      "Sid": "RequireMFAForSensitiveActions",
      "Effect": "Deny",
      "Action": [
        "iam:CreateAccessKey",
        "iam:DeleteAccessKey",
        "iam:CreateUser",
        "iam:DeleteUser",
        "s3:DeleteBucket"
      ],
      "Resource": "*",
      "Condition": {
        "BoolIfExists": {
          "aws:MultiFactorAuthPresent": "false"
        }
      }
    }
  ]
}

# cloudformation/config-rules.yaml
Resources:
  S3BucketEncryptionRule:
    Type: AWS::Config::ConfigRule
    Properties:
      ConfigRuleName: s3-bucket-encryption-enabled
      Description: Checks that S3 buckets have encryption enabled
      Source:
        Owner: AWS
        SourceIdentifier: S3_BUCKET_SERVER_SIDE_ENCRYPTION_ENABLED
      Scope:
        ComplianceResourceTypes:
          - AWS::S3::Bucket

  RDSEncryptionRule:
    Type: AWS::Config::ConfigRule
    Properties:
      ConfigRuleName: rds-storage-encrypted
      Description: Checks that RDS instances have encryption enabled
      Source:
        Owner: AWS
        SourceIdentifier: RDS_STORAGE_ENCRYPTED
      Scope:
        ComplianceResourceTypes:
          - AWS::RDS::DBInstance

  SecurityGroupPublicAccessRule:
    Type: AWS::Config::ConfigRule
    Properties:
      ConfigRuleName: restricted-ssh
      Description: Checks that security groups do not allow unrestricted SSH access
      Source:
        Owner: AWS
        SourceIdentifier: INCOMING_SSH_DISABLED
      Scope:
        ComplianceResourceTypes:
          - AWS::EC2::SecurityGroup

RemediationConfiguration:
  ConfigRuleName: s3-bucket-encryption-enabled
  TargetType: SSM_DOCUMENT
  TargetIdentifier: AWS-EnableS3BucketEncryption
  Parameters:
    AutomationAssumeRole:
      StaticValue:
        Values:
          - arn:aws:iam::123456789012:role/ConfigRemediationRole
    BucketName:
      ResourceValue:
        Value: RESOURCE_ID
    SSEAlgorithm:
      StaticValue:
        Values:
          - AES256
  Automatic: true
  MaximumAutomaticAttempts: 3
  RetryAttemptSeconds: 60

# scripts/advanced-drift-detector.py
import boto3
import subprocess
import json
from datetime import datetime

def detect_drift():
    """Run Terraform drift detection and categorize by severity"""
    result = subprocess.run(
        ['terraform', 'plan', '-refresh-only', '-detailed-exitcode', '-json'],
        capture_output=True,
        text=True
    )

    if result.returncode == 0:
        print("✅ No drift detected")
        return None
    elif result.returncode == 2:
        # Parse JSON output to categorize drift
        drift_data = [json.loads(line) for line in result.stdout.splitlines() if line.strip()]

        critical_drift = []
        warning_drift = []
        info_drift = []

        for entry in drift_data:
            if entry.get('type') == 'resource_drift':
                resource = entry['change']['resource']
                change_type = entry['change']['action']

                # Categorize by resource type and change
                if resource['type'] == 'aws_security_group_rule':
                    if '0.0.0.0/0' in str(entry['change'].get('after', {})):
                        critical_drift.append({
                            'resource': resource['addr'],
                            'issue': 'Security group opened to internet',
                            'severity': 'CRITICAL'
                        })
                elif resource['type'] == 'aws_db_instance':
                    if entry['change'].get('after', {}).get('storage_encrypted') == False:
                        critical_drift.append({
                            'resource': resource['addr'],
                            'issue': 'Database encryption disabled',
                            'severity': 'CRITICAL'
                        })
                elif resource['type'] == 'aws_autoscaling_group':
                    info_drift.append({
                        'resource': resource['addr'],
                        'issue': 'Auto-scaling capacity changed',
                        'severity': 'INFO'
                    })
                else:
                    warning_drift.append({
                        'resource': resource['addr'],
                        'issue': 'Configuration drift detected',
                        'severity': 'WARNING'
                    })

        return {
            'critical': critical_drift,
            'warning': warning_drift,
            'info': info_drift
        }
    else:
        print(f"❌ Error running terraform plan: {result.stderr}")
        return None

def send_alert(drift_summary):
    """Send drift alerts to appropriate channels based on severity"""
    if not drift_summary:
        return

    # Critical drift: Page on-call + Slack
    if drift_summary['critical']:
        send_pagerduty_alert(drift_summary['critical'])
        send_slack_alert('#security-incidents', drift_summary['critical'], severity='critical')

    # Warning drift: Slack only
    if drift_summary['warning']:
        send_slack_alert('#infrastructure-alerts', drift_summary['warning'], severity='warning')

    # Info drift: Log to dashboard
    if drift_summary['info']:
        log_to_dashboard(drift_summary['info'])

def send_pagerduty_alert(critical_issues):
    """Trigger PagerDuty incident for critical drift"""
    import requests

    payload = {
        "routing_key": "YOUR_PAGERDUTY_ROUTING_KEY",
        "event_action": "trigger",
        "payload": {
            "summary": f"CRITICAL: {len(critical_issues)} infrastructure drift issues detected",
            "severity": "critical",
            "source": "Terraform Drift Detection",
            "custom_details": {
                "issues": critical_issues
            }
        }
    }

    requests.post('https://events.pagerduty.com/v2/enqueue', json=payload)

def send_slack_alert(channel, issues, severity='warning'):
    """Send Slack notification for drift"""
    import requests

    color_map = {
        'critical': 'danger',
        'warning': 'warning',
        'info': 'good'
    }

    payload = {
        "channel": channel,
        "attachments": [{
            "color": color_map[severity],
            "title": f"{severity.upper()}: Infrastructure Drift Detected",
            "text": f"Detected {len(issues)} drift issue(s)",
            "fields": [
                {
                    "title": issue['resource'],
                    "value": issue['issue'],
                    "short": False
                }
                for issue in issues[:5]  # Show first 5
            ],
            "footer": "Terraform Drift Detection",
            "ts": int(datetime.now().timestamp())
        }]
    }

    requests.post(SLACK_WEBHOOK_URL, json=payload)

if __name__ == '__main__':
    drift = detect_drift()
    if drift:
        send_alert(drift)

# .github/workflows/infracost.yml
name: Infracost
on: [pull_request]

jobs:
  infracost:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Infracost
        uses: infracost/actions/setup@v2
        with:
          api-key: ${{ secrets.INFRACOST_API_KEY }}

      - name: Generate Infracost JSON
        run: infracost breakdown --path . --format json --out-file infracost.json

      - name: Check Cost Threshold
        run: |
          MONTHLY_COST=$(jq '.totalMonthlyCost | tonumber' infracost.json)
          THRESHOLD=5000  # $5,000/month

          if (( $(echo "$MONTHLY_COST > $THRESHOLD" | bc -l) )); then
            echo "❌ Cost exceeds threshold: \$$MONTHLY_COST > \$$THRESHOLD"
            exit 1
          else
            echo "✅ Cost within threshold: \$$MONTHLY_COST <= \$$THRESHOLD"
          fi

      - name: Post Cost Comment to PR
        uses: infracost/actions/comment@v1
        with:
          path: infracost.json
          behavior: update