implementing-ot-incident-response-playbook

What this skill does

# Implementing OT Incident Response Playbook

## When to Use

- When building OT-specific incident response procedures for the first time
- When existing IT IR playbooks do not address ICS/SCADA-specific requirements
- When preparing for OT ransomware scenarios like EKANS or LockerGoga
- When aligning IR procedures with IEC 62443 and NERC CIP incident reporting requirements
- When conducting post-incident reviews to improve OT IR capabilities

**Do not use** for IT-only incident response without OT components (use standard NIST 800-61 playbooks), for day-to-day OT security monitoring (see implementing-dragos-platform-for-ot-monitoring), or for tabletop exercise design (see performing-ics-tabletop-exercise).

## Prerequisites

- OT asset inventory with criticality ratings and safety system identification
- Defined roles: OT IR Lead, IT SOC Analyst, Plant Operations Manager, Process Safety Engineer
- Communication plan including out-of-band channels (OT incidents may compromise IT communications)
- Known-good backups of PLC programs, HMI configurations, and historian data
- Contact information for ICS vendors, Dragos/Claroty support, and CISA ICS-CERT

## Workflow

### Step 1: Define OT-Specific Incident Classification and Response Procedures

```python
#!/usr/bin/env python3
"""OT Incident Response Playbook Engine.

Implements structured OT incident response procedures following
SANS PICERL lifecycle with ICS-specific considerations for safety,
availability, and cross-team coordination.
"""

import json
import sys
from datetime import datetime
from enum import Enum
from typing import Dict, List, Optional


class OTIncidentSeverity(Enum):
    SEV1_SAFETY = "SEV1-SAFETY"  # Safety system compromise
    SEV2_PROCESS = "SEV2-PROCESS"  # Active process manipulation
    SEV3_ACCESS = "SEV3-ACCESS"  # Unauthorized OT access
    SEV4_RECON = "SEV4-RECON"  # Reconnaissance in OT network
    SEV5_IT_SPILLOVER = "SEV5-IT-SPILLOVER"  # IT incident with OT exposure


class OTIncidentCategory(Enum):
    RANSOMWARE = "ransomware"
    MALWARE_ICS = "malware_ics_specific"
    UNAUTHORIZED_ACCESS = "unauthorized_ot_access"
    PROCESS_MANIPULATION = "process_manipulation"
    SIS_COMPROMISE = "safety_system_compromise"
    DATA_EXFILTRATION = "ot_data_exfiltration"
    SUPPLY_CHAIN = "supply_chain_compromise"
    INSIDER_THREAT = "insider_threat"


# PICERL phase definitions for OT
PICERL_PHASES = {
    "preparation": {
        "description": "Readiness activities before an incident occurs",
        "ot_specific": [
            "Maintain offline backups of all PLC programs and HMI configurations",
            "Document safe shutdown procedures for each process area",
            "Establish out-of-band communication (satellite phone, analog radio)",
            "Pre-stage forensic tools that work in air-gapped OT networks",
            "Maintain spare PLCs and engineering workstations",
            "Conduct quarterly OT tabletop exercises",
        ],
    },
    "identification": {
        "description": "Detect and confirm the OT security incident",
        "ot_specific": [
            "Correlate OT IDS alerts with process anomalies from historian data",
            "Verify if process deviations are cyber-caused vs operational",
            "Check Safety Instrumented Systems (SIS) status and integrity",
            "Review engineering workstation logs for unauthorized access",
            "Examine PLC mode changes (RUN/STOP/PROGRAM transitions)",
            "Assess whether the incident is IT-only or has crossed into OT",
        ],
    },
    "containment": {
        "description": "Limit the spread and impact of the incident",
        "ot_specific": [
            "NEVER shut down OT systems without plant operations approval",
            "Isolate affected segments at the industrial firewall (not by powering off)",
            "Switch PLCs to LOCAL/MANUAL mode if remote manipulation is suspected",
            "Disconnect IT-OT conduits at the DMZ while maintaining intra-OT communication",
            "Preserve forensic evidence before any remediation actions",
            "Maintain safety system operation throughout containment",
        ],
    },
    "eradication": {
        "description": "Remove the threat from OT systems",
        "ot_specific": [
            "Compare running PLC programs against known-good backups",
            "Rebuild compromised engineering workstations from golden images",
            "Verify historian data integrity for evidence of manipulation",
            "Check for persistence mechanisms in OT-specific locations (startup scripts, scheduled tasks on HMIs)",
            "Validate firmware integrity on PLCs and RTUs",
            "Coordinate with ICS vendor for rootkit-level remediation if needed",
        ],
    },
    "recovery": {
        "description": "Restore OT operations to normal",
        "ot_specific": [
            "Restore PLC programs from verified offline backups",
            "Bring processes back online in stages with engineering oversight",
            "Monitor process variables closely during restart for anomalies",
            "Validate safety system functionality before resuming automatic operation",
            "Re-enable IT-OT connectivity only after OT is verified clean",
            "Document any process variable drift during the incident",
        ],
    },
    "lessons_learned": {
        "description": "Post-incident review and improvement",
        "ot_specific": [
            "Conduct joint IT/OT post-incident review within 2 weeks",
            "Update detection rules based on observed attack techniques",
            "Revise network segmentation if lateral movement was successful",
            "Update PLC backup schedules based on recovery time experienced",
            "Report to CISA ICS-CERT and sector ISAC as required",
            "Test updated playbook within 90 days",
        ],
    },
}


class OTIncident:
    """Represents an active OT security incident."""

    def __init__(self, title: str, severity: OTIncidentSeverity,
                 category: OTIncidentCategory, affected_systems: List[str]):
        self.id = f"OT-IR-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
        self.title = title
        self.severity = severity
        self.category = category
        self.affected_systems = affected_systems
        self.created = datetime.now().isoformat()
        self.current_phase = "identification"
        self.timeline = []
        self.decisions = []
        self.containment_actions = []

    def log_event(self, phase: str, action: str, actor: str, notes: str = ""):
        """Log an incident response action."""
        entry = {
            "timestamp": datetime.now().isoformat(),
            "phase": phase,
            "action": action,
            "actor": actor,
            "notes": notes,
        }
        self.timeline.append(entry)
        return entry

    def log_decision(self, decision: str, rationale: str, approved_by: str):
        """Log a critical decision during incident response."""
        entry = {
            "timestamp": datetime.now().isoformat(),
            "decision": decision,
            "rationale": rationale,
            "approved_by": approved_by,
        }
        self.decisions.append(entry)
        return entry


class OTPlaybookEngine:
    """Executes OT incident response playbooks."""

    def __init__(self):
        self.playbooks = self._build_playbooks()

    def _build_playbooks(self) -> Dict:
        """Build category-specific OT IR playbooks."""
        return {
            OTIncidentCategory.RANSOMWARE: {
                "name": "OT Ransomware Response",
                "reference": "SANS ICS Ransomware Defense Playbook",
                "immediate_actions": [
                    "DO NOT pay ransom without executive and legal approval",
                    "Disconnect IT-OT conduit at DMZ firewalls immediately",
                    "Verify SIS/safety syst