Security Checklists

Checklists ensure nothing gets missed in the rush to deploy. This lesson provides comprehensive security checklists for development and pre-deployment verification.

Development Phase Checklist

┌─────────────────────────────────────────────────────────────┐
│                 Development Security Checklist               │
│                                                             │
│ ☐ Input Validation                                          │
│   ☐ Length limits enforced                                  │
│   ☐ Pattern detection for injection attempts                │
│   ☐ Content classification enabled                          │
│   ☐ Rate limiting implemented                               │
│                                                             │
│ ☐ Output Handling                                           │
│   ☐ HTML/XSS sanitization                                   │
│   ☐ PII detection and redaction                             │
│   ☐ Content moderation active                               │
│   ☐ Response length limits                                  │
│                                                             │
│ ☐ System Prompt Protection                                  │
│   ☐ No sensitive data in prompts                            │
│   ☐ Leak detection implemented                              │
│   ☐ Prompt boundaries clearly marked                        │
└─────────────────────────────────────────────────────────────┘

Automated Checklist Verification

from dataclasses import dataclass
from typing import List, Optional, Callable
from enum import Enum

class CheckStatus(Enum):
    PASS = "pass"
    FAIL = "fail"
    WARN = "warn"
    SKIP = "skip"

@dataclass
class CheckItem:
    id: str
    name: str
    description: str
    category: str
    check_function: Optional[Callable[[], bool]] = None
    severity: str = "high"  # high, medium, low

@dataclass
class CheckResult:
    item: CheckItem
    status: CheckStatus
    message: str

class SecurityChecklist:
    """Automated security checklist verification."""

    def __init__(self):
        self.items: List[CheckItem] = []
        self.results: List[CheckResult] = []

    def add_item(self, item: CheckItem):
        """Add a check item to the list."""
        self.items.append(item)

    def run_all_checks(self) -> List[CheckResult]:
        """Run all security checks."""
        self.results = []

        for item in self.items:
            result = self._run_check(item)
            self.results.append(result)

        return self.results

    def _run_check(self, item: CheckItem) -> CheckResult:
        """Run a single check."""
        if item.check_function is None:
            return CheckResult(
                item=item,
                status=CheckStatus.SKIP,
                message="No automated check available"
            )

        try:
            passed = item.check_function()
            return CheckResult(
                item=item,
                status=CheckStatus.PASS if passed else CheckStatus.FAIL,
                message="Check passed" if passed else "Check failed"
            )
        except Exception as e:
            return CheckResult(
                item=item,
                status=CheckStatus.FAIL,
                message=f"Error during check: {str(e)}"
            )

    def get_summary(self) -> dict:
        """Get check results summary."""
        return {
            "total": len(self.results),
            "passed": sum(1 for r in self.results if r.status == CheckStatus.PASS),
            "failed": sum(1 for r in self.results if r.status == CheckStatus.FAIL),
            "warnings": sum(1 for r in self.results if r.status == CheckStatus.WARN),
            "skipped": sum(1 for r in self.results if r.status == CheckStatus.SKIP),
        }

    def generate_report(self) -> str:
        """Generate markdown report."""
        lines = ["# Security Checklist Report\n"]

        # Group by category
        by_category = {}
        for result in self.results:
            cat = result.item.category
            if cat not in by_category:
                by_category[cat] = []
            by_category[cat].append(result)

        for category, results in by_category.items():
            lines.append(f"\n## {category}\n")
            for result in results:
                icon = {
                    CheckStatus.PASS: "✅",
                    CheckStatus.FAIL: "❌",
                    CheckStatus.WARN: "⚠️",
                    CheckStatus.SKIP: "⏭️",
                }[result.status]

                lines.append(f"- {icon} **{result.item.name}**")
                lines.append(f"  - {result.message}")

        # Summary
        summary = self.get_summary()
        lines.append(f"\n## Summary\n")
        lines.append(f"- Total: {summary['total']}")
        lines.append(f"- Passed: {summary['passed']}")
        lines.append(f"- Failed: {summary['failed']}")
        lines.append(f"- Warnings: {summary['warnings']}")

        return "\n".join(lines)

Pre-Deployment Checklist

def create_pre_deployment_checklist(config: dict) -> SecurityChecklist:
    """Create pre-deployment security checklist."""
    checklist = SecurityChecklist()

    # Input Validation Checks
    checklist.add_item(CheckItem(
        id="input-001",
        name="Input Length Limits",
        description="Maximum input length is enforced",
        category="Input Validation",
        check_function=lambda: config.get("max_input_length", 0) > 0,
        severity="high"
    ))

    checklist.add_item(CheckItem(
        id="input-002",
        name="Injection Pattern Detection",
        description="Patterns for detecting injection attempts are configured",
        category="Input Validation",
        check_function=lambda: len(config.get("injection_patterns", [])) > 0,
        severity="high"
    ))

    checklist.add_item(CheckItem(
        id="input-003",
        name="Rate Limiting",
        description="Rate limiting is enabled and configured",
        category="Input Validation",
        check_function=lambda: config.get("rate_limit_enabled", False),
        severity="high"
    ))

    # Output Security Checks
    checklist.add_item(CheckItem(
        id="output-001",
        name="XSS Sanitization",
        description="Output is sanitized for HTML contexts",
        category="Output Security",
        check_function=lambda: config.get("sanitize_html", False),
        severity="high"
    ))

    checklist.add_item(CheckItem(
        id="output-002",
        name="PII Redaction",
        description="PII patterns are detected and redacted",
        category="Output Security",
        check_function=lambda: config.get("pii_redaction", False),
        severity="high"
    ))

    checklist.add_item(CheckItem(
        id="output-003",
        name="Content Moderation",
        description="Content moderation is active",
        category="Output Security",
        check_function=lambda: config.get("content_moderation", False),
        severity="medium"
    ))

    # Guardrails Checks
    checklist.add_item(CheckItem(
        id="guard-001",
        name="NeMo Guardrails Configured",
        description="Guardrail configuration files exist and are valid",
        category="Guardrails",
        severity="high"
    ))

    checklist.add_item(CheckItem(
        id="guard-002",
        name="Safety Classifier Enabled",
        description="LLaMA Guard or similar classifier is active",
        category="Guardrails",
        check_function=lambda: config.get("safety_classifier", False),
        severity="high"
    ))

    # Monitoring Checks
    checklist.add_item(CheckItem(
        id="monitor-001",
        name="Logging Enabled",
        description="Security events are being logged",
        category="Monitoring",
        check_function=lambda: config.get("logging_enabled", False),
        severity="high"
    ))

    checklist.add_item(CheckItem(
        id="monitor-002",
        name="Alerting Configured",
        description="Alerts are configured for security events",
        category="Monitoring",
        check_function=lambda: config.get("alerting_enabled", False),
        severity="medium"
    ))

    # Authentication Checks
    checklist.add_item(CheckItem(
        id="auth-001",
        name="API Authentication",
        description="API endpoints require authentication",
        category="Authentication",
        check_function=lambda: config.get("require_auth", False),
        severity="high"
    ))

    checklist.add_item(CheckItem(
        id="auth-002",
        name="API Keys Secured",
        description="API keys are stored securely (not in code)",
        category="Authentication",
        severity="high"
    ))

    return checklist

# Usage
app_config = {
    "max_input_length": 4000,
    "injection_patterns": ["ignore.*instructions", "you are now"],
    "rate_limit_enabled": True,
    "sanitize_html": True,
    "pii_redaction": True,
    "content_moderation": True,
    "safety_classifier": True,
    "logging_enabled": True,
    "alerting_enabled": True,
    "require_auth": True,
}

checklist = create_pre_deployment_checklist(app_config)
results = checklist.run_all_checks()
report = checklist.generate_report()
print(report)

Quick Reference Checklist

Category	Check	Priority
Input	Length limits enforced	High
Input	Injection patterns blocked	High
Input	Rate limiting active	High
Output	HTML sanitization	High
Output	PII redaction enabled	High
Output	Content moderation	Medium
Guardrails	NeMo/LLaMA Guard active	High
Guardrails	Topic rails configured	Medium
Monitoring	Security logging	High
Monitoring	Alerting configured	Medium
Auth	API authentication required	High
Auth	Secrets management	High
Agent	Least privilege permissions	High
Agent	Tool sandboxing	High
Agent	Human approval for actions	Medium

Key Takeaway: Use checklists before every deployment. Automate checks where possible, but always review critical items manually. :::