Red Team Toolkit Overview

Professional AI red teaming requires specialized tools. This lesson introduces the major open-source frameworks available in 2025, helping you choose the right tool for your testing needs.

The Modern AI Red Team Arsenal

┌─────────────────────────────────────────────────────────────┐
│                    AI Red Team Tools (2025)                  │
├─────────────────────────────────────────────────────────────┤
│                                                             │
│  ┌─────────────┐  ┌─────────────┐  ┌─────────────┐         │
│  │  DeepTeam   │  │   Garak     │  │   PyRIT     │         │
│  │ (Confident) │  │  (NVIDIA)   │  │ (Microsoft) │         │
│  ├─────────────┤  ├─────────────┤  ├─────────────┤         │
│  │ 40+ vulns   │  │ 100+ probes │  │ Multi-turn  │         │
│  │ 10+ attacks │  │ CLI-first   │  │ Orchestrate │         │
│  │ Python API  │  │ Extensible  │  │ Azure focus │         │
│  └─────────────┘  └─────────────┘  └─────────────┘         │
│                                                             │
│  ┌─────────────────────────────────────────────────┐       │
│  │              Promptfoo (Fuzzing)                 │       │
│  ├─────────────────────────────────────────────────┤       │
│  │ • YAML-based test cases • CI/CD integration     │       │
│  │ • Provider agnostic • Assertion library         │       │
│  └─────────────────────────────────────────────────┘       │
└─────────────────────────────────────────────────────────────┘

Tool Comparison Matrix

Tool	Strengths	Best For	Learning Curve
DeepTeam	Comprehensive vulns, Python API	General red teaming	Medium
Garak	100+ attack probes, CLI-first	Quick scanning	Low
PyRIT	Multi-turn orchestration	Advanced attacks	High
Promptfoo	CI/CD integration, YAML config	Continuous testing	Low

Installation Overview

All tools are pip-installable and work on Windows, macOS, and Linux:

# Install all red teaming tools
# Run in your terminal (any platform):
# pip install deepteam garak pyrit promptfoo

# Verify installations
import subprocess
import sys

def verify_installations():
    """Check that all tools are installed correctly."""
    tools = {
        "deepteam": "deepteam",
        "garak": "garak",
        "pyrit": "pyrit",
    }

    results = {}
    for name, package in tools.items():
        try:
            __import__(package)
            results[name] = "Installed"
        except ImportError:
            results[name] = "Not found"

    return results

# Check your setup
status = verify_installations()
for tool, state in status.items():
    print(f"{tool}: {state}")

Choosing the Right Tool

from enum import Enum
from dataclasses import dataclass
from typing import List

class TestingGoal(Enum):
    QUICK_SCAN = "quick_scan"
    COMPREHENSIVE = "comprehensive"
    MULTI_TURN = "multi_turn"
    CI_CD = "ci_cd"

@dataclass
class ToolRecommendation:
    """Recommend tools based on testing goals."""
    goal: TestingGoal
    primary_tool: str
    secondary_tools: List[str]
    rationale: str

def recommend_tool(goal: TestingGoal) -> ToolRecommendation:
    """Get tool recommendation for your goal."""
    recommendations = {
        TestingGoal.QUICK_SCAN: ToolRecommendation(
            goal=goal,
            primary_tool="Garak",
            secondary_tools=["Promptfoo"],
            rationale="CLI-first with 100+ probes for rapid assessment"
        ),
        TestingGoal.COMPREHENSIVE: ToolRecommendation(
            goal=goal,
            primary_tool="DeepTeam",
            secondary_tools=["Garak", "PyRIT"],
            rationale="40+ vulnerability classes with Python API"
        ),
        TestingGoal.MULTI_TURN: ToolRecommendation(
            goal=goal,
            primary_tool="PyRIT",
            secondary_tools=["DeepTeam"],
            rationale="Orchestrates complex multi-turn attack chains"
        ),
        TestingGoal.CI_CD: ToolRecommendation(
            goal=goal,
            primary_tool="Promptfoo",
            secondary_tools=["Garak"],
            rationale="YAML config, easy pipeline integration"
        ),
    }
    return recommendations[goal]

# Example: Comprehensive testing
rec = recommend_tool(TestingGoal.COMPREHENSIVE)
print(f"Primary: {rec.primary_tool}")
print(f"Why: {rec.rationale}")

Environment Setup Best Practices

from pathlib import Path
import os

def setup_red_team_environment():
    """Configure environment for red teaming."""
    # Create project structure (cross-platform)
    project_root = Path.cwd() / "red-team-project"
    directories = [
        project_root / "attacks",
        project_root / "results",
        project_root / "reports",
        project_root / "configs",
    ]

    for directory in directories:
        directory.mkdir(parents=True, exist_ok=True)
        print(f"Created: {directory}")

    # Create .env template
    env_template = project_root / ".env.example"
    env_content = """# API Keys for target models
OPENAI_API_KEY=your-key-here
ANTHROPIC_API_KEY=your-key-here
AZURE_OPENAI_ENDPOINT=your-endpoint

# Red team configuration
RED_TEAM_LOG_LEVEL=INFO
RED_TEAM_OUTPUT_DIR=./results
"""
    env_template.write_text(env_content)
    print(f"Created: {env_template}")

    return project_root

# Setup your environment
project = setup_red_team_environment()

Key Insight: Start with Garak for quick scans, graduate to DeepTeam for comprehensive testing, and use PyRIT for advanced multi-turn attacks.

Next, we'll install and configure DeepTeam for vulnerability scanning. :::