Claude Sonnet 4.6 for Production AI Agents: Real-World Benchmarks

// OpenMOSS task dispatch — model selection layer

interface AgentTask {
  agentId: string;
  taskType: string;
  payload: Record<string, unknown>;
  toolCount: number;
  priority: 'high' | 'normal' | 'low';
}

function selectModel(task: AgentTask): string {
  // Escalate to Opus for complex tool-heavy tasks
  if (task.toolCount >= 5) return 'claude-opus-4-6';
  if (task.priority === 'high' && task.toolCount >= 4) {
    return 'claude-opus-4-6';
  }
  // Default: Sonnet for everything else
  return 'claude-sonnet-4-6';
}

async function dispatchTask(task: AgentTask) {
  const model = selectModel(task);
  const response = await fetch('http://localhost:6565/api/tasks', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ ...task, model }),
  });
  return response.json();
}

// Latency measurement wrapper for Claude API calls

import Anthropic from '@anthropic-ai/sdk';

const client = new Anthropic();

interface LatencyMetric {
  taskId: string;
  model: string;
  ttft: number;        // time to first token (ms)
  totalTime: number;   // full response time (ms)
  inputTokens: number;
  outputTokens: number;
}

async function callWithMetrics(
  taskId: string,
  model: string,
  messages: Anthropic.MessageParam[],
  tools: Anthropic.Tool[]
): Promise<{ response: Anthropic.Message; metrics: LatencyMetric }> {
  const start = performance.now();
  let ttft = 0;

  // Use streaming to capture TTFT
  const stream = await client.messages.stream({
    model,
    max_tokens: 4096,
    messages,
    tools,
  });

  stream.on('text', () => {
    if (ttft === 0) ttft = performance.now() - start;
  });

  const response = await stream.finalMessage();
  const totalTime = performance.now() - start;

  const metrics: LatencyMetric = {
    taskId,
    model,
    ttft: Math.round(ttft),
    totalTime: Math.round(totalTime),
    inputTokens: response.usage.input_tokens,
    outputTokens: response.usage.output_tokens,
  };

  // Log to Supabase for trend analysis
  await logMetric(metrics);

  return { response, metrics };
}

// Cost tracking per agent task

// Pricing as of March 2026 (per million tokens)
const PRICING = {
  'claude-sonnet-4-6': { input: 3.00, output: 15.00 },
  'claude-opus-4-6':   { input: 15.00, output: 75.00 },
} as const;

type ModelKey = keyof typeof PRICING;

function calculateCost(
  model: ModelKey,
  inputTokens: number,
  outputTokens: number
): number {
  const rates = PRICING[model];
  const inputCost = (inputTokens / 1_000_000) * rates.input;
  const outputCost = (outputTokens / 1_000_000) * rates.output;
  return inputCost + outputCost;
}

// Track daily spend per agent
async function trackAgentSpend(
  agentId: string,
  model: ModelKey,
  usage: { input_tokens: number; output_tokens: number }
) {
  const cost = calculateCost(model, usage.input_tokens, usage.output_tokens);
  await supabase.from('agent_costs').insert({
    agent_id: agentId,
    model,
    input_tokens: usage.input_tokens,
    output_tokens: usage.output_tokens,
    cost_usd: cost,
    recorded_at: new Date().toISOString(),
  });
}

// Tool call retry logic with failure classification

type ToolCallFailure =
  | 'wrong_tool'
  | 'malformed_args'
  | 'missing_call'
  | 'hallucinated_tool';

function classifyToolFailure(
  response: Anthropic.Message,
  expectedTools: string[]
): ToolCallFailure | null {
  const toolUses = response.content.filter(
    (b): b is Anthropic.ToolUseBlock => b.type === 'tool_use'
  );

  // No tool call when one was expected
  if (toolUses.length === 0) return 'missing_call';

  for (const toolUse of toolUses) {
    // Tool name not in expected list
    if (!expectedTools.includes(toolUse.name)) {
      return expectedTools.some(t => t.includes(toolUse.name.split('_')[0]))
        ? 'wrong_tool'
        : 'hallucinated_tool';
    }

    // Validate JSON arguments
    try {
      const args = toolUse.input as Record<string, unknown>;
      if (typeof args !== 'object' || args === null) return 'malformed_args';
    } catch {
      return 'malformed_args';
    }
  }

  return null; // success
}

async function callWithRetry(
  params: Anthropic.MessageCreateParams,
  expectedTools: string[],
  maxRetries = 2
): Promise<Anthropic.Message> {
  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    const response = await client.messages.create(params);
    const failure = classifyToolFailure(response, expectedTools);

    if (!failure) return response;

    if (attempt === maxRetries) {
      throw new Error(`Tool call failed after ${maxRetries} retries: ${failure}`);
    }

    // Add corrective message for next attempt
    const correction = failure === 'missing_call'
      ? 'You must use one of the provided tools to respond. Do not answer in plain text.'
      : `Tool call failed (${failure}). Please try again with the correct tool and valid arguments.`;

    params.messages = [
      ...params.messages,
      { role: 'assistant', content: response.content },
      { role: 'user', content: correction },
    ];
  }

  throw new Error('Unreachable');
}

// Model routing with confidence-based escalation

interface RoutingContext {
  taskType: string;
  toolCount: number;
  isHighStakes: boolean;    // irreversible action?
  isAmbiguous: boolean;     // instructions unclear?
  dailyVolume: number;      // tasks/day for this type
  priorFailureRate: number; // from metrics DB
}

function routeToModel(ctx: RoutingContext): 'claude-sonnet-4-6' | 'claude-opus-4-6' {
  // Hard rules: always Opus
  if (ctx.toolCount >= 5) return 'claude-opus-4-6';
  if (ctx.isHighStakes && ctx.toolCount >= 3) return 'claude-opus-4-6';
  if (ctx.isAmbiguous) return 'claude-opus-4-6';

  // Escalate based on observed failure rate
  if (ctx.priorFailureRate > 0.08) return 'claude-opus-4-6';

  // Default: Sonnet for everything else
  return 'claude-sonnet-4-6';
}

// In practice: start with Sonnet, escalate on failure
async function callWithEscalation(
  task: AgentTask,
  params: Anthropic.MessageCreateParams
): Promise<Anthropic.Message> {
  const initialModel = 'claude-sonnet-4-6';

  try {
    return await callWithRetry({ ...params, model: initialModel }, task.expectedTools, 1);
  } catch {
    // Escalate to Opus on repeated Sonnet failure
    console.log(`Escalating ${task.taskId} to Opus after Sonnet failure`);
    return await callWithRetry({ ...params, model: 'claude-opus-4-6' }, task.expectedTools, 2);
  }
}