Prompt Caching & Cost Control

lib/docs-bot.ts

import Anthropic from '@anthropic-ai/sdk';

const client = new Anthropic();
const PRODUCT_DOCS = '...10,000 tokens of help-center articles...'; // stable, reused every request
const question = 'How do I reset my password?';

const response = await client.messages.create({
  model: 'claude-sonnet-5',
  max_tokens: 1024,
  system: [
    { type: 'text', text: 'You answer questions about Taco Tracker using only the docs below.' },
    {
      type: 'text',
      text: PRODUCT_DOCS,
      cache_control: { type: 'ephemeral' }, // cache everything up to and including this block
    },
  ],
  messages: [{ role: 'user', content: question }], // changes every time: goes AFTER the breakpoint
});

const u = response.usage;
console.log({
  uncached: u.input_tokens,
  cacheWrite: u.cache_creation_input_tokens, // first request: written to cache
  cacheRead: u.cache_read_input_tokens,      // later requests: should be > 0
});

Choosing a TTL

// Default: 5-minute cache. Good for chat bursts.
cache_control: { type: 'ephemeral' }

// 1-hour cache. Costs more to write, pays off when traffic is spread out.
cache_control: { type: 'ephemeral', ttl: '1h' }

Automatic caching

// Don't want to pick breakpoints? One top-level cache_control
// caches up to the last cacheable block automatically.
const response = await client.messages.create({
  model: 'claude-sonnet-5',
  max_tokens: 1024,
  cache_control: { type: 'ephemeral' },
  system: PRODUCT_DOCS,
  messages: [{ role: 'user', content: question }],
});

lib/docs-bot-aisdk.ts

import { anthropic } from '@ai-sdk/anthropic';
import { generateText } from 'ai';

const PRODUCT_DOCS = '...';
const question = 'How do I reset my password?';

const result = await generateText({
  model: anthropic('claude-sonnet-5'),
  instructions: {
    role: 'system',
    content: `Answer using only these docs:\n\n${PRODUCT_DOCS}`,
    providerOptions: {
      anthropic: { cacheControl: { type: 'ephemeral' } }, // breakpoint on the system prompt
    },
  },
  prompt: question,
});

console.log('cache read tokens:', result.usage.inputTokenDetails.cacheReadTokens);
console.log('cache write tokens:', result.usage.inputTokenDetails.cacheWriteTokens);

lib/run-task.ts

import { generateText } from 'ai';

type Task = 'classify' | 'summarize' | 'plan-refactor';

// Cheap, fast model for easy jobs; big model only where it pays off.
const MODEL_FOR: Record<Task, string> = {
  classify: 'anthropic/claude-haiku-4.5',
  summarize: 'anthropic/claude-sonnet-5',
  'plan-refactor': 'anthropic/claude-opus-5.5',
};

export async function runTask(task: Task, input: string, userId: string) {
  const { text, usage } = await generateText({
    model: MODEL_FOR[task],
    prompt: input,
    maxOutputTokens: task === 'classify' ? 50 : 2000, // cap output: output tokens cost the most
    providerOptions: { gateway: { user: userId, tags: [task] } }, // spend per user and per feature
  });
  console.log(task, usage.inputTokens, usage.outputTokens);
  return text;
}

Terminal

# Cap AI Gateway spend for one project (pick your own number)
vercel ai-gateway budgets set project my-project --limit 200

# See what's configured
vercel ai-gateway budgets ls