Claude tool use for media processing

View Markdown

Claude can transcode, resize, and reformat media files mid-conversation when you wire up Ittybit’s Task API as a tool. A user says “make that video a 720p MP4” and Claude calls Ittybit to do it.

Define the tool

Describe a process_media tool with an input_schema that matches Ittybit’s task body. Claude uses this schema to decide when and how to call the tool.

const tools = [
  {
    name: "process_media",
    description:
      "Process a media file (video, audio, or image). " +
      "Transcode, resize, change format, adjust quality.",
    input_schema: {
      type: "object",
      properties: {
        kind: {
          type: "string",
          enum: ["video", "audio", "image"],
          description: "The type of media to process",
        },
        input: {
          type: "string",
          description: "URL of the source media file",
        },
        options: {
          type: "object",
          properties: {
            width: { type: "number", description: "Output width in pixels" },
            height: { type: "number", description: "Output height in pixels" },
            format: {
              type: "string",
              description: "Output format (mp4, webm, webp, png, mp3, etc.)",
            },
            codec: { type: "string", description: "Codec (h264, h265, av1, etc.)" },
            quality: {
              type: "string",
              enum: ["low", "medium", "high"],
              description: "Output quality preset",
            },
          },
        },
      },
      required: ["kind", "input"],
    },
  },
];

tools = [
    {
        "name": "process_media",
        "description": (
            "Process a media file (video, audio, or image). "
            "Transcode, resize, change format, adjust quality."
        ),
        "input_schema": {
            "type": "object",
            "properties": {
                "kind": {
                    "type": "string",
                    "enum": ["video", "audio", "image"],
                    "description": "The type of media to process",
                },
                "input": {
                    "type": "string",
                    "description": "URL of the source media file",
                },
                "options": {
                    "type": "object",
                    "properties": {
                        "width": {"type": "number", "description": "Output width in pixels"},
                        "height": {"type": "number", "description": "Output height in pixels"},
                        "format": {
                            "type": "string",
                            "description": "Output format (mp4, webm, webp, png, mp3, etc.)",
                        },
                        "codec": {"type": "string", "description": "Codec (h264, h265, av1, etc.)"},
                        "quality": {
                            "type": "string",
                            "enum": ["low", "medium", "high"],
                            "description": "Output quality preset",
                        },
                    },
                },
            },
            "required": ["kind", "input"],
        },
    },
]

Send a message with tools

Pass the tools array when calling the Messages API. Claude will decide whether to use the tool based on the conversation.

import Anthropic from "@anthropic-ai/sdk";

const client = new Anthropic();

const response = await client.messages.create({
model: "claude-sonnet-4-20250514",
max_tokens: 1024,
tools,
messages: [
{
role: "user",
content:
"Here's a video: https://example.com/video.mov — can you convert it to a 720p MP4?",
},
],
});

import anthropic

client = anthropic.Anthropic()

response = client.messages.create(
    model="claude-sonnet-4-20250514",
    max_tokens=1024,
    tools=tools,
    messages=[
        {
            "role": "user",
            "content": "Here's a video: https://example.com/video.mov — can you convert it to a 720p MP4?",
        },
    ],
)

Handle tool use

When Claude decides to use the tool, the response contains a tool_use content block. Extract the input, call Ittybit, and return the result.

for (const block of response.content) {
  if (block.type === "tool_use" && block.name === "process_media") {
    // Call Ittybit with Claude's chosen parameters
    const task = await fetch("https://api.ittybit.com/jobs", {
      method: "POST",
      headers: {
        Authorization: `Bearer ${process.env.ITTYBIT_API_KEY}`,
        "Content-Type": "application/json",
      },
      body: JSON.stringify(block.input),
    });
    const result = await task.json();

    // Send the result back to Claude
    const followUp = await client.messages.create({
      model: "claude-sonnet-4-20250514",
      max_tokens: 1024,
      tools,
      messages: [
        {
          role: "user",
          content:
            "Here's a video: https://example.com/video.mov — can you convert it to a 720p MP4?",
        },
        { role: "assistant", content: response.content },
        {
          role: "user",
          content: [
            {
              type: "tool_result",
              tool_use_id: block.id,
              content: JSON.stringify(result),
            },
          ],
        },
      ],
    });

    console.log(followUp.content);

}
}

for block in response.content:
    if block.type == "tool_use" and block.name == "process_media":
        # Call Ittybit with Claude's chosen parameters
        import requests

        task = requests.post(
            "https://api.ittybit.com/jobs",
            headers={"Authorization": f"Bearer {ITTYBIT_API_KEY}"},
            json=block.input,
        )
        result = task.json()

        # Send the result back to Claude
        follow_up = client.messages.create(
            model="claude-sonnet-4-20250514",
            max_tokens=1024,
            tools=tools,
            messages=[
                {
                    "role": "user",
                    "content": "Here's a video: https://example.com/video.mov — can you convert it to a 720p MP4?",
                },
                {"role": "assistant", "content": response.content},
                {
                    "role": "user",
                    "content": [
                        {
                            "type": "tool_result",
                            "tool_use_id": block.id,
                            "content": json.dumps(result),
                        },
                    ],
                },
            ],
        )

        print(follow_up.content)

What Claude sends to Ittybit

Given the prompt above, Claude produces a tool call like this:

{
  "kind": "video",
  "input": "https://example.com/video.mov",
  "options": {
    "width": 1280,
    "height": 720,
    "format": "mp4",
    "codec": "h264"
  }
}

That goes straight to POST /jobs. The response comes back as:

{
  "id": "task_abc123",
  "object": "task",
  "kind": "video",
  "status": "queued"
}

Claude then summarizes the result for the user: the task is queued, here’s the ID, it’ll be ready shortly.

Polling for completion

Tasks run asynchronously. If you want Claude to report the final output URL, add a second tool that polls for task status.

const allTools = [
  ...tools,
  {
    name: "check_task",
    description: "Check the status of a media processing task",
    input_schema: {
      type: "object",
      properties: {
        task_id: { type: "string", description: "The task ID to check" },
      },
      required: ["task_id"],
    },
  },
];

// When Claude calls check_task:
const status = await fetch(
`https://api.ittybit.com/jobs/${taskId}`,
{
headers: {
Authorization: `Bearer ${process.env.ITTYBIT_API_KEY}`,
},
},
);
const result = await status.json();