Chat-driven media dashboard with Mistral

View Markdown

Most media dashboards force non-technical users through forms, dropdowns, and modal dialogs for every operation. A conversational interface flips this: a content manager types “resize all product photos to 800x800” and a Mistral agent translates that into Ittybit task batches behind the scenes. No UI to learn, no settings to remember.

Install dependencies

npm install @mistralai/mistralai

pip install mistralai requests

Define the tools

The agent needs two tools: process_media for creating tasks and list_tasks for checking on them. This lets users both dispatch work and ask about progress in the same conversation.

const tools = [
  {
    type: "function" as const,
    function: {
      name: "process_media",
      description:
        "Create a media processing task via Ittybit. Handles resizing, " +
        "format conversion, transcoding, thumbnails, and adaptive streams. " +
        "Call this once per file that needs processing.",
      parameters: {
        type: "object",
        properties: {
          input: {
            type: "string",
            description: "URL of the source media file",
          },
          kind: {
            type: "string",
            enum: ["video", "audio", "image", "adaptive_video"],
            description: "Type of processing task",
          },
          options: {
            type: "object",
            description:
              "Processing options: width, height, format, codec, quality, start, end",
          },
        },
        required: ["input", "kind"],
      },
    },
  },
  {
    type: "function" as const,
    function: {
      name: "list_tasks",
      description:
        "List recent media processing tasks. Use this to check status, " +
        "find completed outputs, or see what's currently running.",
      parameters: {
        type: "object",
        properties: {
          status: {
            type: "string",
            enum: ["queued", "processing", "succeeded", "failed"],
            description: "Filter tasks by status",
          },
          limit: {
            type: "number",
            description: "Number of tasks to return (default 10)",
          },
        },
      },
    },
  },
];

tools = [
    {
        "type": "function",
        "function": {
            "name": "process_media",
            "description": (
                "Create a media processing task via Ittybit. Handles resizing, "
                "format conversion, transcoding, thumbnails, and adaptive streams. "
                "Call this once per file that needs processing."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "input": {
                        "type": "string",
                        "description": "URL of the source media file",
                    },
                    "kind": {
                        "type": "string",
                        "enum": ["video", "audio", "image", "adaptive_video"],
                        "description": "Type of processing task",
                    },
                    "options": {
                        "type": "object",
                        "description": (
                            "Processing options: width, height, format, "
                            "codec, quality, start, end"
                        ),
                    },
                },
                "required": ["input", "kind"],
            },
        },
    },
    {
        "type": "function",
        "function": {
            "name": "list_tasks",
            "description": (
                "List recent media processing tasks. Use this to check status, "
                "find completed outputs, or see what's currently running."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "status": {
                        "type": "string",
                        "enum": ["queued", "processing", "succeeded", "failed"],
                        "description": "Filter tasks by status",
                    },
                    "limit": {
                        "type": "number",
                        "description": "Number of tasks to return (default 10)",
                    },
                },
            },
        },
    },
]

Implement the tool handlers

Each tool maps to an Ittybit API endpoint. process_media creates a task, list_tasks fetches recent tasks with optional filters.

const ITTYBIT_BASE = "https://api.ittybit.com";

const headers = {
Authorization: `Bearer ${process.env.ITTYBIT_API_KEY}`,
"Content-Type": "application/json",
};

async function processMedia(args: {
input: string;
kind: string;
options?: Record<string, unknown>;
}) {
const res = await fetch(`${ITTYBIT_BASE}/tasks`, {
method: "POST",
headers,
body: JSON.stringify({
input: args.input,
kind: args.kind,
options: args.options ?? {},
}),
});
return res.json();
}

async function listTasks(args: {
status?: string;
limit?: number;
}) {
const params = new URLSearchParams();
if (args.status) params.set("status", args.status);
if (args.limit) params.set("limit", String(args.limit));

const res = await fetch(
`${ITTYBIT_BASE}/tasks?${params.toString()}`,
{ headers },
);
return res.json();
}

const toolHandlers: Record<string, (args: any) => Promise<any>> = {
process_media: processMedia,
list_tasks: listTasks,
};

import os
import requests

ITTYBIT_BASE = "https://api.ittybit.com"
ITTYBIT_HEADERS = {
    "Authorization": f"Bearer {os.environ['ITTYBIT_API_KEY']}",
}


def process_media(args: dict) -> dict:
    res = requests.post(
        f"{ITTYBIT_BASE}/tasks",
        headers=ITTYBIT_HEADERS,
        json={
            "input": args["input"],
            "kind": args["kind"],
            "options": args.get("options", {}),
        },
    )
    return res.json()


def list_tasks(args: dict) -> dict:
    params = {}
    if "status" in args:
        params["status"] = args["status"]
    if "limit" in args:
        params["limit"] = args["limit"]

    res = requests.get(
        f"{ITTYBIT_BASE}/tasks",
        headers=ITTYBIT_HEADERS,
        params=params,
    )
    return res.json()


tool_handlers = {
    "process_media": process_media,
    "list_tasks": list_tasks,
}

Build the agent loop

The core of the dashboard: a multi-turn conversation loop that handles tool calls from Mistral, executes them, feeds results back, and lets the model continue until it has a final answer for the user.

import { Mistral } from "@mistralai/mistralai";

const mistral = new Mistral({
apiKey: process.env.MISTRAL_API_KEY,
});

const SYSTEM_PROMPT = `You are a media operations assistant for a content team.
You help non-technical users manage their media files through conversation.

When a user asks you to process files, use the process_media tool.
When they ask about task status or recent activity, use list_tasks.
For batch operations across multiple files, call process_media once per file.

Always confirm what you did: how many tasks were created, what kind, and
what options were applied. If something fails, explain why in plain language.`;

type Message = {
role: string;
content?: string;
name?: string;
toolCalls?: any[];
toolCallId?: string;
};

async function chat(
messages: Message[],
): Promise<{ reply: string; tasks: any[] }> {
const response = await mistral.chat.complete({
model: "mistral-large-latest",
messages,
tools,
});

const choice = response.choices?.[0];
if (!choice) {
return { reply: "No response from model.", tasks: [] };
}

// If no tool calls, return the text response
if (
choice.finishReason !== "tool_calls" ||
!choice.message.toolCalls?.length
) {
return {
reply: choice.message.content as string,
tasks: [],
};
}

// Execute each tool call
const allTasks: any[] = [];
messages.push(choice.message as any);

for (const toolCall of choice.message.toolCalls) {
const handler = toolHandlers[toolCall.function.name];
const args = JSON.parse(toolCall.function.arguments);
const result = await handler(args);

    if (toolCall.function.name === "process_media") {
      allTasks.push(result);
    }

    messages.push({
      role: "tool",
      name: toolCall.function.name,
      content: JSON.stringify(result),
      toolCallId: toolCall.id,
    });

}

// Let the model continue -- it may make more tool calls
// or produce a final summary
const next = await chat(messages);
return {
reply: next.reply,
tasks: [...allTasks, ...next.tasks],
};
}

async function runAgent(userMessage: string) {
const messages: Message[] = [
{ role: "system", content: SYSTEM_PROMPT },
{ role: "user", content: userMessage },
];

return chat(messages);
}

import json
from mistralai import Mistral

client = Mistral(api_key=os.environ["MISTRAL_API_KEY"])

SYSTEM_PROMPT = (
    "You are a media operations assistant for a content team. "
    "You help non-technical users manage their media files through conversation.\n\n"
    "When a user asks you to process files, use the process_media tool. "
    "When they ask about task status or recent activity, use list_tasks. "
    "For batch operations across multiple files, call process_media once per file.\n\n"
    "Always confirm what you did: how many tasks were created, what kind, and "
    "what options were applied. If something fails, explain why in plain language."
)


def chat(messages: list) -> dict:
    response = client.chat.complete(
        model="mistral-large-latest",
        messages=messages,
        tools=tools,
    )

    choice = response.choices[0]

    if (
        choice.finish_reason != "tool_calls"
        or not choice.message.tool_calls
    ):
        return {"reply": choice.message.content, "tasks": []}

    all_tasks = []
    messages.append(choice.message)

    for tool_call in choice.message.tool_calls:
        handler = tool_handlers[tool_call.function.name]
        args = json.loads(tool_call.function.arguments)
        result = handler(args)

        if tool_call.function.name == "process_media":
            all_tasks.append(result)

        messages.append(
            {
                "role": "tool",
                "name": tool_call.function.name,
                "content": json.dumps(result),
                "tool_call_id": tool_call.id,
            }
        )

    # Let the model continue -- it may make more tool calls
    # or produce a final summary
    next_result = chat(messages)
    return {
        "reply": next_result["reply"],
        "tasks": all_tasks + next_result["tasks"],
    }


def run_agent(user_message: str) -> dict:
    messages = [
        {"role": "system", "content": SYSTEM_PROMPT},
        {"role": "user", "content": user_message},
    ]
    return chat(messages)

Multi-turn conversation

A real dashboard session looks like this. The user starts with a batch resize, checks status, then kicks off another operation — all in natural language.

// Turn 1: Batch resize product photos
const turn1 = await runAgent(
  "Resize all these product photos to 800x800:\n" +
  "- https://cdn.example.com/products/jacket-front.jpg\n" +
  "- https://cdn.example.com/products/jacket-back.jpg\n" +
  "- https://cdn.example.com/products/jacket-detail.jpg"
);
console.log(turn1.reply);
// "I've created 3 image tasks to resize your product photos to 800x800.
//  Each one will be processed as a square crop. Task IDs: task_a1, task_a2, task_a3."
console.log(`Tasks created: ${turn1.tasks.length}`);

// Turn 2: Check on progress
const turn2 = await runAgent(
"How are those resize tasks doing?"
);
console.log(turn2.reply);
// "All 3 tasks have succeeded. Your resized images are ready."

// Turn 3: Process a video
const turn3 = await runAgent(
"Now take this demo video and make a 720p MP4 and a thumbnail: " +
"https://cdn.example.com/videos/product-demo.mov"
);
console.log(turn3.reply);
// "Done — I created 2 tasks:
// 1. Video transcode to 720p MP4 (task_b1)
// 2. Thumbnail extraction as JPG (task_b2)"
console.log(`Tasks created: ${turn3.tasks.length}`);

# Turn 1: Batch resize product photos
turn1 = run_agent(
    "Resize all these product photos to 800x800:\n"
    "- https://cdn.example.com/products/jacket-front.jpg\n"
    "- https://cdn.example.com/products/jacket-back.jpg\n"
    "- https://cdn.example.com/products/jacket-detail.jpg"
)
print(turn1["reply"])
# "I've created 3 image tasks to resize your product photos to 800x800.
#  Each one will be processed as a square crop. Task IDs: task_a1, task_a2, task_a3."
print(f"Tasks created: {len(turn1['tasks'])}")

# Turn 2: Check on progress
turn2 = run_agent(
    "How are those resize tasks doing?"
)
print(turn2["reply"])
# "All 3 tasks have succeeded. Your resized images are ready."

# Turn 3: Process a video
turn3 = run_agent(
    "Now take this demo video and make a 720p MP4 and a thumbnail: "
    "https://cdn.example.com/videos/product-demo.mov"
)
print(turn3["reply"])
# "Done — I created 2 tasks:
#  1. Video transcode to 720p MP4 (task_b1)
#  2. Thumbnail extraction as JPG (task_b2)"
print(f"Tasks created: {len(turn3['tasks'])}")

For the batch resize, Mistral issues three parallel process_media calls in a single response — one per image URL. For the video request, it recognizes two distinct operations (transcode and thumbnail) and calls the tool twice with different kind values.

Persistent conversation

The examples above start fresh each turn. To maintain context across a session — so the agent remembers file URLs and task IDs from earlier messages — accumulate messages in a shared array.

class DashboardSession {
  private messages: Message[] = [
    { role: "system", content: SYSTEM_PROMPT },
  ];

async send(userMessage: string) {
this.messages.push({ role: "user", content: userMessage });

    const result = await chat(this.messages);

    this.messages.push({
      role: "assistant",
      content: result.reply,
    });

    return result;

}
}

// Usage
const session = new DashboardSession();

await session.send(
"Resize https://cdn.example.com/hero.jpg to 1200x630"
);

// The agent remembers the previous file
await session.send(
"Also make a 400x400 square version of that same image"
);

// Ask about everything so far
await session.send("Show me all my recent tasks");

class DashboardSession:
    def __init__(self):
        self.messages = [
            {"role": "system", "content": SYSTEM_PROMPT},
        ]

    def send(self, user_message: str) -> dict:
        self.messages.append(
            {"role": "user", "content": user_message}
        )

        result = chat(self.messages)

        self.messages.append(
            {"role": "assistant", "content": result["reply"]}
        )

        return result


# Usage
session = DashboardSession()

session.send(
    "Resize https://cdn.example.com/hero.jpg to 1200x630"
)

# The agent remembers the previous file
session.send(
    "Also make a 400x400 square version of that same image"
)

# Ask about everything so far
session.send("Show me all my recent tasks")