Describe an image

Grab one camera frame, describe it with the platform’s cloud model via client.platform.ai, then ask a question about the same frame with prompt. Edit QUESTION to ask your own. Check image_usable before acting on a result. AI covers cost and how this differs from on-robot models.

Python

from olo import Client
from olo.platform import ImageDescription

RGB_TOPIC = "wrist_rgbd/color_image"
# Edit to ask your own question about the frame.
QUESTION = "Which objects are closest to the camera?"


def print_result(title: str, result: ImageDescription) -> None:
    print(title)
    if result.image_usable:
        print(f"  {result.description}")
    else:
        print(f"  Image unusable: {result.unusable_reason}")


with Client() as client:
    robot = client.robot()
    rgb_topic = robot.core.topic(RGB_TOPIC, "sensor_msgs/msg/Image")
    image = robot.core.get_image(rgb_topic)

    scene = client.platform.ai.describe_image(image)
    print(f"Model: {scene.model}")
    print_result("Scene:", scene)

    answer = client.platform.ai.describe_image(image, prompt=QUESTION)
    print_result(f"Q: {QUESTION}", answer)

TypeScript

import type { ImageDescription } from "olo";
import { connect } from "olo/web";

const RGB_TOPIC = "wrist_rgbd/color_image";
// Edit to ask your own question about the frame.
const QUESTION = "Which objects are closest to the camera?";

function printResult(title: string, result: ImageDescription): void {
  console.log(title);
  if (result.imageUsable) {
    console.log(`  ${result.description}`);
  } else {
    console.log(`  Image unusable: ${result.unusableReason}`);
  }
}

const client = connect();
const robot = await client.robot();
const rgbTopic = robot.core.topic(RGB_TOPIC, "sensor_msgs/msg/Image");
const image = await robot.core.getImage(rgbTopic);

const scene = await client.platform.ai.describeImage(image);
console.log(`Model: ${scene.model}`);
printResult("Scene:", scene);

const answer = await client.platform.ai.describeImage(image, { prompt: QUESTION });
printResult(`Q: ${QUESTION}`, answer);