Detect objects

Grab a camera frame, ask the platform’s cloud model where the named objects are via client.platform.ai, then turn each box into a 3D point with the aligned depth image when the robot has one; RGB-only robots get 2D boxes. Edit TARGET to name what to find, for example “fire extinguishers” or “open cabinet doors”. AI covers cost and how this differs from on-robot models.

Boxes come from a general-purpose model, not a trained detector: they are approximate, labels are free text, and score is always 1.0. The 3D point is the median depth over the box, so it lands on the object’s visible surface rather than its centre. Expect it to be pulled towards the background for see-through objects (chair frames), objects partly out of frame, or objects behind something nearer; narrow the region or use a mask for those.

Python

from olo import Client
from olo.vision import camera_info_to_intrinsics, deproject_region

RGB_TOPIC = "wrist_rgbd/color_image"
# Leave empty for a colour-only camera; boxes then stay 2D.
DEPTH_TOPIC = "wrist_rgbd/depth_image"
CAMERA_INFO_TOPIC = "wrist_rgbd/camera_info"
# Name what to find, e.g. "fire extinguishers"; empty returns every notable object.
TARGET = ""

with Client() as client:
    robot = client.robot()
    image = robot.core.get_image(robot.core.topic(RGB_TOPIC, "sensor_msgs/msg/Image"))
    depth = intrinsics = None
    if DEPTH_TOPIC:
        depth = robot.core.get_image(robot.core.topic(DEPTH_TOPIC, "sensor_msgs/msg/Image"))
        camera_info = robot.core.get_latest(robot.core.topic(CAMERA_INFO_TOPIC, "sensor_msgs/msg/CameraInfo"))
        intrinsics = camera_info_to_intrinsics(camera_info)
    else:
        print("No depth topic; reporting 2D boxes only")

    # Boxes come back in this image's pixels.
    result = client.platform.ai.detect_objects(image, prompt=TARGET)
    print(f"Model: {result.model}")
    if not result.image_usable:
        print(f"Image unusable: {result.unusable_reason}")
    print(f"Found {len(result.detections)} object(s)")

    for detection in result.detections:
        x_min, y_min, x_max, y_max = detection.box.to_xyxy()
        line = f"  {detection.label}: box ({x_min:.0f}, {y_min:.0f})-({x_max:.0f}, {y_max:.0f})"
        if depth is not None and intrinsics is not None:
            try:
                # Median depth over the box, deprojected into the camera optical frame.
                observation = deproject_region(depth, detection.box, intrinsics)
                point = observation.point
                line += f" at ({point.x:.2f}, {point.y:.2f}, {point.z:.2f}) m in {observation.frame_id}"
            except ValueError:
                line += " (no valid depth in box)"
        print(line)

TypeScript

import { connect } from "olo/web";
import { cameraInfoToIntrinsics, deprojectRegion } from "olo/vision";

const RGB_TOPIC = "wrist_rgbd/color_image";
// Leave empty for a colour-only camera; boxes then stay 2D.
const DEPTH_TOPIC: string = "wrist_rgbd/depth_image";
const CAMERA_INFO_TOPIC = "wrist_rgbd/camera_info";
// Name what to find, e.g. "fire extinguishers"; empty returns every notable object.
const TARGET = "";

const client = connect();
const robot = await client.robot();
const image = await robot.core.getImage(robot.core.topic(RGB_TOPIC, "sensor_msgs/msg/Image"));
let depth;
let intrinsics;
if (DEPTH_TOPIC) {
  depth = await robot.core.getImage(robot.core.topic(DEPTH_TOPIC, "sensor_msgs/msg/Image"));
  const cameraInfo = await robot.core.getLatest(robot.core.topic(CAMERA_INFO_TOPIC, "sensor_msgs/msg/CameraInfo"));
  intrinsics = cameraInfoToIntrinsics(cameraInfo);
} else {
  console.log("No depth topic; reporting 2D boxes only");
}

// Boxes come back in this image's pixels.
const result = await client.platform.ai.detectObjects(image, { prompt: TARGET });
console.log(`Model: ${result.model}`);
if (!result.imageUsable) {
  console.log(`Image unusable: ${result.unusableReason}`);
}
console.log(`Found ${result.detections.length} object(s)`);

for (const detection of result.detections) {
  const [xMin, yMin, xMax, yMax] = detection.box.toXyxy();
  let line = `  ${detection.label}: box (${xMin.toFixed(0)}, ${yMin.toFixed(0)})-(${xMax.toFixed(0)}, ${yMax.toFixed(0)})`;
  if (depth && intrinsics) {
    try {
      // Median depth over the box, deprojected into the camera optical frame.
      const observation = deprojectRegion(depth, detection.box, intrinsics);
      const { x, y, z } = observation.point;
      line += ` at (${x.toFixed(2)}, ${y.toFixed(2)}, ${z.toFixed(2)}) m in ${observation.frameId}`;
    } catch {
      line += " (no valid depth in box)";
    }
  }
  console.log(line);
}