ARISE-Initiative
Camera pose does not change in offscreen rendering
System Info
Ubuntu 18.04.6
Robosuite 1.5.1
Mujoco 2.1.0
Information
I'm rendering the samples of the environment in offscreen modality since the machine I'm working on does not have a monitor. I need to move the camera position and orientation to suit my task.
Reproduction
The following is the code I'm using to do it:
import robosuite as suite
from robosuite.wrappers import GymWrapper
from typing import Tuple
import numpy as np
from scipy.spatial.transform import Rotation as R
# Function to move the camera relative to the initial position
def move_camera_relative(env, cam_id, initial_position, position_offset, orientation_offset):
# Update position
env.sim.model.cam_pos[cam_id] = initial_position + position_offset
# Update orientation using quaternion multiplication
current_quat = R.from_quat(env.sim.model.cam_quat[cam_id])
offset_quat = R.from_euler('xyz', orientation_offset, degrees=False) # Small rotations
new_orientation = current_quat * offset_quat # Combine rotations
# Set new orientation
env.sim.model.cam_quat[cam_id] = new_orientation.as_quat()
def make_env(name: str, image_size: Tuple[int, int], robot_name:str, camera_name: str, max_episode_steps: int, action_repeat: int):
# Create the Pick and Place Can environment with the Panda robot
env = suite.make(
env_name="PickPlaceCan",
robots="Panda",
render_camera= "agentview", # point of view
has_renderer=False, # Set to True to visualize
has_offscreen_renderer=True, # No screen rendering
use_camera_obs=False, # Set to True if you want images and duplicates
use_object_obs=False, # observation of the objects
renderer = "mujoco", # to avoid flipped and noisy images
camera_heights=224, # image height
camera_widths=224, # image width
horizon=max_episode_steps * action_repeat,
)
env.reset()
# Get the camera ID
cam_id = env.sim.model.camera_name2id(camera_name)
# Get initial position and orientation from model
initial_position = np.copy(env.sim.model.cam_pos[cam_id])
initial_orientation = np.copy(env.sim.model.cam_quat[cam_id]) # Model instead of Data
print(f"INITIAL CAMERA POS: {initial_position}, ORIENT: {initial_orientation}")
# Move the camera gradually
position_offset = np.array([0.9, 0.5, 0.5]) # Move up slightly
orientation_offset = np.array([0.5, 0.5, 0.5]) # Small rotation
move_camera_relative(env, cam_id, initial_position, position_offset, orientation_offset)
new_position = np.copy(env.sim.model.cam_pos[cam_id])
new_orientation = np.copy(env.sim.model.cam_quat[cam_id])
print(f"UPDATED CAMERA POS: {new_position}, ORIENT: {new_orientation}")
# Wrap the environment for Gym-style API
gym_env = GymWrapper(env)
env.step()
image = gym_env.sim.render(
width=224,
height=224,
camera_name="agentview",
)
Expected behavior
The output on the bash of the poses before and after the modification are correct
INITIAL CAMERA POS: [1. 0. 1.75], ORIENT: [0.65309797 0.27104066 0.27104066 0.65309797]
UPDATED CAMERA POS: [1.5 0.5 2.25], ORIENT: [0.68476004 0.37498769 0.50714498 0.36509163]
However, when I render and save the observation, the position and orientation of the camera are unchanged, and the rendered view is the same as the original "agentview".
Original "agentview" render
Render moving the camera with my code
Following your pointer, the updated version of the code is as follows. However, the results are always the same, rendering the scene always from the same point of view.
import robosuite as suite
from robosuite.wrappers import GymWrapper
from robosuite.utils.camera_utils import CameraMover
from typing import Tuple
import numpy as np
from scipy.spatial.transform import Rotation as R
def make_env(name: str, image_size: Tuple[int, int], robot_name:str, camera_name: str, max_episode_steps: int, action_repeat: int):
# Create the Pick and Place Can environment with the Panda robot
env = suite.make(
env_name=name,
robots=robot_name,
render_camera= camera_name, # point of view
camera_names=[camera_name],
has_renderer=False, # Set to True to visualize
has_offscreen_renderer=True, # No screen rendering
use_camera_obs=False, # Set to True if you want images and duplicates
use_object_obs=False, # observation of the objects
renderer = "mujoco", # to avoid flipped and noisy images
camera_heights=image_size[0], # image height
camera_widths=image_size[1], # image width
horizon=max_episode_steps * action_repeat,
)
env.reset()
# Get the camera ID
cam_id = env.sim.model.camera_name2id(camera_name)
# Get initial position and orientation from model
initial_position = np.copy(env.sim.model.cam_pos[cam_id])
initial_orientation = np.copy(env.sim.model.cam_quat[cam_id]) # Model instead of Data
print(f"INITIAL CAMERA POS: {initial_position}, ORIENT: {initial_orientation}")
# Move the camera gradually
position_offset = np.array([0.9, 0.5, 0.5]) # Move up slightly
orientation_offset = np.array([0.25, 0.25, 0.25, 0.25]) # Small rotation
# move camera pose
cam_mover = CameraMover(env)
cam_mover.set_camera_pose(pos=position_offset, quat=orientation_offset)
new_position = np.copy(env.sim.model.cam_pos[cam_id])
new_orientation = np.copy(env.sim.model.cam_quat[cam_id])
print(f"UPDATED CAMERA POS: {new_position}, ORIENT: {new_orientation}")
# Wrap the environment for Gym-style API
gym_env = GymWrapper(env)
env.step()
image = gym_env.sim.render(
width=224,
height=224,
camera_name="agentview",
)
Seems like the underlying camera has changed:
INITIAL CAMERA POS: [0.5 0. 1.35], ORIENT: [0.65309797 0.27104066 0.27104066 0.65309797] UPDATED CAMERA POS: [-0.0565177 1.27612 1.48796 ], ORIENT: [0.00990507 0.00687796 0.59122299 0.80641798]
Maybe test w/o the gym wrapper?
the other way is to update things in _load_arena --- example