[Python] Why do projected keypoints slightly deviate from ground truth in Pyrender camera...

To test a camera calibration library, I wrote a Python script using Pyrender, where I set up a pinhole camera to capture a grid circle board. The code is available at: https://github.com/tanjoe/camsim.



When pressing Enter in the running app, it saves the scene as an image and also records all keypoint pixel positions in a JSON file.

Simulation Script:

import os
import datetime
import cv2
import pyrender
import trimesh
import json
import numpy as np
import pyglet
from pyrender.constants import GLTF


def projectWorldToImage(
camera: pyrender.IntrinsicsCamera, camera_pose: np.ndarray, world_point: np.ndarray
) -> tuple[float, float]:
# Transform the point to camera coordinates
point_3d_camera = np.linalg.inv(camera_pose) @ world_point
point_3d_camera = point_3d_camera[:3]
X, Y, Z = point_3d_camera

# Project the 3D point to 2D image coordinates
x = camera.cx + (camera.fx * X) / -Z
# Flip the Y-axis to follow OpenCV convention
y = camera.cy - (camera.fy * Y) / -Z

return (x, y)


class MyViewer(pyrender.Viewer):
def __init__(
self,
scene: pyrender.Scene,
camera: pyrender.IntrinsicsCamera,
interested_points: list[np.ndarray],
viewport_size: tuple[int, int],
render_flags=None,
viewer_flags=None,
registered_keys=None,
run_in_thread=False,
**kwargs,
):
self.camera = camera
self.interested_points = interested_points
super().__init__(
scene,
viewport_size,
render_flags,
viewer_flags,
registered_keys,
run_in_thread,
**kwargs,
)

def on_key_press(self, symbol, modifiers):
if symbol == pyglet.window.key.ENTER:
timestamp = datetime.datetime.now().strftime("%H%M%S")
pyglet.image.get_buffer_manager().get_color_buffer().save(
f"output/{timestamp}.png"
)

# Get the current camera pose
camera_pose = self.scene.get_pose(self.scene.main_camera_node)
print(f"Camera pose: {camera_pose}")
np.savetxt(f"output/{timestamp}-camera_pose.txt", camera_pose)

loc_truth = []
for point in self.interested_points:
loc_truth.append(projectWorldToImage(self.camera, camera_pose, point))
with open(f"output/{timestamp}-loc_truth.json", "w") as truth_file:
json.dump(loc_truth, truth_file, indent=4)

return super().on_key_press(symbol, modifiers)


def createBoard(image_path: str) -> pyrender.Mesh:
# Create a texture from the image
image = cv2.imread(image_path)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
sampler = pyrender.Sampler(
magFilter=GLTF.NEAREST,
minFilter=None,
wrapS=GLTF.CLAMP_TO_EDGE,
wrapT=GLTF.CLAMP_TO_EDGE,
)
texture = pyrender.Texture(
sampler=sampler,
source=image,
source_channels="RGB",
width=image.shape[1],
height=image.shape[0],
)

# Create a material using the texture
material = pyrender.MetallicRoughnessMaterial(
baseColorTexture=texture,
emissiveTexture=texture,
emissiveFactor=[0.9, 0.9, 0.9],
doubleSided=False,
smooth=False,
)

# Create a plane mesh with UV coordinates
# Compute vertices based on image resolution to keep the aspect ratio
half_w = image.shape[1] / 1000.0 / 2.0
half_h = image.shape[0] / 1000.0 / 2.0
vertices = np.array(
[
[-half_w, -half_h, 0], # Bottom-left
[half_w, -half_h, 0], # Bottom-right
[-half_w, half_h, 0], # Top-left
[half_w, half_h, 0], # Top-right
],
dtype=np.float64,
)

faces = np.array(
[
[0, 1, 2], # First triangle
[1, 3, 2], # Second triangle
],
dtype=np.uint32,
)

# Define UV coordinates for texture mapping
uv_coords = np.array(
[
[0, 0], # Bottom-left
[1, 0], # Bottom-right
[0, 1], # Top-left
[1, 1], # Top-right
],
dtype=np.float64,
)

# Create a Trimesh object with vertices, faces, and UV coordinates
plane = trimesh.Trimesh(
vertices=vertices,
faces=faces,
visual=trimesh.visual.TextureVisuals(uv=uv_coords, image=image),
)

# Create a Pyrender mesh from the Trimesh object
plane_mesh = pyrender.Mesh.from_trimesh(plane, material=material, smooth=False)
return plane_mesh


def computeCircleCoordinates(json_path: str, board_mesh: pyrender.Mesh) -> np.ndarray:
coordinates: list[list[float]] = []
with open(json_path, "r") as content:
board_info = json.load(content)
image_size = board_info["image_size"]
centers = board_info["centers"]

width_ratio = board_mesh.extents[0] / image_size[0]
height_ratio = board_mesh.extents[1] / image_size[1]
start_x = board_mesh.bounds[0][0]
start_y = board_mesh.bounds[0][1]

for c in centers:
# Flip y to make it follows OpenGL convention (+Y should be upward)
c[1] = image_size[1] - c[1]
x = start_x + c[0] * width_ratio
y = start_y + c[1] * height_ratio
coordinates.append([x, y, 0])
return np.array(coordinates)


def main() -> None:
plane_mesh = createBoard("resource/board.png")
plane_pose = np.eye(4)
plane_pose[2, 3] = -10
plane_node = pyrender.Node(mesh=plane_mesh, matrix=plane_pose)

centers = computeCircleCoordinates("resource/board.json", plane_mesh)
transformed_centers = []
for center in centers:
center_homo = np.append(center, 1)
transformed_centers.append(plane_pose @ center_homo)

# Create a camera at the origin looking down the z-axis
view_width = 1920
view_height = 1080
camera = pyrender.IntrinsicsCamera(
fx=view_width, fy=view_width, cx=(view_width / 2), cy=(view_height / 2)
)
camera_pose = np.eye(4)
camera_node = pyrender.Node(camera=camera, matrix=camera_pose)

# Create a scene
scene = pyrender.Scene(bg_color=[0, 0, 0, 1])
scene.add_node(plane_node)
scene.add_node(camera_node)

# Render the scene using the interactive viewer
os.makedirs("./output", exist_ok=True)
MyViewer(
scene,
camera=camera,
interested_points=transformed_centers,
viewport_size=(view_width, view_height),
)


if __name__ == "__main__":
main()


To verify if the computed truth values are accurate, I took several images with the camera facing directly toward the board, with no rotation. Then, I wrote another script to load the captured image and overlay all keypoints on it.

Verification Script:

import os
import cv2
import json
import matplotlib.pyplot as plt
import typer
from pathlib import Path

def selectImageFromDir(img_dir: Path) -> Path:
if not img_dir.exists():
print(f"Error: '{img_dir}' directory not found.")
raise typer.Exit(code=1)

image_files = [f for f in os.listdir(img_dir) if f.endswith(".png")]
if not image_files:
print(f"Error: No .png images found in the '{img_dir}' directory.")
raise typer.Exit(code=1)

print("Available images:")
for i, file in enumerate(image_files):
print(f"{i + 1}. {file}")

while True:
try:
choice = int(input("Enter the number of the image to process: ")) - 1
if 0 <= choice < len(image_files):
image_filename = image_files[choice]
break
else:
print("Invalid choice. Please enter a number from the list.")
except ValueError:
print("Invalid input. Please enter a number.")

image_path = img_dir / image_filename
return image_path


def draw_crosses_on_image(image_path: str, truth_json_path: str):
"""Loads an image, draws circles based on JSON data, and displays it using matplotlib."""
image = cv2.imread(image_path)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)

with open(truth_json_path, "r") as f:
truth_data = json.load(f)

fig, ax = plt.subplots()
# ax.imshow(image, interpolation="nearest")
ax.imshow(image)

cross_size = 2
truth_cross_color = (0, 1, 0)
for i, center in enumerate(truth_data):
x, y = float(center[0]), float(center[1])
# Draw a cross for the truth position
ax.plot(
[x - cross_size, x + cross_size],
[y, y],
color=truth_cross_color,
linewidth=1,
)
ax.plot(
[x, x],
[y - cross_size, y + cross_size],
color=truth_cross_color,
linewidth=1,
)
ax.text(
x,
y - 5,
str(i),
color="yellow",
fontsize=8,
ha="center",
va="center",
)

ax.set_axis_off()
plt.show()


def main():
"""Draws circles on a selected image from the 'output' directory based on its corresponding JSON data."""
output_dir = Path("output")
image_path = selectImageFromDir(output_dir)
truth_json_path = output_dir / (image_path.stem + "-loc_truth.json")

if not image_path.exists():
print(f"Error: Image file '{image_path}' not found.")
raise typer.Exit(code=1)

if not truth_json_path.exists():
print(f"Error: Corresponding JSON file '{truth_json_path}' not found.")
raise typer.Exit(code=1)

draw_crosses_on_image(str(image_path), str(truth_json_path))


if __name__ == "__main__":
typer.run(main)


I expect all keypoints to align almost perfectly with the centers of circles in the image. However, there is a small deviation between them. The issue becomes more noticeable when the camera is zoomed closer to the board.

An example:



Zoom to one of the cross:



I understand that numerical errors exist when computing truth values, but since all floating-point calculations use 64-bit precision, I believe the error should be minimal and not account for the approximately 0.5-pixel difference observed.

Where might I have made a mistake?

Continue reading...