I am trying to visualize a webcam feed on a plane in Open3D (testable Code below). After I run the script, the memory usage starts increasing after 3-5minutes non stop. When I comment the line modify_geometry_material, it works fine (no memory increase is noticed), although the texture is not updated until I move the camera. I tried self.app.post_to_main_thread(self.vis, lambda: self.vis.modify_geometry_material(self.geometries[0]['name'], self.geometries[0]['material'])), the texture is updated accordingly, but there is an increase of memory usage after a few minutes. Any ideas to achieve this real-time VideoCapture texture on a 3D plane? import open3d as o3d import numpy as np import cv2 import open3d.visualization.rendering as rendering import open3d.visualization.gui as gui import time import threading def create_video_plane(width, height): # Vertices for a rectangle in the XY plane, centered at (0,0,0) vertices = np.array([ [-width/2, -height/2, 0], [ width/2, -height/2, 0], [ width/2, height/2, 0], [-width/2, height/2, 0] ]) triangles = np.array([ [0, 1, 2], [2, 3, 0] ]) # UV coordinates for each vertex (must match vertices order) uvs = np.array([ [0, 0], # vertex 0 [1, 0], # vertex 1 [1, 1], # vertex 2 [0, 1], # vertex 3 ]) mesh = o3d.geometry.TriangleMesh() mesh.vertices = o3d.utility.Vector3dVector(vertices) mesh.triangles = o3d.utility.Vector3iVector(triangles) mesh.triangle_uvs = o3d.utility.Vector2dVector([ [0, 0], [1, 0], [1, 1], # triangle 0 [1, 1], [0, 1], [0, 0] # triangle 1 ]) mesh.compute_vertex_normals() return mesh class TestApp: def __init__(self): self.rgb_frame_width = 3840 self.rgb_frame_height = 2160 self.ratio = self.rgb_frame_width / self.rgb_frame_height self.window_width = 1280 self.window_height = int(self.window_width / self.ratio) self.app = o3d.visualization.gui.Application.instance self.app.initialize() self.vis = o3d.visualization.O3DVisualizer("Test") self.app.add_window(self.vis) self.frame = None self.image_texture = o3d.geometry.Image(np.zeros((self.rgb_frame_height, self.rgb_frame_width, 3), dtype=np.uint8)) self.plane = create_video_plane(self.rgb_frame_width, self.rgb_frame_height) self.plane.translate([0, 0, self.rgb_frame_width/100]) self.image_added = False #VideoFrameSingleton().start(video_source=0, K=np.eye(3), dist_coeffs=np.zeros(5), fps=30, rgb_frame_width=self.rgb_frame_width, rgb_frame_height=self.rgb_frame_height) self.cap = cv2.VideoCapture(0) self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.rgb_frame_width) self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.rgb_frame_height) self.cap.set(cv2.CAP_PROP_FPS, 30) self.cap.set(cv2.CAP_PROP_AUTOFOCUS, 0) self.is_running = True self._thread = threading.Thread(target=self.video_loop, daemon=True) self._thread.start() video_material = rendering.MaterialRecord() video_material.shader = "defaultUnlit" video_material.base_color = [1.0, 1.0, 1.0, 1.0] video_material.albedo_img = self.image_texture self.geometries = [] self.geometries.append({"name":"video_plane", "geometry":self.plane, "material":video_material}) threading.Thread(target=self.update_thread, daemon=True).start() self.app.run() def video_loop(self): while self.is_running: ret, frame = self.cap.read() if ret: frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frame = np.ascontiguousarray(frame) self.frame = frame # Store only the raw frame time.sleep(1.0 / 30) def update_thread(self): while(self.frame is None): time.sleep(0.1) while(self.frame is not None): self.geometries[0]['material'].albedo_img = o3d.geometry.Image(self.frame) if not self.image_added: self.vis.add_geometry(self.geometries[0]['name'], self.geometries[0]['geometry'], self.geometries[0]['material']) self.image_added = True else: self.app.post_to_main_thread(self.vis, lambda: self.vis.modify_geometry_material(self.geometries[0]['name'], self.geometries[0]['material'])) #self.vis.modify_geometry_material(self.geometries[0]['name'], self.geometries[0]['material']) time.sleep(1/20) app = TestApp() Continue reading...
