From 919d247934669e084e2967eee9f5aa3eed70f3ba Mon Sep 17 00:00:00 2001 From: imterah Date: Fri, 20 Jun 2025 20:36:21 -0400 Subject: [PATCH] chore: Add initial implementation of spatial virtual displays --- main.py | 6 +-- render.py | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 117 insertions(+), 11 deletions(-) diff --git a/main.py b/main.py index 107a2b2..ca51df5 100755 --- a/main.py +++ b/main.py @@ -21,7 +21,7 @@ import libunreal default_configuration: dict[str, str | int] = { "display_angle": 45, - "display_pixel_spacing": 45, + "display_spacing": 1, "display_count": 3, "allow_unsupported_devices": False, "allow_unsupported_vendors": False, @@ -164,7 +164,7 @@ def main(): # Raylib gets confused if there's multiple dri devices so we initialize the window before anything logger.info("Initializing XR headset") - pyray.set_target_fps(edid.max_refresh_rate) + pyray.set_target_fps(edid.max_refresh_rate*2) # we need more headroom... pyray.init_window(edid.max_width, edid.max_height, "UnrealXR") logger.info("Initializing virtual displays") @@ -181,7 +181,7 @@ def main(): atexit.register(lambda: card.close()) logger.info("Initialized displays. Entering rendering loop") - render_loop(edid, cards) + render_loop(edid, configuration, cards) if __name__ == "__main__": print("Welcome to UnrealXR!\n") diff --git a/render.py b/render.py index 2a07d7d..21cdffe 100644 --- a/render.py +++ b/render.py @@ -1,12 +1,21 @@ +from dataclasses import dataclass +from io import BufferedWriter +from sys import int_info +from typing import Union +import ctypes import time import math from loguru import logger +from raylib import rl import PyEvdi import pyray from libunreal import UnrealXRDisplayMetadata, MCUCallbackWrapper, start_mcu_event_listener +vertical_size = 0.0 +horizontal_sizing_constant = 1 + previous_pitch = 0.0 previous_yaw = 0.0 previous_roll = 0.0 @@ -19,6 +28,15 @@ has_gotten_pitch_callback_before = False has_gotten_yaw_callback_before = False has_gotten_roll_callback_before = False +@dataclass +class RectMetadata: + card: PyEvdi.Card + buffer_ptr: pyray.ffi.CData | None + texture: Union[pyray.Texture, None] + model: Union[pyray.Model, None] + angle: int + relative_position: int + def pitch_callback(new_pitch: float): global current_pitch global previous_pitch @@ -64,7 +82,20 @@ def text_message(message: str): def stub_brightness_function(brightness: int): pass -def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Card]): +def find_max_vertical_size(fovy_deg: float, distance: float) -> float: + fovy_rad = math.radians(fovy_deg) + return 2 * distance * math.tan(fovy_rad / 2) + +def find_optimal_horizonal_res(vertical_display_res: int, horizontal_display_res: int) -> float: + aspect_ratio = horizontal_display_res/vertical_display_res + horizontal_size = vertical_size * aspect_ratio + horizontal_size = horizontal_size * horizontal_sizing_constant + + return horizontal_size + +def render_loop(display_metadata: UnrealXRDisplayMetadata, config: dict[str, str | int], cards: list[PyEvdi.Card]): + global vertical_size + global core_mesh logger.info("Starting sensor event listener") mcu_callbacks = MCUCallbackWrapper(roll_callback, pitch_callback, yaw_callback, text_message, stub_brightness_function, stub_brightness_function) @@ -78,13 +109,16 @@ def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Ca logger.info("Initialized sensors") camera = pyray.Camera3D() - camera.position = pyray.Vector3(10.0, 10.0, 10.0) - camera.target = pyray.Vector3(0.0, 0.0, 0.0) - camera.up = pyray.Vector3(0.0, 1.0, 0.0) camera.fovy = 45.0 + + vertical_size = find_max_vertical_size(camera.fovy, 5.0) + + camera.position = pyray.Vector3(0.0, vertical_size/2, 5.0) + camera.target = pyray.Vector3(0.0, vertical_size/2, 0.0) + camera.up = pyray.Vector3(0.0, 1.0, 0.0) camera.projection = pyray.CameraProjection.CAMERA_PERSPECTIVE - cube_position = pyray.Vector3(0.0, 0.0, 0.0) + core_mesh = pyray.gen_mesh_plane(find_optimal_horizonal_res(display_metadata.max_height, display_metadata.max_width), vertical_size, 1, 1) movement_vector = pyray.Vector3() look_vector = pyray.Vector3() @@ -93,7 +127,7 @@ def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Ca has_sensor_init_delay_quirk = False sensor_init_start_time = time.time() - if "z_vector_disabled" in display_metadata.device_quirks: + if "z_vector_disabled" in display_metadata.device_quirks and bool(display_metadata.device_quirks["z_vector_disabled"]): logger.warning("QUIRK: The Z vector has been disabled for your specific device") has_z_vector_disabled_quirk = True @@ -103,6 +137,58 @@ def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Ca sensor_init_start_time = time.time() has_sensor_init_delay_quirk = True + rects: list[RectMetadata] = [] + + if int(config["display_count"]) >= 2: + display_angle = int(config["display_angle"]) + display_spacing = int(config["display_spacing"]) + total_displays = int(config["display_count"]) + + highest_possible_angle_on_both_sides = (total_displays-1)*display_angle + highest_possible_pixel_spacing_on_both_sides = (total_displays-1)*display_spacing + + for i in range(total_displays): + current_angle = (-highest_possible_angle_on_both_sides)+(display_angle*i) + current_display_spacing = (-highest_possible_pixel_spacing_on_both_sides)+(display_spacing*i) + + rect_metadata = RectMetadata(cards[i], None, None, None, current_angle, current_display_spacing) + + has_acquired_fb = False + + def fb_acquire_handler(evdi_buffer: PyEvdi.Buffer): + nonlocal has_acquired_fb + + if has_acquired_fb: + return + + has_acquired_fb = True + logger.info(f"Acquired buffer for card #{i+1} with ID {evdi_buffer.id}") + + address = ctypes.pythonapi.PyCapsule_GetPointer + address.restype = ctypes.c_void_p + address.argtypes = [ctypes.py_object, ctypes.c_char_p] + + buffer_void_ptr = address(evdi_buffer.bytes, None) + rect_metadata.buffer_ptr = pyray.ffi.cast("void *", buffer_void_ptr) + + pyray_image = pyray.Image() + + pyray_image.data = rect_metadata.buffer_ptr + pyray_image.width = display_metadata.max_width + pyray_image.height = display_metadata.max_height + pyray_image.mipmaps = 1 + pyray_image.format = pyray.PixelFormat.PIXELFORMAT_UNCOMPRESSED_R8G8B8A8 + + rect_metadata.texture = pyray.load_texture_from_image(pyray_image) + rect_metadata.model = pyray.load_model_from_mesh(core_mesh) + + pyray.set_material_texture(rect_metadata.model.materials[0], pyray.MaterialMapIndex.MATERIAL_MAP_ALBEDO, rect_metadata.texture) + + cards[i].acquire_framebuffer_handler = fb_acquire_handler + cards[i].handle_events(1000) + + rects.append(rect_metadata) + while not pyray.window_should_close(): if has_sensor_init_delay_quirk: if time.time() - sensor_init_start_time >= int(display_metadata.device_quirks["sensor_init_delay"]): @@ -111,7 +197,7 @@ def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Ca has_sensor_init_delay_quirk = False else: look_vector.x = (current_yaw-previous_yaw)*6.5 - look_vector.y = (current_pitch-previous_pitch)*6.5 + look_vector.y = -(current_pitch-previous_pitch)*6.5 if not has_z_vector_disabled_quirk: look_vector.z = (current_roll-previous_roll)*6.5 @@ -121,7 +207,27 @@ def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Ca pyray.begin_drawing() pyray.clear_background(pyray.BLACK) pyray.begin_mode_3d(camera) - pyray.draw_cube(cube_position, 2.0, 2.0, 2.0, pyray.ORANGE) + + for rect_count in range(len(rects)): + rect = rects[rect_count] + + if rect.buffer_ptr is None or rect.texture is None or rect.model is None: + continue + + cards[rect_count].handle_events(1) + pyray.update_texture(rect.texture, rect.buffer_ptr) + + pyray.draw_model_ex( + rect.model, + pyray.Vector3(0, vertical_size/2, 0), + pyray.Vector3(1, 0, 0), # rotate around X to make it vertical + 90, + pyray.Vector3(1, 1, 1), + pyray.WHITE + ) + + break + pyray.end_mode_3d() pyray.end_drawing()