chore: Add initial implementation of spatial virtual displays

main.py

@@ -21,7 +21,7 @@ import libunreal
 
 default_configuration: dict[str, str | int] = {
     "display_angle": 45,
-    "display_pixel_spacing": 45,
+    "display_spacing": 1,
     "display_count": 3,
     "allow_unsupported_devices": False,
     "allow_unsupported_vendors": False,
@@ -164,7 +164,7 @@ def main():
 
     # Raylib gets confused if there's multiple dri devices so we initialize the window before anything
     logger.info("Initializing XR headset")
-    pyray.set_target_fps(edid.max_refresh_rate)
+    pyray.set_target_fps(edid.max_refresh_rate*2) # we need more headroom...
     pyray.init_window(edid.max_width, edid.max_height, "UnrealXR")
 
     logger.info("Initializing virtual displays")
@@ -181,7 +181,7 @@ def main():
         atexit.register(lambda: card.close())
 
     logger.info("Initialized displays. Entering rendering loop")
-    render_loop(edid, cards)
+    render_loop(edid, configuration, cards)
 
 if __name__ == "__main__":
     print("Welcome to UnrealXR!\n")

render.py

@@ -1,12 +1,21 @@
+from dataclasses import dataclass
+from io import BufferedWriter
+from sys import int_info
+from typing import Union
+import ctypes
 import time
 import math
 
 from loguru import logger
+from raylib import rl
 import PyEvdi
 import pyray
 
 from libunreal import UnrealXRDisplayMetadata, MCUCallbackWrapper, start_mcu_event_listener
 
+vertical_size = 0.0
+horizontal_sizing_constant = 1
+
 previous_pitch = 0.0
 previous_yaw = 0.0
 previous_roll = 0.0
@@ -19,6 +28,15 @@ has_gotten_pitch_callback_before = False
 has_gotten_yaw_callback_before = False
 has_gotten_roll_callback_before = False
 
+@dataclass
+class RectMetadata:
+    card: PyEvdi.Card
+    buffer_ptr: pyray.ffi.CData | None
+    texture: Union[pyray.Texture, None]
+    model: Union[pyray.Model, None]
+    angle: int
+    relative_position: int
+
 def pitch_callback(new_pitch: float):
     global current_pitch
     global previous_pitch
@@ -64,7 +82,20 @@ def text_message(message: str):
 def stub_brightness_function(brightness: int):
     pass
 
-def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Card]):
+def find_max_vertical_size(fovy_deg: float, distance: float) -> float:
+    fovy_rad = math.radians(fovy_deg)
+    return 2 * distance * math.tan(fovy_rad / 2)
+
+def find_optimal_horizonal_res(vertical_display_res: int, horizontal_display_res: int) -> float:
+    aspect_ratio = horizontal_display_res/vertical_display_res
+    horizontal_size = vertical_size * aspect_ratio
+    horizontal_size = horizontal_size * horizontal_sizing_constant
+
+    return horizontal_size
+
+def render_loop(display_metadata: UnrealXRDisplayMetadata, config: dict[str, str | int], cards: list[PyEvdi.Card]):
+    global vertical_size
+    global core_mesh
     logger.info("Starting sensor event listener")
 
     mcu_callbacks = MCUCallbackWrapper(roll_callback, pitch_callback, yaw_callback, text_message, stub_brightness_function, stub_brightness_function)
@@ -78,13 +109,16 @@ def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Ca
     logger.info("Initialized sensors")
 
     camera = pyray.Camera3D()
-    camera.position = pyray.Vector3(10.0, 10.0, 10.0)
-    camera.target = pyray.Vector3(0.0, 0.0, 0.0)
-    camera.up = pyray.Vector3(0.0, 1.0, 0.0)
     camera.fovy = 45.0
+
+    vertical_size = find_max_vertical_size(camera.fovy, 5.0)
+
+    camera.position = pyray.Vector3(0.0, vertical_size/2, 5.0)
+    camera.target = pyray.Vector3(0.0, vertical_size/2, 0.0)
+    camera.up = pyray.Vector3(0.0, 1.0, 0.0)
     camera.projection = pyray.CameraProjection.CAMERA_PERSPECTIVE
 
-    cube_position = pyray.Vector3(0.0, 0.0, 0.0)
+    core_mesh = pyray.gen_mesh_plane(find_optimal_horizonal_res(display_metadata.max_height, display_metadata.max_width), vertical_size, 1, 1)
 
     movement_vector = pyray.Vector3()
     look_vector = pyray.Vector3()
@@ -93,7 +127,7 @@ def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Ca
     has_sensor_init_delay_quirk = False
     sensor_init_start_time = time.time()
 
-    if "z_vector_disabled" in display_metadata.device_quirks:
+    if "z_vector_disabled" in display_metadata.device_quirks and bool(display_metadata.device_quirks["z_vector_disabled"]):
         logger.warning("QUIRK: The Z vector has been disabled for your specific device")
         has_z_vector_disabled_quirk = True
 
@@ -103,6 +137,58 @@ def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Ca
         sensor_init_start_time = time.time()
         has_sensor_init_delay_quirk = True
 
+    rects: list[RectMetadata] = []
+
+    if int(config["display_count"]) >= 2:
+        display_angle = int(config["display_angle"])
+        display_spacing = int(config["display_spacing"])
+        total_displays = int(config["display_count"])
+
+        highest_possible_angle_on_both_sides = (total_displays-1)*display_angle
+        highest_possible_pixel_spacing_on_both_sides = (total_displays-1)*display_spacing
+
+        for i in range(total_displays):
+            current_angle = (-highest_possible_angle_on_both_sides)+(display_angle*i)
+            current_display_spacing = (-highest_possible_pixel_spacing_on_both_sides)+(display_spacing*i)
+
+            rect_metadata = RectMetadata(cards[i], None, None, None, current_angle, current_display_spacing)
+
+            has_acquired_fb = False
+
+            def fb_acquire_handler(evdi_buffer: PyEvdi.Buffer):
+                nonlocal has_acquired_fb
+
+                if has_acquired_fb:
+                    return
+
+                has_acquired_fb = True
+                logger.info(f"Acquired buffer for card #{i+1} with ID {evdi_buffer.id}")
+
+                address = ctypes.pythonapi.PyCapsule_GetPointer
+                address.restype = ctypes.c_void_p
+                address.argtypes = [ctypes.py_object, ctypes.c_char_p]
+
+                buffer_void_ptr = address(evdi_buffer.bytes, None)
+                rect_metadata.buffer_ptr = pyray.ffi.cast("void *", buffer_void_ptr)
+
+                pyray_image = pyray.Image()
+
+                pyray_image.data = rect_metadata.buffer_ptr
+                pyray_image.width = display_metadata.max_width
+                pyray_image.height = display_metadata.max_height
+                pyray_image.mipmaps = 1
+                pyray_image.format = pyray.PixelFormat.PIXELFORMAT_UNCOMPRESSED_R8G8B8A8
+
+                rect_metadata.texture = pyray.load_texture_from_image(pyray_image)
+                rect_metadata.model = pyray.load_model_from_mesh(core_mesh)
+
+                pyray.set_material_texture(rect_metadata.model.materials[0], pyray.MaterialMapIndex.MATERIAL_MAP_ALBEDO, rect_metadata.texture)
+
+            cards[i].acquire_framebuffer_handler = fb_acquire_handler
+            cards[i].handle_events(1000)
+
+            rects.append(rect_metadata)
+
     while not pyray.window_should_close():
         if has_sensor_init_delay_quirk:
             if time.time() - sensor_init_start_time >= int(display_metadata.device_quirks["sensor_init_delay"]):
@@ -111,7 +197,7 @@ def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Ca
                 has_sensor_init_delay_quirk = False
         else:
             look_vector.x = (current_yaw-previous_yaw)*6.5
-            look_vector.y = (current_pitch-previous_pitch)*6.5
+            look_vector.y = -(current_pitch-previous_pitch)*6.5
 
             if not has_z_vector_disabled_quirk:
                 look_vector.z = (current_roll-previous_roll)*6.5
@@ -121,7 +207,27 @@ def render_loop(display_metadata: UnrealXRDisplayMetadata, cards: list[PyEvdi.Ca
         pyray.begin_drawing()
         pyray.clear_background(pyray.BLACK)
         pyray.begin_mode_3d(camera)
-        pyray.draw_cube(cube_position, 2.0, 2.0, 2.0, pyray.ORANGE)
+
+        for rect_count in range(len(rects)):
+            rect = rects[rect_count]
+
+            if rect.buffer_ptr is None or rect.texture is None or rect.model is None:
+                continue
+
+            cards[rect_count].handle_events(1)
+            pyray.update_texture(rect.texture, rect.buffer_ptr)
+
+            pyray.draw_model_ex(
+                rect.model,
+                pyray.Vector3(0, vertical_size/2, 0),
+                pyray.Vector3(1, 0, 0), # rotate around X to make it vertical
+                90,
+                pyray.Vector3(1, 1, 1),
+                pyray.WHITE
+            )
+
+            break
+
         pyray.end_mode_3d()
         pyray.end_drawing()