feature: Get initial 3D working

libunreal/__init__.py

@@ -1,5 +1,7 @@
 from libunreal.supported_devices import *
+from libunreal.mcu_driver import *
 from libunreal.edid import *
+
 from sys import platform
 
 if platform == "linux" or platform == "linux2":

libunreal/edid.py

@@ -1,11 +1,12 @@
 from loguru import logger
 
-import dataclasses
+from dataclasses import dataclass
 import uuid
 
-@dataclasses.dataclass
+@dataclass
 class EvdiDisplaySpec:
     edid: bytes
+    device_vendor: str
     max_width: int
     max_height: int
     max_refresh_rate: int

libunreal/linux/__init__.py

@@ -72,7 +72,7 @@ def fetch_xr_glass_edid(allow_unsupported_devices) -> EvdiDisplaySpec:
 
                                 max_refresh = int(manufacturer_supported_devices[edid.name]["max_refresh"])
 
-                            return EvdiDisplaySpec(raw_edid_file, max_width, max_height, max_refresh, card_device, monitor.replace(f"{card_device}-", ""))
+                            return EvdiDisplaySpec(raw_edid_file, edid.manufacturer_pnp_id, max_width, max_height, max_refresh, card_device, monitor.replace(f"{card_device}-", ""))
 
     raise ValueError("Could not find supported device. Check if the device is plugged in. If it is plugged in and working correctly, check the README or open an issue.")
 

libunreal/mcu_driver.py

@@ -0,0 +1,145 @@
+from tempfile import TemporaryDirectory
+from dataclasses import dataclass
+from os import path, environ
+from typing import Callable
+from loguru import logger
+from shutil import which
+from time import sleep
+from enum import Enum
+import subprocess
+import threading
+import socket
+import struct
+import signal
+import atexit
+import os
+
+class MCUCommandTypes(Enum):
+    ROLL = 0
+    PITCH = 1
+    YAW = 2
+    GENERIC_MESSAGE = 3
+    BRIGHTNESS_UP = 4
+    BRIGHTNESS_DOWN = 5
+
+@dataclass
+class MCUCallbackWrapper:
+    OnRollUpdate: Callable[[float], None]
+    OnPitchUpdate: Callable[[float], None]
+    OnYawUpdate: Callable[[float], None]
+    OnTextMessageRecieved: Callable[[str], None]
+    OnBrightnessUp: Callable[[int], None]
+    OnBrightnessDown: Callable[[int], None]
+
+vendor_to_driver_table: dict[str, str] = {
+    "MRG": "xreal_ar_driver",
+}
+
+def find_executable_path_from_driver_name(driver_name) -> str:
+    # First try the normal driver path
+    try:
+        driver_path = path.join("drivers", driver_name)
+
+        file = open(driver_path)
+        file.close()
+
+        return driver_path
+    except OSError:
+        # Then search the system path
+        driver_path = which(driver_name)
+
+        if driver_path == "":
+            raise OSError("Could not find driver executable in driver directory or in PATH")
+
+        return driver_path
+
+def start_mcu_event_listener(driver_vendor: str, events: MCUCallbackWrapper):
+    driver_executable = find_executable_path_from_driver_name(vendor_to_driver_table[driver_vendor])
+
+    created_temp_dir = TemporaryDirectory()
+    sock_path = path.join(created_temp_dir.name, "mcu_socket")
+
+    def on_socket_event(sock: socket.socket):
+        while True:
+            message_type = sock.recv(1)
+
+            if message_type[0] == MCUCommandTypes.ROLL.value:
+                roll_data = sock.recv(4)
+                roll_value = struct.unpack("!f", roll_data)[0]
+
+                if not isinstance(roll_value, float):
+                    logger.warning("Expected roll value to be a float but got other type instead")
+                    continue
+
+                events.OnRollUpdate(roll_value)
+            elif message_type[0] == MCUCommandTypes.PITCH.value:
+                pitch_data = sock.recv(4)
+                pitch_value = struct.unpack("!f", pitch_data)[0]
+
+                if not isinstance(pitch_value, float):
+                    logger.warning("Expected pitch value to be a float but got other type instead")
+                    continue
+
+                events.OnPitchUpdate(pitch_value)
+            elif message_type[0] == MCUCommandTypes.YAW.value:
+                yaw_data = sock.recv(4)
+                yaw_value = struct.unpack("!f", yaw_data)[0]
+
+                if not isinstance(yaw_value, float):
+                    logger.warning("Expected yaw value to be a float but got other type instead")
+                    continue
+
+                events.OnYawUpdate(yaw_value)
+            elif message_type[0] == MCUCommandTypes.GENERIC_MESSAGE.value:
+                length_bytes = sock.recv(4)
+
+                msg_len = struct.unpack("!I", length_bytes)[0]
+                msg_bytes = sock.recv(msg_len)
+
+                msg = msg_bytes.decode("utf-8", errors="replace")
+                events.OnTextMessageRecieved(msg)
+            elif message_type[0] == MCUCommandTypes.BRIGHTNESS_UP.value:
+                brightness_bytes = sock.recv(1)
+                events.OnBrightnessUp(int.from_bytes(brightness_bytes, byteorder='big'))
+            elif message_type[0] == MCUCommandTypes.BRIGHTNESS_DOWN.value:
+                brightness_bytes = sock.recv(1)
+                events.OnBrightnessDown(int.from_bytes(brightness_bytes, byteorder='big'))
+            else:
+                logger.warning(f"Unknown message type recieved: {str(message_type[0])}")
+
+    def start_socket_handout():
+        server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        server.bind(sock_path)
+        server.listen(1)
+
+        sock: socket.socket | None = None
+
+        while True:
+            sock, _ = server.accept()
+
+            threaded_connection_processing = threading.Thread(target=on_socket_event, args=(sock,), daemon=True)
+            threaded_connection_processing.start()
+
+        created_temp_dir.cleanup()
+
+    def spawn_child_process():
+        custom_env = environ.copy()
+        custom_env["UNREALXR_NREAL_DRIVER_SOCK"] = sock_path
+
+        process = subprocess.Popen([driver_executable], env=custom_env)
+
+        def kill_child():
+            if process.pid is None:
+                pass
+            else:
+                os.kill(process.pid, signal.SIGTERM)
+
+        atexit.register(kill_child)
+
+    threaded_socket_handout = threading.Thread(target=start_socket_handout, daemon=True)
+    threaded_socket_handout.start()
+
+    sleep(0.01) # Give the socket server time to initialize
+
+    threaded_child_process = threading.Thread(target=spawn_child_process, daemon=True)
+    threaded_child_process.start()

main.py

@@ -134,7 +134,7 @@ def main():
             if max_refresh == 0:
                 raise ValueError("Could not determine maximum refresh rate from EDID file, and the refresh rate overrides aren't set!")
 
-            edid = libunreal.EvdiDisplaySpec(edid_file, max_width, max_height, max_refresh, "", "")
+            edid = libunreal.EvdiDisplaySpec(edid_file, parsed_edid_file.name if parsed_edid_file.name else "", max_width, max_height, max_refresh, "", "")
     else:
         edid = libunreal.fetch_xr_glass_edid(configuration["allow_unsupported_devices"])
 

modules/nreal-driver

@@ -1 +1 @@
-Subproject commit 682daf43853702b61df02d12ed3870937f528107
+Subproject commit 36ad789c9b5893653c523d4af6e24120ec19ab33

render.py

@@ -1,14 +1,111 @@
+from time import sleep
+import math
+
 from loguru import logger
 import PyEvdi
 import pyray
 
-from libunreal import EvdiDisplaySpec
+from libunreal import EvdiDisplaySpec, MCUCallbackWrapper, start_mcu_event_listener
+
+previous_pitch = 0.0
+previous_yaw = 0.0
+previous_roll = 0.0
+
+current_pitch = 0.0
+current_yaw = 0.0
+current_roll = 0.0
+
+has_gotten_pitch_callback_before = False
+has_gotten_yaw_callback_before = False
+has_gotten_roll_callback_before = False
+
+def pitch_callback(new_pitch: float):
+    global current_pitch
+    global previous_pitch
+    global has_gotten_pitch_callback_before
+
+    if not has_gotten_pitch_callback_before:
+        has_gotten_pitch_callback_before = True
+        previous_pitch = new_pitch
+        current_pitch = new_pitch
+    else:
+        previous_pitch = current_pitch
+        current_pitch = new_pitch
+
+def yaw_callback(new_yaw: float):
+    global current_yaw
+    global previous_yaw
+    global has_gotten_yaw_callback_before
+
+    if not has_gotten_yaw_callback_before:
+        has_gotten_yaw_callback_before = True
+        previous_yaw = new_yaw
+        current_yaw = new_yaw
+    else:
+        previous_yaw = current_yaw
+        current_yaw = new_yaw
+
+def roll_callback(new_roll: float):
+    global current_roll
+    global previous_roll
+    global has_gotten_roll_callback_before
+
+    if not has_gotten_roll_callback_before:
+        has_gotten_roll_callback_before = True
+        previous_roll = new_roll
+        roll = new_roll
+    else:
+        previous_roll = current_roll
+        current_roll = new_roll
+
+def text_message(message: str):
+    logger.debug(f"Got message from AR's MCU: {message}")
+
+def stub_brightness_function(brightness: int):
+    pass
 
 def render_loop(display_metadata: EvdiDisplaySpec, cards: list[PyEvdi.Card]):
+    logger.info("Starting sensor event listener")
+
+    mcu_callbacks = MCUCallbackWrapper(roll_callback, pitch_callback, yaw_callback, text_message, stub_brightness_function, stub_brightness_function)
+    start_mcu_event_listener(display_metadata.device_vendor, mcu_callbacks)
+
+    logger.info("Beginning sensor initialization. Awaiting first sensor update")
+
+    while (not has_gotten_pitch_callback_before) or (not has_gotten_yaw_callback_before) or (not has_gotten_roll_callback_before):
+        sleep(0.01)
+
+    logger.info("Initialized sensors")
+
+    camera = pyray.Camera3D()
+    camera.position = pyray.Vector3(10.0, 10.0, 10.0)
+    camera.target = pyray.Vector3(0.0, 0.0, 0.0)
+    camera.up = pyray.Vector3(0.0, 1.0, 0.0)
+    camera.fovy = 45.0
+    camera.projection = pyray.CameraProjection.CAMERA_PERSPECTIVE
+
+    cube_position = pyray.Vector3(0.0, 0.0, 0.0)
+
+    movement_vector = pyray.Vector3()
+    look_vector = pyray.Vector3()
+
+    logger.error("QUIRK: Waiting 10 seconds before reading sensors due to sensor drift bugs")
+    sleep(10)
+    logger.error("Continuing...")
+
     while not pyray.window_should_close():
+        look_vector.x = (current_yaw-previous_yaw)*6.5
+        look_vector.y = (current_pitch-previous_pitch)*6.5
+        # the Z vector is more trouble than its worth so it just doesn't get accounted for...
+        #look_vector.z = (current_roll-previous_roll)*6.5
+
+        pyray.update_camera_pro(camera, movement_vector, look_vector, 0.0)
+
         pyray.begin_drawing()
         pyray.clear_background(pyray.BLACK)
-        pyray.draw_text("Hello world from Python!", 190, 200, 96, pyray.WHITE)
+        pyray.begin_mode_3d(camera)
+        pyray.draw_cube(cube_position, 2.0, 2.0, 2.0, pyray.ORANGE)
+        pyray.end_mode_3d()
         pyray.end_drawing()
 
     logger.info("Goodbye!")

shell.nix

@@ -33,7 +33,7 @@
   ];
 
   shellHook = ''
-    export LD_LIBRARY_PATH="$PWD/evdi/library:${pkgs.lib.makeLibraryPath [ pkgs.libGL ]}:$LD_LIBRARY_PATH"
+    export LD_LIBRARY_PATH="$PWD/modules/evdi/library:${pkgs.lib.makeLibraryPath [ pkgs.libGL ]}:$LD_LIBRARY_PATH"
     mkdir -p "$PWD/data/config" "$PWD/data/data"
     export UNREALXR_CONFIG_PATH="$PWD/data/config"
     export UNREALXR_DATA_PATH="$PWD/data/data"