diff --git a/game/AvatarStream/scripts/TrackingUI.gd b/game/AvatarStream/scripts/TrackingUI.gd index e4043de..8ed04de 100644 --- a/game/AvatarStream/scripts/TrackingUI.gd +++ b/game/AvatarStream/scripts/TrackingUI.gd @@ -60,9 +60,10 @@ func _on_ResolutionOptionButton_item_selected(index): var width = int(resolution[0]) var height = int(resolution[1]) # godot_cmio.set_resolution(width, height) - # TODO: Inform VirtualCameraSender about resolution change? - # For now, it resizes to 640x360 fixed. - pass + + var sender = get_tree().get_root().find_child("VirtualCameraSender", true, false) + if sender: + sender.set_resolution(width, height) var control_hints = { "VirtualCamButton": "Starts or stops the virtual camera.", diff --git a/game/AvatarStream/scripts/VirtualCameraSender.gd b/game/AvatarStream/scripts/VirtualCameraSender.gd index a82896c..60cbcbb 100644 --- a/game/AvatarStream/scripts/VirtualCameraSender.gd +++ b/game/AvatarStream/scripts/VirtualCameraSender.gd @@ -8,6 +8,9 @@ var frame_rate := 30.0 var time_since_last_frame := 0.0 var connected := false +var target_width := 640 +var target_height := 360 + func _ready(): # Only enable on desktop platforms if not (OS.get_name() in ["Windows", "macOS", "Linux", "FreeBSD", "NetBSD", "OpenBSD", "BSD"]): @@ -26,6 +29,11 @@ func connect_to_server(): print("Failed to connect to Virtual Camera Server.") connected = false +func set_resolution(width: int, height: int): + target_width = width + target_height = height + print("Virtual Camera resolution set to: " + str(width) + "x" + str(height)) + func _process(delta): # Check connection status tcp_client.poll() @@ -52,7 +60,7 @@ func send_frame(): var image = viewport.get_texture().get_image() # Resize to something reasonable for streaming - image.resize(640, 360) + image.resize(target_width, target_height) # Convert to RGB8 if needed (pyvirtualcam expects RGB) if image.get_format() != Image.FORMAT_RGB8: @@ -62,6 +70,7 @@ func send_frame(): var buffer = image.get_data() if buffer.size() > 0: - # Send size first (4 bytes), then data - tcp_client.put_32(buffer.size()) + # Send width (4 bytes), height (4 bytes), then data + tcp_client.put_32(target_width) + tcp_client.put_32(target_height) tcp_client.put_data(buffer) diff --git a/game/AvatarStream/scripts/python/holistic_tracker.py b/game/AvatarStream/scripts/python/holistic_tracker.py index 19d3651..cdc5070 100644 --- a/game/AvatarStream/scripts/python/holistic_tracker.py +++ b/game/AvatarStream/scripts/python/holistic_tracker.py @@ -43,24 +43,21 @@ def virtual_camera_loop(): with conn: while True: - # Read size (4 bytes) - size_data = b'' - while len(size_data) < 4: - packet = conn.recv(4 - len(size_data)) + # Read width (4 bytes) and height (4 bytes) + header_data = b'' + while len(header_data) < 8: + packet = conn.recv(8 - len(header_data)) if not packet: break - size_data += packet + header_data += packet - if not size_data: + if not header_data: break - size = int.from_bytes(size_data, byteorder='little') # Godot uses little endian usually? Or big? - # Godot StreamPeerTCP put_32 is likely little endian on intel. But network byte order is usually big. - # Godot documentation says put_32 puts a 32-bit integer. - # Let's assume little endian for now, or check documentation. - # Actually StreamPeer uses big-endian by default? - # StreamPeer.big_endian property defaults to false. - # So it is little endian. + width = int.from_bytes(header_data[0:4], byteorder='little') + height = int.from_bytes(header_data[4:8], byteorder='little') + + size = width * height * 3 # Read image data img_data = b'' @@ -73,45 +70,9 @@ def virtual_camera_loop(): if len(img_data) != size: break - # Parse Raw Image Data (RGB) - # Assumed resolution 640x360. - # In a more robust implementation, width/height should be part of the protocol. - # For now we rely on the buffer size to infer or hardcode if needed. - # 640*360*3 = 691200 - - # If size matches 640x360x3, we just reshape. - # If it's a JPG (legacy), we detect header? - # Let's assume the client sends raw RGB. - try: frame = np.frombuffer(img_data, dtype=np.uint8) - - # Heuristic to detect if it's still JPG (starts with 0xFFD8) - if len(frame) > 2 and frame[0] == 0xFF and frame[1] == 0xD8: - frame = cv2.imdecode(frame, cv2.IMREAD_COLOR) - if frame is not None: - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - else: - # Assume 640x360 RGB - # If the size is different, we might fail reshaping. - # Let's calculate based on aspect ratio 16:9? - # W * H * 3 = Size. W * (9/16 W) * 3 = Size => W^2 = Size / 3 * 16 / 9 - # For now, hardcode 640x360 as target - target_w, target_h = 640, 360 - if size == target_w * target_h * 3: - frame = frame.reshape((target_h, target_w, 3)) - else: - # Try to guess - # If not 640x360, maybe square? - side = int(np.sqrt(size / 3)) - if side * side * 3 == size: - frame = frame.reshape((side, side, 3)) - else: - # Fallback or error - continue - - if frame is None: - continue + frame = frame.reshape((height, width, 3)) h, w, c = frame.shape @@ -119,6 +80,12 @@ def virtual_camera_loop(): print(f"Frame decode error: {e}") continue + # Update camera if resolution changed + if cam is not None and (cam.width != w or cam.height != h): + print(f"Resolution changed from {cam.width}x{cam.height} to {w}x{h}. Restarting Virtual Camera.") + cam.close() + cam = None + if cam is None: cam = pyvirtualcam.Camera(width=w, height=h, fps=30) print(f"Virtual Camera started: {w}x{h} @ {cam.fps}fps") diff --git a/game/AvatarStream/scripts/python/tests/test_protocol.py b/game/AvatarStream/scripts/python/tests/test_protocol.py index 247277b..112a4aa 100644 --- a/game/AvatarStream/scripts/python/tests/test_protocol.py +++ b/game/AvatarStream/scripts/python/tests/test_protocol.py @@ -51,14 +51,17 @@ def test_video_frame_protocol(self): def server_thread_func(): conn, addr = server_sock.accept() with conn: - # Read size - size_data = b'' - while len(size_data) < 4: - packet = conn.recv(4 - len(size_data)) + # Read width and height + header_data = b'' + while len(header_data) < 8: + packet = conn.recv(8 - len(header_data)) if not packet: break - size_data += packet + header_data += packet - size = int.from_bytes(size_data, byteorder='little') + width = int.from_bytes(header_data[0:4], byteorder='little') + height = int.from_bytes(header_data[4:8], byteorder='little') + + size = width * height * 3 # Read data data = b'' @@ -69,7 +72,7 @@ def server_thread_func(): # Verify data expected_size = 640 * 360 * 3 - if size == expected_size: + if size == expected_size and width == 640 and height == 360: frame = np.frombuffer(data, dtype=np.uint8) if len(frame) == expected_size: return True @@ -87,7 +90,8 @@ def server_thread_func(): w, h = 640, 360 frame_data = bytearray(os.urandom(w * h * 3)) - client.sendall(struct.pack('