release

Author: chengxuxin

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Xuxin Cheng, Jialong Li, Shiqi Yang, Ge Yang and Xiaolong Wang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,85 @@
+## 📺 Open-TeleVision: open-source tele-operation with vision 🤖
+Xuxin Cheng*, Jialong Li*, Shiqi Yang, Ge Yang, 
+Xiaolong Wang
+
+UC San Diego, MIT
+
+<p align="center">
+<img src="./img/television.jpg" width="80%"/>
+</p>
+
+## Introduction
+Stream head, hand, wrist data from VisionPro or Meta Quest 3. Stream real-time stereo video from camera to VR Devices. 
+
+## Installation
+
+```bash
+conda create -n tv python=3.8
+conda activate tv
+pip install 'vuer[all]==0.0.31rc7 opencv-python numpy'
+```
+Install zed sdk: https://www.stereolabs.com/developers/release/
+
+## Local streaming
+Apple does not allow WebXR on non-https connections. To test the application locally, we need to create a self-signed certificate and install it on the client. You need a ubuntu machine and a router. Connect the VisionPro and the ubuntu machine to the same router. 
+1. install mkcert: https://github.com/FiloSottile/mkcert
+2. check local ip address: 
+
+```
+ifconfig | grep inet
+```
+Suppose the local ip address of the ubuntu machine is `192.168.8.102`.
+
+3. create certificate: 
+
+```
+mkcert -install && mkcert -cert-file cert.pem -key-file key.pem 192.168.8.102 localhost 127.0.0.1
+```
+
+4. open firewall on server
+```
+sudo iptables -A INPUT -p tcp --dport 8012 -j ACCEPT
+sudo iptables-save
+sudo iptables -L
+```
+or can be done with `ufw`:
+```
+sudo ufw allow 8012
+```
+5.
+```python
+self.app = Vuer(host='0.0.0.0', cert="", key="")
+```
+
+6. install ca-certificates on VisionPro
+```
+mkcert -CAROOT
+```
+Copy the rootCA.pem via AirDrop to VisionPro and install it.
+
+Settings > General > About > Certificate Trust Settings. Under "Enable full trust for root certificates", turn on trust for the certificate.
+
+settings > Apps > Safari > Advanced > Feature Flags > Enable WebXR Related Features
+
+7. open the browser on Safari on VisionPro and go to `https://192.168.8.102:8012?ws=wss://192.168.8.102:8012`
+
+8. Click `Enter VR` and ``Allow`` to start the VR session.
+
+## Network Streaming
+For Meta Quest3, installation of the certificate is not trivial. We need to use a network streaming solution. We use `ngrok` to create a secure tunnel to the server. This method will work for both VisionPro and Meta Quest3.
+
+1. Install ngrok: https://ngrok.com/download
+2. Run ngrok
+```
+ngrok http 8012
+```
+3. Copy the https address and open the browser on Meta Quest3 and go to the address.
+
+## Citation
+```
+@misc{cheng2022open,
+  title={Open-TeleVision: open-source tele-operation with vision},
+  author={Cheng, Xuxin and Li, Jialong and Yang, Shiqi and Yang, Ge and Wang, Xiaolong},
+  year={2024}
+}
+```

TeleVision.py

@@ -0,0 +1,141 @@
+from vuer import Vuer
+from vuer.schemas import ImageBackground, Hands
+from multiprocessing import Process, Array, Value, shared_memory
+import numpy as np
+import asyncio
+
+class TeleVision:
+    def __init__(self, img_shape, stereo=False):
+        self.stereo = stereo
+
+        self.app = Vuer(host='0.0.0.0', cert="./cert.pem", key="./key.pem", queries=dict(grid=False))
+
+        self.app.add_handler("HAND_MOVE")(self.on_hand_move)
+        self.app.add_handler("CAMERA_MOVE")(self.on_cam_move)
+        self.app.spawn(start=False)(self.main)
+
+        self.img_shape = (2*img_shape[0], img_shape[1], 3)
+        self.img_height, self.img_width = img_shape[:2]
+        self.shm = shared_memory.SharedMemory(create=True, size=np.prod(self.img_shape) * np.uint8().itemsize)
+        self.shm_name = self.shm.name
+        self.shared_image = np.ndarray(self.img_shape, dtype=np.uint8, buffer=self.shm.buf)
+        self.shared_image[:] = np.zeros(self.img_shape, dtype=np.uint8)
+
+        self.left_hand_shared = Array('d', 16, lock=True)
+        self.right_hand_shared = Array('d', 16, lock=True)
+        self.left_landmarks_shared = Array('d', 75, lock=True)
+        self.right_landmarks_shared = Array('d', 75, lock=True)
+        
+        self.head_matrix_shared = Array('d', 16, lock=True)
+        self.aspect_shared = Value('d', 1.0, lock=True)
+
+        self.process = Process(target=self.run)
+        self.process.start()
+
+    def run(self):
+        self.app.run()
+
+    async def on_cam_move(self, event, session):
+        try:
+            with self.head_matrix_shared.get_lock():  # Use the lock to ensure thread-safe updates
+                self.head_matrix_shared[:] = event.value["camera"]["matrix"]
+            with self.aspect_shared.get_lock():
+                self.aspect_shared.value = event.value['camera']['aspect']
+        except:
+            pass
+
+    async def on_hand_move(self, event, session):
+        try:
+            with self.left_hand_shared.get_lock():  # Use the lock to ensure thread-safe updates
+                self.left_hand_shared[:] = event.value["leftHand"]
+            with self.right_hand_shared.get_lock():
+                self.right_hand_shared[:] = event.value["rightHand"]
+            with self.left_landmarks_shared.get_lock():
+                self.left_landmarks_shared[:] = np.array(event.value["leftLandmarks"]).flatten()
+            with self.right_landmarks_shared.get_lock():
+                self.right_landmarks_shared[:] = np.array(event.value["rightLandmarks"]).flatten()
+        except: 
+            pass
+
+    async def main(self, session, fps=60):
+        session.upsert @ Hands(fps=fps, stream=True, key="hands")
+        while True:
+            display_image = self.shared_image
+
+            if not self.stereo:
+                session.upsert(
+                ImageBackground(
+                    display_image[:self.img_height],
+                    format="jpeg",
+                    quality=80,
+                    key="left-image",
+                    interpolate=True,
+                    aspect=1.778,
+                    distanceToCamera=2,
+                    position=[0, -0.5, -2],
+                    rotation=[0, 0, 0],
+                ),
+                to="bgChildren",
+                )
+            else:
+                session.upsert(
+                [ImageBackground(
+                    display_image[:self.img_height],
+                    format="jpeg",
+                    quality=40,
+                    key="left-image",
+                    interpolate=True,
+                    aspect=1.778,
+                    distanceToCamera=2,
+                    layers=1
+                ),
+                ImageBackground(
+                    display_image[self.img_height:],
+                    format="jpeg",
+                    quality=40,
+                    key="right-image",
+                    interpolate=True,
+                    aspect=1.778,
+                    distanceToCamera=2,
+                    layers=2
+                )],
+                to="bgChildren",
+                )
+            await asyncio.sleep(1/fps)
+
+    def modify_shared_image(self, img, random=False):
+        assert img.shape == self.img_shape, f"Image shape must be {self.img_shape}, got {img.shape}"
+        existing_shm = shared_memory.SharedMemory(name=self.shm_name)
+        shared_image = np.ndarray(self.img_shape, dtype=np.uint8, buffer=existing_shm.buf)
+        shared_image[:] = img[:] if not random else np.random.randint(0, 256, self.img_shape, dtype=np.uint8)
+        existing_shm.close()
+
+    @property
+    def left_hand(self):
+        with self.left_hand_shared.get_lock():
+            return np.array(self.left_hand_shared[:]).reshape(4, 4, order="F")
+    
+    @property
+    def right_hand(self):
+        with self.right_hand_shared.get_lock():
+            return np.array(self.right_hand_shared[:]).reshape(4, 4, order="F")
+    
+    @property
+    def left_landmarks(self):
+        with self.left_landmarks_shared.get_lock():
+            return np.array(self.left_landmarks_shared[:]).reshape(25, 3)
+    
+    @property
+    def right_landmarks(self):
+        with self.right_landmarks_shared.get_lock():
+            return np.array(self.right_landmarks_shared[:]).reshape(25, 3)
+
+    @property
+    def head_matrix(self):
+        with self.head_matrix_shared.get_lock():
+            return np.array(self.head_matrix_shared[:]).reshape(4, 4, order="F")
+
+    @property
+    def aspect(self):
+        with self.aspect_shared.get_lock():
+            return float(self.aspect_shared.value)

img/television.jpg

@@ -0,0 +1,48 @@
+import numpy as np
+np.set_printoptions(precision=2, suppress=True)
+
+import time
+import cv2
+from TeleVision import OpenTeleVision
+import pyzed.sl as sl
+
+grd_yup2grd_zup = np.array([[0, 0, -1, 0],
+                            [-1, 0, 0, 0],
+                            [0, 1, 0, 0],
+                            [0, 0, 0, 1]])
+
+resolution = (720, 1280)
+
+# Create a Camera object
+zed = sl.Camera()
+
+# Create a InitParameters object and set configuration parameters
+init_params = sl.InitParameters()
+init_params.camera_resolution = sl.RESOLUTION.HD720 # Use HD720 opr HD1200 video mode, depending on camera type.
+init_params.camera_fps = 60  # Set fps at 30
+
+# Open the camera
+err = zed.open(init_params)
+if err != sl.ERROR_CODE.SUCCESS:
+    print("Camera Open : "+repr(err)+". Exit program.")
+    exit()
+
+image_left = sl.Mat()
+image_right = sl.Mat()
+runtime_parameters = sl.RuntimeParameters()
+tv = OpenTeleVision(resolution)
+
+while True :
+    start = time.time()
+    
+    if zed.grab(runtime_parameters) == sl.ERROR_CODE.SUCCESS:
+        zed.retrieve_image(image_left, sl.VIEW.LEFT)
+        zed.retrieve_image(image_right, sl.VIEW.RIGHT)
+
+    rgb_left = cv2.cvtColor(image_left.numpy(), cv2.COLOR_BGRA2RGB)
+    rgb_right = cv2.cvtColor(image_right.numpy(), cv2.COLOR_BGRA2RGB)
+    
+    tv.modify_shared_image(np.vstack((rgb_left, rgb_right)))
+    
+    end = time.time()
+zed.close()