Real-time demo using Azure Kinect DK

[psemalah]

Thanks for the authors contriubtion.
I am now using the Azure Kinect DK to replace the realsense for the real-time demo. I modify the dataset.py

class KinectDataset(BaseDataset):
    def __init__(self, args, path, config):
        super().__init__(args, path, config)
        
        # Configure the Kinect camera settings
        self.kinect = PyK4A(
            Config(
                color_resolution=pyk4a.ColorResolution.RES_720P,
                camera_fps=pyk4a.FPS.FPS_5,
                depth_mode=pyk4a.DepthMode.NFOV_2X2BINNED,
                synchronized_images_only=True,
            )
        )
        self.kinect.start()
        self.kinect.whitebalance = 4500
        assert self.kinect.whitebalance == 4500
        self.kinect.whitebalance = 4510
        assert self.kinect.whitebalance == 4510
        # Retrieve and set intrinsic parameters
        calib  = self.kinect.calibration
        #print(dir(calib))
    # Get the color camera intrinsic parameters
        color_camera_matrix = calib.get_camera_matrix(CalibrationType.COLOR)
        self.fx = color_camera_matrix[0, 0]
        self.fy = color_camera_matrix[1, 1]
        self.cx = color_camera_matrix[0, 2]
        self.cy = color_camera_matrix[1, 2]
        color_res = calib.color_resolution
        if color_res == ColorResolution.RES_720P:
            self.width = 1280
            self.height = 720
        elif color_res == ColorResolution.RES_1080P:
            self.width = 1920
            self.height = 1080
        elif color_res == ColorResolution.RES_3072P:
            self.width = 3840
            self.height = 2160
        else:
            # Handle other resolutions or set default values
            self.width = 640  # Default width
            self.height = 480  # Default height
        self.K = np.array(
            [[self.fx, 0.0, self.cx], [0.0, self.fy, self.cy], [0.0, 0.0, 1.0]]
        )
        self.has_depth = config["Dataset"]["sensor_type"] == "depth"

        # FOV calculations (optional, using a similar approach as before)
        self.fovx = np.degrees(2 * np.arctan(self.width / (2 * self.fx)))
        self.fovy = np.degrees(2 * np.arctan(self.height / (2 * self.fy)))
        
        # Handle image undistortion if necessary
        self.dist_coeffs = calib.get_distortion_coefficients(CalibrationType.COLOR)
        if self.dist_coeffs.any():
            self.map1x, self.map1y = cv2.initUndistortRectifyMap(
                self.K, self.dist_coeffs, np.eye(3), self.K, (self.width, self.height), cv2.CV_32FC1
            )

        
            
    def __getitem__(self, idx):
        pose = torch.eye(4, device=self.device, dtype=self.dtype)
        frame, image, depth = None, None, None
        
        capture = self.kinect.get_capture()
        

        if capture.transformed_depth is not None:
            print("transformed_Depth is available")
            #cv2.imshow("Transformed Depth", colorize(capture.transformed_depth, (None, 5000)))
            depth_scale = self.kinect.calibration.depth_mode.value  # Use the depth mode value instead
            # Apply the depth scale
            depth = capture.transformed_depth.astype(np.float32) * depth_scale
            depth[depth < 0] = 0
            np.nan_to_num(depth, nan=1000)
        else:
            print("transformed_Depth is not available")
            depth = None
            frame = capture.color
        
        frame = capture.color
        frame = cv2.cvtColor(frame, cv2.COLOR_BGRA2RGB)
        if self.dist_coeffs.any():
            frame = cv2.remap(frame, self.map1x, self.map1y, cv2.INTER_LINEAR)

        image = (
            torch.from_numpy(frame / 255.0)
            .clamp(0.0, 1.0)
            .permute(2, 0, 1)
            .to(device=self.device, dtype=self.dtype)
        )

        return image, depth, pose

And I also modified

def load_dataset(args, path, config):
    if config["Dataset"]["type"] == "tum":
        return TUMDataset(args, path, config)
    elif config["Dataset"]["type"] == "replica":
        return ReplicaDataset(args, path, config)
    elif config["Dataset"]["type"] == "euroc":
        return EurocDataset(args, path, config)
    elif config["Dataset"]["type"] == "realsense":
        return RealsenseDataset(args, path, config)
    elif config["Dataset"]["type"] == "azure":
        return KinectDataset(args, path, config)
    else:
        raise ValueError("Unknown dataset type")

I test the above code it works well, however, when I run the demo

I got

FEngine (64 bits) created at 0x7b1b0d000010 (threading is enabled)
FEngine resolved backend: OpenGL
transformed_Depth is available
/home/mz/MonoGS/utils/slam_utils.py:16: UserWarning: Applied workaround for CuDNN issue, install nvrtc.so (Triggered internally at ../aten/src/ATen/native/cudnn/Conv_v8.cpp:80.)
img_grad_v = normalizer * torch.nn.functional.conv2d(
MonoGS: Resetting the system
Process Process-3:
Traceback (most recent call last):
File "/home/mz/.conda/envs/monogs/lib/python3.8/multiprocessing/process.py", line 315, in _bootstrap
self.run()
File "/home/mz/.conda/envs/monogs/lib/python3.8/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/mz/MonoGS/utils/slam_backend.py", line 408, in run
self.initialize_map(cur_frame_idx, viewpoint)
File "/home/mz/MonoGS/utils/slam_backend.py", line 101, in initialize_map
render_pkg["render"],
TypeError: 'NoneType' object is not subscriptable

But very few time the UI can show the iamge and depth but rendered very strange thing that I can't identify. Do you have any ideas. Thanks so much.