update: app

app.py

@@ -23,6 +23,7 @@ import cv2
 import tqdm
 import numpy as np
 import gradio as gr
+from tools.util import *
 
 from detectron2.config import get_cfg
 
@@ -150,10 +151,16 @@ def greet_sailvos3d(rgb_input, depth_map_input, rage_matrices_input, class_candi
     
     Depth_Semantic_SAM_Mask = read_image('outputs/Depth_Semantic_SAM_Mask.png')
     RGB_Semantic_SAM_Mask = read_image('outputs/RGB_Semantic_SAM_Mask.png')
+    Depth_Semantic_SAM = read_image('outputs/Depth_Semantic_SAM.png')
+    RGB_Semantic_SAM = read_image('outputs/RGB_Semantic_SAM.png')
+    two_image_to_gif(Depth_Semantic_SAM_Mask, Depth_Semantic_SAM, 'Depth_Semantic_SAM_2D')
+    two_image_to_gif(RGB_Semantic_SAM_Mask, RGB_Semantic_SAM, 'RGB_Semantic_SAM_2D')
+    Depth_Semantic_SAM_2D = 'outputs/Depth_Semantic_SAM_2D.mp4'
+    RGB_Semantic_SAM_2D = 'outputs/RGB_Semantic_SAM_2D.mp4'
     Depth_map = read_image('outputs/Depth_rendered.png')
-    Depth_Semantic_SAM_Mask_gif = 'outputs/depth_3d_sam_mask.mp4'
-    RGB_Semantic_SAM_Mask_gif = 'outputs/rgb_3d_sam_mask.mp4'
-    return RGB_Semantic_SAM_Mask, RGB_Semantic_SAM_Mask_gif, Depth_map, Depth_Semantic_SAM_Mask, Depth_Semantic_SAM_Mask_gif
+    Depth_Semantic_SAM_Mask_gif = 'outputs/Depth_3D_All.mp4'
+    RGB_Semantic_SAM_Mask_gif = 'outputs/RGB_3D_All.mp4'
+    return RGB_Semantic_SAM_2D, RGB_Semantic_SAM_Mask_gif, Depth_map, Depth_Semantic_SAM_2D, Depth_Semantic_SAM_Mask_gif
 
 def greet_scannet(rgb_input, depth_map_input, class_candidates):
     rgb_input = rgb_input
@@ -192,10 +199,16 @@ def greet_scannet(rgb_input, depth_map_input, class_candidates):
 
     Depth_Semantic_SAM_Mask = read_image('outputs/Depth_Semantic_SAM_Mask.png')
     RGB_Semantic_SAM_Mask = read_image('outputs/RGB_Semantic_SAM_Mask.png')
+    Depth_Semantic_SAM = read_image('outputs/Depth_Semantic_SAM.png')
+    RGB_Semantic_SAM = read_image('outputs/RGB_Semantic_SAM.png')
+    two_image_to_gif(Depth_Semantic_SAM_Mask, Depth_Semantic_SAM, 'Depth_Semantic_SAM_2D')
+    two_image_to_gif(RGB_Semantic_SAM_Mask, RGB_Semantic_SAM, 'RGB_Semantic_SAM_2D')
+    Depth_Semantic_SAM_2D = 'outputs/Depth_Semantic_SAM_2D.mp4'
+    RGB_Semantic_SAM_2D = 'outputs/RGB_Semantic_SAM_2D.mp4'
     Depth_map = read_image('outputs/Depth_rendered.png')
-    Depth_Semantic_SAM_Mask_gif = 'outputs/depth_3d_sam_mask.mp4'
-    RGB_Semantic_SAM_Mask_gif = 'outputs/rgb_3d_sam_mask.mp4'
-    return RGB_Semantic_SAM_Mask, RGB_Semantic_SAM_Mask_gif, Depth_map, Depth_Semantic_SAM_Mask, Depth_Semantic_SAM_Mask_gif
+    Depth_Semantic_SAM_Mask_gif = 'outputs/Depth_3D_All.mp4'
+    RGB_Semantic_SAM_Mask_gif = 'outputs/RGB_3D_All.mp4'
+    return RGB_Semantic_SAM_2D, RGB_Semantic_SAM_Mask_gif, Depth_map, Depth_Semantic_SAM_2D, Depth_Semantic_SAM_Mask_gif
 
 
 with gr.Blocks(analytics_enabled=False) as segrgbd_iface:
@@ -211,20 +224,22 @@ with gr.Blocks(analytics_enabled=False) as segrgbd_iface:
                     with gr.Column():
                         with gr.Row():
                             Input_RGB_Component = gr.Image(label = 'RGB_Input', type = 'filepath').style(width=320, height=200)
-                            Depth_Map_Output_Component = gr.Image(label = "Depth_Map").style(width=320, height=200)
+                            Depth_Map_Output_Component = gr.Image(label = "Vis_Depth_Map").style(width=320, height=200)
                         with gr.Row():
-                            Depth_Map_Input_Component = gr.File(label = 'Depth_map')
+                            Depth_Map_Input_Component = gr.File(label = 'input_Depth_map')
                             Component_2D_to_3D_Projection_Parameters = gr.File(label = '2D_to_3D_Projection_Parameters')
                         with gr.Row():
                             Class_Candidates_Component = gr.Text(label = 'Class_Candidates')
                         vc_end_btn = gr.Button("Send")
                     with gr.Tab(label='Result'):
                         with gr.Row():
-                            RGB_Semantic_SAM_Mask_Component = gr.Image(label = "RGB_Semantic_SAM_Mask").style(width=320, height=200)
+                            RGB_Semantic_SAM_Mask_Component = gr.Video(label = "RGB_Semantic_SAM_Mask").style(width=320, height=200)
                             RGB_Semantic_SAM_Mask_3D_Component = gr.Video(label = "Video_3D_RGB_Semantic_SAM_Mask").style(width=320, height=200)
                         with gr.Row():
-                            Depth_Semantic_SAM_Mask_Component = gr.Image(label = "Depth_Semantic_SAM_Mask").style(width=320, height=200)
+                            Depth_Semantic_SAM_Mask_Component = gr.Video(label = "Depth_Semantic_SAM_Mask").style(width=320, height=200)
                             Depth_Semantic_SAM_Mask_3D_Component = gr.Video(label = "Video_3D_Depth_Semantic_SAM_Mask").style(width=320, height=200)
+                        with gr.Row():
+                            gr.Markdown("<b> It takes around 2 to 5 minutes to get the final results. The framework initialization, SAM segmentation, zero-shot semantic segmentation and 3D results rendering take long time.</b>")
                 gr.Examples(examples=[
                         [
                             'UI/sailvos3d/ex1/inputs/rgb_000160.bmp',
@@ -252,18 +267,20 @@ with gr.Blocks(analytics_enabled=False) as segrgbd_iface:
                     with gr.Column():
                         with gr.Row():
                             Input_RGB_Component = gr.Image(label = 'RGB_Input', type = 'filepath').style(width=320, height=200)
-                            Depth_Map_Output_Component = gr.Image(label = "Depth_Map").style(width=320, height=200)
+                            Depth_Map_Output_Component = gr.Image(label = "Vis_Depth_Map").style(width=320, height=200)
                         with gr.Row():
-                            Depth_Map_Input_Component = gr.File(label = "Depth_Map")
+                            Depth_Map_Input_Component = gr.File(label = "Input_Depth_Map")
                             Class_Candidates_Component = gr.Text(label = 'Class_Candidates')
                         vc_end_btn = gr.Button("Send")
                     with gr.Tab(label='Result'):
                         with gr.Row():
-                            RGB_Semantic_SAM_Mask_Component = gr.Image(label = "RGB_Semantic_SAM_Mask").style(width=320, height=200)
+                            RGB_Semantic_SAM_Mask_Component = gr.Video(label = "RGB_Semantic_SAM_Mask").style(width=320, height=200)
                             RGB_Semantic_SAM_Mask_3D_Component = gr.Video(label = "Video_3D_RGB_Semantic_SAM_Mask").style(width=320, height=200)
                         with gr.Row():
-                            Depth_Semantic_SAM_Mask_Component = gr.Image(label = "Depth_Semantic_SAM_Mask").style(width=320, height=200)
+                            Depth_Semantic_SAM_Mask_Component = gr.Video(label = "Depth_Semantic_SAM_Mask").style(width=320, height=200)
                             Depth_Semantic_SAM_Mask_3D_Component = gr.Video(label = "Video_3D_Depth_Semantic_SAM_Mask").style(width=320, height=200)
+                        with gr.Row():
+                            gr.Markdown("<b> It takes around 2 to 5 minutes to get the final results. The framework initialization, SAM segmentation, zero-shot semantic segmentation and 3D results rendering take long time.</b>")
                 gr.Examples(examples=[
                         [
                             'UI/scannetv2/examples/scene0000_00/color/1660.jpg',

open_vocab_seg/utils/predictor.py

@@ -7,6 +7,7 @@ import torchvision
 import imageio
 from tqdm import tqdm
 import os
+import cv2
 
 from pytorch3d.structures import Pointclouds
 from pytorch3d.renderer import look_at_view_transform
@@ -19,7 +20,6 @@ from segment_anything import sam_model_registry, SamAutomaticMaskGenerator, SamP
 import matplotlib.pyplot as plt
 import matplotlib as mpl
 from .pcd_rendering import unproject_pts_pt, get_coord_grids_pt, create_pcd_renderer
-import cv2
 
 
 class OVSegPredictor(DefaultPredictor):
@@ -179,7 +179,7 @@ class VisualizationDemo(object):
         if "sem_seg" in predictions:
             r = predictions["sem_seg"]
             pred_mask = r.argmax(dim=0).to('cpu')
-            pred_mask = np.array(pred_mask, dtype=int)
+            pred_mask = np.array(pred_mask, dtype=np.int)
 
             vis_output = visualizer.draw_sem_seg(
                 pred_mask
@@ -223,6 +223,7 @@ class VisualizationDemo(object):
         )
         print('Using SAM to generate segments for the RGB image')
         masks_rgb = mask_generator_2.generate(image)
+        masks_rgb = sorted(masks_rgb, key=(lambda x: x['area']), reverse=True)
 
         print('Using SAM to generate segments for the Depth map')
         d, world_coord = self.project_2d_to_3d(depth_map_path, rage_matrices_path)
@@ -233,15 +234,16 @@ class VisualizationDemo(object):
         plt.axis('off')
         plt.savefig('outputs/Depth_rendered.png', bbox_inches='tight', pad_inches=0.0)
         masks_depth = mask_generator_2.generate(image_depth.astype(np.uint8)[:,:,:-1])
+        masks_depth = sorted(masks_depth, key=(lambda x: x['area']), reverse=True)
 
         if "sem_seg" in predictions:
             r = predictions["sem_seg"]
             pred_mask = r.argmax(dim=0).to('cpu')
-            pred_mask = np.array(pred_mask, dtype=int)
+            pred_mask = np.array(pred_mask, dtype=np.int)
             
             pred_mask_sam_rgb = pred_mask.copy()
             for mask in masks_rgb:
-                cls_tmp, cls_num = np.unique(pred_mask_sam_rgb[mask['segmentation']], return_counts=True)
+                cls_tmp, cls_num = np.unique(pred_mask[mask['segmentation']], return_counts=True)
                 pred_mask_sam_rgb[mask['segmentation']] = cls_tmp[np.argmax(cls_num)]
                 mask['class'] = cls_tmp[np.argmax(cls_num)]
 
@@ -254,7 +256,7 @@ class VisualizationDemo(object):
 
             pred_mask_sam_depth = pred_mask.copy()
             for mask in masks_depth:
-                cls_tmp, cls_num = np.unique(pred_mask_sam_depth[mask['segmentation']], return_counts=True)
+                cls_tmp, cls_num = np.unique(pred_mask[mask['segmentation']], return_counts=True)
                 pred_mask_sam_depth[mask['segmentation']] = cls_tmp[np.argmax(cls_num)]
                 mask['class'] = cls_tmp[np.argmax(cls_num)]
 
@@ -400,7 +402,8 @@ class VisualizationDemo(object):
         num_frames = 45
         degrees = np.linspace(120, 220, num_frames)
         
-        total = ['rgb_3d_sam_mask', 'depth_3d_sam_mask']
+        total = ['rgb_3d_sam', 'depth_3d_sam', 'rgb_3d_sam_mask', 'depth_3d_sam_mask']
+        frames_all = {}
         
         for j, name in enumerate(total):
             img = torch.from_numpy(xyzrgb[name][:, 3:] / 255.).to(device).float()
@@ -415,13 +418,21 @@ class VisualizationDemo(object):
                 result = result.permute(0, 3, 1, 2)
                 frame = (255. * result.detach().cpu().squeeze().permute(1, 2, 0).numpy()).astype(np.uint8)
                 frames.append(frame)
+            
+            frames_all[name] = frames
 
             # video_out_file = '{}.gif'.format(name)
-            # imageio.mimwrite(os.path.join('outputs', video_out_file), frames, duration=20)
+            # imageio.mimwrite(os.path.join('outputs', video_out_file), frames, fps=25)
             
             video_out_file = '{}.mp4'.format(name)
             imageio.mimwrite(os.path.join('outputs', video_out_file), frames, fps=25, quality=8)
-
+            
+        video_out_file = '{}.mp4'.format('RGB_3D_All')
+        imageio.mimwrite(os.path.join('outputs', video_out_file), frames_all['rgb_3d_sam_mask']+frames_all['rgb_3d_sam'], fps=25, quality=8)
+        
+        video_out_file = '{}.mp4'.format('Depth_3D_All')
+        imageio.mimwrite(os.path.join('outputs', video_out_file), frames_all['depth_3d_sam_mask']+frames_all['depth_3d_sam'], fps=25, quality=8)
+            
 class VisualizationDemoIndoor(VisualizationDemo):
     def __init__(self, cfg, instance_mode=ColorMode.IMAGE, parallel=False):
         super().__init__(cfg, instance_mode, parallel)
@@ -472,6 +483,7 @@ class VisualizationDemoIndoor(VisualizationDemo):
         )
         print('Using SAM to generate segments for the RGB image')
         masks_rgb = mask_generator_2.generate(image)
+        masks_rgb = sorted(masks_rgb, key=(lambda x: x['area']), reverse=True)
 
         print('Using SAM to generate segments for the Depth map')
         d = np.full(depth_img.shape, 0, dtype=float)
@@ -483,6 +495,7 @@ class VisualizationDemoIndoor(VisualizationDemo):
         plt.axis('off')
         plt.savefig('outputs/Depth_rendered.png')
         masks_depth = mask_generator_2.generate(colored_depth.astype(np.uint8)[:,:,:-1])
+        masks_depth = sorted(masks_depth, key=(lambda x: x['area']), reverse=True)
 
         if "sem_seg" in predictions:
             r = predictions["sem_seg"]
@@ -541,7 +554,7 @@ class VisualizationDemoIndoor(VisualizationDemo):
 
         output3D = {}
         output3D['rgb_3d_sem'] = np.stack((uv_depth, output2D['sem_seg_on_rgb'].get_image()), axis=2).reshape((depth_img.shape[0], depth_img.shape[1], 6))
-        output3D['depth_3d_sem'] = np.stack((uv_depth, output2D['sem_seg_on_rgb'].get_image()), axis=2).reshape((depth_img.shape[0], depth_img.shape[1], 6))
+        output3D['depth_3d_sem'] = np.stack((uv_depth, output2D['sem_seg_on_depth'].get_image()), axis=2).reshape((depth_img.shape[0], depth_img.shape[1], 6))
         output3D['rgb_3d_sam'] = np.stack((uv_depth, output2D['sam_seg_on_rgb'].get_image()), axis=2).reshape((depth_img.shape[0], depth_img.shape[1], 6))
         output3D['depth_3d_sam'] = np.stack((uv_depth, output2D['sam_seg_on_depth'].get_image()), axis=2).reshape((depth_img.shape[0], depth_img.shape[1], 6))
         
@@ -585,6 +598,7 @@ class VisualizationDemoIndoor(VisualizationDemo):
         )
         print('Using SAM to generate segments for the RGB image')
         masks_rgb = mask_generator_2.generate(image)
+        masks_rgb = sorted(masks_rgb, key=(lambda x: x['area']), reverse=True)
 
         print('Using SAM to generate segments for the Depth map')  
         d = np.full(depth_img.shape, 0, dtype=float)
@@ -596,6 +610,7 @@ class VisualizationDemoIndoor(VisualizationDemo):
         plt.axis('off')
         plt.savefig('outputs/Depth_rendered.png')
         masks_depth = mask_generator_2.generate(colored_depth.astype(np.uint8)[:,:,:-1])
+        masks_depth = sorted(masks_depth, key=(lambda x: x['area']), reverse=True)
 
         if "sem_seg" in predictions:
             r = predictions["sem_seg"]
@@ -656,7 +671,7 @@ class VisualizationDemoIndoor(VisualizationDemo):
 
         output3D = {}
         output3D['rgb_3d_sem'] = np.stack((uv_depth, output2D['sem_seg_on_rgb'].get_image()), axis=2).reshape((depth_img.shape[0], depth_img.shape[1], 6))
-        output3D['depth_3d_sem'] = np.stack((uv_depth, output2D['sem_seg_on_rgb'].get_image()), axis=2).reshape((depth_img.shape[0], depth_img.shape[1], 6))
+        output3D['depth_3d_sem'] = np.stack((uv_depth, output2D['sem_seg_on_depth'].get_image()), axis=2).reshape((depth_img.shape[0], depth_img.shape[1], 6))
         output3D['rgb_3d_sam'] = np.stack((uv_depth, output2D['sam_seg_on_rgb'].get_image()), axis=2).reshape((depth_img.shape[0], depth_img.shape[1], 6))
         output3D['depth_3d_sam'] = np.stack((uv_depth, output2D['sam_seg_on_depth'].get_image()), axis=2).reshape((depth_img.shape[0], depth_img.shape[1], 6))
 
@@ -743,9 +758,10 @@ class VisualizationDemoIndoor(VisualizationDemo):
         radius = 1.5 / min(h, w) * 2.0
 
 
-        total = ['rgb_3d_sam_mask', 'depth_3d_sam_mask']
+        total = ['rgb_3d_sam', 'depth_3d_sam', 'rgb_3d_sam_mask', 'depth_3d_sam_mask']
         num_frames = 45
         degrees = np.linspace(120, 220, num_frames)
+        frames_all = {}
         for j, name in enumerate(total):
             img = torch.from_numpy(xyzrgb[name][:, :, 3:] / 255.).to(device).float()
             pcd = Pointclouds(points=[pts], features=[img.squeeze().reshape(-1, 3)])
@@ -761,11 +777,17 @@ class VisualizationDemoIndoor(VisualizationDemo):
                 result = result.permute(0, 3, 1, 2)
                 frame = (255. * result.detach().cpu().squeeze().permute(1, 2, 0).numpy()).astype(np.uint8)
                 frames.append(frame)
+            
+            frames_all[name] = frames
 
-            # video_out_file = '{}.gif'.format(name)
+            # video_out_file = '{}.mp4'.format(name)
             # imageio.mimwrite(os.path.join('outputs', video_out_file), frames, fps=25)
             
             video_out_file = '{}.mp4'.format(name)
             imageio.mimwrite(os.path.join('outputs', video_out_file), frames, fps=25, quality=8)
-
-            
+        
+        video_out_file = '{}.mp4'.format('RGB_3D_All')
+        imageio.mimwrite(os.path.join('outputs', video_out_file), frames_all['rgb_3d_sam_mask']+frames_all['rgb_3d_sam'], fps=25, quality=8)
+        
+        video_out_file = '{}.mp4'.format('Depth_3D_All')
+        imageio.mimwrite(os.path.join('outputs', video_out_file), frames_all['depth_3d_sam_mask']+frames_all['depth_3d_sam'], fps=25, quality=8)

tools/util.py

@@ -0,0 +1,296 @@
+import numpy as np
+import torch
+import os
+import copy
+from PIL import Image
+import json
+import imageio
+# import clip
+
+
+SCANNET_COLOR_MAP_20 = {-1: (0., 0., 0.), 0: (174., 199., 232.), 1: (152., 223., 138.), 2: (31., 119., 180.), 3: (255., 187., 120.), 4: (188., 189., 34.), 5: (140., 86., 75.),
+                        6: (255., 152., 150.), 7: (214., 39., 40.), 8: (197., 176., 213.), 9: (148., 103., 189.), 10: (196., 156., 148.), 11: (23., 190., 207.), 12: (247., 182., 210.), 
+                        13: (219., 219., 141.), 14: (255., 127., 14.), 15: (158., 218., 229.), 16: (44., 160., 44.), 17: (112., 128., 144.), 18: (227., 119., 194.), 19: (82., 84., 163.)}
+
+class Voxelize(object):
+    def __init__(self,
+                 voxel_size=0.05,
+                 hash_type="fnv",
+                 mode='train',
+                 keys=("coord", "normal", "color", "label"),
+                 return_discrete_coord=False,
+                 return_min_coord=False):
+        self.voxel_size = voxel_size
+        self.hash = self.fnv_hash_vec if hash_type == "fnv" else self.ravel_hash_vec
+        assert mode in ["train", "test"]
+        self.mode = mode
+        self.keys = keys
+        self.return_discrete_coord = return_discrete_coord
+        self.return_min_coord = return_min_coord
+
+    def __call__(self, data_dict):
+        assert "coord" in data_dict.keys()
+        discrete_coord = np.floor(data_dict["coord"] / np.array(self.voxel_size)).astype(np.int)
+        min_coord = discrete_coord.min(0) * np.array(self.voxel_size)
+        discrete_coord -= discrete_coord.min(0)
+        key = self.hash(discrete_coord)
+        idx_sort = np.argsort(key)
+        key_sort = key[idx_sort]
+        _, inverse, count = np.unique(key_sort, return_inverse=True, return_counts=True)
+        if self.mode == 'train':  # train mode
+            # idx_select = np.cumsum(np.insert(count, 0, 0)[0:-1]) + np.random.randint(0, count.max(), count.size) % count
+            idx_select = np.cumsum(np.insert(count, 0, 0)[0:-1])
+            idx_unique = idx_sort[idx_select]
+            if self.return_discrete_coord:
+                data_dict["discrete_coord"] = discrete_coord[idx_unique]
+            if self.return_min_coord:
+                data_dict["min_coord"] = min_coord.reshape([1, 3])
+            for key in self.keys:
+                data_dict[key] = data_dict[key][idx_unique]
+            return data_dict
+
+        elif self.mode == 'test':  # test mode
+            data_part_list = []
+            for i in range(count.max()):
+                idx_select = np.cumsum(np.insert(count, 0, 0)[0:-1]) + i % count
+                idx_part = idx_sort[idx_select]
+                data_part = dict(index=idx_part)
+                for key in data_dict.keys():
+                    if key in self.keys:
+                        data_part[key] = data_dict[key][idx_part]
+                    else:
+                        data_part[key] = data_dict[key]
+                if self.return_discrete_coord:
+                    data_part["discrete_coord"] = discrete_coord[idx_part]
+                if self.return_min_coord:
+                    data_part["min_coord"] = min_coord.reshape([1, 3])
+                data_part_list.append(data_part)
+            return data_part_list
+        else:
+            raise NotImplementedError
+
+    @staticmethod
+    def ravel_hash_vec(arr):
+        """
+        Ravel the coordinates after subtracting the min coordinates.
+        """
+        assert arr.ndim == 2
+        arr = arr.copy()
+        arr -= arr.min(0)
+        arr = arr.astype(np.uint64, copy=False)
+        arr_max = arr.max(0).astype(np.uint64) + 1
+
+        keys = np.zeros(arr.shape[0], dtype=np.uint64)
+        # Fortran style indexing
+        for j in range(arr.shape[1] - 1):
+            keys += arr[:, j]
+            keys *= arr_max[j + 1]
+        keys += arr[:, -1]
+        return keys
+
+    @staticmethod
+    def fnv_hash_vec(arr):
+        """
+        FNV64-1A
+        """
+        assert arr.ndim == 2
+        # Floor first for negative coordinates
+        arr = arr.copy()
+        arr = arr.astype(np.uint64, copy=False)
+        hashed_arr = np.uint64(14695981039346656037) * np.ones(arr.shape[0], dtype=np.uint64)
+        for j in range(arr.shape[1]):
+            hashed_arr *= np.uint64(1099511628211)
+            hashed_arr = np.bitwise_xor(hashed_arr, arr[:, j])
+        return hashed_arr
+
+
+def overlap_percentage(mask1, mask2):
+    intersection = np.logical_and(mask1, mask2)
+    area_intersection = np.sum(intersection)
+
+    area_mask1 = np.sum(mask1)
+    area_mask2 = np.sum(mask2)
+
+    smaller_area = min(area_mask1, area_mask2)
+
+    return area_intersection / smaller_area
+
+
+def remove_samll_masks(masks, ratio=0.8):
+    filtered_masks = []
+    skip_masks = set()
+
+    for i, mask1_dict in enumerate(masks):
+        if i in skip_masks:
+            continue
+
+        should_keep = True
+        for j, mask2_dict in enumerate(masks):
+            if i == j or j in skip_masks:
+                continue
+            mask1 = mask1_dict["segmentation"]
+            mask2 = mask2_dict["segmentation"]
+            overlap = overlap_percentage(mask1, mask2)
+            if overlap > ratio:
+                if np.sum(mask1) < np.sum(mask2):
+                    should_keep = False
+                    break
+                else:
+                    skip_masks.add(j)
+
+        if should_keep:
+            filtered_masks.append(mask1)
+
+    return filtered_masks
+
+
+def to_numpy(x):
+    if isinstance(x, torch.Tensor):
+        x = x.clone().detach().cpu().numpy()
+    assert isinstance(x, np.ndarray)
+    return x
+
+
+def save_point_cloud(coord, color=None, file_path="pc.ply", logger=None):
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+    coord = to_numpy(coord)
+    if color is not None:
+        color = to_numpy(color)
+    pcd = o3d.geometry.PointCloud()
+    pcd.points = o3d.utility.Vector3dVector(coord)
+    pcd.colors = o3d.utility.Vector3dVector(np.ones_like(coord) if color is None else color)
+    o3d.io.write_point_cloud(file_path, pcd)
+    if logger is not None:
+        logger.info(f"Save Point Cloud to: {file_path}")
+
+
+def remove_small_group(group_ids, th):
+    unique_elements, counts = np.unique(group_ids, return_counts=True)
+    result = group_ids.copy()
+    for i, count in enumerate(counts):
+        if count < th:
+            result[group_ids == unique_elements[i]] = -1
+    
+    return result
+
+
+def pairwise_indices(length):
+    return [[i, i + 1] if i + 1 < length else [i] for i in range(0, length, 2)]
+
+
+def num_to_natural(group_ids):
+    '''
+    Change the group number to natural number arrangement
+    '''
+    if np.all(group_ids == -1):
+        return group_ids
+    array = copy.deepcopy(group_ids)
+    unique_values = np.unique(array[array != -1])
+    mapping = np.full(np.max(unique_values) + 2, -1)
+    mapping[unique_values + 1] = np.arange(len(unique_values))
+    array = mapping[array + 1]
+    return array
+
+
+def get_matching_indices(source, pcd_tree, search_voxel_size, K=None):
+    match_inds = []
+    for i, point in enumerate(source.points):
+        [_, idx, _] = pcd_tree.search_radius_vector_3d(point, search_voxel_size)
+        if K is not None:
+            idx = idx[:K]
+        for j in idx:
+            # match_inds[i, j] = 1
+            match_inds.append((i, j))
+    return match_inds
+
+
+def visualize_3d(data_dict, text_feat_path, save_path):
+    text_feat = torch.load(text_feat_path)
+    group_logits = np.einsum('nc,mc->nm', data_dict["group_feat"], text_feat)
+    group_labels = np.argmax(group_logits, axis=-1)
+    labels = group_labels[data_dict["group"]]
+    labels[data_dict["group"] == -1] = -1
+    visualize_pcd(data_dict["coord"], data_dict["color"], labels, save_path)
+
+
+def visualize_pcd(coord, pcd_color, labels, save_path):
+    # alpha = 0.5
+    label_color = np.array([SCANNET_COLOR_MAP_20[label] for label in labels])
+    # overlay = (pcd_color * (1-alpha) + label_color * alpha).astype(np.uint8) / 255
+    label_color = label_color / 255
+    save_point_cloud(coord, label_color, save_path)
+
+
+def visualize_2d(img_color, labels, img_size, save_path):
+    import matplotlib.pyplot as plt
+    # from skimage.segmentation import mark_boundaries
+    # from skimage.color import label2rgb
+    label_names = ["wall", "floor", "cabinet", "bed", "chair",
+           "sofa", "table", "door", "window", "bookshelf",
+           "picture", "counter", "desk", "curtain", "refridgerator",
+           "shower curtain", "toilet", "sink", "bathtub", "other"]
+    colors = np.array(list(SCANNET_COLOR_MAP_20.values()))[1:]
+    segmentation_color = np.zeros((img_size[0], img_size[1], 3))
+    for i, color in enumerate(colors):
+        segmentation_color[labels == i] = color
+    alpha = 1
+    overlay = (img_color * (1-alpha) + segmentation_color * alpha).astype(np.uint8)
+    fig, ax = plt.subplots()
+    ax.imshow(overlay)
+    patches = [plt.plot([], [], 's', color=np.array(color)/255, label=label)[0] for label, color in zip(label_names, colors)]
+    plt.legend(handles=patches, bbox_to_anchor=(0.5, -0.1), loc='upper center', ncol=4, fontsize='small')
+    plt.savefig(save_path, bbox_inches='tight')
+    plt.show()
+
+
+def visualize_partition(coord, group_id, save_path):
+    group_id = group_id.reshape(-1)
+    num_groups = group_id.max() + 1
+    group_colors = np.random.rand(num_groups, 3)
+    group_colors = np.vstack((group_colors, np.array([0,0,0])))
+    color = group_colors[group_id]
+    save_point_cloud(coord, color, save_path)
+
+
+def delete_invalid_group(group, group_feat):
+    indices = np.unique(group[group != -1])
+    group = num_to_natural(group)
+    group_feat = group_feat[indices]
+    return group, group_feat
+
+def group_sem_voting(semantic_label, seg_result, instance_num=0):
+    if instance_num == 0:
+        instance_num = seg_result.max() + 1
+    seg_labels = []
+    sem_map = -1 * torch.ones_like(semantic_label)
+    for n in range(instance_num):
+        mask = (seg_result == n)
+        if mask.sum() == 0: 
+            sem_map[mask] = -1
+            seg_labels.append(-1)
+            continue
+        seg_label_n_cover, seg_label_n_nums  = torch.unique(semantic_label[mask], return_counts=True)
+        seg_label_n = seg_label_n_cover[seg_label_n_nums.max(-1)[1]]
+        seg_labels.append(seg_label_n)
+        sem_map[mask] = seg_label_n
+    
+    return sem_map
+
+def two_image_to_gif(image_1, image_2, name):
+    num_begin = 30
+    num_frames = 30
+    num_end = 30
+    frames = []
+    for i in range(num_begin):
+        frames.append(image_1)
+    for i in range(num_frames):
+        image_tmp = image_1 + (image_2 - image_1) * (i / (num_frames - 1))
+        frames.append(image_tmp.astype(np.uint8))
+    for i in range(num_end):
+        frames.append(image_2)
+        
+    # video_out_file = '{}.gif'.format(name)
+    # imageio.mimwrite(os.path.join('outputs', video_out_file), frames, fps=25)
+    
+    video_out_file = '{}.mp4'.format(name)
+    imageio.mimwrite(os.path.join('outputs', video_out_file), frames, fps=25, quality=8)