add point filter; check if we can upload gradio_cached_examples

.gitattributes

@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.JPG filter=lfs diff=lfs merge=lfs -text
 *.mp4 filter=lfs diff=lfs merge=lfs -text
 vggsfm_code/examples/ filter=lfs diff=lfs merge=lfs -text
+gradio_cached_examples/ filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,6 +1,6 @@
 ---
 title: VGGSfM
-emoji: 🏢
+emoji: 🏛️ 
 colorFrom: yellow
 colorTo: blue
 sdk: gradio

app.py

@@ -14,12 +14,13 @@ from datetime import datetime
 
 from vggsfm_code.hf_demo import demo_fn
 from omegaconf import DictConfig, OmegaConf
-from viz_utils.viz_fn import add_camera
+from viz_utils.viz_fn import add_camera, apply_density_filter_np
 import glob
 # 
 from scipy.spatial.transform import Rotation
 import PIL
 import gc
+import open3d as o3d
 
 # import spaces
 
@@ -30,15 +31,13 @@ def vggsfm_demo(
     query_frame_num,
     max_query_pts=4096,
 ):
+
+    import time
+    start_time = time.time()
     gc.collect()
     torch.cuda.empty_cache()
 
-    if input_video is not None:            
-        if not isinstance(input_video, str):
-            input_video = input_video["video"]["path"]
-    
-    cfg_file = "vggsfm_code/cfgs/demo.yaml"
-    cfg = OmegaConf.load(cfg_file)
+    debug = False
 
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
 
@@ -52,58 +51,72 @@ def vggsfm_demo(
     target_dir_images = target_dir + "/images"
     os.makedirs(target_dir_images)
 
-    if input_image is not None:
-        if len(input_image)<3:
-            return None, "Please input at least three frames"
 
-        input_image = sorted(input_image)
-        input_image = input_image[:max_input_image]
+    if debug:
+        predictions = torch.load("predictions_scene2.pth")
+    else:
+        
+        if input_video is not None:            
+            if not isinstance(input_video, str):
+                input_video = input_video["video"]["path"]
         
-        # Copy files to the new directory
-        for file_name in input_image:
-            shutil.copy(file_name, target_dir_images)
-    elif input_video is not None:
-        vs = cv2.VideoCapture(input_video)
+        cfg_file = "vggsfm_code/cfgs/demo.yaml"
+        cfg = OmegaConf.load(cfg_file)
 
-        fps = vs.get(cv2.CAP_PROP_FPS)
+        if input_image is not None:
+            if len(input_image)<3:
+                return None, "Please input at least three frames"
 
+            input_image = sorted(input_image)
+            input_image = input_image[:max_input_image]
+            
+            # Copy files to the new directory
+            for file_name in input_image:
+                shutil.copy(file_name, target_dir_images)
+        elif input_video is not None:
+            vs = cv2.VideoCapture(input_video)
 
-        frame_rate = 1
-        frame_interval = int(fps * frame_rate)
-        
-        video_frame_num = 0
-        count = 0 
-        
-        while video_frame_num<=max_input_image:
-            (gotit, frame) = vs.read()
-            count +=1
+            fps = vs.get(cv2.CAP_PROP_FPS)
 
-            if not gotit:
-                break
+
+            frame_rate = 1
+            frame_interval = int(fps * frame_rate)
+            
+            video_frame_num = 0
+            count = 0 
             
-            if count % frame_interval == 0:
-                cv2.imwrite(target_dir_images+"/"+f"{video_frame_num:06}.png", frame)
-                video_frame_num+=1
+            while video_frame_num<=max_input_image:
+                (gotit, frame) = vs.read()
+                count +=1
+
+                if not gotit:
+                    break
                 
-        if video_frame_num<3:
-            return None, "Please input at least three frames"
-    else:
-        return None, "Input format incorrect"
+                if count % frame_interval == 0:
+                    cv2.imwrite(target_dir_images+"/"+f"{video_frame_num:06}.png", frame)
+                    video_frame_num+=1
+                    
+            if video_frame_num<3:
+                return None, "Please input at least three frames"
+        else:
+            return None, "Input format incorrect"
+            
+        cfg.query_frame_num = query_frame_num
+        cfg.max_query_pts = max_query_pts
+        print(f"Files have been copied to {target_dir_images}")
+        cfg.SCENE_DIR = target_dir
         
-    cfg.query_frame_num = query_frame_num
-    cfg.max_query_pts = max_query_pts
-    print(f"Files have been copied to {target_dir_images}")
-    cfg.SCENE_DIR = target_dir
-    
-    # try:
-    predictions = demo_fn(cfg)
-    # except:
-    # return None, "Something seems to be incorrect. Please verify that your inputs are formatted correctly. If the issue persists, kindly create a GitHub issue for further assistance."
+        # try:
+        predictions = demo_fn(cfg)
+        # except:
+        # return None, "Something seems to be incorrect. Please verify that your inputs are formatted correctly. If the issue persists, kindly create a GitHub issue for further assistance."
     
     glbscene = vggsfm_predictions_to_glb(predictions)
     
     glbfile = target_dir + "/glbscene.glb"
-    glbscene.export(file_obj=glbfile)    
+    glbscene.export(file_obj=glbfile) 
+    # glbscene.export(file_obj=glbfile, line_settings= {'point_size': 20})    
+
 
     del predictions
     gc.collect()
@@ -111,12 +124,18 @@ def vggsfm_demo(
     
     print(input_image)
     print(input_video)
+    end_time = time.time()
+    execution_time = end_time - start_time
+    print(f"Execution time: {execution_time} seconds")
+
     return glbfile, "Success"
 
 
 
 
-def vggsfm_predictions_to_glb(predictions):
+def vggsfm_predictions_to_glb(predictions, sphere=False):
+    # del predictions['reconstruction']
+    # torch.save(predictions, "predictions_scene2.pth")
     # learned from https://github.com/naver/dust3r/blob/main/dust3r/viz.py
     points3D = predictions["points3D"].cpu().numpy()
     points3D_rgb = predictions["points3D_rgb"].cpu().numpy()
@@ -124,13 +143,34 @@ def vggsfm_predictions_to_glb(predictions):
     
     extrinsics_opencv = predictions["extrinsics_opencv"].cpu().numpy()
     intrinsics_opencv = predictions["intrinsics_opencv"].cpu().numpy()
+    
+        
     raw_image_paths = predictions["raw_image_paths"]
     images = predictions["images"].permute(0,2,3,1).cpu().numpy()
     images = (images*255).astype(np.uint8)
     
     glbscene = trimesh.Scene()
-    point_cloud = trimesh.PointCloud(points3D, colors=points3D_rgb)
-    glbscene.add_geometry(point_cloud)
+    
+    if True:
+        pcd = o3d.geometry.PointCloud()
+        pcd.points = o3d.utility.Vector3dVector(points3D)
+        pcd.colors = o3d.utility.Vector3dVector(points3D_rgb)
+        
+        cl, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=1.0)
+        filtered_pcd = pcd.select_by_index(ind)
+
+        print(f"Filter out {len(points3D) - len(filtered_pcd.points)} 3D points")
+        points3D = np.asarray(filtered_pcd.points)
+        points3D_rgb = np.asarray(filtered_pcd.colors)
+
+    
+    if sphere:
+        # TOO SLOW
+        print("testing sphere")
+        # point_size = 0.02  
+    else: 
+        point_cloud = trimesh.PointCloud(points3D, colors=points3D_rgb)
+        glbscene.add_geometry(point_cloud)
 
 
     camera_edge_colors = [(255, 0, 0), (0, 0, 255), (0, 255, 0), (255, 0, 255), (255, 204, 0), (0, 204, 204),
@@ -160,27 +200,21 @@ def vggsfm_predictions_to_glb(predictions):
     glbscene.apply_transform(np.linalg.inv(np.linalg.inv(extrinsics_opencv_4x4[0]) @ opengl_mat @ rot))
 
     # Calculate the bounding box center and apply the translation
-    bounding_box = glbscene.bounds
-    center = (bounding_box[0] + bounding_box[1]) / 2
-    translation = np.eye(4)
-    translation[:3, 3] = -center
+    # bounding_box = glbscene.bounds
+    # center = (bounding_box[0] + bounding_box[1]) / 2
+    # translation = np.eye(4)
+    # translation[:3, 3] = -center
 
-    glbscene.apply_transform(translation)
+    # glbscene.apply_transform(translation)
     # glbfile = "glbscene.glb"
     # glbscene.export(file_obj=glbfile)    
     return glbscene
 
 apple_video = "vggsfm_code/examples/videos/apple_video.mp4"
-# os.path.join(os.path.dirname(__file__), "apple_video.mp4")
 british_museum_video = "vggsfm_code/examples/videos/british_museum_video.mp4"
-
-# os.path.join(os.path.dirname(__file__), "british_museum_video.mp4")
 cake_video = "vggsfm_code/examples/videos/cake_video.mp4"
-
 bonsai_video = "vggsfm_code/examples/videos/bonsai_video.mp4"
-
-# os.path.join(os.path.dirname(__file__), "cake_video.mp4")
-
+face_video =  "vggsfm_code/examples/videos/in2n_face_video.mp4"
 
 
 apple_images = glob.glob(f'vggsfm_code/examples/apple/images/*')
@@ -188,6 +222,7 @@ bonsai_images = glob.glob(f'vggsfm_code/examples/bonsai/images/*')
 cake_images = glob.glob(f'vggsfm_code/examples/cake/images/*')
 british_museum_images = glob.glob(f'vggsfm_code/examples/british_museum/images/*')
 
+face_images = glob.glob(f'vggsfm_code/examples/in2n_face/images/*')
 
 
 
@@ -221,7 +256,7 @@ with gr.Blocks() as demo:
                                          info="More query points usually lead to denser reconstruction at lower speeds.")
         
         with gr.Column(scale=3):
-            reconstruction_output = gr.Model3D(label="Reconstruction", height=520)
+            reconstruction_output = gr.Model3D(label="Reconstruction", height=520, zoom_speed=1, pan_speed=1)
             log_output = gr.Textbox(label="Log")
 
     with gr.Row():
@@ -232,6 +267,7 @@ with gr.Blocks() as demo:
     
     
     examples = [
+        [face_video, face_images, 4, 2048],
         [british_museum_video, british_museum_images, 1, 4096],
         [apple_video, apple_images, 6, 2048],
         [bonsai_video, bonsai_images, 3, 2048],

gradio_cached_examples/17/log.csv

@@ -0,0 +1,5 @@
+Reconstruction,Log,flag,username,timestamp
+"{""path"": ""gradio_cached_examples/17/Reconstruction/bbd7de840562d63202f0/glbscene.glb"", ""url"": ""/file=/tmp/gradio/fca5f8328739501143569c8f61c49f859d6050aa/glbscene.glb"", ""size"": null, ""orig_name"": ""glbscene.glb"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}",Success,,,2024-06-27 20:41:21.350514
+"{""path"": ""gradio_cached_examples/17/Reconstruction/945304e792b0f852dd99/glbscene.glb"", ""url"": ""/file=/tmp/gradio/4cc4bf183538e927e644e048332dcdaff12da3cf/glbscene.glb"", ""size"": null, ""orig_name"": ""glbscene.glb"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}",Success,,,2024-06-27 20:41:41.130464
+"{""path"": ""gradio_cached_examples/17/Reconstruction/acbcb0d82e2838cc056a/glbscene.glb"", ""url"": ""/file=/tmp/gradio/ba332ab61cbd09dd2e7c53f11d876cb561b990cd/glbscene.glb"", ""size"": null, ""orig_name"": ""glbscene.glb"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}",Success,,,2024-06-27 20:42:49.325264
+"{""path"": ""gradio_cached_examples/17/Reconstruction/eff9890930c07374da16/glbscene.glb"", ""url"": ""/file=/tmp/gradio/803adcd2ead20dcc56157d3e0fe98a7b7cd12bd7/glbscene.glb"", ""size"": null, ""orig_name"": ""glbscene.glb"", ""mime_type"": null, ""is_stream"": false, ""meta"": {""_type"": ""gradio.FileData""}}",Success,,,2024-06-27 20:43:17.587081

images_to_videos.py

@@ -2,7 +2,7 @@ import cv2
 import os
 
 # Parameters
-name = "apple"
+name = "in2n_face"
 folder_path = f'vggsfm_code/examples/{name}/images'  # Update with the path to your images
 video_path = f'vggsfm_code/examples/videos/{name}_video.mp4'
 fps = 1  # frames per second

requirements.txt

@@ -10,3 +10,4 @@ numpy==1.26.3
 pycolmap==0.6.1
 https://huggingface.co/facebook/VGGSfM/resolve/main/poselib-2.0.2-cp310-cp310-linux_x86_64.whl
 trimesh
+open3d

vggsfm_code/examples/videos/british_museum_video.mp4

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4fbbde1a54deaadb5144a3bcecdd2c404fe950312f3b8f2b9628ba49067053df
-size 407548
+oid sha256:39bbcf545761bbeeb4e3cba24622783e93d608fcfe4217ca4575b4bf81178166
+size 929009

vggsfm_code/examples/videos/in2n_face_video.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:746e08978e78446b494e70fc04579beaab3210f6e39b3a72a385bd7d22a112ff
+size 2781740

viz_utils/viz_fn.py

@@ -19,6 +19,46 @@ from datetime import datetime
 from scipy.spatial.transform import Rotation
 import PIL
 
+from scipy.spatial import cKDTree
+
+
+
+def get_density_np(pcl, K=0.005):
+    if isinstance(K, float):
+        K = max(int(K * pcl.shape[0]), 1)
+    
+    tree = cKDTree(pcl)
+    dists, _ = tree.query(pcl, k=K+1)  # K+1 because the point itself is included
+    
+    dists = dists[:, 1:]  # Remove the zero distance to itself
+    D = np.sqrt(dists).sum(axis=1)
+    
+    return D
+
+def apply_density_filter_np(pts, feats=None, density_filter=0.9, density_K=100):
+    """
+    :param pts: ndarray of shape (N, 3) representing the point cloud.
+    :param feats: ndarray of corresponding features for the point cloud.
+    :param density_filter: Float, the percentage of points to keep based on density.
+    :param density_K: Int, number of nearest neighbors to consider for density calculation.
+    :return: Filtered points and their corresponding features.
+    """
+    # Calculate densities
+    D = get_density_np(pts, K=density_K)
+
+    # Apply the density filter
+    topk_k = max(int((1 - density_filter) * pts.shape[0]), 1)
+    val = np.partition(D, topk_k)[topk_k]
+    ok = (D <= val)
+    
+    # Filter points and features
+    filtered_pts = pts[ok]
+    if feats is not None:
+        filtered_feats = feats[ok]
+    else:
+        filtered_feats = feats
+    return filtered_pts, filtered_feats
+
 
 def add_camera(scene, pose_c2w, edge_color, image=None, 
                   focal=None, imsize=None,