Depth V1 working... probably broken elsewhere

README.md

@@ -5,7 +5,7 @@ colorFrom: yellow
 colorTo: purple
 sdk: gradio
 python_version: 3.10.13
-sdk_version: 5.16.0
+sdk_version: 5.16.1
 app_file: app.py
 pinned: true
 short_description: Transform Your Images into Mesmerizing Hexagon Grids

app.py

@@ -1,5 +1,5 @@
 import gradio as gr
-import spaces
+
 import os
 # Import constants
 import numpy as np
@@ -10,7 +10,6 @@ from PIL import Image, ImageFilter
 import cv2
 import utils.constants as constants
 
-
 from haishoku.haishoku import Haishoku
 
 from tempfile import NamedTemporaryFile
@@ -20,7 +19,7 @@ import random
 from transformers import AutoTokenizer , DPTImageProcessor, DPTForDepthEstimation
 from pathlib import Path
 import logging
-logging.getLogger("transformers.modeling_utils").setLevel(logging.ERROR)
+#logging.getLogger("transformers.modeling_utils").setLevel(logging.ERROR)
 import gc
 
 IS_SHARED_SPACE = constants.IS_SHARED_SPACE
@@ -88,6 +87,7 @@ PIPELINE_CLASSES = {
     "FluxControlPipeline": FluxControlPipeline
 }
 
+import spaces
 #from utils.depth_estimation import generate_depth_and_3d
 
 
@@ -299,7 +299,11 @@ class Condition(object):
         type_id = torch.ones_like(ids[:, :1]) * self.type_id
         return tokens, ids, type_id
 
-@spaces.GPU(duration=150, progress=gr.Progress(track_tqdm=True))
+@spaces.GPU(duration=140, progress=gr.Progress(track_tqdm=True))
+def generate_image(pipe, generate_params, progress=gr.Progress(track_tqdm=True)):
+    return pipe(**generate_params)
+
+@spaces.GPU()
 def generate_image_lowmem(
     text,
     neg_prompt=None,
@@ -501,7 +505,7 @@ def generate_image_lowmem(
         generate_params = {k: v for k, v in generate_params.items() if v is not None}
         print(f"generate_params: {generate_params}")
         # Generate the image
-        result = pipe(**generate_params)
+        result = generate_image(pipe,generate_params)
         image = result.images[0]
         # Clean up
         del result
@@ -697,7 +701,7 @@ depth_model = DPTForDepthEstimation.from_pretrained("Intel/dpt-large", ignore_mi
 
 @spaces.GPU()
 def estimate_depth(image):
-
+ 
 
     # Ensure image is in RGB mode
     if image.mode != "RGB":
@@ -740,15 +744,15 @@ def estimate_depth(image):
     return depth_pil, output
 
 @spaces.GPU()
-def create_3d_model(rgb_image, depth_array, voxel_size_factor=0.01):
+def create_3d_model(rgb_image, depth_array, voxel_size_factor=0.01, depth=10):
     import open3d as o3d
     depth_o3d = o3d.geometry.Image(depth_array.astype(np.float32))
-    rgb_o3d = o3d.geometry.Image(np.array(rgb_image))
+    rgb_o3d = o3d.geometry.Image(np.array(rgb_image))    
 
     rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(
         rgb_o3d,
         depth_o3d,
-        convert_rgb_to_intensity=True
+        convert_rgb_to_intensity=False
     )
 
     # Create a point cloud from the RGBD image
@@ -765,16 +769,46 @@ def create_3d_model(rgb_image, depth_array, voxel_size_factor=0.01):
         rgbd_image,
         camera_intrinsic
     )
+    # ########
+    print("normals\n")
+    pcd.normals = o3d.utility.Vector3dVector(
+        np.zeros((1, 3))
+    )  # invalidate existing normals
+    pcd.estimate_normals(
+        search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.01, max_nn=30)
+    )
+    pcd.orient_normals_towards_camera_location(
+        camera_location=np.array([0.0, 0.0, 832.0])
+    )
+    pcd.transform([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]])
+    pcd.transform([[-1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]])
+    # ########
+    print(f"run Poisson surface reconstruction: depth {depth}")
+    with o3d.utility.VerbosityContextManager(o3d.utility.VerbosityLevel.Debug) as cm:
+        mesh_raw, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(
+            pcd, depth=depth, width=0, scale=1.1, linear_fit=True
+        )
 
     # Voxel downsample
-    voxel_size = max(pcd.get_max_bound() - pcd.get_min_bound()) * voxel_size_factor
-    voxel_grid = o3d.geometry.VoxelGrid.create_from_point_cloud(pcd, voxel_size=voxel_size)
+    voxel_size = max(pcd.get_max_bound() - pcd.get_min_bound()) / (max(rgb_image.size) * voxel_size_factor)
+    print(f"voxel_size = {voxel_size:e}")
+    #voxel_grid = o3d.geometry.VoxelGrid.create_from_point_cloud(pcd, voxel_size=voxel_size)
+    mesh = mesh_raw.simplify_vertex_clustering(
+        voxel_size=voxel_size,
+        contraction=o3d.geometry.SimplificationContraction.Average,
+    )
+
+    # vertices_to_remove = densities < np.quantile(densities, 0.001)
+    # mesh.remove_vertices_by_mask(vertices_to_remove)
+    bbox = pcd.get_axis_aligned_bounding_box()
+    mesh_crop = mesh.crop(bbox)
 
     # Save the 3D model to a temporary file
     temp_dir = Path.cwd() / "temp_models"
     temp_dir.mkdir(exist_ok=True)
-    model_path = temp_dir / "model.ply"
-    o3d.io.write_voxel_grid(str(model_path), voxel_grid)
+    model_path = temp_dir / "model.gltf"
+    #o3d.io.write_voxel_grid(str(model_path), voxel_grid)
+    o3d.io.write_triangle_mesh(str(model_path), mesh_crop, write_triangle_uvs=True)
 
     return str(model_path)
 
@@ -869,7 +903,7 @@ with gr.Blocks(css_paths="style_20250128.css", title=title, theme='Surn/beeuty')
                 key="imgInput",
                 image_mode=None,
                 format="PNG", 
-                show_download_button=True,
+                show_download_button=True
             )
 
         # New code to convert input image to RGBA PNG
@@ -968,14 +1002,13 @@ with gr.Blocks(css_paths="style_20250128.css", title=title, theme='Surn/beeuty')
                     # Update map_options to a Dropdown with choices from constants.PROMPTS keys
                     with gr.Row():
                         with gr.Column():
-                                map_options = gr.Dropdown(
-                                label="Map Options",
-                                choices=list(constants.PROMPTS.keys()),
-                                value="Alien Landscape",
-                                elem_classes="solid",
-                                scale=0
+                            map_options = gr.Dropdown(
+                            label="Map Options",
+                            choices=list(constants.PROMPTS.keys()),
+                            value="Alien Landscape",
+                            elem_classes="solid",
+                            scale=0
                             )
-                        with gr.Column():
                             # Add Dropdown for sizing of Images, height and width based on selection. Options are 16x9, 16x10, 4x5, 1x1
                             # The values of height and width are based on common resolutions for each aspect ratio
                             # Default to 16x9, 912x512
@@ -1112,7 +1145,7 @@ with gr.Blocks(css_paths="style_20250128.css", title=title, theme='Surn/beeuty')
             generate_depth_button = gr.Button("Generate Depth Map and 3D Model From Selected Image", elem_classes="solid", variant="secondary")
         with gr.Row():
             depth_map_output = gr.Image(label="Depth Map", image_mode="L", elem_classes="centered solid imgcontainer", format="PNG", type="filepath", key="ImgDepth")
-            model_output = gr.Model3D(label="3D Model", clear_color=[1.0, 1.0, 1.0, 0.25], key="Img3D", elem_classes="centered solid imgcontainer")
+            model_output = gr.Model3D(label="3D Model", clear_color=[1.0, 1.0, 1.0, 1.0], key="Img3D", elem_classes="centered solid imgcontainer")
     with gr.Row():
         gr.Examples(examples=[
             ["assets//examples//hex_map_p1.png", False, True, -32,-31,80,80,-1.8,0,35,0,1,"#FFD0D0", 15],
@@ -1213,23 +1246,22 @@ with gr.Blocks(css_paths="style_20250128.css", title=title, theme='Surn/beeuty')
         scroll_to_output=True
     )
 
-
+    hexaGrid.queue(default_concurrency_limit=1,max_size=12,api_open=False)
+    hexaGrid.launch(allowed_paths=["assets","/","./assets","images","./images", "./images/prerendered"], favicon_path="./assets/favicon.ico", max_file_size="10mb")
 
 if __name__ == "__main__":
     logging.basicConfig(
         format="[%(levelname)s] %(asctime)s %(message)s", level=logging.INFO
     )
     logging.info("Environment Variables: %s" % os.environ)
-        # if _get_output(["nvcc", "--version"]) is None:
-        #     logging.info("Installing CUDA toolkit...")
-        #     install_cuda_toolkit()
-        # else:
-        #     logging.info("Detected CUDA: %s" % _get_output(["nvcc", "--version"]))
+    # if _get_output(["nvcc", "--version"]) is None:
+    #     logging.info("Installing CUDA toolkit...")
+    #     install_cuda_toolkit()
+    # else:
+    #     logging.info("Detected CUDA: %s" % _get_output(["nvcc", "--version"]))
 
-        # logging.info("Installing CUDA extensions...")
-        # setup_runtime_env()
-        #main(os.getenv("DEBUG") == "1")
-        #main()
+    # logging.info("Installing CUDA extensions...")
+    # setup_runtime_env()
+    #main(os.getenv("DEBUG") == "1")
+    #main()
 
-    hexaGrid.queue(default_concurrency_limit=1,max_size=12,api_open=False)
-    hexaGrid.launch(allowed_paths=["assets","/","./assets","images","./images", "./images/prerendered"], favicon_path="./assets/favicon.ico", max_file_size="10mb")

utils/image_utils.py

@@ -7,7 +7,6 @@ import numpy as np
 #from decimal import ROUND_CEILING
 from PIL import Image, ImageChops, ImageDraw, ImageEnhance, ImageFilter, ImageDraw, ImageOps, ImageMath
 from typing import List, Union
-#import numpy as np
 #import math
 from pathlib import Path
 from utils.constants import default_lut_example_img

web-ui.bat

@@ -1,5 +1,5 @@
 set NVIDIA_VISIBLE_DEVICES=0
 set CUDA_VISIBLE_DEVICES=0
 set CUDA_MODULE_LOADING=LAZY
-python -m app
+py -m app
 pause