diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..288ac77985c117c220d3047266074117571c4ed7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,223 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/bear.gif filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/cherry.gif filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/teddy.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/bear_background.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/bear_composite.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/boy_background.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/boy_composite.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/corgi_background.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/corgi_composite.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/stairs_background.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/stairs_composite.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/tinycudann-1.7.post70240121-cp310-cp310-linux_x86_64.whl filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/outpaint_0.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/outpaint_1.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1000-val.mp4 filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test.mp4 filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/0.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/1.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/10.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/100.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/101.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/102.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/103.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/104.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/105.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/106.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/107.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/108.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/109.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/11.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/110.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/111.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/112.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/113.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/114.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/115.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/116.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/117.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/118.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/119.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/12.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/13.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/14.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/15.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/16.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/17.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/18.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/19.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/2.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/20.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/21.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/22.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/23.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/24.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/25.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/26.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/27.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/28.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/29.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/3.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/30.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/31.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/32.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/33.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/34.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/35.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/36.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/37.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/38.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/39.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/4.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/40.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/41.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/42.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/43.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/44.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/45.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/46.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/47.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/48.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/49.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/5.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/50.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/51.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/52.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/53.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/54.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/55.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/56.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/57.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/58.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/59.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/6.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/60.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/61.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/62.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/63.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/64.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/65.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/66.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/67.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/68.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/69.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/7.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/70.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/71.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/72.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/73.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/74.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/75.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/76.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/77.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/78.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/79.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/8.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/80.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-val.mp4 filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it500-val.mp4 filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat.mp4 filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/0.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/1.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/10.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/100.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/101.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/102.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/103.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/104.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/105.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/106.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/107.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/108.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/109.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/11.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/110.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/111.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/112.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/113.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/114.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/115.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/116.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/117.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/118.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/119.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/12.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/13.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/14.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/15.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/16.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/17.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/18.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/19.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/2.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/20.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/21.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/22.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/23.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/24.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/25.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/26.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/27.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/28.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/29.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/3.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/30.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/31.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/32.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/33.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/34.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/35.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/36.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/37.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/38.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/39.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/4.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/40.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/41.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/42.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/43.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/44.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/45.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/46.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/47.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/48.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/49.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/5.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/50.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/51.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/52.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/53.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/54.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/55.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/56.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/57.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/58.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/59.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/6.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/60.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/61.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/62.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/63.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/64.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/65.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/66.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/67.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/68.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/69.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/7.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/70.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/71.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/72.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/73.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/74.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/75.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/76.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/77.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/78.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/79.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/8.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/80.png filter=lfs diff=lfs merge=lfs -text +000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/point_cloud.ply filter=lfs diff=lfs merge=lfs -text diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/ckpts/epoch=0-step=1000.ckpt b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/ckpts/epoch=0-step=1000.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..ff62761de5112d572562912a4a0404ce4c031eec --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/ckpts/epoch=0-step=1000.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05b53135d3920aa7616777d0b9040ed4a12f5060d43f700c361aae7805f9d248 +size 28888900 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/ckpts/last.ckpt b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/ckpts/last.ckpt new file mode 100644 index 0000000000000000000000000000000000000000..44c81746b95f09c0d66a50bdce7d5c6572621797 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/ckpts/last.ckpt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2762c6bc7f087ebcf67488e4e6b106a8d7975d5143b26d3294b8a8f75b65a777 +size 28888900 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/cmd.txt b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/cmd.txt new file mode 100644 index 0000000000000000000000000000000000000000..e6cbdfb10020bd88996e52f38ba79ce1275b1d8b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/cmd.txt @@ -0,0 +1,2 @@ +python launch.py --config custom/threestudio-3dgs/configs/scene_lang.yaml --train --gpu 2 exp_root_dir=outputs/mira_video_clips/000000000/000000000017.1 tag=3DitScene system.geometry.geometry_convert_from=depth:/mnt/hdd1/wufan/datasets/MiraData/data/video_frames/000000000/000000000017.1/0.jpg system.geometry.ooi_bbox=[599,250,692,452] system.prompt_processor.prompt=It is night time in a city with tall buildings and neon lights illuminating the streets. system.empty_prompt= The background is a city at night with tall buildings, out of focus system.side_prompt= The background is a city at night with tall buildings, out of focus +Namespace(config='custom/threestudio-3dgs/configs/scene_lang.yaml', gpu='2', train=True, validate=False, test=False, export=False, save_dir=None, gradio=False, verbose=False, typecheck=False) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/.gitignore b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..85ee96fd179d93e152a232ccd37bf16383654ba3 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/.gitignore @@ -0,0 +1,7 @@ +ckpts/ +outputs/ +.threestudio_cache/ + +*.pyc +*.DS_Store + diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/.gitmodules b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/.gitmodules new file mode 100644 index 0000000000000000000000000000000000000000..aa4d11114eb8fcb403a223ebfd12d352429746ac --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/.gitmodules @@ -0,0 +1,15 @@ +[submodule "submodules/MobileSAM-lang"] + path = submodules/MobileSAM-lang + url = https://github.com/zqh0253/MobileSAM-lang.git +[submodule "submodules/segment-anything-langsplat"] + path = submodules/segment-anything-langsplat + url = https://github.com/zqh0253/segment-anything-langsplat.git +[submodule "submodules/simple-knn"] + path = submodules/simple-knn + url = https://github.com/DSaurus/simple-knn.git +[submodule "submodules/diff-gaussian-rasterization"] + path = submodules/diff-gaussian-rasterization + url = https://github.com/zqh0253/diff-gaussian-rasterization-lang +[submodule "submodules/langsplat-rasterization"] + path = submodules/langsplat-rasterization + url = https://github.com/minghanqin/langsplat-rasterization.git diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/README.md b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/README.md new file mode 100644 index 0000000000000000000000000000000000000000..306e92d07aee828a97fafae32b47a9247a137be0 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/README.md @@ -0,0 +1,83 @@ +# 3DitScene: Editing Any Scene via Language-guided Disentangled Gaussian Splatting + +[![Project Page](https://img.shields.io/badge/Project-Website-green)](https://zqh0253.github.io/3DitScene/) +[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/qihang/3Dit-Scene/) +[![arXiv](https://img.shields.io/badge/arXiv-2405.18424-b31b1b.svg)](https://arxiv.org/abs/2405.18424) + + + + + + + + + + + +
Move the bear, and rotate the cameraMove / remove the girl, and rotate the camera
+ +## Installation + ++ Install `Python >= 3.8`. ++ Install `torch >= 1.12`. We have tested on `torch==2.0.1+cu118`, but other versions should also work fine. ++ Clone our repo: +``` +git clone https://github.com/zqh0253/3DitScene.git --recursive +``` ++ Install dependencies: +``` +pip install -r requirements.txt +``` ++ Install submodules: +``` +pip install ./submodules/segment-anything-langsplat +pip install ./submodules/MobileSAM-lang +pip install ./submodules/langsplat-rasterization +pip install ./submodules/simple-knn +``` ++ Prepare weights for `SAM`: +``` +mkdir ckpts +wget https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth -O ./ckpts/sam_vit_h_4b8939.pth +cp submodules/MobileSAM-lang/weights/mobile_sam.pt ./ckpts/ +``` + +## Usage + +Run the following command to launch the optimization procedure: +``` +python -u launch.py --config custom/threestudio-3dgs/configs/scene_lang.yaml --train --gpu 0 tag=3DitScene +system.geometry.geometry_convert_from=depth:${IMGPATH} system.geometry.ooi_bbox=${BBOX} +system.prompt_processor.prompt="${PROMPT}" system.empty_prompt="${EMPTY_PROMPT}" system.side_prompt="${SIDE_PROMPT}" +``` +You should specify the image path `IMGPATH`, the bounding box of the interested object `BBOX`, and the promtps: `PROMPT`, `EMPTY_PROMPT`, `SIDE_PROMPT`. These prompts describe the image itself, the background area behind the image, and the content of the novel view region, respectively. + +Here we provide an image (`./assets/teddy.png`) as example: +``` +python -u launch.py --config custom/threestudio-3dgs/configs/scene_lang.yaml --train --gpu 0 tag=3DitScene +system.geometry.geometry_convert_from=depth:assets/teddy.png system.geometry.ooi_bbox=[122,119,387,495] +system.prompt_processor.prompt="a teddy bear in Times Square" system.empty_prompt="Times Square, out of focus" system.side_prompt="Times Square, out of focus" +``` + +## Huggingface demo + +We provide a huggingface demo. You have two options to explore our demo: +(1) Visit our [online Hugging Face space](https://huggingface.co/spaces/qihang/3Dit-Scene). +(2) Deploy it locally by following these steps: ++ Install the necessary packages and download required files as specified in our [Dockerfile](https://huggingface.co/spaces/qihang/3Dit-Scene/blob/main/Dockerfile), ++ Run the following command to launch the service at `localhost:10091`: +``` +python gradio_app_single_process.py --listen --hf-space --port 10091 +``` + +## Citation + +If you find our work useful, please consider citing: +``` +inproceedings{zhang20243DitScene, + author = {Qihang Zhang and Yinghao Xu and Chaoyang Wang and Hsin-Ying Lee and Gordon Wetzstein and Bolei Zhou and Ceyuan Yang}, + title = {{3DitScene}: Editing Any Scene via Language-guided Disentangled Gaussian Splatting}, + booktitle = {arXiv}, + year = {2024} +} +``` diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/bear.gif b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/bear.gif new file mode 100644 index 0000000000000000000000000000000000000000..0b22157999e2d70a1dcc3bea57e9fd241b816c7e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/bear.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef075fb5f74ea8fc690b0b68d3abd88d151cdc246ac2df27767fc4cbb24227f9 +size 6431590 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/cherry.gif b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/cherry.gif new file mode 100644 index 0000000000000000000000000000000000000000..108a2eaff6cab04c0bc5d2e72002b96a69708816 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/cherry.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0069d4bdd8da45627cbba25f3c49a102220aff23cc8738417a13101d42a3b25 +size 7390642 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/teddy.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/teddy.png new file mode 100644 index 0000000000000000000000000000000000000000..b3109a702c52ea76a0018ccffe87d7bcad3c980c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/assets/teddy.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d73779e3f37a6e8e6171d30019a12851cdd1a69c5bdf2ff1c1b0b8ade8e1db6 +size 389675 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/check_output.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/check_output.py new file mode 100644 index 0000000000000000000000000000000000000000..2a0c0fc7bf68c7fffb84133c78c96ea944cf930b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/check_output.py @@ -0,0 +1,42 @@ +import os +os.environ["MKL_THREADING_LAYER"] = "GNU" + +import numpy as np +import pandas as pd + + +# Load the CSV file +csv_path = "/mnt/hdd1/wufan/datasets/MiraData/data/data_list/miradata_v1_9k_subset_shard_0.csv" +missing_csv_path = "/mnt/hdd1/wufan/datasets/MiraData/data/data_list/miradata_v1_9k_output_missing.csv" +df = pd.read_csv(csv_path) + +output_path = "/mnt/hdd1/wufan/projects/3DitScene/outputs/mira_video_clips" +# Iterate through each row +save_dict = [] +for index, row in df.iterrows(): + + # Construct the image path from file_path + # video_clips/000005007/000005007658.0.mp4 + # '/mnt/hdd1/wufan/projects/3DitScene/outputs/mira_video_clips/000005007/000005007658.0/gs-sds-generation/save/it500-val.mp4' + file_path = row["file_path"].replace("video_clips/", "").replace(".mp4", "/gs-sds-generation") + file_dir = f"/mnt/hdd1/wufan/projects/3DitScene/outputs/mira_video_clips/{file_path}" + + if os.path.exists(file_dir): + for item in os.listdir(file_dir): + file_path = os.path.join(file_dir, item) + + # Check if 'it500-val.mp4' exists in the directory + if not os.path.exists(os.path.join(file_path, "save/it500-val.mp4")) and os.path.isfile(file_path) and os.path.getsize(file_path) > 0: + save_dict.append(row) # Append the current item if the file doesn't exist + else: + save_dict.append(row) + + + # Check if the image exists before proceeding + + +results_df = pd.DataFrame(save_dict) +# Save results to CSV +results_df.to_csv(missing_csv_path, index=False) + + diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/.gitignore b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..b841bd73f6f94f42888e318460563a26fa78f7e7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/.gitignore @@ -0,0 +1,196 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +# End of https://www.toptal.com/developers/gitignore/api/python + +.vscode/ +.threestudio_cache/ +outputs/ +outputs-gradio/ + +# pretrained model weights +*.ckpt +*.pt +*.pth + +# wandb +wandb/ + +custom/* + +load/tets/256_tets.npz + +diff-gaussian-rasterization/ +simple-knn/ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/.pre-commit-config.yaml b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4687d3c76bd4d5b6489c321f53adbd17a4f5b2df --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +default_language_version: + python: python3 + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: check-ast + - id: check-merge-conflict + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + + - repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black + language_version: python3 + + - repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + exclude: README.md + args: ["--profile", "black"] + + # temporarily disable static type checking + # - repo: https://github.com/pre-commit/mirrors-mypy + # rev: v1.2.0 + # hooks: + # - id: mypy + # args: ["--ignore-missing-imports", "--scripts-are-modules", "--pretty"] diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/README.md b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fe564f41025380e3527a404e37b274ac7c52ff69 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/README.md @@ -0,0 +1,129 @@ +# threestudio-3dgs + + + + + + + + +The Gaussian Splatting extension for threestudio. This extension is writen by [Ruizhi Shao](https://github.com/DSaurus) and [Youtian Lin](https://github.com/Linyou). To use it, please install [threestudio](https://github.com/threestudio-project/threestudio) first and then install this extension in threestudio `custom` directory. + +## Advanced Gaussian Splatting Installation (Recommend) +``` +cd custom +git clone https://github.com/DSaurus/threestudio-3dgs.git +cd threestudio-3dgs +git clone --recursive https://github.com/ashawkey/diff-gaussian-rasterization +git clone https://github.com/DSaurus/simple-knn.git +pip install ./diff-gaussian-rasterization +pip install ./simple-knn +``` + +## Native Gaussian Splatting Installation +``` +cd custom +git clone https://github.com/DSaurus/threestudio-3dgs.git +cd threestudio-3dgs +git clone git@github.com:graphdeco-inria/gaussian-splatting.git --recursive +cd gaussian-splatting/submodules +python -m pip install diff-gaussian-rasterization/. +python -m pip install simple-knn/ + +# If you want to export mesh, please install pymeshlab +pip install pymeshlab +``` + + +## Quick Start +``` +# Native Gaussian Splatting + SDS Loss +python launch.py --config custom/threestudio-3dgs/configs/gaussian_splatting.yaml --train --gpu 0 system.prompt_processor.prompt="a delicious hamburger" + +# Advanced Gaussian Splatting with background + SDS Loss +python launch.py --config custom/threestudio-3dgs/configs/gaussian_splatting_background.yaml --train --gpu 0 system.prompt_processor.prompt="a delicious hamburger" + +# Advanced Gaussian Splatting with background and shading + SDS Loss +python launch.py --config custom/threestudio-3dgs/configs/gaussian_splatting_shading.yaml --train --gpu 0 system.prompt_processor.prompt="a delicious hamburger" +``` + +## Gaussian Splatting + MVDream +Please first install [MVDream extension](https://github.com/DSaurus/threestudio-mvdream), then you can run the following script: +``` +# Advanced Gaussian Splatting with background and shading + MVDream +python launch.py --config custom/threestudio-3dgs/configs/gaussian_splatting_mvdream.yaml --train --gpu 0 system.prompt_processor.prompt="an astronaut riding a horse" +``` + +## Gaussian Splatting + Zero-123 +``` +# Advanced Gaussian Splatting + Zero-123 +python launch.py --config custom/threestudio-3dgs/configs/gaussian_splatting_zero123.yaml --train --gpu 0 data.image_path=./load/images/anya_front_rgba.png +``` + +## Resume from checkpoints +``` +# resume training from the last checkpoint, you may replace last.ckpt with any other checkpoints +python launch.py --config path/to/trial/dir/configs/parsed.yaml --train --gpu 0 resume=path/to/trial/dir/ckpts/last.ckpt +``` + +## Load from PLY +``` +# load from Gaussian Splatting ply file +python launch.py --config custom/threestudio-3dgs/configs/gaussian_splatting.yaml --train --gpu 0 system.prompt_processor.prompt="a delicious hamburger" system.geometry.geometry_conver_from=path/to/poinc_cloud.ply + +# only load points position and color from ply file +python launch.py --config custom/threestudio-3dgs/configs/gaussian_splatting.yaml --train --gpu 0 system.prompt_processor.prompt="a delicious hamburger" system.geometry.geometry_conver_from=path/to/poinc_cloud.ply system.geometry.load_ply_only_vertex=true +``` + +If you want to use shap-e initialization, please install [threestudio-shap-e extension](https://github.com/DSaurus/threestudio-shap-e) first. +``` +# load from shap-e initialization +python launch.py --config custom/threestudio-3dgs/configs/gaussian_splatting.yaml --train --gpu 0 system.prompt_processor.prompt="a delicious hamburger" system.geometry.geometry_convert_from="shap-e:a delicious hamburger" +``` + +If you want to use LRM initialization, please install [threestudio-lrm extension](https://github.com/Adamdad/threestudio-lrm) first. +``` +# load from lrm initialization +python launch.py --config custom/threestudio-3dgs/configs/gaussian_splatting.yaml --train --gpu 0 system.prompt_processor.prompt="a delicious hamburger" system.geometry.geometry_convert_from="lrm:a delicious hamburger" +``` + +## Export +You can use the following script to export Gaussian Splatting ply file and mesh obj. +``` +python launch.py --config path/to/config --export --gpu 0 system.prompt_processor.prompt="a delicious hamburger" resume=path/to/last.ckpt +``` + +## Citation +``` +@Article{kerbl3Dgaussians, + author = {Kerbl, Bernhard and Kopanas, Georgios and Leimk{\"u}hler, Thomas and Drettakis, George}, + title = {3D Gaussian Splatting for Real-Time Radiance Field Rendering}, + journal = {ACM Transactions on Graphics}, + number = {4}, + volume = {42}, + month = {July}, + year = {2023}, + url = {https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/} +} +``` + +## Acknowledgement +Please also consider citing these work about 3D Gaussian Splatting generation. Their open-source code inspires this project.. + +``` +@article{tang2023dreamgaussian, + title={DreamGaussian: Generative Gaussian Splatting for Efficient 3D Content Creation}, + author={Tang, Jiaxiang and Ren, Jiawei and Zhou, Hang and Liu, Ziwei and Zeng, Gang}, + journal={arXiv preprint arXiv:2309.16653}, + year={2023} +} +``` + +``` +@article{GaussianDreamer, + title={GaussianDreamer: Fast Generation from Text to 3D Gaussian Splatting with Point Cloud Priors}, + author={Taoran Yi and Jiemin Fang and Guanjun Wu and Lingxi Xie and Xiaopeng Zhang and Wenyu Liu and Qi Tian and Xinggang Wang}, + journal={arxiv:2310.08529}, + year={2023} +} +``` diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d5bd87f042fc7d25329055c4b72117883ea9c857 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/__init__.py @@ -0,0 +1,25 @@ +import threestudio +from packaging.version import Version + +if hasattr(threestudio, "__version__") and Version(threestudio.__version__) >= Version( + "0.2.1" +): + pass +else: + if hasattr(threestudio, "__version__"): + print(f"[INFO] threestudio version: {threestudio.__version__}") + raise ValueError( + "threestudio version must be >= 0.2.0, please update threestudio by pulling the latest version from github" + ) + + +from .background import gaussian_mvdream_background +from .geometry import exporter, gaussian_base, gaussian_io +from .material import gaussian_material +from .renderer import ( + diff_gaussian_rasterizer, + diff_gaussian_rasterizer_advanced, + diff_gaussian_rasterizer_background, + diff_gaussian_rasterizer_shading, +) +from .system import gaussian_mvdream, gaussian_splatting, gaussian_zero123, scene_lang diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/background/gaussian_mvdream_background.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/background/gaussian_mvdream_background.py new file mode 100644 index 0000000000000000000000000000000000000000..32a9bb7a9676fda517fc3b307a5328864b62df26 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/background/gaussian_mvdream_background.py @@ -0,0 +1,72 @@ +import random +from dataclasses import dataclass, field + +import threestudio +import torch +import torch.nn as nn +import torch.nn.functional as F +from threestudio.models.background.base import BaseBackground +from threestudio.models.networks import get_encoding, get_mlp +from threestudio.utils.ops import get_activation +from threestudio.utils.typing import * + + +@threestudio.register("gaussian-mvdream-neural-environment-map-background") +class NeuralEnvironmentMapBackground(BaseBackground): + @dataclass + class Config(BaseBackground.Config): + n_output_dims: int = 3 + color_activation: str = "sigmoid" + dir_encoding_config: dict = field( + default_factory=lambda: {"otype": "SphericalHarmonics", "degree": 3} + ) + mlp_network_config: dict = field( + default_factory=lambda: { + "otype": "VanillaMLP", + "activation": "ReLU", + "n_neurons": 16, + "n_hidden_layers": 2, + } + ) + random_aug: bool = False + random_aug_prob: float = 0.5 + eval_color: Optional[Tuple[float, float, float]] = None + + # multi-view diffusion + share_aug_bg: bool = False + + cfg: Config + + def configure(self) -> None: + self.encoding = get_encoding(3, self.cfg.dir_encoding_config) + self.network = get_mlp( + self.encoding.n_output_dims, + self.cfg.n_output_dims, + self.cfg.mlp_network_config, + ) + + def forward(self, dirs: Float[Tensor, "B H W 3"]) -> Float[Tensor, "B H W Nc"]: + if not self.training and self.cfg.eval_color is not None: + return torch.ones(*dirs.shape[:-1], self.cfg.n_output_dims).to( + dirs + ) * torch.as_tensor(self.cfg.eval_color).to(dirs) + # viewdirs must be normalized before passing to this function + dirs = (dirs + 1.0) / 2.0 # (-1, 1) => (0, 1) + dirs_embd = self.encoding(dirs.view(-1, 3)) + color = self.network(dirs_embd).view(*dirs.shape[:-1], self.cfg.n_output_dims) + color = get_activation(self.cfg.color_activation)(color) + if ( + self.training + and self.cfg.random_aug + and random.random() < self.cfg.random_aug_prob + ): + # use random background color with probability random_aug_prob + n_color = 1 if self.cfg.share_aug_bg else dirs.shape[0] + value = random.random() < 0.5 + color = color * 0 + ( # prevent checking for unused parameters in DDP + torch.ones(n_color, 1, 1, self.cfg.n_output_dims) + .to(dirs) + .expand(*dirs.shape[:-1], -1) + * value + ) + return color diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting.yaml b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8105a9204ca9aee749bbfb40588046e0a6a79a9b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting.yaml @@ -0,0 +1,96 @@ +name: "gs-sds-generation" +tag: "${rmspace:${system.prompt_processor.prompt},_}" +exp_root_dir: "outputs" +seed: 0 + +data_type: "random-camera-datamodule" +data: + batch_size: 4 + width: 512 + height: 512 + camera_distance_range: [2.5, 2.5] + fovy_range: [60, 70] + elevation_range: [-20, 90] + light_sample_strategy: "dreamfusion" + eval_camera_distance: 2.5 + eval_fovy_deg: 70 + rays_d_normalize: false + +system_type: "gaussian-splatting-system" +system: + + geometry_type: "gaussian-splatting" + geometry: + position_lr: [0, 0.001, 0.00002, 1000] + scale_lr: 0.005 + feature_lr: 0.01 + opacity_lr: 0.05 + rotation_lr: 0.005 + densification_interval: 300 + prune_interval: 300 + opacity_reset_interval: 50000000 + densify_from_iter: 500 + densify_until_iter: ${trainer.max_steps} + prune_from_iter: 500 + prune_until_iter: ${trainer.max_steps} + densify_grad_threshold: 0.01 + min_opac_prune: 0.005 + split_thresh: 0.02 + radii2d_thresh: 1000 + + init_num_pts: 4096 + pc_init_radius: 0.8 + opacity_init: 0.2 + + renderer_type: "diff-gaussian-rasterizer" + renderer: + debug: false + invert_bg_prob: 0.5 + + material_type: "no-material" # unused + material: + n_output_dims: 0 + + background_type: "solid-color-background" # unused + + prompt_processor_type: "stable-diffusion-prompt-processor" + prompt_processor: + pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + prompt: ??? + + guidance_type: "stable-diffusion-guidance" + guidance: + pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + guidance_scale: 100.0 + weighting_strategy: sds + min_step_percent: 0.02 + max_step_percent: [1000, 0.98, 0.5, 1001] + + exporter_type: "gaussian-mesh-exporter" + + loggers: + wandb: + enable: false + project: 'threestudio' + name: None + + loss: + lambda_sds: 0.1 + lambda_position: 1.0 + lambda_opacity: 0.0001 + lambda_scales: 0.0001 + lambda_tv_loss: 1.0 + lambda_depth_tv_loss: 1.0 + +trainer: + max_steps: 5000 + log_every_n_steps: 1 + num_sanity_val_steps: 0 + val_check_interval: 100 + enable_progress_bar: true + precision: 32-true + +checkpoint: + save_last: true # save at each validation time + save_top_k: -1 + every_n_train_steps: ${trainer.max_steps} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_background.yaml b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_background.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c2115391adba2fc5341739eed6676651d4932de9 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_background.yaml @@ -0,0 +1,111 @@ +name: "gs-sds-generation-background" +tag: "${rmspace:${system.prompt_processor.prompt},_}" +exp_root_dir: "outputs" +seed: 0 + +data_type: "random-camera-datamodule" +data: + batch_size: 4 + width: 512 + height: 512 + camera_distance_range: [2.5, 2.5] + fovy_range: [60, 70] + elevation_range: [-20, 90] + light_sample_strategy: "dreamfusion" + eval_camera_distance: 2.5 + eval_fovy_deg: 70 + rays_d_normalize: false + +system_type: "gaussian-splatting-system" +system: + + geometry_type: "gaussian-splatting" + geometry: + position_lr: [0, 0.001, 0.00002, 1000] + scale_lr: 0.005 + feature_lr: 0.01 + opacity_lr: 0.05 + rotation_lr: 0.005 + densification_interval: 300 + prune_interval: 300 + opacity_reset_interval: 50000000 + densify_from_iter: 500 + densify_until_iter: 10000 + prune_from_iter: 500 + prune_until_iter: ${trainer.max_steps} + densify_grad_threshold: 0.01 + min_opac_prune: 0.005 + split_thresh: 0.02 + radii2d_thresh: 1000 + + init_num_pts: 4096 + pc_init_radius: 0.8 + opacity_init: 0.2 + + renderer_type: "diff-gaussian-rasterizer-background" + renderer: + debug: false + + material_type: "no-material" # unused + material: + n_output_dims: 0 + + background_type: "gaussian-mvdream-neural-environment-map-background" + background: + color_activation: sigmoid + random_aug: true + random_aug_prob: 0.8 + + prompt_processor_type: "stable-diffusion-prompt-processor" + prompt_processor: + pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + prompt: ??? + + guidance_type: "stable-diffusion-guidance" + guidance: + pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + guidance_scale: 100.0 + weighting_strategy: sds + min_step_percent: 0.02 + max_step_percent: [1000, 0.98, 0.5, 1001] + + exporter_type: "gaussian-mesh-exporter" + + loggers: + wandb: + enable: false + project: 'threestudio' + name: None + + loss: + lambda_sds: 0.1 + lambda_position: 1.0 + lambda_opacity: 0.0001 + lambda_scales: 0.0001 + lambda_tv_loss: 1.0 + lambda_depth_tv_loss: 1.0 + + optimizer: + name: Adam + args: + lr: 0.01 + betas: [0.9, 0.99] + eps: 1.e-15 + params: + background: + lr: 0.001 + + + +trainer: + max_steps: 5000 + log_every_n_steps: 1 + num_sanity_val_steps: 0 + val_check_interval: 100 + enable_progress_bar: true + precision: 32-true + +checkpoint: + save_last: true # save at each validation time + save_top_k: -1 + every_n_train_steps: ${trainer.max_steps} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_mvdream.yaml b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_mvdream.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d5c9c7e52a45a900e100ea59837583e3f8c01298 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_mvdream.yaml @@ -0,0 +1,131 @@ +name: "gs-sds-mvdream" +tag: "${rmspace:${system.prompt_processor.prompt},_}" +exp_root_dir: "outputs" +seed: 0 + +data_type: "mvdream-random-multiview-camera-datamodule" +data: + batch_size: [4,4] + n_view: 4 + # 0-4999: 64x64, >=5000: 256x256 + width: [256, 256] + height: [256, 256] + resolution_milestones: [1000] + camera_distance_range: [0.8, 1.0] # relative + fovy_range: [15, 60] + elevation_range: [0, 30] + camera_perturb: 0. + center_perturb: 0. + up_perturb: 0. + n_val_views: 4 + eval_camera_distance: 3.0 + eval_fovy_deg: 40. + rays_d_normalize: false + +system_type: "gaussian-splatting-mvdream-system" +system: + geometry_type: "gaussian-splatting" + geometry: + position_lr: [0, 0.0001, 0.00001, 1500] + scale_lr: [0, 0.01, 0.001, 1500] + feature_lr: [0, 0.005, 0.001, 6000] + opacity_lr: 0.05 + rotation_lr: 0.001 + pred_normal: false + normal_lr: 0.005 + densification_interval: 300 + prune_interval: 300 + opacity_reset_interval: 100000 + densify_from_iter: 1500 + densify_until_iter: ${trainer.max_steps} + prune_from_iter: 1500 + prune_until_iter: ${trainer.max_steps} + densify_grad_threshold: 0.01 + min_opac_prune: 0.01 + split_thresh: 0.02 + radii2d_thresh: 1000 + + sphere: False + color_clip: [0, 0.01, 0.02, 1500, 0.5, 4000, 1.0, 7000] + + init_num_pts: 4096 + pc_init_radius: 0.5 + opacity_init: 0.05 + max_num: 100000 + + renderer_type: "diff-gaussian-rasterizer-shading" + renderer: + debug: false + + material_type: "gaussian-diffuse-with-point-light-material" + material: + ambient_only_steps: 3000 + textureless_prob: 0.0 + ambient_light_color: [0.9, 0.9, 0.9] + diffuse_light_color: [0.1, 0.1, 0.1] + soft_shading: true + + background_type: "gaussian-mvdream-neural-environment-map-background" + background: + color_activation: sigmoid + random_aug: true + share_aug_bg: true + random_aug_prob: 0.95 + + prompt_processor_type: "stable-diffusion-prompt-processor" + prompt_processor: + pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + prompt: ??? + negative_prompt: "ugly, bad anatomy, blurry, pixelated obscure, unnatural colors, poor lighting, dull, and unclear, cropped, lowres, low quality, artifacts, duplicate, morbid, mutilated, poorly drawn face, deformed, dehydrated, bad proportions" + front_threshold: 30. + back_threshold: 30. + + guidance_type: "mvdream-multiview-diffusion-guidance" + guidance: + model_name: "sd-v2.1-base-4view" + ckpt_path: null # path to a pre-downloaded checkpoint file (null for loading from URL) + guidance_scale: 50.0 + min_step_percent: [0, 0.98, 0.02, 7000] # (start_iter, start_val, end_val, end_iter) + max_step_percent: [0, 0.98, 0.50, 7000] + recon_loss: true + recon_std_rescale: 0.5 + + exporter_type: "gaussian-mesh-exporter" + + loggers: + wandb: + enable: false + project: 'threestudio' + name: None + + loss: + lambda_sds: 0.1 + lambda_position: 1.0 + lambda_opacity: 0.0001 + lambda_scales: 0.0001 + lambda_sparsity: 1.0 + lambda_tv_loss: 0.0 + lambda_depth_tv_loss: 1.0 + + optimizer: + name: Adam + args: + lr: 0.01 + betas: [0.9, 0.99] + eps: 1.e-6 + params: + background: + lr: 0.0001 + +trainer: + max_steps: 10000 + log_every_n_steps: 1 + num_sanity_val_steps: 0 + val_check_interval: 100 + enable_progress_bar: true + precision: 32-true + +checkpoint: + save_last: true # save at each validation time + save_top_k: -1 + every_n_train_steps: ${trainer.max_steps} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_shading.yaml b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_shading.yaml new file mode 100644 index 0000000000000000000000000000000000000000..734edde00b35dfbec99acab5c6eb151466abdec3 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_shading.yaml @@ -0,0 +1,115 @@ +name: "gs-sds-generation-shading" +tag: "${rmspace:${system.prompt_processor.prompt},_}" +exp_root_dir: "outputs" +seed: 0 + +data_type: "random-camera-datamodule" +data: + batch_size: 4 + width: 512 + height: 512 + camera_distance_range: [2.5, 2.5] + fovy_range: [60, 70] + elevation_range: [-20, 90] + light_sample_strategy: "dreamfusion" + eval_camera_distance: 2.5 + eval_fovy_deg: 70 + rays_d_normalize: false + +system_type: "gaussian-splatting-system" +system: + + geometry_type: "gaussian-splatting" + geometry: + position_lr: [0, 0.001, 0.00002, 1000] + scale_lr: 0.005 + feature_lr: 0.01 + opacity_lr: 0.05 + rotation_lr: 0.005 + densification_interval: 300 + prune_interval: 300 + opacity_reset_interval: 50000000 + densify_from_iter: 500 + densify_until_iter: ${trainer.max_steps} + prune_from_iter: 500 + prune_until_iter: ${trainer.max_steps} + densify_grad_threshold: 0.01 + min_opac_prune: 0.005 + split_thresh: 0.02 + radii2d_thresh: 1000 + + init_num_pts: 4096 + pc_init_radius: 0.8 + opacity_init: 0.2 + + renderer_type: "diff-gaussian-rasterizer-shading" + renderer: + debug: false + + material_type: "gaussian-diffuse-with-point-light-material" + material: + ambient_only_steps: 2000 + textureless_prob: 0.0 + ambient_light_color: [1.0, 1.0, 1.0] + diffuse_light_color: [0.0, 0.0, 0.0] + soft_shading: true + + background_type: "gaussian-mvdream-neural-environment-map-background" + background: + color_activation: sigmoid + random_aug: true + random_aug_prob: 0.8 + + prompt_processor_type: "stable-diffusion-prompt-processor" + prompt_processor: + pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + prompt: ??? + + guidance_type: "stable-diffusion-guidance" + guidance: + pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + guidance_scale: 100.0 + weighting_strategy: sds + min_step_percent: 0.02 + max_step_percent: [2000, 0.98, 0.5, 2001] + + exporter_type: "gaussian-mesh-exporter" + + loggers: + wandb: + enable: false + project: 'threestudio' + name: None + + loss: + lambda_sds: 0.1 + lambda_position: 1.0 + lambda_opacity: 0.0001 + lambda_scales: 0.0001 + lambda_tv_loss: 1.0 + lambda_depth_tv_loss: 1.0 + + optimizer: + name: Adam + args: + lr: 0.01 + betas: [0.9, 0.99] + eps: 1.e-15 + params: + background: + lr: 0.001 + + + +trainer: + max_steps: 5000 + log_every_n_steps: 1 + num_sanity_val_steps: 0 + val_check_interval: 100 + enable_progress_bar: true + precision: 32-true + +checkpoint: + save_last: true # save at each validation time + save_top_k: -1 + every_n_train_steps: ${trainer.max_steps} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_zero123.yaml b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_zero123.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd8fcf11362917fd99d885bb09e889168b5bcf0d --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/gaussian_splatting_zero123.yaml @@ -0,0 +1,144 @@ +name: "gs-zero123-sai" +tag: "${data.random_camera.height}_${rmspace:${basename:${data.image_path}},_}" +exp_root_dir: "outputs" +seed: 0 + +data_type: "single-image-datamodule" +data: # threestudio/data/image.py -> SingleImageDataModuleConfig + image_path: ./load/images/hamburger_rgba.png + height: [128, 256, 512] + width: [128, 256, 512] + resolution_milestones: [200, 300] + default_elevation_deg: 5.0 + default_azimuth_deg: 0.0 + default_camera_distance: 3.8 + default_fovy_deg: 20.0 + requires_depth: ${cmaxgt0orcmaxgt0:${system.loss.lambda_depth},${system.loss.lambda_depth_rel}} + requires_normal: ${cmaxgt0:${system.loss.lambda_normal}} + random_camera: # threestudio/data/uncond.py -> RandomCameraDataModuleConfig + height: 256 + width: 256 + batch_size: 4 + eval_height: 512 + eval_width: 512 + eval_batch_size: 1 + elevation_range: [-10, 80] + azimuth_range: [-180, 180] + camera_distance_range: [3.8, 3.8] + fovy_range: [20.0, 20.0] # Zero123 has fixed fovy + progressive_until: 0 + camera_perturb: 0.0 + center_perturb: 0.0 + up_perturb: 0.0 + light_position_perturb: 1.0 + light_distance_range: [7.5, 10.0] + eval_elevation_deg: ${data.default_elevation_deg} + eval_camera_distance: ${data.default_camera_distance} + eval_fovy_deg: ${data.default_fovy_deg} + light_sample_strategy: "dreamfusion" + batch_uniform_azimuth: False + n_val_views: 30 + n_test_views: 120 + +system_type: "gaussian-splatting-zero123-system" +system: + geometry_type: "gaussian-splatting" + geometry: + position_lr: [0, 0.001, 0.00002, 1000] + scale_lr: [0, 0.01, 0.001, 1000] + feature_lr: 0.01 + opacity_lr: 0.05 + rotation_lr: 0.001 + densification_interval: 100 + prune_interval: 100 + opacity_reset_interval: 100000 + densify_from_iter: 0 + densify_until_iter: ${trainer.max_steps} + prune_from_iter: 0 + prune_until_iter: ${trainer.max_steps} + densify_grad_threshold: 0.01 + min_opac_prune: 0.005 + split_thresh: 0.02 + radii2d_thresh: 1000 + + sphere: False + + init_num_pts: 4096 + pc_init_radius: 0.5 + opacity_init: 0.05 + max_num: 500000 + + exporter_type: "gaussian-mesh-exporter" + + renderer_type: "diff-gaussian-rasterizer-advanced" + renderer: + debug: false + invert_bg_prob: 1.0 + + material_type: "no-material" # unused + material: + n_output_dims: 0 + + background_type: "solid-color-background" # unused + + prompt_processor_type: "dummy-prompt-processor" # Zero123 doesn't use prompts + prompt_processor: + pretrained_model_name_or_path: "" + prompt: "" + + guidance_type: "stable-zero123-guidance" + guidance: + pretrained_config: "./load/zero123/sd-objaverse-finetune-c_concat-256.yaml" + pretrained_model_name_or_path: "./load/zero123/stable_zero123.ckpt" + vram_O: ${not:${gt0:${system.freq.guidance_eval}}} + cond_image_path: ${data.image_path} + cond_elevation_deg: ${data.default_elevation_deg} + cond_azimuth_deg: ${data.default_azimuth_deg} + cond_camera_distance: ${data.default_camera_distance} + guidance_scale: 3.0 + min_step_percent: [50, 0.7, 0.3, 200] # (start_iter, start_val, end_val, end_iter) + max_step_percent: [50, 0.98, 0.8, 200] + + freq: + ref_only_steps: 0 + guidance_eval: 0 + + loggers: + wandb: + enable: false + project: "threestudio" + name: None + + loss: + lambda_sds: 0.1 + lambda_rgb: [100, 500., 1000., 400] + lambda_mask: 50. + lambda_depth: 0. # 0.05 + lambda_depth_rel: 0. # [0, 0, 0.05, 100] + lambda_normal: 0. # [0, 0, 0.05, 100] + lambda_normal_smooth: 0. + lambda_3d_normal_smooth: 0. + + optimizer: + name: Adam + args: + lr: 0.01 + betas: [0.9, 0.99] + eps: 1.e-8 + params: + background: + lr: 0.001 + + +trainer: + max_steps: 5000 + log_every_n_steps: 1 + num_sanity_val_steps: 0 + val_check_interval: 100 + enable_progress_bar: true + precision: 32 + +checkpoint: + save_last: true # save at each validation time + save_top_k: -1 + every_n_train_steps: 100 # ${trainer.max_steps} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/scene_lang.yaml b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/scene_lang.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a34efe32c6274a769a2fd5f05760671206ffc98 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/configs/scene_lang.yaml @@ -0,0 +1,138 @@ +name: "gs-sds-generation" +tag: "${rmspace:${system.prompt_processor.prompt},_}" +exp_root_dir: "outputs" +seed: 0 +# resume: "/mnt/hdd1/wufan/projects/3DitScene/outputs/gs-sds-generation/3DitScene@20250204-120500/ckpts/last.ckpt" + +data_type: "random-camera-datamodule" +data: + rotate_traj: false # WU + random_traj: false # WU + batch_size: 1 + width: 512 + height: 512 + camera_distance_range: [2.5, 2.5] + fovy_range: [60, 60] + elevation_range: [0, 0] # The vertical angle of the camera relative to the object, in degrees. + light_sample_strategy: "dreamfusion" + eval_camera_distance: 2.5 + eval_fovy_deg: 60 # The field of view (FOV) in the vertical direction, in degrees. + eval_elevation_deg: 0 + rays_d_normalize: false + center_perturb: 0 + up_perturb: 0 + camera_perturb: 0 + azimuth_range: [-15, 15] # The range of horizontal rotation angles during training + val_azimuth_range: [-15, 15] # The range of horizontal rotation angles during validation + insert_zero: true + +system_type: "scene-lang-system" +system: + encoder_hidden_dims: [256, 128, 32, 3] + decoder_hidden_dims: [32, 128, 256, 512] + xyz_noise_ratio: [1000, 0.0, 0.0, 3000] + drop_ooi_ratio: 0.3 + crop_with_lang: true + densify: false + + geometry_type: "gaussian-splatting" + geometry: + ooi_bbox: [360,370,730,590] + geometry_convert_from: depth:assets/anime.png + position_lr: [0, 0.001, 0.00002, 1000] + scaling_lr: 0.05 + feature_lr: 0.01 + opacity_lr: 0.05 + rotation_lr: 0.005 + lang_lr: 0.0003 + densification_interval: 300 + prune_interval: 300 + opacity_reset_interval: 50000000 + densify_from_iter: 500 + densify_until_iter: ${trainer.max_steps} + prune_from_iter: 500 + prune_until_iter: ${trainer.max_steps} + densify_grad_threshold: 0.01 + min_opac_prune: 0.005 + split_thresh: 0.02 + radii2d_thresh: 1000 + + init_num_pts: 4096 + pc_init_radius: 0.8 + opacity_init: 0.2 + + empty_prompt: ${system.empty_prompt} + prompt: ${system.prompt_processor.prompt} + max_scaling: 0.2 + + renderer_type: "diff-gaussian-rasterizer" + renderer: + debug: false + invert_bg_prob: 0.5 + + material_type: "no-material" # unused + material: + n_output_dims: 0 + + background_type: "solid-color-background" # unused + + prompt_processor_type: "stable-diffusion-prompt-processor" + prompt_processor: + pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + # pretrained_model_name_or_path: "/mnt/petrelfs/zhangqihang/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1-base" + prompt: ??? + empty_prompt: "empty" + + guidance_type: "stable-diffusion-guidance" + guidance: + pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + # pretrained_model_name_or_path: "/mnt/petrelfs/zhangqihang/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1-base" + guidance_scale: 5.0 + weighting_strategy: sds + min_step_percent: 0.02 + max_step_percent: [0, 0.5, 0.1, 1000] + csd: false + + # guidance_type: "stable-diffusion-vsd-guidance" + # guidance: + # pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + # pretrained_model_name_or_path_lora: "stabilityai/stable-diffusion-2-1" + # guidance_scale: 7.5 + # min_step_percent: 0.02 + + exporter_type: "gaussian-mesh-exporter" + + sam_clip: + use_mobile_sam: True + + loggers: + wandb: + enable: false + project: '3ditscene' + name: "${tag}" + + loss: + lambda_sds: 0.01 + lambda_ref: 1000 + lambda_depth: 0.0 + lambda_position: 1.0 + lambda_opacity: 0.0001 + lambda_scales: 0.0001 + lambda_tv_loss: 1.0 + lambda_depth_tv_loss: 1.0 + lambda_scaling: 0.0 + +trainer: + max_steps: 1500 + log_every_n_steps: 1 + num_sanity_val_steps: 110 + val_check_interval: 500 + enable_progress_bar: true + precision: 32-true + +checkpoint: + save_last: true # save at each validation time + save_top_k: -1 + every_n_train_steps: 1000 + save_weights_only: true + # every_n_train_steps: ${trainer.max_steps} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/exporter.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/exporter.py new file mode 100644 index 0000000000000000000000000000000000000000..2f83e69b0c81a4c0b3a5598c8ff6dbeca69e05b4 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/exporter.py @@ -0,0 +1,44 @@ +from dataclasses import dataclass, field + +import cv2 +import numpy as np +import threestudio +import torch +from threestudio.models.background.base import BaseBackground +from threestudio.models.exporters.base import Exporter, ExporterOutput +from threestudio.models.geometry.base import BaseGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.mesh import Mesh +from threestudio.utils.rasterize import NVDiffRasterizerContext +from threestudio.utils.typing import * + + +@threestudio.register("gaussian-mesh-exporter") +class MeshExporter(Exporter): + @dataclass + class Config(Exporter.Config): + fmt: str = "obj" + save_name: str = "model" + save_video: bool = True + + cfg: Config + + def configure( + self, + geometry: BaseGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + super().configure(geometry, material, background) + + def __call__(self) -> List[ExporterOutput]: + mesh: Mesh = self.geometry.extract_mesh() + return self.export_obj(mesh) + + def export_obj(self, mesh: Mesh) -> List[ExporterOutput]: + params = {"mesh": mesh} + return [ + ExporterOutput( + save_name=f"{self.cfg.save_name}.obj", save_type="obj", params=params + ) + ] diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_base.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_base.py new file mode 100644 index 0000000000000000000000000000000000000000..e1c8cde208c05e3112bcdb9e546d9f6d399593b6 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_base.py @@ -0,0 +1,1469 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# +import math +import os +import random +import sys +import argparse +from dataclasses import dataclass, field +from datetime import datetime +from typing import NamedTuple + +import numpy as np +import cv2 +from PIL import Image +import threestudio +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from transformers import pipeline +from plyfile import PlyData, PlyElement +from simple_knn._C import distCUDA2 +import diffusers +from diffusers import StableDiffusionInpaintPipeline, AutoPipelineForInpainting +from threestudio.models.geometry.base import BaseGeometry +from threestudio.utils.misc import C +from threestudio.utils.typing import * +from segment_anything import sam_model_registry, SamPredictor +import matplotlib.pyplot as plt + +from .gaussian_io import GaussianIO +import imageio + +from scipy.spatial.transform import Rotation as R + +REORDER_MTX = torch.tensor([ + [0,0,0,1], + [1,0,0,0], + [0,1,0,0], + [0,0,1,0] +]).cuda().float() + +def build_rotation(r): + norm = torch.sqrt( + r[:, 0] * r[:, 0] + r[:, 1] * r[:, 1] + r[:, 2] * r[:, 2] + r[:, 3] * r[:, 3] + ) + + q = r / norm[:, None] + + R = torch.zeros((q.size(0), 3, 3), device="cuda") + r = q[:, 0] + x = q[:, 1] + y = q[:, 2] + z = q[:, 3] + + R[:, 0, 0] = 1 - 2 * (y * y + z * z) + R[:, 0, 1] = 2 * (x * y - r * z) + R[:, 0, 2] = 2 * (x * z + r * y) + R[:, 1, 0] = 2 * (x * y + r * z) + R[:, 1, 1] = 1 - 2 * (x * x + z * z) + R[:, 1, 2] = 2 * (y * z - r * x) + R[:, 2, 0] = 2 * (x * z - r * y) + R[:, 2, 1] = 2 * (y * z + r * x) + R[:, 2, 2] = 1 - 2 * (x * x + y * y) + return R + +def rotation_matrix(angle_x, angle_y, angle_z): + # Convert angles to radians + rad_x = torch.deg2rad(torch.tensor(angle_x)) + rad_y = torch.deg2rad(torch.tensor(angle_y)) + rad_z = torch.deg2rad(torch.tensor(angle_z)) + + # Compute sine and cosine of the angles + cos_x = torch.cos(rad_x) + sin_x = torch.sin(rad_x) + cos_y = torch.cos(rad_y) + sin_y = torch.sin(rad_y) + cos_z = torch.cos(rad_z) + sin_z = torch.sin(rad_z) + + # Construct the rotation matrix + Rx = torch.tensor([[1, 0, 0], + [0, cos_x, -sin_x], + [0, sin_x, cos_x]]) + + Ry = torch.tensor([[cos_y, 0, sin_y], + [0, 1, 0], + [-sin_y, 0, cos_y]]) + + Rz = torch.tensor([[cos_z, -sin_z, 0], + [sin_z, cos_z, 0], + [0, 0, 1]]) + + # Combine the rotation matrices + rotation_matrix = Rz @ Ry @ Rx + + return rotation_matrix + +# from scipy.spatial import KDTree +# +# def distCUDA2(points): +# points_np = points.detach().cpu().float().numpy() +# dists, inds = KDTree(points_np).query(points_np, k=4) +# meanDists = (dists[:, 1:] ** 2).mean(1) +# +# return torch.tensor(meanDists, dtype=points.dtype, device=points.device) + +sys.path.append('./utils/GeoWizard/geowizard') +from models.geowizard_pipeline import DepthNormalEstimationPipeline + +C0 = 0.28209479177387814 + +def propagate(canvas): + H, W = canvas.shape + dx = [0, 1, 0, -1] + dy = [1, 0, -1, 0] + count = np.zeros_like(canvas) + + while 1: + curr_mask = canvas > 0 + if sum(sum(curr_mask)) == H * W: + break + expand_mask = (cv2.blur(curr_mask.astype(np.float32), (3, 3)) > 0) + x, y = np.where(np.logical_and(expand_mask, ~curr_mask)) + old_canvas = canvas.copy() + + for xx, yy in zip(x, y): + for i in range(4): + ref_x = xx + dx[i] + ref_y = yy + dy[i] + if 0<=ref_x 0] = -1e10 # abnormal values... make weights 0 + + return torch.exp(power) + + +def build_rotation(r): + norm = torch.sqrt( + r[:, 0] * r[:, 0] + r[:, 1] * r[:, 1] + r[:, 2] * r[:, 2] + r[:, 3] * r[:, 3] + ) + + q = r / norm[:, None] + + R = torch.zeros((q.size(0), 3, 3), device="cuda") + + r = q[:, 0] + x = q[:, 1] + y = q[:, 2] + z = q[:, 3] + + R[:, 0, 0] = 1 - 2 * (y * y + z * z) + R[:, 0, 1] = 2 * (x * y - r * z) + R[:, 0, 2] = 2 * (x * z + r * y) + R[:, 1, 0] = 2 * (x * y + r * z) + R[:, 1, 1] = 1 - 2 * (x * x + z * z) + R[:, 1, 2] = 2 * (y * z - r * x) + R[:, 2, 0] = 2 * (x * z - r * y) + R[:, 2, 1] = 2 * (y * z + r * x) + R[:, 2, 2] = 1 - 2 * (x * x + y * y) + return R + + +def build_scaling_rotation(s, r): + L = torch.zeros((s.shape[0], 3, 3), dtype=torch.float, device="cuda") + R = build_rotation(r) + + L[:, 0, 0] = s[:, 0] + L[:, 1, 1] = s[:, 1] + L[:, 2, 2] = s[:, 2] + + L = R @ L + return L + + +def safe_state(silent): + old_f = sys.stdout + + class F: + def __init__(self, silent): + self.silent = silent + + def write(self, x): + if not self.silent: + if x.endswith("\n"): + old_f.write( + x.replace( + "\n", + " [{}]\n".format( + str(datetime.now().strftime("%d/%m %H:%M:%S")) + ), + ) + ) + else: + old_f.write(x) + + def flush(self): + old_f.flush() + + sys.stdout = F(silent) + + random.seed(0) + np.random.seed(0) + torch.manual_seed(0) + torch.cuda.set_device(torch.device("cuda:0")) + + +class BasicPointCloud(NamedTuple): + points: np.array + colors: np.array + normals: np.array + + +class Camera(NamedTuple): + FoVx: torch.Tensor + FoVy: torch.Tensor + camera_center: torch.Tensor + image_width: int + image_height: int + world_view_transform: torch.Tensor + full_proj_transform: torch.Tensor + +def fill_mask(mask): + mask = np.array(mask) + canvas = np.zeros_like(mask) + H, W = mask.shape + for i in range(H): + for p in range(0, W): + if mask[i, p]: + canvas[i, p] = 1 + else: + break + for p in range(W-1, 0, -1): + if mask[i, p]: + canvas[i, p] = 1 + else: + break + + for i in range(W): + for p in range(0, H): + if mask[p, i]: + canvas[p, i] = 1 + else: + break + for p in range(H-1, 0, -1): + if mask[p, i]: + canvas[p, i] = 1 + else: + break + mask = np.logical_and(mask, canvas) + return Image.fromarray(mask) + +def parse_wh(wh): + try: + W, H = wh + except: + W = H = wh + return W, H + +@threestudio.register("gaussian-splatting") +class GaussianBaseModel(BaseGeometry, GaussianIO): + @dataclass + class Config(BaseGeometry.Config): + max_num: int = 500000 + sh_degree: int = 0 + position_lr: Any = 0.001 + # scale_lr: Any = 0.003 + feature_lr: Any = 0.01 + opacity_lr: Any = 0.05 + scaling_lr: Any = 0.005 + rotation_lr: Any = 0.005 + pred_normal: bool = False + normal_lr: Any = 0.001 + lang_lr: float = 0.005 + + densification_interval: int = 50 + prune_interval: int = 50 + opacity_reset_interval: int = 100000 + densify_from_iter: int = 100 + prune_from_iter: int = 100 + densify_until_iter: int = 2000 + prune_until_iter: int = 2000 + densify_grad_threshold: Any = 0.01 + min_opac_prune: Any = 0.005 + split_thresh: Any = 0.02 + radii2d_thresh: Any = 1000 + + sphere: bool = False + prune_big_points: bool = False + color_clip: Any = 2.0 + + geometry_convert_from: str = "" + load_ply_only_vertex: bool = False + init_num_pts: int = 100 + pc_init_radius: float = 0.8 + opacity_init: float = 0.1 + + img_resolution: Any = 512 + + shap_e_guidance_config: dict = field(default_factory=dict) + + max_scaling: float = 100 + sam_ckpt_path: str = "ckpts/sam_vit_h_4b8939.pth" + ooi_bbox: Any = None + + prompt: Any = None + empty_prompt: Any = None + lang_beta_1: float = 0.9 + lang_beta_2: float = 0.999 + + inference_only: bool = False + pc_max_resolution: int = 512 + + use_sdxl_for_inpaint: bool = False + + cfg: Config + + def setup_functions(self): + def build_covariance_from_scaling_rotation(scaling, scaling_modifier, rotation): + L = build_scaling_rotation(scaling_modifier * scaling, rotation) + actual_covariance = L @ L.transpose(1, 2) + symm = strip_symmetric(actual_covariance) + return symm + + self.scaling_activation = torch.exp + self.scaling_inverse_activation = torch.log + + self.covariance_activation = build_covariance_from_scaling_rotation + + self.opacity_activation = torch.sigmoid + self.inverse_opacity_activation = inverse_sigmoid + + self.rotation_activation = torch.nn.functional.normalize + self.color_clip = C(self.cfg.color_clip, 0, 0) + + self.fixed_xyz = None + self.fixed_rot = None + + if not self.cfg.inference_only: + sam = sam_model_registry["vit_h"](checkpoint=self.cfg.sam_ckpt_path).to('cuda') + self.predictor = SamPredictor(sam) + + def project_pc(self, c2w, H=256, W=None): + if W is None: + W = H + B = c2w.shape[0] + + mask = torch.zeros([B, H, W], device='cuda') + depth_canvas = torch.zeros([B, H, W], device='cuda') + + # for pc in [self.bg_point_cloud, self.point_cloud]: + pc_cam = torch.einsum('bxy,ny->bnx', torch.linalg.inv(c2w), self.point_cloud) + depth = -1 * pc_cam[..., 2].view(pc_cam.shape[0], -1) + pc_cam = (pc_cam / pc_cam[..., 2:3])[..., :3] + pc_2d = torch.einsum('xy,bny->bnx', self.proj_mtx, pc_cam).clamp(0, 1) + pc_2d[..., 0] = pc_2d[..., 0] * (W-1) + pc_2d[..., 1] = pc_2d[..., 1] * (H-1) + pc_2d = pc_2d.long() + for i in range(pc_2d.shape[0]): + x = (W - pc_2d[i, :, 0]).clamp(0, W-1) + y = (pc_2d[i, :, 1]).clamp(0, H-1) + unique_id = x * H + y + map_2d = np.zeros((W+1)*(H+1)) + 1e8 + np.minimum.at(map_2d, unique_id.cpu(), depth[i].cpu()) + map_2d[map_2d==1e8] = 0 + positive_unique_id = np.where(map_2d>0)[0] + x, y = positive_unique_id // H, positive_unique_id % H + mask[i, y, x] = 1.0 + depth_canvas[i, y, x] = torch.tensor(map_2d[positive_unique_id], device='cuda', dtype=torch.float) + # depth_canvas[i, y, x] = depth[i] + + # pc_cam = torch.einsum('bxy,hwy->bhwx', torch.linalg.inv(c2w), self.point_cloud) + # depth = -1 * pc_cam[..., 2].view(pc_cam.shape[0], -1) + # pc_cam = (pc_cam / pc_cam[..., 2:3])[..., :3] + # pc_2d = torch.einsum('xy,bhwy->bhwx', self.proj_mtx, pc_cam).clamp(0, 1) + # pc_2d[..., 0] = pc_2d[..., 0] * (W-1) + # pc_2d[..., 1] = pc_2d[..., 1] * (H-1) + # pc_2d = (pc_2d.long()).view(pc_2d.shape[0], -1, pc_2d.shape[-1]) + + + # mask = self.blur_kernel(mask) > 0 + mask = torchvision.transforms.functional.gaussian_blur(mask, 3) > 0 + # mask = mask > 0 + return mask, depth_canvas + + def img2pc_inpaint(self, img, c2w=None, gt_depth=None, mask=None, proj_func=None): + W, H = parse_wh(self.cfg.img_resolution) + if max(W, H) > self.cfg.pc_max_resolution: + W, H = int(W / max(W, H) * self.cfg.pc_max_resolution), int(H / max(W, H) * self.cfg.pc_max_resolution) + + with torch.no_grad(): + self.geowizard_pipe.to('cuda') + depth = self.geowizard_pipe( + img, + denoising_steps = 25, + ensemble_size = 3, + processing_res = 768, + match_input_res = False, + domain = 'outdoor', + color_map = 'Spectral', + gt_depth = gt_depth, mask = mask, + show_progress_bar = True)['depth_np'] + self.geowizard_pipe.to('cpu') + ret_depth = depth.copy() + depth = torch.from_numpy(depth)[None] + depth = torch.nn.functional.interpolate(depth[None], size=(H, W), mode='bilinear', align_corners=True).squeeze() + + depth = depth.cpu().numpy() + if proj_func is None: + depth = depth * 20 + 5 + else: + depth = proj_func(depth) + + depth = depth * -1 + x, y = np.meshgrid(np.arange(W, dtype=np.float32), np.arange(H, dtype=np.float32), indexing='xy') + x = x / float(W-1) + y = y / float(H-1) + xyz = np.stack((x, y, np.ones_like(x)), 0).transpose(1, 2, 0) + xyz[..., 0] = 1 - xyz[..., 0] + + fov = 60 / 180 * np.pi + proj_mtx = np.array([ + [1 / (2 * np.tan(fov/2)), 0, 1/2], + [0, 1 / (2 * np.tan(fov/2)), 1/2], + [0, 0, 1], + ]) + self.proj_mtx = torch.from_numpy(proj_mtx).cuda().float() + if c2w is None: + c2w = np.array([0.0000, 0.0000, 1.0000, 2.5000, 1.0000, 0.0000, -0.0000, 0.0000, -0.0000, 1.0000, -0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 1.0000]).reshape(4, 4) + else: + c2w = c2w[0].cpu().numpy() + xyz = np.einsum('ab,hwb->hwa', np.linalg.inv(proj_mtx), xyz) + xyz = xyz * depth[..., None] + xyz = np.concatenate([xyz, np.ones_like(x)[..., None]], 2) + xyz = np.einsum('ab,hwb->hwa', c2w, xyz) + return xyz, ret_depth + + def inpaint(self, img, mask, prompt): + # inpaint using base pipe + N = 512 + img = img.convert("RGB").resize((N, N)) + mask = mask.convert("RGB").resize((N, N)) + self.base_inpainting_pipe.to("cuda") + img = self.base_inpainting_pipe(prompt=prompt, image=img, mask_image=mask, guidance_scale=7.5).images[0] + self.base_inpainting_pipe.to("cpu") + torch.cuda.empty_cache() + + if self.cfg.use_sdxl_for_inpaint: + # inpaint using sdxl pipe + N = 1024 + img = img.convert("RGB").resize((N, N)) + mask = mask.convert("RGB").resize((N, N)) + self.sdxl_inpainting_pipe.to("cuda") + img = self.sdxl_inpainting_pipe(prompt=prompt, image=img, mask_image=mask, guidance_scale=7.5, num_inference_steps=20, strength=0.99).images[0] + self.sdxl_inpainting_pipe.to("cpu") + + return img + + def configure(self) -> None: + super().configure() + self.active_sh_degree = 0 + self.max_sh_degree = self.cfg.sh_degree + self._xyz = torch.empty(0) + self._features_dc = torch.empty(0) + self._features_rest = torch.empty(0) + self._scaling = torch.empty(0) + self._rotation = torch.empty(0) + self._opacity = torch.empty(0) + self._opacity_mask = None + self.max_radii2D = torch.empty(0) + self.xyz_gradient_accum = torch.empty(0) + self.denom = torch.empty(0) + self.noise_ratio = 0.0 + if self.cfg.pred_normal: + self._normal = torch.empty(0) + self.optimizer = None + self.setup_functions() + self.save_path = None + self.fixed_xyz = None + self.fixed_rot = None + + if self.cfg.inference_only: + return + # setup GeoWizard + geowizard_checkpoint_path = 'lemonaddie/geowizard' + self.geowizard_pipe = DepthNormalEstimationPipeline.from_pretrained( + geowizard_checkpoint_path, torch_dtype=torch.float32) + + self.base_inpainting_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting", torch_dtype=torch.float16) + # self.base_inpainting_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting", torch_dtype=torch.float16, safety_checker=None) + if self.cfg.use_sdxl_for_inpaint: + self.sdxl_inpainting_pipe = AutoPipelineForInpainting.from_pretrained("diffusers/stable-diffusion-xl-1.0-inpainting-0.1", torch_dtype=torch.float16, variant="fp16") + self.sdxl_inpainting_pipe.scheduler = diffusers.EulerDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler") + + if self.cfg.geometry_convert_from.startswith("depth:"): + # estimate depth + W, H = parse_wh(self.cfg.img_resolution) + if max(W, H) > self.cfg.pc_max_resolution: + W, H = int(W / max(W, H) * self.cfg.pc_max_resolution), int(H / max(W, H) * self.cfg.pc_max_resolution) + img = self.cfg.geometry_convert_from[len("depth:"):] + raw_img = img = Image.open(img).convert("RGB") + img = img.resize((W, H)) + + bg_xyz, bg_color = [], [] + + with torch.no_grad(): + self.predictor.set_image(np.array(raw_img)) + self.ooi_masks = [] + total_inp_ooi_masks = None + total_ooi_masks = [] + for i in range(len(self.cfg.ooi_bbox) // 4): + bbox = np.array(self.cfg.ooi_bbox[4*i:4*i+4]) + masks, _, _ = self.predictor.predict( + point_coords=None, + point_labels=None, + box=bbox[None, :], + multimask_output=False, + ) + # plt.imshow(masks[0]) + # plt.savefig(os.path.join(self.save_path, f'mask_{i}.png')) + ooi_masks = np.array(Image.fromarray(masks[0]).resize((W, H), Image.NEAREST)) + ooi_masks = (cv2.blur(ooi_masks.astype(np.float32), (5, 5)) > 0) + inp_ooi_masks = (cv2.blur(ooi_masks.astype(np.float32), (7, 7)) > 0) + if i == 0: + total_inp_ooi_masks = inp_ooi_masks + else: + total_inp_ooi_masks += inp_ooi_masks + total_ooi_masks.append(ooi_masks) + + total_inp_ooi_masks = total_inp_ooi_masks > 0 + original_wh = parse_wh(self.cfg.img_resolution) + bg_image = self.inpaint(img=img, mask=Image.fromarray(total_inp_ooi_masks), prompt=self.cfg.empty_prompt).resize((original_wh)) + self.bg_image = np.array(bg_image) + self.bg_image_mask = np.array(Image.fromarray(total_inp_ooi_masks).resize((original_wh))) + + xyz, depth = self.img2pc_inpaint(img) + self.point_cloud = torch.from_numpy(xyz).cuda().float().reshape(-1, 4) + + for ooi_masks in total_ooi_masks: + transit_masks = np.logical_and(cv2.blur(ooi_masks.astype(np.float32), (3, 3)) > 0, ~ooi_masks) + depth_tensor = torch.from_numpy(depth)[None, None].cuda() * 2 - 1 + self.ooi_masks.append(torch.tensor(ooi_masks.reshape(-1).astype(np.uint8), device='cuda').float().bool()) + ooi_masks = cv2.blur(ooi_masks.astype(np.float32), (9, 9)) > 0 + mask = torch.from_numpy(ooi_masks.astype(np.float32))[None, None].cuda() + bg_xyz_pc, _ = self.img2pc_inpaint(bg_image, gt_depth=depth_tensor, mask=1-mask) + + bg_xyz.append(bg_xyz_pc[ooi_masks]) + bg_color.append(np.array(bg_image.resize((W, H)))[ooi_masks] / 255) + + # xyz = xyz[..., :3].reshape(-1, 3) + xyz = xyz.reshape(-1, 4) + color = np.array(img).reshape(-1, 3) / 255 + bg_xyz = np.concatenate(bg_xyz, 0) + additional_pts_num = bg_xyz.shape[0] + xyz = np.concatenate([xyz, bg_xyz], 0) + self.point_cloud = torch.from_numpy(xyz).cuda().float() + + color = np.concatenate([color, np.concatenate(bg_color, 0)], 0) + for i in range(len(self.ooi_masks)): + self.register_buffer(f"ooi_masks_{i}", torch.cat([self.ooi_masks[i], torch.zeros([additional_pts_num], device='cuda').bool()]) ) + self.ooi_masks[i] = getattr(self, f"ooi_masks_{i}") + self.register_buffer(f"_delete_mask", torch.ones_like(self.ooi_masks[0].float())) + + # project to 3D space + xyz = xyz[:, :3] + color = color + pcd = BasicPointCloud( + points=xyz, colors=color, normals=np.zeros((xyz.shape[0], 3)) + ) + self.create_from_pcd(pcd, 10) + self.training_setup() + + elif self.cfg.geometry_convert_from.startswith("shap-e:"): + shap_e_guidance = threestudio.find("shap-e-guidance")( + self.cfg.shap_e_guidance_config + ) + prompt = self.cfg.geometry_convert_from[len("shap-e:") :] + xyz, color = shap_e_guidance(prompt) + + pcd = BasicPointCloud( + points=xyz, colors=color, normals=np.zeros((xyz.shape[0], 3)) + ) + self.create_from_pcd(pcd, 10) + self.training_setup() + + # Support Initialization from OpenLRM, Please see https://github.com/Adamdad/threestudio-lrm + elif self.cfg.geometry_convert_from.startswith("lrm:"): + lrm_guidance = threestudio.find("lrm-guidance")( + self.cfg.shap_e_guidance_config + ) + prompt = self.cfg.geometry_convert_from[len("lrm:") :] + xyz, color = lrm_guidance(prompt) + + pcd = BasicPointCloud( + points=xyz, colors=color, normals=np.zeros((xyz.shape[0], 3)) + ) + self.create_from_pcd(pcd, 10) + self.training_setup() + + elif os.path.exists(self.cfg.geometry_convert_from): + threestudio.info( + "Loading point cloud from %s" % self.cfg.geometry_convert_from + ) + if self.cfg.geometry_convert_from.endswith(".ckpt"): + ckpt_dict = torch.load(self.cfg.geometry_convert_from) + num_pts = ckpt_dict["state_dict"]["geometry._xyz"].shape[0] + pcd = BasicPointCloud( + points=np.zeros((num_pts, 3)), + colors=np.zeros((num_pts, 3)), + normals=np.zeros((num_pts, 3)), + ) + self.create_from_pcd(pcd, 10) + self.training_setup() + new_ckpt_dict = {} + for key in self.state_dict(): + if ckpt_dict["state_dict"].__contains__("geometry." + key): + new_ckpt_dict[key] = ckpt_dict["state_dict"]["geometry." + key] + else: + new_ckpt_dict[key] = self.state_dict()[key] + self.load_state_dict(new_ckpt_dict) + elif self.cfg.geometry_convert_from.endswith(".ply"): + if self.cfg.load_ply_only_vertex: + plydata = PlyData.read(self.cfg.geometry_convert_from) + vertices = plydata["vertex"] + positions = np.vstack( + [vertices["x"], vertices["y"], vertices["z"]] + ).T + if vertices.__contains__("red"): + colors = ( + np.vstack( + [vertices["red"], vertices["green"], vertices["blue"]] + ).T + / 255.0 + ) + else: + shs = np.random.random((positions.shape[0], 3)) / 255.0 + C0 = 0.28209479177387814 + colors = shs * C0 + 0.5 + normals = np.zeros_like(positions) + pcd = BasicPointCloud( + points=positions, colors=colors, normals=normals + ) + self.create_from_pcd(pcd, 10) + else: + self.load_ply(self.cfg.geometry_convert_from) + self.training_setup() + else: + threestudio.info("Geometry not found, initilization with random points") + num_pts = self.cfg.init_num_pts + phis = np.random.random((num_pts,)) * 2 * np.pi + costheta = np.random.random((num_pts,)) * 2 - 1 + thetas = np.arccos(costheta) + mu = np.random.random((num_pts,)) + radius = self.cfg.pc_init_radius * np.cbrt(mu) + x = radius * np.sin(thetas) * np.cos(phis) + y = radius * np.sin(thetas) * np.sin(phis) + z = radius * np.cos(thetas) + xyz = np.stack((x, y, z), axis=1) + + shs = np.random.random((num_pts, 3)) / 255.0 + C0 = 0.28209479177387814 + color = shs * C0 + 0.5 + pcd = BasicPointCloud( + points=xyz, colors=color, normals=np.zeros((num_pts, 3)) + ) + + self.create_from_pcd(pcd, 10) + self.training_setup() + + def add_pc_from_novel_view(self, rgb, mask, depth, c2w, save_path=None): + W, H = parse_wh(self.cfg.img_resolution) + if max(W, H) > self.cfg.pc_max_resolution: + W, H = int(W / max(W, H) * self.cfg.pc_max_resolution), int(H / max(W, H) * self.cfg.pc_max_resolution) + # depth estimation -> add points. + mask = fill_mask(mask) + blur_mask = Image.fromarray(cv2.blur(np.array(mask).astype(np.float32), (7, 7)) > 0) + res = self.inpaint(img=rgb, mask=blur_mask, prompt=self.side_prompt) + + self.geowizard_pipe.to('cuda') + depth_unaligned = self.geowizard_pipe( + res, + denoising_steps = 25, + ensemble_size = 3, + processing_res = 768, + match_input_res = False, + domain = 'outdoor', + color_map = 'Spectral', + gt_depth = None, mask = None, + show_progress_bar = True)['depth_np'] + self.geowizard_pipe.to('cpu') + prev_depth = depth_unaligned[~np.array(mask.resize((768,768)))] + # inpaint the depth map + depth_nd = depth[0].cpu().numpy().astype(np.uint8) + inpaint_mask = np.logical_and(~np.array(mask) , depth[0].cpu().numpy().astype(np.uint8)==0 ).astype(np.uint8) + l, r = depth[depth>0].min().item(), depth.max().item() + depth = (depth - l) / (r - l) * 255 + depth = cv2.inpaint(depth[0].cpu().numpy().astype(np.uint8), inpaint_mask, 3, cv2.INPAINT_TELEA) + depth = torch.tensor(depth)[None].cuda().float() / 255 + reproj_func = lambda x: (x - prev_depth.min().item()) / (prev_depth.max().item() - prev_depth.min().item()) * (r-l) + l + depth = depth * (prev_depth.max() - prev_depth.min()) + prev_depth.min() + depth_tensor = torch.nn.functional.interpolate(depth[None].cuda(), 768, mode='nearest') * 2 - 1 + + _masks = cv2.blur(np.array(mask.resize((768, 768))).astype(float), (20, 20)) > 0 + mask_tensor = torch.from_numpy(_masks.astype(np.float32))[None, None].cuda() + bg_xyz_pc, _ = self.img2pc_inpaint(res, gt_depth=depth_tensor, mask=1-mask_tensor, proj_func=reproj_func, c2w=c2w) + + mask = np.array(Image.fromarray(_masks).resize((W, H))) + new_xyz = bg_xyz_pc[mask][:, :3] + res = res.resize((W, H)) + new_color = np.array(res)[mask] / 255 + pcd = BasicPointCloud(points=new_xyz, colors=new_color, normals=np.zeros((new_xyz.shape[0], 3))) + self.merge_from_pcd(pcd, 10) + + original_wh = parse_wh(self.cfg.img_resolution) + return res.resize((original_wh)), Image.fromarray(_masks).resize((original_wh)) + + @property + def get_scaling(self): + if self.cfg.sphere: + return self.scaling_activation( + torch.mean(self._scaling, dim=-1).unsqueeze(-1).repeat(1, 3) + ).clip(0, self.cfg.max_scaling) + return self.scaling_activation(self._scaling).clip(0, self.cfg.max_scaling) + + @property + def get_rotation(self): + return self.rotation_activation(self._rotation) + + @property + def get_language_feature(self): + return self._language_feature + + @property + def get_xyz(self): + ret = self._xyz + if self.noise_ratio > 0.0: + offset = torch.zeros_like(ret) + for idx in range(len(self.ooi_masks)): + ooi_masks = getattr(self, f"ooi_masks_{idx}") + offset[ooi_masks] = torch.rand(3, device='cuda') * self.noise_ratio + return ret + + @property + def get_features(self): + features_dc = self._features_dc + features_dc = features_dc.clip(-self.color_clip, self.color_clip) + features_rest = self._features_rest + return torch.cat((features_dc, features_rest), dim=1) + + @property + def get_opacity(self): + if self._opacity_mask is None: + ret = self.opacity_activation(self._opacity) + else: + ret = self.opacity_activation(self._opacity) * self._opacity_mask.unsqueeze(-1) + + if self._delete_mask is None: + return ret + else: + return ret * self._delete_mask.unsqueeze(-1) + + @property + def get_normal(self): + if self.cfg.pred_normal: + return self._normal + else: + raise ValueError("Normal is not predicted") + + def recover_xyzrot(self): + self._xyz = torch.nn.Parameter(self.fixed_xyz) + self._rotation = torch.nn.Parameter(self.fixed_rot) + + def random_rotate(self, rotate_aug_scale, apply_rotate): + if self.fixed_xyz is None: + self.fixed_xyz = self.get_xyz.data + self.fixed_rot = self.get_rotation.data + + if apply_rotate: + ooi_mask = self.ooi_masks_0.view(-1).byte().to(device='cuda').float() + + rotate = random.randint(-rotate_aug_scale, rotate_aug_scale) + rot_matrix = rotation_matrix(0, 0, rotate).cuda() + prev_xyz = self.fixed_xyz.clone() + ooi_xyz = prev_xyz[ooi_mask.bool()] + mean = ooi_xyz.mean(0) + ooi_xyz = ooi_xyz - mean + after_xyz = torch.einsum('ab,nb->na', rot_matrix, ooi_xyz) + mean + prev_xyz[ooi_mask.bool()] = after_xyz + self._xyz = torch.nn.Parameter(prev_xyz) + + prev_rotation = self.fixed_rot.clone() + prev_rotation_mtx = build_rotation(prev_rotation) + after_rotation_mtx = torch.einsum('ab,nbc->nac', rot_matrix, prev_rotation_mtx) + after_rotation = torch.from_numpy(R.from_matrix(after_rotation_mtx.detach().cpu()).as_quat()).cuda().float() + after_rotation = torch.einsum('ab,nb->na', REORDER_MTX, after_rotation) + prev_rotation[ooi_mask.bool()] = after_rotation[ooi_mask.bool()] + self._rotation = torch.nn.Parameter(prev_rotation) + else: + self.recover_xyzrot() + + def get_covariance(self, scaling_modifier=1): + return self.covariance_activation( + self.get_scaling, scaling_modifier, self._rotation + ) + + def create_from_pcd(self, pcd: BasicPointCloud, spatial_lr_scale: float): + self.spatial_lr_scale = spatial_lr_scale + fused_point_cloud = torch.tensor(np.asarray(pcd.points)).float().cuda() + fused_color = RGB2SH(torch.tensor(np.asarray(pcd.colors)).float().cuda()) + features = ( + torch.zeros((fused_color.shape[0], 3, (self.max_sh_degree + 1) ** 2)) + .float() + .cuda() + ) + features[:, :3, 0] = fused_color + features[:, 3:, 1:] = 0.0 + + threestudio.info( + f"Number of points at initialisation:{fused_point_cloud.shape[0]}" + ) + + dist2 = torch.clamp_min( + distCUDA2(torch.from_numpy(np.asarray(pcd.points)).float().cuda()), + 0.0000001, + ) + scales = torch.log(torch.sqrt(dist2))[..., None].repeat(1, 3) + rots = torch.zeros((fused_point_cloud.shape[0], 4), device="cuda") + rots[:, 0] = 1 + + opacities = inverse_sigmoid( + self.cfg.opacity_init + * torch.ones( + (fused_point_cloud.shape[0], 1), dtype=torch.float, device="cuda" + ) + ) + + self._xyz = nn.Parameter(fused_point_cloud.requires_grad_(True)) + self._features_dc = nn.Parameter( + features[:, :, 0:1].transpose(1, 2).contiguous().requires_grad_(True) + ) + self._features_rest = nn.Parameter( + features[:, :, 1:].transpose(1, 2).contiguous().requires_grad_(True) + ) + self._scaling = nn.Parameter(scales.requires_grad_(True)) + self._rotation = nn.Parameter(rots.requires_grad_(True)) + self._opacity = nn.Parameter(opacities.requires_grad_(True)) + if self.cfg.pred_normal: + normals = torch.zeros((fused_point_cloud.shape[0], 3), device="cuda") + self._normal = nn.Parameter(normals.requires_grad_(True)) + self.max_radii2D = torch.zeros((self._xyz.shape[0]), device="cuda") + + self.fused_point_cloud = fused_point_cloud.cpu().clone().detach() + self.features = features.cpu().clone().detach() + self.scales = scales.cpu().clone().detach() + self.rots = rots.cpu().clone().detach() + self.opacities = opacities.cpu().clone().detach() + + language_feature = torch.zeros((self._xyz.shape[0], 3), device="cuda") + self._language_feature = torch.nn.Parameter(language_feature.requires_grad_(True)) + + def merge_from_pcd(self, pcd: BasicPointCloud, spatial_lr_scale: float): + self.spatial_lr_scale = spatial_lr_scale + fused_point_cloud = torch.tensor(np.asarray(pcd.points)).float().cuda() + fused_color = RGB2SH(torch.tensor(np.asarray(pcd.colors)).float().cuda()) + features = ( + torch.zeros((fused_color.shape[0], 3, (self.max_sh_degree + 1) ** 2)) + .float() + .cuda() + ) + features[:, :3, 0] = fused_color + features[:, 3:, 1:] = 0.0 + + threestudio.info( + f"Number of points at merging:{fused_point_cloud.shape[0]}" + ) + + dist2 = torch.clamp_min( + distCUDA2(torch.from_numpy(np.asarray(pcd.points)).float().cuda()), + 0.0000001, + ) + scales = torch.log(torch.sqrt(dist2))[..., None].repeat(1, 3) + rots = torch.zeros((fused_point_cloud.shape[0], 4), device="cuda") + rots[:, 0] = 1 + + opacities = inverse_sigmoid( + self.cfg.opacity_init + * torch.ones( + (fused_point_cloud.shape[0], 1), dtype=torch.float, device="cuda" + ) + ) + self.densification_postfix( + fused_point_cloud, + features[:, :, 0:1].transpose(1, 2).contiguous(), + features[:, :, 1:].transpose(1, 2).contiguous(), + opacities, + scales, + rots, + None, + torch.zeros((fused_point_cloud.shape[0], 3), device="cuda") + ) + + for idx in range(len(self.ooi_masks)): + # self.ooi_masks[idx] = torch.cat([self.ooi_masks[idx], torch.ones([fused_point_cloud.shape[0]], device='cuda') > 0]) + self.register_buffer(f"ooi_masks_{idx}", torch.cat([getattr(self, f"ooi_masks_{idx}"), torch.zeros([fused_point_cloud.shape[0]], device='cuda').bool()]) ) + self.ooi_masks[idx] = getattr(self, f"ooi_masks_{idx}") + self.register_buffer(f"_delete_mask", torch.ones_like(self.ooi_masks[0].float())) + + # self._xyz = torch.nn.Parameter(torch.cat([self._xyz, fused_point_cloud],0),requires_grad=True) + # self._features_dc = torch.nn.Parameter(torch.cat([self._features_dc, features[:, :, 0:1].transpose(1, 2).contiguous()],0),requires_grad=True) + # self._features_rest = torch.nn.Parameter(torch.cat([self._features_rest, features[:, :, 1:].transpose(1, 2).contiguous()],0),requires_grad=True) + # self._scaling = torch.nn.Parameter(torch.cat([self._scaling, scales],0),requires_grad=True) + # self._rotation = torch.nn.Parameter(torch.cat([self._rotation, rots],0),requires_grad=True) + # self._opacity = torch.nn.Parameter(torch.cat([self._opacity, opacities],0),requires_grad=True) + + # if self.cfg.pred_normal: + # normals = torch.zeros((fused_point_cloud.shape[0], 3), device="cuda") + # self._normal = nn.Parameter(normals.requires_grad_(True)) + # self.max_radii2D = torch.zeros((self._xyz.shape[0]), device="cuda") + + # self.fused_point_cloud = fused_point_cloud.cpu().clone().detach() + # self.features = features.cpu().clone().detach() + # self.scales = scales.cpu().clone().detach() + # self.rots = rots.cpu().clone().detach() + # self.opacities = opacities.cpu().clone().detach() + + # language_feature = torch.zeros((fused_point_cloud.shape[0], 3), device="cuda") + # self._language_feature = torch.nn.Parameter(torch.cat([self._language_feature, language_feature], 0), requires_grad=True) + # self.training_setup() + + + def lang_training_setup(self): + training_args = self.cfg + l = [ + {'params': [self._language_feature], 'lr': C(training_args.lang_lr, 0, 0)}, + ] + self._xyz.requires_grad_(False) + self._features_dc.requires_grad_(False) + self._features_rest.requires_grad_(False) + self._scaling.requires_grad_(False) + self._rotation.requires_grad_(False) + self._opacity.requires_grad_(False) + self._language_feature.requires_grad_(True) + # self.lang_optimizer = torch.optim.SGD(l, lr=0.0) + self.lang_optimizer = torch.optim.Adam(l, lr=0.0, eps=1e-15, betas=(self.cfg.lang_beta_1, self.cfg.lang_beta_2)) + self.optimize_params = ["lang"] + self.optimize_list = l + + def after_lang(self): + self._xyz.requires_grad_(True) + self._features_dc.requires_grad_(True) + self._features_rest.requires_grad_(True) + self._scaling.requires_grad_(True) + self._rotation.requires_grad_(True) + self._opacity.requires_grad_(True) + self._language_feature.requires_grad_(False) + + def training_setup(self): + self._xyz.requires_grad_(True) + self._features_dc.requires_grad_(True) + self._features_rest.requires_grad_(True) + self._scaling.requires_grad_(True) + self._rotation.requires_grad_(True) + self._opacity.requires_grad_(True) + self._language_feature.requires_grad_(False) + training_args = self.cfg + self.xyz_gradient_accum = torch.zeros((self.get_xyz.shape[0], 1), device="cuda") + self.denom = torch.zeros((self.get_xyz.shape[0], 1), device="cuda") + + l = [ + { + "params": [self._xyz], + "lr": C(training_args.position_lr, 0, 0), + "name": "xyz", + }, + { + "params": [self._features_dc], + "lr": C(training_args.feature_lr, 0, 0), + "name": "f_dc", + }, + { + "params": [self._features_rest], + "lr": C(training_args.feature_lr, 0, 0) / 20.0, + "name": "f_rest", + }, + { + "params": [self._opacity], + "lr": C(training_args.opacity_lr, 0, 0), + "name": "opacity", + }, + { + "params": [self._scaling], + "lr": C(training_args.scaling_lr, 0, 0), + "name": "scaling", + }, + { + "params": [self._rotation], + "lr": C(training_args.rotation_lr, 0, 0), + "name": "rotation", + }, + {'params': [self._language_feature], 'lr': C(training_args.lang_lr, 0, 0), "name": "language_feature"}, + ] + if self.cfg.pred_normal: + l.append( + { + "params": [self._normal], + "lr": C(training_args.normal_lr, 0, 0), + "name": "normal", + }, + ) + + self.optimize_params = [ + "xyz", + "f_dc", + "f_rest", + "opacity", + "scaling", + "rotation", + "language_feature" + ] + self.optimize_list = l + self.optimizer = torch.optim.Adam(l, lr=0.0, eps=1e-15) + self.lang_optimizer = None + + def merge_optimizer(self, net_optimizer): + l = self.optimize_list + for param in net_optimizer.param_groups: + l.append( + { + "params": param["params"], + "lr": param["lr"], + } + ) + self.optimizer = torch.optim.Adam(l, lr=0.0) + return self.optimizer + + def update_learning_rate(self, iteration): + """Learning rate scheduling per step""" + for param_group in self.optimizer.param_groups: + if not ("name" in param_group): + continue + if param_group["name"] == "xyz": + param_group["lr"] = C( + self.cfg.position_lr, 0, iteration, interpolation="exp" + ) + if param_group["name"] == "scaling": + param_group["lr"] = C( + self.cfg.scaling_lr, 0, iteration, interpolation="exp" + ) + if param_group["name"] == "f_dc": + param_group["lr"] = C( + self.cfg.feature_lr, 0, iteration, interpolation="exp" + ) + if param_group["name"] == "f_rest": + param_group["lr"] = ( + C(self.cfg.feature_lr, 0, iteration, interpolation="exp") / 20.0 + ) + if param_group["name"] == "opacity": + param_group["lr"] = C( + self.cfg.opacity_lr, 0, iteration, interpolation="exp" + ) + if param_group["name"] == "rotation": + param_group["lr"] = C( + self.cfg.rotation_lr, 0, iteration, interpolation="exp" + ) + if param_group["name"] == "normal": + param_group["lr"] = C( + self.cfg.normal_lr, 0, iteration, interpolation="exp" + ) + if self.lang_optimizer is not None: + for param_group in self.lang_optimizer.param_groups: + if not ("name" in param_group): + continue + if param_group["name"] == "language_feature": + param_group["lr"] = C( + self.cfg.lang_lr, 0, iteration, interpolation="exp" + ) + self.color_clip = C(self.cfg.color_clip, 0, iteration) + + def reset_opacity(self): + # opacities_new = inverse_sigmoid(torch.min(self.get_opacity, torch.ones_like(self.get_opacity)*0.01)) + opacities_new = inverse_sigmoid(self.get_opacity * 0.9) + optimizable_tensors = self.replace_tensor_to_optimizer(opacities_new, "opacity") + self._opacity = optimizable_tensors["opacity"] + + def to(self, device="cpu"): + self._xyz = self._xyz.to(device) + self._features_dc = self._features_dc.to(device) + self._features_rest = self._features_rest.to(device) + self._opacity = self._opacity.to(device) + self._scaling = self._scaling.to(device) + self._rotation = self._rotation.to(device) + self._normal = self._normal.to(device) + self._language_feature = self._language_feature.to(device) + + def replace_tensor_to_optimizer(self, tensor, name): + optimizable_tensors = {} + for group in self.optimizer.param_groups: + if ("name" in group) and group["name"] == name: + stored_state = self.optimizer.state.get(group["params"][0], None) + stored_state["exp_avg"] = torch.zeros_like(tensor) + stored_state["exp_avg_sq"] = torch.zeros_like(tensor) + + del self.optimizer.state[group["params"][0]] + group["params"][0] = nn.Parameter(tensor.requires_grad_(True)) + self.optimizer.state[group["params"][0]] = stored_state + + optimizable_tensors[group["name"]] = group["params"][0] + return optimizable_tensors + + def _prune_optimizer(self, mask): + optimizable_tensors = {} + for group in self.optimizer.param_groups: + if ("name" in group) and (group["name"] in self.optimize_params): + stored_state = self.optimizer.state.get(group["params"][0], None) + if stored_state is not None: + stored_state["exp_avg"] = stored_state["exp_avg"][mask] + stored_state["exp_avg_sq"] = stored_state["exp_avg_sq"][mask] + + del self.optimizer.state[group["params"][0]] + group["params"][0] = nn.Parameter( + (group["params"][0][mask].requires_grad_(True)) + ) + self.optimizer.state[group["params"][0]] = stored_state + + optimizable_tensors[group["name"]] = group["params"][0] + else: + group["params"][0] = nn.Parameter( + group["params"][0][mask].requires_grad_(True) + ) + optimizable_tensors[group["name"]] = group["params"][0] + return optimizable_tensors + + def prune_points(self, mask): + valid_points_mask = ~mask + optimizable_tensors = self._prune_optimizer(valid_points_mask) + + self._xyz = optimizable_tensors["xyz"] + self._features_dc = optimizable_tensors["f_dc"] + self._features_rest = optimizable_tensors["f_rest"] + self._opacity = optimizable_tensors["opacity"] + self._scaling = optimizable_tensors["scaling"] + self._rotation = optimizable_tensors["rotation"] + self._language_feature = optimizable_tensors["language_feature"] + if self.cfg.pred_normal: + self._normal = optimizable_tensors["normal"] + + self.xyz_gradient_accum = self.xyz_gradient_accum[valid_points_mask] + + self.denom = self.denom[valid_points_mask] + self.max_radii2D = self.max_radii2D[valid_points_mask] + + def cat_tensors_to_optimizer(self, tensors_dict): + optimizable_tensors = {} + for group in self.optimizer.param_groups: + if ("name" in group) and (group["name"] in self.optimize_params): + extension_tensor = tensors_dict[group["name"]] + stored_state = self.optimizer.state.get(group["params"][0], None) + if stored_state is not None: + stored_state["exp_avg"] = torch.cat( + (stored_state["exp_avg"], torch.zeros_like(extension_tensor)), + dim=0, + ) + stored_state["exp_avg_sq"] = torch.cat( + ( + stored_state["exp_avg_sq"], + torch.zeros_like(extension_tensor), + ), + dim=0, + ) + + del self.optimizer.state[group["params"][0]] + group["params"][0] = nn.Parameter( + torch.cat( + (group["params"][0], extension_tensor), dim=0 + ).requires_grad_(True) + ) + self.optimizer.state[group["params"][0]] = stored_state + + optimizable_tensors[group["name"]] = group["params"][0] + else: + group["params"][0] = nn.Parameter( + torch.cat( + (group["params"][0], extension_tensor), dim=0 + ).requires_grad_(True) + ) + optimizable_tensors[group["name"]] = group["params"][0] + + return optimizable_tensors + + def densification_postfix( + self, + new_xyz, + new_features_dc, + new_features_rest, + new_opacities, + new_scaling, + new_rotation, + new_normal=None, + new_language_feature=None + ): + d = { + "xyz": new_xyz, + "f_dc": new_features_dc, + "f_rest": new_features_rest, + "opacity": new_opacities, + "scaling": new_scaling, + "rotation": new_rotation, + "language_feature": new_language_feature, + } + if self.cfg.pred_normal: + d.update({"normal": new_normal}) + + optimizable_tensors = self.cat_tensors_to_optimizer(d) + self._xyz = optimizable_tensors["xyz"] + self._features_dc = optimizable_tensors["f_dc"] + self._features_rest = optimizable_tensors["f_rest"] + self._opacity = optimizable_tensors["opacity"] + self._scaling = optimizable_tensors["scaling"] + self._rotation = optimizable_tensors["rotation"] + self._language_feature = optimizable_tensors["language_feature"] + if self.cfg.pred_normal: + self._normal = optimizable_tensors["normal"] + + self.xyz_gradient_accum = torch.zeros((self._xyz.shape[0], 1), device="cuda") + self.denom = torch.zeros((self._xyz.shape[0], 1), device="cuda") + self.max_radii2D = torch.zeros((self._xyz.shape[0]), device="cuda") + + def densify_and_split(self, grads, grad_threshold, N=2): + n_init_points = self._xyz.shape[0] + # Extract points that satisfy the gradient condition + padded_grad = torch.zeros((n_init_points), device="cuda") + padded_grad[: grads.shape[0]] = grads.squeeze() + selected_pts_mask = torch.where(padded_grad >= grad_threshold, True, False) + selected_pts_mask = torch.logical_and( + selected_pts_mask, + torch.norm(self.get_scaling, dim=1) > self.cfg.split_thresh, + ) + + # divide N to enhance robustness + stds = self.get_scaling[selected_pts_mask].repeat(N, 1) / N + means = torch.zeros((stds.size(0), 3), device="cuda") + samples = torch.normal(mean=means, std=stds) + rots = build_rotation(self._rotation[selected_pts_mask]).repeat(N, 1, 1) + new_xyz = torch.bmm(rots, samples.unsqueeze(-1)).squeeze(-1) + self._xyz[ + selected_pts_mask + ].repeat(N, 1) + new_scaling = self.scaling_inverse_activation( + self.get_scaling[selected_pts_mask].repeat(N, 1) / (0.8 * N) + ) + new_rotation = self._rotation[selected_pts_mask].repeat(N, 1) + new_features_dc = self._features_dc[selected_pts_mask].repeat(N, 1, 1) + new_features_rest = self._features_rest[selected_pts_mask].repeat(N, 1, 1) + new_opacity = self._opacity[selected_pts_mask].repeat(N, 1) + new_language_feature = self._language_feature[selected_pts_mask].repeat(N,1) + if self.cfg.pred_normal: + new_normal = self._normal[selected_pts_mask].repeat(N, 1) + else: + new_normal = None + + self.densification_postfix( + new_xyz, + new_features_dc, + new_features_rest, + new_opacity, + new_scaling, + new_rotation, + new_normal, + new_language_feature + ) + + prune_filter = torch.cat( + ( + selected_pts_mask, + torch.zeros(N * selected_pts_mask.sum(), device="cuda", dtype=bool), + ) + ) + self.prune_points(prune_filter) + + def densify_and_clone(self, grads, grad_threshold): + # Extract points that satisfy the gradient condition + selected_pts_mask = torch.where( + torch.norm(grads, dim=-1) >= grad_threshold, True, False + ) + selected_pts_mask = torch.logical_and( + selected_pts_mask, + torch.norm(self.get_scaling, dim=1) <= self.cfg.split_thresh, + ) + + new_xyz = self._xyz[selected_pts_mask] + new_features_dc = self._features_dc[selected_pts_mask] + new_features_rest = self._features_rest[selected_pts_mask] + new_opacities = self._opacity[selected_pts_mask] + new_scaling = self._scaling[selected_pts_mask] + new_rotation = self._rotation[selected_pts_mask] + new_language_feature = self._language_feature[selected_pts_mask] + if self.cfg.pred_normal: + new_normal = self._normal[selected_pts_mask] + else: + new_normal = None + + self.densification_postfix( + new_xyz, + new_features_dc, + new_features_rest, + new_opacities, + new_scaling, + new_rotation, + new_normal, + new_language_feature + ) + + def densify(self, max_grad): + grads = self.xyz_gradient_accum / self.denom + grads[grads.isnan()] = 0.0 + + self.densify_and_clone(grads, max_grad) + self.densify_and_split(grads, max_grad) + + def prune(self, min_opacity, max_screen_size): + prune_mask = (self.get_opacity < min_opacity).squeeze() + if self.cfg.prune_big_points: + big_points_vs = self.max_radii2D > (torch.mean(self.max_radii2D) * 3) + prune_mask = torch.logical_or(prune_mask, big_points_vs) + self.prune_points(prune_mask) + + torch.cuda.empty_cache() + + def add_densification_stats(self, viewspace_point_tensor, update_filter): + self.xyz_gradient_accum[update_filter] += torch.norm( + viewspace_point_tensor.grad[update_filter, :2], dim=-1, keepdim=True + ) + self.denom[update_filter] += 1 + + @torch.no_grad() + def update_states( + self, + iteration, + visibility_filter, + radii, + viewspace_point_tensor, + ): + if self._xyz.shape[0] >= self.cfg.max_num + 100: + prune_mask = torch.randperm(self._xyz.shape[0]).to(self._xyz.device) + prune_mask = prune_mask > self.cfg.max_num + self.prune_points(prune_mask) + return + # Keep track of max radii in image-space for pruning + # loop over batch + bs = len(viewspace_point_tensor) + for i in range(bs): + radii_i = radii[i] + visibility_filter_i = visibility_filter[i] + viewspace_point_tensor_i = viewspace_point_tensor[i] + self.max_radii2D = torch.max(self.max_radii2D, radii_i.float()) + + self.add_densification_stats(viewspace_point_tensor_i, visibility_filter_i) + + if ( + iteration > self.cfg.prune_from_iter + and iteration < self.cfg.prune_until_iter + and iteration % self.cfg.prune_interval == 0 + ): + self.prune(self.cfg.min_opac_prune, self.cfg.radii2d_thresh) + if iteration % self.cfg.opacity_reset_interval == 0: + self.reset_opacity() + + if ( + iteration > self.cfg.densify_from_iter + and iteration < self.cfg.densify_until_iter + and iteration % self.cfg.densification_interval == 0 + ): + self.densify(self.cfg.densify_grad_threshold) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_base.py.bak b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_base.py.bak new file mode 100644 index 0000000000000000000000000000000000000000..a6a8748ae49e90c2e698004c4b08f6a74e9f6ad3 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_base.py.bak @@ -0,0 +1,1492 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# +import math +import os +import random +import sys +import argparse +from dataclasses import dataclass, field +from datetime import datetime +from typing import NamedTuple + +import numpy as np +import cv2 +from PIL import Image +import threestudio +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision +from transformers import pipeline +from plyfile import PlyData, PlyElement +from simple_knn._C import distCUDA2 +import diffusers +from diffusers import StableDiffusionInpaintPipeline, AutoPipelineForInpainting +from threestudio.models.geometry.base import BaseGeometry +from threestudio.utils.misc import C +from threestudio.utils.typing import * +from segment_anything import sam_model_registry, SamPredictor +import matplotlib.pyplot as plt + +from .gaussian_io import GaussianIO +import imageio + +from scipy.spatial.transform import Rotation as R + +REORDER_MTX = torch.tensor([ + [0,0,0,1], + [1,0,0,0], + [0,1,0,0], + [0,0,1,0] +]).cuda().float() + +def build_rotation(r): + norm = torch.sqrt( + r[:, 0] * r[:, 0] + r[:, 1] * r[:, 1] + r[:, 2] * r[:, 2] + r[:, 3] * r[:, 3] + ) + + q = r / norm[:, None] + + R = torch.zeros((q.size(0), 3, 3), device="cuda") + r = q[:, 0] + x = q[:, 1] + y = q[:, 2] + z = q[:, 3] + + R[:, 0, 0] = 1 - 2 * (y * y + z * z) + R[:, 0, 1] = 2 * (x * y - r * z) + R[:, 0, 2] = 2 * (x * z + r * y) + R[:, 1, 0] = 2 * (x * y + r * z) + R[:, 1, 1] = 1 - 2 * (x * x + z * z) + R[:, 1, 2] = 2 * (y * z - r * x) + R[:, 2, 0] = 2 * (x * z - r * y) + R[:, 2, 1] = 2 * (y * z + r * x) + R[:, 2, 2] = 1 - 2 * (x * x + y * y) + return R + +def rotation_matrix(angle_x, angle_y, angle_z): + # Convert angles to radians + rad_x = torch.deg2rad(torch.tensor(angle_x)) + rad_y = torch.deg2rad(torch.tensor(angle_y)) + rad_z = torch.deg2rad(torch.tensor(angle_z)) + + # Compute sine and cosine of the angles + cos_x = torch.cos(rad_x) + sin_x = torch.sin(rad_x) + cos_y = torch.cos(rad_y) + sin_y = torch.sin(rad_y) + cos_z = torch.cos(rad_z) + sin_z = torch.sin(rad_z) + + # Construct the rotation matrix + Rx = torch.tensor([[1, 0, 0], + [0, cos_x, -sin_x], + [0, sin_x, cos_x]]) + + Ry = torch.tensor([[cos_y, 0, sin_y], + [0, 1, 0], + [-sin_y, 0, cos_y]]) + + Rz = torch.tensor([[cos_z, -sin_z, 0], + [sin_z, cos_z, 0], + [0, 0, 1]]) + + # Combine the rotation matrices + rotation_matrix = Rz @ Ry @ Rx + + return rotation_matrix + +# from scipy.spatial import KDTree +# +# def distCUDA2(points): +# points_np = points.detach().cpu().float().numpy() +# dists, inds = KDTree(points_np).query(points_np, k=4) +# meanDists = (dists[:, 1:] ** 2).mean(1) +# +# return torch.tensor(meanDists, dtype=points.dtype, device=points.device) + +sys.path.append('./GeoWizard/geowizard') +from models.geowizard_pipeline import DepthNormalEstimationPipeline + +C0 = 0.28209479177387814 + +def propagate(canvas): + H, W = canvas.shape + dx = [0, 1, 0, -1] + dy = [1, 0, -1, 0] + count = np.zeros_like(canvas) + + while 1: + curr_mask = canvas > 0 + if sum(sum(curr_mask)) == H * W: + break + expand_mask = (cv2.blur(curr_mask.astype(np.float32), (3, 3)) > 0) + x, y = np.where(np.logical_and(expand_mask, ~curr_mask)) + old_canvas = canvas.copy() + + for xx, yy in zip(x, y): + for i in range(4): + ref_x = xx + dx[i] + ref_y = yy + dy[i] + if 0<=ref_x 0] = -1e10 # abnormal values... make weights 0 + + return torch.exp(power) + + +def build_rotation(r): + norm = torch.sqrt( + r[:, 0] * r[:, 0] + r[:, 1] * r[:, 1] + r[:, 2] * r[:, 2] + r[:, 3] * r[:, 3] + ) + + q = r / norm[:, None] + + R = torch.zeros((q.size(0), 3, 3), device="cuda") + + r = q[:, 0] + x = q[:, 1] + y = q[:, 2] + z = q[:, 3] + + R[:, 0, 0] = 1 - 2 * (y * y + z * z) + R[:, 0, 1] = 2 * (x * y - r * z) + R[:, 0, 2] = 2 * (x * z + r * y) + R[:, 1, 0] = 2 * (x * y + r * z) + R[:, 1, 1] = 1 - 2 * (x * x + z * z) + R[:, 1, 2] = 2 * (y * z - r * x) + R[:, 2, 0] = 2 * (x * z - r * y) + R[:, 2, 1] = 2 * (y * z + r * x) + R[:, 2, 2] = 1 - 2 * (x * x + y * y) + return R + + +def build_scaling_rotation(s, r): + L = torch.zeros((s.shape[0], 3, 3), dtype=torch.float, device="cuda") + R = build_rotation(r) + + L[:, 0, 0] = s[:, 0] + L[:, 1, 1] = s[:, 1] + L[:, 2, 2] = s[:, 2] + + L = R @ L + return L + + +def safe_state(silent): + old_f = sys.stdout + + class F: + def __init__(self, silent): + self.silent = silent + + def write(self, x): + if not self.silent: + if x.endswith("\n"): + old_f.write( + x.replace( + "\n", + " [{}]\n".format( + str(datetime.now().strftime("%d/%m %H:%M:%S")) + ), + ) + ) + else: + old_f.write(x) + + def flush(self): + old_f.flush() + + sys.stdout = F(silent) + + random.seed(0) + np.random.seed(0) + torch.manual_seed(0) + torch.cuda.set_device(torch.device("cuda:0")) + + +class BasicPointCloud(NamedTuple): + points: np.array + colors: np.array + normals: np.array + + +class Camera(NamedTuple): + FoVx: torch.Tensor + FoVy: torch.Tensor + camera_center: torch.Tensor + image_width: int + image_height: int + world_view_transform: torch.Tensor + full_proj_transform: torch.Tensor + +def fill_mask(mask): + mask = np.array(mask) + canvas = np.zeros_like(mask) + H, W = mask.shape + for i in range(H): + for p in range(0, W): + if mask[i, p]: + canvas[i, p] = 1 + else: + break + for p in range(W-1, 0, -1): + if mask[i, p]: + canvas[i, p] = 1 + else: + break + + for i in range(W): + for p in range(0, H): + if mask[p, i]: + canvas[p, i] = 1 + else: + break + for p in range(H-1, 0, -1): + if mask[p, i]: + canvas[p, i] = 1 + else: + break + mask = np.logical_and(mask, canvas) + return Image.fromarray(mask) + +def parse_wh(wh): + try: + W, H = wh + except: + H = W = wh + return W, H + +@threestudio.register("gaussian-splatting") +class GaussianBaseModel(BaseGeometry, GaussianIO): + @dataclass + class Config(BaseGeometry.Config): + max_num: int = 500000 + sh_degree: int = 0 + position_lr: Any = 0.001 + # scale_lr: Any = 0.003 + feature_lr: Any = 0.01 + opacity_lr: Any = 0.05 + scaling_lr: Any = 0.005 + rotation_lr: Any = 0.005 + pred_normal: bool = False + normal_lr: Any = 0.001 + lang_lr: float = 0.005 + + densification_interval: int = 50 + prune_interval: int = 50 + opacity_reset_interval: int = 100000 + densify_from_iter: int = 100 + prune_from_iter: int = 100 + densify_until_iter: int = 2000 + prune_until_iter: int = 2000 + densify_grad_threshold: Any = 0.01 + min_opac_prune: Any = 0.005 + split_thresh: Any = 0.02 + radii2d_thresh: Any = 1000 + + sphere: bool = False + prune_big_points: bool = False + color_clip: Any = 2.0 + + geometry_convert_from: str = "" + load_ply_only_vertex: bool = False + init_num_pts: int = 100 + pc_init_radius: float = 0.8 + opacity_init: float = 0.1 + + img_resolution: Any = 512 + + shap_e_guidance_config: dict = field(default_factory=dict) + + max_scaling: float = 100 + sam_ckpt_path: str = "ckpts/sam_vit_h_4b8939.pth" + ooi_bbox: Any = None + + prompt: Any = None + empty_prompt: Any = None + novel_view_gradual: bool = False + lang_beta_1: float = 0.9 + lang_beta_2: float = 0.999 + + inference_only: bool = False + + cfg: Config + + def setup_functions(self): + def build_covariance_from_scaling_rotation(scaling, scaling_modifier, rotation): + L = build_scaling_rotation(scaling_modifier * scaling, rotation) + actual_covariance = L @ L.transpose(1, 2) + symm = strip_symmetric(actual_covariance) + return symm + + self.scaling_activation = torch.exp + self.scaling_inverse_activation = torch.log + + self.covariance_activation = build_covariance_from_scaling_rotation + + self.opacity_activation = torch.sigmoid + self.inverse_opacity_activation = inverse_sigmoid + + self.rotation_activation = torch.nn.functional.normalize + self.color_clip = C(self.cfg.color_clip, 0, 0) + + self.fixed_xyz = None + self.fixed_rot = None + + if not self.cfg.inference_only: + sam = sam_model_registry["vit_h"](checkpoint=self.cfg.sam_ckpt_path).to('cuda') + self.predictor = SamPredictor(sam) + + def project_pc(self, c2w, H=None, W=None): + W, H = parse_wh(self.cfg.img_resolution) + # if W is None: + # W = H + assert self.point_cloud is not None + pc_cam = torch.einsum('bxy,hwy->bhwx', torch.linalg.inv(c2w), self.point_cloud) + depth = -1 * pc_cam[..., 2].view(pc_cam.shape[0], -1) + pc_cam = (pc_cam / pc_cam[..., 2:3])[..., :3] + pc_2d = torch.einsum('xy,bhwy->bhwx', self.proj_mtx, pc_cam).clamp(0, 1) + pc_2d[..., 0] = pc_2d[..., 0] * (W-1) + pc_2d[..., 1] = pc_2d[..., 1] * (H-1) + pc_2d = (pc_2d.long()).view(pc_2d.shape[0], -1, pc_2d.shape[-1]) + + mask = torch.zeros([pc_2d.shape[0], H, W], device='cuda') + depth_canvas = torch.zeros([pc_2d.shape[0], H, W], device='cuda') + for i in range(pc_2d.shape[0]): + x = (W - pc_2d[i, :, 0]).clamp(0, W-1) + y = (pc_2d[i, :, 1]).clamp(0, H-1) + mask[i, y, x] = 1.0 + depth_canvas[i, y, x] = depth[i] + + mask = torchvision.transforms.functional.gaussian_blur(mask, 3) > 0 + return mask, depth_canvas + + def img2pc_inpaint(self, img, c2w=None, gt_depth=None, mask=None, proj_func=None): + W, H = parse_wh(self.cfg.img_resolution) + with torch.no_grad(): + depth = self.geowizard_pipe( + img, + denoising_steps = 25, + ensemble_size = 3, + processing_res = 768, + match_input_res = False, + domain = 'outdoor', + color_map = 'Spectral', + gt_depth = gt_depth, mask = mask, + show_progress_bar = True)['depth_np'] + ret_depth = depth.copy() + depth = torch.from_numpy(depth)[None] + depth = torch.nn.functional.interpolate(depth[None], size=(H, W), mode='bilinear', align_corners=True).squeeze() + + depth = depth.cpu().numpy() + if proj_func is None: + depth = depth * 20 + 5 + else: + depth = proj_func(depth) + + depth = depth * -1 + x, y = np.meshgrid(np.arange(W, dtype=np.float32), np.arange(H, dtype=np.float32), indexing='xy') + x = x / float(W-1) + y = y / float(H-1) + xyz = np.stack((x, y, np.ones_like(x)), 0).transpose(1, 2, 0) + xyz[..., 0] = 1 - xyz[..., 0] + + fov = 60 / 180 * np.pi + proj_mtx = np.array([ + [1 / (2 * np.tan(fov/2)), 0, 1/2], + [0, 1 / (2 * np.tan(fov/2)), 1/2], + [0, 0, 1], + ]) + self.proj_mtx = torch.from_numpy(proj_mtx).cuda().float() + if c2w is None: + c2w = np.array([0.0000, -0.3420, 0.9397, 2.3492, 1.0000, 0.0000, -0.0000, 0.0000, -0.0000, 0.9397, 0.3420, 0.8551, 0.0000, 0.0000, 0.0000, 1.0000]).reshape(4, 4) + c2w = np.array([0.0000, 0.0000, 1.0000, 2.5000, 1.0000, 0.0000, -0.0000, 0.0000, -0.0000, 1.0000, -0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 1.0000]).reshape(4, 4) + else: + c2w = c2w[0].cpu().numpy() + xyz = np.einsum('ab,hwb->hwa', np.linalg.inv(proj_mtx), xyz) + xyz = xyz * depth[..., None] + xyz = np.concatenate([xyz, np.ones_like(x)[..., None]], 2) + xyz = np.einsum('ab,hwb->hwa', c2w, xyz) + return xyz, ret_depth + + def img2pc(self, img, transit_mask=None, fg_transit_l=None, fg_transit_r=None, c2w=None, fg_depth=None): + H, W = parse_hw(self.cfg.img_resolution) + with torch.no_grad(): + depth = self.geowizard_pipe( + img, + denoising_steps = 25, + ensemble_size = 3, + processing_res = 768, + match_input_res = True, + domain = 'outdoor', + color_map = 'Spectral', + show_progress_bar = True)['depth_np'] + depth = torch.from_numpy(depth)[None] + depth = torch.nn.functional.interpolate(depth[None], size=(W, H), mode='bilinear', align_corners=True).squeeze() + + + depth = depth.cpu().numpy() + if fg_depth is None: + if fg_transit_l is None: + l, r = np.quantile(depth, 0.05), np.quantile(depth, 0.95) + depth = (depth - l) / (r - l) * 20 * 0.9 + 2 + 5 / 90 + ret_depth = depth.copy() + else: + transit_l, transit_r = depth[transit_mask].min(), depth[transit_mask].max() + depth = (depth - transit_l) / (transit_r - transit_l) * (fg_transit_r - fg_transit_l) + fg_transit_l + ret_depth = depth + else: + delta = fg_depth[0] - depth + delta[~transit_mask] = 0 + delta = propagate(delta) + depth = depth + delta + ret_depth = depth.copy() + depth = depth * -1 + x, y = np.meshgrid(np.arange(H, dtype=np.float32), np.arange(W, dtype=np.float32), indexing='xy') + x = x / float(H-1) + y = y / float(W-1) + xyz = np.stack((x, y, np.ones_like(x)), 0).transpose(1, 2, 0) + xyz[..., 0] = 1 - xyz[..., 0] + + fov = 60 / 180 * np.pi + proj_mtx = np.array([ + [1 / (2 * np.tan(fov/2)), 0, 1/2], + [0, 1 / (2 * np.tan(fov/2)), 1/2], + [0, 0, 1], + ]) + self.proj_mtx = torch.from_numpy(proj_mtx).cuda().float() + if c2w is None: + c2w = np.array([0.0000, -0.3420, 0.9397, 2.3492, 1.0000, 0.0000, -0.0000, 0.0000, -0.0000, 0.9397, 0.3420, 0.8551, 0.0000, 0.0000, 0.0000, 1.0000]).reshape(4, 4) + c2w = np.array([0.0000, 0.0000, 1.0000, 2.5000, 1.0000, 0.0000, -0.0000, 0.0000, -0.0000, 1.0000, -0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 1.0000]).reshape(4, 4) + else: + c2w = c2w[0].cpu().numpy() + xyz = np.einsum('ab,hwb->hwa', np.linalg.inv(proj_mtx), xyz) + xyz = xyz * depth[..., None] + xyz = np.concatenate([xyz, np.ones_like(x)[..., None]], 2) + xyz = np.einsum('ab,hwb->hwa', c2w, xyz) + return xyz, ret_depth + + def inpaint(self, img, mask, prompt): + # inpaint using base pipe + N = 512 + img = img.convert("RGB").resize((N, N)) + mask = mask.convert("RGB").resize((N, N)) + self.base_inpainting_pipe.to("cuda") + img = self.base_inpainting_pipe(prompt=prompt, image=img, mask_image=mask, guidance_scale=7.5).images[0] + self.base_inpainting_pipe.to("cpu") + torch.cuda.empty_cache() + + # inpaint using sdxl pipe + N = 1024 + img = img.convert("RGB").resize((N, N)) + mask = mask.convert("RGB").resize((N, N)) + self.sdxl_inpainting_pipe.to("cuda") + img = self.sdxl_inpainting_pipe(prompt=prompt, image=img, mask_image=mask, guidance_scale=7.5, num_inference_steps=20, strength=0.99).images[0] + self.sdxl_inpainting_pipe.to("cpu") + + return img + + def configure(self) -> None: + super().configure() + self.active_sh_degree = 0 + self.max_sh_degree = self.cfg.sh_degree + self._xyz = torch.empty(0) + self._features_dc = torch.empty(0) + self._features_rest = torch.empty(0) + self._scaling = torch.empty(0) + self._rotation = torch.empty(0) + self._opacity = torch.empty(0) + self._opacity_mask = None + self.max_radii2D = torch.empty(0) + self.xyz_gradient_accum = torch.empty(0) + self.denom = torch.empty(0) + self.noise_ratio = 0.0 + if self.cfg.pred_normal: + self._normal = torch.empty(0) + self.optimizer = None + self.setup_functions() + self.save_path = None + self.fixed_xyz = None + self.fixed_rot = None + + if self.cfg.inference_only: + return + # setup GeoWizard + geowizard_checkpoint_path = 'lemonaddie/geowizard' + self.geowizard_pipe = DepthNormalEstimationPipeline.from_pretrained( + geowizard_checkpoint_path, torch_dtype=torch.float32).to(torch.device("cuda")) + + self.base_inpainting_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting", torch_dtype=torch.float16) + # self.base_inpainting_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting", torch_dtype=torch.float16, safety_checker=None) + self.sdxl_inpainting_pipe = AutoPipelineForInpainting.from_pretrained("diffusers/stable-diffusion-xl-1.0-inpainting-0.1", torch_dtype=torch.float16, variant="fp16") + self.sdxl_inpainting_pipe.scheduler = diffusers.EulerDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", subfolder="scheduler") + + if self.cfg.geometry_convert_from.startswith("depth:"): + # estimate depth + W, H = parse_wh(self.cfg.img_resolution) + mask_H, mask_W = H, W + if max(H, W) > 1024: + mask_H, mask_W = int(H / max(H, W) * 1024), int(W / max(H, W) * 1024) + img = self.cfg.geometry_convert_from[len("depth:"):] + raw_img = img = Image.open(img).convert("RGB") + img = img.resize((W, H)) + + bg_xyz, bg_color = [], [] + + with torch.no_grad(): + self.predictor.set_image(np.array(raw_img)) + self.ooi_masks = [] + total_inp_ooi_masks = None + total_ooi_masks = [] + for i in range(len(self.cfg.ooi_bbox) // 4): + bbox = np.array(self.cfg.ooi_bbox[4*i:4*i+4]) + masks, _, _ = self.predictor.predict( + point_coords=None, + point_labels=None, + box=bbox[None, :], + multimask_output=False, + ) + # plt.imshow(masks[0]) + # plt.savefig(os.path.join(self.save_path, f'mask_{i}.png')) + ooi_masks = np.array(Image.fromarray(masks[0]).resize((W, H), Image.NEAREST)) + ooi_masks = (cv2.blur(ooi_masks.astype(np.float32), (5, 5)) > 0) + inp_ooi_masks = (cv2.blur(ooi_masks.astype(np.float32), (7, 7)) > 0) + if i == 0: + total_inp_ooi_masks = inp_ooi_masks + else: + total_inp_ooi_masks += inp_ooi_masks + total_ooi_masks.append(ooi_masks) + + total_inp_ooi_masks = total_inp_ooi_masks > 0 + bg_image = self.inpaint(img=img, mask=Image.fromarray(total_inp_ooi_masks), prompt=self.cfg.empty_prompt).resize((W, H)) + self.bg_image = np.array(bg_image) + self.bg_image_mask = np.array(Image.fromarray(total_inp_ooi_masks).resize((W, H))) + + xyz, depth = self.img2pc_inpaint(img) + self.point_cloud = torch.from_numpy(xyz).cuda().float() + + for ooi_masks in total_ooi_masks: + transit_masks = np.logical_and(cv2.blur(ooi_masks.astype(np.float32), (3, 3)) > 0, ~ooi_masks) + depth_tensor = torch.from_numpy(depth)[None, None].cuda() * 2 - 1 + self.ooi_masks.append(torch.tensor(ooi_masks.reshape(-1).astype(np.uint8), device='cuda').float().bool()) + ooi_masks = cv2.blur(ooi_masks.astype(np.float32), (9, 9)) > 0 + mask = torch.from_numpy(ooi_masks.astype(np.float32))[None, None].cuda() + bg_xyz_pc, _ = self.img2pc_inpaint(bg_image, gt_depth=depth_tensor, mask=1-mask) + + bg_xyz.append(bg_xyz_pc[ooi_masks][:, :3]) + bg_color.append(np.array(bg_image)[ooi_masks] / 255) + + xyz = xyz[..., :3].reshape(-1, 3) + color = np.array(img).reshape(-1, 3) / 255 + additional_pts_num = sum([len(each) for each in bg_xyz]) + xyz = np.concatenate([xyz, np.concatenate(bg_xyz, 0)], 0) + color = np.concatenate([color, np.concatenate(bg_color, 0)], 0) + for i in range(len(self.ooi_masks)): + self.register_buffer(f"ooi_masks_{i}", torch.cat([self.ooi_masks[i], torch.zeros([additional_pts_num], device='cuda').bool()]) ) + self.ooi_masks[i] = getattr(self, f"ooi_masks_{i}") + self.register_buffer(f"_delete_mask", torch.ones_like(self.ooi_masks[0].float())) + + # project to 3D space + xyz = xyz + color = color + pcd = BasicPointCloud( + points=xyz, colors=color, normals=np.zeros((xyz.shape[0], 3)) + ) + self.create_from_pcd(pcd, 10) + self.training_setup() + + elif self.cfg.geometry_convert_from.startswith("shap-e:"): + shap_e_guidance = threestudio.find("shap-e-guidance")( + self.cfg.shap_e_guidance_config + ) + prompt = self.cfg.geometry_convert_from[len("shap-e:") :] + xyz, color = shap_e_guidance(prompt) + + pcd = BasicPointCloud( + points=xyz, colors=color, normals=np.zeros((xyz.shape[0], 3)) + ) + self.create_from_pcd(pcd, 10) + self.training_setup() + + # Support Initialization from OpenLRM, Please see https://github.com/Adamdad/threestudio-lrm + elif self.cfg.geometry_convert_from.startswith("lrm:"): + lrm_guidance = threestudio.find("lrm-guidance")( + self.cfg.shap_e_guidance_config + ) + prompt = self.cfg.geometry_convert_from[len("lrm:") :] + xyz, color = lrm_guidance(prompt) + + pcd = BasicPointCloud( + points=xyz, colors=color, normals=np.zeros((xyz.shape[0], 3)) + ) + self.create_from_pcd(pcd, 10) + self.training_setup() + + elif os.path.exists(self.cfg.geometry_convert_from): + threestudio.info( + "Loading point cloud from %s" % self.cfg.geometry_convert_from + ) + if self.cfg.geometry_convert_from.endswith(".ckpt"): + ckpt_dict = torch.load(self.cfg.geometry_convert_from) + num_pts = ckpt_dict["state_dict"]["geometry._xyz"].shape[0] + pcd = BasicPointCloud( + points=np.zeros((num_pts, 3)), + colors=np.zeros((num_pts, 3)), + normals=np.zeros((num_pts, 3)), + ) + self.create_from_pcd(pcd, 10) + self.training_setup() + new_ckpt_dict = {} + for key in self.state_dict(): + if ckpt_dict["state_dict"].__contains__("geometry." + key): + new_ckpt_dict[key] = ckpt_dict["state_dict"]["geometry." + key] + else: + new_ckpt_dict[key] = self.state_dict()[key] + self.load_state_dict(new_ckpt_dict) + elif self.cfg.geometry_convert_from.endswith(".ply"): + if self.cfg.load_ply_only_vertex: + plydata = PlyData.read(self.cfg.geometry_convert_from) + vertices = plydata["vertex"] + positions = np.vstack( + [vertices["x"], vertices["y"], vertices["z"]] + ).T + if vertices.__contains__("red"): + colors = ( + np.vstack( + [vertices["red"], vertices["green"], vertices["blue"]] + ).T + / 255.0 + ) + else: + shs = np.random.random((positions.shape[0], 3)) / 255.0 + C0 = 0.28209479177387814 + colors = shs * C0 + 0.5 + normals = np.zeros_like(positions) + pcd = BasicPointCloud( + points=positions, colors=colors, normals=normals + ) + self.create_from_pcd(pcd, 10) + else: + self.load_ply(self.cfg.geometry_convert_from) + self.training_setup() + else: + threestudio.info("Geometry not found, initilization with random points") + num_pts = self.cfg.init_num_pts + phis = np.random.random((num_pts,)) * 2 * np.pi + costheta = np.random.random((num_pts,)) * 2 - 1 + thetas = np.arccos(costheta) + mu = np.random.random((num_pts,)) + radius = self.cfg.pc_init_radius * np.cbrt(mu) + x = radius * np.sin(thetas) * np.cos(phis) + y = radius * np.sin(thetas) * np.sin(phis) + z = radius * np.cos(thetas) + xyz = np.stack((x, y, z), axis=1) + + shs = np.random.random((num_pts, 3)) / 255.0 + C0 = 0.28209479177387814 + color = shs * C0 + 0.5 + pcd = BasicPointCloud( + points=xyz, colors=color, normals=np.zeros((num_pts, 3)) + ) + + self.create_from_pcd(pcd, 10) + self.training_setup() + + def add_pc_from_novel_view(self, rgb, mask, depth, c2w, save_path=None): + W, H = parse_wh(self.cfg.img_resolution) + # depth estimation -> add points. + mask = fill_mask(mask) + mask_array = np.array(mask) + blur_mask = Image.fromarray(cv2.blur(np.array(mask).astype(np.float32), (7, 7)) > 0) + res = self.inpaint(img=rgb, mask=blur_mask, prompt=self.side_prompt) + + depth_unaligned = self.geowizard_pipe( + res, + denoising_steps = 25, + ensemble_size = 3, + processing_res = 768, + match_input_res = False, + domain = 'outdoor', + color_map = 'Spectral', + gt_depth = None, mask = None, + show_progress_bar = True)['depth_np'] + prev_depth = depth_unaligned[~np.array(mask.resize((768,768)))] + # inpaint the depth map + depth_array = depth[0].cpu().numpy().astype(np.uint8) + inpaint_mask = (~mask_array & (depth_array == 0)).astype(np.uint8) + # inpaint_mask = np.logical_and(~np.array(mask.resize((512, 512), Image.NEAREST)) , depth[0].cpu().numpy().astype(np.uint8)==0 ).astype(np.uint8) + l, r = depth[depth>0].min().item(), depth.max().item() + depth = (depth - l) / (r - l) * 255 + depth = cv2.inpaint(depth_array, inpaint_mask, 3, cv2.INPAINT_TELEA) + depth = torch.tensor(depth)[None].cuda().float() / 255 + reproj_func = lambda x: (x - prev_depth.min().item()) / (prev_depth.max().item() - prev_depth.min().item()) * (r-l) + l + depth = depth * (prev_depth.max() - prev_depth.min()) + prev_depth.min() + depth_tensor = torch.nn.functional.interpolate(depth[None].cuda(), 768, mode='nearest') * 2 - 1 + + _masks = cv2.blur(np.array(mask.resize((768, 768))).astype(float), (20, 20)) > 0 + mask_tensor = torch.from_numpy(_masks.astype(np.float32))[None, None].cuda() + bg_xyz_pc, _ = self.img2pc_inpaint(res, gt_depth=depth_tensor, mask=1-mask_tensor, proj_func=reproj_func, c2w=c2w) + + new_xyz = bg_xyz_pc[mask_array][:, :3] + res = res.resize((W, H)) + new_color = np.array(res)[mask_array] / 255 + pcd = BasicPointCloud(points=new_xyz, colors=new_color, normals=np.zeros((new_xyz.shape[0], 3))) + self.merge_from_pcd(pcd, 10) + + save_pc(save_path, new_xyz, new_color) + return res, mask + + @property + def get_scaling(self): + if self.cfg.sphere: + return self.scaling_activation( + torch.mean(self._scaling, dim=-1).unsqueeze(-1).repeat(1, 3) + ).clip(0, self.cfg.max_scaling) + return self.scaling_activation(self._scaling).clip(0, self.cfg.max_scaling) + + @property + def get_rotation(self): + return self.rotation_activation(self._rotation) + + @property + def get_language_feature(self): + return self._language_feature + + @property + def get_xyz(self): + ret = self._xyz + if self.noise_ratio > 0.0: + offset = torch.zeros_like(ret) + for idx in range(len(self.ooi_masks)): + ooi_masks = getattr(self, f"ooi_masks_{idx}") + offset[ooi_masks] = torch.rand(3, device='cuda') * self.noise_ratio + return ret + + @property + def get_features(self): + features_dc = self._features_dc + features_dc = features_dc.clip(-self.color_clip, self.color_clip) + features_rest = self._features_rest + return torch.cat((features_dc, features_rest), dim=1) + + @property + def get_opacity(self): + if self._opacity_mask is None: + ret = self.opacity_activation(self._opacity) + else: + ret = self.opacity_activation(self._opacity) * self._opacity_mask.unsqueeze(-1) + + if self._delete_mask is None: + return ret + else: + return ret * self._delete_mask.unsqueeze(-1) + + @property + def get_normal(self): + if self.cfg.pred_normal: + return self._normal + else: + raise ValueError("Normal is not predicted") + + def recover_xyzrot(self): + self._xyz = torch.nn.Parameter(self.fixed_xyz) + self._rotation = torch.nn.Parameter(self.fixed_rot) + + def random_rotate(self, rotate_aug_scale, apply_rotate): + if self.fixed_xyz is None: + self.fixed_xyz = self.get_xyz.data + self.fixed_rot = self.get_rotation.data + + if apply_rotate: + ooi_mask = self.ooi_masks_0.view(-1).byte().to(device='cuda').float() + + rotate = random.randint(-rotate_aug_scale, rotate_aug_scale) + rot_matrix = rotation_matrix(0, 0, rotate).cuda() + prev_xyz = self.fixed_xyz.clone() + ooi_xyz = prev_xyz[ooi_mask.bool()] + mean = ooi_xyz.mean(0) + ooi_xyz = ooi_xyz - mean + after_xyz = torch.einsum('ab,nb->na', rot_matrix, ooi_xyz) + mean + prev_xyz[ooi_mask.bool()] = after_xyz + self._xyz = torch.nn.Parameter(prev_xyz) + + prev_rotation = self.fixed_rot.clone() + prev_rotation_mtx = build_rotation(prev_rotation) + after_rotation_mtx = torch.einsum('ab,nbc->nac', rot_matrix, prev_rotation_mtx) + after_rotation = torch.from_numpy(R.from_matrix(after_rotation_mtx.detach().cpu()).as_quat()).cuda().float() + after_rotation = torch.einsum('ab,nb->na', REORDER_MTX, after_rotation) + prev_rotation[ooi_mask.bool()] = after_rotation[ooi_mask.bool()] + self._rotation = torch.nn.Parameter(prev_rotation) + else: + self.recover_xyzrot() + + def get_covariance(self, scaling_modifier=1): + return self.covariance_activation( + self.get_scaling, scaling_modifier, self._rotation + ) + + def create_from_pcd(self, pcd: BasicPointCloud, spatial_lr_scale: float): + self.spatial_lr_scale = spatial_lr_scale + fused_point_cloud = torch.tensor(np.asarray(pcd.points)).float().cuda() + fused_color = RGB2SH(torch.tensor(np.asarray(pcd.colors)).float().cuda()) + features = ( + torch.zeros((fused_color.shape[0], 3, (self.max_sh_degree + 1) ** 2)) + .float() + .cuda() + ) + features[:, :3, 0] = fused_color + features[:, 3:, 1:] = 0.0 + + threestudio.info( + f"Number of points at initialisation:{fused_point_cloud.shape[0]}" + ) + + dist2 = torch.clamp_min( + distCUDA2(torch.from_numpy(np.asarray(pcd.points)).float().cuda()), + 0.0000001, + ) + scales = torch.log(torch.sqrt(dist2))[..., None].repeat(1, 3) + rots = torch.zeros((fused_point_cloud.shape[0], 4), device="cuda") + rots[:, 0] = 1 + + opacities = inverse_sigmoid( + self.cfg.opacity_init + * torch.ones( + (fused_point_cloud.shape[0], 1), dtype=torch.float, device="cuda" + ) + ) + + self._xyz = nn.Parameter(fused_point_cloud.requires_grad_(True)) + self._features_dc = nn.Parameter( + features[:, :, 0:1].transpose(1, 2).contiguous().requires_grad_(True) + ) + self._features_rest = nn.Parameter( + features[:, :, 1:].transpose(1, 2).contiguous().requires_grad_(True) + ) + self._scaling = nn.Parameter(scales.requires_grad_(True)) + self._rotation = nn.Parameter(rots.requires_grad_(True)) + self._opacity = nn.Parameter(opacities.requires_grad_(True)) + if self.cfg.pred_normal: + normals = torch.zeros((fused_point_cloud.shape[0], 3), device="cuda") + self._normal = nn.Parameter(normals.requires_grad_(True)) + self.max_radii2D = torch.zeros((self._xyz.shape[0]), device="cuda") + + self.fused_point_cloud = fused_point_cloud.cpu().clone().detach() + self.features = features.cpu().clone().detach() + self.scales = scales.cpu().clone().detach() + self.rots = rots.cpu().clone().detach() + self.opacities = opacities.cpu().clone().detach() + + language_feature = torch.zeros((self._xyz.shape[0], 3), device="cuda") + self._language_feature = torch.nn.Parameter(language_feature.requires_grad_(True)) + + def merge_from_pcd(self, pcd: BasicPointCloud, spatial_lr_scale: float): + self.spatial_lr_scale = spatial_lr_scale + fused_point_cloud = torch.tensor(np.asarray(pcd.points)).float().cuda() + fused_color = RGB2SH(torch.tensor(np.asarray(pcd.colors)).float().cuda()) + features = ( + torch.zeros((fused_color.shape[0], 3, (self.max_sh_degree + 1) ** 2)) + .float() + .cuda() + ) + features[:, :3, 0] = fused_color + features[:, 3:, 1:] = 0.0 + + threestudio.info( + f"Number of points at merging:{fused_point_cloud.shape[0]}" + ) + + dist2 = torch.clamp_min( + distCUDA2(torch.from_numpy(np.asarray(pcd.points)).float().cuda()), + 0.0000001, + ) + scales = torch.log(torch.sqrt(dist2))[..., None].repeat(1, 3) + rots = torch.zeros((fused_point_cloud.shape[0], 4), device="cuda") + rots[:, 0] = 1 + + opacities = inverse_sigmoid( + self.cfg.opacity_init + * torch.ones( + (fused_point_cloud.shape[0], 1), dtype=torch.float, device="cuda" + ) + ) + self.densification_postfix( + fused_point_cloud, + features[:, :, 0:1].transpose(1, 2).contiguous(), + features[:, :, 1:].transpose(1, 2).contiguous(), + opacities, + scales, + rots, + None, + torch.zeros((fused_point_cloud.shape[0], 3), device="cuda") + ) + + for idx in range(len(self.ooi_masks)): + # self.ooi_masks[idx] = torch.cat([self.ooi_masks[idx], torch.ones([fused_point_cloud.shape[0]], device='cuda') > 0]) + self.register_buffer(f"ooi_masks_{idx}", torch.cat([getattr(self, f"ooi_masks_{idx}"), torch.zeros([fused_point_cloud.shape[0]], device='cuda').bool()]) ) + self.ooi_masks[idx] = getattr(self, f"ooi_masks_{idx}") + self.register_buffer(f"_delete_mask", torch.ones_like(self.ooi_masks[0].float())) + + # self._xyz = torch.nn.Parameter(torch.cat([self._xyz, fused_point_cloud],0),requires_grad=True) + # self._features_dc = torch.nn.Parameter(torch.cat([self._features_dc, features[:, :, 0:1].transpose(1, 2).contiguous()],0),requires_grad=True) + # self._features_rest = torch.nn.Parameter(torch.cat([self._features_rest, features[:, :, 1:].transpose(1, 2).contiguous()],0),requires_grad=True) + # self._scaling = torch.nn.Parameter(torch.cat([self._scaling, scales],0),requires_grad=True) + # self._rotation = torch.nn.Parameter(torch.cat([self._rotation, rots],0),requires_grad=True) + # self._opacity = torch.nn.Parameter(torch.cat([self._opacity, opacities],0),requires_grad=True) + + # if self.cfg.pred_normal: + # normals = torch.zeros((fused_point_cloud.shape[0], 3), device="cuda") + # self._normal = nn.Parameter(normals.requires_grad_(True)) + # self.max_radii2D = torch.zeros((self._xyz.shape[0]), device="cuda") + + # self.fused_point_cloud = fused_point_cloud.cpu().clone().detach() + # self.features = features.cpu().clone().detach() + # self.scales = scales.cpu().clone().detach() + # self.rots = rots.cpu().clone().detach() + # self.opacities = opacities.cpu().clone().detach() + + # language_feature = torch.zeros((fused_point_cloud.shape[0], 3), device="cuda") + # self._language_feature = torch.nn.Parameter(torch.cat([self._language_feature, language_feature], 0), requires_grad=True) + # self.training_setup() + + + def lang_training_setup(self): + training_args = self.cfg + l = [ + {'params': [self._language_feature], 'lr': C(training_args.lang_lr, 0, 0)}, + ] + self._xyz.requires_grad_(False) + self._features_dc.requires_grad_(False) + self._features_rest.requires_grad_(False) + self._scaling.requires_grad_(False) + self._rotation.requires_grad_(False) + self._opacity.requires_grad_(False) + self._language_feature.requires_grad_(True) + # self.lang_optimizer = torch.optim.SGD(l, lr=0.0) + self.lang_optimizer = torch.optim.Adam(l, lr=0.0, eps=1e-15, betas=(self.cfg.lang_beta_1, self.cfg.lang_beta_2)) + self.optimize_params = ["lang"] + self.optimize_list = l + + def after_lang(self): + self._xyz.requires_grad_(True) + self._features_dc.requires_grad_(True) + self._features_rest.requires_grad_(True) + self._scaling.requires_grad_(True) + self._rotation.requires_grad_(True) + self._opacity.requires_grad_(True) + self._language_feature.requires_grad_(False) + + def training_setup(self): + self._xyz.requires_grad_(True) + self._features_dc.requires_grad_(True) + self._features_rest.requires_grad_(True) + self._scaling.requires_grad_(True) + self._rotation.requires_grad_(True) + self._opacity.requires_grad_(True) + self._language_feature.requires_grad_(False) + training_args = self.cfg + self.xyz_gradient_accum = torch.zeros((self.get_xyz.shape[0], 1), device="cuda") + self.denom = torch.zeros((self.get_xyz.shape[0], 1), device="cuda") + + l = [ + { + "params": [self._xyz], + "lr": C(training_args.position_lr, 0, 0), + "name": "xyz", + }, + { + "params": [self._features_dc], + "lr": C(training_args.feature_lr, 0, 0), + "name": "f_dc", + }, + { + "params": [self._features_rest], + "lr": C(training_args.feature_lr, 0, 0) / 20.0, + "name": "f_rest", + }, + { + "params": [self._opacity], + "lr": C(training_args.opacity_lr, 0, 0), + "name": "opacity", + }, + { + "params": [self._scaling], + "lr": C(training_args.scaling_lr, 0, 0), + "name": "scaling", + }, + { + "params": [self._rotation], + "lr": C(training_args.rotation_lr, 0, 0), + "name": "rotation", + }, + {'params': [self._language_feature], 'lr': C(training_args.lang_lr, 0, 0), "name": "language_feature"}, + ] + if self.cfg.pred_normal: + l.append( + { + "params": [self._normal], + "lr": C(training_args.normal_lr, 0, 0), + "name": "normal", + }, + ) + + self.optimize_params = [ + "xyz", + "f_dc", + "f_rest", + "opacity", + "scaling", + "rotation", + "language_feature" + ] + self.optimize_list = l + self.optimizer = torch.optim.Adam(l, lr=0.0, eps=1e-15) + self.lang_optimizer = None + + def merge_optimizer(self, net_optimizer): + l = self.optimize_list + for param in net_optimizer.param_groups: + l.append( + { + "params": param["params"], + "lr": param["lr"], + } + ) + self.optimizer = torch.optim.Adam(l, lr=0.0) + return self.optimizer + + def update_learning_rate(self, iteration): + """Learning rate scheduling per step""" + for param_group in self.optimizer.param_groups: + if not ("name" in param_group): + continue + if param_group["name"] == "xyz": + param_group["lr"] = C( + self.cfg.position_lr, 0, iteration, interpolation="exp" + ) + if param_group["name"] == "scaling": + param_group["lr"] = C( + self.cfg.scaling_lr, 0, iteration, interpolation="exp" + ) + if param_group["name"] == "f_dc": + param_group["lr"] = C( + self.cfg.feature_lr, 0, iteration, interpolation="exp" + ) + if param_group["name"] == "f_rest": + param_group["lr"] = ( + C(self.cfg.feature_lr, 0, iteration, interpolation="exp") / 20.0 + ) + if param_group["name"] == "opacity": + param_group["lr"] = C( + self.cfg.opacity_lr, 0, iteration, interpolation="exp" + ) + if param_group["name"] == "rotation": + param_group["lr"] = C( + self.cfg.rotation_lr, 0, iteration, interpolation="exp" + ) + if param_group["name"] == "normal": + param_group["lr"] = C( + self.cfg.normal_lr, 0, iteration, interpolation="exp" + ) + if self.lang_optimizer is not None: + for param_group in self.lang_optimizer.param_groups: + if not ("name" in param_group): + continue + if param_group["name"] == "language_feature": + param_group["lr"] = C( + self.cfg.lang_lr, 0, iteration, interpolation="exp" + ) + self.color_clip = C(self.cfg.color_clip, 0, iteration) + + def reset_opacity(self): + # opacities_new = inverse_sigmoid(torch.min(self.get_opacity, torch.ones_like(self.get_opacity)*0.01)) + opacities_new = inverse_sigmoid(self.get_opacity * 0.9) + optimizable_tensors = self.replace_tensor_to_optimizer(opacities_new, "opacity") + self._opacity = optimizable_tensors["opacity"] + + def to(self, device="cpu"): + self._xyz = self._xyz.to(device) + self._features_dc = self._features_dc.to(device) + self._features_rest = self._features_rest.to(device) + self._opacity = self._opacity.to(device) + self._scaling = self._scaling.to(device) + self._rotation = self._rotation.to(device) + self._normal = self._normal.to(device) + self._language_feature = self._language_feature.to(device) + + def replace_tensor_to_optimizer(self, tensor, name): + optimizable_tensors = {} + for group in self.optimizer.param_groups: + if ("name" in group) and group["name"] == name: + stored_state = self.optimizer.state.get(group["params"][0], None) + stored_state["exp_avg"] = torch.zeros_like(tensor) + stored_state["exp_avg_sq"] = torch.zeros_like(tensor) + + del self.optimizer.state[group["params"][0]] + group["params"][0] = nn.Parameter(tensor.requires_grad_(True)) + self.optimizer.state[group["params"][0]] = stored_state + + optimizable_tensors[group["name"]] = group["params"][0] + return optimizable_tensors + + def _prune_optimizer(self, mask): + optimizable_tensors = {} + for group in self.optimizer.param_groups: + if ("name" in group) and (group["name"] in self.optimize_params): + stored_state = self.optimizer.state.get(group["params"][0], None) + if stored_state is not None: + stored_state["exp_avg"] = stored_state["exp_avg"][mask] + stored_state["exp_avg_sq"] = stored_state["exp_avg_sq"][mask] + + del self.optimizer.state[group["params"][0]] + group["params"][0] = nn.Parameter( + (group["params"][0][mask].requires_grad_(True)) + ) + self.optimizer.state[group["params"][0]] = stored_state + + optimizable_tensors[group["name"]] = group["params"][0] + else: + group["params"][0] = nn.Parameter( + group["params"][0][mask].requires_grad_(True) + ) + optimizable_tensors[group["name"]] = group["params"][0] + return optimizable_tensors + + def prune_points(self, mask): + valid_points_mask = ~mask + optimizable_tensors = self._prune_optimizer(valid_points_mask) + + self._xyz = optimizable_tensors["xyz"] + self._features_dc = optimizable_tensors["f_dc"] + self._features_rest = optimizable_tensors["f_rest"] + self._opacity = optimizable_tensors["opacity"] + self._scaling = optimizable_tensors["scaling"] + self._rotation = optimizable_tensors["rotation"] + self._language_feature = optimizable_tensors["language_feature"] + if self.cfg.pred_normal: + self._normal = optimizable_tensors["normal"] + + self.xyz_gradient_accum = self.xyz_gradient_accum[valid_points_mask] + + self.denom = self.denom[valid_points_mask] + self.max_radii2D = self.max_radii2D[valid_points_mask] + + def cat_tensors_to_optimizer(self, tensors_dict): + optimizable_tensors = {} + for group in self.optimizer.param_groups: + if ("name" in group) and (group["name"] in self.optimize_params): + extension_tensor = tensors_dict[group["name"]] + stored_state = self.optimizer.state.get(group["params"][0], None) + if stored_state is not None: + stored_state["exp_avg"] = torch.cat( + (stored_state["exp_avg"], torch.zeros_like(extension_tensor)), + dim=0, + ) + stored_state["exp_avg_sq"] = torch.cat( + ( + stored_state["exp_avg_sq"], + torch.zeros_like(extension_tensor), + ), + dim=0, + ) + + del self.optimizer.state[group["params"][0]] + group["params"][0] = nn.Parameter( + torch.cat( + (group["params"][0], extension_tensor), dim=0 + ).requires_grad_(True) + ) + self.optimizer.state[group["params"][0]] = stored_state + + optimizable_tensors[group["name"]] = group["params"][0] + else: + group["params"][0] = nn.Parameter( + torch.cat( + (group["params"][0], extension_tensor), dim=0 + ).requires_grad_(True) + ) + optimizable_tensors[group["name"]] = group["params"][0] + + return optimizable_tensors + + def densification_postfix( + self, + new_xyz, + new_features_dc, + new_features_rest, + new_opacities, + new_scaling, + new_rotation, + new_normal=None, + new_language_feature=None + ): + d = { + "xyz": new_xyz, + "f_dc": new_features_dc, + "f_rest": new_features_rest, + "opacity": new_opacities, + "scaling": new_scaling, + "rotation": new_rotation, + "language_feature": new_language_feature, + } + if self.cfg.pred_normal: + d.update({"normal": new_normal}) + + optimizable_tensors = self.cat_tensors_to_optimizer(d) + self._xyz = optimizable_tensors["xyz"] + self._features_dc = optimizable_tensors["f_dc"] + self._features_rest = optimizable_tensors["f_rest"] + self._opacity = optimizable_tensors["opacity"] + self._scaling = optimizable_tensors["scaling"] + self._rotation = optimizable_tensors["rotation"] + self._language_feature = optimizable_tensors["language_feature"] + if self.cfg.pred_normal: + self._normal = optimizable_tensors["normal"] + + self.xyz_gradient_accum = torch.zeros((self._xyz.shape[0], 1), device="cuda") + self.denom = torch.zeros((self._xyz.shape[0], 1), device="cuda") + self.max_radii2D = torch.zeros((self._xyz.shape[0]), device="cuda") + + def densify_and_split(self, grads, grad_threshold, N=2): + n_init_points = self._xyz.shape[0] + # Extract points that satisfy the gradient condition + padded_grad = torch.zeros((n_init_points), device="cuda") + padded_grad[: grads.shape[0]] = grads.squeeze() + selected_pts_mask = torch.where(padded_grad >= grad_threshold, True, False) + selected_pts_mask = torch.logical_and( + selected_pts_mask, + torch.norm(self.get_scaling, dim=1) > self.cfg.split_thresh, + ) + + # divide N to enhance robustness + stds = self.get_scaling[selected_pts_mask].repeat(N, 1) / N + means = torch.zeros((stds.size(0), 3), device="cuda") + samples = torch.normal(mean=means, std=stds) + rots = build_rotation(self._rotation[selected_pts_mask]).repeat(N, 1, 1) + new_xyz = torch.bmm(rots, samples.unsqueeze(-1)).squeeze(-1) + self._xyz[ + selected_pts_mask + ].repeat(N, 1) + new_scaling = self.scaling_inverse_activation( + self.get_scaling[selected_pts_mask].repeat(N, 1) / (0.8 * N) + ) + new_rotation = self._rotation[selected_pts_mask].repeat(N, 1) + new_features_dc = self._features_dc[selected_pts_mask].repeat(N, 1, 1) + new_features_rest = self._features_rest[selected_pts_mask].repeat(N, 1, 1) + new_opacity = self._opacity[selected_pts_mask].repeat(N, 1) + new_language_feature = self._language_feature[selected_pts_mask].repeat(N,1) + if self.cfg.pred_normal: + new_normal = self._normal[selected_pts_mask].repeat(N, 1) + else: + new_normal = None + + self.densification_postfix( + new_xyz, + new_features_dc, + new_features_rest, + new_opacity, + new_scaling, + new_rotation, + new_normal, + new_language_feature + ) + + prune_filter = torch.cat( + ( + selected_pts_mask, + torch.zeros(N * selected_pts_mask.sum(), device="cuda", dtype=bool), + ) + ) + self.prune_points(prune_filter) + + def densify_and_clone(self, grads, grad_threshold): + # Extract points that satisfy the gradient condition + selected_pts_mask = torch.where( + torch.norm(grads, dim=-1) >= grad_threshold, True, False + ) + selected_pts_mask = torch.logical_and( + selected_pts_mask, + torch.norm(self.get_scaling, dim=1) <= self.cfg.split_thresh, + ) + + new_xyz = self._xyz[selected_pts_mask] + new_features_dc = self._features_dc[selected_pts_mask] + new_features_rest = self._features_rest[selected_pts_mask] + new_opacities = self._opacity[selected_pts_mask] + new_scaling = self._scaling[selected_pts_mask] + new_rotation = self._rotation[selected_pts_mask] + new_language_feature = self._language_feature[selected_pts_mask] + if self.cfg.pred_normal: + new_normal = self._normal[selected_pts_mask] + else: + new_normal = None + + self.densification_postfix( + new_xyz, + new_features_dc, + new_features_rest, + new_opacities, + new_scaling, + new_rotation, + new_normal, + new_language_feature + ) + + def densify(self, max_grad): + grads = self.xyz_gradient_accum / self.denom + grads[grads.isnan()] = 0.0 + + self.densify_and_clone(grads, max_grad) + self.densify_and_split(grads, max_grad) + + def prune(self, min_opacity, max_screen_size): + prune_mask = (self.get_opacity < min_opacity).squeeze() + if self.cfg.prune_big_points: + big_points_vs = self.max_radii2D > (torch.mean(self.max_radii2D) * 3) + prune_mask = torch.logical_or(prune_mask, big_points_vs) + self.prune_points(prune_mask) + + torch.cuda.empty_cache() + + def add_densification_stats(self, viewspace_point_tensor, update_filter): + self.xyz_gradient_accum[update_filter] += torch.norm( + viewspace_point_tensor.grad[update_filter, :2], dim=-1, keepdim=True + ) + self.denom[update_filter] += 1 + + @torch.no_grad() + def update_states( + self, + iteration, + visibility_filter, + radii, + viewspace_point_tensor, + ): + if self._xyz.shape[0] >= self.cfg.max_num + 100: + prune_mask = torch.randperm(self._xyz.shape[0]).to(self._xyz.device) + prune_mask = prune_mask > self.cfg.max_num + self.prune_points(prune_mask) + return + # Keep track of max radii in image-space for pruning + # loop over batch + bs = len(viewspace_point_tensor) + for i in range(bs): + radii_i = radii[i] + visibility_filter_i = visibility_filter[i] + viewspace_point_tensor_i = viewspace_point_tensor[i] + self.max_radii2D = torch.max(self.max_radii2D, radii_i.float()) + + self.add_densification_stats(viewspace_point_tensor_i, visibility_filter_i) + + if ( + iteration > self.cfg.prune_from_iter + and iteration < self.cfg.prune_until_iter + and iteration % self.cfg.prune_interval == 0 + ): + self.prune(self.cfg.min_opac_prune, self.cfg.radii2d_thresh) + if iteration % self.cfg.opacity_reset_interval == 0: + self.reset_opacity() + + if ( + iteration > self.cfg.densify_from_iter + and iteration < self.cfg.densify_until_iter + and iteration % self.cfg.densification_interval == 0 + ): + self.densify(self.cfg.densify_grad_threshold) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_dynamic.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_dynamic.py new file mode 100644 index 0000000000000000000000000000000000000000..e6c5820b28f2676de49150a6d25e4501dfb3542a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_dynamic.py @@ -0,0 +1,77 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# +import math +import os +import random +import sys +from dataclasses import dataclass, field +from datetime import datetime +from typing import NamedTuple + +import numpy as np +import threestudio +import torch +import torch.nn as nn +import torch.nn.functional as F +from plyfile import PlyData, PlyElement +from simple_knn._C import distCUDA2 +from threestudio.models.geometry.base import BaseGeometry +from threestudio.utils.misc import C +from threestudio.utils.typing import * + +from .gaussian_base import GaussianBaseModel + + +@threestudio.register("gaussian-splatting-dynamic") +class GaussianDynamicModel(GaussianBaseModel): + @dataclass + class Config(GaussianBaseModel.Config): + flow: bool = True + num_frames: int = 10 + delta_pos_lr: float = 0.001 + delta_rot_lr: float = 0.0001 + + cfg: Config + + def configure(self) -> None: + super().configure() + self._delta_xyz = torch.empty(0) + self._delta_rot = torch.empty(0) + self.time_index = 0 + + def training_setup(self): + super().training_setup() + l = self.optimize_list + training_args = self.cfg + l.append( + { + "params": [self._delta_xyz], + "lr": C(training_args.delta_pos_lr, 0, 0), + "name": "normal", + }, + ) + l.append( + { + "params": [self._delta_rot], + "lr": C(training_args.delta_rot_lr, 0, 0), + "name": "normal", + }, + ) + + @property + def get_rotation(self): + return self.rotation_activation( + self._rotation + self._delta_rot[self.time_index] + ) + + @property + def get_xyz(self): + return self._xyz + self._delta_xyz[self.time_index] diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_io.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_io.py new file mode 100644 index 0000000000000000000000000000000000000000..60a48464582eac392ad40adc4e247af64a0ccc56 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/gaussian_io.py @@ -0,0 +1,327 @@ +# +# Copyright (C) 2023, Inria +# GRAPHDECO research group, https://team.inria.fr/graphdeco +# All rights reserved. +# +# This software is free for non-commercial, research and evaluation use +# under the terms of the LICENSE.md file. +# +# For inquiries contact george.drettakis@inria.fr +# +import os +import random +import sys +from dataclasses import dataclass, field +from datetime import datetime +from typing import NamedTuple + +import mcubes +import numpy as np +import threestudio +import torch +import torch.nn as nn +import torch.nn.functional as F +from plyfile import PlyData, PlyElement +from simple_knn._C import distCUDA2 +from threestudio.models.geometry.base import BaseGeometry +from threestudio.models.mesh import Mesh +from threestudio.utils.typing import * +from tqdm import tqdm + +from .mesh_utils import * + + +def gaussian_3d_coeff(xyzs, covs): + # xyzs: [N, 3] + # covs: [N, 6] + x, y, z = xyzs[:, 0], xyzs[:, 1], xyzs[:, 2] + a, b, c, d, e, f = ( + covs[:, 0], + covs[:, 1], + covs[:, 2], + covs[:, 3], + covs[:, 4], + covs[:, 5], + ) + + # eps must be small enough !!! + inv_det = 1 / ( + a * d * f + 2 * e * c * b - e**2 * a - c**2 * d - b**2 * f + 1e-24 + ) + inv_a = (d * f - e**2) * inv_det + inv_b = (e * c - b * f) * inv_det + inv_c = (e * b - c * d) * inv_det + inv_d = (a * f - c**2) * inv_det + inv_e = (b * c - e * a) * inv_det + inv_f = (a * d - b**2) * inv_det + + power = ( + -0.5 * (x**2 * inv_a + y**2 * inv_d + z**2 * inv_f) + - x * y * inv_b + - x * z * inv_c + - y * z * inv_e + ) + + power[power > 0] = -1e10 # abnormal values... make weights 0 + + return torch.exp(power) + + +@threestudio.register("gaussian-splatting-io") +class GaussianIO: + def construct_list_of_attributes(self): + l = ["x", "y", "z", "nx", "ny", "nz"] + # All channels except the 3 DC + for i in range(self._features_dc.shape[1] * self._features_dc.shape[2]): + l.append("f_dc_{}".format(i)) + for i in range(self._features_rest.shape[1] * self._features_rest.shape[2]): + l.append("f_rest_{}".format(i)) + l.append("opacity") + for i in range(self._scaling.shape[1]): + l.append("scale_{}".format(i)) + for i in range(self._rotation.shape[1]): + l.append("rot_{}".format(i)) + return l + + def save_ply(self, path): + xyz = self._xyz.detach().cpu().numpy() + normals = np.zeros_like(xyz) + f_dc = ( + self._features_dc.detach() + .transpose(1, 2) + .flatten(start_dim=1) + .contiguous() + .cpu() + .numpy() + ) + f_rest = ( + self._features_rest.detach() + .transpose(1, 2) + .flatten(start_dim=1) + .contiguous() + .cpu() + .numpy() + ) + opacities = self._opacity.detach().cpu().numpy() + scale = self._scaling.detach().cpu().numpy() + rotation = self._rotation.detach().cpu().numpy() + + dtype_full = [ + (attribute, "f4") for attribute in self.construct_list_of_attributes() + ] + + elements = np.empty(xyz.shape[0], dtype=dtype_full) + attributes = np.concatenate( + (xyz, normals, f_dc, f_rest, opacities, scale, rotation), axis=1 + ) + elements[:] = list(map(tuple, attributes)) + el = PlyElement.describe(elements, "vertex") + PlyData([el]).write(path) + + def load_ply(self, path): + plydata = PlyData.read(path) + + xyz = np.stack( + ( + np.asarray(plydata.elements[0]["x"]), + np.asarray(plydata.elements[0]["y"]), + np.asarray(plydata.elements[0]["z"]), + ), + axis=1, + ) + opacities = np.asarray(plydata.elements[0]["opacity"])[..., np.newaxis] + + features_dc = np.zeros((xyz.shape[0], 3, 1)) + features_dc[:, 0, 0] = np.asarray(plydata.elements[0]["f_dc_0"]) + features_dc[:, 1, 0] = np.asarray(plydata.elements[0]["f_dc_1"]) + features_dc[:, 2, 0] = np.asarray(plydata.elements[0]["f_dc_2"]) + + if self.max_sh_degree > 0: + extra_f_names = [ + p.name + for p in plydata.elements[0].properties + if p.name.startswith("f_rest_") + ] + extra_f_names = sorted(extra_f_names, key=lambda x: int(x.split("_")[-1])) + assert len(extra_f_names) == 3 * (self.max_sh_degree + 1) ** 2 - 3 + features_extra = np.zeros((xyz.shape[0], len(extra_f_names))) + for idx, attr_name in enumerate(extra_f_names): + features_extra[:, idx] = np.asarray(plydata.elements[0][attr_name]) + # Reshape (P,F*SH_coeffs) to (P, F, SH_coeffs except DC) + features_extra = features_extra.reshape( + (features_extra.shape[0], 3, (self.max_sh_degree + 1) ** 2 - 1) + ) + + scale_names = [ + p.name + for p in plydata.elements[0].properties + if p.name.startswith("scale_") + ] + scale_names = sorted(scale_names, key=lambda x: int(x.split("_")[-1])) + scales = np.zeros((xyz.shape[0], len(scale_names))) + for idx, attr_name in enumerate(scale_names): + scales[:, idx] = np.asarray(plydata.elements[0][attr_name]) + + rot_names = [ + p.name for p in plydata.elements[0].properties if p.name.startswith("rot") + ] + rot_names = sorted(rot_names, key=lambda x: int(x.split("_")[-1])) + rots = np.zeros((xyz.shape[0], len(rot_names))) + for idx, attr_name in enumerate(rot_names): + rots[:, idx] = np.asarray(plydata.elements[0][attr_name]) + + self._xyz = nn.Parameter( + torch.tensor(xyz, dtype=torch.float, device="cuda").requires_grad_(True) + ) + self._features_dc = nn.Parameter( + torch.tensor(features_dc, dtype=torch.float, device="cuda") + .transpose(1, 2) + .contiguous() + .requires_grad_(True) + ) + if self.max_sh_degree > 0: + self._features_rest = nn.Parameter( + torch.tensor(features_extra, dtype=torch.float, device="cuda") + .transpose(1, 2) + .contiguous() + .requires_grad_(True) + ) + else: + self._features_rest = nn.Parameter( + torch.tensor(features_dc, dtype=torch.float, device="cuda")[:, :, 1:] + .transpose(1, 2) + .contiguous() + .requires_grad_(True) + ) + self._opacity = nn.Parameter( + torch.tensor(opacities, dtype=torch.float, device="cuda").requires_grad_( + True + ) + ) + self._scaling = nn.Parameter( + torch.tensor(scales, dtype=torch.float, device="cuda").requires_grad_(True) + ) + self._rotation = nn.Parameter( + torch.tensor(rots, dtype=torch.float, device="cuda").requires_grad_(True) + ) + self.max_radii2D = torch.zeros((self._xyz.shape[0]), device="cuda") + self.active_sh_degree = self.max_sh_degree + + @torch.no_grad() + def extract_fields(self, resolution=128, num_blocks=16, relax_ratio=1.5): + # resolution: resolution of field + + block_size = 2 / num_blocks + + assert resolution % block_size == 0 + split_size = resolution // num_blocks + + opacities = self.get_opacity + + # pre-filter low opacity gaussians to save computation + mask = (opacities > 0.005).squeeze(1) + + opacities = opacities[mask] + xyzs = self.get_xyz[mask] + stds = self.get_scaling[mask] + + # normalize to ~ [-1, 1] + mn, mx = xyzs.amin(0), xyzs.amax(0) + self.center = (mn + mx) / 2 + self.scale = 1.8 / (mx - mn).amax().item() + + xyzs = (xyzs - self.center) * self.scale + stds = stds * self.scale + + covs = self.covariance_activation(stds, 1, self._rotation[mask]) + + # tile + device = opacities.device + occ = torch.zeros([resolution] * 3, dtype=torch.float32, device=device) + + X = torch.linspace(-1, 1, resolution).split(split_size) + Y = torch.linspace(-1, 1, resolution).split(split_size) + Z = torch.linspace(-1, 1, resolution).split(split_size) + + # loop blocks (assume max size of gaussian is small than relax_ratio * block_size !!!) + for xi, xs in tqdm(enumerate(X)): + for yi, ys in enumerate(Y): + for zi, zs in enumerate(Z): + xx, yy, zz = torch.meshgrid(xs, ys, zs) + # sample points [M, 3] + pts = torch.cat( + [xx.reshape(-1, 1), yy.reshape(-1, 1), zz.reshape(-1, 1)], + dim=-1, + ).to(device) + # in-tile gaussians mask + vmin, vmax = pts.amin(0), pts.amax(0) + vmin -= block_size * relax_ratio + vmax += block_size * relax_ratio + mask = (xyzs < vmax).all(-1) & (xyzs > vmin).all(-1) + # if hit no gaussian, continue to next block + if not mask.any(): + continue + mask_xyzs = xyzs[mask] # [L, 3] + mask_covs = covs[mask] # [L, 6] + mask_opas = opacities[mask].view(1, -1) # [L, 1] --> [1, L] + + # query per point-gaussian pair. + g_pts = pts.unsqueeze(1).repeat( + 1, mask_covs.shape[0], 1 + ) - mask_xyzs.unsqueeze( + 0 + ) # [M, L, 3] + g_covs = mask_covs.unsqueeze(0).repeat( + pts.shape[0], 1, 1 + ) # [M, L, 6] + + # batch on gaussian to avoid OOM + batch_g = 1024 + val = 0 + for start in range(0, g_covs.shape[1], batch_g): + end = min(start + batch_g, g_covs.shape[1]) + w = gaussian_3d_coeff( + g_pts[:, start:end].reshape(-1, 3), + g_covs[:, start:end].reshape(-1, 6), + ).reshape( + pts.shape[0], -1 + ) # [M, l] + val += (mask_opas[:, start:end] * w).sum(-1) + + # kiui.lo(val, mask_opas, w) + + occ[ + xi * split_size : xi * split_size + len(xs), + yi * split_size : yi * split_size + len(ys), + zi * split_size : zi * split_size + len(zs), + ] = val.reshape(len(xs), len(ys), len(zs)) + + # kiui.lo(occ, verbose=1) + + return occ + + def extract_mesh(self, density_thresh=0.8, resolution=128, decimate_target=1e5): + occ = self.extract_fields(resolution).detach().cpu().numpy() + + vertices, triangles = mcubes.marching_cubes(occ, density_thresh) + vertices = vertices / (resolution - 1.0) * 2 - 1 + + # transform back to the original space + vertices = vertices / self.scale + self.center.detach().cpu().numpy() + + vertices, triangles = clean_mesh( + vertices, triangles, remesh=True, remesh_size=0.015 + ) + if decimate_target > 0 and triangles.shape[0] > decimate_target: + vertices, triangles = decimate_mesh(vertices, triangles, decimate_target) + + v = torch.from_numpy(vertices.astype(np.float32)).contiguous().cuda() + f = torch.from_numpy(triangles.astype(np.int32)).contiguous().cuda() + + threestudio.info( + f"marching cubes result: {v.shape} ({v.min().item()}-{v.max().item()}), {f.shape}" + ) + + mesh = Mesh(v_pos=v, t_pos_idx=f) + + return mesh diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/mesh_utils.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/mesh_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c44dcc86044c5b787a3f0956f89523a63a89993d --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/geometry/mesh_utils.py @@ -0,0 +1,150 @@ +import numpy as np +import threestudio + + +def poisson_mesh_reconstruction(points, normals=None): + # points/normals: [N, 3] np.ndarray + + import open3d as o3d + + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(points) + + # outlier removal + pcd, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=10) + + # normals + if normals is None: + pcd.estimate_normals() + else: + pcd.normals = o3d.utility.Vector3dVector(normals[ind]) + + # visualize + o3d.visualization.draw_geometries([pcd], point_show_normal=False) + + mesh, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson( + pcd, depth=9 + ) + vertices_to_remove = densities < np.quantile(densities, 0.1) + mesh.remove_vertices_by_mask(vertices_to_remove) + + # visualize + o3d.visualization.draw_geometries([mesh]) + + vertices = np.asarray(mesh.vertices) + triangles = np.asarray(mesh.triangles) + + print( + f"[INFO] poisson mesh reconstruction: {points.shape} --> {vertices.shape} / {triangles.shape}" + ) + + return vertices, triangles + + +def decimate_mesh( + verts, faces, target, backend="pymeshlab", remesh=False, optimalplacement=True +): + # optimalplacement: default is True, but for flat mesh must turn False to prevent spike artifect. + + _ori_vert_shape = verts.shape + _ori_face_shape = faces.shape + + if backend == "pyfqmr": + import pyfqmr + + solver = pyfqmr.Simplify() + solver.setMesh(verts, faces) + solver.simplify_mesh(target_count=target, preserve_border=False, verbose=False) + verts, faces, normals = solver.getMesh() + else: + import pymeshlab as pml + + m = pml.Mesh(verts, faces) + ms = pml.MeshSet() + ms.add_mesh(m, "mesh") # will copy! + + # filters + # ms.meshing_decimation_clustering(threshold=pml.PercentageValue(1)) + ms.meshing_decimation_quadric_edge_collapse( + targetfacenum=int(target), optimalplacement=optimalplacement + ) + + if remesh: + # ms.apply_coord_taubin_smoothing() + ms.meshing_isotropic_explicit_remeshing( + iterations=3, targetlen=pml.PercentageValue(1) + ) + + # extract mesh + m = ms.current_mesh() + verts = m.vertex_matrix() + faces = m.face_matrix() + + print( + f"[INFO] mesh decimation: {_ori_vert_shape} --> {verts.shape}, {_ori_face_shape} --> {faces.shape}" + ) + + return verts, faces + + +def clean_mesh( + verts, + faces, + v_pct=1, + min_f=64, + min_d=20, + repair=True, + remesh=True, + remesh_size=0.01, +): + # verts: [N, 3] + # faces: [N, 3] + import pymeshlab as pml + + _ori_vert_shape = verts.shape + _ori_face_shape = faces.shape + + m = pml.Mesh(verts, faces) + ms = pml.MeshSet() + ms.add_mesh(m, "mesh") # will copy! + + # filters + ms.meshing_remove_unreferenced_vertices() # verts not refed by any faces + + if v_pct > 0: + ms.meshing_merge_close_vertices( + threshold=pml.PercentageValue(v_pct) + ) # 1/10000 of bounding box diagonal + + ms.meshing_remove_duplicate_faces() # faces defined by the same verts + ms.meshing_remove_null_faces() # faces with area == 0 + + if min_d > 0: + ms.meshing_remove_connected_component_by_diameter( + mincomponentdiag=pml.PercentageValue(min_d) + ) + + if min_f > 0: + ms.meshing_remove_connected_component_by_face_number(mincomponentsize=min_f) + + if repair: + # ms.meshing_remove_t_vertices(method=0, threshold=40, repeat=True) + ms.meshing_repair_non_manifold_edges(method=0) + ms.meshing_repair_non_manifold_vertices(vertdispratio=0) + + if remesh: + # ms.apply_coord_taubin_smoothing() + ms.meshing_isotropic_explicit_remeshing( + iterations=3, targetlen=pml.PureValue(remesh_size) + ) + + # extract mesh + m = ms.current_mesh() + verts = m.vertex_matrix() + faces = m.face_matrix() + + print( + f"[INFO] mesh cleaning: {_ori_vert_shape} --> {verts.shape}, {_ori_face_shape} --> {faces.shape}" + ) + + return verts, faces diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/material/gaussian_material.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/material/gaussian_material.py new file mode 100644 index 0000000000000000000000000000000000000000..273d6cf1522b8110aff9699c32c1d6a2b51d852e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/material/gaussian_material.py @@ -0,0 +1,116 @@ +import random +from dataclasses import dataclass, field + +import threestudio +import torch +import torch.nn as nn +import torch.nn.functional as F +from threestudio.models.materials.base import BaseMaterial +from threestudio.utils.ops import dot, get_activation +from threestudio.utils.typing import * + + +@threestudio.register("gaussian-diffuse-with-point-light-material") +class GaussianDiffuseWithPointLightMaterial(BaseMaterial): + @dataclass + class Config(BaseMaterial.Config): + ambient_light_color: Tuple[float, float, float] = (0.1, 0.1, 0.1) + diffuse_light_color: Tuple[float, float, float] = (0.9, 0.9, 0.9) + ambient_only_steps: int = 1000 + diffuse_prob: float = 0.75 + textureless_prob: float = 0.5 + soft_shading: bool = False + + cfg: Config + + def configure(self) -> None: + self.requires_normal = True + + self.ambient_light_color: Float[Tensor, "3"] + self.register_buffer( + "ambient_light_color", + torch.as_tensor(self.cfg.ambient_light_color, dtype=torch.float32), + ) + self.diffuse_light_color: Float[Tensor, "3"] + self.register_buffer( + "diffuse_light_color", + torch.as_tensor(self.cfg.diffuse_light_color, dtype=torch.float32), + ) + self.ambient_only = False + + def forward( + self, + positions: Float[Tensor, "B ... 3"], + shading_normal: Float[Tensor, "B ... 3"], + light_positions: Float[Tensor, "B ... 3"], + albedo: Float[Tensor, "B ... 3"], + ambient_ratio: Optional[float] = None, + shading: Optional[str] = None, + **kwargs, + ) -> Float[Tensor, "B ... 3"]: + if ambient_ratio is not None: + # if ambient ratio is specified, use it + diffuse_light_color = (1 - ambient_ratio) * torch.ones_like( + self.diffuse_light_color + ) + ambient_light_color = ambient_ratio * torch.ones_like( + self.ambient_light_color + ) + elif self.training and self.cfg.soft_shading: + # otherwise if in training and soft shading is enabled, random a ambient ratio + diffuse_light_color = torch.full_like( + self.diffuse_light_color, random.random() + ) + ambient_light_color = 1.0 - diffuse_light_color + else: + # otherwise use the default fixed values + diffuse_light_color = self.diffuse_light_color + ambient_light_color = self.ambient_light_color + + light_directions: Float[Tensor, "B ... 3"] = F.normalize( + light_positions - positions, dim=-1 + ) + diffuse_light: Float[Tensor, "B ... 3"] = ( + dot(shading_normal, light_directions).clamp(min=0.0) * diffuse_light_color + ) + textureless_color = diffuse_light + ambient_light_color + # clamp albedo to [0, 1] to compute shading + color = albedo.clamp(0.0, 1.0) * textureless_color + + if shading is None: + if self.training: + # adopt the same type of augmentation for the whole batch + if self.ambient_only or random.random() > self.cfg.diffuse_prob: + shading = "albedo" + elif random.random() < self.cfg.textureless_prob: + shading = "textureless" + else: + shading = "diffuse" + else: + if self.ambient_only: + shading = "albedo" + else: + # return shaded color by default in evaluation + shading = "diffuse" + + # multiply by 0 to prevent checking for unused parameters in DDP + if shading == "albedo": + return albedo + textureless_color * 0 + elif shading == "textureless": + return albedo * 0 + textureless_color + elif shading == "diffuse": + return color + else: + raise ValueError(f"Unknown shading type {shading}") + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + if global_step < self.cfg.ambient_only_steps: + self.ambient_only = True + else: + self.ambient_only = False + + def export(self, features: Float[Tensor, "*N Nf"], **kwargs) -> Dict[str, Any]: + albedo = get_activation(self.cfg.albedo_activation)(features[..., :3]).clamp( + 0.0, 1.0 + ) + return {"albedo": albedo} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer.py new file mode 100644 index 0000000000000000000000000000000000000000..90921c52cd416f0e445b7ad1e32cfad3e45fd3a7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer.py @@ -0,0 +1,151 @@ +import math +from dataclasses import dataclass + +import numpy as np +import threestudio +import torch +import torch.nn.functional as F +from diff_gaussian_rasterization import ( + GaussianRasterizationSettings, + GaussianRasterizer, +) +from threestudio.models.background.base import BaseBackground +from threestudio.models.geometry.base import BaseGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.renderers.base import Rasterizer +from threestudio.utils.typing import * + +from .gaussian_batch_renderer import GaussianBatchRenderer + + +@threestudio.register("diff-gaussian-rasterizer") +class DiffGaussian(Rasterizer, GaussianBatchRenderer): + @dataclass + class Config(Rasterizer.Config): + debug: bool = False + invert_bg_prob: float = 1.0 + back_ground_color: Tuple[float, float, float] = (1, 1, 1) + + cfg: Config + + def configure( + self, + geometry: BaseGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + threestudio.info( + "[Note] Gaussian Splatting doesn't support material and background now." + ) + super().configure(geometry, material, background) + self.background_tensor = torch.tensor( + self.cfg.back_ground_color, dtype=torch.float32, device="cuda" + ) + + def forward( + self, + viewpoint_camera, + bg_color: torch.Tensor, + scaling_modifier=1.0, + override_color=None, + **kwargs + ) -> Dict[str, Any]: + """ + Render the scene. + + Background tensor (bg_color) must be on GPU! + """ + + if self.training: + invert_bg_color = np.random.rand() > self.cfg.invert_bg_prob + else: + invert_bg_color = True + + bg_color = bg_color if not invert_bg_color else (1.0 - bg_color) + + pc = self.geometry + # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means + screenspace_points = ( + torch.zeros_like( + pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda" + ) + + 0 + ) + try: + screenspace_points.retain_grad() + except: + pass + + # Set up rasterization configuration + tanfovx = math.tan(viewpoint_camera.FoVx * 0.5) + tanfovy = math.tan(viewpoint_camera.FoVy * 0.5) + + raster_settings = GaussianRasterizationSettings( + image_height=int(viewpoint_camera.image_height), + image_width=int(viewpoint_camera.image_width), + tanfovx=tanfovx, + tanfovy=tanfovy, + bg=bg_color, + scale_modifier=scaling_modifier, + viewmatrix=viewpoint_camera.world_view_transform, + projmatrix=viewpoint_camera.full_proj_transform, + sh_degree=pc.active_sh_degree, + campos=viewpoint_camera.camera_center, + prefiltered=False, + debug=False, + include_feature=True + ) + + rasterizer = GaussianRasterizer(raster_settings=raster_settings) + + means3D = pc.get_xyz + means2D = screenspace_points + opacity = pc.get_opacity + + # If precomputed 3d covariance is provided, use it. If not, then it will be computed from + # scaling / rotation by the rasterizer. + scales = None + rotations = None + cov3D_precomp = None + scales = pc.get_scaling + rotations = pc.get_rotation + + # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors + # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. + shs = None + colors_precomp = None + if override_color is None: + shs = pc.get_features + else: + colors_precomp = override_color + + language_feature_precomp = pc.get_language_feature + language_feature_precomp = language_feature_precomp/ (language_feature_precomp.norm(dim=-1, keepdim=True) + 1e-9) + + # Rasterize visible Gaussians to image, obtain their radii (on screen). + result_list = rasterizer( + means3D=means3D, + means2D=means2D, + shs=shs, + colors_precomp=colors_precomp, + language_feature_precomp = language_feature_precomp, + opacities=opacity, + scales=scales, + rotations=rotations, + cov3D_precomp=cov3D_precomp, + ) + rendered_image, rendered_feature, radii = result_list[0], result_list[1], result_list[2] + + # Retain gradients of the 2D (screen-space) means for batch dim + if self.training: + screenspace_points.retain_grad() + + # Those Gaussians that were frustum culled or had a radius of 0 were not visible. + # They will be excluded from value updates used in the splitting criteria. + return { + "render": rendered_image.clamp(0, 1), + "lang": rendered_feature, + "viewspace_points": screenspace_points, + "visibility_filter": radii > 0, + "radii": radii, + } diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer_advanced.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer_advanced.py new file mode 100644 index 0000000000000000000000000000000000000000..6470ded624306ab00ba8e71971ea2674e9ee19c3 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer_advanced.py @@ -0,0 +1,152 @@ +import math +from dataclasses import dataclass + +import numpy as np +import threestudio +import torch +import torch.nn.functional as F +from diff_gaussian_rasterization import ( + GaussianRasterizationSettings, + GaussianRasterizer, +) +from threestudio.models.background.base import BaseBackground +from threestudio.models.geometry.base import BaseGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.renderers.base import Rasterizer +from threestudio.utils.typing import * + +from .gaussian_batch_renderer import GaussianBatchRenderer + + +@threestudio.register("diff-gaussian-rasterizer-advanced") +class DiffGaussian(Rasterizer, GaussianBatchRenderer): + @dataclass + class Config(Rasterizer.Config): + debug: bool = False + invert_bg_prob: float = 1.0 + back_ground_color: Tuple[float, float, float] = (1, 1, 1) + + cfg: Config + + def configure( + self, + geometry: BaseGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + threestudio.info( + "[Note] Gaussian Splatting doesn't support material and background now." + ) + super().configure(geometry, material, background) + self.background_tensor = torch.tensor( + self.cfg.back_ground_color, dtype=torch.float32, device="cuda" + ) + + def forward( + self, + viewpoint_camera, + bg_color: torch.Tensor, + scaling_modifier=1.0, + override_color=None, + **kwargs + ) -> Dict[str, Any]: + """ + Render the scene. + + Background tensor (bg_color) must be on GPU! + """ + + if self.training: + invert_bg_color = np.random.rand() > self.cfg.invert_bg_prob + else: + invert_bg_color = True + + bg_color = bg_color if not invert_bg_color else (1.0 - bg_color) + + pc = self.geometry + # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means + screenspace_points = ( + torch.zeros_like( + pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda" + ) + + 0 + ) + try: + screenspace_points.retain_grad() + except: + pass + + # Set up rasterization configuration + tanfovx = math.tan(viewpoint_camera.FoVx * 0.5) + tanfovy = math.tan(viewpoint_camera.FoVy * 0.5) + + raster_settings = GaussianRasterizationSettings( + image_height=int(viewpoint_camera.image_height), + image_width=int(viewpoint_camera.image_width), + tanfovx=tanfovx, + tanfovy=tanfovy, + bg=bg_color, + scale_modifier=scaling_modifier, + viewmatrix=viewpoint_camera.world_view_transform, + projmatrix=viewpoint_camera.full_proj_transform, + sh_degree=pc.active_sh_degree, + campos=viewpoint_camera.camera_center, + prefiltered=False, + debug=False, + include_feature=True, + ) + + rasterizer = GaussianRasterizer(raster_settings=raster_settings) + + means3D = pc.get_xyz + means2D = screenspace_points + opacity = pc.get_opacity + + # If precomputed 3d covariance is provided, use it. If not, then it will be computed from + # scaling / rotation by the rasterizer. + scales = None + rotations = None + cov3D_precomp = None + scales = pc.get_scaling + rotations = pc.get_rotation + + # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors + # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. + shs = None + colors_precomp = None + if override_color is None: + shs = pc.get_features + else: + colors_precomp = override_color + + language_feature_precomp = pc.get_language_feature + language_feature_precomp = language_feature_precomp/ (language_feature_precomp.norm(dim=-1, keepdim=True) + 1e-9) + + # Rasterize visible Gaussians to image, obtain their radii (on screen). + rendered_image, rendered_feature, radii, rendered_depth, rendered_alpha = rasterizer( + means3D=means3D, + means2D=means2D, + shs=shs, + colors_precomp=colors_precomp, + language_feature_precomp=language_feature_precomp, + opacities=opacity, + scales=scales, + rotations=rotations, + cov3D_precomp=cov3D_precomp, + ) + + # Retain gradients of the 2D (screen-space) means for batch dim + if self.training: + screenspace_points.retain_grad() + + # Those Gaussians that were frustum culled or had a radius of 0 were not visible. + # They will be excluded from value updates used in the splitting criteria. + print(rendered_feature.mean()) + return { + "render": rendered_image.clamp(0, 1), + "depth": rendered_depth, + "mask": rendered_alpha, + "viewspace_points": screenspace_points, + "visibility_filter": radii > 0, + "radii": radii, + } diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer_background.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer_background.py new file mode 100644 index 0000000000000000000000000000000000000000..0fdb3b77878aba22c921c62f738b935b37660c97 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer_background.py @@ -0,0 +1,145 @@ +import math +from dataclasses import dataclass + +import numpy as np +import threestudio +import torch +import torch.nn.functional as F +from diff_gaussian_rasterization import ( + GaussianRasterizationSettings, + GaussianRasterizer, +) +from threestudio.models.background.base import BaseBackground +from threestudio.models.geometry.base import BaseGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.renderers.base import Rasterizer +from threestudio.utils.typing import * + +from .gaussian_batch_renderer import GaussianBatchRenderer + + +@threestudio.register("diff-gaussian-rasterizer-background") +class DiffGaussian(Rasterizer, GaussianBatchRenderer): + @dataclass + class Config(Rasterizer.Config): + debug: bool = False + back_ground_color: Tuple[float, float, float] = (1, 1, 1) + + cfg: Config + + def configure( + self, + geometry: BaseGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + threestudio.info( + "[Note] diff-gaussian-rasterizer-background doesn't support material." + ) + super().configure(geometry, material, background) + self.background_tensor = torch.tensor( + self.cfg.back_ground_color, dtype=torch.float32, device="cuda" + ) + + def forward( + self, + viewpoint_camera, + bg_color: torch.Tensor, + scaling_modifier=1.0, + override_color=None, + **kwargs + ) -> Dict[str, Any]: + """ + Render the scene. + + Background tensor (bg_color) must be on GPU! + """ + # use neural background + bg_color = bg_color * 0 + + pc = self.geometry + # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means + screenspace_points = ( + torch.zeros_like( + pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda" + ) + + 0 + ) + try: + screenspace_points.retain_grad() + except: + pass + + # Set up rasterization configuration + tanfovx = math.tan(viewpoint_camera.FoVx * 0.5) + tanfovy = math.tan(viewpoint_camera.FoVy * 0.5) + + raster_settings = GaussianRasterizationSettings( + image_height=int(viewpoint_camera.image_height), + image_width=int(viewpoint_camera.image_width), + tanfovx=tanfovx, + tanfovy=tanfovy, + bg=bg_color, + scale_modifier=scaling_modifier, + viewmatrix=viewpoint_camera.world_view_transform, + projmatrix=viewpoint_camera.full_proj_transform, + sh_degree=pc.active_sh_degree, + campos=viewpoint_camera.camera_center, + prefiltered=False, + debug=False, + ) + + rasterizer = GaussianRasterizer(raster_settings=raster_settings) + + means3D = pc.get_xyz + means2D = screenspace_points + opacity = pc.get_opacity + + # If precomputed 3d covariance is provided, use it. If not, then it will be computed from + # scaling / rotation by the rasterizer. + scales = None + rotations = None + cov3D_precomp = None + scales = pc.get_scaling + rotations = pc.get_rotation + + # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors + # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. + shs = None + colors_precomp = None + if override_color is None: + shs = pc.get_features + else: + colors_precomp = override_color + + # Rasterize visible Gaussians to image, obtain their radii (on screen). + rays_d = kwargs["rays_d"][kwargs["batch_idx"]] + comp_rgb_bg = self.background(dirs=rays_d.unsqueeze(0)) + + rendered_image, radii, rendered_depth, rendered_alpha = rasterizer( + means3D=means3D, + means2D=means2D, + shs=shs, + colors_precomp=colors_precomp, + opacities=opacity, + scales=scales, + rotations=rotations, + cov3D_precomp=cov3D_precomp, + ) + _, H, W = rendered_image.shape + rendered_image = rendered_image + (1 - rendered_alpha) * comp_rgb_bg.reshape( + H, W, 3 + ).permute(2, 0, 1) + + # Retain gradients of the 2D (screen-space) means for batch dim + if self.training: + screenspace_points.retain_grad() + + # Those Gaussians that were frustum culled or had a radius of 0 were not visible. + # They will be excluded from value updates used in the splitting criteria. + return { + "render": rendered_image.clamp(0, 1), + "viewspace_points": screenspace_points, + "visibility_filter": radii > 0, + "radii": radii, + } diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer_shading.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer_shading.py new file mode 100644 index 0000000000000000000000000000000000000000..b810f7f4965d69082c80e584576f8822a32ae25d --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/diff_gaussian_rasterizer_shading.py @@ -0,0 +1,226 @@ +import math +from dataclasses import dataclass + +import numpy as np +import threestudio +import torch +import torch.nn.functional as F +from diff_gaussian_rasterization import ( + GaussianRasterizationSettings, + GaussianRasterizer, +) +from threestudio.models.background.base import BaseBackground +from threestudio.models.geometry.base import BaseGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.renderers.base import Rasterizer +from threestudio.utils.typing import * + +from ..material.gaussian_material import GaussianDiffuseWithPointLightMaterial +from .gaussian_batch_renderer import GaussianBatchRenderer + + +class Depth2Normal(torch.nn.Module): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.delzdelxkernel = torch.tensor( + [ + [0.00000, 0.00000, 0.00000], + [-1.00000, 0.00000, 1.00000], + [0.00000, 0.00000, 0.00000], + ] + ) + self.delzdelykernel = torch.tensor( + [ + [0.00000, -1.00000, 0.00000], + [0.00000, 0.00000, 0.00000], + [0.0000, 1.00000, 0.00000], + ] + ) + + def forward(self, x): + B, C, H, W = x.shape + delzdelxkernel = self.delzdelxkernel.view(1, 1, 3, 3).to(x.device) + delzdelx = F.conv2d( + x.reshape(B * C, 1, H, W), delzdelxkernel, padding=1 + ).reshape(B, C, H, W) + delzdelykernel = self.delzdelykernel.view(1, 1, 3, 3).to(x.device) + delzdely = F.conv2d( + x.reshape(B * C, 1, H, W), delzdelykernel, padding=1 + ).reshape(B, C, H, W) + normal = -torch.cross(delzdelx, delzdely, dim=1) + return normal + + +@threestudio.register("diff-gaussian-rasterizer-shading") +class DiffGaussian(Rasterizer, GaussianBatchRenderer): + @dataclass + class Config(Rasterizer.Config): + debug: bool = False + back_ground_color: Tuple[float, float, float] = (1, 1, 1) + + cfg: Config + + def configure( + self, + geometry: BaseGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + if not isinstance(material, GaussianDiffuseWithPointLightMaterial): + raise NotImplementedError( + "diff-gaussian-rasterizer-shading only support Gaussian material." + ) + super().configure(geometry, material, background) + self.normal_module = Depth2Normal() + self.background_tensor = torch.tensor( + self.cfg.back_ground_color, dtype=torch.float32, device="cuda" + ) + + def forward( + self, + viewpoint_camera, + bg_color: torch.Tensor, + scaling_modifier=1.0, + override_color=None, + **kwargs + ) -> Dict[str, Any]: + """ + Render the scene. + + Background tensor (bg_color) must be on GPU! + """ + # use neural background + bg_color = bg_color * 0 + + pc = self.geometry + # Create zero tensor. We will use it to make pytorch return gradients of the 2D (screen-space) means + screenspace_points = ( + torch.zeros_like( + pc.get_xyz, dtype=pc.get_xyz.dtype, requires_grad=True, device="cuda" + ) + + 0 + ) + try: + screenspace_points.retain_grad() + except: + pass + + # Set up rasterization configuration + tanfovx = math.tan(viewpoint_camera.FoVx * 0.5) + tanfovy = math.tan(viewpoint_camera.FoVy * 0.5) + + raster_settings = GaussianRasterizationSettings( + image_height=int(viewpoint_camera.image_height), + image_width=int(viewpoint_camera.image_width), + tanfovx=tanfovx, + tanfovy=tanfovy, + bg=bg_color, + scale_modifier=scaling_modifier, + viewmatrix=viewpoint_camera.world_view_transform, + projmatrix=viewpoint_camera.full_proj_transform, + sh_degree=pc.active_sh_degree, + campos=viewpoint_camera.camera_center, + prefiltered=False, + debug=False, + ) + + rasterizer = GaussianRasterizer(raster_settings=raster_settings) + + means3D = pc.get_xyz + means2D = screenspace_points + opacity = pc.get_opacity + + # If precomputed 3d covariance is provided, use it. If not, then it will be computed from + # scaling / rotation by the rasterizer. + scales = None + rotations = None + cov3D_precomp = None + scales = pc.get_scaling + rotations = pc.get_rotation + + # If precomputed colors are provided, use them. Otherwise, if it is desired to precompute colors + # from SHs in Python, do it. If not, then SH -> RGB conversion will be done by rasterizer. + shs = None + colors_precomp = None + if override_color is None: + shs = pc.get_features + else: + colors_precomp = override_color + + # Rasterize visible Gaussians to image, obtain their radii (on screen). + batch_idx = kwargs["batch_idx"] + rays_d = kwargs["rays_d"][batch_idx] + rays_o = kwargs["rays_o"][batch_idx] + # rays_d_flatten: Float[Tensor, "Nr 3"] = rays_d.unsqueeze(0) + + comp_rgb_bg = self.background(dirs=rays_d.unsqueeze(0)) + + rendered_image, radii, rendered_depth, rendered_alpha = rasterizer( + means3D=means3D, + means2D=means2D, + shs=shs, + colors_precomp=colors_precomp, + opacities=opacity, + scales=scales, + rotations=rotations, + cov3D_precomp=cov3D_precomp, + ) + _, H, W = rendered_image.shape + + xyz_map = rays_o + rendered_depth.permute(1, 2, 0) * rays_d + normal_map = self.normal_module(xyz_map.permute(2, 0, 1).unsqueeze(0))[0] + normal_map = F.normalize(normal_map, dim=0) + if pc.cfg.pred_normal: + pred_normal_map, _, _, _ = rasterizer( + means3D=means3D, + means2D=torch.zeros_like(means2D), + shs=pc.get_normal.unsqueeze(1), + colors_precomp=None, + opacities=opacity, + scales=scales, + rotations=rotations, + cov3D_precomp=cov3D_precomp, + ) + else: + pred_normal_map = None + + light_positions = kwargs["light_positions"][batch_idx, None, None, :].expand( + H, W, -1 + ) + + if pred_normal_map is not None: + shading_normal = pred_normal_map.permute(1, 2, 0).detach() * 2 - 1 + shading_normal = F.normalize(shading_normal, dim=2) + else: + shading_normal = normal_map.permute(1, 2, 0) + rgb_fg = self.material( + positions=xyz_map, + shading_normal=shading_normal, + albedo=(rendered_image / (rendered_alpha + 1e-6)).permute(1, 2, 0), + light_positions=light_positions, + ).permute(2, 0, 1) + rendered_image = rgb_fg * rendered_alpha + ( + 1 - rendered_alpha + ) * comp_rgb_bg.reshape(H, W, 3).permute(2, 0, 1) + normal_map = normal_map * 0.5 * rendered_alpha + 0.5 + mask = rendered_alpha > 0.99 + normal_mask = mask.repeat(3, 1, 1) + normal_map[~normal_mask] = normal_map[~normal_mask].detach() + rendered_depth[~mask] = rendered_depth[~mask].detach() + + # Retain gradients of the 2D (screen-space) means for batch dim + if self.training: + screenspace_points.retain_grad() + + # Those Gaussians that were frustum culled or had a radius of 0 were not visible. + # They will be excluded from value updates used in the splitting criteria. + return { + "render": rendered_image.clamp(0, 1), + "normal": normal_map, + "pred_normal": pred_normal_map, + "mask": rendered_alpha, + "depth": rendered_depth, + "viewspace_points": screenspace_points, + "visibility_filter": radii > 0, + "radii": radii, + } diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/gaussian_batch_renderer.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/gaussian_batch_renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..2503c6ab9cc6d339a20ddf7c42488ca3505531ce --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/renderer/gaussian_batch_renderer.py @@ -0,0 +1,92 @@ +import torch +from threestudio.utils.ops import get_cam_info_gaussian +from torch.cuda.amp import autocast + +from ..geometry.gaussian_base import BasicPointCloud, Camera + + +class GaussianBatchRenderer: + def batch_forward(self, batch): + bs = batch["c2w"].shape[0] + renders = [] + viewspace_points = [] + visibility_filters = [] + radiis = [] + normals = [] + pred_normals = [] + depths = [] + masks = [] + langs = [] + for batch_idx in range(bs): + batch["batch_idx"] = batch_idx + fovy = batch["fovy"][batch_idx] + w2c, proj, cam_p, cam_proj = get_cam_info_gaussian( + c2w=batch["c2w"][batch_idx], fovx=fovy, fovy=fovy, znear=0.1, zfar=100 + ) + + viewpoint_cam = Camera( + FoVx=fovy, + FoVy=fovy, + image_width=batch["width"], + image_height=batch["height"], + world_view_transform=w2c, + full_proj_transform=proj, + camera_center=cam_p, + ) + + with autocast(enabled=False): + render_pkg = self.forward( + viewpoint_cam, self.background_tensor, **batch + ) + renders.append(render_pkg["render"]) + viewspace_points.append(render_pkg["viewspace_points"]) + visibility_filters.append(render_pkg["visibility_filter"]) + radiis.append(render_pkg["radii"]) + if render_pkg.__contains__("normal"): + normals.append(render_pkg["normal"]) + if ( + render_pkg.__contains__("pred_normal") + and render_pkg["pred_normal"] is not None + ): + pred_normals.append(render_pkg["pred_normal"]) + if render_pkg.__contains__("depth"): + depths.append(render_pkg["depth"]) + if render_pkg.__contains__("mask"): + masks.append(render_pkg["mask"]) + if render_pkg.__contains__("lang"): + langs.append(render_pkg["lang"]) + + outputs = { + "comp_rgb": torch.stack(renders, dim=0).permute(0, 2, 3, 1), + "lang": torch.stack(langs, dim=0).permute(0, 2, 3, 1), + "viewspace_points": viewspace_points, + "visibility_filter": visibility_filters, + "radii": radiis, + } + if len(normals) > 0: + outputs.update( + { + "comp_normal": torch.stack(normals, dim=0).permute(0, 2, 3, 1), + } + ) + if len(pred_normals) > 0: + outputs.update( + { + "comp_pred_normal": torch.stack(pred_normals, dim=0).permute( + 0, 2, 3, 1 + ), + } + ) + if len(depths) > 0: + outputs.update( + { + "comp_depth": torch.stack(depths, dim=0).permute(0, 2, 3, 1), + } + ) + if len(masks) > 0: + outputs.update( + { + "comp_mask": torch.stack(masks, dim=0).permute(0, 2, 3, 1), + } + ) + return outputs diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/gaussian_mvdream.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/gaussian_mvdream.py new file mode 100644 index 0000000000000000000000000000000000000000..e82cdd04471f27a53e7c018200091ab66e7e528b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/gaussian_mvdream.py @@ -0,0 +1,249 @@ +import os +from dataclasses import dataclass, field + +import numpy as np +import threestudio +import torch +from threestudio.systems.base import BaseLift3DSystem +from threestudio.systems.utils import parse_optimizer, parse_scheduler +from threestudio.utils.loss import tv_loss +from threestudio.utils.typing import * + +from ..geometry.gaussian_base import BasicPointCloud + + +@threestudio.register("gaussian-splatting-mvdream-system") +class MVDreamSystem(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + visualize_samples: bool = False + + cfg: Config + + def configure(self) -> None: + # set up geometry, material, background, renderer + super().configure() + self.automatic_optimization = False + + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.prompt_utils = self.prompt_processor() + + def configure_optimizers(self): + optim = self.geometry.optimizer + if hasattr(self, "merged_optimizer"): + return [optim] + if hasattr(self.cfg.optimizer, "name"): + net_optim = parse_optimizer(self.cfg.optimizer, self) + optim = self.geometry.merge_optimizer(net_optim) + self.merged_optimizer = True + else: + self.merged_optimizer = False + return [optim] + + def on_load_checkpoint(self, checkpoint): + num_pts = checkpoint["state_dict"]["geometry._xyz"].shape[0] + pcd = BasicPointCloud( + points=np.zeros((num_pts, 3)), + colors=np.zeros((num_pts, 3)), + normals=np.zeros((num_pts, 3)), + ) + self.geometry.create_from_pcd(pcd, 10) + self.geometry.training_setup() + return + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + self.geometry.update_learning_rate(self.global_step) + outputs = self.renderer.batch_forward(batch) + return outputs + + def training_step(self, batch, batch_idx): + opt = self.optimizers() + out = self(batch) + + visibility_filter = out["visibility_filter"] + radii = out["radii"] + guidance_inp = out["comp_rgb"] + viewspace_point_tensor = out["viewspace_points"] + guidance_out = self.guidance( + guidance_inp, self.prompt_utils, **batch, rgb_as_latents=False + ) + + loss_sds = 0.0 + loss = 0.0 + + self.log( + "gauss_num", + int(self.geometry.get_xyz.shape[0]), + on_step=True, + on_epoch=True, + prog_bar=True, + logger=True, + ) + + for name, value in guidance_out.items(): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss_sds += value * self.C( + self.cfg.loss[name.replace("loss_", "lambda_")] + ) + + xyz_mean = None + if self.cfg.loss["lambda_position"] > 0.0: + xyz_mean = self.geometry.get_xyz.norm(dim=-1) + loss_position = xyz_mean.mean() + self.log(f"train/loss_position", loss_position) + loss += self.C(self.cfg.loss["lambda_position"]) * loss_position + + if self.cfg.loss["lambda_opacity"] > 0.0: + scaling = self.geometry.get_scaling.norm(dim=-1) + loss_opacity = ( + scaling.detach().unsqueeze(-1) * self.geometry.get_opacity + ).sum() + self.log(f"train/loss_opacity", loss_opacity) + loss += self.C(self.cfg.loss["lambda_opacity"]) * loss_opacity + + if self.cfg.loss["lambda_sparsity"] > 0.0: + loss_sparsity = (out["comp_mask"] ** 2 + 0.01).sqrt().mean() + self.log("train/loss_sparsity", loss_sparsity) + loss += loss_sparsity * self.C(self.cfg.loss.lambda_sparsity) + + if self.cfg.loss["lambda_scales"] > 0.0: + scale_sum = torch.sum(self.geometry.get_scaling) + self.log(f"train/scales", scale_sum) + loss += self.C(self.cfg.loss["lambda_scales"]) * scale_sum + + if self.cfg.loss["lambda_tv_loss"] > 0.0: + loss_tv = self.C(self.cfg.loss["lambda_tv_loss"]) * tv_loss( + out["comp_rgb"].permute(0, 3, 1, 2) + ) + self.log(f"train/loss_tv", loss_tv) + loss += loss_tv + + if ( + out.__contains__("comp_depth") + and self.cfg.loss["lambda_depth_tv_loss"] > 0.0 + ): + loss_depth_tv = self.C(self.cfg.loss["lambda_depth_tv_loss"]) * ( + tv_loss(out["comp_depth"].permute(0, 3, 1, 2)) + ) + self.log(f"train/loss_depth_tv", loss_depth_tv) + loss += loss_depth_tv + + if out.__contains__("comp_pred_normal"): + loss_pred_normal = torch.nn.functional.mse_loss( + out["comp_pred_normal"], out["comp_normal"].detach() + ) + loss += loss_pred_normal + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + loss_sds.backward(retain_graph=True) + iteration = self.global_step + self.geometry.update_states( + iteration, + visibility_filter, + radii, + viewspace_point_tensor, + ) + if loss > 0: + loss.backward() + opt.step() + opt.zero_grad(set_to_none=True) + + return {"loss": loss_sds} + + def validation_step(self, batch, batch_idx): + out = self(batch) + # import pdb; pdb.set_trace() + self.save_image_grid( + f"it{self.global_step}-{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + ( + [ + { + "type": "rgb", + "img": out["comp_pred_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_pred_normal" in out + else [] + ), + name="validation_step", + step=self.global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + ( + [ + { + "type": "rgb", + "img": out["comp_pred_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_pred_normal" in out + else [] + ), + name="test_step", + step=self.global_step, + ) + if batch["index"][0] == 0: + save_path = self.get_save_path("point_cloud.ply") + self.geometry.save_ply(save_path) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/gaussian_splatting.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/gaussian_splatting.py new file mode 100644 index 0000000000000000000000000000000000000000..337a97dda8ad06a3799d4e56561997a272f94580 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/gaussian_splatting.py @@ -0,0 +1,223 @@ +import math +from dataclasses import dataclass + +import numpy as np +import threestudio +import torch +from threestudio.systems.base import BaseLift3DSystem +from threestudio.systems.utils import parse_optimizer, parse_scheduler +from threestudio.utils.loss import tv_loss +from threestudio.utils.ops import get_cam_info_gaussian +from threestudio.utils.typing import * +from torch.cuda.amp import autocast + +from ..geometry.gaussian_base import BasicPointCloud, Camera + + +@threestudio.register("gaussian-splatting-system") +class GaussianSplatting(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + visualize_samples: bool = False + + cfg: Config + + def configure(self) -> None: + # set up geometry, material, background, renderer + super().configure() + self.automatic_optimization = False + + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.prompt_utils = self.prompt_processor() + + def configure_optimizers(self): + optim = self.geometry.optimizer + if hasattr(self, "merged_optimizer"): + return [optim] + if hasattr(self.cfg.optimizer, "name"): + net_optim = parse_optimizer(self.cfg.optimizer, self) + optim = self.geometry.merge_optimizer(net_optim) + self.merged_optimizer = True + else: + self.merged_optimizer = False + return [optim] + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + self.geometry.update_learning_rate(self.global_step) + outputs = self.renderer.batch_forward(batch) + return outputs + + def on_fit_start(self) -> None: + super().on_fit_start() + + def training_step(self, batch, batch_idx): + opt = self.optimizers() + out = self(batch) + + visibility_filter = out["visibility_filter"] + radii = out["radii"] + guidance_inp = out["comp_rgb"] + # import pdb; pdb.set_trace() + viewspace_point_tensor = out["viewspace_points"] + guidance_out = self.guidance( + guidance_inp, self.prompt_utils, **batch, rgb_as_latents=False + ) + + loss_sds = 0.0 + loss = 0.0 + + self.log( + "gauss_num", + int(self.geometry.get_xyz.shape[0]), + on_step=True, + on_epoch=True, + prog_bar=True, + logger=True, + ) + + for name, value in guidance_out.items(): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss_sds += value * self.C( + self.cfg.loss[name.replace("loss_", "lambda_")] + ) + + xyz_mean = None + if self.cfg.loss["lambda_position"] > 0.0: + xyz_mean = self.geometry.get_xyz.norm(dim=-1) + loss_position = xyz_mean.mean() + self.log(f"train/loss_position", loss_position) + loss += self.C(self.cfg.loss["lambda_position"]) * loss_position + + if self.cfg.loss["lambda_opacity"] > 0.0: + scaling = self.geometry.get_scaling.norm(dim=-1) + loss_opacity = ( + scaling.detach().unsqueeze(-1) * self.geometry.get_opacity + ).sum() + self.log(f"train/loss_opacity", loss_opacity) + loss += self.C(self.cfg.loss["lambda_opacity"]) * loss_opacity + + if self.cfg.loss["lambda_scales"] > 0.0: + scale_sum = torch.sum(self.geometry.get_scaling) + self.log(f"train/scales", scale_sum) + loss += self.C(self.cfg.loss["lambda_scales"]) * scale_sum + + if self.cfg.loss["lambda_tv_loss"] > 0.0: + loss_tv = self.C(self.cfg.loss["lambda_tv_loss"]) * tv_loss( + out["comp_rgb"].permute(0, 3, 1, 2) + ) + self.log(f"train/loss_tv", loss_tv) + loss += loss_tv + + if ( + out.__contains__("comp_depth") + and self.cfg.loss["lambda_depth_tv_loss"] > 0.0 + ): + loss_depth_tv = self.C(self.cfg.loss["lambda_depth_tv_loss"]) * ( + tv_loss(out["comp_normal"].permute(0, 3, 1, 2)) + + tv_loss(out["comp_depth"].permute(0, 3, 1, 2)) + ) + self.log(f"train/loss_depth_tv", loss_depth_tv) + loss += loss_depth_tv + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + loss_sds.backward(retain_graph=True) + iteration = self.global_step + self.geometry.update_states( + iteration, + visibility_filter, + radii, + viewspace_point_tensor, + ) + if loss > 0: + loss.backward() + opt.step() + opt.zero_grad(set_to_none=True) + + return {"loss": loss_sds} + + def validation_step(self, batch, batch_idx): + out = self(batch) + # import pdb; pdb.set_trace() + self.save_image_grid( + f"it{self.global_step}-{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ), + name="validation_step", + step=self.global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ), + name="test_step", + step=self.global_step, + ) + if batch["index"][0] == 0: + save_path = self.get_save_path("point_cloud.ply") + self.geometry.save_ply(save_path) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.global_step}-test", + f"it{self.global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.global_step, + ) + + def on_load_checkpoint(self, ckpt_dict) -> None: + num_pts = ckpt_dict["state_dict"]["geometry._xyz"].shape[0] + pcd = BasicPointCloud( + points=np.zeros((num_pts, 3)), + colors=np.zeros((num_pts, 3)), + normals=np.zeros((num_pts, 3)), + ) + self.geometry.create_from_pcd(pcd, 10) + self.geometry.training_setup() + super().on_load_checkpoint(ckpt_dict) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/gaussian_zero123.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/gaussian_zero123.py new file mode 100644 index 0000000000000000000000000000000000000000..0a59f1541e773fe75b34fb6f268ac8632811b3ad --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/gaussian_zero123.py @@ -0,0 +1,339 @@ +import os +import random +from dataclasses import dataclass, field + +import numpy as np +import threestudio +import torch +import torch.nn.functional as F +from threestudio.systems.base import BaseLift3DSystem +from threestudio.systems.utils import parse_optimizer, parse_scheduler +from threestudio.utils.loss import tv_loss +from threestudio.utils.ops import get_cam_info_gaussian +from threestudio.utils.typing import * +from torch.cuda.amp import autocast +from torchmetrics import PearsonCorrCoef + +from ..geometry.gaussian_base import BasicPointCloud, Camera + + +@threestudio.register("gaussian-splatting-zero123-system") +class Zero123(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + freq: dict = field(default_factory=dict) + refinement: bool = False + ambient_ratio_min: float = 0.5 + back_ground_color: Tuple[float, float, float] = (1, 1, 1) + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + self.automatic_optimization = False + + def configure_optimizers(self): + optim = self.geometry.optimizer + if hasattr(self, "merged_optimizer"): + return [optim] + if hasattr(self.cfg.optimizer, "name"): + net_optim = parse_optimizer(self.cfg.optimizer, self) + optim = self.geometry.merge_optimizer(net_optim) + self.merged_optimizer = True + else: + self.merged_optimizer = False + return [optim] + + def on_load_checkpoint(self, checkpoint): + num_pts = checkpoint["state_dict"]["geometry._xyz"].shape[0] + pcd = BasicPointCloud( + points=np.zeros((num_pts, 3)), + colors=np.zeros((num_pts, 3)), + normals=np.zeros((num_pts, 3)), + ) + self.geometry.create_from_pcd(pcd, 10) + self.geometry.training_setup() + return + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + self.geometry.update_learning_rate(self.global_step) + outputs = self.renderer.batch_forward(batch) + return outputs + + def on_fit_start(self) -> None: + super().on_fit_start() + # no prompt processor + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + # visualize all training images + all_images = self.trainer.datamodule.train_dataloader().dataset.get_all_images() + self.save_image_grid( + "all_training_images.png", + [ + {"type": "rgb", "img": image, "kwargs": {"data_format": "HWC"}} + for image in all_images + ], + name="on_fit_start", + step=self.true_global_step, + ) + + self.pearson = PearsonCorrCoef().to(self.device) + + def training_substep(self, batch, batch_idx, guidance: str): + """ + Args: + guidance: one of "ref" (reference image supervision), "zero123" + """ + if guidance == "ref": + ambient_ratio = 1.0 + shading = "diffuse" + batch["shading"] = shading + elif guidance == "zero123": + batch = batch["random_camera"] + ambient_ratio = ( + self.cfg.ambient_ratio_min + + (1 - self.cfg.ambient_ratio_min) * random.random() + ) + + batch["ambient_ratio"] = ambient_ratio + + out = self(batch) + loss_prefix = f"loss_{guidance}_" + + loss_terms = {} + + def set_loss(name, value): + loss_terms[f"{loss_prefix}{name}"] = value + + guidance_eval = ( + guidance == "zero123" + and self.cfg.freq.guidance_eval > 0 + and self.true_global_step % self.cfg.freq.guidance_eval == 0 + ) + + if guidance == "ref": + gt_mask = batch["mask"] + gt_rgb = batch["rgb"] + + # color loss + gt_rgb = gt_rgb * gt_mask.float() + set_loss("rgb", F.mse_loss(gt_rgb, out["comp_rgb"] * gt_mask.float())) + + # mask loss + set_loss("mask", F.mse_loss(gt_mask.float(), out["comp_mask"])) + + # depth loss + if self.C(self.cfg.loss.lambda_depth) > 0: + valid_gt_depth = batch["ref_depth"][gt_mask.squeeze(-1)].unsqueeze(1) + valid_pred_depth = out["comp_depth"][gt_mask].unsqueeze(1) + with torch.no_grad(): + A = torch.cat( + [valid_gt_depth, torch.ones_like(valid_gt_depth)], dim=-1 + ) # [B, 2] + X = torch.linalg.lstsq(A, valid_pred_depth).solution # [2, 1] + valid_gt_depth = A @ X # [B, 1] + set_loss("depth", F.mse_loss(valid_gt_depth, valid_pred_depth)) + + # relative depth loss + if self.C(self.cfg.loss.lambda_depth_rel) > 0: + valid_gt_depth = batch["ref_depth"][gt_mask.squeeze(-1)] # [B,] + valid_pred_depth = out["comp_depth"][gt_mask] # [B,] + set_loss( + "depth_rel", 1 - self.pearson(valid_pred_depth, valid_gt_depth) + ) + + # normal loss + if self.C(self.cfg.loss.lambda_normal) > 0: + valid_gt_normal = ( + 1 - 2 * batch["ref_normal"][gt_mask.squeeze(-1)] + ) # [B, 3] + valid_pred_normal = ( + 2 * out["comp_normal"][gt_mask.squeeze(-1)] - 1 + ) # [B, 3] + set_loss( + "normal", + 1 - F.cosine_similarity(valid_pred_normal, valid_gt_normal).mean(), + ) + elif guidance == "zero123": + # zero123 + guidance_out = self.guidance( + out["comp_rgb"], + **batch, + rgb_as_latents=False, + guidance_eval=guidance_eval, + ) + # claforte: TODO: rename the loss_terms keys + set_loss("sds", guidance_out["loss_sds"]) + + if self.C(self.cfg.loss.lambda_normal_smooth) > 0: + if "comp_normal" not in out: + raise ValueError( + "comp_normal is required for 2D normal smooth loss, no comp_normal is found in the output." + ) + normal = out["comp_normal"] + set_loss( + "normal_smooth", + (normal[:, 1:, :, :] - normal[:, :-1, :, :]).square().mean() + + (normal[:, :, 1:, :] - normal[:, :, :-1, :]).square().mean(), + ) + + loss = 0.0 + for name, value in loss_terms.items(): + self.log(f"train/{name}", value) + if name.startswith(loss_prefix): + loss_weighted = value * self.C( + self.cfg.loss[name.replace(loss_prefix, "lambda_")] + ) + self.log(f"train/{name}_w", loss_weighted) + loss += loss_weighted + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + self.log(f"train/loss_{guidance}", loss) + + out.update({"loss": loss}) + return out + + def training_step(self, batch, batch_idx): + opt = self.optimizers() + + if self.cfg.freq.get("ref_or_zero123", "accumulate") == "accumulate": + do_ref = True + do_zero123 = True + elif self.cfg.freq.get("ref_or_zero123", "accumulate") == "alternate": + do_ref = ( + self.true_global_step < self.cfg.freq.ref_only_steps + or self.true_global_step % self.cfg.freq.n_ref == 0 + ) + do_zero123 = not do_ref + + total_loss = 0.0 + if do_zero123: + out = self.training_substep(batch, batch_idx, guidance="zero123") + total_loss += out["loss"] + + if do_ref: + out = self.training_substep(batch, batch_idx, guidance="ref") + total_loss += out["loss"] + + self.log("train/loss", total_loss, prog_bar=True) + + visibility_filter = out["visibility_filter"] + radii = out["radii"] + guidance_inp = out["comp_rgb"] + viewspace_point_tensor = out["viewspace_points"] + + total_loss.backward() + iteration = self.global_step + self.geometry.update_states( + iteration, + visibility_filter, + radii, + viewspace_point_tensor, + ) + opt.step() + opt.zero_grad(set_to_none=True) + + return {"loss": total_loss} + + def validation_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-val/{batch['index'][0]}.png", + ( + [ + { + "type": "rgb", + "img": batch["rgb"][0], + "kwargs": {"data_format": "HWC"}, + } + ] + if "rgb" in batch + else [] + ) + + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ), + # claforte: TODO: don't hardcode the frame numbers to record... read them from cfg instead. + name=f"validation_step_batchidx_{batch_idx}" + if batch_idx in [0, 7, 15, 23, 29] + else None, + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + filestem = f"it{self.true_global_step}-val" + self.save_img_sequence( + filestem, + filestem, + "(\d+)\.png", + save_format="mp4", + fps=30, + name="validation_epoch_end", + step=self.true_global_step, + ) + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + ( + [ + { + "type": "rgb", + "img": batch["rgb"][0], + "kwargs": {"data_format": "HWC"}, + } + ] + if "rgb" in batch + else [] + ) + + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ), + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/scene_lang.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/scene_lang.py new file mode 100644 index 0000000000000000000000000000000000000000..7ade6801ed66d5986a09989c3f2eae7294b58972 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/system/scene_lang.py @@ -0,0 +1,528 @@ +import math +from dataclasses import dataclass, field + +import os +import collections +import random +import numpy as np +import threestudio +import torch +import cv2 +from sklearn.cluster import KMeans +import torchvision +from PIL import Image +from transformers import pipeline +from threestudio.systems.base import BaseLift3DSystem +from threestudio.systems.utils import parse_optimizer, parse_scheduler +from threestudio.utils.loss import tv_loss +from threestudio.utils.ops import get_cam_info_gaussian +from threestudio.utils.typing import * +from torch.cuda.amp import autocast +from tqdm.contrib import tenumerate +from tqdm import tqdm, trange + +from ..geometry.gaussian_base import BasicPointCloud, Camera +from ..utils.sam_clip import SamClip +from ..utils.ae import Autoencoder_dataset, Autoencoder +from torch.utils.data import Dataset, DataLoader + +def l2_loss(network_output, gt): + return ((network_output - gt) ** 2).mean() + +def cos_loss(network_output, gt): + return 1 - torch.nn.functional.cosine_similarity(network_output, gt, dim=0).mean() + + +@threestudio.register("scene-lang-system") +class SceneLang(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + visualize_samples: bool = False + + distill_lang_freq: int = 800 + outpaint_step: int = 300 + sam_clip: dict = field(default_factory=dict) + encoder_hidden_dims: Optional[List] = field(default_factory=list) + decoder_hidden_dims: Optional[List] = field(default_factory=list) + ae_epoch: int = 100 + distill_lang_epoch: int = 100 + sam_clip_ae_lr: float = 3e-4 + densify: bool = True + distill_interval: int = 2 + xyz_noise_ratio: Any = None + drop_ooi_ratio: Any = field(default_factory=dict) + empty_prompt: str = "empty" + side_prompt: str = "empty" + crop_with_lang: bool = True + rotate_aug_scale: int = 15 + + cfg: Config + + def configure(self) -> None: + # set up geometry, material, background, renderer + super().configure() + self.automatic_optimization = False + + self.geometry.prompt = self.cfg.prompt_processor.prompt + self.geometry.empty_prompt = self.cfg.empty_prompt + self.geometry.side_prompt = self.cfg.side_prompt + + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.prompt_utils = self.prompt_processor() + + self.cfg.prompt_processor.prompt = self.cfg.empty_prompt + self.bg_prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.bg_prompt_utils = self.bg_prompt_processor() + + self.sam_clip = SamClip(self.cfg.sam_clip) + self.sam_clip_ae = Autoencoder(self.cfg.encoder_hidden_dims, self.cfg.decoder_hidden_dims).cuda() + + def configure_optimizers(self): + optim = self.geometry.optimizer + if hasattr(self, "merged_optimizer"): + return [optim] + if hasattr(self.cfg.optimizer, "name"): + net_optim = parse_optimizer(self.cfg.optimizer, self) + optim = self.geometry.merge_optimizer(net_optim) + self.merged_optimizer = True + else: + self.merged_optimizer = False + return [optim] + + def on_save_checkpoint(self, checkpoint): + if 'optimizer_states' in checkpoint.keys(): + del checkpoint['optimizer_states'] + + del_keys = [k for k in checkpoint['state_dict'].keys() if 'sam' in k] + for k in del_keys: + del checkpoint['state_dict'][k] + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + self.geometry.update_learning_rate(self.global_step) + outputs = self.renderer.batch_forward(batch) + return outputs + + def on_fit_start(self) -> None: + super().on_fit_start() + + def training_step(self, batch, batch_idx): + self.geometry.noise_ratio = self.C(self.cfg.xyz_noise_ratio) + if random.random() < self.C(self.cfg.drop_ooi_ratio): + self.geometry._opacity_mask = (sum(self.geometry.ooi_masks)==0).float() + else: + self.geometry._opacity_mask = None + + if self.true_global_step > 0 and self.true_global_step == self.cfg.distill_lang_freq : # finish rgb phase + self.distill_language_feature() + + if self.true_global_step == self.cfg.outpaint_step: + self.outpaint() + + apply_rotate = False + if self.true_global_step > self.cfg.distill_lang_freq: + apply_rotate = random.random() < 0.5 + self.geometry.random_rotate(self.cfg.rotate_aug_scale, apply_rotate) + + opt = self.optimizers() + out = self(batch) + + visibility_filter = out["visibility_filter"] + radii = out["radii"] + guidance_inp = out["comp_rgb"] + viewspace_point_tensor = out["viewspace_points"] + if self.geometry._opacity_mask is None: + pu = self.prompt_utils + else: + pu = self.bg_prompt_utils + guidance_out = self.guidance( + guidance_inp, pu, **batch, rgb_as_latents=False + ) + + loss_sds = 0.0 + loss = 0.0 + + self.log( + "gauss_num", + int(self.geometry.get_xyz.shape[0]), + on_step=True, + on_epoch=True, + prog_bar=True, + logger=True, + ) + + if self.cfg.loss["lambda_ref"] > 0.0: + ref_img = self.cfg.geometry.geometry_convert_from[len("depth:"):] + ref_img = torch.tensor(np.array(Image.open(ref_img).resize((self.dataset.cfg.width, self.dataset.cfg.height)))[None] / 255, device = out['comp_rgb'].device) + bg_ref_img = torch.tensor(self.geometry.bg_image[None] / 255, device = out['comp_rgb'].device) + bg_ref_img_mask = torch.from_numpy(self.geometry.bg_image_mask[None, ..., None].astype(float)).cuda() + + if self.geometry._opacity_mask is None: + if not apply_rotate: + l1loss = torch.nn.L1Loss()(out['comp_rgb'][0:1], ref_img) # only calculate the first view (zero view) + self.log(f"train/recon_front_view", l1loss) + loss += l1loss * self.cfg.loss["lambda_ref"] + + if self.true_global_step > self.cfg.outpaint_step: + for view_idx in [0, -1]: + self.geometry._opacity_mask = None + sample = self.trainer.val_dataloaders.dataset[view_idx] + for k in sample.keys(): + try: + sample[k] = sample[k].cuda()[None] + except: + pass + output = self(sample) + rgb = output['comp_rgb'] + target = self.outpaint_view[view_idx] + # loss += torch.nn.L1Loss()(rgb, target) * self.cfg.loss["lambda_ref"] + loss += (torch.nn.L1Loss(reduction='none')(rgb, target) * self.outpaint_mask[view_idx]).mean() * self.cfg.loss["lambda_ref"] + else: + ratio = bg_ref_img_mask.sum() / bg_ref_img_mask.shape[1] / bg_ref_img_mask.shape[2] + l1loss = torch.nn.L1Loss(reduction='none')(out['comp_rgb'][0:1], bg_ref_img) * bg_ref_img_mask # only calculate the first view (zero view) + l1loss = l1loss.mean() / ratio + loss += l1loss * self.cfg.loss["lambda_ref"] + + if self.cfg.loss["lambda_scaling"] > 0.0: + scaling_loss = self.geometry.get_scaling.mean() + loss += scaling_loss * self.cfg.loss["lambda_scaling"] + + for name, value in guidance_out.items(): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss_sds += value * self.C( + self.cfg.loss[name.replace("loss_", "lambda_")] + ) + + loss = loss + loss_sds + iteration = self.global_step + opt.zero_grad() + if loss > 0: + loss.backward(retain_graph=True) + if self.cfg.densify: + self.geometry.update_states( + iteration, + visibility_filter, + radii, + viewspace_point_tensor, + ) + opt.step() + opt.zero_grad(set_to_none=True) + + self.log("train/loss", loss) + return {"loss": loss} + + def validation_step(self, batch, batch_idx): + self.geometry._opacity_mask = None + out = self(batch) + mask, _ = self.geometry.project_pc(batch['c2w'], H=self.dataset.cfg.height, W=self.dataset.cfg.width) + self.save_image_grid( + f"it{self.global_step}-val/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ), + name="validation_step", + step=self.global_step, + ) + + def on_validation_epoch_end(self): + self.save_img_sequence( + f"it{self.global_step}-val", + f"it{self.global_step}-val", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="val", + step=self.global_step, + delete_images=True, + ) + + def test_step(self, batch, batch_idx): + # remove the random rotation effect! + self.geometry.recover_xyzrot() + out = self(batch) + self.save_image_grid( + f"it{self.global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + [ + { + "type": "rgb", + "img": out["lang"][0], + "kwargs": {"data_format": "HWC", "data_range": (out["lang"][0].min().item(), out["lang"][0].max().item())}, + }, + ], + name="test_step", + step=self.global_step, + ) + if batch["index"][0] == 0: + save_path = self.get_save_path("point_cloud.ply") + self.geometry.save_ply(save_path) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.global_step}-test", + f"it{self.global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.global_step, + ) + + def on_load_checkpoint(self, ckpt_dict) -> None: + for key in self.state_dict().keys(): + if 'sam' in key: + ckpt_dict["state_dict"][key] = self.state_dict()[key] + + num_pts = ckpt_dict["state_dict"]["geometry._xyz"].shape[0] + pcd = BasicPointCloud( + points=np.zeros((num_pts, 3)), + colors=np.zeros((num_pts, 3)), + normals=np.zeros((num_pts, 3)), + ) + self.geometry.create_from_pcd(pcd, 10) + self.geometry.training_setup() + super().on_load_checkpoint(ckpt_dict) + + def outpaint(self) -> None: + threestudio.info("Start outpainting.") + self.outpaint_view = dict() + self.outpaint_mask = dict() + cnt = 0 + for view_idx in [0, -1]: + self.geometry._opacity_mask = None + sample = self.trainer.val_dataloaders.dataset[view_idx] + for k in sample.keys(): + try: + sample[k] = sample[k].cuda()[None] + except: + pass + output = self(sample) + rgb = (output['comp_rgb'][0] * 255).detach().cpu().numpy().astype(np.uint8) + rgb = Image.fromarray(rgb) + mask, depth = self.geometry.project_pc(sample['c2w'], H=512, W=512) + mask = ~mask[0].cpu().numpy() + mask = Image.fromarray(mask) + c2w = sample['c2w'] + rgb, mask = self.geometry.add_pc_from_novel_view(rgb, mask, depth, c2w, save_path=os.path.join(self._save_dir[:-4], f'{cnt}.ply')) + rgb.save(os.path.join(self._save_dir[:-4], f"outpaint_{cnt}.png")) + mask.save(os.path.join(self._save_dir[:-4], f"mask_{cnt}.png")) + cnt += 1 + self.outpaint_view[view_idx] = torch.tensor(np.array(rgb), device='cuda')[None] / 255 + self.outpaint_mask[view_idx] = torch.tensor(np.array(mask).astype(float), device='cuda')[None, ..., None] + + def distill_language_feature(self) -> None: + threestudio.info("Start distilling language feature.") + self.geometry._opacity_mask = None + total_embed = [] + total_feat = [] + total_flag = [] + + for idx in trange(0, len(self.trainer.val_dataloaders.dataset), self.cfg.distill_interval): + sample = self.trainer.val_dataloaders.dataset[idx] + for k in sample.keys(): + try: + sample[k] = sample[k].cuda()[None] + except: + pass + output = self(sample) + rgb = output['comp_rgb'] #shape: 1, 512, 512, 3 + rgb = (rgb.permute(0, 3, 1, 2) * 255).type(torch.uint8) + + try: + embed, seg, mask= self.sam_clip(rgb) # feat's shape: N * H * W + total_embed.append(embed) + total_feat.append(seg) + total_flag.append(idx) + except: + threestudio.info(f'except caught during language distillation at {idx}') + pass + + # train VAE + threestudio.info("Start training autoencoder.") + dataset = Autoencoder_dataset(torch.cat(total_embed, 0).float().numpy()) + dataloader = DataLoader(dataset, batch_size=32, shuffle=True, num_workers=0, drop_last=False) + optimizer = torch.optim.Adam(self.sam_clip_ae.parameters(), lr=self.cfg.sam_clip_ae_lr) + + self.sam_clip_ae.train() + for epoch in tqdm(range(self.cfg.ae_epoch)): + for idx, data in enumerate(dataloader): + data = data.cuda() + mid = self.sam_clip_ae.encode(data) + _data = self.sam_clip_ae.decode(mid) + l2loss = l2_loss(_data, data) + cosloss = cos_loss(_data, data) + loss = l2loss + cosloss * 0.001 + optimizer.zero_grad() + loss.backward() + optimizer.step() + + self.sam_clip_ae.eval() + mids = dict() + with torch.no_grad(): + zero_tensor = torch.zeros([1, 512], dtype=float) + for idx, seg, embed in zip(total_flag, total_feat, total_embed): + embeds = torch.cat([embed, zero_tensor], 0).float().cuda() + embeds = self.sam_clip_ae.encode(embeds) + mid = embeds[seg[:]].squeeze(0).reshape(self.dataset.cfg.height, self.dataset.cfg.width, -1) + mids[idx] = mid + rgb = ((mid - mid.min()) / (mid.max() - mid.min())).cpu() + if self.sam_clip.cfg.vis_pca_feature: + self.save_image_grid(f"it{self.global_step}-ae/{idx}.png", + [ + { + "type": "rgb", + "img": rgb, + "kwargs": {"data_format": "HWC"}, + }, + ], + name="ae", + step=self.global_step, + ) + + if self.sam_clip.cfg.vis_pca_feature: + self.save_img_sequence( + f"it{self.global_step}-ae", + f"it{self.global_step}-ae", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="ae", + step=self.global_step, + ) + + threestudio.info("Start training Lang feature.") + # distill lang feature + self.geometry.lang_training_setup() + opt = self.geometry.lang_optimizer + + idx_list = list(mids.keys()) + sample_dict = dict() + + for idx, sample in enumerate(self.trainer.val_dataloaders.dataset): + for k in sample.keys(): + try: + sample[k] = sample[k].cuda()[None] + except: + pass + sample_dict[idx] = sample + + for epoch in trange(self.cfg.distill_lang_epoch): + random.shuffle(idx_list) + for idx in idx_list: + sample = sample_dict[idx] + lang = self(sample)["lang"] + mid = mids[idx][None] + loss = l2_loss(mid, lang) + opt.zero_grad() + loss.backward() + opt.step() + if (epoch + 1) % 30 == 0: + opt.state = collections.defaultdict(dict) + + self.renderer.training=False + with torch.no_grad(): + lang_min, lang_max = None, None + for idx, sample in sample_dict.items(): + lang = self(sample)["lang"][0] + if lang_min is None: + lang_min, lang_max = lang.min().item(), lang.max().item() + self.save_image_grid(f"it{self.global_step}-feat/{idx}.png", + [ + { + "type": "rgb", + "img": lang, + "kwargs": {"data_format": "HWC", "data_range": (lang_min, lang_max)}, + }, + ], + name=f"feat", + step=self.global_step, + ) + self.renderer.training=True + + self.save_img_sequence( + f"it{self.global_step}-feat", + f"it{self.global_step}-feat", + "(\d+)\.png", + save_format="mp4", + fps=30, + name=f"feat", + step=self.global_step, + ) + + self.geometry.training_setup() + + threestudio.info("Use Lang feature to crop pts") + if self.cfg.crop_with_lang: + p = 2 + if self.geometry._delete_mask is None: + self.geometry._delete_mask = torch.ones_like(self.geometry.ooi_masks[0]) + for ooi_idx, ooi_mask in enumerate(self.geometry.ooi_masks): + threestudio.info(self.geometry.ooi_masks[ooi_idx].sum()) + idx = torch.arange(len(ooi_mask), device='cuda')[ooi_mask.bool()] + lang_feat = self.geometry.get_language_feature[ooi_mask.bool()] + lang_feat = lang_feat / (lang_feat.norm(2, dim=-1, keepdim=True) + 0.1) + + original_ooi_mask = ooi_mask.clone() + # filter with color by KMeans + kmeans = KMeans(n_init='auto', n_clusters=10) + kmeans.fit(lang_feat.detach().cpu()) + labels = kmeans.labels_ + _ = [(labels==i).sum() for i in np.unique(labels)] + max_label = _.index(max(_)) + dist = ((kmeans.cluster_centers_ - kmeans.cluster_centers_[max_label:max_label+1]) **2).sum(-1)**.5 + + for label, num in enumerate(_): + if dist[label] > 0.3: + ooi_mask[idx[labels == label]] = False + self.geometry._delete_mask[idx[labels == label]] = 0. + + p = 1 + # filter with color by Gaussian + mean, std = lang_feat.mean(0), lang_feat.std(0) + outlier = torch.logical_or(lang_feat < mean - p * std, lang_feat > mean + p * std).sum(-1) > 0 + ooi_mask[idx[outlier]] = False + self.geometry._delete_mask[idx[outlier]] = 0. + + p = 3 + # filter with RGB by Gaussian + rgb =self.geometry.get_features[original_ooi_mask.bool()][:, 0] + mean, std = rgb.mean(0), rgb.std(0) + outlier = torch.logical_or(rgb < mean - p * std, rgb > mean + p * std).sum(-1) > 0 + ooi_mask[idx[outlier]] = False + self.geometry._delete_mask[idx[outlier]] = 0. + + def load_state_dict(self, state_dict, strict=True): + i = 0 + while 1: + if f'geometry.ooi_masks_{i}' not in state_dict.keys(): + break + self.geometry.register_buffer(f'ooi_masks_{i}', state_dict[f'geometry.ooi_masks_{i}']) + i += 1 + self.geometry.register_buffer('_delete_mask', state_dict['geometry._delete_mask']) + return super().load_state_dict(state_dict, strict) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/utils/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/utils/ae.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/utils/ae.py new file mode 100644 index 0000000000000000000000000000000000000000..03c268170cb942238ad52f05f03dd477439968be --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/utils/ae.py @@ -0,0 +1,63 @@ +import os +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import Dataset + +class Autoencoder_dataset(Dataset): + def __init__(self, data): + self.data = data + + def __getitem__(self, index): + data = torch.tensor(self.data[index]) + return data + + def __len__(self): + return self.data.shape[0] + + +class Autoencoder(nn.Module): + def __init__(self, encoder_hidden_dims, decoder_hidden_dims): + super(Autoencoder, self).__init__() + encoder_layers = [] + for i in range(len(encoder_hidden_dims)): + if i == 0: + encoder_layers.append(nn.Linear(512, encoder_hidden_dims[i])) + else: + encoder_layers.append(torch.nn.GroupNorm(2, encoder_hidden_dims[i-1])) + # encoder_layers.append(torch.nn.BatchNorm1d(encoder_hidden_dims[i-1])) + encoder_layers.append(nn.ReLU()) + encoder_layers.append(nn.Linear(encoder_hidden_dims[i-1], encoder_hidden_dims[i])) + self.encoder = nn.ModuleList(encoder_layers) + + decoder_layers = [] + for i in range(len(decoder_hidden_dims)): + if i == 0: + decoder_layers.append(nn.Linear(encoder_hidden_dims[-1], decoder_hidden_dims[i])) + else: + encoder_layers.append(torch.nn.GroupNorm(2, decoder_hidden_dims[i-1])) + # encoder_layers.append(torch.nn.BatchNorm1d(decoder_hidden_dims[i-1])) + decoder_layers.append(nn.ReLU()) + decoder_layers.append(nn.Linear(decoder_hidden_dims[i-1], decoder_hidden_dims[i])) + self.decoder = nn.ModuleList(decoder_layers) + + def forward(self, x): + for m in self.encoder: + x = m(x) + x = x / x.norm(2, dim=-1, keepdim=True) + for m in self.decoder: + x = m(x) + # x = x / x.norm(2, dim=-1, keepdim=True) + return x + + def encode(self, x): + for m in self.encoder: + x = m(x) + x = x / x.norm(2, dim=-1, keepdim=True) + return x + + def decode(self, x): + for m in self.decoder: + x = m(x) + # x = x / x.norm(2, dim=-1, keepdim=True) + return x diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/utils/sam_clip.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/utils/sam_clip.py new file mode 100644 index 0000000000000000000000000000000000000000..5d5c17fd2c407f0b18884172be57b57f67c43958 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/custom/threestudio-3dgs/utils/sam_clip.py @@ -0,0 +1,366 @@ +from dataclasses import dataclass, field +import pytorch_lightning as pl +from threestudio.utils.config import parse_structured +from threestudio.utils.base import Updateable, update_if_possible +from threestudio.utils.saving import SaverMixin +from threestudio.utils.typing import * + +import open_clip +import torch +import torchvision +from torch import nn +import cv2 +import numpy as np +from sklearn.decomposition import PCA + +from segment_anything import SamAutomaticMaskGenerator, sam_model_registry +from mobile_sam import sam_model_registry as m_sam_model_registry +from mobile_sam import SamAutomaticMaskGenerator as m_SamAutomaticMaskGenerator +from mobile_sam import SamPredictor as m_SamPredictor + +@dataclass +class OpenCLIPNetworkConfig: + _target: Type = field(default_factory=lambda: OpenCLIPNetwork) + clip_model_type: str = "ViT-B-16" + clip_model_pretrained: str = "laion2b_s34b_b88k" + clip_n_dims: int = 512 + negatives: Tuple[str] = ("object", "things", "stuff", "texture") + positives: Tuple[str] = ("",) + +class OpenCLIPNetwork(nn.Module): + def __init__(self, config: OpenCLIPNetworkConfig): + super().__init__() + self.config = config + self.process = torchvision.transforms.Compose( + [ + torchvision.transforms.Resize((224, 224)), + torchvision.transforms.Normalize( + mean=[0.48145466, 0.4578275, 0.40821073], + std=[0.26862954, 0.26130258, 0.27577711], + ), + ] + ) + model, _, _ = open_clip.create_model_and_transforms( + self.config.clip_model_type, # e.g., ViT-B-16 + pretrained=self.config.clip_model_pretrained, # e.g., laion2b_s34b_b88k + precision="fp16", + ) + model.eval() + self.tokenizer = open_clip.get_tokenizer(self.config.clip_model_type) + self.model = model.to("cuda") + self.clip_n_dims = self.config.clip_n_dims + + self.positives = self.config.positives + self.negatives = self.config.negatives + with torch.no_grad(): + tok_phrases = torch.cat([self.tokenizer(phrase) for phrase in self.positives]).to("cuda") + self.pos_embeds = model.encode_text(tok_phrases) + tok_phrases = torch.cat([self.tokenizer(phrase) for phrase in self.negatives]).to("cuda") + self.neg_embeds = model.encode_text(tok_phrases) + self.pos_embeds /= self.pos_embeds.norm(dim=-1, keepdim=True) + self.neg_embeds /= self.neg_embeds.norm(dim=-1, keepdim=True) + + assert ( + self.pos_embeds.shape[1] == self.neg_embeds.shape[1] + ), "Positive and negative embeddings must have the same dimensionality" + assert ( + self.pos_embeds.shape[1] == self.clip_n_dims + ), "Embedding dimensionality must match the model dimensionality" + + @property + def name(self) -> str: + return "openclip_{}_{}".format(self.config.clip_model_type, self.config.clip_model_pretrained) + + @property + def embedding_dim(self) -> int: + return self.config.clip_n_dims + + def gui_cb(self,element): + self.set_positives(element.value.split(";")) + + def set_positives(self, text_list): + self.positives = text_list + with torch.no_grad(): + tok_phrases = torch.cat([self.tokenizer(phrase) for phrase in self.positives]).to("cuda") + self.pos_embeds = self.model.encode_text(tok_phrases) + self.pos_embeds /= self.pos_embeds.norm(dim=-1, keepdim=True) + + def get_relevancy(self, embed: torch.Tensor, positive_id: int) -> torch.Tensor: + phrases_embeds = torch.cat([self.pos_embeds, self.neg_embeds], dim=0) + p = phrases_embeds.to(embed.dtype) # phrases x 512 + output = torch.mm(embed, p.T) # rays x phrases + positive_vals = output[..., positive_id : positive_id + 1] # rays x 1 + negative_vals = output[..., len(self.positives) :] # rays x N_phrase + repeated_pos = positive_vals.repeat(1, len(self.negatives)) # rays x N_phrase + + sims = torch.stack((repeated_pos, negative_vals), dim=-1) # rays x N-phrase x 2 + softmax = torch.softmax(10 * sims, dim=-1) # rays x n-phrase x 2 + best_id = softmax[..., 0].argmin(dim=1) # rays x 2 + return torch.gather(softmax, 1, best_id[..., None, None].expand(best_id.shape[0], len(self.negatives), 2))[:, 0, :] + + def encode_image(self, input): + processed_input = self.process(input).half() + return self.model.encode_image(processed_input) + +def get_seg_img(mask, image): + image = image.copy() + image[mask['segmentation']==0] = np.array([0, 0, 0], dtype=np.uint8) + x,y,w,h = np.int32(mask['bbox']) + seg_img = image[y:y+h, x:x+w, ...] + return seg_img + +def pad_img(img): + h, w, _ = img.shape + l = max(w,h) + pad = np.zeros((l,l,3), dtype=np.uint8) + if h > w: + pad[:,(h-w)//2:(h-w)//2 + w, :] = img + else: + pad[(w-h)//2:(w-h)//2 + h, :, :] = img + return pad + +def filter(keep: torch.Tensor, masks_result) -> None: + keep = keep.int().cpu().numpy() + result_keep = [] + for i, m in enumerate(masks_result): + if i in keep: result_keep.append(m) + return result_keep + +def sava_numpy(save_path, data): + save_path_s = save_path + '_s.npy' + save_path_f = save_path + '_f.npy' + np.save(save_path_s, data['seg_maps'].numpy()) + np.save(save_path_f, data['feature'].numpy()) + +def mask_nms(masks, scores, iou_thr=0.7, score_thr=0.1, inner_thr=0.2, **kwargs): + """ + Perform mask non-maximum suppression (NMS) on a set of masks based on their scores. + + Args: + masks (torch.Tensor): has shape (num_masks, H, W) + scores (torch.Tensor): The scores of the masks, has shape (num_masks,) + iou_thr (float, optional): The threshold for IoU. + score_thr (float, optional): The threshold for the mask scores. + inner_thr (float, optional): The threshold for the overlap rate. + **kwargs: Additional keyword arguments. + Returns: + selected_idx (torch.Tensor): A tensor representing the selected indices of the masks after NMS. + """ + + scores, idx = scores.sort(0, descending=True) + num_masks = idx.shape[0] + + masks_ord = masks[idx.view(-1), :] + masks_area = torch.sum(masks_ord, dim=(1, 2), dtype=torch.float) + + iou_matrix = torch.zeros((num_masks,) * 2, dtype=torch.float, device=masks.device) + inner_iou_matrix = torch.zeros((num_masks,) * 2, dtype=torch.float, device=masks.device) + for i in range(num_masks): + for j in range(i, num_masks): + intersection = torch.sum(torch.logical_and(masks_ord[i], masks_ord[j]), dtype=torch.float) + union = torch.sum(torch.logical_or(masks_ord[i], masks_ord[j]), dtype=torch.float) + iou = intersection / union + iou_matrix[i, j] = iou + # select mask pairs that may have a severe internal relationship + if intersection / masks_area[i] < 0.5 and intersection / masks_area[j] >= 0.85: + inner_iou = 1 - (intersection / masks_area[j]) * (intersection / masks_area[i]) + inner_iou_matrix[i, j] = inner_iou + if intersection / masks_area[i] >= 0.85 and intersection / masks_area[j] < 0.5: + inner_iou = 1 - (intersection / masks_area[j]) * (intersection / masks_area[i]) + inner_iou_matrix[j, i] = inner_iou + + iou_matrix.triu_(diagonal=1) + iou_max, _ = iou_matrix.max(dim=0) + inner_iou_matrix_u = torch.triu(inner_iou_matrix, diagonal=1) + inner_iou_max_u, _ = inner_iou_matrix_u.max(dim=0) + inner_iou_matrix_l = torch.tril(inner_iou_matrix, diagonal=1) + inner_iou_max_l, _ = inner_iou_matrix_l.max(dim=0) + + keep = iou_max <= iou_thr + keep_conf = scores > score_thr + keep_inner_u = inner_iou_max_u <= 1 - inner_thr + keep_inner_l = inner_iou_max_l <= 1 - inner_thr + + # If there are no masks with scores above threshold, the top 3 masks are selected + if keep_conf.sum() == 0: + index = scores.topk(3).indices + keep_conf[index, 0] = True + if keep_inner_u.sum() == 0: + index = scores.topk(3).indices + keep_inner_u[index, 0] = True + if keep_inner_l.sum() == 0: + index = scores.topk(3).indices + keep_inner_l[index, 0] = True + keep *= keep_conf + keep *= keep_inner_u + keep *= keep_inner_l + + selected_idx = idx[keep] + return selected_idx + +def masks_update(*args, **kwargs): + # remove redundant masks based on the scores and overlap rate between masks + masks_new = () + for masks_lvl in (args): + seg_pred = torch.from_numpy(np.stack([m['segmentation'] for m in masks_lvl], axis=0)) + iou_pred = torch.from_numpy(np.stack([m['predicted_iou'] for m in masks_lvl], axis=0)) + stability = torch.from_numpy(np.stack([m['stability_score'] for m in masks_lvl], axis=0)) + + scores = stability * iou_pred + keep_mask_nms = mask_nms(seg_pred, scores, **kwargs) + masks_lvl = filter(keep_mask_nms, masks_lvl) + + masks_new += (masks_lvl,) + return masks_new + +def sam_encoder(image, mask_generator): + image = image.detach().cpu() + image = cv2.cvtColor(image[0].permute(1,2,0).numpy().astype(np.uint8), cv2.COLOR_BGR2RGB) + # pre-compute masks + masks_l = mask_generator.generate(image) + # pre-compute postprocess + masks_l = masks_update(masks_l, iou_thr=0.8, score_thr=0.7, inner_thr=0.5)[0] + + def mask2segmap(masks, image): + seg_img_list = [] + seg_map = -np.ones(image.shape[:2], dtype=np.int32) + for i in range(len(masks)): + mask = masks[i] + seg_img = get_seg_img(mask, image) + pad_seg_img = cv2.resize(pad_img(seg_img), (224,224)) + seg_img_list.append(pad_seg_img) + + seg_map[masks[i]['segmentation']] = i + seg_imgs = np.stack(seg_img_list, axis=0) # b,H,W,3 + seg_imgs = (torch.from_numpy(seg_imgs.astype("float32")).permute(0,3,1,2) / 255.0).to('cuda') + + return seg_imgs, seg_map + + seg_images, seg_maps = {}, {} + seg_images['l'], seg_maps['l'] = mask2segmap(masks_l, image) + + # 0:default 1:s 2:m 3:l + return seg_images, seg_maps + +class SamClip(pl.LightningModule, Updateable, SaverMixin): + @dataclass + class Config: + clip_model_type: str = "ViT-B-16" + clip_model_pretrained: str = "laion2b_s34b_b88k" + clip_n_dims: int = 512 + sam_ckpt_path: str = "ckpts/sam_vit_h_4b8939.pth" + feature_level: int = 3 + vis_pca_feature: bool = True + use_mobile_sam: bool = True + + cfg: Config + + def __init__(self, cfg) -> None: + super().__init__() + self.cfg = parse_structured(self.Config, cfg) + self.model = OpenCLIPNetwork(OpenCLIPNetworkConfig) + self.clip_n_dims = self.cfg.clip_n_dims + self.tokenizer = open_clip.get_tokenizer(self.cfg.clip_model_type) + sam = sam_model_registry["vit_h"](checkpoint=self.cfg.sam_ckpt_path).to('cuda') + self.mask_generator = SamAutomaticMaskGenerator( + model=sam, + points_per_side=32, + points_per_batch=64, + pred_iou_thresh=0.7, + box_nms_thresh=0.7, + stability_score_thresh=0.85, + crop_n_layers=1, + crop_n_points_downscale_factor=1, + min_mask_region_area=100, + ) + + model_type = "vit_t" + sam_checkpoint = "./ckpts/mobile_sam.pt" + device = "cuda" if torch.cuda.is_available() else "cpu" + mobile_sam = m_sam_model_registry[model_type](checkpoint=sam_checkpoint) + mobile_sam.to(device=device) + mobile_sam.eval() + self.m_mask_generator = m_SamAutomaticMaskGenerator(mobile_sam) + + # self.estimator = PCA(n_components=3) + # self.has_fit = False + + self.mask_generator.predictor.model.to('cuda') + self.m_mask_generator.predictor.model.to('cuda') + + def _embed_clip_sam_tiles(self, image, sam_encoder): + aug_imgs = torch.cat([image]) + if self.cfg.use_mobile_sam: + seg_images, seg_map = sam_encoder(aug_imgs, self.m_mask_generator) + else: + seg_images, seg_map = sam_encoder(aug_imgs, self.mask_generator) + + clip_embeds = {} + # types = ['default', 's', 'm', 'l'] + types = ['l'] + for mode in types: + tiles = seg_images[mode] + tiles = tiles.to("cuda") + with torch.no_grad(): + clip_embed = self.model.encode_image(tiles) + clip_embed /= clip_embed.norm(dim=-1, keepdim=True) + clip_embeds[mode] = clip_embed.detach().cpu().half() + + return clip_embeds, seg_map + + def forward(self, img): + embed_size=512 + seg_maps = [] + total_lengths = [] + timer = 0 + img_embeds = torch.zeros((len(img), 100, embed_size)) + + seg_maps = torch.zeros((len(img), 1, *img.shape[2:])) + img_embed, seg_map = self._embed_clip_sam_tiles(img, sam_encoder) + + lengths = [len(v) for k, v in img_embed.items()] + total_length = sum(lengths) + # total_lengths.append(total_length) + + # if total_length > img_embeds.shape[1]: + # pad = total_length - img_embeds.shape[1] + # img_embeds = torch.cat([ + # img_embeds, + # torch.zeros((len(image_list), pad, embed_size)) + # ], dim=1) + + # img_embed = torch.cat([v for k, v in img_embed.items()], dim=0) + # assert img_embed.shape[0] == total_length + img_embeds[0, :total_length] = img_embed['l'] + + # seg_map_tensor = [] + # lengths_cumsum = lengths.copy() + # for j in range(1, len(lengths)): + # lengths_cumsum[j] += lengths_cumsum[j-1] + # for j, (k, v) in enumerate(seg_map.items()): + # if j == 0: + # seg_map_tensor.append(torch.from_numpy(v)) + # continue + # assert v.max() == lengths[j] - 1, f"{j}, {v.max()}, {lengths[j]-1}" + # v[v != -1] += lengths_cumsum[j-1] + # seg_map_tensor.append(torch.from_numpy(v)) + # seg_map = torch.stack(seg_map_tensor, dim=0) + seg_maps[0] = torch.from_numpy(seg_map['l']) + + # self.mask_generator.predictor.model.to('cpu') + feature_map = img_embeds[0] # 300, 512 + seg_map = seg_maps[0] # 4, 512, 512 + + image_height, image_width = seg_map.shape[1:] + y, x = torch.meshgrid(torch.arange(0, image_height), torch.arange(0, image_width)) + x = x.reshape(-1, 1) + y = y.reshape(-1, 1) + seg = seg_map[:, y, x].squeeze(-1).long() + mask = seg != -1 + point_feature1 = feature_map[seg[:]].squeeze(0) + mask = mask[:].reshape(1, image_height, image_width) + return img_embed['l'], seg, mask + # point_feature = point_feature1.reshape(image_height, image_width, -1).permute(2, 0, 1) + + # return img_embed['l'], point_feature, mask + diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/bear_background.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/bear_background.png new file mode 100644 index 0000000000000000000000000000000000000000..ba1db72d9083c057fea7b3c41eae07860d5992d9 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/bear_background.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:950496f640077d2d1b3f28cf8f2ecaeb56bc641b2c19f6a8107e6d428f5da17f +size 1244763 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/bear_composite.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/bear_composite.png new file mode 100644 index 0000000000000000000000000000000000000000..84d509318af58e50fc65dcba2843332c8ae4a282 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/bear_composite.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1445582663ddb516915adbfac9f33ba2d95e554d76a1f4b164ef9f119061be74 +size 1130670 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/bear_layers.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/bear_layers.png new file mode 100644 index 0000000000000000000000000000000000000000..ec4469e7e71acf64392b8903ac620cc4cad54bb9 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/bear_layers.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/boy_background.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/boy_background.png new file mode 100644 index 0000000000000000000000000000000000000000..5b3d754d8f4df5148732508d3cd792045d32f86f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/boy_background.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7221341ebcc6084cf6ef9521324bea45658cebb9a3a4de487ef0f17bd83235a +size 1256424 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/boy_composite.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/boy_composite.png new file mode 100644 index 0000000000000000000000000000000000000000..2f464e41f3fac0c10328ef23564e4c3b26a26743 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/boy_composite.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f9a04bee8f5de415251558a4401c7f597ee3e6c2dc989ddf974a9104243c8dc +size 1172283 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/boy_layers.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/boy_layers.png new file mode 100644 index 0000000000000000000000000000000000000000..3db8526628551ac113e5eb1d6ff55e1205c31df5 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/boy_layers.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/corgi_background.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/corgi_background.png new file mode 100644 index 0000000000000000000000000000000000000000..5609e977a29f0461000aa03eb6e6e835a9f30d86 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/corgi_background.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e7c7c2ab126d4d26c2258160d859e03291ef745fae2e05540ac60bc8976e7d9 +size 1340031 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/corgi_composite.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/corgi_composite.png new file mode 100644 index 0000000000000000000000000000000000000000..03e5691f3203f389ab096015f8f31e77f4149145 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/corgi_composite.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c00e3156ad6e929df5abe236b7d98772b13142551e740866317e5e829f3bf03 +size 1224236 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/corgi_layers.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/corgi_layers.png new file mode 100644 index 0000000000000000000000000000000000000000..342e5f9986eafdd315ef14521701ecef07edcc50 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/corgi_layers.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/stairs_background.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/stairs_background.png new file mode 100644 index 0000000000000000000000000000000000000000..45cfd33a9cb9646cd16b9a7b92452ebc883e4d78 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/stairs_background.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffc8a084a1eda8b67b9086fedef36a70348b72cd0c4fa3c8340fdd5c1c862e1a +size 2290972 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/stairs_composite.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/stairs_composite.png new file mode 100644 index 0000000000000000000000000000000000000000..777b1b0b3a9a1c9bed58e28c4396a54ef8b420c0 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/stairs_composite.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f3702eeb88177ea28eb73c42a1642b071835a19c19bf67a897dadd6c016a38b +size 2288382 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/stairs_layers.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/stairs_layers.png new file mode 100644 index 0000000000000000000000000000000000000000..2e970fd41c3a7d40c9826ddc5d3d282466dc148a Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/examples/stairs_layers.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/gradio_app_single_process.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/gradio_app_single_process.py new file mode 100644 index 0000000000000000000000000000000000000000..0e17c5b4b77938559a4b15f9d16ad971f5207e03 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/gradio_app_single_process.py @@ -0,0 +1,854 @@ +import argparse +import glob +import os +import re +import signal +import subprocess +import tempfile +import time +import logging +import threading +import gc +from dataclasses import dataclass +from datetime import datetime +from typing import Optional + +import gradio as gr +import numpy as np +from scipy.ndimage import label, find_objects +import psutil +import trimesh +from PIL import Image +import torch +import time, traceback +from typing import NamedTuple +import contextlib +import importlib +import sys +import threestudio +from pytorch_lightning import Trainer +from tqdm import tqdm, trange +from torchvision import transforms +import imageio + +from threestudio.utils.config import load_config, ExperimentConfig, parse_structured +from threestudio.utils.typing import * + +from utils.tools import rotation_matrix, build_rotation, prune, REORDER_MTX +from scipy.spatial.transform import Rotation as R + +INTERACTIVE_N = 8 + +class BasicPointCloud(NamedTuple): + points: np.array + colors: np.array + normals: np.array + +def load_custom_module(module_path): + module_name = os.path.basename(module_path) + if os.path.isfile(module_path): + sp = os.path.splitext(module_path) + module_name = sp[0] + try: + if os.path.isfile(module_path): + module_spec = importlib.util.spec_from_file_location( + module_name, module_path + ) + else: + module_spec = importlib.util.spec_from_file_location( + module_name, os.path.join(module_path, "__init__.py") + ) + + module = importlib.util.module_from_spec(module_spec) + sys.modules[module_name] = module + module_spec.loader.exec_module(module) + return True + except Exception as e: + print(traceback.format_exc()) + print(f"Cannot import {module_path} module for custom nodes:", e) + return False + + +def load_custom_modules(): + node_paths = ["custom"] + node_import_times = [] + for custom_node_path in node_paths: + possible_modules = os.listdir(custom_node_path) + if "__pycache__" in possible_modules: + possible_modules.remove("__pycache__") + + for possible_module in possible_modules: + module_path = os.path.join(custom_node_path, possible_module) + if ( + os.path.isfile(module_path) + and os.path.splitext(module_path)[1] != ".py" + ): + continue + if module_path.endswith("_disabled"): + continue + time_before = time.perf_counter() + success = load_custom_module(module_path) + node_import_times.append( + (time.perf_counter() - time_before, module_path, success) + ) + + if len(node_import_times) > 0: + print("\nImport times for custom modules:") + for n in sorted(node_import_times): + if n[2]: + import_message = "" + else: + import_message = " (IMPORT FAILED)" + print("{:6.1f} seconds{}:".format(n[0], import_message), n[1]) + print() + +def set_system_status(system, ckpt_path): + if ckpt_path is None: + return + ckpt = torch.load(ckpt_path, map_location="cpu") + system.set_resume_status(ckpt["epoch"], ckpt["global_step"]) + +def load_ckpt(ckpt_name): + if cfg.resume is None: + if not cfg.use_timestamp and os.path.isfile(ckpt_name): + print('load_' + ckpt_name) + cfg.resume = ckpt_name + + set_system_status(system, cfg.resume) + ckpt = torch.load(ckpt_name) + num_pts = ckpt["state_dict"]["geometry._xyz"].shape[0] + pcd = BasicPointCloud( + points=np.zeros((num_pts, 3)), + colors=np.zeros((num_pts, 3)), + normals=np.zeros((num_pts, 3)), + ) + system.geometry.create_from_pcd(pcd, 10) + system.geometry.training_setup() + + o1, o2 = system.load_state_dict(ckpt['state_dict'], strict=False) + + system.to('cuda') + system.renderer.training=False + +def tail(f, window=20): + # Returns the last `window` lines of file `f`. + if window == 0: + return [] + + BUFSIZ = 1024 + f.seek(0, 2) + remaining_bytes = f.tell() + size = window + 1 + block = -1 + data = [] + + while size > 0 and remaining_bytes > 0: + if remaining_bytes - BUFSIZ > 0: + # Seek back one whole BUFSIZ + f.seek(block * BUFSIZ, 2) + # read BUFFER + bunch = f.read(BUFSIZ) + else: + # file too small, start from beginning + f.seek(0, 0) + # only read what was not read + bunch = f.read(remaining_bytes) + + bunch = bunch.decode("utf-8") + data.insert(0, bunch) + size -= bunch.count("\n") + remaining_bytes -= BUFSIZ + block -= 1 + + return "\n".join("".join(data).splitlines()[-window:]) + + +@dataclass +class ExperimentStatus: + pid: Optional[int] = None + progress: str = "" + log: str = "" + output_image: Optional[str] = None + output_video: Optional[str] = None + output_feat_video: Optional[str] = None + + def tolist(self): + return [ + self.progress, + self.log, + self.output_video, + self.output_feat_video, + ] + + +EXP_ROOT_DIR = "outputs-gradio" +DEFAULT_PROMPT = "a countryside" +DEFAULT_PROMPT_SIDE = "a countryside" +DEFAULT_PROMPT_EMPTY = "empty" + +EXAMPLE_PROMPT_LIST = [ + "a teddy bear at Times Square", + "Color photo of a corgi made of transparent glass, standing on the riverside in Yosemite National Park", + "two men climbing stairs", + "a boy standing near the window", +] + +model_name_config = [ + ("3Dit Scene", "custom/threestudio-3dgs/configs/scene_lang.yaml"), +] + +config_path = "custom/threestudio-3dgs/configs/scene_lang.yaml" +with open(config_path) as f: + config_yaml = f.read() +config_obj = load_config( + config_yaml, + # set name and tag to dummy values to avoid creating new directories + cli_args=[ + "name=dummy", + "tag=dummy", + "use_timestamp=false", + f"exp_root_dir={EXP_ROOT_DIR}", + "system.prompt_processor.prompt=placeholder", + ], + from_string=True, +) + +def get_current_status(save_root, trial_dir, alive_path): + status = ExperimentStatus() + + # write the current timestamp to the alive file + # the watcher will know the last active time of this process from this timestamp + if os.path.exists(os.path.dirname(alive_path)): + alive_fp = open(alive_path, "w") + alive_fp.seek(0) + alive_fp.write(str(time.time())) + alive_fp.flush() + + log_path = os.path.join(trial_dir, "logs") + progress_path = os.path.join(trial_dir, "progress") + save_path = os.path.join(trial_dir, "save") + + # read current progress from the progress file + # the progress file is created by GradioCallback + if os.path.exists(progress_path): + status.progress = open(progress_path).read() + else: + status.progress = "Setting up everything ..." + + # read the last 10 lines of the log file + if os.path.exists(log_path): + status.log = tail(open(log_path, "rb"), window=10) + else: + status.log = "" + + # get the validation image and testing video if they exist + if os.path.exists(save_path): + videos = [f for f in glob.glob(os.path.join(save_path, "*.mp4")) if 'val' in f or 'test' in f] + steps = [ + int(re.match(r"it(\d+)-(val|test)\.mp4", os.path.basename(f)).group(1)) + for f in videos + ] + videos = sorted(list(zip(videos, steps)), key=lambda x: x[1]) + if len(videos) > 0: + status.output_video = videos[-1][0] + + videos = [f for f in glob.glob(os.path.join(save_path, "*.mp4")) if 'feat' in f] + steps = [ + int(re.match(r"it(\d+)-feat\.mp4", os.path.basename(f)).group(1)) + for f in videos + ] + videos = sorted(list(zip(videos, steps)), key=lambda x: x[1]) + if len(videos) > 0: + status.output_feat_video = videos[-1][0] + + return status + +def create_bounding_boxes(mask): + labeled_array, num_features = label(mask) + bounding_boxes = find_objects(labeled_array) + + return bounding_boxes + +def train(extras, name, tag, config_path): + os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + env_gpus_str = os.environ.get("CUDA_VISIBLE_DEVICES", None) + env_gpus = list(env_gpus_str.split(",")) if env_gpus_str else [] + selected_gpus = [0] + # Always rely on CUDA_VISIBLE_DEVICES if specific GPU ID(s) are specified. + # As far as Pytorch Lightning is concerned, we always use all available GPUs + # (possibly filtered by CUDA_VISIBLE_DEVICES). + devices = -1 + gpu = '0' + if len(env_gpus) > 0: + # CUDA_VISIBLE_DEVICES was set already, e.g. within SLURM srun or higher-level script. + n_gpus = len(env_gpus) + else: + selected_gpus = list(gpu.split(",")) + n_gpus = len(selected_gpus) + os.environ["CUDA_VISIBLE_DEVICES"] = gpu + + import pytorch_lightning as pl + import torch + from pytorch_lightning import Trainer + from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint + from pytorch_lightning.loggers import CSVLogger, TensorBoardLogger + from pytorch_lightning.utilities.rank_zero import rank_zero_only + import threestudio + from threestudio.systems.base import BaseSystem + from threestudio.utils.callbacks import ( + CodeSnapshotCallback, + ConfigSnapshotCallback, + CustomProgressBar, + ProgressCallback, + ) + from threestudio.utils.config import ExperimentConfig, load_config + from threestudio.utils.misc import get_rank + from threestudio.utils.typing import Optional + + logger = logging.getLogger("pytorch_lightning") + + for handler in logger.handlers: + if handler.stream == sys.stderr: # type: ignore + handler.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) + + # parse YAML config to OmegaConf + cfg: ExperimentConfig + cfg = load_config(config_path, cli_args=extras, n_gpus=n_gpus) + + # set a different seed for each device + pl.seed_everything(cfg.seed + get_rank(), workers=True) + + system.cfg = parse_structured(system.Config, cfg.system) + system.configure() + + # set up dm + dm = threestudio.find(cfg.data_type)(cfg.data) + + system.set_save_dir(os.path.join(cfg.trial_dir, "save")) + fh = logging.FileHandler(os.path.join(cfg.trial_dir, "logs")) + fh.setLevel(logging.INFO) + fh.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) + logger.addHandler(fh) + callbacks = [] + callbacks += [ + ModelCheckpoint( + dirpath=os.path.join(cfg.trial_dir, "ckpts"), **cfg.checkpoint + ), + LearningRateMonitor(logging_interval="step"), + CodeSnapshotCallback( + os.path.join(cfg.trial_dir, "code"), use_version=False + ), + ConfigSnapshotCallback( + config_path, + cfg, + os.path.join(cfg.trial_dir, "configs"), + use_version=False, + ), + ] + callbacks += [ + ProgressCallback(save_path=os.path.join(cfg.trial_dir, "progress")) + ] + def write_to_text(file, lines): + with open(file, "w") as f: + for line in lines: + f.write(line + "\n") + loggers = [] + rank_zero_only( + lambda: os.makedirs(os.path.join(cfg.trial_dir, "tb_logs"), exist_ok=True) + )() + loggers += [ + TensorBoardLogger(cfg.trial_dir, name="tb_logs"), + CSVLogger(cfg.trial_dir, name="csv_logs"), + ] + system.get_loggers() + rank_zero_only( + lambda: write_to_text( + os.path.join(cfg.trial_dir, "cmd.txt"), + ["python " + " ".join(sys.argv), str(args)], + ) + )() + trainer = Trainer( + callbacks=callbacks, + logger=loggers, + inference_mode=False, + accelerator="gpu", + devices=devices, + **cfg.trainer, + ) + + trainer.fit(system, datamodule=dm, ckpt_path=cfg.resume) + + +def launch( + host='0.0.0.0', + port=10092, + listen=False, + self_deploy=False, + save_root=".", + dm=None, + system=None, +): + dataloader = dm.test_dataloader() + iter_dataloader = iter(dataloader) + sampled_data = [] + for i in range(120): + data = next(iter_dataloader) + for k in data.keys(): + try: + data[k] = data[k].cuda() + except: + pass + sampled_data.append(data) + + def run( + image, + prompt: str, + side_prompt: str, + empty_prompt: str, + seed: int, + max_steps: int, + ): + gc.collect() + torch.cuda.empty_cache() + if prompt in EXAMPLE_PROMPT_LIST: + print(os.path.join("examples_cache/", prompt, "rgb.mp4")) + print(os.path.join("examples_cache/", prompt, "feat.mp4")) + status_list = ["Loaded!", "load log", os.path.join("examples_cache/", prompt, "rgb.mp4"), os.path.join("examples_cache/", prompt, "feat.mp4")] + yield status_list + [ + gr.update(value="Run", variant="primary", visible=True), + gr.update(visible=False), + ] + [gr.update(interactive=True) for _ in range(INTERACTIVE_N)] + else: + save_root = '.' + mask = np.array(image['layers'][0])[..., 3] + bbox = create_bounding_boxes(mask) + if len(bbox) == 0: + status_list = ["You need to use the brush to mark the content of interest over the image!", "load log", None, None] + yield status_list + [ + gr.update(value="Run", variant="primary", visible=True), + gr.update(visible=False), + ] + [gr.update(interactive=True) for _ in range(INTERACTIVE_N)] + else: + bbox_str = "" + for each in bbox: + bbox_str = bbox_str + f"{each[1].start},{each[0].start},{each[1].stop},{each[0].stop}," + bbox_str = '[' + bbox_str + ']' + + # update status every 1 second + status_update_interval = 1 + + # save the config to a temporary file + config_file = tempfile.NamedTemporaryFile() + + with open(config_file.name, "w") as f: + f.write(config_yaml) + + # manually assign the output directory, name and tag so that we know the trial directory + name = os.path.basename(config_path).split(".")[0] + tag = prompt + '_' + datetime.now().strftime("%Y%m%d-%H%M%S") + trial_dir = os.path.join(save_root, EXP_ROOT_DIR, name, tag) + alive_path = os.path.join(trial_dir, "alive") + img_path = os.path.join(save_root, EXP_ROOT_DIR, f"{name}-{tag}.png") + Image.fromarray(np.array(image['background'])[...,:3]).save(img_path) + + width, height = image['background'].size + extras = [ + f'name="{name}"', + f'tag="{tag}"', + # "trainer.enable_progress_bar=false", + f"exp_root_dir={os.path.join(save_root, EXP_ROOT_DIR)}", + "use_timestamp=false", + f'system.prompt_processor.prompt="{prompt}"', + f'system.empty_prompt="{empty_prompt}"', + f'system.side_prompt="{side_prompt}"', + f"system.guidance.guidance_scale=5", + f"seed={seed}", + f"trainer.max_steps={max_steps}", + "trainer.num_sanity_val_steps=120", + "trainer.val_check_interval=100", + f"system.geometry.ooi_bbox={bbox_str}", + f"system.geometry.geometry_convert_from=depth:{img_path}", + f"system.geometry.img_resolution=[{width},{height}]", + f"data.width={width}", f"data.height={height}", f"data.eval_width={width}", f"data.eval_height={height}", + # system.outpaint_step=500 system.crop_with_lang=True system.guidance.max_step_percent=[0,0.5,0.1,1000] system.geometry.max_scaling=0.2 + ] + thread = threading.Thread(target=train, args=(extras, name, tag, config_file.name)) + try: + thread.start() + + while thread.is_alive(): + thread.join(timeout=1) + status = get_current_status(save_root, trial_dir, alive_path) + + yield status.tolist() + [ + gr.update(visible=False), + gr.update(value="Stop", variant="stop", visible=True), + ] + [gr.update(interactive=False) for _ in range(INTERACTIVE_N)] + + status.progress = 'Finished!' + load_ckpt(os.path.join(trial_dir, 'ckpts/last.ckpt')) + + yield status.tolist() + [ + gr.update(value="Run", variant="primary", visible=True), + gr.update(visible=False), + ] + [gr.update(interactive=True) for _ in range(INTERACTIVE_N)] + except: + del thread + gc.collect() + torch.cuda.empty_cache() + finally: + gc.collect() + torch.cuda.empty_cache() + + def stop_run(pid): + return [ + gr.update( + value="Please Refresh the Page", + variant="secondary", + visible=True, + interactive=False, + ), + gr.update(visible=False), + ] + + def inference_image(x_offset, y_offset, z_offset, rotate, prompt): + gc.collect() + torch.cuda.empty_cache() + if prompt in EXAMPLE_PROMPT_LIST: + load_ckpt(os.path.join("examples_cache/", prompt, "last.ckpt")) + # prune(system) + + system.geometry._opacity_mask = None + xyz_bak = system.geometry.get_xyz.data + rot_bak = system.geometry.get_rotation.data + + offset = torch.zeros(len(system.geometry._xyz), 3) + ooi_mask = system.geometry.ooi_masks_0.view(-1).byte().to(device='cuda').float() + + offset[ooi_mask.bool(), 0] = x_offset + offset[ooi_mask.bool(), 1] = y_offset + offset[ooi_mask.bool(), 2] = z_offset + system.geometry._xyz = torch.nn.Parameter(system.geometry._xyz + offset.cuda()) + + rot_matrix = rotation_matrix(0, 0, rotate).cuda() + prev_xyz = system.geometry.get_xyz.data + ooi_xyz = prev_xyz[ooi_mask.bool()] + mean = ooi_xyz.mean(0) + ooi_xyz = ooi_xyz - mean + after_xyz = torch.einsum('ab,nb->na', rot_matrix, ooi_xyz) + mean + prev_xyz[ooi_mask.bool()] = after_xyz + # system.geometry._xyz = torch.nn.Parameter(system.geometry._xyz + offset.cuda()) + system.geometry._xyz = torch.nn.Parameter(prev_xyz) + + prev_rotation = system.geometry.get_rotation.data + prev_rotation_mtx = build_rotation(prev_rotation) + after_rotation_mtx = torch.einsum('ab,nbc->nac', rot_matrix, prev_rotation_mtx) + after_rotation = torch.from_numpy(R.from_matrix(after_rotation_mtx.detach().cpu()).as_quat()).cuda().float() + after_rotation = torch.einsum('ab,nb->na', REORDER_MTX, after_rotation) + prev_rotation[ooi_mask.bool()] = after_rotation[ooi_mask.bool()] + system.geometry._rotation = torch.nn.Parameter(prev_rotation) + + with torch.no_grad(): + res = system(sampled_data[59]) + rgb = res['comp_rgb'][0].cpu().numpy() + rgb = (rgb * 255).astype(np.uint8) + + system.geometry._xyz = torch.nn.Parameter(xyz_bak.cuda()) + system.geometry._rotation = torch.nn.Parameter(rot_bak.cuda()) + return rgb + + def inference_video(x_offset, y_offset, z_offset, rotate, prompt): + gc.collect() + torch.cuda.empty_cache() + if prompt in EXAMPLE_PROMPT_LIST: + load_ckpt(os.path.join("examples_cache/", prompt, "last.ckpt")) + # prune(system) + + system.geometry._opacity_mask = None + video_writer = imageio.get_writer('result.mp4', fps=40) + xyz_bak = system.geometry.get_xyz.data + rot_bak = system.geometry.get_rotation.data + + offset = torch.zeros(len(system.geometry._xyz), 3) + ooi_mask = system.geometry.ooi_masks_0.view(-1).byte().to(device='cuda').float() + + offset[ooi_mask.bool(), 0] = x_offset + offset[ooi_mask.bool(), 1] = y_offset + offset[ooi_mask.bool(), 2] = z_offset + system.geometry._xyz = torch.nn.Parameter(system.geometry._xyz + offset.cuda()) + + rot_matrix = rotation_matrix(0, 0, rotate).cuda() + prev_xyz = system.geometry.get_xyz.data + ooi_xyz = prev_xyz[ooi_mask.bool()] + mean = ooi_xyz.mean(0) + ooi_xyz = ooi_xyz - mean + after_xyz = torch.einsum('ab,nb->na', rot_matrix, ooi_xyz) + mean + prev_xyz[ooi_mask.bool()] = after_xyz + system.geometry._xyz = torch.nn.Parameter(prev_xyz) + + prev_rotation = system.geometry.get_rotation.data + prev_rotation_mtx = build_rotation(prev_rotation) + after_rotation_mtx = torch.einsum('ab,nbc->nac', rot_matrix, prev_rotation_mtx) + after_rotation = torch.from_numpy(R.from_matrix(after_rotation_mtx.detach().cpu()).as_quat()).cuda().float() + after_rotation = torch.einsum('ab,nb->na', REORDER_MTX, after_rotation) + prev_rotation[ooi_mask.bool()] = after_rotation[ooi_mask.bool()] + system.geometry._rotation = torch.nn.Parameter(prev_rotation) + + with torch.no_grad(): + for i in range(120): + res = system(sampled_data[i]) + rgb = res['comp_rgb'][0].cpu().numpy() + rgb = (rgb * 255).astype(np.uint8) + video_writer.append_data(rgb) + video_writer.close() + + system.geometry._xyz = torch.nn.Parameter(xyz_bak.cuda()) + system.geometry._rotation = torch.nn.Parameter(rot_bak.cuda()) + return "result.mp4" + + self_deploy = self_deploy or "TS_SELF_DEPLOY" in os.environ + + css = """ + #examples {color: black !important} + .dark #examples {color: white !important} + #config-accordion, #logs-accordion {color: black !important;} + .dark #config-accordion, .dark #logs-accordion {color: white !important;} + .stop {background: darkred !important;} + """ + + with gr.Blocks( + title="3Dit Scene - Web Demo", + theme=gr.themes.Monochrome(), + css=css, + ) as demo: + with gr.Row(equal_height=True): + header = """ + # Web demo for 3DitScene: Editing Any Scene via Language-guided Disentangled Gaussian Splatting + +
+ + Duplicate Space +
+ + ### Usage + - Input an image and use the brush to mark the content of interest. + - Input the text prompt describing the image, the novel views content, and the background content. + - Hit the `Run` button to start optimization. + - After optimization, the interactive panel will be activated. You can control the object using the slider. + - Optionally, you could skip the optimization process by loading examples, and interact with them. + - If you experience long waiting time, consider duplicate this space. + - **IMPORTANT NOTE: Keep this tab active when running the model.** + """ + gr.Markdown(header) + + with gr.Row(equal_height=False): + pid = gr.State() + with gr.Column(scale=1): + # generation status + status = gr.Textbox( + value="Hit the Run button to start.", + label="Status", + lines=1, + max_lines=1, + ) + + # brush = gr.Brush(colors="rgba(100, 0, 0, 100)", default_color="rgba(100, 0, 0, 100)", color_mode="fixed") + # brush = gr.Brush(colors=['#FFFFFF']) + reference_image = gr.ImageEditor(label="Reference image", sources="upload", type="pil", transforms=None, layers=False) + + # prompt input + input_prompt = gr.Textbox(value=DEFAULT_PROMPT, label="Describe the image:") + side_prompt = gr.Textbox(value=DEFAULT_PROMPT_SIDE, label="Prompt the left and right areas in the novel view:") + empty_prompt = gr.Textbox(value=DEFAULT_PROMPT_EMPTY, label="Prompt the area behind the foreground object:") + + with gr.Row(): + # seed slider + seed_input = gr.Slider( + minimum=0, maximum=2147483647, value=0, step=1, label="Seed" + ) + + max_steps_input = gr.Slider( + minimum=100, + maximum=10000 if self_deploy else 2000, + value=3000 if self_deploy else 1000, + step=1, + label="Number of training steps", + ) + + run_btn = gr.Button(value="Run", variant="primary") + stop_btn = gr.Button(value="Stop", variant="stop", visible=False) + + with gr.Column(scale=1): + with gr.Accordion( + "See terminal logs", open=False, elem_id="logs-accordion", visible=False + ): + # logs + logs = gr.Textbox(label="Logs", lines=10) + + # validation image display + # output_image = gr.Image(value=None, label="Image") + + # testing video display + output_title = gr.Textbox( + value="Rendered RGB and semantic video", + label="The videos will keep updating during optimization. Stay tuned!", + lines=1, + max_lines=1, + ) + with gr.Row(): + output_video = gr.Video(value=None, label="Video") + output_feat_video = gr.Video(value=None, label="Video") + + interactive_status = gr.Textbox( + value="The panel is not interactive until training is finished or an example is loaded.", + label="Interactive Panel", + lines=1, + max_lines=1, + ) + + x_offset = gr.Slider( + minimum=-1, + maximum=1, + value=0, + step=0.1, + label="X offset", + visible=True, + interactive=False + ) + + y_offset = gr.Slider( + minimum=-1, + maximum=1, + value=0, + step=0.1, + label="Y offset", + visible=True, + interactive=False + ) + + z_offset = gr.Slider( + minimum=-1, + maximum=1, + value=0, + step=0.1, + label="Z offset", + visible=True, + interactive=False + ) + + rotate = gr.Slider( + minimum=-15, + maximum=15, + value=0, + step=1, + label="Rotation over Z", + visible=True, + interactive=False + ) + + with gr.Row(): + inference_image_btn = gr.Button(value="inference single frame", variant="primary", visible=True, interactive=False) + inference_video_btn = gr.Button(value="inference Video", variant="primary", visible=True, interactive=False) + + with gr.Row(): + edit_image = gr.Image(value=None, label="Edited image", interactive=False) + edit_video = gr.Video(value=None, label="Edited video", interactive=False) + + + interactive_component_list = [x_offset, y_offset, z_offset, rotate, inference_image_btn, inference_video_btn, edit_image, edit_video] + + inputs = [ + reference_image, + input_prompt, + side_prompt, + empty_prompt, + seed_input, + max_steps_input, + ] + + outputs = [ + # pid, + status, + logs, + # output_image, + output_video, + output_feat_video, + run_btn, + stop_btn, + ] + interactive_component_list + + run_event = run_btn.click( + fn=run, + inputs=inputs, + outputs=outputs, + concurrency_limit=1, + ) + + stop_btn.click( + fn=stop_run, + inputs=[pid], + outputs=[run_btn, stop_btn], + cancels=[run_event], + queue=False, + ) + + inference_image_btn.click( + fn=inference_image, + inputs=[x_offset, y_offset, z_offset, rotate, input_prompt], + outputs=edit_image, + ) + + inference_video_btn.click( + fn=inference_video, + inputs=[x_offset, y_offset, z_offset, rotate, input_prompt], + outputs=edit_video, + ) + + gr.Examples( + examples=[ + [{"background": "examples/bear_background.png", "layers": ["examples/bear_layers.png"], "composite": "examples/bear_composite.png"}, "a teddy bear at Times Square", "Times Square", "Times Square", 1, 1500, False], + [{"background": "examples/corgi_background.png", "layers": ["examples/corgi_layers.png"], "composite": "examples/corgi_composite.png"}, "Color photo of a corgi made of transparent glass, standing on the riverside in Yosemite National Park", "riverside in Yosemite National Park", "riverside in Yosemite National Park", 1, 1500, False], + [{"background": "examples/stairs_background.png", "layers": ["examples/stairs_layers.png"], "composite": "examples/stairs_composite.png"}, "two men climbing stairs", "stairs", "stairs", 1, 1500, False], + [{"background": "examples/boy_background.png", "layers": ["examples/boy_layers.png"], "composite": "examples/boy_composite.png"}, "a boy standing near the window", "a house", "empty", 1, 1500, False], + ], + inputs=inputs, + outputs=outputs, + fn=run, + cache_examples=True, + elem_id="examples" + ) + + launch_args = {"server_port": port} + if listen: + launch_args["server_name"] = "0.0.0.0" + + print('launch!', launch_args, flush=True) + + demo.queue(default_concurrency_limit=1).launch(**launch_args) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + args, extra = parser.parse_known_args() + + load_custom_modules() + + extras = ['tag=inference', 'system.prompt_processor.prompt=a bedroom, anime style', 'system.geometry.inference_only=true'] + cfg = load_config(config_path, cli_args=extras, n_gpus=1) + dm = threestudio.find(cfg.data_type)(cfg.data) + dm.setup(stage='test') + system = threestudio.find(cfg.system_type)(cfg.system, resumed=False) + + parser.add_argument("--listen", action="store_true") + parser.add_argument("--hf-space", action="store_true") + parser.add_argument("--self-deploy", action="store_true") + parser.add_argument("--save-root", type=str, default=".") + parser.add_argument("--port", type=int, default=7860) + args = parser.parse_args() + launch( + port=args.port, + listen=args.listen, + self_deploy=args.self_deploy, + save_root=args.save_root, + dm=dm, + system=system, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/launch.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/launch.py new file mode 100644 index 0000000000000000000000000000000000000000..23db40f1cbf59caee493bf221cb521764e3aad65 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/launch.py @@ -0,0 +1,312 @@ +import argparse +import contextlib +import importlib +import logging +import os +import sys +import time +import traceback + + +class ColoredFilter(logging.Filter): + """ + A logging filter to add color to certain log levels. + """ + + RESET = "\033[0m" + RED = "\033[31m" + GREEN = "\033[32m" + YELLOW = "\033[33m" + BLUE = "\033[34m" + MAGENTA = "\033[35m" + CYAN = "\033[36m" + + COLORS = { + "WARNING": YELLOW, + "INFO": GREEN, + "DEBUG": BLUE, + "CRITICAL": MAGENTA, + "ERROR": RED, + } + + RESET = "\x1b[0m" + + def __init__(self): + super().__init__() + + def filter(self, record): + if record.levelname in self.COLORS: + color_start = self.COLORS[record.levelname] + record.levelname = f"{color_start}[{record.levelname}]" + record.msg = f"{record.msg}{self.RESET}" + return True + + +def load_custom_module(module_path): + module_name = os.path.basename(module_path) + if os.path.isfile(module_path): + sp = os.path.splitext(module_path) + module_name = sp[0] + try: + if os.path.isfile(module_path): + module_spec = importlib.util.spec_from_file_location( + module_name, module_path + ) + else: + module_spec = importlib.util.spec_from_file_location( + module_name, os.path.join(module_path, "__init__.py") + ) + + module = importlib.util.module_from_spec(module_spec) + sys.modules[module_name] = module + module_spec.loader.exec_module(module) + return True + except Exception as e: + print(traceback.format_exc()) + print(f"Cannot import {module_path} module for custom nodes:", e) + return False + + +def load_custom_modules(): + node_paths = ["custom"] + node_import_times = [] + for custom_node_path in node_paths: + possible_modules = os.listdir(custom_node_path) + if "__pycache__" in possible_modules: + possible_modules.remove("__pycache__") + + for possible_module in possible_modules: + module_path = os.path.join(custom_node_path, possible_module) + if ( + os.path.isfile(module_path) + and os.path.splitext(module_path)[1] != ".py" + ): + continue + if module_path.endswith("_disabled"): + continue + time_before = time.perf_counter() + success = load_custom_module(module_path) + node_import_times.append( + (time.perf_counter() - time_before, module_path, success) + ) + + if len(node_import_times) > 0: + print("\nImport times for custom modules:") + for n in sorted(node_import_times): + if n[2]: + import_message = "" + else: + import_message = " (IMPORT FAILED)" + print("{:6.1f} seconds{}:".format(n[0], import_message), n[1]) + print() + + +def main(args, extras) -> None: + # set CUDA_VISIBLE_DEVICES if needed, then import pytorch-lightning + os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + env_gpus_str = os.environ.get("CUDA_VISIBLE_DEVICES", None) + env_gpus = list(env_gpus_str.split(",")) if env_gpus_str else [] + selected_gpus = [0] + + # Always rely on CUDA_VISIBLE_DEVICES if specific GPU ID(s) are specified. + # As far as Pytorch Lightning is concerned, we always use all available GPUs + # (possibly filtered by CUDA_VISIBLE_DEVICES). + devices = -1 + if len(env_gpus) > 0: + # CUDA_VISIBLE_DEVICES was set already, e.g. within SLURM srun or higher-level script. + n_gpus = len(env_gpus) + else: + selected_gpus = list(args.gpu.split(",")) + n_gpus = len(selected_gpus) + os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu + + import pytorch_lightning as pl + import torch + from pytorch_lightning import Trainer + from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint + from pytorch_lightning.loggers import CSVLogger, TensorBoardLogger + from pytorch_lightning.utilities.rank_zero import rank_zero_only + + if args.typecheck: + from jaxtyping import install_import_hook + + install_import_hook("threestudio", "typeguard.typechecked") + + import threestudio + from threestudio.systems.base import BaseSystem + from threestudio.utils.callbacks import ( + CodeSnapshotCallback, + ConfigSnapshotCallback, + CustomProgressBar, + ProgressCallback, + ) + from threestudio.utils.config import ExperimentConfig, load_config + from threestudio.utils.misc import get_rank + from threestudio.utils.typing import Optional + + logger = logging.getLogger("pytorch_lightning") + if args.verbose: + logger.setLevel(logging.DEBUG) + + for handler in logger.handlers: + if handler.stream == sys.stderr: # type: ignore + if not args.gradio: + handler.setFormatter(logging.Formatter("%(levelname)s %(message)s")) + handler.addFilter(ColoredFilter()) + else: + handler.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) + + load_custom_modules() + + # parse YAML config to OmegaConf + cfg: ExperimentConfig + cfg = load_config(args.config, cli_args=extras, n_gpus=n_gpus) + + # if args.save_dir is not None: + # cfg.exp_root_dir = args.save_dir + + # set a different seed for each device + pl.seed_everything(cfg.seed + get_rank(), workers=True) + + dm = threestudio.find(cfg.data_type)(cfg.data) + system: BaseSystem = threestudio.find(cfg.system_type)( + cfg.system, resumed=cfg.resume is not None + ) + system.set_save_dir(os.path.join(cfg.trial_dir, "save")) + + if args.gradio: + fh = logging.FileHandler(os.path.join(cfg.trial_dir, "logs")) + fh.setLevel(logging.INFO) + if args.verbose: + fh.setLevel(logging.DEBUG) + fh.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) + logger.addHandler(fh) + + callbacks = [] + if args.train: + callbacks += [ + ModelCheckpoint( + dirpath=os.path.join(cfg.trial_dir, "ckpts"), **cfg.checkpoint + ), + LearningRateMonitor(logging_interval="step"), + CodeSnapshotCallback( + os.path.join(cfg.trial_dir, "code"), use_version=False + ), + ConfigSnapshotCallback( + args.config, + cfg, + os.path.join(cfg.trial_dir, "configs"), + use_version=False, + ), + ] + if args.gradio: + callbacks += [ + ProgressCallback(save_path=os.path.join(cfg.trial_dir, "progress")) + ] + else: + callbacks += [CustomProgressBar(refresh_rate=1)] + + def write_to_text(file, lines): + with open(file, "w") as f: + for line in lines: + f.write(line + "\n") + + loggers = [] + if args.train: + # make tensorboard logging dir to suppress warning + rank_zero_only( + lambda: os.makedirs(os.path.join(cfg.trial_dir, "tb_logs"), exist_ok=True) + )() + loggers += [ + TensorBoardLogger(cfg.trial_dir, name="tb_logs"), + CSVLogger(cfg.trial_dir, name="csv_logs"), + ] + system.get_loggers() + rank_zero_only( + lambda: write_to_text( + os.path.join(cfg.trial_dir, "cmd.txt"), + ["python " + " ".join(sys.argv), str(args)], + ) + )() + + trainer = Trainer( + callbacks=callbacks, + logger=loggers, + inference_mode=False, + accelerator="gpu", + devices=devices, + **cfg.trainer, + ) + + def set_system_status(system: BaseSystem, ckpt_path: Optional[str]): + if ckpt_path is None: + return + ckpt = torch.load(ckpt_path, map_location="cpu") + system.set_resume_status(ckpt["epoch"], ckpt["global_step"]) + + if args.train: + trainer.fit(system, datamodule=dm, ckpt_path=cfg.resume) + trainer.test(system, datamodule=dm) + # if args.gradio: + # # also export assets if in gradio mode + # trainer.predict(system, datamodule=dm) + elif args.validate: + # manually set epoch and global_step as they cannot be automatically resumed + set_system_status(system, cfg.resume) + trainer.validate(system, datamodule=dm, ckpt_path=cfg.resume) + elif args.test: + # manually set epoch and global_step as they cannot be automatically resumed + set_system_status(system, cfg.resume) + trainer.test(system, datamodule=dm, ckpt_path=cfg.resume) + elif args.export: + set_system_status(system, cfg.resume) + trainer.predict(system, datamodule=dm, ckpt_path=cfg.resume) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True, help="path to config file") + parser.add_argument( + "--gpu", + default="0", + help="GPU(s) to be used. 0 means use the 1st available GPU. " + "1,2 means use the 2nd and 3rd available GPU. " + "If CUDA_VISIBLE_DEVICES is set before calling `launch.py`, " + "this argument is ignored and all available GPUs are always used.", + ) + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--train", action="store_true") + group.add_argument("--validate", action="store_true") + group.add_argument("--test", action="store_true") + group.add_argument("--export", action="store_true") + + # WU + # group.add_argument( + # "--rotate", action="store_true" + # ) + parser.add_argument( + "--save_dir", type=str, default=None + ) + + parser.add_argument( + "--gradio", action="store_true", help="if true, run in gradio mode" + ) + + parser.add_argument( + "--verbose", action="store_true", help="if true, set logging level to DEBUG" + ) + + parser.add_argument( + "--typecheck", + action="store_true", + help="whether to enable dynamic type checking", + ) + + args, extras = parser.parse_known_args() + + if args.gradio: + # FIXME: no effect, stdout is not captured + with contextlib.redirect_stdout(sys.stderr): + main(args, extras) + else: + main(args, extras) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/requirements.txt b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..d63f087c8b250fdaf20cdb30b65ab2b10182e8e6 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/requirements.txt @@ -0,0 +1,49 @@ +xformers==0.0.22 +# torch==2.0.1+cu118 torchvision==0.15.2+cu118 lightning==2.0.0 +lightning==2.0.0 +omegaconf==2.3.0 +jaxtyping +typeguard +# git+https://github.com/KAIR-BAIR/nerfacc.git@v0.5.2 +# git+https://github.com/NVlabs/tiny-cuda-nn/#subdirectory=bindings/torch +diffusers==0.25.0 +transformers==4.28.1 +accelerate +opencv-python +tensorboard +matplotlib +imageio>=2.28.0 +imageio[ffmpeg] +git+https://github.com/NVlabs/nvdiffrast.git +libigl +xatlas +trimesh[easy] +networkx +pysdf +PyMCubes +wandb +gradio==4.27.0 +git+https://github.com/ashawkey/envlight.git +torchmetrics +plyfile +scikit-learn + +# deepfloyd +xformers +bitsandbytes==0.38.1 +sentencepiece +safetensors +huggingface_hub + +# for zero123 +einops +kornia +taming-transformers-rom1504 +git+https://github.com/openai/CLIP.git + +#controlnet +controlnet_aux + +# bug fixed +huggingface_hub==0.24.6 +open-clip-torch diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/run.sh b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..a40157dc93789f02f0231017410c2c4a3ab45a3e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/run.sh @@ -0,0 +1,3 @@ +python -u launch.py --config custom/threestudio-3dgs/configs/scene_lang.yaml --train --gpu 0 tag=3DitScene \ +system.geometry.geometry_convert_from=depth:assets/teddy.png system.geometry.ooi_bbox=[122,119,387,495] \ +system.prompt_processor.prompt="a teddy bear in Times Square" system.empty_prompt="Times Square, out of focus" system.side_prompt="Times Square, out of focus" \ No newline at end of file diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/run_miradata.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/run_miradata.py new file mode 100644 index 0000000000000000000000000000000000000000..c3c1f47e4ef3e16b70512b3e7d9dc807be9a3a33 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/run_miradata.py @@ -0,0 +1,62 @@ +import os +import argparse +import pandas as pd + +# Set MKL threading layer +os.environ["MKL_THREADING_LAYER"] = "GNU" + +# Argument parser +parser = argparse.ArgumentParser(description="Process video frames and generate commands.") + +parser.add_argument("--csv_path", type=str, required=True, help="Path to the CSV file") +parser.add_argument("--gpu", type=int, default=0, help="GPU ID to use") + +args = parser.parse_args() + +# Load the CSV file +df = pd.read_csv(args.csv_path) + +# Iterate through each row +for index, row in df.iterrows(): + # Construct the image path from file_path + file_path = row["file_path"].replace("video_clips", "video_frames").replace(".mp4", "") + image_path = f"/mnt/hdd1/wufan/datasets/MiraData/data/{file_path}/0.jpg" + + # Check if the image exists before proceeding + if not os.path.exists(image_path): + print(f"Image not found: {image_path}. Skipping...") + continue + + # Extract prompt and context + prompt = row.get('short_caption', '') + prompt_context = row.get("context", '') + + # Create the save directory path + save_dir = "outputs/" + row["file_path"].replace("video_clips", "mira_video_clips").replace(".mp4", "") + + # Create directory if it doesn't exist + os.makedirs(save_dir, exist_ok=True) + + # Extract bounding box coordinates, ensuring they are valid integers + try: + x1 = str(int(row['x1'])) + y1 = str(int(row['y1'])) + x2 = str(int(row['x2'])) + y2 = str(int(row['y2'])) + except (ValueError, KeyError) as e: + print(f"Invalid bounding box for {image_path}. Skipping... Error: {e}") + continue + + # Construct the command + command = ( + f"python -u launch.py --config custom/threestudio-3dgs/configs/scene_lang.yaml --train --gpu {args.gpu} " + f"exp_root_dir={save_dir} tag=3DitScene " + f"system.geometry.geometry_convert_from=depth:{image_path} " + f"system.geometry.ooi_bbox=[{x1},{y1},{x2},{y2}] " + f"system.prompt_processor.prompt=\"{prompt}\" " + f"system.empty_prompt=\"{prompt_context}, out of focus\" " + f"system.side_prompt=\"{prompt_context}, out of focus\"" + ) + + print(f"Running command for index {index}:\n{command}\n") + os.system(command) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9b7b83bef2e3eb8f90010a8036a535fa5c022514 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/__init__.py @@ -0,0 +1,55 @@ +__modules__ = {} +__version__ = "0.2.3" + + +def register(name): + def decorator(cls): + if name in __modules__: + raise ValueError( + f"Module {name} already exists! Names of extensions conflict!" + ) + else: + __modules__[name] = cls + return cls + + return decorator + + +def find(name): + if ":" in name: + main_name, sub_name = name.split(":") + if "," in sub_name: + name_list = sub_name.split(",") + else: + name_list = [sub_name] + name_list.append(main_name) + NewClass = type( + f"{main_name}.{sub_name}", + tuple([__modules__[name] for name in name_list]), + {}, + ) + return NewClass + return __modules__[name] + + +### grammar sugar for logging utilities ### +import logging + +logger = logging.getLogger("pytorch_lightning") + +from pytorch_lightning.utilities.rank_zero import ( + rank_zero_debug, + rank_zero_info, + rank_zero_only, +) + +debug = rank_zero_debug +info = rank_zero_info + + +@rank_zero_only +def warn(*args, **kwargs): + logger.warn(*args, **kwargs) + + +from . import data, models, systems diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ce2e5cc7a9d6e37016c430e93208a3f1210fea20 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/__init__.py @@ -0,0 +1 @@ +from . import co3d, image, multiview, uncond diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/co3d.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/co3d.py new file mode 100644 index 0000000000000000000000000000000000000000..4916888b89f77963bf8af945b8ef779c02531681 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/co3d.py @@ -0,0 +1,713 @@ +import gzip +import json +import os +import warnings +from dataclasses import dataclass, field +from typing import List + +import cv2 +import numpy as np +import pytorch_lightning as pl +import torch +import torchvision.transforms.functional as TF +from PIL import Image +from torch.utils.data import DataLoader, Dataset, IterableDataset + +from threestudio import register +from threestudio.data.uncond import ( + RandomCameraDataModuleConfig, + RandomCameraDataset, + RandomCameraIterableDataset, +) +from threestudio.utils.config import parse_structured +from threestudio.utils.misc import get_rank +from threestudio.utils.ops import ( + get_mvp_matrix, + get_projection_matrix, + get_ray_directions, + get_rays, +) +from threestudio.utils.typing import * + + +def _load_16big_png_depth(depth_png) -> np.ndarray: + with Image.open(depth_png) as depth_pil: + # the image is stored with 16-bit depth but PIL reads it as I (32 bit). + # we cast it to uint16, then reinterpret as float16, then cast to float32 + depth = ( + np.frombuffer(np.array(depth_pil, dtype=np.uint16), dtype=np.float16) + .astype(np.float32) + .reshape((depth_pil.size[1], depth_pil.size[0])) + ) + return depth + + +def _load_depth(path, scale_adjustment) -> np.ndarray: + if not path.lower().endswith(".png"): + raise ValueError('unsupported depth file name "%s"' % path) + + d = _load_16big_png_depth(path) * scale_adjustment + d[~np.isfinite(d)] = 0.0 + return d[None] # fake feature channel + + +# Code adapted from https://github.com/eldar/snes/blob/473ff2b1f6/3rdparty/co3d/dataset/co3d_dataset.py +def _get_1d_bounds(arr): + nz = np.flatnonzero(arr) + return nz[0], nz[-1] + + +def get_bbox_from_mask(mask, thr, decrease_quant=0.05): + # bbox in xywh + masks_for_box = np.zeros_like(mask) + while masks_for_box.sum() <= 1.0: + masks_for_box = (mask > thr).astype(np.float32) + thr -= decrease_quant + if thr <= 0.0: + warnings.warn(f"Empty masks_for_bbox (thr={thr}) => using full image.") + + x0, x1 = _get_1d_bounds(masks_for_box.sum(axis=-2)) + y0, y1 = _get_1d_bounds(masks_for_box.sum(axis=-1)) + + return x0, y0, x1 - x0, y1 - y0 + + +def get_clamp_bbox(bbox, box_crop_context=0.0, impath=""): + # box_crop_context: rate of expansion for bbox + # returns possibly expanded bbox xyxy as float + + # increase box size + if box_crop_context > 0.0: + c = box_crop_context + bbox = bbox.astype(np.float32) + bbox[0] -= bbox[2] * c / 2 + bbox[1] -= bbox[3] * c / 2 + bbox[2] += bbox[2] * c + bbox[3] += bbox[3] * c + + if (bbox[2:] <= 1.0).any(): + warnings.warn(f"squashed image {impath}!!") + return None + + # bbox[2:] = np.clip(bbox[2:], 2, ) + bbox[2:] = np.maximum(bbox[2:], 2) + bbox[2:] += bbox[0:2] + 1 # convert to [xmin, ymin, xmax, ymax] + # +1 because upper bound is not inclusive + + return bbox + + +def crop_around_box(tensor, bbox, impath=""): + bbox[[0, 2]] = np.clip(bbox[[0, 2]], 0.0, tensor.shape[-2]) + bbox[[1, 3]] = np.clip(bbox[[1, 3]], 0.0, tensor.shape[-3]) + bbox = bbox.round().astype(np.longlong) + return tensor[bbox[1] : bbox[3], bbox[0] : bbox[2], ...] + + +def resize_image(image, height, width, mode="bilinear"): + if image.shape[:2] == (height, width): + return image, 1.0, np.ones_like(image[..., :1]) + + image = torch.from_numpy(image).permute(2, 0, 1) + minscale = min(height / image.shape[-2], width / image.shape[-1]) + imre = torch.nn.functional.interpolate( + image[None], + scale_factor=minscale, + mode=mode, + align_corners=False if mode == "bilinear" else None, + recompute_scale_factor=True, + )[0] + + # pyre-fixme[19]: Expected 1 positional argument. + imre_ = torch.zeros(image.shape[0], height, width) + imre_[:, 0 : imre.shape[1], 0 : imre.shape[2]] = imre + # pyre-fixme[6]: For 2nd param expected `int` but got `Optional[int]`. + # pyre-fixme[6]: For 3rd param expected `int` but got `Optional[int]`. + mask = torch.zeros(1, height, width) + mask[:, 0 : imre.shape[1], 0 : imre.shape[2]] = 1.0 + return imre_.permute(1, 2, 0).numpy(), minscale, mask.permute(1, 2, 0).numpy() + + +# Code adapted from https://github.com/POSTECH-CVLab/PeRFception/data_util/co3d.py +def similarity_from_cameras(c2w, fix_rot=False, radius=1.0): + """ + Get a similarity transform to normalize dataset + from c2w (OpenCV convention) cameras + :param c2w: (N, 4) + :return T (4,4) , scale (float) + """ + t = c2w[:, :3, 3] + R = c2w[:, :3, :3] + + # (1) Rotate the world so that z+ is the up axis + # we estimate the up axis by averaging the camera up axes + ups = np.sum(R * np.array([0, -1.0, 0]), axis=-1) + world_up = np.mean(ups, axis=0) + world_up /= np.linalg.norm(world_up) + + up_camspace = np.array([0.0, 0.0, 1.0]) + c = (up_camspace * world_up).sum() + cross = np.cross(world_up, up_camspace) + skew = np.array( + [ + [0.0, -cross[2], cross[1]], + [cross[2], 0.0, -cross[0]], + [-cross[1], cross[0], 0.0], + ] + ) + if c > -1: + R_align = np.eye(3) + skew + (skew @ skew) * 1 / (1 + c) + else: + # In the unlikely case the original data has y+ up axis, + # rotate 180-deg about x axis + R_align = np.array([[-1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]) + + if fix_rot: + R_align = np.eye(3) + R = np.eye(3) + else: + R = R_align @ R + fwds = np.sum(R * np.array([0, 0.0, 1.0]), axis=-1) + t = (R_align @ t[..., None])[..., 0] + + # (2) Recenter the scene using camera center rays + # find the closest point to the origin for each camera's center ray + nearest = t + (fwds * -t).sum(-1)[:, None] * fwds + + # median for more robustness + translate = -np.median(nearest, axis=0) + + # translate = -np.mean(t, axis=0) # DEBUG + + transform = np.eye(4) + transform[:3, 3] = translate + transform[:3, :3] = R_align + + # (3) Rescale the scene using camera distances + scale = radius / np.median(np.linalg.norm(t + translate, axis=-1)) + + return transform, scale + + +@dataclass +class Co3dDataModuleConfig: + root_dir: str = "" + batch_size: int = 1 + height: int = 256 + width: int = 256 + load_preprocessed: bool = False + cam_scale_factor: float = 0.95 + max_num_frames: int = 300 + v2_mode: bool = True + use_mask: bool = True + box_crop: bool = True + box_crop_mask_thr: float = 0.4 + box_crop_context: float = 0.3 + train_num_rays: int = -1 + train_views: Optional[list] = None + train_split: str = "train" + val_split: str = "val" + test_split: str = "test" + scale_radius: float = 1.0 + use_random_camera: bool = True + random_camera: dict = field(default_factory=dict) + rays_noise_scale: float = 0.0 + render_path: str = "circle" + + +class Co3dDatasetBase: + def setup(self, cfg, split): + self.split = split + self.rank = get_rank() + self.cfg: Co3dDataModuleConfig = cfg + + if self.cfg.use_random_camera: + random_camera_cfg = parse_structured( + RandomCameraDataModuleConfig, self.cfg.get("random_camera", {}) + ) + if split == "train": + self.random_pose_generator = RandomCameraIterableDataset( + random_camera_cfg + ) + else: + self.random_pose_generator = RandomCameraDataset( + random_camera_cfg, split + ) + + self.use_mask = self.cfg.use_mask + cam_scale_factor = self.cfg.cam_scale_factor + + assert os.path.exists(self.cfg.root_dir), f"{self.cfg.root_dir} doesn't exist!" + + cam_trans = np.diag(np.array([-1, -1, 1, 1], dtype=np.float32)) + scene_number = self.cfg.root_dir.split("/")[-1] + json_path = os.path.join(self.cfg.root_dir, "..", "frame_annotations.jgz") + with gzip.open(json_path, "r") as fp: + all_frames_data = json.load(fp) + + frame_data, images, intrinsics, extrinsics, image_sizes = [], [], [], [], [] + masks = [] + depths = [] + + for temporal_data in all_frames_data: + if temporal_data["sequence_name"] == scene_number: + frame_data.append(temporal_data) + + self.all_directions = [] + self.all_fg_masks = [] + for frame in frame_data: + if "unseen" in frame["meta"]["frame_type"]: + continue + img = cv2.imread( + os.path.join(self.cfg.root_dir, "..", "..", frame["image"]["path"]) + ) + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) / 255.0 + + # TODO: use estimated depth + depth = _load_depth( + os.path.join(self.cfg.root_dir, "..", "..", frame["depth"]["path"]), + frame["depth"]["scale_adjustment"], + )[0] + + H, W = frame["image"]["size"] + image_size = np.array([H, W]) + fxy = np.array(frame["viewpoint"]["focal_length"]) + cxy = np.array(frame["viewpoint"]["principal_point"]) + R = np.array(frame["viewpoint"]["R"]) + T = np.array(frame["viewpoint"]["T"]) + + if self.cfg.v2_mode: + min_HW = min(W, H) + image_size_half = np.array([W * 0.5, H * 0.5], dtype=np.float32) + scale_arr = np.array([min_HW * 0.5, min_HW * 0.5], dtype=np.float32) + fxy_x = fxy * scale_arr + prp_x = np.array([W * 0.5, H * 0.5], dtype=np.float32) - cxy * scale_arr + cxy = (image_size_half - prp_x) / image_size_half + fxy = fxy_x / image_size_half + + scale_arr = np.array([W * 0.5, H * 0.5], dtype=np.float32) + focal = fxy * scale_arr + prp = -1.0 * (cxy - 1.0) * scale_arr + + pose = np.eye(4) + pose[:3, :3] = R + pose[:3, 3:] = -R @ T[..., None] + # original camera: x left, y up, z in (Pytorch3D) + # transformed camera: x right, y down, z in (OpenCV) + pose = pose @ cam_trans + intrinsic = np.array( + [ + [focal[0], 0.0, prp[0], 0.0], + [0.0, focal[1], prp[1], 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ] + ) + + if any([np.all(pose == _pose) for _pose in extrinsics]): + continue + + image_sizes.append(image_size) + intrinsics.append(intrinsic) + extrinsics.append(pose) + images.append(img) + depths.append(depth) + self.all_directions.append(get_ray_directions(W, H, focal, prp)) + + # vis_utils.vis_depth_pcd([depth], [pose], intrinsic, [(img * 255).astype(np.uint8)]) + + if self.use_mask: + mask = np.array( + Image.open( + os.path.join( + self.cfg.root_dir, "..", "..", frame["mask"]["path"] + ) + ) + ) + mask = mask.astype(np.float32) / 255.0 # (h, w) + else: + mask = torch.ones_like(img[..., 0]) + self.all_fg_masks.append(mask) + + intrinsics = np.stack(intrinsics) + extrinsics = np.stack(extrinsics) + image_sizes = np.stack(image_sizes) + self.all_directions = torch.stack(self.all_directions, dim=0) + self.all_fg_masks = np.stack(self.all_fg_masks, 0) + + H_median, W_median = np.median( + np.stack([image_size for image_size in image_sizes]), axis=0 + ) + + H_inlier = np.abs(image_sizes[:, 0] - H_median) / H_median < 0.1 + W_inlier = np.abs(image_sizes[:, 1] - W_median) / W_median < 0.1 + inlier = np.logical_and(H_inlier, W_inlier) + dists = np.linalg.norm( + extrinsics[:, :3, 3] - np.median(extrinsics[:, :3, 3], axis=0), axis=-1 + ) + med = np.median(dists) + good_mask = dists < (med * 5.0) + inlier = np.logical_and(inlier, good_mask) + + if inlier.sum() != 0: + intrinsics = intrinsics[inlier] + extrinsics = extrinsics[inlier] + image_sizes = image_sizes[inlier] + images = [images[i] for i in range(len(inlier)) if inlier[i]] + depths = [depths[i] for i in range(len(inlier)) if inlier[i]] + self.all_directions = self.all_directions[inlier] + self.all_fg_masks = self.all_fg_masks[inlier] + + extrinsics = np.stack(extrinsics) + T, sscale = similarity_from_cameras(extrinsics, radius=self.cfg.scale_radius) + extrinsics = T @ extrinsics + + extrinsics[:, :3, 3] *= sscale * cam_scale_factor + + depths = [depth * sscale * cam_scale_factor for depth in depths] + + num_frames = len(extrinsics) + + if self.cfg.max_num_frames < num_frames: + num_frames = self.cfg.max_num_frames + extrinsics = extrinsics[:num_frames] + intrinsics = intrinsics[:num_frames] + image_sizes = image_sizes[:num_frames] + images = images[:num_frames] + depths = depths[:num_frames] + self.all_directions = self.all_directions[:num_frames] + self.all_fg_masks = self.all_fg_masks[:num_frames] + + if self.cfg.box_crop: + print("cropping...") + crop_masks = [] + crop_imgs = [] + crop_depths = [] + crop_directions = [] + crop_xywhs = [] + max_sl = 0 + for i in range(num_frames): + bbox_xywh = np.array( + get_bbox_from_mask(self.all_fg_masks[i], self.cfg.box_crop_mask_thr) + ) + clamp_bbox_xywh = get_clamp_bbox(bbox_xywh, self.cfg.box_crop_context) + max_sl = max(clamp_bbox_xywh[2] - clamp_bbox_xywh[0], max_sl) + max_sl = max(clamp_bbox_xywh[3] - clamp_bbox_xywh[1], max_sl) + mask = crop_around_box(self.all_fg_masks[i][..., None], clamp_bbox_xywh) + img = crop_around_box(images[i], clamp_bbox_xywh) + depth = crop_around_box(depths[i][..., None], clamp_bbox_xywh) + + # resize to the same shape + mask, _, _ = resize_image(mask, self.cfg.height, self.cfg.width) + depth, _, _ = resize_image(depth, self.cfg.height, self.cfg.width) + img, scale, _ = resize_image(img, self.cfg.height, self.cfg.width) + fx, fy, cx, cy = ( + intrinsics[i][0, 0], + intrinsics[i][1, 1], + intrinsics[i][0, 2], + intrinsics[i][1, 2], + ) + + crop_masks.append(mask) + crop_imgs.append(img) + crop_depths.append(depth) + crop_xywhs.append(clamp_bbox_xywh) + crop_directions.append( + get_ray_directions( + self.cfg.height, + self.cfg.width, + (fx * scale, fy * scale), + ( + (cx - clamp_bbox_xywh[0]) * scale, + (cy - clamp_bbox_xywh[1]) * scale, + ), + ) + ) + + # # pad all images to the same shape + # for i in range(num_frames): + # uh = (max_sl - crop_imgs[i].shape[0]) // 2 # h + # dh = max_sl - crop_imgs[i].shape[0] - uh + # lw = (max_sl - crop_imgs[i].shape[1]) // 2 + # rw = max_sl - crop_imgs[i].shape[1] - lw + # crop_masks[i] = np.pad(crop_masks[i], pad_width=((uh, dh), (lw, rw), (0, 0)), mode='constant', constant_values=0.) + # crop_imgs[i] = np.pad(crop_imgs[i], pad_width=((uh, dh), (lw, rw), (0, 0)), mode='constant', constant_values=1.) + # crop_depths[i] = np.pad(crop_depths[i], pad_width=((uh, dh), (lw, rw), (0, 0)), mode='constant', constant_values=0.) + # fx, fy, cx, cy = intrinsics[i][0, 0], intrinsics[i][1, 1], intrinsics[i][0, 2], intrinsics[i][1, 2] + # crop_directions.append(get_ray_directions(max_sl, max_sl, (fx, fy), (cx - crop_xywhs[i][0] + lw, cy - crop_xywhs[i][1] + uh))) + # self.w, self.h = max_sl, max_sl + + images = crop_imgs + depths = crop_depths + self.all_fg_masks = np.stack(crop_masks, 0) + self.all_directions = torch.from_numpy(np.stack(crop_directions, 0)) + + # self.width, self.height = self.w, self.h + + self.all_c2w = torch.from_numpy( + ( + extrinsics + @ np.diag(np.array([1, -1, -1, 1], dtype=np.float32))[None, ...] + )[..., :3, :4] + ) + self.all_images = torch.from_numpy(np.stack(images, axis=0)) + self.all_depths = torch.from_numpy(np.stack(depths, axis=0)) + + # self.all_c2w = [] + # self.all_images = [] + # for i in range(num_frames): + # # convert to: x right, y up, z back (OpenGL) + # c2w = torch.from_numpy(extrinsics[i] @ np.diag(np.array([1, -1, -1, 1], dtype=np.float32)))[:3, :4] + # self.all_c2w.append(c2w) + # img = torch.from_numpy(images[i]) + # self.all_images.append(img) + + # TODO: save data for fast loading next time + if self.cfg.load_preprocessed and os.path.exists( + self.cfg.root_dir, "nerf_preprocessed.npy" + ): + pass + + i_all = np.arange(num_frames) + + if self.cfg.train_views is None: + i_test = i_all[::10] + i_val = i_test + i_train = np.array([i for i in i_all if not i in i_test]) + else: + # use provided views + i_train = self.cfg.train_views + i_test = np.array([i for i in i_all if not i in i_train]) + i_val = i_test + + if self.split == "train": + print("[INFO] num of train views: ", len(i_train)) + print("[INFO] train view ids = ", i_train) + + i_split = {"train": i_train, "val": i_val, "test": i_all} + + # if self.split == 'test': + # self.all_c2w = create_spheric_poses(self.all_c2w[:,:,3], n_steps=self.cfg.n_test_traj_steps) + # self.all_images = torch.zeros((self.cfg.n_test_traj_steps, self.h, self.w, 3), dtype=torch.float32) + # self.all_fg_masks = torch.zeros((self.cfg.n_test_traj_steps, self.h, self.w), dtype=torch.float32) + # self.directions = self.directions[0].to(self.rank) + # else: + self.all_images, self.all_c2w = ( + self.all_images[i_split[self.split]], + self.all_c2w[i_split[self.split]], + ) + self.all_directions = self.all_directions[i_split[self.split]].to(self.rank) + self.all_fg_masks = torch.from_numpy(self.all_fg_masks)[i_split[self.split]] + self.all_depths = self.all_depths[i_split[self.split]] + # if render_random_pose: + # render_poses = random_pose(extrinsics[i_all], 50) + # elif render_scene_interp: + # render_poses = pose_interp(extrinsics[i_all], interp_fac) + # render_poses = spherical_poses(sscale * cam_scale_factor * np.eye(4)) + + # near, far = 0., 1. + # ndc_coeffs = (-1., -1.) + + self.all_c2w, self.all_images, self.all_fg_masks = ( + self.all_c2w.float().to(self.rank), + self.all_images.float().to(self.rank), + self.all_fg_masks.float().to(self.rank), + ) + + # self.all_c2w, self.all_images, self.all_fg_masks = \ + # self.all_c2w.float(), \ + # self.all_images.float(), \ + # self.all_fg_masks.float() + + self.all_depths = self.all_depths.float().to(self.rank) + + def get_all_images(self): + return self.all_images + + +class Co3dDataset(Dataset, Co3dDatasetBase): + def __init__(self, cfg, split): + self.setup(cfg, split) + + def __len__(self): + if self.split == "test": + if self.cfg.render_path == "circle": + return len(self.random_pose_generator) + else: + return len(self.all_images) + else: + return len(self.random_pose_generator) + # return len(self.all_images) + + def prepare_data(self, index): + # prepare batch data here + c2w = self.all_c2w[index] + light_positions = c2w[..., :3, -1] + directions = self.all_directions[index] + rays_o, rays_d = get_rays( + directions, c2w, keepdim=True, noise_scale=self.cfg.rays_noise_scale + ) + rgb = self.all_images[index] + depth = self.all_depths[index] + mask = self.all_fg_masks[index] + + # TODO: get projection matrix and mvp matrix + # proj_mtx = get_projection_matrix() + + batch = { + "rays_o": rays_o, + "rays_d": rays_d, + "mvp_mtx": 0, + "camera_positions": c2w[..., :3, -1], + "light_positions": light_positions, + "elevation": 0, + "azimuth": 0, + "camera_distances": 0, + "rgb": rgb, + "depth": depth, + "mask": mask, + } + + # c2w = self.all_c2w[index] + # return { + # 'index': index, + # 'c2w': c2w, + # 'light_positions': c2w[:3, -1], + # 'H': self.h, + # 'W': self.w + # } + + return batch + + def __getitem__(self, index): + if self.split == "test": + if self.cfg.render_path == "circle": + return self.random_pose_generator[index] + else: + return self.prepare_data(index) + else: + return self.random_pose_generator[index] + + +class Co3dIterableDataset(IterableDataset, Co3dDatasetBase): + def __init__(self, cfg, split): + self.setup(cfg, split) + self.idx = 0 + self.image_perm = torch.randperm(len(self.all_images)) + + def __iter__(self): + while True: + yield {} + + def collate(self, batch) -> Dict[str, Any]: + idx = self.image_perm[self.idx] + # prepare batch data here + c2w = self.all_c2w[idx][None] + light_positions = c2w[..., :3, -1] + directions = self.all_directions[idx][None] + rays_o, rays_d = get_rays( + directions, c2w, keepdim=True, noise_scale=self.cfg.rays_noise_scale + ) + rgb = self.all_images[idx][None] + depth = self.all_depths[idx][None] + mask = self.all_fg_masks[idx][None] + + if ( + self.cfg.train_num_rays != -1 + and self.cfg.train_num_rays < self.cfg.height * self.cfg.width + ): + _, height, width, _ = rays_o.shape + x = torch.randint( + 0, width, size=(self.cfg.train_num_rays,), device=rays_o.device + ) + y = torch.randint( + 0, height, size=(self.cfg.train_num_rays,), device=rays_o.device + ) + + rays_o = rays_o[:, y, x].unsqueeze(-2) + rays_d = rays_d[:, y, x].unsqueeze(-2) + directions = directions[:, y, x].unsqueeze(-2) + rgb = rgb[:, y, x].unsqueeze(-2) + mask = mask[:, y, x].unsqueeze(-2) + depth = depth[:, y, x].unsqueeze(-2) + + # TODO: get projection matrix and mvp matrix + # proj_mtx = get_projection_matrix() + + batch = { + "rays_o": rays_o, + "rays_d": rays_d, + "mvp_mtx": None, + "camera_positions": c2w[..., :3, -1], + "light_positions": light_positions, + "elevation": None, + "azimuth": None, + "camera_distances": None, + "rgb": rgb, + "depth": depth, + "mask": mask, + } + + if self.cfg.use_random_camera: + batch["random_camera"] = self.random_pose_generator.collate(None) + + # prepare batch data in system + # c2w = self.all_c2w[idx][None] + + # batch = { + # 'index': torch.tensor([idx]), + # 'c2w': c2w, + # 'light_positions': c2w[..., :3, -1], + # 'H': self.h, + # 'W': self.w + # } + + self.idx += 1 + if self.idx == len(self.all_images): + self.idx = 0 + self.image_perm = torch.randperm(len(self.all_images)) + # self.idx = (self.idx + 1) % len(self.all_images) + + return batch + + +@register("co3d-datamodule") +class Co3dDataModule(pl.LightningDataModule): + def __init__(self, cfg: Optional[Union[dict, DictConfig]] = None) -> None: + super().__init__() + self.cfg = parse_structured(Co3dDataModuleConfig, cfg) + + def setup(self, stage=None): + if stage in [None, "fit"]: + self.train_dataset = Co3dIterableDataset(self.cfg, self.cfg.train_split) + if stage in [None, "fit", "validate"]: + self.val_dataset = Co3dDataset(self.cfg, self.cfg.val_split) + if stage in [None, "test", "predict"]: + self.test_dataset = Co3dDataset(self.cfg, self.cfg.test_split) + + def prepare_data(self): + pass + + def general_loader(self, dataset, batch_size, collate_fn=None) -> DataLoader: + sampler = None + return DataLoader( + dataset, + num_workers=0, + batch_size=batch_size, + # pin_memory=True, + collate_fn=collate_fn, + ) + + def train_dataloader(self): + return self.general_loader( + self.train_dataset, batch_size=1, collate_fn=self.train_dataset.collate + ) + + def val_dataloader(self): + return self.general_loader(self.val_dataset, batch_size=1) + + def test_dataloader(self): + return self.general_loader(self.test_dataset, batch_size=1) + + def predict_dataloader(self): + return self.general_loader(self.test_dataset, batch_size=1) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/image.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/image.py new file mode 100644 index 0000000000000000000000000000000000000000..033c528f2100f70371b66894e66e4edaf4261f63 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/image.py @@ -0,0 +1,350 @@ +import bisect +import math +import os +from dataclasses import dataclass, field + +import cv2 +import numpy as np +import pytorch_lightning as pl +import torch +import torch.nn.functional as F +from torch.utils.data import DataLoader, Dataset, IterableDataset + +import threestudio +from threestudio import register +from threestudio.data.uncond import ( + RandomCameraDataModuleConfig, + RandomCameraDataset, + RandomCameraIterableDataset, +) +from threestudio.utils.base import Updateable +from threestudio.utils.config import parse_structured +from threestudio.utils.misc import get_rank +from threestudio.utils.ops import ( + get_mvp_matrix, + get_projection_matrix, + get_ray_directions, + get_rays, +) +from threestudio.utils.typing import * + + +@dataclass +class SingleImageDataModuleConfig: + # height and width should be Union[int, List[int]] + # but OmegaConf does not support Union of containers + height: Any = 96 + width: Any = 96 + resolution_milestones: List[int] = field(default_factory=lambda: []) + default_elevation_deg: float = 0.0 + default_azimuth_deg: float = -180.0 + default_camera_distance: float = 1.2 + default_fovy_deg: float = 60.0 + image_path: str = "" + use_random_camera: bool = True + random_camera: dict = field(default_factory=dict) + rays_noise_scale: float = 2e-3 + batch_size: int = 1 + requires_depth: bool = False + requires_normal: bool = False + + rays_d_normalize: bool = True + + +class SingleImageDataBase: + def setup(self, cfg, split): + self.split = split + self.rank = get_rank() + self.cfg: SingleImageDataModuleConfig = cfg + + if self.cfg.use_random_camera: + random_camera_cfg = parse_structured( + RandomCameraDataModuleConfig, self.cfg.get("random_camera", {}) + ) + if split == "train": + self.random_pose_generator = RandomCameraIterableDataset( + random_camera_cfg + ) + else: + self.random_pose_generator = RandomCameraDataset( + random_camera_cfg, split + ) + + elevation_deg = torch.FloatTensor([self.cfg.default_elevation_deg]) + azimuth_deg = torch.FloatTensor([self.cfg.default_azimuth_deg]) + camera_distance = torch.FloatTensor([self.cfg.default_camera_distance]) + + elevation = elevation_deg * math.pi / 180 + azimuth = azimuth_deg * math.pi / 180 + camera_position: Float[Tensor, "1 3"] = torch.stack( + [ + camera_distance * torch.cos(elevation) * torch.cos(azimuth), + camera_distance * torch.cos(elevation) * torch.sin(azimuth), + camera_distance * torch.sin(elevation), + ], + dim=-1, + ) + + center: Float[Tensor, "1 3"] = torch.zeros_like(camera_position) + up: Float[Tensor, "1 3"] = torch.as_tensor([0, 0, 1], dtype=torch.float32)[None] + + light_position: Float[Tensor, "1 3"] = camera_position + lookat: Float[Tensor, "1 3"] = F.normalize(center - camera_position, dim=-1) + right: Float[Tensor, "1 3"] = F.normalize(torch.cross(lookat, up), dim=-1) + up = F.normalize(torch.cross(right, lookat), dim=-1) + self.c2w: Float[Tensor, "1 3 4"] = torch.cat( + [torch.stack([right, up, -lookat], dim=-1), camera_position[:, :, None]], + dim=-1, + ) + self.c2w4x4: Float[Tensor, "B 4 4"] = torch.cat( + [self.c2w, torch.zeros_like(self.c2w[:, :1])], dim=1 + ) + self.c2w4x4[:, 3, 3] = 1.0 + + self.camera_position = camera_position + self.light_position = light_position + self.elevation_deg, self.azimuth_deg = elevation_deg, azimuth_deg + self.camera_distance = camera_distance + self.fovy = torch.deg2rad(torch.FloatTensor([self.cfg.default_fovy_deg])) + + self.heights: List[int] = ( + [self.cfg.height] if isinstance(self.cfg.height, int) else self.cfg.height + ) + self.widths: List[int] = ( + [self.cfg.width] if isinstance(self.cfg.width, int) else self.cfg.width + ) + assert len(self.heights) == len(self.widths) + self.resolution_milestones: List[int] + if len(self.heights) == 1 and len(self.widths) == 1: + if len(self.cfg.resolution_milestones) > 0: + threestudio.warn( + "Ignoring resolution_milestones since height and width are not changing" + ) + self.resolution_milestones = [-1] + else: + assert len(self.heights) == len(self.cfg.resolution_milestones) + 1 + self.resolution_milestones = [-1] + self.cfg.resolution_milestones + + self.directions_unit_focals = [ + get_ray_directions(H=height, W=width, focal=1.0) + for (height, width) in zip(self.heights, self.widths) + ] + self.focal_lengths = [ + 0.5 * height / torch.tan(0.5 * self.fovy) for height in self.heights + ] + + self.height: int = self.heights[0] + self.width: int = self.widths[0] + self.directions_unit_focal = self.directions_unit_focals[0] + self.focal_length = self.focal_lengths[0] + self.set_rays() + self.load_images() + self.prev_height = self.height + + def set_rays(self): + # get directions by dividing directions_unit_focal by focal length + directions: Float[Tensor, "1 H W 3"] = self.directions_unit_focal[None] + directions[:, :, :, :2] = directions[:, :, :, :2] / self.focal_length + + rays_o, rays_d = get_rays( + directions, + self.c2w, + keepdim=True, + noise_scale=self.cfg.rays_noise_scale, + normalize=self.cfg.rays_d_normalize, + ) + + proj_mtx: Float[Tensor, "4 4"] = get_projection_matrix( + self.fovy, self.width / self.height, 0.1, 100.0 + ) # FIXME: hard-coded near and far + mvp_mtx: Float[Tensor, "4 4"] = get_mvp_matrix(self.c2w, proj_mtx) + + self.rays_o, self.rays_d = rays_o, rays_d + self.mvp_mtx = mvp_mtx + + def load_images(self): + # load image + assert os.path.exists( + self.cfg.image_path + ), f"Could not find image {self.cfg.image_path}!" + rgba = cv2.cvtColor( + cv2.imread(self.cfg.image_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGRA2RGBA + ) + rgba = ( + cv2.resize( + rgba, (self.width, self.height), interpolation=cv2.INTER_AREA + ).astype(np.float32) + / 255.0 + ) + rgb = rgba[..., :3] + self.rgb: Float[Tensor, "1 H W 3"] = ( + torch.from_numpy(rgb).unsqueeze(0).contiguous().to(self.rank) + ) + self.mask: Float[Tensor, "1 H W 1"] = ( + torch.from_numpy(rgba[..., 3:] > 0.5).unsqueeze(0).to(self.rank) + ) + print( + f"[INFO] single image dataset: load image {self.cfg.image_path} {self.rgb.shape}" + ) + + # load depth + if self.cfg.requires_depth: + depth_path = self.cfg.image_path.replace("_rgba.png", "_depth.png") + assert os.path.exists(depth_path) + depth = cv2.imread(depth_path, cv2.IMREAD_UNCHANGED) + depth = cv2.resize( + depth, (self.width, self.height), interpolation=cv2.INTER_AREA + ) + self.depth: Float[Tensor, "1 H W 1"] = ( + torch.from_numpy(depth.astype(np.float32) / 255.0) + .unsqueeze(0) + .to(self.rank) + ) + print( + f"[INFO] single image dataset: load depth {depth_path} {self.depth.shape}" + ) + else: + self.depth = None + + # load normal + if self.cfg.requires_normal: + normal_path = self.cfg.image_path.replace("_rgba.png", "_normal.png") + assert os.path.exists(normal_path) + normal = cv2.imread(normal_path, cv2.IMREAD_UNCHANGED) + normal = cv2.resize( + normal, (self.width, self.height), interpolation=cv2.INTER_AREA + ) + self.normal: Float[Tensor, "1 H W 3"] = ( + torch.from_numpy(normal.astype(np.float32) / 255.0) + .unsqueeze(0) + .to(self.rank) + ) + print( + f"[INFO] single image dataset: load normal {normal_path} {self.normal.shape}" + ) + else: + self.normal = None + + def get_all_images(self): + return self.rgb + + def update_step_(self, epoch: int, global_step: int, on_load_weights: bool = False): + size_ind = bisect.bisect_right(self.resolution_milestones, global_step) - 1 + self.height = self.heights[size_ind] + if self.height == self.prev_height: + return + + self.prev_height = self.height + self.width = self.widths[size_ind] + self.directions_unit_focal = self.directions_unit_focals[size_ind] + self.focal_length = self.focal_lengths[size_ind] + threestudio.debug(f"Training height: {self.height}, width: {self.width}") + self.set_rays() + self.load_images() + + +class SingleImageIterableDataset(IterableDataset, SingleImageDataBase, Updateable): + def __init__(self, cfg: Any, split: str) -> None: + super().__init__() + self.setup(cfg, split) + + def collate(self, batch) -> Dict[str, Any]: + batch = { + "rays_o": self.rays_o, + "rays_d": self.rays_d, + "mvp_mtx": self.mvp_mtx, + "camera_positions": self.camera_position, + "light_positions": self.light_position, + "elevation": self.elevation_deg, + "azimuth": self.azimuth_deg, + "camera_distances": self.camera_distance, + "rgb": self.rgb, + "ref_depth": self.depth, + "ref_normal": self.normal, + "mask": self.mask, + "height": self.height, + "width": self.width, + "c2w": self.c2w4x4, + "fovy": self.fovy, + } + if self.cfg.use_random_camera: + batch["random_camera"] = self.random_pose_generator.collate(None) + + return batch + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + self.update_step_(epoch, global_step, on_load_weights) + self.random_pose_generator.update_step(epoch, global_step, on_load_weights) + + def __iter__(self): + while True: + yield {} + + +class SingleImageDataset(Dataset, SingleImageDataBase): + def __init__(self, cfg: Any, split: str) -> None: + super().__init__() + self.setup(cfg, split) + + def __len__(self): + return len(self.random_pose_generator) + + def __getitem__(self, index): + return self.random_pose_generator[index] + # if index == 0: + # return { + # 'rays_o': self.rays_o[0], + # 'rays_d': self.rays_d[0], + # 'mvp_mtx': self.mvp_mtx[0], + # 'camera_positions': self.camera_position[0], + # 'light_positions': self.light_position[0], + # 'elevation': self.elevation_deg[0], + # 'azimuth': self.azimuth_deg[0], + # 'camera_distances': self.camera_distance[0], + # 'rgb': self.rgb[0], + # 'depth': self.depth[0], + # 'mask': self.mask[0] + # } + # else: + # return self.random_pose_generator[index - 1] + + +@register("single-image-datamodule") +class SingleImageDataModule(pl.LightningDataModule): + cfg: SingleImageDataModuleConfig + + def __init__(self, cfg: Optional[Union[dict, DictConfig]] = None) -> None: + super().__init__() + self.cfg = parse_structured(SingleImageDataModuleConfig, cfg) + + def setup(self, stage=None) -> None: + if stage in [None, "fit"]: + self.train_dataset = SingleImageIterableDataset(self.cfg, "train") + if stage in [None, "fit", "validate"]: + self.val_dataset = SingleImageDataset(self.cfg, "val") + if stage in [None, "test", "predict"]: + self.test_dataset = SingleImageDataset(self.cfg, "test") + + def prepare_data(self): + pass + + def general_loader(self, dataset, batch_size, collate_fn=None) -> DataLoader: + return DataLoader( + dataset, num_workers=0, batch_size=batch_size, collate_fn=collate_fn + ) + + def train_dataloader(self) -> DataLoader: + return self.general_loader( + self.train_dataset, + batch_size=self.cfg.batch_size, + collate_fn=self.train_dataset.collate, + ) + + def val_dataloader(self) -> DataLoader: + return self.general_loader(self.val_dataset, batch_size=1) + + def test_dataloader(self) -> DataLoader: + return self.general_loader(self.test_dataset, batch_size=1) + + def predict_dataloader(self) -> DataLoader: + return self.general_loader(self.test_dataset, batch_size=1) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/multiview.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/multiview.py new file mode 100644 index 0000000000000000000000000000000000000000..8b722624a0846df3859215504480e42e9b1fc149 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/multiview.py @@ -0,0 +1,435 @@ +import json +import math +import os +import random +from dataclasses import dataclass + +import cv2 +import numpy as np +import pytorch_lightning as pl +import torch +import torch.nn.functional as F +from scipy.spatial.transform import Rotation as Rot +from scipy.spatial.transform import Slerp +from torch.utils.data import DataLoader, Dataset, IterableDataset +from tqdm import tqdm + +import threestudio +from threestudio import register +from threestudio.utils.config import parse_structured +from threestudio.utils.ops import get_mvp_matrix, get_ray_directions, get_rays +from threestudio.utils.typing import * + + +def convert_pose(C2W): + flip_yz = torch.eye(4) + flip_yz[1, 1] = -1 + flip_yz[2, 2] = -1 + C2W = torch.matmul(C2W, flip_yz) + return C2W + + +def convert_proj(K, H, W, near, far): + return [ + [2 * K[0, 0] / W, -2 * K[0, 1] / W, (W - 2 * K[0, 2]) / W, 0], + [0, -2 * K[1, 1] / H, (H - 2 * K[1, 2]) / H, 0], + [0, 0, (-far - near) / (far - near), -2 * far * near / (far - near)], + [0, 0, -1, 0], + ] + + +def inter_pose(pose_0, pose_1, ratio): + pose_0 = pose_0.detach().cpu().numpy() + pose_1 = pose_1.detach().cpu().numpy() + pose_0 = np.linalg.inv(pose_0) + pose_1 = np.linalg.inv(pose_1) + rot_0 = pose_0[:3, :3] + rot_1 = pose_1[:3, :3] + rots = Rot.from_matrix(np.stack([rot_0, rot_1])) + key_times = [0, 1] + slerp = Slerp(key_times, rots) + rot = slerp(ratio) + pose = np.diag([1.0, 1.0, 1.0, 1.0]) + pose = pose.astype(np.float32) + pose[:3, :3] = rot.as_matrix() + pose[:3, 3] = ((1.0 - ratio) * pose_0 + ratio * pose_1)[:3, 3] + pose = np.linalg.inv(pose) + return pose + + +@dataclass +class MultiviewsDataModuleConfig: + dataroot: str = "" + train_downsample_resolution: int = 4 + eval_downsample_resolution: int = 4 + train_data_interval: int = 1 + eval_data_interval: int = 1 + batch_size: int = 1 + eval_batch_size: int = 1 + camera_layout: str = "around" + camera_distance: float = -1 + eval_interpolation: Optional[Tuple[int, int, int]] = None # (0, 1, 30) + + rays_d_normalize: bool = True + + +class MultiviewIterableDataset(IterableDataset): + def __init__(self, cfg: Any) -> None: + super().__init__() + self.cfg: MultiviewsDataModuleConfig = cfg + + assert self.cfg.batch_size == 1 + scale = self.cfg.train_downsample_resolution + + camera_dict = json.load( + open(os.path.join(self.cfg.dataroot, "transforms.json"), "r") + ) + assert camera_dict["camera_model"] == "OPENCV" + + frames = camera_dict["frames"] + frames = frames[:: self.cfg.train_data_interval] + frames_proj = [] + frames_c2w = [] + frames_position = [] + frames_direction = [] + frames_img = [] + + self.frame_w = frames[0]["w"] // scale + self.frame_h = frames[0]["h"] // scale + threestudio.info("Loading frames...") + self.n_frames = len(frames) + + c2w_list = [] + for frame in tqdm(frames): + extrinsic: Float[Tensor, "4 4"] = torch.as_tensor( + frame["transform_matrix"], dtype=torch.float32 + ) + c2w = extrinsic + c2w_list.append(c2w) + c2w_list = torch.stack(c2w_list, dim=0) + + if self.cfg.camera_layout == "around": + c2w_list[:, :3, 3] -= torch.mean(c2w_list[:, :3, 3], dim=0).unsqueeze(0) + elif self.cfg.camera_layout == "front": + assert self.cfg.camera_distance > 0 + c2w_list[:, :3, 3] -= torch.mean(c2w_list[:, :3, 3], dim=0).unsqueeze(0) + z_vector = torch.zeros(c2w_list.shape[0], 3, 1) + z_vector[:, 2, :] = -1 + rot_z_vector = c2w_list[:, :3, :3] @ z_vector + rot_z_vector = torch.mean(rot_z_vector, dim=0).unsqueeze(0) + c2w_list[:, :3, 3] -= rot_z_vector[:, :, 0] * self.cfg.camera_distance + else: + raise ValueError( + f"Unknown camera layout {self.cfg.camera_layout}. Now support only around and front." + ) + + for idx, frame in tqdm(enumerate(frames)): + intrinsic: Float[Tensor, "4 4"] = torch.eye(4) + intrinsic[0, 0] = frame["fl_x"] / scale + intrinsic[1, 1] = frame["fl_y"] / scale + intrinsic[0, 2] = frame["cx"] / scale + intrinsic[1, 2] = frame["cy"] / scale + + frame_path = os.path.join(self.cfg.dataroot, frame["file_path"]) + img = cv2.imread(frame_path)[:, :, ::-1].copy() + img = cv2.resize(img, (self.frame_w, self.frame_h)) + img: Float[Tensor, "H W 3"] = torch.FloatTensor(img) / 255 + frames_img.append(img) + + direction: Float[Tensor, "H W 3"] = get_ray_directions( + self.frame_h, + self.frame_w, + (intrinsic[0, 0], intrinsic[1, 1]), + (intrinsic[0, 2], intrinsic[1, 2]), + use_pixel_centers=False, + ) + + c2w = c2w_list[idx] + camera_position: Float[Tensor, "3"] = c2w[:3, 3:].reshape(-1) + + near = 0.1 + far = 1000.0 + proj = convert_proj(intrinsic, self.frame_h, self.frame_w, near, far) + proj: Float[Tensor, "4 4"] = torch.FloatTensor(proj) + frames_proj.append(proj) + frames_c2w.append(c2w) + frames_position.append(camera_position) + frames_direction.append(direction) + threestudio.info("Loaded frames.") + + self.frames_proj: Float[Tensor, "B 4 4"] = torch.stack(frames_proj, dim=0) + self.frames_c2w: Float[Tensor, "B 4 4"] = torch.stack(frames_c2w, dim=0) + self.frames_position: Float[Tensor, "B 3"] = torch.stack(frames_position, dim=0) + self.frames_direction: Float[Tensor, "B H W 3"] = torch.stack( + frames_direction, dim=0 + ) + self.frames_img: Float[Tensor, "B H W 3"] = torch.stack(frames_img, dim=0) + + self.rays_o, self.rays_d = get_rays( + self.frames_direction, + self.frames_c2w, + keepdim=True, + normalize=self.cfg.rays_d_normalize, + ) + self.mvp_mtx: Float[Tensor, "B 4 4"] = get_mvp_matrix( + self.frames_c2w, self.frames_proj + ) + self.light_positions: Float[Tensor, "B 3"] = torch.zeros_like( + self.frames_position + ) + + def __iter__(self): + while True: + yield {} + + def collate(self, batch): + index = torch.randint(0, self.n_frames, (1,)).item() + return { + "index": index, + "rays_o": self.rays_o[index : index + 1], + "rays_d": self.rays_d[index : index + 1], + "mvp_mtx": self.mvp_mtx[index : index + 1], + "c2w": self.frames_c2w[index : index + 1], + "camera_positions": self.frames_position[index : index + 1], + "light_positions": self.light_positions[index : index + 1], + "gt_rgb": self.frames_img[index : index + 1], + "height": self.frame_h, + "width": self.frame_w, + } + + +class MultiviewDataset(Dataset): + def __init__(self, cfg: Any, split: str) -> None: + super().__init__() + self.cfg: MultiviewsDataModuleConfig = cfg + + assert self.cfg.eval_batch_size == 1 + scale = self.cfg.eval_downsample_resolution + + camera_dict = json.load( + open(os.path.join(self.cfg.dataroot, "transforms.json"), "r") + ) + assert camera_dict["camera_model"] == "OPENCV" + + frames = camera_dict["frames"] + frames = frames[:: self.cfg.eval_data_interval] + frames_proj = [] + frames_c2w = [] + frames_position = [] + frames_direction = [] + frames_img = [] + + self.frame_w = frames[0]["w"] // scale + self.frame_h = frames[0]["h"] // scale + threestudio.info("Loading frames...") + self.n_frames = len(frames) + + c2w_list = [] + for frame in tqdm(frames): + extrinsic: Float[Tensor, "4 4"] = torch.as_tensor( + frame["transform_matrix"], dtype=torch.float32 + ) + c2w = extrinsic + c2w_list.append(c2w) + c2w_list = torch.stack(c2w_list, dim=0) + + if self.cfg.camera_layout == "around": + c2w_list[:, :3, 3] -= torch.mean(c2w_list[:, :3, 3], dim=0).unsqueeze(0) + elif self.cfg.camera_layout == "front": + assert self.cfg.camera_distance > 0 + c2w_list[:, :3, 3] -= torch.mean(c2w_list[:, :3, 3], dim=0).unsqueeze(0) + z_vector = torch.zeros(c2w_list.shape[0], 3, 1) + z_vector[:, 2, :] = -1 + rot_z_vector = c2w_list[:, :3, :3] @ z_vector + rot_z_vector = torch.mean(rot_z_vector, dim=0).unsqueeze(0) + c2w_list[:, :3, 3] -= rot_z_vector[:, :, 0] * self.cfg.camera_distance + else: + raise ValueError( + f"Unknown camera layout {self.cfg.camera_layout}. Now support only around and front." + ) + + if not (self.cfg.eval_interpolation is None): + idx0 = self.cfg.eval_interpolation[0] + idx1 = self.cfg.eval_interpolation[1] + eval_nums = self.cfg.eval_interpolation[2] + frame = frames[idx0] + intrinsic: Float[Tensor, "4 4"] = torch.eye(4) + intrinsic[0, 0] = frame["fl_x"] / scale + intrinsic[1, 1] = frame["fl_y"] / scale + intrinsic[0, 2] = frame["cx"] / scale + intrinsic[1, 2] = frame["cy"] / scale + for ratio in np.linspace(0, 1, eval_nums): + img: Float[Tensor, "H W 3"] = torch.zeros( + (self.frame_h, self.frame_w, 3) + ) + frames_img.append(img) + direction: Float[Tensor, "H W 3"] = get_ray_directions( + self.frame_h, + self.frame_w, + (intrinsic[0, 0], intrinsic[1, 1]), + (intrinsic[0, 2], intrinsic[1, 2]), + use_pixel_centers=False, + ) + + c2w = torch.FloatTensor( + inter_pose(c2w_list[idx0], c2w_list[idx1], ratio) + ) + camera_position: Float[Tensor, "3"] = c2w[:3, 3:].reshape(-1) + + near = 0.1 + far = 1000.0 + proj = convert_proj(intrinsic, self.frame_h, self.frame_w, near, far) + proj: Float[Tensor, "4 4"] = torch.FloatTensor(proj) + frames_proj.append(proj) + frames_c2w.append(c2w) + frames_position.append(camera_position) + frames_direction.append(direction) + else: + for idx, frame in tqdm(enumerate(frames)): + intrinsic: Float[Tensor, "4 4"] = torch.eye(4) + intrinsic[0, 0] = frame["fl_x"] / scale + intrinsic[1, 1] = frame["fl_y"] / scale + intrinsic[0, 2] = frame["cx"] / scale + intrinsic[1, 2] = frame["cy"] / scale + + frame_path = os.path.join(self.cfg.dataroot, frame["file_path"]) + img = cv2.imread(frame_path)[:, :, ::-1].copy() + img = cv2.resize(img, (self.frame_w, self.frame_h)) + img: Float[Tensor, "H W 3"] = torch.FloatTensor(img) / 255 + frames_img.append(img) + + direction: Float[Tensor, "H W 3"] = get_ray_directions( + self.frame_h, + self.frame_w, + (intrinsic[0, 0], intrinsic[1, 1]), + (intrinsic[0, 2], intrinsic[1, 2]), + use_pixel_centers=False, + ) + + c2w = c2w_list[idx] + camera_position: Float[Tensor, "3"] = c2w[:3, 3:].reshape(-1) + + near = 0.1 + far = 1000.0 + K = intrinsic + proj = [ + [ + 2 * K[0, 0] / self.frame_w, + -2 * K[0, 1] / self.frame_w, + (self.frame_w - 2 * K[0, 2]) / self.frame_w, + 0, + ], + [ + 0, + -2 * K[1, 1] / self.frame_h, + (self.frame_h - 2 * K[1, 2]) / self.frame_h, + 0, + ], + [ + 0, + 0, + (-far - near) / (far - near), + -2 * far * near / (far - near), + ], + [0, 0, -1, 0], + ] + proj: Float[Tensor, "4 4"] = torch.FloatTensor(proj) + frames_proj.append(proj) + frames_c2w.append(c2w) + frames_position.append(camera_position) + frames_direction.append(direction) + threestudio.info("Loaded frames.") + + self.frames_proj: Float[Tensor, "B 4 4"] = torch.stack(frames_proj, dim=0) + self.frames_c2w: Float[Tensor, "B 4 4"] = torch.stack(frames_c2w, dim=0) + self.frames_position: Float[Tensor, "B 3"] = torch.stack(frames_position, dim=0) + self.frames_direction: Float[Tensor, "B H W 3"] = torch.stack( + frames_direction, dim=0 + ) + self.frames_img: Float[Tensor, "B H W 3"] = torch.stack(frames_img, dim=0) + + self.rays_o, self.rays_d = get_rays( + self.frames_direction, + self.frames_c2w, + keepdim=True, + normalize=self.cfg.rays_d_normalize, + ) + self.mvp_mtx: Float[Tensor, "B 4 4"] = get_mvp_matrix( + self.frames_c2w, self.frames_proj + ) + self.light_positions: Float[Tensor, "B 3"] = torch.zeros_like( + self.frames_position + ) + + def __len__(self): + return self.frames_proj.shape[0] + + def __getitem__(self, index): + return { + "index": index, + "rays_o": self.rays_o[index], + "rays_d": self.rays_d[index], + "mvp_mtx": self.mvp_mtx[index], + "c2w": self.frames_c2w[index], + "camera_positions": self.frames_position[index], + "light_positions": self.light_positions[index], + "gt_rgb": self.frames_img[index], + } + + def __iter__(self): + while True: + yield {} + + def collate(self, batch): + batch = torch.utils.data.default_collate(batch) + batch.update({"height": self.frame_h, "width": self.frame_w}) + return batch + + +@register("multiview-camera-datamodule") +class MultiviewDataModule(pl.LightningDataModule): + cfg: MultiviewsDataModuleConfig + + def __init__(self, cfg: Optional[Union[dict, DictConfig]] = None) -> None: + super().__init__() + self.cfg = parse_structured(MultiviewsDataModuleConfig, cfg) + + def setup(self, stage=None) -> None: + if stage in [None, "fit"]: + self.train_dataset = MultiviewIterableDataset(self.cfg) + if stage in [None, "fit", "validate"]: + self.val_dataset = MultiviewDataset(self.cfg, "val") + if stage in [None, "test", "predict"]: + self.test_dataset = MultiviewDataset(self.cfg, "test") + + def prepare_data(self): + pass + + def general_loader(self, dataset, batch_size, collate_fn=None) -> DataLoader: + return DataLoader( + dataset, + num_workers=1, # type: ignore + batch_size=batch_size, + collate_fn=collate_fn, + ) + + def train_dataloader(self) -> DataLoader: + return self.general_loader( + self.train_dataset, batch_size=None, collate_fn=self.train_dataset.collate + ) + + def val_dataloader(self) -> DataLoader: + return self.general_loader( + self.val_dataset, batch_size=1, collate_fn=self.val_dataset.collate + ) + # return self.general_loader(self.train_dataset, batch_size=None, collate_fn=self.train_dataset.collate) + + def test_dataloader(self) -> DataLoader: + return self.general_loader( + self.test_dataset, batch_size=1, collate_fn=self.test_dataset.collate + ) + + def predict_dataloader(self) -> DataLoader: + return self.general_loader( + self.test_dataset, batch_size=1, collate_fn=self.test_dataset.collate + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/uncond.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/uncond.py new file mode 100644 index 0000000000000000000000000000000000000000..0e286c4c603b50a29db483cf1a00c25893eb5169 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/data/uncond.py @@ -0,0 +1,754 @@ +import bisect +import math +import random +from dataclasses import dataclass, field + +import numpy as np +import pytorch_lightning as pl +import torch +import torch.nn.functional as F +from torch.utils.data import DataLoader, Dataset, IterableDataset + +import threestudio +from threestudio import register +from threestudio.utils.base import Updateable +from threestudio.utils.config import parse_structured +from threestudio.utils.misc import get_device +from threestudio.utils.ops import ( + get_full_projection_matrix, + get_mvp_matrix, + get_projection_matrix, + get_ray_directions, + get_rays, +) +from threestudio.utils.typing import * + + +@dataclass +class RandomCameraDataModuleConfig: + # height, width, and batch_size should be Union[int, List[int]] + # but OmegaConf does not support Union of containers + height: Any = 64 + width: Any = 64 + batch_size: Any = 1 + resolution_milestones: List[int] = field(default_factory=lambda: []) + eval_height: int = 512 + eval_width: int = 512 + eval_batch_size: int = 1 + n_val_views: int = 120 + n_test_views: int = 120 + elevation_range: Tuple[float, float] = (-10, 90) + azimuth_range: Tuple[float, float] = (-180, 180) + val_azimuth_range: Tuple[float, float] = (0, 360) + camera_distance_range: Tuple[float, float] = (1, 1.5) + fovy_range: Tuple[float, float] = ( + 40, + 70, + ) # in degrees, in vertical direction (along height) + camera_perturb: float = 0.1 + center_perturb: float = 0.2 + up_perturb: float = 0.02 + light_position_perturb: float = 1.0 + light_distance_range: Tuple[float, float] = (0.8, 1.5) + eval_elevation_deg: float = 15.0 + eval_camera_distance: float = 1.5 + eval_fovy_deg: float = 70.0 + light_sample_strategy: str = "dreamfusion" + batch_uniform_azimuth: bool = True + progressive_until: int = 0 # progressive ranges for elevation, azimuth, r, fovy + + rays_d_normalize: bool = True + insert_zero: bool = False + + # WU + rotate_traj: bool = False + random_traj: bool = False + + +class RandomCameraIterableDataset(IterableDataset, Updateable): + def __init__(self, cfg: Any) -> None: + super().__init__() + self.cfg: RandomCameraDataModuleConfig = cfg + self.heights: List[int] = ( + [self.cfg.height] if isinstance(self.cfg.height, int) else self.cfg.height + ) + self.widths: List[int] = ( + [self.cfg.width] if isinstance(self.cfg.width, int) else self.cfg.width + ) + self.batch_sizes: List[int] = ( + [self.cfg.batch_size] + if isinstance(self.cfg.batch_size, int) + else self.cfg.batch_size + ) + assert len(self.heights) == len(self.widths) == len(self.batch_sizes) + self.resolution_milestones: List[int] + if ( + len(self.heights) == 1 + and len(self.widths) == 1 + and len(self.batch_sizes) == 1 + ): + if len(self.cfg.resolution_milestones) > 0: + threestudio.warn( + "Ignoring resolution_milestones since height and width are not changing" + ) + self.resolution_milestones = [-1] + else: + assert len(self.heights) == len(self.cfg.resolution_milestones) + 1 + self.resolution_milestones = [-1] + self.cfg.resolution_milestones + + self.directions_unit_focals = [ + get_ray_directions(H=height, W=width, focal=1.0) + for (height, width) in zip(self.heights, self.widths) + ] + self.height: int = self.heights[0] + self.width: int = self.widths[0] + self.batch_size: int = self.batch_sizes[0] + self.directions_unit_focal = self.directions_unit_focals[0] + self.elevation_range = self.cfg.elevation_range + self.azimuth_range = self.cfg.azimuth_range + self.camera_distance_range = self.cfg.camera_distance_range + self.fovy_range = self.cfg.fovy_range + self.insert_zero = self.cfg.insert_zero + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + size_ind = bisect.bisect_right(self.resolution_milestones, global_step) - 1 + self.height = self.heights[size_ind] + self.width = self.widths[size_ind] + self.batch_size = self.batch_sizes[size_ind] + self.directions_unit_focal = self.directions_unit_focals[size_ind] + threestudio.debug( + f"Training height: {self.height}, width: {self.width}, batch_size: {self.batch_size}" + ) + # progressive view + self.progressive_view(global_step) + + def __iter__(self): + while True: + yield {} + + def progressive_view(self, global_step): + r = min(1.0, global_step / (self.cfg.progressive_until + 1)) + self.elevation_range = [ + (1 - r) * self.cfg.eval_elevation_deg + r * self.cfg.elevation_range[0], + (1 - r) * self.cfg.eval_elevation_deg + r * self.cfg.elevation_range[1], + ] + self.azimuth_range = [ + (1 - r) * 0.0 + r * self.cfg.azimuth_range[0], + (1 - r) * 0.0 + r * self.cfg.azimuth_range[1], + ] + # self.camera_distance_range = [ + # (1 - r) * self.cfg.eval_camera_distance + # + r * self.cfg.camera_distance_range[0], + # (1 - r) * self.cfg.eval_camera_distance + # + r * self.cfg.camera_distance_range[1], + # ] + # self.fovy_range = [ + # (1 - r) * self.cfg.eval_fovy_deg + r * self.cfg.fovy_range[0], + # (1 - r) * self.cfg.eval_fovy_deg + r * self.cfg.fovy_range[1], + # ] + + def collate(self, batch) -> Dict[str, Any]: + # sample elevation angles + elevation_deg: Float[Tensor, "B"] + elevation: Float[Tensor, "B"] + if random.random() < 0.5: + # sample elevation angles uniformly with a probability 0.5 (biased towards poles) + elevation_deg = ( + torch.rand(self.batch_size) + * (self.elevation_range[1] - self.elevation_range[0]) + + self.elevation_range[0] + ) + if self.insert_zero: + elevation_deg = torch.cat([torch.tensor([self.cfg.eval_elevation_deg]), elevation_deg]) + elevation = elevation_deg * math.pi / 180 + else: + # otherwise sample uniformly on sphere + elevation_range_percent = [ + self.elevation_range[0] / 180.0 * math.pi, + self.elevation_range[1] / 180.0 * math.pi, + ] + # inverse transform sampling + elevation = torch.asin( + ( + torch.rand(self.batch_size) + * ( + math.sin(elevation_range_percent[1]) + - math.sin(elevation_range_percent[0]) + ) + + math.sin(elevation_range_percent[0]) + ) + ) + elevation_deg = elevation / math.pi * 180.0 + if self.insert_zero: + elevation_deg = torch.cat([torch.tensor([self.cfg.eval_elevation_deg]), elevation_deg]) + elevation = elevation_deg / 180.0 * math.pi + + # sample azimuth angles from a uniform distribution bounded by azimuth_range + azimuth_deg: Float[Tensor, "B"] + if self.cfg.batch_uniform_azimuth: + # ensures sampled azimuth angles in a batch cover the whole range + azimuth_deg = ( + torch.rand(self.batch_size) + torch.arange(self.batch_size) + ) / self.batch_size * ( + self.azimuth_range[1] - self.azimuth_range[0] + ) + self.azimuth_range[ + 0 + ] + else: + # simple random sampling + azimuth_deg = ( + torch.rand(self.batch_size) + * (self.azimuth_range[1] - self.azimuth_range[0]) + + self.azimuth_range[0] + ) + if self.insert_zero: + azimuth_deg = torch.cat([torch.tensor([0.0]), azimuth_deg]) + azimuth = azimuth_deg * math.pi / 180 + + # sample distances from a uniform distribution bounded by distance_range + camera_distances: Float[Tensor, "B"] = ( + torch.rand(self.batch_size) + * (self.camera_distance_range[1] - self.camera_distance_range[0]) + + self.camera_distance_range[0] + ) + if self.insert_zero: + camera_distances = torch.cat([torch.tensor([self.cfg.eval_camera_distance]), camera_distances]) + + # convert spherical coordinates to cartesian coordinates + # right hand coordinate system, x back, y right, z up + # elevation in (-90, 90), azimuth from +x to +y in (-180, 180) + camera_positions: Float[Tensor, "B 3"] = torch.stack( + [ + camera_distances * torch.cos(elevation) * torch.cos(azimuth), + camera_distances * torch.cos(elevation) * torch.sin(azimuth), + camera_distances * torch.sin(elevation), + ], + dim=-1, + ) + + # default scene center at origin + center: Float[Tensor, "B 3"] = torch.zeros_like(camera_positions) + # default camera up direction as +z + up: Float[Tensor, "B 3"] = torch.as_tensor([0, 0, 1], dtype=torch.float32)[ + None, : + ].repeat(self.batch_size + self.insert_zero, 1) + + # sample camera perturbations from a uniform distribution [-camera_perturb, camera_perturb] + camera_perturb: Float[Tensor, "B 3"] = ( + torch.rand(self.batch_size, 3) * 2 * self.cfg.camera_perturb + - self.cfg.camera_perturb + ) + if self.insert_zero: + camera_perturb = torch.cat([torch.zeros([1, 3]), camera_perturb]) + + camera_positions = camera_positions + camera_perturb + # sample center perturbations from a normal distribution with mean 0 and std center_perturb + center_perturb: Float[Tensor, "B 3"] = ( + torch.randn(self.batch_size, 3) * self.cfg.center_perturb + ) + if self.insert_zero: + center_perturb = torch.cat([torch.zeros([1, 3]), center_perturb]) + center = center + center_perturb + # sample up perturbations from a normal distribution with mean 0 and std up_perturb + up_perturb: Float[Tensor, "B 3"] = ( + torch.randn(self.batch_size, 3) * self.cfg.up_perturb + ) + if self.insert_zero: + up_perturb = torch.cat([torch.zeros([1, 3]), up_perturb]) + up = up + up_perturb + + # sample fovs from a uniform distribution bounded by fov_range + fovy_deg: Float[Tensor, "B"] = ( + torch.rand(self.batch_size) * (self.fovy_range[1] - self.fovy_range[0]) + + self.fovy_range[0] + ) + if self.insert_zero: + fovy_deg = torch.cat([torch.tensor([self.cfg.eval_fovy_deg]), fovy_deg]) + + fovy = fovy_deg * math.pi / 180 + + # sample light distance from a uniform distribution bounded by light_distance_range + light_distances: Float[Tensor, "B"] = ( + torch.rand(self.batch_size + self.insert_zero) + * (self.cfg.light_distance_range[1] - self.cfg.light_distance_range[0]) + + self.cfg.light_distance_range[0] + ) + + if self.cfg.light_sample_strategy == "dreamfusion": + # sample light direction from a normal distribution with mean camera_position and std light_position_perturb + light_direction: Float[Tensor, "B 3"] = F.normalize( + camera_positions + + torch.randn(self.batch_size + self.insert_zero, 3) * self.cfg.light_position_perturb, + dim=-1, + ) + # get light position by scaling light direction by light distance + light_positions: Float[Tensor, "B 3"] = ( + light_direction * light_distances[:, None] + ) + elif self.cfg.light_sample_strategy == "magic3d": + # sample light direction within restricted angle range (pi/3) + local_z = F.normalize(camera_positions, dim=-1) + local_x = F.normalize( + torch.stack( + [local_z[:, 1], -local_z[:, 0], torch.zeros_like(local_z[:, 0])], + dim=-1, + ), + dim=-1, + ) + local_y = F.normalize(torch.cross(local_z, local_x, dim=-1), dim=-1) + rot = torch.stack([local_x, local_y, local_z], dim=-1) + light_azimuth = ( + torch.rand(self.batch_size + self.insert_zero) * math.pi * 2 - math.pi + ) # [-pi, pi] + light_elevation = ( + torch.rand(self.batch_size + self.insert_zero) * math.pi / 3 + math.pi / 6 + ) # [pi/6, pi/2] + light_positions_local = torch.stack( + [ + light_distances + * torch.cos(light_elevation) + * torch.cos(light_azimuth), + light_distances + * torch.cos(light_elevation) + * torch.sin(light_azimuth), + light_distances * torch.sin(light_elevation), + ], + dim=-1, + ) + light_positions = (rot @ light_positions_local[:, :, None])[:, :, 0] + else: + raise ValueError( + f"Unknown light sample strategy: {self.cfg.light_sample_strategy}" + ) + + lookat: Float[Tensor, "B 3"] = F.normalize(center - camera_positions, dim=-1) + right: Float[Tensor, "B 3"] = F.normalize(torch.cross(lookat, up), dim=-1) + up = F.normalize(torch.cross(right, lookat), dim=-1) + c2w3x4: Float[Tensor, "B 3 4"] = torch.cat( + [torch.stack([right, up, -lookat], dim=-1), camera_positions[:, :, None]], + dim=-1, + ) + c2w: Float[Tensor, "B 4 4"] = torch.cat( + [c2w3x4, torch.zeros_like(c2w3x4[:, :1])], dim=1 + ) + c2w[:, 3, 3] = 1.0 + + # get directions by dividing directions_unit_focal by focal length + focal_length: Float[Tensor, "B"] = 0.5 * self.height / torch.tan(0.5 * fovy) + directions: Float[Tensor, "B H W 3"] = self.directions_unit_focal[ + None, :, :, : + ].repeat(self.batch_size + self.insert_zero, 1, 1, 1) + directions[:, :, :, :2] = ( + directions[:, :, :, :2] / focal_length[:, None, None, None] + ) + + # Importance note: the returned rays_d MUST be normalized! + rays_o, rays_d = get_rays( + directions, c2w, keepdim=True, normalize=self.cfg.rays_d_normalize + ) + + self.proj_mtx: Float[Tensor, "B 4 4"] = get_projection_matrix( + fovy, self.width / self.height, 0.01, 100.0 + ) # FIXME: hard-coded near and far + mvp_mtx: Float[Tensor, "B 4 4"] = get_mvp_matrix(c2w, self.proj_mtx) + self.fovy = fovy + + return { + "rays_o": rays_o, + "rays_d": rays_d, + "mvp_mtx": mvp_mtx, + "camera_positions": camera_positions, + "c2w": c2w, + "light_positions": light_positions, + "elevation": elevation_deg, + "azimuth": azimuth_deg, + "camera_distances": camera_distances, + "height": self.height, + "width": self.width, + "fovy": self.fovy, + "proj_mtx": self.proj_mtx, + } + + +class RandomCameraDataset(Dataset): + def __init__(self, cfg: Any, split: str) -> None: + super().__init__() + self.cfg: RandomCameraDataModuleConfig = cfg + self.split = split + + if split == "val": + self.n_views = self.cfg.n_val_views + else: + self.n_views = self.cfg.n_test_views + l, r = self.cfg.val_azimuth_range + + azimuth_deg: Float[Tensor, "B"] + if self.split == "val": + # make sure the first and last view are not the same + azimuth_deg = torch.linspace(l, r, self.n_views + 1)[: self.n_views] + else: + azimuth_deg = torch.linspace(l, r, self.n_views) + elevation_deg: Float[Tensor, "B"] = torch.full_like( + azimuth_deg, self.cfg.eval_elevation_deg + ) + camera_distances: Float[Tensor, "B"] = torch.full_like( + elevation_deg, self.cfg.eval_camera_distance + ) + + elevation = elevation_deg * math.pi / 180 + azimuth = azimuth_deg * math.pi / 180 + + # convert spherical coordinates to cartesian coordinates + # right hand coordinate system, x back, y right, z up + # elevation in (-90, 90), azimuth from +x to +y in (-180, 180) + camera_positions: Float[Tensor, "B 3"] = torch.stack( + [ + camera_distances * torch.cos(elevation) * torch.cos(azimuth), + camera_distances * torch.cos(elevation) * torch.sin(azimuth), + camera_distances * torch.sin(elevation), + ], + dim=-1, + ) + + # default scene center at origin + center: Float[Tensor, "B 3"] = torch.zeros_like(camera_positions) + # default camera up direction as +z + up: Float[Tensor, "B 3"] = torch.as_tensor([0, 0, 1], dtype=torch.float32)[ + None, : + ].repeat(self.cfg.eval_batch_size, 1) + + fovy_deg: Float[Tensor, "B"] = torch.full_like( + elevation_deg, self.cfg.eval_fovy_deg + ) + fovy = fovy_deg * math.pi / 180 + light_positions: Float[Tensor, "B 3"] = camera_positions + + lookat: Float[Tensor, "B 3"] = F.normalize(center - camera_positions, dim=-1) + right: Float[Tensor, "B 3"] = F.normalize(torch.cross(lookat, up), dim=-1) + up = F.normalize(torch.cross(right, lookat), dim=-1) + c2w3x4: Float[Tensor, "B 3 4"] = torch.cat( + [torch.stack([right, up, -lookat], dim=-1), camera_positions[:, :, None]], + dim=-1, + ) + c2w: Float[Tensor, "B 4 4"] = torch.cat( + [c2w3x4, torch.zeros_like(c2w3x4[:, :1])], dim=1 + ) + c2w[:, 3, 3] = 1.0 + + + + # get directions by dividing directions_unit_focal by focal length + focal_length: Float[Tensor, "B"] = ( + 0.5 * self.cfg.eval_height / torch.tan(0.5 * fovy) + ) + directions_unit_focal = get_ray_directions( + H=self.cfg.eval_height, W=self.cfg.eval_width, focal=1.0 + ) + directions: Float[Tensor, "B H W 3"] = directions_unit_focal[ + None, :, :, : + ].repeat(self.n_views, 1, 1, 1) + directions[:, :, :, :2] = ( + directions[:, :, :, :2] / focal_length[:, None, None, None] + ) + + rays_o, rays_d = get_rays( + directions, c2w, keepdim=True, normalize=self.cfg.rays_d_normalize + ) + self.proj_mtx: Float[Tensor, "B 4 4"] = get_projection_matrix( + fovy, self.cfg.eval_width / self.cfg.eval_height, 0.01, 100.0 + ) # FIXME: hard-coded near and far + mvp_mtx: Float[Tensor, "B 4 4"] = get_mvp_matrix(c2w, self.proj_mtx) + + self.rays_o, self.rays_d = rays_o, rays_d + self.mvp_mtx = mvp_mtx + self.c2w = c2w + self.camera_positions = camera_positions + self.right = right + self.up = up + self.center = center + self.light_positions = light_positions + self.elevation, self.azimuth = elevation, azimuth + self.elevation_deg, self.azimuth_deg = elevation_deg, azimuth_deg + self.camera_distances = camera_distances + self.fovy = fovy + + def rotate_around(self, index, ratio=0, scale=0.1): + # l, r = self.cfg.val_azimuth_range + + # azimuth_deg: Float[Tensor, "B"] + # if self.split == "val": + # # make sure the first and last view are not the same + # azimuth_deg = torch.linspace(l, r, self.n_views + 1)[: self.n_views] + # else: + # azimuth_deg = torch.linspace(l, r, self.n_views) + # elevation_deg: Float[Tensor, "B"] = torch.full_like( + # azimuth_deg, self.cfg.eval_elevation_deg + # ) + # camera_distances: Float[Tensor, "B"] = torch.full_like( + # elevation_deg, self.cfg.eval_camera_distance + # ) + + # elevation = elevation_deg * math.pi / 180 + # azimuth = azimuth_deg * math.pi / 180 + + # # convert spherical coordinates to cartesian coordinates + # # right hand coordinate system, x back, y right, z up + # # elevation in (-90, 90), azimuth from +x to +y in (-180, 180) + # camera_positions: Float[Tensor, "B 3"] = torch.stack( + # [ + # camera_distances * torch.cos(elevation) * torch.cos(azimuth), + # camera_distances * torch.cos(elevation) * torch.sin(azimuth), + # camera_distances * torch.sin(elevation), + # ], + # dim=-1, + # ) + + # # default scene center at origin + # center: Float[Tensor, "B 3"] = torch.zeros_like(camera_positions) + # # default camera up direction as +z + # up: Float[Tensor, "B 3"] = torch.as_tensor([0, 0, 1], dtype=torch.float32)[ + # None, : + # ].repeat(self.cfg.eval_batch_size, 1) + + # fovy_deg: Float[Tensor, "B"] = torch.full_like( + # elevation_deg, self.cfg.eval_fovy_deg + # ) + # fovy = fovy_deg * math.pi / 180 + # light_positions: Float[Tensor, "B 3"] = camera_positions + + # lookat: Float[Tensor, "B 3"] = F.normalize(center - camera_positions, dim=-1) + # right: Float[Tensor, "B 3"] = F.normalize(torch.cross(lookat, up), dim=-1) + # up = F.normalize(torch.cross(right, lookat), dim=-1) + + # orbit trajectory + right_ratio = scale * np.sin(ratio) + # up_ratio = scale * (np.cos(ratio) - 1) + up_ratio = scale * (np.cos(ratio)) + camera_positions = self.camera_positions + right_ratio * self.right + up_ratio * self.up + + lookat: Float[Tensor, "B 3"] = F.normalize(self.center - camera_positions, dim=-1) + right: Float[Tensor, "B 3"] = F.normalize(torch.cross(lookat, self.up), dim=-1) + up = F.normalize(torch.cross(right, lookat), dim=-1) + + c2w3x4: Float[Tensor, "B 3 4"] = torch.cat( + [torch.stack([right, up, -lookat], dim=-1), camera_positions[:, :, None]], + dim=-1, + ) + c2w: Float[Tensor, "B 4 4"] = torch.cat( + [c2w3x4, torch.zeros_like(c2w3x4[:, :1])], dim=1 + ) + c2w[:, 3, 3] = 1.0 + + # get directions by dividing directions_unit_focal by focal length + focal_length: Float[Tensor, "B"] = ( + 0.5 * self.cfg.eval_height / torch.tan(0.5 * self.fovy) + ) + directions_unit_focal = get_ray_directions( + H=self.cfg.eval_height, W=self.cfg.eval_width, focal=1.0 + ) + directions: Float[Tensor, "B H W 3"] = directions_unit_focal[ + None, :, :, : + ].repeat(self.n_views, 1, 1, 1) + directions[:, :, :, :2] = ( + directions[:, :, :, :2] / focal_length[:, None, None, None] + ) + + rays_o, rays_d = get_rays( + directions, c2w, keepdim=True, normalize=self.cfg.rays_d_normalize + ) + proj_mtx: Float[Tensor, "B 4 4"] = get_projection_matrix( + self.fovy, self.cfg.eval_width / self.cfg.eval_height, 0.01, 100.0 + ) # FIXME: hard-coded near and far + mvp_mtx: Float[Tensor, "B 4 4"] = get_mvp_matrix(c2w, proj_mtx) + + return { + "index": index, + "rays_o": rays_o[index], + "rays_d": rays_d[index], + "mvp_mtx": mvp_mtx[index], + "c2w": c2w[index], + "camera_positions": camera_positions[index], + "light_positions": self.light_positions[index], + "elevation": self.elevation_deg[index], + "azimuth": self.azimuth_deg[index], + "camera_distances": self.camera_distances[index], + "height": self.cfg.eval_height, + "width": self.cfg.eval_width, + "fovy": self.fovy[index], + "proj_mtx": proj_mtx[index], + } + + def random_trajectory(self, index, ratio=0, scale=0.1, freq=1.0, noise_intensity=0.05): + """ + Generate a smooth random camera trajectory around the scene. + + Args: + index (int): Index for batch processing. + ratio (float): Controls the progress along the trajectory (0 to 1). + scale (float): Scale for the base rotation trajectory. + freq (float): Frequency of sinusoidal perturbations for smooth randomness. + noise_intensity (float): Intensity of Perlin noise for random deviations. + + Returns: + dict: Updated camera parameters and ray information. + """ + from noise import pnoise1 # Perlin noise for smooth randomness + + + # Base sinusoidal movement for orbiting + right_ratio = scale * np.sin(2 * np.pi * ratio) + up_ratio = scale * np.cos(2 * np.pi * ratio) + + # Adding smooth random perturbations using Perlin noise + noise_seed = ratio * freq + random_right = noise_intensity * pnoise1(noise_seed) + random_up = noise_intensity * pnoise1(noise_seed + 50) + + # Apply smooth random perturbations to the camera position + camera_positions = self.camera_positions + (right_ratio + random_right) * self.right + (up_ratio + random_up) * self.up + + # Calculate the look-at vector (camera looks at the center) + lookat = F.normalize(self.center - camera_positions, dim=-1) + + # Recalculate the right and up vectors to maintain orthogonality + right = F.normalize(torch.cross(lookat, self.up), dim=-1) + up = F.normalize(torch.cross(right, lookat), dim=-1) + + # Construct the camera-to-world (c2w) matrix + c2w3x4 = torch.cat( + [torch.stack([right, up, -lookat], dim=-1), camera_positions[:, :, None]], + dim=-1 + ) + c2w = torch.cat( + [c2w3x4, torch.zeros_like(c2w3x4[:, :1])], dim=1 + ) + c2w[:, 3, 3] = 1.0 # Homogeneous coordinate + + # Compute focal length based on field of view (fovy) + focal_length = 0.5 * self.cfg.eval_height / torch.tan(0.5 * self.fovy) + + # Get ray directions in camera space + directions_unit_focal = get_ray_directions( + H=self.cfg.eval_height, W=self.cfg.eval_width, focal=1.0 + ) + directions = directions_unit_focal[None, :, :, :].repeat(self.n_views, 1, 1, 1) + directions[:, :, :, :2] /= focal_length[:, None, None, None] + + # Transform rays from camera to world coordinates + rays_o, rays_d = get_rays( + directions, c2w, keepdim=True, normalize=self.cfg.rays_d_normalize + ) + + # Create projection and MVP matrices + proj_mtx = get_projection_matrix( + self.fovy, self.cfg.eval_width / self.cfg.eval_height, 0.01, 100.0 + ) + mvp_mtx = get_mvp_matrix(c2w, proj_mtx) + + # Return the updated camera state + return { + "index": index, + "rays_o": rays_o[index], + "rays_d": rays_d[index], + "mvp_mtx": mvp_mtx[index], + "c2w": c2w[index], + "camera_positions": camera_positions[index], + "light_positions": camera_positions[index], # Light follows the camera + "elevation": self.elevation_deg[index], + "azimuth": self.azimuth_deg[index], + "camera_distances": self.camera_distances[index], + "height": self.cfg.eval_height, + "width": self.cfg.eval_width, + "fovy": self.fovy[index], + "proj_mtx": proj_mtx[index], + } + + def __len__(self): + return self.n_views + + def __getitem__(self, index): + + if self.cfg.rotate_traj: + frame_n = 120 + return self.rotate_around(index, index/frame_n*2*np.pi, scale=0.2) + + elif self.cfg.random_traj: + frame_n = 120 + return self.random_trajectory(index, ratio=0.01 + + , scale=0.1, freq=1.0, noise_intensity=0.05) + + else: + return { + "index": index, + "rays_o": self.rays_o[index], + "rays_d": self.rays_d[index], + "mvp_mtx": self.mvp_mtx[index], + "c2w": self.c2w[index], + "camera_positions": self.camera_positions[index], + "light_positions": self.light_positions[index], + "elevation": self.elevation_deg[index], + "azimuth": self.azimuth_deg[index], + "camera_distances": self.camera_distances[index], + "height": self.cfg.eval_height, + "width": self.cfg.eval_width, + "fovy": self.fovy[index], + "proj_mtx": self.proj_mtx[index], + } + + def collate(self, batch): + batch = torch.utils.data.default_collate(batch) + batch.update({"height": self.cfg.eval_height, "width": self.cfg.eval_width}) + return batch + + +@register("random-camera-datamodule") +class RandomCameraDataModule(pl.LightningDataModule): + cfg: RandomCameraDataModuleConfig + + def __init__(self, cfg: Optional[Union[dict, DictConfig]] = None) -> None: + super().__init__() + self.cfg = parse_structured(RandomCameraDataModuleConfig, cfg) + + def setup(self, stage=None) -> None: + if stage in [None, "fit"]: + self.train_dataset = RandomCameraIterableDataset(self.cfg) + if stage in [None, "fit", "validate"]: + self.val_dataset = RandomCameraDataset(self.cfg, "val") + if stage in [None, "test", "predict"]: + self.test_dataset = RandomCameraDataset(self.cfg, "test") + + def prepare_data(self): + pass + + def general_loader(self, dataset, batch_size, collate_fn=None) -> DataLoader: + return DataLoader( + dataset, + # very important to disable multi-processing if you want to change self attributes at runtime! + # (for example setting self.width and self.height in update_step) + num_workers=0, # type: ignore + batch_size=batch_size, + collate_fn=collate_fn, + ) + + def train_dataloader(self) -> DataLoader: + return self.general_loader( + self.train_dataset, batch_size=None, collate_fn=self.train_dataset.collate + ) + + def val_dataloader(self) -> DataLoader: + return self.general_loader( + self.val_dataset, batch_size=1, collate_fn=self.val_dataset.collate + ) + # return self.general_loader(self.train_dataset, batch_size=None, collate_fn=self.train_dataset.collate) + + def test_dataloader(self) -> DataLoader: + return self.general_loader( + self.test_dataset, batch_size=1, collate_fn=self.test_dataset.collate + ) + + def predict_dataloader(self) -> DataLoader: + return self.general_loader( + self.test_dataset, batch_size=1, collate_fn=self.test_dataset.collate + ) + diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..369a3774ab97d1dbd84856eeabcf27c52768b304 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/__init__.py @@ -0,0 +1,9 @@ +from . import ( + background, + exporters, + geometry, + guidance, + materials, + prompt_processors, + # renderers, +) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c637e6b7775f8565250c371e300924bbaf09842c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/__init__.py @@ -0,0 +1,6 @@ +from . import ( + base, + neural_environment_map_background, + solid_color_background, + textured_background, +) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/base.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/base.py new file mode 100644 index 0000000000000000000000000000000000000000..8911bd12ebb0655c87db981afee93f5000bfd0af --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/base.py @@ -0,0 +1,24 @@ +import random +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.utils.base import BaseModule +from threestudio.utils.typing import * + + +class BaseBackground(BaseModule): + @dataclass + class Config(BaseModule.Config): + pass + + cfg: Config + + def configure(self): + pass + + def forward(self, dirs: Float[Tensor, "B H W 3"]) -> Float[Tensor, "B H W Nc"]: + raise NotImplementedError diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/neural_environment_map_background.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/neural_environment_map_background.py new file mode 100644 index 0000000000000000000000000000000000000000..33adfc0a1d348585509aed604228accc5a87f951 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/neural_environment_map_background.py @@ -0,0 +1,67 @@ +import random +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.background.base import BaseBackground +from threestudio.models.networks import get_encoding, get_mlp +from threestudio.utils.ops import get_activation +from threestudio.utils.typing import * + + +@threestudio.register("neural-environment-map-background") +class NeuralEnvironmentMapBackground(BaseBackground): + @dataclass + class Config(BaseBackground.Config): + n_output_dims: int = 3 + color_activation: str = "sigmoid" + dir_encoding_config: dict = field( + default_factory=lambda: {"otype": "SphericalHarmonics", "degree": 3} + ) + mlp_network_config: dict = field( + default_factory=lambda: { + "otype": "VanillaMLP", + "activation": "ReLU", + "n_neurons": 16, + "n_hidden_layers": 2, + } + ) + random_aug: bool = False + random_aug_prob: float = 0.5 + eval_color: Optional[Tuple[float, float, float]] = None + + cfg: Config + + def configure(self) -> None: + self.encoding = get_encoding(3, self.cfg.dir_encoding_config) + self.network = get_mlp( + self.encoding.n_output_dims, + self.cfg.n_output_dims, + self.cfg.mlp_network_config, + ) + + def forward(self, dirs: Float[Tensor, "B H W 3"]) -> Float[Tensor, "B H W Nc"]: + if not self.training and self.cfg.eval_color is not None: + return torch.ones(*dirs.shape[:-1], self.cfg.n_output_dims).to( + dirs + ) * torch.as_tensor(self.cfg.eval_color).to(dirs) + # viewdirs must be normalized before passing to this function + dirs = (dirs + 1.0) / 2.0 # (-1, 1) => (0, 1) + dirs_embd = self.encoding(dirs.view(-1, 3)) + color = self.network(dirs_embd).view(*dirs.shape[:-1], self.cfg.n_output_dims) + color = get_activation(self.cfg.color_activation)(color) + if ( + self.training + and self.cfg.random_aug + and random.random() < self.cfg.random_aug_prob + ): + # use random background color with probability random_aug_prob + color = color * 0 + ( # prevent checking for unused parameters in DDP + torch.rand(dirs.shape[0], 1, 1, self.cfg.n_output_dims) + .to(dirs) + .expand(*dirs.shape[:-1], -1) + ) + return color diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/solid_color_background.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/solid_color_background.py new file mode 100644 index 0000000000000000000000000000000000000000..0b68d5b4901fb7d9d52ab4d3e6c2302650edcc87 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/solid_color_background.py @@ -0,0 +1,51 @@ +import random +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.background.base import BaseBackground +from threestudio.utils.typing import * + + +@threestudio.register("solid-color-background") +class SolidColorBackground(BaseBackground): + @dataclass + class Config(BaseBackground.Config): + n_output_dims: int = 3 + color: Tuple = (1.0, 1.0, 1.0) + learned: bool = False + random_aug: bool = False + random_aug_prob: float = 0.5 + + cfg: Config + + def configure(self) -> None: + self.env_color: Float[Tensor, "Nc"] + if self.cfg.learned: + self.env_color = nn.Parameter( + torch.as_tensor(self.cfg.color, dtype=torch.float32) + ) + else: + self.register_buffer( + "env_color", torch.as_tensor(self.cfg.color, dtype=torch.float32) + ) + + def forward(self, dirs: Float[Tensor, "B H W 3"]) -> Float[Tensor, "B H W Nc"]: + color = torch.ones(*dirs.shape[:-1], self.cfg.n_output_dims).to( + dirs + ) * self.env_color.to(dirs) + if ( + self.training + and self.cfg.random_aug + and random.random() < self.cfg.random_aug_prob + ): + # use random background color with probability random_aug_prob + color = color * 0 + ( # prevent checking for unused parameters in DDP + torch.rand(dirs.shape[0], 1, 1, self.cfg.n_output_dims) + .to(dirs) + .expand(*dirs.shape[:-1], -1) + ) + return color diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/textured_background.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/textured_background.py new file mode 100644 index 0000000000000000000000000000000000000000..ee0a0ca889678eb492dd81412b2281cdf6e56946 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/background/textured_background.py @@ -0,0 +1,54 @@ +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.background.base import BaseBackground +from threestudio.utils.ops import get_activation +from threestudio.utils.typing import * + + +@threestudio.register("textured-background") +class TexturedBackground(BaseBackground): + @dataclass + class Config(BaseBackground.Config): + n_output_dims: int = 3 + height: int = 64 + width: int = 64 + color_activation: str = "sigmoid" + + cfg: Config + + def configure(self) -> None: + self.texture = nn.Parameter( + torch.randn((1, self.cfg.n_output_dims, self.cfg.height, self.cfg.width)) + ) + + def spherical_xyz_to_uv(self, dirs: Float[Tensor, "*B 3"]) -> Float[Tensor, "*B 2"]: + x, y, z = dirs[..., 0], dirs[..., 1], dirs[..., 2] + xy = (x**2 + y**2) ** 0.5 + u = torch.atan2(xy, z) / torch.pi + v = torch.atan2(y, x) / (torch.pi * 2) + 0.5 + uv = torch.stack([u, v], -1) + return uv + + def forward(self, dirs: Float[Tensor, "*B 3"]) -> Float[Tensor, "*B Nc"]: + dirs_shape = dirs.shape[:-1] + uv = self.spherical_xyz_to_uv(dirs.reshape(-1, dirs.shape[-1])) + uv = 2 * uv - 1 # rescale to [-1, 1] for grid_sample + uv = uv.reshape(1, -1, 1, 2) + color = ( + F.grid_sample( + self.texture, + uv, + mode="bilinear", + padding_mode="reflection", + align_corners=False, + ) + .reshape(self.cfg.n_output_dims, -1) + .T.reshape(*dirs_shape, self.cfg.n_output_dims) + ) + color = get_activation(self.cfg.color_activation)(color) + return color diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/estimators.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/estimators.py new file mode 100644 index 0000000000000000000000000000000000000000..c4c6c5ea224d23bea86a895d6e39d45011803edf --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/estimators.py @@ -0,0 +1,118 @@ +from typing import Callable, List, Optional, Tuple + +try: + from typing import Literal +except ImportError: + from typing_extensions import Literal + +import torch +from nerfacc.data_specs import RayIntervals +from nerfacc.estimators.base import AbstractEstimator +from nerfacc.pdf import importance_sampling, searchsorted +from nerfacc.volrend import render_transmittance_from_density +from torch import Tensor + + +class ImportanceEstimator(AbstractEstimator): + def __init__( + self, + ) -> None: + super().__init__() + + @torch.no_grad() + def sampling( + self, + prop_sigma_fns: List[Callable], + prop_samples: List[int], + num_samples: int, + # rendering options + n_rays: int, + near_plane: float, + far_plane: float, + sampling_type: Literal["uniform", "lindisp"] = "uniform", + # training options + stratified: bool = False, + requires_grad: bool = False, + ) -> Tuple[Tensor, Tensor]: + """Sampling with CDFs from proposal networks. + + Args: + prop_sigma_fns: Proposal network evaluate functions. It should be a list + of functions that take in samples {t_starts (n_rays, n_samples), + t_ends (n_rays, n_samples)} and returns the post-activation densities + (n_rays, n_samples). + prop_samples: Number of samples to draw from each proposal network. Should + be the same length as `prop_sigma_fns`. + num_samples: Number of samples to draw in the end. + n_rays: Number of rays. + near_plane: Near plane. + far_plane: Far plane. + sampling_type: Sampling type. Either "uniform" or "lindisp". Default to + "lindisp". + stratified: Whether to use stratified sampling. Default to `False`. + + Returns: + A tuple of {Tensor, Tensor}: + + - **t_starts**: The starts of the samples. Shape (n_rays, num_samples). + - **t_ends**: The ends of the samples. Shape (n_rays, num_samples). + + """ + assert len(prop_sigma_fns) == len(prop_samples), ( + "The number of proposal networks and the number of samples " + "should be the same." + ) + cdfs = torch.cat( + [ + torch.zeros((n_rays, 1), device=self.device), + torch.ones((n_rays, 1), device=self.device), + ], + dim=-1, + ) + intervals = RayIntervals(vals=cdfs) + + for level_fn, level_samples in zip(prop_sigma_fns, prop_samples): + intervals, _ = importance_sampling( + intervals, cdfs, level_samples, stratified + ) + t_vals = _transform_stot( + sampling_type, intervals.vals, near_plane, far_plane + ) + t_starts = t_vals[..., :-1] + t_ends = t_vals[..., 1:] + + with torch.set_grad_enabled(requires_grad): + sigmas = level_fn(t_starts, t_ends) + assert sigmas.shape == t_starts.shape + trans, _ = render_transmittance_from_density(t_starts, t_ends, sigmas) + cdfs = 1.0 - torch.cat([trans, torch.zeros_like(trans[:, :1])], dim=-1) + + intervals, _ = importance_sampling(intervals, cdfs, num_samples, stratified) + t_vals_fine = _transform_stot( + sampling_type, intervals.vals, near_plane, far_plane + ) + + t_vals = torch.cat([t_vals, t_vals_fine], dim=-1) + t_vals, _ = torch.sort(t_vals, dim=-1) + + t_starts_ = t_vals[..., :-1] + t_ends_ = t_vals[..., 1:] + + return t_starts_, t_ends_ + + +def _transform_stot( + transform_type: Literal["uniform", "lindisp"], + s_vals: torch.Tensor, + t_min: torch.Tensor, + t_max: torch.Tensor, +) -> torch.Tensor: + if transform_type == "uniform": + _contract_fn, _icontract_fn = lambda x: x, lambda x: x + elif transform_type == "lindisp": + _contract_fn, _icontract_fn = lambda x: 1 / x, lambda x: 1 / x + else: + raise ValueError(f"Unknown transform_type: {transform_type}") + s_min, s_max = _contract_fn(t_min), _contract_fn(t_max) + icontract_fn = lambda s: _icontract_fn(s * s_max + (1 - s) * s_min) + return icontract_fn(s_vals) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/exporters/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/exporters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..add385e29f6db22395bcfb3f09d2f80a0c9e4da0 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/exporters/__init__.py @@ -0,0 +1 @@ +from . import base, mesh_exporter diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/exporters/base.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/exporters/base.py new file mode 100644 index 0000000000000000000000000000000000000000..592d103774ab59ce340b55eabaa0f5704a2e2a94 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/exporters/base.py @@ -0,0 +1,59 @@ +from dataclasses import dataclass + +import threestudio +from threestudio.models.background.base import BaseBackground +from threestudio.models.geometry.base import BaseImplicitGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.utils.base import BaseObject +from threestudio.utils.typing import * + + +@dataclass +class ExporterOutput: + save_name: str + save_type: str + params: Dict[str, Any] + + +class Exporter(BaseObject): + @dataclass + class Config(BaseObject.Config): + save_video: bool = False + + cfg: Config + + def configure( + self, + geometry: BaseImplicitGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + @dataclass + class SubModules: + geometry: BaseImplicitGeometry + material: BaseMaterial + background: BaseBackground + + self.sub_modules = SubModules(geometry, material, background) + + @property + def geometry(self) -> BaseImplicitGeometry: + return self.sub_modules.geometry + + @property + def material(self) -> BaseMaterial: + return self.sub_modules.material + + @property + def background(self) -> BaseBackground: + return self.sub_modules.background + + def __call__(self, *args, **kwargs) -> List[ExporterOutput]: + raise NotImplementedError + + +@threestudio.register("dummy-exporter") +class DummyExporter(Exporter): + def __call__(self, *args, **kwargs) -> List[ExporterOutput]: + # DummyExporter does not export anything + return [] diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/exporters/mesh_exporter.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/exporters/mesh_exporter.py new file mode 100644 index 0000000000000000000000000000000000000000..2c67826f89dd882e7ed99e5e3b045a489790fc23 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/exporters/mesh_exporter.py @@ -0,0 +1,175 @@ +from dataclasses import dataclass, field + +import cv2 +import numpy as np +import torch + +import threestudio +from threestudio.models.background.base import BaseBackground +from threestudio.models.exporters.base import Exporter, ExporterOutput +from threestudio.models.geometry.base import BaseImplicitGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.mesh import Mesh +from threestudio.utils.rasterize import NVDiffRasterizerContext +from threestudio.utils.typing import * + + +@threestudio.register("mesh-exporter") +class MeshExporter(Exporter): + @dataclass + class Config(Exporter.Config): + fmt: str = "obj-mtl" # in ['obj-mtl', 'obj'], TODO: fbx + save_name: str = "model" + save_normal: bool = False + save_uv: bool = True + save_texture: bool = True + texture_size: int = 1024 + texture_format: str = "jpg" + xatlas_chart_options: dict = field(default_factory=dict) + xatlas_pack_options: dict = field(default_factory=dict) + context_type: str = "gl" + + cfg: Config + + def configure( + self, + geometry: BaseImplicitGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + super().configure(geometry, material, background) + self.ctx = NVDiffRasterizerContext(self.cfg.context_type, self.device) + + def __call__(self) -> List[ExporterOutput]: + mesh: Mesh = self.geometry.isosurface() + + if self.cfg.fmt == "obj-mtl": + return self.export_obj_with_mtl(mesh) + elif self.cfg.fmt == "obj": + return self.export_obj(mesh) + else: + raise ValueError(f"Unsupported mesh export format: {self.cfg.fmt}") + + def export_obj_with_mtl(self, mesh: Mesh) -> List[ExporterOutput]: + params = { + "mesh": mesh, + "save_mat": True, + "save_normal": self.cfg.save_normal, + "save_uv": self.cfg.save_uv, + "save_vertex_color": False, + "map_Kd": None, # Base Color + "map_Ks": None, # Specular + "map_Bump": None, # Normal + # ref: https://en.wikipedia.org/wiki/Wavefront_.obj_file#Physically-based_Rendering + "map_Pm": None, # Metallic + "map_Pr": None, # Roughness + "map_format": self.cfg.texture_format, + } + + if self.cfg.save_uv: + mesh.unwrap_uv(self.cfg.xatlas_chart_options, self.cfg.xatlas_pack_options) + + if self.cfg.save_texture: + threestudio.info("Exporting textures ...") + assert self.cfg.save_uv, "save_uv must be True when save_texture is True" + # clip space transform + uv_clip = mesh.v_tex * 2.0 - 1.0 + # pad to four component coordinate + uv_clip4 = torch.cat( + ( + uv_clip, + torch.zeros_like(uv_clip[..., 0:1]), + torch.ones_like(uv_clip[..., 0:1]), + ), + dim=-1, + ) + # rasterize + rast, _ = self.ctx.rasterize_one( + uv_clip4, mesh.t_tex_idx, (self.cfg.texture_size, self.cfg.texture_size) + ) + + hole_mask = ~(rast[:, :, 3] > 0) + + def uv_padding(image): + uv_padding_size = self.cfg.xatlas_pack_options.get("padding", 2) + inpaint_image = ( + cv2.inpaint( + (image.detach().cpu().numpy() * 255).astype(np.uint8), + (hole_mask.detach().cpu().numpy() * 255).astype(np.uint8), + uv_padding_size, + cv2.INPAINT_TELEA, + ) + / 255.0 + ) + return torch.from_numpy(inpaint_image).to(image) + + # Interpolate world space position + gb_pos, _ = self.ctx.interpolate_one( + mesh.v_pos, rast[None, ...], mesh.t_pos_idx + ) + gb_pos = gb_pos[0] + + # Sample out textures from MLP + geo_out = self.geometry.export(points=gb_pos) + mat_out = self.material.export(points=gb_pos, **geo_out) + + threestudio.info( + "Perform UV padding on texture maps to avoid seams, may take a while ..." + ) + + if "albedo" in mat_out: + params["map_Kd"] = uv_padding(mat_out["albedo"]) + else: + threestudio.warn( + "save_texture is True but no albedo texture found, using default white texture" + ) + if "metallic" in mat_out: + params["map_Pm"] = uv_padding(mat_out["metallic"]) + if "roughness" in mat_out: + params["map_Pr"] = uv_padding(mat_out["roughness"]) + if "bump" in mat_out: + params["map_Bump"] = uv_padding(mat_out["bump"]) + # TODO: map_Ks + return [ + ExporterOutput( + save_name=f"{self.cfg.save_name}.obj", save_type="obj", params=params + ) + ] + + def export_obj(self, mesh: Mesh) -> List[ExporterOutput]: + params = { + "mesh": mesh, + "save_mat": False, + "save_normal": self.cfg.save_normal, + "save_uv": self.cfg.save_uv, + "save_vertex_color": False, + "map_Kd": None, # Base Color + "map_Ks": None, # Specular + "map_Bump": None, # Normal + # ref: https://en.wikipedia.org/wiki/Wavefront_.obj_file#Physically-based_Rendering + "map_Pm": None, # Metallic + "map_Pr": None, # Roughness + "map_format": self.cfg.texture_format, + } + + if self.cfg.save_uv: + mesh.unwrap_uv(self.cfg.xatlas_chart_options, self.cfg.xatlas_pack_options) + + if self.cfg.save_texture: + threestudio.info("Exporting textures ...") + geo_out = self.geometry.export(points=mesh.v_pos) + mat_out = self.material.export(points=mesh.v_pos, **geo_out) + + if "albedo" in mat_out: + mesh.set_vertex_color(mat_out["albedo"]) + params["save_vertex_color"] = True + else: + threestudio.warn( + "save_texture is True but no albedo texture found, not saving vertex color" + ) + + return [ + ExporterOutput( + save_name=f"{self.cfg.save_name}.obj", save_type="obj", params=params + ) + ] diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..19185d262915e45ac79aa3d5c623f1c3798d8529 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/__init__.py @@ -0,0 +1,8 @@ +from . import ( + base, + custom_mesh, + implicit_sdf, + implicit_volume, + tetrahedra_sdf_grid, + volume_grid, +) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/base.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/base.py new file mode 100644 index 0000000000000000000000000000000000000000..8190fb1fead49c4729c60f6f1f84a618954f3c71 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/base.py @@ -0,0 +1,209 @@ +from dataclasses import dataclass, field + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.isosurface import ( + IsosurfaceHelper, + MarchingCubeCPUHelper, + MarchingTetrahedraHelper, +) +from threestudio.models.mesh import Mesh +from threestudio.utils.base import BaseModule +from threestudio.utils.ops import chunk_batch, scale_tensor +from threestudio.utils.typing import * + + +def contract_to_unisphere( + x: Float[Tensor, "... 3"], bbox: Float[Tensor, "2 3"], unbounded: bool = False +) -> Float[Tensor, "... 3"]: + if unbounded: + x = scale_tensor(x, bbox, (0, 1)) + x = x * 2 - 1 # aabb is at [-1, 1] + mag = x.norm(dim=-1, keepdim=True) + mask = mag.squeeze(-1) > 1 + x[mask] = (2 - 1 / mag[mask]) * (x[mask] / mag[mask]) + x = x / 4 + 0.5 # [-inf, inf] is at [0, 1] + else: + x = scale_tensor(x, bbox, (0, 1)) + return x + + +class BaseGeometry(BaseModule): + @dataclass + class Config(BaseModule.Config): + pass + + cfg: Config + + @staticmethod + def create_from( + other: "BaseGeometry", cfg: Optional[Union[dict, DictConfig]] = None, **kwargs + ) -> "BaseGeometry": + raise TypeError( + f"Cannot create {BaseGeometry.__name__} from {other.__class__.__name__}" + ) + + def export(self, *args, **kwargs) -> Dict[str, Any]: + return {} + + +class BaseImplicitGeometry(BaseGeometry): + @dataclass + class Config(BaseGeometry.Config): + radius: float = 1.0 + isosurface: bool = True + isosurface_method: str = "mt" + isosurface_resolution: int = 128 + isosurface_threshold: Union[float, str] = 0.0 + isosurface_chunk: int = 0 + isosurface_coarse_to_fine: bool = True + isosurface_deformable_grid: bool = False + isosurface_remove_outliers: bool = True + isosurface_outlier_n_faces_threshold: Union[int, float] = 0.01 + + cfg: Config + + def configure(self) -> None: + self.bbox: Float[Tensor, "2 3"] + self.register_buffer( + "bbox", + torch.as_tensor( + [ + [-self.cfg.radius, -self.cfg.radius, -self.cfg.radius], + [self.cfg.radius, self.cfg.radius, self.cfg.radius], + ], + dtype=torch.float32, + ), + ) + self.isosurface_helper: Optional[IsosurfaceHelper] = None + self.unbounded: bool = False + + def _initilize_isosurface_helper(self): + if self.cfg.isosurface and self.isosurface_helper is None: + if self.cfg.isosurface_method == "mc-cpu": + self.isosurface_helper = MarchingCubeCPUHelper( + self.cfg.isosurface_resolution + ).to(self.device) + elif self.cfg.isosurface_method == "mt": + self.isosurface_helper = MarchingTetrahedraHelper( + self.cfg.isosurface_resolution, + f"load/tets/{self.cfg.isosurface_resolution}_tets.npz", + ).to(self.device) + else: + raise AttributeError( + "Unknown isosurface method {self.cfg.isosurface_method}" + ) + + def forward( + self, points: Float[Tensor, "*N Di"], output_normal: bool = False + ) -> Dict[str, Float[Tensor, "..."]]: + raise NotImplementedError + + def forward_field( + self, points: Float[Tensor, "*N Di"] + ) -> Tuple[Float[Tensor, "*N 1"], Optional[Float[Tensor, "*N 3"]]]: + # return the value of the implicit field, could be density / signed distance + # also return a deformation field if the grid vertices can be optimized + raise NotImplementedError + + def forward_level( + self, field: Float[Tensor, "*N 1"], threshold: float + ) -> Float[Tensor, "*N 1"]: + # return the value of the implicit field, where the zero level set represents the surface + raise NotImplementedError + + def _isosurface(self, bbox: Float[Tensor, "2 3"], fine_stage: bool = False) -> Mesh: + def batch_func(x): + # scale to bbox as the input vertices are in [0, 1] + field, deformation = self.forward_field( + scale_tensor( + x.to(bbox.device), self.isosurface_helper.points_range, bbox + ), + ) + field = field.to( + x.device + ) # move to the same device as the input (could be CPU) + if deformation is not None: + deformation = deformation.to(x.device) + return field, deformation + + assert self.isosurface_helper is not None + + field, deformation = chunk_batch( + batch_func, + self.cfg.isosurface_chunk, + self.isosurface_helper.grid_vertices, + ) + + threshold: float + + if isinstance(self.cfg.isosurface_threshold, float): + threshold = self.cfg.isosurface_threshold + elif self.cfg.isosurface_threshold == "auto": + eps = 1.0e-5 + threshold = field[field > eps].mean().item() + threestudio.info( + f"Automatically determined isosurface threshold: {threshold}" + ) + else: + raise TypeError( + f"Unknown isosurface_threshold {self.cfg.isosurface_threshold}" + ) + + level = self.forward_level(field, threshold) + mesh: Mesh = self.isosurface_helper(level, deformation=deformation) + mesh.v_pos = scale_tensor( + mesh.v_pos, self.isosurface_helper.points_range, bbox + ) # scale to bbox as the grid vertices are in [0, 1] + mesh.add_extra("bbox", bbox) + + if self.cfg.isosurface_remove_outliers: + # remove outliers components with small number of faces + # only enabled when the mesh is not differentiable + mesh = mesh.remove_outlier(self.cfg.isosurface_outlier_n_faces_threshold) + + return mesh + + def isosurface(self) -> Mesh: + if not self.cfg.isosurface: + raise NotImplementedError( + "Isosurface is not enabled in the current configuration" + ) + self._initilize_isosurface_helper() + if self.cfg.isosurface_coarse_to_fine: + threestudio.debug("First run isosurface to get a tight bounding box ...") + with torch.no_grad(): + mesh_coarse = self._isosurface(self.bbox) + vmin, vmax = mesh_coarse.v_pos.amin(dim=0), mesh_coarse.v_pos.amax(dim=0) + vmin_ = (vmin - (vmax - vmin) * 0.1).max(self.bbox[0]) + vmax_ = (vmax + (vmax - vmin) * 0.1).min(self.bbox[1]) + threestudio.debug("Run isosurface again with the tight bounding box ...") + mesh = self._isosurface(torch.stack([vmin_, vmax_], dim=0), fine_stage=True) + else: + mesh = self._isosurface(self.bbox) + return mesh + + +class BaseExplicitGeometry(BaseGeometry): + @dataclass + class Config(BaseGeometry.Config): + radius: float = 1.0 + + cfg: Config + + def configure(self) -> None: + self.bbox: Float[Tensor, "2 3"] + self.register_buffer( + "bbox", + torch.as_tensor( + [ + [-self.cfg.radius, -self.cfg.radius, -self.cfg.radius], + [self.cfg.radius, self.cfg.radius, self.cfg.radius], + ], + dtype=torch.float32, + ), + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/custom_mesh.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/custom_mesh.py new file mode 100644 index 0000000000000000000000000000000000000000..08ebc6306792632cd0dbc36748130ddfcbfef661 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/custom_mesh.py @@ -0,0 +1,178 @@ +import os +from dataclasses import dataclass, field + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.geometry.base import ( + BaseExplicitGeometry, + BaseGeometry, + contract_to_unisphere, +) +from threestudio.models.mesh import Mesh +from threestudio.models.networks import get_encoding, get_mlp +from threestudio.utils.ops import scale_tensor +from threestudio.utils.typing import * + + +@threestudio.register("custom-mesh") +class CustomMesh(BaseExplicitGeometry): + @dataclass + class Config(BaseExplicitGeometry.Config): + n_input_dims: int = 3 + n_feature_dims: int = 3 + pos_encoding_config: dict = field( + default_factory=lambda: { + "otype": "HashGrid", + "n_levels": 16, + "n_features_per_level": 2, + "log2_hashmap_size": 19, + "base_resolution": 16, + "per_level_scale": 1.447269237440378, + } + ) + mlp_network_config: dict = field( + default_factory=lambda: { + "otype": "VanillaMLP", + "activation": "ReLU", + "output_activation": "none", + "n_neurons": 64, + "n_hidden_layers": 1, + } + ) + shape_init: str = "" + shape_init_params: Optional[Any] = None + shape_init_mesh_up: str = "+z" + shape_init_mesh_front: str = "+x" + + cfg: Config + + def configure(self) -> None: + super().configure() + + self.encoding = get_encoding( + self.cfg.n_input_dims, self.cfg.pos_encoding_config + ) + self.feature_network = get_mlp( + self.encoding.n_output_dims, + self.cfg.n_feature_dims, + self.cfg.mlp_network_config, + ) + + # Initialize custom mesh + if self.cfg.shape_init.startswith("mesh:"): + assert isinstance(self.cfg.shape_init_params, float) + mesh_path = self.cfg.shape_init[5:] + if not os.path.exists(mesh_path): + raise ValueError(f"Mesh file {mesh_path} does not exist.") + + import trimesh + + scene = trimesh.load(mesh_path) + if isinstance(scene, trimesh.Trimesh): + mesh = scene + elif isinstance(scene, trimesh.scene.Scene): + mesh = trimesh.Trimesh() + for obj in scene.geometry.values(): + mesh = trimesh.util.concatenate([mesh, obj]) + else: + raise ValueError(f"Unknown mesh type at {mesh_path}.") + + # move to center + centroid = mesh.vertices.mean(0) + mesh.vertices = mesh.vertices - centroid + + # align to up-z and front-x + dirs = ["+x", "+y", "+z", "-x", "-y", "-z"] + dir2vec = { + "+x": np.array([1, 0, 0]), + "+y": np.array([0, 1, 0]), + "+z": np.array([0, 0, 1]), + "-x": np.array([-1, 0, 0]), + "-y": np.array([0, -1, 0]), + "-z": np.array([0, 0, -1]), + } + if ( + self.cfg.shape_init_mesh_up not in dirs + or self.cfg.shape_init_mesh_front not in dirs + ): + raise ValueError( + f"shape_init_mesh_up and shape_init_mesh_front must be one of {dirs}." + ) + if self.cfg.shape_init_mesh_up[1] == self.cfg.shape_init_mesh_front[1]: + raise ValueError( + "shape_init_mesh_up and shape_init_mesh_front must be orthogonal." + ) + z_, x_ = ( + dir2vec[self.cfg.shape_init_mesh_up], + dir2vec[self.cfg.shape_init_mesh_front], + ) + y_ = np.cross(z_, x_) + std2mesh = np.stack([x_, y_, z_], axis=0).T + mesh2std = np.linalg.inv(std2mesh) + + # scaling + scale = np.abs(mesh.vertices).max() + mesh.vertices = mesh.vertices / scale * self.cfg.shape_init_params + mesh.vertices = np.dot(mesh2std, mesh.vertices.T).T + + v_pos = torch.tensor(mesh.vertices, dtype=torch.float32).to(self.device) + t_pos_idx = torch.tensor(mesh.faces, dtype=torch.int64).to(self.device) + self.mesh = Mesh(v_pos=v_pos, t_pos_idx=t_pos_idx) + self.register_buffer( + "v_buffer", + v_pos, + ) + self.register_buffer( + "t_buffer", + t_pos_idx, + ) + + else: + raise ValueError( + f"Unknown shape initialization type: {self.cfg.shape_init}" + ) + print(self.mesh.v_pos.device) + + def isosurface(self) -> Mesh: + if hasattr(self, "mesh"): + return self.mesh + elif hasattr(self, "v_buffer"): + self.mesh = Mesh(v_pos=self.v_buffer, t_pos_idx=self.t_buffer) + return self.mesh + else: + raise ValueError(f"custom mesh is not initialized") + + def forward( + self, points: Float[Tensor, "*N Di"], output_normal: bool = False + ) -> Dict[str, Float[Tensor, "..."]]: + assert ( + output_normal == False + ), f"Normal output is not supported for {self.__class__.__name__}" + points_unscaled = points # points in the original scale + points = contract_to_unisphere(points, self.bbox) # points normalized to (0, 1) + enc = self.encoding(points.view(-1, self.cfg.n_input_dims)) + features = self.feature_network(enc).view( + *points.shape[:-1], self.cfg.n_feature_dims + ) + return {"features": features} + + def export(self, points: Float[Tensor, "*N Di"], **kwargs) -> Dict[str, Any]: + out: Dict[str, Any] = {} + if self.cfg.n_feature_dims == 0: + return out + points_unscaled = points + points = contract_to_unisphere(points_unscaled, self.bbox) + enc = self.encoding(points.reshape(-1, self.cfg.n_input_dims)) + features = self.feature_network(enc).view( + *points.shape[:-1], self.cfg.n_feature_dims + ) + out.update( + { + "features": features, + } + ) + return out diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/implicit_sdf.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/implicit_sdf.py new file mode 100644 index 0000000000000000000000000000000000000000..95892918d66d10c53e29a6633f6d4469ddb1996c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/implicit_sdf.py @@ -0,0 +1,413 @@ +import os +from dataclasses import dataclass, field + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.geometry.base import BaseImplicitGeometry, contract_to_unisphere +from threestudio.models.mesh import Mesh +from threestudio.models.networks import get_encoding, get_mlp +from threestudio.utils.misc import broadcast, get_rank +from threestudio.utils.typing import * + + +@threestudio.register("implicit-sdf") +class ImplicitSDF(BaseImplicitGeometry): + @dataclass + class Config(BaseImplicitGeometry.Config): + n_input_dims: int = 3 + n_feature_dims: int = 3 + pos_encoding_config: dict = field( + default_factory=lambda: { + "otype": "HashGrid", + "n_levels": 16, + "n_features_per_level": 2, + "log2_hashmap_size": 19, + "base_resolution": 16, + "per_level_scale": 1.447269237440378, + } + ) + mlp_network_config: dict = field( + default_factory=lambda: { + "otype": "VanillaMLP", + "activation": "ReLU", + "output_activation": "none", + "n_neurons": 64, + "n_hidden_layers": 1, + } + ) + normal_type: Optional[ + str + ] = "finite_difference" # in ['pred', 'finite_difference', 'finite_difference_laplacian'] + finite_difference_normal_eps: Union[ + float, str + ] = 0.01 # in [float, "progressive"] + shape_init: Optional[str] = None + shape_init_params: Optional[Any] = None + shape_init_mesh_up: str = "+z" + shape_init_mesh_front: str = "+x" + force_shape_init: bool = False + sdf_bias: Union[float, str] = 0.0 + sdf_bias_params: Optional[Any] = None + + # no need to removal outlier for SDF + isosurface_remove_outliers: bool = False + + cfg: Config + + def configure(self) -> None: + super().configure() + self.encoding = get_encoding( + self.cfg.n_input_dims, self.cfg.pos_encoding_config + ) + self.sdf_network = get_mlp( + self.encoding.n_output_dims, 1, self.cfg.mlp_network_config + ) + + if self.cfg.n_feature_dims > 0: + self.feature_network = get_mlp( + self.encoding.n_output_dims, + self.cfg.n_feature_dims, + self.cfg.mlp_network_config, + ) + + if self.cfg.normal_type == "pred": + self.normal_network = get_mlp( + self.encoding.n_output_dims, 3, self.cfg.mlp_network_config + ) + if self.cfg.isosurface_deformable_grid: + assert ( + self.cfg.isosurface_method == "mt" + ), "isosurface_deformable_grid only works with mt" + self.deformation_network = get_mlp( + self.encoding.n_output_dims, 3, self.cfg.mlp_network_config + ) + + self.finite_difference_normal_eps: Optional[float] = None + + def initialize_shape(self) -> None: + if self.cfg.shape_init is None and not self.cfg.force_shape_init: + return + + # do not initialize shape if weights are provided + if self.cfg.weights is not None and not self.cfg.force_shape_init: + return + + if self.cfg.sdf_bias != 0.0: + threestudio.warn( + "shape_init and sdf_bias are both specified, which may lead to unexpected results." + ) + + get_gt_sdf: Callable[[Float[Tensor, "N 3"]], Float[Tensor, "N 1"]] + assert isinstance(self.cfg.shape_init, str) + if self.cfg.shape_init == "ellipsoid": + assert ( + isinstance(self.cfg.shape_init_params, Sized) + and len(self.cfg.shape_init_params) == 3 + ) + size = torch.as_tensor(self.cfg.shape_init_params).to(self.device) + + def func(points_rand: Float[Tensor, "N 3"]) -> Float[Tensor, "N 1"]: + return ((points_rand / size) ** 2).sum( + dim=-1, keepdim=True + ).sqrt() - 1.0 # pseudo signed distance of an ellipsoid + + get_gt_sdf = func + elif self.cfg.shape_init == "sphere": + assert isinstance(self.cfg.shape_init_params, float) + radius = self.cfg.shape_init_params + + def func(points_rand: Float[Tensor, "N 3"]) -> Float[Tensor, "N 1"]: + return (points_rand**2).sum(dim=-1, keepdim=True).sqrt() - radius + + get_gt_sdf = func + elif self.cfg.shape_init.startswith("mesh:"): + assert isinstance(self.cfg.shape_init_params, float) + mesh_path = self.cfg.shape_init[5:] + if not os.path.exists(mesh_path): + raise ValueError(f"Mesh file {mesh_path} does not exist.") + + import trimesh + + scene = trimesh.load(mesh_path) + if isinstance(scene, trimesh.Trimesh): + mesh = scene + elif isinstance(scene, trimesh.scene.Scene): + mesh = trimesh.Trimesh() + for obj in scene.geometry.values(): + mesh = trimesh.util.concatenate([mesh, obj]) + else: + raise ValueError(f"Unknown mesh type at {mesh_path}.") + + # move to center + centroid = mesh.vertices.mean(0) + mesh.vertices = mesh.vertices - centroid + + # align to up-z and front-x + dirs = ["+x", "+y", "+z", "-x", "-y", "-z"] + dir2vec = { + "+x": np.array([1, 0, 0]), + "+y": np.array([0, 1, 0]), + "+z": np.array([0, 0, 1]), + "-x": np.array([-1, 0, 0]), + "-y": np.array([0, -1, 0]), + "-z": np.array([0, 0, -1]), + } + if ( + self.cfg.shape_init_mesh_up not in dirs + or self.cfg.shape_init_mesh_front not in dirs + ): + raise ValueError( + f"shape_init_mesh_up and shape_init_mesh_front must be one of {dirs}." + ) + if self.cfg.shape_init_mesh_up[1] == self.cfg.shape_init_mesh_front[1]: + raise ValueError( + "shape_init_mesh_up and shape_init_mesh_front must be orthogonal." + ) + z_, x_ = ( + dir2vec[self.cfg.shape_init_mesh_up], + dir2vec[self.cfg.shape_init_mesh_front], + ) + y_ = np.cross(z_, x_) + std2mesh = np.stack([x_, y_, z_], axis=0).T + mesh2std = np.linalg.inv(std2mesh) + + # scaling + scale = np.abs(mesh.vertices).max() + mesh.vertices = mesh.vertices / scale * self.cfg.shape_init_params + mesh.vertices = np.dot(mesh2std, mesh.vertices.T).T + + from pysdf import SDF + + sdf = SDF(mesh.vertices, mesh.faces) + + def func(points_rand: Float[Tensor, "N 3"]) -> Float[Tensor, "N 1"]: + # add a negative signed here + # as in pysdf the inside of the shape has positive signed distance + return torch.from_numpy(-sdf(points_rand.cpu().numpy())).to( + points_rand + )[..., None] + + get_gt_sdf = func + + else: + raise ValueError( + f"Unknown shape initialization type: {self.cfg.shape_init}" + ) + + # Initialize SDF to a given shape when no weights are provided or force_shape_init is True + optim = torch.optim.Adam(self.parameters(), lr=1e-3) + from tqdm import tqdm + + for _ in tqdm( + range(1000), + desc=f"Initializing SDF to a(n) {self.cfg.shape_init}:", + disable=get_rank() != 0, + ): + points_rand = ( + torch.rand((10000, 3), dtype=torch.float32).to(self.device) * 2.0 - 1.0 + ) + sdf_gt = get_gt_sdf(points_rand) + sdf_pred = self.forward_sdf(points_rand) + loss = F.mse_loss(sdf_pred, sdf_gt) + optim.zero_grad() + loss.backward() + optim.step() + + # explicit broadcast to ensure param consistency across ranks + for param in self.parameters(): + broadcast(param, src=0) + + def get_shifted_sdf( + self, points: Float[Tensor, "*N Di"], sdf: Float[Tensor, "*N 1"] + ) -> Float[Tensor, "*N 1"]: + sdf_bias: Union[float, Float[Tensor, "*N 1"]] + if self.cfg.sdf_bias == "ellipsoid": + assert ( + isinstance(self.cfg.sdf_bias_params, Sized) + and len(self.cfg.sdf_bias_params) == 3 + ) + size = torch.as_tensor(self.cfg.sdf_bias_params).to(points) + sdf_bias = ((points / size) ** 2).sum( + dim=-1, keepdim=True + ).sqrt() - 1.0 # pseudo signed distance of an ellipsoid + elif self.cfg.sdf_bias == "sphere": + assert isinstance(self.cfg.sdf_bias_params, float) + radius = self.cfg.sdf_bias_params + sdf_bias = (points**2).sum(dim=-1, keepdim=True).sqrt() - radius + elif isinstance(self.cfg.sdf_bias, float): + sdf_bias = self.cfg.sdf_bias + else: + raise ValueError(f"Unknown sdf bias {self.cfg.sdf_bias}") + return sdf + sdf_bias + + def forward( + self, points: Float[Tensor, "*N Di"], output_normal: bool = False + ) -> Dict[str, Float[Tensor, "..."]]: + grad_enabled = torch.is_grad_enabled() + + if output_normal and self.cfg.normal_type == "analytic": + torch.set_grad_enabled(True) + points.requires_grad_(True) + + points_unscaled = points # points in the original scale + points = contract_to_unisphere( + points, self.bbox, self.unbounded + ) # points normalized to (0, 1) + + enc = self.encoding(points.view(-1, self.cfg.n_input_dims)) + sdf = self.sdf_network(enc).view(*points.shape[:-1], 1) + sdf = self.get_shifted_sdf(points_unscaled, sdf) + output = {"sdf": sdf} + + if self.cfg.n_feature_dims > 0: + features = self.feature_network(enc).view( + *points.shape[:-1], self.cfg.n_feature_dims + ) + output.update({"features": features}) + + if output_normal: + if ( + self.cfg.normal_type == "finite_difference" + or self.cfg.normal_type == "finite_difference_laplacian" + ): + assert self.finite_difference_normal_eps is not None + eps: float = self.finite_difference_normal_eps + if self.cfg.normal_type == "finite_difference_laplacian": + offsets: Float[Tensor, "6 3"] = torch.as_tensor( + [ + [eps, 0.0, 0.0], + [-eps, 0.0, 0.0], + [0.0, eps, 0.0], + [0.0, -eps, 0.0], + [0.0, 0.0, eps], + [0.0, 0.0, -eps], + ] + ).to(points_unscaled) + points_offset: Float[Tensor, "... 6 3"] = ( + points_unscaled[..., None, :] + offsets + ).clamp(-self.cfg.radius, self.cfg.radius) + sdf_offset: Float[Tensor, "... 6 1"] = self.forward_sdf( + points_offset + ) + sdf_grad = ( + 0.5 + * (sdf_offset[..., 0::2, 0] - sdf_offset[..., 1::2, 0]) + / eps + ) + else: + offsets: Float[Tensor, "3 3"] = torch.as_tensor( + [[eps, 0.0, 0.0], [0.0, eps, 0.0], [0.0, 0.0, eps]] + ).to(points_unscaled) + points_offset: Float[Tensor, "... 3 3"] = ( + points_unscaled[..., None, :] + offsets + ).clamp(-self.cfg.radius, self.cfg.radius) + sdf_offset: Float[Tensor, "... 3 1"] = self.forward_sdf( + points_offset + ) + sdf_grad = (sdf_offset[..., 0::1, 0] - sdf) / eps + normal = F.normalize(sdf_grad, dim=-1) + elif self.cfg.normal_type == "pred": + normal = self.normal_network(enc).view(*points.shape[:-1], 3) + normal = F.normalize(normal, dim=-1) + sdf_grad = normal + elif self.cfg.normal_type == "analytic": + sdf_grad = -torch.autograd.grad( + sdf, + points_unscaled, + grad_outputs=torch.ones_like(sdf), + create_graph=True, + )[0] + normal = F.normalize(sdf_grad, dim=-1) + if not grad_enabled: + sdf_grad = sdf_grad.detach() + normal = normal.detach() + else: + raise AttributeError(f"Unknown normal type {self.cfg.normal_type}") + output.update( + {"normal": normal, "shading_normal": normal, "sdf_grad": sdf_grad} + ) + return output + + def forward_sdf(self, points: Float[Tensor, "*N Di"]) -> Float[Tensor, "*N 1"]: + points_unscaled = points + points = contract_to_unisphere(points_unscaled, self.bbox, self.unbounded) + + sdf = self.sdf_network( + self.encoding(points.reshape(-1, self.cfg.n_input_dims)) + ).reshape(*points.shape[:-1], 1) + sdf = self.get_shifted_sdf(points_unscaled, sdf) + return sdf + + def forward_field( + self, points: Float[Tensor, "*N Di"] + ) -> Tuple[Float[Tensor, "*N 1"], Optional[Float[Tensor, "*N 3"]]]: + points_unscaled = points + points = contract_to_unisphere(points_unscaled, self.bbox, self.unbounded) + enc = self.encoding(points.reshape(-1, self.cfg.n_input_dims)) + sdf = self.sdf_network(enc).reshape(*points.shape[:-1], 1) + sdf = self.get_shifted_sdf(points_unscaled, sdf) + deformation: Optional[Float[Tensor, "*N 3"]] = None + if self.cfg.isosurface_deformable_grid: + deformation = self.deformation_network(enc).reshape(*points.shape[:-1], 3) + return sdf, deformation + + def forward_level( + self, field: Float[Tensor, "*N 1"], threshold: float + ) -> Float[Tensor, "*N 1"]: + return field - threshold + + def export(self, points: Float[Tensor, "*N Di"], **kwargs) -> Dict[str, Any]: + out: Dict[str, Any] = {} + if self.cfg.n_feature_dims == 0: + return out + points_unscaled = points + points = contract_to_unisphere(points_unscaled, self.bbox, self.unbounded) + enc = self.encoding(points.reshape(-1, self.cfg.n_input_dims)) + features = self.feature_network(enc).view( + *points.shape[:-1], self.cfg.n_feature_dims + ) + out.update( + { + "features": features, + } + ) + return out + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + if ( + self.cfg.normal_type == "finite_difference" + or self.cfg.normal_type == "finite_difference_laplacian" + ): + if isinstance(self.cfg.finite_difference_normal_eps, float): + self.finite_difference_normal_eps = ( + self.cfg.finite_difference_normal_eps + ) + elif self.cfg.finite_difference_normal_eps == "progressive": + # progressive finite difference eps from Neuralangelo + # https://arxiv.org/abs/2306.03092 + hg_conf: Any = self.cfg.pos_encoding_config + assert ( + hg_conf.otype == "ProgressiveBandHashGrid" + ), "finite_difference_normal_eps=progressive only works with ProgressiveBandHashGrid" + current_level = min( + hg_conf.start_level + + max(global_step - hg_conf.start_step, 0) // hg_conf.update_steps, + hg_conf.n_levels, + ) + grid_res = hg_conf.base_resolution * hg_conf.per_level_scale ** ( + current_level - 1 + ) + grid_size = 2 * self.cfg.radius / grid_res + if grid_size != self.finite_difference_normal_eps: + threestudio.info( + f"Update finite_difference_normal_eps to {grid_size}" + ) + self.finite_difference_normal_eps = grid_size + else: + raise ValueError( + f"Unknown finite_difference_normal_eps={self.cfg.finite_difference_normal_eps}" + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/implicit_volume.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/implicit_volume.py new file mode 100644 index 0000000000000000000000000000000000000000..cfee001744b1035d16e446fc4c51914bc5ea40d1 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/implicit_volume.py @@ -0,0 +1,285 @@ +from dataclasses import dataclass, field + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.geometry.base import ( + BaseGeometry, + BaseImplicitGeometry, + contract_to_unisphere, +) +from threestudio.models.networks import get_encoding, get_mlp +from threestudio.utils.ops import get_activation +from threestudio.utils.typing import * + + +@threestudio.register("implicit-volume") +class ImplicitVolume(BaseImplicitGeometry): + @dataclass + class Config(BaseImplicitGeometry.Config): + n_input_dims: int = 3 + n_feature_dims: int = 3 + density_activation: Optional[str] = "softplus" + density_bias: Union[float, str] = "blob_magic3d" + density_blob_scale: float = 10.0 + density_blob_std: float = 0.5 + pos_encoding_config: dict = field( + default_factory=lambda: { + "otype": "HashGrid", + "n_levels": 16, + "n_features_per_level": 2, + "log2_hashmap_size": 19, + "base_resolution": 16, + "per_level_scale": 1.447269237440378, + } + ) + mlp_network_config: dict = field( + default_factory=lambda: { + "otype": "VanillaMLP", + "activation": "ReLU", + "output_activation": "none", + "n_neurons": 64, + "n_hidden_layers": 1, + } + ) + normal_type: Optional[ + str + ] = "finite_difference" # in ['pred', 'finite_difference', 'finite_difference_laplacian'] + finite_difference_normal_eps: float = 0.01 + + # automatically determine the threshold + isosurface_threshold: Union[float, str] = 25.0 + + # 4D Gaussian Annealing + anneal_density_blob_std_config: Optional[dict] = None + + cfg: Config + + def configure(self) -> None: + super().configure() + self.encoding = get_encoding( + self.cfg.n_input_dims, self.cfg.pos_encoding_config + ) + self.density_network = get_mlp( + self.encoding.n_output_dims, 1, self.cfg.mlp_network_config + ) + if self.cfg.n_feature_dims > 0: + self.feature_network = get_mlp( + self.encoding.n_output_dims, + self.cfg.n_feature_dims, + self.cfg.mlp_network_config, + ) + if self.cfg.normal_type == "pred": + self.normal_network = get_mlp( + self.encoding.n_output_dims, 3, self.cfg.mlp_network_config + ) + + def get_activated_density( + self, points: Float[Tensor, "*N Di"], density: Float[Tensor, "*N 1"] + ) -> Tuple[Float[Tensor, "*N 1"], Float[Tensor, "*N 1"]]: + density_bias: Union[float, Float[Tensor, "*N 1"]] + if self.cfg.density_bias == "blob_dreamfusion": + # pre-activation density bias + density_bias = ( + self.cfg.density_blob_scale + * torch.exp( + -0.5 * (points**2).sum(dim=-1) / self.cfg.density_blob_std**2 + )[..., None] + ) + elif self.cfg.density_bias == "blob_magic3d": + # pre-activation density bias + density_bias = ( + self.cfg.density_blob_scale + * ( + 1 + - torch.sqrt((points**2).sum(dim=-1)) / self.cfg.density_blob_std + )[..., None] + ) + elif isinstance(self.cfg.density_bias, float): + density_bias = self.cfg.density_bias + else: + raise ValueError(f"Unknown density bias {self.cfg.density_bias}") + raw_density: Float[Tensor, "*N 1"] = density + density_bias + density = get_activation(self.cfg.density_activation)(raw_density) + return raw_density, density + + def forward( + self, points: Float[Tensor, "*N Di"], output_normal: bool = False + ) -> Dict[str, Float[Tensor, "..."]]: + grad_enabled = torch.is_grad_enabled() + + if output_normal and self.cfg.normal_type == "analytic": + torch.set_grad_enabled(True) + points.requires_grad_(True) + + points_unscaled = points # points in the original scale + points = contract_to_unisphere( + points, self.bbox, self.unbounded + ) # points normalized to (0, 1) + + enc = self.encoding(points.view(-1, self.cfg.n_input_dims)) + density = self.density_network(enc).view(*points.shape[:-1], 1) + raw_density, density = self.get_activated_density(points_unscaled, density) + + output = { + "density": density, + } + + if self.cfg.n_feature_dims > 0: + features = self.feature_network(enc).view( + *points.shape[:-1], self.cfg.n_feature_dims + ) + output.update({"features": features}) + + if output_normal: + if ( + self.cfg.normal_type == "finite_difference" + or self.cfg.normal_type == "finite_difference_laplacian" + ): + # TODO: use raw density + eps = self.cfg.finite_difference_normal_eps + if self.cfg.normal_type == "finite_difference_laplacian": + offsets: Float[Tensor, "6 3"] = torch.as_tensor( + [ + [eps, 0.0, 0.0], + [-eps, 0.0, 0.0], + [0.0, eps, 0.0], + [0.0, -eps, 0.0], + [0.0, 0.0, eps], + [0.0, 0.0, -eps], + ] + ).to(points_unscaled) + points_offset: Float[Tensor, "... 6 3"] = ( + points_unscaled[..., None, :] + offsets + ).clamp(-self.cfg.radius, self.cfg.radius) + density_offset: Float[Tensor, "... 6 1"] = self.forward_density( + points_offset + ) + normal = ( + -0.5 + * (density_offset[..., 0::2, 0] - density_offset[..., 1::2, 0]) + / eps + ) + else: + offsets: Float[Tensor, "3 3"] = torch.as_tensor( + [[eps, 0.0, 0.0], [0.0, eps, 0.0], [0.0, 0.0, eps]] + ).to(points_unscaled) + points_offset: Float[Tensor, "... 3 3"] = ( + points_unscaled[..., None, :] + offsets + ).clamp(-self.cfg.radius, self.cfg.radius) + density_offset: Float[Tensor, "... 3 1"] = self.forward_density( + points_offset + ) + normal = -(density_offset[..., 0::1, 0] - density) / eps + normal = F.normalize(normal, dim=-1) + elif self.cfg.normal_type == "pred": + normal = self.normal_network(enc).view(*points.shape[:-1], 3) + normal = F.normalize(normal, dim=-1) + elif self.cfg.normal_type == "analytic": + normal = -torch.autograd.grad( + density, + points_unscaled, + grad_outputs=torch.ones_like(density), + create_graph=True, + )[0] + normal = F.normalize(normal, dim=-1) + if not grad_enabled: + normal = normal.detach() + else: + raise AttributeError(f"Unknown normal type {self.cfg.normal_type}") + output.update({"normal": normal, "shading_normal": normal}) + + torch.set_grad_enabled(grad_enabled) + return output + + def forward_density(self, points: Float[Tensor, "*N Di"]) -> Float[Tensor, "*N 1"]: + points_unscaled = points + points = contract_to_unisphere(points_unscaled, self.bbox, self.unbounded) + + density = self.density_network( + self.encoding(points.reshape(-1, self.cfg.n_input_dims)) + ).reshape(*points.shape[:-1], 1) + + _, density = self.get_activated_density(points_unscaled, density) + return density + + def forward_field( + self, points: Float[Tensor, "*N Di"] + ) -> Tuple[Float[Tensor, "*N 1"], Optional[Float[Tensor, "*N 3"]]]: + if self.cfg.isosurface_deformable_grid: + threestudio.warn( + f"{self.__class__.__name__} does not support isosurface_deformable_grid. Ignoring." + ) + density = self.forward_density(points) + return density, None + + def forward_level( + self, field: Float[Tensor, "*N 1"], threshold: float + ) -> Float[Tensor, "*N 1"]: + return -(field - threshold) + + def export(self, points: Float[Tensor, "*N Di"], **kwargs) -> Dict[str, Any]: + out: Dict[str, Any] = {} + if self.cfg.n_feature_dims == 0: + return out + points_unscaled = points + points = contract_to_unisphere(points_unscaled, self.bbox, self.unbounded) + enc = self.encoding(points.reshape(-1, self.cfg.n_input_dims)) + features = self.feature_network(enc).view( + *points.shape[:-1], self.cfg.n_feature_dims + ) + out.update( + { + "features": features, + } + ) + return out + + @staticmethod + @torch.no_grad() + def create_from( + other: BaseGeometry, + cfg: Optional[Union[dict, DictConfig]] = None, + copy_net: bool = True, + **kwargs, + ) -> "ImplicitVolume": + if isinstance(other, ImplicitVolume): + instance = ImplicitVolume(cfg, **kwargs) + instance.encoding.load_state_dict(other.encoding.state_dict()) + instance.density_network.load_state_dict(other.density_network.state_dict()) + if copy_net: + if ( + instance.cfg.n_feature_dims > 0 + and other.cfg.n_feature_dims == instance.cfg.n_feature_dims + ): + instance.feature_network.load_state_dict( + other.feature_network.state_dict() + ) + if ( + instance.cfg.normal_type == "pred" + and other.cfg.normal_type == "pred" + ): + instance.normal_network.load_state_dict( + other.normal_network.state_dict() + ) + return instance + else: + raise TypeError( + f"Cannot create {ImplicitVolume.__name__} from {other.__class__.__name__}" + ) + + def update_step( + self, epoch: int, global_step: int, on_load_weights: bool = False + ) -> None: + if self.cfg.anneal_density_blob_std_config is not None: + min_step = self.cfg.anneal_density_blob_std_config.min_anneal_step + max_step = self.cfg.anneal_density_blob_std_config.max_anneal_step + if global_step >= min_step and global_step <= max_step: + end_val = self.cfg.anneal_density_blob_std_config.end_val + start_val = self.cfg.anneal_density_blob_std_config.start_val + self.density_blob_std = start_val + (global_step - min_step) * ( + end_val - start_val + ) / (max_step - min_step) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/tetrahedra_sdf_grid.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/tetrahedra_sdf_grid.py new file mode 100644 index 0000000000000000000000000000000000000000..71f5c07bb74620a672e3268bebf47c58882a1c00 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/tetrahedra_sdf_grid.py @@ -0,0 +1,369 @@ +import os +from dataclasses import dataclass, field + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.geometry.base import ( + BaseExplicitGeometry, + BaseGeometry, + contract_to_unisphere, +) +from threestudio.models.geometry.implicit_sdf import ImplicitSDF +from threestudio.models.geometry.implicit_volume import ImplicitVolume +from threestudio.models.isosurface import MarchingTetrahedraHelper +from threestudio.models.mesh import Mesh +from threestudio.models.networks import get_encoding, get_mlp +from threestudio.utils.misc import broadcast +from threestudio.utils.ops import scale_tensor +from threestudio.utils.typing import * + + +@threestudio.register("tetrahedra-sdf-grid") +class TetrahedraSDFGrid(BaseExplicitGeometry): + @dataclass + class Config(BaseExplicitGeometry.Config): + isosurface_resolution: int = 128 + isosurface_deformable_grid: bool = True + isosurface_remove_outliers: bool = False + isosurface_outlier_n_faces_threshold: Union[int, float] = 0.01 + + n_input_dims: int = 3 + n_feature_dims: int = 3 + pos_encoding_config: dict = field( + default_factory=lambda: { + "otype": "HashGrid", + "n_levels": 16, + "n_features_per_level": 2, + "log2_hashmap_size": 19, + "base_resolution": 16, + "per_level_scale": 1.447269237440378, + } + ) + mlp_network_config: dict = field( + default_factory=lambda: { + "otype": "VanillaMLP", + "activation": "ReLU", + "output_activation": "none", + "n_neurons": 64, + "n_hidden_layers": 1, + } + ) + shape_init: Optional[str] = None + shape_init_params: Optional[Any] = None + shape_init_mesh_up: str = "+z" + shape_init_mesh_front: str = "+x" + force_shape_init: bool = False + geometry_only: bool = False + fix_geometry: bool = False + + cfg: Config + + def configure(self) -> None: + super().configure() + + # this should be saved to state_dict, register as buffer + self.isosurface_bbox: Float[Tensor, "2 3"] + self.register_buffer("isosurface_bbox", self.bbox.clone()) + + self.isosurface_helper = MarchingTetrahedraHelper( + self.cfg.isosurface_resolution, + f"load/tets/{self.cfg.isosurface_resolution}_tets.npz", + ) + + self.sdf: Float[Tensor, "Nv 1"] + self.deformation: Optional[Float[Tensor, "Nv 3"]] + + if not self.cfg.fix_geometry: + self.register_parameter( + "sdf", + nn.Parameter( + torch.zeros( + (self.isosurface_helper.grid_vertices.shape[0], 1), + dtype=torch.float32, + ) + ), + ) + if self.cfg.isosurface_deformable_grid: + self.register_parameter( + "deformation", + nn.Parameter( + torch.zeros_like(self.isosurface_helper.grid_vertices) + ), + ) + else: + self.deformation = None + else: + self.register_buffer( + "sdf", + torch.zeros( + (self.isosurface_helper.grid_vertices.shape[0], 1), + dtype=torch.float32, + ), + ) + if self.cfg.isosurface_deformable_grid: + self.register_buffer( + "deformation", + torch.zeros_like(self.isosurface_helper.grid_vertices), + ) + else: + self.deformation = None + + if not self.cfg.geometry_only: + self.encoding = get_encoding( + self.cfg.n_input_dims, self.cfg.pos_encoding_config + ) + self.feature_network = get_mlp( + self.encoding.n_output_dims, + self.cfg.n_feature_dims, + self.cfg.mlp_network_config, + ) + + self.mesh: Optional[Mesh] = None + + def initialize_shape(self) -> None: + if self.cfg.shape_init is None and not self.cfg.force_shape_init: + return + + # do not initialize shape if weights are provided + if self.cfg.weights is not None and not self.cfg.force_shape_init: + return + + get_gt_sdf: Callable[[Float[Tensor, "N 3"]], Float[Tensor, "N 1"]] + assert isinstance(self.cfg.shape_init, str) + if self.cfg.shape_init == "ellipsoid": + assert ( + isinstance(self.cfg.shape_init_params, Sized) + and len(self.cfg.shape_init_params) == 3 + ) + size = torch.as_tensor(self.cfg.shape_init_params).to(self.device) + + def func(points_rand: Float[Tensor, "N 3"]) -> Float[Tensor, "N 1"]: + return ((points_rand / size) ** 2).sum( + dim=-1, keepdim=True + ).sqrt() - 1.0 # pseudo signed distance of an ellipsoid + + get_gt_sdf = func + elif self.cfg.shape_init == "sphere": + assert isinstance(self.cfg.shape_init_params, float) + radius = self.cfg.shape_init_params + + def func(points_rand: Float[Tensor, "N 3"]) -> Float[Tensor, "N 1"]: + return (points_rand**2).sum(dim=-1, keepdim=True).sqrt() - radius + + get_gt_sdf = func + elif self.cfg.shape_init.startswith("mesh:"): + assert isinstance(self.cfg.shape_init_params, float) + mesh_path = self.cfg.shape_init[5:] + if not os.path.exists(mesh_path): + raise ValueError(f"Mesh file {mesh_path} does not exist.") + + import trimesh + + mesh = trimesh.load(mesh_path) + + # move to center + centroid = mesh.vertices.mean(0) + mesh.vertices = mesh.vertices - centroid + + # align to up-z and front-x + dirs = ["+x", "+y", "+z", "-x", "-y", "-z"] + dir2vec = { + "+x": np.array([1, 0, 0]), + "+y": np.array([0, 1, 0]), + "+z": np.array([0, 0, 1]), + "-x": np.array([-1, 0, 0]), + "-y": np.array([0, -1, 0]), + "-z": np.array([0, 0, -1]), + } + if ( + self.cfg.shape_init_mesh_up not in dirs + or self.cfg.shape_init_mesh_front not in dirs + ): + raise ValueError( + f"shape_init_mesh_up and shape_init_mesh_front must be one of {dirs}." + ) + if self.cfg.shape_init_mesh_up[1] == self.cfg.shape_init_mesh_front[1]: + raise ValueError( + "shape_init_mesh_up and shape_init_mesh_front must be orthogonal." + ) + z_, x_ = ( + dir2vec[self.cfg.shape_init_mesh_up], + dir2vec[self.cfg.shape_init_mesh_front], + ) + y_ = np.cross(z_, x_) + std2mesh = np.stack([x_, y_, z_], axis=0).T + mesh2std = np.linalg.inv(std2mesh) + + # scaling + scale = np.abs(mesh.vertices).max() + mesh.vertices = mesh.vertices / scale * self.cfg.shape_init_params + mesh.vertices = np.dot(mesh2std, mesh.vertices.T).T + + from pysdf import SDF + + sdf = SDF(mesh.vertices, mesh.faces) + + def func(points_rand: Float[Tensor, "N 3"]) -> Float[Tensor, "N 1"]: + # add a negative signed here + # as in pysdf the inside of the shape has positive signed distance + return torch.from_numpy(-sdf(points_rand.cpu().numpy())).to( + points_rand + )[..., None] + + get_gt_sdf = func + + else: + raise ValueError( + f"Unknown shape initialization type: {self.cfg.shape_init}" + ) + + sdf_gt = get_gt_sdf( + scale_tensor( + self.isosurface_helper.grid_vertices, + self.isosurface_helper.points_range, + self.isosurface_bbox, + ) + ) + self.sdf.data = sdf_gt + + # explicit broadcast to ensure param consistency across ranks + for param in self.parameters(): + broadcast(param, src=0) + + def isosurface(self) -> Mesh: + # return cached mesh if fix_geometry is True to save computation + if self.cfg.fix_geometry and self.mesh is not None: + return self.mesh + mesh = self.isosurface_helper(self.sdf, self.deformation) + mesh.v_pos = scale_tensor( + mesh.v_pos, self.isosurface_helper.points_range, self.isosurface_bbox + ) + if self.cfg.isosurface_remove_outliers: + mesh = mesh.remove_outlier(self.cfg.isosurface_outlier_n_faces_threshold) + self.mesh = mesh + return mesh + + def forward( + self, points: Float[Tensor, "*N Di"], output_normal: bool = False + ) -> Dict[str, Float[Tensor, "..."]]: + if self.cfg.geometry_only: + return {} + assert ( + output_normal == False + ), f"Normal output is not supported for {self.__class__.__name__}" + points_unscaled = points # points in the original scale + points = contract_to_unisphere(points, self.bbox) # points normalized to (0, 1) + enc = self.encoding(points.view(-1, self.cfg.n_input_dims)) + features = self.feature_network(enc).view( + *points.shape[:-1], self.cfg.n_feature_dims + ) + return {"features": features} + + @staticmethod + @torch.no_grad() + def create_from( + other: BaseGeometry, + cfg: Optional[Union[dict, DictConfig]] = None, + copy_net: bool = True, + **kwargs, + ) -> "TetrahedraSDFGrid": + if isinstance(other, TetrahedraSDFGrid): + instance = TetrahedraSDFGrid(cfg, **kwargs) + assert instance.cfg.isosurface_resolution == other.cfg.isosurface_resolution + instance.isosurface_bbox = other.isosurface_bbox.clone() + instance.sdf.data = other.sdf.data.clone() + if ( + instance.cfg.isosurface_deformable_grid + and other.cfg.isosurface_deformable_grid + ): + assert ( + instance.deformation is not None and other.deformation is not None + ) + instance.deformation.data = other.deformation.data.clone() + if ( + not instance.cfg.geometry_only + and not other.cfg.geometry_only + and copy_net + ): + instance.encoding.load_state_dict(other.encoding.state_dict()) + instance.feature_network.load_state_dict( + other.feature_network.state_dict() + ) + return instance + elif isinstance(other, ImplicitVolume): + instance = TetrahedraSDFGrid(cfg, **kwargs) + if other.cfg.isosurface_method != "mt": + other.cfg.isosurface_method = "mt" + threestudio.warn( + f"Override isosurface_method of the source geometry to 'mt'" + ) + if other.cfg.isosurface_resolution != instance.cfg.isosurface_resolution: + other.cfg.isosurface_resolution = instance.cfg.isosurface_resolution + threestudio.warn( + f"Override isosurface_resolution of the source geometry to {instance.cfg.isosurface_resolution}" + ) + mesh = other.isosurface() + instance.isosurface_bbox = mesh.extras["bbox"] + instance.sdf.data = ( + mesh.extras["grid_level"].to(instance.sdf.data).clamp(-1, 1) + ) + if not instance.cfg.geometry_only and copy_net: + instance.encoding.load_state_dict(other.encoding.state_dict()) + instance.feature_network.load_state_dict( + other.feature_network.state_dict() + ) + return instance + elif isinstance(other, ImplicitSDF): + instance = TetrahedraSDFGrid(cfg, **kwargs) + if other.cfg.isosurface_method != "mt": + other.cfg.isosurface_method = "mt" + threestudio.warn( + f"Override isosurface_method of the source geometry to 'mt'" + ) + if other.cfg.isosurface_resolution != instance.cfg.isosurface_resolution: + other.cfg.isosurface_resolution = instance.cfg.isosurface_resolution + threestudio.warn( + f"Override isosurface_resolution of the source geometry to {instance.cfg.isosurface_resolution}" + ) + mesh = other.isosurface() + instance.isosurface_bbox = mesh.extras["bbox"] + instance.sdf.data = mesh.extras["grid_level"].to(instance.sdf.data) + if ( + instance.cfg.isosurface_deformable_grid + and other.cfg.isosurface_deformable_grid + ): + assert instance.deformation is not None + instance.deformation.data = mesh.extras["grid_deformation"].to( + instance.deformation.data + ) + if not instance.cfg.geometry_only and copy_net: + instance.encoding.load_state_dict(other.encoding.state_dict()) + instance.feature_network.load_state_dict( + other.feature_network.state_dict() + ) + return instance + else: + raise TypeError( + f"Cannot create {TetrahedraSDFGrid.__name__} from {other.__class__.__name__}" + ) + + def export(self, points: Float[Tensor, "*N Di"], **kwargs) -> Dict[str, Any]: + out: Dict[str, Any] = {} + if self.cfg.geometry_only or self.cfg.n_feature_dims == 0: + return out + points_unscaled = points + points = contract_to_unisphere(points_unscaled, self.bbox) + enc = self.encoding(points.reshape(-1, self.cfg.n_input_dims)) + features = self.feature_network(enc).view( + *points.shape[:-1], self.cfg.n_feature_dims + ) + out.update( + { + "features": features, + } + ) + return out diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/volume_grid.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/volume_grid.py new file mode 100644 index 0000000000000000000000000000000000000000..ae258a117802fe4b48cf59c2e756a42b3723a5a4 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/geometry/volume_grid.py @@ -0,0 +1,190 @@ +from dataclasses import dataclass, field + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.geometry.base import BaseImplicitGeometry, contract_to_unisphere +from threestudio.utils.ops import get_activation +from threestudio.utils.typing import * + + +@threestudio.register("volume-grid") +class VolumeGrid(BaseImplicitGeometry): + @dataclass + class Config(BaseImplicitGeometry.Config): + grid_size: Tuple[int, int, int] = field(default_factory=lambda: (100, 100, 100)) + n_feature_dims: int = 3 + density_activation: Optional[str] = "softplus" + density_bias: Union[float, str] = "blob" + density_blob_scale: float = 5.0 + density_blob_std: float = 0.5 + normal_type: Optional[ + str + ] = "finite_difference" # in ['pred', 'finite_difference', 'finite_difference_laplacian'] + + # automatically determine the threshold + isosurface_threshold: Union[float, str] = "auto" + + cfg: Config + + def configure(self) -> None: + super().configure() + self.grid_size = self.cfg.grid_size + + self.grid = nn.Parameter( + torch.zeros(1, self.cfg.n_feature_dims + 1, *self.grid_size) + ) + if self.cfg.density_bias == "blob": + self.register_buffer("density_scale", torch.tensor(0.0)) + else: + self.density_scale = nn.Parameter(torch.tensor(0.0)) + + if self.cfg.normal_type == "pred": + self.normal_grid = nn.Parameter(torch.zeros(1, 3, *self.grid_size)) + + def get_density_bias(self, points: Float[Tensor, "*N Di"]): + if self.cfg.density_bias == "blob": + # density_bias: Float[Tensor, "*N 1"] = self.cfg.density_blob_scale * torch.exp(-0.5 * (points ** 2).sum(dim=-1) / self.cfg.density_blob_std ** 2)[...,None] + density_bias: Float[Tensor, "*N 1"] = ( + self.cfg.density_blob_scale + * ( + 1 + - torch.sqrt((points.detach() ** 2).sum(dim=-1)) + / self.cfg.density_blob_std + )[..., None] + ) + return density_bias + elif isinstance(self.cfg.density_bias, float): + return self.cfg.density_bias + else: + raise AttributeError(f"Unknown density bias {self.cfg.density_bias}") + + def get_trilinear_feature( + self, points: Float[Tensor, "*N Di"], grid: Float[Tensor, "1 Df G1 G2 G3"] + ) -> Float[Tensor, "*N Df"]: + points_shape = points.shape[:-1] + df = grid.shape[1] + di = points.shape[-1] + out = F.grid_sample( + grid, points.view(1, 1, 1, -1, di), align_corners=False, mode="bilinear" + ) + out = out.reshape(df, -1).T.reshape(*points_shape, df) + return out + + def forward( + self, points: Float[Tensor, "*N Di"], output_normal: bool = False + ) -> Dict[str, Float[Tensor, "..."]]: + points_unscaled = points # points in the original scale + points = contract_to_unisphere( + points, self.bbox, self.unbounded + ) # points normalized to (0, 1) + points = points * 2 - 1 # convert to [-1, 1] for grid sample + + out = self.get_trilinear_feature(points, self.grid) + density, features = out[..., 0:1], out[..., 1:] + density = density * torch.exp(self.density_scale) # exp scaling in DreamFusion + + # breakpoint() + density = get_activation(self.cfg.density_activation)( + density + self.get_density_bias(points_unscaled) + ) + + output = { + "density": density, + "features": features, + } + + if output_normal: + if ( + self.cfg.normal_type == "finite_difference" + or self.cfg.normal_type == "finite_difference_laplacian" + ): + eps = 1.0e-3 + if self.cfg.normal_type == "finite_difference_laplacian": + offsets: Float[Tensor, "6 3"] = torch.as_tensor( + [ + [eps, 0.0, 0.0], + [-eps, 0.0, 0.0], + [0.0, eps, 0.0], + [0.0, -eps, 0.0], + [0.0, 0.0, eps], + [0.0, 0.0, -eps], + ] + ).to(points_unscaled) + points_offset: Float[Tensor, "... 6 3"] = ( + points_unscaled[..., None, :] + offsets + ).clamp(-self.cfg.radius, self.cfg.radius) + density_offset: Float[Tensor, "... 6 1"] = self.forward_density( + points_offset + ) + normal = ( + -0.5 + * (density_offset[..., 0::2, 0] - density_offset[..., 1::2, 0]) + / eps + ) + else: + offsets: Float[Tensor, "3 3"] = torch.as_tensor( + [[eps, 0.0, 0.0], [0.0, eps, 0.0], [0.0, 0.0, eps]] + ).to(points_unscaled) + points_offset: Float[Tensor, "... 3 3"] = ( + points_unscaled[..., None, :] + offsets + ).clamp(-self.cfg.radius, self.cfg.radius) + density_offset: Float[Tensor, "... 3 1"] = self.forward_density( + points_offset + ) + normal = -(density_offset[..., 0::1, 0] - density) / eps + normal = F.normalize(normal, dim=-1) + elif self.cfg.normal_type == "pred": + normal = self.get_trilinear_feature(points, self.normal_grid) + normal = F.normalize(normal, dim=-1) + else: + raise AttributeError(f"Unknown normal type {self.cfg.normal_type}") + output.update({"normal": normal, "shading_normal": normal}) + return output + + def forward_density(self, points: Float[Tensor, "*N Di"]) -> Float[Tensor, "*N 1"]: + points_unscaled = points + points = contract_to_unisphere(points_unscaled, self.bbox, self.unbounded) + points = points * 2 - 1 # convert to [-1, 1] for grid sample + + out = self.get_trilinear_feature(points, self.grid) + density = out[..., 0:1] + density = density * torch.exp(self.density_scale) + + density = get_activation(self.cfg.density_activation)( + density + self.get_density_bias(points_unscaled) + ) + return density + + def forward_field( + self, points: Float[Tensor, "*N Di"] + ) -> Tuple[Float[Tensor, "*N 1"], Optional[Float[Tensor, "*N 3"]]]: + if self.cfg.isosurface_deformable_grid: + threestudio.warn( + f"{self.__class__.__name__} does not support isosurface_deformable_grid. Ignoring." + ) + density = self.forward_density(points) + return density, None + + def forward_level( + self, field: Float[Tensor, "*N 1"], threshold: float + ) -> Float[Tensor, "*N 1"]: + return -(field - threshold) + + def export(self, points: Float[Tensor, "*N Di"], **kwargs) -> Dict[str, Any]: + out: Dict[str, Any] = {} + if self.cfg.n_feature_dims == 0: + return out + points_unscaled = points + points = contract_to_unisphere(points, self.bbox, self.unbounded) + points = points * 2 - 1 # convert to [-1, 1] for grid sample + features = self.get_trilinear_feature(points, self.grid)[..., 1:] + out.update( + { + "features": features, + } + ) + return out diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1aee36c3262d88986e7052bbf4e5603de3e4605a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/__init__.py @@ -0,0 +1,11 @@ +from . import ( + controlnet_guidance, + deep_floyd_guidance, + instructpix2pix_guidance, + stable_diffusion_guidance, + stable_diffusion_unified_guidance, + stable_diffusion_vsd_guidance, + # stable_zero123_guidance, + # zero123_guidance, + # zero123_unified_guidance, +) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/controlnet_guidance.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/controlnet_guidance.py new file mode 100644 index 0000000000000000000000000000000000000000..ac014c715a129fd4f21f40fc067c622cdc385948 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/controlnet_guidance.py @@ -0,0 +1,427 @@ +import os +from dataclasses import dataclass + +import cv2 +import numpy as np +import torch +import torch.nn.functional as F +from controlnet_aux import CannyDetector, NormalBaeDetector +from diffusers import ControlNetModel, DDIMScheduler, StableDiffusionControlNetPipeline +from diffusers.utils.import_utils import is_xformers_available +from tqdm import tqdm + +import threestudio +from threestudio.models.prompt_processors.base import PromptProcessorOutput +from threestudio.utils.base import BaseObject +from threestudio.utils.misc import C, parse_version +from threestudio.utils.typing import * + + +@threestudio.register("stable-diffusion-controlnet-guidance") +class ControlNetGuidance(BaseObject): + @dataclass + class Config(BaseObject.Config): + cache_dir: Optional[str] = None + pretrained_model_name_or_path: str = "SG161222/Realistic_Vision_V2.0" + ddim_scheduler_name_or_path: str = "runwayml/stable-diffusion-v1-5" + control_type: str = "normal" # normal/canny + + enable_memory_efficient_attention: bool = False + enable_sequential_cpu_offload: bool = False + enable_attention_slicing: bool = False + enable_channels_last_format: bool = False + guidance_scale: float = 7.5 + condition_scale: float = 1.5 + grad_clip: Optional[ + Any + ] = None # field(default_factory=lambda: [0, 2.0, 8.0, 1000]) + half_precision_weights: bool = True + + fixed_size: int = -1 + + min_step_percent: float = 0.02 + max_step_percent: float = 0.98 + + diffusion_steps: int = 20 + + use_sds: bool = False + + # Canny threshold + canny_lower_bound: int = 50 + canny_upper_bound: int = 100 + + cfg: Config + + def configure(self) -> None: + threestudio.info(f"Loading ControlNet ...") + + controlnet_name_or_path: str + if self.cfg.control_type == "normal": + controlnet_name_or_path = "lllyasviel/control_v11p_sd15_normalbae" + elif self.cfg.control_type == "canny": + controlnet_name_or_path = "lllyasviel/control_v11p_sd15_canny" + + self.weights_dtype = ( + torch.float16 if self.cfg.half_precision_weights else torch.float32 + ) + + pipe_kwargs = { + "safety_checker": None, + "feature_extractor": None, + "requires_safety_checker": False, + "torch_dtype": self.weights_dtype, + "cache_dir": self.cfg.cache_dir, + } + + controlnet = ControlNetModel.from_pretrained( + controlnet_name_or_path, + torch_dtype=self.weights_dtype, + cache_dir=self.cfg.cache_dir, + ) + self.pipe = StableDiffusionControlNetPipeline.from_pretrained( + self.cfg.pretrained_model_name_or_path, controlnet=controlnet, **pipe_kwargs + ).to(self.device) + self.scheduler = DDIMScheduler.from_pretrained( + self.cfg.ddim_scheduler_name_or_path, + subfolder="scheduler", + torch_dtype=self.weights_dtype, + cache_dir=self.cfg.cache_dir, + ) + self.scheduler.set_timesteps(self.cfg.diffusion_steps) + + if self.cfg.enable_memory_efficient_attention: + if parse_version(torch.__version__) >= parse_version("2"): + threestudio.info( + "PyTorch2.0 uses memory efficient attention by default." + ) + elif not is_xformers_available(): + threestudio.warn( + "xformers is not available, memory efficient attention is not enabled." + ) + else: + self.pipe.enable_xformers_memory_efficient_attention() + + if self.cfg.enable_sequential_cpu_offload: + self.pipe.enable_sequential_cpu_offload() + + if self.cfg.enable_attention_slicing: + self.pipe.enable_attention_slicing(1) + + if self.cfg.enable_channels_last_format: + self.pipe.unet.to(memory_format=torch.channels_last) + + # Create model + self.vae = self.pipe.vae.eval() + self.unet = self.pipe.unet.eval() + self.controlnet = self.pipe.controlnet.eval() + + if self.cfg.control_type == "normal": + self.preprocessor = NormalBaeDetector.from_pretrained( + "lllyasviel/Annotators" + ) + self.preprocessor.model.to(self.device) + elif self.cfg.control_type == "canny": + self.preprocessor = CannyDetector() + + for p in self.vae.parameters(): + p.requires_grad_(False) + for p in self.unet.parameters(): + p.requires_grad_(False) + + self.num_train_timesteps = self.scheduler.config.num_train_timesteps + self.set_min_max_steps() # set to default value + + self.alphas: Float[Tensor, "..."] = self.scheduler.alphas_cumprod.to( + self.device + ) + + self.grad_clip_val: Optional[float] = None + + threestudio.info(f"Loaded ControlNet!") + + @torch.cuda.amp.autocast(enabled=False) + def set_min_max_steps(self, min_step_percent=0.02, max_step_percent=0.98): + self.min_step = int(self.num_train_timesteps * min_step_percent) + self.max_step = int(self.num_train_timesteps * max_step_percent) + + @torch.cuda.amp.autocast(enabled=False) + def forward_controlnet( + self, + latents: Float[Tensor, "..."], + t: Float[Tensor, "..."], + image_cond: Float[Tensor, "..."], + condition_scale: float, + encoder_hidden_states: Float[Tensor, "..."], + ) -> Float[Tensor, "..."]: + return self.controlnet( + latents.to(self.weights_dtype), + t.to(self.weights_dtype), + encoder_hidden_states=encoder_hidden_states.to(self.weights_dtype), + controlnet_cond=image_cond.to(self.weights_dtype), + conditioning_scale=condition_scale, + return_dict=False, + ) + + @torch.cuda.amp.autocast(enabled=False) + def forward_control_unet( + self, + latents: Float[Tensor, "..."], + t: Float[Tensor, "..."], + encoder_hidden_states: Float[Tensor, "..."], + cross_attention_kwargs, + down_block_additional_residuals, + mid_block_additional_residual, + ) -> Float[Tensor, "..."]: + input_dtype = latents.dtype + return self.unet( + latents.to(self.weights_dtype), + t.to(self.weights_dtype), + encoder_hidden_states=encoder_hidden_states.to(self.weights_dtype), + cross_attention_kwargs=cross_attention_kwargs, + down_block_additional_residuals=down_block_additional_residuals, + mid_block_additional_residual=mid_block_additional_residual, + ).sample.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def encode_images( + self, imgs: Float[Tensor, "B 3 H W"] + ) -> Float[Tensor, "B 4 DH DW"]: + input_dtype = imgs.dtype + imgs = imgs * 2.0 - 1.0 + posterior = self.vae.encode(imgs.to(self.weights_dtype)).latent_dist + latents = posterior.sample() * self.vae.config.scaling_factor + return latents.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def encode_cond_images( + self, imgs: Float[Tensor, "B 3 H W"] + ) -> Float[Tensor, "B 4 DH DW"]: + input_dtype = imgs.dtype + imgs = imgs * 2.0 - 1.0 + posterior = self.vae.encode(imgs.to(self.weights_dtype)).latent_dist + latents = posterior.mode() + uncond_image_latents = torch.zeros_like(latents) + latents = torch.cat([latents, latents, uncond_image_latents], dim=0) + return latents.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def decode_latents( + self, latents: Float[Tensor, "B 4 DH DW"] + ) -> Float[Tensor, "B 3 H W"]: + input_dtype = latents.dtype + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents.to(self.weights_dtype)).sample + image = (image * 0.5 + 0.5).clamp(0, 1) + return image.to(input_dtype) + + def edit_latents( + self, + text_embeddings: Float[Tensor, "BB 77 768"], + latents: Float[Tensor, "B 4 DH DW"], + image_cond: Float[Tensor, "B 3 H W"], + t: Int[Tensor, "B"], + ) -> Float[Tensor, "B 4 DH DW"]: + self.scheduler.config.num_train_timesteps = t.item() + self.scheduler.set_timesteps(self.cfg.diffusion_steps) + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) + latents = self.scheduler.add_noise(latents, noise, t) # type: ignore + + # sections of code used from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py + threestudio.debug("Start editing...") + for i, t in enumerate(self.scheduler.timesteps): + # predict the noise residual with unet, NO grad! + with torch.no_grad(): + # pred noise + latent_model_input = torch.cat([latents] * 2) + ( + down_block_res_samples, + mid_block_res_sample, + ) = self.forward_controlnet( + latent_model_input, + t, + encoder_hidden_states=text_embeddings, + image_cond=image_cond, + condition_scale=self.cfg.condition_scale, + ) + + noise_pred = self.forward_control_unet( + latent_model_input, + t, + encoder_hidden_states=text_embeddings, + cross_attention_kwargs=None, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + ) + # perform classifier-free guidance + noise_pred_text, noise_pred_uncond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + # get previous sample, continue loop + latents = self.scheduler.step(noise_pred, t, latents).prev_sample + threestudio.debug("Editing finished.") + return latents + + def prepare_image_cond(self, cond_rgb: Float[Tensor, "B H W C"]): + if self.cfg.control_type == "normal": + cond_rgb = ( + (cond_rgb[0].detach().cpu().numpy() * 255).astype(np.uint8).copy() + ) + detected_map = self.preprocessor(cond_rgb) + control = ( + torch.from_numpy(np.array(detected_map)).float().to(self.device) / 255.0 + ) + control = control.unsqueeze(0) + control = control.permute(0, 3, 1, 2) + elif self.cfg.control_type == "canny": + cond_rgb = ( + (cond_rgb[0].detach().cpu().numpy() * 255).astype(np.uint8).copy() + ) + blurred_img = cv2.blur(cond_rgb, ksize=(5, 5)) + detected_map = self.preprocessor( + blurred_img, self.cfg.canny_lower_bound, self.cfg.canny_upper_bound + ) + control = ( + torch.from_numpy(np.array(detected_map)).float().to(self.device) / 255.0 + ) + control = control.unsqueeze(-1).repeat(1, 1, 3) + control = control.unsqueeze(0) + control = control.permute(0, 3, 1, 2) + + return control + + def compute_grad_sds( + self, + text_embeddings: Float[Tensor, "BB 77 768"], + latents: Float[Tensor, "B 4 DH DW"], + image_cond: Float[Tensor, "B 3 H W"], + t: Int[Tensor, "B"], + ): + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) # TODO: use torch generator + latents_noisy = self.scheduler.add_noise(latents, noise, t) + # pred noise + latent_model_input = torch.cat([latents_noisy] * 2) + down_block_res_samples, mid_block_res_sample = self.forward_controlnet( + latent_model_input, + t, + encoder_hidden_states=text_embeddings, + image_cond=image_cond, + condition_scale=self.cfg.condition_scale, + ) + + noise_pred = self.forward_control_unet( + latent_model_input, + t, + encoder_hidden_states=text_embeddings, + cross_attention_kwargs=None, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + ) + + # perform classifier-free guidance + noise_pred_text, noise_pred_uncond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + + w = (1 - self.alphas[t]).view(-1, 1, 1, 1) + grad = w * (noise_pred - noise) + return grad + + def __call__( + self, + rgb: Float[Tensor, "B H W C"], + cond_rgb: Float[Tensor, "B H W C"], + prompt_utils: PromptProcessorOutput, + **kwargs, + ): + batch_size, H, W, _ = rgb.shape + assert batch_size == 1 + assert rgb.shape[:-1] == cond_rgb.shape[:-1] + + rgb_BCHW = rgb.permute(0, 3, 1, 2) + latents: Float[Tensor, "B 4 DH DW"] + if self.cfg.fixed_size > 0: + RH, RW = self.cfg.fixed_size, self.cfg.fixed_size + else: + RH, RW = H // 8 * 8, W // 8 * 8 + rgb_BCHW_HW8 = F.interpolate( + rgb_BCHW, (RH, RW), mode="bilinear", align_corners=False + ) + latents = self.encode_images(rgb_BCHW_HW8) + + image_cond = self.prepare_image_cond(cond_rgb) + image_cond = F.interpolate( + image_cond, (RH, RW), mode="bilinear", align_corners=False + ) + + temp = torch.zeros(1).to(rgb.device) + text_embeddings = prompt_utils.get_text_embeddings(temp, temp, temp, False) + + # timestep ~ U(0.02, 0.98) to avoid very high/low noise level + t = torch.randint( + self.min_step, + self.max_step + 1, + [batch_size], + dtype=torch.long, + device=self.device, + ) + + if self.cfg.use_sds: + grad = self.compute_grad_sds(text_embeddings, latents, image_cond, t) + grad = torch.nan_to_num(grad) + if self.grad_clip_val is not None: + grad = grad.clamp(-self.grad_clip_val, self.grad_clip_val) + target = (latents - grad).detach() + loss_sds = 0.5 * F.mse_loss(latents, target, reduction="sum") / batch_size + return { + "loss_sds": loss_sds, + "grad_norm": grad.norm(), + "min_step": self.min_step, + "max_step": self.max_step, + } + else: + edit_latents = self.edit_latents(text_embeddings, latents, image_cond, t) + edit_images = self.decode_latents(edit_latents) + edit_images = F.interpolate(edit_images, (H, W), mode="bilinear") + + return {"edit_images": edit_images.permute(0, 2, 3, 1)} + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + # clip grad for stable training as demonstrated in + # Debiasing Scores and Prompts of 2D Diffusion for Robust Text-to-3D Generation + # http://arxiv.org/abs/2303.15413 + if self.cfg.grad_clip is not None: + self.grad_clip_val = C(self.cfg.grad_clip, epoch, global_step) + + self.set_min_max_steps( + min_step_percent=C(self.cfg.min_step_percent, epoch, global_step), + max_step_percent=C(self.cfg.max_step_percent, epoch, global_step), + ) + + +if __name__ == "__main__": + from threestudio.utils.config import ExperimentConfig, load_config + from threestudio.utils.typing import Optional + + cfg = load_config("configs/debugging/controlnet-normal.yaml") + guidance = threestudio.find(cfg.system.guidance_type)(cfg.system.guidance) + prompt_processor = threestudio.find(cfg.system.prompt_processor_type)( + cfg.system.prompt_processor + ) + + rgb_image = cv2.imread("assets/face.jpg")[:, :, ::-1].copy() / 255 + rgb_image = torch.FloatTensor(rgb_image).unsqueeze(0).to(guidance.device) + prompt_utils = prompt_processor() + guidance_out = guidance(rgb_image, rgb_image, prompt_utils) + edit_image = ( + (guidance_out["edit_images"][0].detach().cpu().clip(0, 1).numpy() * 255) + .astype(np.uint8)[:, :, ::-1] + .copy() + ) + os.makedirs(".threestudio_cache", exist_ok=True) + cv2.imwrite(".threestudio_cache/edit_image.jpg", edit_image) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/deep_floyd_guidance.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/deep_floyd_guidance.py new file mode 100644 index 0000000000000000000000000000000000000000..fa6ecea5c68803de85e64d533dfe4167ce06520b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/deep_floyd_guidance.py @@ -0,0 +1,469 @@ +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers import IFPipeline +from diffusers.utils.import_utils import is_xformers_available +from tqdm import tqdm + +import threestudio +from threestudio.models.prompt_processors.base import PromptProcessorOutput +from threestudio.utils.base import BaseObject +from threestudio.utils.misc import C, parse_version +from threestudio.utils.ops import perpendicular_component +from threestudio.utils.typing import * + + +@threestudio.register("deep-floyd-guidance") +class DeepFloydGuidance(BaseObject): + @dataclass + class Config(BaseObject.Config): + pretrained_model_name_or_path: str = "DeepFloyd/IF-I-XL-v1.0" + # FIXME: xformers error + enable_memory_efficient_attention: bool = False + enable_sequential_cpu_offload: bool = False + enable_attention_slicing: bool = False + enable_channels_last_format: bool = True + guidance_scale: float = 20.0 + grad_clip: Optional[ + Any + ] = None # field(default_factory=lambda: [0, 2.0, 8.0, 1000]) + half_precision_weights: bool = True + + min_step_percent: float = 0.02 + max_step_percent: float = 0.98 + + weighting_strategy: str = "sds" + + view_dependent_prompting: bool = True + + """Maximum number of batch items to evaluate guidance for (for debugging) and to save on disk. -1 means save all items.""" + max_items_eval: int = 4 + + cfg: Config + + def configure(self) -> None: + threestudio.info(f"Loading Deep Floyd ...") + + self.weights_dtype = ( + torch.float16 if self.cfg.half_precision_weights else torch.float32 + ) + + # Create model + self.pipe = IFPipeline.from_pretrained( + self.cfg.pretrained_model_name_or_path, + text_encoder=None, + safety_checker=None, + watermarker=None, + feature_extractor=None, + requires_safety_checker=False, + variant="fp16" if self.cfg.half_precision_weights else None, + torch_dtype=self.weights_dtype, + ).to(self.device) + + if self.cfg.enable_memory_efficient_attention: + if parse_version(torch.__version__) >= parse_version("2"): + threestudio.info( + "PyTorch2.0 uses memory efficient attention by default." + ) + elif not is_xformers_available(): + threestudio.warn( + "xformers is not available, memory efficient attention is not enabled." + ) + else: + threestudio.warn( + f"Use DeepFloyd with xformers may raise error, see https://github.com/deep-floyd/IF/issues/52 to track this problem." + ) + self.pipe.enable_xformers_memory_efficient_attention() + + if self.cfg.enable_sequential_cpu_offload: + self.pipe.enable_sequential_cpu_offload() + + if self.cfg.enable_attention_slicing: + self.pipe.enable_attention_slicing(1) + + if self.cfg.enable_channels_last_format: + self.pipe.unet.to(memory_format=torch.channels_last) + + self.unet = self.pipe.unet.eval() + + for p in self.unet.parameters(): + p.requires_grad_(False) + + self.scheduler = self.pipe.scheduler + + self.num_train_timesteps = self.scheduler.config.num_train_timesteps + self.set_min_max_steps() # set to default value + + self.alphas: Float[Tensor, "..."] = self.scheduler.alphas_cumprod.to( + self.device + ) + + self.grad_clip_val: Optional[float] = None + + threestudio.info(f"Loaded Deep Floyd!") + + @torch.cuda.amp.autocast(enabled=False) + def set_min_max_steps(self, min_step_percent=0.02, max_step_percent=0.98): + self.min_step = int(self.num_train_timesteps * min_step_percent) + self.max_step = int(self.num_train_timesteps * max_step_percent) + + @torch.cuda.amp.autocast(enabled=False) + def forward_unet( + self, + latents: Float[Tensor, "..."], + t: Float[Tensor, "..."], + encoder_hidden_states: Float[Tensor, "..."], + ) -> Float[Tensor, "..."]: + input_dtype = latents.dtype + return self.unet( + latents.to(self.weights_dtype), + t.to(self.weights_dtype), + encoder_hidden_states=encoder_hidden_states.to(self.weights_dtype), + ).sample.to(input_dtype) + + def __call__( + self, + rgb: Float[Tensor, "B H W C"], + prompt_utils: PromptProcessorOutput, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + rgb_as_latents=False, + guidance_eval=False, + **kwargs, + ): + batch_size = rgb.shape[0] + + rgb_BCHW = rgb.permute(0, 3, 1, 2) + + assert rgb_as_latents == False, f"No latent space in {self.__class__.__name__}" + rgb_BCHW = rgb_BCHW * 2.0 - 1.0 # scale to [-1, 1] to match the diffusion range + latents = F.interpolate( + rgb_BCHW, (64, 64), mode="bilinear", align_corners=False + ) + + # timestep ~ U(0.02, 0.98) to avoid very high/low noise level + t = torch.randint( + self.min_step, + self.max_step + 1, + [batch_size], + dtype=torch.long, + device=self.device, + ) + + if prompt_utils.use_perp_neg: + ( + text_embeddings, + neg_guidance_weights, + ) = prompt_utils.get_text_embeddings_perp_neg( + elevation, azimuth, camera_distances, self.cfg.view_dependent_prompting + ) + with torch.no_grad(): + noise = torch.randn_like(latents) + latents_noisy = self.scheduler.add_noise(latents, noise, t) + latent_model_input = torch.cat([latents_noisy] * 4, dim=0) + noise_pred = self.forward_unet( + latent_model_input, + torch.cat([t] * 4), + encoder_hidden_states=text_embeddings, + ) # (4B, 6, 64, 64) + + noise_pred_text, _ = noise_pred[:batch_size].split(3, dim=1) + noise_pred_uncond, _ = noise_pred[batch_size : batch_size * 2].split( + 3, dim=1 + ) + noise_pred_neg, _ = noise_pred[batch_size * 2 :].split(3, dim=1) + + e_pos = noise_pred_text - noise_pred_uncond + accum_grad = 0 + n_negative_prompts = neg_guidance_weights.shape[-1] + for i in range(n_negative_prompts): + e_i_neg = noise_pred_neg[i::n_negative_prompts] - noise_pred_uncond + accum_grad += neg_guidance_weights[:, i].view( + -1, 1, 1, 1 + ) * perpendicular_component(e_i_neg, e_pos) + + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + e_pos + accum_grad + ) + else: + neg_guidance_weights = None + text_embeddings = prompt_utils.get_text_embeddings( + elevation, azimuth, camera_distances, self.cfg.view_dependent_prompting + ) + # predict the noise residual with unet, NO grad! + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) # TODO: use torch generator + latents_noisy = self.scheduler.add_noise(latents, noise, t) + # pred noise + latent_model_input = torch.cat([latents_noisy] * 2, dim=0) + noise_pred = self.forward_unet( + latent_model_input, + torch.cat([t] * 2), + encoder_hidden_states=text_embeddings, + ) # (2B, 6, 64, 64) + + # perform guidance (high scale from paper!) + noise_pred_text, noise_pred_uncond = noise_pred.chunk(2) + noise_pred_text, predicted_variance = noise_pred_text.split(3, dim=1) + noise_pred_uncond, _ = noise_pred_uncond.split(3, dim=1) + noise_pred = noise_pred_text + self.cfg.guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + + """ + # thresholding, experimental + if self.cfg.thresholding: + assert batch_size == 1 + noise_pred = torch.cat([noise_pred, predicted_variance], dim=1) + noise_pred = custom_ddpm_step(self.scheduler, + noise_pred, int(t.item()), latents_noisy, **self.pipe.prepare_extra_step_kwargs(None, 0.0) + ) + """ + + if self.cfg.weighting_strategy == "sds": + # w(t), sigma_t^2 + w = (1 - self.alphas[t]).view(-1, 1, 1, 1) + elif self.cfg.weighting_strategy == "uniform": + w = 1 + elif self.cfg.weighting_strategy == "fantasia3d": + w = (self.alphas[t] ** 0.5 * (1 - self.alphas[t])).view(-1, 1, 1, 1) + else: + raise ValueError( + f"Unknown weighting strategy: {self.cfg.weighting_strategy}" + ) + + grad = w * (noise_pred - noise) + grad = torch.nan_to_num(grad) + # clip grad for stable training? + if self.grad_clip_val is not None: + grad = grad.clamp(-self.grad_clip_val, self.grad_clip_val) + + # loss = SpecifyGradient.apply(latents, grad) + # SpecifyGradient is not straghtforward, use a reparameterization trick instead + target = (latents - grad).detach() + # d(loss)/d(latents) = latents - target = latents - (latents - grad) = grad + loss_sds = 0.5 * F.mse_loss(latents, target, reduction="sum") / batch_size + + guidance_out = { + "loss_sds": loss_sds, + "grad_norm": grad.norm(), + "min_step": self.min_step, + "max_step": self.max_step, + } + + if guidance_eval: + guidance_eval_utils = { + "use_perp_neg": prompt_utils.use_perp_neg, + "neg_guidance_weights": neg_guidance_weights, + "text_embeddings": text_embeddings, + "t_orig": t, + "latents_noisy": latents_noisy, + "noise_pred": torch.cat([noise_pred, predicted_variance], dim=1), + } + guidance_eval_out = self.guidance_eval(**guidance_eval_utils) + texts = [] + for n, e, a, c in zip( + guidance_eval_out["noise_levels"], elevation, azimuth, camera_distances + ): + texts.append( + f"n{n:.02f}\ne{e.item():.01f}\na{a.item():.01f}\nc{c.item():.02f}" + ) + guidance_eval_out.update({"texts": texts}) + guidance_out.update({"eval": guidance_eval_out}) + + return guidance_out + + @torch.cuda.amp.autocast(enabled=False) + @torch.no_grad() + def get_noise_pred( + self, + latents_noisy, + t, + text_embeddings, + use_perp_neg=False, + neg_guidance_weights=None, + ): + batch_size = latents_noisy.shape[0] + if use_perp_neg: + latent_model_input = torch.cat([latents_noisy] * 4, dim=0) + noise_pred = self.forward_unet( + latent_model_input, + torch.cat([t.reshape(1)] * 4).to(self.device), + encoder_hidden_states=text_embeddings, + ) # (4B, 6, 64, 64) + + noise_pred_text, _ = noise_pred[:batch_size].split(3, dim=1) + noise_pred_uncond, _ = noise_pred[batch_size : batch_size * 2].split( + 3, dim=1 + ) + noise_pred_neg, _ = noise_pred[batch_size * 2 :].split(3, dim=1) + + e_pos = noise_pred_text - noise_pred_uncond + accum_grad = 0 + n_negative_prompts = neg_guidance_weights.shape[-1] + for i in range(n_negative_prompts): + e_i_neg = noise_pred_neg[i::n_negative_prompts] - noise_pred_uncond + accum_grad += neg_guidance_weights[:, i].view( + -1, 1, 1, 1 + ) * perpendicular_component(e_i_neg, e_pos) + + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + e_pos + accum_grad + ) + else: + latent_model_input = torch.cat([latents_noisy] * 2, dim=0) + noise_pred = self.forward_unet( + latent_model_input, + torch.cat([t.reshape(1)] * 2).to(self.device), + encoder_hidden_states=text_embeddings, + ) # (2B, 6, 64, 64) + + # perform guidance (high scale from paper!) + noise_pred_text, noise_pred_uncond = noise_pred.chunk(2) + noise_pred_text, predicted_variance = noise_pred_text.split(3, dim=1) + noise_pred_uncond, _ = noise_pred_uncond.split(3, dim=1) + noise_pred = noise_pred_text + self.cfg.guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + + return torch.cat([noise_pred, predicted_variance], dim=1) + + @torch.cuda.amp.autocast(enabled=False) + @torch.no_grad() + def guidance_eval( + self, + t_orig, + text_embeddings, + latents_noisy, + noise_pred, + use_perp_neg=False, + neg_guidance_weights=None, + ): + # use only 50 timesteps, and find nearest of those to t + self.scheduler.set_timesteps(50) + self.scheduler.timesteps_gpu = self.scheduler.timesteps.to(self.device) + bs = ( + min(self.cfg.max_items_eval, latents_noisy.shape[0]) + if self.cfg.max_items_eval > 0 + else latents_noisy.shape[0] + ) # batch size + large_enough_idxs = self.scheduler.timesteps_gpu.expand([bs, -1]) > t_orig[ + :bs + ].unsqueeze( + -1 + ) # sized [bs,50] > [bs,1] + idxs = torch.min(large_enough_idxs, dim=1)[1] + t = self.scheduler.timesteps_gpu[idxs] + + fracs = list((t / self.scheduler.config.num_train_timesteps).cpu().numpy()) + imgs_noisy = (latents_noisy[:bs] / 2 + 0.5).permute(0, 2, 3, 1) + + # get prev latent + latents_1step = [] + pred_1orig = [] + for b in range(bs): + step_output = self.scheduler.step( + noise_pred[b : b + 1], t[b], latents_noisy[b : b + 1] + ) + latents_1step.append(step_output["prev_sample"]) + pred_1orig.append(step_output["pred_original_sample"]) + latents_1step = torch.cat(latents_1step) + pred_1orig = torch.cat(pred_1orig) + imgs_1step = (latents_1step / 2 + 0.5).permute(0, 2, 3, 1) + imgs_1orig = (pred_1orig / 2 + 0.5).permute(0, 2, 3, 1) + + latents_final = [] + for b, i in enumerate(idxs): + latents = latents_1step[b : b + 1] + text_emb = ( + text_embeddings[ + [b, b + len(idxs), b + 2 * len(idxs), b + 3 * len(idxs)], ... + ] + if use_perp_neg + else text_embeddings[[b, b + len(idxs)], ...] + ) + neg_guid = neg_guidance_weights[b : b + 1] if use_perp_neg else None + for t in tqdm(self.scheduler.timesteps[i + 1 :], leave=False): + # pred noise + noise_pred = self.get_noise_pred( + latents, t, text_emb, use_perp_neg, neg_guid + ) + # get prev latent + latents = self.scheduler.step(noise_pred, t, latents)["prev_sample"] + latents_final.append(latents) + + latents_final = torch.cat(latents_final) + imgs_final = (latents_final / 2 + 0.5).permute(0, 2, 3, 1) + + return { + "bs": bs, + "noise_levels": fracs, + "imgs_noisy": imgs_noisy, + "imgs_1step": imgs_1step, + "imgs_1orig": imgs_1orig, + "imgs_final": imgs_final, + } + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + # clip grad for stable training as demonstrated in + # Debiasing Scores and Prompts of 2D Diffusion for Robust Text-to-3D Generation + # http://arxiv.org/abs/2303.15413 + if self.cfg.grad_clip is not None: + self.grad_clip_val = C(self.cfg.grad_clip, epoch, global_step) + + self.set_min_max_steps( + min_step_percent=C(self.cfg.min_step_percent, epoch, global_step), + max_step_percent=C(self.cfg.max_step_percent, epoch, global_step), + ) + + +""" +# used by thresholding, experimental +def custom_ddpm_step(ddpm, model_output: torch.FloatTensor, timestep: int, sample: torch.FloatTensor, generator=None, return_dict: bool = True): + self = ddpm + t = timestep + + prev_t = self.previous_timestep(t) + + if model_output.shape[1] == sample.shape[1] * 2 and self.variance_type in ["learned", "learned_range"]: + model_output, predicted_variance = torch.split(model_output, sample.shape[1], dim=1) + else: + predicted_variance = None + + # 1. compute alphas, betas + alpha_prod_t = self.alphas_cumprod[t].item() + alpha_prod_t_prev = self.alphas_cumprod[prev_t].item() if prev_t >= 0 else 1.0 + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + current_alpha_t = alpha_prod_t / alpha_prod_t_prev + current_beta_t = 1 - current_alpha_t + + # 2. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or" + " `v_prediction` for the DDPMScheduler." + ) + + # 3. Clip or threshold "predicted x_0" + if self.config.thresholding: + pred_original_sample = self._threshold_sample(pred_original_sample) + elif self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + noise_thresholded = (sample - (alpha_prod_t ** 0.5) * pred_original_sample) / (beta_prod_t ** 0.5) + return noise_thresholded +""" diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/instructpix2pix_guidance.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/instructpix2pix_guidance.py new file mode 100644 index 0000000000000000000000000000000000000000..8fa19faa0d8a8d97cb62a761074610c9ea91530e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/instructpix2pix_guidance.py @@ -0,0 +1,347 @@ +from dataclasses import dataclass + +import cv2 +import numpy as np +import torch +import torch.nn.functional as F +from diffusers import DDIMScheduler, StableDiffusionInstructPix2PixPipeline +from diffusers.utils.import_utils import is_xformers_available +from tqdm import tqdm + +import threestudio +from threestudio.models.prompt_processors.base import PromptProcessorOutput +from threestudio.utils.base import BaseObject +from threestudio.utils.misc import C, parse_version +from threestudio.utils.typing import * + + +@threestudio.register("stable-diffusion-instructpix2pix-guidance") +class InstructPix2PixGuidance(BaseObject): + @dataclass + class Config(BaseObject.Config): + cache_dir: Optional[str] = None + ddim_scheduler_name_or_path: str = "CompVis/stable-diffusion-v1-4" + ip2p_name_or_path: str = "timbrooks/instruct-pix2pix" + + enable_memory_efficient_attention: bool = False + enable_sequential_cpu_offload: bool = False + enable_attention_slicing: bool = False + enable_channels_last_format: bool = False + guidance_scale: float = 7.5 + condition_scale: float = 1.5 + grad_clip: Optional[ + Any + ] = None # field(default_factory=lambda: [0, 2.0, 8.0, 1000]) + half_precision_weights: bool = True + + fixed_size: int = -1 + + min_step_percent: float = 0.02 + max_step_percent: float = 0.98 + + diffusion_steps: int = 20 + + use_sds: bool = False + + cfg: Config + + def configure(self) -> None: + threestudio.info(f"Loading InstructPix2Pix ...") + + self.weights_dtype = ( + torch.float16 if self.cfg.half_precision_weights else torch.float32 + ) + + pipe_kwargs = { + "safety_checker": None, + "feature_extractor": None, + "requires_safety_checker": False, + "torch_dtype": self.weights_dtype, + "cache_dir": self.cfg.cache_dir, + } + + self.pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained( + self.cfg.ip2p_name_or_path, **pipe_kwargs + ).to(self.device) + self.scheduler = DDIMScheduler.from_pretrained( + self.cfg.ddim_scheduler_name_or_path, + subfolder="scheduler", + torch_dtype=self.weights_dtype, + cache_dir=self.cfg.cache_dir, + ) + self.scheduler.set_timesteps(self.cfg.diffusion_steps) + + if self.cfg.enable_memory_efficient_attention: + if parse_version(torch.__version__) >= parse_version("2"): + threestudio.info( + "PyTorch2.0 uses memory efficient attention by default." + ) + elif not is_xformers_available(): + threestudio.warn( + "xformers is not available, memory efficient attention is not enabled." + ) + else: + self.pipe.enable_xformers_memory_efficient_attention() + + if self.cfg.enable_sequential_cpu_offload: + self.pipe.enable_sequential_cpu_offload() + + if self.cfg.enable_attention_slicing: + self.pipe.enable_attention_slicing(1) + + if self.cfg.enable_channels_last_format: + self.pipe.unet.to(memory_format=torch.channels_last) + + # Create model + self.vae = self.pipe.vae.eval() + self.unet = self.pipe.unet.eval() + + for p in self.vae.parameters(): + p.requires_grad_(False) + for p in self.unet.parameters(): + p.requires_grad_(False) + + self.num_train_timesteps = self.scheduler.config.num_train_timesteps + self.set_min_max_steps() # set to default value + + self.alphas: Float[Tensor, "..."] = self.scheduler.alphas_cumprod.to( + self.device + ) + + self.grad_clip_val: Optional[float] = None + + threestudio.info(f"Loaded InstructPix2Pix!") + + @torch.cuda.amp.autocast(enabled=False) + def set_min_max_steps(self, min_step_percent=0.02, max_step_percent=0.98): + self.min_step = int(self.num_train_timesteps * min_step_percent) + self.max_step = int(self.num_train_timesteps * max_step_percent) + + @torch.cuda.amp.autocast(enabled=False) + def forward_unet( + self, + latents: Float[Tensor, "..."], + t: Float[Tensor, "..."], + encoder_hidden_states: Float[Tensor, "..."], + ) -> Float[Tensor, "..."]: + input_dtype = latents.dtype + return self.unet( + latents.to(self.weights_dtype), + t.to(self.weights_dtype), + encoder_hidden_states=encoder_hidden_states.to(self.weights_dtype), + ).sample.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def encode_images( + self, imgs: Float[Tensor, "B 3 H W"] + ) -> Float[Tensor, "B 4 DH DW"]: + input_dtype = imgs.dtype + imgs = imgs * 2.0 - 1.0 + posterior = self.vae.encode(imgs.to(self.weights_dtype)).latent_dist + latents = posterior.sample() * self.vae.config.scaling_factor + return latents.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def encode_cond_images( + self, imgs: Float[Tensor, "B 3 H W"] + ) -> Float[Tensor, "B 4 DH DW"]: + input_dtype = imgs.dtype + imgs = imgs * 2.0 - 1.0 + posterior = self.vae.encode(imgs.to(self.weights_dtype)).latent_dist + latents = posterior.mode() + uncond_image_latents = torch.zeros_like(latents) + latents = torch.cat([latents, latents, uncond_image_latents], dim=0) + return latents.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def decode_latents( + self, latents: Float[Tensor, "B 4 DH DW"] + ) -> Float[Tensor, "B 3 H W"]: + input_dtype = latents.dtype + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents.to(self.weights_dtype)).sample + image = (image * 0.5 + 0.5).clamp(0, 1) + return image.to(input_dtype) + + def edit_latents( + self, + text_embeddings: Float[Tensor, "BB 77 768"], + latents: Float[Tensor, "B 4 DH DW"], + image_cond_latents: Float[Tensor, "B 4 DH DW"], + t: Int[Tensor, "B"], + ) -> Float[Tensor, "B 4 DH DW"]: + self.scheduler.config.num_train_timesteps = t.item() + self.scheduler.set_timesteps(self.cfg.diffusion_steps) + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) + latents = self.scheduler.add_noise(latents, noise, t) # type: ignore + threestudio.debug("Start editing...") + # sections of code used from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py + for i, t in enumerate(self.scheduler.timesteps): + # predict the noise residual with unet, NO grad! + with torch.no_grad(): + # pred noise + latent_model_input = torch.cat([latents] * 3) + latent_model_input = torch.cat( + [latent_model_input, image_cond_latents], dim=1 + ) + + noise_pred = self.forward_unet( + latent_model_input, t, encoder_hidden_states=text_embeddings + ) + + # perform classifier-free guidance + noise_pred_text, noise_pred_image, noise_pred_uncond = noise_pred.chunk( + 3 + ) + noise_pred = ( + noise_pred_uncond + + self.cfg.guidance_scale * (noise_pred_text - noise_pred_image) + + self.cfg.condition_scale * (noise_pred_image - noise_pred_uncond) + ) + + # get previous sample, continue loop + latents = self.scheduler.step(noise_pred, t, latents).prev_sample + threestudio.debug("Editing finished.") + return latents + + def compute_grad_sds( + self, + text_embeddings: Float[Tensor, "BB 77 768"], + latents: Float[Tensor, "B 4 DH DW"], + image_cond_latents: Float[Tensor, "B 4 DH DW"], + t: Int[Tensor, "B"], + ): + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) # TODO: use torch generator + latents_noisy = self.scheduler.add_noise(latents, noise, t) + # pred noise + latent_model_input = torch.cat([latents_noisy] * 3) + latent_model_input = torch.cat( + [latent_model_input, image_cond_latents], dim=1 + ) + + noise_pred = self.forward_unet( + latent_model_input, t, encoder_hidden_states=text_embeddings + ) + + noise_pred_text, noise_pred_image, noise_pred_uncond = noise_pred.chunk(3) + noise_pred = ( + noise_pred_uncond + + self.cfg.guidance_scale * (noise_pred_text - noise_pred_image) + + self.cfg.condition_scale * (noise_pred_image - noise_pred_uncond) + ) + + w = (1 - self.alphas[t]).view(-1, 1, 1, 1) + grad = w * (noise_pred - noise) + return grad + + def __call__( + self, + rgb: Float[Tensor, "B H W C"], + cond_rgb: Float[Tensor, "B H W C"], + prompt_utils: PromptProcessorOutput, + **kwargs, + ): + batch_size, H, W, _ = rgb.shape + + rgb_BCHW = rgb.permute(0, 3, 1, 2) + latents: Float[Tensor, "B 4 DH DW"] + if self.cfg.fixed_size > 0: + RH, RW = self.cfg.fixed_size, self.cfg.fixed_size + else: + RH, RW = H // 8 * 8, W // 8 * 8 + rgb_BCHW_HW8 = F.interpolate( + rgb_BCHW, (RH, RW), mode="bilinear", align_corners=False + ) + latents = self.encode_images(rgb_BCHW_HW8) + + cond_rgb_BCHW = cond_rgb.permute(0, 3, 1, 2) + cond_rgb_BCHW_HW8 = F.interpolate( + cond_rgb_BCHW, + (RH, RW), + mode="bilinear", + align_corners=False, + ) + cond_latents = self.encode_cond_images(cond_rgb_BCHW_HW8) + + temp = torch.zeros(1).to(rgb.device) + text_embeddings = prompt_utils.get_text_embeddings(temp, temp, temp, False) + text_embeddings = torch.cat( + [text_embeddings, text_embeddings[-1:]], dim=0 + ) # [positive, negative, negative] + + # timestep ~ U(0.02, 0.98) to avoid very high/low noise level + t = torch.randint( + self.min_step, + self.max_step + 1, + [batch_size], + dtype=torch.long, + device=self.device, + ) + + if self.cfg.use_sds: + grad = self.compute_grad_sds(text_embeddings, latents, cond_latents, t) + grad = torch.nan_to_num(grad) + if self.grad_clip_val is not None: + grad = grad.clamp(-self.grad_clip_val, self.grad_clip_val) + target = (latents - grad).detach() + loss_sds = 0.5 * F.mse_loss(latents, target, reduction="sum") / batch_size + return { + "loss_sds": loss_sds, + "grad_norm": grad.norm(), + "min_step": self.min_step, + "max_step": self.max_step, + } + else: + edit_latents = self.edit_latents(text_embeddings, latents, cond_latents, t) + edit_images = self.decode_latents(edit_latents) + edit_images = F.interpolate(edit_images, (H, W), mode="bilinear") + + return {"edit_images": edit_images.permute(0, 2, 3, 1)} + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + # clip grad for stable training as demonstrated in + # Debiasing Scores and Prompts of 2D Diffusion for Robust Text-to-3D Generation + # http://arxiv.org/abs/2303.15413 + if self.cfg.grad_clip is not None: + self.grad_clip_val = C(self.cfg.grad_clip, epoch, global_step) + + self.set_min_max_steps( + min_step_percent=C(self.cfg.min_step_percent, epoch, global_step), + max_step_percent=C(self.cfg.max_step_percent, epoch, global_step), + ) + + +if __name__ == "__main__": + from threestudio.utils.config import ExperimentConfig, load_config + from threestudio.utils.typing import Optional + + cfg = load_config("configs/debugging/instructpix2pix.yaml") + guidance = threestudio.find(cfg.system.guidance_type)(cfg.system.guidance) + prompt_processor = threestudio.find(cfg.system.prompt_processor_type)( + cfg.system.prompt_processor + ) + rgb_image = cv2.imread("assets/face.jpg")[:, :, ::-1].copy() / 255 + rgb_image = torch.FloatTensor(rgb_image).unsqueeze(0).to(guidance.device) + prompt_utils = prompt_processor() + guidance_out = guidance(rgb_image, rgb_image, prompt_utils) + edit_image = ( + ( + guidance_out["edit_images"][0] + .permute(1, 2, 0) + .detach() + .cpu() + .clip(0, 1) + .numpy() + * 255 + ) + .astype(np.uint8)[:, :, ::-1] + .copy() + ) + import os + + os.makedirs(".threestudio_cache", exist_ok=True) + cv2.imwrite(".threestudio_cache/edit_image.jpg", edit_image) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_diffusion_guidance.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_diffusion_guidance.py new file mode 100644 index 0000000000000000000000000000000000000000..7b6465dcda1ff125fd02eca4b099ef1458baf3d8 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_diffusion_guidance.py @@ -0,0 +1,647 @@ +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers import DDIMScheduler, DDPMScheduler, StableDiffusionPipeline +from diffusers.utils.import_utils import is_xformers_available +from tqdm import tqdm + +import threestudio +from threestudio.models.prompt_processors.base import PromptProcessorOutput +from threestudio.utils.base import BaseObject +from threestudio.utils.misc import C, cleanup, parse_version +from threestudio.utils.ops import perpendicular_component +from threestudio.utils.typing import * + + +@threestudio.register("stable-diffusion-guidance") +class StableDiffusionGuidance(BaseObject): + @dataclass + class Config(BaseObject.Config): + pretrained_model_name_or_path: str = "runwayml/stable-diffusion-v1-5" + enable_memory_efficient_attention: bool = False + enable_sequential_cpu_offload: bool = False + enable_attention_slicing: bool = False + enable_channels_last_format: bool = False + guidance_scale: float = 100.0 + grad_clip: Optional[ + Any + ] = None # field(default_factory=lambda: [0, 2.0, 8.0, 1000]) + half_precision_weights: bool = True + + min_step_percent: float = 0.02 + max_step_percent: float = 0.98 + sqrt_anneal: bool = False # sqrt anneal proposed in HiFA: https://hifa-team.github.io/HiFA-site/ + trainer_max_steps: int = 25000 + use_img_loss: bool = False # image-space SDS proposed in HiFA: https://hifa-team.github.io/HiFA-site/ + + use_sjc: bool = False + var_red: bool = True + weighting_strategy: str = "sds" + csd: bool = False + + token_merging: bool = False + token_merging_params: Optional[dict] = field(default_factory=dict) + + view_dependent_prompting: bool = True + + """Maximum number of batch items to evaluate guidance for (for debugging) and to save on disk. -1 means save all items.""" + max_items_eval: int = 4 + + cfg: Config + + def configure(self) -> None: + threestudio.info(f"Loading Stable Diffusion ...") + + self.weights_dtype = ( + torch.float16 if self.cfg.half_precision_weights else torch.float32 + ) + + pipe_kwargs = { + "tokenizer": None, + "safety_checker": None, + "feature_extractor": None, + "requires_safety_checker": False, + "torch_dtype": self.weights_dtype, + } + self.pipe = StableDiffusionPipeline.from_pretrained( + self.cfg.pretrained_model_name_or_path, + **pipe_kwargs, + ).to(self.device) + + if self.cfg.enable_memory_efficient_attention: + if parse_version(torch.__version__) >= parse_version("2"): + threestudio.info( + "PyTorch2.0 uses memory efficient attention by default." + ) + elif not is_xformers_available(): + threestudio.warn( + "xformers is not available, memory efficient attention is not enabled." + ) + else: + self.pipe.enable_xformers_memory_efficient_attention() + + if self.cfg.enable_sequential_cpu_offload: + self.pipe.enable_sequential_cpu_offload() + + if self.cfg.enable_attention_slicing: + self.pipe.enable_attention_slicing(1) + + if self.cfg.enable_channels_last_format: + self.pipe.unet.to(memory_format=torch.channels_last) + + del self.pipe.text_encoder + cleanup() + + # Create model + self.vae = self.pipe.vae.eval() + self.unet = self.pipe.unet.eval() + + for p in self.vae.parameters(): + p.requires_grad_(False) + for p in self.unet.parameters(): + p.requires_grad_(False) + + if self.cfg.token_merging: + import tomesd + + tomesd.apply_patch(self.unet, **self.cfg.token_merging_params) + + if self.cfg.use_sjc: + # score jacobian chaining use DDPM + self.scheduler = DDPMScheduler.from_pretrained( + self.cfg.pretrained_model_name_or_path, + subfolder="scheduler", + torch_dtype=self.weights_dtype, + beta_start=0.00085, + beta_end=0.0120, + beta_schedule="scaled_linear", + ) + else: + self.scheduler = DDIMScheduler.from_pretrained( + self.cfg.pretrained_model_name_or_path, + subfolder="scheduler", + torch_dtype=self.weights_dtype, + ) + + self.num_train_timesteps = self.scheduler.config.num_train_timesteps + self.set_min_max_steps() # set to default value + + self.alphas: Float[Tensor, "..."] = self.scheduler.alphas_cumprod.to( + self.device + ) + if self.cfg.use_sjc: + # score jacobian chaining need mu + self.us: Float[Tensor, "..."] = torch.sqrt((1 - self.alphas) / self.alphas) + + self.grad_clip_val: Optional[float] = None + + threestudio.info(f"Loaded Stable Diffusion!") + + @torch.cuda.amp.autocast(enabled=False) + def set_min_max_steps(self, min_step_percent=0.02, max_step_percent=0.98): + self.min_step = int(self.num_train_timesteps * min_step_percent) + self.max_step = int(self.num_train_timesteps * max_step_percent) + + @torch.cuda.amp.autocast(enabled=False) + def forward_unet( + self, + latents: Float[Tensor, "..."], + t: Float[Tensor, "..."], + encoder_hidden_states: Float[Tensor, "..."], + ) -> Float[Tensor, "..."]: + input_dtype = latents.dtype + return self.unet( + latents.to(self.weights_dtype), + t.to(self.weights_dtype), + encoder_hidden_states=encoder_hidden_states.to(self.weights_dtype), + ).sample.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def encode_images( + self, imgs: Float[Tensor, "B 3 512 512"] + ) -> Float[Tensor, "B 4 64 64"]: + input_dtype = imgs.dtype + imgs = imgs * 2.0 - 1.0 + posterior = self.vae.encode(imgs.to(self.weights_dtype)).latent_dist + latents = posterior.sample() * self.vae.config.scaling_factor + return latents.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def decode_latents( + self, + latents: Float[Tensor, "B 4 H W"], + latent_height: int = 64, + latent_width: int = 64, + ) -> Float[Tensor, "B 3 512 512"]: + input_dtype = latents.dtype + latents = F.interpolate( + latents, (latent_height, latent_width), mode="bilinear", align_corners=False + ) + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents.to(self.weights_dtype)).sample + image = (image * 0.5 + 0.5).clamp(0, 1) + return image.to(input_dtype) + + def compute_grad_sds( + self, + latents: Float[Tensor, "B 4 64 64"], + image: Float[Tensor, "B 3 512 512"], + t: Int[Tensor, "B"], + prompt_utils: PromptProcessorOutput, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + ): + batch_size = elevation.shape[0] + + if prompt_utils.use_perp_neg: + ( + text_embeddings, + neg_guidance_weights, + ) = prompt_utils.get_text_embeddings_perp_neg( + elevation, azimuth, camera_distances, self.cfg.view_dependent_prompting + ) + with torch.no_grad(): + noise = torch.randn_like(latents) + latents_noisy = self.scheduler.add_noise(latents, noise, t) + latent_model_input = torch.cat([latents_noisy] * 4, dim=0) + noise_pred = self.forward_unet( + latent_model_input, + torch.cat([t] * 4), + encoder_hidden_states=text_embeddings, + ) # (4B, 3, 64, 64) + + noise_pred_text = noise_pred[:batch_size] + noise_pred_uncond = noise_pred[batch_size : batch_size * 2] + noise_pred_neg = noise_pred[batch_size * 2 :] + + e_pos = noise_pred_text - noise_pred_uncond + accum_grad = 0 + n_negative_prompts = neg_guidance_weights.shape[-1] + for i in range(n_negative_prompts): + e_i_neg = noise_pred_neg[i::n_negative_prompts] - noise_pred_uncond + accum_grad += neg_guidance_weights[:, i].view( + -1, 1, 1, 1 + ) * perpendicular_component(e_i_neg, e_pos) + + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + e_pos + accum_grad + ) + else: + neg_guidance_weights = None + text_embeddings = prompt_utils.get_text_embeddings( + elevation, azimuth, camera_distances, self.cfg.view_dependent_prompting + ) + # predict the noise residual with unet, NO grad! + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) # TODO: use torch generator + latents_noisy = self.scheduler.add_noise(latents, noise, t) + # pred noise + latent_model_input = torch.cat([latents_noisy] * 2, dim=0) + noise_pred = self.forward_unet( + latent_model_input, + torch.cat([t] * 2), + encoder_hidden_states=text_embeddings, + ) + + # perform guidance (high scale from paper!) + noise_pred_text, noise_pred_uncond = noise_pred.chunk(2) + noise_pred = noise_pred_text + self.cfg.guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + + if self.cfg.weighting_strategy == "sds": + # w(t), sigma_t^2 + w = (1 - self.alphas[t]).view(-1, 1, 1, 1) + elif self.cfg.weighting_strategy == "uniform": + w = 1 + elif self.cfg.weighting_strategy == "fantasia3d": + w = (self.alphas[t] ** 0.5 * (1 - self.alphas[t])).view(-1, 1, 1, 1) + else: + raise ValueError( + f"Unknown weighting strategy: {self.cfg.weighting_strategy}" + ) + + alpha = (self.alphas[t] ** 0.5).view(-1, 1, 1, 1) + sigma = ((1 - self.alphas[t]) ** 0.5).view(-1, 1, 1, 1) + latents_denoised = (latents_noisy - sigma * noise_pred) / alpha + image_denoised = self.decode_latents(latents_denoised) + + if self.cfg.csd: + grad = w * (noise_pred_text - noise_pred_uncond) + else: + grad = w * (noise_pred - noise) + # image-space SDS proposed in HiFA: https://hifa-team.github.io/HiFA-site/ + if self.cfg.use_img_loss: + grad_img = w * (image - image_denoised) * alpha / sigma + else: + grad_img = None + + guidance_eval_utils = { + "use_perp_neg": prompt_utils.use_perp_neg, + "neg_guidance_weights": neg_guidance_weights, + "text_embeddings": text_embeddings, + "t_orig": t, + "latents_noisy": latents_noisy, + "noise_pred": noise_pred, + } + + return grad, grad_img, guidance_eval_utils + + def compute_grad_sjc( + self, + latents: Float[Tensor, "B 4 64 64"], + t: Int[Tensor, "B"], + prompt_utils: PromptProcessorOutput, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + ): + batch_size = elevation.shape[0] + + sigma = self.us[t] + sigma = sigma.view(-1, 1, 1, 1) + + if prompt_utils.use_perp_neg: + ( + text_embeddings, + neg_guidance_weights, + ) = prompt_utils.get_text_embeddings_perp_neg( + elevation, azimuth, camera_distances, self.cfg.view_dependent_prompting + ) + with torch.no_grad(): + noise = torch.randn_like(latents) + y = latents + zs = y + sigma * noise + scaled_zs = zs / torch.sqrt(1 + sigma**2) + # pred noise + latent_model_input = torch.cat([scaled_zs] * 4, dim=0) + noise_pred = self.forward_unet( + latent_model_input, + torch.cat([t] * 4), + encoder_hidden_states=text_embeddings, + ) # (4B, 3, 64, 64) + + noise_pred_text = noise_pred[:batch_size] + noise_pred_uncond = noise_pred[batch_size : batch_size * 2] + noise_pred_neg = noise_pred[batch_size * 2 :] + + e_pos = noise_pred_text - noise_pred_uncond + accum_grad = 0 + n_negative_prompts = neg_guidance_weights.shape[-1] + for i in range(n_negative_prompts): + e_i_neg = noise_pred_neg[i::n_negative_prompts] - noise_pred_uncond + accum_grad += neg_guidance_weights[:, i].view( + -1, 1, 1, 1 + ) * perpendicular_component(e_i_neg, e_pos) + + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + e_pos + accum_grad + ) + else: + neg_guidance_weights = None + text_embeddings = prompt_utils.get_text_embeddings( + elevation, azimuth, camera_distances, self.cfg.view_dependent_prompting + ) + # predict the noise residual with unet, NO grad! + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) # TODO: use torch generator + y = latents + + zs = y + sigma * noise + scaled_zs = zs / torch.sqrt(1 + sigma**2) + + # pred noise + latent_model_input = torch.cat([scaled_zs] * 2, dim=0) + noise_pred = self.forward_unet( + latent_model_input, + torch.cat([t] * 2), + encoder_hidden_states=text_embeddings, + ) + + # perform guidance (high scale from paper!) + noise_pred_text, noise_pred_uncond = noise_pred.chunk(2) + noise_pred = noise_pred_text + self.cfg.guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + + Ds = zs - sigma * noise_pred + + if self.cfg.var_red: + grad = -(Ds - y) / sigma + else: + grad = -(Ds - zs) / sigma + + guidance_eval_utils = { + "use_perp_neg": prompt_utils.use_perp_neg, + "neg_guidance_weights": neg_guidance_weights, + "text_embeddings": text_embeddings, + "t_orig": t, + "latents_noisy": scaled_zs, + "noise_pred": noise_pred, + } + + return grad, guidance_eval_utils + + def __call__( + self, + rgb: Float[Tensor, "B H W C"], + prompt_utils: PromptProcessorOutput, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + rgb_as_latents=False, + guidance_eval=False, + **kwargs, + ): + batch_size = rgb.shape[0] + + rgb_BCHW = rgb.permute(0, 3, 1, 2) + latents: Float[Tensor, "B 4 64 64"] + rgb_BCHW_512 = F.interpolate( + rgb_BCHW, (512, 512), mode="bilinear", align_corners=False + ) + if rgb_as_latents: + latents = F.interpolate( + rgb_BCHW, (64, 64), mode="bilinear", align_corners=False + ) + else: + # encode image into latents with vae + latents = self.encode_images(rgb_BCHW_512) + + # timestep ~ U(0.02, 0.98) to avoid very high/low noise level + t = torch.randint( + self.min_step, + self.max_step + 1, + [batch_size], + dtype=torch.long, + device=self.device, + ) + + if self.cfg.use_sjc: + grad, guidance_eval_utils = self.compute_grad_sjc( + latents, t, prompt_utils, elevation, azimuth, camera_distances + ) + grad_img = torch.tensor([0.0], dtype=grad.dtype).to(grad.device) + else: + grad, grad_img, guidance_eval_utils = self.compute_grad_sds( + latents, + rgb_BCHW_512, + t, + prompt_utils, + elevation, + azimuth, + camera_distances, + ) + + grad = torch.nan_to_num(grad) + + # clip grad for stable training? + if self.grad_clip_val is not None: + grad = grad.clamp(-self.grad_clip_val, self.grad_clip_val) + + # loss = SpecifyGradient.apply(latents, grad) + # SpecifyGradient is not straghtforward, use a reparameterization trick instead + target = (latents - grad).detach() + # d(loss)/d(latents) = latents - target = latents - (latents - grad) = grad + + if 'mask' in kwargs: + mask = kwargs["mask"][:, None] + mask = torch.nn.functional.interpolate(mask.float(), size=64, align_corners=True, mode='bilinear') + loss_sds = 0.5 * ((latents-target)**2 * mask).sum() / batch_size + else: + loss_sds = 0.5 * F.mse_loss(latents, target, reduction="sum") / batch_size + + guidance_out = { + "loss_sds": loss_sds, + "grad_norm": grad.norm(), + "min_step": self.min_step, + "max_step": self.max_step, + } + + if self.cfg.use_img_loss: + grad_img = torch.nan_to_num(grad_img) + if self.grad_clip_val is not None: + grad_img = grad_img.clamp(-self.grad_clip_val, self.grad_clip_val) + target_img = (rgb_BCHW_512 - grad_img).detach() + loss_sds_img = ( + 0.5 * F.mse_loss(rgb_BCHW_512, target_img, reduction="sum") / batch_size + ) + guidance_out["loss_sds_img"] = loss_sds_img + + if guidance_eval: + guidance_eval_out = self.guidance_eval(**guidance_eval_utils) + texts = [] + for n, e, a, c in zip( + guidance_eval_out["noise_levels"], elevation, azimuth, camera_distances + ): + texts.append( + f"n{n:.02f}\ne{e.item():.01f}\na{a.item():.01f}\nc{c.item():.02f}" + ) + guidance_eval_out.update({"texts": texts}) + guidance_out.update({"eval": guidance_eval_out}) + + return guidance_out + + @torch.cuda.amp.autocast(enabled=False) + @torch.no_grad() + def get_noise_pred( + self, + latents_noisy, + t, + text_embeddings, + use_perp_neg=False, + neg_guidance_weights=None, + ): + batch_size = latents_noisy.shape[0] + + if use_perp_neg: + # pred noise + latent_model_input = torch.cat([latents_noisy] * 4, dim=0) + noise_pred = self.forward_unet( + latent_model_input, + torch.cat([t.reshape(1)] * 4).to(self.device), + encoder_hidden_states=text_embeddings, + ) # (4B, 3, 64, 64) + + noise_pred_text = noise_pred[:batch_size] + noise_pred_uncond = noise_pred[batch_size : batch_size * 2] + noise_pred_neg = noise_pred[batch_size * 2 :] + + e_pos = noise_pred_text - noise_pred_uncond + accum_grad = 0 + n_negative_prompts = neg_guidance_weights.shape[-1] + for i in range(n_negative_prompts): + e_i_neg = noise_pred_neg[i::n_negative_prompts] - noise_pred_uncond + accum_grad += neg_guidance_weights[:, i].view( + -1, 1, 1, 1 + ) * perpendicular_component(e_i_neg, e_pos) + + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + e_pos + accum_grad + ) + else: + # pred noise + latent_model_input = torch.cat([latents_noisy] * 2, dim=0) + noise_pred = self.forward_unet( + latent_model_input, + torch.cat([t.reshape(1)] * 2).to(self.device), + encoder_hidden_states=text_embeddings, + ) + # perform guidance (high scale from paper!) + noise_pred_text, noise_pred_uncond = noise_pred.chunk(2) + noise_pred = noise_pred_text + self.cfg.guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + + return noise_pred + + @torch.cuda.amp.autocast(enabled=False) + @torch.no_grad() + def guidance_eval( + self, + t_orig, + text_embeddings, + latents_noisy, + noise_pred, + use_perp_neg=False, + neg_guidance_weights=None, + ): + # use only 50 timesteps, and find nearest of those to t + self.scheduler.set_timesteps(50) + self.scheduler.timesteps_gpu = self.scheduler.timesteps.to(self.device) + bs = ( + min(self.cfg.max_items_eval, latents_noisy.shape[0]) + if self.cfg.max_items_eval > 0 + else latents_noisy.shape[0] + ) # batch size + large_enough_idxs = self.scheduler.timesteps_gpu.expand([bs, -1]) > t_orig[ + :bs + ].unsqueeze( + -1 + ) # sized [bs,50] > [bs,1] + idxs = torch.min(large_enough_idxs, dim=1)[1] + t = self.scheduler.timesteps_gpu[idxs] + + fracs = list((t / self.scheduler.config.num_train_timesteps).cpu().numpy()) + imgs_noisy = self.decode_latents(latents_noisy[:bs]).permute(0, 2, 3, 1) + + # get prev latent + latents_1step = [] + pred_1orig = [] + for b in range(bs): + step_output = self.scheduler.step( + noise_pred[b : b + 1], t[b], latents_noisy[b : b + 1], eta=1 + ) + latents_1step.append(step_output["prev_sample"]) + pred_1orig.append(step_output["pred_original_sample"]) + latents_1step = torch.cat(latents_1step) + pred_1orig = torch.cat(pred_1orig) + imgs_1step = self.decode_latents(latents_1step).permute(0, 2, 3, 1) + imgs_1orig = self.decode_latents(pred_1orig).permute(0, 2, 3, 1) + + latents_final = [] + for b, i in enumerate(idxs): + latents = latents_1step[b : b + 1] + text_emb = ( + text_embeddings[ + [b, b + len(idxs), b + 2 * len(idxs), b + 3 * len(idxs)], ... + ] + if use_perp_neg + else text_embeddings[[b, b + len(idxs)], ...] + ) + neg_guid = neg_guidance_weights[b : b + 1] if use_perp_neg else None + for t in tqdm(self.scheduler.timesteps[i + 1 :], leave=False): + # pred noise + noise_pred = self.get_noise_pred( + latents, t, text_emb, use_perp_neg, neg_guid + ) + # get prev latent + latents = self.scheduler.step(noise_pred, t, latents, eta=1)[ + "prev_sample" + ] + latents_final.append(latents) + + latents_final = torch.cat(latents_final) + imgs_final = self.decode_latents(latents_final).permute(0, 2, 3, 1) + + return { + "bs": bs, + "noise_levels": fracs, + "imgs_noisy": imgs_noisy, + "imgs_1step": imgs_1step, + "imgs_1orig": imgs_1orig, + "imgs_final": imgs_final, + } + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + # clip grad for stable training as demonstrated in + # Debiasing Scores and Prompts of 2D Diffusion for Robust Text-to-3D Generation + # http://arxiv.org/abs/2303.15413 + if self.cfg.grad_clip is not None: + self.grad_clip_val = C(self.cfg.grad_clip, epoch, global_step) + + if self.cfg.sqrt_anneal: + percentage = ( + float(global_step) / self.cfg.trainer_max_steps + ) ** 0.5 # progress percentage + if type(self.cfg.max_step_percent) not in [float, int]: + max_step_percent = self.cfg.max_step_percent[1] + else: + max_step_percent = self.cfg.max_step_percent + curr_percent = ( + max_step_percent - C(self.cfg.min_step_percent, epoch, global_step) + ) * (1 - percentage) + C(self.cfg.min_step_percent, epoch, global_step) + self.set_min_max_steps( + min_step_percent=curr_percent, + max_step_percent=curr_percent, + ) + else: + self.set_min_max_steps( + min_step_percent=C(self.cfg.min_step_percent, epoch, global_step), + max_step_percent=C(self.cfg.max_step_percent, epoch, global_step), + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_diffusion_unified_guidance.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_diffusion_unified_guidance.py new file mode 100644 index 0000000000000000000000000000000000000000..5774b1d421b3e637f5c3aa89e615126eb0d1cf53 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_diffusion_unified_guidance.py @@ -0,0 +1,779 @@ +import random +from contextlib import contextmanager +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers import ( + AutoencoderKL, + ControlNetModel, + DDPMScheduler, + DPMSolverSinglestepScheduler, + StableDiffusionPipeline, + UNet2DConditionModel, +) +from diffusers.loaders import AttnProcsLayers +from diffusers.models.attention_processor import LoRAAttnProcessor +from diffusers.models.embeddings import TimestepEmbedding +from diffusers.utils.import_utils import is_xformers_available +from tqdm import tqdm + +import threestudio +from threestudio.models.networks import ToDTypeWrapper +from threestudio.models.prompt_processors.base import PromptProcessorOutput +from threestudio.utils.base import BaseModule +from threestudio.utils.misc import C, cleanup, enable_gradient, parse_version +from threestudio.utils.ops import perpendicular_component +from threestudio.utils.typing import * + + +@threestudio.register("stable-diffusion-unified-guidance") +class StableDiffusionUnifiedGuidance(BaseModule): + @dataclass + class Config(BaseModule.Config): + # guidance type, in ["sds", "vsd"] + guidance_type: str = "sds" + + pretrained_model_name_or_path: str = "runwayml/stable-diffusion-v1-5" + guidance_scale: float = 100.0 + weighting_strategy: str = "dreamfusion" + view_dependent_prompting: bool = True + + min_step_percent: Any = 0.02 + max_step_percent: Any = 0.98 + grad_clip: Optional[Any] = None + + return_rgb_1step_orig: bool = False + return_rgb_multistep_orig: bool = False + n_rgb_multistep_orig_steps: int = 4 + + # TODO + # controlnet + controlnet_model_name_or_path: Optional[str] = None + preprocessor: Optional[str] = None + control_scale: float = 1.0 + + # TODO + # lora + lora_model_name_or_path: Optional[str] = None + + # efficiency-related configurations + half_precision_weights: bool = True + enable_memory_efficient_attention: bool = False + enable_sequential_cpu_offload: bool = False + enable_attention_slicing: bool = False + enable_channels_last_format: bool = False + token_merging: bool = False + token_merging_params: Optional[dict] = field(default_factory=dict) + + # VSD configurations, only used when guidance_type is "vsd" + vsd_phi_model_name_or_path: Optional[str] = None + vsd_guidance_scale_phi: float = 1.0 + vsd_use_lora: bool = True + vsd_lora_cfg_training: bool = False + vsd_lora_n_timestamp_samples: int = 1 + vsd_use_camera_condition: bool = True + # camera condition type, in ["extrinsics", "mvp", "spherical"] + vsd_camera_condition_type: Optional[str] = "extrinsics" + + # HiFA configurations: https://hifa-team.github.io/HiFA-site/ + sqrt_anneal: bool = ( + False # requires setting min_step_percent=0.3 to work properly + ) + trainer_max_steps: int = 25000 + use_img_loss: bool = True # works with most cases + + cfg: Config + + def configure(self) -> None: + self.min_step: Optional[int] = None + self.max_step: Optional[int] = None + self.grad_clip_val: Optional[float] = None + + @dataclass + class NonTrainableModules: + pipe: StableDiffusionPipeline + pipe_phi: Optional[StableDiffusionPipeline] = None + controlnet: Optional[ControlNetModel] = None + + self.weights_dtype = ( + torch.float16 if self.cfg.half_precision_weights else torch.float32 + ) + + threestudio.info(f"Loading Stable Diffusion ...") + + pipe_kwargs = { + "tokenizer": None, + "safety_checker": None, + "feature_extractor": None, + "requires_safety_checker": False, + "torch_dtype": self.weights_dtype, + } + pipe = StableDiffusionPipeline.from_pretrained( + self.cfg.pretrained_model_name_or_path, + **pipe_kwargs, + ).to(self.device) + self.prepare_pipe(pipe) + self.configure_pipe_token_merging(pipe) + + # phi network for VSD + # introduce two trainable modules: + # - self.camera_embedding + # - self.lora_layers + pipe_phi = None + + # if the phi network shares the same unet with the pretrain network + # we need to pass additional cross attention kwargs to the unet + self.vsd_share_model = ( + self.cfg.guidance_type == "vsd" + and self.cfg.vsd_phi_model_name_or_path is None + ) + if self.cfg.guidance_type == "vsd": + if self.cfg.vsd_phi_model_name_or_path is None: + pipe_phi = pipe + else: + pipe_phi = StableDiffusionPipeline.from_pretrained( + self.cfg.vsd_phi_model_name_or_path, + **pipe_kwargs, + ).to(self.device) + self.prepare_pipe(pipe_phi) + self.configure_pipe_token_merging(pipe_phi) + + # set up camera embedding + if self.cfg.vsd_use_camera_condition: + if self.cfg.vsd_camera_condition_type in ["extrinsics", "mvp"]: + self.camera_embedding_dim = 16 + elif self.cfg.vsd_camera_condition_type == "spherical": + self.camera_embedding_dim = 4 + else: + raise ValueError("Invalid camera condition type!") + + # FIXME: hard-coded output dim + self.camera_embedding = ToDTypeWrapper( + TimestepEmbedding(self.camera_embedding_dim, 1280), + self.weights_dtype, + ).to(self.device) + pipe_phi.unet.class_embedding = self.camera_embedding + + if self.cfg.vsd_use_lora: + # set up LoRA layers + lora_attn_procs = {} + for name in pipe_phi.unet.attn_processors.keys(): + cross_attention_dim = ( + None + if name.endswith("attn1.processor") + else pipe_phi.unet.config.cross_attention_dim + ) + if name.startswith("mid_block"): + hidden_size = pipe_phi.unet.config.block_out_channels[-1] + elif name.startswith("up_blocks"): + block_id = int(name[len("up_blocks.")]) + hidden_size = list( + reversed(pipe_phi.unet.config.block_out_channels) + )[block_id] + elif name.startswith("down_blocks"): + block_id = int(name[len("down_blocks.")]) + hidden_size = pipe_phi.unet.config.block_out_channels[block_id] + + lora_attn_procs[name] = LoRAAttnProcessor( + hidden_size=hidden_size, cross_attention_dim=cross_attention_dim + ) + + pipe_phi.unet.set_attn_processor(lora_attn_procs) + + self.lora_layers = AttnProcsLayers(pipe_phi.unet.attn_processors).to( + self.device + ) + self.lora_layers._load_state_dict_pre_hooks.clear() + self.lora_layers._state_dict_hooks.clear() + + threestudio.info(f"Loaded Stable Diffusion!") + + # controlnet + controlnet = None + if self.cfg.controlnet_model_name_or_path is not None: + threestudio.info(f"Loading ControlNet ...") + + controlnet = ControlNetModel.from_pretrained( + self.cfg.controlnet_model_name_or_path, + torch_dtype=self.weights_dtype, + ).to(self.device) + controlnet.eval() + enable_gradient(controlnet, enabled=False) + + threestudio.info(f"Loaded ControlNet!") + + self.scheduler = DDPMScheduler.from_config(pipe.scheduler.config) + self.num_train_timesteps = self.scheduler.config.num_train_timesteps + + # q(z_t|x) = N(alpha_t x, sigma_t^2 I) + # in DDPM, alpha_t = sqrt(alphas_cumprod_t), sigma_t^2 = 1 - alphas_cumprod_t + self.alphas_cumprod: Float[Tensor, "T"] = self.scheduler.alphas_cumprod.to( + self.device + ) + self.alphas: Float[Tensor, "T"] = self.alphas_cumprod**0.5 + self.sigmas: Float[Tensor, "T"] = (1 - self.alphas_cumprod) ** 0.5 + # log SNR + self.lambdas: Float[Tensor, "T"] = self.sigmas / self.alphas + + self._non_trainable_modules = NonTrainableModules( + pipe=pipe, + pipe_phi=pipe_phi, + controlnet=controlnet, + ) + + @property + def pipe(self) -> StableDiffusionPipeline: + return self._non_trainable_modules.pipe + + @property + def pipe_phi(self) -> StableDiffusionPipeline: + if self._non_trainable_modules.pipe_phi is None: + raise RuntimeError("phi model is not available.") + return self._non_trainable_modules.pipe_phi + + @property + def controlnet(self) -> ControlNetModel: + if self._non_trainable_modules.controlnet is None: + raise RuntimeError("ControlNet model is not available.") + return self._non_trainable_modules.controlnet + + def prepare_pipe(self, pipe: StableDiffusionPipeline): + if self.cfg.enable_memory_efficient_attention: + if parse_version(torch.__version__) >= parse_version("2"): + threestudio.info( + "PyTorch2.0 uses memory efficient attention by default." + ) + elif not is_xformers_available(): + threestudio.warn( + "xformers is not available, memory efficient attention is not enabled." + ) + else: + pipe.enable_xformers_memory_efficient_attention() + + if self.cfg.enable_sequential_cpu_offload: + pipe.enable_sequential_cpu_offload() + + if self.cfg.enable_attention_slicing: + pipe.enable_attention_slicing(1) + + if self.cfg.enable_channels_last_format: + pipe.unet.to(memory_format=torch.channels_last) + + # FIXME: pipe.__call__ requires text_encoder.dtype + # pipe.text_encoder.to("meta") + cleanup() + + pipe.vae.eval() + pipe.unet.eval() + + enable_gradient(pipe.vae, enabled=False) + enable_gradient(pipe.unet, enabled=False) + + # disable progress bar + pipe.set_progress_bar_config(disable=True) + + def configure_pipe_token_merging(self, pipe: StableDiffusionPipeline): + if self.cfg.token_merging: + import tomesd + + tomesd.apply_patch(pipe.unet, **self.cfg.token_merging_params) + + @torch.cuda.amp.autocast(enabled=False) + def forward_unet( + self, + unet: UNet2DConditionModel, + latents: Float[Tensor, "..."], + t: Int[Tensor, "..."], + encoder_hidden_states: Float[Tensor, "..."], + class_labels: Optional[Float[Tensor, "..."]] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + down_block_additional_residuals: Optional[Float[Tensor, "..."]] = None, + mid_block_additional_residual: Optional[Float[Tensor, "..."]] = None, + velocity_to_epsilon: bool = False, + ) -> Float[Tensor, "..."]: + input_dtype = latents.dtype + pred = unet( + latents.to(unet.dtype), + t.to(unet.dtype), + encoder_hidden_states=encoder_hidden_states.to(unet.dtype), + class_labels=class_labels, + cross_attention_kwargs=cross_attention_kwargs, + down_block_additional_residuals=down_block_additional_residuals, + mid_block_additional_residual=mid_block_additional_residual, + ).sample + if velocity_to_epsilon: + pred = latents * self.sigmas[t].view(-1, 1, 1, 1) + pred * self.alphas[ + t + ].view(-1, 1, 1, 1) + return pred.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def vae_encode( + self, vae: AutoencoderKL, imgs: Float[Tensor, "B 3 H W"], mode=False + ) -> Float[Tensor, "B 4 Hl Wl"]: + # expect input in [-1, 1] + input_dtype = imgs.dtype + posterior = vae.encode(imgs.to(vae.dtype)).latent_dist + if mode: + latents = posterior.mode() + else: + latents = posterior.sample() + latents = latents * vae.config.scaling_factor + return latents.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def vae_decode( + self, vae: AutoencoderKL, latents: Float[Tensor, "B 4 Hl Wl"] + ) -> Float[Tensor, "B 3 H W"]: + # output in [0, 1] + input_dtype = latents.dtype + latents = 1 / vae.config.scaling_factor * latents + image = vae.decode(latents.to(vae.dtype)).sample + image = (image * 0.5 + 0.5).clamp(0, 1) + return image.to(input_dtype) + + @contextmanager + def disable_unet_class_embedding(self, unet: UNet2DConditionModel): + class_embedding = unet.class_embedding + try: + unet.class_embedding = None + yield unet + finally: + unet.class_embedding = class_embedding + + @contextmanager + def set_scheduler( + self, pipe: StableDiffusionPipeline, scheduler_class: Any, **kwargs + ): + scheduler_orig = pipe.scheduler + pipe.scheduler = scheduler_class.from_config(scheduler_orig.config, **kwargs) + yield pipe + pipe.scheduler = scheduler_orig + + def get_eps_pretrain( + self, + latents_noisy: Float[Tensor, "B 4 Hl Wl"], + t: Int[Tensor, "B"], + prompt_utils: PromptProcessorOutput, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + ) -> Float[Tensor, "B 4 Hl Wl"]: + batch_size = latents_noisy.shape[0] + + if prompt_utils.use_perp_neg: + ( + text_embeddings, + neg_guidance_weights, + ) = prompt_utils.get_text_embeddings_perp_neg( + elevation, azimuth, camera_distances, self.cfg.view_dependent_prompting + ) + with torch.no_grad(): + with self.disable_unet_class_embedding(self.pipe.unet) as unet: + noise_pred = self.forward_unet( + unet, + torch.cat([latents_noisy] * 4, dim=0), + torch.cat([t] * 4, dim=0), + encoder_hidden_states=text_embeddings, + cross_attention_kwargs={"scale": 0.0} + if self.vsd_share_model + else None, + velocity_to_epsilon=self.pipe.scheduler.config.prediction_type + == "v_prediction", + ) # (4B, 3, Hl, Wl) + + noise_pred_text = noise_pred[:batch_size] + noise_pred_uncond = noise_pred[batch_size : batch_size * 2] + noise_pred_neg = noise_pred[batch_size * 2 :] + + e_pos = noise_pred_text - noise_pred_uncond + accum_grad = 0 + n_negative_prompts = neg_guidance_weights.shape[-1] + for i in range(n_negative_prompts): + e_i_neg = noise_pred_neg[i::n_negative_prompts] - noise_pred_uncond + accum_grad += neg_guidance_weights[:, i].view( + -1, 1, 1, 1 + ) * perpendicular_component(e_i_neg, e_pos) + + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + e_pos + accum_grad + ) + else: + text_embeddings = prompt_utils.get_text_embeddings( + elevation, azimuth, camera_distances, self.cfg.view_dependent_prompting + ) + with torch.no_grad(): + with self.disable_unet_class_embedding(self.pipe.unet) as unet: + noise_pred = self.forward_unet( + unet, + torch.cat([latents_noisy] * 2, dim=0), + torch.cat([t] * 2, dim=0), + encoder_hidden_states=text_embeddings, + cross_attention_kwargs={"scale": 0.0} + if self.vsd_share_model + else None, + velocity_to_epsilon=self.pipe.scheduler.config.prediction_type + == "v_prediction", + ) + + noise_pred_text, noise_pred_uncond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + + return noise_pred + + def get_eps_phi( + self, + latents_noisy: Float[Tensor, "B 4 Hl Wl"], + t: Int[Tensor, "B"], + prompt_utils: PromptProcessorOutput, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + camera_condition: Float[Tensor, "B ..."], + ) -> Float[Tensor, "B 4 Hl Wl"]: + batch_size = latents_noisy.shape[0] + + # not using view-dependent prompting in LoRA + text_embeddings, _ = prompt_utils.get_text_embeddings( + elevation, azimuth, camera_distances, view_dependent_prompting=False + ).chunk(2) + with torch.no_grad(): + noise_pred = self.forward_unet( + self.pipe_phi.unet, + torch.cat([latents_noisy] * 2, dim=0), + torch.cat([t] * 2, dim=0), + encoder_hidden_states=torch.cat([text_embeddings] * 2, dim=0), + class_labels=torch.cat( + [ + camera_condition.view(batch_size, -1), + torch.zeros_like(camera_condition.view(batch_size, -1)), + ], + dim=0, + ) + if self.cfg.vsd_use_camera_condition + else None, + cross_attention_kwargs={"scale": 1.0}, + velocity_to_epsilon=self.pipe_phi.scheduler.config.prediction_type + == "v_prediction", + ) + + noise_pred_camera, noise_pred_uncond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.cfg.vsd_guidance_scale_phi * ( + noise_pred_camera - noise_pred_uncond + ) + + return noise_pred + + def train_phi( + self, + latents: Float[Tensor, "B 4 Hl Wl"], + prompt_utils: PromptProcessorOutput, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + camera_condition: Float[Tensor, "B ..."], + ): + B = latents.shape[0] + latents = latents.detach().repeat( + self.cfg.vsd_lora_n_timestamp_samples, 1, 1, 1 + ) + + num_train_timesteps = self.pipe_phi.scheduler.config.num_train_timesteps + t = torch.randint( + int(num_train_timesteps * 0.0), + int(num_train_timesteps * 1.0), + [B * self.cfg.vsd_lora_n_timestamp_samples], + dtype=torch.long, + device=self.device, + ) + + noise = torch.randn_like(latents) + latents_noisy = self.pipe_phi.scheduler.add_noise(latents, noise, t) + if self.pipe_phi.scheduler.config.prediction_type == "epsilon": + target = noise + elif self.pipe_phi.scheduler.prediction_type == "v_prediction": + target = self.pipe_phi.scheduler.get_velocity(latents, noise, t) + else: + raise ValueError( + f"Unknown prediction type {self.pipe_phi.scheduler.prediction_type}" + ) + + # not using view-dependent prompting in LoRA + text_embeddings, _ = prompt_utils.get_text_embeddings( + elevation, azimuth, camera_distances, view_dependent_prompting=False + ).chunk(2) + + if ( + self.cfg.vsd_use_camera_condition + and self.cfg.vsd_lora_cfg_training + and random.random() < 0.1 + ): + camera_condition = torch.zeros_like(camera_condition) + + noise_pred = self.forward_unet( + self.pipe_phi.unet, + latents_noisy, + t, + encoder_hidden_states=text_embeddings.repeat( + self.cfg.vsd_lora_n_timestamp_samples, 1, 1 + ), + class_labels=camera_condition.view(B, -1).repeat( + self.cfg.vsd_lora_n_timestamp_samples, 1 + ) + if self.cfg.vsd_use_camera_condition + else None, + cross_attention_kwargs={"scale": 1.0}, + ) + return F.mse_loss(noise_pred.float(), target.float(), reduction="mean") + + def forward( + self, + rgb: Float[Tensor, "B H W C"], + prompt_utils: PromptProcessorOutput, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + mvp_mtx: Float[Tensor, "B 4 4"], + c2w: Float[Tensor, "B 4 4"], + rgb_as_latents=False, + **kwargs, + ): + batch_size = rgb.shape[0] + + rgb_BCHW = rgb.permute(0, 3, 1, 2) + latents: Float[Tensor, "B 4 Hl Wl"] + rgb_BCHW_512 = F.interpolate( + rgb_BCHW, (512, 512), mode="bilinear", align_corners=False + ) + if rgb_as_latents: + # treat input rgb as latents + # input rgb should be in range [-1, 1] + latents = F.interpolate( + rgb_BCHW, (64, 64), mode="bilinear", align_corners=False + ) + else: + # treat input rgb as rgb + # input rgb should be in range [0, 1] + # encode image into latents with vae + latents = self.vae_encode(self.pipe.vae, rgb_BCHW_512 * 2.0 - 1.0) + + # sample timestep + # use the same timestep for each batch + assert self.min_step is not None and self.max_step is not None + t = torch.randint( + self.min_step, + self.max_step + 1, + [1], + dtype=torch.long, + device=self.device, + ).repeat(batch_size) + + # sample noise + noise = torch.randn_like(latents) + latents_noisy = self.scheduler.add_noise(latents, noise, t) + + eps_pretrain = self.get_eps_pretrain( + latents_noisy, t, prompt_utils, elevation, azimuth, camera_distances + ) + + latents_1step_orig = ( + 1 + / self.alphas[t].view(-1, 1, 1, 1) + * (latents_noisy - self.sigmas[t].view(-1, 1, 1, 1) * eps_pretrain) + ).detach() + + if self.cfg.guidance_type == "sds": + eps_phi = noise + elif self.cfg.guidance_type == "vsd": + if self.cfg.vsd_camera_condition_type == "extrinsics": + camera_condition = c2w + elif self.cfg.vsd_camera_condition_type == "mvp": + camera_condition = mvp_mtx + elif self.cfg.vsd_camera_condition_type == "spherical": + camera_condition = torch.stack( + [ + torch.deg2rad(elevation), + torch.sin(torch.deg2rad(azimuth)), + torch.cos(torch.deg2rad(azimuth)), + camera_distances, + ], + dim=-1, + ) + else: + raise ValueError( + f"Unknown camera_condition_type {self.cfg.vsd_camera_condition_type}" + ) + eps_phi = self.get_eps_phi( + latents_noisy, + t, + prompt_utils, + elevation, + azimuth, + camera_distances, + camera_condition, + ) + + loss_train_phi = self.train_phi( + latents, + prompt_utils, + elevation, + azimuth, + camera_distances, + camera_condition, + ) + + if self.cfg.weighting_strategy == "dreamfusion": + w = (1.0 - self.alphas[t]).view(-1, 1, 1, 1) + elif self.cfg.weighting_strategy == "uniform": + w = 1.0 + elif self.cfg.weighting_strategy == "fantasia3d": + w = (self.alphas[t] ** 0.5 * (1 - self.alphas[t])).view(-1, 1, 1, 1) + else: + raise ValueError( + f"Unknown weighting strategy: {self.cfg.weighting_strategy}" + ) + + grad = w * (eps_pretrain - eps_phi) + + # compute decoded image if needed for visualization/img loss + if self.cfg.return_rgb_1step_orig or self.cfg.use_img_loss: + with torch.no_grad(): + image_denoised_pretrain = self.vae_decode( + self.pipe.vae, latents_1step_orig + ) + rgb_1step_orig = image_denoised_pretrain.permute(0, 2, 3, 1) + + if self.grad_clip_val is not None: + grad = grad.clamp(-self.grad_clip_val, self.grad_clip_val) + + # reparameterization trick: + # d(loss)/d(latents) = latents - target = latents - (latents - grad) = grad + target = (latents - grad).detach() + loss_sd = 0.5 * F.mse_loss(latents, target, reduction="sum") / batch_size + + guidance_out = { + "loss_sd": loss_sd, + "grad_norm": grad.norm(), + "timesteps": t, + "min_step": self.min_step, + "max_step": self.max_step, + "latents": latents, + "latents_1step_orig": latents_1step_orig, + "rgb": rgb_BCHW.permute(0, 2, 3, 1), + "weights": w, + "lambdas": self.lambdas[t], + } + + # image-space loss proposed in HiFA: https://hifa-team.github.io/HiFA-site + if self.cfg.use_img_loss: + if self.cfg.guidance_type == "vsd": + latents_denoised_est = ( + latents_noisy - self.sigmas[t] * eps_phi + ) / self.alphas[t].view(-1, 1, 1, 1) + image_denoised_est = self.vae_decode( + self.pipe.vae, latents_denoised_est + ) + else: + image_denoised_est = rgb_BCHW_512 + grad_img = ( + w + * (image_denoised_est - image_denoised_pretrain) + * self.alphas[t].view(-1, 1, 1, 1) + / self.sigmas[t].view(-1, 1, 1, 1) + ) + if self.grad_clip_val is not None: + grad_img = grad_img.clamp(-self.grad_clip_val, self.grad_clip_val) + target_img = (rgb_BCHW_512 - grad_img).detach() + loss_sd_img = ( + 0.5 * F.mse_loss(rgb_BCHW_512, target_img, reduction="sum") / batch_size + ) + guidance_out.update({"loss_sd_img": loss_sd_img}) + + if self.cfg.return_rgb_1step_orig: + guidance_out.update({"rgb_1step_orig": rgb_1step_orig}) + + if self.cfg.return_rgb_multistep_orig: + with self.set_scheduler( + self.pipe, + DPMSolverSinglestepScheduler, + solver_order=1, + num_train_timesteps=int(t[0]), + ) as pipe: + text_embeddings = prompt_utils.get_text_embeddings( + elevation, + azimuth, + camera_distances, + self.cfg.view_dependent_prompting, + ) + text_embeddings_cond, text_embeddings_uncond = text_embeddings.chunk(2) + with torch.cuda.amp.autocast(enabled=False): + latents_multistep_orig = pipe( + num_inference_steps=self.cfg.n_rgb_multistep_orig_steps, + guidance_scale=self.cfg.guidance_scale, + eta=1.0, + latents=latents_noisy.to(pipe.unet.dtype), + prompt_embeds=text_embeddings_cond.to(pipe.unet.dtype), + negative_prompt_embeds=text_embeddings_uncond.to( + pipe.unet.dtype + ), + cross_attention_kwargs={"scale": 0.0} + if self.vsd_share_model + else None, + output_type="latent", + ).images.to(latents.dtype) + with torch.no_grad(): + rgb_multistep_orig = self.vae_decode( + self.pipe.vae, latents_multistep_orig + ) + guidance_out.update( + { + "latents_multistep_orig": latents_multistep_orig, + "rgb_multistep_orig": rgb_multistep_orig.permute(0, 2, 3, 1), + } + ) + + if self.cfg.guidance_type == "vsd": + guidance_out.update( + { + "loss_train_phi": loss_train_phi, + } + ) + + return guidance_out + + @torch.cuda.amp.autocast(enabled=False) + def set_min_max_steps(self, min_step_percent=0.02, max_step_percent=0.98): + self.min_step = int(self.num_train_timesteps * min_step_percent) + self.max_step = int(self.num_train_timesteps * max_step_percent) + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + # clip grad for stable training as demonstrated in + # Debiasing Scores and Prompts of 2D Diffusion for Robust Text-to-3D Generation + # http://arxiv.org/abs/2303.15413 + if self.cfg.grad_clip is not None: + self.grad_clip_val = C(self.cfg.grad_clip, epoch, global_step) + + if self.cfg.sqrt_anneal: + percentage = ( + float(global_step) / self.cfg.trainer_max_steps + ) ** 0.5 # progress percentage + if type(self.cfg.max_step_percent) not in [float, int]: + max_step_percent = self.cfg.max_step_percent[1] + else: + max_step_percent = self.cfg.max_step_percent + curr_percent = ( + max_step_percent - C(self.cfg.min_step_percent, epoch, global_step) + ) * (1 - percentage) + C(self.cfg.min_step_percent, epoch, global_step) + self.set_min_max_steps( + min_step_percent=curr_percent, + max_step_percent=curr_percent, + ) + else: + self.set_min_max_steps( + min_step_percent=C(self.cfg.min_step_percent, epoch, global_step), + max_step_percent=C(self.cfg.max_step_percent, epoch, global_step), + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_diffusion_vsd_guidance.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_diffusion_vsd_guidance.py new file mode 100644 index 0000000000000000000000000000000000000000..8a380baf6b82bd323f1884ff54b3378649aca8af --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_diffusion_vsd_guidance.py @@ -0,0 +1,723 @@ +import random +from contextlib import contextmanager +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers import ( + DDPMScheduler, + DPMSolverMultistepScheduler, + StableDiffusionPipeline, + UNet2DConditionModel, +) +from diffusers.loaders import AttnProcsLayers +from diffusers.models.attention_processor import LoRAAttnProcessor +from diffusers.models.embeddings import TimestepEmbedding +from diffusers.utils.import_utils import is_xformers_available + +import threestudio +from threestudio.models.prompt_processors.base import PromptProcessorOutput +from threestudio.utils.base import BaseModule +from threestudio.utils.misc import C, cleanup, parse_version +from threestudio.utils.typing import * + + +class ToWeightsDType(nn.Module): + def __init__(self, module: nn.Module, dtype: torch.dtype): + super().__init__() + self.module = module + self.dtype = dtype + + def forward(self, x: Float[Tensor, "..."]) -> Float[Tensor, "..."]: + return self.module(x).to(self.dtype) + + +@threestudio.register("stable-diffusion-vsd-guidance") +class StableDiffusionVSDGuidance(BaseModule): + @dataclass + class Config(BaseModule.Config): + pretrained_model_name_or_path: str = "stabilityai/stable-diffusion-2-1-base" + pretrained_model_name_or_path_lora: str = "stabilityai/stable-diffusion-2-1" + enable_memory_efficient_attention: bool = False + enable_sequential_cpu_offload: bool = False + enable_attention_slicing: bool = False + enable_channels_last_format: bool = False + guidance_scale: float = 7.5 + guidance_scale_lora: float = 1.0 + grad_clip: Optional[ + Any + ] = None # field(default_factory=lambda: [0, 2.0, 8.0, 1000]) + half_precision_weights: bool = True + lora_cfg_training: bool = True + lora_n_timestamp_samples: int = 1 + + min_step_percent: float = 0.02 + max_step_percent: float = 0.98 + sqrt_anneal: bool = False # sqrt anneal proposed in HiFA: https://hifa-team.github.io/HiFA-site/ + trainer_max_steps: int = 25000 + use_img_loss: bool = False # image-space SDS proposed in HiFA: https://hifa-team.github.io/HiFA-site/ + + view_dependent_prompting: bool = True + camera_condition_type: str = "extrinsics" + + cfg: Config + + def configure(self) -> None: + threestudio.info(f"Loading Stable Diffusion ...") + + self.weights_dtype = ( + torch.float16 if self.cfg.half_precision_weights else torch.float32 + ) + + pipe_kwargs = { + "tokenizer": None, + "safety_checker": None, + "feature_extractor": None, + "requires_safety_checker": False, + "torch_dtype": self.weights_dtype, + } + + pipe_lora_kwargs = { + "tokenizer": None, + "safety_checker": None, + "feature_extractor": None, + "requires_safety_checker": False, + "torch_dtype": self.weights_dtype, + } + + @dataclass + class SubModules: + pipe: StableDiffusionPipeline + pipe_lora: StableDiffusionPipeline + + pipe = StableDiffusionPipeline.from_pretrained( + self.cfg.pretrained_model_name_or_path, + **pipe_kwargs, + ).to(self.device) + if ( + self.cfg.pretrained_model_name_or_path + == self.cfg.pretrained_model_name_or_path_lora + ): + self.single_model = True + pipe_lora = pipe + else: + self.single_model = False + pipe_lora = StableDiffusionPipeline.from_pretrained( + self.cfg.pretrained_model_name_or_path_lora, + **pipe_lora_kwargs, + ).to(self.device) + del pipe_lora.vae + cleanup() + pipe_lora.vae = pipe.vae + self.submodules = SubModules(pipe=pipe, pipe_lora=pipe_lora) + + if self.cfg.enable_memory_efficient_attention: + if parse_version(torch.__version__) >= parse_version("2"): + threestudio.info( + "PyTorch2.0 uses memory efficient attention by default." + ) + elif not is_xformers_available(): + threestudio.warn( + "xformers is not available, memory efficient attention is not enabled." + ) + else: + self.pipe.enable_xformers_memory_efficient_attention() + self.pipe_lora.enable_xformers_memory_efficient_attention() + + if self.cfg.enable_sequential_cpu_offload: + self.pipe.enable_sequential_cpu_offload() + self.pipe_lora.enable_sequential_cpu_offload() + + if self.cfg.enable_attention_slicing: + self.pipe.enable_attention_slicing(1) + self.pipe_lora.enable_attention_slicing(1) + + if self.cfg.enable_channels_last_format: + self.pipe.unet.to(memory_format=torch.channels_last) + self.pipe_lora.unet.to(memory_format=torch.channels_last) + + del self.pipe.text_encoder + if not self.single_model: + del self.pipe_lora.text_encoder + cleanup() + + for p in self.vae.parameters(): + p.requires_grad_(False) + for p in self.unet.parameters(): + p.requires_grad_(False) + for p in self.unet_lora.parameters(): + p.requires_grad_(False) + + # FIXME: hard-coded dims + self.camera_embedding = ToWeightsDType( + TimestepEmbedding(16, 1280), self.weights_dtype + ).to(self.device) + self.unet_lora.class_embedding = self.camera_embedding + + # set up LoRA layers + lora_attn_procs = {} + for name in self.unet_lora.attn_processors.keys(): + cross_attention_dim = ( + None + if name.endswith("attn1.processor") + else self.unet_lora.config.cross_attention_dim + ) + if name.startswith("mid_block"): + hidden_size = self.unet_lora.config.block_out_channels[-1] + elif name.startswith("up_blocks"): + block_id = int(name[len("up_blocks.")]) + hidden_size = list(reversed(self.unet_lora.config.block_out_channels))[ + block_id + ] + elif name.startswith("down_blocks"): + block_id = int(name[len("down_blocks.")]) + hidden_size = self.unet_lora.config.block_out_channels[block_id] + + lora_attn_procs[name] = LoRAAttnProcessor( + hidden_size=hidden_size, cross_attention_dim=cross_attention_dim + ) + + self.unet_lora.set_attn_processor(lora_attn_procs) + + self.lora_layers = AttnProcsLayers(self.unet_lora.attn_processors).to( + self.device + ) + self.lora_layers._load_state_dict_pre_hooks.clear() + self.lora_layers._state_dict_hooks.clear() + + self.scheduler = DDPMScheduler.from_pretrained( + self.cfg.pretrained_model_name_or_path, + subfolder="scheduler", + torch_dtype=self.weights_dtype, + ) + + self.scheduler_lora = DDPMScheduler.from_pretrained( + self.cfg.pretrained_model_name_or_path_lora, + subfolder="scheduler", + torch_dtype=self.weights_dtype, + ) + + self.scheduler_sample = DPMSolverMultistepScheduler.from_config( + self.pipe.scheduler.config + ) + self.scheduler_lora_sample = DPMSolverMultistepScheduler.from_config( + self.pipe_lora.scheduler.config + ) + + self.pipe.scheduler = self.scheduler + self.pipe_lora.scheduler = self.scheduler_lora + + self.num_train_timesteps = self.scheduler.config.num_train_timesteps + self.set_min_max_steps() # set to default value + + self.alphas: Float[Tensor, "..."] = self.scheduler.alphas_cumprod.to( + self.device + ) + + self.grad_clip_val: Optional[float] = None + + threestudio.info(f"Loaded Stable Diffusion!") + + @torch.cuda.amp.autocast(enabled=False) + def set_min_max_steps(self, min_step_percent=0.02, max_step_percent=0.98): + self.min_step = int(self.num_train_timesteps * min_step_percent) + self.max_step = int(self.num_train_timesteps * max_step_percent) + + @property + def pipe(self): + return self.submodules.pipe + + @property + def pipe_lora(self): + return self.submodules.pipe_lora + + @property + def unet(self): + return self.submodules.pipe.unet + + @property + def unet_lora(self): + return self.submodules.pipe_lora.unet + + @property + def vae(self): + return self.submodules.pipe.vae + + @property + def vae_lora(self): + return self.submodules.pipe_lora.vae + + @torch.no_grad() + @torch.cuda.amp.autocast(enabled=False) + def _sample( + self, + pipe: StableDiffusionPipeline, + sample_scheduler: DPMSolverMultistepScheduler, + text_embeddings: Float[Tensor, "BB N Nf"], + num_inference_steps: int, + guidance_scale: float, + num_images_per_prompt: int = 1, + height: Optional[int] = None, + width: Optional[int] = None, + class_labels: Optional[Float[Tensor, "BB 16"]] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + ) -> Float[Tensor, "B H W 3"]: + vae_scale_factor = 2 ** (len(pipe.vae.config.block_out_channels) - 1) + height = height or pipe.unet.config.sample_size * vae_scale_factor + width = width or pipe.unet.config.sample_size * vae_scale_factor + batch_size = text_embeddings.shape[0] // 2 + device = self.device + + sample_scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = sample_scheduler.timesteps + num_channels_latents = pipe.unet.config.in_channels + + latents = pipe.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + self.weights_dtype, + device, + generator, + ) + + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) + latent_model_input = sample_scheduler.scale_model_input( + latent_model_input, t + ) + + # predict the noise residual + if class_labels is None: + with self.disable_unet_class_embedding(pipe.unet) as unet: + noise_pred = unet( + latent_model_input, + t, + encoder_hidden_states=text_embeddings.to(self.weights_dtype), + cross_attention_kwargs=cross_attention_kwargs, + ).sample + else: + noise_pred = pipe.unet( + latent_model_input, + t, + encoder_hidden_states=text_embeddings.to(self.weights_dtype), + class_labels=class_labels, + cross_attention_kwargs=cross_attention_kwargs, + ).sample + + noise_pred_text, noise_pred_uncond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * ( + noise_pred_text - noise_pred_uncond + ) + + # compute the previous noisy sample x_t -> x_t-1 + latents = sample_scheduler.step(noise_pred, t, latents).prev_sample + + latents = 1 / pipe.vae.config.scaling_factor * latents + images = pipe.vae.decode(latents).sample + images = (images / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + images = images.permute(0, 2, 3, 1).float() + return images + + def sample( + self, + prompt_utils: PromptProcessorOutput, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + seed: int = 0, + **kwargs, + ) -> Float[Tensor, "N H W 3"]: + # view-dependent text embeddings + text_embeddings_vd = prompt_utils.get_text_embeddings( + elevation, + azimuth, + camera_distances, + view_dependent_prompting=self.cfg.view_dependent_prompting, + ) + cross_attention_kwargs = {"scale": 0.0} if self.single_model else None + generator = torch.Generator(device=self.device).manual_seed(seed) + + return self._sample( + pipe=self.pipe, + sample_scheduler=self.scheduler_sample, + text_embeddings=text_embeddings_vd, + num_inference_steps=25, + guidance_scale=self.cfg.guidance_scale, + cross_attention_kwargs=cross_attention_kwargs, + generator=generator, + ) + + def sample_lora( + self, + prompt_utils: PromptProcessorOutput, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + mvp_mtx: Float[Tensor, "B 4 4"], + c2w: Float[Tensor, "B 4 4"], + seed: int = 0, + **kwargs, + ) -> Float[Tensor, "N H W 3"]: + # input text embeddings, view-independent + text_embeddings = prompt_utils.get_text_embeddings( + elevation, azimuth, camera_distances, view_dependent_prompting=False + ) + + if self.cfg.camera_condition_type == "extrinsics": + camera_condition = c2w + elif self.cfg.camera_condition_type == "mvp": + camera_condition = mvp_mtx + else: + raise ValueError( + f"Unknown camera_condition_type {self.cfg.camera_condition_type}" + ) + + B = elevation.shape[0] + camera_condition_cfg = torch.cat( + [ + camera_condition.view(B, -1), + torch.zeros_like(camera_condition.view(B, -1)), + ], + dim=0, + ) + + generator = torch.Generator(device=self.device).manual_seed(seed) + return self._sample( + sample_scheduler=self.scheduler_lora_sample, + pipe=self.pipe_lora, + text_embeddings=text_embeddings, + num_inference_steps=25, + guidance_scale=self.cfg.guidance_scale_lora, + class_labels=camera_condition_cfg, + cross_attention_kwargs={"scale": 1.0}, + generator=generator, + ) + + @torch.cuda.amp.autocast(enabled=False) + def forward_unet( + self, + unet: UNet2DConditionModel, + latents: Float[Tensor, "..."], + t: Float[Tensor, "..."], + encoder_hidden_states: Float[Tensor, "..."], + class_labels: Optional[Float[Tensor, "B 16"]] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Float[Tensor, "..."]: + input_dtype = latents.dtype + return unet( + latents.to(self.weights_dtype), + t.to(self.weights_dtype), + encoder_hidden_states=encoder_hidden_states.to(self.weights_dtype), + class_labels=class_labels, + cross_attention_kwargs=cross_attention_kwargs, + ).sample.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def encode_images( + self, imgs: Float[Tensor, "B 3 512 512"] + ) -> Float[Tensor, "B 4 64 64"]: + input_dtype = imgs.dtype + imgs = imgs * 2.0 - 1.0 + posterior = self.vae.encode(imgs.to(self.weights_dtype)).latent_dist + latents = posterior.sample() * self.vae.config.scaling_factor + return latents.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def decode_latents( + self, + latents: Float[Tensor, "B 4 H W"], + latent_height: int = 64, + latent_width: int = 64, + ) -> Float[Tensor, "B 3 512 512"]: + input_dtype = latents.dtype + latents = F.interpolate( + latents, (latent_height, latent_width), mode="bilinear", align_corners=False + ) + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents.to(self.weights_dtype)).sample + image = (image * 0.5 + 0.5).clamp(0, 1) + return image.to(input_dtype) + + @contextmanager + def disable_unet_class_embedding(self, unet: UNet2DConditionModel): + class_embedding = unet.class_embedding + try: + unet.class_embedding = None + yield unet + finally: + unet.class_embedding = class_embedding + + def compute_grad_vsd( + self, + latents: Float[Tensor, "B 4 64 64"], + text_embeddings_vd: Float[Tensor, "BB 77 768"], + text_embeddings: Float[Tensor, "BB 77 768"], + camera_condition: Float[Tensor, "B 4 4"], + ): + B = latents.shape[0] + + with torch.no_grad(): + # random timestamp + t = torch.randint( + self.min_step, + self.max_step + 1, + [B], + dtype=torch.long, + device=self.device, + ) + + # add noise + noise = torch.randn_like(latents) + latents_noisy = self.scheduler.add_noise(latents, noise, t) + # pred noise + latent_model_input = torch.cat([latents_noisy] * 2, dim=0) + with self.disable_unet_class_embedding(self.unet) as unet: + cross_attention_kwargs = {"scale": 0.0} if self.single_model else None + noise_pred_pretrain = self.forward_unet( + unet, + latent_model_input, + torch.cat([t] * 2), + encoder_hidden_states=text_embeddings_vd, + cross_attention_kwargs=cross_attention_kwargs, + ) + + # use view-independent text embeddings in LoRA + text_embeddings_cond, _ = text_embeddings.chunk(2) + noise_pred_est = self.forward_unet( + self.unet_lora, + latent_model_input, + torch.cat([t] * 2), + encoder_hidden_states=torch.cat([text_embeddings_cond] * 2), + class_labels=torch.cat( + [ + camera_condition.view(B, -1), + torch.zeros_like(camera_condition.view(B, -1)), + ], + dim=0, + ), + cross_attention_kwargs={"scale": 1.0}, + ) + + ( + noise_pred_pretrain_text, + noise_pred_pretrain_uncond, + ) = noise_pred_pretrain.chunk(2) + + # NOTE: guidance scale definition here is aligned with diffusers, but different from other guidance + noise_pred_pretrain = noise_pred_pretrain_uncond + self.cfg.guidance_scale * ( + noise_pred_pretrain_text - noise_pred_pretrain_uncond + ) + + # TODO: more general cases + assert self.scheduler.config.prediction_type == "epsilon" + if self.scheduler_lora.config.prediction_type == "v_prediction": + alphas_cumprod = self.scheduler_lora.alphas_cumprod.to( + device=latents_noisy.device, dtype=latents_noisy.dtype + ) + alpha_t = alphas_cumprod[t] ** 0.5 + sigma_t = (1 - alphas_cumprod[t]) ** 0.5 + + noise_pred_est = latent_model_input * torch.cat([sigma_t] * 2, dim=0).view( + -1, 1, 1, 1 + ) + noise_pred_est * torch.cat([alpha_t] * 2, dim=0).view(-1, 1, 1, 1) + + ( + noise_pred_est_camera, + noise_pred_est_uncond, + ) = noise_pred_est.chunk(2) + + # NOTE: guidance scale definition here is aligned with diffusers, but different from other guidance + noise_pred_est = noise_pred_est_uncond + self.cfg.guidance_scale_lora * ( + noise_pred_est_camera - noise_pred_est_uncond + ) + + w = (1 - self.alphas[t]).view(-1, 1, 1, 1) + + grad = w * (noise_pred_pretrain - noise_pred_est) + + alpha = (self.alphas[t] ** 0.5).view(-1, 1, 1, 1) + sigma = ((1 - self.alphas[t]) ** 0.5).view(-1, 1, 1, 1) + # image-space SDS proposed in HiFA: https://hifa-team.github.io/HiFA-site/ + if self.cfg.use_img_loss: + latents_denoised_pretrain = ( + latents_noisy - sigma * noise_pred_pretrain + ) / alpha + latents_denoised_est = (latents_noisy - sigma * noise_pred_est) / alpha + image_denoised_pretrain = self.decode_latents(latents_denoised_pretrain) + image_denoised_est = self.decode_latents(latents_denoised_est) + grad_img = ( + w * (image_denoised_est - image_denoised_pretrain) * alpha / sigma + ) + else: + grad_img = None + return grad, grad_img + + def train_lora( + self, + latents: Float[Tensor, "B 4 64 64"], + text_embeddings: Float[Tensor, "BB 77 768"], + camera_condition: Float[Tensor, "B 4 4"], + ): + B = latents.shape[0] + latents = latents.detach().repeat(self.cfg.lora_n_timestamp_samples, 1, 1, 1) + + t = torch.randint( + int(self.num_train_timesteps * 0.0), + int(self.num_train_timesteps * 1.0), + [B * self.cfg.lora_n_timestamp_samples], + dtype=torch.long, + device=self.device, + ) + + noise = torch.randn_like(latents) + noisy_latents = self.scheduler_lora.add_noise(latents, noise, t) + if self.scheduler_lora.config.prediction_type == "epsilon": + target = noise + elif self.scheduler_lora.config.prediction_type == "v_prediction": + target = self.scheduler_lora.get_velocity(latents, noise, t) + else: + raise ValueError( + f"Unknown prediction type {self.scheduler_lora.config.prediction_type}" + ) + # use view-independent text embeddings in LoRA + text_embeddings_cond, _ = text_embeddings.chunk(2) + if self.cfg.lora_cfg_training and random.random() < 0.1: + camera_condition = torch.zeros_like(camera_condition) + noise_pred = self.forward_unet( + self.unet_lora, + noisy_latents, + t, + encoder_hidden_states=text_embeddings_cond.repeat( + self.cfg.lora_n_timestamp_samples, 1, 1 + ), + class_labels=camera_condition.view(B, -1).repeat( + self.cfg.lora_n_timestamp_samples, 1 + ), + cross_attention_kwargs={"scale": 1.0}, + ) + return F.mse_loss(noise_pred.float(), target.float(), reduction="mean") + + def get_latents( + self, rgb_BCHW: Float[Tensor, "B C H W"], rgb_as_latents=False + ) -> Float[Tensor, "B 4 64 64"]: + rgb_BCHW_512 = F.interpolate( + rgb_BCHW, (512, 512), mode="bilinear", align_corners=False + ) + if rgb_as_latents: + latents = F.interpolate( + rgb_BCHW, (64, 64), mode="bilinear", align_corners=False + ) + else: + # encode image into latents with vae + latents = self.encode_images(rgb_BCHW_512) + return latents, rgb_BCHW_512 + + def forward( + self, + rgb: Float[Tensor, "B H W C"], + prompt_utils: PromptProcessorOutput, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + mvp_mtx: Float[Tensor, "B 4 4"], + c2w: Float[Tensor, "B 4 4"], + rgb_as_latents=False, + **kwargs, + ): + batch_size = rgb.shape[0] + + rgb_BCHW = rgb.permute(0, 3, 1, 2) + latents, rgb_BCHW_512 = self.get_latents( + rgb_BCHW, rgb_as_latents=rgb_as_latents + ) + + # view-dependent text embeddings + text_embeddings_vd = prompt_utils.get_text_embeddings( + elevation, + azimuth, + camera_distances, + view_dependent_prompting=self.cfg.view_dependent_prompting, + ) + + # input text embeddings, view-independent + text_embeddings = prompt_utils.get_text_embeddings( + elevation, azimuth, camera_distances, view_dependent_prompting=False + ) + + if self.cfg.camera_condition_type == "extrinsics": + camera_condition = c2w + elif self.cfg.camera_condition_type == "mvp": + camera_condition = mvp_mtx + else: + raise ValueError( + f"Unknown camera_condition_type {self.cfg.camera_condition_type}" + ) + + grad, grad_img = self.compute_grad_vsd( + latents, text_embeddings_vd, text_embeddings, camera_condition + ) + + grad = torch.nan_to_num(grad) + # clip grad for stable training? + if self.grad_clip_val is not None: + grad = grad.clamp(-self.grad_clip_val, self.grad_clip_val) + + # reparameterization trick + # d(loss)/d(latents) = latents - target = latents - (latents - grad) = grad + target = (latents - grad).detach() + loss_vsd = 0.5 * F.mse_loss(latents, target, reduction="sum") / batch_size + loss_lora = self.train_lora(latents, text_embeddings, camera_condition) + + loss_dict = { + "loss_vsd": loss_vsd, + "loss_lora": loss_lora, + "grad_norm": grad.norm(), + "min_step": self.min_step, + "max_step": self.max_step, + } + + if self.cfg.use_img_loss: + grad_img = torch.nan_to_num(grad_img) + if self.grad_clip_val is not None: + grad_img = grad_img.clamp(-self.grad_clip_val, self.grad_clip_val) + target_img = (rgb_BCHW_512 - grad_img).detach() + loss_vsd_img = ( + 0.5 * F.mse_loss(rgb_BCHW_512, target_img, reduction="sum") / batch_size + ) + loss_dict["loss_vsd_img"] = loss_vsd_img + + return loss_dict + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + # clip grad for stable training as demonstrated in + # Debiasing Scores and Prompts of 2D Diffusion for Robust Text-to-3D Generation + # http://arxiv.org/abs/2303.15413 + if self.cfg.grad_clip is not None: + self.grad_clip_val = C(self.cfg.grad_clip, epoch, global_step) + + if self.cfg.sqrt_anneal: + percentage = ( + float(global_step) / self.cfg.trainer_max_steps + ) ** 0.5 # progress percentage + if type(self.cfg.max_step_percent) not in [float, int]: + max_step_percent = self.cfg.max_step_percent[1] + else: + max_step_percent = self.cfg.max_step_percent + curr_percent = ( + max_step_percent - C(self.cfg.min_step_percent, epoch, global_step) + ) * (1 - percentage) + C(self.cfg.min_step_percent, epoch, global_step) + self.set_min_max_steps( + min_step_percent=curr_percent, + max_step_percent=curr_percent, + ) + else: + self.set_min_max_steps( + min_step_percent=C(self.cfg.min_step_percent, epoch, global_step), + max_step_percent=C(self.cfg.max_step_percent, epoch, global_step), + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_zero123_guidance.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_zero123_guidance.py new file mode 100644 index 0000000000000000000000000000000000000000..6d545908397a3daf93a846860a7fcec90d99841a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/stable_zero123_guidance.py @@ -0,0 +1,340 @@ +import importlib +import os +from dataclasses import dataclass, field + +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers import DDIMScheduler, DDPMScheduler, StableDiffusionPipeline +from diffusers.utils.import_utils import is_xformers_available +from omegaconf import OmegaConf +from tqdm import tqdm + +import threestudio +from threestudio.utils.base import BaseObject +from threestudio.utils.misc import C, parse_version +from threestudio.utils.typing import * + + +def get_obj_from_str(string, reload=False): + module, cls = string.rsplit(".", 1) + if reload: + module_imp = importlib.import_module(module) + importlib.reload(module_imp) + return getattr(importlib.import_module(module, package=None), cls) + + +def instantiate_from_config(config): + if not "target" in config: + if config == "__is_first_stage__": + return None + elif config == "__is_unconditional__": + return None + raise KeyError("Expected key `target` to instantiate.") + return get_obj_from_str(config["target"])(**config.get("params", dict())) + + +# load model +def load_model_from_config(config, ckpt, device, vram_O=True, verbose=False): + pl_sd = torch.load(ckpt, map_location="cpu") + + if "global_step" in pl_sd and verbose: + print(f'[INFO] Global Step: {pl_sd["global_step"]}') + + sd = pl_sd["state_dict"] + + model = instantiate_from_config(config.model) + m, u = model.load_state_dict(sd, strict=False) + + if len(m) > 0 and verbose: + print("[INFO] missing keys: \n", m) + if len(u) > 0 and verbose: + print("[INFO] unexpected keys: \n", u) + + # manually load ema and delete it to save GPU memory + if model.use_ema: + if verbose: + print("[INFO] loading EMA...") + model.model_ema.copy_to(model.model) + del model.model_ema + + if vram_O: + # we don't need decoder + del model.first_stage_model.decoder + + torch.cuda.empty_cache() + + model.eval().to(device) + + return model + + +@threestudio.register("stable-zero123-guidance") +class StableZero123Guidance(BaseObject): + @dataclass + class Config(BaseObject.Config): + pretrained_model_name_or_path: str = "load/zero123/stable-zero123.ckpt" + pretrained_config: str = "load/zero123/sd-objaverse-finetune-c_concat-256.yaml" + vram_O: bool = True + + cond_image_path: str = "load/images/hamburger_rgba.png" + cond_elevation_deg: float = 0.0 + cond_azimuth_deg: float = 0.0 + cond_camera_distance: float = 1.2 + + guidance_scale: float = 5.0 + + grad_clip: Optional[ + Any + ] = None # field(default_factory=lambda: [0, 2.0, 8.0, 1000]) + half_precision_weights: bool = False + + min_step_percent: float = 0.02 + max_step_percent: float = 0.98 + + cfg: Config + + def configure(self) -> None: + threestudio.info(f"Loading Stable Zero123 ...") + + self.config = OmegaConf.load(self.cfg.pretrained_config) + # TODO: seems it cannot load into fp16... + self.weights_dtype = torch.float32 + self.model = load_model_from_config( + self.config, + self.cfg.pretrained_model_name_or_path, + device=self.device, + vram_O=self.cfg.vram_O, + ) + + for p in self.model.parameters(): + p.requires_grad_(False) + + # timesteps: use diffuser for convenience... hope it's alright. + self.num_train_timesteps = self.config.model.params.timesteps + + self.scheduler = DDIMScheduler( + self.num_train_timesteps, + self.config.model.params.linear_start, + self.config.model.params.linear_end, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + steps_offset=1, + ) + + self.num_train_timesteps = self.scheduler.config.num_train_timesteps + self.set_min_max_steps() # set to default value + + self.alphas: Float[Tensor, "..."] = self.scheduler.alphas_cumprod.to( + self.device + ) + + self.grad_clip_val: Optional[float] = None + + self.prepare_embeddings(self.cfg.cond_image_path) + + threestudio.info(f"Loaded Stable Zero123!") + + @torch.cuda.amp.autocast(enabled=False) + def set_min_max_steps(self, min_step_percent=0.02, max_step_percent=0.98): + self.min_step = int(self.num_train_timesteps * min_step_percent) + self.max_step = int(self.num_train_timesteps * max_step_percent) + + @torch.cuda.amp.autocast(enabled=False) + def prepare_embeddings(self, image_path: str) -> None: + # load cond image for zero123 + assert os.path.exists(image_path) + rgba = cv2.cvtColor( + cv2.imread(image_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGRA2RGBA + ) + rgba = ( + cv2.resize(rgba, (256, 256), interpolation=cv2.INTER_AREA).astype( + np.float32 + ) + / 255.0 + ) + rgb = rgba[..., :3] * rgba[..., 3:] + (1 - rgba[..., 3:]) + self.rgb_256: Float[Tensor, "1 3 H W"] = ( + torch.from_numpy(rgb) + .unsqueeze(0) + .permute(0, 3, 1, 2) + .contiguous() + .to(self.device) + ) + self.c_crossattn, self.c_concat = self.get_img_embeds(self.rgb_256) + + @torch.cuda.amp.autocast(enabled=False) + @torch.no_grad() + def get_img_embeds( + self, + img: Float[Tensor, "B 3 256 256"], + ) -> Tuple[Float[Tensor, "B 1 768"], Float[Tensor, "B 4 32 32"]]: + img = img * 2.0 - 1.0 + c_crossattn = self.model.get_learned_conditioning(img.to(self.weights_dtype)) + c_concat = self.model.encode_first_stage(img.to(self.weights_dtype)).mode() + return c_crossattn, c_concat + + @torch.cuda.amp.autocast(enabled=False) + def encode_images( + self, imgs: Float[Tensor, "B 3 256 256"] + ) -> Float[Tensor, "B 4 32 32"]: + input_dtype = imgs.dtype + imgs = imgs * 2.0 - 1.0 + latents = self.model.get_first_stage_encoding( + self.model.encode_first_stage(imgs.to(self.weights_dtype)) + ) + return latents.to(input_dtype) # [B, 4, 32, 32] Latent space image + + @torch.cuda.amp.autocast(enabled=False) + def decode_latents( + self, + latents: Float[Tensor, "B 4 H W"], + ) -> Float[Tensor, "B 3 512 512"]: + input_dtype = latents.dtype + image = self.model.decode_first_stage(latents) + image = (image * 0.5 + 0.5).clamp(0, 1) + return image.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + @torch.no_grad() + def get_cond( + self, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + c_crossattn=None, + c_concat=None, + **kwargs, + ) -> dict: + T = torch.stack( + [ + torch.deg2rad( + (90 - elevation) - (90 - self.cfg.cond_elevation_deg) + ), # Zero123 polar is 90-elevation + torch.sin(torch.deg2rad(azimuth - self.cfg.cond_azimuth_deg)), + torch.cos(torch.deg2rad(azimuth - self.cfg.cond_azimuth_deg)), + torch.deg2rad( + 90 - torch.full_like(elevation, self.cfg.cond_elevation_deg) + ), + ], + dim=-1, + )[:, None, :].to(self.device) + cond = {} + clip_emb = self.model.cc_projection( + torch.cat( + [ + (self.c_crossattn if c_crossattn is None else c_crossattn).repeat( + len(T), 1, 1 + ), + T, + ], + dim=-1, + ) + ) + cond["c_crossattn"] = [ + torch.cat([torch.zeros_like(clip_emb).to(self.device), clip_emb], dim=0) + ] + cond["c_concat"] = [ + torch.cat( + [ + torch.zeros_like(self.c_concat) + .repeat(len(T), 1, 1, 1) + .to(self.device), + (self.c_concat if c_concat is None else c_concat).repeat( + len(T), 1, 1, 1 + ), + ], + dim=0, + ) + ] + return cond + + def __call__( + self, + rgb: Float[Tensor, "B H W C"], + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + rgb_as_latents=False, + **kwargs, + ): + batch_size = rgb.shape[0] + + rgb_BCHW = rgb.permute(0, 3, 1, 2) + latents: Float[Tensor, "B 4 64 64"] + if rgb_as_latents: + latents = ( + F.interpolate(rgb_BCHW, (32, 32), mode="bilinear", align_corners=False) + * 2 + - 1 + ) + else: + rgb_BCHW_512 = F.interpolate( + rgb_BCHW, (256, 256), mode="bilinear", align_corners=False + ) + # encode image into latents with vae + latents = self.encode_images(rgb_BCHW_512) + + cond = self.get_cond(elevation, azimuth, camera_distances) + + # timestep ~ U(0.02, 0.98) to avoid very high/low noise level + t = torch.randint( + self.min_step, + self.max_step + 1, + [batch_size], + dtype=torch.long, + device=self.device, + ) + + # predict the noise residual with unet, NO grad! + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) # TODO: use torch generator + latents_noisy = self.scheduler.add_noise(latents, noise, t) + # pred noise + x_in = torch.cat([latents_noisy] * 2) + t_in = torch.cat([t] * 2) + noise_pred = self.model.apply_model(x_in, t_in, cond) + + # perform guidance + noise_pred_uncond, noise_pred_cond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + noise_pred_cond - noise_pred_uncond + ) + + w = (1 - self.alphas[t]).reshape(-1, 1, 1, 1) + grad = w * (noise_pred - noise) + grad = torch.nan_to_num(grad) + # clip grad for stable training? + if self.grad_clip_val is not None: + grad = grad.clamp(-self.grad_clip_val, self.grad_clip_val) + + # loss = SpecifyGradient.apply(latents, grad) + # SpecifyGradient is not straghtforward, use a reparameterization trick instead + target = (latents - grad).detach() + # d(loss)/d(latents) = latents - target = latents - (latents - grad) = grad + loss_sds = 0.5 * F.mse_loss(latents, target, reduction="sum") / batch_size + + guidance_out = { + "loss_sds": loss_sds, + "grad_norm": grad.norm(), + "min_step": self.min_step, + "max_step": self.max_step, + } + + return guidance_out + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + # clip grad for stable training as demonstrated in + # Debiasing Scores and Prompts of 2D Diffusion for Robust Text-to-3D Generation + # http://arxiv.org/abs/2303.15413 + if self.cfg.grad_clip is not None: + self.grad_clip_val = C(self.cfg.grad_clip, epoch, global_step) + + self.set_min_max_steps( + min_step_percent=C(self.cfg.min_step_percent, epoch, global_step), + max_step_percent=C(self.cfg.max_step_percent, epoch, global_step), + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/zero123_guidance.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/zero123_guidance.py new file mode 100644 index 0000000000000000000000000000000000000000..d1e2697b2574bbbb010ce37eed0530f941ce4f6f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/zero123_guidance.py @@ -0,0 +1,491 @@ +import importlib +import os +from dataclasses import dataclass, field + +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers import DDIMScheduler, DDPMScheduler, StableDiffusionPipeline +from diffusers.utils.import_utils import is_xformers_available +from omegaconf import OmegaConf +from tqdm import tqdm + +import threestudio +from threestudio.utils.base import BaseObject +from threestudio.utils.misc import C, parse_version +from threestudio.utils.typing import * + + +def get_obj_from_str(string, reload=False): + module, cls = string.rsplit(".", 1) + if reload: + module_imp = importlib.import_module(module) + importlib.reload(module_imp) + return getattr(importlib.import_module(module, package=None), cls) + + +def instantiate_from_config(config): + if not "target" in config: + if config == "__is_first_stage__": + return None + elif config == "__is_unconditional__": + return None + raise KeyError("Expected key `target` to instantiate.") + return get_obj_from_str(config["target"])(**config.get("params", dict())) + + +# load model +def load_model_from_config(config, ckpt, device, vram_O=True, verbose=False): + pl_sd = torch.load(ckpt, map_location="cpu") + + if "global_step" in pl_sd and verbose: + print(f'[INFO] Global Step: {pl_sd["global_step"]}') + + sd = pl_sd["state_dict"] + + model = instantiate_from_config(config.model) + m, u = model.load_state_dict(sd, strict=False) + + if len(m) > 0 and verbose: + print("[INFO] missing keys: \n", m) + if len(u) > 0 and verbose: + print("[INFO] unexpected keys: \n", u) + + # manually load ema and delete it to save GPU memory + if model.use_ema: + if verbose: + print("[INFO] loading EMA...") + model.model_ema.copy_to(model.model) + del model.model_ema + + if vram_O: + # we don't need decoder + del model.first_stage_model.decoder + + torch.cuda.empty_cache() + + model.eval().to(device) + + return model + + +@threestudio.register("zero123-guidance") +class Zero123Guidance(BaseObject): + @dataclass + class Config(BaseObject.Config): + pretrained_model_name_or_path: str = "load/zero123/105000.ckpt" + pretrained_config: str = "load/zero123/sd-objaverse-finetune-c_concat-256.yaml" + vram_O: bool = True + + cond_image_path: str = "load/images/hamburger_rgba.png" + cond_elevation_deg: float = 0.0 + cond_azimuth_deg: float = 0.0 + cond_camera_distance: float = 1.2 + + guidance_scale: float = 5.0 + + grad_clip: Optional[ + Any + ] = None # field(default_factory=lambda: [0, 2.0, 8.0, 1000]) + half_precision_weights: bool = False + + min_step_percent: float = 0.02 + max_step_percent: float = 0.98 + + """Maximum number of batch items to evaluate guidance for (for debugging) and to save on disk. -1 means save all items.""" + max_items_eval: int = 4 + + cfg: Config + + def configure(self) -> None: + threestudio.info(f"Loading Zero123 ...") + + self.config = OmegaConf.load(self.cfg.pretrained_config) + # TODO: seems it cannot load into fp16... + self.weights_dtype = torch.float32 + self.model = load_model_from_config( + self.config, + self.cfg.pretrained_model_name_or_path, + device=self.device, + vram_O=self.cfg.vram_O, + ) + + for p in self.model.parameters(): + p.requires_grad_(False) + + # timesteps: use diffuser for convenience... hope it's alright. + self.num_train_timesteps = self.config.model.params.timesteps + + self.scheduler = DDIMScheduler( + self.num_train_timesteps, + self.config.model.params.linear_start, + self.config.model.params.linear_end, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + steps_offset=1, + ) + + self.num_train_timesteps = self.scheduler.config.num_train_timesteps + self.set_min_max_steps() # set to default value + + self.alphas: Float[Tensor, "..."] = self.scheduler.alphas_cumprod.to( + self.device + ) + + self.grad_clip_val: Optional[float] = None + + self.prepare_embeddings(self.cfg.cond_image_path) + + threestudio.info(f"Loaded Zero123!") + + @torch.cuda.amp.autocast(enabled=False) + def set_min_max_steps(self, min_step_percent=0.02, max_step_percent=0.98): + self.min_step = int(self.num_train_timesteps * min_step_percent) + self.max_step = int(self.num_train_timesteps * max_step_percent) + + @torch.cuda.amp.autocast(enabled=False) + def prepare_embeddings(self, image_path: str) -> None: + # load cond image for zero123 + assert os.path.exists(image_path) + rgba = cv2.cvtColor( + cv2.imread(image_path, cv2.IMREAD_UNCHANGED), cv2.COLOR_BGRA2RGBA + ) + rgba = ( + cv2.resize(rgba, (256, 256), interpolation=cv2.INTER_AREA).astype( + np.float32 + ) + / 255.0 + ) + rgb = rgba[..., :3] * rgba[..., 3:] + (1 - rgba[..., 3:]) + self.rgb_256: Float[Tensor, "1 3 H W"] = ( + torch.from_numpy(rgb) + .unsqueeze(0) + .permute(0, 3, 1, 2) + .contiguous() + .to(self.device) + ) + self.c_crossattn, self.c_concat = self.get_img_embeds(self.rgb_256) + + @torch.cuda.amp.autocast(enabled=False) + @torch.no_grad() + def get_img_embeds( + self, + img: Float[Tensor, "B 3 256 256"], + ) -> Tuple[Float[Tensor, "B 1 768"], Float[Tensor, "B 4 32 32"]]: + img = img * 2.0 - 1.0 + c_crossattn = self.model.get_learned_conditioning(img.to(self.weights_dtype)) + c_concat = self.model.encode_first_stage(img.to(self.weights_dtype)).mode() + return c_crossattn, c_concat + + @torch.cuda.amp.autocast(enabled=False) + def encode_images( + self, imgs: Float[Tensor, "B 3 256 256"] + ) -> Float[Tensor, "B 4 32 32"]: + input_dtype = imgs.dtype + imgs = imgs * 2.0 - 1.0 + latents = self.model.get_first_stage_encoding( + self.model.encode_first_stage(imgs.to(self.weights_dtype)) + ) + return latents.to(input_dtype) # [B, 4, 32, 32] Latent space image + + @torch.cuda.amp.autocast(enabled=False) + def decode_latents( + self, + latents: Float[Tensor, "B 4 H W"], + ) -> Float[Tensor, "B 3 512 512"]: + input_dtype = latents.dtype + image = self.model.decode_first_stage(latents) + image = (image * 0.5 + 0.5).clamp(0, 1) + return image.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + @torch.no_grad() + def get_cond( + self, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + c_crossattn=None, + c_concat=None, + **kwargs, + ) -> dict: + T = torch.stack( + [ + torch.deg2rad( + (90 - elevation) - (90 - self.cfg.cond_elevation_deg) + ), # Zero123 polar is 90-elevation + torch.sin(torch.deg2rad(azimuth - self.cfg.cond_azimuth_deg)), + torch.cos(torch.deg2rad(azimuth - self.cfg.cond_azimuth_deg)), + camera_distances - self.cfg.cond_camera_distance, + ], + dim=-1, + )[:, None, :].to(self.device) + cond = {} + clip_emb = self.model.cc_projection( + torch.cat( + [ + (self.c_crossattn if c_crossattn is None else c_crossattn).repeat( + len(T), 1, 1 + ), + T, + ], + dim=-1, + ) + ) + cond["c_crossattn"] = [ + torch.cat([torch.zeros_like(clip_emb).to(self.device), clip_emb], dim=0) + ] + cond["c_concat"] = [ + torch.cat( + [ + torch.zeros_like(self.c_concat) + .repeat(len(T), 1, 1, 1) + .to(self.device), + (self.c_concat if c_concat is None else c_concat).repeat( + len(T), 1, 1, 1 + ), + ], + dim=0, + ) + ] + return cond + + def __call__( + self, + rgb: Float[Tensor, "B H W C"], + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + rgb_as_latents=False, + guidance_eval=False, + **kwargs, + ): + batch_size = rgb.shape[0] + + rgb_BCHW = rgb.permute(0, 3, 1, 2) + latents: Float[Tensor, "B 4 64 64"] + if rgb_as_latents: + latents = ( + F.interpolate(rgb_BCHW, (32, 32), mode="bilinear", align_corners=False) + * 2 + - 1 + ) + else: + rgb_BCHW_512 = F.interpolate( + rgb_BCHW, (256, 256), mode="bilinear", align_corners=False + ) + # encode image into latents with vae + latents = self.encode_images(rgb_BCHW_512) + + cond = self.get_cond(elevation, azimuth, camera_distances) + + # timestep ~ U(0.02, 0.98) to avoid very high/low noise level + t = torch.randint( + self.min_step, + self.max_step + 1, + [batch_size], + dtype=torch.long, + device=self.device, + ) + + # predict the noise residual with unet, NO grad! + with torch.no_grad(): + # add noise + noise = torch.randn_like(latents) # TODO: use torch generator + latents_noisy = self.scheduler.add_noise(latents, noise, t) + # pred noise + x_in = torch.cat([latents_noisy] * 2) + t_in = torch.cat([t] * 2) + noise_pred = self.model.apply_model(x_in, t_in, cond) + + # perform guidance + noise_pred_uncond, noise_pred_cond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + noise_pred_cond - noise_pred_uncond + ) + + w = (1 - self.alphas[t]).reshape(-1, 1, 1, 1) + grad = w * (noise_pred - noise) + grad = torch.nan_to_num(grad) + # clip grad for stable training? + if self.grad_clip_val is not None: + grad = grad.clamp(-self.grad_clip_val, self.grad_clip_val) + + # loss = SpecifyGradient.apply(latents, grad) + # SpecifyGradient is not straghtforward, use a reparameterization trick instead + target = (latents - grad).detach() + # d(loss)/d(latents) = latents - target = latents - (latents - grad) = grad + loss_sds = 0.5 * F.mse_loss(latents, target, reduction="sum") / batch_size + + guidance_out = { + "loss_sds": loss_sds, + "grad_norm": grad.norm(), + "min_step": self.min_step, + "max_step": self.max_step, + } + + if guidance_eval: + guidance_eval_utils = { + "cond": cond, + "t_orig": t, + "latents_noisy": latents_noisy, + "noise_pred": noise_pred, + } + guidance_eval_out = self.guidance_eval(**guidance_eval_utils) + texts = [] + for n, e, a, c in zip( + guidance_eval_out["noise_levels"], elevation, azimuth, camera_distances + ): + texts.append( + f"n{n:.02f}\ne{e.item():.01f}\na{a.item():.01f}\nc{c.item():.02f}" + ) + guidance_eval_out.update({"texts": texts}) + guidance_out.update({"eval": guidance_eval_out}) + + return guidance_out + + @torch.cuda.amp.autocast(enabled=False) + @torch.no_grad() + def guidance_eval(self, cond, t_orig, latents_noisy, noise_pred): + # use only 50 timesteps, and find nearest of those to t + self.scheduler.set_timesteps(50) + self.scheduler.timesteps_gpu = self.scheduler.timesteps.to(self.device) + bs = ( + min(self.cfg.max_items_eval, latents_noisy.shape[0]) + if self.cfg.max_items_eval > 0 + else latents_noisy.shape[0] + ) # batch size + large_enough_idxs = self.scheduler.timesteps_gpu.expand([bs, -1]) > t_orig[ + :bs + ].unsqueeze( + -1 + ) # sized [bs,50] > [bs,1] + idxs = torch.min(large_enough_idxs, dim=1)[1] + t = self.scheduler.timesteps_gpu[idxs] + + fracs = list((t / self.scheduler.config.num_train_timesteps).cpu().numpy()) + imgs_noisy = self.decode_latents(latents_noisy[:bs]).permute(0, 2, 3, 1) + + # get prev latent + latents_1step = [] + pred_1orig = [] + for b in range(bs): + step_output = self.scheduler.step( + noise_pred[b : b + 1], t[b], latents_noisy[b : b + 1], eta=1 + ) + latents_1step.append(step_output["prev_sample"]) + pred_1orig.append(step_output["pred_original_sample"]) + latents_1step = torch.cat(latents_1step) + pred_1orig = torch.cat(pred_1orig) + imgs_1step = self.decode_latents(latents_1step).permute(0, 2, 3, 1) + imgs_1orig = self.decode_latents(pred_1orig).permute(0, 2, 3, 1) + + latents_final = [] + for b, i in enumerate(idxs): + latents = latents_1step[b : b + 1] + c = { + "c_crossattn": [cond["c_crossattn"][0][[b, b + len(idxs)], ...]], + "c_concat": [cond["c_concat"][0][[b, b + len(idxs)], ...]], + } + for t in tqdm(self.scheduler.timesteps[i + 1 :], leave=False): + # pred noise + x_in = torch.cat([latents] * 2) + t_in = torch.cat([t.reshape(1)] * 2).to(self.device) + noise_pred = self.model.apply_model(x_in, t_in, c) + # perform guidance + noise_pred_uncond, noise_pred_cond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + noise_pred_cond - noise_pred_uncond + ) + # get prev latent + latents = self.scheduler.step(noise_pred, t, latents, eta=1)[ + "prev_sample" + ] + latents_final.append(latents) + + latents_final = torch.cat(latents_final) + imgs_final = self.decode_latents(latents_final).permute(0, 2, 3, 1) + + return { + "bs": bs, + "noise_levels": fracs, + "imgs_noisy": imgs_noisy, + "imgs_1step": imgs_1step, + "imgs_1orig": imgs_1orig, + "imgs_final": imgs_final, + } + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + # clip grad for stable training as demonstrated in + # Debiasing Scores and Prompts of 2D Diffusion for Robust Text-to-3D Generation + # http://arxiv.org/abs/2303.15413 + if self.cfg.grad_clip is not None: + self.grad_clip_val = C(self.cfg.grad_clip, epoch, global_step) + + self.set_min_max_steps( + min_step_percent=C(self.cfg.min_step_percent, epoch, global_step), + max_step_percent=C(self.cfg.max_step_percent, epoch, global_step), + ) + + # verification - requires `vram_O = False` in load_model_from_config + @torch.no_grad() + def generate( + self, + image, # image tensor [1, 3, H, W] in [0, 1] + elevation=0, + azimuth=0, + camera_distances=0, # new view params + c_crossattn=None, + c_concat=None, + scale=3, + ddim_steps=50, + post_process=True, + ddim_eta=1, + ): + if c_crossattn is None: + c_crossattn, c_concat = self.get_img_embeds(image) + + cond = self.get_cond( + elevation, azimuth, camera_distances, c_crossattn, c_concat + ) + + imgs = self.gen_from_cond(cond, scale, ddim_steps, post_process, ddim_eta) + + return imgs + + # verification - requires `vram_O = False` in load_model_from_config + @torch.no_grad() + def gen_from_cond( + self, + cond, + scale=3, + ddim_steps=50, + post_process=True, + ddim_eta=1, + ): + # produce latents loop + B = cond["c_crossattn"][0].shape[0] // 2 + latents = torch.randn((B, 4, 32, 32), device=self.device) + self.scheduler.set_timesteps(ddim_steps) + + for t in self.scheduler.timesteps: + x_in = torch.cat([latents] * 2) + t_in = torch.cat([t.reshape(1).repeat(B)] * 2).to(self.device) + + noise_pred = self.model.apply_model(x_in, t_in, cond) + noise_pred_uncond, noise_pred_cond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + scale * ( + noise_pred_cond - noise_pred_uncond + ) + + latents = self.scheduler.step(noise_pred, t, latents, eta=ddim_eta)[ + "prev_sample" + ] + + imgs = self.decode_latents(latents) + imgs = imgs.cpu().numpy().transpose(0, 2, 3, 1) if post_process else imgs + + return imgs diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/zero123_unified_guidance.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/zero123_unified_guidance.py new file mode 100644 index 0000000000000000000000000000000000000000..8274b51d9cf2a5a4f1a1435dd2df9aafd4f413f9 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/guidance/zero123_unified_guidance.py @@ -0,0 +1,716 @@ +import os +import random +import sys +from contextlib import contextmanager +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from diffusers import ( + AutoencoderKL, + DDPMScheduler, + DPMSolverSinglestepScheduler, + UNet2DConditionModel, +) +from diffusers.loaders import AttnProcsLayers +from diffusers.models.attention_processor import LoRAAttnProcessor +from diffusers.models.embeddings import TimestepEmbedding +from PIL import Image +from tqdm import tqdm + +import threestudio +from extern.zero123 import Zero123Pipeline +from threestudio.models.networks import ToDTypeWrapper +from threestudio.models.prompt_processors.base import PromptProcessorOutput +from threestudio.utils.base import BaseModule +from threestudio.utils.misc import C, cleanup, enable_gradient, parse_version +from threestudio.utils.typing import * + + +@threestudio.register("zero123-unified-guidance") +class Zero123UnifiedGuidance(BaseModule): + @dataclass + class Config(BaseModule.Config): + # guidance type, in ["sds", "vsd"] + guidance_type: str = "sds" + + pretrained_model_name_or_path: str = "bennyguo/zero123-diffusers" + guidance_scale: float = 5.0 + weighting_strategy: str = "dreamfusion" + + min_step_percent: Any = 0.02 + max_step_percent: Any = 0.98 + grad_clip: Optional[Any] = None + + return_rgb_1step_orig: bool = False + return_rgb_multistep_orig: bool = False + n_rgb_multistep_orig_steps: int = 4 + + cond_image_path: str = "" + cond_elevation_deg: float = 0.0 + cond_azimuth_deg: float = 0.0 + cond_camera_distance: float = 1.2 + + # efficiency-related configurations + half_precision_weights: bool = True + + # VSD configurations, only used when guidance_type is "vsd" + vsd_phi_model_name_or_path: Optional[str] = None + vsd_guidance_scale_phi: float = 1.0 + vsd_use_lora: bool = True + vsd_lora_cfg_training: bool = False + vsd_lora_n_timestamp_samples: int = 1 + vsd_use_camera_condition: bool = True + # camera condition type, in ["extrinsics", "mvp", "spherical"] + vsd_camera_condition_type: Optional[str] = "extrinsics" + + cfg: Config + + def configure(self) -> None: + self.min_step: Optional[int] = None + self.max_step: Optional[int] = None + self.grad_clip_val: Optional[float] = None + + @dataclass + class NonTrainableModules: + pipe: Zero123Pipeline + pipe_phi: Optional[Zero123Pipeline] = None + + self.weights_dtype = ( + torch.float16 if self.cfg.half_precision_weights else torch.float32 + ) + + threestudio.info(f"Loading Zero123 ...") + + # need to make sure the pipeline file is in path + sys.path.append("extern/") + + pipe_kwargs = { + "safety_checker": None, + "requires_safety_checker": False, + "variant": "fp16" if self.cfg.half_precision_weights else None, + "torch_dtype": self.weights_dtype, + } + pipe = Zero123Pipeline.from_pretrained( + self.cfg.pretrained_model_name_or_path, + **pipe_kwargs, + ).to(self.device) + self.prepare_pipe(pipe) + + # phi network for VSD + # introduce two trainable modules: + # - self.camera_embedding + # - self.lora_layers + pipe_phi = None + + # if the phi network shares the same unet with the pretrain network + # we need to pass additional cross attention kwargs to the unet + self.vsd_share_model = ( + self.cfg.guidance_type == "vsd" + and self.cfg.vsd_phi_model_name_or_path is None + ) + if self.cfg.guidance_type == "vsd": + if self.cfg.vsd_phi_model_name_or_path is None: + pipe_phi = pipe + else: + pipe_phi = Zero123Pipeline.from_pretrained( + self.cfg.vsd_phi_model_name_or_path, + **pipe_kwargs, + ).to(self.device) + self.prepare_pipe(pipe_phi) + + # set up camera embedding + if self.cfg.vsd_use_camera_condition: + if self.cfg.vsd_camera_condition_type in ["extrinsics", "mvp"]: + self.camera_embedding_dim = 16 + elif self.cfg.vsd_camera_condition_type == "spherical": + self.camera_embedding_dim = 4 + else: + raise ValueError("Invalid camera condition type!") + + # FIXME: hard-coded output dim + self.camera_embedding = ToDTypeWrapper( + TimestepEmbedding(self.camera_embedding_dim, 1280), + self.weights_dtype, + ).to(self.device) + pipe_phi.unet.class_embedding = self.camera_embedding + + if self.cfg.vsd_use_lora: + # set up LoRA layers + lora_attn_procs = {} + for name in pipe_phi.unet.attn_processors.keys(): + cross_attention_dim = ( + None + if name.endswith("attn1.processor") + else pipe_phi.unet.config.cross_attention_dim + ) + if name.startswith("mid_block"): + hidden_size = pipe_phi.unet.config.block_out_channels[-1] + elif name.startswith("up_blocks"): + block_id = int(name[len("up_blocks.")]) + hidden_size = list( + reversed(pipe_phi.unet.config.block_out_channels) + )[block_id] + elif name.startswith("down_blocks"): + block_id = int(name[len("down_blocks.")]) + hidden_size = pipe_phi.unet.config.block_out_channels[block_id] + + lora_attn_procs[name] = LoRAAttnProcessor( + hidden_size=hidden_size, cross_attention_dim=cross_attention_dim + ) + + pipe_phi.unet.set_attn_processor(lora_attn_procs) + + self.lora_layers = AttnProcsLayers(pipe_phi.unet.attn_processors).to( + self.device + ) + self.lora_layers._load_state_dict_pre_hooks.clear() + self.lora_layers._state_dict_hooks.clear() + + threestudio.info(f"Loaded Stable Diffusion!") + + self.scheduler = DDPMScheduler.from_config(pipe.scheduler.config) + self.num_train_timesteps = self.scheduler.config.num_train_timesteps + + # q(z_t|x) = N(alpha_t x, sigma_t^2 I) + # in DDPM, alpha_t = sqrt(alphas_cumprod_t), sigma_t^2 = 1 - alphas_cumprod_t + self.alphas_cumprod: Float[Tensor, "T"] = self.scheduler.alphas_cumprod.to( + self.device + ) + self.alphas: Float[Tensor, "T"] = self.alphas_cumprod**0.5 + self.sigmas: Float[Tensor, "T"] = (1 - self.alphas_cumprod) ** 0.5 + # log SNR + self.lambdas: Float[Tensor, "T"] = self.sigmas / self.alphas + + self._non_trainable_modules = NonTrainableModules( + pipe=pipe, + pipe_phi=pipe_phi, + ) + + # self.clip_image_embeddings and self.image_latents + self.prepare_image_embeddings() + + @property + def pipe(self) -> Zero123Pipeline: + return self._non_trainable_modules.pipe + + @property + def pipe_phi(self) -> Zero123Pipeline: + if self._non_trainable_modules.pipe_phi is None: + raise RuntimeError("phi model is not available.") + return self._non_trainable_modules.pipe_phi + + def prepare_pipe(self, pipe: Zero123Pipeline): + cleanup() + + pipe.image_encoder.eval() + pipe.vae.eval() + pipe.unet.eval() + pipe.clip_camera_projection.eval() + + enable_gradient(pipe.image_encoder, enabled=False) + enable_gradient(pipe.vae, enabled=False) + enable_gradient(pipe.unet, enabled=False) + enable_gradient(pipe.clip_camera_projection, enabled=False) + + # disable progress bar + pipe.set_progress_bar_config(disable=True) + + def prepare_image_embeddings(self) -> None: + if not os.path.exists(self.cfg.cond_image_path): + raise RuntimeError( + f"Condition image not found at {self.cfg.cond_image_path}" + ) + image = Image.open(self.cfg.cond_image_path).convert("RGBA").resize((256, 256)) + image = ( + TF.to_tensor(image) + .unsqueeze(0) + .to(device=self.device, dtype=self.weights_dtype) + ) + # rgba -> rgb, apply white background + image = image[:, :3] * image[:, 3:4] + (1 - image[:, 3:4]) + + with torch.no_grad(): + self.clip_image_embeddings: Float[ + Tensor, "1 1 D" + ] = self.extract_clip_image_embeddings(image) + + # encoded latents should be multiplied with vae.config.scaling_factor + # but zero123 was not trained this way + self.image_latents: Float[Tensor, "1 4 Hl Wl"] = ( + self.vae_encode(self.pipe.vae, image * 2.0 - 1.0, mode=True) + / self.pipe.vae.config.scaling_factor + ) + + def extract_clip_image_embeddings( + self, images: Float[Tensor, "B 3 H W"] + ) -> Float[Tensor, "B 1 D"]: + # expect images in [0, 1] + images_pil = [TF.to_pil_image(image) for image in images] + images_processed = self.pipe.feature_extractor( + images=images_pil, return_tensors="pt" + ).pixel_values.to(device=self.device, dtype=self.weights_dtype) + clip_image_embeddings = self.pipe.image_encoder(images_processed).image_embeds + return clip_image_embeddings.to(images.dtype) + + def get_image_camera_embeddings( + self, + elevation_deg: Float[Tensor, "B"], + azimuth_deg: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + ) -> Float[Tensor, "B 1 D"]: + batch_size = elevation_deg.shape[0] + camera_embeddings: Float[Tensor, "B 1 4"] = torch.stack( + [ + torch.deg2rad(self.cfg.cond_elevation_deg - elevation_deg), + torch.sin(torch.deg2rad(azimuth_deg - self.cfg.cond_azimuth_deg)), + torch.cos(torch.deg2rad(azimuth_deg - self.cfg.cond_azimuth_deg)), + camera_distances - self.cfg.cond_camera_distance, + ], + dim=-1, + )[:, None, :] + + image_camera_embeddings = self.pipe.clip_camera_projection( + torch.cat( + [ + self.clip_image_embeddings.repeat(batch_size, 1, 1), + camera_embeddings, + ], + dim=-1, + ).to(self.weights_dtype) + ) + + return image_camera_embeddings + + @torch.cuda.amp.autocast(enabled=False) + def forward_unet( + self, + unet: UNet2DConditionModel, + latents: Float[Tensor, "..."], + t: Int[Tensor, "..."], + encoder_hidden_states: Float[Tensor, "..."], + class_labels: Optional[Float[Tensor, "..."]] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + down_block_additional_residuals: Optional[Float[Tensor, "..."]] = None, + mid_block_additional_residual: Optional[Float[Tensor, "..."]] = None, + velocity_to_epsilon: bool = False, + ) -> Float[Tensor, "..."]: + input_dtype = latents.dtype + pred = unet( + latents.to(unet.dtype), + t.to(unet.dtype), + encoder_hidden_states=encoder_hidden_states.to(unet.dtype), + class_labels=class_labels, + cross_attention_kwargs=cross_attention_kwargs, + down_block_additional_residuals=down_block_additional_residuals, + mid_block_additional_residual=mid_block_additional_residual, + ).sample + if velocity_to_epsilon: + pred = latents * self.sigmas[t].view(-1, 1, 1, 1) + pred * self.alphas[ + t + ].view(-1, 1, 1, 1) + return pred.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def vae_encode( + self, vae: AutoencoderKL, imgs: Float[Tensor, "B 3 H W"], mode=False + ) -> Float[Tensor, "B 4 Hl Wl"]: + # expect input in [-1, 1] + input_dtype = imgs.dtype + posterior = vae.encode(imgs.to(vae.dtype)).latent_dist + if mode: + latents = posterior.mode() + else: + latents = posterior.sample() + latents = latents * vae.config.scaling_factor + return latents.to(input_dtype) + + @torch.cuda.amp.autocast(enabled=False) + def vae_decode( + self, vae: AutoencoderKL, latents: Float[Tensor, "B 4 Hl Wl"] + ) -> Float[Tensor, "B 3 H W"]: + # output in [0, 1] + input_dtype = latents.dtype + latents = 1 / vae.config.scaling_factor * latents + image = vae.decode(latents.to(vae.dtype)).sample + image = (image * 0.5 + 0.5).clamp(0, 1) + return image.to(input_dtype) + + @contextmanager + def disable_unet_class_embedding(self, unet: UNet2DConditionModel): + class_embedding = unet.class_embedding + try: + unet.class_embedding = None + yield unet + finally: + unet.class_embedding = class_embedding + + @contextmanager + def set_scheduler(self, pipe: Zero123Pipeline, scheduler_class: Any, **kwargs): + scheduler_orig = pipe.scheduler + pipe.scheduler = scheduler_class.from_config(scheduler_orig.config, **kwargs) + yield pipe + pipe.scheduler = scheduler_orig + + def get_eps_pretrain( + self, + latents_noisy: Float[Tensor, "B 4 Hl Wl"], + t: Int[Tensor, "B"], + image_camera_embeddings: Float[Tensor, "B 1 D"], + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + ) -> Float[Tensor, "B 4 Hl Wl"]: + batch_size = latents_noisy.shape[0] + + with torch.no_grad(): + with self.disable_unet_class_embedding(self.pipe.unet) as unet: + noise_pred = self.forward_unet( + unet, + torch.cat( + [ + torch.cat([latents_noisy] * 2, dim=0), + torch.cat( + [ + self.image_latents.repeat(batch_size, 1, 1, 1), + torch.zeros_like(self.image_latents).repeat( + batch_size, 1, 1, 1 + ), + ], + dim=0, + ), + ], + dim=1, + ), + torch.cat([t] * 2, dim=0), + encoder_hidden_states=torch.cat( + [ + image_camera_embeddings, + torch.zeros_like(image_camera_embeddings), + ], + dim=0, + ), + cross_attention_kwargs={"scale": 0.0} + if self.vsd_share_model + else None, + velocity_to_epsilon=self.pipe.scheduler.config.prediction_type + == "v_prediction", + ) + + noise_pred_image, noise_pred_uncond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.cfg.guidance_scale * ( + noise_pred_image - noise_pred_uncond + ) + + return noise_pred + + def get_eps_phi( + self, + latents_noisy: Float[Tensor, "B 4 Hl Wl"], + t: Int[Tensor, "B"], + image_camera_embeddings: Float[Tensor, "B 1 D"], + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + camera_condition: Float[Tensor, "B ..."], + ) -> Float[Tensor, "B 4 Hl Wl"]: + batch_size = latents_noisy.shape[0] + + with torch.no_grad(): + noise_pred = self.forward_unet( + self.pipe_phi.unet, + torch.cat( + [ + torch.cat([latents_noisy] * 2, dim=0), + torch.cat( + [self.image_latents.repeat(batch_size, 1, 1, 1)] * 2, + dim=0, + ), + ], + dim=1, + ), + torch.cat([t] * 2, dim=0), + encoder_hidden_states=torch.cat([image_camera_embeddings] * 2, dim=0), + class_labels=torch.cat( + [ + camera_condition.view(batch_size, -1), + torch.zeros_like(camera_condition.view(batch_size, -1)), + ], + dim=0, + ) + if self.cfg.vsd_use_camera_condition + else None, + cross_attention_kwargs={"scale": 1.0}, + velocity_to_epsilon=self.pipe_phi.scheduler.config.prediction_type + == "v_prediction", + ) + + noise_pred_camera, noise_pred_uncond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.cfg.vsd_guidance_scale_phi * ( + noise_pred_camera - noise_pred_uncond + ) + + return noise_pred + + def train_phi( + self, + latents: Float[Tensor, "B 4 Hl Wl"], + image_camera_embeddings: Float[Tensor, "B 1 D"], + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + camera_condition: Float[Tensor, "B ..."], + ): + B = latents.shape[0] + latents = latents.detach().repeat( + self.cfg.vsd_lora_n_timestamp_samples, 1, 1, 1 + ) + + num_train_timesteps = self.pipe_phi.scheduler.config.num_train_timesteps + t = torch.randint( + int(num_train_timesteps * 0.0), + int(num_train_timesteps * 1.0), + [B * self.cfg.vsd_lora_n_timestamp_samples], + dtype=torch.long, + device=self.device, + ) + + noise = torch.randn_like(latents) + latents_noisy = self.pipe_phi.scheduler.add_noise(latents, noise, t) + if self.pipe_phi.scheduler.config.prediction_type == "epsilon": + target = noise + elif self.pipe_phi.scheduler.prediction_type == "v_prediction": + target = self.pipe_phi.scheduler.get_velocity(latents, noise, t) + else: + raise ValueError( + f"Unknown prediction type {self.pipe_phi.scheduler.prediction_type}" + ) + + if ( + self.cfg.vsd_use_camera_condition + and self.cfg.vsd_lora_cfg_training + and random.random() < 0.1 + ): + camera_condition = torch.zeros_like(camera_condition) + + noise_pred = self.forward_unet( + self.pipe_phi.unet, + torch.cat([latents_noisy, self.image_latents.repeat(B, 1, 1, 1)], dim=1), + t, + encoder_hidden_states=image_camera_embeddings.repeat( + self.cfg.vsd_lora_n_timestamp_samples, 1, 1 + ), + class_labels=camera_condition.view(B, -1).repeat( + self.cfg.vsd_lora_n_timestamp_samples, 1 + ) + if self.cfg.vsd_use_camera_condition + else None, + cross_attention_kwargs={"scale": 1.0}, + ) + return F.mse_loss(noise_pred.float(), target.float(), reduction="mean") + + def forward( + self, + rgb: Float[Tensor, "B H W C"], + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + mvp_mtx: Float[Tensor, "B 4 4"], + c2w: Float[Tensor, "B 4 4"], + rgb_as_latents=False, + **kwargs, + ): + batch_size = rgb.shape[0] + + rgb_BCHW = rgb.permute(0, 3, 1, 2) + latents: Float[Tensor, "B 4 32 32"] + if rgb_as_latents: + # treat input rgb as latents + # input rgb should be in range [-1, 1] + latents = F.interpolate( + rgb_BCHW, (32, 32), mode="bilinear", align_corners=False + ) + else: + # treat input rgb as rgb + # input rgb should be in range [0, 1] + rgb_BCHW = F.interpolate( + rgb_BCHW, (256, 256), mode="bilinear", align_corners=False + ) + # encode image into latents with vae + latents = self.vae_encode(self.pipe.vae, rgb_BCHW * 2.0 - 1.0) + + # sample timestep + # use the same timestep for each batch + assert self.min_step is not None and self.max_step is not None + t = torch.randint( + self.min_step, + self.max_step + 1, + [1], + dtype=torch.long, + device=self.device, + ).repeat(batch_size) + + # sample noise + noise = torch.randn_like(latents) + latents_noisy = self.scheduler.add_noise(latents, noise, t) + + # image-camera feature condition + image_camera_embeddings = self.get_image_camera_embeddings( + elevation, azimuth, camera_distances + ) + + eps_pretrain = self.get_eps_pretrain( + latents_noisy, + t, + image_camera_embeddings, + elevation, + azimuth, + camera_distances, + ) + + latents_1step_orig = ( + 1 + / self.alphas[t].view(-1, 1, 1, 1) + * (latents_noisy - self.sigmas[t].view(-1, 1, 1, 1) * eps_pretrain) + ).detach() + + if self.cfg.guidance_type == "sds": + eps_phi = noise + elif self.cfg.guidance_type == "vsd": + if self.cfg.vsd_camera_condition_type == "extrinsics": + camera_condition = c2w + elif self.cfg.vsd_camera_condition_type == "mvp": + camera_condition = mvp_mtx + elif self.cfg.vsd_camera_condition_type == "spherical": + camera_condition = torch.stack( + [ + torch.deg2rad(elevation), + torch.sin(torch.deg2rad(azimuth)), + torch.cos(torch.deg2rad(azimuth)), + camera_distances, + ], + dim=-1, + ) + else: + raise ValueError( + f"Unknown camera_condition_type {self.cfg.vsd_camera_condition_type}" + ) + eps_phi = self.get_eps_phi( + latents_noisy, + t, + image_camera_embeddings, + elevation, + azimuth, + camera_distances, + camera_condition, + ) + + loss_train_phi = self.train_phi( + latents, + image_camera_embeddings, + elevation, + azimuth, + camera_distances, + camera_condition, + ) + + if self.cfg.weighting_strategy == "dreamfusion": + w = (1.0 - self.alphas[t]).view(-1, 1, 1, 1) + elif self.cfg.weighting_strategy == "uniform": + w = 1.0 + elif self.cfg.weighting_strategy == "fantasia3d": + w = (self.alphas[t] ** 0.5 * (1 - self.alphas[t])).view(-1, 1, 1, 1) + else: + raise ValueError( + f"Unknown weighting strategy: {self.cfg.weighting_strategy}" + ) + + grad = w * (eps_pretrain - eps_phi) + + if self.grad_clip_val is not None: + grad = grad.clamp(-self.grad_clip_val, self.grad_clip_val) + + # reparameterization trick: + # d(loss)/d(latents) = latents - target = latents - (latents - grad) = grad + target = (latents - grad).detach() + loss_sd = 0.5 * F.mse_loss(latents, target, reduction="sum") / batch_size + + guidance_out = { + "loss_sd": loss_sd, + "grad_norm": grad.norm(), + "timesteps": t, + "min_step": self.min_step, + "max_step": self.max_step, + "latents": latents, + "latents_1step_orig": latents_1step_orig, + "rgb": rgb_BCHW.permute(0, 2, 3, 1), + "weights": w, + "lambdas": self.lambdas[t], + } + + if self.cfg.return_rgb_1step_orig: + with torch.no_grad(): + rgb_1step_orig = self.vae_decode( + self.pipe.vae, latents_1step_orig + ).permute(0, 2, 3, 1) + guidance_out.update({"rgb_1step_orig": rgb_1step_orig}) + + if self.cfg.return_rgb_multistep_orig: + with self.set_scheduler( + self.pipe, + DPMSolverSinglestepScheduler, + solver_order=1, + num_train_timesteps=int(t[0]), + ) as pipe: + with torch.cuda.amp.autocast(enabled=False): + latents_multistep_orig = pipe( + num_inference_steps=self.cfg.n_rgb_multistep_orig_steps, + guidance_scale=self.cfg.guidance_scale, + eta=1.0, + latents=latents_noisy.to(pipe.unet.dtype), + image_camera_embeddings=image_camera_embeddings.to( + pipe.unet.dtype + ), + image_latents=self.image_latents.repeat(batch_size, 1, 1, 1).to( + pipe.unet.dtype + ), + cross_attention_kwargs={"scale": 0.0} + if self.vsd_share_model + else None, + output_type="latent", + ).images.to(latents.dtype) + with torch.no_grad(): + rgb_multistep_orig = self.vae_decode( + self.pipe.vae, latents_multistep_orig + ) + guidance_out.update( + { + "latents_multistep_orig": latents_multistep_orig, + "rgb_multistep_orig": rgb_multistep_orig.permute(0, 2, 3, 1), + } + ) + + if self.cfg.guidance_type == "vsd": + guidance_out.update( + { + "loss_train_phi": loss_train_phi, + } + ) + + return guidance_out + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + # clip grad for stable training as demonstrated in + # Debiasing Scores and Prompts of 2D Diffusion for Robust Text-to-3D Generation + # http://arxiv.org/abs/2303.15413 + if self.cfg.grad_clip is not None: + self.grad_clip_val = C(self.cfg.grad_clip, epoch, global_step) + + self.min_step = int( + self.num_train_timesteps * C(self.cfg.min_step_percent, epoch, global_step) + ) + self.max_step = int( + self.num_train_timesteps * C(self.cfg.max_step_percent, epoch, global_step) + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/isosurface.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/isosurface.py new file mode 100644 index 0000000000000000000000000000000000000000..3a7f149c81c6315703637d740239193269fbbb9b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/isosurface.py @@ -0,0 +1,253 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.mesh import Mesh +from threestudio.utils.typing import * + + +class IsosurfaceHelper(nn.Module): + points_range: Tuple[float, float] = (0, 1) + + @property + def grid_vertices(self) -> Float[Tensor, "N 3"]: + raise NotImplementedError + + +class MarchingCubeCPUHelper(IsosurfaceHelper): + def __init__(self, resolution: int) -> None: + super().__init__() + self.resolution = resolution + import mcubes + + self.mc_func: Callable = mcubes.marching_cubes + self._grid_vertices: Optional[Float[Tensor, "N3 3"]] = None + self._dummy: Float[Tensor, "..."] + self.register_buffer( + "_dummy", torch.zeros(0, dtype=torch.float32), persistent=False + ) + + @property + def grid_vertices(self) -> Float[Tensor, "N3 3"]: + if self._grid_vertices is None: + # keep the vertices on CPU so that we can support very large resolution + x, y, z = ( + torch.linspace(*self.points_range, self.resolution), + torch.linspace(*self.points_range, self.resolution), + torch.linspace(*self.points_range, self.resolution), + ) + x, y, z = torch.meshgrid(x, y, z, indexing="ij") + verts = torch.cat( + [x.reshape(-1, 1), y.reshape(-1, 1), z.reshape(-1, 1)], dim=-1 + ).reshape(-1, 3) + self._grid_vertices = verts + return self._grid_vertices + + def forward( + self, + level: Float[Tensor, "N3 1"], + deformation: Optional[Float[Tensor, "N3 3"]] = None, + ) -> Mesh: + if deformation is not None: + threestudio.warn( + f"{self.__class__.__name__} does not support deformation. Ignoring." + ) + level = -level.view(self.resolution, self.resolution, self.resolution) + v_pos, t_pos_idx = self.mc_func( + level.detach().cpu().numpy(), 0.0 + ) # transform to numpy + v_pos, t_pos_idx = ( + torch.from_numpy(v_pos).float().to(self._dummy.device), + torch.from_numpy(t_pos_idx.astype(np.int64)).long().to(self._dummy.device), + ) # transform back to torch tensor on CUDA + v_pos = v_pos / (self.resolution - 1.0) + return Mesh(v_pos=v_pos, t_pos_idx=t_pos_idx) + + +class MarchingTetrahedraHelper(IsosurfaceHelper): + def __init__(self, resolution: int, tets_path: str): + super().__init__() + self.resolution = resolution + self.tets_path = tets_path + + self.triangle_table: Float[Tensor, "..."] + self.register_buffer( + "triangle_table", + torch.as_tensor( + [ + [-1, -1, -1, -1, -1, -1], + [1, 0, 2, -1, -1, -1], + [4, 0, 3, -1, -1, -1], + [1, 4, 2, 1, 3, 4], + [3, 1, 5, -1, -1, -1], + [2, 3, 0, 2, 5, 3], + [1, 4, 0, 1, 5, 4], + [4, 2, 5, -1, -1, -1], + [4, 5, 2, -1, -1, -1], + [4, 1, 0, 4, 5, 1], + [3, 2, 0, 3, 5, 2], + [1, 3, 5, -1, -1, -1], + [4, 1, 2, 4, 3, 1], + [3, 0, 4, -1, -1, -1], + [2, 0, 1, -1, -1, -1], + [-1, -1, -1, -1, -1, -1], + ], + dtype=torch.long, + ), + persistent=False, + ) + self.num_triangles_table: Integer[Tensor, "..."] + self.register_buffer( + "num_triangles_table", + torch.as_tensor( + [0, 1, 1, 2, 1, 2, 2, 1, 1, 2, 2, 1, 2, 1, 1, 0], dtype=torch.long + ), + persistent=False, + ) + self.base_tet_edges: Integer[Tensor, "..."] + self.register_buffer( + "base_tet_edges", + torch.as_tensor([0, 1, 0, 2, 0, 3, 1, 2, 1, 3, 2, 3], dtype=torch.long), + persistent=False, + ) + + tets = np.load(self.tets_path) + self._grid_vertices: Float[Tensor, "..."] + self.register_buffer( + "_grid_vertices", + torch.from_numpy(tets["vertices"]).float(), + persistent=False, + ) + self.indices: Integer[Tensor, "..."] + self.register_buffer( + "indices", torch.from_numpy(tets["indices"]).long(), persistent=False + ) + + self._all_edges: Optional[Integer[Tensor, "Ne 2"]] = None + + def normalize_grid_deformation( + self, grid_vertex_offsets: Float[Tensor, "Nv 3"] + ) -> Float[Tensor, "Nv 3"]: + return ( + (self.points_range[1] - self.points_range[0]) + / (self.resolution) # half tet size is approximately 1 / self.resolution + * torch.tanh(grid_vertex_offsets) + ) # FIXME: hard-coded activation + + @property + def grid_vertices(self) -> Float[Tensor, "Nv 3"]: + return self._grid_vertices + + @property + def all_edges(self) -> Integer[Tensor, "Ne 2"]: + if self._all_edges is None: + # compute edges on GPU, or it would be VERY SLOW (basically due to the unique operation) + edges = torch.tensor( + [0, 1, 0, 2, 0, 3, 1, 2, 1, 3, 2, 3], + dtype=torch.long, + device=self.indices.device, + ) + _all_edges = self.indices[:, edges].reshape(-1, 2) + _all_edges_sorted = torch.sort(_all_edges, dim=1)[0] + _all_edges = torch.unique(_all_edges_sorted, dim=0) + self._all_edges = _all_edges + return self._all_edges + + def sort_edges(self, edges_ex2): + with torch.no_grad(): + order = (edges_ex2[:, 0] > edges_ex2[:, 1]).long() + order = order.unsqueeze(dim=1) + + a = torch.gather(input=edges_ex2, index=order, dim=1) + b = torch.gather(input=edges_ex2, index=1 - order, dim=1) + + return torch.stack([a, b], -1) + + def _forward(self, pos_nx3, sdf_n, tet_fx4): + with torch.no_grad(): + occ_n = sdf_n > 0 + occ_fx4 = occ_n[tet_fx4.reshape(-1)].reshape(-1, 4) + occ_sum = torch.sum(occ_fx4, -1) + valid_tets = (occ_sum > 0) & (occ_sum < 4) + occ_sum = occ_sum[valid_tets] + + # find all vertices + all_edges = tet_fx4[valid_tets][:, self.base_tet_edges].reshape(-1, 2) + all_edges = self.sort_edges(all_edges) + unique_edges, idx_map = torch.unique(all_edges, dim=0, return_inverse=True) + + unique_edges = unique_edges.long() + mask_edges = occ_n[unique_edges.reshape(-1)].reshape(-1, 2).sum(-1) == 1 + mapping = ( + torch.ones( + (unique_edges.shape[0]), dtype=torch.long, device=pos_nx3.device + ) + * -1 + ) + mapping[mask_edges] = torch.arange( + mask_edges.sum(), dtype=torch.long, device=pos_nx3.device + ) + idx_map = mapping[idx_map] # map edges to verts + + interp_v = unique_edges[mask_edges] + edges_to_interp = pos_nx3[interp_v.reshape(-1)].reshape(-1, 2, 3) + edges_to_interp_sdf = sdf_n[interp_v.reshape(-1)].reshape(-1, 2, 1) + edges_to_interp_sdf[:, -1] *= -1 + + denominator = edges_to_interp_sdf.sum(1, keepdim=True) + + edges_to_interp_sdf = torch.flip(edges_to_interp_sdf, [1]) / denominator + verts = (edges_to_interp * edges_to_interp_sdf).sum(1) + + idx_map = idx_map.reshape(-1, 6) + + v_id = torch.pow(2, torch.arange(4, dtype=torch.long, device=pos_nx3.device)) + tetindex = (occ_fx4[valid_tets] * v_id.unsqueeze(0)).sum(-1) + num_triangles = self.num_triangles_table[tetindex] + + # Generate triangle indices + faces = torch.cat( + ( + torch.gather( + input=idx_map[num_triangles == 1], + dim=1, + index=self.triangle_table[tetindex[num_triangles == 1]][:, :3], + ).reshape(-1, 3), + torch.gather( + input=idx_map[num_triangles == 2], + dim=1, + index=self.triangle_table[tetindex[num_triangles == 2]][:, :6], + ).reshape(-1, 3), + ), + dim=0, + ) + + return verts, faces + + def forward( + self, + level: Float[Tensor, "N3 1"], + deformation: Optional[Float[Tensor, "N3 3"]] = None, + ) -> Mesh: + if deformation is not None: + grid_vertices = self.grid_vertices + self.normalize_grid_deformation( + deformation + ) + else: + grid_vertices = self.grid_vertices + + v_pos, t_pos_idx = self._forward(grid_vertices, level, self.indices) + + mesh = Mesh( + v_pos=v_pos, + t_pos_idx=t_pos_idx, + # extras + grid_vertices=grid_vertices, + tet_edges=self.all_edges, + grid_level=level, + grid_deformation=deformation, + ) + + return mesh diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..85d50ba19d3c44f141d5aa0a5edde29aead83064 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/__init__.py @@ -0,0 +1,9 @@ +from . import ( + base, + diffuse_with_point_light_material, + hybrid_rgb_latent_material, + neural_radiance_material, + no_material, + pbr_material, + sd_latent_adapter_material, +) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/base.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/base.py new file mode 100644 index 0000000000000000000000000000000000000000..9df8f3048d092f2b58fd5e396d87915c8ca98224 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/base.py @@ -0,0 +1,29 @@ +import random +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.utils.base import BaseModule +from threestudio.utils.typing import * + + +class BaseMaterial(BaseModule): + @dataclass + class Config(BaseModule.Config): + pass + + cfg: Config + requires_normal: bool = False + requires_tangent: bool = False + + def configure(self): + pass + + def forward(self, *args, **kwargs) -> Float[Tensor, "*B 3"]: + raise NotImplementedError + + def export(self, *args, **kwargs) -> Dict[str, Any]: + return {} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/diffuse_with_point_light_material.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/diffuse_with_point_light_material.py new file mode 100644 index 0000000000000000000000000000000000000000..abf06717cd82ee720f128ddbfe993a924013e562 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/diffuse_with_point_light_material.py @@ -0,0 +1,120 @@ +import random +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.materials.base import BaseMaterial +from threestudio.utils.ops import dot, get_activation +from threestudio.utils.typing import * + + +@threestudio.register("diffuse-with-point-light-material") +class DiffuseWithPointLightMaterial(BaseMaterial): + @dataclass + class Config(BaseMaterial.Config): + ambient_light_color: Tuple[float, float, float] = (0.1, 0.1, 0.1) + diffuse_light_color: Tuple[float, float, float] = (0.9, 0.9, 0.9) + ambient_only_steps: int = 1000 + diffuse_prob: float = 0.75 + textureless_prob: float = 0.5 + albedo_activation: str = "sigmoid" + soft_shading: bool = False + + cfg: Config + + def configure(self) -> None: + self.requires_normal = True + + self.ambient_light_color: Float[Tensor, "3"] + self.register_buffer( + "ambient_light_color", + torch.as_tensor(self.cfg.ambient_light_color, dtype=torch.float32), + ) + self.diffuse_light_color: Float[Tensor, "3"] + self.register_buffer( + "diffuse_light_color", + torch.as_tensor(self.cfg.diffuse_light_color, dtype=torch.float32), + ) + self.ambient_only = False + + def forward( + self, + features: Float[Tensor, "B ... Nf"], + positions: Float[Tensor, "B ... 3"], + shading_normal: Float[Tensor, "B ... 3"], + light_positions: Float[Tensor, "B ... 3"], + ambient_ratio: Optional[float] = None, + shading: Optional[str] = None, + **kwargs, + ) -> Float[Tensor, "B ... 3"]: + albedo = get_activation(self.cfg.albedo_activation)(features[..., :3]) + + if ambient_ratio is not None: + # if ambient ratio is specified, use it + diffuse_light_color = (1 - ambient_ratio) * torch.ones_like( + self.diffuse_light_color + ) + ambient_light_color = ambient_ratio * torch.ones_like( + self.ambient_light_color + ) + elif self.training and self.cfg.soft_shading: + # otherwise if in training and soft shading is enabled, random a ambient ratio + diffuse_light_color = torch.full_like( + self.diffuse_light_color, random.random() + ) + ambient_light_color = 1.0 - diffuse_light_color + else: + # otherwise use the default fixed values + diffuse_light_color = self.diffuse_light_color + ambient_light_color = self.ambient_light_color + + light_directions: Float[Tensor, "B ... 3"] = F.normalize( + light_positions - positions, dim=-1 + ) + diffuse_light: Float[Tensor, "B ... 3"] = ( + dot(shading_normal, light_directions).clamp(min=0.0) * diffuse_light_color + ) + textureless_color = diffuse_light + ambient_light_color + # clamp albedo to [0, 1] to compute shading + color = albedo.clamp(0.0, 1.0) * textureless_color + + if shading is None: + if self.training: + # adopt the same type of augmentation for the whole batch + if self.ambient_only or random.random() > self.cfg.diffuse_prob: + shading = "albedo" + elif random.random() < self.cfg.textureless_prob: + shading = "textureless" + else: + shading = "diffuse" + else: + if self.ambient_only: + shading = "albedo" + else: + # return shaded color by default in evaluation + shading = "diffuse" + + # multiply by 0 to prevent checking for unused parameters in DDP + if shading == "albedo": + return albedo + textureless_color * 0 + elif shading == "textureless": + return albedo * 0 + textureless_color + elif shading == "diffuse": + return color + else: + raise ValueError(f"Unknown shading type {shading}") + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + if global_step < self.cfg.ambient_only_steps: + self.ambient_only = True + else: + self.ambient_only = False + + def export(self, features: Float[Tensor, "*N Nf"], **kwargs) -> Dict[str, Any]: + albedo = get_activation(self.cfg.albedo_activation)(features[..., :3]).clamp( + 0.0, 1.0 + ) + return {"albedo": albedo} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/hybrid_rgb_latent_material.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/hybrid_rgb_latent_material.py new file mode 100644 index 0000000000000000000000000000000000000000..f5f2c5193678151506db30d0ecb04c68228f0d4a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/hybrid_rgb_latent_material.py @@ -0,0 +1,36 @@ +import random +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.networks import get_encoding, get_mlp +from threestudio.utils.ops import dot, get_activation +from threestudio.utils.typing import * + + +@threestudio.register("hybrid-rgb-latent-material") +class HybridRGBLatentMaterial(BaseMaterial): + @dataclass + class Config(BaseMaterial.Config): + n_output_dims: int = 3 + color_activation: str = "sigmoid" + requires_normal: bool = True + + cfg: Config + + def configure(self) -> None: + self.requires_normal = self.cfg.requires_normal + + def forward( + self, features: Float[Tensor, "B ... Nf"], **kwargs + ) -> Float[Tensor, "B ... Nc"]: + assert ( + features.shape[-1] == self.cfg.n_output_dims + ), f"Expected {self.cfg.n_output_dims} output dims, only got {features.shape[-1]} dims input." + color = features + color[..., :3] = get_activation(self.cfg.color_activation)(color[..., :3]) + return color diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/neural_radiance_material.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/neural_radiance_material.py new file mode 100644 index 0000000000000000000000000000000000000000..c9dcc50ffb394e0c883b79d091c0de3ffeaa4718 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/neural_radiance_material.py @@ -0,0 +1,54 @@ +import random +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.networks import get_encoding, get_mlp +from threestudio.utils.ops import dot, get_activation +from threestudio.utils.typing import * + + +@threestudio.register("neural-radiance-material") +class NeuralRadianceMaterial(BaseMaterial): + @dataclass + class Config(BaseMaterial.Config): + input_feature_dims: int = 8 + color_activation: str = "sigmoid" + dir_encoding_config: dict = field( + default_factory=lambda: {"otype": "SphericalHarmonics", "degree": 3} + ) + mlp_network_config: dict = field( + default_factory=lambda: { + "otype": "FullyFusedMLP", + "activation": "ReLU", + "n_neurons": 16, + "n_hidden_layers": 2, + } + ) + + cfg: Config + + def configure(self) -> None: + self.encoding = get_encoding(3, self.cfg.dir_encoding_config) + self.n_input_dims = self.cfg.input_feature_dims + self.encoding.n_output_dims # type: ignore + self.network = get_mlp(self.n_input_dims, 3, self.cfg.mlp_network_config) + + def forward( + self, + features: Float[Tensor, "*B Nf"], + viewdirs: Float[Tensor, "*B 3"], + **kwargs, + ) -> Float[Tensor, "*B 3"]: + # viewdirs and normals must be normalized before passing to this function + viewdirs = (viewdirs + 1.0) / 2.0 # (-1, 1) => (0, 1) + viewdirs_embd = self.encoding(viewdirs.view(-1, 3)) + network_inp = torch.cat( + [features.view(-1, features.shape[-1]), viewdirs_embd], dim=-1 + ) + color = self.network(network_inp).view(*features.shape[:-1], 3) + color = get_activation(self.cfg.color_activation)(color) + return color diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/no_material.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/no_material.py new file mode 100644 index 0000000000000000000000000000000000000000..402a9518753ba93b78a43a688ff62099a05e15d3 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/no_material.py @@ -0,0 +1,63 @@ +import random +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.networks import get_encoding, get_mlp +from threestudio.utils.ops import dot, get_activation +from threestudio.utils.typing import * + + +@threestudio.register("no-material") +class NoMaterial(BaseMaterial): + @dataclass + class Config(BaseMaterial.Config): + n_output_dims: int = 3 + color_activation: str = "sigmoid" + input_feature_dims: Optional[int] = None + mlp_network_config: Optional[dict] = None + requires_normal: bool = False + + cfg: Config + + def configure(self) -> None: + self.use_network = False + if ( + self.cfg.input_feature_dims is not None + and self.cfg.mlp_network_config is not None + ): + self.network = get_mlp( + self.cfg.input_feature_dims, + self.cfg.n_output_dims, + self.cfg.mlp_network_config, + ) + self.use_network = True + self.requires_normal = self.cfg.requires_normal + + def forward( + self, features: Float[Tensor, "B ... Nf"], **kwargs + ) -> Float[Tensor, "B ... Nc"]: + if not self.use_network: + assert ( + features.shape[-1] == self.cfg.n_output_dims + ), f"Expected {self.cfg.n_output_dims} output dims, only got {features.shape[-1]} dims input." + color = get_activation(self.cfg.color_activation)(features) + else: + color = self.network(features.view(-1, features.shape[-1])).view( + *features.shape[:-1], self.cfg.n_output_dims + ) + color = get_activation(self.cfg.color_activation)(color) + return color + + def export(self, features: Float[Tensor, "*N Nf"], **kwargs) -> Dict[str, Any]: + color = self(features, **kwargs).clamp(0, 1) + assert color.shape[-1] >= 3, "Output color must have at least 3 channels" + if color.shape[-1] > 3: + threestudio.warn( + "Output color has >3 channels, treating the first 3 as RGB" + ) + return {"albedo": color[..., :3]} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/pbr_material.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/pbr_material.py new file mode 100644 index 0000000000000000000000000000000000000000..c81f67bf980778214e2a9c3afd1bff2d7caba304 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/pbr_material.py @@ -0,0 +1,143 @@ +import random +from dataclasses import dataclass, field + +import envlight +import numpy as np +import nvdiffrast.torch as dr +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.materials.base import BaseMaterial +from threestudio.utils.ops import get_activation +from threestudio.utils.typing import * + + +@threestudio.register("pbr-material") +class PBRMaterial(BaseMaterial): + @dataclass + class Config(BaseMaterial.Config): + material_activation: str = "sigmoid" + environment_texture: str = "load/lights/mud_road_puresky_1k.hdr" + environment_scale: float = 2.0 + min_metallic: float = 0.0 + max_metallic: float = 0.9 + min_roughness: float = 0.08 + max_roughness: float = 0.9 + use_bump: bool = True + + cfg: Config + + def configure(self) -> None: + self.requires_normal = True + self.requires_tangent = self.cfg.use_bump + + self.light = envlight.EnvLight( + self.cfg.environment_texture, scale=self.cfg.environment_scale + ) + + FG_LUT = torch.from_numpy( + np.fromfile("load/lights/bsdf_256_256.bin", dtype=np.float32).reshape( + 1, 256, 256, 2 + ) + ) + self.register_buffer("FG_LUT", FG_LUT) + + def forward( + self, + features: Float[Tensor, "*B Nf"], + viewdirs: Float[Tensor, "*B 3"], + shading_normal: Float[Tensor, "B ... 3"], + tangent: Optional[Float[Tensor, "B ... 3"]] = None, + **kwargs, + ) -> Float[Tensor, "*B 3"]: + prefix_shape = features.shape[:-1] + + material: Float[Tensor, "*B Nf"] = get_activation(self.cfg.material_activation)( + features + ) + albedo = material[..., :3] + metallic = ( + material[..., 3:4] * (self.cfg.max_metallic - self.cfg.min_metallic) + + self.cfg.min_metallic + ) + roughness = ( + material[..., 4:5] * (self.cfg.max_roughness - self.cfg.min_roughness) + + self.cfg.min_roughness + ) + + if self.cfg.use_bump: + assert tangent is not None + # perturb_normal is a delta to the initialization [0, 0, 1] + perturb_normal = (material[..., 5:8] * 2 - 1) + torch.tensor( + [0, 0, 1], dtype=material.dtype, device=material.device + ) + perturb_normal = F.normalize(perturb_normal.clamp(-1, 1), dim=-1) + + # apply normal perturbation in tangent space + bitangent = F.normalize(torch.cross(tangent, shading_normal), dim=-1) + shading_normal = ( + tangent * perturb_normal[..., 0:1] + - bitangent * perturb_normal[..., 1:2] + + shading_normal * perturb_normal[..., 2:3] + ) + shading_normal = F.normalize(shading_normal, dim=-1) + + v = -viewdirs + n_dot_v = (shading_normal * v).sum(-1, keepdim=True) + reflective = n_dot_v * shading_normal * 2 - v + + diffuse_albedo = (1 - metallic) * albedo + + fg_uv = torch.cat([n_dot_v, roughness], -1).clamp(0, 1) + fg = dr.texture( + self.FG_LUT, + fg_uv.reshape(1, -1, 1, 2).contiguous(), + filter_mode="linear", + boundary_mode="clamp", + ).reshape(*prefix_shape, 2) + F0 = (1 - metallic) * 0.04 + metallic * albedo + specular_albedo = F0 * fg[:, 0:1] + fg[:, 1:2] + + diffuse_light = self.light(shading_normal) + specular_light = self.light(reflective, roughness) + + color = diffuse_albedo * diffuse_light + specular_albedo * specular_light + color = color.clamp(0.0, 1.0) + + return color + + def export(self, features: Float[Tensor, "*N Nf"], **kwargs) -> Dict[str, Any]: + material: Float[Tensor, "*N Nf"] = get_activation(self.cfg.material_activation)( + features + ) + albedo = material[..., :3] + metallic = ( + material[..., 3:4] * (self.cfg.max_metallic - self.cfg.min_metallic) + + self.cfg.min_metallic + ) + roughness = ( + material[..., 4:5] * (self.cfg.max_roughness - self.cfg.min_roughness) + + self.cfg.min_roughness + ) + + out = { + "albedo": albedo, + "metallic": metallic, + "roughness": roughness, + } + + if self.cfg.use_bump: + perturb_normal = (material[..., 5:8] * 2 - 1) + torch.tensor( + [0, 0, 1], dtype=material.dtype, device=material.device + ) + perturb_normal = F.normalize(perturb_normal.clamp(-1, 1), dim=-1) + perturb_normal = (perturb_normal + 1) / 2 + out.update( + { + "bump": perturb_normal, + } + ) + + return out diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/sd_latent_adapter_material.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/sd_latent_adapter_material.py new file mode 100644 index 0000000000000000000000000000000000000000..046cabb68b02f3da39b55412e31198846f4115bb --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/materials/sd_latent_adapter_material.py @@ -0,0 +1,42 @@ +import random +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.materials.base import BaseMaterial +from threestudio.utils.typing import * + + +@threestudio.register("sd-latent-adapter-material") +class StableDiffusionLatentAdapterMaterial(BaseMaterial): + @dataclass + class Config(BaseMaterial.Config): + pass + + cfg: Config + + def configure(self) -> None: + adapter = nn.Parameter( + torch.as_tensor( + [ + # R G B + [0.298, 0.207, 0.208], # L1 + [0.187, 0.286, 0.173], # L2 + [-0.158, 0.189, 0.264], # L3 + [-0.184, -0.271, -0.473], # L4 + ] + ) + ) + self.register_parameter("adapter", adapter) + + def forward( + self, features: Float[Tensor, "B ... 4"], **kwargs + ) -> Float[Tensor, "B ... 3"]: + assert features.shape[-1] == 4 + color = features @ self.adapter + color = (color + 1) / 2 + color = color.clamp(0.0, 1.0) + return color diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/mesh.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/mesh.py new file mode 100644 index 0000000000000000000000000000000000000000..232fa7c9517881b679944c0071667af0b5a00d7f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/mesh.py @@ -0,0 +1,309 @@ +from __future__ import annotations + +import numpy as np +import torch +import torch.nn.functional as F + +import threestudio +from threestudio.utils.ops import dot +from threestudio.utils.typing import * + + +class Mesh: + def __init__( + self, v_pos: Float[Tensor, "Nv 3"], t_pos_idx: Integer[Tensor, "Nf 3"], **kwargs + ) -> None: + self.v_pos: Float[Tensor, "Nv 3"] = v_pos + self.t_pos_idx: Integer[Tensor, "Nf 3"] = t_pos_idx + self._v_nrm: Optional[Float[Tensor, "Nv 3"]] = None + self._v_tng: Optional[Float[Tensor, "Nv 3"]] = None + self._v_tex: Optional[Float[Tensor, "Nt 3"]] = None + self._t_tex_idx: Optional[Float[Tensor, "Nf 3"]] = None + self._v_rgb: Optional[Float[Tensor, "Nv 3"]] = None + self._edges: Optional[Integer[Tensor, "Ne 2"]] = None + self.extras: Dict[str, Any] = {} + for k, v in kwargs.items(): + self.add_extra(k, v) + + def add_extra(self, k, v) -> None: + self.extras[k] = v + + def remove_outlier(self, outlier_n_faces_threshold: Union[int, float]) -> Mesh: + if self.requires_grad: + threestudio.debug("Mesh is differentiable, not removing outliers") + return self + + # use trimesh to first split the mesh into connected components + # then remove the components with less than n_face_threshold faces + import trimesh + + # construct a trimesh object + mesh = trimesh.Trimesh( + vertices=self.v_pos.detach().cpu().numpy(), + faces=self.t_pos_idx.detach().cpu().numpy(), + ) + + # split the mesh into connected components + components = mesh.split(only_watertight=False) + # log the number of faces in each component + threestudio.debug( + "Mesh has {} components, with faces: {}".format( + len(components), [c.faces.shape[0] for c in components] + ) + ) + + n_faces_threshold: int + if isinstance(outlier_n_faces_threshold, float): + # set the threshold to the number of faces in the largest component multiplied by outlier_n_faces_threshold + n_faces_threshold = int( + max([c.faces.shape[0] for c in components]) * outlier_n_faces_threshold + ) + else: + # set the threshold directly to outlier_n_faces_threshold + n_faces_threshold = outlier_n_faces_threshold + + # log the threshold + threestudio.debug( + "Removing components with less than {} faces".format(n_faces_threshold) + ) + + # remove the components with less than n_face_threshold faces + components = [c for c in components if c.faces.shape[0] >= n_faces_threshold] + + # log the number of faces in each component after removing outliers + threestudio.debug( + "Mesh has {} components after removing outliers, with faces: {}".format( + len(components), [c.faces.shape[0] for c in components] + ) + ) + # merge the components + mesh = trimesh.util.concatenate(components) + + # convert back to our mesh format + v_pos = torch.from_numpy(mesh.vertices).to(self.v_pos) + t_pos_idx = torch.from_numpy(mesh.faces).to(self.t_pos_idx) + + clean_mesh = Mesh(v_pos, t_pos_idx) + # keep the extras unchanged + + if len(self.extras) > 0: + clean_mesh.extras = self.extras + threestudio.debug( + f"The following extra attributes are inherited from the original mesh unchanged: {list(self.extras.keys())}" + ) + return clean_mesh + + @property + def requires_grad(self): + return self.v_pos.requires_grad + + @property + def v_nrm(self): + if self._v_nrm is None: + self._v_nrm = self._compute_vertex_normal() + return self._v_nrm + + @property + def v_tng(self): + if self._v_tng is None: + self._v_tng = self._compute_vertex_tangent() + return self._v_tng + + @property + def v_tex(self): + if self._v_tex is None: + self._v_tex, self._t_tex_idx = self._unwrap_uv() + return self._v_tex + + @property + def t_tex_idx(self): + if self._t_tex_idx is None: + self._v_tex, self._t_tex_idx = self._unwrap_uv() + return self._t_tex_idx + + @property + def v_rgb(self): + return self._v_rgb + + @property + def edges(self): + if self._edges is None: + self._edges = self._compute_edges() + return self._edges + + def _compute_vertex_normal(self): + i0 = self.t_pos_idx[:, 0] + i1 = self.t_pos_idx[:, 1] + i2 = self.t_pos_idx[:, 2] + + v0 = self.v_pos[i0, :] + v1 = self.v_pos[i1, :] + v2 = self.v_pos[i2, :] + + face_normals = torch.cross(v1 - v0, v2 - v0) + + # Splat face normals to vertices + v_nrm = torch.zeros_like(self.v_pos) + v_nrm.scatter_add_(0, i0[:, None].repeat(1, 3), face_normals) + v_nrm.scatter_add_(0, i1[:, None].repeat(1, 3), face_normals) + v_nrm.scatter_add_(0, i2[:, None].repeat(1, 3), face_normals) + + # Normalize, replace zero (degenerated) normals with some default value + v_nrm = torch.where( + dot(v_nrm, v_nrm) > 1e-20, v_nrm, torch.as_tensor([0.0, 0.0, 1.0]).to(v_nrm) + ) + v_nrm = F.normalize(v_nrm, dim=1) + + if torch.is_anomaly_enabled(): + assert torch.all(torch.isfinite(v_nrm)) + + return v_nrm + + def _compute_vertex_tangent(self): + vn_idx = [None] * 3 + pos = [None] * 3 + tex = [None] * 3 + for i in range(0, 3): + pos[i] = self.v_pos[self.t_pos_idx[:, i]] + tex[i] = self.v_tex[self.t_tex_idx[:, i]] + # t_nrm_idx is always the same as t_pos_idx + vn_idx[i] = self.t_pos_idx[:, i] + + tangents = torch.zeros_like(self.v_nrm) + tansum = torch.zeros_like(self.v_nrm) + + # Compute tangent space for each triangle + uve1 = tex[1] - tex[0] + uve2 = tex[2] - tex[0] + pe1 = pos[1] - pos[0] + pe2 = pos[2] - pos[0] + + nom = pe1 * uve2[..., 1:2] - pe2 * uve1[..., 1:2] + denom = uve1[..., 0:1] * uve2[..., 1:2] - uve1[..., 1:2] * uve2[..., 0:1] + + # Avoid division by zero for degenerated texture coordinates + tang = nom / torch.where( + denom > 0.0, torch.clamp(denom, min=1e-6), torch.clamp(denom, max=-1e-6) + ) + + # Update all 3 vertices + for i in range(0, 3): + idx = vn_idx[i][:, None].repeat(1, 3) + tangents.scatter_add_(0, idx, tang) # tangents[n_i] = tangents[n_i] + tang + tansum.scatter_add_( + 0, idx, torch.ones_like(tang) + ) # tansum[n_i] = tansum[n_i] + 1 + tangents = tangents / tansum + + # Normalize and make sure tangent is perpendicular to normal + tangents = F.normalize(tangents, dim=1) + tangents = F.normalize(tangents - dot(tangents, self.v_nrm) * self.v_nrm) + + if torch.is_anomaly_enabled(): + assert torch.all(torch.isfinite(tangents)) + + return tangents + + def _unwrap_uv( + self, xatlas_chart_options: dict = {}, xatlas_pack_options: dict = {} + ): + threestudio.info("Using xatlas to perform UV unwrapping, may take a while ...") + + import xatlas + + atlas = xatlas.Atlas() + atlas.add_mesh( + self.v_pos.detach().cpu().numpy(), + self.t_pos_idx.cpu().numpy(), + ) + co = xatlas.ChartOptions() + po = xatlas.PackOptions() + for k, v in xatlas_chart_options.items(): + setattr(co, k, v) + for k, v in xatlas_pack_options.items(): + setattr(po, k, v) + atlas.generate(co, po) + vmapping, indices, uvs = atlas.get_mesh(0) + vmapping = ( + torch.from_numpy( + vmapping.astype(np.uint64, casting="same_kind").view(np.int64) + ) + .to(self.v_pos.device) + .long() + ) + uvs = torch.from_numpy(uvs).to(self.v_pos.device).float() + indices = ( + torch.from_numpy( + indices.astype(np.uint64, casting="same_kind").view(np.int64) + ) + .to(self.v_pos.device) + .long() + ) + return uvs, indices + + def unwrap_uv( + self, xatlas_chart_options: dict = {}, xatlas_pack_options: dict = {} + ): + self._v_tex, self._t_tex_idx = self._unwrap_uv( + xatlas_chart_options, xatlas_pack_options + ) + + def set_vertex_color(self, v_rgb): + assert v_rgb.shape[0] == self.v_pos.shape[0] + self._v_rgb = v_rgb + + def _compute_edges(self): + # Compute edges + edges = torch.cat( + [ + self.t_pos_idx[:, [0, 1]], + self.t_pos_idx[:, [1, 2]], + self.t_pos_idx[:, [2, 0]], + ], + dim=0, + ) + edges = edges.sort()[0] + edges = torch.unique(edges, dim=0) + return edges + + def normal_consistency(self) -> Float[Tensor, ""]: + edge_nrm: Float[Tensor, "Ne 2 3"] = self.v_nrm[self.edges] + nc = ( + 1.0 - torch.cosine_similarity(edge_nrm[:, 0], edge_nrm[:, 1], dim=-1) + ).mean() + return nc + + def _laplacian_uniform(self): + # from stable-dreamfusion + # https://github.com/ashawkey/stable-dreamfusion/blob/8fb3613e9e4cd1ded1066b46e80ca801dfb9fd06/nerf/renderer.py#L224 + verts, faces = self.v_pos, self.t_pos_idx + + V = verts.shape[0] + F = faces.shape[0] + + # Neighbor indices + ii = faces[:, [1, 2, 0]].flatten() + jj = faces[:, [2, 0, 1]].flatten() + adj = torch.stack([torch.cat([ii, jj]), torch.cat([jj, ii])], dim=0).unique( + dim=1 + ) + adj_values = torch.ones(adj.shape[1]).to(verts) + + # Diagonal indices + diag_idx = adj[0] + + # Build the sparse matrix + idx = torch.cat((adj, torch.stack((diag_idx, diag_idx), dim=0)), dim=1) + values = torch.cat((-adj_values, adj_values)) + + # The coalesce operation sums the duplicate indices, resulting in the + # correct diagonal + return torch.sparse_coo_tensor(idx, values, (V, V)).coalesce() + + def laplacian(self) -> Float[Tensor, ""]: + with torch.no_grad(): + L = self._laplacian_uniform() + loss = L.mm(self.v_pos) + loss = loss.norm(dim=1) + loss = loss.mean() + return loss diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/networks.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/networks.py new file mode 100644 index 0000000000000000000000000000000000000000..a1124102e6cd597a6c54e2500c765dfc82d383a4 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/networks.py @@ -0,0 +1,411 @@ +import math + +# import tinycudann as tcnn +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.utils.base import Updateable +from threestudio.utils.config import config_to_primitive +from threestudio.utils.misc import get_rank +from threestudio.utils.ops import get_activation +from threestudio.utils.typing import * + + +class ProgressiveBandFrequency(nn.Module, Updateable): + def __init__(self, in_channels: int, config: dict): + super().__init__() + self.N_freqs = config["n_frequencies"] + self.in_channels, self.n_input_dims = in_channels, in_channels + self.funcs = [torch.sin, torch.cos] + self.freq_bands = 2 ** torch.linspace(0, self.N_freqs - 1, self.N_freqs) + self.n_output_dims = self.in_channels * (len(self.funcs) * self.N_freqs) + self.n_masking_step = config.get("n_masking_step", 0) + self.update_step( + None, None + ) # mask should be updated at the beginning each step + + def forward(self, x): + out = [] + for freq, mask in zip(self.freq_bands, self.mask): + for func in self.funcs: + out += [func(freq * x) * mask] + return torch.cat(out, -1) + + def update_step(self, epoch, global_step, on_load_weights=False): + if self.n_masking_step <= 0 or global_step is None: + self.mask = torch.ones(self.N_freqs, dtype=torch.float32) + else: + self.mask = ( + 1.0 + - torch.cos( + math.pi + * ( + global_step / self.n_masking_step * self.N_freqs + - torch.arange(0, self.N_freqs) + ).clamp(0, 1) + ) + ) / 2.0 + threestudio.debug( + f"Update mask: {global_step}/{self.n_masking_step} {self.mask}" + ) + + +class TCNNEncoding(nn.Module): + def __init__(self, in_channels, config, dtype=torch.float32) -> None: + super().__init__() + self.n_input_dims = in_channels + with torch.cuda.device(get_rank()): + self.encoding = tcnn.Encoding(in_channels, config, dtype=dtype) + self.n_output_dims = self.encoding.n_output_dims + + def forward(self, x): + return self.encoding(x) + + +# 4D implicit decomposition of space and time (4D-fy) +class TCNNEncodingSpatialTime(nn.Module): + def __init__( + self, in_channels, config, dtype=torch.float32, init_time_zero=False + ) -> None: + super().__init__() + self.n_input_dims = in_channels + config["otype"] = "HashGrid" + self.num_frames = 1 # config["num_frames"] + self.static = config["static"] + self.cfg = config_to_primitive(config) + self.cfg_time = self.cfg + self.n_key_frames = config.get("n_key_frames", 1) + with torch.cuda.device(get_rank()): + self.encoding = tcnn.Encoding(self.n_input_dims, self.cfg, dtype=dtype) + self.encoding_time = tcnn.Encoding( + self.n_input_dims + 1, self.cfg_time, dtype=dtype + ) + self.n_output_dims = self.encoding.n_output_dims + self.frame_time = None + if self.static: + self.set_temp_param_grad(requires_grad=False) + self.use_key_frame = config.get("use_key_frame", False) + self.is_video = True + self.update_occ_grid = False + + def set_temp_param_grad(self, requires_grad=False): + self.set_param_grad(self.encoding_time, requires_grad=requires_grad) + + def set_param_grad(self, param_list, requires_grad=False): + if isinstance(param_list, nn.Parameter): + param_list.requires_grad = requires_grad + else: + for param in param_list.parameters(): + param.requires_grad = requires_grad + + def forward(self, x): + # TODO frame_time only supports batch_size == 1 cases + if self.update_occ_grid and not isinstance(self.frame_time, float): + frame_time = self.frame_time + else: + if (self.static or not self.training) and self.frame_time is None: + frame_time = torch.zeros( + (self.num_frames, 1), device=x.device, dtype=x.dtype + ).expand(x.shape[0], 1) + else: + if self.frame_time is None: + frame_time = 0.0 + else: + frame_time = self.frame_time + frame_time = ( + torch.ones((self.num_frames, 1), device=x.device, dtype=x.dtype) + * frame_time + ).expand(x.shape[0], 1) + frame_time = frame_time.view(-1, 1) + enc_space = self.encoding(x) + x_frame_time = torch.cat((x, frame_time), 1) + enc_space_time = self.encoding_time(x_frame_time) + enc = enc_space + enc_space_time + return enc + + +class ProgressiveBandHashGrid(nn.Module, Updateable): + def __init__(self, in_channels, config, dtype=torch.float32): + super().__init__() + self.n_input_dims = in_channels + encoding_config = config.copy() + encoding_config["otype"] = "Grid" + encoding_config["type"] = "Hash" + with torch.cuda.device(get_rank()): + self.encoding = tcnn.Encoding(in_channels, encoding_config, dtype=dtype) + self.n_output_dims = self.encoding.n_output_dims + self.n_level = config["n_levels"] + self.n_features_per_level = config["n_features_per_level"] + self.start_level, self.start_step, self.update_steps = ( + config["start_level"], + config["start_step"], + config["update_steps"], + ) + self.current_level = self.start_level + self.mask = torch.zeros( + self.n_level * self.n_features_per_level, + dtype=torch.float32, + device=get_rank(), + ) + + def forward(self, x): + enc = self.encoding(x) + enc = enc * self.mask + return enc + + def update_step(self, epoch, global_step, on_load_weights=False): + current_level = min( + self.start_level + + max(global_step - self.start_step, 0) // self.update_steps, + self.n_level, + ) + if current_level > self.current_level: + threestudio.debug(f"Update current level to {current_level}") + self.current_level = current_level + self.mask[: self.current_level * self.n_features_per_level] = 1.0 + + +class CompositeEncoding(nn.Module, Updateable): + def __init__(self, encoding, include_xyz=False, xyz_scale=2.0, xyz_offset=-1.0): + super(CompositeEncoding, self).__init__() + self.encoding = encoding + self.include_xyz, self.xyz_scale, self.xyz_offset = ( + include_xyz, + xyz_scale, + xyz_offset, + ) + self.n_output_dims = ( + int(self.include_xyz) * self.encoding.n_input_dims + + self.encoding.n_output_dims + ) + + def forward(self, x, *args): + return ( + self.encoding(x, *args) + if not self.include_xyz + else torch.cat( + [x * self.xyz_scale + self.xyz_offset, self.encoding(x, *args)], dim=-1 + ) + ) + + +def get_encoding(n_input_dims: int, config) -> nn.Module: + # input suppose to be range [0, 1] + encoding: nn.Module + if config.otype == "ProgressiveBandFrequency": + encoding = ProgressiveBandFrequency(n_input_dims, config_to_primitive(config)) + elif config.otype == "ProgressiveBandHashGrid": + encoding = ProgressiveBandHashGrid(n_input_dims, config_to_primitive(config)) + elif config.otype == "HashGridSpatialTime": + encoding = TCNNEncodingSpatialTime(n_input_dims, config) # 4D-fy encoding + else: + encoding = TCNNEncoding(n_input_dims, config_to_primitive(config)) + encoding = CompositeEncoding( + encoding, + include_xyz=config.get("include_xyz", False), + xyz_scale=2.0, + xyz_offset=-1.0, + ) # FIXME: hard coded + return encoding + + +class VanillaMLP(nn.Module): + def __init__(self, dim_in: int, dim_out: int, config: dict): + super().__init__() + self.n_neurons, self.n_hidden_layers = ( + config["n_neurons"], + config["n_hidden_layers"], + ) + layers = [ + self.make_linear(dim_in, self.n_neurons, is_first=True, is_last=False), + self.make_activation(), + ] + for i in range(self.n_hidden_layers - 1): + layers += [ + self.make_linear( + self.n_neurons, self.n_neurons, is_first=False, is_last=False + ), + self.make_activation(), + ] + layers += [ + self.make_linear(self.n_neurons, dim_out, is_first=False, is_last=True) + ] + self.layers = nn.Sequential(*layers) + self.output_activation = get_activation(config.get("output_activation", None)) + + def forward(self, x): + # disable autocast + # strange that the parameters will have empty gradients if autocast is enabled in AMP + with torch.cuda.amp.autocast(enabled=False): + x = self.layers(x) + x = self.output_activation(x) + return x + + def make_linear(self, dim_in, dim_out, is_first, is_last): + layer = nn.Linear(dim_in, dim_out, bias=False) + return layer + + def make_activation(self): + return nn.ReLU(inplace=True) + + +class SphereInitVanillaMLP(nn.Module): + def __init__(self, dim_in, dim_out, config): + super().__init__() + self.n_neurons, self.n_hidden_layers = ( + config["n_neurons"], + config["n_hidden_layers"], + ) + self.sphere_init, self.weight_norm = True, True + self.sphere_init_radius = config["sphere_init_radius"] + self.sphere_init_inside_out = config["inside_out"] + + self.layers = [ + self.make_linear(dim_in, self.n_neurons, is_first=True, is_last=False), + self.make_activation(), + ] + for i in range(self.n_hidden_layers - 1): + self.layers += [ + self.make_linear( + self.n_neurons, self.n_neurons, is_first=False, is_last=False + ), + self.make_activation(), + ] + self.layers += [ + self.make_linear(self.n_neurons, dim_out, is_first=False, is_last=True) + ] + self.layers = nn.Sequential(*self.layers) + self.output_activation = get_activation(config.get("output_activation", None)) + + def forward(self, x): + # disable autocast + # strange that the parameters will have empty gradients if autocast is enabled in AMP + with torch.cuda.amp.autocast(enabled=False): + x = self.layers(x) + x = self.output_activation(x) + return x + + def make_linear(self, dim_in, dim_out, is_first, is_last): + layer = nn.Linear(dim_in, dim_out, bias=True) + + if is_last: + if not self.sphere_init_inside_out: + torch.nn.init.constant_(layer.bias, -self.sphere_init_radius) + torch.nn.init.normal_( + layer.weight, + mean=math.sqrt(math.pi) / math.sqrt(dim_in), + std=0.0001, + ) + else: + torch.nn.init.constant_(layer.bias, self.sphere_init_radius) + torch.nn.init.normal_( + layer.weight, + mean=-math.sqrt(math.pi) / math.sqrt(dim_in), + std=0.0001, + ) + elif is_first: + torch.nn.init.constant_(layer.bias, 0.0) + torch.nn.init.constant_(layer.weight[:, 3:], 0.0) + torch.nn.init.normal_( + layer.weight[:, :3], 0.0, math.sqrt(2) / math.sqrt(dim_out) + ) + else: + torch.nn.init.constant_(layer.bias, 0.0) + torch.nn.init.normal_(layer.weight, 0.0, math.sqrt(2) / math.sqrt(dim_out)) + + if self.weight_norm: + layer = nn.utils.weight_norm(layer) + return layer + + def make_activation(self): + return nn.Softplus(beta=100) + + +class TCNNNetwork(nn.Module): + def __init__(self, dim_in: int, dim_out: int, config: dict) -> None: + super().__init__() + with torch.cuda.device(get_rank()): + self.network = tcnn.Network(dim_in, dim_out, config) + + def forward(self, x): + return self.network(x).float() # transform to float32 + + +def get_mlp(n_input_dims, n_output_dims, config) -> nn.Module: + network: nn.Module + if config.otype == "VanillaMLP": + network = VanillaMLP(n_input_dims, n_output_dims, config_to_primitive(config)) + elif config.otype == "SphereInitVanillaMLP": + network = SphereInitVanillaMLP( + n_input_dims, n_output_dims, config_to_primitive(config) + ) + else: + assert ( + config.get("sphere_init", False) is False + ), "sphere_init=True only supported by VanillaMLP" + network = TCNNNetwork(n_input_dims, n_output_dims, config_to_primitive(config)) + return network + + +class NetworkWithInputEncoding(nn.Module, Updateable): + def __init__(self, encoding, network): + super().__init__() + self.encoding, self.network = encoding, network + + def forward(self, x): + return self.network(self.encoding(x)) + + +class TCNNNetworkWithInputEncoding(nn.Module): + def __init__( + self, + n_input_dims: int, + n_output_dims: int, + encoding_config: dict, + network_config: dict, + ) -> None: + super().__init__() + with torch.cuda.device(get_rank()): + self.network_with_input_encoding = tcnn.NetworkWithInputEncoding( + n_input_dims=n_input_dims, + n_output_dims=n_output_dims, + encoding_config=encoding_config, + network_config=network_config, + ) + + def forward(self, x): + return self.network_with_input_encoding(x).float() # transform to float32 + + +def create_network_with_input_encoding( + n_input_dims: int, n_output_dims: int, encoding_config, network_config +) -> nn.Module: + # input suppose to be range [0, 1] + network_with_input_encoding: nn.Module + if encoding_config.otype in [ + "VanillaFrequency", + "ProgressiveBandHashGrid", + ] or network_config.otype in ["VanillaMLP", "SphereInitVanillaMLP"]: + encoding = get_encoding(n_input_dims, encoding_config) + network = get_mlp(encoding.n_output_dims, n_output_dims, network_config) + network_with_input_encoding = NetworkWithInputEncoding(encoding, network) + else: + network_with_input_encoding = TCNNNetworkWithInputEncoding( + n_input_dims=n_input_dims, + n_output_dims=n_output_dims, + encoding_config=config_to_primitive(encoding_config), + network_config=config_to_primitive(network_config), + ) + return network_with_input_encoding + + +class ToDTypeWrapper(nn.Module): + def __init__(self, module: nn.Module, dtype: torch.dtype): + super().__init__() + self.module = module + self.dtype = dtype + + def forward(self, x: Float[Tensor, "..."]) -> Float[Tensor, "..."]: + return self.module(x).to(self.dtype) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..eb45c70acda72d3c6759a63dd11eaba864412bd3 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/__init__.py @@ -0,0 +1,6 @@ +from . import ( + base, + deepfloyd_prompt_processor, + dummy_prompt_processor, + stable_diffusion_prompt_processor, +) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/base.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/base.py new file mode 100644 index 0000000000000000000000000000000000000000..9a402daf5cada7eaa5004b4279a55829b33642e6 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/base.py @@ -0,0 +1,523 @@ +import json +import os +from dataclasses import dataclass, field + +import torch +import torch.multiprocessing as mp +import torch.nn as nn +import torch.nn.functional as F +from pytorch_lightning.utilities.rank_zero import rank_zero_only +from transformers import AutoTokenizer, BertForMaskedLM + +import threestudio +from threestudio.utils.base import BaseObject +from threestudio.utils.misc import barrier, cleanup, get_rank +from threestudio.utils.ops import shifted_cosine_decay, shifted_expotional_decay +from threestudio.utils.typing import * + + +def hash_prompt(model: str, prompt: str) -> str: + import hashlib + + identifier = f"{model}-{prompt}" + return hashlib.md5(identifier.encode()).hexdigest() + + +@dataclass +class DirectionConfig: + name: str + prompt: Callable[[str], str] + negative_prompt: Callable[[str], str] + condition: Callable[ + [Float[Tensor, "B"], Float[Tensor, "B"], Float[Tensor, "B"]], + Float[Tensor, "B"], + ] + + +@dataclass +class PromptProcessorOutput: + text_embeddings: Float[Tensor, "N Nf"] + uncond_text_embeddings: Float[Tensor, "N Nf"] + text_embeddings_vd: Float[Tensor, "Nv N Nf"] + uncond_text_embeddings_vd: Float[Tensor, "Nv N Nf"] + directions: List[DirectionConfig] + direction2idx: Dict[str, int] + use_perp_neg: bool + perp_neg_f_sb: Tuple[float, float, float] + perp_neg_f_fsb: Tuple[float, float, float] + perp_neg_f_fs: Tuple[float, float, float] + perp_neg_f_sf: Tuple[float, float, float] + prompt: str + prompts_vd: List[str] + + def get_text_embeddings( + self, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + view_dependent_prompting: bool = True, + ) -> Float[Tensor, "BB N Nf"]: + batch_size = elevation.shape[0] + + if view_dependent_prompting: + # Get direction + direction_idx = torch.zeros_like(elevation, dtype=torch.long) + for d in self.directions: + direction_idx[ + d.condition(elevation, azimuth, camera_distances) + ] = self.direction2idx[d.name] + + # Get text embeddings + text_embeddings = self.text_embeddings_vd[direction_idx] # type: ignore + uncond_text_embeddings = self.uncond_text_embeddings_vd[direction_idx] # type: ignore + else: + text_embeddings = self.text_embeddings.expand(batch_size, -1, -1) # type: ignore + uncond_text_embeddings = self.uncond_text_embeddings.expand( # type: ignore + batch_size, -1, -1 + ) + + # IMPORTANT: we return (cond, uncond), which is in different order than other implementations! + return torch.cat([text_embeddings, uncond_text_embeddings], dim=0) + + def get_text_embeddings_perp_neg( + self, + elevation: Float[Tensor, "B"], + azimuth: Float[Tensor, "B"], + camera_distances: Float[Tensor, "B"], + view_dependent_prompting: bool = True, + ) -> Tuple[Float[Tensor, "BBBB N Nf"], Float[Tensor, "B 2"]]: + assert ( + view_dependent_prompting + ), "Perp-Neg only works with view-dependent prompting" + + batch_size = elevation.shape[0] + + direction_idx = torch.zeros_like(elevation, dtype=torch.long) + for d in self.directions: + direction_idx[ + d.condition(elevation, azimuth, camera_distances) + ] = self.direction2idx[d.name] + # 0 - side view + # 1 - front view + # 2 - back view + # 3 - overhead view + + pos_text_embeddings = [] + neg_text_embeddings = [] + neg_guidance_weights = [] + uncond_text_embeddings = [] + + side_emb = self.text_embeddings_vd[0] + front_emb = self.text_embeddings_vd[1] + back_emb = self.text_embeddings_vd[2] + overhead_emb = self.text_embeddings_vd[3] + + for idx, ele, azi, dis in zip( + direction_idx, elevation, azimuth, camera_distances + ): + azi = shift_azimuth_deg(azi) # to (-180, 180) + uncond_text_embeddings.append( + self.uncond_text_embeddings_vd[idx] + ) # should be "" + if idx.item() == 3: # overhead view + pos_text_embeddings.append(overhead_emb) # side view + # dummy + neg_text_embeddings += [ + self.uncond_text_embeddings_vd[idx], + self.uncond_text_embeddings_vd[idx], + ] + neg_guidance_weights += [0.0, 0.0] + else: # interpolating views + if torch.abs(azi) < 90: + # front-side interpolation + # 0 - complete side, 1 - complete front + r_inter = 1 - torch.abs(azi) / 90 + pos_text_embeddings.append( + r_inter * front_emb + (1 - r_inter) * side_emb + ) + neg_text_embeddings += [front_emb, side_emb] + neg_guidance_weights += [ + -shifted_expotional_decay(*self.perp_neg_f_fs, r_inter), + -shifted_expotional_decay(*self.perp_neg_f_sf, 1 - r_inter), + ] + else: + # side-back interpolation + # 0 - complete back, 1 - complete side + r_inter = 2.0 - torch.abs(azi) / 90 + pos_text_embeddings.append( + r_inter * side_emb + (1 - r_inter) * back_emb + ) + neg_text_embeddings += [side_emb, front_emb] + neg_guidance_weights += [ + -shifted_expotional_decay(*self.perp_neg_f_sb, r_inter), + -shifted_expotional_decay(*self.perp_neg_f_fsb, r_inter), + ] + + text_embeddings = torch.cat( + [ + torch.stack(pos_text_embeddings, dim=0), + torch.stack(uncond_text_embeddings, dim=0), + torch.stack(neg_text_embeddings, dim=0), + ], + dim=0, + ) + + return text_embeddings, torch.as_tensor( + neg_guidance_weights, device=elevation.device + ).reshape(batch_size, 2) + + +def shift_azimuth_deg(azimuth: Float[Tensor, "..."]) -> Float[Tensor, "..."]: + # shift azimuth angle (in degrees), to [-180, 180] + return (azimuth + 180) % 360 - 180 + + +class PromptProcessor(BaseObject): + @dataclass + class Config(BaseObject.Config): + prompt: str = "a hamburger" + + # manually assigned view-dependent prompts + prompt_front: Optional[str] = None + prompt_side: Optional[str] = None + prompt_back: Optional[str] = None + prompt_overhead: Optional[str] = None + + negative_prompt: str = "" + pretrained_model_name_or_path: str = "runwayml/stable-diffusion-v1-5" + overhead_threshold: float = 60.0 + front_threshold: float = 45.0 + back_threshold: float = 45.0 + view_dependent_prompt_front: bool = False + use_cache: bool = True + spawn: bool = True + + # perp neg + use_perp_neg: bool = False + # a*e(-b*r) + c + # a * e(-b) + c = 0 + perp_neg_f_sb: Tuple[float, float, float] = (1, 0.5, -0.606) + perp_neg_f_fsb: Tuple[float, float, float] = (1, 0.5, +0.967) + perp_neg_f_fs: Tuple[float, float, float] = ( + 4, + 0.5, + -2.426, + ) # f_fs(1) = 0, a, b > 0 + perp_neg_f_sf: Tuple[float, float, float] = (4, 0.5, -2.426) + + # prompt debiasing + use_prompt_debiasing: bool = False + pretrained_model_name_or_path_prompt_debiasing: str = "bert-base-uncased" + # index of words that can potentially be removed + prompt_debiasing_mask_ids: Optional[List[int]] = None + + cfg: Config + + @rank_zero_only + def configure_text_encoder(self) -> None: + raise NotImplementedError + + @rank_zero_only + def destroy_text_encoder(self) -> None: + raise NotImplementedError + + def configure(self) -> None: + self._cache_dir = ".threestudio_cache/text_embeddings" # FIXME: hard-coded path + + # view-dependent text embeddings + self.directions: List[DirectionConfig] + if self.cfg.view_dependent_prompt_front: + self.directions = [ + DirectionConfig( + "side", + lambda s: f"side view of {s}", + lambda s: s, + lambda ele, azi, dis: torch.ones_like(ele, dtype=torch.bool), + ), + DirectionConfig( + "front", + lambda s: f"front view of {s}", + lambda s: s, + lambda ele, azi, dis: ( + shift_azimuth_deg(azi) > -self.cfg.front_threshold + ) + & (shift_azimuth_deg(azi) < self.cfg.front_threshold), + ), + DirectionConfig( + "back", + lambda s: f"backside view of {s}", + lambda s: s, + lambda ele, azi, dis: ( + shift_azimuth_deg(azi) > 180 - self.cfg.back_threshold + ) + | (shift_azimuth_deg(azi) < -180 + self.cfg.back_threshold), + ), + DirectionConfig( + "overhead", + lambda s: f"overhead view of {s}", + lambda s: s, + lambda ele, azi, dis: ele > self.cfg.overhead_threshold, + ), + ] + else: + self.directions = [ + DirectionConfig( + "side", + lambda s: f"{s}, side view", + lambda s: s, + lambda ele, azi, dis: torch.ones_like(ele, dtype=torch.bool), + ), + DirectionConfig( + "front", + lambda s: f"{s}, front view", + lambda s: s, + lambda ele, azi, dis: ( + shift_azimuth_deg(azi) > -self.cfg.front_threshold + ) + & (shift_azimuth_deg(azi) < self.cfg.front_threshold), + ), + DirectionConfig( + "back", + lambda s: f"{s}, back view", + lambda s: s, + lambda ele, azi, dis: ( + shift_azimuth_deg(azi) > 180 - self.cfg.back_threshold + ) + | (shift_azimuth_deg(azi) < -180 + self.cfg.back_threshold), + ), + DirectionConfig( + "overhead", + lambda s: f"{s}, overhead view", + lambda s: s, + lambda ele, azi, dis: ele > self.cfg.overhead_threshold, + ), + ] + + self.direction2idx = {d.name: i for i, d in enumerate(self.directions)} + + if os.path.exists("load/prompt_library.json"): + with open(os.path.join("load/prompt_library.json"), "r") as f: + self.prompt_library = json.load(f) + else: + self.prompt_library = {} + # use provided prompt or find prompt in library + self.prompt = self.preprocess_prompt(self.cfg.prompt) + # use provided negative prompt + self.negative_prompt = self.cfg.negative_prompt + + threestudio.info( + f"Using prompt [{self.prompt}] and negative prompt [{self.negative_prompt}]" + ) + + # view-dependent prompting + if self.cfg.use_prompt_debiasing: + assert ( + self.cfg.prompt_side is None + and self.cfg.prompt_back is None + and self.cfg.prompt_overhead is None + ), "Do not manually assign prompt_side, prompt_back or prompt_overhead when using prompt debiasing" + prompts = self.get_debiased_prompt(self.prompt) + self.prompts_vd = [ + d.prompt(prompt) for d, prompt in zip(self.directions, prompts) + ] + else: + self.prompts_vd = [ + self.cfg.get(f"prompt_{d.name}", None) or d.prompt(self.prompt) # type: ignore + for d in self.directions + ] + + prompts_vd_display = " ".join( + [ + f"[{d.name}]:[{prompt}]" + for prompt, d in zip(self.prompts_vd, self.directions) + ] + ) + threestudio.info(f"Using view-dependent prompts {prompts_vd_display}") + + self.negative_prompts_vd = [ + d.negative_prompt(self.negative_prompt) for d in self.directions + ] + + self.prepare_text_embeddings() + self.load_text_embeddings() + + @staticmethod + def spawn_func(pretrained_model_name_or_path, prompts, cache_dir): + raise NotImplementedError + + @rank_zero_only + def prepare_text_embeddings(self): + os.makedirs(self._cache_dir, exist_ok=True) + + all_prompts = ( + [self.prompt] + + [self.negative_prompt] + + self.prompts_vd + + self.negative_prompts_vd + ) + prompts_to_process = [] + for prompt in all_prompts: + if self.cfg.use_cache: + # some text embeddings are already in cache + # do not process them + cache_path = os.path.join( + self._cache_dir, + f"{hash_prompt(self.cfg.pretrained_model_name_or_path, prompt)}.pt", + ) + if os.path.exists(cache_path): + threestudio.debug( + f"Text embeddings for model {self.cfg.pretrained_model_name_or_path} and prompt [{prompt}] are already in cache, skip processing." + ) + continue + prompts_to_process.append(prompt) + + if len(prompts_to_process) > 0: + if self.cfg.spawn: + ctx = mp.get_context("spawn") + subprocess = ctx.Process( + target=self.spawn_func, + args=( + self.cfg.pretrained_model_name_or_path, + prompts_to_process, + self._cache_dir, + ), + ) + subprocess.start() + subprocess.join() + assert subprocess.exitcode == 0, "prompt embedding process failed!" + else: + self.spawn_func( + self.cfg.pretrained_model_name_or_path, + prompts_to_process, + self._cache_dir, + ) + cleanup() + + def load_text_embeddings(self): + # synchronize, to ensure the text embeddings have been computed and saved to cache + barrier() + self.text_embeddings = self.load_from_cache(self.prompt)[None, ...] + self.uncond_text_embeddings = self.load_from_cache(self.negative_prompt)[ + None, ... + ] + self.text_embeddings_vd = torch.stack( + [self.load_from_cache(prompt) for prompt in self.prompts_vd], dim=0 + ) + self.uncond_text_embeddings_vd = torch.stack( + [self.load_from_cache(prompt) for prompt in self.negative_prompts_vd], dim=0 + ) + threestudio.debug(f"Loaded text embeddings.") + + def load_from_cache(self, prompt): + cache_path = os.path.join( + self._cache_dir, + f"{hash_prompt(self.cfg.pretrained_model_name_or_path, prompt)}.pt", + ) + if not os.path.exists(cache_path): + raise FileNotFoundError( + f"Text embedding file {cache_path} for model {self.cfg.pretrained_model_name_or_path} and prompt [{prompt}] not found." + ) + return torch.load(cache_path, map_location=self.device) + + def preprocess_prompt(self, prompt: str) -> str: + if prompt.startswith("lib:"): + # find matches in the library + candidate = None + keywords = prompt[4:].lower().split("_") + for prompt in self.prompt_library["dreamfusion"]: + if all([k in prompt.lower() for k in keywords]): + if candidate is not None: + raise ValueError( + f"Multiple prompts matched with keywords {keywords} in library" + ) + candidate = prompt + if candidate is None: + raise ValueError( + f"Cannot find prompt with keywords {keywords} in library" + ) + threestudio.info("Find matched prompt in library: " + candidate) + return candidate + else: + return prompt + + def get_text_embeddings( + self, prompt: Union[str, List[str]], negative_prompt: Union[str, List[str]] + ) -> Tuple[Float[Tensor, "B ..."], Float[Tensor, "B ..."]]: + raise NotImplementedError + + def get_debiased_prompt(self, prompt: str) -> List[str]: + os.environ["TOKENIZERS_PARALLELISM"] = "false" + + tokenizer = AutoTokenizer.from_pretrained( + self.cfg.pretrained_model_name_or_path_prompt_debiasing + ) + model = BertForMaskedLM.from_pretrained( + self.cfg.pretrained_model_name_or_path_prompt_debiasing + ) + + views = [d.name for d in self.directions] + view_ids = tokenizer(" ".join(views), return_tensors="pt").input_ids[0] + view_ids = view_ids[1:5] + + def modulate(prompt): + prompt_vd = f"This image is depicting a [MASK] view of {prompt}" + tokens = tokenizer( + prompt_vd, + padding="max_length", + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + mask_idx = torch.where(tokens.input_ids == tokenizer.mask_token_id)[1] + + logits = model(**tokens).logits + logits = F.softmax(logits[0, mask_idx], dim=-1) + logits = logits[0, view_ids] + probes = logits / logits.sum() + return probes + + prompts = [prompt.split(" ") for _ in range(4)] + full_probe = modulate(prompt) + n_words = len(prompt.split(" ")) + prompt_debiasing_mask_ids = ( + self.cfg.prompt_debiasing_mask_ids + if self.cfg.prompt_debiasing_mask_ids is not None + else list(range(n_words)) + ) + words_to_debias = [prompt.split(" ")[idx] for idx in prompt_debiasing_mask_ids] + threestudio.info(f"Words that can potentially be removed: {words_to_debias}") + for idx in prompt_debiasing_mask_ids: + words = prompt.split(" ") + prompt_ = " ".join(words[:idx] + words[(idx + 1) :]) + part_probe = modulate(prompt_) + + pmi = full_probe / torch.lerp(part_probe, full_probe, 0.5) + for i in range(pmi.shape[0]): + if pmi[i].item() < 0.95: + prompts[i][idx] = "" + + debiased_prompts = [" ".join([word for word in p if word]) for p in prompts] + for d, debiased_prompt in zip(views, debiased_prompts): + threestudio.info(f"Debiased prompt of the {d} view is [{debiased_prompt}]") + + del tokenizer, model + cleanup() + + return debiased_prompts + + def __call__(self) -> PromptProcessorOutput: + return PromptProcessorOutput( + text_embeddings=self.text_embeddings, + uncond_text_embeddings=self.uncond_text_embeddings, + prompt=self.prompt, + text_embeddings_vd=self.text_embeddings_vd, + uncond_text_embeddings_vd=self.uncond_text_embeddings_vd, + prompts_vd=self.prompts_vd, + directions=self.directions, + direction2idx=self.direction2idx, + use_perp_neg=self.cfg.use_perp_neg, + perp_neg_f_sb=self.cfg.perp_neg_f_sb, + perp_neg_f_fsb=self.cfg.perp_neg_f_fsb, + perp_neg_f_fs=self.cfg.perp_neg_f_fs, + perp_neg_f_sf=self.cfg.perp_neg_f_sf, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/deepfloyd_prompt_processor.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/deepfloyd_prompt_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..6dd6eef95fb0cf7d35dd0533279b6b60231e7b38 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/deepfloyd_prompt_processor.py @@ -0,0 +1,95 @@ +import json +import os +from dataclasses import dataclass + +import torch +import torch.nn as nn +from diffusers import IFPipeline +from transformers import T5EncoderModel, T5Tokenizer + +import threestudio +from threestudio.models.prompt_processors.base import PromptProcessor, hash_prompt +from threestudio.utils.misc import cleanup +from threestudio.utils.typing import * + + +@threestudio.register("deep-floyd-prompt-processor") +class DeepFloydPromptProcessor(PromptProcessor): + @dataclass + class Config(PromptProcessor.Config): + pretrained_model_name_or_path: str = "DeepFloyd/IF-I-XL-v1.0" + + cfg: Config + + ### these functions are unused, kept for debugging ### + def configure_text_encoder(self) -> None: + os.environ["TOKENIZERS_PARALLELISM"] = "false" + self.text_encoder = T5EncoderModel.from_pretrained( + self.cfg.pretrained_model_name_or_path, + subfolder="text_encoder", + load_in_8bit=True, + variant="8bit", + device_map="auto", + ) # FIXME: behavior of auto device map in multi-GPU training + self.pipe = IFPipeline.from_pretrained( + self.cfg.pretrained_model_name_or_path, + text_encoder=self.text_encoder, # pass the previously instantiated 8bit text encoder + unet=None, + ) + + def destroy_text_encoder(self) -> None: + del self.text_encoder + del self.pipe + cleanup() + + def get_text_embeddings( + self, prompt: Union[str, List[str]], negative_prompt: Union[str, List[str]] + ) -> Tuple[Float[Tensor, "B 77 4096"], Float[Tensor, "B 77 4096"]]: + text_embeddings, uncond_text_embeddings = self.pipe.encode_prompt( + prompt=prompt, negative_prompt=negative_prompt, device=self.device + ) + return text_embeddings, uncond_text_embeddings + + ### + + @staticmethod + def spawn_func(pretrained_model_name_or_path, prompts, cache_dir): + max_length = 77 + tokenizer = T5Tokenizer.from_pretrained( + pretrained_model_name_or_path, subfolder="tokenizer" + ) + text_encoder = T5EncoderModel.from_pretrained( + pretrained_model_name_or_path, + subfolder="text_encoder", + torch_dtype=torch.float16, # suppress warning + load_in_8bit=True, + variant="8bit", + device_map="auto", + ) + with torch.no_grad(): + text_inputs = tokenizer( + prompts, + padding="max_length", + max_length=max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + attention_mask = text_inputs.attention_mask + text_embeddings = text_encoder( + text_input_ids.to(text_encoder.device), + attention_mask=attention_mask.to(text_encoder.device), + ) + text_embeddings = text_embeddings[0] + + for prompt, embedding in zip(prompts, text_embeddings): + torch.save( + embedding, + os.path.join( + cache_dir, + f"{hash_prompt(pretrained_model_name_or_path, prompt)}.pt", + ), + ) + + del text_encoder diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/dummy_prompt_processor.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/dummy_prompt_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..cfbc6c43b84f607338785a1e265361bfed9d7424 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/dummy_prompt_processor.py @@ -0,0 +1,18 @@ +import json +import os +from dataclasses import dataclass + +import threestudio +from threestudio.models.prompt_processors.base import PromptProcessor, hash_prompt +from threestudio.utils.misc import cleanup +from threestudio.utils.typing import * + + +@threestudio.register("dummy-prompt-processor") +class DummyPromptProcessor(PromptProcessor): + @dataclass + class Config(PromptProcessor.Config): + pretrained_model_name_or_path: str = "" + prompt: str = "" + + cfg: Config diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/stable_diffusion_prompt_processor.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/stable_diffusion_prompt_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..91b9eebe20f4cbf42ae12f8cbe59c38da51d3494 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/prompt_processors/stable_diffusion_prompt_processor.py @@ -0,0 +1,102 @@ +import json +import os +from dataclasses import dataclass + +import torch +import torch.nn as nn +from transformers import AutoTokenizer, CLIPTextModel + +import threestudio +from threestudio.models.prompt_processors.base import PromptProcessor, hash_prompt +from threestudio.utils.misc import cleanup +from threestudio.utils.typing import * + + +@threestudio.register("stable-diffusion-prompt-processor") +class StableDiffusionPromptProcessor(PromptProcessor): + @dataclass + class Config(PromptProcessor.Config): + pass + + cfg: Config + + ### these functions are unused, kept for debugging ### + def configure_text_encoder(self) -> None: + self.tokenizer = AutoTokenizer.from_pretrained( + self.cfg.pretrained_model_name_or_path, subfolder="tokenizer" + ) + os.environ["TOKENIZERS_PARALLELISM"] = "false" + self.text_encoder = CLIPTextModel.from_pretrained( + self.cfg.pretrained_model_name_or_path, subfolder="text_encoder" + ).to(self.device) + + for p in self.text_encoder.parameters(): + p.requires_grad_(False) + + def destroy_text_encoder(self) -> None: + del self.tokenizer + del self.text_encoder + cleanup() + + def get_text_embeddings( + self, prompt: Union[str, List[str]], negative_prompt: Union[str, List[str]] + ) -> Tuple[Float[Tensor, "B 77 768"], Float[Tensor, "B 77 768"]]: + if isinstance(prompt, str): + prompt = [prompt] + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + # Tokenize text and get embeddings + tokens = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + return_tensors="pt", + ) + uncond_tokens = self.tokenizer( + negative_prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + return_tensors="pt", + ) + + with torch.no_grad(): + text_embeddings = self.text_encoder(tokens.input_ids.to(self.device))[0] + uncond_text_embeddings = self.text_encoder( + uncond_tokens.input_ids.to(self.device) + )[0] + + return text_embeddings, uncond_text_embeddings + + ### + + @staticmethod + def spawn_func(pretrained_model_name_or_path, prompts, cache_dir): + os.environ["TOKENIZERS_PARALLELISM"] = "false" + tokenizer = AutoTokenizer.from_pretrained( + pretrained_model_name_or_path, subfolder="tokenizer" + ) + text_encoder = CLIPTextModel.from_pretrained( + pretrained_model_name_or_path, + subfolder="text_encoder", + device_map="auto", + ) + + with torch.no_grad(): + tokens = tokenizer( + prompts, + padding="max_length", + max_length=tokenizer.model_max_length, + return_tensors="pt", + ) + text_embeddings = text_encoder(tokens.input_ids.to(text_encoder.device))[0] + + for prompt, embedding in zip(prompts, text_embeddings): + torch.save( + embedding, + os.path.join( + cache_dir, + f"{hash_prompt(pretrained_model_name_or_path, prompt)}.pt", + ), + ) + + del text_encoder diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d33e0fc01ef99e454ac31487f45ec6bcdc8ab82e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/__init__.py @@ -0,0 +1,9 @@ +from . import ( + base, + deferred_volume_renderer, + gan_volume_renderer, + nerf_volume_renderer, + neus_volume_renderer, + nvdiff_rasterizer, + patch_renderer, +) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/base.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/base.py new file mode 100644 index 0000000000000000000000000000000000000000..5642f6e8db3f00b6a8852a10d7301367ab6279e0 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/base.py @@ -0,0 +1,80 @@ +from dataclasses import dataclass + +# import nerfacc +import torch +import torch.nn.functional as F + +import threestudio +from threestudio.models.background.base import BaseBackground +from threestudio.models.geometry.base import BaseImplicitGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.utils.base import BaseModule +from threestudio.utils.typing import * + + +class Renderer(BaseModule): + @dataclass + class Config(BaseModule.Config): + radius: float = 1.0 + + cfg: Config + + def configure( + self, + geometry: BaseImplicitGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + # keep references to submodules using namedtuple, avoid being registered as modules + @dataclass + class SubModules: + geometry: BaseImplicitGeometry + material: BaseMaterial + background: BaseBackground + + self.sub_modules = SubModules(geometry, material, background) + + # set up bounding box + self.bbox: Float[Tensor, "2 3"] + self.register_buffer( + "bbox", + torch.as_tensor( + [ + [-self.cfg.radius, -self.cfg.radius, -self.cfg.radius], + [self.cfg.radius, self.cfg.radius, self.cfg.radius], + ], + dtype=torch.float32, + ), + ) + + def forward(self, *args, **kwargs) -> Dict[str, Any]: + raise NotImplementedError + + @property + def geometry(self) -> BaseImplicitGeometry: + return self.sub_modules.geometry + + @property + def material(self) -> BaseMaterial: + return self.sub_modules.material + + @property + def background(self) -> BaseBackground: + return self.sub_modules.background + + def set_geometry(self, geometry: BaseImplicitGeometry) -> None: + self.sub_modules.geometry = geometry + + def set_material(self, material: BaseMaterial) -> None: + self.sub_modules.material = material + + def set_background(self, background: BaseBackground) -> None: + self.sub_modules.background = background + + +class VolumeRenderer(Renderer): + pass + + +class Rasterizer(Renderer): + pass diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/deferred_volume_renderer.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/deferred_volume_renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..4c8f2ac8f461927692cb8cae8ad03a4940900f02 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/deferred_volume_renderer.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass + +import torch +import torch.nn.functional as F + +import threestudio +from threestudio.models.renderers.base import VolumeRenderer + + +class DeferredVolumeRenderer(VolumeRenderer): + pass diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/gan_volume_renderer.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/gan_volume_renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..fed55a5c4338de163b126b0655e633b23e203f1e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/gan_volume_renderer.py @@ -0,0 +1,159 @@ +from dataclasses import dataclass + +import torch +import torch.nn.functional as F + +import threestudio +from threestudio.models.background.base import BaseBackground +from threestudio.models.geometry.base import BaseImplicitGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.renderers.base import VolumeRenderer +from threestudio.utils.GAN.discriminator import NLayerDiscriminator, weights_init +from threestudio.utils.GAN.distribution import DiagonalGaussianDistribution +from threestudio.utils.GAN.mobilenet import MobileNetV3 as GlobalEncoder +from threestudio.utils.GAN.vae import Decoder as Generator +from threestudio.utils.GAN.vae import Encoder as LocalEncoder +from threestudio.utils.typing import * + + +@threestudio.register("gan-volume-renderer") +class GANVolumeRenderer(VolumeRenderer): + @dataclass + class Config(VolumeRenderer.Config): + base_renderer_type: str = "" + base_renderer: Optional[VolumeRenderer.Config] = None + + cfg: Config + + def configure( + self, + geometry: BaseImplicitGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + self.base_renderer = threestudio.find(self.cfg.base_renderer_type)( + self.cfg.base_renderer, + geometry=geometry, + material=material, + background=background, + ) + self.ch_mult = [1, 2, 4] + self.generator = Generator( + ch=64, + out_ch=3, + ch_mult=self.ch_mult, + num_res_blocks=1, + attn_resolutions=[], + dropout=0.0, + resamp_with_conv=True, + in_channels=7, + resolution=512, + z_channels=4, + ) + self.local_encoder = LocalEncoder( + ch=32, + out_ch=3, + ch_mult=self.ch_mult, + num_res_blocks=1, + attn_resolutions=[], + dropout=0.0, + resamp_with_conv=True, + in_channels=3, + resolution=512, + z_channels=4, + ) + self.global_encoder = GlobalEncoder(n_class=64) + self.discriminator = NLayerDiscriminator( + input_nc=3, n_layers=3, use_actnorm=False, ndf=64 + ).apply(weights_init) + + def forward( + self, + rays_o: Float[Tensor, "B H W 3"], + rays_d: Float[Tensor, "B H W 3"], + light_positions: Float[Tensor, "B 3"], + bg_color: Optional[Tensor] = None, + gt_rgb: Float[Tensor, "B H W 3"] = None, + multi_level_guidance: Bool = False, + **kwargs + ) -> Dict[str, Float[Tensor, "..."]]: + B, H, W, _ = rays_o.shape + if gt_rgb is not None and multi_level_guidance: + generator_level = torch.randint(0, 3, (1,)).item() + interval_x = torch.randint(0, 8, (1,)).item() + interval_y = torch.randint(0, 8, (1,)).item() + int_rays_o = rays_o[:, interval_y::8, interval_x::8] + int_rays_d = rays_d[:, interval_y::8, interval_x::8] + out = self.base_renderer( + int_rays_o, int_rays_d, light_positions, bg_color, **kwargs + ) + comp_int_rgb = out["comp_rgb"][..., :3] + comp_gt_rgb = gt_rgb[:, interval_y::8, interval_x::8] + else: + generator_level = 0 + scale_ratio = 2 ** (len(self.ch_mult) - 1) + rays_o = torch.nn.functional.interpolate( + rays_o.permute(0, 3, 1, 2), + (H // scale_ratio, W // scale_ratio), + mode="bilinear", + ).permute(0, 2, 3, 1) + rays_d = torch.nn.functional.interpolate( + rays_d.permute(0, 3, 1, 2), + (H // scale_ratio, W // scale_ratio), + mode="bilinear", + ).permute(0, 2, 3, 1) + + out = self.base_renderer(rays_o, rays_d, light_positions, bg_color, **kwargs) + comp_rgb = out["comp_rgb"][..., :3] + latent = out["comp_rgb"][..., 3:] + out["comp_lr_rgb"] = comp_rgb.clone() + + posterior = DiagonalGaussianDistribution(latent.permute(0, 3, 1, 2)) + if multi_level_guidance: + z_map = posterior.sample() + else: + z_map = posterior.mode() + lr_rgb = comp_rgb.permute(0, 3, 1, 2) + + if generator_level == 0: + g_code_rgb = self.global_encoder(F.interpolate(lr_rgb, (224, 224))) + comp_gan_rgb = self.generator(torch.cat([lr_rgb, z_map], dim=1), g_code_rgb) + elif generator_level == 1: + g_code_rgb = self.global_encoder( + F.interpolate(gt_rgb.permute(0, 3, 1, 2), (224, 224)) + ) + comp_gan_rgb = self.generator(torch.cat([lr_rgb, z_map], dim=1), g_code_rgb) + elif generator_level == 2: + g_code_rgb = self.global_encoder( + F.interpolate(gt_rgb.permute(0, 3, 1, 2), (224, 224)) + ) + l_code_rgb = self.local_encoder(gt_rgb.permute(0, 3, 1, 2)) + posterior = DiagonalGaussianDistribution(l_code_rgb) + z_map = posterior.sample() + comp_gan_rgb = self.generator(torch.cat([lr_rgb, z_map], dim=1), g_code_rgb) + + comp_rgb = F.interpolate(comp_rgb.permute(0, 3, 1, 2), (H, W), mode="bilinear") + comp_gan_rgb = F.interpolate(comp_gan_rgb, (H, W), mode="bilinear") + out.update( + { + "posterior": posterior, + "comp_gan_rgb": comp_gan_rgb.permute(0, 2, 3, 1), + "comp_rgb": comp_rgb.permute(0, 2, 3, 1), + "generator_level": generator_level, + } + ) + + if gt_rgb is not None and multi_level_guidance: + out.update({"comp_int_rgb": comp_int_rgb, "comp_gt_rgb": comp_gt_rgb}) + return out + + def update_step( + self, epoch: int, global_step: int, on_load_weights: bool = False + ) -> None: + self.base_renderer.update_step(epoch, global_step, on_load_weights) + + def train(self, mode=True): + return self.base_renderer.train(mode) + + def eval(self): + return self.base_renderer.eval() diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/nerf_volume_renderer.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/nerf_volume_renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..c96f1e534551c9719d055e2e7810cbce91e4a035 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/nerf_volume_renderer.py @@ -0,0 +1,470 @@ +from dataclasses import dataclass, field +from functools import partial + +# import nerfacc +import torch +import torch.nn.functional as F + +import threestudio +from threestudio.models.background.base import BaseBackground +# from threestudio.models.estimators import ImportanceEstimator +from threestudio.models.geometry.base import BaseImplicitGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.networks import create_network_with_input_encoding +from threestudio.models.renderers.base import VolumeRenderer +from threestudio.systems.utils import parse_optimizer, parse_scheduler_to_instance +from threestudio.utils.ops import chunk_batch, get_activation, validate_empty_rays +from threestudio.utils.typing import * + + +@threestudio.register("nerf-volume-renderer") +class NeRFVolumeRenderer(VolumeRenderer): + @dataclass + class Config(VolumeRenderer.Config): + num_samples_per_ray: int = 512 + eval_chunk_size: int = 160000 + randomized: bool = True + + near_plane: float = 0.0 + far_plane: float = 1e10 + + return_comp_normal: bool = False + return_normal_perturb: bool = False + + # in ["occgrid", "proposal", "importance"] + estimator: str = "occgrid" + + # for occgrid + grid_prune: bool = True + prune_alpha_threshold: bool = True + + # for proposal + proposal_network_config: Optional[dict] = None + prop_optimizer_config: Optional[dict] = None + prop_scheduler_config: Optional[dict] = None + num_samples_per_ray_proposal: int = 64 + + # for importance + num_samples_per_ray_importance: int = 64 + + cfg: Config + + def configure( + self, + geometry: BaseImplicitGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + super().configure(geometry, material, background) + if self.cfg.estimator == "occgrid": + self.estimator = nerfacc.OccGridEstimator( + roi_aabb=self.bbox.view(-1), resolution=32, levels=1 + ) + if not self.cfg.grid_prune: + self.estimator.occs.fill_(True) + self.estimator.binaries.fill_(True) + self.render_step_size = ( + 1.732 * 2 * self.cfg.radius / self.cfg.num_samples_per_ray + ) + self.randomized = self.cfg.randomized + elif self.cfg.estimator == "importance": + self.estimator = ImportanceEstimator() + elif self.cfg.estimator == "proposal": + self.prop_net = create_network_with_input_encoding( + **self.cfg.proposal_network_config + ) + self.prop_optim = parse_optimizer( + self.cfg.prop_optimizer_config, self.prop_net + ) + self.prop_scheduler = ( + parse_scheduler_to_instance( + self.cfg.prop_scheduler_config, self.prop_optim + ) + if self.cfg.prop_scheduler_config is not None + else None + ) + self.estimator = nerfacc.PropNetEstimator( + self.prop_optim, self.prop_scheduler + ) + + def get_proposal_requires_grad_fn( + target: float = 5.0, num_steps: int = 1000 + ): + schedule = lambda s: min(s / num_steps, 1.0) * target + + steps_since_last_grad = 0 + + def proposal_requires_grad_fn(step: int) -> bool: + nonlocal steps_since_last_grad + target_steps_since_last_grad = schedule(step) + requires_grad = steps_since_last_grad > target_steps_since_last_grad + if requires_grad: + steps_since_last_grad = 0 + steps_since_last_grad += 1 + return requires_grad + + return proposal_requires_grad_fn + + self.proposal_requires_grad_fn = get_proposal_requires_grad_fn() + self.randomized = self.cfg.randomized + else: + raise NotImplementedError( + "Unknown estimator, should be one of ['occgrid', 'proposal', 'importance']." + ) + + # for proposal + self.vars_in_forward = {} + + def forward( + self, + rays_o: Float[Tensor, "B H W 3"], + rays_d: Float[Tensor, "B H W 3"], + light_positions: Float[Tensor, "B 3"], + bg_color: Optional[Tensor] = None, + **kwargs + ) -> Dict[str, Float[Tensor, "..."]]: + batch_size, height, width = rays_o.shape[:3] + rays_o_flatten: Float[Tensor, "Nr 3"] = rays_o.reshape(-1, 3) + rays_d_flatten: Float[Tensor, "Nr 3"] = rays_d.reshape(-1, 3) + light_positions_flatten: Float[Tensor, "Nr 3"] = ( + light_positions.reshape(-1, 1, 1, 3) + .expand(-1, height, width, -1) + .reshape(-1, 3) + ) + n_rays = rays_o_flatten.shape[0] + + if self.cfg.estimator == "occgrid": + if not self.cfg.grid_prune: + with torch.no_grad(): + ray_indices, t_starts_, t_ends_ = self.estimator.sampling( + rays_o_flatten, + rays_d_flatten, + sigma_fn=None, + near_plane=self.cfg.near_plane, + far_plane=self.cfg.far_plane, + render_step_size=self.render_step_size, + alpha_thre=0.0, + stratified=self.randomized, + cone_angle=0.0, + early_stop_eps=0, + ) + else: + + def sigma_fn(t_starts, t_ends, ray_indices): + t_starts, t_ends = t_starts[..., None], t_ends[..., None] + t_origins = rays_o_flatten[ray_indices] + t_positions = (t_starts + t_ends) / 2.0 + t_dirs = rays_d_flatten[ray_indices] + positions = t_origins + t_dirs * t_positions + if self.training: + sigma = self.geometry.forward_density(positions)[..., 0] + else: + sigma = chunk_batch( + self.geometry.forward_density, + self.cfg.eval_chunk_size, + positions, + )[..., 0] + return sigma + + with torch.no_grad(): + ray_indices, t_starts_, t_ends_ = self.estimator.sampling( + rays_o_flatten, + rays_d_flatten, + sigma_fn=sigma_fn if self.cfg.prune_alpha_threshold else None, + near_plane=self.cfg.near_plane, + far_plane=self.cfg.far_plane, + render_step_size=self.render_step_size, + alpha_thre=0.01 if self.cfg.prune_alpha_threshold else 0.0, + stratified=self.randomized, + cone_angle=0.0, + ) + elif self.cfg.estimator == "proposal": + + def prop_sigma_fn( + t_starts: Float[Tensor, "Nr Ns"], + t_ends: Float[Tensor, "Nr Ns"], + proposal_network, + ): + t_origins: Float[Tensor, "Nr 1 3"] = rays_o_flatten.unsqueeze(-2) + t_dirs: Float[Tensor, "Nr 1 3"] = rays_d_flatten.unsqueeze(-2) + positions: Float[Tensor, "Nr Ns 3"] = ( + t_origins + t_dirs * (t_starts + t_ends)[..., None] / 2.0 + ) + aabb_min, aabb_max = self.bbox[0], self.bbox[1] + positions = (positions - aabb_min) / (aabb_max - aabb_min) + selector = ((positions > 0.0) & (positions < 1.0)).all(dim=-1) + density_before_activation = ( + proposal_network(positions.view(-1, 3)) + .view(*positions.shape[:-1], 1) + .to(positions) + ) + density: Float[Tensor, "Nr Ns 1"] = ( + get_activation("shifted_trunc_exp")(density_before_activation) + * selector[..., None] + ) + return density.squeeze(-1) + + t_starts_, t_ends_ = self.estimator.sampling( + prop_sigma_fns=[partial(prop_sigma_fn, proposal_network=self.prop_net)], + prop_samples=[self.cfg.num_samples_per_ray_proposal], + num_samples=self.cfg.num_samples_per_ray, + n_rays=n_rays, + near_plane=self.cfg.near_plane, + far_plane=self.cfg.far_plane, + sampling_type="uniform", + stratified=self.randomized, + requires_grad=self.vars_in_forward["requires_grad"], + ) + ray_indices = ( + torch.arange(n_rays, device=rays_o_flatten.device) + .unsqueeze(-1) + .expand(-1, t_starts_.shape[1]) + ) + ray_indices = ray_indices.flatten() + t_starts_ = t_starts_.flatten() + t_ends_ = t_ends_.flatten() + elif self.cfg.estimator == "importance": + + def prop_sigma_fn( + t_starts: Float[Tensor, "Nr Ns"], + t_ends: Float[Tensor, "Nr Ns"], + proposal_network, + ): + t_origins: Float[Tensor, "Nr 1 3"] = rays_o_flatten.unsqueeze(-2) + t_dirs: Float[Tensor, "Nr 1 3"] = rays_d_flatten.unsqueeze(-2) + positions: Float[Tensor, "Nr Ns 3"] = ( + t_origins + t_dirs * (t_starts + t_ends)[..., None] / 2.0 + ) + with torch.no_grad(): + geo_out = chunk_batch( + proposal_network, + self.cfg.eval_chunk_size, + positions.reshape(-1, 3), + output_normal=False, + ) + density = geo_out["density"] + return density.reshape(positions.shape[:2]) + + t_starts_, t_ends_ = self.estimator.sampling( + prop_sigma_fns=[partial(prop_sigma_fn, proposal_network=self.geometry)], + prop_samples=[self.cfg.num_samples_per_ray_importance], + num_samples=self.cfg.num_samples_per_ray, + n_rays=n_rays, + near_plane=self.cfg.near_plane, + far_plane=self.cfg.far_plane, + sampling_type="uniform", + stratified=self.randomized, + ) + ray_indices = ( + torch.arange(n_rays, device=rays_o_flatten.device) + .unsqueeze(-1) + .expand(-1, t_starts_.shape[1]) + ) + ray_indices = ray_indices.flatten() + t_starts_ = t_starts_.flatten() + t_ends_ = t_ends_.flatten() + else: + raise NotImplementedError + + ray_indices, t_starts_, t_ends_ = validate_empty_rays( + ray_indices, t_starts_, t_ends_ + ) + ray_indices = ray_indices.long() + t_starts, t_ends = t_starts_[..., None], t_ends_[..., None] + t_origins = rays_o_flatten[ray_indices] + t_dirs = rays_d_flatten[ray_indices] + t_light_positions = light_positions_flatten[ray_indices] + t_positions = (t_starts + t_ends) / 2.0 + positions = t_origins + t_dirs * t_positions + t_intervals = t_ends - t_starts + + if self.training: + geo_out = self.geometry( + positions, output_normal=self.material.requires_normal + ) + rgb_fg_all = self.material( + viewdirs=t_dirs, + positions=positions, + light_positions=t_light_positions, + **geo_out, + **kwargs + ) + comp_rgb_bg = self.background(dirs=rays_d) + else: + geo_out = chunk_batch( + self.geometry, + self.cfg.eval_chunk_size, + positions, + output_normal=self.material.requires_normal, + ) + rgb_fg_all = chunk_batch( + self.material, + self.cfg.eval_chunk_size, + viewdirs=t_dirs, + positions=positions, + light_positions=t_light_positions, + **geo_out + ) + comp_rgb_bg = chunk_batch( + self.background, self.cfg.eval_chunk_size, dirs=rays_d + ) + + weights: Float[Tensor, "Nr 1"] + weights_, trans_, _ = nerfacc.render_weight_from_density( + t_starts[..., 0], + t_ends[..., 0], + geo_out["density"][..., 0], + ray_indices=ray_indices, + n_rays=n_rays, + ) + if self.training and self.cfg.estimator == "proposal": + self.vars_in_forward["trans"] = trans_.reshape(n_rays, -1) + + weights = weights_[..., None] + opacity: Float[Tensor, "Nr 1"] = nerfacc.accumulate_along_rays( + weights[..., 0], values=None, ray_indices=ray_indices, n_rays=n_rays + ) + depth: Float[Tensor, "Nr 1"] = nerfacc.accumulate_along_rays( + weights[..., 0], values=t_positions, ray_indices=ray_indices, n_rays=n_rays + ) + comp_rgb_fg: Float[Tensor, "Nr Nc"] = nerfacc.accumulate_along_rays( + weights[..., 0], values=rgb_fg_all, ray_indices=ray_indices, n_rays=n_rays + ) + + # populate depth and opacity to each point + weights_normalized = weights / opacity.clamp(min=1e-5)[ray_indices] # num_pts + # z-variance loss from HiFA: https://hifa-team.github.io/HiFA-site/ + z_mean: Float[Tensor, "Nr 1"] = nerfacc.accumulate_along_rays( + weights_normalized[..., 0], + values=t_positions, + ray_indices=ray_indices, + n_rays=n_rays, + ) + z_variance_unmasked = nerfacc.accumulate_along_rays( + weights_normalized[..., 0], + values=(t_positions - z_mean[ray_indices]) ** 2, + ray_indices=ray_indices, + n_rays=n_rays, + ) + z_variance = z_variance_unmasked * (opacity > 0.5).float() + + if bg_color is None: + bg_color = comp_rgb_bg + else: + if bg_color.shape[:-1] == (batch_size,): + # e.g. constant random color used for Zero123 + # [bs,3] -> [bs, 1, 1, 3]): + bg_color = bg_color.unsqueeze(1).unsqueeze(1) + # -> [bs, height, width, 3]): + bg_color = bg_color.expand(-1, height, width, -1) + + if bg_color.shape[:-1] == (batch_size, height, width): + bg_color = bg_color.reshape(batch_size * height * width, -1) + + comp_rgb = comp_rgb_fg + bg_color * (1.0 - opacity) + + out = { + "comp_rgb": comp_rgb.view(batch_size, height, width, -1), + "comp_rgb_fg": comp_rgb_fg.view(batch_size, height, width, -1), + "comp_rgb_bg": comp_rgb_bg.view(batch_size, height, width, -1), + "opacity": opacity.view(batch_size, height, width, 1), + "depth": depth.view(batch_size, height, width, 1), + "z_variance": z_variance.view(batch_size, height, width, 1), + } + + if self.training: + out.update( + { + "weights": weights, + "t_points": t_positions, + "t_intervals": t_intervals, + "t_dirs": t_dirs, + "ray_indices": ray_indices, + "points": positions, + **geo_out, + } + ) + if "normal" in geo_out: + if self.cfg.return_comp_normal: + comp_normal: Float[Tensor, "Nr 3"] = nerfacc.accumulate_along_rays( + weights[..., 0], + values=geo_out["normal"], + ray_indices=ray_indices, + n_rays=n_rays, + ) + comp_normal = F.normalize(comp_normal, dim=-1) + comp_normal = ( + (comp_normal + 1.0) / 2.0 * opacity + ) # for visualization + out.update( + { + "comp_normal": comp_normal.view( + batch_size, height, width, 3 + ), + } + ) + if self.cfg.return_normal_perturb: + normal_perturb = self.geometry( + positions + torch.randn_like(positions) * 1e-2, + output_normal=self.material.requires_normal, + )["normal"] + out.update({"normal_perturb": normal_perturb}) + else: + if "normal" in geo_out: + comp_normal = nerfacc.accumulate_along_rays( + weights[..., 0], + values=geo_out["normal"], + ray_indices=ray_indices, + n_rays=n_rays, + ) + comp_normal = F.normalize(comp_normal, dim=-1) + comp_normal = (comp_normal + 1.0) / 2.0 * opacity # for visualization + out.update( + { + "comp_normal": comp_normal.view(batch_size, height, width, 3), + } + ) + + return out + + def update_step( + self, epoch: int, global_step: int, on_load_weights: bool = False + ) -> None: + if self.cfg.estimator == "occgrid": + if self.cfg.grid_prune: + + def occ_eval_fn(x): + density = self.geometry.forward_density(x) + # approximate for 1 - torch.exp(-density * self.render_step_size) based on taylor series + return density * self.render_step_size + + if self.training and not on_load_weights: + self.estimator.update_every_n_steps( + step=global_step, occ_eval_fn=occ_eval_fn + ) + elif self.cfg.estimator == "proposal": + if self.training: + requires_grad = self.proposal_requires_grad_fn(global_step) + self.vars_in_forward["requires_grad"] = requires_grad + else: + self.vars_in_forward["requires_grad"] = False + + def update_step_end(self, epoch: int, global_step: int) -> None: + if self.cfg.estimator == "proposal" and self.training: + self.estimator.update_every_n_steps( + self.vars_in_forward["trans"], + self.vars_in_forward["requires_grad"], + loss_scaler=1.0, + ) + + def train(self, mode=True): + self.randomized = mode and self.cfg.randomized + if self.cfg.estimator == "proposal": + self.prop_net.train() + return super().train(mode=mode) + + def eval(self): + self.randomized = False + if self.cfg.estimator == "proposal": + self.prop_net.eval() + return super().eval() diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/neus_volume_renderer.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/neus_volume_renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..0960176c0f3fe53559a9de10f0ecf97a8be8e03b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/neus_volume_renderer.py @@ -0,0 +1,390 @@ +from dataclasses import dataclass +from functools import partial + +import nerfacc +import torch +import torch.nn as nn +import torch.nn.functional as F + +import threestudio +from threestudio.models.background.base import BaseBackground +from threestudio.models.estimators import ImportanceEstimator +from threestudio.models.geometry.base import BaseImplicitGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.renderers.base import VolumeRenderer +from threestudio.utils.ops import chunk_batch, validate_empty_rays +from threestudio.utils.typing import * + + +def volsdf_density(sdf, inv_std): + inv_std = inv_std.clamp(0.0, 80.0) + beta = 1 / inv_std + alpha = inv_std + return alpha * (0.5 + 0.5 * sdf.sign() * torch.expm1(-sdf.abs() / beta)) + + +class LearnedVariance(nn.Module): + def __init__(self, init_val): + super(LearnedVariance, self).__init__() + self.register_parameter("_inv_std", nn.Parameter(torch.tensor(init_val))) + + @property + def inv_std(self): + val = torch.exp(self._inv_std * 10.0) + return val + + def forward(self, x): + return torch.ones_like(x) * self.inv_std.clamp(1.0e-6, 1.0e6) + + +@threestudio.register("neus-volume-renderer") +class NeuSVolumeRenderer(VolumeRenderer): + @dataclass + class Config(VolumeRenderer.Config): + num_samples_per_ray: int = 512 + randomized: bool = True + eval_chunk_size: int = 160000 + learned_variance_init: float = 0.3 + cos_anneal_end_steps: int = 0 + use_volsdf: bool = False + + near_plane: float = 0.0 + far_plane: float = 1e10 + + # in ['occgrid', 'importance'] + estimator: str = "occgrid" + + # for occgrid + grid_prune: bool = True + prune_alpha_threshold: bool = True + + # for importance + num_samples_per_ray_importance: int = 64 + + cfg: Config + + def configure( + self, + geometry: BaseImplicitGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + super().configure(geometry, material, background) + self.variance = LearnedVariance(self.cfg.learned_variance_init) + if self.cfg.estimator == "occgrid": + self.estimator = nerfacc.OccGridEstimator( + roi_aabb=self.bbox.view(-1), resolution=32, levels=1 + ) + if not self.cfg.grid_prune: + self.estimator.occs.fill_(True) + self.estimator.binaries.fill_(True) + self.render_step_size = ( + 1.732 * 2 * self.cfg.radius / self.cfg.num_samples_per_ray + ) + self.randomized = self.cfg.randomized + elif self.cfg.estimator == "importance": + self.estimator = ImportanceEstimator() + else: + raise NotImplementedError( + "unknown estimator, should be in ['occgrid', 'importance']" + ) + self.cos_anneal_ratio = 1.0 + + def get_alpha(self, sdf, normal, dirs, dists): + inv_std = self.variance(sdf) + if self.cfg.use_volsdf: + alpha = torch.abs(dists.detach()) * volsdf_density(sdf, inv_std) + else: + true_cos = (dirs * normal).sum(-1, keepdim=True) + # "cos_anneal_ratio" grows from 0 to 1 in the beginning training iterations. The anneal strategy below makes + # the cos value "not dead" at the beginning training iterations, for better convergence. + iter_cos = -( + F.relu(-true_cos * 0.5 + 0.5) * (1.0 - self.cos_anneal_ratio) + + F.relu(-true_cos) * self.cos_anneal_ratio + ) # always non-positive + + # Estimate signed distances at section points + estimated_next_sdf = sdf + iter_cos * dists * 0.5 + estimated_prev_sdf = sdf - iter_cos * dists * 0.5 + + prev_cdf = torch.sigmoid(estimated_prev_sdf * inv_std) + next_cdf = torch.sigmoid(estimated_next_sdf * inv_std) + + p = prev_cdf - next_cdf + c = prev_cdf + + alpha = ((p + 1e-5) / (c + 1e-5)).clip(0.0, 1.0) + return alpha + + def forward( + self, + rays_o: Float[Tensor, "B H W 3"], + rays_d: Float[Tensor, "B H W 3"], + light_positions: Float[Tensor, "B 3"], + bg_color: Optional[Tensor] = None, + **kwargs + ) -> Dict[str, Float[Tensor, "..."]]: + batch_size, height, width = rays_o.shape[:3] + rays_o_flatten: Float[Tensor, "Nr 3"] = rays_o.reshape(-1, 3) + rays_d_flatten: Float[Tensor, "Nr 3"] = rays_d.reshape(-1, 3) + light_positions_flatten: Float[Tensor, "Nr 3"] = ( + light_positions.reshape(-1, 1, 1, 3) + .expand(-1, height, width, -1) + .reshape(-1, 3) + ) + n_rays = rays_o_flatten.shape[0] + + if self.cfg.estimator == "occgrid": + + def alpha_fn(t_starts, t_ends, ray_indices): + t_starts, t_ends = t_starts[..., None], t_ends[..., None] + t_origins = rays_o_flatten[ray_indices] + t_positions = (t_starts + t_ends) / 2.0 + t_dirs = rays_d_flatten[ray_indices] + positions = t_origins + t_dirs * t_positions + if self.training: + sdf = self.geometry.forward_sdf(positions)[..., 0] + else: + sdf = chunk_batch( + self.geometry.forward_sdf, + self.cfg.eval_chunk_size, + positions, + )[..., 0] + + inv_std = self.variance(sdf) + if self.cfg.use_volsdf: + alpha = self.render_step_size * volsdf_density(sdf, inv_std) + else: + estimated_next_sdf = sdf - self.render_step_size * 0.5 + estimated_prev_sdf = sdf + self.render_step_size * 0.5 + prev_cdf = torch.sigmoid(estimated_prev_sdf * inv_std) + next_cdf = torch.sigmoid(estimated_next_sdf * inv_std) + p = prev_cdf - next_cdf + c = prev_cdf + alpha = ((p + 1e-5) / (c + 1e-5)).clip(0.0, 1.0) + + return alpha + + if not self.cfg.grid_prune: + with torch.no_grad(): + ray_indices, t_starts_, t_ends_ = self.estimator.sampling( + rays_o_flatten, + rays_d_flatten, + alpha_fn=None, + near_plane=self.cfg.near_plane, + far_plane=self.cfg.far_plane, + render_step_size=self.render_step_size, + alpha_thre=0.0, + stratified=self.randomized, + cone_angle=0.0, + early_stop_eps=0, + ) + else: + with torch.no_grad(): + ray_indices, t_starts_, t_ends_ = self.estimator.sampling( + rays_o_flatten, + rays_d_flatten, + alpha_fn=alpha_fn if self.cfg.prune_alpha_threshold else None, + near_plane=self.cfg.near_plane, + far_plane=self.cfg.far_plane, + render_step_size=self.render_step_size, + alpha_thre=0.01 if self.cfg.prune_alpha_threshold else 0.0, + stratified=self.randomized, + cone_angle=0.0, + ) + elif self.cfg.estimator == "importance": + + def prop_sigma_fn( + t_starts: Float[Tensor, "Nr Ns"], + t_ends: Float[Tensor, "Nr Ns"], + proposal_network, + ): + if self.cfg.use_volsdf: + t_origins: Float[Tensor, "Nr 1 3"] = rays_o_flatten.unsqueeze(-2) + t_dirs: Float[Tensor, "Nr 1 3"] = rays_d_flatten.unsqueeze(-2) + positions: Float[Tensor, "Nr Ns 3"] = ( + t_origins + t_dirs * (t_starts + t_ends)[..., None] / 2.0 + ) + with torch.no_grad(): + geo_out = chunk_batch( + proposal_network, + self.cfg.eval_chunk_size, + positions.reshape(-1, 3), + output_normal=False, + ) + inv_std = self.variance(geo_out["sdf"]) + density = volsdf_density(geo_out["sdf"], inv_std) + return density.reshape(positions.shape[:2]) + else: + raise ValueError( + "Currently only VolSDF supports importance sampling." + ) + + t_starts_, t_ends_ = self.estimator.sampling( + prop_sigma_fns=[partial(prop_sigma_fn, proposal_network=self.geometry)], + prop_samples=[self.cfg.num_samples_per_ray_importance], + num_samples=self.cfg.num_samples_per_ray, + n_rays=n_rays, + near_plane=self.cfg.near_plane, + far_plane=self.cfg.far_plane, + sampling_type="uniform", + stratified=self.randomized, + ) + ray_indices = ( + torch.arange(n_rays, device=rays_o_flatten.device) + .unsqueeze(-1) + .expand(-1, t_starts_.shape[1]) + ) + ray_indices = ray_indices.flatten() + t_starts_ = t_starts_.flatten() + t_ends_ = t_ends_.flatten() + else: + raise NotImplementedError + + ray_indices, t_starts_, t_ends_ = validate_empty_rays( + ray_indices, t_starts_, t_ends_ + ) + ray_indices = ray_indices.long() + t_starts, t_ends = t_starts_[..., None], t_ends_[..., None] + t_origins = rays_o_flatten[ray_indices] + t_dirs = rays_d_flatten[ray_indices] + t_light_positions = light_positions_flatten[ray_indices] + t_positions = (t_starts + t_ends) / 2.0 + positions = t_origins + t_dirs * t_positions + t_intervals = t_ends - t_starts + + if self.training: + geo_out = self.geometry(positions, output_normal=True) + rgb_fg_all = self.material( + viewdirs=t_dirs, + positions=positions, + light_positions=t_light_positions, + **geo_out, + **kwargs + ) + comp_rgb_bg = self.background(dirs=rays_d) + else: + geo_out = chunk_batch( + self.geometry, + self.cfg.eval_chunk_size, + positions, + output_normal=True, + ) + rgb_fg_all = chunk_batch( + self.material, + self.cfg.eval_chunk_size, + viewdirs=t_dirs, + positions=positions, + light_positions=t_light_positions, + **geo_out + ) + comp_rgb_bg = chunk_batch( + self.background, self.cfg.eval_chunk_size, dirs=rays_d + ) + + # grad or normal? + alpha: Float[Tensor, "Nr 1"] = self.get_alpha( + geo_out["sdf"], geo_out["normal"], t_dirs, t_intervals + ) + + weights: Float[Tensor, "Nr 1"] + weights_, _ = nerfacc.render_weight_from_alpha( + alpha[..., 0], + ray_indices=ray_indices, + n_rays=n_rays, + ) + weights = weights_[..., None] + opacity: Float[Tensor, "Nr 1"] = nerfacc.accumulate_along_rays( + weights[..., 0], values=None, ray_indices=ray_indices, n_rays=n_rays + ) + depth: Float[Tensor, "Nr 1"] = nerfacc.accumulate_along_rays( + weights[..., 0], values=t_positions, ray_indices=ray_indices, n_rays=n_rays + ) + comp_rgb_fg: Float[Tensor, "Nr Nc"] = nerfacc.accumulate_along_rays( + weights[..., 0], values=rgb_fg_all, ray_indices=ray_indices, n_rays=n_rays + ) + + if bg_color is None: + bg_color = comp_rgb_bg + + if bg_color.shape[:-1] == (batch_size, height, width): + bg_color = bg_color.reshape(batch_size * height * width, -1) + + comp_rgb = comp_rgb_fg + bg_color * (1.0 - opacity) + + out = { + "comp_rgb": comp_rgb.view(batch_size, height, width, -1), + "comp_rgb_fg": comp_rgb_fg.view(batch_size, height, width, -1), + "comp_rgb_bg": comp_rgb_bg.view(batch_size, height, width, -1), + "opacity": opacity.view(batch_size, height, width, 1), + "depth": depth.view(batch_size, height, width, 1), + } + + if self.training: + out.update( + { + "weights": weights, + "t_points": t_positions, + "t_intervals": t_intervals, + "t_dirs": t_dirs, + "ray_indices": ray_indices, + "points": positions, + **geo_out, + } + ) + else: + if "normal" in geo_out: + comp_normal: Float[Tensor, "Nr 3"] = nerfacc.accumulate_along_rays( + weights[..., 0], + values=geo_out["normal"], + ray_indices=ray_indices, + n_rays=n_rays, + ) + comp_normal = F.normalize(comp_normal, dim=-1) + comp_normal = (comp_normal + 1.0) / 2.0 * opacity # for visualization + out.update( + { + "comp_normal": comp_normal.view(batch_size, height, width, 3), + } + ) + out.update({"inv_std": self.variance.inv_std}) + return out + + def update_step( + self, epoch: int, global_step: int, on_load_weights: bool = False + ) -> None: + self.cos_anneal_ratio = ( + 1.0 + if self.cfg.cos_anneal_end_steps == 0 + else min(1.0, global_step / self.cfg.cos_anneal_end_steps) + ) + if self.cfg.estimator == "occgrid": + if self.cfg.grid_prune: + + def occ_eval_fn(x): + sdf = self.geometry.forward_sdf(x) + inv_std = self.variance(sdf) + if self.cfg.use_volsdf: + alpha = self.render_step_size * volsdf_density(sdf, inv_std) + else: + estimated_next_sdf = sdf - self.render_step_size * 0.5 + estimated_prev_sdf = sdf + self.render_step_size * 0.5 + prev_cdf = torch.sigmoid(estimated_prev_sdf * inv_std) + next_cdf = torch.sigmoid(estimated_next_sdf * inv_std) + p = prev_cdf - next_cdf + c = prev_cdf + alpha = ((p + 1e-5) / (c + 1e-5)).clip(0.0, 1.0) + return alpha + + if self.training and not on_load_weights: + self.estimator.update_every_n_steps( + step=global_step, occ_eval_fn=occ_eval_fn + ) + + def train(self, mode=True): + self.randomized = mode and self.cfg.randomized + return super().train(mode=mode) + + def eval(self): + self.randomized = False + return super().eval() diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/nvdiff_rasterizer.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/nvdiff_rasterizer.py new file mode 100644 index 0000000000000000000000000000000000000000..b73e9e7846197aa7c247ff54e53848a641eef34c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/nvdiff_rasterizer.py @@ -0,0 +1,111 @@ +from dataclasses import dataclass + +# import nerfacc +import torch +import torch.nn.functional as F + +import threestudio +from threestudio.models.background.base import BaseBackground +from threestudio.models.geometry.base import BaseImplicitGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.renderers.base import Rasterizer, VolumeRenderer +from threestudio.utils.misc import get_device +from threestudio.utils.rasterize import NVDiffRasterizerContext +from threestudio.utils.typing import * + + +@threestudio.register("nvdiff-rasterizer") +class NVDiffRasterizer(Rasterizer): + @dataclass + class Config(VolumeRenderer.Config): + context_type: str = "gl" + + cfg: Config + + def configure( + self, + geometry: BaseImplicitGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + super().configure(geometry, material, background) + self.ctx = NVDiffRasterizerContext(self.cfg.context_type, get_device()) + + def forward( + self, + mvp_mtx: Float[Tensor, "B 4 4"], + camera_positions: Float[Tensor, "B 3"], + light_positions: Float[Tensor, "B 3"], + height: int, + width: int, + render_rgb: bool = True, + **kwargs + ) -> Dict[str, Any]: + batch_size = mvp_mtx.shape[0] + mesh = self.geometry.isosurface() + + v_pos_clip: Float[Tensor, "B Nv 4"] = self.ctx.vertex_transform( + mesh.v_pos, mvp_mtx + ) + rast, _ = self.ctx.rasterize(v_pos_clip, mesh.t_pos_idx, (height, width)) + mask = rast[..., 3:] > 0 + mask_aa = self.ctx.antialias(mask.float(), rast, v_pos_clip, mesh.t_pos_idx) + + out = {"opacity": mask_aa, "mesh": mesh} + + gb_normal, _ = self.ctx.interpolate_one(mesh.v_nrm, rast, mesh.t_pos_idx) + gb_normal = F.normalize(gb_normal, dim=-1) + gb_normal_aa = torch.lerp( + torch.zeros_like(gb_normal), (gb_normal + 1.0) / 2.0, mask.float() + ) + gb_normal_aa = self.ctx.antialias( + gb_normal_aa, rast, v_pos_clip, mesh.t_pos_idx + ) + out.update({"comp_normal": gb_normal_aa}) # in [0, 1] + + # TODO: make it clear whether to compute the normal, now we compute it in all cases + # consider using: require_normal_computation = render_normal or (render_rgb and material.requires_normal) + # or + # render_normal = render_normal or (render_rgb and material.requires_normal) + + if render_rgb: + selector = mask[..., 0] + + gb_pos, _ = self.ctx.interpolate_one(mesh.v_pos, rast, mesh.t_pos_idx) + gb_viewdirs = F.normalize( + gb_pos - camera_positions[:, None, None, :], dim=-1 + ) + gb_light_positions = light_positions[:, None, None, :].expand( + -1, height, width, -1 + ) + + positions = gb_pos[selector] + geo_out = self.geometry(positions, output_normal=False) + + extra_geo_info = {} + if self.material.requires_normal: + extra_geo_info["shading_normal"] = gb_normal[selector] + if self.material.requires_tangent: + gb_tangent, _ = self.ctx.interpolate_one( + mesh.v_tng, rast, mesh.t_pos_idx + ) + gb_tangent = F.normalize(gb_tangent, dim=-1) + extra_geo_info["tangent"] = gb_tangent[selector] + + rgb_fg = self.material( + viewdirs=gb_viewdirs[selector], + positions=positions, + light_positions=gb_light_positions[selector], + **extra_geo_info, + **geo_out + ) + gb_rgb_fg = torch.zeros(batch_size, height, width, 3).to(rgb_fg) + gb_rgb_fg[selector] = rgb_fg + + gb_rgb_bg = self.background(dirs=gb_viewdirs) + gb_rgb = torch.lerp(gb_rgb_bg, gb_rgb_fg, mask.float()) + gb_rgb_aa = self.ctx.antialias(gb_rgb, rast, v_pos_clip, mesh.t_pos_idx) + + out.update({"comp_rgb": gb_rgb_aa, "comp_rgb_bg": gb_rgb_bg}) + + return out diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/patch_renderer.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/patch_renderer.py new file mode 100644 index 0000000000000000000000000000000000000000..d595791e2ad6df42b263224794a33ac7824fc3a3 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/models/renderers/patch_renderer.py @@ -0,0 +1,106 @@ +from dataclasses import dataclass + +import torch +import torch.nn.functional as F + +import threestudio +from threestudio.models.background.base import BaseBackground +from threestudio.models.geometry.base import BaseImplicitGeometry +from threestudio.models.materials.base import BaseMaterial +from threestudio.models.renderers.base import VolumeRenderer +from threestudio.utils.typing import * + + +@threestudio.register("patch-renderer") +class PatchRenderer(VolumeRenderer): + @dataclass + class Config(VolumeRenderer.Config): + patch_size: int = 128 + base_renderer_type: str = "" + base_renderer: Optional[VolumeRenderer.Config] = None + global_detach: bool = False + global_downsample: int = 4 + + cfg: Config + + def configure( + self, + geometry: BaseImplicitGeometry, + material: BaseMaterial, + background: BaseBackground, + ) -> None: + self.base_renderer = threestudio.find(self.cfg.base_renderer_type)( + self.cfg.base_renderer, + geometry=geometry, + material=material, + background=background, + ) + + def forward( + self, + rays_o: Float[Tensor, "B H W 3"], + rays_d: Float[Tensor, "B H W 3"], + light_positions: Float[Tensor, "B 3"], + bg_color: Optional[Tensor] = None, + **kwargs + ) -> Dict[str, Float[Tensor, "..."]]: + B, H, W, _ = rays_o.shape + + if self.base_renderer.training: + downsample = self.cfg.global_downsample + global_rays_o = torch.nn.functional.interpolate( + rays_o.permute(0, 3, 1, 2), + (H // downsample, W // downsample), + mode="bilinear", + ).permute(0, 2, 3, 1) + global_rays_d = torch.nn.functional.interpolate( + rays_d.permute(0, 3, 1, 2), + (H // downsample, W // downsample), + mode="bilinear", + ).permute(0, 2, 3, 1) + out_global = self.base_renderer( + global_rays_o, global_rays_d, light_positions, bg_color, **kwargs + ) + + PS = self.cfg.patch_size + patch_x = torch.randint(0, W - PS, (1,)).item() + patch_y = torch.randint(0, H - PS, (1,)).item() + patch_rays_o = rays_o[:, patch_y : patch_y + PS, patch_x : patch_x + PS] + patch_rays_d = rays_d[:, patch_y : patch_y + PS, patch_x : patch_x + PS] + out = self.base_renderer( + patch_rays_o, patch_rays_d, light_positions, bg_color, **kwargs + ) + + valid_patch_key = [] + for key in out: + if torch.is_tensor(out[key]): + if len(out[key].shape) == len(out["comp_rgb"].shape): + if out[key][..., 0].shape == out["comp_rgb"][..., 0].shape: + valid_patch_key.append(key) + for key in valid_patch_key: + out_global[key] = F.interpolate( + out_global[key].permute(0, 3, 1, 2), (H, W), mode="bilinear" + ).permute(0, 2, 3, 1) + if self.cfg.global_detach: + out_global[key] = out_global[key].detach() + out_global[key][ + :, patch_y : patch_y + PS, patch_x : patch_x + PS + ] = out[key] + out = out_global + else: + out = self.base_renderer( + rays_o, rays_d, light_positions, bg_color, **kwargs + ) + + return out + + def update_step( + self, epoch: int, global_step: int, on_load_weights: bool = False + ) -> None: + self.base_renderer.update_step(epoch, global_step, on_load_weights) + + def train(self, mode=True): + return self.base_renderer.train(mode) + + def eval(self): + return self.base_renderer.eval() diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/make_training_vid.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/make_training_vid.py new file mode 100644 index 0000000000000000000000000000000000000000..2dee971043147d52fc400106b3c18d6fa1720e20 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/make_training_vid.py @@ -0,0 +1,77 @@ +# make_training_vid("outputs/zero123/64_teddy_rgba.png@20230627-195615", frames_per_vid=30, fps=20, max_iters=200) +import argparse +import glob +import os + +import imageio +import numpy as np +from PIL import Image, ImageDraw +from tqdm import tqdm + + +def draw_text_in_image(img, texts): + img = Image.fromarray(img) + draw = ImageDraw.Draw(img) + black, white = (0, 0, 0), (255, 255, 255) + for i, text in enumerate(texts): + draw.text((2, (img.size[1] // len(texts)) * i + 1), f"{text}", white) + draw.text((0, (img.size[1] // len(texts)) * i + 1), f"{text}", white) + draw.text((2, (img.size[1] // len(texts)) * i - 1), f"{text}", white) + draw.text((0, (img.size[1] // len(texts)) * i - 1), f"{text}", white) + draw.text((1, (img.size[1] // len(texts)) * i), f"{text}", black) + return np.asarray(img) + + +def make_training_vid(exp, frames_per_vid=1, fps=3, max_iters=None, max_vids=None): + # exp = "/admin/home-vikram/git/threestudio/outputs/zero123/64_teddy_rgba.png@20230627-195615" + files = glob.glob(os.path.join(exp, "save", "*.mp4")) + if os.path.join(exp, "save", "training_vid.mp4") in files: + files.remove(os.path.join(exp, "save", "training_vid.mp4")) + its = [int(os.path.basename(file).split("-")[0].split("it")[-1]) for file in files] + it_sort = np.argsort(its) + files = list(np.array(files)[it_sort]) + its = list(np.array(its)[it_sort]) + max_vids = max_iters // its[0] if max_iters is not None else max_vids + files, its = files[:max_vids], its[:max_vids] + frames, i = [], 0 + for it, file in tqdm(zip(its, files), total=len(files)): + vid = imageio.mimread(file) + for _ in range(frames_per_vid): + frame = vid[i % len(vid)] + frame = draw_text_in_image(frame, [str(it)]) + frames.append(frame) + i += 1 + # Save + imageio.mimwrite(os.path.join(exp, "save", "training_vid.mp4"), frames, fps=fps) + + +def join(file1, file2, name): + # file1 = "/admin/home-vikram/git/threestudio/outputs/zero123/OLD_64_dragon2_rgba.png@20230629-023028/save/it200-val.mp4" + # file2 = "/admin/home-vikram/git/threestudio/outputs/zero123/64_dragon2_rgba.png@20230628-152734/save/it200-val.mp4" + vid1 = imageio.mimread(file1) + vid2 = imageio.mimread(file2) + frames = [] + for f1, f2 in zip(vid1, vid2): + frames.append( + np.concatenate([f1[:, : f1.shape[0]], f2[:, : f2.shape[0]]], axis=1) + ) + imageio.mimwrite(name, frames) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--exp", help="directory of experiment") + parser.add_argument( + "--frames_per_vid", type=int, default=1, help="# of frames from each val vid" + ) + parser.add_argument("--fps", type=int, help="max # of iters to save") + parser.add_argument("--max_iters", type=int, help="max # of iters to save") + parser.add_argument( + "--max_vids", + type=int, + help="max # of val videos to save. Will be overridden by max_iters", + ) + args = parser.parse_args() + make_training_vid( + args.exp, args.frames_per_vid, args.fps, args.max_iters, args.max_vids + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123.sh b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123.sh new file mode 100644 index 0000000000000000000000000000000000000000..6f21bd7abf95fd92479fea5c2612aa84ad6612a4 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123.sh @@ -0,0 +1,13 @@ +NAME="dragon2" + +# Phase 1 - 64x64 +python launch.py --config configs/zero123.yaml --train --gpu 7 data.image_path=./load/images/${NAME}_rgba.png use_timestamp=False name=${NAME} tag=Phase1 # system.freq.guidance_eval=0 system.loggers.wandb.enable=false system.loggers.wandb.project="zero123" system.loggers.wandb.name=${NAME}_Phase1 + +# Phase 1.5 - 512 refine +python launch.py --config configs/zero123-geometry.yaml --train --gpu 4 data.image_path=./load/images/${NAME}_rgba.png system.geometry_convert_from=./outputs/${NAME}/Phase1/ckpts/last.ckpt use_timestamp=False name=${NAME} tag=Phase1p5 # system.freq.guidance_eval=0 system.loggers.wandb.enable=false system.loggers.wandb.project="zero123" system.loggers.wandb.name=${NAME}_Phase1p5 + +# Phase 2 - dreamfusion +python launch.py --config configs/experimental/imagecondition_zero123nerf.yaml --train --gpu 5 data.image_path=./load/images/${NAME}_rgba.png system.prompt_processor.prompt="A 3D model of a friendly dragon" system.weights="/admin/home-vikram/git/threestudio/outputs/${NAME}/Phase1/ckpts/last.ckpt" name=${NAME} tag=Phase2 # system.freq.guidance_eval=0 system.loggers.wandb.enable=false system.loggers.wandb.project="zero123" system.loggers.wandb.name=${NAME}_Phase2 + +# Phase 2 - SDF + dreamfusion +python launch.py --config configs/experimental/imagecondition_zero123nerf_refine.yaml --train --gpu 5 data.image_path=./load/images/${NAME}_rgba.png system.prompt_processor.prompt="A 3D model of a friendly dragon" system.geometry_convert_from="/admin/home-vikram/git/threestudio/outputs/${NAME}/Phase1/ckpts/last.ckpt" name=${NAME} tag=Phase2_refine # system.freq.guidance_eval=0 system.loggers.wandb.enable=false system.loggers.wandb.project="zero123" system.loggers.wandb.name=${NAME}_Phase2_refine diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_comparison.sh b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_comparison.sh new file mode 100644 index 0000000000000000000000000000000000000000..7f9f60a76a61dd1e5992fea157fb28427e112736 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_comparison.sh @@ -0,0 +1,23 @@ +# with standard zero123 +threestudio/scripts/run_zero123_phase.sh 6 anya_front 105000 0 + +# with zero123XL (not released yet!) +threestudio/scripts/run_zero123_phase.sh 1 anya_front XL_20230604 0 +threestudio/scripts/run_zero123_phase.sh 2 baby_phoenix_on_ice XL_20230604 20 +threestudio/scripts/run_zero123_phase.sh 3 beach_house_1 XL_20230604 50 +threestudio/scripts/run_zero123_phase.sh 4 bollywood_actress XL_20230604 0 +threestudio/scripts/run_zero123_phase.sh 5 beach_house_2 XL_20230604 30 +threestudio/scripts/run_zero123_phase.sh 6 hamburger XL_20230604 10 +threestudio/scripts/run_zero123_phase.sh 7 cactus XL_20230604 8 +threestudio/scripts/run_zero123_phase.sh 0 catstatue XL_20230604 50 +threestudio/scripts/run_zero123_phase.sh 1 church_ruins XL_20230604 0 +threestudio/scripts/run_zero123_phase.sh 2 firekeeper XL_20230604 10 +threestudio/scripts/run_zero123_phase.sh 3 futuristic_car XL_20230604 20 +threestudio/scripts/run_zero123_phase.sh 4 mona_lisa XL_20230604 10 +threestudio/scripts/run_zero123_phase.sh 5 teddy XL_20230604 20 + +# set guidance_eval to 0, to greatly speed up training +threestudio/scripts/run_zero123_phase.sh 7 anya_front XL_20230604 0 system.freq.guidance_eval=0 + +# disable wandb for faster training (or if you don't want to use it) +threestudio/scripts/run_zero123_phase.sh 7 anya_front XL_20230604 0 system.loggers.wandb.enable=false system.freq.guidance_eval=0 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_phase.sh b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_phase.sh new file mode 100644 index 0000000000000000000000000000000000000000..03c14612eaf2c2e32cde75250aa57de61f3730c7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_phase.sh @@ -0,0 +1,14 @@ + +GPU_ID=$1 # e.g. 0 +IMAGE_PREFIX=$2 # e.g. "anya_front" +ZERO123_PREFIX=$3 # e.g. "zero123-xl" +ELEVATION=$4 # e.g. 0 +REST=${@:5:99} # e.g. "system.guidance.min_step_percent=0.1 system.guidance.max_step_percent=0.9" + +# change this config if you don't use wandb or want to speed up training +python launch.py --config configs/zero123.yaml --train --gpu $GPU_ID system.loggers.wandb.enable=true system.loggers.wandb.project="claforte-noise_atten" \ + system.loggers.wandb.name="${IMAGE_PREFIX}_zero123_${ZERO123_PREFIX}...fov20_${REST}" \ + data.image_path=./load/images/${IMAGE_PREFIX}_rgba.png system.freq.guidance_eval=37 \ + system.guidance.pretrained_model_name_or_path="./load/zero123/${ZERO123_PREFIX}.ckpt" \ + system.guidance.cond_elevation_deg=$ELEVATION \ + ${REST} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_phase2.sh b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_phase2.sh new file mode 100644 index 0000000000000000000000000000000000000000..c6b964233910ce3dcd3bfdf7978d4bc9b498102a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_phase2.sh @@ -0,0 +1,5 @@ +# Reconstruct Anya using latest Zero123XL, in <2000 steps. +python launch.py --config configs/zero123.yaml --train --gpu 0 system.loggers.wandb.enable=true system.loggers.wandb.project="voletiv-anya-new" system.loggers.wandb.name="claforte_params" data.image_path=./load/images/anya_front_rgba.png system.freq.ref_or_zero123="accumulate" system.freq.guidance_eval=13 system.guidance.pretrained_model_name_or_path="./load/zero123/zero123-xl.ckpt" + +# PHASE 2 +python launch.py --config configs/experimental/imagecondition_zero123nerf.yaml --train --gpu 0 system.prompt_processor.prompt="A DSLR 3D photo of a cute anime schoolgirl stands proudly with her arms in the air, pink hair ( unreal engine 5 trending on Artstation Ghibli 4k )" system.weights=outputs/zero123/128_anya_front_rgba.png@20230623-145711/ckpts/last.ckpt system.freq.guidance_eval=13 system.loggers.wandb.enable=true system.loggers.wandb.project="voletiv-anya-new" data.image_path=./load/images/anya_front_rgba.png system.loggers.wandb.name="anya" data.random_camera.progressive_until=500 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_sbatch.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_sbatch.py new file mode 100644 index 0000000000000000000000000000000000000000..8ec2642df4149319c4ef7d0c87ce4c64ebecba9d --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/run_zero123_sbatch.py @@ -0,0 +1,33 @@ +import os +import time + +files = [ + "~/git/threestudio/load/images/dog1_rgba.png", + "~/git/threestudio/load/images/dragon2_rgba.png", +] + +for file in files: + name = os.path.basename(file).split("_rgba.png")[0] + with open( + os.path.expanduser("~/git/threestudio/threestudio/scripts/zero123_sbatch.sh"), + "w", + ) as f: + f.write("#!/bin/bash\n") + f.write(f"#SBATCH --job-name=vikky_{name}\n") + f.write("#SBATCH --account=mod3d\n") + f.write("#SBATCH --partition=g40\n") + f.write("#SBATCH --gpus=1\n") + f.write("#SBATCH --time=0-00:07:00\n") + f.write("conda activate three\n") + f.write("cd ~/git/threestudio/\n") + f.write(f"NAME={name}\n") + # Phase 1 + f.write( + "python launch.py --config configs/zero123.yaml --train data.image_path=./load/images/${NAME}_rgba.png use_timestamp=true name=${NAME} tag=Phase1 system.loggers.wandb.enable=false system.loggers.wandb.project='zero123' system.loggers.wandb.name=${NAME}_Phase1\n" + ) + # # Phase 1.5 + # f.write( + # "python launch.py --config configs/zero123-geometry.yaml --train data.image_path=./load/images/${NAME}_rgba.png system.geometry_convert_from=./outputs/${NAME}/Phase1/ckpts/last.ckpt use_timestamp=False name=${NAME} tag=Phase1p5 system.loggers.wandb.enable=true system.loggers.wandb.project='zero123' system.loggers.wandb.name=${NAME}_Phase1p5\n" + # ) + os.system("sbatch ~/git/threestudio/threestudio/scripts/zero123_sbatch.sh") + time.sleep(1) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/zero123_demo.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/zero123_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..03c9e5ff7e56de2f422f99a18bc1e118485457e5 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/zero123_demo.py @@ -0,0 +1,61 @@ +# 1. Generate using StableDiffusionXL https://clipdrop.co/stable-diffusion + +# 2. Remove background https://clipdrop.co/remove-background + +# 3. Resize to 512x512 https://www.iloveimg.com/resize-image + +# (OPTIONAL) +# 4. Estimate depth and normal https://omnidata.vision/demo/ (I used Omnidata Normal (with X-TC & 3DCC), and MiDaS Depth) + + +# (OPTIONAL) +# 5. Convert depth image from RGB to greyscale +def depth_rgb_to_grey(depth_filename): + # depth_filename = "image_depth.png" + import cv2 + import numpy as np + + # import shutil + # shutil.copyfile(depth_filename, depth_filename.replace("_depth", "_depth_orig")) + depth = cv2.imread(depth_filename) + depth = cv2.cvtColor(depth, cv2.COLOR_BGR2GRAY) + mask = ( + cv2.resize( + cv2.imread(depth_filename.replace("_depth", "_rgba"), cv2.IMREAD_UNCHANGED)[ + :, :, -1 + ], + depth.shape, + ) + > 0 + ) + # depth[mask] = (depth[mask] - depth.min()) / (depth.max() - depth.min() + 1e-9) + depth = (depth - depth.min()) / (depth.max() - depth.min() + 1e-9) + depth[~mask] = 0 + depth = (depth * 255).astype(np.uint8) + cv2.imwrite(depth_filename, depth) + + +# (OPTIONAL) +# 6. Mask normal +def normal_mask(normal_filename): + # filename = "image_normal.png" + import cv2 + + # import shutil + # shutil.copyfile(normal_filename, normal_filename.replace("_normal", "_normal_orig")) + normal = cv2.imread(normal_filename) + mask = ( + cv2.resize( + cv2.imread( + normal_filename.replace("_normal", "_rgba"), cv2.IMREAD_UNCHANGED + )[:, :, -1], + normal.shape[:2], + ) + > 0 + ) + normal[~mask] = 0 + cv2.imwrite(normal_filename, normal) + + +# 5. Run Zero123 +# python launch.py --config configs/zero123.yaml --train data.image_path=./load/images/grootplant_rgba.png diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/zero123_sbatch.sh b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/zero123_sbatch.sh new file mode 100644 index 0000000000000000000000000000000000000000..bac234180c0585be4529f42ceeb6357b4935408a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/scripts/zero123_sbatch.sh @@ -0,0 +1,10 @@ +#!/bin/bash +#SBATCH --job-name=vikky +#SBATCH --account=mod3d +#SBATCH --partition=g40 +#SBATCH --gpus=1 +#SBATCH --time=0-00:07:00 +conda activate three +cd ~/git/threestudio/ +NAME="dog1" +python launch.py --config configs/zero123.yaml --train data.image_path=./load/images/${NAME}_rgba.png use_timestamp=False name=${NAME} tag=Phase1 system.loggers.wandb.enable=true system.loggers.wandb.project='zero123' system.loggers.wandb.name=${NAME}_Phase1 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..edbe7bf28c3ccd2b8d9c41e595a780911168bc5c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/__init__.py @@ -0,0 +1,15 @@ +from . import ( + control4d_multiview, + dreamfusion, + fantasia3d, + imagedreamfusion, + instructnerf2nerf, + latentnerf, + magic3d, + magic123, + prolificdreamer, + sjc, + textmesh, + zero123, + zero123_simple, +) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/base.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/base.py new file mode 100644 index 0000000000000000000000000000000000000000..73faac60960aad98d74d5981c43d267873d30d49 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/base.py @@ -0,0 +1,402 @@ +import os +from dataclasses import dataclass, field + +import pytorch_lightning as pl +import torch.nn.functional as F + +import threestudio +from threestudio.models.exporters.base import Exporter, ExporterOutput +from threestudio.systems.utils import parse_optimizer, parse_scheduler +from threestudio.utils.base import ( + Updateable, + update_end_if_possible, + update_if_possible, +) +from threestudio.utils.config import parse_structured +from threestudio.utils.misc import ( + C, + cleanup, + find_last_path, + get_device, + load_module_weights, +) +from threestudio.utils.saving import SaverMixin +from threestudio.utils.typing import * + + +class BaseSystem(pl.LightningModule, Updateable, SaverMixin): + @dataclass + class Config: + loggers: dict = field(default_factory=dict) + loss: dict = field(default_factory=dict) + optimizer: dict = field(default_factory=dict) + scheduler: Optional[dict] = None + weights: Optional[str] = None + weights_ignore_modules: Optional[List[str]] = None + cleanup_after_validation_step: bool = False + cleanup_after_test_step: bool = False + + cfg: Config + + def __init__(self, cfg, resumed=False) -> None: + super().__init__() + self.cfg = parse_structured(self.Config, cfg) + self._save_dir: Optional[str] = None + self._resumed: bool = resumed + self._resumed_eval: bool = False + self._resumed_eval_status: dict = {"global_step": 0, "current_epoch": 0} + if "loggers" in cfg: + self.create_loggers(cfg.loggers) + + self.configure() + if self.cfg.weights is not None: + self.load_weights(self.cfg.weights, self.cfg.weights_ignore_modules) + self.post_configure() + + def load_weights(self, weights: str, ignore_modules: Optional[List[str]] = None): + state_dict, epoch, global_step = load_module_weights( + weights, ignore_modules=ignore_modules, map_location="cpu" + ) + self.load_state_dict(state_dict, strict=False) + # restore step-dependent states + self.do_update_step(epoch, global_step, on_load_weights=True) + + def set_resume_status(self, current_epoch: int, global_step: int): + # restore correct epoch and global step in eval + self._resumed_eval = True + self._resumed_eval_status["current_epoch"] = current_epoch + self._resumed_eval_status["global_step"] = global_step + + @property + def resumed(self): + # whether from resumed checkpoint + return self._resumed + + @property + def true_global_step(self): + if self._resumed_eval: + return self._resumed_eval_status["global_step"] + else: + return self.global_step + + @property + def true_current_epoch(self): + if self._resumed_eval: + return self._resumed_eval_status["current_epoch"] + else: + return self.current_epoch + + def configure(self) -> None: + pass + + def post_configure(self) -> None: + """ + executed after weights are loaded + """ + pass + + def C(self, value: Any) -> float: + return C(value, self.true_current_epoch, self.true_global_step) + + def configure_optimizers(self): + optim = parse_optimizer(self.cfg.optimizer, self) + ret = { + "optimizer": optim, + } + if self.cfg.scheduler is not None: + ret.update( + { + "lr_scheduler": parse_scheduler(self.cfg.scheduler, optim), + } + ) + return ret + + def training_step(self, batch, batch_idx): + raise NotImplementedError + + def validation_step(self, batch, batch_idx): + raise NotImplementedError + + def on_train_batch_end(self, outputs, batch, batch_idx): + self.dataset = self.trainer.train_dataloader.dataset + update_end_if_possible( + self.dataset, self.true_current_epoch, self.true_global_step + ) + self.do_update_step_end(self.true_current_epoch, self.true_global_step) + + def on_validation_batch_end(self, outputs, batch, batch_idx): + self.dataset = self.trainer.val_dataloaders.dataset + update_end_if_possible( + self.dataset, self.true_current_epoch, self.true_global_step + ) + self.do_update_step_end(self.true_current_epoch, self.true_global_step) + if self.cfg.cleanup_after_validation_step: + # cleanup to save vram + cleanup() + + def on_validation_epoch_end(self): + raise NotImplementedError + + def test_step(self, batch, batch_idx): + raise NotImplementedError + + def on_test_batch_end(self, outputs, batch, batch_idx): + self.dataset = self.trainer.test_dataloaders.dataset + update_end_if_possible( + self.dataset, self.true_current_epoch, self.true_global_step + ) + self.do_update_step_end(self.true_current_epoch, self.true_global_step) + if self.cfg.cleanup_after_test_step: + # cleanup to save vram + cleanup() + + def on_test_epoch_end(self): + pass + + def predict_step(self, batch, batch_idx): + raise NotImplementedError + + def on_predict_batch_end(self, outputs, batch, batch_idx): + self.dataset = self.trainer.predict_dataloaders.dataset + update_end_if_possible( + self.dataset, self.true_current_epoch, self.true_global_step + ) + self.do_update_step_end(self.true_current_epoch, self.true_global_step) + if self.cfg.cleanup_after_test_step: + # cleanup to save vram + cleanup() + + def on_predict_epoch_end(self): + pass + + def preprocess_data(self, batch, stage): + pass + + """ + Implementing on_after_batch_transfer of DataModule does the same. + But on_after_batch_transfer does not support DP. + """ + + def on_train_batch_start(self, batch, batch_idx, unused=0): + self.preprocess_data(batch, "train") + self.dataset = self.trainer.train_dataloader.dataset + update_if_possible(self.dataset, self.true_current_epoch, self.true_global_step) + self.do_update_step(self.true_current_epoch, self.true_global_step) + + def on_validation_batch_start(self, batch, batch_idx, dataloader_idx=0): + self.preprocess_data(batch, "validation") + self.dataset = self.trainer.val_dataloaders.dataset + update_if_possible(self.dataset, self.true_current_epoch, self.true_global_step) + self.do_update_step(self.true_current_epoch, self.true_global_step) + + def on_test_batch_start(self, batch, batch_idx, dataloader_idx=0): + self.preprocess_data(batch, "test") + self.dataset = self.trainer.test_dataloaders.dataset + update_if_possible(self.dataset, self.true_current_epoch, self.true_global_step) + self.do_update_step(self.true_current_epoch, self.true_global_step) + + def on_predict_batch_start(self, batch, batch_idx, dataloader_idx=0): + self.preprocess_data(batch, "predict") + self.dataset = self.trainer.predict_dataloaders.dataset + update_if_possible(self.dataset, self.true_current_epoch, self.true_global_step) + self.do_update_step(self.true_current_epoch, self.true_global_step) + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + pass + + def on_before_optimizer_step(self, optimizer): + """ + # some gradient-related debugging goes here, example: + from lightning.pytorch.utilities import grad_norm + norms = grad_norm(self.geometry, norm_type=2) + print(norms) + """ + pass + + +class BaseLift3DSystem(BaseSystem): + @dataclass + class Config(BaseSystem.Config): + geometry_type: str = "" + geometry: dict = field(default_factory=dict) + geometry_convert_from: Optional[str] = None + geometry_convert_inherit_texture: bool = False + # used to override configurations of the previous geometry being converted from, + # for example isosurface_threshold + geometry_convert_override: dict = field(default_factory=dict) + + material_type: str = "" + material: dict = field(default_factory=dict) + + background_type: str = "" + background: dict = field(default_factory=dict) + + renderer_type: str = "" + renderer: dict = field(default_factory=dict) + + guidance_type: str = "" + guidance: dict = field(default_factory=dict) + + prompt_processor_type: str = "" + prompt_processor: dict = field(default_factory=dict) + + # geometry export configurations, no need to specify in training + exporter_type: str = "mesh-exporter" + exporter: dict = field(default_factory=dict) + + cfg: Config + + def configure(self) -> None: + self.cfg.geometry_convert_from = find_last_path(self.cfg.geometry_convert_from) + self.cfg.weights = find_last_path(self.cfg.weights) + if ( + self.cfg.geometry_convert_from # from_coarse must be specified + and not self.cfg.weights # not initialized from coarse when weights are specified + and not self.resumed # not initialized from coarse when resumed from checkpoints + ): + threestudio.info("Initializing geometry from a given checkpoint ...") + from threestudio.utils.config import load_config, parse_structured + + prev_cfg = load_config( + os.path.join( + os.path.dirname(self.cfg.geometry_convert_from), + "../configs/parsed.yaml", + ) + ) # TODO: hard-coded relative path + prev_system_cfg: BaseLift3DSystem.Config = parse_structured( + self.Config, prev_cfg.system + ) + prev_geometry_cfg = prev_system_cfg.geometry + prev_geometry_cfg.update(self.cfg.geometry_convert_override) + prev_geometry = threestudio.find(prev_system_cfg.geometry_type)( + prev_geometry_cfg + ) + state_dict, epoch, global_step = load_module_weights( + self.cfg.geometry_convert_from, + module_name="geometry", + map_location="cpu", + ) + prev_geometry.load_state_dict(state_dict, strict=False) + # restore step-dependent states + prev_geometry.do_update_step(epoch, global_step, on_load_weights=True) + # convert from coarse stage geometry + prev_geometry = prev_geometry.to(get_device()) + self.geometry = threestudio.find(self.cfg.geometry_type).create_from( + prev_geometry, + self.cfg.geometry, + copy_net=self.cfg.geometry_convert_inherit_texture, + ) + del prev_geometry + cleanup() + else: + self.geometry = threestudio.find(self.cfg.geometry_type)(self.cfg.geometry) + + self.material = threestudio.find(self.cfg.material_type)(self.cfg.material) + self.background = threestudio.find(self.cfg.background_type)( + self.cfg.background + ) + self.renderer = threestudio.find(self.cfg.renderer_type)( + self.cfg.renderer, + geometry=self.geometry, + material=self.material, + background=self.background, + ) + + def on_fit_start(self) -> None: + if self._save_dir is not None: + threestudio.info(f"Validation results will be saved to {self._save_dir}") + else: + threestudio.warn( + f"Saving directory not set for the system, visualization results will not be saved" + ) + + def on_test_end(self) -> None: + if self._save_dir is not None: + threestudio.info(f"Test results saved to {self._save_dir}") + + def on_predict_start(self) -> None: + self.exporter: Exporter = threestudio.find(self.cfg.exporter_type)( + self.cfg.exporter, + geometry=self.geometry, + material=self.material, + background=self.background, + ) + + def predict_step(self, batch, batch_idx): + if self.exporter.cfg.save_video: + self.test_step(batch, batch_idx) + + def on_predict_epoch_end(self) -> None: + if self.exporter.cfg.save_video: + self.on_test_epoch_end() + exporter_output: List[ExporterOutput] = self.exporter() + for out in exporter_output: + save_func_name = f"save_{out.save_type}" + if not hasattr(self, save_func_name): + raise ValueError(f"{save_func_name} not supported by the SaverMixin") + save_func = getattr(self, save_func_name) + save_func(f"it{self.true_global_step}-export/{out.save_name}", **out.params) + + def on_predict_end(self) -> None: + if self._save_dir is not None: + threestudio.info(f"Export assets saved to {self._save_dir}") + + def guidance_evaluation_save(self, comp_rgb, guidance_eval_out): + B, size = comp_rgb.shape[:2] + resize = lambda x: F.interpolate( + x.permute(0, 3, 1, 2), (size, size), mode="bilinear", align_corners=False + ).permute(0, 2, 3, 1) + filename = f"it{self.true_global_step}-train.png" + + def merge12(x): + return x.reshape(-1, *x.shape[2:]) + + self.save_image_grid( + filename, + [ + { + "type": "rgb", + "img": merge12(comp_rgb), + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": merge12(resize(guidance_eval_out["imgs_noisy"])), + "kwargs": {"data_format": "HWC"}, + } + ] + ) + + ( + [ + { + "type": "rgb", + "img": merge12(resize(guidance_eval_out["imgs_1step"])), + "kwargs": {"data_format": "HWC"}, + } + ] + ) + + ( + [ + { + "type": "rgb", + "img": merge12(resize(guidance_eval_out["imgs_1orig"])), + "kwargs": {"data_format": "HWC"}, + } + ] + ) + + ( + [ + { + "type": "rgb", + "img": merge12(resize(guidance_eval_out["imgs_final"])), + "kwargs": {"data_format": "HWC"}, + } + ] + ), + name="train_step", + step=self.true_global_step, + texts=guidance_eval_out["texts"], + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/control4d_multiview.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/control4d_multiview.py new file mode 100644 index 0000000000000000000000000000000000000000..8cfd9cf534adfd33c3202a9448c6fd1316c8618e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/control4d_multiview.py @@ -0,0 +1,286 @@ +import os +from dataclasses import dataclass, field + +import torch +import torch.nn.functional as F + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.systems.utils import parse_optimizer +from threestudio.utils.GAN.loss import discriminator_loss, generator_loss +from threestudio.utils.misc import cleanup, get_device +from threestudio.utils.ops import binary_cross_entropy, dot +from threestudio.utils.perceptual import PerceptualLoss +from threestudio.utils.typing import * + + +@threestudio.register("control4d-multiview-system") +class Control4D(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + per_editing_step: int = 20 + start_editing_step: int = 2000 + + cfg: Config + + def configure(self) -> None: + # override the default configure function + self.material = threestudio.find(self.cfg.material_type)(self.cfg.material) + self.background = threestudio.find(self.cfg.background_type)( + self.cfg.background + ) + self.geometry = threestudio.find(self.cfg.geometry_type)(self.cfg.geometry) + + self.renderer = threestudio.find(self.cfg.renderer_type)( + self.cfg.renderer, + geometry=self.geometry, + material=self.material, + background=self.background, + ) + p_config = {} + self.perceptual_loss = threestudio.find("perceptual-loss")(p_config) + self.edit_frames = {} + self.per_editing_step = self.cfg.per_editing_step + self.start_editing_step = self.cfg.start_editing_step + + self.automatic_optimization = False + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + render_out = self.renderer(**batch) + return { + **render_out, + } + + def on_fit_start(self) -> None: + super().on_fit_start() + # only used in training + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + def training_step(self, batch, batch_idx): + optimizer_g, optimizer_d = self.optimizers() + self.toggle_optimizer(optimizer_g) + + if torch.is_tensor(batch["index"]): + batch_index = batch["index"].item() + else: + batch_index = batch["index"] + batch["multi_level_guidance"] = True + + origin_gt_rgb = batch["gt_rgb"] + B, H, W, C = origin_gt_rgb.shape + if batch_index in self.edit_frames: + gt_rgb = self.edit_frames[batch_index].to(batch["gt_rgb"].device) + gt_rgb = torch.nn.functional.interpolate( + gt_rgb.permute(0, 3, 1, 2), (H, W), mode="bilinear", align_corners=False + ).permute(0, 2, 3, 1) + batch["gt_rgb"] = gt_rgb + else: + gt_rgb = origin_gt_rgb + out = self(batch) + if self.per_editing_step > 0 and self.global_step > self.start_editing_step: + prompt_utils = self.prompt_processor() + if ( + not batch_index in self.edit_frames + or self.global_step % self.per_editing_step == 0 + ): + result = self.guidance(out["comp_gan_rgb"], origin_gt_rgb, prompt_utils) + self.edit_frames[batch_index] = result["edit_images"].detach().cpu() + + loss = 0.0 + # loss of generator level 0 + loss_l1 = F.l1_loss(out["comp_int_rgb"], out["comp_gt_rgb"]) + loss_p = 0.0 + loss_kl = out["posterior"].kl().mean() + loss_G = generator_loss( + self.renderer.discriminator, + gt_rgb.permute(0, 3, 1, 2), + out["comp_gan_rgb"].permute(0, 3, 1, 2), + ) + + generator_level = out["generator_level"] + + level_ratio = 1.0 if generator_level == 2 else 0.1 + loss_l1 += F.l1_loss(out["comp_gan_rgb"], gt_rgb) * level_ratio + lr_gan_rgb = F.interpolate( + out["comp_gan_rgb"].permute(0, 3, 1, 2), (H // 4, W // 4), mode="area" + ) + lr_rgb = F.interpolate( + out["comp_rgb"].permute(0, 3, 1, 2), (H // 4, W // 4), mode="area" + ).detach() + loss_l1 += F.l1_loss(lr_gan_rgb, lr_rgb).sum() * level_ratio * 0.25 + + level_ratio = 1.0 if generator_level >= 1 else 0.1 + loss_p += ( + self.perceptual_loss( + out["comp_gan_rgb"].permute(0, 3, 1, 2).contiguous(), + gt_rgb.permute(0, 3, 1, 2).contiguous(), + ).sum() + * level_ratio + ) + + guidance_out = { + "loss_l1": loss_l1, + "loss_p": loss_p, + "loss_G": loss_G, + "loss_kl": loss_kl, + } + + for name, value in guidance_out.items(): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss += value * self.C(self.cfg.loss[name.replace("loss_", "lambda_")]) + + if self.C(self.cfg.loss.lambda_orient) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for orientation loss, no normal is found in the output." + ) + loss_orient = ( + out["weights"].detach() + * dot(out["normal"], out["t_dirs"]).clamp_min(0.0) ** 2 + ).sum() / (out["opacity"] > 0).sum() + self.log("train/loss_orient", loss_orient) + loss += loss_orient * self.C(self.cfg.loss.lambda_orient) + + loss_sparsity = (out["opacity"] ** 2 + 0.01).sqrt().mean() + self.log("train/loss_sparsity", loss_sparsity) + loss += loss_sparsity * self.C(self.cfg.loss.lambda_sparsity) + + opacity_clamped = out["opacity"].clamp(1.0e-3, 1.0 - 1.0e-3) + loss_opaque = binary_cross_entropy(opacity_clamped, opacity_clamped) + self.log("train/loss_opaque", loss_opaque) + loss += loss_opaque * self.C(self.cfg.loss.lambda_opaque) + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + self.manual_backward(loss) + optimizer_g.step() + optimizer_g.zero_grad() + self.untoggle_optimizer(optimizer_g) + + self.toggle_optimizer(optimizer_d) + loss_D = discriminator_loss( + self.renderer.discriminator, + gt_rgb.permute(0, 3, 1, 2), + out["comp_gan_rgb"].permute(0, 3, 1, 2), + ) + loss_D *= self.C(self.cfg.loss["lambda_D"]) + self.log("train/loss_D", loss_D) + self.manual_backward(loss_D) + optimizer_d.step() + optimizer_d.zero_grad() + self.untoggle_optimizer(optimizer_d) + + def validation_step(self, batch, batch_idx): + out = self(batch) + if torch.is_tensor(batch["index"]): + batch_index = batch["index"].item() + else: + batch_index = batch["index"] + if batch_index in self.edit_frames: + B, H, W, C = batch["gt_rgb"].shape + rgb = torch.nn.functional.interpolate( + self.edit_frames[batch_index].permute(0, 3, 1, 2), (H, W) + ).permute(0, 2, 3, 1)[0] + else: + rgb = batch["gt_rgb"][0] + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}.jpg", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + [ + { + "type": "rgb", + "img": out["comp_gan_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ] + + [ + { + "type": "rgb", + "img": rgb, + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + }, + ], + name="validation_step", + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_gan_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) + + def configure_optimizers(self): + optimizer_g = parse_optimizer(self.cfg.optimizer, self) + optimizer_d = parse_optimizer(self.cfg.optimizer.optimizer_dis, self) + return [optimizer_g, optimizer_d], [] diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/dreamfusion.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/dreamfusion.py new file mode 100644 index 0000000000000000000000000000000000000000..4e594b6ebeed82f1cb1005372c21cc49b4cb422c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/dreamfusion.py @@ -0,0 +1,162 @@ +from dataclasses import dataclass, field + +import torch + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.ops import binary_cross_entropy, dot +from threestudio.utils.typing import * + + +@threestudio.register("dreamfusion-system") +class DreamFusion(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + pass + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + render_out = self.renderer(**batch) + return { + **render_out, + } + + def on_fit_start(self) -> None: + super().on_fit_start() + # only used in training + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + def training_step(self, batch, batch_idx): + out = self(batch) + prompt_utils = self.prompt_processor() + guidance_out = self.guidance( + out["comp_rgb"], prompt_utils, **batch, rgb_as_latents=False + ) + + loss = 0.0 + + for name, value in guidance_out.items(): + if not (type(value) is torch.Tensor and value.numel() > 1): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss += value * self.C(self.cfg.loss[name.replace("loss_", "lambda_")]) + + if self.C(self.cfg.loss.lambda_orient) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for orientation loss, no normal is found in the output." + ) + loss_orient = ( + out["weights"].detach() + * dot(out["normal"], out["t_dirs"]).clamp_min(0.0) ** 2 + ).sum() / (out["opacity"] > 0).sum() + self.log("train/loss_orient", loss_orient) + loss += loss_orient * self.C(self.cfg.loss.lambda_orient) + + loss_sparsity = (out["opacity"] ** 2 + 0.01).sqrt().mean() + self.log("train/loss_sparsity", loss_sparsity) + loss += loss_sparsity * self.C(self.cfg.loss.lambda_sparsity) + + opacity_clamped = out["opacity"].clamp(1.0e-3, 1.0 - 1.0e-3) + loss_opaque = binary_cross_entropy(opacity_clamped, opacity_clamped) + self.log("train/loss_opaque", loss_opaque) + loss += loss_opaque * self.C(self.cfg.loss.lambda_opaque) + + # z-variance loss proposed in HiFA: https://hifa-team.github.io/HiFA-site/ + if "z_variance" in out and "lambda_z_variance" in self.cfg.loss: + loss_z_variance = out["z_variance"][out["opacity"] > 0.5].mean() + self.log("train/loss_z_variance", loss_z_variance) + loss += loss_z_variance * self.C(self.cfg.loss.lambda_z_variance) + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + return {"loss": loss} + + def validation_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="validation_step", + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/fantasia3d.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/fantasia3d.py new file mode 100644 index 0000000000000000000000000000000000000000..26d8d635b519b751809a498d112e5f45f656a20a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/fantasia3d.py @@ -0,0 +1,168 @@ +from dataclasses import dataclass, field + +import torch +import torch.nn.functional as F + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.ops import binary_cross_entropy, dot +from threestudio.utils.typing import * + + +@threestudio.register("fantasia3d-system") +class Fantasia3D(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + latent_steps: int = 1000 + texture: bool = False + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + render_out = self.renderer(**batch, render_rgb=self.cfg.texture) + return { + **render_out, + } + + def on_fit_start(self) -> None: + super().on_fit_start() + # only used in training + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + if not self.cfg.texture: + # initialize SDF + # FIXME: what if using other geometry types? + self.geometry.initialize_shape() + + def training_step(self, batch, batch_idx): + loss = 0.0 + + out = self(batch) + prompt_utils = self.prompt_processor() + + if not self.cfg.texture: # geometry training + if self.true_global_step < self.cfg.latent_steps: + guidance_inp = torch.cat( + [out["comp_normal"] * 2.0 - 1.0, out["opacity"]], dim=-1 + ) + guidance_out = self.guidance( + guidance_inp, prompt_utils, **batch, rgb_as_latents=True + ) + else: + guidance_inp = out["comp_normal"] + guidance_out = self.guidance( + guidance_inp, prompt_utils, **batch, rgb_as_latents=False + ) + + loss_normal_consistency = out["mesh"].normal_consistency() + self.log("train/loss_normal_consistency", loss_normal_consistency) + loss += loss_normal_consistency * self.C( + self.cfg.loss.lambda_normal_consistency + ) + else: # texture training + guidance_inp = out["comp_rgb"] + if isinstance( + self.guidance, + threestudio.models.guidance.controlnet_guidance.ControlNetGuidance, + ): + cond_inp = out["comp_normal"] + guidance_out = self.guidance( + guidance_inp, cond_inp, prompt_utils, **batch, rgb_as_latents=False + ) + else: + guidance_out = self.guidance( + guidance_inp, prompt_utils, **batch, rgb_as_latents=False + ) + + for name, value in guidance_out.items(): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss += value * self.C(self.cfg.loss[name.replace("loss_", "lambda_")]) + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + return {"loss": loss} + + def validation_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}.png", + ( + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if self.cfg.texture + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + }, + ], + name="validation_step", + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + ( + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if self.cfg.texture + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + }, + ], + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/imagedreamfusion.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/imagedreamfusion.py new file mode 100644 index 0000000000000000000000000000000000000000..63108a35d8bba73cf6d758fc0de99713c83b2631 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/imagedreamfusion.py @@ -0,0 +1,387 @@ +import os +import random +import shutil +from dataclasses import dataclass, field + +import torch +import torch.nn.functional as F +from torchmetrics import PearsonCorrCoef + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.ops import binary_cross_entropy, dot +from threestudio.utils.typing import * + + +@threestudio.register("image-condition-dreamfusion-system") +class ImageConditionDreamFusion(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + freq: dict = field(default_factory=dict) + refinement: bool = False + ambient_ratio_min: float = 0.5 + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + render_out = self.renderer(**batch) + return { + **render_out, + } + + def on_fit_start(self) -> None: + super().on_fit_start() + # only used in training + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + # visualize all training images + all_images = self.trainer.datamodule.train_dataloader().dataset.get_all_images() + self.save_image_grid( + "all_training_images.png", + [ + {"type": "rgb", "img": image, "kwargs": {"data_format": "HWC"}} + for image in all_images + ], + name="on_fit_start", + step=self.true_global_step, + ) + + self.pearson = PearsonCorrCoef().to(self.device) + + def training_substep(self, batch, batch_idx, guidance: str): + """ + Args: + guidance: one of "ref" (reference image supervision), "guidance" + """ + if guidance == "ref": + # bg_color = torch.rand_like(batch['rays_o']) + ambient_ratio = 1.0 + shading = "diffuse" + batch["shading"] = shading + elif guidance == "guidance": + batch = batch["random_camera"] + ambient_ratio = ( + self.cfg.ambient_ratio_min + + (1 - self.cfg.ambient_ratio_min) * random.random() + ) + + batch["bg_color"] = None + batch["ambient_ratio"] = ambient_ratio + + out = self(batch) + loss_prefix = f"loss_{guidance}_" + + loss_terms = {} + + def set_loss(name, value): + loss_terms[f"{loss_prefix}{name}"] = value + + guidance_eval = ( + guidance == "guidance" + and self.cfg.freq.guidance_eval > 0 + and self.true_global_step % self.cfg.freq.guidance_eval == 0 + ) + + if guidance == "ref": + gt_mask = batch["mask"] + gt_rgb = batch["rgb"] + + # color loss + gt_rgb = gt_rgb * gt_mask.float() + out["comp_rgb_bg"] * ( + 1 - gt_mask.float() + ) + set_loss("rgb", F.mse_loss(gt_rgb, out["comp_rgb"])) + + # mask loss + set_loss("mask", F.mse_loss(gt_mask.float(), out["opacity"])) + + # depth loss + if self.C(self.cfg.loss.lambda_depth) > 0: + valid_gt_depth = batch["ref_depth"][gt_mask.squeeze(-1)].unsqueeze(1) + valid_pred_depth = out["depth"][gt_mask].unsqueeze(1) + with torch.no_grad(): + A = torch.cat( + [valid_gt_depth, torch.ones_like(valid_gt_depth)], dim=-1 + ) # [B, 2] + X = torch.linalg.lstsq(A, valid_pred_depth).solution # [2, 1] + valid_gt_depth = A @ X # [B, 1] + set_loss("depth", F.mse_loss(valid_gt_depth, valid_pred_depth)) + + # relative depth loss + if self.C(self.cfg.loss.lambda_depth_rel) > 0: + valid_gt_depth = batch["ref_depth"][gt_mask.squeeze(-1)] # [B,] + valid_pred_depth = out["depth"][gt_mask] # [B,] + set_loss( + "depth_rel", 1 - self.pearson(valid_pred_depth, valid_gt_depth) + ) + + # normal loss + if self.C(self.cfg.loss.lambda_normal) > 0: + valid_gt_normal = ( + 1 - 2 * batch["ref_normal"][gt_mask.squeeze(-1)] + ) # [B, 3] + valid_pred_normal = ( + 2 * out["comp_normal"][gt_mask.squeeze(-1)] - 1 + ) # [B, 3] + set_loss( + "normal", + 1 - F.cosine_similarity(valid_pred_normal, valid_gt_normal).mean(), + ) + elif guidance == "guidance": + self.guidance.set_min_max_steps( + self.C(self.guidance.cfg.min_step_percent), + self.C(self.guidance.cfg.max_step_percent), + ) + prompt_utils = self.prompt_processor() + guidance_out = self.guidance( + out["comp_rgb"], + prompt_utils, + **batch, + rgb_as_latents=False, + guidance_eval=guidance_eval, + ) + set_loss("sds", guidance_out["loss_sds"]) + + if self.C(self.cfg.loss.lambda_normal_smooth) > 0: + if "comp_normal" not in out: + raise ValueError( + "comp_normal is required for 2D normal smooth loss, no comp_normal is found in the output." + ) + normal = out["comp_normal"] + set_loss( + "normal_smooth", + (normal[:, 1:, :, :] - normal[:, :-1, :, :]).square().mean() + + (normal[:, :, 1:, :] - normal[:, :, :-1, :]).square().mean(), + ) + + if self.C(self.cfg.loss.lambda_3d_normal_smooth) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for normal smooth loss, no normal is found in the output." + ) + if "normal_perturb" not in out: + raise ValueError( + "normal_perturb is required for normal smooth loss, no normal_perturb is found in the output." + ) + normals = out["normal"] + normals_perturb = out["normal_perturb"] + set_loss("3d_normal_smooth", (normals - normals_perturb).abs().mean()) + + if not self.cfg.refinement: + if self.C(self.cfg.loss.lambda_orient) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for orientation loss, no normal is found in the output." + ) + set_loss( + "orient", + ( + out["weights"].detach() + * dot(out["normal"], out["t_dirs"]).clamp_min(0.0) ** 2 + ).sum() + / (out["opacity"] > 0).sum(), + ) + + if guidance != "ref" and self.C(self.cfg.loss.lambda_sparsity) > 0: + set_loss("sparsity", (out["opacity"] ** 2 + 0.01).sqrt().mean()) + + if self.C(self.cfg.loss.lambda_opaque) > 0: + opacity_clamped = out["opacity"].clamp(1.0e-3, 1.0 - 1.0e-3) + set_loss( + "opaque", binary_cross_entropy(opacity_clamped, opacity_clamped) + ) + else: + if self.C(self.cfg.loss.lambda_normal_consistency) > 0: + set_loss("normal_consistency", out["mesh"].normal_consistency()) + if self.C(self.cfg.loss.lambda_laplacian_smoothness) > 0: + set_loss("laplacian_smoothness", out["mesh"].laplacian()) + + loss = 0.0 + for name, value in loss_terms.items(): + self.log(f"train/{name}", value) + if name.startswith(loss_prefix): + loss_weighted = value * self.C( + self.cfg.loss[name.replace(loss_prefix, "lambda_")] + ) + self.log(f"train/{name}_w", loss_weighted) + loss += loss_weighted + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + self.log(f"train/loss_{guidance}", loss) + + if guidance_eval: + self.guidance_evaluation_save( + out["comp_rgb"].detach()[: guidance_out["eval"]["bs"]], + guidance_out["eval"], + ) + + return {"loss": loss} + + def training_step(self, batch, batch_idx): + total_loss = 0.0 + + # guidance + if self.true_global_step > self.cfg.freq.ref_only_steps: + out = self.training_substep(batch, batch_idx, guidance="guidance") + total_loss += out["loss"] + + # ref + out = self.training_substep(batch, batch_idx, guidance="ref") + total_loss += out["loss"] + + self.log("train/loss", total_loss, prog_bar=True) + + # sch = self.lr_schedulers() + # sch.step() + + return {"loss": total_loss} + + def validation_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-val/{batch['index'][0]}.png", + ( + [ + { + "type": "rgb", + "img": batch["rgb"][0], + "kwargs": {"data_format": "HWC"}, + } + ] + if "rgb" in batch + else [] + ) + + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + ( + [ + { + "type": "grayscale", + "img": out["depth"][0], + "kwargs": {}, + } + ] + if "depth" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name=f"validation_step_batchidx_{batch_idx}" + if batch_idx in [0, 7, 15, 23, 29] + else None, + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + filestem = f"it{self.true_global_step}-val" + self.save_img_sequence( + filestem, + filestem, + "(\d+)\.png", + save_format="mp4", + fps=30, + name="validation_epoch_end", + step=self.true_global_step, + ) + shutil.rmtree( + os.path.join(self.get_save_dir(), f"it{self.true_global_step}-val") + ) + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + ( + [ + { + "type": "rgb", + "img": batch["rgb"][0], + "kwargs": {"data_format": "HWC"}, + } + ] + if "rgb" in batch + else [] + ) + + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + ( + [ + { + "type": "grayscale", + "img": out["depth"][0], + "kwargs": {}, + } + ] + if "depth" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) + shutil.rmtree( + os.path.join(self.get_save_dir(), f"it{self.true_global_step}-test") + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/instructnerf2nerf.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/instructnerf2nerf.py new file mode 100644 index 0000000000000000000000000000000000000000..f6e3ecde194291a94d681c2e4594373e8ac1442e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/instructnerf2nerf.py @@ -0,0 +1,213 @@ +import os +from dataclasses import dataclass, field + +import torch + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.misc import cleanup, get_device +from threestudio.utils.ops import binary_cross_entropy, dot +from threestudio.utils.perceptual import PerceptualLoss +from threestudio.utils.typing import * + + +@threestudio.register("instructnerf2nerf-system") +class Instructnerf2nerf(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + per_editing_step: int = 10 + start_editing_step: int = 1000 + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + self.edit_frames = {} + p_config = {} + self.perceptual_loss = threestudio.find("perceptual-loss")(p_config) + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + render_out = self.renderer(**batch) + return { + **render_out, + } + + def on_fit_start(self) -> None: + super().on_fit_start() + # only used in training + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + def training_step(self, batch, batch_idx): + if torch.is_tensor(batch["index"]): + batch_index = batch["index"].item() + else: + batch_index = batch["index"] + origin_gt_rgb = batch["gt_rgb"] + B, H, W, C = origin_gt_rgb.shape + if batch_index in self.edit_frames: + gt_rgb = self.edit_frames[batch_index].to(batch["gt_rgb"].device) + gt_rgb = torch.nn.functional.interpolate( + gt_rgb.permute(0, 3, 1, 2), (H, W), mode="bilinear", align_corners=False + ).permute(0, 2, 3, 1) + batch["gt_rgb"] = gt_rgb + else: + gt_rgb = origin_gt_rgb + out = self(batch) + if ( + self.cfg.per_editing_step > 0 + and self.global_step > self.cfg.start_editing_step + ): + prompt_utils = self.prompt_processor() + if ( + not batch_index in self.edit_frames + or self.global_step % self.cfg.per_editing_step == 0 + ): + self.renderer.eval() + full_out = self(batch) + self.renderer.train() + result = self.guidance( + full_out["comp_rgb"], origin_gt_rgb, prompt_utils + ) + self.edit_frames[batch_index] = result["edit_images"].detach().cpu() + + loss = 0.0 + guidance_out = { + "loss_l1": torch.nn.functional.l1_loss(out["comp_rgb"], gt_rgb), + "loss_p": self.perceptual_loss( + out["comp_rgb"].permute(0, 3, 1, 2).contiguous(), + gt_rgb.permute(0, 3, 1, 2).contiguous(), + ).sum(), + } + + for name, value in guidance_out.items(): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss += value * self.C(self.cfg.loss[name.replace("loss_", "lambda_")]) + + if self.C(self.cfg.loss.lambda_orient) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for orientation loss, no normal is found in the output." + ) + loss_orient = ( + out["weights"].detach() + * dot(out["normal"], out["t_dirs"]).clamp_min(0.0) ** 2 + ).sum() / (out["opacity"] > 0).sum() + self.log("train/loss_orient", loss_orient) + loss += loss_orient * self.C(self.cfg.loss.lambda_orient) + + loss_sparsity = (out["opacity"] ** 2 + 0.01).sqrt().mean() + self.log("train/loss_sparsity", loss_sparsity) + loss += loss_sparsity * self.C(self.cfg.loss.lambda_sparsity) + + opacity_clamped = out["opacity"].clamp(1.0e-3, 1.0 - 1.0e-3) + loss_opaque = binary_cross_entropy(opacity_clamped, opacity_clamped) + self.log("train/loss_opaque", loss_opaque) + loss += loss_opaque * self.C(self.cfg.loss.lambda_opaque) + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + return {"loss": loss} + + def validation_step(self, batch, batch_idx): + out = self(batch) + if torch.is_tensor(batch["index"]): + batch_index = batch["index"].item() + else: + batch_index = batch["index"] + if batch_index in self.edit_frames: + B, H, W, C = batch["gt_rgb"].shape + rgb = torch.nn.functional.interpolate( + self.edit_frames[batch_index].permute(0, 3, 1, 2), (H, W) + ).permute(0, 2, 3, 1)[0] + else: + rgb = batch["gt_rgb"][0] + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ] + + [ + { + "type": "rgb", + "img": rgb, + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + }, + ], + name="validation_step", + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/latentnerf.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/latentnerf.py new file mode 100644 index 0000000000000000000000000000000000000000..48804c2ed7fe89c1fd6443d6ed1b3405ae8d5a7f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/latentnerf.py @@ -0,0 +1,181 @@ +from dataclasses import dataclass, field + +import torch + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.ops import ShapeLoss, binary_cross_entropy, dot +from threestudio.utils.typing import * + + +@threestudio.register("latentnerf-system") +class LatentNeRF(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + guide_shape: Optional[str] = None + refinement: bool = False + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + + if self.training or not self.cfg.refinement: + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + if self.cfg.guide_shape is not None: + self.shape_loss = ShapeLoss(self.cfg.guide_shape) + + def forward(self, batch: Dict[str, Any], decode: bool = False) -> Dict[str, Any]: + render_out = self.renderer(**batch) + out = { + **render_out, + } + if decode: + if self.cfg.refinement: + out["decoded_rgb"] = out["comp_rgb"] + else: + out["decoded_rgb"] = self.guidance.decode_latents( + out["comp_rgb"].permute(0, 3, 1, 2) + ).permute(0, 2, 3, 1) + return out + + def on_fit_start(self) -> None: + super().on_fit_start() + # only used in training + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + + def training_step(self, batch, batch_idx): + out = self(batch) + prompt_utils = self.prompt_processor() + guidance_out = self.guidance( + out["comp_rgb"], + prompt_utils, + **batch, + rgb_as_latents=not self.cfg.refinement, + ) + + loss = 0.0 + + for name, value in guidance_out.items(): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss += value * self.C(self.cfg.loss[name.replace("loss_", "lambda_")]) + + if self.C(self.cfg.loss.lambda_orient) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for orientation loss, no normal is found in the output." + ) + loss_orient = ( + out["weights"].detach() + * dot(out["normal"], out["t_dirs"]).clamp_min(0.0) ** 2 + ).sum() / (out["opacity"] > 0).sum() + self.log("train/loss_orient", loss_orient) + loss += loss_orient * self.C(self.cfg.loss.lambda_orient) + + loss_sparsity = (out["opacity"] ** 2 + 0.01).sqrt().mean() + self.log("train/loss_sparsity", loss_sparsity) + loss += loss_sparsity * self.C(self.cfg.loss.lambda_sparsity) + + opacity_clamped = out["opacity"].clamp(1.0e-3, 1.0 - 1.0e-3) + loss_opaque = binary_cross_entropy(opacity_clamped, opacity_clamped) + self.log("train/loss_opaque", loss_opaque) + loss += loss_opaque * self.C(self.cfg.loss.lambda_opaque) + + if ( + self.cfg.guide_shape is not None + and self.C(self.cfg.loss.lambda_shape) > 0 + and out["points"].shape[0] > 0 + ): + loss_shape = self.shape_loss(out["points"], out["density"]) + self.log("train/loss_shape", loss_shape) + loss += loss_shape * self.C(self.cfg.loss.lambda_shape) + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + return {"loss": loss} + + def validation_step(self, batch, batch_idx): + out = self(batch, decode=True) + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["decoded_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="validation_step", + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch, decode=True) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["decoded_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/magic123.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/magic123.py new file mode 100644 index 0000000000000000000000000000000000000000..12551518e1d799220c967bd525f7d191903ef332 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/magic123.py @@ -0,0 +1,230 @@ +from dataclasses import dataclass, field + +import torch +import torch.nn.functional as F + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.ops import binary_cross_entropy, dot +from threestudio.utils.typing import * + + +@threestudio.register("magic123-system") +class Magic123(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + refinement: bool = False + guidance_3d_type: str = "" + guidance_3d: dict = field(default_factory=dict) + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + self.guidance_3d = threestudio.find(self.cfg.guidance_3d_type)( + self.cfg.guidance_3d + ) + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + render_out = self.renderer(**batch) + return { + **render_out, + } + + def on_fit_start(self) -> None: + super().on_fit_start() + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + + def training_step(self, batch, batch_idx): + out_input = self(batch) + out = self(batch["random_camera"]) + prompt_utils = self.prompt_processor() + guidance_out = self.guidance( + out["comp_rgb"], + prompt_utils, + **batch["random_camera"], + rgb_as_latents=False, + ) + guidance_3d_out = self.guidance_3d( + out["comp_rgb"], + **batch["random_camera"], + rgb_as_latents=False, + ) + + loss = 0.0 + + loss_rgb = F.mse_loss( + out_input["comp_rgb"], + batch["rgb"] * batch["mask"].float() + + out_input["comp_rgb_bg"] * (1.0 - batch["mask"].float()), + ) + self.log("train/loss_rgb", loss_rgb) + loss += loss_rgb * self.C(self.cfg.loss.lambda_rgb) + + loss_mask = F.binary_cross_entropy( + out_input["opacity"].clamp(1.0e-5, 1.0 - 1.0e-5), + batch["mask"].float(), + ) + self.log("train/loss_mask", loss_mask) + loss += loss_mask * self.C(self.cfg.loss.lambda_mask) + + for name, value in guidance_out.items(): + if not (isinstance(value, torch.Tensor) and len(value.shape) > 0): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss += value * self.C(self.cfg.loss[name.replace("loss_", "lambda_")]) + + for name, value in guidance_3d_out.items(): + if not (isinstance(value, torch.Tensor) and len(value.shape) > 0): + self.log(f"train/{name}_3d", value) + if name.startswith("loss_"): + loss += value * self.C( + self.cfg.loss[name.replace("loss_", "lambda_3d_")] + ) + + if not self.cfg.refinement: + if self.C(self.cfg.loss.lambda_orient) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for orientation loss, no normal is found in the output." + ) + loss_orient = ( + out["weights"].detach() + * dot(out["normal"], out["t_dirs"]).clamp_min(0.0) ** 2 + ).sum() / (out["opacity"] > 0).sum() + self.log("train/loss_orient", loss_orient) + loss += loss_orient * self.C(self.cfg.loss.lambda_orient) + + if self.C(self.cfg.loss.lambda_normal_smoothness_2d) > 0: + if "comp_normal" not in out: + raise ValueError( + "comp_normal is required for 2D normal smoothness loss, no comp_normal is found in the output." + ) + normal = out["comp_normal"] + loss_normal_smoothness_2d = ( + normal[:, 1:, :, :] - normal[:, :-1, :, :] + ).square().mean() + ( + normal[:, :, 1:, :] - normal[:, :, :-1, :] + ).square().mean() + self.log("trian/loss_normal_smoothness_2d", loss_normal_smoothness_2d) + loss += loss_normal_smoothness_2d * self.C( + self.cfg.loss.lambda_normal_smoothness_2d + ) + + loss_sparsity = (out["opacity"] ** 2 + 0.01).sqrt().mean() + self.log("train/loss_sparsity", loss_sparsity) + loss += loss_sparsity * self.C(self.cfg.loss.lambda_sparsity) + + opacity_clamped = out["opacity"].clamp(1.0e-3, 1.0 - 1.0e-3) + loss_opaque = binary_cross_entropy(opacity_clamped, opacity_clamped) + self.log("train/loss_opaque", loss_opaque) + loss += loss_opaque * self.C(self.cfg.loss.lambda_opaque) + + # z variance loss proposed in HiFA: http://arxiv.org/abs/2305.18766 + # helps reduce floaters and produce solid geometry + if "z_variance" in out and "lambda_z_variance" in self.cfg.loss: + loss_z_variance = out["z_variance"][out["opacity"] > 0.5].mean() + self.log("train/loss_z_variance", loss_z_variance) + loss += loss_z_variance * self.C(self.cfg.loss.lambda_z_variance) + else: + loss_normal_consistency = out["mesh"].normal_consistency() + self.log("train/loss_normal_consistency", loss_normal_consistency) + loss += loss_normal_consistency * self.C( + self.cfg.loss.lambda_normal_consistency + ) + + if self.C(self.cfg.loss.lambda_laplacian_smoothness) > 0: + loss_laplacian_smoothness = out["mesh"].laplacian() + self.log("train/loss_laplacian_smoothness", loss_laplacian_smoothness) + loss += loss_laplacian_smoothness * self.C( + self.cfg.loss.lambda_laplacian_smoothness + ) + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + return {"loss": loss} + + def validation_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="validation_step", + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/magic3d.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/magic3d.py new file mode 100644 index 0000000000000000000000000000000000000000..9b1666aa7b945435824809cd24d0aa75957cc319 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/magic3d.py @@ -0,0 +1,164 @@ +import os +from dataclasses import dataclass, field + +import torch + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.misc import cleanup, get_device +from threestudio.utils.ops import binary_cross_entropy, dot +from threestudio.utils.typing import * + + +@threestudio.register("magic3d-system") +class Magic3D(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + refinement: bool = False + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + render_out = self.renderer(**batch) + return { + **render_out, + } + + def on_fit_start(self) -> None: + super().on_fit_start() + # only used in training + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + def training_step(self, batch, batch_idx): + out = self(batch) + prompt_utils = self.prompt_processor() + guidance_out = self.guidance( + out["comp_rgb"], prompt_utils, **batch, rgb_as_latents=False + ) + + loss = 0.0 + + for name, value in guidance_out.items(): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss += value * self.C(self.cfg.loss[name.replace("loss_", "lambda_")]) + + if not self.cfg.refinement: + if self.C(self.cfg.loss.lambda_orient) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for orientation loss, no normal is found in the output." + ) + loss_orient = ( + out["weights"].detach() + * dot(out["normal"], out["t_dirs"]).clamp_min(0.0) ** 2 + ).sum() / (out["opacity"] > 0).sum() + self.log("train/loss_orient", loss_orient) + loss += loss_orient * self.C(self.cfg.loss.lambda_orient) + + loss_sparsity = (out["opacity"] ** 2 + 0.01).sqrt().mean() + self.log("train/loss_sparsity", loss_sparsity) + loss += loss_sparsity * self.C(self.cfg.loss.lambda_sparsity) + + opacity_clamped = out["opacity"].clamp(1.0e-3, 1.0 - 1.0e-3) + loss_opaque = binary_cross_entropy(opacity_clamped, opacity_clamped) + self.log("train/loss_opaque", loss_opaque) + loss += loss_opaque * self.C(self.cfg.loss.lambda_opaque) + else: + loss_normal_consistency = out["mesh"].normal_consistency() + self.log("train/loss_normal_consistency", loss_normal_consistency) + loss += loss_normal_consistency * self.C( + self.cfg.loss.lambda_normal_consistency + ) + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + return {"loss": loss} + + def validation_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="validation_step", + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/optimizers.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/optimizers.py new file mode 100644 index 0000000000000000000000000000000000000000..cfc426e37043e4a6a3159a7ed35f7df7c912b180 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/optimizers.py @@ -0,0 +1,315 @@ +# Copyright 2022 Garena Online Private Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import List + +import torch +from torch import Tensor +from torch.optim.optimizer import Optimizer + + +class Adan(Optimizer): + """ + Implements a pytorch variant of Adan + Adan was proposed in + Adan: Adaptive Nesterov Momentum Algorithm for + Faster Optimizing Deep Models[J].arXiv preprint arXiv:2208.06677, 2022. + https://arxiv.org/abs/2208.06677 + Arguments: + params (iterable): iterable of parameters to optimize or + dicts defining parameter groups. + lr (float, optional): learning rate. (default: 1e-3) + betas (Tuple[float, float, flot], optional): coefficients used for + first- and second-order moments. (default: (0.98, 0.92, 0.99)) + eps (float, optional): term added to the denominator to improve + numerical stability. (default: 1e-8) + weight_decay (float, optional): decoupled weight decay + (L2 penalty) (default: 0) + max_grad_norm (float, optional): value used to clip + global grad norm (default: 0.0 no clip) + no_prox (bool): how to perform the decoupled weight decay + (default: False) + foreach (bool): if True would use torch._foreach implementation. + It's faster but uses slightly more memory. (default: True) + """ + + def __init__( + self, + params, + lr=1e-3, + betas=(0.98, 0.92, 0.99), + eps=1e-8, + weight_decay=0.0, + max_grad_norm=0.0, + no_prox=False, + foreach: bool = True, + ): + if not 0.0 <= max_grad_norm: + raise ValueError("Invalid Max grad norm: {}".format(max_grad_norm)) + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1])) + if not 0.0 <= betas[2] < 1.0: + raise ValueError("Invalid beta parameter at index 2: {}".format(betas[2])) + defaults = dict( + lr=lr, + betas=betas, + eps=eps, + weight_decay=weight_decay, + max_grad_norm=max_grad_norm, + no_prox=no_prox, + foreach=foreach, + ) + super().__init__(params, defaults) + + def __setstate__(self, state): + super(Adan, self).__setstate__(state) + for group in self.param_groups: + group.setdefault("no_prox", False) + + @torch.no_grad() + def restart_opt(self): + for group in self.param_groups: + group["step"] = 0 + for p in group["params"]: + if p.requires_grad: + state = self.state[p] + # State initialization + + # Exponential moving average of gradient values + state["exp_avg"] = torch.zeros_like(p) + # Exponential moving average of squared gradient values + state["exp_avg_sq"] = torch.zeros_like(p) + # Exponential moving average of gradient difference + state["exp_avg_diff"] = torch.zeros_like(p) + + @torch.no_grad() + def step(self, closure=None): + """Performs a single optimization step.""" + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + if self.defaults["max_grad_norm"] > 0: + device = self.param_groups[0]["params"][0].device + global_grad_norm = torch.zeros(1, device=device) + + max_grad_norm = torch.tensor(self.defaults["max_grad_norm"], device=device) + for group in self.param_groups: + for p in group["params"]: + if p.grad is not None: + grad = p.grad + global_grad_norm.add_(grad.pow(2).sum()) + + global_grad_norm = torch.sqrt(global_grad_norm) + + clip_global_grad_norm = torch.clamp( + max_grad_norm / (global_grad_norm + group["eps"]), max=1.0 + ).item() + else: + clip_global_grad_norm = 1.0 + + for group in self.param_groups: + params_with_grad = [] + grads = [] + exp_avgs = [] + exp_avg_sqs = [] + exp_avg_diffs = [] + neg_pre_grads = [] + + beta1, beta2, beta3 = group["betas"] + # assume same step across group now to simplify things + # per parameter step can be easily support + # by making it tensor, or pass list into kernel + if "step" in group: + group["step"] += 1 + else: + group["step"] = 1 + + bias_correction1 = 1.0 - beta1 ** group["step"] + bias_correction2 = 1.0 - beta2 ** group["step"] + bias_correction3 = 1.0 - beta3 ** group["step"] + + for p in group["params"]: + if p.grad is None: + continue + params_with_grad.append(p) + grads.append(p.grad) + + state = self.state[p] + if len(state) == 0: + state["exp_avg"] = torch.zeros_like(p) + state["exp_avg_sq"] = torch.zeros_like(p) + state["exp_avg_diff"] = torch.zeros_like(p) + + if "neg_pre_grad" not in state or group["step"] == 1: + state["neg_pre_grad"] = p.grad.clone().mul_(-clip_global_grad_norm) + + exp_avgs.append(state["exp_avg"]) + exp_avg_sqs.append(state["exp_avg_sq"]) + exp_avg_diffs.append(state["exp_avg_diff"]) + neg_pre_grads.append(state["neg_pre_grad"]) + + kwargs = dict( + params=params_with_grad, + grads=grads, + exp_avgs=exp_avgs, + exp_avg_sqs=exp_avg_sqs, + exp_avg_diffs=exp_avg_diffs, + neg_pre_grads=neg_pre_grads, + beta1=beta1, + beta2=beta2, + beta3=beta3, + bias_correction1=bias_correction1, + bias_correction2=bias_correction2, + bias_correction3_sqrt=math.sqrt(bias_correction3), + lr=group["lr"], + weight_decay=group["weight_decay"], + eps=group["eps"], + no_prox=group["no_prox"], + clip_global_grad_norm=clip_global_grad_norm, + ) + + if group["foreach"]: + _multi_tensor_adan(**kwargs) + else: + _single_tensor_adan(**kwargs) + + return loss + + +def _single_tensor_adan( + params: List[Tensor], + grads: List[Tensor], + exp_avgs: List[Tensor], + exp_avg_sqs: List[Tensor], + exp_avg_diffs: List[Tensor], + neg_pre_grads: List[Tensor], + *, + beta1: float, + beta2: float, + beta3: float, + bias_correction1: float, + bias_correction2: float, + bias_correction3_sqrt: float, + lr: float, + weight_decay: float, + eps: float, + no_prox: bool, + clip_global_grad_norm: Tensor, +): + for i, param in enumerate(params): + grad = grads[i] + exp_avg = exp_avgs[i] + exp_avg_sq = exp_avg_sqs[i] + exp_avg_diff = exp_avg_diffs[i] + neg_grad_or_diff = neg_pre_grads[i] + + grad.mul_(clip_global_grad_norm) + + # for memory saving, we use `neg_grad_or_diff` + # to get some temp variable in a inplace way + neg_grad_or_diff.add_(grad) + + exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) # m_t + exp_avg_diff.mul_(beta2).add_(neg_grad_or_diff, alpha=1 - beta2) # diff_t + + neg_grad_or_diff.mul_(beta2).add_(grad) + exp_avg_sq.mul_(beta3).addcmul_( + neg_grad_or_diff, neg_grad_or_diff, value=1 - beta3 + ) # n_t + + denom = ((exp_avg_sq).sqrt() / bias_correction3_sqrt).add_(eps) + step_size_diff = lr * beta2 / bias_correction2 + step_size = lr / bias_correction1 + + if no_prox: + param.mul_(1 - lr * weight_decay) + param.addcdiv_(exp_avg, denom, value=-step_size) + param.addcdiv_(exp_avg_diff, denom, value=-step_size_diff) + else: + param.addcdiv_(exp_avg, denom, value=-step_size) + param.addcdiv_(exp_avg_diff, denom, value=-step_size_diff) + param.div_(1 + lr * weight_decay) + + neg_grad_or_diff.zero_().add_(grad, alpha=-1.0) + + +def _multi_tensor_adan( + params: List[Tensor], + grads: List[Tensor], + exp_avgs: List[Tensor], + exp_avg_sqs: List[Tensor], + exp_avg_diffs: List[Tensor], + neg_pre_grads: List[Tensor], + *, + beta1: float, + beta2: float, + beta3: float, + bias_correction1: float, + bias_correction2: float, + bias_correction3_sqrt: float, + lr: float, + weight_decay: float, + eps: float, + no_prox: bool, + clip_global_grad_norm: Tensor, +): + if len(params) == 0: + return + + torch._foreach_mul_(grads, clip_global_grad_norm) + + # for memory saving, we use `neg_pre_grads` + # to get some temp variable in a inplace way + torch._foreach_add_(neg_pre_grads, grads) + + torch._foreach_mul_(exp_avgs, beta1) + torch._foreach_add_(exp_avgs, grads, alpha=1 - beta1) # m_t + + torch._foreach_mul_(exp_avg_diffs, beta2) + torch._foreach_add_(exp_avg_diffs, neg_pre_grads, alpha=1 - beta2) # diff_t + + torch._foreach_mul_(neg_pre_grads, beta2) + torch._foreach_add_(neg_pre_grads, grads) + torch._foreach_mul_(exp_avg_sqs, beta3) + torch._foreach_addcmul_( + exp_avg_sqs, neg_pre_grads, neg_pre_grads, value=1 - beta3 + ) # n_t + + denom = torch._foreach_sqrt(exp_avg_sqs) + torch._foreach_div_(denom, bias_correction3_sqrt) + torch._foreach_add_(denom, eps) + + step_size_diff = lr * beta2 / bias_correction2 + step_size = lr / bias_correction1 + + if no_prox: + torch._foreach_mul_(params, 1 - lr * weight_decay) + torch._foreach_addcdiv_(params, exp_avgs, denom, value=-step_size) + torch._foreach_addcdiv_(params, exp_avg_diffs, denom, value=-step_size_diff) + else: + torch._foreach_addcdiv_(params, exp_avgs, denom, value=-step_size) + torch._foreach_addcdiv_(params, exp_avg_diffs, denom, value=-step_size_diff) + torch._foreach_div_(params, 1 + lr * weight_decay) + torch._foreach_zero_(neg_pre_grads) + torch._foreach_add_(neg_pre_grads, grads, alpha=-1.0) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/prolificdreamer.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/prolificdreamer.py new file mode 100644 index 0000000000000000000000000000000000000000..a29e9d24d2668fc083d024ec95ea359d56912f71 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/prolificdreamer.py @@ -0,0 +1,234 @@ +import os +from dataclasses import dataclass, field + +import torch + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.misc import cleanup, get_device +from threestudio.utils.ops import binary_cross_entropy, dot +from threestudio.utils.typing import * + + +@threestudio.register("prolificdreamer-system") +class ProlificDreamer(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + # in ['coarse', 'geometry', 'texture'] + stage: str = "coarse" + visualize_samples: bool = False + + cfg: Config + + def configure(self) -> None: + # set up geometry, material, background, renderer + super().configure() + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.prompt_utils = self.prompt_processor() + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + if self.cfg.stage == "geometry": + render_out = self.renderer(**batch, render_rgb=False) + else: + render_out = self.renderer(**batch) + return { + **render_out, + } + + def on_fit_start(self) -> None: + super().on_fit_start() + + def training_step(self, batch, batch_idx): + out = self(batch) + + if self.cfg.stage == "geometry": + guidance_inp = out["comp_normal"] + guidance_out = self.guidance( + guidance_inp, self.prompt_utils, **batch, rgb_as_latents=False + ) + else: + guidance_inp = out["comp_rgb"] + guidance_out = self.guidance( + guidance_inp, self.prompt_utils, **batch, rgb_as_latents=False + ) + + loss = 0.0 + + for name, value in guidance_out.items(): + if not (type(value) is torch.Tensor and value.numel() > 1): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss += value * self.C(self.cfg.loss[name.replace("loss_", "lambda_")]) + + if self.cfg.stage == "coarse": + if self.C(self.cfg.loss.lambda_orient) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for orientation loss, no normal is found in the output." + ) + loss_orient = ( + out["weights"].detach() + * dot(out["normal"], out["t_dirs"]).clamp_min(0.0) ** 2 + ).sum() / (out["opacity"] > 0).sum() + self.log("train/loss_orient", loss_orient) + loss += loss_orient * self.C(self.cfg.loss.lambda_orient) + + loss_sparsity = (out["opacity"] ** 2 + 0.01).sqrt().mean() + self.log("train/loss_sparsity", loss_sparsity) + loss += loss_sparsity * self.C(self.cfg.loss.lambda_sparsity) + + opacity_clamped = out["opacity"].clamp(1.0e-3, 1.0 - 1.0e-3) + loss_opaque = binary_cross_entropy(opacity_clamped, opacity_clamped) + self.log("train/loss_opaque", loss_opaque) + loss += loss_opaque * self.C(self.cfg.loss.lambda_opaque) + + # z variance loss proposed in HiFA: http://arxiv.org/abs/2305.18766 + # helps reduce floaters and produce solid geometry + if "z_variance" in out: + loss_z_variance = out["z_variance"][out["opacity"] > 0.5].mean() + self.log("train/loss_z_variance", loss_z_variance) + loss += loss_z_variance * self.C(self.cfg.loss.lambda_z_variance) + + # sdf loss + if "sdf_grad" in out: + loss_eikonal = ( + (torch.linalg.norm(out["sdf_grad"], ord=2, dim=-1) - 1.0) ** 2 + ).mean() + self.log("train/loss_eikonal", loss_eikonal) + loss += loss_eikonal * self.C(self.cfg.loss.lambda_eikonal) + self.log("train/inv_std", out["inv_std"], prog_bar=True) + + elif self.cfg.stage == "geometry": + loss_normal_consistency = out["mesh"].normal_consistency() + self.log("train/loss_normal_consistency", loss_normal_consistency) + loss += loss_normal_consistency * self.C( + self.cfg.loss.lambda_normal_consistency + ) + + if self.C(self.cfg.loss.lambda_laplacian_smoothness) > 0: + loss_laplacian_smoothness = out["mesh"].laplacian() + self.log("train/loss_laplacian_smoothness", loss_laplacian_smoothness) + loss += loss_laplacian_smoothness * self.C( + self.cfg.loss.lambda_laplacian_smoothness + ) + elif self.cfg.stage == "texture": + pass + else: + raise ValueError(f"Unknown stage {self.cfg.stage}") + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + return {"loss": loss} + + def validation_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}.png", + ( + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + if "comp_rgb" in out + else [] + ) + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="validation_step", + step=self.true_global_step, + ) + + if self.cfg.visualize_samples: + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}-sample.png", + [ + { + "type": "rgb", + "img": self.guidance.sample( + self.prompt_utils, **batch, seed=self.global_step + )[0], + "kwargs": {"data_format": "HWC"}, + }, + { + "type": "rgb", + "img": self.guidance.sample_lora(self.prompt_utils, **batch)[0], + "kwargs": {"data_format": "HWC"}, + }, + ], + name="validation_step_samples", + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + ( + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + if "comp_rgb" in out + else [] + ) + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/sjc.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/sjc.py new file mode 100644 index 0000000000000000000000000000000000000000..102c86a42d5d5602a5e78b04fff22e471884b308 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/sjc.py @@ -0,0 +1,200 @@ +from dataclasses import dataclass, field + +import numpy as np +import torch + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.typing import * + + +@threestudio.register("sjc-system") +class ScoreJacobianChaining(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + subpixel_rendering: bool = True + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + def forward(self, batch: Dict[str, Any], decode: bool = False) -> Dict[str, Any]: + render_out = self.renderer(**batch) + out = { + **render_out, + } + if decode: + if self.cfg.subpixel_rendering: + latent_height, latent_width = 128, 128 + else: + latent_height, latent_width = 64, 64 + out["decoded_rgb"] = self.guidance.decode_latents( + out["comp_rgb"].permute(0, 3, 1, 2), + latent_height=latent_height, + latent_width=latent_width, + ).permute(0, 2, 3, 1) + return out + + def on_fit_start(self) -> None: + super().on_fit_start() + # only used in training + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + + def on_test_start(self) -> None: + # check if guidance is initialized, such as when loading from checkpoint + if not hasattr(self, "guidance"): + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + def training_step(self, batch, batch_idx): + out = self(batch) + prompt_utils = self.prompt_processor() + guidance_out = self.guidance( + out["comp_rgb"], prompt_utils, **batch, rgb_as_latents=True + ) + + loss = 0.0 + + for name, value in guidance_out.items(): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss += value * self.C(self.cfg.loss[name.replace("loss_", "lambda_")]) + + loss_emptiness = ( + self.C(self.cfg.loss.lambda_emptiness) + * torch.log(1 + self.cfg.loss.emptiness_scale * out["weights"]).mean() + ) + + self.log("train/loss_emptiness", loss_emptiness) + loss += loss_emptiness + + # About the depth loss, see https://github.com/pals-ttic/sjc/issues/21 + if self.C(self.cfg.loss.lambda_depth) > 0: + _, h, w, _ = out["comp_rgb"].shape + comp_depth = (out["depth"] + 10 * (1 - out["opacity"])).squeeze(-1) + center_h = int(self.cfg.loss.center_ratio * h) + center_w = int(self.cfg.loss.center_ratio * w) + border_h = (h - center_h) // 2 + border_w = (h - center_w) // 2 + center_depth = comp_depth[ + ..., border_h : border_h + center_h, border_w : border_w + center_w + ] + center_depth_mean = center_depth.mean() + border_depth_mean = (comp_depth.sum() - center_depth.sum()) / ( + h * w - center_h * center_w + ) + log_input = center_depth_mean - border_depth_mean + 1e-12 + loss_depth = ( + torch.sign(log_input) + * torch.log(log_input) + * self.C(self.cfg.loss.lambda_depth) + ) + + self.log("train/loss_depth", loss_depth) + loss += loss_depth + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + return {"loss": loss} + + def vis_depth(self, pred_depth): + depth = pred_depth.detach().cpu().numpy() + depth = np.log(1.0 + depth + 1e-12) / np.log(1 + 10.0) + return depth + + def validation_step(self, batch, batch_idx): + out = self(batch, decode=True) + comp_depth = out["depth"] + 10 * (1 - out["opacity"]) # 10 for background + vis_depth = self.vis_depth(comp_depth.squeeze(-1)) + + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["decoded_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ] + + [ + { + "type": "grayscale", + "img": vis_depth[0], + "kwargs": {"cmap": "spectral", "data_range": (0, 1)}, + }, + ], + align=512, + name="validation_step", + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch, decode=True) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["decoded_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + align=512, + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/textmesh.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/textmesh.py new file mode 100644 index 0000000000000000000000000000000000000000..d5f0d65e5682327aca157680fe25a3376cc294ce --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/textmesh.py @@ -0,0 +1,160 @@ +from dataclasses import dataclass, field + +import torch + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.ops import binary_cross_entropy, dot +from threestudio.utils.typing import * + + +@threestudio.register("textmesh-system") +class TextMesh(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + pass + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + render_out = self.renderer(**batch) + return { + **render_out, + } + + def on_fit_start(self) -> None: + super().on_fit_start() + # only used in training + self.prompt_processor = threestudio.find(self.cfg.prompt_processor_type)( + self.cfg.prompt_processor + ) + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + # initialize SDF + self.geometry.initialize_shape() + + def training_step(self, batch, batch_idx): + out = self(batch) + prompt_utils = self.prompt_processor() + guidance_out = self.guidance( + out["comp_rgb"], prompt_utils, **batch, rgb_as_latents=False + ) + + loss = 0.0 + + for name, value in guidance_out.items(): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss += value * self.C(self.cfg.loss[name.replace("loss_", "lambda_")]) + + if self.C(self.cfg.loss.lambda_orient) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for orientation loss, no normal is found in the output." + ) + loss_orient = ( + out["weights"].detach() + * dot(out["normal"], out["t_dirs"]).clamp_min(0.0) ** 2 + ).sum() / (out["opacity"] > 0).sum() + self.log("train/loss_orient", loss_orient) + loss += loss_orient * self.C(self.cfg.loss.lambda_orient) + + loss_sparsity = (out["opacity"] ** 2 + 0.01).sqrt().mean() + self.log("train/loss_sparsity", loss_sparsity) + loss += loss_sparsity * self.C(self.cfg.loss.lambda_sparsity) + + opacity_clamped = out["opacity"].clamp(1.0e-3, 1.0 - 1.0e-3) + loss_opaque = binary_cross_entropy(opacity_clamped, opacity_clamped) + self.log("train/loss_opaque", loss_opaque) + loss += loss_opaque * self.C(self.cfg.loss.lambda_opaque) + + loss_eikonal = ( + (torch.linalg.norm(out["sdf_grad"], ord=2, dim=-1) - 1.0) ** 2 + ).mean() + self.log("train/loss_eikonal", loss_eikonal) + loss += loss_eikonal * self.C(self.cfg.loss.lambda_eikonal) + + self.log("train/inv_std", out["inv_std"], prog_bar=True) + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + return {"loss": loss} + + def validation_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/utils.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..732b36900815a591e4d957537cee3a50d0b929d7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/utils.py @@ -0,0 +1,104 @@ +import sys +import warnings +from bisect import bisect_right + +import torch +import torch.nn as nn +from torch.optim import lr_scheduler + +import threestudio + + +def get_scheduler(name): + if hasattr(lr_scheduler, name): + return getattr(lr_scheduler, name) + else: + raise NotImplementedError + + +def getattr_recursive(m, attr): + for name in attr.split("."): + m = getattr(m, name) + return m + + +def get_parameters(model, name): + module = getattr_recursive(model, name) + if isinstance(module, nn.Module): + return module.parameters() + elif isinstance(module, nn.Parameter): + return module + return [] + + +def parse_optimizer(config, model): + if hasattr(config, "params"): + params = [ + {"params": get_parameters(model, name), "name": name, **args} + for name, args in config.params.items() + ] + threestudio.debug(f"Specify optimizer params: {config.params}") + else: + params = model.parameters() + if config.name in ["FusedAdam"]: + import apex + + optim = getattr(apex.optimizers, config.name)(params, **config.args) + elif config.name in ["Adan"]: + from threestudio.systems import optimizers + + optim = getattr(optimizers, config.name)(params, **config.args) + else: + optim = getattr(torch.optim, config.name)(params, **config.args) + return optim + + +def parse_scheduler_to_instance(config, optimizer): + if config.name == "ChainedScheduler": + schedulers = [ + parse_scheduler_to_instance(conf, optimizer) for conf in config.schedulers + ] + scheduler = lr_scheduler.ChainedScheduler(schedulers) + elif config.name == "Sequential": + schedulers = [ + parse_scheduler_to_instance(conf, optimizer) for conf in config.schedulers + ] + scheduler = lr_scheduler.SequentialLR( + optimizer, schedulers, milestones=config.milestones + ) + else: + scheduler = getattr(lr_scheduler, config.name)(optimizer, **config.args) + return scheduler + + +def parse_scheduler(config, optimizer): + interval = config.get("interval", "epoch") + assert interval in ["epoch", "step"] + if config.name == "SequentialLR": + scheduler = { + "scheduler": lr_scheduler.SequentialLR( + optimizer, + [ + parse_scheduler(conf, optimizer)["scheduler"] + for conf in config.schedulers + ], + milestones=config.milestones, + ), + "interval": interval, + } + elif config.name == "ChainedScheduler": + scheduler = { + "scheduler": lr_scheduler.ChainedScheduler( + [ + parse_scheduler(conf, optimizer)["scheduler"] + for conf in config.schedulers + ] + ), + "interval": interval, + } + else: + scheduler = { + "scheduler": get_scheduler(config.name)(optimizer, **config.args), + "interval": interval, + } + return scheduler diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/zero123.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/zero123.py new file mode 100644 index 0000000000000000000000000000000000000000..320b3e6a1da71b08331faee07b0029f3179e2839 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/zero123.py @@ -0,0 +1,390 @@ +import os +import random +import shutil +from dataclasses import dataclass, field + +import torch +import torch.nn.functional as F +from PIL import Image, ImageDraw +from torchmetrics import PearsonCorrCoef + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.ops import binary_cross_entropy, dot +from threestudio.utils.typing import * + + +@threestudio.register("zero123-system") +class Zero123(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + freq: dict = field(default_factory=dict) + refinement: bool = False + ambient_ratio_min: float = 0.5 + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + render_out = self.renderer(**batch) + return { + **render_out, + } + + def on_fit_start(self) -> None: + super().on_fit_start() + # no prompt processor + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + # visualize all training images + all_images = self.trainer.datamodule.train_dataloader().dataset.get_all_images() + self.save_image_grid( + "all_training_images.png", + [ + {"type": "rgb", "img": image, "kwargs": {"data_format": "HWC"}} + for image in all_images + ], + name="on_fit_start", + step=self.true_global_step, + ) + + self.pearson = PearsonCorrCoef().to(self.device) + + def training_substep(self, batch, batch_idx, guidance: str): + """ + Args: + guidance: one of "ref" (reference image supervision), "zero123" + """ + if guidance == "ref": + # bg_color = torch.rand_like(batch['rays_o']) + ambient_ratio = 1.0 + shading = "diffuse" + batch["shading"] = shading + elif guidance == "zero123": + batch = batch["random_camera"] + ambient_ratio = ( + self.cfg.ambient_ratio_min + + (1 - self.cfg.ambient_ratio_min) * random.random() + ) + + batch["bg_color"] = None + batch["ambient_ratio"] = ambient_ratio + + out = self(batch) + loss_prefix = f"loss_{guidance}_" + + loss_terms = {} + + def set_loss(name, value): + loss_terms[f"{loss_prefix}{name}"] = value + + guidance_eval = ( + guidance == "zero123" + and self.cfg.freq.guidance_eval > 0 + and self.true_global_step % self.cfg.freq.guidance_eval == 0 + ) + + if guidance == "ref": + gt_mask = batch["mask"] + gt_rgb = batch["rgb"] + + # color loss + gt_rgb = gt_rgb * gt_mask.float() + out["comp_rgb_bg"] * ( + 1 - gt_mask.float() + ) + set_loss("rgb", F.mse_loss(gt_rgb, out["comp_rgb"])) + + # mask loss + set_loss("mask", F.mse_loss(gt_mask.float(), out["opacity"])) + + # depth loss + if self.C(self.cfg.loss.lambda_depth) > 0: + valid_gt_depth = batch["ref_depth"][gt_mask.squeeze(-1)].unsqueeze(1) + valid_pred_depth = out["depth"][gt_mask].unsqueeze(1) + with torch.no_grad(): + A = torch.cat( + [valid_gt_depth, torch.ones_like(valid_gt_depth)], dim=-1 + ) # [B, 2] + X = torch.linalg.lstsq(A, valid_pred_depth).solution # [2, 1] + valid_gt_depth = A @ X # [B, 1] + set_loss("depth", F.mse_loss(valid_gt_depth, valid_pred_depth)) + + # relative depth loss + if self.C(self.cfg.loss.lambda_depth_rel) > 0: + valid_gt_depth = batch["ref_depth"][gt_mask.squeeze(-1)] # [B,] + valid_pred_depth = out["depth"][gt_mask] # [B,] + set_loss( + "depth_rel", 1 - self.pearson(valid_pred_depth, valid_gt_depth) + ) + + # normal loss + if self.C(self.cfg.loss.lambda_normal) > 0: + valid_gt_normal = ( + 1 - 2 * batch["ref_normal"][gt_mask.squeeze(-1)] + ) # [B, 3] + valid_pred_normal = ( + 2 * out["comp_normal"][gt_mask.squeeze(-1)] - 1 + ) # [B, 3] + set_loss( + "normal", + 1 - F.cosine_similarity(valid_pred_normal, valid_gt_normal).mean(), + ) + elif guidance == "zero123": + # zero123 + guidance_out = self.guidance( + out["comp_rgb"], + **batch, + rgb_as_latents=False, + guidance_eval=guidance_eval, + ) + # claforte: TODO: rename the loss_terms keys + set_loss("sds", guidance_out["loss_sds"]) + + if self.C(self.cfg.loss.lambda_normal_smooth) > 0: + if "comp_normal" not in out: + raise ValueError( + "comp_normal is required for 2D normal smooth loss, no comp_normal is found in the output." + ) + normal = out["comp_normal"] + set_loss( + "normal_smooth", + (normal[:, 1:, :, :] - normal[:, :-1, :, :]).square().mean() + + (normal[:, :, 1:, :] - normal[:, :, :-1, :]).square().mean(), + ) + + if self.C(self.cfg.loss.lambda_3d_normal_smooth) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for normal smooth loss, no normal is found in the output." + ) + if "normal_perturb" not in out: + raise ValueError( + "normal_perturb is required for normal smooth loss, no normal_perturb is found in the output." + ) + normals = out["normal"] + normals_perturb = out["normal_perturb"] + set_loss("3d_normal_smooth", (normals - normals_perturb).abs().mean()) + + if not self.cfg.refinement: + if self.C(self.cfg.loss.lambda_orient) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for orientation loss, no normal is found in the output." + ) + set_loss( + "orient", + ( + out["weights"].detach() + * dot(out["normal"], out["t_dirs"]).clamp_min(0.0) ** 2 + ).sum() + / (out["opacity"] > 0).sum(), + ) + + if guidance != "ref" and self.C(self.cfg.loss.lambda_sparsity) > 0: + set_loss("sparsity", (out["opacity"] ** 2 + 0.01).sqrt().mean()) + + if self.C(self.cfg.loss.lambda_opaque) > 0: + opacity_clamped = out["opacity"].clamp(1.0e-3, 1.0 - 1.0e-3) + set_loss( + "opaque", binary_cross_entropy(opacity_clamped, opacity_clamped) + ) + else: + if self.C(self.cfg.loss.lambda_normal_consistency) > 0: + set_loss("normal_consistency", out["mesh"].normal_consistency()) + if self.C(self.cfg.loss.lambda_laplacian_smoothness) > 0: + set_loss("laplacian_smoothness", out["mesh"].laplacian()) + + loss = 0.0 + for name, value in loss_terms.items(): + self.log(f"train/{name}", value) + if name.startswith(loss_prefix): + loss_weighted = value * self.C( + self.cfg.loss[name.replace(loss_prefix, "lambda_")] + ) + self.log(f"train/{name}_w", loss_weighted) + loss += loss_weighted + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + self.log(f"train/loss_{guidance}", loss) + + if guidance_eval: + self.guidance_evaluation_save( + out["comp_rgb"].detach()[: guidance_out["eval"]["bs"]], + guidance_out["eval"], + ) + + return {"loss": loss} + + def training_step(self, batch, batch_idx): + if self.cfg.freq.get("ref_or_zero123", "accumulate") == "accumulate": + do_ref = True + do_zero123 = True + elif self.cfg.freq.get("ref_or_zero123", "accumulate") == "alternate": + do_ref = ( + self.true_global_step < self.cfg.freq.ref_only_steps + or self.true_global_step % self.cfg.freq.n_ref == 0 + ) + do_zero123 = not do_ref + + total_loss = 0.0 + if do_zero123: + out = self.training_substep(batch, batch_idx, guidance="zero123") + total_loss += out["loss"] + + if do_ref: + out = self.training_substep(batch, batch_idx, guidance="ref") + total_loss += out["loss"] + + self.log("train/loss", total_loss, prog_bar=True) + + # sch = self.lr_schedulers() + # sch.step() + + return {"loss": total_loss} + + def validation_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-val/{batch['index'][0]}.png", + ( + [ + { + "type": "rgb", + "img": batch["rgb"][0], + "kwargs": {"data_format": "HWC"}, + } + ] + if "rgb" in batch + else [] + ) + + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + ( + [ + { + "type": "grayscale", + "img": out["depth"][0], + "kwargs": {}, + } + ] + if "depth" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + # claforte: TODO: don't hardcode the frame numbers to record... read them from cfg instead. + name=f"validation_step_batchidx_{batch_idx}" + if batch_idx in [0, 7, 15, 23, 29] + else None, + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + filestem = f"it{self.true_global_step}-val" + self.save_img_sequence( + filestem, + filestem, + "(\d+)\.png", + save_format="mp4", + fps=30, + name="validation_epoch_end", + step=self.true_global_step, + ) + shutil.rmtree( + os.path.join(self.get_save_dir(), f"it{self.true_global_step}-val") + ) + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + ( + [ + { + "type": "rgb", + "img": batch["rgb"][0], + "kwargs": {"data_format": "HWC"}, + } + ] + if "rgb" in batch + else [] + ) + + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + ( + [ + { + "type": "grayscale", + "img": out["depth"][0], + "kwargs": {}, + } + ] + if "depth" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) + shutil.rmtree( + os.path.join(self.get_save_dir(), f"it{self.true_global_step}-test") + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/zero123_simple.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/zero123_simple.py new file mode 100644 index 0000000000000000000000000000000000000000..f1f5d054b5eb68e0e5e376fa04047d81cd7c0c64 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/systems/zero123_simple.py @@ -0,0 +1,207 @@ +from dataclasses import dataclass, field + +import torch + +import threestudio +from threestudio.systems.base import BaseLift3DSystem +from threestudio.utils.ops import binary_cross_entropy, dot +from threestudio.utils.typing import * + + +@threestudio.register("zero123-simple-system") +class Zero123Simple(BaseLift3DSystem): + @dataclass + class Config(BaseLift3DSystem.Config): + pass + + cfg: Config + + def configure(self): + # create geometry, material, background, renderer + super().configure() + self.guidance = threestudio.find(self.cfg.guidance_type)(self.cfg.guidance) + + def forward(self, batch: Dict[str, Any]) -> Dict[str, Any]: + render_out = self.renderer(**batch) + return { + **render_out, + } + + def on_fit_start(self) -> None: + super().on_fit_start() + + def training_step(self, batch, batch_idx): + out = self(batch["random_camera"]) + guidance_out = self.guidance( + out["comp_rgb"], + **batch["random_camera"], + rgb_as_latents=False, + ) + + loss = 0.0 + + for name, value in guidance_out.items(): + if not (isinstance(value, torch.Tensor) and len(value.shape) > 0): + self.log(f"train/{name}", value) + if name.startswith("loss_"): + loss += value * self.C(self.cfg.loss[name.replace("loss_", "lambda_")]) + + if self.C(self.cfg.loss.lambda_orient) > 0: + if "normal" not in out: + raise ValueError( + "Normal is required for orientation loss, no normal is found in the output." + ) + loss_orient = ( + out["weights"].detach() + * dot(out["normal"], out["t_dirs"]).clamp_min(0.0) ** 2 + ).sum() / (out["opacity"] > 0).sum() + self.log("train/loss_orient", loss_orient) + loss += loss_orient * self.C(self.cfg.loss.lambda_orient) + + if self.C(self.cfg.loss.lambda_normal_smoothness_2d) > 0: + if "comp_normal" not in out: + raise ValueError( + "comp_normal is required for 2D normal smoothness loss, no comp_normal is found in the output." + ) + normal = out["comp_normal"] + loss_normal_smoothness_2d = ( + normal[:, 1:, :, :] - normal[:, :-1, :, :] + ).square().mean() + ( + normal[:, :, 1:, :] - normal[:, :, :-1, :] + ).square().mean() + self.log("trian/loss_normal_smoothness_2d", loss_normal_smoothness_2d) + loss += loss_normal_smoothness_2d * self.C( + self.cfg.loss.lambda_normal_smoothness_2d + ) + + loss_sparsity = (out["opacity"] ** 2 + 0.01).sqrt().mean() + self.log("train/loss_sparsity", loss_sparsity) + loss += loss_sparsity * self.C(self.cfg.loss.lambda_sparsity) + + opacity_clamped = out["opacity"].clamp(1.0e-3, 1.0 - 1.0e-3) + loss_opaque = binary_cross_entropy(opacity_clamped, opacity_clamped) + self.log("train/loss_opaque", loss_opaque) + loss += loss_opaque * self.C(self.cfg.loss.lambda_opaque) + + for name, value in self.cfg.loss.items(): + self.log(f"train_params/{name}", self.C(value)) + + if self.true_global_step % 50 == 0: + self.save_image_grid( + f"it{self.true_global_step}-train-t{int(guidance_out['timesteps'][0])}.png", + ( + [ + { + "type": "rgb", + "img": guidance_out["rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + if "rgb" in guidance_out + else [] + ) + + ( + [ + { + "type": "rgb", + "img": guidance_out["rgb_1step_orig"][0], + "kwargs": {"data_format": "HWC"}, + } + ] + if "rgb_1step_orig" in guidance_out + else [] + ) + + ( + [ + { + "type": "rgb", + "img": guidance_out["rgb_multistep_orig"][0], + "kwargs": {"data_format": "HWC"}, + } + ] + if "rgb_multistep_orig" in guidance_out + else [] + ), + ) + + return {"loss": loss} + + def validation_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="validation_step", + step=self.true_global_step, + ) + + def on_validation_epoch_end(self): + pass + + def test_step(self, batch, batch_idx): + out = self(batch) + self.save_image_grid( + f"it{self.true_global_step}-test/{batch['index'][0]}.png", + [ + { + "type": "rgb", + "img": out["comp_rgb"][0], + "kwargs": {"data_format": "HWC"}, + }, + ] + + ( + [ + { + "type": "rgb", + "img": out["comp_normal"][0], + "kwargs": {"data_format": "HWC", "data_range": (0, 1)}, + } + ] + if "comp_normal" in out + else [] + ) + + [ + { + "type": "grayscale", + "img": out["opacity"][0, :, :, 0], + "kwargs": {"cmap": None, "data_range": (0, 1)}, + }, + ], + name="test_step", + step=self.true_global_step, + ) + + def on_test_epoch_end(self): + self.save_img_sequence( + f"it{self.true_global_step}-test", + f"it{self.true_global_step}-test", + "(\d+)\.png", + save_format="mp4", + fps=30, + name="test", + step=self.true_global_step, + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/attention.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..c9a1a95faa33a9b6c176e280875ba88dee25fd93 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/attention.py @@ -0,0 +1,278 @@ +import math +from inspect import isfunction + +import torch +import torch.nn.functional as F +from einops import rearrange, repeat +from torch import einsum, nn + +from threestudio.utils.GAN.network_util import checkpoint + + +def exists(val): + return val is not None + + +def uniq(arr): + return {el: True for el in arr}.keys() + + +def default(val, d): + if exists(val): + return val + return d() if isfunction(d) else d + + +def max_neg_value(t): + return -torch.finfo(t.dtype).max + + +def init_(tensor): + dim = tensor.shape[-1] + std = 1 / math.sqrt(dim) + tensor.uniform_(-std, std) + return tensor + + +# feedforward +class GEGLU(nn.Module): + def __init__(self, dim_in, dim_out): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out * 2) + + def forward(self, x): + x, gate = self.proj(x).chunk(2, dim=-1) + return x * F.gelu(gate) + + +class FeedForward(nn.Module): + def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.0): + super().__init__() + inner_dim = int(dim * mult) + dim_out = default(dim_out, dim) + project_in = ( + nn.Sequential(nn.Linear(dim, inner_dim), nn.GELU()) + if not glu + else GEGLU(dim, inner_dim) + ) + + self.net = nn.Sequential( + project_in, nn.Dropout(dropout), nn.Linear(inner_dim, dim_out) + ) + + def forward(self, x): + return self.net(x) + + +def zero_module(module): + """ + Zero out the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().zero_() + return module + + +def Normalize(in_channels): + return torch.nn.GroupNorm( + num_groups=32, num_channels=in_channels, eps=1e-6, affine=True + ) + + +class LinearAttention(nn.Module): + def __init__(self, dim, heads=4, dim_head=32): + super().__init__() + self.heads = heads + hidden_dim = dim_head * heads + self.to_qkv = nn.Conv2d(dim, hidden_dim * 3, 1, bias=False) + self.to_out = nn.Conv2d(hidden_dim, dim, 1) + + def forward(self, x): + b, c, h, w = x.shape + qkv = self.to_qkv(x) + q, k, v = rearrange( + qkv, "b (qkv heads c) h w -> qkv b heads c (h w)", heads=self.heads, qkv=3 + ) + k = k.softmax(dim=-1) + context = torch.einsum("bhdn,bhen->bhde", k, v) + out = torch.einsum("bhde,bhdn->bhen", context, q) + out = rearrange( + out, "b heads c (h w) -> b (heads c) h w", heads=self.heads, h=h, w=w + ) + return self.to_out(out) + + +class SpatialSelfAttention(nn.Module): + def __init__(self, in_channels): + super().__init__() + self.in_channels = in_channels + + self.norm = Normalize(in_channels) + self.q = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0 + ) + self.k = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0 + ) + self.v = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0 + ) + self.proj_out = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0 + ) + + def forward(self, x): + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + b, c, h, w = q.shape + q = rearrange(q, "b c h w -> b (h w) c") + k = rearrange(k, "b c h w -> b c (h w)") + w_ = torch.einsum("bij,bjk->bik", q, k) + + w_ = w_ * (int(c) ** (-0.5)) + w_ = torch.nn.functional.softmax(w_, dim=2) + + # attend to values + v = rearrange(v, "b c h w -> b c (h w)") + w_ = rearrange(w_, "b i j -> b j i") + h_ = torch.einsum("bij,bjk->bik", v, w_) + h_ = rearrange(h_, "b c (h w) -> b c h w", h=h) + h_ = self.proj_out(h_) + + return x + h_ + + +class CrossAttention(nn.Module): + def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0.0): + super().__init__() + inner_dim = dim_head * heads + context_dim = default(context_dim, query_dim) + + self.scale = dim_head**-0.5 + self.heads = heads + + self.to_q = nn.Linear(query_dim, inner_dim, bias=False) + self.to_k = nn.Linear(context_dim, inner_dim, bias=False) + self.to_v = nn.Linear(context_dim, inner_dim, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, query_dim), nn.Dropout(dropout) + ) + + def forward(self, x, context=None, mask=None): + h = self.heads + + q = self.to_q(x) + context = default(context, x) + k = self.to_k(context) + v = self.to_v(context) + + q, k, v = map(lambda t: rearrange(t, "b n (h d) -> (b h) n d", h=h), (q, k, v)) + + sim = einsum("b i d, b j d -> b i j", q, k) * self.scale + + if exists(mask): + mask = rearrange(mask, "b ... -> b (...)") + max_neg_value = -torch.finfo(sim.dtype).max + mask = repeat(mask, "b j -> (b h) () j", h=h) + sim.masked_fill_(~mask, max_neg_value) + + # attention, what we cannot get enough of + attn = sim.softmax(dim=-1) + + out = einsum("b i j, b j d -> b i d", attn, v) + out = rearrange(out, "(b h) n d -> b n (h d)", h=h) + return self.to_out(out) + + +class BasicTransformerBlock(nn.Module): + def __init__( + self, + dim, + n_heads, + d_head, + dropout=0.0, + context_dim=None, + gated_ff=True, + checkpoint=True, + ): + super().__init__() + self.attn1 = CrossAttention( + query_dim=dim, heads=n_heads, dim_head=d_head, dropout=dropout + ) # is a self-attention + self.ff = FeedForward(dim, dropout=dropout, glu=gated_ff) + self.attn2 = CrossAttention( + query_dim=dim, + context_dim=context_dim, + heads=n_heads, + dim_head=d_head, + dropout=dropout, + ) # is self-attn if context is none + self.norm1 = nn.LayerNorm(dim) + self.norm2 = nn.LayerNorm(dim) + self.norm3 = nn.LayerNorm(dim) + self.checkpoint = checkpoint + + def forward(self, x, context=None): + return checkpoint( + self._forward, (x, context), self.parameters(), self.checkpoint + ) + + def _forward(self, x, context=None): + x = self.attn1(self.norm1(x)) + x + x = self.attn2(self.norm2(x), context=context) + x + x = self.ff(self.norm3(x)) + x + return x + + +class SpatialTransformer(nn.Module): + """ + Transformer block for image-like data. + First, project the input (aka embedding) + and reshape to b, t, d. + Then apply standard transformer action. + Finally, reshape to image + """ + + def __init__( + self, in_channels, n_heads, d_head, depth=1, dropout=0.0, context_dim=None + ): + super().__init__() + self.in_channels = in_channels + inner_dim = n_heads * d_head + self.norm = Normalize(in_channels) + + self.proj_in = nn.Conv2d( + in_channels, inner_dim, kernel_size=1, stride=1, padding=0 + ) + + self.transformer_blocks = nn.ModuleList( + [ + BasicTransformerBlock( + inner_dim, n_heads, d_head, dropout=dropout, context_dim=context_dim + ) + for d in range(depth) + ] + ) + + self.proj_out = zero_module( + nn.Conv2d(inner_dim, in_channels, kernel_size=1, stride=1, padding=0) + ) + + def forward(self, x, context=None): + # note: if no context is given, cross-attention defaults to self-attention + b, c, h, w = x.shape + x_in = x + x = self.norm(x) + x = self.proj_in(x) + x = rearrange(x, "b c h w -> b (h w) c") + for block in self.transformer_blocks: + x = block(x, context=context) + x = rearrange(x, "b (h w) c -> b c h w", h=h, w=w) + x = self.proj_out(x) + return x + x_in diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/discriminator.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/discriminator.py new file mode 100644 index 0000000000000000000000000000000000000000..d3fdfe7beb90f759d340e3c446f04784980e1876 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/discriminator.py @@ -0,0 +1,217 @@ +import functools + +import torch +import torch.nn as nn + + +def count_params(model): + total_params = sum(p.numel() for p in model.parameters()) + return total_params + + +class ActNorm(nn.Module): + def __init__( + self, num_features, logdet=False, affine=True, allow_reverse_init=False + ): + assert affine + super().__init__() + self.logdet = logdet + self.loc = nn.Parameter(torch.zeros(1, num_features, 1, 1)) + self.scale = nn.Parameter(torch.ones(1, num_features, 1, 1)) + self.allow_reverse_init = allow_reverse_init + + self.register_buffer("initialized", torch.tensor(0, dtype=torch.uint8)) + + def initialize(self, input): + with torch.no_grad(): + flatten = input.permute(1, 0, 2, 3).contiguous().view(input.shape[1], -1) + mean = ( + flatten.mean(1) + .unsqueeze(1) + .unsqueeze(2) + .unsqueeze(3) + .permute(1, 0, 2, 3) + ) + std = ( + flatten.std(1) + .unsqueeze(1) + .unsqueeze(2) + .unsqueeze(3) + .permute(1, 0, 2, 3) + ) + + self.loc.data.copy_(-mean) + self.scale.data.copy_(1 / (std + 1e-6)) + + def forward(self, input, reverse=False): + if reverse: + return self.reverse(input) + if len(input.shape) == 2: + input = input[:, :, None, None] + squeeze = True + else: + squeeze = False + + _, _, height, width = input.shape + + if self.training and self.initialized.item() == 0: + self.initialize(input) + self.initialized.fill_(1) + + h = self.scale * (input + self.loc) + + if squeeze: + h = h.squeeze(-1).squeeze(-1) + + if self.logdet: + log_abs = torch.log(torch.abs(self.scale)) + logdet = height * width * torch.sum(log_abs) + logdet = logdet * torch.ones(input.shape[0]).to(input) + return h, logdet + + return h + + def reverse(self, output): + if self.training and self.initialized.item() == 0: + if not self.allow_reverse_init: + raise RuntimeError( + "Initializing ActNorm in reverse direction is " + "disabled by default. Use allow_reverse_init=True to enable." + ) + else: + self.initialize(output) + self.initialized.fill_(1) + + if len(output.shape) == 2: + output = output[:, :, None, None] + squeeze = True + else: + squeeze = False + + h = output / self.scale - self.loc + + if squeeze: + h = h.squeeze(-1).squeeze(-1) + return h + + +class AbstractEncoder(nn.Module): + def __init__(self): + super().__init__() + + def encode(self, *args, **kwargs): + raise NotImplementedError + + +class Labelator(AbstractEncoder): + """Net2Net Interface for Class-Conditional Model""" + + def __init__(self, n_classes, quantize_interface=True): + super().__init__() + self.n_classes = n_classes + self.quantize_interface = quantize_interface + + def encode(self, c): + c = c[:, None] + if self.quantize_interface: + return c, None, [None, None, c.long()] + return c + + +class SOSProvider(AbstractEncoder): + # for unconditional training + def __init__(self, sos_token, quantize_interface=True): + super().__init__() + self.sos_token = sos_token + self.quantize_interface = quantize_interface + + def encode(self, x): + # get batch size from data and replicate sos_token + c = torch.ones(x.shape[0], 1) * self.sos_token + c = c.long().to(x.device) + if self.quantize_interface: + return c, None, [None, None, c] + return c + + +def weights_init(m): + classname = m.__class__.__name__ + if classname.find("Conv") != -1: + nn.init.normal_(m.weight.data, 0.0, 0.02) + elif classname.find("BatchNorm") != -1: + nn.init.normal_(m.weight.data, 1.0, 0.02) + nn.init.constant_(m.bias.data, 0) + + +class NLayerDiscriminator(nn.Module): + """Defines a PatchGAN discriminator as in Pix2Pix + --> see https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix/blob/master/models/networks.py + """ + + def __init__(self, input_nc=3, ndf=64, n_layers=3, use_actnorm=False): + """Construct a PatchGAN discriminator + Parameters: + input_nc (int) -- the number of channels in input images + ndf (int) -- the number of filters in the last conv layer + n_layers (int) -- the number of conv layers in the discriminator + norm_layer -- normalization layer + """ + super(NLayerDiscriminator, self).__init__() + if not use_actnorm: + norm_layer = nn.BatchNorm2d + else: + norm_layer = ActNorm + if ( + type(norm_layer) == functools.partial + ): # no need to use bias as BatchNorm2d has affine parameters + use_bias = norm_layer.func != nn.BatchNorm2d + else: + use_bias = norm_layer != nn.BatchNorm2d + + kw = 4 + padw = 1 + sequence = [ + nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), + nn.LeakyReLU(0.2, True), + ] + nf_mult = 1 + nf_mult_prev = 1 + for n in range(1, n_layers): # gradually increase the number of filters + nf_mult_prev = nf_mult + nf_mult = min(2**n, 8) + sequence += [ + nn.Conv2d( + ndf * nf_mult_prev, + ndf * nf_mult, + kernel_size=kw, + stride=2, + padding=padw, + bias=use_bias, + ), + norm_layer(ndf * nf_mult), + nn.LeakyReLU(0.2, True), + ] + + nf_mult_prev = nf_mult + nf_mult = min(2**n_layers, 8) + sequence += [ + nn.Conv2d( + ndf * nf_mult_prev, + ndf * nf_mult, + kernel_size=kw, + stride=1, + padding=padw, + bias=use_bias, + ), + norm_layer(ndf * nf_mult), + nn.LeakyReLU(0.2, True), + ] + + sequence += [ + nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw) + ] # output 1 channel prediction map + self.main = nn.Sequential(*sequence) + + def forward(self, input): + """Standard forward.""" + return self.main(input) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/distribution.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/distribution.py new file mode 100644 index 0000000000000000000000000000000000000000..016be35523187ea366db9ade391fe8ee276db60b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/distribution.py @@ -0,0 +1,102 @@ +import numpy as np +import torch + + +class AbstractDistribution: + def sample(self): + raise NotImplementedError() + + def mode(self): + raise NotImplementedError() + + +class DiracDistribution(AbstractDistribution): + def __init__(self, value): + self.value = value + + def sample(self): + return self.value + + def mode(self): + return self.value + + +class DiagonalGaussianDistribution(object): + def __init__(self, parameters, deterministic=False): + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) + self.logvar = torch.clamp(self.logvar, -30.0, 20.0) + self.deterministic = deterministic + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + if self.deterministic: + self.var = self.std = torch.zeros_like(self.mean).to( + device=self.parameters.device + ) + + def sample(self): + x = self.mean + self.std * torch.randn(self.mean.shape).to( + device=self.parameters.device + ) + return x + + def kl(self, other=None): + if self.deterministic: + return torch.Tensor([0.0]) + else: + if other is None: + return 0.5 * torch.sum( + torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, + dim=[1, 2, 3], + ) + else: + return 0.5 * torch.sum( + torch.pow(self.mean - other.mean, 2) / other.var + + self.var / other.var + - 1.0 + - self.logvar + + other.logvar, + dim=[1, 2, 3], + ) + + def nll(self, sample, dims=[1, 2, 3]): + if self.deterministic: + return torch.Tensor([0.0]) + logtwopi = np.log(2.0 * np.pi) + return 0.5 * torch.sum( + logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, + dim=dims, + ) + + def mode(self): + return self.mean + + +def normal_kl(mean1, logvar1, mean2, logvar2): + """ + source: https://github.com/openai/guided-diffusion/blob/27c20a8fab9cb472df5d6bdd6c8d11c8f430b924/guided_diffusion/losses.py#L12 + Compute the KL divergence between two gaussians. + Shapes are automatically broadcasted, so batches can be compared to + scalars, among other use cases. + """ + tensor = None + for obj in (mean1, logvar1, mean2, logvar2): + if isinstance(obj, torch.Tensor): + tensor = obj + break + assert tensor is not None, "at least one argument must be a Tensor" + + # Force variances to be Tensors. Broadcasting helps convert scalars to + # Tensors, but it does not work for torch.exp(). + logvar1, logvar2 = [ + x if isinstance(x, torch.Tensor) else torch.tensor(x).to(tensor) + for x in (logvar1, logvar2) + ] + + return 0.5 * ( + -1.0 + + logvar2 + - logvar1 + + torch.exp(logvar1 - logvar2) + + ((mean1 - mean2) ** 2) * torch.exp(-logvar2) + ) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/loss.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/loss.py new file mode 100644 index 0000000000000000000000000000000000000000..7758065d532d1768e675eb33775b62a4bdcf3bb9 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/loss.py @@ -0,0 +1,35 @@ +import torch +import torch.nn.functional as F + + +def generator_loss(discriminator, inputs, reconstructions, cond=None): + if cond is None: + logits_fake = discriminator(reconstructions.contiguous()) + else: + logits_fake = discriminator( + torch.cat((reconstructions.contiguous(), cond), dim=1) + ) + g_loss = -torch.mean(logits_fake) + return g_loss + + +def hinge_d_loss(logits_real, logits_fake): + loss_real = torch.mean(F.relu(1.0 - logits_real)) + loss_fake = torch.mean(F.relu(1.0 + logits_fake)) + d_loss = 0.5 * (loss_real + loss_fake) + return d_loss + + +def discriminator_loss(discriminator, inputs, reconstructions, cond=None): + if cond is None: + logits_real = discriminator(inputs.contiguous().detach()) + logits_fake = discriminator(reconstructions.contiguous().detach()) + else: + logits_real = discriminator( + torch.cat((inputs.contiguous().detach(), cond), dim=1) + ) + logits_fake = discriminator( + torch.cat((reconstructions.contiguous().detach(), cond), dim=1) + ) + d_loss = hinge_d_loss(logits_real, logits_fake).mean() + return d_loss diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/mobilenet.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/mobilenet.py new file mode 100644 index 0000000000000000000000000000000000000000..1f5f8efdfa546ab7ab6c3d59e2592e8c5fcfe039 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/mobilenet.py @@ -0,0 +1,254 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ["MobileNetV3", "mobilenetv3"] + + +def conv_bn( + inp, + oup, + stride, + conv_layer=nn.Conv2d, + norm_layer=nn.BatchNorm2d, + nlin_layer=nn.ReLU, +): + return nn.Sequential( + conv_layer(inp, oup, 3, stride, 1, bias=False), + norm_layer(oup), + nlin_layer(inplace=True), + ) + + +def conv_1x1_bn( + inp, oup, conv_layer=nn.Conv2d, norm_layer=nn.BatchNorm2d, nlin_layer=nn.ReLU +): + return nn.Sequential( + conv_layer(inp, oup, 1, 1, 0, bias=False), + norm_layer(oup), + nlin_layer(inplace=True), + ) + + +class Hswish(nn.Module): + def __init__(self, inplace=True): + super(Hswish, self).__init__() + self.inplace = inplace + + def forward(self, x): + return x * F.relu6(x + 3.0, inplace=self.inplace) / 6.0 + + +class Hsigmoid(nn.Module): + def __init__(self, inplace=True): + super(Hsigmoid, self).__init__() + self.inplace = inplace + + def forward(self, x): + return F.relu6(x + 3.0, inplace=self.inplace) / 6.0 + + +class SEModule(nn.Module): + def __init__(self, channel, reduction=4): + super(SEModule, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Sequential( + nn.Linear(channel, channel // reduction, bias=False), + nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel, bias=False), + Hsigmoid() + # nn.Sigmoid() + ) + + def forward(self, x): + b, c, _, _ = x.size() + y = self.avg_pool(x).view(b, c) + y = self.fc(y).view(b, c, 1, 1) + return x * y.expand_as(x) + + +class Identity(nn.Module): + def __init__(self, channel): + super(Identity, self).__init__() + + def forward(self, x): + return x + + +def make_divisible(x, divisible_by=8): + import numpy as np + + return int(np.ceil(x * 1.0 / divisible_by) * divisible_by) + + +class MobileBottleneck(nn.Module): + def __init__(self, inp, oup, kernel, stride, exp, se=False, nl="RE"): + super(MobileBottleneck, self).__init__() + assert stride in [1, 2] + assert kernel in [3, 5] + padding = (kernel - 1) // 2 + self.use_res_connect = stride == 1 and inp == oup + + conv_layer = nn.Conv2d + norm_layer = nn.BatchNorm2d + if nl == "RE": + nlin_layer = nn.ReLU # or ReLU6 + elif nl == "HS": + nlin_layer = Hswish + else: + raise NotImplementedError + if se: + SELayer = SEModule + else: + SELayer = Identity + + self.conv = nn.Sequential( + # pw + conv_layer(inp, exp, 1, 1, 0, bias=False), + norm_layer(exp), + nlin_layer(inplace=True), + # dw + conv_layer(exp, exp, kernel, stride, padding, groups=exp, bias=False), + norm_layer(exp), + SELayer(exp), + nlin_layer(inplace=True), + # pw-linear + conv_layer(exp, oup, 1, 1, 0, bias=False), + norm_layer(oup), + ) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileNetV3(nn.Module): + def __init__( + self, n_class=1000, input_size=224, dropout=0.0, mode="small", width_mult=1.0 + ): + super(MobileNetV3, self).__init__() + input_channel = 16 + last_channel = 1280 + if mode == "large": + # refer to Table 1 in paper + mobile_setting = [ + # k, exp, c, se, nl, s, + [3, 16, 16, False, "RE", 1], + [3, 64, 24, False, "RE", 2], + [3, 72, 24, False, "RE", 1], + [5, 72, 40, True, "RE", 2], + [5, 120, 40, True, "RE", 1], + [5, 120, 40, True, "RE", 1], + [3, 240, 80, False, "HS", 2], + [3, 200, 80, False, "HS", 1], + [3, 184, 80, False, "HS", 1], + [3, 184, 80, False, "HS", 1], + [3, 480, 112, True, "HS", 1], + [3, 672, 112, True, "HS", 1], + [5, 672, 160, True, "HS", 2], + [5, 960, 160, True, "HS", 1], + [5, 960, 160, True, "HS", 1], + ] + elif mode == "small": + # refer to Table 2 in paper + mobile_setting = [ + # k, exp, c, se, nl, s, + [3, 16, 16, True, "RE", 2], + [3, 72, 24, False, "RE", 2], + [3, 88, 24, False, "RE", 1], + [5, 96, 40, True, "HS", 2], + [5, 240, 40, True, "HS", 1], + [5, 240, 40, True, "HS", 1], + [5, 120, 48, True, "HS", 1], + [5, 144, 48, True, "HS", 1], + [5, 288, 96, True, "HS", 2], + [5, 576, 96, True, "HS", 1], + [5, 576, 96, True, "HS", 1], + ] + else: + raise NotImplementedError + + # building first layer + assert input_size % 32 == 0 + last_channel = ( + make_divisible(last_channel * width_mult) + if width_mult > 1.0 + else last_channel + ) + self.features = [conv_bn(3, input_channel, 2, nlin_layer=Hswish)] + self.classifier = [] + + # building mobile blocks + for k, exp, c, se, nl, s in mobile_setting: + output_channel = make_divisible(c * width_mult) + exp_channel = make_divisible(exp * width_mult) + self.features.append( + MobileBottleneck( + input_channel, output_channel, k, s, exp_channel, se, nl + ) + ) + input_channel = output_channel + + # building last several layers + if mode == "large": + last_conv = make_divisible(960 * width_mult) + self.features.append( + conv_1x1_bn(input_channel, last_conv, nlin_layer=Hswish) + ) + self.features.append(nn.AdaptiveAvgPool2d(1)) + self.features.append(nn.Conv2d(last_conv, last_channel, 1, 1, 0)) + self.features.append(Hswish(inplace=True)) + elif mode == "small": + last_conv = make_divisible(576 * width_mult) + self.features.append( + conv_1x1_bn(input_channel, last_conv, nlin_layer=Hswish) + ) + # self.features.append(SEModule(last_conv)) # refer to paper Table2, but I think this is a mistake + self.features.append(nn.AdaptiveAvgPool2d(1)) + self.features.append(nn.Conv2d(last_conv, last_channel, 1, 1, 0)) + self.features.append(Hswish(inplace=True)) + else: + raise NotImplementedError + + # make it nn.Sequential + self.features = nn.Sequential(*self.features) + + # building classifier + self.classifier = nn.Sequential( + nn.Dropout(p=dropout), # refer to paper section 6 + nn.Linear(last_channel, n_class), + ) + + self._initialize_weights() + + def forward(self, x): + x = self.features(x) + x = x.mean(3).mean(2) + x = self.classifier(x) + return x + + def _initialize_weights(self): + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode="fan_out") + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.BatchNorm2d): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + if m.bias is not None: + nn.init.zeros_(m.bias) + + +def mobilenetv3(pretrained=False, **kwargs): + model = MobileNetV3(**kwargs) + if pretrained: + state_dict = torch.load("mobilenetv3_small_67.4.pth.tar") + model.load_state_dict(state_dict, strict=True) + # raise NotImplementedError + return model diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/network_util.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/network_util.py new file mode 100644 index 0000000000000000000000000000000000000000..671df1bda38dca4e85d0ebaa13af8b3df72b78b1 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/network_util.py @@ -0,0 +1,296 @@ +# adopted from +# https://github.com/openai/improved-diffusion/blob/main/improved_diffusion/gaussian_diffusion.py +# and +# https://github.com/lucidrains/denoising-diffusion-pytorch/blob/7706bdfc6f527f58d33f84b7b522e61e6e3164b3/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py +# and +# https://github.com/openai/guided-diffusion/blob/0ba878e517b276c45d1195eb29f6f5f72659a05b/guided_diffusion/nn.py +# +# thanks! + + +import math +import os + +import numpy as np +import torch +import torch.nn as nn +from einops import repeat + +from threestudio.utils.GAN.util import instantiate_from_config + + +def make_beta_schedule( + schedule, n_timestep, linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3 +): + if schedule == "linear": + betas = ( + torch.linspace( + linear_start**0.5, linear_end**0.5, n_timestep, dtype=torch.float64 + ) + ** 2 + ) + + elif schedule == "cosine": + timesteps = ( + torch.arange(n_timestep + 1, dtype=torch.float64) / n_timestep + cosine_s + ) + alphas = timesteps / (1 + cosine_s) * np.pi / 2 + alphas = torch.cos(alphas).pow(2) + alphas = alphas / alphas[0] + betas = 1 - alphas[1:] / alphas[:-1] + betas = np.clip(betas, a_min=0, a_max=0.999) + + elif schedule == "sqrt_linear": + betas = torch.linspace( + linear_start, linear_end, n_timestep, dtype=torch.float64 + ) + elif schedule == "sqrt": + betas = ( + torch.linspace(linear_start, linear_end, n_timestep, dtype=torch.float64) + ** 0.5 + ) + else: + raise ValueError(f"schedule '{schedule}' unknown.") + return betas.numpy() + + +def make_ddim_timesteps( + ddim_discr_method, num_ddim_timesteps, num_ddpm_timesteps, verbose=True +): + if ddim_discr_method == "uniform": + c = num_ddpm_timesteps // num_ddim_timesteps + ddim_timesteps = np.asarray(list(range(0, num_ddpm_timesteps, c))) + elif ddim_discr_method == "quad": + ddim_timesteps = ( + (np.linspace(0, np.sqrt(num_ddpm_timesteps * 0.8), num_ddim_timesteps)) ** 2 + ).astype(int) + else: + raise NotImplementedError( + f'There is no ddim discretization method called "{ddim_discr_method}"' + ) + + # assert ddim_timesteps.shape[0] == num_ddim_timesteps + # add one to get the final alpha values right (the ones from first scale to data during sampling) + steps_out = ddim_timesteps + 1 + if verbose: + print(f"Selected timesteps for ddim sampler: {steps_out}") + return steps_out + + +def make_ddim_sampling_parameters(alphacums, ddim_timesteps, eta, verbose=True): + # select alphas for computing the variance schedule + alphas = alphacums[ddim_timesteps] + alphas_prev = np.asarray([alphacums[0]] + alphacums[ddim_timesteps[:-1]].tolist()) + + # according the the formula provided in https://arxiv.org/abs/2010.02502 + sigmas = eta * np.sqrt( + (1 - alphas_prev) / (1 - alphas) * (1 - alphas / alphas_prev) + ) + if verbose: + print( + f"Selected alphas for ddim sampler: a_t: {alphas}; a_(t-1): {alphas_prev}" + ) + print( + f"For the chosen value of eta, which is {eta}, " + f"this results in the following sigma_t schedule for ddim sampler {sigmas}" + ) + return sigmas, alphas, alphas_prev + + +def betas_for_alpha_bar(num_diffusion_timesteps, alpha_bar, max_beta=0.999): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, + which defines the cumulative product of (1-beta) over time from t = [0,1]. + :param num_diffusion_timesteps: the number of betas to produce. + :param alpha_bar: a lambda that takes an argument t from 0 to 1 and + produces the cumulative product of (1-beta) up to that + part of the diffusion process. + :param max_beta: the maximum beta to use; use values lower than 1 to + prevent singularities. + """ + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta)) + return np.array(betas) + + +def extract_into_tensor(a, t, x_shape): + b, *_ = t.shape + out = a.gather(-1, t) + return out.reshape(b, *((1,) * (len(x_shape) - 1))) + + +def checkpoint(func, inputs, params, flag): + """ + Evaluate a function without caching intermediate activations, allowing for + reduced memory at the expense of extra compute in the backward pass. + :param func: the function to evaluate. + :param inputs: the argument sequence to pass to `func`. + :param params: a sequence of parameters `func` depends on but does not + explicitly take as arguments. + :param flag: if False, disable gradient checkpointing. + """ + if flag: + args = tuple(inputs) + tuple(params) + return CheckpointFunction.apply(func, len(inputs), *args) + else: + return func(*inputs) + + +class CheckpointFunction(torch.autograd.Function): + @staticmethod + def forward(ctx, run_function, length, *args): + ctx.run_function = run_function + ctx.input_tensors = list(args[:length]) + ctx.input_params = list(args[length:]) + + with torch.no_grad(): + output_tensors = ctx.run_function(*ctx.input_tensors) + return output_tensors + + @staticmethod + def backward(ctx, *output_grads): + ctx.input_tensors = [x.detach().requires_grad_(True) for x in ctx.input_tensors] + with torch.enable_grad(): + # Fixes a bug where the first op in run_function modifies the + # Tensor storage in place, which is not allowed for detach()'d + # Tensors. + shallow_copies = [x.view_as(x) for x in ctx.input_tensors] + output_tensors = ctx.run_function(*shallow_copies) + input_grads = torch.autograd.grad( + output_tensors, + ctx.input_tensors + ctx.input_params, + output_grads, + allow_unused=True, + ) + del ctx.input_tensors + del ctx.input_params + del output_tensors + return (None, None) + input_grads + + +def timestep_embedding(timesteps, dim, max_period=10000, repeat_only=False): + """ + Create sinusoidal timestep embeddings. + :param timesteps: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param dim: the dimension of the output. + :param max_period: controls the minimum frequency of the embeddings. + :return: an [N x dim] Tensor of positional embeddings. + """ + if not repeat_only: + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=torch.float32) + / half + ).to(device=timesteps.device) + args = timesteps[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat( + [embedding, torch.zeros_like(embedding[:, :1])], dim=-1 + ) + else: + embedding = repeat(timesteps, "b -> b d", d=dim) + return embedding + + +def zero_module(module): + """ + Zero out the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().zero_() + return module + + +def scale_module(module, scale): + """ + Scale the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().mul_(scale) + return module + + +def mean_flat(tensor): + """ + Take the mean over all non-batch dimensions. + """ + return tensor.mean(dim=list(range(1, len(tensor.shape)))) + + +def normalization(channels): + """ + Make a standard normalization layer. + :param channels: number of input channels. + :return: an nn.Module for normalization. + """ + return GroupNorm32(32, channels) + + +# PyTorch 1.7 has SiLU, but we support PyTorch 1.5. +class SiLU(nn.Module): + def forward(self, x): + return x * torch.sigmoid(x) + + +class GroupNorm32(nn.GroupNorm): + def forward(self, x): + return super().forward(x.float()).type(x.dtype) + + +def conv_nd(dims, *args, **kwargs): + """ + Create a 1D, 2D, or 3D convolution module. + """ + if dims == 1: + return nn.Conv1d(*args, **kwargs) + elif dims == 2: + return nn.Conv2d(*args, **kwargs) + elif dims == 3: + return nn.Conv3d(*args, **kwargs) + raise ValueError(f"unsupported dimensions: {dims}") + + +def linear(*args, **kwargs): + """ + Create a linear module. + """ + return nn.Linear(*args, **kwargs) + + +def avg_pool_nd(dims, *args, **kwargs): + """ + Create a 1D, 2D, or 3D average pooling module. + """ + if dims == 1: + return nn.AvgPool1d(*args, **kwargs) + elif dims == 2: + return nn.AvgPool2d(*args, **kwargs) + elif dims == 3: + return nn.AvgPool3d(*args, **kwargs) + raise ValueError(f"unsupported dimensions: {dims}") + + +class HybridConditioner(nn.Module): + def __init__(self, c_concat_config, c_crossattn_config): + super().__init__() + self.concat_conditioner = instantiate_from_config(c_concat_config) + self.crossattn_conditioner = instantiate_from_config(c_crossattn_config) + + def forward(self, c_concat, c_crossattn): + c_concat = self.concat_conditioner(c_concat) + c_crossattn = self.crossattn_conditioner(c_crossattn) + return {"c_concat": [c_concat], "c_crossattn": [c_crossattn]} + + +def noise_like(shape, device, repeat=False): + repeat_noise = lambda: torch.randn((1, *shape[1:]), device=device).repeat( + shape[0], *((1,) * (len(shape) - 1)) + ) + noise = lambda: torch.randn(shape, device=device) + return repeat_noise() if repeat else noise() diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/util.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/util.py new file mode 100644 index 0000000000000000000000000000000000000000..33a86d6a1c20746acb0795a61ed7dd9e6aa860c5 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/util.py @@ -0,0 +1,208 @@ +import importlib +import multiprocessing as mp +from collections import abc +from functools import partial +from inspect import isfunction +from queue import Queue +from threading import Thread + +import numpy as np +import torch +from einops import rearrange +from PIL import Image, ImageDraw, ImageFont + + +def log_txt_as_img(wh, xc, size=10): + # wh a tuple of (width, height) + # xc a list of captions to plot + b = len(xc) + txts = list() + for bi in range(b): + txt = Image.new("RGB", wh, color="white") + draw = ImageDraw.Draw(txt) + font = ImageFont.truetype("data/DejaVuSans.ttf", size=size) + nc = int(40 * (wh[0] / 256)) + lines = "\n".join( + xc[bi][start : start + nc] for start in range(0, len(xc[bi]), nc) + ) + + try: + draw.text((0, 0), lines, fill="black", font=font) + except UnicodeEncodeError: + print("Cant encode string for logging. Skipping.") + + txt = np.array(txt).transpose(2, 0, 1) / 127.5 - 1.0 + txts.append(txt) + txts = np.stack(txts) + txts = torch.tensor(txts) + return txts + + +def ismap(x): + if not isinstance(x, torch.Tensor): + return False + return (len(x.shape) == 4) and (x.shape[1] > 3) + + +def isimage(x): + if not isinstance(x, torch.Tensor): + return False + return (len(x.shape) == 4) and (x.shape[1] == 3 or x.shape[1] == 1) + + +def exists(x): + return x is not None + + +def default(val, d): + if exists(val): + return val + return d() if isfunction(d) else d + + +def mean_flat(tensor): + """ + https://github.com/openai/guided-diffusion/blob/27c20a8fab9cb472df5d6bdd6c8d11c8f430b924/guided_diffusion/nn.py#L86 + Take the mean over all non-batch dimensions. + """ + return tensor.mean(dim=list(range(1, len(tensor.shape)))) + + +def count_params(model, verbose=False): + total_params = sum(p.numel() for p in model.parameters()) + if verbose: + print(f"{model.__class__.__name__} has {total_params * 1.e-6:.2f} M params.") + return total_params + + +def instantiate_from_config(config): + if not "target" in config: + if config == "__is_first_stage__": + return None + elif config == "__is_unconditional__": + return None + raise KeyError("Expected key `target` to instantiate.") + return get_obj_from_str(config["target"])(**config.get("params", dict())) + + +def get_obj_from_str(string, reload=False): + module, cls = string.rsplit(".", 1) + if reload: + module_imp = importlib.import_module(module) + importlib.reload(module_imp) + return getattr(importlib.import_module(module, package=None), cls) + + +def _do_parallel_data_prefetch(func, Q, data, idx, idx_to_fn=False): + # create dummy dataset instance + + # run prefetching + if idx_to_fn: + res = func(data, worker_id=idx) + else: + res = func(data) + Q.put([idx, res]) + Q.put("Done") + + +def parallel_data_prefetch( + func: callable, + data, + n_proc, + target_data_type="ndarray", + cpu_intensive=True, + use_worker_id=False, +): + # if target_data_type not in ["ndarray", "list"]: + # raise ValueError( + # "Data, which is passed to parallel_data_prefetch has to be either of type list or ndarray." + # ) + if isinstance(data, np.ndarray) and target_data_type == "list": + raise ValueError("list expected but function got ndarray.") + elif isinstance(data, abc.Iterable): + if isinstance(data, dict): + print( + f'WARNING:"data" argument passed to parallel_data_prefetch is a dict: Using only its values and disregarding keys.' + ) + data = list(data.values()) + if target_data_type == "ndarray": + data = np.asarray(data) + else: + data = list(data) + else: + raise TypeError( + f"The data, that shall be processed parallel has to be either an np.ndarray or an Iterable, but is actually {type(data)}." + ) + + if cpu_intensive: + Q = mp.Queue(1000) + proc = mp.Process + else: + Q = Queue(1000) + proc = Thread + # spawn processes + if target_data_type == "ndarray": + arguments = [ + [func, Q, part, i, use_worker_id] + for i, part in enumerate(np.array_split(data, n_proc)) + ] + else: + step = ( + int(len(data) / n_proc + 1) + if len(data) % n_proc != 0 + else int(len(data) / n_proc) + ) + arguments = [ + [func, Q, part, i, use_worker_id] + for i, part in enumerate( + [data[i : i + step] for i in range(0, len(data), step)] + ) + ] + processes = [] + for i in range(n_proc): + p = proc(target=_do_parallel_data_prefetch, args=arguments[i]) + processes += [p] + + # start processes + print(f"Start prefetching...") + import time + + start = time.time() + gather_res = [[] for _ in range(n_proc)] + try: + for p in processes: + p.start() + + k = 0 + while k < n_proc: + # get result + res = Q.get() + if res == "Done": + k += 1 + else: + gather_res[res[0]] = res[1] + + except Exception as e: + print("Exception: ", e) + for p in processes: + p.terminate() + + raise e + finally: + for p in processes: + p.join() + print(f"Prefetching complete. [{time.time() - start} sec.]") + + if target_data_type == "ndarray": + if not isinstance(gather_res[0], np.ndarray): + return np.concatenate([np.asarray(r) for r in gather_res], axis=0) + + # order outputs + return np.concatenate(gather_res, axis=0) + elif target_data_type == "list": + out = [] + for r in gather_res: + out.extend(r) + return out + else: + return gather_res diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/vae.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/vae.py new file mode 100644 index 0000000000000000000000000000000000000000..2b4741f8e03587664739adb348689054aac38c59 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/GAN/vae.py @@ -0,0 +1,1028 @@ +# pytorch_diffusion + derived encoder decoder +import math + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange + +from threestudio.utils.GAN.attention import LinearAttention +from threestudio.utils.GAN.util import instantiate_from_config + + +def get_timestep_embedding(timesteps, embedding_dim): + """ + This matches the implementation in Denoising Diffusion Probabilistic Models: + From Fairseq. + Build sinusoidal embeddings. + This matches the implementation in tensor2tensor, but differs slightly + from the description in Section 3.5 of "Attention Is All You Need". + """ + assert len(timesteps.shape) == 1 + + half_dim = embedding_dim // 2 + emb = math.log(10000) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=torch.float32) * -emb) + emb = emb.to(device=timesteps.device) + emb = timesteps.float()[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1, 0, 0)) + return emb + + +def nonlinearity(x): + # swish + return x * torch.sigmoid(x) + + +def Normalize(in_channels, num_groups=32): + return torch.nn.BatchNorm2d(num_features=in_channels) + + +class Upsample(nn.Module): + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=1, padding=1 + ) + + def forward(self, x): + x = torch.nn.functional.interpolate(x, scale_factor=2.0, mode="nearest") + if self.with_conv: + x = self.conv(x) + return x + + +class Downsample(nn.Module): + def __init__(self, in_channels, with_conv): + super().__init__() + self.with_conv = with_conv + if self.with_conv: + # no asymmetric padding in torch conv, must do it ourselves + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=3, stride=2, padding=0 + ) + + def forward(self, x): + if self.with_conv: + pad = (0, 1, 0, 1) + x = torch.nn.functional.pad(x, pad, mode="constant", value=0) + x = self.conv(x) + else: + x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2) + return x + + +class ResnetBlock(nn.Module): + def __init__( + self, + *, + in_channels, + out_channels=None, + conv_shortcut=False, + dropout, + temb_channels=512, + ): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.use_conv_shortcut = conv_shortcut + + self.norm1 = Normalize(in_channels) + self.conv1 = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=1, padding=1 + ) + if temb_channels > 0: + self.temb_proj = torch.nn.Linear(temb_channels, out_channels) + self.norm2 = Normalize(out_channels) + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = torch.nn.Conv2d( + out_channels, out_channels, kernel_size=3, stride=1, padding=1 + ) + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=1, padding=1 + ) + else: + self.nin_shortcut = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=1, stride=1, padding=0 + ) + + def forward(self, x, temb): + h = x + h = self.norm1(h) + h = nonlinearity(h) + h = self.conv1(h) + + if temb is not None: + h = h + self.temb_proj(nonlinearity(temb))[:, :, None, None] + + h = self.norm2(h) + h = nonlinearity(h) + h = self.dropout(h) + h = self.conv2(h) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + x = self.conv_shortcut(x) + else: + x = self.nin_shortcut(x) + + return x + h + + +class LinAttnBlock(LinearAttention): + """to match AttnBlock usage""" + + def __init__(self, in_channels): + super().__init__(dim=in_channels, heads=1, dim_head=in_channels) + + +class AttnBlock(nn.Module): + def __init__(self, in_channels): + super().__init__() + self.in_channels = in_channels + + self.norm = Normalize(in_channels) + self.q = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0 + ) + self.k = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0 + ) + self.v = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0 + ) + self.proj_out = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=1, stride=1, padding=0 + ) + + def forward(self, x): + h_ = x + h_ = self.norm(h_) + q = self.q(h_) + k = self.k(h_) + v = self.v(h_) + + # compute attention + b, c, h, w = q.shape + q = q.reshape(b, c, h * w) + q = q.permute(0, 2, 1) # b,hw,c + k = k.reshape(b, c, h * w) # b,c,hw + w_ = torch.bmm(q, k) # b,hw,hw w[b,i,j]=sum_c q[b,i,c]k[b,c,j] + w_ = w_ * (int(c) ** (-0.5)) + w_ = torch.nn.functional.softmax(w_, dim=2) + + # attend to values + v = v.reshape(b, c, h * w) + w_ = w_.permute(0, 2, 1) # b,hw,hw (first hw of k, second of q) + h_ = torch.bmm(v, w_) # b, c,hw (hw of q) h_[b,c,j] = sum_i v[b,c,i] w_[b,i,j] + h_ = h_.reshape(b, c, h, w) + + h_ = self.proj_out(h_) + + return x + h_ + + +def make_attn(in_channels, attn_type="vanilla"): + assert attn_type in ["vanilla", "linear", "none"], f"attn_type {attn_type} unknown" + print(f"making attention of type '{attn_type}' with {in_channels} in_channels") + if attn_type == "vanilla": + return AttnBlock(in_channels) + elif attn_type == "none": + return nn.Identity(in_channels) + else: + return LinAttnBlock(in_channels) + + +class Model(nn.Module): + def __init__( + self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + use_timestep=True, + use_linear_attn=False, + attn_type="vanilla", + ): + super().__init__() + if use_linear_attn: + attn_type = "linear" + self.ch = ch + self.temb_ch = self.ch * 4 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + + self.use_timestep = use_timestep + if self.use_timestep: + # timestep embedding + self.temb = nn.Module() + self.temb.dense = nn.ModuleList( + [ + torch.nn.Linear(self.ch, self.temb_ch), + torch.nn.Linear(self.temb_ch, self.temb_ch), + ] + ) + + # downsampling + self.conv_in = torch.nn.Conv2d( + in_channels, self.ch, kernel_size=3, stride=1, padding=1 + ) + + curr_res = resolution + in_ch_mult = (1,) + tuple(ch_mult) + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout, + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(make_attn(block_in, attn_type=attn_type)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in, resamp_with_conv) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout, + ) + self.mid.attn_1 = make_attn(block_in, attn_type=attn_type) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout, + ) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + skip_in = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + if i_block == self.num_res_blocks: + skip_in = ch * in_ch_mult[i_level] + block.append( + ResnetBlock( + in_channels=block_in + skip_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout, + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(make_attn(block_in, attn_type=attn_type)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in, resamp_with_conv) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, out_ch, kernel_size=3, stride=1, padding=1 + ) + + def forward(self, x, t=None, context=None): + # assert x.shape[2] == x.shape[3] == self.resolution + if context is not None: + # assume aligned context, cat along channel axis + x = torch.cat((x, context), dim=1) + if self.use_timestep: + # timestep embedding + assert t is not None + temb = get_timestep_embedding(t, self.ch) + temb = self.temb.dense[0](temb) + temb = nonlinearity(temb) + temb = self.temb.dense[1](temb) + else: + temb = None + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1], temb) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h, temb) + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block]( + torch.cat([h, hs.pop()], dim=1), temb + ) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + def get_last_layer(self): + return self.conv_out.weight + + +class Encoder(nn.Module): + def __init__( + self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + double_z=True, + use_linear_attn=False, + attn_type="vanilla", + **ignore_kwargs, + ): + super().__init__() + if use_linear_attn: + attn_type = "linear" + self.ch = ch + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.attn_resolutions = attn_resolutions + + # downsampling + self.conv_in = torch.nn.Conv2d( + in_channels, self.ch, kernel_size=3, stride=1, padding=1 + ) + + curr_res = resolution + in_ch_mult = (1,) + tuple(ch_mult) + self.in_ch_mult = in_ch_mult + self.down = nn.ModuleList() + for i_level in range(self.num_resolutions): + block = nn.ModuleList() + attn = nn.ModuleList() + block_in = ch * in_ch_mult[i_level] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout, + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(make_attn(block_in, attn_type=attn_type)) + down = nn.Module() + down.block = block + down.attn = attn + if i_level != self.num_resolutions - 1: + down.downsample = Downsample(block_in, resamp_with_conv) + curr_res = curr_res // 2 + self.down.append(down) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout, + ) + if len(attn_resolutions) > 0: + self.mid.attn_1 = make_attn(block_in, attn_type=attn_type) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout, + ) + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, + 2 * z_channels if double_z else z_channels, + kernel_size=3, + stride=1, + padding=1, + ) + + def forward(self, x): + # timestep embedding + temb = None + + # downsampling + hs = [self.conv_in(x)] + for i_level in range(self.num_resolutions): + for i_block in range(self.num_res_blocks): + h = self.down[i_level].block[i_block](hs[-1], temb) + if len(self.down[i_level].attn) > 0: + h = self.down[i_level].attn[i_block](h) + hs.append(h) + if i_level != self.num_resolutions - 1: + hs.append(self.down[i_level].downsample(hs[-1])) + + # middle + h = hs[-1] + h = self.mid.block_1(h, temb) + if len(self.attn_resolutions) > 0: + h = self.mid.attn_1(h) + h = self.mid.block_2(h, temb) + + # end + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class Decoder(nn.Module): + def __init__( + self, + *, + ch, + out_ch, + ch_mult=(1, 2, 4, 8), + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + in_channels, + resolution, + z_channels, + give_pre_end=False, + tanh_out=False, + use_linear_attn=False, + attn_type="vanilla", + **ignorekwargs, + ): + super().__init__() + if use_linear_attn: + attn_type = "linear" + self.ch = ch + # self.temb_ch = 3 + self.temb_ch = 64 + # self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + self.resolution = resolution + self.in_channels = in_channels + self.give_pre_end = give_pre_end + self.tanh_out = tanh_out + self.attn_resolutions = attn_resolutions + + # compute in_ch_mult, block_in and curr_res at lowest res + in_ch_mult = (1,) + tuple(ch_mult) + block_in = ch * ch_mult[self.num_resolutions - 1] + curr_res = resolution // 2 ** (self.num_resolutions - 1) + self.z_shape = (1, z_channels, curr_res, curr_res) + print( + "Working with z of shape {} = {} dimensions.".format( + self.z_shape, np.prod(self.z_shape) + ) + ) + + # z to block_in + self.conv_in = torch.nn.Conv2d( + z_channels, block_in, kernel_size=3, stride=1, padding=1 + ) + + self.conv_in3 = torch.nn.Conv2d( + z_channels + 3, block_in, kernel_size=3, stride=1, padding=1 + ) + + # middle + self.mid = nn.Module() + self.mid.block_1 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout, + ) + self.mid.attn_1 = make_attn(block_in, attn_type=attn_type) + self.mid.block_2 = ResnetBlock( + in_channels=block_in, + out_channels=block_in, + temb_channels=self.temb_ch, + dropout=dropout, + ) + + # upsampling + self.up = nn.ModuleList() + for i_level in reversed(range(self.num_resolutions)): + block = nn.ModuleList() + attn = nn.ModuleList() + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout, + ) + ) + block_in = block_out + if curr_res in attn_resolutions: + attn.append(make_attn(block_in, attn_type=attn_type)) + up = nn.Module() + up.block = block + up.attn = attn + if i_level != 0: + up.upsample = Upsample(block_in, resamp_with_conv) + up.rgb_conv = torch.nn.Conv2d( + block_in + 3, 3, kernel_size=3, stride=1, padding=1 + ) + up.rgb_cat_conv = torch.nn.Conv2d( + block_in + 3, block_in, kernel_size=3, stride=1, padding=1 + ) + curr_res = curr_res * 2 + self.up.insert(0, up) # prepend to get consistent order + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, out_ch, kernel_size=3, stride=1, padding=1 + ) + + def forward(self, z, temb=None): + # assert z.shape[1:] == self.z_shape[1:] + self.last_z_shape = z.shape + + # timestep embedding + # temb = None + + # z to block_in + rgb = z[:, :3] + if z.shape[1] == self.z_shape[1] + 3: + h = self.conv_in3(z) + else: + h = self.conv_in(z) + + # middle + # h = self.mid.block_1(h, temb) + # h = self.mid.block_2(h, temb) + + # upsampling + for i_level in reversed(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.up[i_level].block[i_block](h, temb) + if len(self.up[i_level].attn) > 0: + h = self.up[i_level].attn[i_block](h) + if i_level != 0: + h = self.up[i_level].upsample(h) + + # end + if self.give_pre_end: + return h + + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + + rgb = torch.nn.functional.interpolate(rgb, scale_factor=4.0, mode="bilinear") + rgb = torch.sigmoid(torch.logit(rgb, eps=1e-3) + h) + return rgb + + +class SimpleDecoder(nn.Module): + def __init__(self, in_channels, out_channels, *args, **kwargs): + super().__init__() + self.model = nn.ModuleList( + [ + nn.Conv2d(in_channels, in_channels, 1), + ResnetBlock( + in_channels=in_channels, + out_channels=2 * in_channels, + temb_channels=0, + dropout=0.0, + ), + ResnetBlock( + in_channels=2 * in_channels, + out_channels=4 * in_channels, + temb_channels=0, + dropout=0.0, + ), + ResnetBlock( + in_channels=4 * in_channels, + out_channels=2 * in_channels, + temb_channels=0, + dropout=0.0, + ), + nn.Conv2d(2 * in_channels, in_channels, 1), + Upsample(in_channels, with_conv=True), + ] + ) + # end + self.norm_out = Normalize(in_channels) + self.conv_out = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=1, padding=1 + ) + + def forward(self, x): + for i, layer in enumerate(self.model): + if i in [1, 2, 3]: + x = layer(x, None) + else: + x = layer(x) + + h = self.norm_out(x) + h = nonlinearity(h) + x = self.conv_out(h) + return x + + +class UpsampleDecoder(nn.Module): + def __init__( + self, + in_channels, + out_channels, + ch, + num_res_blocks, + resolution, + ch_mult=(2, 2), + dropout=0.0, + ): + super().__init__() + # upsampling + self.temb_ch = 0 + self.num_resolutions = len(ch_mult) + self.num_res_blocks = num_res_blocks + block_in = in_channels + curr_res = resolution // 2 ** (self.num_resolutions - 1) + self.upsample_blocks = nn.ModuleList() + self.rgb_blocks = nn.ModuleList() + for i_level in range(self.num_resolutions): + res_block = [] + block_out = ch * ch_mult[i_level] + for i_block in range(self.num_res_blocks + 1): + res_block.append( + ResnetBlock( + in_channels=block_in, + out_channels=block_out, + temb_channels=self.temb_ch, + dropout=dropout, + ) + ) + block_in = block_out + self.res_blocks.append(nn.ModuleList(res_block)) + if i_level != self.num_resolutions - 1: + self.upsample_blocks.append(Upsample(block_in, True)) + curr_res = curr_res * 2 + + # end + self.norm_out = Normalize(block_in) + self.conv_out = torch.nn.Conv2d( + block_in, out_channels, kernel_size=3, stride=1, padding=1 + ) + + def forward(self, x): + # upsampling + h = x + for k, i_level in enumerate(range(self.num_resolutions)): + for i_block in range(self.num_res_blocks + 1): + h = self.res_blocks[i_level][i_block](h, None) + if i_level != self.num_resolutions - 1: + h = self.upsample_blocks[k](h) + h = self.norm_out(h) + h = nonlinearity(h) + h = self.conv_out(h) + return h + + +class LatentRescaler(nn.Module): + def __init__(self, factor, in_channels, mid_channels, out_channels, depth=2): + super().__init__() + # residual block, interpolate, residual block + self.factor = factor + self.conv_in = nn.Conv2d( + in_channels, mid_channels, kernel_size=3, stride=1, padding=1 + ) + self.res_block1 = nn.ModuleList( + [ + ResnetBlock( + in_channels=mid_channels, + out_channels=mid_channels, + temb_channels=0, + dropout=0.0, + ) + for _ in range(depth) + ] + ) + self.attn = AttnBlock(mid_channels) + self.res_block2 = nn.ModuleList( + [ + ResnetBlock( + in_channels=mid_channels, + out_channels=mid_channels, + temb_channels=0, + dropout=0.0, + ) + for _ in range(depth) + ] + ) + + self.conv_out = nn.Conv2d( + mid_channels, + out_channels, + kernel_size=1, + ) + + def forward(self, x): + x = self.conv_in(x) + for block in self.res_block1: + x = block(x, None) + x = torch.nn.functional.interpolate( + x, + size=( + int(round(x.shape[2] * self.factor)), + int(round(x.shape[3] * self.factor)), + ), + ) + x = self.attn(x) + for block in self.res_block2: + x = block(x, None) + x = self.conv_out(x) + return x + + +class MergedRescaleEncoder(nn.Module): + def __init__( + self, + in_channels, + ch, + resolution, + out_ch, + num_res_blocks, + attn_resolutions, + dropout=0.0, + resamp_with_conv=True, + ch_mult=(1, 2, 4, 8), + rescale_factor=1.0, + rescale_module_depth=1, + ): + super().__init__() + intermediate_chn = ch * ch_mult[-1] + self.encoder = Encoder( + in_channels=in_channels, + num_res_blocks=num_res_blocks, + ch=ch, + ch_mult=ch_mult, + z_channels=intermediate_chn, + double_z=False, + resolution=resolution, + attn_resolutions=attn_resolutions, + dropout=dropout, + resamp_with_conv=resamp_with_conv, + out_ch=None, + ) + self.rescaler = LatentRescaler( + factor=rescale_factor, + in_channels=intermediate_chn, + mid_channels=intermediate_chn, + out_channels=out_ch, + depth=rescale_module_depth, + ) + + def forward(self, x): + x = self.encoder(x) + x = self.rescaler(x) + return x + + +class MergedRescaleDecoder(nn.Module): + def __init__( + self, + z_channels, + out_ch, + resolution, + num_res_blocks, + attn_resolutions, + ch, + ch_mult=(1, 2, 4, 8), + dropout=0.0, + resamp_with_conv=True, + rescale_factor=1.0, + rescale_module_depth=1, + ): + super().__init__() + tmp_chn = z_channels * ch_mult[-1] + self.decoder = Decoder( + out_ch=out_ch, + z_channels=tmp_chn, + attn_resolutions=attn_resolutions, + dropout=dropout, + resamp_with_conv=resamp_with_conv, + in_channels=None, + num_res_blocks=num_res_blocks, + ch_mult=ch_mult, + resolution=resolution, + ch=ch, + ) + self.rescaler = LatentRescaler( + factor=rescale_factor, + in_channels=z_channels, + mid_channels=tmp_chn, + out_channels=tmp_chn, + depth=rescale_module_depth, + ) + + def forward(self, x): + x = self.rescaler(x) + x = self.decoder(x) + return x + + +class Upsampler(nn.Module): + def __init__(self, in_size, out_size, in_channels, out_channels, ch_mult=2): + super().__init__() + assert out_size >= in_size + num_blocks = int(np.log2(out_size // in_size)) + 1 + factor_up = 1.0 + (out_size % in_size) + print( + f"Building {self.__class__.__name__} with in_size: {in_size} --> out_size {out_size} and factor {factor_up}" + ) + self.rescaler = LatentRescaler( + factor=factor_up, + in_channels=in_channels, + mid_channels=2 * in_channels, + out_channels=in_channels, + ) + self.decoder = Decoder( + out_ch=out_channels, + resolution=out_size, + z_channels=in_channels, + num_res_blocks=2, + attn_resolutions=[], + in_channels=None, + ch=in_channels, + ch_mult=[ch_mult for _ in range(num_blocks)], + ) + + def forward(self, x): + x = self.rescaler(x) + x = self.decoder(x) + return x + + +class Resize(nn.Module): + def __init__(self, in_channels=None, learned=False, mode="bilinear"): + super().__init__() + self.with_conv = learned + self.mode = mode + if self.with_conv: + print( + f"Note: {self.__class__.__name} uses learned downsampling and will ignore the fixed {mode} mode" + ) + raise NotImplementedError() + assert in_channels is not None + # no asymmetric padding in torch conv, must do it ourselves + self.conv = torch.nn.Conv2d( + in_channels, in_channels, kernel_size=4, stride=2, padding=1 + ) + + def forward(self, x, scale_factor=1.0): + if scale_factor == 1.0: + return x + else: + x = torch.nn.functional.interpolate( + x, mode=self.mode, align_corners=False, scale_factor=scale_factor + ) + return x + + +class FirstStagePostProcessor(nn.Module): + def __init__( + self, + ch_mult: list, + in_channels, + pretrained_model: nn.Module = None, + reshape=False, + n_channels=None, + dropout=0.0, + pretrained_config=None, + ): + super().__init__() + if pretrained_config is None: + assert ( + pretrained_model is not None + ), 'Either "pretrained_model" or "pretrained_config" must not be None' + self.pretrained_model = pretrained_model + else: + assert ( + pretrained_config is not None + ), 'Either "pretrained_model" or "pretrained_config" must not be None' + self.instantiate_pretrained(pretrained_config) + + self.do_reshape = reshape + + if n_channels is None: + n_channels = self.pretrained_model.encoder.ch + + self.proj_norm = Normalize(in_channels, num_groups=in_channels // 2) + self.proj = nn.Conv2d( + in_channels, n_channels, kernel_size=3, stride=1, padding=1 + ) + + blocks = [] + downs = [] + ch_in = n_channels + for m in ch_mult: + blocks.append( + ResnetBlock( + in_channels=ch_in, out_channels=m * n_channels, dropout=dropout + ) + ) + ch_in = m * n_channels + downs.append(Downsample(ch_in, with_conv=False)) + + self.model = nn.ModuleList(blocks) + self.downsampler = nn.ModuleList(downs) + + def instantiate_pretrained(self, config): + model = instantiate_from_config(config) + self.pretrained_model = model.eval() + # self.pretrained_model.train = False + for param in self.pretrained_model.parameters(): + param.requires_grad = False + + @torch.no_grad() + def encode_with_pretrained(self, x): + c = self.pretrained_model.encode(x) + if isinstance(c, DiagonalGaussianDistribution): + c = c.mode() + return c + + def forward(self, x): + z_fs = self.encode_with_pretrained(x) + z = self.proj_norm(z_fs) + z = self.proj(z) + z = nonlinearity(z) + + for submodel, downmodel in zip(self.model, self.downsampler): + z = submodel(z, temb=None) + z = downmodel(z) + + if self.do_reshape: + z = rearrange(z, "b c h w -> b (h w) c") + return z diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0e44449338cf3ff3bb3d124ef22c8f7bbca760a5 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/__init__.py @@ -0,0 +1 @@ +from . import base diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/base.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/base.py new file mode 100644 index 0000000000000000000000000000000000000000..97f1f6639b2cbbb4cfa4d2a84a300ce769d236a5 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/base.py @@ -0,0 +1,118 @@ +from dataclasses import dataclass + +import torch +import torch.nn as nn + +from threestudio.utils.config import parse_structured +from threestudio.utils.misc import get_device, load_module_weights +from threestudio.utils.typing import * + + +class Configurable: + @dataclass + class Config: + pass + + def __init__(self, cfg: Optional[dict] = None) -> None: + super().__init__() + self.cfg = parse_structured(self.Config, cfg) + + +class Updateable: + def do_update_step( + self, epoch: int, global_step: int, on_load_weights: bool = False + ): + for attr in self.__dir__(): + if attr.startswith("_"): + continue + try: + module = getattr(self, attr) + except: + continue # ignore attributes like property, which can't be retrived using getattr? + if isinstance(module, Updateable): + module.do_update_step( + epoch, global_step, on_load_weights=on_load_weights + ) + self.update_step(epoch, global_step, on_load_weights=on_load_weights) + + def do_update_step_end(self, epoch: int, global_step: int): + for attr in self.__dir__(): + if attr.startswith("_"): + continue + try: + module = getattr(self, attr) + except: + continue # ignore attributes like property, which can't be retrived using getattr? + if isinstance(module, Updateable): + module.do_update_step_end(epoch, global_step) + self.update_step_end(epoch, global_step) + + def update_step(self, epoch: int, global_step: int, on_load_weights: bool = False): + # override this method to implement custom update logic + # if on_load_weights is True, you should be careful doing things related to model evaluations, + # as the models and tensors are not guarenteed to be on the same device + pass + + def update_step_end(self, epoch: int, global_step: int): + pass + + +def update_if_possible(module: Any, epoch: int, global_step: int) -> None: + if isinstance(module, Updateable): + module.do_update_step(epoch, global_step) + + +def update_end_if_possible(module: Any, epoch: int, global_step: int) -> None: + if isinstance(module, Updateable): + module.do_update_step_end(epoch, global_step) + + +class BaseObject(Updateable): + @dataclass + class Config: + pass + + cfg: Config # add this to every subclass of BaseObject to enable static type checking + + def __init__( + self, cfg: Optional[Union[dict, DictConfig]] = None, *args, **kwargs + ) -> None: + super().__init__() + self.cfg = parse_structured(self.Config, cfg) + self.device = get_device() + self.configure(*args, **kwargs) + + def configure(self, *args, **kwargs) -> None: + pass + + +class BaseModule(nn.Module, Updateable): + @dataclass + class Config: + weights: Optional[str] = None + + cfg: Config # add this to every subclass of BaseModule to enable static type checking + + def __init__( + self, cfg: Optional[Union[dict, DictConfig]] = None, *args, **kwargs + ) -> None: + super().__init__() + self.cfg = parse_structured(self.Config, cfg) + self.device = get_device() + self.configure(*args, **kwargs) + if self.cfg.weights is not None: + # format: path/to/weights:module_name + weights_path, module_name = self.cfg.weights.split(":") + state_dict, epoch, global_step = load_module_weights( + weights_path, module_name=module_name, map_location="cpu" + ) + self.load_state_dict(state_dict) + self.do_update_step( + epoch, global_step, on_load_weights=True + ) # restore states + # dummy tensor to indicate model state + self._dummy: Float[Tensor, "..."] + self.register_buffer("_dummy", torch.zeros(0).float(), persistent=False) + + def configure(self, *args, **kwargs) -> None: + pass diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/callbacks.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/callbacks.py new file mode 100644 index 0000000000000000000000000000000000000000..c6a765a30fc8d196d63517701b854663a70ded14 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/callbacks.py @@ -0,0 +1,156 @@ +import os +import shutil +import subprocess + +import pytorch_lightning + +from threestudio.utils.config import dump_config +from threestudio.utils.misc import parse_version + +if parse_version(pytorch_lightning.__version__) > parse_version("1.8"): + from pytorch_lightning.callbacks import Callback +else: + from pytorch_lightning.callbacks.base import Callback + +from pytorch_lightning.callbacks.progress import TQDMProgressBar +from pytorch_lightning.utilities.rank_zero import rank_zero_only, rank_zero_warn + + +class VersionedCallback(Callback): + def __init__(self, save_root, version=None, use_version=True): + self.save_root = save_root + self._version = version + self.use_version = use_version + + @property + def version(self) -> int: + """Get the experiment version. + + Returns: + The experiment version if specified else the next version. + """ + if self._version is None: + self._version = self._get_next_version() + return self._version + + def _get_next_version(self): + existing_versions = [] + if os.path.isdir(self.save_root): + for f in os.listdir(self.save_root): + bn = os.path.basename(f) + if bn.startswith("version_"): + dir_ver = os.path.splitext(bn)[0].split("_")[1].replace("/", "") + existing_versions.append(int(dir_ver)) + if len(existing_versions) == 0: + return 0 + return max(existing_versions) + 1 + + @property + def savedir(self): + if not self.use_version: + return self.save_root + return os.path.join( + self.save_root, + self.version + if isinstance(self.version, str) + else f"version_{self.version}", + ) + + +class CodeSnapshotCallback(VersionedCallback): + def __init__(self, save_root, version=None, use_version=True): + super().__init__(save_root, version, use_version) + + def get_file_list(self): + return [ + b.decode() + for b in set( + subprocess.check_output( + 'git ls-files -- ":!:load/*"', shell=True + ).splitlines() + ) + | set( # hard code, TODO: use config to exclude folders or files + subprocess.check_output( + "git ls-files --others --exclude-standard", shell=True + ).splitlines() + ) + ] + + @rank_zero_only + def save_code_snapshot(self): + os.makedirs(self.savedir, exist_ok=True) + for f in self.get_file_list(): + if not os.path.exists(f) or os.path.isdir(f): + continue + os.makedirs(os.path.join(self.savedir, os.path.dirname(f)), exist_ok=True) + shutil.copyfile(f, os.path.join(self.savedir, f)) + + def on_fit_start(self, trainer, pl_module): + try: + self.save_code_snapshot() + except: + rank_zero_warn( + "Code snapshot is not saved. Please make sure you have git installed and are in a git repository." + ) + + +class ConfigSnapshotCallback(VersionedCallback): + def __init__(self, config_path, config, save_root, version=None, use_version=True): + super().__init__(save_root, version, use_version) + self.config_path = config_path + self.config = config + + @rank_zero_only + def save_config_snapshot(self): + os.makedirs(self.savedir, exist_ok=True) + dump_config(os.path.join(self.savedir, "parsed.yaml"), self.config) + shutil.copyfile(self.config_path, os.path.join(self.savedir, "raw.yaml")) + + def on_fit_start(self, trainer, pl_module): + self.save_config_snapshot() + + +class CustomProgressBar(TQDMProgressBar): + def get_metrics(self, *args, **kwargs): + # don't show the version number + items = super().get_metrics(*args, **kwargs) + items.pop("v_num", None) + return items + + +class ProgressCallback(Callback): + def __init__(self, save_path): + super().__init__() + self.save_path = save_path + self._file_handle = None + + @property + def file_handle(self): + if self._file_handle is None: + self._file_handle = open(self.save_path, "w") + return self._file_handle + + @rank_zero_only + def write(self, msg: str) -> None: + self.file_handle.seek(0) + self.file_handle.truncate() + self.file_handle.write(msg) + self.file_handle.flush() + + @rank_zero_only + def on_train_batch_end(self, trainer, pl_module, *args, **kwargs): + self.write( + f"Generation progress: {pl_module.true_global_step / trainer.max_steps * 100:.2f}%" + ) + + @rank_zero_only + def on_validation_start(self, trainer, pl_module): + self.write(f"Rendering validation image ...") + + @rank_zero_only + def on_test_start(self, trainer, pl_module): + self.write(f"Rendering video ...") + + @rank_zero_only + def on_predict_start(self, trainer, pl_module): + self.write(f"Exporting mesh assets ...") diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/config.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..b205566d4ee19bd01e991e32ba416efbbd4d824a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/config.py @@ -0,0 +1,131 @@ +import os +from dataclasses import dataclass, field +from datetime import datetime + +from omegaconf import OmegaConf + +import threestudio +from threestudio.utils.typing import * + +# ============ Register OmegaConf Recolvers ============= # +OmegaConf.register_new_resolver( + "calc_exp_lr_decay_rate", lambda factor, n: factor ** (1.0 / n) +) +OmegaConf.register_new_resolver("add", lambda a, b: a + b) +OmegaConf.register_new_resolver("sub", lambda a, b: a - b) +OmegaConf.register_new_resolver("mul", lambda a, b: a * b) +OmegaConf.register_new_resolver("div", lambda a, b: a / b) +OmegaConf.register_new_resolver("idiv", lambda a, b: a // b) +OmegaConf.register_new_resolver("basename", lambda p: os.path.basename(p)) +OmegaConf.register_new_resolver("rmspace", lambda s, sub: s.replace(" ", sub)) +OmegaConf.register_new_resolver("tuple2", lambda s: [float(s), float(s)]) +OmegaConf.register_new_resolver("gt0", lambda s: s > 0) +OmegaConf.register_new_resolver("cmaxgt0", lambda s: C_max(s) > 0) +OmegaConf.register_new_resolver("not", lambda s: not s) +OmegaConf.register_new_resolver( + "cmaxgt0orcmaxgt0", lambda a, b: C_max(a) > 0 or C_max(b) > 0 +) +# ======================================================= # + + +def C_max(value: Any) -> float: + if isinstance(value, int) or isinstance(value, float): + pass + else: + value = config_to_primitive(value) + if not isinstance(value, list): + raise TypeError("Scalar specification only supports list, got", type(value)) + if len(value) >= 6: + max_value = value[2] + for i in range(4, len(value), 2): + max_value = max(max_value, value[i]) + value = [value[0], value[1], max_value, value[3]] + if len(value) == 3: + value = [0] + value + assert len(value) == 4 + start_step, start_value, end_value, end_step = value + value = max(start_value, end_value) + return value + + +@dataclass +class ExperimentConfig: + name: str = "default" + description: str = "" + tag: str = "" + seed: int = 0 + use_timestamp: bool = True + timestamp: Optional[str] = None + exp_root_dir: str = "outputs" + + ### these shouldn't be set manually + exp_dir: str = "outputs/default" + trial_name: str = "exp" + trial_dir: str = "outputs/default/exp" + n_gpus: int = 1 + ### + + resume: Optional[str] = None + + data_type: str = "" + data: dict = field(default_factory=dict) + + system_type: str = "" + system: dict = field(default_factory=dict) + + # accept pytorch-lightning trainer parameters + # see https://lightning.ai/docs/pytorch/stable/common/trainer.html#trainer-class-api + trainer: dict = field(default_factory=dict) + + # accept pytorch-lightning checkpoint callback parameters + # see https://lightning.ai/docs/pytorch/stable/api/lightning.pytorch.callbacks.ModelCheckpoint.html#modelcheckpoint + checkpoint: dict = field(default_factory=dict) + + def __post_init__(self): + if not self.tag and not self.use_timestamp: + raise ValueError("Either tag is specified or use_timestamp is True.") + self.trial_name = self.tag + # if resume from an existing config, self.timestamp should not be None + if self.timestamp is None: + self.timestamp = "" + if self.use_timestamp: + if self.n_gpus > 1: + threestudio.warn( + "Timestamp is disabled when using multiple GPUs, please make sure you have a unique tag." + ) + else: + self.timestamp = datetime.now().strftime("@%Y%m%d-%H%M%S") + self.trial_name += self.timestamp + self.exp_dir = os.path.join(self.exp_root_dir, self.name) + self.trial_dir = os.path.join(self.exp_dir, self.trial_name) + os.makedirs(self.trial_dir, exist_ok=True) + + +def load_config(*yamls: str, cli_args: list = [], from_string=False, **kwargs) -> Any: + if from_string: + yaml_confs = [OmegaConf.create(s) for s in yamls] + else: + yaml_confs = [OmegaConf.load(f) for f in yamls] + cli_conf = OmegaConf.from_cli(cli_args) + cfg = OmegaConf.merge(*yaml_confs, cli_conf, kwargs) + OmegaConf.resolve(cfg) + assert isinstance(cfg, DictConfig) + scfg = parse_structured(ExperimentConfig, cfg) + return scfg + + +def config_to_primitive(config, resolve: bool = True) -> Any: + return OmegaConf.to_container(config, resolve=resolve) + + +def dump_config(path: str, config) -> None: + with open(path, "w") as fp: + OmegaConf.save(config=config, f=fp) + + +def parse_structured(fields: Any, cfg: Optional[Union[dict, DictConfig]] = None) -> Any: + scfg = OmegaConf.structured(fields(**cfg)) + return scfg + + + diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/loss.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/loss.py new file mode 100644 index 0000000000000000000000000000000000000000..eb0c7250f9e04900caaef02c02db649dd3200afb --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/loss.py @@ -0,0 +1,16 @@ +import torch + + +def _tensor_size(t): + return t.size()[1] * t.size()[2] * t.size()[3] + + +def tv_loss(x): + batch_size = x.size()[0] + h_x = x.size()[2] + w_x = x.size()[3] + count_h = _tensor_size(x[:, :, 1:, :]) + count_w = _tensor_size(x[:, :, :, 1:]) + h_tv = torch.pow((x[:, :, 1:, :] - x[:, :, : h_x - 1, :]), 2).sum() + w_tv = torch.pow((x[:, :, :, 1:] - x[:, :, :, : w_x - 1]), 2).sum() + return 2 * (h_tv / count_h + w_tv / count_w) / batch_size diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/misc.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..2cfdd630d602f3ef41ed324c1e4a29fac8843f94 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/misc.py @@ -0,0 +1,161 @@ +import gc +import math +import os +import re + +# import tinycudann as tcnn +import torch +from packaging import version + +from threestudio.utils.config import config_to_primitive +from threestudio.utils.typing import * + + +def parse_version(ver: str): + return version.parse(ver) + + +def get_rank(): + # SLURM_PROCID can be set even if SLURM is not managing the multiprocessing, + # therefore LOCAL_RANK needs to be checked first + rank_keys = ("RANK", "LOCAL_RANK", "SLURM_PROCID", "JSM_NAMESPACE_RANK") + for key in rank_keys: + rank = os.environ.get(key) + if rank is not None: + return int(rank) + return 0 + + +def get_device(): + return torch.device(f"cuda:{get_rank()}") + + +def load_module_weights( + path, module_name=None, ignore_modules=None, map_location=None +) -> Tuple[dict, int, int]: + if module_name is not None and ignore_modules is not None: + raise ValueError("module_name and ignore_modules cannot be both set") + if map_location is None: + map_location = get_device() + + ckpt = torch.load(path, map_location=map_location) + state_dict = ckpt["state_dict"] + state_dict_to_load = state_dict + + if ignore_modules is not None: + state_dict_to_load = {} + for k, v in state_dict.items(): + ignore = any( + [k.startswith(ignore_module + ".") for ignore_module in ignore_modules] + ) + if ignore: + continue + state_dict_to_load[k] = v + + if module_name is not None: + state_dict_to_load = {} + for k, v in state_dict.items(): + m = re.match(rf"^{module_name}\.(.*)$", k) + if m is None: + continue + state_dict_to_load[m.group(1)] = v + + return state_dict_to_load, ckpt["epoch"], ckpt["global_step"] + + +def C(value: Any, epoch: int, global_step: int, interpolation="linear") -> float: + if isinstance(value, int) or isinstance(value, float): + pass + else: + value = config_to_primitive(value) + if not isinstance(value, list): + raise TypeError("Scalar specification only supports list, got", type(value)) + if len(value) == 3: + value = [0] + value + if len(value) >= 6: + select_i = 3 + for i in range(3, len(value) - 2, 2): + if global_step >= value[i]: + select_i = i + 2 + if select_i != 3: + start_value, start_step = value[select_i - 3], value[select_i - 2] + else: + start_step, start_value = value[:2] + end_value, end_step = value[select_i - 1], value[select_i] + value = [start_step, start_value, end_value, end_step] + assert len(value) == 4 + start_step, start_value, end_value, end_step = value + if isinstance(end_step, int): + current_step = global_step + elif isinstance(end_step, float): + current_step = epoch + t = max(min(1.0, (current_step - start_step) / (end_step - start_step)), 0.0) + if interpolation == "linear": + value = start_value + (end_value - start_value) * t + elif interpolation == "exp": + value = math.exp(math.log(start_value) * (1 - t) + math.log(end_value) * t) + else: + raise ValueError( + f"Unknown interpolation method: {interpolation}, only support linear and exp" + ) + return value + + +def cleanup(): + gc.collect() + torch.cuda.empty_cache() + # tcnn.free_temporary_memory() + + +def finish_with_cleanup(func: Callable): + def wrapper(*args, **kwargs): + out = func(*args, **kwargs) + cleanup() + return out + + return wrapper + + +def _distributed_available(): + return torch.distributed.is_available() and torch.distributed.is_initialized() + + +def barrier(): + if not _distributed_available(): + return + else: + torch.distributed.barrier() + + +def broadcast(tensor, src=0): + if not _distributed_available(): + return tensor + else: + torch.distributed.broadcast(tensor, src=src) + return tensor + + +def enable_gradient(model, enabled: bool = True) -> None: + for param in model.parameters(): + param.requires_grad_(enabled) + + +def find_last_path(path: str): + if (path is not None) and ("LAST" in path): + path = path.replace(" ", "_") + base_dir_prefix, suffix = path.split("LAST", 1) + base_dir = os.path.dirname(base_dir_prefix) + prefix = os.path.split(base_dir_prefix)[-1] + base_dir_prefix = os.path.join(base_dir, prefix) + all_path = os.listdir(base_dir) + all_path = [os.path.join(base_dir, dir) for dir in all_path] + filtered_path = [dir for dir in all_path if dir.startswith(base_dir_prefix)] + filtered_path.sort(reverse=True) + last_path = filtered_path[0] + new_path = last_path + suffix + if os.path.exists(new_path): + return new_path + else: + raise FileNotFoundError(new_path) + else: + return path diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/ops.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/ops.py new file mode 100644 index 0000000000000000000000000000000000000000..24861eaae9baa50a8b5fe0bdd4e92290f8eef33c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/ops.py @@ -0,0 +1,518 @@ +import math +from collections import defaultdict + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from igl import fast_winding_number_for_meshes, point_mesh_squared_distance, read_obj +from torch.autograd import Function +from torch.cuda.amp import custom_bwd, custom_fwd + +import threestudio +from threestudio.utils.typing import * + + +def dot(x, y): + return torch.sum(x * y, -1, keepdim=True) + + +def reflect(x, n): + return 2 * dot(x, n) * n - x + + +ValidScale = Union[Tuple[float, float], Num[Tensor, "2 D"]] + + +def scale_tensor( + dat: Num[Tensor, "... D"], inp_scale: ValidScale, tgt_scale: ValidScale +): + if inp_scale is None: + inp_scale = (0, 1) + if tgt_scale is None: + tgt_scale = (0, 1) + if isinstance(tgt_scale, Tensor): + assert dat.shape[-1] == tgt_scale.shape[-1] + dat = (dat - inp_scale[0]) / (inp_scale[1] - inp_scale[0]) + dat = dat * (tgt_scale[1] - tgt_scale[0]) + tgt_scale[0] + return dat + + +class _TruncExp(Function): # pylint: disable=abstract-method + # Implementation from torch-ngp: + # https://github.com/ashawkey/torch-ngp/blob/93b08a0d4ec1cc6e69d85df7f0acdfb99603b628/activation.py + @staticmethod + @custom_fwd(cast_inputs=torch.float32) + def forward(ctx, x): # pylint: disable=arguments-differ + ctx.save_for_backward(x) + return torch.exp(x) + + @staticmethod + @custom_bwd + def backward(ctx, g): # pylint: disable=arguments-differ + x = ctx.saved_tensors[0] + return g * torch.exp(torch.clamp(x, max=15)) + + +class SpecifyGradient(Function): + # Implementation from stable-dreamfusion + # https://github.com/ashawkey/stable-dreamfusion + @staticmethod + @custom_fwd + def forward(ctx, input_tensor, gt_grad): + ctx.save_for_backward(gt_grad) + # we return a dummy value 1, which will be scaled by amp's scaler so we get the scale in backward. + return torch.ones([1], device=input_tensor.device, dtype=input_tensor.dtype) + + @staticmethod + @custom_bwd + def backward(ctx, grad_scale): + (gt_grad,) = ctx.saved_tensors + gt_grad = gt_grad * grad_scale + return gt_grad, None + + +trunc_exp = _TruncExp.apply + + +def get_activation(name) -> Callable: + if name is None: + return lambda x: x + name = name.lower() + if name == "none": + return lambda x: x + elif name == "lin2srgb": + return lambda x: torch.where( + x > 0.0031308, + torch.pow(torch.clamp(x, min=0.0031308), 1.0 / 2.4) * 1.055 - 0.055, + 12.92 * x, + ).clamp(0.0, 1.0) + elif name == "exp": + return lambda x: torch.exp(x) + elif name == "shifted_exp": + return lambda x: torch.exp(x - 1.0) + elif name == "trunc_exp": + return trunc_exp + elif name == "shifted_trunc_exp": + return lambda x: trunc_exp(x - 1.0) + elif name == "sigmoid": + return lambda x: torch.sigmoid(x) + elif name == "tanh": + return lambda x: torch.tanh(x) + elif name == "shifted_softplus": + return lambda x: F.softplus(x - 1.0) + elif name == "scale_-11_01": + return lambda x: x * 0.5 + 0.5 + else: + try: + return getattr(F, name) + except AttributeError: + raise ValueError(f"Unknown activation function: {name}") + + +def chunk_batch(func: Callable, chunk_size: int, *args, **kwargs) -> Any: + if chunk_size <= 0: + return func(*args, **kwargs) + B = None + for arg in list(args) + list(kwargs.values()): + if isinstance(arg, torch.Tensor): + B = arg.shape[0] + break + assert ( + B is not None + ), "No tensor found in args or kwargs, cannot determine batch size." + out = defaultdict(list) + out_type = None + # max(1, B) to support B == 0 + for i in range(0, max(1, B), chunk_size): + out_chunk = func( + *[ + arg[i : i + chunk_size] if isinstance(arg, torch.Tensor) else arg + for arg in args + ], + **{ + k: arg[i : i + chunk_size] if isinstance(arg, torch.Tensor) else arg + for k, arg in kwargs.items() + }, + ) + if out_chunk is None: + continue + out_type = type(out_chunk) + if isinstance(out_chunk, torch.Tensor): + out_chunk = {0: out_chunk} + elif isinstance(out_chunk, tuple) or isinstance(out_chunk, list): + chunk_length = len(out_chunk) + out_chunk = {i: chunk for i, chunk in enumerate(out_chunk)} + elif isinstance(out_chunk, dict): + pass + else: + print( + f"Return value of func must be in type [torch.Tensor, list, tuple, dict], get {type(out_chunk)}." + ) + exit(1) + for k, v in out_chunk.items(): + v = v if torch.is_grad_enabled() else v.detach() + out[k].append(v) + + if out_type is None: + return None + + out_merged: Dict[Any, Optional[torch.Tensor]] = {} + for k, v in out.items(): + if all([vv is None for vv in v]): + # allow None in return value + out_merged[k] = None + elif all([isinstance(vv, torch.Tensor) for vv in v]): + out_merged[k] = torch.cat(v, dim=0) + else: + raise TypeError( + f"Unsupported types in return value of func: {[type(vv) for vv in v if not isinstance(vv, torch.Tensor)]}" + ) + + if out_type is torch.Tensor: + return out_merged[0] + elif out_type in [tuple, list]: + return out_type([out_merged[i] for i in range(chunk_length)]) + elif out_type is dict: + return out_merged + + +def get_ray_directions( + H: int, + W: int, + focal: Union[float, Tuple[float, float]], + principal: Optional[Tuple[float, float]] = None, + use_pixel_centers: bool = True, +) -> Float[Tensor, "H W 3"]: + """ + Get ray directions for all pixels in camera coordinate. + Reference: https://www.scratchapixel.com/lessons/3d-basic-rendering/ + ray-tracing-generating-camera-rays/standard-coordinate-systems + + Inputs: + H, W, focal, principal, use_pixel_centers: image height, width, focal length, principal point and whether use pixel centers + Outputs: + directions: (H, W, 3), the direction of the rays in camera coordinate + """ + pixel_center = 0.5 if use_pixel_centers else 0 + + if isinstance(focal, float): + fx, fy = focal, focal + cx, cy = W / 2, H / 2 + else: + fx, fy = focal + assert principal is not None + cx, cy = principal + + i, j = torch.meshgrid( + torch.arange(W, dtype=torch.float32) + pixel_center, + torch.arange(H, dtype=torch.float32) + pixel_center, + indexing="xy", + ) + + directions: Float[Tensor, "H W 3"] = torch.stack( + [(i - cx) / fx, -(j - cy) / fy, -torch.ones_like(i)], -1 + ) + + return directions + + +def get_rays( + directions: Float[Tensor, "... 3"], + c2w: Float[Tensor, "... 4 4"], + keepdim=False, + noise_scale=0.0, + normalize=True, +) -> Tuple[Float[Tensor, "... 3"], Float[Tensor, "... 3"]]: + # Rotate ray directions from camera coordinate to the world coordinate + assert directions.shape[-1] == 3 + + if directions.ndim == 2: # (N_rays, 3) + if c2w.ndim == 2: # (4, 4) + c2w = c2w[None, :, :] + assert c2w.ndim == 3 # (N_rays, 4, 4) or (1, 4, 4) + rays_d = (directions[:, None, :] * c2w[:, :3, :3]).sum(-1) # (N_rays, 3) + rays_o = c2w[:, :3, 3].expand(rays_d.shape) + elif directions.ndim == 3: # (H, W, 3) + assert c2w.ndim in [2, 3] + if c2w.ndim == 2: # (4, 4) + rays_d = (directions[:, :, None, :] * c2w[None, None, :3, :3]).sum( + -1 + ) # (H, W, 3) + rays_o = c2w[None, None, :3, 3].expand(rays_d.shape) + elif c2w.ndim == 3: # (B, 4, 4) + rays_d = (directions[None, :, :, None, :] * c2w[:, None, None, :3, :3]).sum( + -1 + ) # (B, H, W, 3) + rays_o = c2w[:, None, None, :3, 3].expand(rays_d.shape) + elif directions.ndim == 4: # (B, H, W, 3) + assert c2w.ndim == 3 # (B, 4, 4) + rays_d = (directions[:, :, :, None, :] * c2w[:, None, None, :3, :3]).sum( + -1 + ) # (B, H, W, 3) + rays_o = c2w[:, None, None, :3, 3].expand(rays_d.shape) + + # add camera noise to avoid grid-like artifect + # https://github.com/ashawkey/stable-dreamfusion/blob/49c3d4fa01d68a4f027755acf94e1ff6020458cc/nerf/utils.py#L373 + if noise_scale > 0: + rays_o = rays_o + torch.randn(3, device=rays_o.device) * noise_scale + rays_d = rays_d + torch.randn(3, device=rays_d.device) * noise_scale + + if normalize: + rays_d = F.normalize(rays_d, dim=-1) + if not keepdim: + rays_o, rays_d = rays_o.reshape(-1, 3), rays_d.reshape(-1, 3) + + return rays_o, rays_d + + +def get_projection_matrix( + fovy: Float[Tensor, "B"], aspect_wh: float, near: float, far: float +) -> Float[Tensor, "B 4 4"]: + batch_size = fovy.shape[0] + proj_mtx = torch.zeros(batch_size, 4, 4, dtype=torch.float32) + proj_mtx[:, 0, 0] = 1.0 / (torch.tan(fovy / 2.0) * aspect_wh) + proj_mtx[:, 1, 1] = -1.0 / torch.tan( + fovy / 2.0 + ) # add a negative sign here as the y axis is flipped in nvdiffrast output + proj_mtx[:, 2, 2] = -(far + near) / (far - near) + proj_mtx[:, 2, 3] = -2.0 * far * near / (far - near) + proj_mtx[:, 3, 2] = -1.0 + return proj_mtx + + +def get_mvp_matrix( + c2w: Float[Tensor, "B 4 4"], proj_mtx: Float[Tensor, "B 4 4"] +) -> Float[Tensor, "B 4 4"]: + # calculate w2c from c2w: R' = Rt, t' = -Rt * t + # mathematically equivalent to (c2w)^-1 + w2c: Float[Tensor, "B 4 4"] = torch.zeros(c2w.shape[0], 4, 4).to(c2w) + w2c[:, :3, :3] = c2w[:, :3, :3].permute(0, 2, 1) + w2c[:, :3, 3:] = -c2w[:, :3, :3].permute(0, 2, 1) @ c2w[:, :3, 3:] + w2c[:, 3, 3] = 1.0 + # calculate mvp matrix by proj_mtx @ w2c (mv_mtx) + mvp_mtx = proj_mtx @ w2c + return mvp_mtx + + +def get_full_projection_matrix( + c2w: Float[Tensor, "B 4 4"], proj_mtx: Float[Tensor, "B 4 4"] +) -> Float[Tensor, "B 4 4"]: + return (c2w.unsqueeze(0).bmm(proj_mtx.unsqueeze(0))).squeeze(0) + + +# gaussian splatting functions +def convert_pose(C2W): + flip_yz = torch.eye(4, device=C2W.device) + flip_yz[1, 1] = -1 + flip_yz[2, 2] = -1 + C2W = torch.matmul(C2W, flip_yz) + return C2W + + +def get_projection_matrix_gaussian(znear, zfar, fovX, fovY, device="cuda"): + tanHalfFovY = math.tan((fovY / 2)) + tanHalfFovX = math.tan((fovX / 2)) + + top = tanHalfFovY * znear + bottom = -top + right = tanHalfFovX * znear + left = -right + + P = torch.zeros(4, 4, device=device) + + z_sign = 1.0 + + P[0, 0] = 2.0 * znear / (right - left) + P[1, 1] = 2.0 * znear / (top - bottom) + P[0, 2] = (right + left) / (right - left) + P[1, 2] = (top + bottom) / (top - bottom) + P[3, 2] = z_sign + P[2, 2] = z_sign * zfar / (zfar - znear) + P[2, 3] = -(zfar * znear) / (zfar - znear) + return P + + +def get_fov_gaussian(P): + tanHalfFovX = 1 / P[0, 0] + tanHalfFovY = 1 / P[1, 1] + fovY = math.atan(tanHalfFovY) * 2 + fovX = math.atan(tanHalfFovX) * 2 + return fovX, fovY + + +def get_cam_info_gaussian(c2w, fovx, fovy, znear, zfar): + c2w = convert_pose(c2w) + world_view_transform = torch.inverse(c2w) + + world_view_transform = world_view_transform.transpose(0, 1).cuda().float() + projection_matrix = ( + get_projection_matrix_gaussian(znear=znear, zfar=zfar, fovX=fovx, fovY=fovy) + .transpose(0, 1) + .cuda() + ) + full_proj_transform = ( + world_view_transform.unsqueeze(0).bmm(projection_matrix.unsqueeze(0)) + ).squeeze(0) + camera_center = world_view_transform.inverse()[3, :3] + + # return world_view_transform, full_proj_transform, camera_center + return world_view_transform, full_proj_transform, camera_center, projection_matrix + + +def binary_cross_entropy(input, target): + """ + F.binary_cross_entropy is not numerically stable in mixed-precision training. + """ + return -(target * torch.log(input) + (1 - target) * torch.log(1 - input)).mean() + + +def tet_sdf_diff( + vert_sdf: Float[Tensor, "Nv 1"], tet_edges: Integer[Tensor, "Ne 2"] +) -> Float[Tensor, ""]: + sdf_f1x6x2 = vert_sdf[:, 0][tet_edges.reshape(-1)].reshape(-1, 2) + mask = torch.sign(sdf_f1x6x2[..., 0]) != torch.sign(sdf_f1x6x2[..., 1]) + sdf_f1x6x2 = sdf_f1x6x2[mask] + sdf_diff = F.binary_cross_entropy_with_logits( + sdf_f1x6x2[..., 0], (sdf_f1x6x2[..., 1] > 0).float() + ) + F.binary_cross_entropy_with_logits( + sdf_f1x6x2[..., 1], (sdf_f1x6x2[..., 0] > 0).float() + ) + return sdf_diff + + +# Implementation from Latent-NeRF +# https://github.com/eladrich/latent-nerf/blob/f49ecefcd48972e69a28e3116fe95edf0fac4dc8/src/latent_nerf/models/mesh_utils.py +class MeshOBJ: + dx = torch.zeros(3).float() + dx[0] = 1 + dy, dz = dx[[1, 0, 2]], dx[[2, 1, 0]] + dx, dy, dz = dx[None, :], dy[None, :], dz[None, :] + + def __init__(self, v: np.ndarray, f: np.ndarray): + self.v = v + self.f = f + self.dx, self.dy, self.dz = MeshOBJ.dx, MeshOBJ.dy, MeshOBJ.dz + self.v_tensor = torch.from_numpy(self.v) + + vf = self.v[self.f, :] + self.f_center = vf.mean(axis=1) + self.f_center_tensor = torch.from_numpy(self.f_center).float() + + e1 = vf[:, 1, :] - vf[:, 0, :] + e2 = vf[:, 2, :] - vf[:, 0, :] + self.face_normals = np.cross(e1, e2) + self.face_normals = ( + self.face_normals / np.linalg.norm(self.face_normals, axis=-1)[:, None] + ) + self.face_normals_tensor = torch.from_numpy(self.face_normals) + + def normalize_mesh(self, target_scale=0.5): + verts = self.v + + # Compute center of bounding box + # center = torch.mean(torch.column_stack([torch.max(verts, dim=0)[0], torch.min(verts, dim=0)[0]])) + center = verts.mean(axis=0) + verts = verts - center + scale = np.max(np.linalg.norm(verts, axis=1)) + verts = (verts / scale) * target_scale + + return MeshOBJ(verts, self.f) + + def winding_number(self, query: torch.Tensor): + device = query.device + shp = query.shape + query_np = query.detach().cpu().reshape(-1, 3).numpy() + target_alphas = fast_winding_number_for_meshes( + self.v.astype(np.float32), self.f, query_np + ) + return torch.from_numpy(target_alphas).reshape(shp[:-1]).to(device) + + def gaussian_weighted_distance(self, query: torch.Tensor, sigma): + device = query.device + shp = query.shape + query_np = query.detach().cpu().reshape(-1, 3).numpy() + distances, _, _ = point_mesh_squared_distance( + query_np, self.v.astype(np.float32), self.f + ) + distances = torch.from_numpy(distances).reshape(shp[:-1]).to(device) + weight = torch.exp(-(distances / (2 * sigma**2))) + return weight + + +def ce_pq_loss(p, q, weight=None): + def clamp(v, T=0.0001): + return v.clamp(T, 1 - T) + + p = p.view(q.shape) + ce = -1 * (p * torch.log(clamp(q)) + (1 - p) * torch.log(clamp(1 - q))) + if weight is not None: + ce *= weight + return ce.sum() + + +class ShapeLoss(nn.Module): + def __init__(self, guide_shape): + super().__init__() + self.mesh_scale = 0.7 + self.proximal_surface = 0.3 + self.delta = 0.2 + self.shape_path = guide_shape + v, _, _, f, _, _ = read_obj(self.shape_path, float) + mesh = MeshOBJ(v, f) + matrix_rot = np.array([[1, 0, 0], [0, 0, -1], [0, 1, 0]]) @ np.array( + [[0, 0, 1], [0, 1, 0], [-1, 0, 0]] + ) + self.sketchshape = mesh.normalize_mesh(self.mesh_scale) + self.sketchshape = MeshOBJ( + np.ascontiguousarray( + (matrix_rot @ self.sketchshape.v.transpose(1, 0)).transpose(1, 0) + ), + f, + ) + + def forward(self, xyzs, sigmas): + mesh_occ = self.sketchshape.winding_number(xyzs) + if self.proximal_surface > 0: + weight = 1 - self.sketchshape.gaussian_weighted_distance( + xyzs, self.proximal_surface + ) + else: + weight = None + indicator = (mesh_occ > 0.5).float() + nerf_occ = 1 - torch.exp(-self.delta * sigmas) + nerf_occ = nerf_occ.clamp(min=0, max=1.1) + loss = ce_pq_loss( + nerf_occ, indicator, weight=weight + ) # order is important for CE loss + second argument may not be optimized + return loss + + +def shifted_expotional_decay(a, b, c, r): + return a * torch.exp(-b * r) + c + + +def shifted_cosine_decay(a, b, c, r): + return a * torch.cos(b * r + c) + a + + +def perpendicular_component(x: Float[Tensor, "B C H W"], y: Float[Tensor, "B C H W"]): + # get the component of x that is perpendicular to y + eps = torch.ones_like(x[:, 0, 0, 0]) * 1e-6 + return ( + x + - ( + torch.mul(x, y).sum(dim=[1, 2, 3]) + / torch.maximum(torch.mul(y, y).sum(dim=[1, 2, 3]), eps) + ).view(-1, 1, 1, 1) + * y + ) + + +def validate_empty_rays(ray_indices, t_start, t_end): + if ray_indices.nelement() == 0: + threestudio.warn("Empty rays_indices!") + ray_indices = torch.LongTensor([0]).to(ray_indices) + t_start = torch.Tensor([0]).to(ray_indices) + t_end = torch.Tensor([0]).to(ray_indices) + return ray_indices, t_start, t_end diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/perceptual/__init__.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/perceptual/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a4d2c7aa01c18180f20d79b91d9cdac3bf78f5c7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/perceptual/__init__.py @@ -0,0 +1 @@ +from .perceptual import PerceptualLoss diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/perceptual/perceptual.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/perceptual/perceptual.py new file mode 100644 index 0000000000000000000000000000000000000000..403d9a922e58f65024980f531634e5105f83db25 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/perceptual/perceptual.py @@ -0,0 +1,170 @@ +"""Stripped version of https://github.com/richzhang/PerceptualSimilarity/tree/master/models""" + +from collections import namedtuple +from dataclasses import dataclass, field + +import torch +import torch.nn as nn +from torchvision import models + +import threestudio +from threestudio.utils.base import BaseObject +from threestudio.utils.perceptual.utils import get_ckpt_path +from threestudio.utils.typing import * + + +@threestudio.register("perceptual-loss") +class PerceptualLossObject(BaseObject): + @dataclass + class Config(BaseObject.Config): + use_dropout: bool = True + + cfg: Config + + def configure(self) -> None: + self.perceptual_loss = PerceptualLoss(self.cfg.use_dropout).to(self.device) + + def __call__( + self, + x: Float[Tensor, "B 3 256 256"], + y: Float[Tensor, "B 3 256 256"], + ): + return self.perceptual_loss(x, y) + + +class PerceptualLoss(nn.Module): + # Learned perceptual metric + def __init__(self, use_dropout=True): + super().__init__() + self.scaling_layer = ScalingLayer() + self.chns = [64, 128, 256, 512, 512] # vg16 features + self.net = vgg16(pretrained=True, requires_grad=False) + self.lin0 = NetLinLayer(self.chns[0], use_dropout=use_dropout) + self.lin1 = NetLinLayer(self.chns[1], use_dropout=use_dropout) + self.lin2 = NetLinLayer(self.chns[2], use_dropout=use_dropout) + self.lin3 = NetLinLayer(self.chns[3], use_dropout=use_dropout) + self.lin4 = NetLinLayer(self.chns[4], use_dropout=use_dropout) + self.load_from_pretrained() + for param in self.parameters(): + param.requires_grad = False + + def load_from_pretrained(self, name="vgg_lpips"): + ckpt = get_ckpt_path(name, "threestudio/utils/lpips") + self.load_state_dict( + torch.load(ckpt, map_location=torch.device("cpu")), strict=False + ) + print("loaded pretrained LPIPS loss from {}".format(ckpt)) + + @classmethod + def from_pretrained(cls, name="vgg_lpips"): + if name != "vgg_lpips": + raise NotImplementedError + model = cls() + ckpt = get_ckpt_path(name) + model.load_state_dict( + torch.load(ckpt, map_location=torch.device("cpu")), strict=False + ) + return model + + def forward(self, input, target): + in0_input, in1_input = (self.scaling_layer(input), self.scaling_layer(target)) + outs0, outs1 = self.net(in0_input), self.net(in1_input) + feats0, feats1, diffs = {}, {}, {} + lins = [self.lin0, self.lin1, self.lin2, self.lin3, self.lin4] + for kk in range(len(self.chns)): + feats0[kk], feats1[kk] = normalize_tensor(outs0[kk]), normalize_tensor( + outs1[kk] + ) + diffs[kk] = (feats0[kk] - feats1[kk]) ** 2 + + res = [ + spatial_average(lins[kk].model(diffs[kk]), keepdim=True) + for kk in range(len(self.chns)) + ] + val = res[0] + for l in range(1, len(self.chns)): + val += res[l] + return val + + +class ScalingLayer(nn.Module): + def __init__(self): + super(ScalingLayer, self).__init__() + self.register_buffer( + "shift", torch.Tensor([-0.030, -0.088, -0.188])[None, :, None, None] + ) + self.register_buffer( + "scale", torch.Tensor([0.458, 0.448, 0.450])[None, :, None, None] + ) + + def forward(self, inp): + return (inp - self.shift) / self.scale + + +class NetLinLayer(nn.Module): + """A single linear layer which does a 1x1 conv""" + + def __init__(self, chn_in, chn_out=1, use_dropout=False): + super(NetLinLayer, self).__init__() + layers = ( + [ + nn.Dropout(), + ] + if (use_dropout) + else [] + ) + layers += [ + nn.Conv2d(chn_in, chn_out, 1, stride=1, padding=0, bias=False), + ] + self.model = nn.Sequential(*layers) + + +class vgg16(torch.nn.Module): + def __init__(self, requires_grad=False, pretrained=True): + super(vgg16, self).__init__() + vgg_pretrained_features = models.vgg16(pretrained=pretrained).features + self.slice1 = torch.nn.Sequential() + self.slice2 = torch.nn.Sequential() + self.slice3 = torch.nn.Sequential() + self.slice4 = torch.nn.Sequential() + self.slice5 = torch.nn.Sequential() + self.N_slices = 5 + for x in range(4): + self.slice1.add_module(str(x), vgg_pretrained_features[x]) + for x in range(4, 9): + self.slice2.add_module(str(x), vgg_pretrained_features[x]) + for x in range(9, 16): + self.slice3.add_module(str(x), vgg_pretrained_features[x]) + for x in range(16, 23): + self.slice4.add_module(str(x), vgg_pretrained_features[x]) + for x in range(23, 30): + self.slice5.add_module(str(x), vgg_pretrained_features[x]) + if not requires_grad: + for param in self.parameters(): + param.requires_grad = False + + def forward(self, X): + h = self.slice1(X) + h_relu1_2 = h + h = self.slice2(h) + h_relu2_2 = h + h = self.slice3(h) + h_relu3_3 = h + h = self.slice4(h) + h_relu4_3 = h + h = self.slice5(h) + h_relu5_3 = h + vgg_outputs = namedtuple( + "VggOutputs", ["relu1_2", "relu2_2", "relu3_3", "relu4_3", "relu5_3"] + ) + out = vgg_outputs(h_relu1_2, h_relu2_2, h_relu3_3, h_relu4_3, h_relu5_3) + return out + + +def normalize_tensor(x, eps=1e-10): + norm_factor = torch.sqrt(torch.sum(x**2, dim=1, keepdim=True)) + return x / (norm_factor + eps) + + +def spatial_average(x, keepdim=True): + return x.mean([2, 3], keepdim=keepdim) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/perceptual/utils.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/perceptual/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5da48b3158876b3388f24edf395d105db175f980 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/perceptual/utils.py @@ -0,0 +1,154 @@ +import hashlib +import os + +import requests +from tqdm import tqdm + +URL_MAP = {"vgg_lpips": "https://heibox.uni-heidelberg.de/f/607503859c864bc1b30b/?dl=1"} + +CKPT_MAP = {"vgg_lpips": "vgg.pth"} + +MD5_MAP = {"vgg_lpips": "d507d7349b931f0638a25a48a722f98a"} + + +def download(url, local_path, chunk_size=1024): + os.makedirs(os.path.split(local_path)[0], exist_ok=True) + with requests.get(url, stream=True) as r: + total_size = int(r.headers.get("content-length", 0)) + with tqdm(total=total_size, unit="B", unit_scale=True) as pbar: + with open(local_path, "wb") as f: + for data in r.iter_content(chunk_size=chunk_size): + if data: + f.write(data) + pbar.update(chunk_size) + + +def md5_hash(path): + with open(path, "rb") as f: + content = f.read() + return hashlib.md5(content).hexdigest() + + +def get_ckpt_path(name, root, check=False): + assert name in URL_MAP + path = os.path.join(root, CKPT_MAP[name]) + if not os.path.exists(path) or (check and not md5_hash(path) == MD5_MAP[name]): + print("Downloading {} model from {} to {}".format(name, URL_MAP[name], path)) + download(URL_MAP[name], path) + md5 = md5_hash(path) + assert md5 == MD5_MAP[name], md5 + return path + + +class KeyNotFoundError(Exception): + def __init__(self, cause, keys=None, visited=None): + self.cause = cause + self.keys = keys + self.visited = visited + messages = list() + if keys is not None: + messages.append("Key not found: {}".format(keys)) + if visited is not None: + messages.append("Visited: {}".format(visited)) + messages.append("Cause:\n{}".format(cause)) + message = "\n".join(messages) + super().__init__(message) + + +def retrieve( + list_or_dict, key, splitval="/", default=None, expand=True, pass_success=False +): + """Given a nested list or dict return the desired value at key expanding + callable nodes if necessary and :attr:`expand` is ``True``. The expansion + is done in-place. + + Parameters + ---------- + list_or_dict : list or dict + Possibly nested list or dictionary. + key : str + key/to/value, path like string describing all keys necessary to + consider to get to the desired value. List indices can also be + passed here. + splitval : str + String that defines the delimiter between keys of the + different depth levels in `key`. + default : obj + Value returned if :attr:`key` is not found. + expand : bool + Whether to expand callable nodes on the path or not. + + Returns + ------- + The desired value or if :attr:`default` is not ``None`` and the + :attr:`key` is not found returns ``default``. + + Raises + ------ + Exception if ``key`` not in ``list_or_dict`` and :attr:`default` is + ``None``. + """ + + keys = key.split(splitval) + + success = True + try: + visited = [] + parent = None + last_key = None + for key in keys: + if callable(list_or_dict): + if not expand: + raise KeyNotFoundError( + ValueError( + "Trying to get past callable node with expand=False." + ), + keys=keys, + visited=visited, + ) + list_or_dict = list_or_dict() + parent[last_key] = list_or_dict + + last_key = key + parent = list_or_dict + + try: + if isinstance(list_or_dict, dict): + list_or_dict = list_or_dict[key] + else: + list_or_dict = list_or_dict[int(key)] + except (KeyError, IndexError, ValueError) as e: + raise KeyNotFoundError(e, keys=keys, visited=visited) + + visited += [key] + # final expansion of retrieved value + if expand and callable(list_or_dict): + list_or_dict = list_or_dict() + parent[last_key] = list_or_dict + except KeyNotFoundError as e: + if default is None: + raise e + else: + list_or_dict = default + success = False + + if not pass_success: + return list_or_dict + else: + return list_or_dict, success + + +if __name__ == "__main__": + config = { + "keya": "a", + "keyb": "b", + "keyc": { + "cc1": 1, + "cc2": 2, + }, + } + from omegaconf import OmegaConf + + config = OmegaConf.create(config) + print(config) + retrieve(config, "keya") diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/rasterize.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/rasterize.py new file mode 100644 index 0000000000000000000000000000000000000000..a174bc1d95cdc792054417728997be7823b58da2 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/rasterize.py @@ -0,0 +1,78 @@ +import nvdiffrast.torch as dr +import torch + +from threestudio.utils.typing import * + + +class NVDiffRasterizerContext: + def __init__(self, context_type: str, device: torch.device) -> None: + self.device = device + self.ctx = self.initialize_context(context_type, device) + + def initialize_context( + self, context_type: str, device: torch.device + ) -> Union[dr.RasterizeGLContext, dr.RasterizeCudaContext]: + if context_type == "gl": + return dr.RasterizeGLContext(device=device) + elif context_type == "cuda": + return dr.RasterizeCudaContext(device=device) + else: + raise ValueError(f"Unknown rasterizer context type: {context_type}") + + def vertex_transform( + self, verts: Float[Tensor, "Nv 3"], mvp_mtx: Float[Tensor, "B 4 4"] + ) -> Float[Tensor, "B Nv 4"]: + verts_homo = torch.cat( + [verts, torch.ones([verts.shape[0], 1]).to(verts)], dim=-1 + ) + return torch.matmul(verts_homo, mvp_mtx.permute(0, 2, 1)) + + def rasterize( + self, + pos: Float[Tensor, "B Nv 4"], + tri: Integer[Tensor, "Nf 3"], + resolution: Union[int, Tuple[int, int]], + ): + # rasterize in instance mode (single topology) + return dr.rasterize(self.ctx, pos.float(), tri.int(), resolution, grad_db=True) + + def rasterize_one( + self, + pos: Float[Tensor, "Nv 4"], + tri: Integer[Tensor, "Nf 3"], + resolution: Union[int, Tuple[int, int]], + ): + # rasterize one single mesh under a single viewpoint + rast, rast_db = self.rasterize(pos[None, ...], tri, resolution) + return rast[0], rast_db[0] + + def antialias( + self, + color: Float[Tensor, "B H W C"], + rast: Float[Tensor, "B H W 4"], + pos: Float[Tensor, "B Nv 4"], + tri: Integer[Tensor, "Nf 3"], + ) -> Float[Tensor, "B H W C"]: + return dr.antialias(color.float(), rast, pos.float(), tri.int()) + + def interpolate( + self, + attr: Float[Tensor, "B Nv C"], + rast: Float[Tensor, "B H W 4"], + tri: Integer[Tensor, "Nf 3"], + rast_db=None, + diff_attrs=None, + ) -> Float[Tensor, "B H W C"]: + return dr.interpolate( + attr.float(), rast, tri.int(), rast_db=rast_db, diff_attrs=diff_attrs + ) + + def interpolate_one( + self, + attr: Float[Tensor, "Nv C"], + rast: Float[Tensor, "B H W 4"], + tri: Integer[Tensor, "Nf 3"], + rast_db=None, + diff_attrs=None, + ) -> Float[Tensor, "B H W C"]: + return self.interpolate(attr[None, ...], rast, tri, rast_db, diff_attrs) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/saving.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/saving.py new file mode 100644 index 0000000000000000000000000000000000000000..0c42c2cbd1b7777d24f1e8ab7333264008b6964b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/saving.py @@ -0,0 +1,655 @@ +import json +import os +import re +import shutil + +import cv2 +import imageio +import matplotlib.pyplot as plt +import numpy as np +import torch +import trimesh +import wandb +from matplotlib import cm +from matplotlib.colors import LinearSegmentedColormap +from PIL import Image, ImageDraw +from pytorch_lightning.loggers import WandbLogger + +from threestudio.models.mesh import Mesh +from threestudio.utils.typing import * + + +class SaverMixin: + _save_dir: Optional[str] = None + _wandb_logger: Optional[WandbLogger] = None + + def set_save_dir(self, save_dir: str): + self._save_dir = save_dir + + def get_save_dir(self): + if self._save_dir is None: + raise ValueError("Save dir is not set") + return self._save_dir + + def convert_data(self, data): + if data is None: + return None + elif isinstance(data, np.ndarray): + return data + elif isinstance(data, torch.Tensor): + return data.detach().cpu().numpy() + elif isinstance(data, list): + return [self.convert_data(d) for d in data] + elif isinstance(data, dict): + return {k: self.convert_data(v) for k, v in data.items()} + else: + raise TypeError( + "Data must be in type numpy.ndarray, torch.Tensor, list or dict, getting", + type(data), + ) + + def get_save_path(self, filename): + save_path = os.path.join(self.get_save_dir(), filename) + os.makedirs(os.path.dirname(save_path), exist_ok=True) + return save_path + + def create_loggers(self, cfg_loggers: DictConfig) -> None: + if "wandb" in cfg_loggers.keys() and cfg_loggers.wandb.enable: + self._wandb_logger = WandbLogger( + project=cfg_loggers.wandb.project, name=cfg_loggers.wandb.name + ) + + def get_loggers(self) -> List: + if self._wandb_logger: + return [self._wandb_logger] + else: + return [] + + DEFAULT_RGB_KWARGS = {"data_format": "HWC", "data_range": (0, 1)} + DEFAULT_UV_KWARGS = { + "data_format": "HWC", + "data_range": (0, 1), + "cmap": "checkerboard", + } + DEFAULT_GRAYSCALE_KWARGS = {"data_range": None, "cmap": "jet"} + DEFAULT_GRID_KWARGS = {"align": "max"} + + def get_rgb_image_(self, img, data_format, data_range, rgba=False): + img = self.convert_data(img) + assert data_format in ["CHW", "HWC"] + if data_format == "CHW": + img = img.transpose(1, 2, 0) + if img.dtype != np.uint8: + img = img.clip(min=data_range[0], max=data_range[1]) + img = ( + (img - data_range[0]) / (data_range[1] - data_range[0]) * 255.0 + ).astype(np.uint8) + nc = 4 if rgba else 3 + imgs = [img[..., start : start + nc] for start in range(0, img.shape[-1], nc)] + imgs = [ + img_ + if img_.shape[-1] == nc + else np.concatenate( + [ + img_, + np.zeros( + (img_.shape[0], img_.shape[1], nc - img_.shape[2]), + dtype=img_.dtype, + ), + ], + axis=-1, + ) + for img_ in imgs + ] + img = np.concatenate(imgs, axis=1) + if rgba: + img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGRA) + else: + img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) + return img + + def _save_rgb_image( + self, + filename, + img, + data_format, + data_range, + name: Optional[str] = None, + step: Optional[int] = None, + ): + img = self.get_rgb_image_(img, data_format, data_range) + cv2.imwrite(filename, img) + if name and self._wandb_logger: + wandb.log( + { + name: wandb.Image(self.get_save_path(filename)), + "trainer/global_step": step, + } + ) + + def save_rgb_image( + self, + filename, + img, + data_format=DEFAULT_RGB_KWARGS["data_format"], + data_range=DEFAULT_RGB_KWARGS["data_range"], + name: Optional[str] = None, + step: Optional[int] = None, + ) -> str: + save_path = self.get_save_path(filename) + self._save_rgb_image(save_path, img, data_format, data_range, name, step) + return save_path + + def get_uv_image_(self, img, data_format, data_range, cmap): + img = self.convert_data(img) + assert data_format in ["CHW", "HWC"] + if data_format == "CHW": + img = img.transpose(1, 2, 0) + img = img.clip(min=data_range[0], max=data_range[1]) + img = (img - data_range[0]) / (data_range[1] - data_range[0]) + assert cmap in ["checkerboard", "color"] + if cmap == "checkerboard": + n_grid = 64 + mask = (img * n_grid).astype(int) + mask = (mask[..., 0] + mask[..., 1]) % 2 == 0 + img = np.ones((img.shape[0], img.shape[1], 3), dtype=np.uint8) * 255 + img[mask] = np.array([255, 0, 255], dtype=np.uint8) + img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) + elif cmap == "color": + img_ = np.zeros((img.shape[0], img.shape[1], 3), dtype=np.uint8) + img_[..., 0] = (img[..., 0] * 255).astype(np.uint8) + img_[..., 1] = (img[..., 1] * 255).astype(np.uint8) + img_ = cv2.cvtColor(img_, cv2.COLOR_RGB2BGR) + img = img_ + return img + + def save_uv_image( + self, + filename, + img, + data_format=DEFAULT_UV_KWARGS["data_format"], + data_range=DEFAULT_UV_KWARGS["data_range"], + cmap=DEFAULT_UV_KWARGS["cmap"], + ) -> str: + save_path = self.get_save_path(filename) + img = self.get_uv_image_(img, data_format, data_range, cmap) + cv2.imwrite(save_path, img) + return save_path + + def get_grayscale_image_(self, img, data_range, cmap): + img = self.convert_data(img) + img = np.nan_to_num(img) + if data_range is None: + img = (img - img.min()) / (img.max() - img.min()) + else: + img = img.clip(data_range[0], data_range[1]) + img = (img - data_range[0]) / (data_range[1] - data_range[0]) + assert cmap in [None, "jet", "magma", "spectral"] + if cmap == None: + img = (img * 255.0).astype(np.uint8) + img = np.repeat(img[..., None], 3, axis=2) + elif cmap == "jet": + img = (img * 255.0).astype(np.uint8) + img = cv2.applyColorMap(img, cv2.COLORMAP_JET) + elif cmap == "magma": + img = 1.0 - img + base = cm.get_cmap("magma") + num_bins = 256 + colormap = LinearSegmentedColormap.from_list( + f"{base.name}{num_bins}", base(np.linspace(0, 1, num_bins)), num_bins + )(np.linspace(0, 1, num_bins))[:, :3] + a = np.floor(img * 255.0) + b = (a + 1).clip(max=255.0) + f = img * 255.0 - a + a = a.astype(np.uint16).clip(0, 255) + b = b.astype(np.uint16).clip(0, 255) + img = colormap[a] + (colormap[b] - colormap[a]) * f[..., None] + img = (img * 255.0).astype(np.uint8) + elif cmap == "spectral": + colormap = plt.get_cmap("Spectral") + + def blend_rgba(image): + image = image[..., :3] * image[..., -1:] + ( + 1.0 - image[..., -1:] + ) # blend A to RGB + return image + + img = colormap(img) + img = blend_rgba(img) + img = (img * 255).astype(np.uint8) + img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) + return img + + def _save_grayscale_image( + self, + filename, + img, + data_range, + cmap, + name: Optional[str] = None, + step: Optional[int] = None, + ): + img = self.get_grayscale_image_(img, data_range, cmap) + cv2.imwrite(filename, img) + if name and self._wandb_logger: + wandb.log( + { + name: wandb.Image(self.get_save_path(filename)), + "trainer/global_step": step, + } + ) + + def save_grayscale_image( + self, + filename, + img, + data_range=DEFAULT_GRAYSCALE_KWARGS["data_range"], + cmap=DEFAULT_GRAYSCALE_KWARGS["cmap"], + name: Optional[str] = None, + step: Optional[int] = None, + ) -> str: + save_path = self.get_save_path(filename) + self._save_grayscale_image(save_path, img, data_range, cmap, name, step) + return save_path + + def get_image_grid_(self, imgs, align): + if isinstance(imgs[0], list): + return np.concatenate( + [self.get_image_grid_(row, align) for row in imgs], axis=0 + ) + cols = [] + for col in imgs: + assert col["type"] in ["rgb", "uv", "grayscale"] + if col["type"] == "rgb": + rgb_kwargs = self.DEFAULT_RGB_KWARGS.copy() + rgb_kwargs.update(col["kwargs"]) + cols.append(self.get_rgb_image_(col["img"], **rgb_kwargs)) + elif col["type"] == "uv": + uv_kwargs = self.DEFAULT_UV_KWARGS.copy() + uv_kwargs.update(col["kwargs"]) + cols.append(self.get_uv_image_(col["img"], **uv_kwargs)) + elif col["type"] == "grayscale": + grayscale_kwargs = self.DEFAULT_GRAYSCALE_KWARGS.copy() + grayscale_kwargs.update(col["kwargs"]) + cols.append(self.get_grayscale_image_(col["img"], **grayscale_kwargs)) + + if align == "max": + h = max([col.shape[0] for col in cols]) + w = max([col.shape[1] for col in cols]) + elif align == "min": + h = min([col.shape[0] for col in cols]) + w = min([col.shape[1] for col in cols]) + elif isinstance(align, int): + h = align + w = align + elif ( + isinstance(align, tuple) + and isinstance(align[0], int) + and isinstance(align[1], int) + ): + h, w = align + else: + raise ValueError( + f"Unsupported image grid align: {align}, should be min, max, int or (int, int)" + ) + + for i in range(len(cols)): + if cols[i].shape[0] != h or cols[i].shape[1] != w: + cols[i] = cv2.resize(cols[i], (w, h), interpolation=cv2.INTER_LINEAR) + return np.concatenate(cols, axis=1) + + def save_image_grid( + self, + filename, + imgs, + align=DEFAULT_GRID_KWARGS["align"], + name: Optional[str] = None, + step: Optional[int] = None, + texts: Optional[List[float]] = None, + ): + save_path = self.get_save_path(filename) + img = self.get_image_grid_(imgs, align=align) + + if texts is not None: + img = Image.fromarray(img) + draw = ImageDraw.Draw(img) + black, white = (0, 0, 0), (255, 255, 255) + for i, text in enumerate(texts): + draw.text((2, (img.size[1] // len(texts)) * i + 1), f"{text}", white) + draw.text((0, (img.size[1] // len(texts)) * i + 1), f"{text}", white) + draw.text((2, (img.size[1] // len(texts)) * i - 1), f"{text}", white) + draw.text((0, (img.size[1] // len(texts)) * i - 1), f"{text}", white) + draw.text((1, (img.size[1] // len(texts)) * i), f"{text}", black) + img = np.asarray(img) + + cv2.imwrite(save_path, img) + if name and self._wandb_logger: + wandb.log({name: wandb.Image(save_path), "trainer/global_step": step}) + return save_path + + def save_image(self, filename, img) -> str: + save_path = self.get_save_path(filename) + img = self.convert_data(img) + assert img.dtype == np.uint8 or img.dtype == np.uint16 + if img.ndim == 3 and img.shape[-1] == 3: + img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) + elif img.ndim == 3 and img.shape[-1] == 4: + img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGRA) + cv2.imwrite(save_path, img) + return save_path + + def save_cubemap(self, filename, img, data_range=(0, 1), rgba=False) -> str: + save_path = self.get_save_path(filename) + img = self.convert_data(img) + assert img.ndim == 4 and img.shape[0] == 6 and img.shape[1] == img.shape[2] + + imgs_full = [] + for start in range(0, img.shape[-1], 3): + img_ = img[..., start : start + 3] + img_ = np.stack( + [ + self.get_rgb_image_(img_[i], "HWC", data_range, rgba=rgba) + for i in range(img_.shape[0]) + ], + axis=0, + ) + size = img_.shape[1] + placeholder = np.zeros((size, size, 3), dtype=np.float32) + img_full = np.concatenate( + [ + np.concatenate( + [placeholder, img_[2], placeholder, placeholder], axis=1 + ), + np.concatenate([img_[1], img_[4], img_[0], img_[5]], axis=1), + np.concatenate( + [placeholder, img_[3], placeholder, placeholder], axis=1 + ), + ], + axis=0, + ) + imgs_full.append(img_full) + + imgs_full = np.concatenate(imgs_full, axis=1) + cv2.imwrite(save_path, imgs_full) + return save_path + + def save_data(self, filename, data) -> str: + data = self.convert_data(data) + if isinstance(data, dict): + if not filename.endswith(".npz"): + filename += ".npz" + save_path = self.get_save_path(filename) + np.savez(save_path, **data) + else: + if not filename.endswith(".npy"): + filename += ".npy" + save_path = self.get_save_path(filename) + np.save(save_path, data) + return save_path + + def save_state_dict(self, filename, data) -> str: + save_path = self.get_save_path(filename) + torch.save(data, save_path) + return save_path + + def save_img_sequence( + self, + filename, + img_dir, + matcher, + save_format="mp4", + fps=30, + name: Optional[str] = None, + step: Optional[int] = None, + delete_images = False, + ) -> str: + assert save_format in ["gif", "mp4"] + if not filename.endswith(save_format): + filename += f".{save_format}" + save_path = self.get_save_path(filename) + matcher = re.compile(matcher) + img_dir = os.path.join(self.get_save_dir(), img_dir) + imgs = [] + for f in os.listdir(img_dir): + if matcher.search(f): + imgs.append(f) + imgs = sorted(imgs, key=lambda f: int(matcher.search(f).groups()[0])) + imgs = [cv2.imread(os.path.join(img_dir, f)) for f in imgs] + + if save_format == "gif": + imgs = [cv2.cvtColor(i, cv2.COLOR_BGR2RGB) for i in imgs] + imageio.mimsave(save_path, imgs, fps=fps, palettesize=256) + elif save_format == "mp4": + imgs = [cv2.cvtColor(i, cv2.COLOR_BGR2RGB) for i in imgs] + imageio.mimsave(save_path, imgs, fps=fps) + if name and self._wandb_logger: + wandb.log( + { + name: wandb.Video(save_path, format="mp4"), + "trainer/global_step": step, + } + ) + if delete_images: + shutil.rmtree(img_dir) + return save_path + + def save_mesh(self, filename, v_pos, t_pos_idx, v_tex=None, t_tex_idx=None) -> str: + save_path = self.get_save_path(filename) + v_pos = self.convert_data(v_pos) + t_pos_idx = self.convert_data(t_pos_idx) + mesh = trimesh.Trimesh(vertices=v_pos, faces=t_pos_idx) + mesh.export(save_path) + return save_path + + def save_obj( + self, + filename: str, + mesh: Mesh, + save_mat: bool = False, + save_normal: bool = False, + save_uv: bool = False, + save_vertex_color: bool = False, + map_Kd: Optional[Float[Tensor, "H W 3"]] = None, + map_Ks: Optional[Float[Tensor, "H W 3"]] = None, + map_Bump: Optional[Float[Tensor, "H W 3"]] = None, + map_Pm: Optional[Float[Tensor, "H W 1"]] = None, + map_Pr: Optional[Float[Tensor, "H W 1"]] = None, + map_format: str = "jpg", + ) -> List[str]: + save_paths: List[str] = [] + if not filename.endswith(".obj"): + filename += ".obj" + v_pos, t_pos_idx = self.convert_data(mesh.v_pos), self.convert_data( + mesh.t_pos_idx + ) + v_nrm, v_tex, t_tex_idx, v_rgb = None, None, None, None + if save_normal: + v_nrm = self.convert_data(mesh.v_nrm) + if save_uv: + v_tex, t_tex_idx = self.convert_data(mesh.v_tex), self.convert_data( + mesh.t_tex_idx + ) + if save_vertex_color: + v_rgb = self.convert_data(mesh.v_rgb) + matname, mtllib = None, None + if save_mat: + matname = "default" + mtl_filename = filename.replace(".obj", ".mtl") + mtllib = os.path.basename(mtl_filename) + mtl_save_paths = self._save_mtl( + mtl_filename, + matname, + map_Kd=self.convert_data(map_Kd), + map_Ks=self.convert_data(map_Ks), + map_Bump=self.convert_data(map_Bump), + map_Pm=self.convert_data(map_Pm), + map_Pr=self.convert_data(map_Pr), + map_format=map_format, + ) + save_paths += mtl_save_paths + obj_save_path = self._save_obj( + filename, + v_pos, + t_pos_idx, + v_nrm=v_nrm, + v_tex=v_tex, + t_tex_idx=t_tex_idx, + v_rgb=v_rgb, + matname=matname, + mtllib=mtllib, + ) + save_paths.append(obj_save_path) + return save_paths + + def _save_obj( + self, + filename, + v_pos, + t_pos_idx, + v_nrm=None, + v_tex=None, + t_tex_idx=None, + v_rgb=None, + matname=None, + mtllib=None, + ) -> str: + obj_str = "" + if matname is not None: + obj_str += f"mtllib {mtllib}\n" + obj_str += f"g object\n" + obj_str += f"usemtl {matname}\n" + for i in range(len(v_pos)): + obj_str += f"v {v_pos[i][0]} {v_pos[i][1]} {v_pos[i][2]}" + if v_rgb is not None: + obj_str += f" {v_rgb[i][0]} {v_rgb[i][1]} {v_rgb[i][2]}" + obj_str += "\n" + if v_nrm is not None: + for v in v_nrm: + obj_str += f"vn {v[0]} {v[1]} {v[2]}\n" + if v_tex is not None: + for v in v_tex: + obj_str += f"vt {v[0]} {1.0 - v[1]}\n" + + for i in range(len(t_pos_idx)): + obj_str += "f" + for j in range(3): + obj_str += f" {t_pos_idx[i][j] + 1}/" + if v_tex is not None: + obj_str += f"{t_tex_idx[i][j] + 1}" + obj_str += "/" + if v_nrm is not None: + obj_str += f"{t_pos_idx[i][j] + 1}" + obj_str += "\n" + + save_path = self.get_save_path(filename) + with open(save_path, "w") as f: + f.write(obj_str) + return save_path + + def _save_mtl( + self, + filename, + matname, + Ka=(0.0, 0.0, 0.0), + Kd=(1.0, 1.0, 1.0), + Ks=(0.0, 0.0, 0.0), + map_Kd=None, + map_Ks=None, + map_Bump=None, + map_Pm=None, + map_Pr=None, + map_format="jpg", + step: Optional[int] = None, + ) -> List[str]: + mtl_save_path = self.get_save_path(filename) + save_paths = [mtl_save_path] + mtl_str = f"newmtl {matname}\n" + mtl_str += f"Ka {Ka[0]} {Ka[1]} {Ka[2]}\n" + if map_Kd is not None: + map_Kd_save_path = os.path.join( + os.path.dirname(mtl_save_path), f"texture_kd.{map_format}" + ) + mtl_str += f"map_Kd texture_kd.{map_format}\n" + self._save_rgb_image( + map_Kd_save_path, + map_Kd, + data_format="HWC", + data_range=(0, 1), + name=f"{matname}_Kd", + step=step, + ) + save_paths.append(map_Kd_save_path) + else: + mtl_str += f"Kd {Kd[0]} {Kd[1]} {Kd[2]}\n" + if map_Ks is not None: + map_Ks_save_path = os.path.join( + os.path.dirname(mtl_save_path), f"texture_ks.{map_format}" + ) + mtl_str += f"map_Ks texture_ks.{map_format}\n" + self._save_rgb_image( + map_Ks_save_path, + map_Ks, + data_format="HWC", + data_range=(0, 1), + name=f"{matname}_Ks", + step=step, + ) + save_paths.append(map_Ks_save_path) + else: + mtl_str += f"Ks {Ks[0]} {Ks[1]} {Ks[2]}\n" + if map_Bump is not None: + map_Bump_save_path = os.path.join( + os.path.dirname(mtl_save_path), f"texture_nrm.{map_format}" + ) + mtl_str += f"map_Bump texture_nrm.{map_format}\n" + self._save_rgb_image( + map_Bump_save_path, + map_Bump, + data_format="HWC", + data_range=(0, 1), + name=f"{matname}_Bump", + step=step, + ) + save_paths.append(map_Bump_save_path) + if map_Pm is not None: + map_Pm_save_path = os.path.join( + os.path.dirname(mtl_save_path), f"texture_metallic.{map_format}" + ) + mtl_str += f"map_Pm texture_metallic.{map_format}\n" + self._save_grayscale_image( + map_Pm_save_path, + map_Pm, + data_range=(0, 1), + cmap=None, + name=f"{matname}_refl", + step=step, + ) + save_paths.append(map_Pm_save_path) + if map_Pr is not None: + map_Pr_save_path = os.path.join( + os.path.dirname(mtl_save_path), f"texture_roughness.{map_format}" + ) + mtl_str += f"map_Pr texture_roughness.{map_format}\n" + self._save_grayscale_image( + map_Pr_save_path, + map_Pr, + data_range=(0, 1), + cmap=None, + name=f"{matname}_Ns", + step=step, + ) + save_paths.append(map_Pr_save_path) + with open(self.get_save_path(filename), "w") as f: + f.write(mtl_str) + return save_paths + + def save_file(self, filename, src_path) -> str: + save_path = self.get_save_path(filename) + shutil.copyfile(src_path, save_path) + return save_path + + def save_json(self, filename, payload) -> str: + save_path = self.get_save_path(filename) + with open(save_path, "w") as f: + f.write(json.dumps(payload)) + return save_path diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/typing.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/typing.py new file mode 100644 index 0000000000000000000000000000000000000000..dee9f967c21f94db1ad939d7dead156d86748752 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/threestudio/utils/typing.py @@ -0,0 +1,40 @@ +""" +This module contains type annotations for the project, using +1. Python type hints (https://docs.python.org/3/library/typing.html) for Python objects +2. jaxtyping (https://github.com/google/jaxtyping/blob/main/API.md) for PyTorch tensors + +Two types of typing checking can be used: +1. Static type checking with mypy (install with pip and enabled as the default linter in VSCode) +2. Runtime type checking with typeguard (install with pip and triggered at runtime, mainly for tensor dtype and shape checking) +""" + +# Basic types +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Literal, + NamedTuple, + NewType, + Optional, + Sized, + Tuple, + Type, + TypeVar, + Union, +) + +# Tensor dtype +# for jaxtyping usage, see https://github.com/google/jaxtyping/blob/main/API.md +from jaxtyping import Bool, Complex, Float, Inexact, Int, Integer, Num, Shaped, UInt + +# Config type +from omegaconf import DictConfig + +# PyTorch Tensor type +from torch import Tensor + +# Runtime type checking decorator +from typeguard import typechecked as typechecker diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/tinycudann-1.7.post70240121-cp310-cp310-linux_x86_64.whl b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/tinycudann-1.7.post70240121-cp310-cp310-linux_x86_64.whl new file mode 100644 index 0000000000000000000000000000000000000000..ab7fa1610f54bd5888308acbd65cd1dfac1dfa30 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/tinycudann-1.7.post70240121-cp310-cp310-linux_x86_64.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2db01d6778b5c89c0c157a487ff15dfcfef894d3afc1566872f7f9b0b0f65b +size 18271592 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/README.md b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a362ff5b56bdcbccefe09414113e156774361040 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/README.md @@ -0,0 +1,74 @@ +# GeoWizard: Unleashing the Diffusion Priors for 3D Geometry Estimation from a Single Image +### [Project Page](https://fuxiao0719.github.io/projects/geowizard/) | [Paper](https://arxiv.org/abs/2403.12013) | [Hugging Face](https://huggingface.co/spaces/lemonaddie/geowizard) +
+ +> GeoWizard: Unleashing the Diffusion Priors for 3D Geometry Estimation from a Single Image + +> [Xiao Fu*](http://fuxiao0719.github.io/), [Wei Yin*](https://yvanyin.net/), [Mu Hu*](https://github.com/JUGGHM), [Kaixuan Wang](https://wang-kx.github.io/), [Yuexin Ma](https://yuexinma.me/), [Ping Tan](https://ece.hkust.edu.hk/pingtan), [Shaojie Shen](https://uav.hkust.edu.hk/group/), [Dahua Lin†](http://dahua.site/) , [Xiaoxiao Long†](https://www.xxlong.site/) +> * Equal contribution; † Corresponding authors +> Arxiv Preprint, 2024 + +![demo_vid](assets/teaser.png) + +## 🛠️ Setup + +We test our codes under the following environment: `Ubuntu 22.04, Python 3.9.18, CUDA 11.8`. +1. Clone this repository. +```bash +git clone git@github.com:fuxiao0719/GeoWizard.git +cd GeoWizard +``` +2. Install packages +```bash +conda create -n geowizard python=3.9 +conda activate geowizard +pip install -r requirements.txt +cd geowizard +``` + +## 🤖 Usage + +### Run inference for depth & normal + +Place your images in a directory `input/example` (for example, where we have prepared several cases), and run the following inference. The depth and normal outputs will be stored in `output/example`. + +```bash +python run_infer.py \ + --input_dir ${input path} \ + --output_dir ${output path} \ + --ensemble_size ${ensemble size} \ + --denoise_steps ${denoising steps} \ + --domain ${data type} +# e.g. +python run_infer.py \ + --input_dir input/example \ + --output_dir output \ + --ensemble_size 3 \ + --denoise_steps 10 \ + --domain "indoor" +``` + +Inference settings: `--domain`: Data type. Options: "indoor", "outdoor", and "object". Note that "object" is best for background-free objects, like that in objaverse. We find that "indoor" will suit in most scenarios. Default: "indoor". `--ensemble_size` and `--denoise_steps`: trade-off arguments for speed and performance, more ensembles and denoising steps to get higher accuracy. Default: 3 and 10. + +## 📝 TODO List +- [ ] Add inference code for 3D reconstruction. +- [ ] Add training codes. +- [ ] Test on more different local environments. + +## 📚 Related Work +We also encourage readers to follow these concurrent exciting works. +- [Marigold](https://arxiv.org/abs/2312.02145): a finetuned diffusion model for estimating monocular depth. +- [Wonder3D](https://arxiv.org/abs/2310.15008): generate multi-view normal maps and color images and reconstruct high-fidelity textured mesh. +- [HyperHuman](https://arxiv.org/abs/2310.08579): a latent structural diffusion and a structure-guided refiner for high-resolution human generation. +- [GenPercept](https://arxiv.org/abs/2403.06090): a finetuned UNet for a lot of downstream image understanding tasks. + +## 🔗 Citation + +```bibtex +@article{fu2024geowizard, + title={GeoWizard: Unleashing the Diffusion Priors for 3D Geometry Estimation from a Single Image}, + author={Fu, Xiao and Yin, Wei and Hu, Mu and Wang, Kaixuan and Ma, Yuexin and Tan, Ping and Shen, Shaojie and Lin, Dahua and Long, Xiaoxiao}, + journal={arXiv preprint arXiv:2403.12013}, + year={2024} +} +``` \ No newline at end of file diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/attention.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/attention.py new file mode 100644 index 0000000000000000000000000000000000000000..ccbb7eedeb0956e7c7febf2147cf1c3e859ed6fe --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/attention.py @@ -0,0 +1,777 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Some modifications are reimplemented in public environments by Xiao Fu and Mu Hu + + +from typing import Any, Dict, Optional + +import torch +import torch.nn.functional as F +from torch import nn +import xformers + +from diffusers.utils import USE_PEFT_BACKEND +from diffusers.utils.torch_utils import maybe_allow_in_graph +from diffusers.models.activations import GEGLU, GELU, ApproximateGELU +from diffusers.models.attention_processor import Attention +from diffusers.models.embeddings import SinusoidalPositionalEmbedding +from diffusers.models.lora import LoRACompatibleLinear +from diffusers.models.normalization import AdaLayerNorm, AdaLayerNormContinuous, AdaLayerNormZero, RMSNorm + + +def _chunked_feed_forward( + ff: nn.Module, hidden_states: torch.Tensor, chunk_dim: int, chunk_size: int, lora_scale: Optional[float] = None +): + # "feed_forward_chunk_size" can be used to save memory + if hidden_states.shape[chunk_dim] % chunk_size != 0: + raise ValueError( + f"`hidden_states` dimension to be chunked: {hidden_states.shape[chunk_dim]} has to be divisible by chunk size: {chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`." + ) + + num_chunks = hidden_states.shape[chunk_dim] // chunk_size + if lora_scale is None: + ff_output = torch.cat( + [ff(hid_slice) for hid_slice in hidden_states.chunk(num_chunks, dim=chunk_dim)], + dim=chunk_dim, + ) + else: + # TOOD(Patrick): LoRA scale can be removed once PEFT refactor is complete + ff_output = torch.cat( + [ff(hid_slice, scale=lora_scale) for hid_slice in hidden_states.chunk(num_chunks, dim=chunk_dim)], + dim=chunk_dim, + ) + + return ff_output + + +@maybe_allow_in_graph +class GatedSelfAttentionDense(nn.Module): + r""" + A gated self-attention dense layer that combines visual features and object features. + + Parameters: + query_dim (`int`): The number of channels in the query. + context_dim (`int`): The number of channels in the context. + n_heads (`int`): The number of heads to use for attention. + d_head (`int`): The number of channels in each head. + """ + + def __init__(self, query_dim: int, context_dim: int, n_heads: int, d_head: int): + super().__init__() + + # we need a linear projection since we need cat visual feature and obj feature + self.linear = nn.Linear(context_dim, query_dim) + + self.attn = Attention(query_dim=query_dim, heads=n_heads, dim_head=d_head) + self.ff = FeedForward(query_dim, activation_fn="geglu") + + self.norm1 = nn.LayerNorm(query_dim) + self.norm2 = nn.LayerNorm(query_dim) + + self.register_parameter("alpha_attn", nn.Parameter(torch.tensor(0.0))) + self.register_parameter("alpha_dense", nn.Parameter(torch.tensor(0.0))) + + self.enabled = True + + def forward(self, x: torch.Tensor, objs: torch.Tensor) -> torch.Tensor: + if not self.enabled: + return x + + n_visual = x.shape[1] + objs = self.linear(objs) + + x = x + self.alpha_attn.tanh() * self.attn(self.norm1(torch.cat([x, objs], dim=1)))[:, :n_visual, :] + x = x + self.alpha_dense.tanh() * self.ff(self.norm2(x)) + + return x + + +@maybe_allow_in_graph +class BasicTransformerBlock(nn.Module): + r""" + A basic Transformer block. + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + num_embeds_ada_norm (: + obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`. + attention_bias (: + obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter. + only_cross_attention (`bool`, *optional*): + Whether to use only cross-attention layers. In this case two cross attention layers are used. + double_self_attention (`bool`, *optional*): + Whether to use two self-attention layers. In this case no cross attention layers are used. + upcast_attention (`bool`, *optional*): + Whether to upcast the attention computation to float32. This is useful for mixed precision training. + norm_elementwise_affine (`bool`, *optional*, defaults to `True`): + Whether to use learnable elementwise affine parameters for normalization. + norm_type (`str`, *optional*, defaults to `"layer_norm"`): + The normalization layer to use. Can be `"layer_norm"`, `"ada_norm"` or `"ada_norm_zero"`. + final_dropout (`bool` *optional*, defaults to False): + Whether to apply a final dropout after the last feed-forward layer. + attention_type (`str`, *optional*, defaults to `"default"`): + The type of attention to use. Can be `"default"` or `"gated"` or `"gated-text-image"`. + positional_embeddings (`str`, *optional*, defaults to `None`): + The type of positional embeddings to apply to. + num_positional_embeddings (`int`, *optional*, defaults to `None`): + The maximum number of positional embeddings to apply. + """ + + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + dropout=0.0, + cross_attention_dim: Optional[int] = None, + activation_fn: str = "geglu", + num_embeds_ada_norm: Optional[int] = None, + attention_bias: bool = False, + only_cross_attention: bool = False, + double_self_attention: bool = False, + upcast_attention: bool = False, + norm_elementwise_affine: bool = True, + norm_type: str = "layer_norm", # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single' + norm_eps: float = 1e-5, + final_dropout: bool = False, + attention_type: str = "default", + positional_embeddings: Optional[str] = None, + num_positional_embeddings: Optional[int] = None, + ada_norm_continous_conditioning_embedding_dim: Optional[int] = None, + ada_norm_bias: Optional[int] = None, + ff_inner_dim: Optional[int] = None, + ff_bias: bool = True, + attention_out_bias: bool = True, + ): + super().__init__() + self.only_cross_attention = only_cross_attention + + self.use_ada_layer_norm_zero = (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero" + self.use_ada_layer_norm = (num_embeds_ada_norm is not None) and norm_type == "ada_norm" + self.use_ada_layer_norm_single = norm_type == "ada_norm_single" + self.use_layer_norm = norm_type == "layer_norm" + self.use_ada_layer_norm_continuous = norm_type == "ada_norm_continuous" + + if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None: + raise ValueError( + f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to" + f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}." + ) + + if positional_embeddings and (num_positional_embeddings is None): + raise ValueError( + "If `positional_embedding` type is defined, `num_positition_embeddings` must also be defined." + ) + + if positional_embeddings == "sinusoidal": + self.pos_embed = SinusoidalPositionalEmbedding(dim, max_seq_length=num_positional_embeddings) + else: + self.pos_embed = None + + # Define 3 blocks. Each block has its own normalization layer. + # 1. Self-Attn + if self.use_ada_layer_norm: + self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm) + elif self.use_ada_layer_norm_zero: + self.norm1 = AdaLayerNormZero(dim, num_embeds_ada_norm) + elif self.use_ada_layer_norm_continuous: + self.norm1 = AdaLayerNormContinuous( + dim, + ada_norm_continous_conditioning_embedding_dim, + norm_elementwise_affine, + norm_eps, + ada_norm_bias, + "rms_norm", + ) + else: + self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps) + + + self.attn1 = CustomJointAttention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=cross_attention_dim if only_cross_attention else None, + upcast_attention=upcast_attention, + out_bias=attention_out_bias + ) + + # 2. Cross-Attn + if cross_attention_dim is not None or double_self_attention: + # We currently only use AdaLayerNormZero for self attention where there will only be one attention block. + # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during + # the second cross attention block. + + if self.use_ada_layer_norm: + self.norm2 = AdaLayerNorm(dim, num_embeds_ada_norm) + elif self.use_ada_layer_norm_continuous: + self.norm2 = AdaLayerNormContinuous( + dim, + ada_norm_continous_conditioning_embedding_dim, + norm_elementwise_affine, + norm_eps, + ada_norm_bias, + "rms_norm", + ) + else: + self.norm2 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine) + + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim if not double_self_attention else None, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + upcast_attention=upcast_attention, + out_bias=attention_out_bias, + ) # is self-attn if encoder_hidden_states is none + else: + self.norm2 = None + self.attn2 = None + + # 3. Feed-forward + if self.use_ada_layer_norm_continuous: + self.norm3 = AdaLayerNormContinuous( + dim, + ada_norm_continous_conditioning_embedding_dim, + norm_elementwise_affine, + norm_eps, + ada_norm_bias, + "layer_norm", + ) + elif not self.use_ada_layer_norm_single: + self.norm3 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine) + + self.ff = FeedForward( + dim, + dropout=dropout, + activation_fn=activation_fn, + final_dropout=final_dropout, + inner_dim=ff_inner_dim, + bias=ff_bias, + ) + + # 4. Fuser + if attention_type == "gated" or attention_type == "gated-text-image": + self.fuser = GatedSelfAttentionDense(dim, cross_attention_dim, num_attention_heads, attention_head_dim) + + # 5. Scale-shift for PixArt-Alpha. + if self.use_ada_layer_norm_single: + self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5) + + # let chunk size default to None + self._chunk_size = None + self._chunk_dim = 0 + + def set_chunk_feed_forward(self, chunk_size: Optional[int], dim: int = 0): + # Sets chunk feed-forward + self._chunk_size = chunk_size + self._chunk_dim = dim + + def forward( + self, + hidden_states: torch.FloatTensor, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + timestep: Optional[torch.LongTensor] = None, + cross_attention_kwargs: Dict[str, Any] = None, + class_labels: Optional[torch.LongTensor] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + ) -> torch.FloatTensor: + # Notice that normalization is always applied before the real computation in the following blocks. + + # 0. Self-Attention + batch_size = hidden_states.shape[0] + + if self.use_ada_layer_norm: + norm_hidden_states = self.norm1(hidden_states, timestep) + elif self.use_ada_layer_norm_zero: + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1( + hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype + ) + elif self.use_layer_norm: + norm_hidden_states = self.norm1(hidden_states) + elif self.use_ada_layer_norm_continuous: + norm_hidden_states = self.norm1(hidden_states, added_cond_kwargs["pooled_text_emb"]) + elif self.use_ada_layer_norm_single: + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + self.scale_shift_table[None] + timestep.reshape(batch_size, 6, -1) + ).chunk(6, dim=1) + norm_hidden_states = self.norm1(hidden_states) + norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa + norm_hidden_states = norm_hidden_states.squeeze(1) + else: + raise ValueError("Incorrect norm used") + + if self.pos_embed is not None: + norm_hidden_states = self.pos_embed(norm_hidden_states) + + # 1. Retrieve lora scale. + lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + + # 2. Prepare GLIGEN inputs + cross_attention_kwargs = cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {} + gligen_kwargs = cross_attention_kwargs.pop("gligen", None) + + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + if self.use_ada_layer_norm_zero: + attn_output = gate_msa.unsqueeze(1) * attn_output + elif self.use_ada_layer_norm_single: + attn_output = gate_msa * attn_output + + hidden_states = attn_output + hidden_states + if hidden_states.ndim == 4: + hidden_states = hidden_states.squeeze(1) + + # 2.5 GLIGEN Control + if gligen_kwargs is not None: + hidden_states = self.fuser(hidden_states, gligen_kwargs["objs"]) + + # 3. Cross-Attention + if self.attn2 is not None: + if self.use_ada_layer_norm: + norm_hidden_states = self.norm2(hidden_states, timestep) + elif self.use_ada_layer_norm_zero or self.use_layer_norm: + norm_hidden_states = self.norm2(hidden_states) + elif self.use_ada_layer_norm_single: + # For PixArt norm2 isn't applied here: + # https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L70C1-L76C103 + norm_hidden_states = hidden_states + elif self.use_ada_layer_norm_continuous: + norm_hidden_states = self.norm2(hidden_states, added_cond_kwargs["pooled_text_emb"]) + else: + raise ValueError("Incorrect norm") + + if self.pos_embed is not None and self.use_ada_layer_norm_single is False: + norm_hidden_states = self.pos_embed(norm_hidden_states) + + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + **cross_attention_kwargs, + ) + hidden_states = attn_output + hidden_states + + # 4. Feed-forward + if self.use_ada_layer_norm_continuous: + norm_hidden_states = self.norm3(hidden_states, added_cond_kwargs["pooled_text_emb"]) + elif not self.use_ada_layer_norm_single: + norm_hidden_states = self.norm3(hidden_states) + + if self.use_ada_layer_norm_zero: + norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + + if self.use_ada_layer_norm_single: + norm_hidden_states = self.norm2(hidden_states) + norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp + + if self._chunk_size is not None: + # "feed_forward_chunk_size" can be used to save memory + ff_output = _chunked_feed_forward( + self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size, lora_scale=lora_scale + ) + else: + ff_output = self.ff(norm_hidden_states, scale=lora_scale) + + if self.use_ada_layer_norm_zero: + ff_output = gate_mlp.unsqueeze(1) * ff_output + elif self.use_ada_layer_norm_single: + ff_output = gate_mlp * ff_output + + hidden_states = ff_output + hidden_states + if hidden_states.ndim == 4: + hidden_states = hidden_states.squeeze(1) + + return hidden_states + + +class CustomJointAttention(Attention): + def set_use_memory_efficient_attention_xformers( + self, use_memory_efficient_attention_xformers: bool, *args, **kwargs + ): + processor = XFormersJointAttnProcessor() + self.set_processor(processor) + # print("using xformers attention processor") + + +class XFormersJointAttnProcessor: + r""" + Default processor for performing attention-related computations. + """ + + def __call__( + self, + attn: Attention, + hidden_states, + encoder_hidden_states=None, + attention_mask=None, + temb=None, + num_tasks=2 + ): + + residual = hidden_states + + if attn.spatial_norm is not None: + hidden_states = attn.spatial_norm(hidden_states, temb) + + input_ndim = hidden_states.ndim + + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + # from yuancheng; here attention_mask is None + if attention_mask is not None: + # expand our mask's singleton query_tokens dimension: + # [batch*heads, 1, key_tokens] -> + # [batch*heads, query_tokens, key_tokens] + # so that it can be added as a bias onto the attention scores that xformers computes: + # [batch*heads, query_tokens, key_tokens] + # we do this explicitly because xformers doesn't broadcast the singleton dimension for us. + _, query_tokens, _ = hidden_states.shape + attention_mask = attention_mask.expand(-1, query_tokens, -1) + + if attn.group_norm is not None: + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + assert num_tasks == 2 # only support two tasks now + + key_0, key_1 = torch.chunk(key, dim=0, chunks=2) # keys shape (b t) d c + value_0, value_1 = torch.chunk(value, dim=0, chunks=2) + + # key = torch.cat([key_1, key_0], dim=0) + # value = torch.cat([value_1, value_0], dim=0) + + key = torch.cat([key_0, key_1], dim=1) # (b t) 2d c + value = torch.cat([value_0, value_1], dim=1) # (b t) 2d c + key = torch.cat([key]*2, dim=0) # (2 b t) 2d c + value = torch.cat([value]*2, dim=0) # (2 b t) 2d c + + query = attn.head_to_batch_dim(query).contiguous() + key = attn.head_to_batch_dim(key).contiguous() + value = attn.head_to_batch_dim(value).contiguous() + + hidden_states = xformers.ops.memory_efficient_attention(query, key, value, attn_bias=attention_mask) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + if input_ndim == 4: + hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) + + if attn.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / attn.rescale_output_factor + + return hidden_states + + +@maybe_allow_in_graph +class TemporalBasicTransformerBlock(nn.Module): + r""" + A basic Transformer block for video like data. + + Parameters: + dim (`int`): The number of channels in the input and output. + time_mix_inner_dim (`int`): The number of channels for temporal attention. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + """ + + def __init__( + self, + dim: int, + time_mix_inner_dim: int, + num_attention_heads: int, + attention_head_dim: int, + cross_attention_dim: Optional[int] = None, + ): + super().__init__() + self.is_res = dim == time_mix_inner_dim + + self.norm_in = nn.LayerNorm(dim) + + # Define 3 blocks. Each block has its own normalization layer. + # 1. Self-Attn + self.norm_in = nn.LayerNorm(dim) + self.ff_in = FeedForward( + dim, + dim_out=time_mix_inner_dim, + activation_fn="geglu", + ) + + self.norm1 = nn.LayerNorm(time_mix_inner_dim) + self.attn1 = Attention( + query_dim=time_mix_inner_dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + cross_attention_dim=None, + ) + + # 2. Cross-Attn + if cross_attention_dim is not None: + # We currently only use AdaLayerNormZero for self attention where there will only be one attention block. + # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during + # the second cross attention block. + self.norm2 = nn.LayerNorm(time_mix_inner_dim) + self.attn2 = Attention( + query_dim=time_mix_inner_dim, + cross_attention_dim=cross_attention_dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + ) # is self-attn if encoder_hidden_states is none + else: + self.norm2 = None + self.attn2 = None + + # 3. Feed-forward + self.norm3 = nn.LayerNorm(time_mix_inner_dim) + self.ff = FeedForward(time_mix_inner_dim, activation_fn="geglu") + + # let chunk size default to None + self._chunk_size = None + self._chunk_dim = None + + def set_chunk_feed_forward(self, chunk_size: Optional[int], **kwargs): + # Sets chunk feed-forward + self._chunk_size = chunk_size + # chunk dim should be hardcoded to 1 to have better speed vs. memory trade-off + self._chunk_dim = 1 + + def forward( + self, + hidden_states: torch.FloatTensor, + num_frames: int, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + # Notice that normalization is always applied before the real computation in the following blocks. + # 0. Self-Attention + batch_size = hidden_states.shape[0] + + batch_frames, seq_length, channels = hidden_states.shape + batch_size = batch_frames // num_frames + + hidden_states = hidden_states[None, :].reshape(batch_size, num_frames, seq_length, channels) + hidden_states = hidden_states.permute(0, 2, 1, 3) + hidden_states = hidden_states.reshape(batch_size * seq_length, num_frames, channels) + + residual = hidden_states + hidden_states = self.norm_in(hidden_states) + + if self._chunk_size is not None: + hidden_states = _chunked_feed_forward(self.ff_in, hidden_states, self._chunk_dim, self._chunk_size) + else: + hidden_states = self.ff_in(hidden_states) + + if self.is_res: + hidden_states = hidden_states + residual + + norm_hidden_states = self.norm1(hidden_states) + attn_output = self.attn1(norm_hidden_states, encoder_hidden_states=None) + hidden_states = attn_output + hidden_states + + # 3. Cross-Attention + if self.attn2 is not None: + norm_hidden_states = self.norm2(hidden_states) + attn_output = self.attn2(norm_hidden_states, encoder_hidden_states=encoder_hidden_states) + hidden_states = attn_output + hidden_states + + # 4. Feed-forward + norm_hidden_states = self.norm3(hidden_states) + + if self._chunk_size is not None: + ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size) + else: + ff_output = self.ff(norm_hidden_states) + + if self.is_res: + hidden_states = ff_output + hidden_states + else: + hidden_states = ff_output + + hidden_states = hidden_states[None, :].reshape(batch_size, seq_length, num_frames, channels) + hidden_states = hidden_states.permute(0, 2, 1, 3) + hidden_states = hidden_states.reshape(batch_size * num_frames, seq_length, channels) + + return hidden_states + + +class SkipFFTransformerBlock(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + kv_input_dim: int, + kv_input_dim_proj_use_bias: bool, + dropout=0.0, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + attention_out_bias: bool = True, + ): + super().__init__() + if kv_input_dim != dim: + self.kv_mapper = nn.Linear(kv_input_dim, dim, kv_input_dim_proj_use_bias) + else: + self.kv_mapper = None + + self.norm1 = RMSNorm(dim, 1e-06) + + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=cross_attention_dim, + out_bias=attention_out_bias, + ) + + self.norm2 = RMSNorm(dim, 1e-06) + + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + out_bias=attention_out_bias, + ) + + def forward(self, hidden_states, encoder_hidden_states, cross_attention_kwargs): + cross_attention_kwargs = cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {} + + if self.kv_mapper is not None: + encoder_hidden_states = self.kv_mapper(F.silu(encoder_hidden_states)) + + norm_hidden_states = self.norm1(hidden_states) + + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + **cross_attention_kwargs, + ) + + hidden_states = attn_output + hidden_states + + norm_hidden_states = self.norm2(hidden_states) + + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + **cross_attention_kwargs, + ) + + hidden_states = attn_output + hidden_states + + return hidden_states + + +class FeedForward(nn.Module): + r""" + A feed-forward layer. + + Parameters: + dim (`int`): The number of channels in the input. + dim_out (`int`, *optional*): The number of channels in the output. If not given, defaults to `dim`. + mult (`int`, *optional*, defaults to 4): The multiplier to use for the hidden dimension. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + final_dropout (`bool` *optional*, defaults to False): Apply a final dropout. + bias (`bool`, defaults to True): Whether to use a bias in the linear layer. + """ + + def __init__( + self, + dim: int, + dim_out: Optional[int] = None, + mult: int = 4, + dropout: float = 0.0, + activation_fn: str = "geglu", + final_dropout: bool = False, + inner_dim=None, + bias: bool = True, + ): + super().__init__() + if inner_dim is None: + inner_dim = int(dim * mult) + dim_out = dim_out if dim_out is not None else dim + linear_cls = LoRACompatibleLinear if not USE_PEFT_BACKEND else nn.Linear + + if activation_fn == "gelu": + act_fn = GELU(dim, inner_dim, bias=bias) + if activation_fn == "gelu-approximate": + act_fn = GELU(dim, inner_dim, approximate="tanh", bias=bias) + elif activation_fn == "geglu": + act_fn = GEGLU(dim, inner_dim, bias=bias) + elif activation_fn == "geglu-approximate": + act_fn = ApproximateGELU(dim, inner_dim, bias=bias) + + self.net = nn.ModuleList([]) + # project in + self.net.append(act_fn) + # project dropout + self.net.append(nn.Dropout(dropout)) + # project out + self.net.append(linear_cls(inner_dim, dim_out, bias=bias)) + # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout + if final_dropout: + self.net.append(nn.Dropout(dropout)) + + def forward(self, hidden_states: torch.Tensor, scale: float = 1.0) -> torch.Tensor: + compatible_cls = (GEGLU,) if USE_PEFT_BACKEND else (GEGLU, LoRACompatibleLinear) + for module in self.net: + if isinstance(module, compatible_cls): + hidden_states = module(hidden_states, scale) + else: + hidden_states = module(hidden_states) + return hidden_states diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/geowizard_pipeline.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/geowizard_pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..71b3e973489187887257a0ec1524bc2338e3448f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/geowizard_pipeline.py @@ -0,0 +1,476 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +from typing import Any, Dict, Union + +import torch +from torch.utils.data import DataLoader, TensorDataset +import numpy as np +from tqdm.auto import tqdm +from PIL import Image +from diffusers import ( + DiffusionPipeline, + DDIMScheduler, + AutoencoderKL, +) +from models.unet_2d_condition import UNet2DConditionModel +from diffusers.utils import BaseOutput +from transformers import CLIPTextModel, CLIPTokenizer +from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection +import torchvision.transforms.functional as TF +from torchvision.transforms import InterpolationMode + +from utils.image_util import resize_max_res,chw2hwc,colorize_depth_maps +from utils.colormap import kitti_colormap +from utils.depth_ensemble import ensemble_depths +from utils.normal_ensemble import ensemble_normals +from utils.batch_size import find_batch_size +import cv2 + +class DepthNormalPipelineOutput(BaseOutput): + """ + Output class for Marigold monocular depth prediction pipeline. + Args: + depth_np (`np.ndarray`): + Predicted depth map, with depth values in the range of [0, 1]. + depth_colored (`PIL.Image.Image`): + Colorized depth map, with the shape of [3, H, W] and values in [0, 1]. + normal_np (`np.ndarray`): + Predicted normal map, with depth values in the range of [0, 1]. + normal_colored (`PIL.Image.Image`): + Colorized normal map, with the shape of [3, H, W] and values in [0, 1]. + uncertainty (`None` or `np.ndarray`): + Uncalibrated uncertainty(MAD, median absolute deviation) coming from ensembling. + """ + depth_np: np.ndarray + depth_colored: Image.Image + normal_np: np.ndarray + normal_colored: Image.Image + uncertainty: Union[None, np.ndarray] + +class DepthNormalEstimationPipeline(DiffusionPipeline): + # two hyper-parameters + latent_scale_factor = 0.18215 + + def __init__(self, + unet:UNet2DConditionModel, + vae:AutoencoderKL, + scheduler:DDIMScheduler, + image_encoder:CLIPVisionModelWithProjection, + feature_extractor:CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + unet=unet, + vae=vae, + scheduler=scheduler, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + ) + self.img_embed = None + + @torch.no_grad() + def __call__(self, + input_image:Image, + denoising_steps: int = 10, + ensemble_size: int = 10, + processing_res: int = 768, + match_input_res:bool =True, + batch_size:int = 0, + domain: str = "indoor", + color_map: str="Spectral", + show_progress_bar:bool = True, + ensemble_kwargs: Dict = None, + gt_depth = None, + mask = None + ) -> DepthNormalPipelineOutput: + + # inherit from thea Diffusion Pipeline + device = self.device + input_size = input_image.size + + # adjust the input resolution. + if not match_input_res: + assert ( + processing_res is not None + )," Value Error: `resize_output_back` is only valid with " + + assert processing_res >=0 + assert denoising_steps >=1 + assert ensemble_size >=1 + + # --------------- Image Processing ------------------------ + # Resize image + if processing_res >0: + input_image = resize_max_res( + input_image, max_edge_resolution=processing_res + ) + + # Convert the image to RGB, to 1. reomve the alpha channel. + input_image = input_image.convert("RGB") + image = np.array(input_image) + + # Normalize RGB Values. + rgb = np.transpose(image,(2,0,1)) + rgb_norm = rgb / 255.0 * 2.0 - 1.0 # [0, 255] -> [-1, 1] + rgb_norm = torch.from_numpy(rgb_norm).to(self.dtype) + rgb_norm = rgb_norm.to(device) + + assert rgb_norm.min() >= -1.0 and rgb_norm.max() <= 1.0 + + # ----------------- predicting depth ----------------- + duplicated_rgb = torch.stack([rgb_norm] * ensemble_size) + single_rgb_dataset = TensorDataset(duplicated_rgb) + + # find the batch size + if batch_size>0: + _bs = batch_size + else: + _bs = 1 + + single_rgb_loader = DataLoader(single_rgb_dataset, batch_size=_bs, shuffle=False) + + # predicted the depth + depth_pred_ls = [] + normal_pred_ls = [] + + if show_progress_bar: + iterable_bar = tqdm( + single_rgb_loader, desc=" " * 2 + "Inference batches", leave=False + ) + else: + iterable_bar = single_rgb_loader + + for batch in iterable_bar: + (batched_image, )= batch # here the image is still around 0-1 + + if mask is None: + depth_pred_raw, normal_pred_raw = self.single_infer( + input_rgb=batched_image, + num_inference_steps=denoising_steps, + domain=domain, + show_pbar=show_progress_bar, + ) + else: + depth_pred_raw, normal_pred_raw = self.single_infer_inpaint( + input_rgb=batched_image, + gt_depth=gt_depth, + mask=mask, + num_inference_steps=denoising_steps, + domain=domain, + show_pbar=show_progress_bar, + ) + depth_pred_ls.append(depth_pred_raw.detach().clone()) + normal_pred_ls.append(normal_pred_raw.detach().clone()) + + depth_preds = torch.concat(depth_pred_ls, axis=0).squeeze() #(10,224,768) + normal_preds = torch.concat(normal_pred_ls, axis=0).squeeze() + torch.cuda.empty_cache() # clear vram cache for ensembling + + # ----------------- Test-time ensembling ----------------- + if ensemble_size > 1: + depth_pred, pred_uncert = ensemble_depths( + depth_preds, **(ensemble_kwargs or {}) + ) + normal_pred = ensemble_normals(normal_preds) + else: + depth_pred = depth_preds + normal_pred = normal_preds + pred_uncert = None + + # ----------------- Post processing ----------------- + # Scale prediction to [0, 1] + min_d = torch.min(depth_pred) + max_d = torch.max(depth_pred) + depth_pred = (depth_pred - min_d) / (max_d - min_d) + + # Convert to numpy + depth_pred = depth_pred.cpu().numpy().astype(np.float32) + normal_pred = normal_pred.cpu().numpy().astype(np.float32) + + # Resize back to original resolution + if match_input_res: + pred_img = Image.fromarray(depth_pred) + pred_img = pred_img.resize(input_size) + depth_pred = np.asarray(pred_img) + normal_pred = cv2.resize(chw2hwc(normal_pred), input_size, interpolation = cv2.INTER_NEAREST) + else: + normal_pred = chw2hwc(normal_pred) + + # Clip output range: current size is the original size + depth_pred = depth_pred.clip(0, 1) + normal_pred = normal_pred.clip(-1, 1) + + # Colorize + depth_colored = colorize_depth_maps( + depth_pred, 0, 1, cmap=color_map + ).squeeze() # [3, H, W], value in (0, 1) + depth_colored = (depth_colored * 255).astype(np.uint8) + depth_colored_hwc = chw2hwc(depth_colored) + depth_colored_img = Image.fromarray(depth_colored_hwc) + + normal_colored = ((normal_pred + 1)/2 * 255).astype(np.uint8) + normal_colored_img = Image.fromarray(normal_colored) + + return DepthNormalPipelineOutput( + depth_np = depth_pred, + depth_colored = depth_colored_img, + normal_np = normal_pred, + normal_colored = normal_colored_img, + uncertainty=pred_uncert, + ) + + def __encode_img_embed(self, rgb): + """ + Encode clip embeddings for img + """ + clip_image_mean = torch.as_tensor(self.feature_extractor.image_mean)[:,None,None].to(device=self.device, dtype=self.dtype) + clip_image_std = torch.as_tensor(self.feature_extractor.image_std)[:,None,None].to(device=self.device, dtype=self.dtype) + + img_in_proc = TF.resize((rgb +1)/2, + (self.feature_extractor.crop_size['height'], self.feature_extractor.crop_size['width']), + interpolation=InterpolationMode.BICUBIC, + antialias=True + ) + # do the normalization in float32 to preserve precision + img_in_proc = ((img_in_proc.float() - clip_image_mean) / clip_image_std).to(self.dtype) + img_embed = self.image_encoder(img_in_proc).image_embeds.unsqueeze(1).to(self.dtype) + + self.img_embed = img_embed + + + @torch.no_grad() + def single_infer_inpaint(self,input_rgb:torch.Tensor, + gt_depth, mask, + num_inference_steps:int, + domain:str, + show_pbar:bool,): + + device = input_rgb.device + gt_depth_latent = self.encode_depth(gt_depth) + mask = torch.nn.functional.interpolate(mask, size=gt_depth_latent.shape[2:], mode='nearest') + + # Set timesteps: inherit from the diffuison pipeline + self.scheduler.set_timesteps(num_inference_steps, device=device) # here the numbers of the steps is only 10. + timesteps = self.scheduler.timesteps # [T] + + # encode image + rgb_latent = self.encode_RGB(input_rgb) + + # Initial geometric maps (Guassian noise) + geo_latent = torch.randn(rgb_latent.shape, device=device, dtype=self.dtype).repeat(2,1,1,1) + rgb_latent = rgb_latent.repeat(2,1,1,1) + + # Batched img embedding + if self.img_embed is None: + self.__encode_img_embed(input_rgb) + + batch_img_embed = self.img_embed.repeat( + (rgb_latent.shape[0], 1, 1) + ) # [B, 1, 768] + + # hybrid switcher + geo_class = torch.tensor([[0., 1.], [1, 0]], device=device, dtype=self.dtype) + geo_embedding = torch.cat([torch.sin(geo_class), torch.cos(geo_class)], dim=-1) + + if domain == "indoor": + domain_class = torch.tensor([[1., 0., 0]], device=device, dtype=self.dtype).repeat(2,1) + elif domain == "outdoor": + domain_class = torch.tensor([[0., 1., 0]], device=device, dtype=self.dtype).repeat(2,1) + elif domain == "object": + domain_class = torch.tensor([[0., 0., 1]], device=device, dtype=self.dtype).repeat(2,1) + domain_embedding = torch.cat([torch.sin(domain_class), torch.cos(domain_class)], dim=-1) + + class_embedding = torch.cat((geo_embedding, domain_embedding), dim=-1) + + # Denoising loop + if show_pbar: + iterable = tqdm( + enumerate(timesteps), + total=len(timesteps), + leave=False, + desc=" " * 4 + "Diffusion denoising", + ) + else: + iterable = enumerate(timesteps) + + add_noise = torch.randn_like(gt_depth_latent) + for i, t in iterable: + gt_depth_latent_noised = self.scheduler.add_noise(gt_depth_latent, add_noise, t) + geo_latent[0:1] = gt_depth_latent_noised * mask + geo_latent[0:1] * (1-mask) + + unet_input = torch.cat([rgb_latent, geo_latent], dim=1) + + # predict the noise residual + noise_pred = self.unet( + unet_input, t.repeat(2), encoder_hidden_states=batch_img_embed, class_labels=class_embedding + ).sample # [B, 4, h, w] + + # compute the previous noisy sample x_t -> x_t-1 + geo_latent = self.scheduler.step(noise_pred, t, geo_latent).prev_sample + + geo_latent = geo_latent + torch.cuda.empty_cache() + + depth = self.decode_depth(geo_latent[0][None]) + depth = torch.clip(depth, -1.0, 1.0) + depth = (depth + 1.0) / 2.0 + + normal = self.decode_normal(geo_latent[1][None]) + normal /= (torch.norm(normal, p=2, dim=1, keepdim=True)+1e-5) + normal *= -1. + + return depth, normal + + @torch.no_grad() + def single_infer(self,input_rgb:torch.Tensor, + num_inference_steps:int, + domain:str, + show_pbar:bool,): + + device = input_rgb.device + + # Set timesteps: inherit from the diffuison pipeline + self.scheduler.set_timesteps(num_inference_steps, device=device) # here the numbers of the steps is only 10. + timesteps = self.scheduler.timesteps # [T] + + # encode image + rgb_latent = self.encode_RGB(input_rgb) + + # Initial geometric maps (Guassian noise) + geo_latent = torch.randn(rgb_latent.shape, device=device, dtype=self.dtype).repeat(2,1,1,1) + rgb_latent = rgb_latent.repeat(2,1,1,1) + + # Batched img embedding + if self.img_embed is None: + self.__encode_img_embed(input_rgb) + + batch_img_embed = self.img_embed.repeat( + (rgb_latent.shape[0], 1, 1) + ) # [B, 1, 768] + + # hybrid switcher + geo_class = torch.tensor([[0., 1.], [1, 0]], device=device, dtype=self.dtype) + geo_embedding = torch.cat([torch.sin(geo_class), torch.cos(geo_class)], dim=-1) + + if domain == "indoor": + domain_class = torch.tensor([[1., 0., 0]], device=device, dtype=self.dtype).repeat(2,1) + elif domain == "outdoor": + domain_class = torch.tensor([[0., 1., 0]], device=device, dtype=self.dtype).repeat(2,1) + elif domain == "object": + domain_class = torch.tensor([[0., 0., 1]], device=device, dtype=self.dtype).repeat(2,1) + domain_embedding = torch.cat([torch.sin(domain_class), torch.cos(domain_class)], dim=-1) + + class_embedding = torch.cat((geo_embedding, domain_embedding), dim=-1) + + # Denoising loop + if show_pbar: + iterable = tqdm( + enumerate(timesteps), + total=len(timesteps), + leave=False, + desc=" " * 4 + "Diffusion denoising", + ) + else: + iterable = enumerate(timesteps) + + for i, t in iterable: + unet_input = torch.cat([rgb_latent, geo_latent], dim=1) + + # predict the noise residual + noise_pred = self.unet( + unet_input, t.repeat(2), encoder_hidden_states=batch_img_embed, class_labels=class_embedding + ).sample # [B, 4, h, w] + + # compute the previous noisy sample x_t -> x_t-1 + geo_latent = self.scheduler.step(noise_pred, t, geo_latent).prev_sample + + geo_latent = geo_latent + torch.cuda.empty_cache() + + depth = self.decode_depth(geo_latent[0][None]) + depth = torch.clip(depth, -1.0, 1.0) + depth = (depth + 1.0) / 2.0 + + normal = self.decode_normal(geo_latent[1][None]) + normal /= (torch.norm(normal, p=2, dim=1, keepdim=True)+1e-5) + normal *= -1. + + return depth, normal + + + def encode_RGB(self, rgb_in: torch.Tensor) -> torch.Tensor: + """ + Encode RGB image into latent. + Args: + rgb_in (`torch.Tensor`): + Input RGB image to be encoded. + Returns: + `torch.Tensor`: Image latent. + """ + + # encode + h = self.vae.encoder(rgb_in) + + moments = self.vae.quant_conv(h) + mean, logvar = torch.chunk(moments, 2, dim=1) + # scale latent + rgb_latent = mean * self.latent_scale_factor + + return rgb_latent + + def encode_depth(self, depth_in: torch.Tensor) -> torch.Tensor: + """ + Encode depth image into latent. + + Args: + depth_in (`torch.Tensor`): + Input depth image to be encoded. + + Returns: + `torch.Tensor`: Image latent. + """ + # encode + h = self.vae.encoder(depth_in.repeat(1, 3, 1, 1)) + moments = self.vae.quant_conv(h) + mean, logvar = torch.chunk(moments, 2, dim=1) + # scale latent + depth_latent = mean * self.latent_scale_factor + return depth_latent + + def decode_depth(self, depth_latent: torch.Tensor) -> torch.Tensor: + """ + Decode depth latent into depth map. + Args: + depth_latent (`torch.Tensor`): + Depth latent to be decoded. + Returns: + `torch.Tensor`: Decoded depth map. + """ + + # scale latent + depth_latent = depth_latent / self.latent_scale_factor + # decode + z = self.vae.post_quant_conv(depth_latent) + stacked = self.vae.decoder(z) + # mean of output channels + depth_mean = stacked.mean(dim=1, keepdim=True) + return depth_mean + + def decode_normal(self, normal_latent: torch.Tensor) -> torch.Tensor: + """ + Decode normal latent into normal map. + Args: + normal_latent (`torch.Tensor`): + Depth latent to be decoded. + Returns: + `torch.Tensor`: Decoded normal map. + """ + + # scale latent + normal_latent = normal_latent / self.latent_scale_factor + # decode + z = self.vae.post_quant_conv(normal_latent) + normal = self.vae.decoder(z) + return normal + diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/transformer_2d.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/transformer_2d.py new file mode 100644 index 0000000000000000000000000000000000000000..2b28b316bfb645d84921335050efa25e15d8b2b8 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/transformer_2d.py @@ -0,0 +1,463 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Some modifications are reimplemented in public environments by Xiao Fu and Mu Hu + +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.embeddings import ImagePositionalEmbeddings +from diffusers.utils import USE_PEFT_BACKEND, BaseOutput, deprecate, is_torch_version +from models.attention import BasicTransformerBlock +from diffusers.models.embeddings import PatchEmbed, PixArtAlphaTextProjection +from diffusers.models.lora import LoRACompatibleConv, LoRACompatibleLinear +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import AdaLayerNormSingle + + +@dataclass +class Transformer2DModelOutput(BaseOutput): + """ + The output of [`Transformer2DModel`]. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `(batch size, num_vector_embeds - 1, num_latent_pixels)` if [`Transformer2DModel`] is discrete): + The hidden states output conditioned on the `encoder_hidden_states` input. If discrete, returns probability + distributions for the unnoised latent pixels. + """ + + sample: torch.FloatTensor + + +class Transformer2DModel(ModelMixin, ConfigMixin): + """ + A 2D Transformer model for image-like data. + + Parameters: + num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention. + attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head. + in_channels (`int`, *optional*): + The number of channels in the input and output (specify if the input is **continuous**). + num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The number of `encoder_hidden_states` dimensions to use. + sample_size (`int`, *optional*): The width of the latent images (specify if the input is **discrete**). + This is fixed during training since it is used to learn a number of position embeddings. + num_vector_embeds (`int`, *optional*): + The number of classes of the vector embeddings of the latent pixels (specify if the input is **discrete**). + Includes the class for the masked latent pixel. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to use in feed-forward. + num_embeds_ada_norm ( `int`, *optional*): + The number of diffusion steps used during training. Pass if at least one of the norm_layers is + `AdaLayerNorm`. This is fixed during training since it is used to learn a number of embeddings that are + added to the hidden states. + + During inference, you can denoise for up to but not more steps than `num_embeds_ada_norm`. + attention_bias (`bool`, *optional*): + Configure if the `TransformerBlocks` attention should contain a bias parameter. + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + num_attention_heads: int = 16, + attention_head_dim: int = 88, + in_channels: Optional[int] = None, + out_channels: Optional[int] = None, + num_layers: int = 1, + dropout: float = 0.0, + norm_num_groups: int = 32, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + sample_size: Optional[int] = None, + num_vector_embeds: Optional[int] = None, + patch_size: Optional[int] = None, + activation_fn: str = "geglu", + num_embeds_ada_norm: Optional[int] = None, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + double_self_attention: bool = False, + upcast_attention: bool = False, + norm_type: str = "layer_norm", + norm_elementwise_affine: bool = True, + norm_eps: float = 1e-5, + attention_type: str = "default", + caption_channels: int = None, + ): + super().__init__() + self.use_linear_projection = use_linear_projection + self.num_attention_heads = num_attention_heads + self.attention_head_dim = attention_head_dim + inner_dim = num_attention_heads * attention_head_dim + + conv_cls = nn.Conv2d if USE_PEFT_BACKEND else LoRACompatibleConv + linear_cls = nn.Linear if USE_PEFT_BACKEND else LoRACompatibleLinear + + # 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)` + # Define whether input is continuous or discrete depending on configuration + self.is_input_continuous = (in_channels is not None) and (patch_size is None) + self.is_input_vectorized = num_vector_embeds is not None + self.is_input_patches = in_channels is not None and patch_size is not None + + if norm_type == "layer_norm" and num_embeds_ada_norm is not None: + deprecation_message = ( + f"The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or" + " incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config." + " Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect" + " results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it" + " would be very nice if you could open a Pull request for the `transformer/config.json` file" + ) + deprecate("norm_type!=num_embeds_ada_norm", "1.0.0", deprecation_message, standard_warn=False) + norm_type = "ada_norm" + + if self.is_input_continuous and self.is_input_vectorized: + raise ValueError( + f"Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make" + " sure that either `in_channels` or `num_vector_embeds` is None." + ) + elif self.is_input_vectorized and self.is_input_patches: + raise ValueError( + f"Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make" + " sure that either `num_vector_embeds` or `num_patches` is None." + ) + elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches: + raise ValueError( + f"Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:" + f" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None." + ) + + # 2. Define input layers + if self.is_input_continuous: + self.in_channels = in_channels + + self.norm = torch.nn.GroupNorm(num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True) + if use_linear_projection: + self.proj_in = linear_cls(in_channels, inner_dim) + else: + self.proj_in = conv_cls(in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + elif self.is_input_vectorized: + assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size" + assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed" + + self.height = sample_size + self.width = sample_size + self.num_vector_embeds = num_vector_embeds + self.num_latent_pixels = self.height * self.width + + self.latent_image_embedding = ImagePositionalEmbeddings( + num_embed=num_vector_embeds, embed_dim=inner_dim, height=self.height, width=self.width + ) + elif self.is_input_patches: + assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size" + + self.height = sample_size + self.width = sample_size + + self.patch_size = patch_size + interpolation_scale = self.config.sample_size // 64 # => 64 (= 512 pixart) has interpolation scale 1 + interpolation_scale = max(interpolation_scale, 1) + self.pos_embed = PatchEmbed( + height=sample_size, + width=sample_size, + patch_size=patch_size, + in_channels=in_channels, + embed_dim=inner_dim, + interpolation_scale=interpolation_scale, + ) + + # 3. Define transformers blocks + self.transformer_blocks = nn.ModuleList( + [ + BasicTransformerBlock( + inner_dim, + num_attention_heads, + attention_head_dim, + dropout=dropout, + cross_attention_dim=cross_attention_dim, + activation_fn=activation_fn, + num_embeds_ada_norm=num_embeds_ada_norm, + attention_bias=attention_bias, + only_cross_attention=only_cross_attention, + double_self_attention=double_self_attention, + upcast_attention=upcast_attention, + norm_type=norm_type, + norm_elementwise_affine=norm_elementwise_affine, + norm_eps=norm_eps, + attention_type=attention_type, + ) + for d in range(num_layers) + ] + ) + + # 4. Define output layers + self.out_channels = in_channels if out_channels is None else out_channels + if self.is_input_continuous: + # TODO: should use out_channels for continuous projections + if use_linear_projection: + self.proj_out = linear_cls(inner_dim, in_channels) + else: + self.proj_out = conv_cls(inner_dim, in_channels, kernel_size=1, stride=1, padding=0) + elif self.is_input_vectorized: + self.norm_out = nn.LayerNorm(inner_dim) + self.out = nn.Linear(inner_dim, self.num_vector_embeds - 1) + elif self.is_input_patches and norm_type != "ada_norm_single": + self.norm_out = nn.LayerNorm(inner_dim, elementwise_affine=False, eps=1e-6) + self.proj_out_1 = nn.Linear(inner_dim, 2 * inner_dim) + self.proj_out_2 = nn.Linear(inner_dim, patch_size * patch_size * self.out_channels) + elif self.is_input_patches and norm_type == "ada_norm_single": + self.norm_out = nn.LayerNorm(inner_dim, elementwise_affine=False, eps=1e-6) + self.scale_shift_table = nn.Parameter(torch.randn(2, inner_dim) / inner_dim**0.5) + self.proj_out = nn.Linear(inner_dim, patch_size * patch_size * self.out_channels) + + # 5. PixArt-Alpha blocks. + self.adaln_single = None + self.use_additional_conditions = False + if norm_type == "ada_norm_single": + self.use_additional_conditions = self.config.sample_size == 128 + # TODO(Sayak, PVP) clean this, for now we use sample size to determine whether to use + # additional conditions until we find better name + self.adaln_single = AdaLayerNormSingle(inner_dim, use_additional_conditions=self.use_additional_conditions) + + self.caption_projection = None + if caption_channels is not None: + self.caption_projection = PixArtAlphaTextProjection(in_features=caption_channels, hidden_size=inner_dim) + + self.gradient_checkpointing = False + + def _set_gradient_checkpointing(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + timestep: Optional[torch.LongTensor] = None, + added_cond_kwargs: Dict[str, torch.Tensor] = None, + class_labels: Optional[torch.LongTensor] = None, + cross_attention_kwargs: Dict[str, Any] = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + return_dict: bool = True, + ): + """ + The [`Transformer2DModel`] forward method. + + Args: + hidden_states (`torch.LongTensor` of shape `(batch size, num latent pixels)` if discrete, `torch.FloatTensor` of shape `(batch size, channel, height, width)` if continuous): + Input `hidden_states`. + encoder_hidden_states ( `torch.FloatTensor` of shape `(batch size, sequence len, embed dims)`, *optional*): + Conditional embeddings for cross attention layer. If not given, cross-attention defaults to + self-attention. + timestep ( `torch.LongTensor`, *optional*): + Used to indicate denoising step. Optional timestep to be applied as an embedding in `AdaLayerNorm`. + class_labels ( `torch.LongTensor` of shape `(batch size, num classes)`, *optional*): + Used to indicate class labels conditioning. Optional class labels to be applied as an embedding in + `AdaLayerZeroNorm`. + cross_attention_kwargs ( `Dict[str, Any]`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + attention_mask ( `torch.Tensor`, *optional*): + An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask + is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large + negative values to the attention scores corresponding to "discard" tokens. + encoder_attention_mask ( `torch.Tensor`, *optional*): + Cross-attention mask applied to `encoder_hidden_states`. Two formats supported: + + * Mask `(batch, sequence_length)` True = keep, False = discard. + * Bias `(batch, 1, sequence_length)` 0 = keep, -10000 = discard. + + If `ndim == 2`: will be interpreted as a mask, then converted into a bias consistent with the format + above. This bias will be added to the cross-attention scores. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain + tuple. + + Returns: + If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a + `tuple` where the first element is the sample tensor. + """ + # ensure attention_mask is a bias, and give it a singleton query_tokens dimension. + # we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward. + # we can tell by counting dims; if ndim == 2: it's a mask rather than a bias. + # expects mask of shape: + # [batch, key_tokens] + # adds singleton query_tokens dimension: + # [batch, 1, key_tokens] + # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes: + # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn) + # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn) + + if attention_mask is not None and attention_mask.ndim == 2: + # assume that mask is expressed as: + # (1 = keep, 0 = discard) + # convert mask into a bias that can be added to attention scores: + # (keep = +0, discard = -10000.0) + attention_mask = (1 - attention_mask.to(hidden_states.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # convert encoder_attention_mask to a bias the same way we do for attention_mask + if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2: + encoder_attention_mask = (1 - encoder_attention_mask.to(hidden_states.dtype)) * -10000.0 + encoder_attention_mask = encoder_attention_mask.unsqueeze(1) + + # Retrieve lora scale. + lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + + # 1. Input + if self.is_input_continuous: + batch, _, height, width = hidden_states.shape + residual = hidden_states + + hidden_states = self.norm(hidden_states) + if not self.use_linear_projection: + hidden_states = ( + self.proj_in(hidden_states, scale=lora_scale) + if not USE_PEFT_BACKEND + else self.proj_in(hidden_states) + ) + inner_dim = hidden_states.shape[1] + hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * width, inner_dim) + else: + inner_dim = hidden_states.shape[1] + hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * width, inner_dim) + hidden_states = ( + self.proj_in(hidden_states, scale=lora_scale) + if not USE_PEFT_BACKEND + else self.proj_in(hidden_states) + ) + + elif self.is_input_vectorized: + hidden_states = self.latent_image_embedding(hidden_states) + elif self.is_input_patches: + height, width = hidden_states.shape[-2] // self.patch_size, hidden_states.shape[-1] // self.patch_size + hidden_states = self.pos_embed(hidden_states) + + if self.adaln_single is not None: + if self.use_additional_conditions and added_cond_kwargs is None: + raise ValueError( + "`added_cond_kwargs` cannot be None when using additional conditions for `adaln_single`." + ) + batch_size = hidden_states.shape[0] + timestep, embedded_timestep = self.adaln_single( + timestep, added_cond_kwargs, batch_size=batch_size, hidden_dtype=hidden_states.dtype + ) + + # 2. Blocks + if self.caption_projection is not None: + batch_size = hidden_states.shape[0] + encoder_hidden_states = self.caption_projection(encoder_hidden_states) + encoder_hidden_states = encoder_hidden_states.view(batch_size, -1, hidden_states.shape[-1]) + + for block in self.transformer_blocks: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + attention_mask, + encoder_hidden_states, + encoder_attention_mask, + timestep, + cross_attention_kwargs, + class_labels, + **ckpt_kwargs, + ) + else: + hidden_states = block( + hidden_states, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + timestep=timestep, + cross_attention_kwargs=cross_attention_kwargs, + class_labels=class_labels, + ) + + # 3. Output + if self.is_input_continuous: + if not self.use_linear_projection: + hidden_states = hidden_states.reshape(batch, height, width, inner_dim).permute(0, 3, 1, 2).contiguous() + hidden_states = ( + self.proj_out(hidden_states, scale=lora_scale) + if not USE_PEFT_BACKEND + else self.proj_out(hidden_states) + ) + else: + hidden_states = ( + self.proj_out(hidden_states, scale=lora_scale) + if not USE_PEFT_BACKEND + else self.proj_out(hidden_states) + ) + hidden_states = hidden_states.reshape(batch, height, width, inner_dim).permute(0, 3, 1, 2).contiguous() + + output = hidden_states + residual + elif self.is_input_vectorized: + hidden_states = self.norm_out(hidden_states) + logits = self.out(hidden_states) + # (batch, self.num_vector_embeds - 1, self.num_latent_pixels) + logits = logits.permute(0, 2, 1) + + # log(p(x_0)) + output = F.log_softmax(logits.double(), dim=1).float() + + if self.is_input_patches: + if self.config.norm_type != "ada_norm_single": + conditioning = self.transformer_blocks[0].norm1.emb( + timestep, class_labels, hidden_dtype=hidden_states.dtype + ) + shift, scale = self.proj_out_1(F.silu(conditioning)).chunk(2, dim=1) + hidden_states = self.norm_out(hidden_states) * (1 + scale[:, None]) + shift[:, None] + hidden_states = self.proj_out_2(hidden_states) + elif self.config.norm_type == "ada_norm_single": + shift, scale = (self.scale_shift_table[None] + embedded_timestep[:, None]).chunk(2, dim=1) + hidden_states = self.norm_out(hidden_states) + # Modulation + hidden_states = hidden_states * (1 + scale) + shift + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states.squeeze(1) + + # unpatchify + if self.adaln_single is None: + height = width = int(hidden_states.shape[1] ** 0.5) + hidden_states = hidden_states.reshape( + shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels) + ) + hidden_states = torch.einsum("nhwpqc->nchpwq", hidden_states) + output = hidden_states.reshape( + shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size) + ) + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/unet_2d_blocks.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/unet_2d_blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..994036ec97e423f203c980734a14300fe78ebfb4 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/unet_2d_blocks.py @@ -0,0 +1,3494 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Some modifications are reimplemented in public environments by Xiao Fu and Mu Hu + +from typing import Any, Dict, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from diffusers.utils import is_torch_version, logging +from diffusers.utils.torch_utils import apply_freeu +from diffusers.models.activations import get_activation +from diffusers.models.attention_processor import Attention, AttnAddedKVProcessor, AttnAddedKVProcessor2_0 +from diffusers.models.dual_transformer_2d import DualTransformer2DModel +from diffusers.models.normalization import AdaGroupNorm +from diffusers.models.resnet import Downsample2D, FirDownsample2D, FirUpsample2D, KDownsample2D, KUpsample2D, ResnetBlock2D, Upsample2D +from models.transformer_2d import Transformer2DModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def get_down_block( + down_block_type: str, + num_layers: int, + in_channels: int, + out_channels: int, + temb_channels: int, + add_downsample: bool, + resnet_eps: float, + resnet_act_fn: str, + transformer_layers_per_block: int = 1, + num_attention_heads: Optional[int] = None, + resnet_groups: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + downsample_padding: Optional[int] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + attention_type: str = "default", + resnet_skip_time_act: bool = False, + resnet_out_scale_factor: float = 1.0, + cross_attention_norm: Optional[str] = None, + attention_head_dim: Optional[int] = None, + downsample_type: Optional[str] = None, + dropout: float = 0.0, +): + # If attn head dim is not defined, we default it to the number of heads + if attention_head_dim is None: + logger.warn( + f"It is recommended to provide `attention_head_dim` when calling `get_down_block`. Defaulting `attention_head_dim` to {num_attention_heads}." + ) + attention_head_dim = num_attention_heads + + down_block_type = down_block_type[7:] if down_block_type.startswith("UNetRes") else down_block_type + if down_block_type == "DownBlock2D": + return DownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "ResnetDownsampleBlock2D": + return ResnetDownsampleBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + ) + elif down_block_type == "AttnDownBlock2D": + if add_downsample is False: + downsample_type = None + else: + downsample_type = downsample_type or "conv" # default to 'conv' + return AttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + downsample_type=downsample_type, + ) + elif down_block_type == "CrossAttnDownBlock2D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnDownBlock2D") + return CrossAttnDownBlock2D( + num_layers=num_layers, + transformer_layers_per_block=transformer_layers_per_block, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + ) + elif down_block_type == "SimpleCrossAttnDownBlock2D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for SimpleCrossAttnDownBlock2D") + return SimpleCrossAttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif down_block_type == "SkipDownBlock2D": + return SkipDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "AttnSkipDownBlock2D": + return AttnSkipDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "DownEncoderBlock2D": + return DownEncoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "AttnDownEncoderBlock2D": + return AttnDownEncoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "KDownBlock2D": + return KDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + ) + elif down_block_type == "KCrossAttnDownBlock2D": + return KCrossAttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + add_self_attention=True if not add_downsample else False, + ) + raise ValueError(f"{down_block_type} does not exist.") + + +def get_up_block( + up_block_type: str, + num_layers: int, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + add_upsample: bool, + resnet_eps: float, + resnet_act_fn: str, + resolution_idx: Optional[int] = None, + transformer_layers_per_block: int = 1, + num_attention_heads: Optional[int] = None, + resnet_groups: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + attention_type: str = "default", + resnet_skip_time_act: bool = False, + resnet_out_scale_factor: float = 1.0, + cross_attention_norm: Optional[str] = None, + attention_head_dim: Optional[int] = None, + upsample_type: Optional[str] = None, + dropout: float = 0.0, +) -> nn.Module: + # If attn head dim is not defined, we default it to the number of heads + if attention_head_dim is None: + logger.warn( + f"It is recommended to provide `attention_head_dim` when calling `get_up_block`. Defaulting `attention_head_dim` to {num_attention_heads}." + ) + attention_head_dim = num_attention_heads + + up_block_type = up_block_type[7:] if up_block_type.startswith("UNetRes") else up_block_type + if up_block_type == "UpBlock2D": + return UpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == "ResnetUpsampleBlock2D": + return ResnetUpsampleBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + ) + elif up_block_type == "CrossAttnUpBlock2D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnUpBlock2D") + return CrossAttnUpBlock2D( + num_layers=num_layers, + transformer_layers_per_block=transformer_layers_per_block, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + ) + elif up_block_type == "SimpleCrossAttnUpBlock2D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for SimpleCrossAttnUpBlock2D") + return SimpleCrossAttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif up_block_type == "AttnUpBlock2D": + if add_upsample is False: + upsample_type = None + else: + upsample_type = upsample_type or "conv" # default to 'conv' + + return AttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + upsample_type=upsample_type, + ) + elif up_block_type == "SkipUpBlock2D": + return SkipUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == "AttnSkipUpBlock2D": + return AttnSkipUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == "UpDecoderBlock2D": + return UpDecoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + temb_channels=temb_channels, + ) + elif up_block_type == "AttnUpDecoderBlock2D": + return AttnUpDecoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + temb_channels=temb_channels, + ) + elif up_block_type == "KUpBlock2D": + return KUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + ) + elif up_block_type == "KCrossAttnUpBlock2D": + return KCrossAttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + ) + + raise ValueError(f"{up_block_type} does not exist.") + + +class AutoencoderTinyBlock(nn.Module): + """ + Tiny Autoencoder block used in [`AutoencoderTiny`]. It is a mini residual module consisting of plain conv + ReLU + blocks. + + Args: + in_channels (`int`): The number of input channels. + out_channels (`int`): The number of output channels. + act_fn (`str`): + ` The activation function to use. Supported values are `"swish"`, `"mish"`, `"gelu"`, and `"relu"`. + + Returns: + `torch.FloatTensor`: A tensor with the same shape as the input tensor, but with the number of channels equal to + `out_channels`. + """ + + def __init__(self, in_channels: int, out_channels: int, act_fn: str): + super().__init__() + act_fn = get_activation(act_fn) + self.conv = nn.Sequential( + nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1), + act_fn, + nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1), + act_fn, + nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1), + ) + self.skip = ( + nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False) + if in_channels != out_channels + else nn.Identity() + ) + self.fuse = nn.ReLU() + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + return self.fuse(self.conv(x) + self.skip(x)) + + +class UNetMidBlock2D(nn.Module): + """ + A 2D UNet mid-block [`UNetMidBlock2D`] with multiple residual blocks and optional attention blocks. + + Args: + in_channels (`int`): The number of input channels. + temb_channels (`int`): The number of temporal embedding channels. + dropout (`float`, *optional*, defaults to 0.0): The dropout rate. + num_layers (`int`, *optional*, defaults to 1): The number of residual blocks. + resnet_eps (`float`, *optional*, 1e-6 ): The epsilon value for the resnet blocks. + resnet_time_scale_shift (`str`, *optional*, defaults to `default`): + The type of normalization to apply to the time embeddings. This can help to improve the performance of the + model on tasks with long-range temporal dependencies. + resnet_act_fn (`str`, *optional*, defaults to `swish`): The activation function for the resnet blocks. + resnet_groups (`int`, *optional*, defaults to 32): + The number of groups to use in the group normalization layers of the resnet blocks. + attn_groups (`Optional[int]`, *optional*, defaults to None): The number of groups for the attention blocks. + resnet_pre_norm (`bool`, *optional*, defaults to `True`): + Whether to use pre-normalization for the resnet blocks. + add_attention (`bool`, *optional*, defaults to `True`): Whether to add attention blocks. + attention_head_dim (`int`, *optional*, defaults to 1): + Dimension of a single attention head. The number of attention heads is determined based on this value and + the number of input channels. + output_scale_factor (`float`, *optional*, defaults to 1.0): The output scale factor. + + Returns: + `torch.FloatTensor`: The output of the last residual block, which is a tensor of shape `(batch_size, + in_channels, height, width)`. + + """ + + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", # default, spatial + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + attn_groups: Optional[int] = None, + resnet_pre_norm: bool = True, + add_attention: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + ): + super().__init__() + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + self.add_attention = add_attention + + if attn_groups is None: + attn_groups = resnet_groups if resnet_time_scale_shift == "default" else None + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + + if attention_head_dim is None: + logger.warn( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {in_channels}." + ) + attention_head_dim = in_channels + + for _ in range(num_layers): + if self.add_attention: + attentions.append( + Attention( + in_channels, + heads=in_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=attn_groups, + spatial_norm_dim=temb_channels if resnet_time_scale_shift == "spatial" else None, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + else: + attentions.append(None) + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if attn is not None: + hidden_states = attn(hidden_states, temb=temb) + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +class UNetMidBlock2DCrossAttn(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + output_scale_factor: float = 1.0, + cross_attention_dim: int = 1280, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + upcast_attention: bool = False, + attention_type: str = "default", + ): + super().__init__() + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + + # support for variable transformer layers per block + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + + for i in range(num_layers): + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + hidden_states = self.resnets[0](hidden_states, temb, scale=lora_scale) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + else: + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = resnet(hidden_states, temb, scale=lora_scale) + + return hidden_states + + +class UNetMidBlock2DSimpleCrossAttn(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + cross_attention_dim: int = 1280, + skip_time_act: bool = False, + only_cross_attention: bool = False, + cross_attention_norm: Optional[str] = None, + ): + super().__init__() + + self.has_cross_attention = True + + self.attention_head_dim = attention_head_dim + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + + self.num_heads = in_channels // self.attention_head_dim + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ] + attentions = [] + + for _ in range(num_layers): + processor = ( + AttnAddedKVProcessor2_0() if hasattr(F, "scaled_dot_product_attention") else AttnAddedKVProcessor() + ) + + attentions.append( + Attention( + query_dim=in_channels, + cross_attention_dim=in_channels, + heads=self.num_heads, + dim_head=self.attention_head_dim, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + ) + ) + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + lora_scale = cross_attention_kwargs.get("scale", 1.0) + + if attention_mask is None: + # if encoder_hidden_states is defined: we are doing cross-attn, so we should use cross-attn mask. + mask = None if encoder_hidden_states is None else encoder_attention_mask + else: + # when attention_mask is defined: we don't even check for encoder_attention_mask. + # this is to maintain compatibility with UnCLIP, which uses 'attention_mask' param for cross-attn masks. + # TODO: UnCLIP should express cross-attn mask via encoder_attention_mask param instead of via attention_mask. + # then we can simplify this whole if/else block to: + # mask = attention_mask if encoder_hidden_states is None else encoder_attention_mask + mask = attention_mask + + hidden_states = self.resnets[0](hidden_states, temb, scale=lora_scale) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + # attn + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + + # resnet + hidden_states = resnet(hidden_states, temb, scale=lora_scale) + + return hidden_states + + +class AttnDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + downsample_padding: int = 1, + downsample_type: str = "conv", + ): + super().__init__() + resnets = [] + attentions = [] + self.downsample_type = downsample_type + + if attention_head_dim is None: + logger.warn( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if downsample_type == "conv": + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + elif downsample_type == "resnet": + self.downsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + down=True, + ) + ] + ) + else: + self.downsamplers = None + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + + lora_scale = cross_attention_kwargs.get("scale", 1.0) + + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + cross_attention_kwargs.update({"scale": lora_scale}) + hidden_states = resnet(hidden_states, temb, scale=lora_scale) + hidden_states = attn(hidden_states, **cross_attention_kwargs) + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + if self.downsample_type == "resnet": + hidden_states = downsampler(hidden_states, temb=temb, scale=lora_scale) + else: + hidden_states = downsampler(hidden_states, scale=lora_scale) + + output_states += (hidden_states,) + + return hidden_states, output_states + + +class CrossAttnDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + downsample_padding: int = 1, + add_downsample: bool = True, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + attention_type: str = "default", + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + additional_residuals: Optional[torch.FloatTensor] = None, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + output_states = () + + lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + + blocks = list(zip(self.resnets, self.attentions)) + + for i, (resnet, attn) in enumerate(blocks): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + else: + hidden_states = resnet(hidden_states, temb, scale=lora_scale) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + + # apply additional residuals to the output of the last pair of resnet and attention blocks + if i == len(blocks) - 1 and additional_residuals is not None: + hidden_states = hidden_states + additional_residuals + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, scale=lora_scale) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class DownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + downsample_padding: int = 1, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None, scale: float = 1.0 + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb, scale=scale) + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, scale=scale) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class DownEncoderBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + downsample_padding: int = 1, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + else: + self.downsamplers = None + + def forward(self, hidden_states: torch.FloatTensor, scale: float = 1.0) -> torch.FloatTensor: + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb=None, scale=scale) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, scale) + + return hidden_states + + +class AttnDownEncoderBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + downsample_padding: int = 1, + ): + super().__init__() + resnets = [] + attentions = [] + + if attention_head_dim is None: + logger.warn( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + else: + self.downsamplers = None + + def forward(self, hidden_states: torch.FloatTensor, scale: float = 1.0) -> torch.FloatTensor: + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb=None, scale=scale) + cross_attention_kwargs = {"scale": scale} + hidden_states = attn(hidden_states, **cross_attention_kwargs) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, scale) + + return hidden_states + + +class AttnSkipDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = np.sqrt(2.0), + add_downsample: bool = True, + ): + super().__init__() + self.attentions = nn.ModuleList([]) + self.resnets = nn.ModuleList([]) + + if attention_head_dim is None: + logger.warn( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + self.resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(in_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + self.attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=32, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + if add_downsample: + self.resnet_down = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + down=True, + kernel="fir", + ) + self.downsamplers = nn.ModuleList([FirDownsample2D(out_channels, out_channels=out_channels)]) + self.skip_conv = nn.Conv2d(3, out_channels, kernel_size=(1, 1), stride=(1, 1)) + else: + self.resnet_down = None + self.downsamplers = None + self.skip_conv = None + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + skip_sample: Optional[torch.FloatTensor] = None, + scale: float = 1.0, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...], torch.FloatTensor]: + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb, scale=scale) + cross_attention_kwargs = {"scale": scale} + hidden_states = attn(hidden_states, **cross_attention_kwargs) + output_states += (hidden_states,) + + if self.downsamplers is not None: + hidden_states = self.resnet_down(hidden_states, temb, scale=scale) + for downsampler in self.downsamplers: + skip_sample = downsampler(skip_sample) + + hidden_states = self.skip_conv(skip_sample) + hidden_states + + output_states += (hidden_states,) + + return hidden_states, output_states, skip_sample + + +class SkipDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_pre_norm: bool = True, + output_scale_factor: float = np.sqrt(2.0), + add_downsample: bool = True, + downsample_padding: int = 1, + ): + super().__init__() + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + self.resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(in_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + if add_downsample: + self.resnet_down = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + down=True, + kernel="fir", + ) + self.downsamplers = nn.ModuleList([FirDownsample2D(out_channels, out_channels=out_channels)]) + self.skip_conv = nn.Conv2d(3, out_channels, kernel_size=(1, 1), stride=(1, 1)) + else: + self.resnet_down = None + self.downsamplers = None + self.skip_conv = None + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + skip_sample: Optional[torch.FloatTensor] = None, + scale: float = 1.0, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...], torch.FloatTensor]: + output_states = () + + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb, scale) + output_states += (hidden_states,) + + if self.downsamplers is not None: + hidden_states = self.resnet_down(hidden_states, temb, scale) + for downsampler in self.downsamplers: + skip_sample = downsampler(skip_sample) + + hidden_states = self.skip_conv(skip_sample) + hidden_states + + output_states += (hidden_states,) + + return hidden_states, output_states, skip_sample + + +class ResnetDownsampleBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + skip_time_act: bool = False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + down=True, + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None, scale: float = 1.0 + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb, scale) + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, temb, scale) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class SimpleCrossAttnDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + skip_time_act: bool = False, + only_cross_attention: bool = False, + cross_attention_norm: Optional[str] = None, + ): + super().__init__() + + self.has_cross_attention = True + + resnets = [] + attentions = [] + + self.attention_head_dim = attention_head_dim + self.num_heads = out_channels // self.attention_head_dim + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ) + + processor = ( + AttnAddedKVProcessor2_0() if hasattr(F, "scaled_dot_product_attention") else AttnAddedKVProcessor() + ) + + attentions.append( + Attention( + query_dim=out_channels, + cross_attention_dim=out_channels, + heads=self.num_heads, + dim_head=attention_head_dim, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + down=True, + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + output_states = () + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + + lora_scale = cross_attention_kwargs.get("scale", 1.0) + + if attention_mask is None: + # if encoder_hidden_states is defined: we are doing cross-attn, so we should use cross-attn mask. + mask = None if encoder_hidden_states is None else encoder_attention_mask + else: + # when attention_mask is defined: we don't even check for encoder_attention_mask. + # this is to maintain compatibility with UnCLIP, which uses 'attention_mask' param for cross-attn masks. + # TODO: UnCLIP should express cross-attn mask via encoder_attention_mask param instead of via attention_mask. + # then we can simplify this whole if/else block to: + # mask = attention_mask if encoder_hidden_states is None else encoder_attention_mask + mask = attention_mask + + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + else: + hidden_states = resnet(hidden_states, temb, scale=lora_scale) + + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, temb, scale=lora_scale) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class KDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_eps: float = 1e-5, + resnet_act_fn: str = "gelu", + resnet_group_size: int = 32, + add_downsample: bool = False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + temb_channels=temb_channels, + groups=groups, + groups_out=groups_out, + eps=resnet_eps, + non_linearity=resnet_act_fn, + time_embedding_norm="ada_group", + conv_shortcut_bias=False, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + # YiYi's comments- might be able to use FirDownsample2D, look into details later + self.downsamplers = nn.ModuleList([KDownsample2D()]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None, scale: float = 1.0 + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb, scale) + + output_states += (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states, output_states + + +class KCrossAttnDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + cross_attention_dim: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_group_size: int = 32, + add_downsample: bool = True, + attention_head_dim: int = 64, + add_self_attention: bool = False, + resnet_eps: float = 1e-5, + resnet_act_fn: str = "gelu", + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + temb_channels=temb_channels, + groups=groups, + groups_out=groups_out, + eps=resnet_eps, + non_linearity=resnet_act_fn, + time_embedding_norm="ada_group", + conv_shortcut_bias=False, + ) + ) + attentions.append( + KAttentionBlock( + out_channels, + out_channels // attention_head_dim, + attention_head_dim, + cross_attention_dim=cross_attention_dim, + temb_channels=temb_channels, + attention_bias=True, + add_self_attention=add_self_attention, + cross_attention_norm="layer_norm", + group_size=resnet_group_size, + ) + ) + + self.resnets = nn.ModuleList(resnets) + self.attentions = nn.ModuleList(attentions) + + if add_downsample: + self.downsamplers = nn.ModuleList([KDownsample2D()]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + output_states = () + lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + else: + hidden_states = resnet(hidden_states, temb, scale=lora_scale) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + + if self.downsamplers is None: + output_states += (None,) + else: + output_states += (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states, output_states + + +class AttnUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: int = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + upsample_type: str = "conv", + ): + super().__init__() + resnets = [] + attentions = [] + + self.upsample_type = upsample_type + + if attention_head_dim is None: + logger.warn( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if upsample_type == "conv": + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + elif upsample_type == "resnet": + self.upsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + up=True, + ) + ] + ) + else: + self.upsamplers = None + + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + scale: float = 1.0, + ) -> torch.FloatTensor: + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + hidden_states = resnet(hidden_states, temb, scale=scale) + cross_attention_kwargs = {"scale": scale} + hidden_states = attn(hidden_states, **cross_attention_kwargs) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + if self.upsample_type == "resnet": + hidden_states = upsampler(hidden_states, temb=temb, scale=scale) + else: + hidden_states = upsampler(hidden_states, scale=scale) + + return hidden_states + + +class CrossAttnUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + attention_type: str = "default", + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + is_freeu_enabled = ( + getattr(self, "s1", None) + and getattr(self, "s2", None) + and getattr(self, "b1", None) + and getattr(self, "b2", None) + ) + + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # FreeU: Only operate on the first two stages + if is_freeu_enabled: + hidden_states, res_hidden_states = apply_freeu( + self.resolution_idx, + hidden_states, + res_hidden_states, + s1=self.s1, + s2=self.s2, + b1=self.b1, + b2=self.b2, + ) + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + else: + hidden_states = resnet(hidden_states, temb, scale=lora_scale) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size, scale=lora_scale) + + return hidden_states + + +class UpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + scale: float = 1.0, + ) -> torch.FloatTensor: + is_freeu_enabled = ( + getattr(self, "s1", None) + and getattr(self, "s2", None) + and getattr(self, "b1", None) + and getattr(self, "b2", None) + ) + + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # FreeU: Only operate on the first two stages + if is_freeu_enabled: + hidden_states, res_hidden_states = apply_freeu( + self.resolution_idx, + hidden_states, + res_hidden_states, + s1=self.s1, + s2=self.s2, + b1=self.b1, + b2=self.b2, + ) + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb, scale=scale) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size, scale=scale) + + return hidden_states + + +class UpDecoderBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", # default, spatial + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + temb_channels: Optional[int] = None, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.resolution_idx = resolution_idx + + def forward( + self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None, scale: float = 1.0 + ) -> torch.FloatTensor: + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb=temb, scale=scale) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class AttnUpDecoderBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + temb_channels: Optional[int] = None, + ): + super().__init__() + resnets = [] + attentions = [] + + if attention_head_dim is None: + logger.warn( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `out_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups if resnet_time_scale_shift != "spatial" else None, + spatial_norm_dim=temb_channels if resnet_time_scale_shift == "spatial" else None, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.resolution_idx = resolution_idx + + def forward( + self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None, scale: float = 1.0 + ) -> torch.FloatTensor: + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb=temb, scale=scale) + cross_attention_kwargs = {"scale": scale} + hidden_states = attn(hidden_states, temb=temb, **cross_attention_kwargs) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, scale=scale) + + return hidden_states + + +class AttnSkipUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = np.sqrt(2.0), + add_upsample: bool = True, + ): + super().__init__() + self.attentions = nn.ModuleList([]) + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + self.resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(resnet_in_channels + res_skip_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + if attention_head_dim is None: + logger.warn( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `out_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + self.attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=32, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + self.upsampler = FirUpsample2D(in_channels, out_channels=out_channels) + if add_upsample: + self.resnet_up = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + up=True, + kernel="fir", + ) + self.skip_conv = nn.Conv2d(out_channels, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + self.skip_norm = torch.nn.GroupNorm( + num_groups=min(out_channels // 4, 32), num_channels=out_channels, eps=resnet_eps, affine=True + ) + self.act = nn.SiLU() + else: + self.resnet_up = None + self.skip_conv = None + self.skip_norm = None + self.act = None + + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + skip_sample=None, + scale: float = 1.0, + ) -> Tuple[torch.FloatTensor, torch.FloatTensor]: + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + hidden_states = resnet(hidden_states, temb, scale=scale) + + cross_attention_kwargs = {"scale": scale} + hidden_states = self.attentions[0](hidden_states, **cross_attention_kwargs) + + if skip_sample is not None: + skip_sample = self.upsampler(skip_sample) + else: + skip_sample = 0 + + if self.resnet_up is not None: + skip_sample_states = self.skip_norm(hidden_states) + skip_sample_states = self.act(skip_sample_states) + skip_sample_states = self.skip_conv(skip_sample_states) + + skip_sample = skip_sample + skip_sample_states + + hidden_states = self.resnet_up(hidden_states, temb, scale=scale) + + return hidden_states, skip_sample + + +class SkipUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_pre_norm: bool = True, + output_scale_factor: float = np.sqrt(2.0), + add_upsample: bool = True, + upsample_padding: int = 1, + ): + super().__init__() + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + self.resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min((resnet_in_channels + res_skip_channels) // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.upsampler = FirUpsample2D(in_channels, out_channels=out_channels) + if add_upsample: + self.resnet_up = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + up=True, + kernel="fir", + ) + self.skip_conv = nn.Conv2d(out_channels, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + self.skip_norm = torch.nn.GroupNorm( + num_groups=min(out_channels // 4, 32), num_channels=out_channels, eps=resnet_eps, affine=True + ) + self.act = nn.SiLU() + else: + self.resnet_up = None + self.skip_conv = None + self.skip_norm = None + self.act = None + + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + skip_sample=None, + scale: float = 1.0, + ) -> Tuple[torch.FloatTensor, torch.FloatTensor]: + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + hidden_states = resnet(hidden_states, temb, scale=scale) + + if skip_sample is not None: + skip_sample = self.upsampler(skip_sample) + else: + skip_sample = 0 + + if self.resnet_up is not None: + skip_sample_states = self.skip_norm(hidden_states) + skip_sample_states = self.act(skip_sample_states) + skip_sample_states = self.skip_conv(skip_sample_states) + + skip_sample = skip_sample + skip_sample_states + + hidden_states = self.resnet_up(hidden_states, temb, scale=scale) + + return hidden_states, skip_sample + + +class ResnetUpsampleBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + skip_time_act: bool = False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + up=True, + ) + ] + ) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + scale: float = 1.0, + ) -> torch.FloatTensor: + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb, scale=scale) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, temb, scale=scale) + + return hidden_states + + +class SimpleCrossAttnUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + skip_time_act: bool = False, + only_cross_attention: bool = False, + cross_attention_norm: Optional[str] = None, + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.attention_head_dim = attention_head_dim + + self.num_heads = out_channels // self.attention_head_dim + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ) + + processor = ( + AttnAddedKVProcessor2_0() if hasattr(F, "scaled_dot_product_attention") else AttnAddedKVProcessor() + ) + + attentions.append( + Attention( + query_dim=out_channels, + cross_attention_dim=out_channels, + heads=self.num_heads, + dim_head=self.attention_head_dim, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + up=True, + ) + ] + ) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + + lora_scale = cross_attention_kwargs.get("scale", 1.0) + if attention_mask is None: + # if encoder_hidden_states is defined: we are doing cross-attn, so we should use cross-attn mask. + mask = None if encoder_hidden_states is None else encoder_attention_mask + else: + # when attention_mask is defined: we don't even check for encoder_attention_mask. + # this is to maintain compatibility with UnCLIP, which uses 'attention_mask' param for cross-attn masks. + # TODO: UnCLIP should express cross-attn mask via encoder_attention_mask param instead of via attention_mask. + # then we can simplify this whole if/else block to: + # mask = attention_mask if encoder_hidden_states is None else encoder_attention_mask + mask = attention_mask + + for resnet, attn in zip(self.resnets, self.attentions): + # resnet + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + else: + hidden_states = resnet(hidden_states, temb, scale=lora_scale) + + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, temb, scale=lora_scale) + + return hidden_states + + +class KUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + resolution_idx: int, + dropout: float = 0.0, + num_layers: int = 5, + resnet_eps: float = 1e-5, + resnet_act_fn: str = "gelu", + resnet_group_size: Optional[int] = 32, + add_upsample: bool = True, + ): + super().__init__() + resnets = [] + k_in_channels = 2 * out_channels + k_out_channels = in_channels + num_layers = num_layers - 1 + + for i in range(num_layers): + in_channels = k_in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=k_out_channels if (i == num_layers - 1) else out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=groups, + groups_out=groups_out, + dropout=dropout, + non_linearity=resnet_act_fn, + time_embedding_norm="ada_group", + conv_shortcut_bias=False, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([KUpsample2D()]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + scale: float = 1.0, + ) -> torch.FloatTensor: + res_hidden_states_tuple = res_hidden_states_tuple[-1] + if res_hidden_states_tuple is not None: + hidden_states = torch.cat([hidden_states, res_hidden_states_tuple], dim=1) + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb, scale=scale) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class KCrossAttnUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + resolution_idx: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_eps: float = 1e-5, + resnet_act_fn: str = "gelu", + resnet_group_size: int = 32, + attention_head_dim: int = 1, # attention dim_head + cross_attention_dim: int = 768, + add_upsample: bool = True, + upcast_attention: bool = False, + ): + super().__init__() + resnets = [] + attentions = [] + + is_first_block = in_channels == out_channels == temb_channels + is_middle_block = in_channels != out_channels + add_self_attention = True if is_first_block else False + + self.has_cross_attention = True + self.attention_head_dim = attention_head_dim + + # in_channels, and out_channels for the block (k-unet) + k_in_channels = out_channels if is_first_block else 2 * out_channels + k_out_channels = in_channels + + num_layers = num_layers - 1 + + for i in range(num_layers): + in_channels = k_in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + if is_middle_block and (i == num_layers - 1): + conv_2d_out_channels = k_out_channels + else: + conv_2d_out_channels = None + + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + conv_2d_out_channels=conv_2d_out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=groups, + groups_out=groups_out, + dropout=dropout, + non_linearity=resnet_act_fn, + time_embedding_norm="ada_group", + conv_shortcut_bias=False, + ) + ) + attentions.append( + KAttentionBlock( + k_out_channels if (i == num_layers - 1) else out_channels, + k_out_channels // attention_head_dim + if (i == num_layers - 1) + else out_channels // attention_head_dim, + attention_head_dim, + cross_attention_dim=cross_attention_dim, + temb_channels=temb_channels, + attention_bias=True, + add_self_attention=add_self_attention, + cross_attention_norm="layer_norm", + upcast_attention=upcast_attention, + ) + ) + + self.resnets = nn.ModuleList(resnets) + self.attentions = nn.ModuleList(attentions) + + if add_upsample: + self.upsamplers = nn.ModuleList([KUpsample2D()]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + res_hidden_states_tuple = res_hidden_states_tuple[-1] + if res_hidden_states_tuple is not None: + hidden_states = torch.cat([hidden_states, res_hidden_states_tuple], dim=1) + + lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + else: + hidden_states = resnet(hidden_states, temb, scale=lora_scale) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +# can potentially later be renamed to `No-feed-forward` attention +class KAttentionBlock(nn.Module): + r""" + A basic Transformer block. + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + attention_bias (`bool`, *optional*, defaults to `False`): + Configure if the attention layers should contain a bias parameter. + upcast_attention (`bool`, *optional*, defaults to `False`): + Set to `True` to upcast the attention computation to `float32`. + temb_channels (`int`, *optional*, defaults to 768): + The number of channels in the token embedding. + add_self_attention (`bool`, *optional*, defaults to `False`): + Set to `True` to add self-attention to the block. + cross_attention_norm (`str`, *optional*, defaults to `None`): + The type of normalization to use for the cross attention. Can be `None`, `layer_norm`, or `group_norm`. + group_size (`int`, *optional*, defaults to 32): + The number of groups to separate the channels into for group normalization. + """ + + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + dropout: float = 0.0, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + upcast_attention: bool = False, + temb_channels: int = 768, # for ada_group_norm + add_self_attention: bool = False, + cross_attention_norm: Optional[str] = None, + group_size: int = 32, + ): + super().__init__() + self.add_self_attention = add_self_attention + + # 1. Self-Attn + if add_self_attention: + self.norm1 = AdaGroupNorm(temb_channels, dim, max(1, dim // group_size)) + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=None, + cross_attention_norm=None, + ) + + # 2. Cross-Attn + self.norm2 = AdaGroupNorm(temb_channels, dim, max(1, dim // group_size)) + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + upcast_attention=upcast_attention, + cross_attention_norm=cross_attention_norm, + ) + + def _to_3d(self, hidden_states: torch.FloatTensor, height: int, weight: int) -> torch.FloatTensor: + return hidden_states.permute(0, 2, 3, 1).reshape(hidden_states.shape[0], height * weight, -1) + + def _to_4d(self, hidden_states: torch.FloatTensor, height: int, weight: int) -> torch.FloatTensor: + return hidden_states.permute(0, 2, 1).reshape(hidden_states.shape[0], -1, height, weight) + + def forward( + self, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + # TODO: mark emb as non-optional (self.norm2 requires it). + # requires assessing impact of change to positional param interface. + emb: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + + # 1. Self-Attention + if self.add_self_attention: + norm_hidden_states = self.norm1(hidden_states, emb) + + height, weight = norm_hidden_states.shape[2:] + norm_hidden_states = self._to_3d(norm_hidden_states, height, weight) + + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + attn_output = self._to_4d(attn_output, height, weight) + + hidden_states = attn_output + hidden_states + + # 2. Cross-Attention/None + norm_hidden_states = self.norm2(hidden_states, emb) + + height, weight = norm_hidden_states.shape[2:] + norm_hidden_states = self._to_3d(norm_hidden_states, height, weight) + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask if encoder_hidden_states is None else encoder_attention_mask, + **cross_attention_kwargs, + ) + attn_output = self._to_4d(attn_output, height, weight) + + hidden_states = attn_output + hidden_states + + return hidden_states diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/unet_2d_condition.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/unet_2d_condition.py new file mode 100644 index 0000000000000000000000000000000000000000..e74bd371a4ad59ce6e2eec81a8910aa7721f9afe --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/models/unet_2d_condition.py @@ -0,0 +1,1214 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Some modifications are reimplemented in public environments by Xiao Fu and Mu Hu + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.utils.checkpoint + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import UNet2DConditionLoadersMixin +from diffusers.utils import USE_PEFT_BACKEND, BaseOutput, deprecate, logging, scale_lora_layers, unscale_lora_layers +from diffusers.models.activations import get_activation +from diffusers.models.attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + Attention, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from diffusers.models.embeddings import ( + GaussianFourierProjection, + ImageHintTimeEmbedding, + ImageProjection, + ImageTimeEmbedding, + PositionNet, + TextImageProjection, + TextImageTimeEmbedding, + TextTimeEmbedding, + TimestepEmbedding, + Timesteps, +) +from diffusers.models.modeling_utils import ModelMixin + +from models.unet_2d_blocks import ( + UNetMidBlock2D, + UNetMidBlock2DCrossAttn, + UNetMidBlock2DSimpleCrossAttn, + get_down_block, + get_up_block, +) + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class UNet2DConditionOutput(BaseOutput): + """ + The output of [`UNet2DConditionModel`]. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + The hidden states output conditioned on `encoder_hidden_states` input. Output of last layer of model. + """ + + sample: torch.FloatTensor = None + + +class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin): + r""" + A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample + shaped output. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + Height and width of input/output sample. + in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 4): Number of channels in the output. + center_input_sample (`bool`, *optional*, defaults to `False`): Whether to center the input sample. + flip_sin_to_cos (`bool`, *optional*, defaults to `False`): + Whether to flip the sin to cos in the time embedding. + freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding. + down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): + The tuple of downsample blocks to use. + mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`): + Block type for middle of UNet, it can be one of `UNetMidBlock2DCrossAttn`, `UNetMidBlock2D`, or + `UNetMidBlock2DSimpleCrossAttn`. If `None`, the mid block layer is skipped. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): + The tuple of upsample blocks to use. + only_cross_attention(`bool` or `Tuple[bool]`, *optional*, default to `False`): + Whether to include self-attention in the basic transformer blocks, see + [`~models.attention.BasicTransformerBlock`]. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. + downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution. + mid_block_scale_factor (`float`, *optional*, defaults to 1.0): The scale factor to use for the mid block. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. + If `None`, normalization and activation layers is skipped in post-processing. + norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization. + cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280): + The dimension of the cross attention features. + transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple]` , *optional*, defaults to 1): + The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for + [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], + [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. + reverse_transformer_layers_per_block : (`Tuple[Tuple]`, *optional*, defaults to None): + The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`], in the upsampling + blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `Tuple[Tuple]` and for + [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], + [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. + encoder_hid_dim (`int`, *optional*, defaults to None): + If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim` + dimension to `cross_attention_dim`. + encoder_hid_dim_type (`str`, *optional*, defaults to `None`): + If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text + embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`. + attention_head_dim (`int`, *optional*, defaults to 8): The dimension of the attention heads. + num_attention_heads (`int`, *optional*): + The number of attention heads. If not defined, defaults to `attention_head_dim` + resnet_time_scale_shift (`str`, *optional*, defaults to `"default"`): Time scale shift config + for ResNet blocks (see [`~models.resnet.ResnetBlock2D`]). Choose from `default` or `scale_shift`. + class_embed_type (`str`, *optional*, defaults to `None`): + The type of class embedding to use which is ultimately summed with the time embeddings. Choose from `None`, + `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`. + addition_embed_type (`str`, *optional*, defaults to `None`): + Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or + "text". "text" will use the `TextTimeEmbedding` layer. + addition_time_embed_dim: (`int`, *optional*, defaults to `None`): + Dimension for the timestep embeddings. + num_class_embeds (`int`, *optional*, defaults to `None`): + Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing + class conditioning with `class_embed_type` equal to `None`. + time_embedding_type (`str`, *optional*, defaults to `positional`): + The type of position embedding to use for timesteps. Choose from `positional` or `fourier`. + time_embedding_dim (`int`, *optional*, defaults to `None`): + An optional override for the dimension of the projected time embedding. + time_embedding_act_fn (`str`, *optional*, defaults to `None`): + Optional activation function to use only once on the time embeddings before they are passed to the rest of + the UNet. Choose from `silu`, `mish`, `gelu`, and `swish`. + timestep_post_act (`str`, *optional*, defaults to `None`): + The second activation function to use in timestep embedding. Choose from `silu`, `mish` and `gelu`. + time_cond_proj_dim (`int`, *optional*, defaults to `None`): + The dimension of `cond_proj` layer in the timestep embedding. + conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer. conv_out_kernel (`int`, + *optional*, default to `3`): The kernel size of `conv_out` layer. projection_class_embeddings_input_dim (`int`, + *optional*): The dimension of the `class_labels` input when + `class_embed_type="projection"`. Required when `class_embed_type="projection"`. + class_embeddings_concat (`bool`, *optional*, defaults to `False`): Whether to concatenate the time + embeddings with the class embeddings. + mid_block_only_cross_attention (`bool`, *optional*, defaults to `None`): + Whether to use cross attention with the mid block when using the `UNetMidBlock2DSimpleCrossAttn`. If + `only_cross_attention` is given as a single boolean and `mid_block_only_cross_attention` is `None`, the + `only_cross_attention` value is used as the value for `mid_block_only_cross_attention`. Default to `False` + otherwise. + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + sample_size: Optional[int] = None, + in_channels: int = 4, + out_channels: int = 4, + center_input_sample: bool = False, + flip_sin_to_cos: bool = True, + freq_shift: int = 0, + down_block_types: Tuple[str] = ( + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "DownBlock2D", + ), + mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn", + up_block_types: Tuple[str] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + only_cross_attention: Union[bool, Tuple[bool]] = False, + block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + layers_per_block: Union[int, Tuple[int]] = 2, + downsample_padding: int = 1, + mid_block_scale_factor: float = 1, + dropout: float = 0.0, + act_fn: str = "silu", + norm_num_groups: Optional[int] = 32, + norm_eps: float = 1e-5, + cross_attention_dim: Union[int, Tuple[int]] = 1280, + transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1, + reverse_transformer_layers_per_block: Optional[Tuple[Tuple[int]]] = None, + encoder_hid_dim: Optional[int] = None, + encoder_hid_dim_type: Optional[str] = None, + attention_head_dim: Union[int, Tuple[int]] = 8, + num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + class_embed_type: Optional[str] = None, + addition_embed_type: Optional[str] = None, + addition_time_embed_dim: Optional[int] = None, + num_class_embeds: Optional[int] = None, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + resnet_skip_time_act: bool = False, + resnet_out_scale_factor: int = 1.0, + time_embedding_type: str = "positional", + time_embedding_dim: Optional[int] = None, + time_embedding_act_fn: Optional[str] = None, + timestep_post_act: Optional[str] = None, + time_cond_proj_dim: Optional[int] = None, + conv_in_kernel: int = 3, + conv_out_kernel: int = 3, + projection_class_embeddings_input_dim: Optional[int] = None, + attention_type: str = "default", + class_embeddings_concat: bool = False, + mid_block_only_cross_attention: Optional[bool] = None, + cross_attention_norm: Optional[str] = None, + addition_embed_type_num_heads=64, + ): + super().__init__() + + self.sample_size = sample_size + + if num_attention_heads is not None: + raise ValueError( + "At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19." + ) + + # If `num_attention_heads` is not defined (which is the case for most models) + # it will default to `attention_head_dim`. This looks weird upon first reading it and it is. + # The reason for this behavior is to correct for incorrectly named variables that were introduced + # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131 + # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking + # which is why we correct for the naming here. + num_attention_heads = num_attention_heads or attention_head_dim + + # Check inputs + if len(down_block_types) != len(up_block_types): + raise ValueError( + f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}." + ) + + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(only_cross_attention, bool) and len(only_cross_attention) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(attention_head_dim, int) and len(attention_head_dim) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `attention_head_dim` as `down_block_types`. `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}." + ) + + if isinstance(cross_attention_dim, list) and len(cross_attention_dim) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `cross_attention_dim` as `down_block_types`. `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(layers_per_block, int) and len(layers_per_block) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `layers_per_block` as `down_block_types`. `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}." + ) + if isinstance(transformer_layers_per_block, list) and reverse_transformer_layers_per_block is None: + for layer_number_per_block in transformer_layers_per_block: + if isinstance(layer_number_per_block, list): + raise ValueError("Must provide 'reverse_transformer_layers_per_block` if using asymmetrical UNet.") + + # input + conv_in_padding = (conv_in_kernel - 1) // 2 + self.conv_in = nn.Conv2d( + in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding + ) + + # time + if time_embedding_type == "fourier": + time_embed_dim = time_embedding_dim or block_out_channels[0] * 2 + if time_embed_dim % 2 != 0: + raise ValueError(f"`time_embed_dim` should be divisible by 2, but is {time_embed_dim}.") + self.time_proj = GaussianFourierProjection( + time_embed_dim // 2, set_W_to_weight=False, log=False, flip_sin_to_cos=flip_sin_to_cos + ) + timestep_input_dim = time_embed_dim + elif time_embedding_type == "positional": + time_embed_dim = time_embedding_dim or block_out_channels[0] * 4 + + self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift) + timestep_input_dim = block_out_channels[0] + else: + raise ValueError( + f"{time_embedding_type} does not exist. Please make sure to use one of `fourier` or `positional`." + ) + + self.time_embedding = TimestepEmbedding( + timestep_input_dim, + time_embed_dim, + act_fn=act_fn, + post_act_fn=timestep_post_act, + cond_proj_dim=time_cond_proj_dim, + ) + + if encoder_hid_dim_type is None and encoder_hid_dim is not None: + encoder_hid_dim_type = "text_proj" + self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type) + logger.info("encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined.") + + if encoder_hid_dim is None and encoder_hid_dim_type is not None: + raise ValueError( + f"`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}." + ) + + if encoder_hid_dim_type == "text_proj": + self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim) + elif encoder_hid_dim_type == "text_image_proj": + # image_embed_dim DOESN'T have to be `cross_attention_dim`. To not clutter the __init__ too much + # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use + # case when `addition_embed_type == "text_image_proj"` (Kadinsky 2.1)` + self.encoder_hid_proj = TextImageProjection( + text_embed_dim=encoder_hid_dim, + image_embed_dim=cross_attention_dim, + cross_attention_dim=cross_attention_dim, + ) + elif encoder_hid_dim_type == "image_proj": + # Kandinsky 2.2 + self.encoder_hid_proj = ImageProjection( + image_embed_dim=encoder_hid_dim, + cross_attention_dim=cross_attention_dim, + ) + elif encoder_hid_dim_type is not None: + raise ValueError( + f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'." + ) + else: + self.encoder_hid_proj = None + + # class embedding + if class_embed_type is None and num_class_embeds is not None: + self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim) + elif class_embed_type == "timestep": + self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim, act_fn=act_fn) + elif class_embed_type == "identity": + self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim) + elif class_embed_type == "projection": + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set" + ) + # The projection `class_embed_type` is the same as the timestep `class_embed_type` except + # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings + # 2. it projects from an arbitrary input dimension. + # + # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations. + # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings. + # As a result, `TimestepEmbedding` can be passed arbitrary vectors. + self.class_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim) + elif class_embed_type == "simple_projection": + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'simple_projection' requires `projection_class_embeddings_input_dim` be set" + ) + self.class_embedding = nn.Linear(projection_class_embeddings_input_dim, time_embed_dim) + else: + self.class_embedding = None + + if addition_embed_type == "text": + if encoder_hid_dim is not None: + text_time_embedding_from_dim = encoder_hid_dim + else: + text_time_embedding_from_dim = cross_attention_dim + + self.add_embedding = TextTimeEmbedding( + text_time_embedding_from_dim, time_embed_dim, num_heads=addition_embed_type_num_heads + ) + elif addition_embed_type == "text_image": + # text_embed_dim and image_embed_dim DON'T have to be `cross_attention_dim`. To not clutter the __init__ too much + # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use + # case when `addition_embed_type == "text_image"` (Kadinsky 2.1)` + self.add_embedding = TextImageTimeEmbedding( + text_embed_dim=cross_attention_dim, image_embed_dim=cross_attention_dim, time_embed_dim=time_embed_dim + ) + elif addition_embed_type == "text_time": + self.add_time_proj = Timesteps(addition_time_embed_dim, flip_sin_to_cos, freq_shift) + self.add_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim) + elif addition_embed_type == "image": + # Kandinsky 2.2 + self.add_embedding = ImageTimeEmbedding(image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim) + elif addition_embed_type == "image_hint": + # Kandinsky 2.2 ControlNet + self.add_embedding = ImageHintTimeEmbedding(image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim) + elif addition_embed_type is not None: + raise ValueError(f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'.") + + if time_embedding_act_fn is None: + self.time_embed_act = None + else: + self.time_embed_act = get_activation(time_embedding_act_fn) + + self.down_blocks = nn.ModuleList([]) + self.up_blocks = nn.ModuleList([]) + + if isinstance(only_cross_attention, bool): + if mid_block_only_cross_attention is None: + mid_block_only_cross_attention = only_cross_attention + + only_cross_attention = [only_cross_attention] * len(down_block_types) + + if mid_block_only_cross_attention is None: + mid_block_only_cross_attention = False + + if isinstance(num_attention_heads, int): + num_attention_heads = (num_attention_heads,) * len(down_block_types) + + if isinstance(attention_head_dim, int): + attention_head_dim = (attention_head_dim,) * len(down_block_types) + + if isinstance(cross_attention_dim, int): + cross_attention_dim = (cross_attention_dim,) * len(down_block_types) + + if isinstance(layers_per_block, int): + layers_per_block = [layers_per_block] * len(down_block_types) + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * len(down_block_types) + + if class_embeddings_concat: + # The time embeddings are concatenated with the class embeddings. The dimension of the + # time embeddings passed to the down, middle, and up blocks is twice the dimension of the + # regular time embeddings + blocks_time_embed_dim = time_embed_dim * 2 + else: + blocks_time_embed_dim = time_embed_dim + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block[i], + transformer_layers_per_block=transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + temb_channels=blocks_time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim[i], + num_attention_heads=num_attention_heads[i], + downsample_padding=downsample_padding, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + attention_head_dim=attention_head_dim[i] if attention_head_dim[i] is not None else output_channel, + dropout=dropout, + ) + self.down_blocks.append(down_block) + + # mid + if mid_block_type == "UNetMidBlock2DCrossAttn": + self.mid_block = UNetMidBlock2DCrossAttn( + transformer_layers_per_block=transformer_layers_per_block[-1], + in_channels=block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + dropout=dropout, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_time_scale_shift=resnet_time_scale_shift, + cross_attention_dim=cross_attention_dim[-1], + num_attention_heads=num_attention_heads[-1], + resnet_groups=norm_num_groups, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + elif mid_block_type == "UNetMidBlock2DSimpleCrossAttn": + self.mid_block = UNetMidBlock2DSimpleCrossAttn( + in_channels=block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + dropout=dropout, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + cross_attention_dim=cross_attention_dim[-1], + attention_head_dim=attention_head_dim[-1], + resnet_groups=norm_num_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + only_cross_attention=mid_block_only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif mid_block_type == "UNetMidBlock2D": + self.mid_block = UNetMidBlock2D( + in_channels=block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + dropout=dropout, + num_layers=0, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_groups=norm_num_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + add_attention=False, + ) + elif mid_block_type is None: + self.mid_block = None + else: + raise ValueError(f"unknown mid_block_type : {mid_block_type}") + + # count how many layers upsample the images + self.num_upsamplers = 0 + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_num_attention_heads = list(reversed(num_attention_heads)) + reversed_layers_per_block = list(reversed(layers_per_block)) + reversed_cross_attention_dim = list(reversed(cross_attention_dim)) + reversed_transformer_layers_per_block = ( + list(reversed(transformer_layers_per_block)) + if reverse_transformer_layers_per_block is None + else reverse_transformer_layers_per_block + ) + only_cross_attention = list(reversed(only_cross_attention)) + + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + is_final_block = i == len(block_out_channels) - 1 + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)] + + # add upsample block for all BUT final layer + if not is_final_block: + add_upsample = True + self.num_upsamplers += 1 + else: + add_upsample = False + + up_block = get_up_block( + up_block_type, + num_layers=reversed_layers_per_block[i] + 1, + transformer_layers_per_block=reversed_transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=blocks_time_embed_dim, + add_upsample=add_upsample, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resolution_idx=i, + resnet_groups=norm_num_groups, + cross_attention_dim=reversed_cross_attention_dim[i], + num_attention_heads=reversed_num_attention_heads[i], + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + attention_head_dim=attention_head_dim[i] if attention_head_dim[i] is not None else output_channel, + dropout=dropout, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_num_groups is not None: + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps + ) + + self.conv_act = get_activation(act_fn) + + else: + self.conv_norm_out = None + self.conv_act = None + + conv_out_padding = (conv_out_kernel - 1) // 2 + self.conv_out = nn.Conv2d( + block_out_channels[0], out_channels, kernel_size=conv_out_kernel, padding=conv_out_padding + ) + + if attention_type in ["gated", "gated-text-image"]: + positive_len = 768 + if isinstance(cross_attention_dim, int): + positive_len = cross_attention_dim + elif isinstance(cross_attention_dim, tuple) or isinstance(cross_attention_dim, list): + positive_len = cross_attention_dim[0] + + feature_type = "text-only" if attention_type == "gated" else "text-image" + self.position_net = PositionNet( + positive_len=positive_len, out_dim=cross_attention_dim, feature_type=feature_type + ) + + @property + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + def set_attn_processor( + self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]], _remove_lora=False + ): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor, _remove_lora=_remove_lora) + else: + module.set_processor(processor.pop(f"{name}.processor"), _remove_lora=_remove_lora) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor, _remove_lora=True) + + def set_attention_slice(self, slice_size): + r""" + Enable sliced attention computation. + + When this option is enabled, the attention module splits the input tensor in slices to compute attention in + several steps. This is useful for saving some memory in exchange for a small decrease in speed. + + Args: + slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`): + When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If + `"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is + provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim` + must be a multiple of `slice_size`. + """ + sliceable_head_dims = [] + + def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): + if hasattr(module, "set_attention_slice"): + sliceable_head_dims.append(module.sliceable_head_dim) + + for child in module.children(): + fn_recursive_retrieve_sliceable_dims(child) + + # retrieve number of attention layers + for module in self.children(): + fn_recursive_retrieve_sliceable_dims(module) + + num_sliceable_layers = len(sliceable_head_dims) + + if slice_size == "auto": + # half the attention head size is usually a good trade-off between + # speed and memory + slice_size = [dim // 2 for dim in sliceable_head_dims] + elif slice_size == "max": + # make smallest slice possible + slice_size = num_sliceable_layers * [1] + + slice_size = num_sliceable_layers * [slice_size] if not isinstance(slice_size, list) else slice_size + + if len(slice_size) != len(sliceable_head_dims): + raise ValueError( + f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different" + f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}." + ) + + for i in range(len(slice_size)): + size = slice_size[i] + dim = sliceable_head_dims[i] + if size is not None and size > dim: + raise ValueError(f"size {size} has to be smaller or equal to {dim}.") + + # Recursively walk through all the children. + # Any children which exposes the set_attention_slice method + # gets the message + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + if hasattr(module, "set_attention_slice"): + module.set_attention_slice(slice_size.pop()) + + for child in module.children(): + fn_recursive_set_attention_slice(child, slice_size) + + reversed_slice_size = list(reversed(slice_size)) + for module in self.children(): + fn_recursive_set_attention_slice(module, reversed_slice_size) + + def _set_gradient_checkpointing(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + + def enable_freeu(self, s1, s2, b1, b2): + r"""Enables the FreeU mechanism from https://arxiv.org/abs/2309.11497. + + The suffixes after the scaling factors represent the stage blocks where they are being applied. + + Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of values that + are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL. + + Args: + s1 (`float`): + Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + s2 (`float`): + Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. + b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. + """ + for i, upsample_block in enumerate(self.up_blocks): + setattr(upsample_block, "s1", s1) + setattr(upsample_block, "s2", s2) + setattr(upsample_block, "b1", b1) + setattr(upsample_block, "b2", b2) + + def disable_freeu(self): + """Disables the FreeU mechanism.""" + freeu_keys = {"s1", "s2", "b1", "b2"} + for i, upsample_block in enumerate(self.up_blocks): + for k in freeu_keys: + if hasattr(upsample_block, k) or getattr(upsample_block, k, None) is not None: + setattr(upsample_block, k, None) + + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, + key, value) are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is 🧪 experimental. + + + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.") + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + def unfuse_qkv_projections(self): + """Disables the fused QKV projection if enabled. + + + + This API is 🧪 experimental. + + + + """ + if self.original_attn_processors is not None: + self.set_attn_processor(self.original_attn_processors) + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + mid_block_additional_residual: Optional[torch.Tensor] = None, + down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[UNet2DConditionOutput, Tuple]: + r""" + The [`UNet2DConditionModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor with the following shape `(batch, channel, height, width)`. + timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input. + encoder_hidden_states (`torch.FloatTensor`): + The encoder hidden states with shape `(batch, sequence_length, feature_dim)`. + class_labels (`torch.Tensor`, *optional*, defaults to `None`): + Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings. + timestep_cond: (`torch.Tensor`, *optional*, defaults to `None`): + Conditional embeddings for timestep. If provided, the embeddings will be summed with the samples passed + through the `self.time_embedding` layer to obtain the timestep embeddings. + attention_mask (`torch.Tensor`, *optional*, defaults to `None`): + An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask + is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large + negative values to the attention scores corresponding to "discard" tokens. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + added_cond_kwargs: (`dict`, *optional*): + A kwargs dictionary containing additional embeddings that if specified are added to the embeddings that + are passed along to the UNet blocks. + down_block_additional_residuals: (`tuple` of `torch.Tensor`, *optional*): + A tuple of tensors that if specified are added to the residuals of down unet blocks. + mid_block_additional_residual: (`torch.Tensor`, *optional*): + A tensor that if specified is added to the residual of the middle unet block. + encoder_attention_mask (`torch.Tensor`): + A cross-attention mask of shape `(batch, sequence_length)` is applied to `encoder_hidden_states`. If + `True` the mask is kept, otherwise if `False` it is discarded. Mask will be converted into a bias, + which adds large negative values to the attention scores corresponding to "discard" tokens. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unet_2d_condition.UNet2DConditionOutput`] instead of a plain + tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttnProcessor`]. + added_cond_kwargs: (`dict`, *optional*): + A kwargs dictionary containin additional embeddings that if specified are added to the embeddings that + are passed along to the UNet blocks. + down_block_additional_residuals (`tuple` of `torch.Tensor`, *optional*): + additional residuals to be added to UNet long skip connections from down blocks to up blocks for + example from ControlNet side model(s) + mid_block_additional_residual (`torch.Tensor`, *optional*): + additional residual to be added to UNet mid block output, for example from ControlNet side model + down_intrablock_additional_residuals (`tuple` of `torch.Tensor`, *optional*): + additional residuals to be added within UNet down blocks, for example from T2I-Adapter side model(s) + + Returns: + [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`: + If `return_dict` is True, an [`~models.unet_2d_condition.UNet2DConditionOutput`] is returned, otherwise + a `tuple` is returned where the first element is the sample tensor. + """ + + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layers). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2**self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + for dim in sample.shape[-2:]: + if dim % default_overall_up_factor != 0: + # Forward upsample size to force interpolation output size. + forward_upsample_size = True + break + + # ensure attention_mask is a bias, and give it a singleton query_tokens dimension + # expects mask of shape: + # [batch, key_tokens] + # adds singleton query_tokens dimension: + # [batch, 1, key_tokens] + # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes: + # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn) + # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn) + if attention_mask is not None: + # assume that mask is expressed as: + # (1 = keep, 0 = discard) + # convert mask into a bias that can be added to attention scores: + # (keep = +0, discard = -10000.0) + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # convert encoder_attention_mask to a bias the same way we do for attention_mask + if encoder_attention_mask is not None: + encoder_attention_mask = (1 - encoder_attention_mask.to(sample.dtype)) * -10000.0 + encoder_attention_mask = encoder_attention_mask.unsqueeze(1) + + # 0. center input if necessary + if self.config.center_input_sample: + sample = 2 * sample - 1.0 + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == "mps" + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=sample.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + aug_emb = None + + if self.class_embedding is not None: + if class_labels is None: + raise ValueError("class_labels should be provided when num_class_embeds > 0") + + if self.config.class_embed_type == "timestep": + class_labels = self.time_proj(class_labels) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # there might be better ways to encapsulate this. + class_labels = class_labels.to(dtype=sample.dtype) + + class_emb = self.class_embedding(class_labels).to(dtype=sample.dtype) + + if self.config.class_embeddings_concat: + emb = torch.cat([emb, class_emb], dim=-1) + else: + emb = emb + class_emb + + if self.config.addition_embed_type == "text": + aug_emb = self.add_embedding(encoder_hidden_states) + elif self.config.addition_embed_type == "text_image": + # Kandinsky 2.1 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`" + ) + + image_embs = added_cond_kwargs.get("image_embeds") + text_embs = added_cond_kwargs.get("text_embeds", encoder_hidden_states) + aug_emb = self.add_embedding(text_embs, image_embs) + elif self.config.addition_embed_type == "text_time": + # SDXL - style + if "text_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`" + ) + text_embeds = added_cond_kwargs.get("text_embeds") + if "time_ids" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`" + ) + time_ids = added_cond_kwargs.get("time_ids") + time_embeds = self.add_time_proj(time_ids.flatten()) + time_embeds = time_embeds.reshape((text_embeds.shape[0], -1)) + add_embeds = torch.concat([text_embeds, time_embeds], dim=-1) + add_embeds = add_embeds.to(emb.dtype) + aug_emb = self.add_embedding(add_embeds) + elif self.config.addition_embed_type == "image": + # Kandinsky 2.2 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`" + ) + image_embs = added_cond_kwargs.get("image_embeds") + aug_emb = self.add_embedding(image_embs) + elif self.config.addition_embed_type == "image_hint": + # Kandinsky 2.2 - style + if "image_embeds" not in added_cond_kwargs or "hint" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'image_hint' which requires the keyword arguments `image_embeds` and `hint` to be passed in `added_cond_kwargs`" + ) + image_embs = added_cond_kwargs.get("image_embeds") + hint = added_cond_kwargs.get("hint") + aug_emb, hint = self.add_embedding(image_embs, hint) + sample = torch.cat([sample, hint], dim=1) + + emb = emb + aug_emb if aug_emb is not None else emb + + if self.time_embed_act is not None: + emb = self.time_embed_act(emb) + + if self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "text_proj": + encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states) + elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "text_image_proj": + # Kadinsky 2.1 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'text_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + + image_embeds = added_cond_kwargs.get("image_embeds") + encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states, image_embeds) + elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "image_proj": + # Kandinsky 2.2 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + image_embeds = added_cond_kwargs.get("image_embeds") + encoder_hidden_states = self.encoder_hid_proj(image_embeds) + elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "ip_image_proj": + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'ip_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + image_embeds = added_cond_kwargs.get("image_embeds") + image_embeds = self.encoder_hid_proj(image_embeds).to(encoder_hidden_states.dtype) + encoder_hidden_states = torch.cat([encoder_hidden_states, image_embeds], dim=1) + + # 2. pre-process + sample = self.conv_in(sample) + + # 2.5 GLIGEN position net + if cross_attention_kwargs is not None and cross_attention_kwargs.get("gligen", None) is not None: + cross_attention_kwargs = cross_attention_kwargs.copy() + gligen_args = cross_attention_kwargs.pop("gligen") + cross_attention_kwargs["gligen"] = {"objs": self.position_net(**gligen_args)} + + # 3. down + lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + if USE_PEFT_BACKEND: + # weight the lora layers by setting `lora_scale` for each PEFT layer + scale_lora_layers(self, lora_scale) + + is_controlnet = mid_block_additional_residual is not None and down_block_additional_residuals is not None + # using new arg down_intrablock_additional_residuals for T2I-Adapters, to distinguish from controlnets + is_adapter = down_intrablock_additional_residuals is not None + # maintain backward compatibility for legacy usage, where + # T2I-Adapter and ControlNet both use down_block_additional_residuals arg + # but can only use one or the other + if not is_adapter and mid_block_additional_residual is None and down_block_additional_residuals is not None: + deprecate( + "T2I should not use down_block_additional_residuals", + "1.3.0", + "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated \ + and will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used \ + for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ", + standard_warn=False, + ) + down_intrablock_additional_residuals = down_block_additional_residuals + is_adapter = True + + down_block_res_samples = (sample,) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: + # For t2i-adapter CrossAttnDownBlock2D + additional_residuals = {} + if is_adapter and len(down_intrablock_additional_residuals) > 0: + additional_residuals["additional_residuals"] = down_intrablock_additional_residuals.pop(0) + + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + **additional_residuals, + ) + else: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb, scale=lora_scale) + if is_adapter and len(down_intrablock_additional_residuals) > 0: + sample += down_intrablock_additional_residuals.pop(0) + + down_block_res_samples += res_samples + + if is_controlnet: + new_down_block_res_samples = () + + for down_block_res_sample, down_block_additional_residual in zip( + down_block_res_samples, down_block_additional_residuals + ): + down_block_res_sample = down_block_res_sample + down_block_additional_residual + new_down_block_res_samples = new_down_block_res_samples + (down_block_res_sample,) + + down_block_res_samples = new_down_block_res_samples + + # 4. mid + if self.mid_block is not None: + if hasattr(self.mid_block, "has_cross_attention") and self.mid_block.has_cross_attention: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + else: + sample = self.mid_block(sample, emb) + + # To support T2I-Adapter-XL + if ( + is_adapter + and len(down_intrablock_additional_residuals) > 0 + and sample.shape == down_intrablock_additional_residuals[0].shape + ): + sample += down_intrablock_additional_residuals.pop(0) + + if is_controlnet: + sample = sample + mid_block_additional_residual + + # 5. up + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets) :] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + upsample_size=upsample_size, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + ) + else: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + upsample_size=upsample_size, + scale=lora_scale, + ) + + # 6. post-process + if self.conv_norm_out: + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + if USE_PEFT_BACKEND: + # remove `lora_scale` from each PEFT layer + unscale_lora_layers(self, lora_scale) + + if not return_dict: + return (sample,) + + return UNet2DConditionOutput(sample=sample) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/run_infer.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/run_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..19b71f9d71a91fccd3a2430f5715ee8c1a98a88e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/run_infer.py @@ -0,0 +1,236 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +import argparse +import os +import logging + +import numpy as np +import torch +from PIL import Image +from tqdm.auto import tqdm +import glob +import json +import cv2 + +import sys +from models.geowizard_pipeline import DepthNormalEstimationPipeline +from utils.seed_all import seed_all +import matplotlib.pyplot as plt +from utils.depth2normal import * + +from diffusers import DiffusionPipeline, DDIMScheduler, AutoencoderKL +from models.unet_2d_condition import UNet2DConditionModel + +from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection +import torchvision.transforms.functional as TF +from torchvision.transforms import InterpolationMode + +if __name__=="__main__": + + logging.basicConfig(level=logging.INFO) + + '''Set the Args''' + parser = argparse.ArgumentParser( + description="Run MonoDepthNormal Estimation using Stable Diffusion." + ) + parser.add_argument( + "--pretrained_model_path", + type=str, + default='lemonaddie/geowizard', + help="pretrained model path from hugging face or local dir", + ) + parser.add_argument( + "--input_dir", type=str, required=True, help="Input directory." + ) + + parser.add_argument( + "--output_dir", type=str, required=True, help="Output directory." + ) + parser.add_argument( + "--domain", + type=str, + default='indoor', + required=True, + help="domain prediction", + ) + + # inference setting + parser.add_argument( + "--denoise_steps", + type=int, + default=10, + help="Diffusion denoising steps, more steps results in higher accuracy but slower inference speed.", + ) + parser.add_argument( + "--ensemble_size", + type=int, + default=10, + help="Number of predictions to be ensembled, more inference gives better results but runs slower.", + ) + parser.add_argument( + "--half_precision", + action="store_true", + help="Run with half-precision (16-bit float), might lead to suboptimal result.", + ) + + # resolution setting + parser.add_argument( + "--processing_res", + type=int, + default=768, + help="Maximum resolution of processing. 0 for using input image resolution. Default: 768.", + ) + parser.add_argument( + "--output_processing_res", + action="store_true", + help="When input is resized, out put depth at resized operating resolution. Default: False.", + ) + + # depth map colormap + parser.add_argument( + "--color_map", + type=str, + default="Spectral", + help="Colormap used to render depth predictions.", + ) + # other settings + parser.add_argument("--seed", type=int, default=None, help="Random seed.") + parser.add_argument( + "--batch_size", + type=int, + default=0, + help="Inference batch size. Default: 0 (will be set automatically).", + ) + + args = parser.parse_args() + + checkpoint_path = args.pretrained_model_path + output_dir = args.output_dir + denoise_steps = args.denoise_steps + ensemble_size = args.ensemble_size + + if ensemble_size>15: + logging.warning("long ensemble steps, low speed..") + + half_precision = args.half_precision + + processing_res = args.processing_res + match_input_res = not args.output_processing_res + domain = args.domain + + color_map = args.color_map + seed = args.seed + batch_size = args.batch_size + + if batch_size==0: + batch_size = 1 # set default batchsize + + # -------------------- Preparation -------------------- + # Random seed + if seed is None: + import time + seed = int(time.time()) + seed_all(seed) + + # Output directories + output_dir_color = os.path.join(output_dir, "depth_colored") + output_dir_npy = os.path.join(output_dir, "depth_npy") + output_dir_normal_npy = os.path.join(output_dir, "normal_npy") + output_dir_normal_color = os.path.join(output_dir, "normal_colored") + os.makedirs(output_dir, exist_ok=True) + os.makedirs(output_dir_color, exist_ok=True) + os.makedirs(output_dir_npy, exist_ok=True) + os.makedirs(output_dir_normal_npy, exist_ok=True) + os.makedirs(output_dir_normal_color, exist_ok=True) + logging.info(f"output dir = {output_dir}") + + # -------------------- Device -------------------- + if torch.cuda.is_available(): + device = torch.device("cuda") + else: + device = torch.device("cpu") + logging.warning("CUDA is not available. Running on CPU will be slow.") + logging.info(f"device = {device}") + + # -------------------- Data -------------------- + input_dir = args.input_dir + test_files = sorted(os.listdir(input_dir)) + n_images = len(test_files) + if n_images > 0: + logging.info(f"Found {n_images} images") + else: + logging.error(f"No image found") + exit(1) + + # -------------------- Model -------------------- + if half_precision: + dtype = torch.float16 + logging.info(f"Running with half precision ({dtype}).") + else: + dtype = torch.float32 + + # declare a pipeline + pipe = DepthNormalEstimationPipeline.from_pretrained(checkpoint_path, torch_dtype=dtype) + + logging.info("loading pipeline whole successfully.") + + try: + pipe.enable_xformers_memory_efficient_attention() + except: + pass # run without xformers + + pipe = pipe.to(device) + + # -------------------- Inference and saving -------------------- + with torch.no_grad(): + os.makedirs(output_dir, exist_ok=True) + + for test_file in tqdm(test_files, desc="Estimating Depth & Normal", leave=True): + rgb_path = os.path.join(input_dir, test_file) + + # Read input image + input_image = Image.open(rgb_path) + + # predict the depth here + pipe_out = pipe(input_image, + denoising_steps = denoise_steps, + ensemble_size= ensemble_size, + processing_res = processing_res, + match_input_res = match_input_res, + domain = domain, + color_map = color_map, + show_progress_bar = True, + ) + + depth_pred: np.ndarray = pipe_out.depth_np + depth_colored: Image.Image = pipe_out.depth_colored + normal_pred: np.ndarray = pipe_out.normal_np + normal_colored: Image.Image = pipe_out.normal_colored + + # Save as npy + rgb_name_base = os.path.splitext(os.path.basename(rgb_path))[0] + pred_name_base = rgb_name_base + "_pred" + npy_save_path = os.path.join(output_dir_npy, f"{pred_name_base}.npy") + if os.path.exists(npy_save_path): + logging.warning(f"Existing file: '{npy_save_path}' will be overwritten") + np.save(npy_save_path, depth_pred) + + normal_npy_save_path = os.path.join(output_dir_normal_npy, f"{pred_name_base}.npy") + if os.path.exists(normal_npy_save_path): + logging.warning(f"Existing file: '{normal_npy_save_path}' will be overwritten") + np.save(normal_npy_save_path, normal_pred) + + # Colorize + depth_colored_save_path = os.path.join(output_dir_color, f"{pred_name_base}_colored.png") + if os.path.exists(depth_colored_save_path): + logging.warning( + f"Existing file: '{depth_colored_save_path}' will be overwritten" + ) + depth_colored.save(depth_colored_save_path) + + normal_colored_save_path = os.path.join(output_dir_normal_color, f"{pred_name_base}_colored.png") + if os.path.exists(normal_colored_save_path): + logging.warning( + f"Existing file: '{normal_colored_save_path}' will be overwritten" + ) + normal_colored.save(normal_colored_save_path) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/batch_size.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/batch_size.py new file mode 100644 index 0000000000000000000000000000000000000000..c7ba2aa12bbc928da1dce37a58e3f525475754cc --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/batch_size.py @@ -0,0 +1,63 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +import torch +import math + + +# Search table for suggested max. inference batch size +bs_search_table = [ + # tested on A100-PCIE-80GB + {"res": 768, "total_vram": 79, "bs": 35, "dtype": torch.float32}, + {"res": 1024, "total_vram": 79, "bs": 20, "dtype": torch.float32}, + # tested on A100-PCIE-40GB + {"res": 768, "total_vram": 39, "bs": 15, "dtype": torch.float32}, + {"res": 1024, "total_vram": 39, "bs": 8, "dtype": torch.float32}, + {"res": 768, "total_vram": 39, "bs": 30, "dtype": torch.float16}, + {"res": 1024, "total_vram": 39, "bs": 15, "dtype": torch.float16}, + # tested on RTX3090, RTX4090 + {"res": 512, "total_vram": 23, "bs": 20, "dtype": torch.float32}, + {"res": 768, "total_vram": 23, "bs": 7, "dtype": torch.float32}, + {"res": 1024, "total_vram": 23, "bs": 3, "dtype": torch.float32}, + {"res": 512, "total_vram": 23, "bs": 40, "dtype": torch.float16}, + {"res": 768, "total_vram": 23, "bs": 18, "dtype": torch.float16}, + {"res": 1024, "total_vram": 23, "bs": 10, "dtype": torch.float16}, + # tested on GTX1080Ti + {"res": 512, "total_vram": 10, "bs": 5, "dtype": torch.float32}, + {"res": 768, "total_vram": 10, "bs": 2, "dtype": torch.float32}, + {"res": 512, "total_vram": 10, "bs": 10, "dtype": torch.float16}, + {"res": 768, "total_vram": 10, "bs": 5, "dtype": torch.float16}, + {"res": 1024, "total_vram": 10, "bs": 3, "dtype": torch.float16}, +] + + +def find_batch_size(ensemble_size: int, input_res: int, dtype: torch.dtype) -> int: + """ + Automatically search for suitable operating batch size. + + Args: + ensemble_size (`int`): + Number of predictions to be ensembled. + input_res (`int`): + Operating resolution of the input image. + + Returns: + `int`: Operating batch size. + """ + if not torch.cuda.is_available(): + return 1 + + total_vram = torch.cuda.mem_get_info()[1] / 1024.0**3 + filtered_bs_search_table = [s for s in bs_search_table if s["dtype"] == dtype] + for settings in sorted( + filtered_bs_search_table, + key=lambda k: (k["res"], -k["total_vram"]), + ): + if input_res <= settings["res"] and total_vram >= settings["total_vram"]: + bs = settings["bs"] + if bs > ensemble_size: + bs = ensemble_size + elif bs > math.ceil(ensemble_size / 2) and bs < ensemble_size: + bs = math.ceil(ensemble_size / 2) + return bs + + return 1 \ No newline at end of file diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/colormap.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/colormap.py new file mode 100644 index 0000000000000000000000000000000000000000..e2f0905b51941fdcc189f13c0038c9c00079657e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/colormap.py @@ -0,0 +1,45 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +import numpy as np +import cv2 + +def kitti_colormap(disparity, maxval=-1): + """ + A utility function to reproduce KITTI fake colormap + Arguments: + - disparity: numpy float32 array of dimension HxW + - maxval: maximum disparity value for normalization (if equal to -1, the maximum value in disparity will be used) + + Returns a numpy uint8 array of shape HxWx3. + """ + if maxval < 0: + maxval = np.max(disparity) + + colormap = np.asarray([[0,0,0,114],[0,0,1,185],[1,0,0,114],[1,0,1,174],[0,1,0,114],[0,1,1,185],[1,1,0,114],[1,1,1,0]]) + weights = np.asarray([8.771929824561404,5.405405405405405,8.771929824561404,5.747126436781609,8.771929824561404,5.405405405405405,8.771929824561404,0]) + cumsum = np.asarray([0,0.114,0.299,0.413,0.587,0.701,0.8859999999999999,0.9999999999999999]) + + colored_disp = np.zeros([disparity.shape[0], disparity.shape[1], 3]) + values = np.expand_dims(np.minimum(np.maximum(disparity/maxval, 0.), 1.), -1) + bins = np.repeat(np.repeat(np.expand_dims(np.expand_dims(cumsum,axis=0),axis=0), disparity.shape[1], axis=1), disparity.shape[0], axis=0) + diffs = np.where((np.repeat(values, 8, axis=-1) - bins) > 0, -1000, (np.repeat(values, 8, axis=-1) - bins)) + index = np.argmax(diffs, axis=-1)-1 + + w = 1-(values[:,:,0]-cumsum[index])*np.asarray(weights)[index] + + + colored_disp[:,:,2] = (w*colormap[index][:,:,0] + (1.-w)*colormap[index+1][:,:,0]) + colored_disp[:,:,1] = (w*colormap[index][:,:,1] + (1.-w)*colormap[index+1][:,:,1]) + colored_disp[:,:,0] = (w*colormap[index][:,:,2] + (1.-w)*colormap[index+1][:,:,2]) + + return (colored_disp*np.expand_dims((disparity>0),-1)*255).astype(np.uint8) + +def read_16bit_gt(path): + """ + A utility function to read KITTI 16bit gt + Arguments: + - path: filepath + Returns a numpy float32 array of shape HxW. + """ + gt = cv2.imread(path,-1).astype(np.float32)/256. + return gt \ No newline at end of file diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/common.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/common.py new file mode 100644 index 0000000000000000000000000000000000000000..fe476a1f601a0a4c76d520d8897222dff05cbdd1 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/common.py @@ -0,0 +1,42 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +import json +import yaml +import logging +import os +import numpy as np +import sys + +def load_loss_scheme(loss_config): + with open(loss_config, 'r') as f: + loss_json = yaml.safe_load(f) + return loss_json + + +DEBUG =0 +logger = logging.getLogger() + + +if DEBUG: + #coloredlogs.install(level='DEBUG') + logger.setLevel(logging.DEBUG) +else: + #coloredlogs.install(level='INFO') + logger.setLevel(logging.INFO) + + +strhdlr = logging.StreamHandler() +logger.addHandler(strhdlr) +formatter = logging.Formatter('%(asctime)s [%(filename)s:%(lineno)d] %(levelname)s %(message)s') +strhdlr.setFormatter(formatter) + + + +def count_parameters(model): + return sum(p.numel() for p in model.parameters() if p.requires_grad) + +def check_path(path): + if not os.path.exists(path): + os.makedirs(path, exist_ok=True) + + diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/dataset_configuration.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/dataset_configuration.py new file mode 100644 index 0000000000000000000000000000000000000000..952a46473e4791f7b80ea009bbdffcf2859595ea --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/dataset_configuration.py @@ -0,0 +1,81 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np +import sys +sys.path.append("..") + +from dataloader.mix_loader import MixDataset +from torch.utils.data import DataLoader +from dataloader import transforms +import os + + +# Get Dataset Here +def prepare_dataset(data_dir=None, + batch_size=1, + test_batch=1, + datathread=4, + logger=None): + + # set the config parameters + dataset_config_dict = dict() + + train_dataset = MixDataset(data_dir=data_dir) + + img_height, img_width = train_dataset.get_img_size() + + datathread = datathread + if os.environ.get('datathread') is not None: + datathread = int(os.environ.get('datathread')) + + if logger is not None: + logger.info("Use %d processes to load data..." % datathread) + + train_loader = DataLoader(train_dataset, batch_size = batch_size, \ + shuffle = True, num_workers = datathread, \ + pin_memory = True) + + num_batches_per_epoch = len(train_loader) + + dataset_config_dict['num_batches_per_epoch'] = num_batches_per_epoch + dataset_config_dict['img_size'] = (img_height,img_width) + + return train_loader, dataset_config_dict + +def depth_scale_shift_normalization(depth): + + bsz = depth.shape[0] + + depth_ = depth[:,0,:,:].reshape(bsz,-1).cpu().numpy() + min_value = torch.from_numpy(np.percentile(a=depth_,q=2,axis=1)).to(depth)[...,None,None,None] + max_value = torch.from_numpy(np.percentile(a=depth_,q=98,axis=1)).to(depth)[...,None,None,None] + + normalized_depth = ((depth - min_value)/(max_value-min_value+1e-5) - 0.5) * 2 + normalized_depth = torch.clip(normalized_depth, -1., 1.) + + return normalized_depth + + + +def resize_max_res_tensor(input_tensor, mode, recom_resolution=768): + assert input_tensor.shape[1]==3 + original_H, original_W = input_tensor.shape[2:] + downscale_factor = min(recom_resolution/original_H, recom_resolution/original_W) + + if mode == 'normal': + resized_input_tensor = F.interpolate(input_tensor, + scale_factor=downscale_factor, + mode='nearest') + else: + resized_input_tensor = F.interpolate(input_tensor, + scale_factor=downscale_factor, + mode='bilinear', + align_corners=False) + + if mode == 'depth': + return resized_input_tensor / downscale_factor + else: + return resized_input_tensor diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/de_normalized.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/de_normalized.py new file mode 100644 index 0000000000000000000000000000000000000000..4e918b0d6df0c5ac08306ce72894a7f81c9e76a4 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/de_normalized.py @@ -0,0 +1,33 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +import numpy as np +from scipy.optimize import least_squares +import torch + +def align_scale_shift(pred, target, clip_max): + mask = (target > 0) & (target < clip_max) + if mask.sum() > 10: + target_mask = target[mask] + pred_mask = pred[mask] + scale, shift = np.polyfit(pred_mask, target_mask, deg=1) + return scale, shift + else: + return 1, 0 + +def align_scale(pred: torch.tensor, target: torch.tensor): + mask = target > 0 + if torch.sum(mask) > 10: + scale = torch.median(target[mask]) / (torch.median(pred[mask]) + 1e-8) + else: + scale = 1 + pred_scale = pred * scale + return pred_scale, scale + +def align_shift(pred: torch.tensor, target: torch.tensor): + mask = target > 0 + if torch.sum(mask) > 10: + shift = torch.median(target[mask]) - (torch.median(pred[mask]) + 1e-8) + else: + shift = 0 + pred_shift = pred + shift + return pred_shift, shift \ No newline at end of file diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/depth2normal.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/depth2normal.py new file mode 100644 index 0000000000000000000000000000000000000000..bca7f1df52ac77c5234231b38bd16dd7dd86ff28 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/depth2normal.py @@ -0,0 +1,186 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +import pickle +import os +import h5py +import numpy as np +import cv2 +import torch +import torch.nn as nn +import glob + + +def init_image_coor(height, width): + x_row = np.arange(0, width) + x = np.tile(x_row, (height, 1)) + x = x[np.newaxis, :, :] + x = x.astype(np.float32) + x = torch.from_numpy(x.copy()).cuda() + u_u0 = x - width/2.0 + + y_col = np.arange(0, height) # y_col = np.arange(0, height) + y = np.tile(y_col, (width, 1)).T + y = y[np.newaxis, :, :] + y = y.astype(np.float32) + y = torch.from_numpy(y.copy()).cuda() + v_v0 = y - height/2.0 + return u_u0, v_v0 + + +def depth_to_xyz(depth, focal_length): + b, c, h, w = depth.shape + u_u0, v_v0 = init_image_coor(h, w) + x = u_u0 * depth / focal_length[0] + y = v_v0 * depth / focal_length[1] + z = depth + pw = torch.cat([x, y, z], 1).permute(0, 2, 3, 1) # [b, h, w, c] + return pw + + +def get_surface_normal(xyz, patch_size=5): + # xyz: [1, h, w, 3] + x, y, z = torch.unbind(xyz, dim=3) + x = torch.unsqueeze(x, 0) + y = torch.unsqueeze(y, 0) + z = torch.unsqueeze(z, 0) + + xx = x * x + yy = y * y + zz = z * z + xy = x * y + xz = x * z + yz = y * z + patch_weight = torch.ones((1, 1, patch_size, patch_size), requires_grad=False).cuda() + xx_patch = nn.functional.conv2d(xx, weight=patch_weight, padding=int(patch_size / 2)) + yy_patch = nn.functional.conv2d(yy, weight=patch_weight, padding=int(patch_size / 2)) + zz_patch = nn.functional.conv2d(zz, weight=patch_weight, padding=int(patch_size / 2)) + xy_patch = nn.functional.conv2d(xy, weight=patch_weight, padding=int(patch_size / 2)) + xz_patch = nn.functional.conv2d(xz, weight=patch_weight, padding=int(patch_size / 2)) + yz_patch = nn.functional.conv2d(yz, weight=patch_weight, padding=int(patch_size / 2)) + ATA = torch.stack([xx_patch, xy_patch, xz_patch, xy_patch, yy_patch, yz_patch, xz_patch, yz_patch, zz_patch], + dim=4) + ATA = torch.squeeze(ATA) + ATA = torch.reshape(ATA, (ATA.size(0), ATA.size(1), 3, 3)) + eps_identity = 1e-6 * torch.eye(3, device=ATA.device, dtype=ATA.dtype)[None, None, :, :].repeat([ATA.size(0), ATA.size(1), 1, 1]) + ATA = ATA + eps_identity + x_patch = nn.functional.conv2d(x, weight=patch_weight, padding=int(patch_size / 2)) + y_patch = nn.functional.conv2d(y, weight=patch_weight, padding=int(patch_size / 2)) + z_patch = nn.functional.conv2d(z, weight=patch_weight, padding=int(patch_size / 2)) + AT1 = torch.stack([x_patch, y_patch, z_patch], dim=4) + AT1 = torch.squeeze(AT1) + AT1 = torch.unsqueeze(AT1, 3) + + patch_num = 4 + patch_x = int(AT1.size(1) / patch_num) + patch_y = int(AT1.size(0) / patch_num) + n_img = torch.randn(AT1.shape).cuda() + overlap = patch_size // 2 + 1 + for x in range(int(patch_num)): + for y in range(int(patch_num)): + left_flg = 0 if x == 0 else 1 + right_flg = 0 if x == patch_num -1 else 1 + top_flg = 0 if y == 0 else 1 + btm_flg = 0 if y == patch_num - 1 else 1 + at1 = AT1[y * patch_y - top_flg * overlap:(y + 1) * patch_y + btm_flg * overlap, + x * patch_x - left_flg * overlap:(x + 1) * patch_x + right_flg * overlap] + ata = ATA[y * patch_y - top_flg * overlap:(y + 1) * patch_y + btm_flg * overlap, + x * patch_x - left_flg * overlap:(x + 1) * patch_x + right_flg * overlap] + # n_img_tmp, _ = torch.solve(at1, ata) + n_img_tmp = torch.linalg.solve(ata, at1) + + n_img_tmp_select = n_img_tmp[top_flg * overlap:patch_y + top_flg * overlap, left_flg * overlap:patch_x + left_flg * overlap, :, :] + n_img[y * patch_y:y * patch_y + patch_y, x * patch_x:x * patch_x + patch_x, :, :] = n_img_tmp_select + + n_img_L2 = torch.sqrt(torch.sum(n_img ** 2, dim=2, keepdim=True)) + n_img_norm = n_img / n_img_L2 + + # re-orient normals consistently + orient_mask = torch.sum(torch.squeeze(n_img_norm) * torch.squeeze(xyz), dim=2) > 0 + n_img_norm[orient_mask] *= -1 + return n_img_norm + +def get_surface_normalv2(xyz, patch_size=5): + """ + xyz: xyz coordinates + patch: [p1, p2, p3, + p4, p5, p6, + p7, p8, p9] + surface_normal = [(p9-p1) x (p3-p7)] + [(p6-p4) - (p8-p2)] + return: normal [h, w, 3, b] + """ + b, h, w, c = xyz.shape + half_patch = patch_size // 2 + xyz_pad = torch.zeros((b, h + patch_size - 1, w + patch_size - 1, c), dtype=xyz.dtype, device=xyz.device) + xyz_pad[:, half_patch:-half_patch, half_patch:-half_patch, :] = xyz + + # xyz_left_top = xyz_pad[:, :h, :w, :] # p1 + # xyz_right_bottom = xyz_pad[:, -h:, -w:, :]# p9 + # xyz_left_bottom = xyz_pad[:, -h:, :w, :] # p7 + # xyz_right_top = xyz_pad[:, :h, -w:, :] # p3 + # xyz_cross1 = xyz_left_top - xyz_right_bottom # p1p9 + # xyz_cross2 = xyz_left_bottom - xyz_right_top # p7p3 + + xyz_left = xyz_pad[:, half_patch:half_patch + h, :w, :] # p4 + xyz_right = xyz_pad[:, half_patch:half_patch + h, -w:, :] # p6 + xyz_top = xyz_pad[:, :h, half_patch:half_patch + w, :] # p2 + xyz_bottom = xyz_pad[:, -h:, half_patch:half_patch + w, :] # p8 + xyz_horizon = xyz_left - xyz_right # p4p6 + xyz_vertical = xyz_top - xyz_bottom # p2p8 + + xyz_left_in = xyz_pad[:, half_patch:half_patch + h, 1:w+1, :] # p4 + xyz_right_in = xyz_pad[:, half_patch:half_patch + h, patch_size-1:patch_size-1+w, :] # p6 + xyz_top_in = xyz_pad[:, 1:h+1, half_patch:half_patch + w, :] # p2 + xyz_bottom_in = xyz_pad[:, patch_size-1:patch_size-1+h, half_patch:half_patch + w, :] # p8 + xyz_horizon_in = xyz_left_in - xyz_right_in # p4p6 + xyz_vertical_in = xyz_top_in - xyz_bottom_in # p2p8 + + n_img_1 = torch.cross(xyz_horizon_in, xyz_vertical_in, dim=3) + n_img_2 = torch.cross(xyz_horizon, xyz_vertical, dim=3) + + # re-orient normals consistently + orient_mask = torch.sum(n_img_1 * xyz, dim=3) > 0 + n_img_1[orient_mask] *= -1 + orient_mask = torch.sum(n_img_2 * xyz, dim=3) > 0 + n_img_2[orient_mask] *= -1 + + n_img1_L2 = torch.sqrt(torch.sum(n_img_1 ** 2, dim=3, keepdim=True)) + n_img1_norm = n_img_1 / (n_img1_L2 + 1e-8) + + n_img2_L2 = torch.sqrt(torch.sum(n_img_2 ** 2, dim=3, keepdim=True)) + n_img2_norm = n_img_2 / (n_img2_L2 + 1e-8) + + # average 2 norms + n_img_aver = n_img1_norm + n_img2_norm + n_img_aver_L2 = torch.sqrt(torch.sum(n_img_aver ** 2, dim=3, keepdim=True)) + n_img_aver_norm = n_img_aver / (n_img_aver_L2 + 1e-8) + # re-orient normals consistently + orient_mask = torch.sum(n_img_aver_norm * xyz, dim=3) > 0 + n_img_aver_norm[orient_mask] *= -1 + n_img_aver_norm_out = n_img_aver_norm.permute((1, 2, 3, 0)) # [h, w, c, b] + + # a = torch.sum(n_img1_norm_out*n_img2_norm_out, dim=2).cpu().numpy().squeeze() + # plt.imshow(np.abs(a), cmap='rainbow') + # plt.show() + return n_img_aver_norm_out#n_img1_norm.permute((1, 2, 3, 0)) + +def surface_normal_from_depth(depth, focal_length, valid_mask=None): + # para depth: depth map, [b, c, h, w] + b, c, h, w = depth.shape + focal_length = focal_length[:, None, None, None] + depth_filter = nn.functional.avg_pool2d(depth, kernel_size=3, stride=1, padding=1) + #depth_filter = nn.functional.avg_pool2d(depth_filter, kernel_size=3, stride=1, padding=1) + xyz = depth_to_xyz(depth_filter, focal_length) + sn_batch = [] + for i in range(b): + xyz_i = xyz[i, :][None, :, :, :] + #normal = get_surface_normalv2(xyz_i) + normal = get_surface_normal(xyz_i) + sn_batch.append(normal) + sn_batch = torch.cat(sn_batch, dim=3).permute((3, 2, 0, 1)) # [b, c, h, w] + + if valid_mask != None: + mask_invalid = (~valid_mask).repeat(1, 3, 1, 1) + sn_batch[mask_invalid] = 0.0 + + return sn_batch + diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/depth_ensemble.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/depth_ensemble.py new file mode 100644 index 0000000000000000000000000000000000000000..170deb2dda88c80313dc710dad12af4195f1b528 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/depth_ensemble.py @@ -0,0 +1,115 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +import numpy as np +import torch + +from scipy.optimize import minimize + +def inter_distances(tensors: torch.Tensor): + """ + To calculate the distance between each two depth maps. + """ + distances = [] + for i, j in torch.combinations(torch.arange(tensors.shape[0])): + arr1 = tensors[i : i + 1] + arr2 = tensors[j : j + 1] + distances.append(arr1 - arr2) + dist = torch.concat(distances, dim=0) + return dist + + +def ensemble_depths(input_images:torch.Tensor, + regularizer_strength: float =0.02, + max_iter: int =2, + tol:float =1e-3, + reduction: str='median', + max_res: int=None): + """ + To ensemble multiple affine-invariant depth images (up to scale and shift), + by aligning estimating the scale and shift + """ + + device = input_images.device + dtype = input_images.dtype + np_dtype = np.float32 + + + original_input = input_images.clone() + n_img = input_images.shape[0] + ori_shape = input_images.shape + + if max_res is not None: + scale_factor = torch.min(max_res / torch.tensor(ori_shape[-2:])) + if scale_factor < 1: + downscaler = torch.nn.Upsample(scale_factor=scale_factor, mode="nearest") + input_images = downscaler(torch.from_numpy(input_images)).numpy() + + # init guess + _min = np.min(input_images.reshape((n_img, -1)).cpu().numpy(), axis=1) # get the min value of each possible depth + _max = np.max(input_images.reshape((n_img, -1)).cpu().numpy(), axis=1) # get the max value of each possible depth + s_init = 1.0 / (_max - _min).reshape((-1, 1, 1)) #(10,1,1) : re-scale'f scale + t_init = (-1 * s_init.flatten() * _min.flatten()).reshape((-1, 1, 1)) #(10,1,1) + + x = np.concatenate([s_init, t_init]).reshape(-1).astype(np_dtype) #(20,) + + input_images = input_images.to(device) + + # objective function + def closure(x): + l = len(x) + s = x[: int(l / 2)] + t = x[int(l / 2) :] + s = torch.from_numpy(s).to(dtype=dtype).to(device) + t = torch.from_numpy(t).to(dtype=dtype).to(device) + + transformed_arrays = input_images * s.view((-1, 1, 1)) + t.view((-1, 1, 1)) + dists = inter_distances(transformed_arrays) + sqrt_dist = torch.sqrt(torch.mean(dists**2)) + + if "mean" == reduction: + pred = torch.mean(transformed_arrays, dim=0) + elif "median" == reduction: + pred = torch.median(transformed_arrays, dim=0).values + else: + raise ValueError + + near_err = torch.sqrt((0 - torch.min(pred)) ** 2) + far_err = torch.sqrt((1 - torch.max(pred)) ** 2) + + err = sqrt_dist + (near_err + far_err) * regularizer_strength + err = err.detach().cpu().numpy().astype(np_dtype) + return err + + res = minimize( + closure, x, method="BFGS", tol=tol, options={"maxiter": max_iter, "disp": False} + ) + x = res.x + l = len(x) + s = x[: int(l / 2)] + t = x[int(l / 2) :] + + # Prediction + s = torch.from_numpy(s).to(dtype=dtype).to(device) + t = torch.from_numpy(t).to(dtype=dtype).to(device) + transformed_arrays = original_input * s.view(-1, 1, 1) + t.view(-1, 1, 1) #[10,H,W] + + + if "mean" == reduction: + aligned_images = torch.mean(transformed_arrays, dim=0) + std = torch.std(transformed_arrays, dim=0) + uncertainty = std + + elif "median" == reduction: + aligned_images = torch.median(transformed_arrays, dim=0).values + # MAD (median absolute deviation) as uncertainty indicator + abs_dev = torch.abs(transformed_arrays - aligned_images) + mad = torch.median(abs_dev, dim=0).values + uncertainty = mad + + # Scale and shift to [0, 1] + _min = torch.min(aligned_images) + _max = torch.max(aligned_images) + aligned_images = (aligned_images - _min) / (_max - _min) + uncertainty /= _max - _min + + return aligned_images, uncertainty \ No newline at end of file diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/image_util.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/image_util.py new file mode 100644 index 0000000000000000000000000000000000000000..a864ca1ef16101b6ff6c9b13e75a784b8a70256f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/image_util.py @@ -0,0 +1,83 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +import matplotlib +import numpy as np +import torch +from PIL import Image + + + + +def resize_max_res(img: Image.Image, max_edge_resolution: int) -> Image.Image: + """ + Resize image to limit maximum edge length while keeping aspect ratio. + Args: + img (`Image.Image`): + Image to be resized. + max_edge_resolution (`int`): + Maximum edge length (pixel). + Returns: + `Image.Image`: Resized image. + """ + + original_width, original_height = img.size + + downscale_factor = min( + max_edge_resolution / original_width, max_edge_resolution / original_height + ) + + new_width = int(original_width * downscale_factor) + new_height = int(original_height * downscale_factor) + + resized_img = img.resize((new_width, new_height)) + return resized_img + + +def colorize_depth_maps( + depth_map, min_depth, max_depth, cmap="Spectral", valid_mask=None +): + """ + Colorize depth maps. + """ + assert len(depth_map.shape) >= 2, "Invalid dimension" + + if isinstance(depth_map, torch.Tensor): + depth = depth_map.detach().clone().squeeze().numpy() + elif isinstance(depth_map, np.ndarray): + depth = depth_map.copy().squeeze() + # reshape to [ (B,) H, W ] + if depth.ndim < 3: + depth = depth[np.newaxis, :, :] + + # colorize + cm = matplotlib.colormaps[cmap] + depth = ((depth - min_depth) / (max_depth - min_depth)).clip(0, 1) + img_colored_np = cm(depth, bytes=False)[:, :, :, 0:3] # value from 0 to 1 + img_colored_np = np.rollaxis(img_colored_np, 3, 1) + + if valid_mask is not None: + if isinstance(depth_map, torch.Tensor): + valid_mask = valid_mask.detach().numpy() + valid_mask = valid_mask.squeeze() # [H, W] or [B, H, W] + if valid_mask.ndim < 3: + valid_mask = valid_mask[np.newaxis, np.newaxis, :, :] + else: + valid_mask = valid_mask[:, np.newaxis, :, :] + valid_mask = np.repeat(valid_mask, 3, axis=1) + img_colored_np[~valid_mask] = 0 + + if isinstance(depth_map, torch.Tensor): + img_colored = torch.from_numpy(img_colored_np).float() + elif isinstance(depth_map, np.ndarray): + img_colored = img_colored_np + + return img_colored + + +def chw2hwc(chw): + assert 3 == len(chw.shape) + if isinstance(chw, torch.Tensor): + hwc = torch.permute(chw, (1, 2, 0)) + elif isinstance(chw, np.ndarray): + hwc = np.moveaxis(chw, 0, -1) + return hwc diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/normal_ensemble.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/normal_ensemble.py new file mode 100644 index 0000000000000000000000000000000000000000..f185122bf03f6da30482f62c4b00f709ba1ca4cc --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/normal_ensemble.py @@ -0,0 +1,22 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +import numpy as np +import torch + +def ensemble_normals(input_images:torch.Tensor): + normal_preds = input_images + + bsz, d, h, w = normal_preds.shape + normal_preds = normal_preds / (torch.norm(normal_preds, p=2, dim=1).unsqueeze(1)+1e-5) + + phi = torch.atan2(normal_preds[:,1,:,:], normal_preds[:,0,:,:]).mean(dim=0) + theta = torch.atan2(torch.norm(normal_preds[:,:2,:,:], p=2, dim=1), normal_preds[:,2,:,:]).mean(dim=0) + normal_pred = torch.zeros((d,h,w)).to(normal_preds) + normal_pred[0,:,:] = torch.sin(theta) * torch.cos(phi) + normal_pred[1,:,:] = torch.sin(theta) * torch.sin(phi) + normal_pred[2,:,:] = torch.cos(theta) + + angle_error = torch.acos(torch.cosine_similarity(normal_pred[None], normal_preds, dim=1)) + normal_idx = torch.argmin(angle_error.reshape(bsz,-1).sum(-1)) + + return normal_preds[normal_idx] \ No newline at end of file diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/seed_all.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/seed_all.py new file mode 100644 index 0000000000000000000000000000000000000000..95795654e36d3fbc14e876dff50f348a1f5a9c0a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/seed_all.py @@ -0,0 +1,33 @@ +# Copyright 2023 Bingxin Ke, ETH Zurich. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# -------------------------------------------------------------------------- +# If you find this code useful, we kindly ask you to cite our paper in your work. +# Please find bibtex at: https://github.com/prs-eth/Marigold#-citation +# More information about the method can be found at https://marigoldmonodepth.github.io +# -------------------------------------------------------------------------- + + +import numpy as np +import random +import torch + + +def seed_all(seed: int = 0): + """ + Set random seeds of all components. + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/surface_normal.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/surface_normal.py new file mode 100644 index 0000000000000000000000000000000000000000..79d3a068aea0f52ef1e45b7f2f6969804e57c90d --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/geowizard/utils/surface_normal.py @@ -0,0 +1,213 @@ +# A reimplemented version in public environments by Xiao Fu and Mu Hu + +import torch +import numpy as np +import torch.nn as nn + + +def init_image_coor(height, width): + x_row = np.arange(0, width) + x = np.tile(x_row, (height, 1)) + x = x[np.newaxis, :, :] + x = x.astype(np.float32) + x = torch.from_numpy(x.copy()).cuda() + u_u0 = x - width/2.0 + + y_col = np.arange(0, height) # y_col = np.arange(0, height) + y = np.tile(y_col, (width, 1)).T + y = y[np.newaxis, :, :] + y = y.astype(np.float32) + y = torch.from_numpy(y.copy()).cuda() + v_v0 = y - height/2.0 + return u_u0, v_v0 + + +def depth_to_xyz(depth, focal_length): + b, c, h, w = depth.shape + u_u0, v_v0 = init_image_coor(h, w) + x = u_u0 * depth / focal_length + y = v_v0 * depth / focal_length + z = depth + pw = torch.cat([x, y, z], 1).permute(0, 2, 3, 1) # [b, h, w, c] + return pw + + +def get_surface_normal(xyz, patch_size=3): + # xyz: [1, h, w, 3] + x, y, z = torch.unbind(xyz, dim=3) + x = torch.unsqueeze(x, 0) + y = torch.unsqueeze(y, 0) + z = torch.unsqueeze(z, 0) + + xx = x * x + yy = y * y + zz = z * z + xy = x * y + xz = x * z + yz = y * z + patch_weight = torch.ones((1, 1, patch_size, patch_size), requires_grad=False).cuda() + xx_patch = nn.functional.conv2d(xx, weight=patch_weight, padding=int(patch_size / 2)) + yy_patch = nn.functional.conv2d(yy, weight=patch_weight, padding=int(patch_size / 2)) + zz_patch = nn.functional.conv2d(zz, weight=patch_weight, padding=int(patch_size / 2)) + xy_patch = nn.functional.conv2d(xy, weight=patch_weight, padding=int(patch_size / 2)) + xz_patch = nn.functional.conv2d(xz, weight=patch_weight, padding=int(patch_size / 2)) + yz_patch = nn.functional.conv2d(yz, weight=patch_weight, padding=int(patch_size / 2)) + ATA = torch.stack([xx_patch, xy_patch, xz_patch, xy_patch, yy_patch, yz_patch, xz_patch, yz_patch, zz_patch], + dim=4) + ATA = torch.squeeze(ATA) + ATA = torch.reshape(ATA, (ATA.size(0), ATA.size(1), 3, 3)) + eps_identity = 1e-6 * torch.eye(3, device=ATA.device, dtype=ATA.dtype)[None, None, :, :].repeat([ATA.size(0), ATA.size(1), 1, 1]) + ATA = ATA + eps_identity + x_patch = nn.functional.conv2d(x, weight=patch_weight, padding=int(patch_size / 2)) + y_patch = nn.functional.conv2d(y, weight=patch_weight, padding=int(patch_size / 2)) + z_patch = nn.functional.conv2d(z, weight=patch_weight, padding=int(patch_size / 2)) + AT1 = torch.stack([x_patch, y_patch, z_patch], dim=4) + AT1 = torch.squeeze(AT1) + AT1 = torch.unsqueeze(AT1, 3) + + patch_num = 4 + patch_x = int(AT1.size(1) / patch_num) + patch_y = int(AT1.size(0) / patch_num) + n_img = torch.randn(AT1.shape).cuda() + overlap = patch_size // 2 + 1 + for x in range(int(patch_num)): + for y in range(int(patch_num)): + left_flg = 0 if x == 0 else 1 + right_flg = 0 if x == patch_num -1 else 1 + top_flg = 0 if y == 0 else 1 + btm_flg = 0 if y == patch_num - 1 else 1 + at1 = AT1[y * patch_y - top_flg * overlap:(y + 1) * patch_y + btm_flg * overlap, + x * patch_x - left_flg * overlap:(x + 1) * patch_x + right_flg * overlap] + ata = ATA[y * patch_y - top_flg * overlap:(y + 1) * patch_y + btm_flg * overlap, + x * patch_x - left_flg * overlap:(x + 1) * patch_x + right_flg * overlap] + n_img_tmp, _ = torch.solve(at1, ata) + + n_img_tmp_select = n_img_tmp[top_flg * overlap:patch_y + top_flg * overlap, left_flg * overlap:patch_x + left_flg * overlap, :, :] + n_img[y * patch_y:y * patch_y + patch_y, x * patch_x:x * patch_x + patch_x, :, :] = n_img_tmp_select + + n_img_L2 = torch.sqrt(torch.sum(n_img ** 2, dim=2, keepdim=True)) + n_img_norm = n_img / n_img_L2 + + # re-orient normals consistently + orient_mask = torch.sum(torch.squeeze(n_img_norm) * torch.squeeze(xyz), dim=2) > 0 + n_img_norm[orient_mask] *= -1 + return n_img_norm + +def get_surface_normalv2(xyz, patch_size=3): + """ + xyz: xyz coordinates + patch: [p1, p2, p3, + p4, p5, p6, + p7, p8, p9] + surface_normal = [(p9-p1) x (p3-p7)] + [(p6-p4) - (p8-p2)] + return: normal [h, w, 3, b] + """ + b, h, w, c = xyz.shape + half_patch = patch_size // 2 + xyz_pad = torch.zeros((b, h + patch_size - 1, w + patch_size - 1, c), dtype=xyz.dtype, device=xyz.device) + xyz_pad[:, half_patch:-half_patch, half_patch:-half_patch, :] = xyz + + # xyz_left_top = xyz_pad[:, :h, :w, :] # p1 + # xyz_right_bottom = xyz_pad[:, -h:, -w:, :]# p9 + # xyz_left_bottom = xyz_pad[:, -h:, :w, :] # p7 + # xyz_right_top = xyz_pad[:, :h, -w:, :] # p3 + # xyz_cross1 = xyz_left_top - xyz_right_bottom # p1p9 + # xyz_cross2 = xyz_left_bottom - xyz_right_top # p7p3 + + xyz_left = xyz_pad[:, half_patch:half_patch + h, :w, :] # p4 + xyz_right = xyz_pad[:, half_patch:half_patch + h, -w:, :] # p6 + xyz_top = xyz_pad[:, :h, half_patch:half_patch + w, :] # p2 + xyz_bottom = xyz_pad[:, -h:, half_patch:half_patch + w, :] # p8 + xyz_horizon = xyz_left - xyz_right # p4p6 + xyz_vertical = xyz_top - xyz_bottom # p2p8 + + xyz_left_in = xyz_pad[:, half_patch:half_patch + h, 1:w+1, :] # p4 + xyz_right_in = xyz_pad[:, half_patch:half_patch + h, patch_size-1:patch_size-1+w, :] # p6 + xyz_top_in = xyz_pad[:, 1:h+1, half_patch:half_patch + w, :] # p2 + xyz_bottom_in = xyz_pad[:, patch_size-1:patch_size-1+h, half_patch:half_patch + w, :] # p8 + xyz_horizon_in = xyz_left_in - xyz_right_in # p4p6 + xyz_vertical_in = xyz_top_in - xyz_bottom_in # p2p8 + + n_img_1 = torch.cross(xyz_horizon_in, xyz_vertical_in, dim=3) + n_img_2 = torch.cross(xyz_horizon, xyz_vertical, dim=3) + + # re-orient normals consistently + orient_mask = torch.sum(n_img_1 * xyz, dim=3) > 0 + n_img_1[orient_mask] *= -1 + orient_mask = torch.sum(n_img_2 * xyz, dim=3) > 0 + n_img_2[orient_mask] *= -1 + + n_img1_L2 = torch.sqrt(torch.sum(n_img_1 ** 2, dim=3, keepdim=True)) + n_img1_norm = n_img_1 / (n_img1_L2 + 1e-8) + + n_img2_L2 = torch.sqrt(torch.sum(n_img_2 ** 2, dim=3, keepdim=True)) + n_img2_norm = n_img_2 / (n_img2_L2 + 1e-8) + + # average 2 norms + n_img_aver = n_img1_norm + n_img2_norm + n_img_aver_L2 = torch.sqrt(torch.sum(n_img_aver ** 2, dim=3, keepdim=True)) + n_img_aver_norm = n_img_aver / (n_img_aver_L2 + 1e-8) + # re-orient normals consistently + orient_mask = torch.sum(n_img_aver_norm * xyz, dim=3) > 0 + n_img_aver_norm[orient_mask] *= -1 + n_img_aver_norm_out = n_img_aver_norm.permute((1, 2, 3, 0)) # [h, w, c, b] + + # a = torch.sum(n_img1_norm_out*n_img2_norm_out, dim=2).cpu().numpy().squeeze() + # plt.imshow(np.abs(a), cmap='rainbow') + # plt.show() + return n_img_aver_norm_out#n_img1_norm.permute((1, 2, 3, 0)) + +def surface_normal_from_depth(depth, focal_length, valid_mask=None): + # para depth: depth map, [b, c, h, w] + b, c, h, w = depth.shape + focal_length = focal_length[:, None, None, None] + depth_filter = nn.functional.avg_pool2d(depth, kernel_size=3, stride=1, padding=1) + depth_filter = nn.functional.avg_pool2d(depth_filter, kernel_size=3, stride=1, padding=1) + xyz = depth_to_xyz(depth_filter, focal_length) + sn_batch = [] + for i in range(b): + xyz_i = xyz[i, :][None, :, :, :] + normal = get_surface_normalv2(xyz_i) + sn_batch.append(normal) + sn_batch = torch.cat(sn_batch, dim=3).permute((3, 2, 0, 1)) # [b, c, h, w] + mask_invalid = (~valid_mask).repeat(1, 3, 1, 1) + sn_batch[mask_invalid] = 0.0 + + return sn_batch + + +def vis_normal(normal): + """ + Visualize surface normal. Transfer surface normal value from [-1, 1] to [0, 255] + @para normal: surface normal, [h, w, 3], numpy.array + """ + n_img_L2 = np.sqrt(np.sum(normal ** 2, axis=2, keepdims=True)) + n_img_norm = normal / (n_img_L2 + 1e-8) + normal_vis = n_img_norm * 127 + normal_vis += 128 + normal_vis = normal_vis.astype(np.uint8) + return normal_vis + +def vis_normal2(normals): + ''' + Montage of normal maps. Vectors are unit length and backfaces thresholded. + ''' + x = normals[:, :, 0] # horizontal; pos right + y = normals[:, :, 1] # depth; pos far + z = normals[:, :, 2] # vertical; pos up + backfacing = (z > 0) + norm = np.sqrt(np.sum(normals**2, axis=2)) + zero = (norm < 1e-5) + x += 1.0; x *= 0.5 + y += 1.0; y *= 0.5 + z = np.abs(z) + x[zero] = 0.0 + y[zero] = 0.0 + z[zero] = 0.0 + normals[:, :, 0] = x # horizontal; pos right + normals[:, :, 1] = y # depth; pos far + normals[:, :, 2] = z # vertical; pos up + return normals + +if __name__ == '__main__': + import cv2, os \ No newline at end of file diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/requirements.txt b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..231b28b7b6f3124805c5775b4993420722128a62 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/GeoWizard/requirements.txt @@ -0,0 +1,20 @@ +torch == 2.0.1 +torchvision == 0.15.2 +xformers == 0.0.21 +diffusers == 0.25.0 +opencv-python +numpy == 1.23.1 +Pillow +h5py +matplotlib==3.7.5 +scikit-image==0.18.1 +gradio==4.11.0 +gradio-imageslider==0.0.16 +trimesh==4.0.5 +tqdm == 4.65.0 +accelerate==0.28.0 +transformers==4.39.1 +imgaug +GitPython==3.1.40 +pygltflib==1.16.1 +fire \ No newline at end of file diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/tools.py b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..9e802cf5fde06ca188727188ec586501140ad3bc --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/code/utils/tools.py @@ -0,0 +1,99 @@ +import torch +import cv2 +from sklearn.cluster import KMeans +import numpy as np + + +REORDER_MTX = torch.tensor([ + [0,0,0,1], + [1,0,0,0], + [0,1,0,0], + [0,0,1,0] +]).cuda().float() + +def build_rotation(r): + norm = torch.sqrt( + r[:, 0] * r[:, 0] + r[:, 1] * r[:, 1] + r[:, 2] * r[:, 2] + r[:, 3] * r[:, 3] + ) + + q = r / norm[:, None] + + R = torch.zeros((q.size(0), 3, 3), device="cuda") + + r = q[:, 0] + x = q[:, 1] + y = q[:, 2] + z = q[:, 3] + + R[:, 0, 0] = 1 - 2 * (y * y + z * z) + R[:, 0, 1] = 2 * (x * y - r * z) + R[:, 0, 2] = 2 * (x * z + r * y) + R[:, 1, 0] = 2 * (x * y + r * z) + R[:, 1, 1] = 1 - 2 * (x * x + z * z) + R[:, 1, 2] = 2 * (y * z - r * x) + R[:, 2, 0] = 2 * (x * z - r * y) + R[:, 2, 1] = 2 * (y * z + r * x) + R[:, 2, 2] = 1 - 2 * (x * x + y * y) + return R + +def rotation_matrix(angle_x, angle_y, angle_z): + # Convert angles to radians + rad_x = torch.deg2rad(torch.tensor(angle_x)) + rad_y = torch.deg2rad(torch.tensor(angle_y)) + rad_z = torch.deg2rad(torch.tensor(angle_z)) + + # Compute sine and cosine of the angles + cos_x = torch.cos(rad_x) + sin_x = torch.sin(rad_x) + cos_y = torch.cos(rad_y) + sin_y = torch.sin(rad_y) + cos_z = torch.cos(rad_z) + sin_z = torch.sin(rad_z) + + # Construct the rotation matrix + Rx = torch.tensor([[1, 0, 0], + [0, cos_x, -sin_x], + [0, sin_x, cos_x]]) + + Ry = torch.tensor([[cos_y, 0, sin_y], + [0, 1, 0], + [-sin_y, 0, cos_y]]) + + Rz = torch.tensor([[cos_z, -sin_z, 0], + [sin_z, cos_z, 0], + [0, 0, 1]]) + + # Combine the rotation matrices + rotation_matrix = Rz @ Ry @ Rx + + return rotation_matrix + +def prune(system, idx=0, p1=3.0, kmeans_t=0.3): + system.sam_clip_ae.eval() + ooi_masks = getattr(system.geometry, f"ooi_masks_{idx}").cpu().numpy() + ooi_masks = cv2.blur(ooi_masks.astype(np.float32), (5, 5)) > 0 + ooi_mask = torch.tensor(ooi_masks.reshape(-1).astype(np.uint8), device='cuda').float() + idx = torch.arange(len(ooi_mask), device='cuda')[ooi_mask.bool()] + + lang_feat = system.geometry.get_language_feature[ooi_mask.bool()] + lang_feat = lang_feat / (lang_feat.norm(2, dim=-1, keepdim=True) + 1e-3) + original_ooi_mask = ooi_mask.clone() + + # filter with color by KMeans + kmeans = KMeans(n_init='auto', n_clusters=10) + kmeans.fit(lang_feat.detach().cpu()) + labels = kmeans.labels_ + _ = [(labels==i).sum() for i in np.unique(labels)] + max_label = _.index(max(_)) + dist = ((kmeans.cluster_centers_ - kmeans.cluster_centers_[max_label:max_label+1]) **2).sum(-1)**.5 + for label, num in enumerate(_): + if dist[label] > kmeans_t: + ooi_mask[idx[labels == label]] = False + system.geometry._delete_mask[idx[labels == label]] = 0. + + p = p1 + # filter Gaussians + mean, std = lang_feat.mean(0), lang_feat.std(0) + outlier = torch.logical_or(lang_feat < mean - p * std, lang_feat > mean + p * std).sum(-1) > 0 + ooi_mask[idx[outlier]] = False + system.geometry._delete_mask[idx[outlier]] = 0. diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/configs/parsed.yaml b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/configs/parsed.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5cdbf135dca94364990406305f80db9a80c82f7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/configs/parsed.yaml @@ -0,0 +1,156 @@ +name: gs-sds-generation +description: '' +tag: 3DitScene +seed: 0 +use_timestamp: true +timestamp: '@20250207-015119' +exp_root_dir: outputs/mira_video_clips/000000000/000000000017.1 +exp_dir: outputs/mira_video_clips/000000000/000000000017.1/gs-sds-generation +trial_name: 3DitScene@20250207-015119 +trial_dir: outputs/mira_video_clips/000000000/000000000017.1/gs-sds-generation/3DitScene@20250207-015119 +n_gpus: 1 +resume: null +data_type: random-camera-datamodule +data: + rotate_traj: false + random_traj: false + batch_size: 1 + width: 512 + height: 512 + camera_distance_range: + - 2.5 + - 2.5 + fovy_range: + - 60 + - 60 + elevation_range: + - 0 + - 0 + light_sample_strategy: dreamfusion + eval_camera_distance: 2.5 + eval_fovy_deg: 60 + eval_elevation_deg: 0 + rays_d_normalize: false + center_perturb: 0 + up_perturb: 0 + camera_perturb: 0 + azimuth_range: + - -15 + - 15 + val_azimuth_range: + - -15 + - 15 + insert_zero: true +system_type: scene-lang-system +system: + encoder_hidden_dims: + - 256 + - 128 + - 32 + - 3 + decoder_hidden_dims: + - 32 + - 128 + - 256 + - 512 + xyz_noise_ratio: + - 1000 + - 0.0 + - 0.0 + - 3000 + drop_ooi_ratio: 0.3 + crop_with_lang: true + densify: false + geometry_type: gaussian-splatting + geometry: + ooi_bbox: + - 599 + - 250 + - 692 + - 452 + geometry_convert_from: depth:/mnt/hdd1/wufan/datasets/MiraData/data/video_frames/000000000/000000000017.1/0.jpg + position_lr: + - 0 + - 0.001 + - 2.0e-05 + - 1000 + scaling_lr: 0.05 + feature_lr: 0.01 + opacity_lr: 0.05 + rotation_lr: 0.005 + lang_lr: 0.0003 + densification_interval: 300 + prune_interval: 300 + opacity_reset_interval: 50000000 + densify_from_iter: 500 + densify_until_iter: 1500 + prune_from_iter: 500 + prune_until_iter: 1500 + densify_grad_threshold: 0.01 + min_opac_prune: 0.005 + split_thresh: 0.02 + radii2d_thresh: 1000 + init_num_pts: 4096 + pc_init_radius: 0.8 + opacity_init: 0.2 + empty_prompt: The background is a city at night with tall buildings, out of focus + prompt: It is night time in a city with tall buildings and neon lights illuminating + the streets. + max_scaling: 0.2 + renderer_type: diff-gaussian-rasterizer + renderer: + debug: false + invert_bg_prob: 0.5 + material_type: no-material + material: + n_output_dims: 0 + background_type: solid-color-background + prompt_processor_type: stable-diffusion-prompt-processor + prompt_processor: + pretrained_model_name_or_path: stabilityai/stable-diffusion-2-1-base + prompt: It is night time in a city with tall buildings and neon lights illuminating + the streets. + empty_prompt: The background is a city at night with tall buildings, out of focus + guidance_type: stable-diffusion-guidance + guidance: + pretrained_model_name_or_path: stabilityai/stable-diffusion-2-1-base + guidance_scale: 5.0 + weighting_strategy: sds + min_step_percent: 0.02 + max_step_percent: + - 0 + - 0.5 + - 0.1 + - 1000 + csd: false + exporter_type: gaussian-mesh-exporter + sam_clip: + use_mobile_sam: true + loggers: + wandb: + enable: false + project: 3ditscene + name: 3DitScene + loss: + lambda_sds: 0.01 + lambda_ref: 1000 + lambda_depth: 0.0 + lambda_position: 1.0 + lambda_opacity: 0.0001 + lambda_scales: 0.0001 + lambda_tv_loss: 1.0 + lambda_depth_tv_loss: 1.0 + lambda_scaling: 0.0 + side_prompt: The background is a city at night with tall buildings, out of focus +trainer: + max_steps: 1500 + log_every_n_steps: 1 + num_sanity_val_steps: 110 + val_check_interval: 500 + enable_progress_bar: true + precision: 32-true +checkpoint: + save_last: true + save_top_k: -1 + every_n_train_steps: 1000 + save_weights_only: true diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/configs/raw.yaml b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/configs/raw.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a34efe32c6274a769a2fd5f05760671206ffc98 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/configs/raw.yaml @@ -0,0 +1,138 @@ +name: "gs-sds-generation" +tag: "${rmspace:${system.prompt_processor.prompt},_}" +exp_root_dir: "outputs" +seed: 0 +# resume: "/mnt/hdd1/wufan/projects/3DitScene/outputs/gs-sds-generation/3DitScene@20250204-120500/ckpts/last.ckpt" + +data_type: "random-camera-datamodule" +data: + rotate_traj: false # WU + random_traj: false # WU + batch_size: 1 + width: 512 + height: 512 + camera_distance_range: [2.5, 2.5] + fovy_range: [60, 60] + elevation_range: [0, 0] # The vertical angle of the camera relative to the object, in degrees. + light_sample_strategy: "dreamfusion" + eval_camera_distance: 2.5 + eval_fovy_deg: 60 # The field of view (FOV) in the vertical direction, in degrees. + eval_elevation_deg: 0 + rays_d_normalize: false + center_perturb: 0 + up_perturb: 0 + camera_perturb: 0 + azimuth_range: [-15, 15] # The range of horizontal rotation angles during training + val_azimuth_range: [-15, 15] # The range of horizontal rotation angles during validation + insert_zero: true + +system_type: "scene-lang-system" +system: + encoder_hidden_dims: [256, 128, 32, 3] + decoder_hidden_dims: [32, 128, 256, 512] + xyz_noise_ratio: [1000, 0.0, 0.0, 3000] + drop_ooi_ratio: 0.3 + crop_with_lang: true + densify: false + + geometry_type: "gaussian-splatting" + geometry: + ooi_bbox: [360,370,730,590] + geometry_convert_from: depth:assets/anime.png + position_lr: [0, 0.001, 0.00002, 1000] + scaling_lr: 0.05 + feature_lr: 0.01 + opacity_lr: 0.05 + rotation_lr: 0.005 + lang_lr: 0.0003 + densification_interval: 300 + prune_interval: 300 + opacity_reset_interval: 50000000 + densify_from_iter: 500 + densify_until_iter: ${trainer.max_steps} + prune_from_iter: 500 + prune_until_iter: ${trainer.max_steps} + densify_grad_threshold: 0.01 + min_opac_prune: 0.005 + split_thresh: 0.02 + radii2d_thresh: 1000 + + init_num_pts: 4096 + pc_init_radius: 0.8 + opacity_init: 0.2 + + empty_prompt: ${system.empty_prompt} + prompt: ${system.prompt_processor.prompt} + max_scaling: 0.2 + + renderer_type: "diff-gaussian-rasterizer" + renderer: + debug: false + invert_bg_prob: 0.5 + + material_type: "no-material" # unused + material: + n_output_dims: 0 + + background_type: "solid-color-background" # unused + + prompt_processor_type: "stable-diffusion-prompt-processor" + prompt_processor: + pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + # pretrained_model_name_or_path: "/mnt/petrelfs/zhangqihang/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1-base" + prompt: ??? + empty_prompt: "empty" + + guidance_type: "stable-diffusion-guidance" + guidance: + pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + # pretrained_model_name_or_path: "/mnt/petrelfs/zhangqihang/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1-base" + guidance_scale: 5.0 + weighting_strategy: sds + min_step_percent: 0.02 + max_step_percent: [0, 0.5, 0.1, 1000] + csd: false + + # guidance_type: "stable-diffusion-vsd-guidance" + # guidance: + # pretrained_model_name_or_path: "stabilityai/stable-diffusion-2-1-base" + # pretrained_model_name_or_path_lora: "stabilityai/stable-diffusion-2-1" + # guidance_scale: 7.5 + # min_step_percent: 0.02 + + exporter_type: "gaussian-mesh-exporter" + + sam_clip: + use_mobile_sam: True + + loggers: + wandb: + enable: false + project: '3ditscene' + name: "${tag}" + + loss: + lambda_sds: 0.01 + lambda_ref: 1000 + lambda_depth: 0.0 + lambda_position: 1.0 + lambda_opacity: 0.0001 + lambda_scales: 0.0001 + lambda_tv_loss: 1.0 + lambda_depth_tv_loss: 1.0 + lambda_scaling: 0.0 + +trainer: + max_steps: 1500 + log_every_n_steps: 1 + num_sanity_val_steps: 110 + val_check_interval: 500 + enable_progress_bar: true + precision: 32-true + +checkpoint: + save_last: true # save at each validation time + save_top_k: -1 + every_n_train_steps: 1000 + save_weights_only: true + # every_n_train_steps: ${trainer.max_steps} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/csv_logs/version_0/hparams.yaml b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/csv_logs/version_0/hparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0967ef424bce6791893e9a57bb952f80fd536e93 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/csv_logs/version_0/hparams.yaml @@ -0,0 +1 @@ +{} diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/csv_logs/version_0/metrics.csv b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/csv_logs/version_0/metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..62ddb50008f1bfb180a377adc9620850d16af12c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/csv_logs/version_0/metrics.csv @@ -0,0 +1,3002 @@ +epoch,gauss_num_epoch,gauss_num_step,lr-Adam/f_dc,lr-Adam/f_rest,lr-Adam/language_feature,lr-Adam/opacity,lr-Adam/rotation,lr-Adam/scaling,lr-Adam/xyz,step,train/grad_norm,train/loss,train/loss_sds,train/max_step,train/min_step,train/recon_front_view +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0010000000000000002,0,,,,,, +0,,265950.0,,,,,,,,0,42.68325424194336,38.733915243085406,455.465087890625,500.0,20.0,0.03417926445954659 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0010000000000000002,1,,,,,, +0,,265950.0,,,,,,,,1,26.468387603759766,32.66113883579425,175.14389038085938,499.0,20.0,0.030909699979669372 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009960956189881043,2,,,,,, +0,,265950.0,,,,,,,,2,34.642520904541016,63.6353360099658,300.02606201171875,499.0,20.0,0.06063507541845884 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009922064821672925,3,,,,,, +0,,265950.0,,,,,,,,3,14.082338333129883,53.30965792797515,49.578060150146484,498.0,20.0,0.0528138773407788 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009883325300184384,4,,,,,, +0,,265950.0,,,,,,,,4,22.451513290405273,26.19865692804037,126.01760864257812,498.0,20.0,0.02493848086545645 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009844737032547945,5,,,,,, +0,,265950.0,,,,,,,,5,20.587684631347656,45.09874114013329,105.96318817138672,498.0,20.0,0.02493848086545645 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009806299428210946,6,,,,,, +0,,265950.0,,,,,,,,6,38.20637893676758,26.656967234807063,364.93182373046875,497.0,20.0,0.02300764901657586 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009768011898926474,7,,,,,, +0,,265950.0,,,,,,,,7,23.738849639892578,32.69910391726515,140.88323974609375,497.0,20.0,0.0312902716056349 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009729873858744314,8,,,,,, +0,,265950.0,,,,,,,,8,34.39173889160156,31.028689714582637,295.69793701171875,496.0,20.0,0.02807171043983288 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000969188472400209,9,,,,,, +0,,265950.0,,,,,,,,9,14.628003120422363,21.59896121824046,53.4946174621582,496.0,20.0,0.021064015074613293 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009654043913316211,10,,,,,, +0,,265950.0,,,,,,,,10,15.720948219299316,21.077054363671586,61.78705596923828,496.0,20.0,0.02045918379444246 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009616350847573032,11,,,,,, +0,,265950.0,,,,,,,,11,22.75542640686035,22.68677399629518,129.45236206054688,495.0,20.0,0.02139225039953157 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009578804949920038,12,,,,,, +0,,265950.0,,,,,,,,12,32.412384033203125,22.58533384956676,262.64068603515625,495.0,20.0,0.01995892694153148 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009541405645756907,13,,,,,, +0,,265950.0,,,,,,,,13,30.392555236816406,21.045937728716698,230.92684936523438,494.0,20.0,0.01873666925413784 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009504152362726808,14,,,,,, +0,,265950.0,,,,,,,,14,21.193540573120117,19.606666931352347,112.29154205322266,494.0,20.0,0.01848375154419872 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009467044530707611,15,,,,,, +0,,265950.0,,,,,,,,15,22.840179443359375,39.06578984568376,130.41844177246094,494.0,20.0,0.01848375154419872 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009430081581803134,16,,,,,, +0,,265950.0,,,,,,,,16,30.69192886352539,19.13743682628126,235.4986114501953,493.0,20.0,0.01678245087390394 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009393262950334503,17,,,,,, +0,,265950.0,,,,,,,,17,41.6754264831543,20.748887729525947,434.21026611328125,493.0,20.0,0.01640678520190754 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009356588072831463,18,,,,,, +0,,265950.0,,,,,,,,18,10.857988357543945,17.411581885501295,29.47397804260254,492.0,20.0,0.017116842102691085 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009320056388023767,19,,,,,, +0,,265950.0,,,,,,,,19,17.861846923828125,33.01082916699109,79.76139068603516,492.0,20.0,0.017116842102691085 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000928366733683256,20,,,,,, +0,,265950.0,,,,,,,,20,37.419151306152344,18.85025934421045,350.0481872558594,492.0,20.0,0.015349777500262085 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009247420362361864,21,,,,,, +0,,265950.0,,,,,,,,21,34.569515228271484,18.89171300905796,298.7628173828125,491.0,20.0,0.015904084787546117 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009211314909890029,22,,,,,, +0,,265950.0,,,,,,,,22,35.92061996459961,33.0498688005617,322.57275390625,491.0,20.0,0.015904084787546117 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000917535042686126,23,,,,,, +0,,265950.0,,,,,,,,23,36.307106018066406,32.93136832462827,329.55145263671875,490.0,20.0,0.015904084787546117 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009139526362877123,24,,,,,, +0,,265950.0,,,,,,,,24,35.02411651611328,17.58680454446944,306.67218017578125,490.0,20.0,0.01452008286668929 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009103842169688175,25,,,,,, +0,,265950.0,,,,,,,,25,20.341655731201172,28.160538035896177,103.44573974609375,490.0,20.0,0.01452008286668929 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009068297301185539,26,,,,,, +0,,265950.0,,,,,,,,26,15.659523010253906,15.348801940184334,61.30516815185547,489.0,20.0,0.014735750287276009 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0009032891213392562,27,,,,,, +0,,265950.0,,,,,,,,27,19.15457534790039,26.473903269084094,91.72444152832031,489.0,20.0,0.014735750287276009 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008997623364456462,28,,,,,, +0,,265950.0,,,,,,,,28,16.341455459594727,15.004156178484287,66.76078796386719,488.0,20.0,0.014336548334608401 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008962493214640079,29,,,,,, +0,,265950.0,,,,,,,,29,36.77142333984375,17.21824589586068,338.03436279296875,488.0,20.0,0.013837902458665854 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008927500226313591,30,,,,,, +0,,265950.0,,,,,,,,30,15.244278907775879,14.11893728893171,58.097007751464844,488.0,20.0,0.013537967240027291 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008892643863946268,31,,,,,, +0,,265950.0,,,,,,,,31,18.78181266784668,14.358131463023371,88.18911743164062,487.0,20.0,0.013476240331622308 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008857923594098315,32,,,,,, +0,,265950.0,,,,,,,,32,28.363462448120117,14.840161363719679,201.1215057373047,487.0,20.0,0.012828946392177321 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008823338885412684,33,,,,,, +0,,265950.0,,,,,,,,33,20.130212783813477,13.909946105461902,101.30636596679688,486.0,20.0,0.01289688243625719 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008788889208606948,34,,,,,, +0,,265950.0,,,,,,,,34,12.51075267791748,23.275077976692007,39.129730224609375,486.0,20.0,0.01289688243625719 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008754574036465205,35,,,,,, +0,,265950.0,,,,,,,,35,33.844932556152344,15.29708086560209,286.369873046875,486.0,20.0,0.012433382144670083 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008720392843829977,36,,,,,, +0,,265950.0,,,,,,,,36,32.11604309082031,14.776263726614866,257.8600769042969,485.0,20.0,0.012197663081549558 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008686345107594254,37,,,,,, +0,,265950.0,,,,,,,,37,34.42442321777344,14.892607813437486,296.26019287109375,485.0,20.0,0.011930005913336778 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008652430306693387,38,,,,,, +0,,265950.0,,,,,,,,38,22.432180404663086,12.967113260563917,125.80069732666016,484.0,20.0,0.01170910633021266 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008618647922097165,39,,,,,, +0,,265950.0,,,,,,,,39,20.799867630004883,12.568953833919814,108.15861511230469,484.0,20.0,0.011487367711406996 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008584997436801912,40,,,,,, +0,,265950.0,,,,,,,,40,29.265487670898438,13.352632017017184,214.1171875,484.0,20.0,0.011211460084796725 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008551478335822487,41,,,,,, +0,,265950.0,,,,,,,,41,33.696651458740234,13.789953339181984,283.8660888671875,483.0,20.0,0.010951292622171486 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008518090106184446,42,,,,,, +0,,265950.0,,,,,,,,42,6.839019775390625,11.204308572696451,11.693046569824219,483.0,20.0,0.011087378110276465 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008484832236916241,43,,,,,, +0,,265950.0,,,,,,,,43,43.67146682739258,15.659776125532883,476.79925537109375,482.0,20.0,0.010891783629042404 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008451704219041287,44,,,,,, +0,,265950.0,,,,,,,,44,27.906335830688477,12.596286044707337,194.69088745117188,482.0,20.0,0.010649377213110963 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008418705545570302,45,,,,,, +0,,265950.0,,,,,,,,45,22.210702896118164,11.463925433566017,123.3288345336914,482.0,20.0,0.010230637145449561 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008385835711493434,46,,,,,, +0,,265950.0,,,,,,,,46,39.826045989990234,14.036272403698261,396.5284729003906,481.0,20.0,0.010070987817745502 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008353094213772584,47,,,,,, +0,,265950.0,,,,,,,,47,21.86585235595703,11.077037320594807,119.52887725830078,481.0,20.0,0.009881748543243427 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008320480551333752,48,,,,,, +0,,265950.0,,,,,,,,48,38.59725570678711,13.629816995049856,372.43701171875,480.0,20.0,0.009905446992303274 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008287994225059268,49,,,,,, +0,,265950.0,,,,,,,,49,33.916988372802734,12.628329215819777,287.59051513671875,480.0,20.0,0.009752424178893507 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008255634737780235,50,,,,,, +0,,265950.0,,,,,,,,50,37.162818908691406,26.0386868846602,345.2687683105469,480.0,20.0,0.009752424178893507 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008223401594268897,51,,,,,, +0,,265950.0,,,,,,,,51,12.743383407592773,21.509089917017135,40.59845733642578,479.0,20.0,0.009752424178893507 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008191294301231038,52,,,,,, +0,,265950.0,,,,,,,,52,35.24991226196289,12.679598518056062,310.63909912109375,479.0,20.0,0.009573207564992097 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008159312367298446,53,,,,,, +0,,265950.0,,,,,,,,53,33.92365646362305,12.504382677184942,287.70367431640625,478.0,20.0,0.009627346105682257 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008127455303021438,54,,,,,, +0,,265950.0,,,,,,,,54,29.602313995361328,21.628032413407574,219.0742645263672,478.0,20.0,0.009627346105682257 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000809572262086128,55,,,,,, +0,,265950.0,,,,,,,,55,37.15911865234375,22.334624810592448,345.2000427246094,478.0,20.0,0.009627346105682257 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008064113835182805,56,,,,,, +0,,265950.0,,,,,,,,56,23.692808151245117,11.005552082489762,140.33729553222656,477.0,20.0,0.009602179198693071 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008032628462246943,57,,,,,, +0,,265950.0,,,,,,,,57,33.96633529663086,12.429734073268627,288.427978515625,477.0,20.0,0.009545454345332835 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0008001266020203324,58,,,,,, +0,,265950.0,,,,,,,,58,26.303083419799805,11.110233700258412,172.96304321289062,476.0,20.0,0.009380603349191823 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007970026029082913,59,,,,,, +0,,265950.0,,,,,,,,59,26.535852432250977,10.949636284151007,176.03785705566406,476.0,20.0,0.009189257804193426 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007938908010790631,60,,,,,, +0,,265950.0,,,,,,,,60,32.76526641845703,11.702519750844354,268.3906555175781,476.0,20.0,0.009018613195668573 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007907911489098111,61,,,,,, +0,,265950.0,,,,,,,,61,29.515716552734375,23.196589535862575,217.7943878173828,475.0,20.0,0.009018613195668573 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007877035989636316,62,,,,,, +0,,265950.0,,,,,,,,62,34.553287506103516,11.96537082792906,298.482421875,475.0,20.0,0.008980546647326034 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007846281039888359,63,,,,,, +0,,265950.0,,,,,,,,63,30.214879989624023,20.39101091362689,228.2347412109375,474.0,20.0,0.008980546647326034 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007815646169182205,64,,,,,, +0,,265950.0,,,,,,,,64,22.374677658081055,10.105215012524885,125.15655517578125,474.0,20.0,0.008853649436925215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007785130908683553,65,,,,,, +0,,265950.0,,,,,,,,65,31.6043701171875,19.760219922783868,249.7090606689453,474.0,20.0,0.008853649436925215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007754734791388557,66,,,,,, +0,,265950.0,,,,,,,,66,10.085225105285645,16.12928442158313,25.42794418334961,473.0,20.0,0.008853649436925215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007724457352116771,67,,,,,, +0,,265950.0,,,,,,,,67,9.112401962280273,15.737837628681284,20.758968353271484,473.0,20.0,0.008853649436925215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007694298127503952,68,,,,,, +0,,265950.0,,,,,,,,68,10.890023231506348,9.302777108554288,29.648149490356445,472.0,20.0,0.009006295618419095 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007664256655995059,69,,,,,, +0,,265950.0,,,,,,,,69,36.255279541015625,12.158936631376358,328.6112976074219,472.0,20.0,0.008872823846037003 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000763433247783708,70,,,,,, +0,,265950.0,,,,,,,,70,31.491973876953125,17.028936898834182,247.93609619140625,472.0,20.0,0.008872823846037003 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007604525135072119,71,,,,,, +0,,265950.0,,,,,,,,71,29.477224349975586,16.988007160499908,217.22669982910156,471.0,20.0,0.008872823846037003 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007574834171530245,72,,,,,, +0,,265950.0,,,,,,,,72,12.424270629882812,9.005166946735187,38.59062194824219,471.0,20.0,0.008619260727252765 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007545259132822654,73,,,,,, +0,,265950.0,,,,,,,,73,28.19713592529297,10.655918498315458,198.76963806152344,470.0,20.0,0.008668222208299284 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007515799566334627,74,,,,,, +0,,265950.0,,,,,,,,74,18.09131622314453,9.147386268861725,81.82392883300781,470.0,20.0,0.008329146997220947 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007486455021218607,75,,,,,, +0,,265950.0,,,,,,,,75,12.769766807556152,13.693224956613156,40.76673889160156,470.0,20.0,0.008329146997220947 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007457225048387342,76,,,,,, +0,,265950.0,,,,,,,,76,32.78950119018555,10.819606617583716,268.787841796875,469.0,20.0,0.008131728247298681 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007428109200506983,77,,,,,, +0,,265950.0,,,,,,,,77,38.52956008911133,15.942518564803379,371.1317138671875,469.0,20.0,0.008131728247298681 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000739910703199022,78,,,,,, +0,,265950.0,,,,,,,,78,10.513257026672363,8.436974844884297,27.632144927978516,468.0,20.0,0.008160653403949163 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007370218098989532,79,,,,,, +0,,265950.0,,,,,,,,79,32.2757682800293,10.542796746402718,260.4312744140625,468.0,20.0,0.007938484088092782 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007341441959390305,80,,,,,, +0,,265950.0,,,,,,,,80,21.37489891052246,9.016786984117246,114.2215805053711,468.0,20.0,0.00787457125535748 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007312778172804112,81,,,,,, +0,,265950.0,,,,,,,,81,21.630189895629883,8.890247385516755,116.96627807617188,467.0,20.0,0.007720584671512237 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007284226300562008,82,,,,,, +0,,265950.0,,,,,,,,82,29.054136276245117,9.718203033513944,211.0357208251953,467.0,20.0,0.007607845987386626 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007255785905707734,83,,,,,, +0,,265950.0,,,,,,,,83,44.529869079589844,12.589971581328586,495.7273254394531,466.0,20.0,0.007632698574889377 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00072274565529911,84,,,,,, +0,,265950.0,,,,,,,,84,6.6534600257873535,7.545219757034042,11.067133903503418,466.0,20.0,0.007434548423065403 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007199237808861298,85,,,,,, +0,,265950.0,,,,,,,,85,29.3704891204834,15.89414414263199,215.65640258789062,465.0,20.0,0.007434548423065403 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007171129241460244,86,,,,,, +0,,265950.0,,,,,,,,86,17.262666702270508,13.118998900828881,74.49991607666016,465.0,20.0,0.007434548423065403 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007143130420616034,87,,,,,, +0,,265950.0,,,,,,,,87,43.5029296875,12.322715448832614,473.126220703125,465.0,20.0,0.007591453241801363 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007115240917836285,88,,,,,, +0,,265950.0,,,,,,,,88,25.46869468688965,9.20975538672224,162.16360473632812,464.0,20.0,0.007588119353664072 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007087460306301606,89,,,,,, +0,,265950.0,,,,,,,,89,41.31194305419922,11.682972782192227,426.669189453125,464.0,20.0,0.007416281097469325 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007059788160859116,90,,,,,, +0,,265950.0,,,,,,,,90,35.69619369506836,10.503351339738689,318.5545654296875,464.0,20.0,0.007317805656831586 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007032224058015844,91,,,,,, +0,,265950.0,,,,,,,,91,35.22724151611328,10.322789954365437,310.2395935058594,463.0,20.0,0.007220394181431476 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0007004767575932322,92,,,,,, +0,,265950.0,,,,,,,,92,38.72806930541992,18.55639811610056,374.9658203125,463.0,20.0,0.007220394181431476 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006977418294416108,93,,,,,, +0,,265950.0,,,,,,,,93,29.28601837158203,9.433312742284386,214.417724609375,462.0,20.0,0.007289135543874352 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006950175794915327,94,,,,,, +0,,265950.0,,,,,,,,94,15.490274429321289,7.883372866384988,59.987152099609375,462.0,20.0,0.007283501350157266 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006923039660512314,95,,,,,, +0,,265950.0,,,,,,,,95,27.677814483642578,9.149759300025135,191.51535034179688,461.0,20.0,0.0072346057966071655 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006896009475917208,96,,,,,, +0,,265950.0,,,,,,,,96,34.12641143798828,9.999724688028692,291.15301513671875,461.0,20.0,0.007088194670175909 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006869084827461569,97,,,,,, +0,,265950.0,,,,,,,,97,26.34781265258789,8.752306600183026,173.55181884765625,461.0,20.0,0.007016788502305524 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006842265303092126,98,,,,,, +0,,265950.0,,,,,,,,98,31.832101821899414,9.392549765387162,253.32066345214844,460.0,20.0,0.006859343302527056 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006815550492364372,99,,,,,, +0,,265950.0,,,,,,,,99,37.91231155395508,10.340767427275367,359.3358154296875,460.0,20.0,0.006747409387419408 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006788939986436361,100,,,,,, +0,,265950.0,,,,,,,,100,17.13890838623047,12.888782058322885,73.435546875,459.0,20.0,0.006747409387419408 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006762433378062411,101,,,,,, +0,,265950.0,,,,,,,,101,37.28112030029297,10.202361985034862,347.4704895019531,459.0,20.0,0.006727657242603222 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006736030261586892,102,,,,,, +0,,265950.0,,,,,,,,102,22.716751098632812,7.998157782603402,129.0126953125,459.0,20.0,0.006708030862857002 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000670973023293799,103,,,,,, +0,,265950.0,,,,,,,,103,24.494585037231445,8.255695624509553,149.99618530273438,458.0,20.0,0.006755733771482209 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006683532889621562,104,,,,,, +0,,265950.0,,,,,,,,104,12.9419584274292,7.03560909506731,41.873573303222656,458.0,20.0,0.006616873382300662 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006657437830714936,105,,,,,, +0,,265950.0,,,,,,,,105,17.778762817382812,7.265586205550579,79.02108764648438,458.0,20.0,0.0064753753624643366 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006631444656860808,106,,,,,, +0,,265950.0,,,,,,,,106,9.23563003540039,10.939054179328508,21.324214935302734,457.0,20.0,0.0064753753624643366 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006605552970261116,107,,,,,, +0,,265950.0,,,,,,,,107,22.190746307373047,7.78728725979642,123.10731506347656,457.0,20.0,0.006556214118698398 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006579762374670949,108,,,,,, +0,,265950.0,,,,,,,,108,36.54587173461914,9.78005809078942,333.9002380371094,456.0,20.0,0.006441055719955071 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006554072475392491,109,,,,,, +0,,265950.0,,,,,,,,109,28.689218521118164,8.298997788903687,205.767822265625,456.0,20.0,0.0062413195662474365 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006528482879268979,110,,,,,, +0,,265950.0,,,,,,,,110,20.839435577392578,7.256102207993065,108.57051086425781,456.0,20.0,0.006170397166107689 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006502993194678667,111,,,,,, +0,,265950.0,,,,,,,,111,28.410350799560547,8.066364017570463,201.78701782226562,455.0,20.0,0.006048493829811063 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006477603031528868,112,,,,,, +0,,265950.0,,,,,,,,112,31.587411880493164,8.463558945425326,249.44114685058594,455.0,20.0,0.005969147476919468 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006452312001249968,113,,,,,, +0,,265950.0,,,,,,,,113,30.265737533569336,12.242332560186558,229.00372314453125,454.0,20.0,0.005969147476919468 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006427119716789453,114,,,,,, +0,,265950.0,,,,,,,,114,32.98884201049805,8.838071080885008,272.06591796875,454.0,20.0,0.006117412063322142 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006402025792606032,115,,,,,, +0,,265950.0,,,,,,,,115,23.69048309326172,10.180095228424767,140.30975341796875,454.0,20.0,0.006117412063322142 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006377029844663714,116,,,,,, +0,,265950.0,,,,,,,,116,10.571578979492188,8.474368747512948,27.93956756591797,453.0,20.0,0.006117412063322142 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006352131490425903,117,,,,,, +0,,265950.0,,,,,,,,117,33.80846405029297,9.280552606720962,285.7530517578125,453.0,20.0,0.00642302225126747 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006327330348849617,118,,,,,, +0,,265950.0,,,,,,,,118,18.51886558532715,9.102520291184277,85.73709106445312,452.0,20.0,0.00642302225126747 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006302626040379576,119,,,,,, +0,,265950.0,,,,,,,,119,16.2149658203125,7.119260280862212,65.73127746582031,452.0,20.0,0.0064619475300458674 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006278018186942426,120,,,,,, +0,,265950.0,,,,,,,,120,20.7990779876709,9.336825154541666,108.15042877197266,452.0,20.0,0.0064619475300458674 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000625350641194099,121,,,,,, +0,,265950.0,,,,,,,,121,15.463083267211914,7.047707999426915,59.77674102783203,451.0,20.0,0.006449940586764409 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006229090340248432,122,,,,,, +0,,265950.0,,,,,,,,122,38.54520797729492,10.029350296140297,371.4332580566406,451.0,20.0,0.006315017715573892 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006204769598202576,123,,,,,, +0,,265950.0,,,,,,,,123,29.800954818725586,10.081239364691385,222.02423095703125,450.0,20.0,0.006315017715573892 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006180543813600164,124,,,,,, +0,,265950.0,,,,,,,,124,16.547473907470703,6.923378946976029,68.4547348022461,450.0,20.0,0.006238831641868912 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006156412615691141,125,,,,,, +0,,265950.0,,,,,,,,125,36.36060333251953,9.476170313989849,330.52337646484375,450.0,20.0,0.006170936597025127 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000613237563517304,126,,,,,, +0,,265950.0,,,,,,,,126,18.247177124023438,6.8887806697362155,83.23987579345703,449.0,20.0,0.006056381957101176 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006108432504185257,127,,,,,, +0,,265950.0,,,,,,,,127,13.130558967590332,6.40210832317591,43.10289764404297,449.0,20.0,0.005971079361040595 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006084582856303457,128,,,,,, +0,,265950.0,,,,,,,,128,40.58843231201172,9.969276940695087,411.85516357421875,448.0,20.0,0.005850725209585468 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006060826326533998,129,,,,,, +0,,265950.0,,,,,,,,129,28.784177780151367,7.821934083918559,207.13223266601562,448.0,20.0,0.005750611881236064 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006037162551308274,130,,,,,, +0,,265950.0,,,,,,,,130,43.304805755615234,10.453183535191027,468.8265380859375,448.0,20.0,0.005764918211552112 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0006013591168477212,131,,,,,, +0,,265950.0,,,,,,,,131,32.41533279418945,11.11612822034097,262.6884765625,447.0,20.0,0.005764918211552112 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005990111817305704,132,,,,,, +0,,265950.0,,,,,,,,132,30.230663299560547,9.690359069542477,228.47325134277344,447.0,20.0,0.005764918211552112 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000596672413846707,133,,,,,, +0,,265950.0,,,,,,,,133,28.467639923095703,8.090764516436202,202.60162353515625,446.0,20.0,0.006064748281084641 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005943427774037619,134,,,,,, +0,,265950.0,,,,,,,,134,24.09848976135254,7.532167548078393,145.18429565429688,446.0,20.0,0.0060803246439875116 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005920222367491091,135,,,,,, +0,,265950.0,,,,,,,,135,21.646114349365234,7.189776991877669,117.1385726928711,446.0,20.0,0.006018391346011275 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005897107563693246,136,,,,,, +0,,265950.0,,,,,,,,136,27.115276336669922,10.411815586806775,183.80953979492188,445.0,20.0,0.006018391346011275 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005874083008896454,137,,,,,, +0,,265950.0,,,,,,,,137,33.08665466308594,10.764641182081622,273.68170166015625,445.0,20.0,0.006018391346011275 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005851148350734213,138,,,,,, +0,,265950.0,,,,,,,,138,26.71845054626465,7.922819454355956,178.46890258789062,444.0,20.0,0.006138130504770995 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005828303238215814,139,,,,,, +0,,265950.0,,,,,,,,139,22.700307846069336,7.391985746309705,128.8260040283203,444.0,20.0,0.00610372576324696 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005805547321720948,140,,,,,, +0,,265950.0,,,,,,,,140,26.636075973510742,8.84470677801005,177.3701629638672,444.0,20.0,0.00610372576324696 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005782880252994357,141,,,,,, +0,,265950.0,,,,,,,,141,16.226665496826172,6.636751891060323,65.82616424560547,443.0,20.0,0.005978490234299153 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005760301685140492,142,,,,,, +0,,265950.0,,,,,,,,142,29.234785079956055,8.126671507341435,213.6681671142578,443.0,20.0,0.005989989950639774 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005737811272618238,143,,,,,, +0,,265950.0,,,,,,,,143,27.749746322631836,7.711939893045902,192.51211547851562,442.0,20.0,0.005786818824091435 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005715408671235579,144,,,,,, +0,,265950.0,,,,,,,,144,28.705150604248047,7.756125967550691,205.99642944335938,442.0,20.0,0.005696161787558015 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005693093538144376,145,,,,,, +0,,265950.0,,,,,,,,145,23.753705978393555,7.001071213198168,141.05963134765625,442.0,20.0,0.005590474961710436 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005670865531835093,146,,,,,, +0,,265950.0,,,,,,,,146,34.82942581176758,8.591344803196206,303.272216796875,441.0,20.0,0.005558622806888833 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005648724312131575,147,,,,,, +0,,265950.0,,,,,,,,147,36.36903381347656,8.78449858754267,330.6766357421875,441.0,20.0,0.005477732315951484 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005626669540185853,148,,,,,, +0,,265950.0,,,,,,,,148,43.105003356933594,10.065047019703844,464.5103454589844,440.0,20.0,0.005419943565114001 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005604700878472933,149,,,,,, +0,,265950.0,,,,,,,,149,27.074893951416016,7.238588710198212,183.2624969482422,440.0,20.0,0.005405963797936249 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005582817990785662,150,,,,,, +0,,265950.0,,,,,,,,150,41.9355354309082,9.781746648779793,439.6473388671875,440.0,20.0,0.0053852732410344314 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005561020542229567,151,,,,,, +0,,265950.0,,,,,,,,151,31.73444175720215,7.952986650305445,251.76870727539062,439.0,20.0,0.005435299568014796 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005539308199217718,152,,,,,, +0,,265950.0,,,,,,,,152,9.924219131469727,10.442858993834976,24.62253189086914,439.0,20.0,0.005435299568014796 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005517680629465647,153,,,,,, +0,,265950.0,,,,,,,,153,25.59379768371582,7.185962856651387,163.7606201171875,438.0,20.0,0.005548356712699971 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000549613750198625,154,,,,,, +0,,265950.0,,,,,,,,154,31.968097686767578,10.451121933713205,255.4898223876953,438.0,20.0,0.005548356712699971 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005474678487084726,155,,,,,, +0,,265950.0,,,,,,,,155,15.262399673461914,6.307126936376801,58.23521423339844,438.0,20.0,0.005724774834573975 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005453303256353507,156,,,,,, +0,,265950.0,,,,,,,,156,22.971956253051758,6.9531139278805885,131.92770385742188,437.0,20.0,0.005633836879769627 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000543201148266729,157,,,,,, +0,,265950.0,,,,,,,,157,34.483604431152344,8.510440485492463,297.27972412109375,437.0,20.0,0.005537643330112215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005410802840177958,158,,,,,, +0,,265950.0,,,,,,,,158,22.012451171875,6.654850124466646,121.13700866699219,436.0,20.0,0.005443480013954866 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000538967700430965,159,,,,,, +0,,265950.0,,,,,,,,159,5.513007164001465,8.771334976426253,7.598311901092529,436.0,20.0,0.005443480013954866 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005368633651753771,160,,,,,, +0,,265950.0,,,,,,,,160,12.209976196289062,5.7981330960375645,37.27088165283203,436.0,20.0,0.0054254242985827305 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005347672460464033,161,,,,,, +0,,265950.0,,,,,,,,161,21.437458038330078,8.811420356188375,114.89115905761719,435.0,20.0,0.0054254242985827305 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005326793109651553,162,,,,,, +0,,265950.0,,,,,,,,162,17.51696014404297,6.170930808843948,76.71096801757812,435.0,20.0,0.005403821116747237 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005305995279779931,163,,,,,, +0,,265950.0,,,,,,,,163,33.44075393676758,8.179228962887468,279.5710144042969,434.0,20.0,0.005383518876064958 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005285278652560339,164,,,,,, +0,,265950.0,,,,,,,,164,34.63035202026367,9.568410871923042,299.8153076171875,434.0,20.0,0.005383518876064958 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005264642910946703,165,,,,,, +0,,265950.0,,,,,,,,165,35.2889404296875,9.350760632428246,311.32733154296875,434.0,20.0,0.005383518876064958 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005244087739130791,166,,,,,, +0,,265950.0,,,,,,,,166,18.754043579101562,6.334427206922824,87.92854309082031,433.0,20.0,0.005455141811777408 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005223612822537403,167,,,,,, +0,,265950.0,,,,,,,,167,21.667736053466797,7.289211140068908,117.37271118164062,433.0,20.0,0.005455141811777408 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005203217847819591,168,,,,,, +0,,265950.0,,,,,,,,168,11.99553108215332,5.8609319639206205,35.97319030761719,432.0,20.0,0.005501200081110028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005182902502853797,169,,,,,, +0,,265950.0,,,,,,,,169,6.809325218200684,6.264088814019899,11.591727256774902,432.0,20.0,0.005501200081110028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005162666476735146,170,,,,,, +0,,265950.0,,,,,,,,170,16.8325138092041,6.136051574769652,70.8333740234375,432.0,20.0,0.00542771784407202 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005142509459772625,171,,,,,, +0,,265950.0,,,,,,,,171,13.82982063293457,5.7895723876592315,47.81597900390625,431.0,20.0,0.005311412602388541 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005122431143484389,172,,,,,, +0,,265950.0,,,,,,,,172,28.072566986083984,7.192038035039337,197.0172576904297,431.0,20.0,0.005221865510587127 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005102431220593019,173,,,,,, +0,,265950.0,,,,,,,,173,32.01354217529297,7.629764126428842,256.21673583984375,430.0,20.0,0.005067596720346689 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005082509385020835,174,,,,,, +0,,265950.0,,,,,,,,174,23.16834259033203,6.330587707862815,134.19302368164062,430.0,20.0,0.004988657556877097 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005062665331885163,175,,,,,, +0,,265950.0,,,,,,,,175,25.376907348632812,6.535373386900705,160.99685668945312,430.0,20.0,0.004925404843848032 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005042898757493771,176,,,,,, +0,,265950.0,,,,,,,,176,27.487003326416016,6.8338473535163216,188.88385009765625,429.0,20.0,0.0049450089431388195 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005023209359340094,177,,,,,, +0,,265950.0,,,,,,,,177,45.90589904785156,10.107900107473057,526.837890625,429.0,20.0,0.0048395213728844345 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0005003596836098705,178,,,,,, +0,,265950.0,,,,,,,,178,37.890830993652344,8.443261752386038,358.92877197265625,428.0,20.0,0.004853973994512502 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004984060887620653,179,,,,,, +0,,265950.0,,,,,,,,179,16.57732582092285,9.238672638341503,68.70191955566406,428.0,20.0,0.004853973994512502 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004964601214928893,180,,,,,, +0,,265950.0,,,,,,,,180,41.661006927490234,11.557532731743521,433.909912109375,428.0,20.0,0.004853973994512502 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004945217520213681,181,,,,,, +0,,265950.0,,,,,,,,181,41.85374069213867,9.798131711540243,437.93389892578125,427.0,20.0,0.005418792970237753 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004925909506828068,182,,,,,, +0,,265950.0,,,,,,,,182,32.862613677978516,8.189774920469018,269.9878234863281,427.0,20.0,0.005489896704679223 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004906676879283281,183,,,,,, +0,,265950.0,,,,,,,,183,28.027538299560547,9.34808150948378,196.38571166992188,426.0,20.0,0.005489896704679223 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004887519343244298,184,,,,,, +0,,265950.0,,,,,,,,184,11.814212799072266,5.88982209919629,34.89390563964844,426.0,20.0,0.005540883038031434 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004868436605525257,185,,,,,, +0,,265950.0,,,,,,,,185,27.843782424926758,9.143218148837516,193.81906127929688,426.0,20.0,0.005540883038031434 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00048494283740850206,186,,,,,, +0,,265950.0,,,,,,,,186,33.60511779785156,9.160613692999805,282.325927734375,425.0,20.0,0.005540883038031434 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004830494358022689,187,,,,,, +0,,265950.0,,,,,,,,187,20.270954132080078,6.984080403904383,102.72789001464844,425.0,20.0,0.0059568015037578985 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00048116342675731547,188,,,,,, +0,,265950.0,,,,,,,,188,31.505495071411133,8.435875428360205,248.1490478515625,424.0,20.0,0.0059568015037578985 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004792847814102645,189,,,,,, +0,,265950.0,,,,,,,,189,14.423304557800293,6.433563766769423,52.007930755615234,424.0,20.0,0.005913484452060714 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000477413471010436,190,,,,,, +0,,265950.0,,,,,,,,190,21.295270919799805,6.907315642307514,113.37214660644531,424.0,20.0,0.005773594171474689 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004755494669193987,191,,,,,, +0,,265950.0,,,,,,,,191,14.845475196838379,6.176770208577809,55.09703063964844,423.0,20.0,0.005625799892644581 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004736927406105413,192,,,,,, +0,,265950.0,,,,,,,,192,26.84942054748535,7.264331733920028,180.22283935546875,423.0,20.0,0.005462103402354171 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004718432636686277,193,,,,,, +0,,265950.0,,,,,,,,193,9.610553741455078,5.526824420133182,23.09068489074707,422.0,20.0,0.005295917575994083 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00047000100778936887,194,,,,,, +0,,265950.0,,,,,,,,194,23.494604110717773,6.508857428906613,137.99911499023438,422.0,20.0,0.005128866255162411 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00046816594477898366,195,,,,,, +0,,265950.0,,,,,,,,195,26.816823959350586,6.856570308779003,179.7855224609375,422.0,20.0,0.005058715170000316 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004663380465537723,196,,,,,, +0,,265950.0,,,,,,,,196,17.360279083251953,5.695036347100785,75.34481811523438,421.0,20.0,0.004941588158795885 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004645172851396822,197,,,,,, +0,,265950.0,,,,,,,,197,9.405638694763184,5.05458706012029,22.11650848388672,421.0,20.0,0.00483342198422212 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004627036326718854,198,,,,,, +0,,265950.0,,,,,,,,198,10.170571327209473,4.9943702272651125,25.860126495361328,420.0,20.0,0.0047357689790008 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004608970613943456,199,,,,,, +0,,265950.0,,,,,,,,199,21.699016571044922,5.825168301288499,117.71183776855469,420.0,20.0,0.0046480499998968975 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00045909754365939886,200,,,,,, +0,,265950.0,,,,,,,,200,29.756576538085938,13.957113436750369,221.3634490966797,420.0,20.0,0.0046480499998968975 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004573050519273262,201,,,,,, +0,,265950.0,,,,,,,,201,30.299213409423828,6.942346006100501,229.5105743408203,419.0,20.0,0.0046472403103760135 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00045551955876593704,202,,,,,, +0,,265950.0,,,,,,,,202,8.945772171020508,9.066535367203931,20.006710052490234,419.0,20.0,0.0046472403103760135 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00045374103685014367,203,,,,,, +0,,265950.0,,,,,,,,203,24.973453521728516,6.307227561714663,155.9183349609375,418.0,20.0,0.004748044202568545 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00045196945896154795,204,,,,,, +0,,265950.0,,,,,,,,204,32.8907356262207,7.517947201004202,270.45013427734375,418.0,20.0,0.004813445810547049 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004502047979880212,205,,,,,, +0,,265950.0,,,,,,,,205,26.282161712646484,7.751281717134514,172.68800354003906,418.0,20.0,0.004813445810547049 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004484470269232915,206,,,,,, +0,,265950.0,,,,,,,,206,27.548763275146484,7.830840126855633,189.73358154296875,417.0,20.0,0.004813445810547049 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000446696118866531,207,,,,,, +0,,265950.0,,,,,,,,207,19.396997451782227,5.927291015361907,94.06088256835938,417.0,20.0,0.004986682215904357 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004449520470219409,208,,,,,, +0,,265950.0,,,,,,,,208,19.338970184326172,6.869984878232231,93.49894714355469,416.0,20.0,0.004986682215904357 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004432147846983434,209,,,,,, +0,,265950.0,,,,,,,,209,38.974117279052734,8.814055697131833,379.7454833984375,416.0,20.0,0.0050166008631474576 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004414843053087756,210,,,,,, +0,,265950.0,,,,,,,,210,30.421398162841797,7.296995852404032,231.3653564453125,416.0,20.0,0.004983342383318338 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00043976058237007754,211,,,,,, +0,,265950.0,,,,,,,,211,31.14910316467285,8.254959479433815,242.566650390625,415.0,20.0,0.004983342383318338 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004380435895024911,212,,,,,, +0,,265950.0,,,,,,,,212,26.533714294433594,6.741681373380101,176.00949096679688,415.0,20.0,0.004981586492322361 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00043633330042925446,213,,,,,, +0,,265950.0,,,,,,,,213,28.375951766967773,6.965711220957734,201.2986602783203,414.0,20.0,0.004952724560954072 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00043462968897620016,214,,,,,, +0,,265950.0,,,,,,,,214,3.4196648597717285,4.881823565640425,2.9235265254974365,414.0,20.0,0.0048525883013540265 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00043293272907135523,215,,,,,, +0,,265950.0,,,,,,,,215,29.052976608276367,7.68887101742082,211.0188751220703,414.0,20.0,0.0048525883013540265 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00043124239474454074,216,,,,,, +0,,265950.0,,,,,,,,216,28.529085159301758,6.948106523477515,203.4771728515625,413.0,20.0,0.004913334842645605 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004295586601269748,217,,,,,, +0,,265950.0,,,,,,,,217,37.773189544677734,8.354595622505087,356.70343017578125,413.0,20.0,0.004787561377967734 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004278814994508796,218,,,,,, +0,,265950.0,,,,,,,,218,12.326604843139648,5.074280016629691,37.986297607421875,412.0,20.0,0.004694417052476402 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004262108870490816,219,,,,,, +0,,265950.0,,,,,,,,219,8.912463188171387,4.809228841086627,19.858001708984375,412.0,20.0,0.004610648829361201 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00042454679735462345,220,,,,,, +0,,265950.0,,,,,,,,220,35.45529556274414,7.727809891685235,314.26947021484375,412.0,20.0,0.004585115180000054 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00042288920490037046,221,,,,,, +0,,265950.0,,,,,,,,221,31.96426773071289,7.022962632328283,255.42860412597656,411.0,20.0,0.00446867662921549 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00042123808431862125,222,,,,,, +0,,265950.0,,,,,,,,222,39.978668212890625,9.426190903205704,399.573486328125,411.0,20.0,0.00446867662921549 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00041959341034071985,223,,,,,, +0,,265950.0,,,,,,,,223,20.818592071533203,5.653213145350372,108.35344696044922,410.0,20.0,0.004569678666209136 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004179551577966692,224,,,,,, +0,,265950.0,,,,,,,,224,27.5214786529541,6.758893852153082,189.35794067382812,410.0,20.0,0.004569678666209136 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004163233016147432,225,,,,,, +0,,265950.0,,,,,,,,225,24.633079528808594,6.304701603650837,151.69715881347656,410.0,20.0,0.0047877300155160715 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00041469781682110875,226,,,,,, +0,,265950.0,,,,,,,,226,15.395917892456055,5.373018779157447,59.25857162475586,409.0,20.0,0.00478043309628849 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004130786785394373,227,,,,,, +0,,265950.0,,,,,,,,227,38.77360534667969,8.469592878685614,375.84814453125,409.0,20.0,0.004711111614571235 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00041146586199052845,228,,,,,, +0,,265950.0,,,,,,,,228,9.371313095092773,4.864395690765178,21.955379486083984,408.0,20.0,0.004644841907229221 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00040985934249192885,229,,,,,, +0,,265950.0,,,,,,,,229,16.78853988647461,5.258647595391756,70.4637680053711,408.0,20.0,0.004554009948716646 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004082590954575552,230,,,,,, +0,,265950.0,,,,,,,,230,21.419736862182617,5.676003663059638,114.70127868652344,408.0,20.0,0.004528990952488349 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004066650963973162,231,,,,,, +0,,265950.0,,,,,,,,231,25.925745010375977,6.10988563519662,168.03607177734375,407.0,20.0,0.004429524960338526 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00040507732091674164,232,,,,,, +0,,265950.0,,,,,,,,232,22.195453643798828,7.899025510017168,123.15953063964844,407.0,20.0,0.004429524960338526 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004034957447166043,233,,,,,, +0,,265950.0,,,,,,,,233,37.95513153076172,8.056373061377407,360.14801025390625,406.0,20.0,0.004454893054205777 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0004019203435925519,234,,,,,, +0,,265950.0,,,,,,,,234,38.42048645019531,8.223460444293966,369.0334167480469,406.0,20.0,0.004533126362644186 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00040035109343473414,235,,,,,, +0,,265950.0,,,,,,,,235,4.912154197692871,4.547118354897983,6.032315254211426,406.0,20.0,0.004486795202206857 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003987879702274354,236,,,,,, +0,,265950.0,,,,,,,,236,26.851648330688477,6.943991105837013,180.25274658203125,405.0,20.0,0.004486795202206857 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003972309500487065,237,,,,,, +0,,265950.0,,,,,,,,237,23.15769386291504,6.015908355855383,134.0697021484375,405.0,20.0,0.004675211305760778 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003956800090699989,238,,,,,, +0,,265950.0,,,,,,,,238,28.543733596801758,6.717760010788736,203.68618774414062,404.0,20.0,0.004680898114273844 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003941351235557985,239,,,,,, +0,,265950.0,,,,,,,,239,37.245296478271484,8.064346028007165,346.8030090332031,404.0,20.0,0.0045963160521160515 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00039259626986326633,240,,,,,, +0,,265950.0,,,,,,,,240,27.875032424926758,6.551025221379162,194.2543487548828,404.0,20.0,0.004608481833966138 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003910634244418706,241,,,,,, +0,,265950.0,,,,,,,,241,35.853111267089844,7.789716805899124,321.36138916015625,403.0,20.0,0.004576103057348709 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003895365638330324,242,,,,,, +0,,265950.0,,,,,,,,242,17.135986328125,5.275711402782361,73.41050720214844,403.0,20.0,0.004541606352218549 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003880156646697632,243,,,,,, +0,,265950.0,,,,,,,,243,10.880547523498535,4.774668410427079,29.596580505371094,402.0,20.0,0.004478702619678483 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003865007036763084,244,,,,,, +0,,265950.0,,,,,,,,244,32.60651397705078,8.689802066358823,265.796142578125,402.0,20.0,0.004478702619678483 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003849916576677895,245,,,,,, +0,,265950.0,,,,,,,,245,27.015600204467773,6.335665875895655,182.46066284179688,402.0,20.0,0.004511059338076747 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000383488503549853,246,,,,,, +0,,265950.0,,,,,,,,246,16.488298416137695,5.229913477402602,67.96599578857422,401.0,20.0,0.0045502535147484875 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00038199121831831187,247,,,,,, +0,,265950.0,,,,,,,,247,28.06425666809082,6.403927840148652,196.900634765625,401.0,20.0,0.004434921540176118 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003804997790587988,248,,,,,, +0,,265950.0,,,,,,,,248,27.331018447875977,6.52754208139691,186.7461395263672,400.0,20.0,0.004434921540176118 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00037901416294641103,249,,,,,, +0,,265950.0,,,,,,,,249,35.0700569152832,7.346726429042726,307.47723388671875,400.0,20.0,0.004434921540176118 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003775343472453631,250,,,,,, +0,,265950.0,,,,,,,,250,23.439861297607422,5.38595089322288,137.35678100585938,400.0,20.0,0.004434921540176118 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003760603093086394,251,,,,,, +0,,265950.0,,,,,,,,251,16.36183738708496,5.72797329078046,66.92743682861328,399.0,20.0,0.0050586989606412995 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003745920265776467,252,,,,,, +0,,265950.0,,,,,,,,252,27.6116886138916,7.01737336291832,190.6013641357422,399.0,20.0,0.005111359754939499 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00037312947658186895,253,,,,,, +0,,265950.0,,,,,,,,253,25.840320587158203,6.900589689841044,166.9305419921875,398.0,20.0,0.005111359754939499 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00037167263693852403,254,,,,,, +0,,265950.0,,,,,,,,254,18.646142959594727,5.730939308841738,86.91966247558594,398.0,20.0,0.004861742715080294 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00037022148535221995,255,,,,,, +0,,265950.0,,,,,,,,255,21.709674835205078,5.900618434639889,117.82748413085938,398.0,20.0,0.004861742715080294 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003687759996146142,256,,,,,, +0,,265950.0,,,,,,,,256,20.46659278869629,5.384154117154568,104.7203598022461,397.0,20.0,0.004861742715080294 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003673361576040759,257,,,,,, +0,,265950.0,,,,,,,,257,35.42508316040039,8.289668839092212,313.7341003417969,397.0,20.0,0.005152327816600756 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00036590193728534305,258,,,,,, +0,,265950.0,,,,,,,,258,20.74295997619629,5.3172037533955985,107.56758117675781,396.0,20.0,0.005152327816600756 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003644733167091902,259,,,,,, +0,,265950.0,,,,,,,,259,27.62245750427246,7.008536531005784,190.75003051757812,396.0,20.0,0.0051010362639769755 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003630502740120878,260,,,,,, +0,,265950.0,,,,,,,,260,36.56535720825195,8.309752580555248,334.25634765625,396.0,20.0,0.004967189189823437 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00036163278741587103,261,,,,,, +0,,265950.0,,,,,,,,261,28.98014259338379,6.943842093129985,209.962158203125,395.0,20.0,0.004844220558782451 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00036022083522740516,262,,,,,, +0,,265950.0,,,,,,,,262,15.616145133972168,5.361482136208366,60.96599578857422,395.0,20.0,0.004751822166401695 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003588143958382543,263,,,,,, +0,,265950.0,,,,,,,,263,30.70318603515625,7.005627597015943,235.67141723632812,394.0,20.0,0.004648913586777295 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00035741344772434783,264,,,,,, +0,,265950.0,,,,,,,,264,14.841007232666016,5.114933127748902,55.06386947631836,394.0,20.0,0.004564294452059204 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00035601796944565665,265,,,,,, +0,,265950.0,,,,,,,,265,28.44429588317871,6.46761139279427,202.26950073242188,394.0,20.0,0.004444916328249592 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00035462793964585894,266,,,,,, +0,,265950.0,,,,,,,,266,18.9840145111084,5.292308665703592,90.09819793701172,393.0,20.0,0.0043913267030227755 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00035324333705201753,267,,,,,, +0,,265950.0,,,,,,,,267,31.469614028930664,6.774434355169694,247.58413696289062,393.0,20.0,0.004298593071371476 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00035186414047425253,268,,,,,, +0,,265950.0,,,,,,,,268,29.946949005126953,8.978755374709143,224.20494079589844,392.0,20.0,0.004298593071371476 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003504903288054178,269,,,,,, +0,,265950.0,,,,,,,,269,40.91755676269531,10.146878054565672,418.56158447265625,392.0,20.0,0.004298593071371476 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003491218810207761,270,,,,,, +0,,265950.0,,,,,,,,270,10.964113235473633,5.25425039908122,30.052946090698242,392.0,20.0,0.004298593071371476 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003477587761776812,271,,,,,, +0,,265950.0,,,,,,,,271,23.65837860107422,5.623916142606374,139.92971801757812,391.0,20.0,0.004298593071371476 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00034640099341525226,272,,,,,, +0,,265950.0,,,,,,,,272,30.630735397338867,6.486818644419689,234.56048583984375,391.0,20.0,0.004298593071371476 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003450485119540601,273,,,,,, +0,,265950.0,,,,,,,,273,31.966047286987305,8.362247419029096,255.45704650878906,390.0,20.0,0.0058076769825514355 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00034370131109580344,274,,,,,, +0,,265950.0,,,,,,,,274,28.50366973876953,7.187289451238062,203.11480712890625,390.0,20.0,0.0058076769825514355 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00034235937022299693,275,,,,,, +0,,265950.0,,,,,,,,275,32.55539321899414,8.479631396785889,264.96337890625,390.0,20.0,0.005829997750774536 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003410226687986533,276,,,,,, +0,,265950.0,,,,,,,,276,11.662156105041504,5.994306902656845,34.00147247314453,389.0,20.0,0.005654292176733307 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003396911863659694,277,,,,,, +0,,265950.0,,,,,,,,277,17.483205795288086,6.145938569078019,76.41561889648438,389.0,20.0,0.005381782406339218 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003383649025480134,278,,,,,, +0,,265950.0,,,,,,,,278,21.94440460205078,6.38103527895092,120.38920593261719,388.0,20.0,0.00517714328638195 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003370437970474129,279,,,,,, +0,,265950.0,,,,,,,,279,25.942302703857422,6.708274753149459,168.25076293945312,388.0,20.0,0.005025767118986555 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003357278496460437,280,,,,,, +0,,265950.0,,,,,,,,280,33.93920135498047,9.485276173510027,287.96734619140625,388.0,20.0,0.005025767118986555 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003344170402047201,281,,,,,, +0,,265950.0,,,,,,,,281,26.556303024291992,6.764429477717479,176.309326171875,387.0,20.0,0.005001336244608958 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00033311134866289064,282,,,,,, +0,,265950.0,,,,,,,,282,34.843509674072266,8.001724725337947,303.51751708984375,387.0,20.0,0.004966549640270199 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003318107550383239,283,,,,,, +0,,265950.0,,,,,,,,283,20.161609649658203,5.869170084910466,101.62261962890625,386.0,20.0,0.004852943912463261 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003305152394268091,284,,,,,, +0,,265950.0,,,,,,,,284,20.047407150268555,5.742855140218224,100.47464752197266,386.0,20.0,0.00473810870314547 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003292247820018491,285,,,,,, +0,,265950.0,,,,,,,,285,25.367033004760742,6.250892750153631,160.87158203125,386.0,20.0,0.004642176977524847 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003279393630143546,286,,,,,, +0,,265950.0,,,,,,,,286,19.37509536743164,5.56716428817837,93.84858703613281,385.0,20.0,0.004628678427353785 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003266589627923484,287,,,,,, +0,,265950.0,,,,,,,,287,16.424171447753906,5.189154856353015,67.43833923339844,385.0,20.0,0.004514771454482288 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00032538356174065545,288,,,,,, +0,,265950.0,,,,,,,,288,20.787172317504883,6.484901661819047,108.02661895751953,384.0,20.0,0.004514771454482288 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00032411314034061245,289,,,,,, +0,,265950.0,,,,,,,,289,36.172298431396484,7.818413714519902,327.10882568359375,384.0,20.0,0.004547325591198369 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00032284767914976027,290,,,,,, +0,,265950.0,,,,,,,,290,16.398242950439453,5.135304019197878,67.2255859375,384.0,20.0,0.004547325591198369 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000321587158801553,291,,,,,, +0,,265950.0,,,,,,,,291,23.278535842895508,5.970683909707289,135.47256469726562,383.0,20.0,0.004615958310418349 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00032033156000505836,292,,,,,, +0,,265950.0,,,,,,,,292,26.653470993041992,5.91685099182093,177.60186767578125,383.0,20.0,0.004615958310418349 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003190808635446633,293,,,,,, +0,,265950.0,,,,,,,,293,15.281689643859863,5.19331348236867,58.38250732421875,382.0,20.0,0.004609488430584154 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00031783505027977983,294,,,,,, +0,,265950.0,,,,,,,,294,12.3596773147583,4.91181110978572,38.190406799316406,382.0,20.0,0.004529907044176742 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00031659410114455276,295,,,,,, +0,,265950.0,,,,,,,,295,5.442988395690918,4.52112021463746,7.406530380249023,382.0,20.0,0.0044470549093448535 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003153579971475654,296,,,,,, +0,,265950.0,,,,,,,,296,21.91683006286621,5.473312502448329,120.08686828613281,381.0,20.0,0.004272443895880947 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00031412671937155264,297,,,,,, +0,,265950.0,,,,,,,,297,31.476993560791016,7.5740713545564375,247.70028686523438,381.0,20.0,0.004272443895880947 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00031290024897310886,298,,,,,, +0,,265950.0,,,,,,,,298,24.209165573120117,6.016695776299955,146.52093505859375,380.0,20.0,0.004272443895880947 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00031167856718240046,299,,,,,, +0,,265950.0,,,,,,,,299,27.86842155456543,5.9782228420787735,194.1622314453125,380.0,20.0,0.004272443895880947 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003104616553028783,300,,,,,, +0,,395377.0,,,,,,,,300,25.121488571166992,8.83906864246668,157.7722930908203,380.0,20.0,0.007261345735400335 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003092494947109922,301,,,,,, +0,,395377.0,,,,,,,,301,17.54111099243164,31.71597472418204,76.92263793945312,379.0,20.0,0.006673435566083305 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00030804206685590337,302,,,,,, +0,,395377.0,,,,,,,,302,25.616445541381836,23.93796282858222,164.05056762695312,379.0,20.0,0.006035703633950566 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003068393532592062,303,,,,,, +0,,395377.0,,,,,,,,303,10.767082214355469,6.534185594410792,28.98251724243164,378.0,20.0,0.006035703633950566 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00030564133551463777,304,,,,,, +0,,395377.0,,,,,,,,304,18.5352725982666,20.26640552610345,85.88908386230469,378.0,20.0,0.0055755765531212715 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0003044479952878041,305,,,,,, +0,,395377.0,,,,,,,,305,24.049278259277344,19.28033974795551,144.59194946289062,378.0,20.0,0.0054130264063529665 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00030325931431589184,306,,,,,, +0,,395377.0,,,,,,,,306,34.03902816772461,19.061099621980745,289.66387939453125,377.0,20.0,0.005273254677038737 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00030207527440739656,307,,,,,, +0,,395377.0,,,,,,,,307,30.73919105529785,17.03766626990538,236.22447204589844,377.0,20.0,0.0051112068182137885 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00030089585744183674,308,,,,,, +0,,395377.0,,,,,,,,308,27.620819091796875,7.287907156623355,190.7274169921875,376.0,20.0,0.0051112068182137885 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00029972104536948245,309,,,,,, +0,,395377.0,,,,,,,,309,26.10537338256836,6.446056915402482,170.37261962890625,376.0,20.0,0.0051112068182137885 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00029855082021107594,310,,,,,, +0,,395377.0,,,,,,,,310,36.472347259521484,7.234239474227309,332.55804443359375,376.0,20.0,0.0051112068182137885 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002973851640575576,311,,,,,, +0,,395377.0,,,,,,,,311,26.528255462646484,19.85267051461956,175.93707275390625,375.0,20.0,0.006137513321262355 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002962240590697914,312,,,,,, +0,,395377.0,,,,,,,,312,29.479337692260742,18.21810391608905,217.25784301757812,375.0,20.0,0.0058635042412845 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00029506748747829285,313,,,,,, +0,,395377.0,,,,,,,,313,38.46181106567383,17.566351954062444,369.82769775390625,374.0,20.0,0.005527684632050176 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002939154315829545,314,,,,,, +0,,395377.0,,,,,,,,314,20.165626525878906,13.827451175709086,101.66313171386719,374.0,20.0,0.005257243923136827 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002927678737527785,315,,,,,, +0,,395377.0,,,,,,,,315,27.404010772705078,9.202706067800527,187.74496459960938,374.0,20.0,0.005257243923136827 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00029162479642560476,316,,,,,, +0,,395377.0,,,,,,,,316,24.62763786315918,14.765860889018432,151.630126953125,373.0,20.0,0.005356766054807093 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002904861821078423,317,,,,,, +0,,395377.0,,,,,,,,317,12.525280952453613,12.871434112921353,39.220664978027344,373.0,20.0,0.005231089050129895 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00028935201337420207,318,,,,,, +0,,395377.0,,,,,,,,318,20.518505096435547,12.47545961888956,105.25226593017578,372.0,20.0,0.005023143214059493 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002882222728674302,319,,,,,, +0,,395377.0,,,,,,,,319,27.16765594482422,12.628607673830654,184.52037048339844,372.0,20.0,0.004825479427633593 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00028709694329804035,320,,,,,, +0,,395377.0,,,,,,,,320,12.107037544250488,10.68169531550856,36.64508819580078,372.0,20.0,0.004724864107095619 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00028597600744405434,321,,,,,, +0,,395377.0,,,,,,,,321,27.295364379882812,11.88290188768079,186.2592315673828,371.0,20.0,0.00462388297799261 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002848594481507312,322,,,,,, +0,,395377.0,,,,,,,,322,21.62116813659668,10.897577108726686,116.86872863769531,371.0,20.0,0.004582279238786199 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002837472483303126,323,,,,,, +0,,395377.0,,,,,,,,323,28.903865814208984,8.594498440583383,208.85836791992188,370.0,20.0,0.004582279238786199 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002826393909617537,324,,,,,, +0,,395377.0,,,,,,,,324,21.206417083740234,11.100857242827315,112.42802429199219,370.0,20.0,0.004554073356189813 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00028153585909046854,325,,,,,, +0,,395377.0,,,,,,,,325,11.326798439025879,9.799477195164433,32.07408905029297,370.0,20.0,0.004550831166104255 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002804366358280676,326,,,,,, +0,,395377.0,,,,,,,,326,29.111129760742188,11.056782396428854,211.8644561767578,369.0,20.0,0.004397620453394507 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002793417043521008,327,,,,,, +0,,395377.0,,,,,,,,327,28.936525344848633,6.783321503536622,209.33062744140625,369.0,20.0,0.004397620453394507 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00027825104790579704,328,,,,,, +0,,395377.0,,,,,,,,328,18.258647918701172,5.055700033853429,83.34455108642578,368.0,20.0,0.004397620453394507 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00027716464979781375,329,,,,,, +0,,395377.0,,,,,,,,329,16.58217430114746,10.954194954356336,68.74211883544922,368.0,20.0,0.004671130826954083 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002760824934019741,330,,,,,, +0,,395377.0,,,,,,,,330,12.368617057800293,9.887386601679875,38.24567413330078,368.0,20.0,0.004583221421570258 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002750045621570183,331,,,,,, +0,,395377.0,,,,,,,,331,32.45249557495117,6.687847074111319,263.2911376953125,367.0,20.0,0.004583221421570258 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002739308395663474,332,,,,,, +0,,395377.0,,,,,,,,332,30.72585105895996,11.59525363521587,236.01947021484375,367.0,20.0,0.004413853404729197 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002728613091977716,333,,,,,, +0,,395377.0,,,,,,,,333,17.190784454345703,9.587763542706885,73.88077545166016,366.0,20.0,0.004359957180229971 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002717959546832585,334,,,,,, +0,,395377.0,,,,,,,,334,20.90888786315918,5.261930505408886,109.29539489746094,366.0,20.0,0.004359957180229971 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002707347597186833,335,,,,,, +0,,395377.0,,,,,,,,335,25.42436408996582,5.606688834365102,161.5995635986328,366.0,20.0,0.004359957180229971 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00026967770806357675,336,,,,,, +0,,395377.0,,,,,,,,336,11.991310119628906,10.301230113928302,35.947879791259766,365.0,20.0,0.004586024661019726 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00026862478354088194,337,,,,,, +0,,395377.0,,,,,,,,337,27.578731536865234,5.41507567674641,190.1466064453125,365.0,20.0,0.004586024661019726 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002675759700367,338,,,,,, +0,,395377.0,,,,,,,,338,20.304519653320312,10.101914405766703,103.0683822631836,364.0,20.0,0.0045868619946677885 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00026653125150004884,339,,,,,, +0,,395377.0,,,,,,,,339,32.359100341796875,11.079949012879274,261.77783203125,364.0,20.0,0.00443007812024745 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002654906119426149,340,,,,,, +0,,395377.0,,,,,,,,340,27.177520751953125,9.944012684357265,184.65440368652344,364.0,20.0,0.004312392039062283 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002644540354385093,341,,,,,, +0,,395377.0,,,,,,,,341,11.654278755187988,8.27043008894478,33.95555114746094,363.0,20.0,0.0042199683041052715 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00026342150612402367,342,,,,,, +0,,395377.0,,,,,,,,342,25.117273330688477,9.5151770413547,157.7193603515625,363.0,20.0,0.0041982251943181665 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002623930081973882,343,,,,,, +0,,395377.0,,,,,,,,343,15.689846992492676,8.340141034977574,61.54281997680664,362.0,20.0,0.0041183867158641274 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00026136852591852785,344,,,,,, +0,,395377.0,,,,,,,,344,11.648089408874512,5.3631437951815615,33.91949462890625,362.0,20.0,0.0041183867158641274 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002603480436088236,345,,,,,, +0,,395377.0,,,,,,,,345,10.723748207092285,8.19172430753186,28.74969482421875,362.0,20.0,0.0041757541302785105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002593315456508733,346,,,,,, +0,,395377.0,,,,,,,,346,31.80260467529297,6.883682656585871,252.85140991210938,361.0,20.0,0.0041757541302785105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002583190164882481,347,,,,,, +0,,395377.0,,,,,,,,347,27.112110137939453,5.6361141710912115,183.76661682128906,361.0,20.0,0.0041757541302785105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00025731044062525955,348,,,,,, +0,,395377.0,,,,,,,,348,31.993520736694336,5.868140593779785,255.89633178710938,360.0,20.0,0.0041757541302785105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002563058026267195,349,,,,,, +0,,395377.0,,,,,,,,349,29.136699676513672,10.65243276417399,212.23681640625,360.0,20.0,0.004491158134856733 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00025530508711770476,350,,,,,, +0,,395377.0,,,,,,,,350,34.968116760253906,11.051853967345142,305.69232177734375,360.0,20.0,0.0044057742942197175 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00025430827878332213,351,,,,,, +0,,395377.0,,,,,,,,351,34.78970718383789,10.583167418694797,302.5809326171875,359.0,20.0,0.004281408787562298 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00025331536236847236,352,,,,,, +0,,395377.0,,,,,,,,352,25.236499786376953,6.433636460156495,159.22023010253906,359.0,20.0,0.004281408787562298 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00025232632267761914,353,,,,,, +0,,395377.0,,,,,,,,353,20.076993942260742,8.965659407792248,100.77141571044922,358.0,20.0,0.004328151935119193 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002513411445745549,354,,,,,, +0,,395377.0,,,,,,,,354,23.145368576049805,9.064418545269888,133.92703247070312,358.0,20.0,0.004302760787310151 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00025035981298216956,355,,,,,, +0,,395377.0,,,,,,,,355,28.797700881958008,9.44687887508633,207.326904296875,358.0,20.0,0.0042188756288836626 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002493823128822199,356,,,,,, +0,,395377.0,,,,,,,,356,9.668769836425781,7.511025576239444,23.371278762817383,357.0,20.0,0.004126904833938063 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00024840862931509964,357,,,,,, +0,,395377.0,,,,,,,,357,31.201129913330078,9.577678983823251,243.37762451171875,357.0,20.0,0.004075700594317817 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00024743874737961045,358,,,,,, +0,,395377.0,,,,,,,,358,34.402854919433594,10.080739337827627,295.88909912109375,356.0,20.0,0.004041247856235977 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00024647265223273393,359,,,,,, +0,,395377.0,,,,,,,,359,30.461448669433594,7.571340574196814,231.9749755859375,356.0,20.0,0.004041247856235977 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00024551032908940503,360,,,,,, +0,,395377.0,,,,,,,,360,23.442272186279297,6.261647936823621,137.38502502441406,356.0,20.0,0.004041247856235977 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002445517632222834,361,,,,,, +0,,395377.0,,,,,,,,361,30.026859283447266,10.240628261165698,225.403076171875,355.0,20.0,0.004398816096603779 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00024359693996153285,362,,,,,, +0,,395377.0,,,,,,,,362,19.951133728027344,8.677206383383265,99.51193237304688,355.0,20.0,0.004386363106563817 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00024264584469459085,363,,,,,, +0,,395377.0,,,,,,,,363,21.71893882751465,8.425377962717272,117.92808532714844,354.0,20.0,0.004253314203832784 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002416984628659496,364,,,,,, +0,,395377.0,,,,,,,,364,18.373271942138672,7.931733526956755,84.3942642211914,354.0,20.0,0.004173947309577696 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00024075477997693114,365,,,,,, +0,,395377.0,,,,,,,,365,23.572811126708984,8.507544256584078,138.9193572998047,354.0,20.0,0.004101023493478613 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00023981478158546579,366,,,,,, +0,,395377.0,,,,,,,,366,34.91096496582031,9.973389968554367,304.69384765625,353.0,20.0,0.004041698680852192 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002388784533058713,367,,,,,, +0,,395377.0,,,,,,,,367,28.120683670043945,8.861200635320898,197.69322204589844,353.0,20.0,0.004012234012825272 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00023794578080863258,368,,,,,, +0,,395377.0,,,,,,,,368,9.109737396240234,6.968220609188294,20.746829986572266,352.0,20.0,0.003983328712811628 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002370167498201828,369,,,,,, +0,,395377.0,,,,,,,,369,31.503406524658203,8.673898944815566,248.11614990234375,352.0,20.0,0.003983328712811628 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00023609134612268337,370,,,,,, +0,,395377.0,,,,,,,,370,26.308731079101562,6.938607964875493,173.03732299804688,352.0,20.0,0.003983328712811628 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002351695555538088,371,,,,,, +0,,395377.0,,,,,,,,371,25.774463653564453,5.750140007113561,166.0807342529297,351.0,20.0,0.003983328712811628 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00023425136400652828,372,,,,,, +0,,395377.0,,,,,,,,372,26.85494613647461,10.06171914513662,180.29702758789062,351.0,20.0,0.004655860175256366 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00023333675742889025,373,,,,,, +0,,395377.0,,,,,,,,373,26.818777084350586,9.689801339603553,179.81170654296875,350.0,20.0,0.00458752863665284 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002324257218238077,374,,,,,, +0,,395377.0,,,,,,,,374,29.64385223388672,8.1805188993847,219.68948364257812,350.0,20.0,0.00458752863665284 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000231518243248842,375,,,,,, +0,,395377.0,,,,,,,,375,25.864578247070312,9.279211476216407,167.24412536621094,350.0,20.0,0.004503580818822856 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002306143078159939,376,,,,,, +0,,395377.0,,,,,,,,376,25.161956787109375,9.048270970531318,158.281005859375,349.0,20.0,0.004416560523884367 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002297139016914854,377,,,,,, +0,,395377.0,,,,,,,,377,33.05397415161133,9.894187995365364,273.14129638671875,349.0,20.0,0.004305833880351426 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00022881701109555243,378,,,,,, +0,,395377.0,,,,,,,,378,33.898983001708984,8.998345452186726,287.2852783203125,348.0,20.0,0.004305833880351426 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00022792362230223195,379,,,,,, +0,,395377.0,,,,,,,,379,25.088151931762695,8.988607608504337,157.35385131835938,348.0,20.0,0.004396658330969185 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00022703372163915237,380,,,,,, +0,,395377.0,,,,,,,,380,29.813766479492188,7.190108228794145,222.21517944335938,348.0,20.0,0.004396658330969185 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002261472954873242,381,,,,,, +0,,395377.0,,,,,,,,381,15.406238555908203,4.758027594084115,59.338043212890625,347.0,20.0,0.004396658330969185 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00022526433028093165,382,,,,,, +0,,395377.0,,,,,,,,382,25.28083038330078,9.388131014130757,159.78009033203125,347.0,20.0,0.004717077710236389 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002243848125071251,383,,,,,, +0,,395377.0,,,,,,,,383,25.84536361694336,9.189724144896442,166.99569702148438,346.0,20.0,0.004665127562541109 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00022350872870581425,384,,,,,, +0,,395377.0,,,,,,,,384,34.27219009399414,10.116588290835665,293.645751953125,346.0,20.0,0.004534394819269557 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00022263606546946247,385,,,,,, +0,,395377.0,,,,,,,,385,31.19963836669922,9.501664685522783,243.35433959960938,346.0,20.0,0.00443645953968112 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00022176680944287967,386,,,,,, +0,,395377.0,,,,,,,,386,17.09606170654297,7.440675781312421,73.06883239746094,345.0,20.0,0.00443645953968112 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00022090094732302233,387,,,,,, +0,,395377.0,,,,,,,,387,14.655499458312988,6.757681319782403,53.69591522216797,345.0,20.0,0.00443645953968112 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00022003846585878428,388,,,,,, +0,,395377.0,,,,,,,,388,21.135990142822266,8.92992343786718,111.68251037597656,344.0,20.0,0.004703891427662593 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00021917935185079834,389,,,,,, +0,,395377.0,,,,,,,,389,20.187297821044922,8.610113438936018,101.88174438476562,344.0,20.0,0.004657130870543596 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002183235921512322,390,,,,,, +0,,395377.0,,,,,,,,390,10.662979125976562,4.636153438529748,28.424781799316406,344.0,20.0,0.004657130870543596 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00021747117366358783,391,,,,,, +0,,395377.0,,,,,,,,391,12.274914741516113,4.333412404088799,37.66838073730469,343.0,20.0,0.004657130870543596 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002166220833425012,392,,,,,, +0,,395377.0,,,,,,,,392,30.463687896728516,10.01071893740464,232.0090789794922,343.0,20.0,0.004714177549140206 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002157763081935408,393,,,,,, +0,,395377.0,,,,,,,,393,22.806793212890625,8.678058349525452,130.03746032714844,342.0,20.0,0.004555458736533784 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002149338352730131,394,,,,,, +0,,395377.0,,,,,,,,394,10.574999809265137,4.595684688233312,27.957656860351562,342.0,20.0,0.004555458736533784 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00021409465168775862,395,,,,,, +0,,395377.0,,,,,,,,395,34.001625061035156,9.990585007308109,289.02764892578125,341.0,20.0,0.004388980445478105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00021325874459496064,396,,,,,, +0,,395377.0,,,,,,,,396,22.085464477539062,8.151621915553918,121.94194030761719,341.0,20.0,0.0043144228910118685 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002124261012019431,397,,,,,, +0,,395377.0,,,,,,,,397,15.905340194702148,5.102219482524656,63.244964599609375,341.0,20.0,0.0043144228910118685 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00021159670876597894,398,,,,,, +0,,395377.0,,,,,,,,398,32.47001266479492,9.455362538202753,263.575439453125,340.0,20.0,0.004251875897697225 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00021077055459409357,399,,,,,, +0,,395377.0,,,,,,,,399,25.290512084960938,8.34067604674365,159.90249633789062,340.0,20.0,0.004196559804263235 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00020994762604286904,400,,,,,, +0,,395377.0,,,,,,,,400,31.13553237915039,6.246782461329485,242.3553466796875,339.0,20.0,0.004196559804263235 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00020912791051825482,401,,,,,, +0,,395377.0,,,,,,,,401,20.824542999267578,4.49379388363176,108.4154052734375,339.0,20.0,0.004196559804263235 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00020831139547536966,402,,,,,, +0,,395377.0,,,,,,,,402,17.42411994934082,7.793660846817939,75.9000015258789,339.0,20.0,0.00430760102183215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002074980684183139,403,,,,,, +0,,395377.0,,,,,,,,403,24.09638786315918,8.298408820462855,145.15895080566406,338.0,20.0,0.004265676930600783 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00020668791689997616,404,,,,,, +0,,395377.0,,,,,,,,404,13.955513954162598,4.248329590455081,48.68909454345703,338.0,20.0,0.004265676930600783 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00020588092852184337,405,,,,,, +0,,395377.0,,,,,,,,405,9.926898002624512,3.7105213219009303,24.635826110839844,337.0,20.0,0.004265676930600783 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000205077090933811,406,,,,,, +0,,395377.0,,,,,,,,406,24.80853271484375,8.711062500159235,153.86581420898438,337.0,20.0,0.004352610400548943 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00020427639183399397,407,,,,,, +0,,395377.0,,,,,,,,407,17.014312744140625,7.629005514667404,72.37171173095703,337.0,20.0,0.004274112533512708 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002034788189685389,408,,,,,, +0,,395377.0,,,,,,,,408,28.089200973510742,5.795180750880132,197.2508087158203,336.0,20.0,0.004274112533512708 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00020268436013143456,409,,,,,, +0,,395377.0,,,,,,,,409,27.144895553588867,8.376218659762515,184.21133422851562,336.0,20.0,0.004108448890538297 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00020189300316432929,410,,,,,, +0,,395377.0,,,,,,,,410,15.050094604492188,7.068112375070878,56.626338958740234,335.0,20.0,0.004052000574017084 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0002011047359563393,411,,,,,, +0,,395377.0,,,,,,,,411,13.636443138122559,3.7758511131559183,46.4881477355957,335.0,20.0,0.004052000574017084 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00020031954644386918,412,,,,,, +0,,395377.0,,,,,,,,412,31.897354125976562,8.917439064066674,254.36029052734375,335.0,20.0,0.003994297986548332 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00019953742261042193,413,,,,,, +0,,395377.0,,,,,,,,413,36.83741760253906,6.392920414877166,339.24884033203125,334.0,20.0,0.003994297986548332 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00019875835248641896,414,,,,,, +0,,395377.0,,,,,,,,414,22.023860931396484,7.64098320068959,121.26262664794922,334.0,20.0,0.004016360888498698 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001979823241490151,415,,,,,, +0,,395377.0,,,,,,,,415,21.514263153076172,7.387694350093162,115.71588134765625,333.0,20.0,0.003972369395524042 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001972093257219168,416,,,,,, +0,,395377.0,,,,,,,,416,24.417484283447266,4.6214535401032855,149.05337524414062,333.0,20.0,0.003972369395524042 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00019643934537519884,417,,,,,, +0,,395377.0,,,,,,,,417,18.763093948364258,7.261777002679869,88.013427734375,333.0,20.0,0.004001799326133381 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001956723713251268,418,,,,,, +0,,395377.0,,,,,,,,418,26.05770492553711,7.947478238837422,169.75100708007812,332.0,20.0,0.00395101240471122 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00019490839183397215,419,,,,,, +0,,395377.0,,,,,,,,419,28.767118453979492,8.134635464013705,206.88677978515625,332.0,20.0,0.0038784151578058652 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00019414739520983622,420,,,,,, +0,,395377.0,,,,,,,,420,27.145278930664062,4.9194624803295675,184.21653747558594,331.0,20.0,0.0038784151578058652 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001933893698064697,421,,,,,, +0,,395377.0,,,,,,,,421,31.238943099975586,8.818563147863834,243.9678955078125,331.0,20.0,0.003957665274634326 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00019263430402309496,422,,,,,, +0,,395377.0,,,,,,,,422,27.734678268432617,8.213146413700814,192.30308532714844,331.0,20.0,0.003960459189682958 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001918821863042269,423,,,,,, +0,,395377.0,,,,,,,,423,16.291973114013672,3.353298103319647,66.35710144042969,330.0,20.0,0.003960459189682958 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00019113300513949973,424,,,,,, +0,,395377.0,,,,,,,,424,29.83962059020996,8.490312450616493,222.60073852539062,330.0,20.0,0.004017189364639646 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00019038674906348595,425,,,,,, +0,,395377.0,,,,,,,,425,17.274925231933594,7.0221538940862995,74.60575866699219,329.0,20.0,0.004023184596160025 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00018964340665552604,426,,,,,, +0,,395377.0,,,,,,,,426,15.866762161254883,6.663369201188139,62.93853759765625,329.0,20.0,0.003931929095669856 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00018890296653954877,427,,,,,, +0,,395377.0,,,,,,,,427,19.180614471435547,3.6935598268540124,91.97398376464844,329.0,20.0,0.003931929095669856 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00018816541738390078,428,,,,,, +0,,395377.0,,,,,,,,428,20.401763916015625,7.3009923433990505,104.05799865722656,328.0,20.0,0.003965948852642077 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00018743074790117143,429,,,,,, +0,,395377.0,,,,,,,,429,7.117210388183594,6.247657561329131,12.663670539855957,328.0,20.0,0.00390892795023901 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00018669894684802046,430,,,,,, +0,,395377.0,,,,,,,,430,14.662271499633789,3.107254719944165,53.74555206298828,327.0,20.0,0.00390892795023901 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001859700030250059,431,,,,,, +0,,395377.0,,,,,,,,431,10.255019187927246,6.233121967373869,26.29135513305664,327.0,20.0,0.003826297137729348 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000185243905276413,432,,,,,, +0,,395377.0,,,,,,,,432,25.37777328491211,7.4888799835439155,161.00784301757812,327.0,20.0,0.003760100098066166 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001845206424900818,433,,,,,, +0,,395377.0,,,,,,,,433,25.710065841674805,7.256467072618193,165.25186157226562,326.0,20.0,0.0036298610701327146 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00018380020359724087,434,,,,,, +0,,395377.0,,,,,,,,434,28.270095825195312,4.806517139745505,199.79959106445312,326.0,20.0,0.0036298610701327146 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00018308257757233303,435,,,,,, +0,,395377.0,,,,,,,,435,10.425483703613281,6.232115345504365,27.172677993774414,325.0,20.0,0.003681058505769003 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00018236775343285046,436,,,,,, +0,,395377.0,,,,,,,,436,26.08955955505371,7.568693338007661,170.1663055419922,325.0,20.0,0.0036923864140051737 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00018165572023916496,437,,,,,, +0,,395377.0,,,,,,,,437,16.518030166625977,3.39134048754231,68.2113265991211,325.0,20.0,0.0036923864140051737 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00018094646709436072,438,,,,,, +0,,395377.0,,,,,,,,438,12.204599380493164,6.173684147617949,37.238059997558594,324.0,20.0,0.0036564648685024994 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00018023998314406766,439,,,,,, +0,,395377.0,,,,,,,,439,28.221908569335938,4.39200210254633,199.1190185546875,324.0,20.0,0.0036564648685024994 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00017953625757629568,440,,,,,, +0,,395377.0,,,,,,,,440,24.047489166259766,7.4359790266787025,144.5704345703125,323.0,20.0,0.0037769795478953235 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00017883527962126746,441,,,,,, +0,,395377.0,,,,,,,,441,18.328218460083008,3.440035165876101,83.98089599609375,323.0,20.0,0.0037769795478953235 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00017813703855125722,442,,,,,, +0,,395377.0,,,,,,,,442,23.39971923828125,7.143467623178553,136.88671875,323.0,20.0,0.0037354342973766512 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00017744152368042183,443,,,,,, +0,,395377.0,,,,,,,,443,16.806180953979492,3.1927463515862624,70.61192321777344,322.0,20.0,0.0037354342973766512 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00017674872436464225,444,,,,,, +0,,395377.0,,,,,,,,444,27.59038734436035,7.8803575341161025,190.30735778808594,322.0,20.0,0.003846653517091134 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00017605863000135565,445,,,,,, +0,,395377.0,,,,,,,,445,16.60442352294922,6.380918436260748,68.92671966552734,321.0,20.0,0.003726747096090087 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001753712300293981,446,,,,,, +0,,395377.0,,,,,,,,446,4.276236057281494,5.5840781435108635,4.5715484619140625,321.0,20.0,0.003647712793345269 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001746865139288383,447,,,,,, +0,,395377.0,,,,,,,,447,15.29521656036377,6.071442323577058,58.48591232299805,321.0,20.0,0.0035772309963710055 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001740044712208201,448,,,,,, +0,,395377.0,,,,,,,,448,20.42019271850586,6.391981839083912,104.24606323242188,320.0,20.0,0.003483975674982428 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00017332509146740068,449,,,,,, +0,,395377.0,,,,,,,,449,15.409798622131348,5.892041781451784,59.36547088623047,320.0,20.0,0.0034737105604158752 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00017264836427139006,450,,,,,, +0,,395377.0,,,,,,,,450,18.958351135253906,4.319187851528957,89.85476684570312,319.0,20.0,0.0034737105604158752 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001719742792761938,451,,,,,, +0,,395377.0,,,,,,,,451,15.546404838562012,6.01351479054638,60.422672271728516,319.0,20.0,0.0034653253879188004 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001713028261656532,452,,,,,, +0,,395377.0,,,,,,,,452,23.815017700195312,4.003280195378977,141.78878784179688,319.0,20.0,0.0034653253879188004 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00017063399466388774,453,,,,,, +0,,395377.0,,,,,,,,453,23.5948486328125,6.792815589002566,139.17922973632812,318.0,20.0,0.0035177090649554953 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016996777453513824,454,,,,,, +0,,395377.0,,,,,,,,454,29.56075668334961,7.468354394099065,218.4595947265625,318.0,20.0,0.0034649543958911637 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016930415558360858,455,,,,,, +0,,395377.0,,,,,,,,455,33.885738372802734,8.132385615182047,287.0608215332031,317.0,20.0,0.0034488307273960518 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000168643127653313,456,,,,,, +0,,395377.0,,,,,,,,456,32.39593505859375,7.83762259735593,262.3741455078125,317.0,20.0,0.0034319632676780482 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001679846806279162,457,,,,,, +0,,395377.0,,,,,,,,457,22.1315860748291,5.3679377667933945,122.45176696777344,317.0,20.0,0.0034319632676780482 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016732880443058332,458,,,,,, +0,,395377.0,,,,,,,,458,10.682491302490234,3.9322167115325395,28.528902053833008,316.0,20.0,0.0034319632676780482 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016667548902382084,459,,,,,, +0,,395377.0,,,,,,,,459,13.735001564025879,6.458489212032491,47.162567138671875,316.0,20.0,0.0038704648082682537 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001660247244093279,460,,,,,, +0,,395377.0,,,,,,,,460,15.31863784790039,3.3598053366377734,58.66516876220703,315.0,20.0,0.0038704648082682537 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016537650062783873,461,,,,,, +0,,395377.0,,,,,,,,461,5.906253337860107,5.9237539055491295,8.72095775604248,315.0,20.0,0.0038554017221055768 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016473080775897343,462,,,,,, +0,,395377.0,,,,,,,,462,16.230592727661133,6.290838698775598,65.85803985595703,315.0,20.0,0.0037522828421295232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016408763592108488,463,,,,,, +0,,395377.0,,,,,,,,463,34.888153076171875,8.468450934227107,304.29583740234375,314.0,20.0,0.003621473033358147 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016344697527110753,464,,,,,, +0,,395377.0,,,,,,,,464,25.143281936645508,6.8913160004008445,158.04615783691406,314.0,20.0,0.003528762189406228 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016280881600440735,465,,,,,, +0,,395377.0,,,,,,,,465,26.559654235839844,6.844603444797804,176.3538055419922,313.0,20.0,0.003528762189406228 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016217314835463033,466,,,,,, +0,,395377.0,,,,,,,,466,22.45360565185547,5.804055025965303,126.04110717773438,313.0,20.0,0.003528762189406228 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016153996259355498,467,,,,,, +0,,395377.0,,,,,,,,467,18.90012550354004,4.30309748248583,89.30368041992188,313.0,20.0,0.003528762189406228 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016090924903094215,468,,,,,, +0,,395377.0,,,,,,,,468,23.11911392211914,4.1138548265137524,133.6233673095703,312.0,20.0,0.003528762189406228 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00016028099801438717,469,,,,,, +0,,395377.0,,,,,,,,469,25.13338279724121,8.538353596665363,157.92172241210938,312.0,20.0,0.004394058089728115 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001596551999291719,470,,,,,, +0,,395377.0,,,,,,,,470,21.472064971923828,5.285661455101393,115.26239013671875,312.0,20.0,0.004394058089728115 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00015903184519811786,471,,,,,, +0,,395377.0,,,,,,,,471,21.53667449951172,7.759309674853645,115.95708465576172,311.0,20.0,0.004325217303751374 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001584109242814394,472,,,,,, +0,,395377.0,,,,,,,,472,14.225669860839844,6.772289015910508,50.59242248535156,311.0,20.0,0.004174884552223043 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001577924276765979,473,,,,,, +0,,395377.0,,,,,,,,473,25.83592987060547,7.607618304988592,166.87380981445312,310.0,20.0,0.003976396580422219 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001571763459181566,474,,,,,, +0,,395377.0,,,,,,,,474,20.481708526611328,6.767887433069436,104.8750991821289,310.0,20.0,0.0038137376591760334 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001565626695776344,475,,,,,, +0,,395377.0,,,,,,,,475,21.554931640625,6.71996381486293,116.15376281738281,310.0,20.0,0.003698794961844502 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001559513892633636,476,,,,,, +0,,395377.0,,,,,,,,476,7.981998443603516,5.651852189996376,15.928072929382324,309.0,20.0,0.0036322791724394824 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00015534249562034478,477,,,,,, +0,,395377.0,,,,,,,,477,15.780851364135742,6.045744915956602,62.258819580078125,309.0,20.0,0.0035728669254844747 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00015473597933010405,478,,,,,, +0,,395377.0,,,,,,,,478,20.572017669677734,6.39615637405321,105.80197143554688,308.0,20.0,0.003541029325037063 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00015413183111055058,479,,,,,, +0,,395377.0,,,,,,,,479,30.511795043945312,7.557097653521872,232.74240112304688,308.0,20.0,0.0034426704814575237 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001535300417158334,480,,,,,, +0,,395377.0,,,,,,,,480,31.279207229614258,7.596626135066982,244.59719848632812,308.0,20.0,0.0034107593093508715 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001529306019362026,481,,,,,, +0,,395377.0,,,,,,,,481,8.78956127166748,5.291136864926997,19.314096450805664,307.0,20.0,0.003370591396157868 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00015233350259786496,482,,,,,, +0,,395377.0,,,,,,,,482,16.295686721801758,5.716248362235242,66.38734436035156,307.0,20.0,0.0033548687288258196 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001517387345628461,483,,,,,, +0,,395377.0,,,,,,,,483,24.079172134399414,6.430016246917902,144.95162963867188,306.0,20.0,0.0033052053241279643 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00015114628872884967,484,,,,,, +0,,395377.0,,,,,,,,484,17.66637420654297,8.560465403746717,78.02519226074219,306.0,20.0,0.0033052053241279643 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00015055615602911808,485,,,,,, +0,,395377.0,,,,,,,,485,18.0765323638916,5.960408155165379,81.69025421142578,306.0,20.0,0.0033746666857347404 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014996832743229381,486,,,,,, +0,,395377.0,,,,,,,,486,28.905033111572266,7.186611061139013,208.875244140625,305.0,20.0,0.0033545073623715803 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014938279394228124,487,,,,,, +0,,395377.0,,,,,,,,487,26.46765899658203,6.75261716046591,175.13424682617188,305.0,20.0,0.0033335856922612735 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014879954659810887,488,,,,,, +0,,395377.0,,,,,,,,488,25.991300582885742,6.59757165104945,168.8869171142578,304.0,20.0,0.003266279294319608 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014821857647379262,489,,,,,, +0,,395377.0,,,,,,,,489,13.71255874633789,5.3502678568067825,47.008567810058594,304.0,20.0,0.0032583430777517334 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014763987467819795,490,,,,,, +0,,395377.0,,,,,,,,490,10.417553901672363,5.0988766293540975,27.13135528564453,304.0,20.0,0.0031857351511570308 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014706343235490558,491,,,,,, +0,,395377.0,,,,,,,,491,31.22119140625,7.244098919360065,243.69068908691406,303.0,20.0,0.0031596120751852313 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001464892406820747,492,,,,,, +0,,395377.0,,,,,,,,492,24.91167640686035,6.287972536223613,155.14788818359375,303.0,20.0,0.003119025363720418 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001459172908723084,493,,,,,, +0,,395377.0,,,,,,,,493,8.335371971130371,4.923239711019906,17.36960792541504,302.0,20.0,0.0031328119266884166 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014534757417251914,494,,,,,, +0,,395377.0,,,,,,,,494,14.883543968200684,5.201491288175736,55.379966735839844,302.0,20.0,0.0030693376753260513 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001447800818637947,495,,,,,, +0,,395377.0,,,,,,,,495,26.866565704345703,6.395950907199411,180.45309448242188,302.0,20.0,0.003023325987324931 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014421480526126483,496,,,,,, +0,,395377.0,,,,,,,,496,23.503328323364258,6.173986291744027,138.1016082763672,301.0,20.0,0.003023325987324931 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014365173571396857,497,,,,,, +0,,395377.0,,,,,,,,497,11.810768127441406,4.4270574776858025,34.87356185913086,301.0,20.0,0.003023325987324931 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014309086460472068,498,,,,,, +0,,395377.0,,,,,,,,498,18.402389526367188,6.119600725247691,84.66197967529297,300.0,20.0,0.003467892479134668 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014253218334998237,499,,,,,, +0,,395377.0,,,,,,,,499,24.961082458496094,6.693768569157488,155.763916015625,300.0,20.0,0.003407123635192166 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001414213562373095,500,,,,,, +0,,395377.0,,,,,,,,500,15.191482543945312,5.511096284219551,57.695281982421875,300.0,20.0,0.0032962603775035 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001414213562373095,501,,,,,, +0,,395377.0,,,,,,,,501,27.19546127319336,6.149720010220519,184.8982696533203,299.0,20.0,0.0032962603775035 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014086919337933987,502,,,,,, +0,,395377.0,,,,,,,,502,29.792287826538086,7.551607151594687,221.8950958251953,299.0,20.0,0.003525917304706406 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00014031918637554835,503,,,,,, +0,,395377.0,,,,,,,,503,30.033672332763672,6.0048359411097545,225.50535583496094,298.0,20.0,0.003525917304706406 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001397713268086591,504,,,,,, +0,,395377.0,,,,,,,,504,16.152156829833984,5.98261772840496,65.22303771972656,298.0,20.0,0.003582191524326048 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013922560629425946,505,,,,,, +0,,395377.0,,,,,,,,505,15.363151550292969,3.7010293757886856,59.00660705566406,298.0,20.0,0.003582191524326048 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013868201648067457,506,,,,,, +0,,395377.0,,,,,,,,506,18.96209716796875,6.190002122224066,89.8902816772461,297.0,20.0,0.0035824723523381068 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013814054904883586,507,,,,,, +0,,395377.0,,,,,,,,507,23.9138240814209,6.627068245304971,142.96774291992188,297.0,20.0,0.0035728142298410703 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013760119571215658,508,,,,,, +0,,395377.0,,,,,,,,508,24.597469329833984,5.052076103579518,151.25888061523438,296.0,20.0,0.0035728142298410703 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013706394821640374,509,,,,,, +0,,395377.0,,,,,,,,509,10.936989784240723,5.574146512730588,29.904436111450195,296.0,20.0,0.0035701703309129794 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013652879833957199,510,,,,,, +0,,395377.0,,,,,,,,510,7.442154884338379,5.297726361598878,13.846417427062988,296.0,20.0,0.0035143279913267276 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013599573789175787,511,,,,,, +0,,395377.0,,,,,,,,511,21.092369079589844,6.094731246711721,111.22200012207031,295.0,20.0,0.0033995818163076244 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001354647587150344,512,,,,,, +0,,395377.0,,,,,,,,512,22.908775329589844,4.558938204085677,131.2030029296875,295.0,20.0,0.0033995818163076244 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013493585268332622,513,,,,,, +0,,395377.0,,,,,,,,513,11.373237609863281,5.5373385104591915,32.33763122558594,294.0,20.0,0.0034534463240834474 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013440901170228556,514,,,,,, +0,,395377.0,,,,,,,,514,23.372983932495117,6.425661436944295,136.5740966796875,294.0,20.0,0.003394313576786911 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013388422770916733,515,,,,,, +0,,395377.0,,,,,,,,515,16.704072952270508,5.545252333508885,69.75651550292969,294.0,20.0,0.0032656797165920475 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013336149267270718,516,,,,,, +0,,395377.0,,,,,,,,516,19.43840980529785,5.744921412007557,94.46294403076172,293.0,20.0,0.0032315690158453377 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013284079859299765,517,,,,,, +0,,395377.0,,,,,,,,517,24.222475051879883,6.2076192449410685,146.68206787109375,293.0,20.0,0.003170792718112267 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013232213750136594,518,,,,,, +0,,395377.0,,,,,,,,518,23.242761611938477,5.986214991938764,135.05648803710938,292.0,20.0,0.003104230623360927 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.000131805501460252,519,,,,,, +0,,395377.0,,,,,,,,519,19.400510787963867,5.542074611990936,94.0949478149414,292.0,20.0,0.003077910331552667 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013129088256308704,520,,,,,, +0,,395377.0,,,,,,,,520,14.904398918151855,4.349373855451846,55.5352783203125,292.0,20.0,0.003077910331552667 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013077827293417254,521,,,,,, +0,,395377.0,,,,,,,,521,24.086721420288086,6.23214935699338,145.04254150390625,291.0,20.0,0.0031299041665844443 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00013026766472855992,522,,,,,, +0,,395377.0,,,,,,,,522,17.36134147644043,5.414687678535813,75.35404968261719,291.0,20.0,0.0030865916503870537 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012975905013192936,523,,,,,, +0,,395377.0,,,,,,,,523,23.61683464050293,5.91587707835554,139.438720703125,290.0,20.0,0.003012590795575091 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012925242136047272,524,,,,,, +0,,395377.0,,,,,,,,524,22.24118995666504,4.1258155043498705,123.66764068603516,290.0,20.0,0.003012590795575091 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001287477706607712,525,,,,,, +0,,395377.0,,,,,,,,525,14.718766212463379,2.991478883321282,54.1605224609375,290.0,20.0,0.003012590795575091 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012824509030967923,526,,,,,, +0,,395377.0,,,,,,,,526,9.724987983703613,5.544801447078548,23.64384651184082,289.0,20.0,0.003448669637091401 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012774437261420512,527,,,,,, +0,,395377.0,,,,,,,,527,19.149816513061523,6.002444539876137,91.67886352539062,289.0,20.0,0.0033559277915420705 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012724560991139353,528,,,,,, +0,,395377.0,,,,,,,,528,24.632291793823242,6.246814697758226,151.6874542236328,288.0,20.0,0.0031757431069317295 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012674879456820847,529,,,,,, +0,,395377.0,,,,,,,,529,16.72164535522461,5.412858386756673,69.90335845947266,288.0,20.0,0.003158126425224197 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012625391898141554,530,,,,,, +0,,395377.0,,,,,,,,530,24.885303497314453,6.2913856711768235,154.819580078125,288.0,20.0,0.0031220766507076806 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012576097557746694,531,,,,,, +0,,395377.0,,,,,,,,531,17.249408721923828,5.386708554162386,74.38552856445312,287.0,20.0,0.0030725602697161715 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012526995681238465,532,,,,,, +0,,395377.0,,,,,,,,532,13.939325332641602,5.043392521842082,48.576194763183594,287.0,20.0,0.0030221979157531776 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012478085517164523,533,,,,,, +0,,395377.0,,,,,,,,533,15.899599075317383,6.598462928612547,63.199310302734375,286.0,20.0,0.0030221979157531776 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001242936631700648,534,,,,,, +0,,395377.0,,,,,,,,534,18.89820098876953,5.700671525845495,89.28549194335938,286.0,20.0,0.0031336270834431464 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001238083733516845,535,,,,,, +0,,395377.0,,,,,,,,535,21.495914459228516,5.26045861878881,115.51860046386719,286.0,20.0,0.0031336270834431464 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012332497828965636,536,,,,,, +0,,395377.0,,,,,,,,536,19.207712173461914,4.072617957194348,92.23405456542969,285.0,20.0,0.0031336270834431464 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001228434705861296,537,,,,,, +0,,395377.0,,,,,,,,537,9.336368560791016,5.328233910499735,21.79194450378418,285.0,20.0,0.003349443782060876 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012236384287213763,538,,,,,, +0,,395377.0,,,,,,,,538,8.214051246643066,5.050327801539609,16.867660522460938,284.0,20.0,0.0032621092782653114 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012188608780748514,539,,,,,, +0,,395377.0,,,,,,,,539,14.465192794799805,5.1397991760210395,52.31045150756836,284.0,20.0,0.0031184797589600983 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012141019808063519,540,,,,,, +0,,395377.0,,,,,,,,540,17.595760345458984,7.063547701232566,77.40269470214844,284.0,20.0,0.0031184797589600983 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00012093616640859853,541,,,,,, +0,,395377.0,,,,,,,,541,28.80750274658203,6.934120956840679,207.4680633544922,283.0,20.0,0.003156674464268959 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001204639855368212,542,,,,,, +0,,395377.0,,,,,,,,542,22.037675857543945,5.94963624898649,121.414794921875,283.0,20.0,0.0031083820477426317 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011999364823907382,543,,,,,, +0,,395377.0,,,,,,,,543,22.373912811279297,7.036471662026785,125.14798736572266,282.0,20.0,0.0031083820477426317 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011952514731734093,544,,,,,, +0,,395377.0,,,,,,,,544,20.382057189941406,5.750647908864887,103.85707092285156,282.0,20.0,0.003135152894747362 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011905847560171116,545,,,,,, +0,,395377.0,,,,,,,,545,23.281543731689453,5.991490480883571,135.507568359375,282.0,20.0,0.003131680050448145 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011859362595026624,546,,,,,, +0,,395377.0,,,,,,,,546,8.866786003112793,4.279120369000606,19.654972076416016,281.0,20.0,0.003131680050448145 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011813059124897423,547,,,,,, +0,,395377.0,,,,,,,,547,19.433609008789062,4.409007166798787,94.4162826538086,281.0,20.0,0.003131680050448145 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011766936441157737,548,,,,,, +0,,395377.0,,,,,,,,548,22.727535247802734,4.096513518545582,129.13522338867188,280.0,20.0,0.003131680050448145 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011720993837948704,549,,,,,, +0,,395377.0,,,,,,,,549,19.19212532043457,6.7198505666823785,92.08442687988281,280.0,20.0,0.003715482333249167 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011675230612167258,550,,,,,, +0,,395377.0,,,,,,,,550,20.622509002685547,6.500464759712375,106.32197570800781,279.0,20.0,0.0036111169325696323 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011629646063455594,551,,,,,, +0,,395377.0,,,,,,,,551,23.152135848999023,6.345522860736207,134.0053253173828,279.0,20.0,0.003430101185258099 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011584239494190357,552,,,,,, +0,,395377.0,,,,,,,,552,10.048052787780762,5.677773614949546,25.240840911865234,279.0,20.0,0.003430101185258099 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011539010209471974,553,,,,,, +0,,395377.0,,,,,,,,553,17.482009887695312,6.250714391085518,76.4051742553711,278.0,20.0,0.0035578575024859515 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011493957517114048,554,,,,,, +0,,395377.0,,,,,,,,554,18.03769302368164,6.13581148380017,81.33958435058594,278.0,20.0,0.003467851758531957 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001144908072763268,555,,,,,, +0,,395377.0,,,,,,,,555,19.6754093170166,6.40348010203537,96.78044128417969,277.0,20.0,0.003467851758531957 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011404379154236036,556,,,,,, +0,,395377.0,,,,,,,,556,14.905928611755371,5.643173120388541,55.54667663574219,277.0,20.0,0.003352526742025179 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011359852112813764,557,,,,,, +0,,395377.0,,,,,,,,557,16.478492736816406,5.68174803203793,67.88517761230469,277.0,20.0,0.0033079116294068593 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011315498921926537,558,,,,,, +0,,395377.0,,,,,,,,558,21.63908576965332,5.8777024682021235,117.0625,276.0,20.0,0.00318333195557352 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011271318902795628,559,,,,,, +0,,395377.0,,,,,,,,559,19.240169525146484,5.541602000359852,92.5460433959961,276.0,20.0,0.0030915511361459974 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011227311379292518,560,,,,,, +0,,395377.0,,,,,,,,560,14.639939308166504,5.157910926176484,53.58195495605469,275.0,20.0,0.0030872852003276234 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011183475677928556,561,,,,,, +0,,395377.0,,,,,,,,561,19.643707275390625,5.4796431935342955,96.46881103515625,275.0,20.0,0.0030170042421340214 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011139811127844641,562,,,,,, +0,,395377.0,,,,,,,,562,18.83831787109375,5.05225324348166,88.72055053710938,275.0,20.0,0.0030170042421340214 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011096317060800986,563,,,,,, +0,,395377.0,,,,,,,,563,20.170743942260742,5.723182182649733,101.71473693847656,274.0,20.0,0.0030769222217187143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011052992811166807,564,,,,,, +0,,395377.0,,,,,,,,564,22.335865020751953,4.543159091179205,124.72271728515625,274.0,20.0,0.0030769222217187143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00011009837715910255,565,,,,,, +0,,395377.0,,,,,,,,565,15.713213920593262,3.3854810797425987,61.72627639770508,274.0,20.0,0.0030769222217187143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001096685111458819,566,,,,,, +0,,395377.0,,,,,,,,566,16.976375579833984,5.6915155821164864,72.04933166503906,273.0,20.0,0.0032886663574131056 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010924032349336091,567,,,,,, +0,,395377.0,,,,,,,,567,14.865457534790039,3.3192207706039336,55.24545669555664,273.0,20.0,0.0032886663574131056 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010881380764858015,568,,,,,, +0,,395377.0,,,,,,,,568,28.001697540283203,6.765541384156833,196.02377319335938,272.0,20.0,0.0032782315397125524 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010838895708416484,569,,,,,, +0,,395377.0,,,,,,,,569,14.6009521484375,5.175520940373662,53.29694747924805,272.0,20.0,0.00319333063793905 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010796576529822613,570,,,,,, +0,,395377.0,,,,,,,,570,7.750654220581055,3.4035870384881712,15.018159866333008,272.0,20.0,0.00319333063793905 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010754422581426082,571,,,,,, +0,,395377.0,,,,,,,,571,14.857428550720215,5.213419097556857,55.185794830322266,271.0,20.0,0.0031538949637419907 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010712433218105247,572,,,,,, +0,,395377.0,,,,,,,,572,22.773035049438477,5.88542651075434,129.65277099609375,271.0,20.0,0.0031418632546893047 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010670607797257263,573,,,,,, +0,,395377.0,,,,,,,,573,13.692291259765625,3.5039442140092927,46.869712829589844,270.0,20.0,0.0031418632546893047 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010628945678788254,574,,,,,, +0,,395377.0,,,,,,,,574,6.897785663604736,2.819997637227945,11.894862174987793,270.0,20.0,0.0031418632546893047 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001058744622510351,575,,,,,, +0,,395377.0,,,,,,,,575,8.646753311157227,5.136335343283385,18.691585540771484,270.0,20.0,0.0032643460049964074 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010546108801097736,576,,,,,, +0,,395377.0,,,,,,,,576,6.73136568069458,4.833240646506013,11.327821731567383,269.0,20.0,0.0031592882877048444 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010504932774145332,577,,,,,, +0,,395377.0,,,,,,,,577,22.51953887939453,5.635821110172002,126.78240966796875,269.0,20.0,0.002971106590247861 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010463917514090724,578,,,,,, +0,,395377.0,,,,,,,,578,18.178590774536133,5.229795122923495,82.61529541015625,268.0,20.0,0.0029327508054775714 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010423062393238653,579,,,,,, +0,,395377.0,,,,,,,,579,21.36969757080078,5.516513162356089,114.1659927368164,268.0,20.0,0.002933742258267541 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010382366786344675,580,,,,,, +0,,395377.0,,,,,,,,580,11.738260269165039,4.610373022657493,34.446685791015625,268.0,20.0,0.0028633157951561635 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010341830070605524,581,,,,,, +0,,395377.0,,,,,,,,581,12.146428108215332,4.662867858612193,36.88392639160156,267.0,20.0,0.0028413490597941913 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010301451625649587,582,,,,,, +0,,395377.0,,,,,,,,582,7.273435592651367,4.35369871574921,13.225715637207031,267.0,20.0,0.002806876566454135 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010261230833527426,583,,,,,, +0,,395377.0,,,,,,,,583,18.01812171936035,4.9872118654844675,81.16317749023438,266.0,20.0,0.0027727520723207695 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001022116707870233,584,,,,,, +0,,395377.0,,,,,,,,584,25.15675163269043,5.686427627915452,158.21554565429688,266.0,20.0,0.002736817987968811 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010181259748040801,585,,,,,, +0,,395377.0,,,,,,,,585,19.072980880737305,4.977783519967778,90.94465637207031,266.0,20.0,0.0027137484210817135 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010141508230803378,586,,,,,, +0,,395377.0,,,,,,,,586,6.419836521148682,4.156645580695073,10.30357551574707,265.0,20.0,0.0027195675928069697 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010101911918635016,587,,,,,, +0,,395377.0,,,,,,,,587,14.223691940307617,4.5232724943998885,50.57835388183594,265.0,20.0,0.002677775254704732 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.00010062470205556061,588,,,,,, +0,,395377.0,,,,,,,,588,18.536718368530273,7.39377453708505,85.90248107910156,264.0,20.0,0.002677775254704732 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,0.0001002318248795271,589,,,,,, +0,,395377.0,,,,,,,,589,17.90736961364746,4.996875618782068,80.16847229003906,264.0,20.0,0.0027781807474488293 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.98404816456797e-05,590,,,,,, +0,,395377.0,,,,,,,,590,10.464017868041992,4.371042701668446,27.373916625976562,264.0,20.0,0.002741997668412789 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.945066636492367e-05,591,,,,,, +0,,395377.0,,,,,,,,591,28.454593658447266,6.677076520060606,202.41598510742188,263.0,20.0,0.002741997668412789 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.906237307154797e-05,592,,,,,, +0,,395377.0,,,,,,,,592,19.855398178100586,5.221951121816517,98.5592041015625,263.0,20.0,0.002796366932134705 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.867559582313416e-05,593,,,,,, +0,,395377.0,,,,,,,,593,19.225297927856445,5.144528563326332,92.40301513671875,262.0,20.0,0.002822219593537065 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.82903287004647e-05,594,,,,,, +0,,395377.0,,,,,,,,594,4.830965042114258,4.100619524109558,5.834555625915527,262.0,20.0,0.0027291254388844164 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.79065658074335e-05,595,,,,,, +0,,395377.0,,,,,,,,595,21.91196632385254,4.642262072223938,120.0335693359375,262.0,20.0,0.0027291254388844164 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.752430127095493e-05,596,,,,,, +0,,395377.0,,,,,,,,596,14.583842277526855,3.684097112226895,53.17211151123047,261.0,20.0,0.0027291254388844164 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.71435292408741e-05,597,,,,,, +0,,395377.0,,,,,,,,597,22.53404998779297,6.0210398974795964,126.94584655761719,261.0,20.0,0.0030527774252974587 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.676424388987738e-05,598,,,,,, +0,,395377.0,,,,,,,,598,13.772932052612305,4.962650976084649,47.42341613769531,260.0,20.0,0.002954693434225119 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.638643941340319e-05,599,,,,,, +0,,395377.0,,,,,,,,599,17.037752151489258,4.8894337144539035,72.57124328613281,260.0,20.0,0.002810745997640034 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.601011002955315e-05,600,,,,,, +0,,395377.0,,,,,,,,600,15.911346435546875,4.53524039835089,63.29273986816406,260.0,20.0,0.002810745997640034 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.563524997900363e-05,601,,,,,, +0,,395377.0,,,,,,,,601,20.037145614624023,5.828169852171342,100.37179565429688,259.0,20.0,0.0030878616472922145 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.52618535249176e-05,602,,,,,, +0,,395377.0,,,,,,,,602,19.077518463134766,5.50592038193235,90.98793029785156,259.0,20.0,0.002951237069193074 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.488991495285699e-05,603,,,,,, +0,,395377.0,,,,,,,,603,19.835906982421875,4.494734476808619,98.36579895019531,258.0,20.0,0.002951237069193074 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.451942857069454e-05,604,,,,,, +0,,395377.0,,,,,,,,604,13.373757362365723,4.920126093204074,44.71434783935547,258.0,20.0,0.002918076920353654 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.415038870852778e-05,605,,,,,, +0,,395377.0,,,,,,,,605,7.165124893188477,2.94397826930145,12.83475399017334,258.0,20.0,0.002918076920353654 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.37827897185915e-05,606,,,,,, +0,,395377.0,,,,,,,,606,13.320664405822754,3.031095338739586,44.36002731323242,257.0,20.0,0.002918076920353654 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.341662597517167e-05,607,,,,,, +0,,395377.0,,,,,,,,607,16.51078224182129,5.781625686020597,68.1514892578125,257.0,20.0,0.003277401918501998 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.305189187451857e-05,608,,,,,, +0,,395377.0,,,,,,,,608,14.731402397155762,5.306125933022636,54.25355529785156,256.0,20.0,0.0031193462157760796 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.268858183476279e-05,609,,,,,, +0,,395377.0,,,,,,,,609,25.7465877532959,5.968759796583303,165.7216796875,256.0,20.0,0.00291110506545123 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.232669029582733e-05,610,,,,,, +0,,395377.0,,,,,,,,610,26.110532760620117,5.055392345593574,170.44000244140625,256.0,20.0,0.00291110506545123 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.196621171934517e-05,611,,,,,, +0,,395377.0,,,,,,,,611,26.507465362548828,6.839882879980758,175.66143798828125,255.0,20.0,0.003299142707549753 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.160714058857208e-05,612,,,,,, +0,,395377.0,,,,,,,,612,25.303199768066406,6.552194193262798,160.06297302246094,255.0,20.0,0.003190741036660203 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.124947140830389e-05,613,,,,,, +0,,395377.0,,,,,,,,613,16.149673461914062,3.714405352905062,65.2030029296875,254.0,20.0,0.003190741036660203 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.089319870479169e-05,614,,,,,, +0,,395377.0,,,,,,,,614,18.55226707458496,5.541127800624588,86.04666137695312,254.0,20.0,0.0031704438863540477 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.053831702565813e-05,615,,,,,, +0,,395377.0,,,,,,,,615,21.930349349975586,5.8261000505184075,120.23503875732422,254.0,20.0,0.0031786195784372683 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,9.018482093981406e-05,616,,,,,, +0,,395377.0,,,,,,,,616,17.066566467285156,5.262696420104113,72.81692504882812,253.0,20.0,0.0031446074171593828 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.983270503737533e-05,617,,,,,, +0,,395377.0,,,,,,,,617,23.95561981201172,5.9322052579529165,143.46792602539062,253.0,20.0,0.003110848585295617 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.948196392958022e-05,618,,,,,, +0,,395377.0,,,,,,,,618,19.535688400268555,3.9224825916061525,95.41078186035156,252.0,20.0,0.003110848585295617 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.913259224870632e-05,619,,,,,, +0,,395377.0,,,,,,,,619,18.62806510925293,3.401405578366942,86.75120544433594,252.0,20.0,0.003110848585295617 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.878458464798933e-05,620,,,,,, +0,,395377.0,,,,,,,,620,20.585813522338867,3.226024334267639,105.94392395019531,252.0,20.0,0.003110848585295617 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.843793580154057e-05,621,,,,,, +0,,395377.0,,,,,,,,621,16.105466842651367,6.497200853037327,64.84651184082031,251.0,20.0,0.003884012839932522 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.809264040426568e-05,622,,,,,, +0,,395377.0,,,,,,,,622,7.036957263946533,5.612006966819356,12.379692077636719,251.0,20.0,0.0037377683177741345 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.774869317178341e-05,623,,,,,, +0,,395377.0,,,,,,,,623,20.664464950561523,6.010146696950972,106.75503540039062,250.0,20.0,0.0034746031259450267 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.740608884034473e-05,624,,,,,, +0,,395377.0,,,,,,,,624,9.735597610473633,5.001287840099793,23.695463180541992,250.0,20.0,0.0033390063467560287 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.706482216675233e-05,625,,,,,, +0,,395377.0,,,,,,,,625,10.334614753723145,5.117085758747264,26.701065063476562,250.0,20.0,0.003320065086618026 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.672488792828028e-05,626,,,,,, +0,,395377.0,,,,,,,,626,20.590808868408203,5.747852302534318,105.99534606933594,249.0,20.0,0.003213007333088016 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.638628092259437e-05,627,,,,,, +0,,395377.0,,,,,,,,627,16.453569412231445,5.163800996271278,67.67997741699219,249.0,20.0,0.0031013647863288593 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.60489959676718e-05,628,,,,,, +0,,395377.0,,,,,,,,628,22.25812339782715,7.197578664723393,123.85601806640625,248.0,20.0,0.0031013647863288593 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.571302790172284e-05,629,,,,,, +0,,395377.0,,,,,,,,629,18.08432388305664,6.054620568603406,81.76068878173828,248.0,20.0,0.0031013647863288593 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.537837158311116e-05,630,,,,,, +0,,395377.0,,,,,,,,630,12.205594062805176,5.68443068751766,37.24412536621094,248.0,20.0,0.0034677599626801465 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.504502189027538e-05,631,,,,,, +0,,395377.0,,,,,,,,631,26.493431091308594,6.887680633358684,175.4754638671875,247.0,20.0,0.003415477196063035 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.47129737216508e-05,632,,,,,, +0,,395377.0,,,,,,,,632,15.369258880615234,5.258120453992144,59.053531646728516,247.0,20.0,0.0032093819249653472 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.438222199559051e-05,633,,,,,, +0,,395377.0,,,,,,,,633,19.43496322631836,5.429663832425985,94.42945098876953,246.0,20.0,0.0030954780411805794 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.405276165028942e-05,634,,,,,, +0,,395377.0,,,,,,,,634,12.425870895385742,5.456869386855198,38.60057067871094,246.0,20.0,0.0030954780411805794 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.372458764370438e-05,635,,,,,, +0,,395377.0,,,,,,,,635,19.942548751831055,5.992156306921671,99.42631530761719,246.0,20.0,0.0032629887485121824 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.339769495347956e-05,636,,,,,, +0,,395377.0,,,,,,,,636,15.462815284729004,5.531136561579047,59.77466583251953,245.0,20.0,0.0032429540797200822 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.307207857686721e-05,637,,,,,, +0,,395377.0,,,,,,,,637,21.73047637939453,5.695358761926548,118.05340576171875,245.0,20.0,0.0030467347711836193 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.27477335306529e-05,638,,,,,, +0,,395377.0,,,,,,,,638,22.044775009155273,5.573150251567195,121.49303436279297,244.0,20.0,0.002957959944339076 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.242465485107832e-05,639,,,,,, +0,,395377.0,,,,,,,,639,7.360058307647705,4.347697007618494,13.542613983154297,244.0,20.0,0.002957959944339076 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.210283759376561e-05,640,,,,,, +0,,395377.0,,,,,,,,640,16.40592384338379,5.552158636991014,67.28858947753906,244.0,20.0,0.0031615693534314745 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.178227683364166e-05,641,,,,,, +0,,395377.0,,,,,,,,641,13.967950820922852,5.314670899894057,48.775917053222656,243.0,20.0,0.0031188312869966294 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.14629676648627e-05,642,,,,,, +0,,395377.0,,,,,,,,642,18.134950637817383,5.287535660335846,82.21913146972656,243.0,20.0,0.0029648009392287114 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.114490520073938e-05,643,,,,,, +0,,395377.0,,,,,,,,643,6.442147254943848,4.340338866789848,10.375314712524414,242.0,20.0,0.0028521495698331932 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.082808457366145e-05,644,,,,,, +0,,395377.0,,,,,,,,644,29.323835372924805,6.415629252822409,214.97183227539062,242.0,20.0,0.002852946226843458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.051250093502404e-05,645,,,,,, +0,,395377.0,,,,,,,,645,18.239776611328125,4.818128416742637,83.17236328125,242.0,20.0,0.002852946226843458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,8.019814945515301e-05,646,,,,,, +0,,395377.0,,,,,,,,646,19.036922454833984,5.505131257992572,90.60110473632812,241.0,20.0,0.002940910547353002 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.988502532323105e-05,647,,,,,, +0,,395377.0,,,,,,,,647,20.357812881469727,5.511218023331192,103.6101303100586,241.0,20.0,0.0029100121020978666 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.957312374722412e-05,648,,,,,, +0,,395377.0,,,,,,,,648,25.022146224975586,5.810238350068417,156.52696228027344,240.0,20.0,0.0028341116082319174 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.926243995380814e-05,649,,,,,, +0,,395377.0,,,,,,,,649,17.927282333374023,4.937675161441863,80.34687042236328,240.0,20.0,0.0027984141935199426 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.895296918829601e-05,650,,,,,, +0,,395377.0,,,,,,,,650,10.921091079711914,4.499107550351147,29.81756019592285,240.0,20.0,0.002817100785699622 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.864470671456422e-05,651,,,,,, +0,,395377.0,,,,,,,,651,11.641522407531738,4.520373733113591,33.88125991821289,239.0,20.0,0.0027941390539582417 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.833764781498181e-05,652,,,,,, +0,,395377.0,,,,,,,,652,19.09964370727539,4.9801471738504635,91.1990966796875,239.0,20.0,0.002735326433515064 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.803178779033633e-05,653,,,,,, +0,,395377.0,,,,,,,,653,8.807564735412598,4.196123181232205,19.393299102783203,238.0,20.0,0.002735326433515064 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.772712195976337e-05,654,,,,,, +0,,395377.0,,,,,,,,654,21.239788055419922,5.3418282454769335,112.78215026855469,238.0,20.0,0.0028467520849304044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.742364566067428e-05,655,,,,,, +0,,395377.0,,,,,,,,655,7.480870246887207,4.291353335042741,13.990854263305664,238.0,20.0,0.0028134591282985124 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.712135424868492e-05,656,,,,,, +0,,395377.0,,,,,,,,656,20.746017456054688,5.067471757862056,107.59931182861328,237.0,20.0,0.0027376052146044503 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.682024309754471e-05,657,,,,,, +0,,395377.0,,,,,,,,657,12.199114799499512,4.285475771275641,37.2046012878418,237.0,20.0,0.002692092630427843 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.652030759906523e-05,658,,,,,, +0,,395377.0,,,,,,,,658,18.13346290588379,4.712765225679055,82.20561218261719,236.0,20.0,0.002654540182119879 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.622154316305106e-05,659,,,,,, +0,,395377.0,,,,,,,,659,15.73222827911377,4.463302260611275,61.87574768066406,236.0,20.0,0.002617836687284525 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.592394521722777e-05,660,,,,,, +0,,395377.0,,,,,,,,660,23.82621192932129,5.209918110580432,141.92208862304688,236.0,20.0,0.002593686294815953 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.562750920717332e-05,661,,,,,, +0,,395377.0,,,,,,,,661,22.996583938598633,5.066294565151144,132.21072387695312,235.0,20.0,0.0025614562966725764 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.533223059624779e-05,662,,,,,, +0,,395377.0,,,,,,,,662,8.344014167785645,3.9357419761231585,17.405641555786133,235.0,20.0,0.0025921954123044174 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.503810486552396e-05,663,,,,,, +0,,395377.0,,,,,,,,663,4.090243816375732,3.7705660291488403,4.182523250579834,234.0,20.0,0.002561181074013871 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.474512751371828e-05,664,,,,,, +0,,395377.0,,,,,,,,664,17.738676071166992,5.177929370305306,78.66516876220703,234.0,20.0,0.002561181074013871 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.445329405712191e-05,665,,,,,, +0,,395377.0,,,,,,,,665,10.769631385803223,4.181596438264183,28.996238708496094,234.0,20.0,0.0026747427162064443 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.416260002953211e-05,666,,,,,, +0,,395377.0,,,,,,,,666,5.0311102867126465,3.846501448561975,6.328017711639404,233.0,20.0,0.0026035611931666874 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.38730409821839e-05,667,,,,,, +0,,395377.0,,,,,,,,667,8.468350410461426,3.8706121279410795,17.928239822387695,233.0,20.0,0.0025496689181229684 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.358461248368212e-05,668,,,,,, +0,,395377.0,,,,,,,,668,14.226434707641602,2.954716890867164,50.597862243652344,232.0,20.0,0.0025496689181229684 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.329731011993303e-05,669,,,,,, +0,,395377.0,,,,,,,,669,25.834190368652344,5.482377630154172,166.85134887695312,232.0,20.0,0.002580501216301995 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.301112949407765e-05,670,,,,,, +0,,395377.0,,,,,,,,670,19.420928955078125,4.640469639076286,94.29312133789062,231.0,20.0,0.0025173446860274153 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.272606622642383e-05,671,,,,,, +0,,395377.0,,,,,,,,671,17.39729118347168,3.38343530800993,75.66644287109375,231.0,20.0,0.0025173446860274153 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.244211595437943e-05,672,,,,,, +0,,395377.0,,,,,,,,672,6.3819499015808105,2.2978855823841924,10.182321548461914,231.0,20.0,0.0025173446860274153 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.215927433238551e-05,673,,,,,, +0,,395377.0,,,,,,,,673,11.495635986328125,4.539125586722562,33.03740692138672,230.0,20.0,0.0028456910135020554 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.18775370318499e-05,674,,,,,, +0,,395377.0,,,,,,,,674,10.939269065856934,4.3142330102047985,29.916900634765625,230.0,20.0,0.002754758886420399 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.159689974108095e-05,675,,,,,, +0,,395377.0,,,,,,,,675,6.049371242523193,3.896865972782777,9.148723602294922,229.0,20.0,0.0026374052950454492 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.131735816522107e-05,676,,,,,, +0,,395377.0,,,,,,,,676,12.874002456665039,4.781832443629983,41.43498229980469,229.0,20.0,0.0026374052950454492 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.103890802618226e-05,677,,,,,, +0,,395377.0,,,,,,,,677,12.40422248840332,4.455480939863502,38.4661865234375,229.0,20.0,0.0027321282582228592 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.076154506257882e-05,678,,,,,, +0,,395377.0,,,,,,,,678,15.515047073364258,4.53920969733667,60.17917251586914,228.0,20.0,0.002679017242742176 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.048526502966412e-05,679,,,,,, +0,,395377.0,,,,,,,,679,22.45805549621582,4.924025086633109,126.091064453125,228.0,20.0,0.0025084839647283048 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,7.02100636992638e-05,680,,,,,, +0,,395377.0,,,,,,,,680,7.975762844085693,3.8379243948449417,15.903196334838867,227.0,20.0,0.0025169726623413494 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.993593685971231e-05,681,,,,,, +0,,395377.0,,,,,,,,681,18.1257266998291,4.522837221196756,82.13550567626953,227.0,20.0,0.0025266981510509686 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.966288031578816e-05,682,,,,,, +0,,395377.0,,,,,,,,682,24.461397171020508,5.1233197882749995,149.58998107910156,227.0,20.0,0.0024864018607636184 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.939088988864903e-05,683,,,,,, +0,,395377.0,,,,,,,,683,14.318248748779297,4.133697409336137,51.25305938720703,226.0,20.0,0.0024822688001153127 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.911996141576928e-05,684,,,,,, +0,,395377.0,,,,,,,,684,7.288775444030762,3.748080907596829,13.281561851501465,226.0,20.0,0.002483518024804065 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.885009075087451e-05,685,,,,,, +0,,395377.0,,,,,,,,685,21.15898895263672,4.6988584247362475,111.92569732666016,225.0,20.0,0.0024668224991948135 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.858127376387942e-05,686,,,,,, +0,,395377.0,,,,,,,,686,16.098844528198242,4.1931792238223835,64.79319763183594,225.0,20.0,0.0024453272995874475 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.831350634082403e-05,687,,,,,, +0,,395377.0,,,,,,,,687,11.611871719360352,3.855576654587243,33.708892822265625,225.0,20.0,0.0024250865447646105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.804678438381081e-05,688,,,,,, +0,,395377.0,,,,,,,,688,15.704887390136719,4.174274602610267,61.66087341308594,224.0,20.0,0.0024250865447646105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.778110381094203e-05,689,,,,,, +0,,395377.0,,,,,,,,689,20.830055236816406,4.847180899782119,108.47279357910156,224.0,20.0,0.002540869056506942 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.751646055625718e-05,690,,,,,, +0,,395377.0,,,,,,,,690,21.96735382080078,4.841090192456932,120.64115905761719,224.0,20.0,0.002503866550039505 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.725285056967096e-05,691,,,,,, +0,,395377.0,,,,,,,,691,8.888158798217773,3.7591762898022774,19.74984359741211,223.0,20.0,0.002480214962197101 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.699026981691079e-05,692,,,,,, +0,,395377.0,,,,,,,,692,19.181148529052734,4.4753778200780285,91.97911071777344,223.0,20.0,0.0024486412319415698 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.67287142794558e-05,693,,,,,, +0,,395377.0,,,,,,,,693,15.680610656738281,3.3213283200980377,61.47039031982422,222.0,20.0,0.0024486412319415698 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.64681799544748e-05,694,,,,,, +0,,395377.0,,,,,,,,694,19.660093307495117,4.7280655938265665,96.62981414794922,222.0,20.0,0.0025587206623026576 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.62086628547652e-05,695,,,,,, +0,,395377.0,,,,,,,,695,10.79262638092041,4.019131120222046,29.120197296142578,222.0,20.0,0.0025820925736901304 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.59501590086921e-05,696,,,,,, +0,,395377.0,,,,,,,,696,14.107183456420898,4.063489733709091,49.75315856933594,221.0,20.0,0.002479401815949798 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.569266446012686e-05,697,,,,,, +0,,395377.0,,,,,,,,697,8.575677871704102,3.7868243534259407,18.385562896728516,221.0,20.0,0.002499768341918436 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.543617526838795e-05,698,,,,,, +0,,395377.0,,,,,,,,698,13.0562105178833,3.9164184875630004,42.61616134643555,220.0,20.0,0.002403159428198763 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.51806875081789e-05,699,,,,,, +0,,395377.0,,,,,,,,699,18.45103645324707,3.6213834949179087,85.11018371582031,220.0,20.0,0.002403159428198763 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.492619726952959e-05,700,,,,,, +0,,395377.0,,,,,,,,700,15.628984451293945,4.328277885876146,61.066287994384766,220.0,20.0,0.0024954944133457513 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.467270065773576e-05,701,,,,,, +0,,395377.0,,,,,,,,701,20.305866241455078,4.5916766747476565,103.08204650878906,219.0,20.0,0.002441582720585432 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.442019379329961e-05,702,,,,,, +0,,395377.0,,,,,,,,702,6.766868591308594,2.227885272075241,11.447628021240234,219.0,20.0,0.002441582720585432 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.416867281187033e-05,703,,,,,, +0,,395377.0,,,,,,,,703,20.54620361328125,4.789494386768609,105.53662109375,218.0,20.0,0.0024961000746569527 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.391813386418504e-05,704,,,,,, +0,,395377.0,,,,,,,,704,13.679205894470215,4.055464474958151,46.78016662597656,218.0,20.0,0.0024413565284118013 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.366857311600984e-05,705,,,,,, +0,,395377.0,,,,,,,,705,16.921222686767578,4.157556914178704,71.58193969726562,218.0,20.0,0.00234369379267853 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.341998674808112e-05,706,,,,,, +0,,395377.0,,,,,,,,706,4.660401344299316,3.5199858357238676,5.429834842681885,217.0,20.0,0.002349241357430806 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.317237095604728e-05,707,,,,,, +0,,395377.0,,,,,,,,707,16.51463508605957,4.117940943388476,68.18328857421875,217.0,20.0,0.0023373675546817293 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.292572195040997e-05,708,,,,,, +0,,395377.0,,,,,,,,708,4.082554340362549,2.8931959762066772,4.166813373565674,216.0,20.0,0.0023373675546817293 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.268003595646689e-05,709,,,,,, +0,,395377.0,,,,,,,,709,18.589519500732422,4.49828194033304,86.39256286621094,216.0,20.0,0.0024267315271300723 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.243530921425345e-05,710,,,,,, +0,,395377.0,,,,,,,,710,18.896589279174805,4.450137706460144,89.270263671875,216.0,20.0,0.002414781073497202 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.219153797848541e-05,711,,,,,, +0,,395377.0,,,,,,,,711,14.565272331237793,3.9769599598298773,53.03679275512695,215.0,20.0,0.002358997542048007 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.194871851850156e-05,712,,,,,, +0,,395377.0,,,,,,,,712,11.576533317565918,3.8032723702222486,33.5040283203125,215.0,20.0,0.002346739624299354 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.170684711820658e-05,713,,,,,, +0,,395377.0,,,,,,,,713,20.420671463012695,4.4393191614761625,104.25094604492188,214.0,20.0,0.0023149474920327474 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.146592007601433e-05,714,,,,,, +0,,395377.0,,,,,,,,714,17.285934448242188,4.184157739024517,74.70087432861328,214.0,20.0,0.002349325984763713 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.122593370479066e-05,715,,,,,, +0,,395377.0,,,,,,,,715,10.737166404724121,3.6753255259266906,28.821687698364258,214.0,20.0,0.0023078928379516154 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.0986884331798133e-05,716,,,,,, +0,,395377.0,,,,,,,,716,10.530336380004883,3.6637297835474616,27.721996307373047,213.0,20.0,0.0023208947367287574 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.074876829863831e-05,717,,,,,, +0,,395377.0,,,,,,,,717,10.746455192565918,3.6897706469536087,28.87157440185547,213.0,20.0,0.0023208947367287574 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.051158196119698e-05,718,,,,,, +0,,395377.0,,,,,,,,718,18.792695999145508,3.541176496077979,88.29135131835938,212.0,20.0,0.0023208947367287574 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.0275321689587845e-05,719,,,,,, +0,,395377.0,,,,,,,,719,14.007993698120117,4.396131513332344,49.05596923828125,212.0,20.0,0.0025725301592066834 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,6.003998386809704e-05,720,,,,,, +0,,395377.0,,,,,,,,720,15.621397972106934,3.2062216819196525,61.007015228271484,212.0,20.0,0.0025725301592066834 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.980556489512785e-05,721,,,,,, +0,,395377.0,,,,,,,,721,17.637741088867188,4.506950954052739,77.77247619628906,211.0,20.0,0.0025092686660743305 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.9572061183145544e-05,722,,,,,, +0,,395377.0,,,,,,,,722,7.083792686462402,3.710722530701073,12.545029640197754,211.0,20.0,0.002438214834902059 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.933946915862262e-05,723,,,,,, +0,,395377.0,,,,,,,,723,10.78743839263916,3.961581588341536,29.092205047607422,210.0,20.0,0.002438214834902059 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.9107785261983654e-05,724,,,,,, +0,,395377.0,,,,,,,,724,13.23082160949707,3.2897547926492257,43.7636604309082,210.0,20.0,0.002438214834902059 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.8877005947551485e-05,725,,,,,, +0,,395377.0,,,,,,,,725,13.811942100524902,2.632335427348771,47.69243621826172,210.0,20.0,0.002438214834902059 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.8647127683492525e-05,726,,,,,, +0,,395377.0,,,,,,,,726,13.207777976989746,4.927972485197888,43.611351013183594,209.0,20.0,0.002850304392624699 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.8418146951762806e-05,727,,,,,, +0,,395377.0,,,,,,,,727,12.997025489807129,4.543177323532771,42.23066711425781,209.0,20.0,0.0026835601783810046 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.819006024805415e-05,728,,,,,, +0,,395377.0,,,,,,,,728,10.001921653747559,3.992676945160569,25.009611129760742,208.0,20.0,0.0024929500907003082 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.796286408174051e-05,729,,,,,, +0,,395377.0,,,,,,,,729,15.410452842712402,4.310599190074091,59.37051773071289,208.0,20.0,0.0024311288598156872 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.7736554975824605e-05,730,,,,,, +0,,395377.0,,,,,,,,730,17.1815128326416,4.417011884584717,73.80110168457031,208.0,20.0,0.0024050105760269336 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.7511129466884755e-05,731,,,,,, +0,,395377.0,,,,,,,,731,15.77780532836914,4.267298097401084,62.23478698730469,207.0,20.0,0.0024122975752508837 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.7286584105021505e-05,732,,,,,, +0,,395377.0,,,,,,,,732,16.39771270751953,4.226951208938471,67.22125244140625,207.0,20.0,0.0023326099830151678 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.706291545380543e-05,733,,,,,, +0,,395377.0,,,,,,,,733,12.305499076843262,10.269540889071836,37.8563232421875,206.0,20.0,0.0023326099830151678 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.684012009022411e-05,734,,,,,, +0,,395377.0,,,,,,,,734,15.881364822387695,4.392469965509415,63.054439544677734,206.0,20.0,0.00243200488511047 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.661819460462991e-05,735,,,,,, +0,,395377.0,,,,,,,,735,7.704119682312012,8.834386555195865,14.83836555480957,206.0,20.0,0.00243200488511047 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.639713560068771e-05,736,,,,,, +0,,395377.0,,,,,,,,736,14.801007270812988,4.3873200503782055,54.7674560546875,205.0,20.0,0.002516573974377627 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.6176939695323106e-05,737,,,,,, +0,,395377.0,,,,,,,,737,16.707195281982422,4.327652283361907,69.78260040283203,205.0,20.0,0.0023974359231584384 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.595760351867012e-05,738,,,,,, +0,,395377.0,,,,,,,,738,14.1290864944458,3.969064089213039,49.90776824951172,204.0,20.0,0.0023177322552299885 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.5739123714020666e-05,739,,,,,, +0,,395377.0,,,,,,,,739,7.312697887420654,3.613463889836034,13.368887901306152,204.0,20.0,0.0023006796151496954 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.5521496937771775e-05,740,,,,,, +0,,395377.0,,,,,,,,740,16.224647521972656,4.050897742795898,65.80979919433594,204.0,20.0,0.0022563986218146763 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.530471985937594e-05,741,,,,,, +0,,395377.0,,,,,,,,741,10.067216873168945,3.5888922459404475,25.33721160888672,203.0,20.0,0.0022306095352757254 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.5088789161288724e-05,742,,,,,, +0,,395377.0,,,,,,,,742,15.513023376464844,3.876019507828476,60.163475036621094,203.0,20.0,0.0021856710017776585 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.4873701538919e-05,743,,,,,, +0,,395377.0,,,,,,,,743,17.74660301208496,4.024132463823752,78.7354736328125,202.0,20.0,0.0021575762644974426 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.465945370057795e-05,744,,,,,, +0,,395377.0,,,,,,,,744,11.557136535644531,6.068744479868203,33.391845703125,202.0,20.0,0.0021575762644974426 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.444604236742875e-05,745,,,,,, +0,,395377.0,,,,,,,,745,10.78028392791748,3.8046021928864273,29.053630828857422,202.0,20.0,0.0023360942320514748 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.4233464273436435e-05,746,,,,,, +0,,395377.0,,,,,,,,746,20.887657165527344,4.492178382391011,109.07356262207031,201.0,20.0,0.00226642375533327 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.4021716165317936e-05,747,,,,,, +0,,395377.0,,,,,,,,747,11.087061882019043,3.5725153808884857,30.730735778808594,201.0,20.0,0.0021980168923518718 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.3810794802491984e-05,748,,,,,, +0,,395377.0,,,,,,,,748,16.00066375732422,3.893630915930804,64.00531768798828,200.0,20.0,0.002185145681411597 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.3600696957029966e-05,749,,,,,, +0,,395377.0,,,,,,,,749,13.859283447265625,3.7352635407497594,48.01993179321289,200.0,20.0,0.00220217691638489 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.339141941360659e-05,750,,,,,, +0,,395377.0,,,,,,,,750,6.344883441925049,3.293400960066686,10.064386367797852,199.0,20.0,0.0021484850284997467 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.318295896944988e-05,751,,,,,, +0,,395377.0,,,,,,,,751,7.501628398895264,3.3478981337053675,14.068607330322266,199.0,20.0,0.0021776994942531843 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.297531243429308e-05,752,,,,,, +0,,395377.0,,,,,,,,752,15.69465446472168,3.7437827498322154,61.580543518066406,199.0,20.0,0.0021126912878458954 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.276847663032541e-05,753,,,,,, +0,,395377.0,,,,,,,,753,8.50290298461914,4.188341402005297,18.074838638305664,198.0,20.0,0.0021126912878458954 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.2562448392143154e-05,754,,,,,, +0,,395377.0,,,,,,,,754,20.2648868560791,4.413593659488783,102.66641235351562,198.0,20.0,0.002299780042895521 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.235722456670206e-05,755,,,,,, +0,,395377.0,,,,,,,,755,13.171239852905273,3.301669777266627,43.37038803100586,197.0,20.0,0.002299780042895521 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.21528020132683e-05,756,,,,,, +0,,395377.0,,,,,,,,756,8.660235404968262,3.5196267263955447,18.749919891357422,197.0,20.0,0.0022620292478162077 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.194917760337048e-05,757,,,,,, +0,,395377.0,,,,,,,,757,18.64171028137207,4.089236154126782,86.87833404541016,197.0,20.0,0.002192884362577103 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.174634822075222e-05,758,,,,,, +0,,395377.0,,,,,,,,758,16.198638916015625,3.793692657788668,65.59896850585938,196.0,20.0,0.0021350209552155946 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.154431076132411e-05,759,,,,,, +0,,395377.0,,,,,,,,759,18.2611083984375,3.9614813760952368,83.36701202392578,196.0,20.0,0.0021150686168477017 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.1343062133116285e-05,760,,,,,, +0,,395377.0,,,,,,,,760,17.90699577331543,3.898242371395254,80.16513061523438,195.0,20.0,0.0020928937981350496 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.114259925623111e-05,761,,,,,, +0,,395377.0,,,,,,,,761,7.09876012802124,3.225396778547613,12.598098754882812,195.0,20.0,0.002098762569085367 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.094291906279603e-05,762,,,,,, +0,,395377.0,,,,,,,,762,11.183862686157227,3.3999877634369744,31.269695281982422,195.0,20.0,0.00209568479475614 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.074401849691674e-05,763,,,,,, +0,,395377.0,,,,,,,,763,15.135168075561523,6.4158194225172895,57.26832580566406,194.0,20.0,0.00209568479475614 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.054589451463003e-05,764,,,,,, +0,,395377.0,,,,,,,,764,21.319416046142578,4.433347171820398,113.62937927246094,194.0,20.0,0.00224187675413507 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.034854408385768e-05,765,,,,,, +0,,395377.0,,,,,,,,765,19.365327835083008,5.520847370050891,93.75397491455078,193.0,20.0,0.00224187675413507 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,5.01519641843601e-05,766,,,,,, +0,,395377.0,,,,,,,,766,9.928537368774414,3.695276375880454,24.643962860107422,193.0,20.0,0.00224187675413507 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.995615180768936e-05,767,,,,,, +0,,395377.0,,,,,,,,767,8.464786529541016,2.6755201319447615,17.91315460205078,193.0,20.0,0.00224187675413507 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.976110395714397e-05,768,,,,,, +0,,395377.0,,,,,,,,768,8.775120735168457,4.177709543532088,19.250686645507812,192.0,20.0,0.002736373677159581 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.9566817647722674e-05,769,,,,,, +0,,395377.0,,,,,,,,769,14.923513412475586,4.365892943379154,55.67781448364258,192.0,20.0,0.002736373677159581 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.937328990607884e-05,770,,,,,, +0,,395377.0,,,,,,,,770,18.754714965820312,5.0090788391322345,87.93482971191406,191.0,20.0,0.002736373677159581 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.918051777047458e-05,771,,,,,, +0,,395377.0,,,,,,,,771,6.058437347412109,4.063023261006638,9.176165580749512,191.0,20.0,0.002773033332075112 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.898849829073628e-05,772,,,,,, +0,,395377.0,,,,,,,,772,16.898771286010742,4.362183318279327,71.3921127319336,191.0,20.0,0.002773033332075112 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.879722852820868e-05,773,,,,,, +0,,395377.0,,,,,,,,773,11.058905601501465,3.403216340450851,30.574848175048828,190.0,20.0,0.002773033332075112 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.860670555570995e-05,774,,,,,, +0,,395377.0,,,,,,,,774,15.301257133483887,4.508542208897047,58.5321159362793,190.0,20.0,0.002761457125681985 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.841692645748737e-05,775,,,,,, +0,,395377.0,,,,,,,,775,13.035795211791992,3.245440068313566,42.48298645019531,189.0,20.0,0.002761457125681985 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.822788832917243e-05,776,,,,,, +0,,395377.0,,,,,,,,776,16.12354850769043,3.5880610114723352,64.99220275878906,189.0,20.0,0.002761457125681985 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.8039588277736046e-05,777,,,,,, +0,,395377.0,,,,,,,,777,13.449020385742188,4.41882541505894,45.21903991699219,189.0,20.0,0.0027677333135401987 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.785202342144511e-05,778,,,,,, +0,,395377.0,,,,,,,,778,11.839510917663574,4.100548518062596,35.04350662231445,188.0,20.0,0.0026716726198498797 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.7665190889817576e-05,779,,,,,, +0,,395377.0,,,,,,,,779,17.31059455871582,4.338410367528626,74.91416931152344,188.0,20.0,0.002552031142062093 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.747908782357902e-05,780,,,,,, +0,,395377.0,,,,,,,,780,12.535136222839355,3.8576948241311864,39.28240966796875,187.0,20.0,0.002552031142062093 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.7293711374618455e-05,781,,,,,, +0,,395377.0,,,,,,,,781,8.420939445495605,3.8505976995880253,17.72805404663086,187.0,20.0,0.0025208416067276987 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.710905870594518e-05,782,,,,,, +0,,395377.0,,,,,,,,782,10.713398933410645,3.809138903598428,28.694231033325195,187.0,20.0,0.002458648885229605 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.692512699164543e-05,783,,,,,, +0,,395377.0,,,,,,,,783,11.83584213256836,3.750252468073332,35.02178955078125,186.0,20.0,0.0023572061552614276 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.674191341683841e-05,784,,,,,, +0,,395377.0,,,,,,,,784,15.235759735107422,3.887987849287302,58.03209686279297,186.0,20.0,0.002291941878165532 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.655941517763397e-05,785,,,,,, +0,,395377.0,,,,,,,,785,8.819697380065918,3.3742058080454465,19.44676399230957,185.0,20.0,0.002291941878165532 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.637762948108948e-05,786,,,,,, +0,,395377.0,,,,,,,,786,10.826911926269531,3.732175526624947,29.305505752563477,185.0,20.0,0.002331483734765714 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.6196553545166724e-05,787,,,,,, +0,,395377.0,,,,,,,,787,16.44413948059082,4.001807181453779,67.60243225097656,185.0,20.0,0.002277549904234072 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.601618459868982e-05,788,,,,,, +0,,395377.0,,,,,,,,788,15.978561401367188,3.839610445042836,63.828609466552734,184.0,20.0,0.002199217644368246 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.5836519881302754e-05,789,,,,,, +0,,395377.0,,,,,,,,789,12.24189567565918,3.5894182558707555,37.465999603271484,184.0,20.0,0.0022082881531081486 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.565755664342684e-05,790,,,,,, +0,,395377.0,,,,,,,,790,6.834939002990723,2.6987535018951623,11.679098129272461,183.0,20.0,0.0022082881531081486 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.547929214621864e-05,791,,,,,, +0,,395377.0,,,,,,,,791,18.762929916381836,3.103180860873059,88.01188659667969,183.0,20.0,0.0022082881531081486 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.530172366152843e-05,792,,,,,, +0,,395377.0,,,,,,,,792,7.753754615783691,3.900070631478288,15.03017807006836,183.0,20.0,0.0024703728880730693 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.5124848471858165e-05,793,,,,,, +0,,395377.0,,,,,,,,793,5.920732021331787,3.659332496057229,8.763766288757324,182.0,20.0,0.002393325683687365 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.4948663870319913e-05,794,,,,,, +0,,395377.0,,,,,,,,794,16.03962516784668,3.996593233932192,64.31739807128906,182.0,20.0,0.0022719200071402495 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.47731671605945e-05,795,,,,,, +0,,395377.0,,,,,,,,795,15.84260082244873,3.96039217346223,62.74700164794922,181.0,20.0,0.002242074809575203 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.459835565689027e-05,796,,,,,, +0,,395377.0,,,,,,,,796,13.863188743591309,3.668336396474544,48.047000885009766,181.0,20.0,0.002242074809575203 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.442422668390169e-05,797,,,,,, +0,,395377.0,,,,,,,,797,12.82424545288086,3.9259460727017235,41.115318298339844,181.0,20.0,0.002337077745003092 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.425077757676886e-05,798,,,,,, +0,,395377.0,,,,,,,,798,14.499483108520508,3.1522281082885804,52.558746337890625,180.0,20.0,0.002337077745003092 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.407800568103645e-05,799,,,,,, +0,,395377.0,,,,,,,,799,8.17711353302002,2.3398987904430606,16.716297149658203,180.0,20.0,0.002337077745003092 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.390590835261313e-05,800,,,,,, +0,,395377.0,,,,,,,,800,19.22334861755371,4.972664175889072,92.38427734375,179.0,20.0,0.0027564778147315845 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,801,,,,,, +0,,395377.0,,,,,,,,801,12.669700622558594,4.245935215100093,40.130332946777344,179.0,20.0,0.0026640810108114824 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,802,,,,,, +0,,395377.0,,,,,,,,802,10.333943367004395,3.922881526459967,26.697595596313477,179.0,20.0,0.0025607830414552425 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,803,,,,,, +0,,395377.0,,,,,,,,803,10.497143745422363,3.7929556971699836,27.547504425048828,178.0,20.0,0.0024575183256419345 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,804,,,,,, +0,,395377.0,,,,,,,,804,17.71806526184082,4.727157157698727,78.4824447631836,178.0,20.0,0.0024575183256419345 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,805,,,,,, +0,,395377.0,,,,,,,,805,14.51591682434082,4.153269714263509,52.67796325683594,177.0,20.0,0.0025025699434598797 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,806,,,,,, +0,,395377.0,,,,,,,,806,14.60742473602295,0.5334421396255493,53.344215393066406,177.0,20.0,0.0025025699434598797 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,807,,,,,, +0,,395377.0,,,,,,,,807,18.121156692504883,4.5680265503155475,82.09407806396484,177.0,20.0,0.0026031907314479604 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,808,,,,,, +0,,395377.0,,,,,,,,808,4.375502109527588,0.047862548381090164,4.7862548828125,176.0,20.0,0.0026031907314479604 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,809,,,,,, +0,,395377.0,,,,,,,,809,11.941215515136719,4.046549017633778,35.648155212402344,176.0,20.0,0.0025876316363366222 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,810,,,,,, +0,,395377.0,,,,,,,,810,10.239974975585938,0.26214271783828735,26.214271545410156,175.0,20.0,0.0025876316363366222 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,811,,,,,, +0,,395377.0,,,,,,,,811,4.458722114562988,3.621687104741446,4.97005033493042,175.0,20.0,0.0025248583091104244 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,812,,,,,, +0,,395377.0,,,,,,,,812,16.066694259643555,3.9077783873642984,64.53466796875,175.0,20.0,0.0025248583091104244 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,813,,,,,, +0,,395377.0,,,,,,,,813,9.510570526123047,3.051134754159636,22.61273765563965,174.0,20.0,0.0025248583091104244 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,814,,,,,, +0,,395377.0,,,,,,,,814,15.104024887084961,0.5703288912773132,57.03289031982422,174.0,20.0,0.0025248583091104244 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,815,,,,,, +0,,395377.0,,,,,,,,815,14.401782989501953,4.445410586982102,51.85283660888672,174.0,20.0,0.002762466429725244 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,816,,,,,, +0,,395377.0,,,,,,,,816,18.284658432006836,3.3630484418002724,83.58218383789062,173.0,20.0,0.002762466429725244 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,817,,,,,, +0,,395377.0,,,,,,,,817,15.093131065368652,4.3055320405413555,56.950653076171875,173.0,20.0,0.00266162159338451 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,818,,,,,, +0,,395377.0,,,,,,,,818,5.712047576904297,3.6941572894640764,8.156871795654297,172.0,20.0,0.0025963839561104094 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,819,,,,,, +0,,395377.0,,,,,,,,819,14.60966682434082,4.001727193260591,53.360595703125,172.0,20.0,0.002507639719062874 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,820,,,,,, +0,,395377.0,,,,,,,,820,10.750067710876465,0.2889098823070526,28.890989303588867,172.0,20.0,0.002507639719062874 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,821,,,,,, +0,,395377.0,,,,,,,,821,12.732442855834961,3.607209088012987,40.52877426147461,171.0,20.0,0.002507639719062874 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,822,,,,,, +0,,395377.0,,,,,,,,822,11.086944580078125,3.112697905621197,30.730083465576172,171.0,20.0,0.002507639719062874 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,823,,,,,, +0,,395377.0,,,,,,,,823,16.078123092651367,0.6462650895118713,64.62651062011719,170.0,20.0,0.002507639719062874 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,824,,,,,, +0,,395377.0,,,,,,,,824,16.096078872680664,0.6477094292640686,64.77094268798828,170.0,20.0,0.002507639719062874 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,825,,,,,, +0,,395377.0,,,,,,,,825,15.489877700805664,2.9964339374173727,59.98408126831055,169.0,20.0,0.002507639719062874 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,826,,,,,, +0,,395377.0,,,,,,,,826,10.793086051940918,5.412198587775966,29.122676849365234,169.0,20.0,0.0034588751693501564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,827,,,,,, +0,,395377.0,,,,,,,,827,9.688244819641113,0.2346552163362503,23.46552276611328,169.0,20.0,0.0034588751693501564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,828,,,,,, +0,,395377.0,,,,,,,,828,9.312118530273438,4.7987770516627295,21.67888832092285,168.0,20.0,0.003203576066540737 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,829,,,,,, +0,,395377.0,,,,,,,,829,14.898842811584473,4.790497622814337,55.49388122558594,168.0,20.0,0.0030026933186271245 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,830,,,,,, +0,,395377.0,,,,,,,,830,7.356316566467285,0.13528847694396973,13.528848648071289,167.0,20.0,0.0030026933186271245 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,831,,,,,, +0,,395377.0,,,,,,,,831,15.458861351013184,4.0665936571441925,59.74409484863281,167.0,20.0,0.0030026933186271245 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,832,,,,,, +0,,395377.0,,,,,,,,832,13.691231727600098,0.4686245322227478,46.86245346069336,167.0,20.0,0.0030026933186271245 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,833,,,,,, +0,,395377.0,,,,,,,,833,6.625,0.10972657054662704,10.972657203674316,166.0,20.0,0.0030026933186271245 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,834,,,,,, +0,,395377.0,,,,,,,,834,12.858235359191895,3.0273653467816133,41.33355712890625,166.0,20.0,0.0030026933186271245 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,835,,,,,, +0,,395377.0,,,,,,,,835,5.424776554107666,2.2528686248186984,7.35705041885376,165.0,20.0,0.0030026933186271245 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,836,,,,,, +0,,395377.0,,,,,,,,836,7.419400215148926,2.061650432322873,13.76187515258789,165.0,20.0,0.0030026933186271245 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,837,,,,,, +0,,395377.0,,,,,,,,837,14.80415153503418,0.5479072332382202,54.79072570800781,165.0,20.0,0.0030026933186271245 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,838,,,,,, +0,,395377.0,,,,,,,,838,10.866501808166504,7.29691621659466,29.520214080810547,164.0,20.0,0.004512832621022702 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,839,,,,,, +0,,395377.0,,,,,,,,839,10.141935348510742,0.2571471631526947,25.7147159576416,164.0,20.0,0.004512832621022702 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,840,,,,,, +0,,395377.0,,,,,,,,840,10.732657432556152,6.444396562335791,28.797481536865234,163.0,20.0,0.004095917903593361 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,841,,,,,, +0,,395377.0,,,,,,,,841,11.538928985595703,5.741365370570664,33.286720275878906,163.0,20.0,0.0036606699534534846 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,842,,,,,, +0,,395377.0,,,,,,,,842,18.321828842163086,0.8392234444618225,83.92234802246094,163.0,20.0,0.0036606699534534846 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,843,,,,,, +0,,395377.0,,,,,,,,843,12.620868682861328,5.618407974059015,39.82157897949219,162.0,20.0,0.0036606699534534846 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,844,,,,,, +0,,395377.0,,,,,,,,844,15.451594352722168,5.670016853422618,59.68794250488281,162.0,20.0,0.0033969883960826243 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,845,,,,,, +0,,395377.0,,,,,,,,845,14.17084789276123,5.475387536929259,50.20323181152344,161.0,20.0,0.0033505500363502777 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,846,,,,,, +0,,395377.0,,,,,,,,846,8.91746997833252,4.73480288204618,19.880319595336914,161.0,20.0,0.0031049644489932436 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,847,,,,,, +0,,395377.0,,,,,,,,847,5.437378406524658,4.398287487504336,7.39127254486084,161.0,20.0,0.0031049644489932436 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,848,,,,,, +0,,395377.0,,,,,,,,848,16.108963012695312,0.6487467885017395,64.87467956542969,160.0,20.0,0.0031049644489932436 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,849,,,,,, +0,,395377.0,,,,,,,,849,9.57080078125,0.22900058329105377,22.90005874633789,160.0,20.0,0.0031049644489932436 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,850,,,,,, +0,,395377.0,,,,,,,,850,12.177746772766113,6.102184809719485,37.074378967285156,159.0,20.0,0.003602593919920599 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,851,,,,,, +0,,395377.0,,,,,,,,851,6.466287612915039,0.10453218221664429,10.453218460083008,159.0,20.0,0.003602593919920599 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,852,,,,,, +0,,395377.0,,,,,,,,852,5.7066240310668945,2.9851988826027314,8.141389846801758,159.0,20.0,0.003602593919920599 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,853,,,,,, +0,,395377.0,,,,,,,,853,8.603642463684082,2.737593925266985,18.505666732788086,158.0,20.0,0.003602593919920599 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,854,,,,,, +0,,395377.0,,,,,,,,854,12.695454597473145,5.790293178244129,40.293643951416016,158.0,20.0,0.003490987187187456 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,855,,,,,, +0,,395377.0,,,,,,,,855,5.394243240356445,0.07274464517831802,7.2744646072387695,157.0,20.0,0.003490987187187456 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,856,,,,,, +0,,395377.0,,,,,,,,856,14.935615539550781,5.167146861414317,55.768150329589844,157.0,20.0,0.00310604024698485 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,857,,,,,, +0,,395377.0,,,,,,,,857,12.861958503723145,0.4135749340057373,41.35749435424805,157.0,20.0,0.00310604024698485 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,858,,,,,, +0,,395377.0,,,,,,,,858,13.060953140258789,0.4264712333679199,42.647125244140625,156.0,20.0,0.00310604024698485 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,859,,,,,, +0,,395377.0,,,,,,,,859,10.48583984375,0.27488207817077637,27.488208770751953,156.0,20.0,0.00310604024698485 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,860,,,,,, +0,,395377.0,,,,,,,,860,9.38824462890625,5.234138939329706,22.034786224365234,155.0,20.0,0.00310604024698485 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,861,,,,,, +0,,395377.0,,,,,,,,861,14.767866134643555,4.881426850828724,54.52246856689453,155.0,20.0,0.00310604024698485 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,862,,,,,, +0,,395377.0,,,,,,,,862,12.757019996643066,3.7803246090240745,40.685394287109375,155.0,20.0,0.00310604024698485 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,863,,,,,, +0,,395377.0,,,,,,,,863,8.772603034973145,0.1923963874578476,19.239639282226562,154.0,20.0,0.00310604024698485 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,864,,,,,, +0,,395377.0,,,,,,,,864,12.868371963500977,0.41398748755455017,41.39875030517578,154.0,20.0,0.00310604024698485 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,865,,,,,, +0,,395377.0,,,,,,,,865,14.917549133300781,7.697846112898883,55.63331604003906,153.0,20.0,0.004426802382074392 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,866,,,,,, +0,,395377.0,,,,,,,,866,13.453282356262207,5.641847722555855,45.24769973754883,153.0,20.0,0.004426802382074392 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,867,,,,,, +0,,395377.0,,,,,,,,867,13.024810791015625,5.144562405184834,42.41142272949219,153.0,20.0,0.004426802382074392 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,868,,,,,, +0,,395377.0,,,,,,,,868,10.940937995910645,6.083627141494974,29.92603302001953,152.0,20.0,0.0037883092258429504 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,869,,,,,, +0,,395377.0,,,,,,,,869,4.233215808868408,0.0448002889752388,4.480029106140137,152.0,20.0,0.0037883092258429504 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,870,,,,,, +0,,395377.0,,,,,,,,870,11.083489418029785,5.060504403317336,30.710933685302734,151.0,20.0,0.0032996666146313165 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,871,,,,,, +0,,395377.0,,,,,,,,871,5.17229700088501,0.06688163429498672,6.688163757324219,151.0,20.0,0.0032996666146313165 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,872,,,,,, +0,,395377.0,,,,,,,,872,10.822447776794434,0.29281342029571533,29.281343460083008,151.0,20.0,0.0032996666146313165 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,873,,,,,, +0,,395377.0,,,,,,,,873,12.480029106140137,4.789247968589887,38.93778610229492,150.0,20.0,0.0032996666146313165 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,874,,,,,, +0,,395377.0,,,,,,,,874,6.000234127044678,5.587327441832599,9.000701904296875,150.0,20.0,0.0035905059503725995 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,875,,,,,, +0,,395377.0,,,,,,,,875,14.232752799987793,0.5064281225204468,50.6428108215332,149.0,20.0,0.0035905059503725995 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,876,,,,,, +0,,395377.0,,,,,,,,876,14.815024375915527,5.712325098733964,54.871238708496094,149.0,20.0,0.003390760047059334 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,877,,,,,, +0,,395377.0,,,,,,,,877,9.418561935424805,4.836487654590848,22.177326202392578,149.0,20.0,0.0031155663603469 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,878,,,,,, +0,,395377.0,,,,,,,,878,10.724438667297363,4.37094563027054,28.75339698791504,148.0,20.0,0.002866573445312094 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,879,,,,,, +0,,395377.0,,,,,,,,879,7.426174640655518,4.127358567725279,13.787017822265625,148.0,20.0,0.002805684555867335 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,880,,,,,, +0,,395377.0,,,,,,,,880,15.418879508972168,4.7514961695006,59.435462951660156,147.0,20.0,0.0028381030368213435 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,881,,,,,, +0,,395377.0,,,,,,,,881,13.871845245361328,0.48107028007507324,48.10702896118164,147.0,20.0,0.0028381030368213435 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,882,,,,,, +0,,395377.0,,,,,,,,882,8.620927810668945,4.594474020729628,18.58009910583496,147.0,20.0,0.0029335255379130745 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,883,,,,,, +0,,395377.0,,,,,,,,883,4.505847454071045,3.7426559099771652,5.075665473937988,146.0,20.0,0.0029335255379130745 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,884,,,,,, +0,,395377.0,,,,,,,,884,12.025955200195312,4.5527922616238925,36.15589904785156,146.0,20.0,0.0028210158964175683 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,885,,,,,, +0,,395377.0,,,,,,,,885,10.39939022064209,4.183348416104958,27.03683090209961,145.0,20.0,0.00267854917182994 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,886,,,,,, +0,,395377.0,,,,,,,,886,12.26856803894043,0.37629440426826477,37.62944030761719,145.0,20.0,0.00267854917182994 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,887,,,,,, +0,,395377.0,,,,,,,,887,13.64963150024414,4.2481129944136455,46.57810974121094,145.0,20.0,0.002589957093417619 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,888,,,,,, +0,,395377.0,,,,,,,,888,13.85445785522461,0.4798649549484253,47.98649597167969,144.0,20.0,0.002589957093417619 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,889,,,,,, +0,,395377.0,,,,,,,,889,11.15561294555664,0.3111192286014557,31.111923217773438,144.0,20.0,0.002589957093417619 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,890,,,,,, +0,,395377.0,,,,,,,,890,5.806703567504883,0.08429450541734695,8.429450988769531,143.0,20.0,0.002589957093417619 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,891,,,,,, +0,,395377.0,,,,,,,,891,9.977890014648438,4.739180281614283,24.889572143554688,143.0,20.0,0.002943542913655249 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,892,,,,,, +0,,395377.0,,,,,,,,892,7.396885395050049,4.419622495845365,13.678478240966797,143.0,20.0,0.002862789984568904 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,893,,,,,, +0,,395377.0,,,,,,,,893,15.612751007080078,4.6721092352327664,60.93949890136719,142.0,20.0,0.002862789984568904 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,894,,,,,, +0,,395377.0,,,,,,,,894,7.323747158050537,3.850668968810143,13.409317970275879,142.0,20.0,0.002567155089477365 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,895,,,,,, +0,,395377.0,,,,,,,,895,8.768510818481445,0.19221694767475128,19.221694946289062,141.0,20.0,0.002567155089477365 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,896,,,,,, +0,,395377.0,,,,,,,,896,6.255778789520264,3.906493077077638,9.783693313598633,141.0,20.0,0.0026028413803964824 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,897,,,,,, +0,,395377.0,,,,,,,,897,5.581198692321777,2.4771351432497744,7.787445068359375,141.0,20.0,0.0026028413803964824 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,898,,,,,, +0,,395377.0,,,,,,,,898,11.827701568603516,4.128444516003215,34.9736328125,140.0,20.0,0.0025527594304007506 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,899,,,,,, +0,,395377.0,,,,,,,,899,14.22025203704834,0.5055389404296875,50.55389404296875,140.0,20.0,0.0025527594304007506 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,900,,,,,, +0,,395377.0,,,,,,,,900,11.657482147216797,3.9301566554942315,33.974220275878906,139.0,20.0,0.0024634663472902947 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,901,,,,,, +0,,395377.0,,,,,,,,901,9.367340087890625,3.644202154168956,21.936763763427734,139.0,20.0,0.00237394155116962 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,902,,,,,, +0,,395377.0,,,,,,,,902,7.890209674835205,0.15563851594924927,15.563851356506348,139.0,20.0,0.00237394155116962 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,903,,,,,, +0,,395377.0,,,,,,,,903,9.608307838439941,3.694696669160259,23.07989501953125,138.0,20.0,0.00237394155116962 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,904,,,,,, +0,,395377.0,,,,,,,,904,12.695388793945312,3.251876264783524,40.29322052001953,138.0,20.0,0.00237394155116962 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,905,,,,,, +0,,395377.0,,,,,,,,905,10.146730422973633,0.2573903501033783,25.739036560058594,137.0,20.0,0.00237394155116962 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,906,,,,,, +0,,395377.0,,,,,,,,906,11.334278106689453,2.5214698202337256,32.1164665222168,137.0,20.0,0.00237394155116962 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,907,,,,,, +0,,395377.0,,,,,,,,907,9.63072395324707,0.23187708854675293,23.18770980834961,137.0,20.0,0.00237394155116962 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,908,,,,,, +0,,395377.0,,,,,,,,908,11.319993019104004,0.3203555941581726,32.035560607910156,136.0,20.0,0.00237394155116962 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,909,,,,,, +0,,395377.0,,,,,,,,909,13.199841499328613,0.43558958172798157,43.5589599609375,136.0,20.0,0.00237394155116962 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,910,,,,,, +0,,395377.0,,,,,,,,910,7.11027193069458,6.499824075724776,12.638992309570312,135.0,20.0,0.00398336332399328 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,911,,,,,, +0,,395377.0,,,,,,,,911,8.631400108337402,4.694001831126161,18.625267028808594,135.0,20.0,0.00398336332399328 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,912,,,,,, +0,,395377.0,,,,,,,,912,9.15661907196045,4.063701043991381,20.960918426513672,135.0,20.0,0.00398336332399328 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,913,,,,,, +0,,395377.0,,,,,,,,913,7.644833087921143,0.1461086869239807,14.610868453979492,134.0,20.0,0.00398336332399328 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,914,,,,,, +0,,395377.0,,,,,,,,914,8.278508186340332,2.5815546672118583,17.1334228515625,134.0,20.0,0.00398336332399328 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,915,,,,,, +0,,395377.0,,,,,,,,915,12.69424057006836,2.874926931327334,40.28593826293945,133.0,20.0,0.00398336332399328 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,916,,,,,, +0,,395377.0,,,,,,,,916,6.350343227386475,0.10081715136766434,10.08171558380127,133.0,20.0,0.00398336332399328 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,917,,,,,, +0,,395377.0,,,,,,,,917,4.786952495574951,5.482044687755157,5.728728294372559,133.0,20.0,0.0036173706053873062 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,918,,,,,, +0,,395377.0,,,,,,,,918,12.415153503417969,0.3853401243686676,38.53401184082031,132.0,20.0,0.0036173706053873062 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,919,,,,,, +0,,395377.0,,,,,,,,919,9.439931869506836,4.24369476689286,22.278078079223633,132.0,20.0,0.0036173706053873062 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,920,,,,,, +0,,395377.0,,,,,,,,920,10.291979789733887,0.2648121118545532,26.481210708618164,131.0,20.0,0.0036173706053873062 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,921,,,,,, +0,,395377.0,,,,,,,,921,10.218568801879883,3.7492575185056944,26.104785919189453,131.0,20.0,0.0036173706053873062 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,922,,,,,, +0,,395377.0,,,,,,,,922,9.597845077514648,4.92816312505742,23.029659271240234,131.0,20.0,0.0032954773455635086 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,923,,,,,, +0,,395377.0,,,,,,,,923,9.634248733520508,4.594884545774307,23.204689025878906,130.0,20.0,0.003126889028306215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,924,,,,,, +0,,395377.0,,,,,,,,924,9.845836639404297,2.828672696271732,24.23512840270996,130.0,20.0,0.003126889028306215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,925,,,,,, +0,,395377.0,,,,,,,,925,3.136563777923584,0.02459508180618286,2.459508180618286,129.0,20.0,0.003126889028306215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,926,,,,,, +0,,395377.0,,,,,,,,926,10.47213077545166,0.27416378259658813,27.416378021240234,129.0,20.0,0.003126889028306215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,927,,,,,, +0,,395377.0,,,,,,,,927,6.2243332862854,0.09685581922531128,9.685582160949707,129.0,20.0,0.003126889028306215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,928,,,,,, +0,,395377.0,,,,,,,,928,10.696489334106445,0.2860372066497803,28.603721618652344,128.0,20.0,0.003126889028306215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,929,,,,,, +0,,395377.0,,,,,,,,929,9.040312767028809,0.2043181210756302,20.431812286376953,128.0,20.0,0.003126889028306215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,930,,,,,, +0,,395377.0,,,,,,,,930,13.949677467346191,4.219352218663798,48.64836883544922,127.0,20.0,0.003126889028306215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,931,,,,,, +0,,395377.0,,,,,,,,931,9.908842086791992,3.5004606809013503,24.546287536621094,127.0,20.0,0.003126889028306215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,932,,,,,, +0,,395377.0,,,,,,,,932,10.732003211975098,2.6835690867171653,28.79397201538086,127.0,20.0,0.003126889028306215 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,933,,,,,, +0,,395377.0,,,,,,,,933,6.959240913391113,7.666602783188881,12.107757568359375,126.0,20.0,0.004267561844437076 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,934,,,,,, +0,,395377.0,,,,,,,,934,8.02113151550293,0.16084636747837067,16.084636688232422,126.0,20.0,0.004267561844437076 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,935,,,,,, +0,,395377.0,,,,,,,,935,9.712990760803223,3.747597598017084,23.585548400878906,125.0,20.0,0.004267561844437076 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,936,,,,,, +0,,395377.0,,,,,,,,936,10.8275146484375,4.089885461551749,29.30876922607422,125.0,20.0,0.004267561844437076 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,937,,,,,, +0,,395377.0,,,,,,,,937,9.846198081970215,3.4374341563199247,24.236900329589844,125.0,20.0,0.004267561844437076 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,938,,,,,, +0,,395377.0,,,,,,,,938,10.038335800170898,2.5904091893871835,25.192047119140625,124.0,20.0,0.004267561844437076 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,939,,,,,, +0,,395377.0,,,,,,,,939,10.92891788482666,0.2986031174659729,29.860313415527344,124.0,20.0,0.004267561844437076 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,940,,,,,, +0,,395377.0,,,,,,,,940,8.856080055236816,5.709435109732089,19.607540130615234,124.0,20.0,0.003594117495693373 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,941,,,,,, +0,,395377.0,,,,,,,,941,10.7036714553833,5.3334020615220465,28.64214515686035,123.0,20.0,0.0034050382281168284 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,942,,,,,, +0,,395377.0,,,,,,,,942,9.570612907409668,0.2289915829896927,22.899158477783203,123.0,20.0,0.0034050382281168284 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,943,,,,,, +0,,395377.0,,,,,,,,943,10.922061920166016,6.368421024586016,29.822858810424805,122.0,20.0,0.0034050382281168284 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,944,,,,,, +0,,395377.0,,,,,,,,944,8.39296817779541,0.17610478401184082,17.6104793548584,122.0,20.0,0.0034050382281168284 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,945,,,,,, +0,,395377.0,,,,,,,,945,7.9355149269104,0.1574310064315796,15.743101119995117,122.0,20.0,0.0034050382281168284 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,946,,,,,, +0,,395377.0,,,,,,,,946,4.936562538146973,5.921296301223684,6.092411994934082,121.0,20.0,0.0034050382281168284 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,947,,,,,, +0,,395377.0,,,,,,,,947,10.465377807617188,6.785580964210142,27.38103485107422,121.0,20.0,0.003924124463746977 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,948,,,,,, +0,,395377.0,,,,,,,,948,7.358617305755615,4.511030977849863,13.537311553955078,120.0,20.0,0.003924124463746977 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,949,,,,,, +0,,395377.0,,,,,,,,949,7.253050327301025,0.13151685893535614,13.15168571472168,120.0,20.0,0.003924124463746977 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,950,,,,,, +0,,395377.0,,,,,,,,950,9.342625617980957,0.21821165084838867,21.821165084838867,120.0,20.0,0.003924124463746977 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,951,,,,,, +0,,395377.0,,,,,,,,951,8.215359687805176,2.7210273005885615,16.87303352355957,119.0,20.0,0.003924124463746977 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,952,,,,,, +0,,395377.0,,,,,,,,952,6.694700241088867,2.634804149468228,11.204752922058105,119.0,20.0,0.003924124463746977 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,953,,,,,, +0,,395377.0,,,,,,,,953,6.30123233795166,5.643174855380776,9.926382064819336,118.0,20.0,0.0034814835823411953 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,954,,,,,, +0,,395377.0,,,,,,,,954,7.077791690826416,0.125237837433815,12.523784637451172,118.0,20.0,0.0034814835823411953 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,955,,,,,, +0,,395377.0,,,,,,,,955,7.654131889343262,4.875544071940671,14.646432876586914,118.0,20.0,0.0031296877694806193 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,956,,,,,, +0,,395377.0,,,,,,,,956,10.222861289978027,4.520503991635399,26.126724243164062,117.0,20.0,0.002908800265303939 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,957,,,,,, +0,,395377.0,,,,,,,,957,9.14730167388916,0.20918282866477966,20.918283462524414,117.0,20.0,0.002908800265303939 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,958,,,,,, +0,,395377.0,,,,,,,,958,7.903411865234375,4.304359118842563,15.61598014831543,116.0,20.0,0.0027727276353264295 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,959,,,,,, +0,,395377.0,,,,,,,,959,6.967930316925049,0.12138013541698456,12.13801383972168,116.0,20.0,0.0027727276353264295 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,960,,,,,, +0,,395377.0,,,,,,,,960,6.226405143737793,4.415104479445437,9.69202995300293,115.0,20.0,0.0027939402825306915 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,961,,,,,, +0,,395377.0,,,,,,,,961,3.8916542530059814,4.11940198421169,3.786242961883545,115.0,20.0,0.002654174005760082 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,962,,,,,, +0,,395377.0,,,,,,,,962,9.475545883178711,6.313229027648272,22.446489334106445,115.0,20.0,0.002654174005760082 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,963,,,,,, +0,,395377.0,,,,,,,,963,4.830445766448975,3.8161674393373186,5.833301544189453,114.0,20.0,0.002525597880249594 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,964,,,,,, +0,,395377.0,,,,,,,,964,8.375203132629395,0.1753600388765335,17.5360050201416,114.0,20.0,0.002525597880249594 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,965,,,,,, +0,,395377.0,,,,,,,,965,7.267153263092041,3.870622906441194,13.202879905700684,113.0,20.0,0.002481083719828021 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,966,,,,,, +0,,395377.0,,,,,,,,966,7.5143842697143555,3.716207865240653,14.116493225097656,113.0,20.0,0.0023735484441654397 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,967,,,,,, +0,,395377.0,,,,,,,,967,9.322823524475098,0.2172876000404358,21.728759765625,113.0,20.0,0.0023735484441654397 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,968,,,,,, +0,,395377.0,,,,,,,,968,12.164704322814941,3.8248131326232886,36.99501037597656,112.0,20.0,0.002307307544287097 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,969,,,,,, +0,,395377.0,,,,,,,,969,7.907354831695557,4.200047577554664,15.63156509399414,112.0,20.0,0.002307307544287097 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,970,,,,,, +0,,395377.0,,,,,,,,970,5.999547481536865,3.5842118108490677,8.998642921447754,111.0,20.0,0.0023355523311814545 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,971,,,,,, +0,,395377.0,,,,,,,,971,8.369436264038086,0.17511864006519318,17.511863708496094,111.0,20.0,0.0023355523311814545 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,972,,,,,, +0,,395377.0,,,,,,,,972,8.775556564331055,3.54994709610831,19.252599716186523,111.0,20.0,0.0022439009097635324 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,973,,,,,, +0,,395377.0,,,,,,,,973,5.597649097442627,0.07833418250083923,7.833418369293213,110.0,20.0,0.0022439009097635324 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,974,,,,,, +0,,395377.0,,,,,,,,974,8.051708221435547,2.8654940837639287,16.207500457763672,110.0,20.0,0.0022439009097635324 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,975,,,,,, +0,,395377.0,,,,,,,,975,11.517350196838379,0.331623375415802,33.16233825683594,109.0,20.0,0.0022439009097635324 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,976,,,,,, +0,,395377.0,,,,,,,,976,8.12733268737793,0.16513381898403168,16.513381958007812,109.0,20.0,0.0022439009097635324 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,977,,,,,, +0,,395377.0,,,,,,,,977,6.191794395446777,2.2188066307747722,9.584579467773438,109.0,20.0,0.0022439009097635324 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,978,,,,,, +0,,395377.0,,,,,,,,978,3.102823495864868,0.02406878024339676,2.4068779945373535,108.0,20.0,0.0022439009097635324 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,979,,,,,, +0,,395377.0,,,,,,,,979,11.227987289428711,4.782784432559952,31.516925811767578,108.0,20.0,0.0028975671451345183 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,980,,,,,, +0,,395377.0,,,,,,,,980,10.757173538208008,4.291733059407644,28.929195404052734,107.0,20.0,0.002645160448830286 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,981,,,,,, +0,,395377.0,,,,,,,,981,6.000405311584473,3.5241442482841068,9.001214981079102,107.0,20.0,0.0023399937917719027 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,982,,,,,, +0,,395377.0,,,,,,,,982,8.878406524658203,0.19706523418426514,19.706523895263672,107.0,20.0,0.0023399937917719027 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,983,,,,,, +0,,395377.0,,,,,,,,983,7.654417991638184,4.128389641118952,14.647528648376465,106.0,20.0,0.002610238247181089 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,984,,,,,, +0,,395377.0,,,,,,,,984,8.699575424194336,3.277334461929536,18.920652389526367,106.0,20.0,0.002610238247181089 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,985,,,,,, +0,,395377.0,,,,,,,,985,7.462568283081055,2.8676383494355164,13.922481536865234,105.0,20.0,0.002610238247181089 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,986,,,,,, +0,,395377.0,,,,,,,,986,5.973041534423828,2.2796269069735944,8.919305801391602,105.0,20.0,0.002610238247181089 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,987,,,,,, +0,,395377.0,,,,,,,,987,4.932908058166504,4.7948175447666515,6.083395957946777,105.0,20.0,0.0030251156666668668 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,988,,,,,, +0,,395377.0,,,,,,,,988,11.876319885253906,0.35261741280555725,35.261741638183594,104.0,20.0,0.0030251156666668668 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,989,,,,,, +0,,395377.0,,,,,,,,989,10.650232315063477,4.368916917335172,28.356861114501953,104.0,20.0,0.002676535082107792 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,990,,,,,, +0,,395377.0,,,,,,,,990,10.143120765686035,0.2572072148323059,25.720722198486328,103.0,20.0,0.002676535082107792 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,991,,,,,, +0,,395377.0,,,,,,,,991,10.13813304901123,0.2569543421268463,25.6954345703125,103.0,20.0,0.002676535082107792 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,992,,,,,, +0,,395377.0,,,,,,,,992,4.302720069885254,3.8659502040284353,4.628350257873535,103.0,20.0,0.0025666456045427746 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,993,,,,,, +0,,395377.0,,,,,,,,993,10.093402862548828,5.3112314072888305,25.469194412231445,102.0,20.0,0.0025666456045427746 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,994,,,,,, +0,,395377.0,,,,,,,,994,8.238142013549805,3.9064817092775455,16.966745376586914,102.0,20.0,0.0025239463568799917 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,995,,,,,, +0,,395377.0,,,,,,,,995,5.32907772064209,4.516285823203823,7.099766731262207,101.0,20.0,0.0025239463568799917 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,996,,,,,, +0,,395377.0,,,,,,,,996,6.573265552520752,3.5950936069254924,10.801955223083496,101.0,20.0,0.002376894526721737 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,997,,,,,, +0,,395377.0,,,,,,,,997,9.23726749420166,0.21331779658794403,21.33177947998047,101.0,20.0,0.002376894526721737 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,998,,,,,, +0,,395377.0,,,,,,,,998,8.467140197753906,0.1792311668395996,17.92311668395996,100.0,20.0,0.002376894526721737 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,999,,,,,, +0,,395377.0,,,,,,,,999,10.136072158813477,0.25684988498687744,25.68498992919922,100.0,20.0,0.002376894526721737 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1000,,,,,, +0,,395377.0,,,,,,,,1000,7.982100009918213,4.097853261330783,15.928482055664062,99.0,20.0,0.002624637539325325 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1001,,,,,, +0,,395377.0,,,,,,,,1001,5.788248062133789,3.1695662799774222,8.375953674316406,99.0,20.0,0.002624637539325325 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1002,,,,,, +0,,395377.0,,,,,,,,1002,4.8660969734191895,0.05919724330306053,5.919724464416504,99.0,20.0,0.002624637539325325 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1003,,,,,, +0,,395377.0,,,,,,,,1003,5.834576606750488,0.08510570228099823,8.510570526123047,99.0,20.0,0.002624637539325325 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1004,,,,,, +0,,395377.0,,,,,,,,1004,7.812748432159424,3.792132031728264,15.259757995605469,99.0,20.0,0.0024723808794329554 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1005,,,,,, +0,,395377.0,,,,,,,,1005,4.927226543426514,3.445116562025614,6.069390296936035,99.0,20.0,0.002343938449973232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1006,,,,,, +0,,395377.0,,,,,,,,1006,8.515851974487305,2.88834316686542,18.129932403564453,99.0,20.0,0.002343938449973232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1007,,,,,, +0,,395377.0,,,,,,,,1007,8.76370906829834,0.19200651347637177,19.200651168823242,99.0,20.0,0.002343938449973232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1008,,,,,, +0,,395377.0,,,,,,,,1008,11.120620727539062,0.30917051434516907,30.91705322265625,99.0,20.0,0.002343938449973232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1009,,,,,, +0,,395377.0,,,,,,,,1009,5.963527679443359,2.130837189514652,8.890915870666504,99.0,20.0,0.002343938449973232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1010,,,,,, +0,,395377.0,,,,,,,,1010,5.146509170532227,1.9083039823223513,6.621639251708984,99.0,20.0,0.002343938449973232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1011,,,,,, +0,,395377.0,,,,,,,,1011,7.595058917999268,5.414454217117313,14.42123031616211,99.0,20.0,0.0033534274678368014 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1012,,,,,, +0,,395377.0,,,,,,,,1012,4.386849880218506,4.811583562072032,4.811112880706787,99.0,20.0,0.003076257308223313 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1013,,,,,, +0,,395377.0,,,,,,,,1013,9.97231388092041,0.24861761927604675,24.86176300048828,99.0,20.0,0.003076257308223313 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1014,,,,,, +0,,395377.0,,,,,,,,1014,10.072709083557129,0.25364866852760315,25.3648681640625,99.0,20.0,0.003076257308223313 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1015,,,,,, +0,,395377.0,,,,,,,,1015,10.24361515045166,0.2623291313648224,26.232913970947266,99.0,20.0,0.003076257308223313 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1016,,,,,, +0,,395377.0,,,,,,,,1016,11.065545082092285,5.150380964408815,30.611570358276367,99.0,20.0,0.003033486102822127 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1017,,,,,, +0,,395377.0,,,,,,,,1017,6.887848854064941,4.814585288010249,11.860614776611328,99.0,20.0,0.002992697979000878 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1018,,,,,, +0,,395377.0,,,,,,,,1018,6.256146430969238,0.09784841537475586,9.784841537475586,99.0,20.0,0.002992697979000878 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1019,,,,,, +0,,395377.0,,,,,,,,1019,5.024600982666016,0.06311653554439545,6.3116536140441895,99.0,20.0,0.002992697979000878 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1020,,,,,, +0,,395377.0,,,,,,,,1020,10.155616760253906,4.463652415943321,25.784137725830078,99.0,20.0,0.002992697979000878 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1021,,,,,, +0,,395377.0,,,,,,,,1021,5.285769462585449,3.8253891706346113,6.984840393066406,99.0,20.0,0.002992697979000878 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1022,,,,,, +0,,395377.0,,,,,,,,1022,5.2370452880859375,0.06856660544872284,6.856660842895508,99.0,20.0,0.002992697979000878 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1023,,,,,, +0,,395377.0,,,,,,,,1023,7.390727996826172,6.3915114339936565,13.655715942382812,99.0,20.0,0.003774716483879892 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1024,,,,,, +0,,395377.0,,,,,,,,1024,8.714517593383789,5.753371717652579,18.985702514648438,99.0,20.0,0.003387450947316675 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1025,,,,,, +0,,395377.0,,,,,,,,1025,8.72407054901123,4.408450488458353,19.027353286743164,99.0,20.0,0.002712822178719693 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1026,,,,,, +0,,395377.0,,,,,,,,1026,4.705924034118652,0.05536430701613426,5.536430835723877,99.0,20.0,0.002712822178719693 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1027,,,,,, +0,,395377.0,,,,,,,,1027,5.65704870223999,4.85903841127368,8.000550270080566,99.0,20.0,0.002997283819683283 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1028,,,,,, +0,,395377.0,,,,,,,,1028,4.8565993309021,0.058966390788555145,5.896639347076416,99.0,20.0,0.002997283819683283 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1029,,,,,, +0,,395377.0,,,,,,,,1029,8.929759979248047,0.19935153424739838,19.93515396118164,99.0,20.0,0.002997283819683283 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1030,,,,,, +0,,395377.0,,,,,,,,1030,3.8245067596435547,4.8341341479775535,3.6567130088806152,99.0,20.0,0.002997283819683283 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1031,,,,,, +0,,395377.0,,,,,,,,1031,6.101334095001221,6.983602247530683,9.306570053100586,99.0,20.0,0.004044302442181309 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1032,,,,,, +0,,395377.0,,,,,,,,1032,9.758770942687988,6.371822050963517,23.808401107788086,99.0,20.0,0.003616869715913232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1033,,,,,, +0,,395377.0,,,,,,,,1033,9.068391799926758,0.20558932423591614,20.55893325805664,99.0,20.0,0.003616869715913232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1034,,,,,, +0,,395377.0,,,,,,,,1034,9.327436447143555,0.2175026684999466,21.750267028808594,99.0,20.0,0.003616869715913232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1035,,,,,, +0,,395377.0,,,,,,,,1035,8.31334114074707,0.17277908325195312,17.277908325195312,99.0,20.0,0.003616869715913232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1036,,,,,, +0,,395377.0,,,,,,,,1036,6.541468620300293,0.10697703063488007,10.69770336151123,99.0,20.0,0.003616869715913232 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1037,,,,,, +0,,395377.0,,,,,,,,1037,6.46543025970459,6.167645446604313,10.450447082519531,99.0,20.0,0.0037339487875104223 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1038,,,,,, +0,,395377.0,,,,,,,,1038,3.5773532390594482,0.03199363872408867,3.199363946914673,99.0,20.0,0.0037339487875104223 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1039,,,,,, +0,,395377.0,,,,,,,,1039,10.821406364440918,0.2927570939064026,29.275711059570312,99.0,20.0,0.0037339487875104223 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1040,,,,,, +0,,395377.0,,,,,,,,1040,8.034911155700684,3.0988613015494044,16.139949798583984,99.0,20.0,0.0037339487875104223 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1041,,,,,, +0,,395377.0,,,,,,,,1041,7.88845682144165,2.7251504615314044,15.556937217712402,99.0,20.0,0.0037339487875104223 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1042,,,,,, +0,,395377.0,,,,,,,,1042,6.251373767852783,5.80546094089738,9.769918441772461,99.0,20.0,0.0035284057950322545 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1043,,,,,, +0,,395377.0,,,,,,,,1043,10.377409934997559,0.2692265808582306,26.922658920288086,99.0,20.0,0.0035284057950322545 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1044,,,,,, +0,,395377.0,,,,,,,,1044,8.51600170135498,0.18130570650100708,18.130571365356445,99.0,20.0,0.0035284057950322545 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1045,,,,,, +0,,395377.0,,,,,,,,1045,8.229162216186523,2.7297176156930623,16.929779052734375,99.0,20.0,0.0035284057950322545 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1046,,,,,, +0,,395377.0,,,,,,,,1046,6.087009906768799,4.288135954704847,9.262922286987305,99.0,20.0,0.0027891534974458744 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1047,,,,,, +0,,395377.0,,,,,,,,1047,6.578552722930908,4.055673593125995,10.81933879852295,99.0,20.0,0.0026677459523466564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1048,,,,,, +0,,395377.0,,,,,,,,1048,3.694960594177246,0.03413183242082596,3.4131834506988525,99.0,20.0,0.0026677459523466564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1049,,,,,, +0,,395377.0,,,,,,,,1049,10.208847045898438,0.26055142283439636,26.05514144897461,99.0,20.0,0.0026677459523466564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1050,,,,,, +0,,395377.0,,,,,,,,1050,11.072260856628418,0.30648747086524963,30.648746490478516,99.0,20.0,0.0026677459523466564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1051,,,,,, +0,,395377.0,,,,,,,,1051,3.221435785293579,0.025944121181964874,2.594412088394165,99.0,20.0,0.0026677459523466564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1052,,,,,, +0,,395377.0,,,,,,,,1052,4.100905418395996,0.04204356670379639,4.204356670379639,99.0,20.0,0.0026677459523466564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1053,,,,,, +0,,395377.0,,,,,,,,1053,6.557579517364502,0.10750462114810944,10.750462532043457,99.0,20.0,0.0026677459523466564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1054,,,,,, +0,,395377.0,,,,,,,,1054,10.059053421020508,0.2529614269733429,25.296142578125,99.0,20.0,0.0026677459523466564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1055,,,,,, +0,,395377.0,,,,,,,,1055,8.014586448669434,3.3600864579785448,16.058399200439453,99.0,20.0,0.0026677459523466564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1056,,,,,, +0,,395377.0,,,,,,,,1056,8.06414794921875,2.944508568646764,16.257619857788086,99.0,20.0,0.0026677459523466564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1057,,,,,, +0,,395377.0,,,,,,,,1057,5.8705668449401855,0.086158886551857,8.615888595581055,99.0,20.0,0.0026677459523466564 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1058,,,,,, +0,,395377.0,,,,,,,,1058,9.594746589660645,7.39869796759182,23.014789581298828,99.0,20.0,0.004324582840021279 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1059,,,,,, +0,,395377.0,,,,,,,,1059,10.662117004394531,2.4193678754181738,28.42018699645996,99.0,20.0,0.004324582840021279 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1060,,,,,, +0,,395377.0,,,,,,,,1060,5.655604362487793,2.131980346314165,7.996464252471924,99.0,20.0,0.004324582840021279 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1061,,,,,, +0,,395377.0,,,,,,,,1061,7.103373050689697,5.615864717911766,12.614477157592773,99.0,20.0,0.003438983083599756 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1062,,,,,, +0,,395377.0,,,,,,,,1062,8.322452545166016,0.17315803468227386,17.31580352783203,99.0,20.0,0.003438983083599756 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1063,,,,,, +0,,395377.0,,,,,,,,1063,7.489521026611328,4.494156225578554,14.023231506347656,99.0,20.0,0.0028620292023470656 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1064,,,,,, +0,,395377.0,,,,,,,,1064,7.435948848724365,4.134382370293288,13.823335647583008,99.0,20.0,0.002680496165519846 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1065,,,,,, +0,,395377.0,,,,,,,,1065,5.654719829559326,0.07993964105844498,7.993964195251465,99.0,20.0,0.002680496165519846 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1066,,,,,, +0,,395377.0,,,,,,,,1066,9.963825225830078,0.24819451570510864,24.8194522857666,99.0,20.0,0.002680496165519846 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1067,,,,,, +0,,395377.0,,,,,,,,1067,7.95497465133667,0.15820403397083282,15.820404052734375,99.0,20.0,0.002680496165519846 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1068,,,,,, +0,,395377.0,,,,,,,,1068,10.343034744262695,0.2674458920955658,26.744590759277344,99.0,20.0,0.002680496165519846 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1069,,,,,, +0,,395377.0,,,,,,,,1069,7.830386638641357,0.1532873809337616,15.32873821258545,99.0,20.0,0.002680496165519846 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1070,,,,,, +0,,395377.0,,,,,,,,1070,3.852463483810425,0.03710368648171425,3.7103686332702637,99.0,20.0,0.002680496165519846 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1071,,,,,, +0,,395377.0,,,,,,,,1071,7.678507328033447,7.040125488783786,14.7398681640625,99.0,20.0,0.00410068065765533 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1072,,,,,, +0,,395377.0,,,,,,,,1072,9.837762832641602,6.751254383633828,24.195396423339844,99.0,20.0,0.0038864392127626046 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1073,,,,,, +0,,395377.0,,,,,,,,1073,8.670609474182129,0.18794867396354675,18.79486846923828,99.0,20.0,0.0038864392127626046 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1074,,,,,, +0,,395377.0,,,,,,,,1074,5.921946048736572,0.0876736119389534,8.767361640930176,99.0,20.0,0.0038864392127626046 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1075,,,,,, +0,,395377.0,,,,,,,,1075,8.94298267364502,5.171700519278529,19.99423599243164,99.0,20.0,0.0038864392127626046 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1076,,,,,, +0,,395377.0,,,,,,,,1076,5.5219807624816895,4.446815447036734,7.6230669021606445,99.0,20.0,0.0038864392127626046 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1077,,,,,, +0,,395377.0,,,,,,,,1077,7.884921073913574,0.15542994439601898,15.542994499206543,99.0,20.0,0.0038864392127626046 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1078,,,,,, +0,,395377.0,,,,,,,,1078,7.489312171936035,5.467121147140236,14.022449493408203,99.0,20.0,0.0033530557896248752 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1079,,,,,, +0,,395377.0,,,,,,,,1079,4.540591716766357,0.051542434841394424,5.154243469238281,99.0,20.0,0.0033530557896248752 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1080,,,,,, +0,,395377.0,,,,,,,,1080,8.260462760925293,3.1557025475516873,17.05881118774414,99.0,20.0,0.0033530557896248752 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1081,,,,,, +0,,395377.0,,,,,,,,1081,7.541139602661133,4.728493121392626,14.21719741821289,99.0,20.0,0.002992759318945674 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1082,,,,,, +0,,395377.0,,,,,,,,1082,9.96446418762207,0.2482263445854187,24.822635650634766,99.0,20.0,0.002992759318945674 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1083,,,,,, +0,,395377.0,,,,,,,,1083,6.222689151763916,0.0968046560883522,9.680465698242188,99.0,20.0,0.002992759318945674 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1084,,,,,, +0,,395377.0,,,,,,,,1084,7.998775005340576,4.1439988295587185,15.995100975036621,99.0,20.0,0.002633972644925397 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1085,,,,,, +0,,395377.0,,,,,,,,1085,3.937540054321289,3.829970523040543,3.8760554790496826,99.0,20.0,0.002533921037424446 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1086,,,,,, +0,,395377.0,,,,,,,,1086,9.020971298217773,3.6596879090279497,20.344482421875,99.0,20.0,0.002335628149380839 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1087,,,,,, +0,,395377.0,,,,,,,,1087,7.845667362213135,3.474655606150208,15.388622283935547,99.0,20.0,0.0022426717363200734 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1088,,,,,, +0,,395377.0,,,,,,,,1088,6.281922340393066,3.4055967913642933,9.865638732910156,99.0,20.0,0.002215875492184761 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1089,,,,,, +0,,395377.0,,,,,,,,1089,9.218823432922363,3.469166399572753,21.246675491333008,99.0,20.0,0.0021769344339069704 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1090,,,,,, +0,,395377.0,,,,,,,,1090,7.570062637329102,3.35507400912169,14.326461791992188,99.0,20.0,0.00216525741407911 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1091,,,,,, +0,,395377.0,,,,,,,,1091,7.597019195556641,3.2901448635035395,14.428674697875977,99.0,20.0,0.002103128800301977 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1092,,,,,, +0,,395377.0,,,,,,,,1092,3.29936146736145,3.123268990998065,2.7214465141296387,99.0,20.0,0.0020860440319448066 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1093,,,,,, +0,,395377.0,,,,,,,,1093,8.646139144897461,0.18688930571079254,18.68893051147461,99.0,20.0,0.0020860440319448066 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1094,,,,,, +0,,395377.0,,,,,,,,1094,9.413018226623535,0.22151225805282593,22.151226043701172,99.0,20.0,0.0020860440319448066 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1095,,,,,, +0,,395377.0,,,,,,,,1095,7.84292459487915,0.15377865731716156,15.377866744995117,99.0,20.0,0.0020860440319448066 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1096,,,,,, +0,,395377.0,,,,,,,,1096,5.752199172973633,0.08271948993206024,8.27194881439209,99.0,20.0,0.0020860440319448066 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1097,,,,,, +0,,395377.0,,,,,,,,1097,9.482818603515625,0.22480960190296173,22.480960845947266,99.0,20.0,0.0020860440319448066 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1098,,,,,, +0,,395377.0,,,,,,,,1098,7.072022914886475,5.740691633026989,12.503377914428711,99.0,20.0,0.0020860440319448066 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1099,,,,,, +0,,395377.0,,,,,,,,1099,11.047493934631348,0.30511778593063354,30.51177978515625,99.0,20.0,0.0020860440319448066 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1100,,,,,, +0,,395377.0,,,,,,,,1100,6.300237655639648,0.09923247992992401,9.923248291015625,99.0,20.0,0.0020860440319448066 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1101,,,,,, +0,,395377.0,,,,,,,,1101,8.571402549743652,5.495554661087148,18.367237091064453,99.0,20.0,0.0033825822292182983 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1102,,,,,, +0,,395377.0,,,,,,,,1102,7.65102481842041,4.989027617247256,14.634544372558594,99.0,20.0,0.003124548724826043 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1103,,,,,, +0,,395377.0,,,,,,,,1103,8.49250602722168,4.2193044689067,18.03066635131836,99.0,20.0,0.0026643350238195044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1104,,,,,, +0,,395377.0,,,,,,,,1104,9.638253211975098,3.8477569863658174,23.223981857299805,99.0,20.0,0.002413210548228166 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1105,,,,,, +0,,395377.0,,,,,,,,1105,9.541377067565918,3.293654290385688,22.75946807861328,99.0,20.0,0.002413210548228166 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1106,,,,,, +0,,395377.0,,,,,,,,1106,6.321164131164551,4.30467622879245,9.989279747009277,99.0,20.0,0.002722727268381206 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1107,,,,,, +0,,395377.0,,,,,,,,1107,9.29690933227539,0.21608127653598785,21.60812759399414,99.0,20.0,0.002722727268381206 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1108,,,,,, +0,,395377.0,,,,,,,,1108,8.949925422668457,0.20025290548801422,20.025291442871094,99.0,20.0,0.002722727268381206 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1109,,,,,, +0,,395377.0,,,,,,,,1109,8.556445121765137,4.914384469712514,18.303186416625977,99.0,20.0,0.003017493409807976 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1110,,,,,, +0,,395377.0,,,,,,,,1110,7.687978744506836,0.14776253700256348,14.776254653930664,99.0,20.0,0.003017493409807976 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1111,,,,,, +0,,395377.0,,,,,,,,1111,7.145952224731445,2.6693279992591776,12.766158103942871,99.0,20.0,0.003017493409807976 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1112,,,,,, +0,,395377.0,,,,,,,,1112,7.162616729736328,2.3938473316114255,12.825770378112793,99.0,20.0,0.003017493409807976 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1113,,,,,, +0,,395377.0,,,,,,,,1113,8.446296691894531,0.1783498078584671,17.83498191833496,99.0,20.0,0.003017493409807976 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1114,,,,,, +0,,395377.0,,,,,,,,1114,5.746436595916748,1.9140677245042592,8.255382537841797,99.0,20.0,0.003017493409807976 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1115,,,,,, +0,,395377.0,,,,,,,,1115,8.09011459350586,0.1636248528957367,16.362485885620117,99.0,20.0,0.003017493409807976 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1116,,,,,, +0,,395377.0,,,,,,,,1116,7.240950584411621,5.213834662891028,13.107841491699219,99.0,20.0,0.003317288929219424 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1117,,,,,, +0,,395377.0,,,,,,,,1117,6.701116561889648,4.737389696362045,11.226241111755371,99.0,20.0,0.0030537461045543127 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1118,,,,,, +0,,395377.0,,,,,,,,1118,7.988185405731201,0.1595277339220047,15.952774047851562,99.0,20.0,0.0030537461045543127 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1119,,,,,, +0,,395377.0,,,,,,,,1119,6.687184810638428,3.8747079384895335,11.179610252380371,99.0,20.0,0.0025509964018778936 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1120,,,,,, +0,,395377.0,,,,,,,,1120,7.523520469665527,3.760432266189005,14.150839805603027,99.0,20.0,0.002448084256352443 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1121,,,,,, +0,,395377.0,,,,,,,,1121,9.626747131347656,0.23168563842773438,23.168563842773438,99.0,20.0,0.002448084256352443 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1122,,,,,, +0,,395377.0,,,,,,,,1122,6.24160099029541,3.671241148806385,9.739395141601562,99.0,20.0,0.002399936324626131 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1123,,,,,, +0,,395377.0,,,,,,,,1123,7.789499759674072,4.159918952388006,15.169076919555664,99.0,20.0,0.002399936324626131 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1124,,,,,, +0,,395377.0,,,,,,,,1124,6.239402770996094,3.7077794471076295,9.732537269592285,99.0,20.0,0.0023843701576178964 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1125,,,,,, +0,,395377.0,,,,,,,,1125,9.482657432556152,3.4518436691380003,22.48019790649414,99.0,20.0,0.0023843701576178964 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1126,,,,,, +0,,395377.0,,,,,,,,1126,8.477651596069336,0.17967642843723297,17.96764373779297,99.0,20.0,0.0023843701576178964 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1127,,,,,, +0,,395377.0,,,,,,,,1127,7.859984397888184,0.15444839000701904,15.444839477539062,99.0,20.0,0.0023843701576178964 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1128,,,,,, +0,,395377.0,,,,,,,,1128,5.084762096405029,3.9424573766073934,6.463700771331787,99.0,20.0,0.0025383618909547243 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1129,,,,,, +0,,395377.0,,,,,,,,1129,8.435914039611816,3.8104417374471633,17.791162490844727,99.0,20.0,0.0024012257615971047 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1130,,,,,, +0,,395377.0,,,,,,,,1130,10.582602500915527,0.2799787223339081,27.99787139892578,99.0,20.0,0.0024012257615971047 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1131,,,,,, +0,,395377.0,,,,,,,,1131,4.394078254699707,3.237355890986536,4.8269805908203125,99.0,20.0,0.0024012257615971047 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1132,,,,,, +0,,395377.0,,,,,,,,1132,5.9570536613464355,3.6925696041238156,8.871622085571289,99.0,20.0,0.002337324889594917 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1133,,,,,, +0,,395377.0,,,,,,,,1133,8.916635513305664,3.5982396732361517,19.876598358154297,99.0,20.0,0.0022444839721163313 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1134,,,,,, +0,,395377.0,,,,,,,,1134,6.611537933349609,2.73758273192989,10.928107261657715,99.0,20.0,0.0022444839721163313 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1135,,,,,, +0,,395377.0,,,,,,,,1135,7.745996475219727,0.1500011533498764,15.000115394592285,99.0,20.0,0.0022444839721163313 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1136,,,,,, +0,,395377.0,,,,,,,,1136,8.999292373657227,3.7557053387986135,20.246814727783203,99.0,20.0,0.0023066410150606185 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1137,,,,,, +0,,395377.0,,,,,,,,1137,7.538639068603516,1.993290693405088,14.207768440246582,99.0,20.0,0.0023066410150606185 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1138,,,,,, +0,,395377.0,,,,,,,,1138,8.173819541931152,3.5370825013245466,16.702829360961914,99.0,20.0,0.0022189307869135994 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1139,,,,,, +0,,395377.0,,,,,,,,1139,10.076658248901367,0.253847599029541,25.384761810302734,99.0,20.0,0.0022189307869135994 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1140,,,,,, +0,,395377.0,,,,,,,,1140,10.135832786560059,3.4113381638378177,25.683774948120117,99.0,20.0,0.0021017296415913483 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1141,,,,,, +0,,395377.0,,,,,,,,1141,6.876063823699951,3.2363379516678297,11.820062637329102,99.0,20.0,0.002104446032897868 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1142,,,,,, +0,,395377.0,,,,,,,,1142,7.400704860687256,2.7733254066394375,13.692607879638672,99.0,20.0,0.002104446032897868 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1143,,,,,, +0,,395377.0,,,,,,,,1143,8.172476768493652,3.1895625343516545,16.697345733642578,99.0,20.0,0.0020435479760659335 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1144,,,,,, +0,,395377.0,,,,,,,,1144,8.694708824157715,3.1604069241163453,18.899490356445312,99.0,20.0,0.0020081779216531673 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1145,,,,,, +0,,395377.0,,,,,,,,1145,5.7892351150512695,2.9680779507148274,8.37881088256836,99.0,20.0,0.00196217730639376 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1146,,,,,, +0,,395377.0,,,,,,,,1146,7.361483097076416,0.13547858595848083,13.547859191894531,99.0,20.0,0.00196217730639376 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1147,,,,,, +0,,395377.0,,,,,,,,1147,2.9712185859680176,3.0945379093408425,2.2070348262786865,99.0,20.0,0.002056437896935008 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1148,,,,,, +0,,395377.0,,,,,,,,1148,9.004822731018066,0.20271709561347961,20.271709442138672,99.0,20.0,0.002056437896935008 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1149,,,,,, +0,,395377.0,,,,,,,,1149,4.399122714996338,3.0354321638162407,4.838069915771484,99.0,20.0,0.00200468195213726 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1150,,,,,, +0,,395377.0,,,,,,,,1150,5.652103424072266,0.07986567914485931,7.986567974090576,99.0,20.0,0.00200468195213726 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1151,,,,,, +0,,395377.0,,,,,,,,1151,8.683083534240723,3.088235089823212,18.84898567199707,99.0,20.0,0.0019411685775152989 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1152,,,,,, +0,,395377.0,,,,,,,,1152,7.86839485168457,0.1547790914773941,15.477909088134766,99.0,20.0,0.0019411685775152989 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1153,,,,,, +0,,395377.0,,,,,,,,1153,6.476962566375732,0.10487761348485947,10.487761497497559,99.0,20.0,0.0019411685775152989 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1154,,,,,, +0,,395377.0,,,,,,,,1154,10.03928279876709,0.25196799635887146,25.196800231933594,99.0,20.0,0.0019411685775152989 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1155,,,,,, +0,,395377.0,,,,,,,,1155,7.749081611633301,0.15012066066265106,15.012065887451172,99.0,20.0,0.0019411685775152989 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1156,,,,,, +0,,395377.0,,,,,,,,1156,5.228504180908203,0.0683431401848793,6.834314346313477,99.0,20.0,0.0019411685775152989 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1157,,,,,, +0,,395377.0,,,,,,,,1157,10.022348403930664,0.25111865997314453,25.111865997314453,99.0,20.0,0.0019411685775152989 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1158,,,,,, +0,,395377.0,,,,,,,,1158,6.469074249267578,0.10462228208780289,10.462228775024414,99.0,20.0,0.0019411685775152989 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1159,,,,,, +0,,395377.0,,,,,,,,1159,8.209592819213867,0.16849349439144135,16.849349975585938,99.0,20.0,0.0019411685775152989 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1160,,,,,, +0,,395377.0,,,,,,,,1160,4.790361404418945,0.05736890807747841,5.73689079284668,99.0,20.0,0.0019411685775152989 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1161,,,,,, +0,,395377.0,,,,,,,,1161,5.829634666442871,4.647366256468533,8.496160507202148,99.0,20.0,0.0030071525682390044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1162,,,,,, +0,,395377.0,,,,,,,,1162,8.472583770751953,0.17946170270442963,17.946170806884766,99.0,20.0,0.0030071525682390044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1163,,,,,, +0,,395377.0,,,,,,,,1163,8.309804916381836,0.1726321429014206,17.263214111328125,99.0,20.0,0.0030071525682390044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1164,,,,,, +0,,395377.0,,,,,,,,1164,7.292313098907471,3.4009623191315357,13.294458389282227,99.0,20.0,0.0030071525682390044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1165,,,,,, +0,,395377.0,,,,,,,,1165,6.4643025398254395,3.8841068069761855,10.446802139282227,99.0,20.0,0.0025451667165609696 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1166,,,,,, +0,,395377.0,,,,,,,,1166,4.84666109085083,2.268781236695247,5.872530937194824,99.0,20.0,0.0025451667165609696 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1167,,,,,, +0,,395377.0,,,,,,,,1167,6.022100925445557,3.523240538026628,9.066425323486328,99.0,20.0,0.002340966859775291 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1168,,,,,, +0,,395377.0,,,,,,,,1168,7.002074718475342,0.1225726306438446,12.25726318359375,99.0,20.0,0.002340966859775291 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1169,,,,,, +0,,395377.0,,,,,,,,1169,10.985490798950195,0.30170249938964844,30.170249938964844,99.0,20.0,0.002340966859775291 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1170,,,,,, +0,,395377.0,,,,,,,,1170,3.4636566638946533,2.99829041616804,2.9992294311523438,99.0,20.0,0.002340966859775291 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1171,,,,,, +0,,395377.0,,,,,,,,1171,7.2398858070373535,0.13103985786437988,13.103986740112305,99.0,20.0,0.002340966859775291 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1172,,,,,, +0,,395377.0,,,,,,,,1172,9.144662857055664,2.970666437176635,20.90621566772461,99.0,20.0,0.002340966859775291 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1173,,,,,, +0,,395377.0,,,,,,,,1173,8.219789505004883,0.1689123511314392,16.8912353515625,99.0,20.0,0.002340966859775291 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1174,,,,,, +0,,395377.0,,,,,,,,1174,7.563967227935791,2.0294452386911117,14.303400039672852,99.0,20.0,0.002340966859775291 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1175,,,,,, +0,,395377.0,,,,,,,,1175,6.397334575653076,5.143120195322657,10.231471061706543,99.0,20.0,0.0031183851391284522 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1176,,,,,, +0,,395377.0,,,,,,,,1176,4.176234245300293,0.04360233247280121,4.360233306884766,99.0,20.0,0.0031183851391284522 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1177,,,,,, +0,,395377.0,,,,,,,,1177,10.273335456848145,4.411437285594781,26.385353088378906,99.0,20.0,0.00269547912340197 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1178,,,,,, +0,,395377.0,,,,,,,,1178,4.8899126052856445,0.0597781166434288,5.977811813354492,99.0,20.0,0.00269547912340197 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1179,,,,,, +0,,395377.0,,,,,,,,1179,6.992398262023926,3.8055924149129488,12.223407745361328,99.0,20.0,0.0024506201768958085 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1180,,,,,, +0,,395377.0,,,,,,,,1180,4.095975399017334,0.0419425368309021,4.194253921508789,99.0,20.0,0.0024506201768958085 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1181,,,,,, +0,,395377.0,,,,,,,,1181,9.716094970703125,4.9907311122536075,23.600624084472656,99.0,20.0,0.0024506201768958085 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1182,,,,,, +0,,395377.0,,,,,,,,1182,9.231828689575195,0.21306666731834412,21.30666732788086,99.0,20.0,0.0024506201768958085 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1183,,,,,, +0,,395377.0,,,,,,,,1183,4.90812873840332,3.932048881452402,6.022430896759033,99.0,20.0,0.0024506201768958085 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1184,,,,,, +0,,395377.0,,,,,,,,1184,3.8775508403778076,2.9846000807002633,3.75885009765625,99.0,20.0,0.0024506201768958085 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1185,,,,,, +0,,395377.0,,,,,,,,1185,7.846090793609619,5.260366327625957,15.390283584594727,99.0,20.0,0.0032057748788786866 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1186,,,,,, +0,,395377.0,,,,,,,,1186,2.639619827270508,4.705119426453297,1.7418980598449707,99.0,20.0,0.002984272141466438 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1187,,,,,, +0,,395377.0,,,,,,,,1187,3.8055694103240967,3.0258132082958675,3.6205894947052,99.0,20.0,0.002984272141466438 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1188,,,,,, +0,,395377.0,,,,,,,,1188,9.137734413146973,3.9126186482869496,20.874547958374023,99.0,20.0,0.0024579264222199843 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1189,,,,,, +0,,395377.0,,,,,,,,1189,2.945023775100708,3.6659888009864354,2.1682910919189453,99.0,20.0,0.0024579264222199843 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1190,,,,,, +0,,395377.0,,,,,,,,1190,9.182048797607422,4.065213263260256,21.077503204345703,99.0,20.0,0.002473943500038759 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1191,,,,,, +0,,395377.0,,,,,,,,1191,9.032796859741211,3.8943428837405607,20.39785385131836,99.0,20.0,0.002385477533548278 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1192,,,,,, +0,,395377.0,,,,,,,,1192,7.27106237411499,3.483037232862774,13.217087745666504,99.0,20.0,0.002199354672426124 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1193,,,,,, +0,,395377.0,,,,,,,,1193,8.32016372680664,2.8523854910981283,17.30628204345703,99.0,20.0,0.002199354672426124 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1194,,,,,, +0,,395377.0,,,,,,,,1194,9.116354942321777,0.20776982605457306,20.7769832611084,99.0,20.0,0.002199354672426124 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1195,,,,,, +0,,395377.0,,,,,,,,1195,6.2574782371521,4.331050953561192,9.789009094238281,99.0,20.0,0.0026505472498536435 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1196,,,,,, +0,,395377.0,,,,,,,,1196,7.857601165771484,0.1543547362089157,15.435473442077637,99.0,20.0,0.0026505472498536435 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1197,,,,,, +0,,395377.0,,,,,,,,1197,5.819211959838867,2.38889094209225,8.46580696105957,99.0,20.0,0.0026505472498536435 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1198,,,,,, +0,,395377.0,,,,,,,,1198,9.699655532836914,2.4007831231643437,23.520828247070312,99.0,20.0,0.0026505472498536435 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1199,,,,,, +0,,395377.0,,,,,,,,1199,8.70744514465332,4.4632856978796625,18.95490074157715,99.0,20.0,0.002712668542815546 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1200,,,,,, +0,,395377.0,,,,,,,,1200,8.45265007019043,3.9884706735782114,17.861825942993164,99.0,20.0,0.002468099879454333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1201,,,,,, +0,,395377.0,,,,,,,,1201,7.377421855926514,2.270874051083454,13.606588363647461,99.0,20.0,0.002468099879454333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1202,,,,,, +0,,395377.0,,,,,,,,1202,4.592196464538574,0.05272066965699196,5.272067070007324,99.0,20.0,0.002468099879454333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1203,,,,,, +0,,395377.0,,,,,,,,1203,7.524899482727051,2.179643711324356,14.156028747558594,99.0,20.0,0.002468099879454333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1204,,,,,, +0,,395377.0,,,,,,,,1204,7.723357200622559,4.835971235572567,14.912561416625977,99.0,20.0,0.0030019799710759736 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1205,,,,,, +0,,395377.0,,,,,,,,1205,9.668227195739746,1.9199726290360954,23.36865234375,99.0,20.0,0.0030019799710759736 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1206,,,,,, +0,,395377.0,,,,,,,,1206,8.262847900390625,0.17068664729595184,17.06866455078125,99.0,20.0,0.0030019799710759736 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1207,,,,,, +0,,395377.0,,,,,,,,1207,8.678921699523926,1.9233489032033633,18.83091926574707,99.0,20.0,0.0030019799710759736 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1208,,,,,, +0,,395377.0,,,,,,,,1208,8.836617469787598,4.613325683187584,19.52145004272461,99.0,20.0,0.0028197627278910634 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1209,,,,,, +0,,395377.0,,,,,,,,1209,4.987597942352295,3.9879805221813056,6.219033241271973,99.0,20.0,0.002558667572677143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1210,,,,,, +0,,395377.0,,,,,,,,1210,7.8010454177856445,1.8763664543193468,15.214078903198242,99.0,20.0,0.002558667572677143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1211,,,,,, +0,,395377.0,,,,,,,,1211,5.265601634979248,0.06931640207767487,6.931640625,99.0,20.0,0.002558667572677143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1212,,,,,, +0,,395377.0,,,,,,,,1212,8.93354606628418,0.19952060282230377,19.95206069946289,99.0,20.0,0.002558667572677143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1213,,,,,, +0,,395377.0,,,,,,,,1213,5.1584038734436035,1.6935778883958827,6.65228271484375,99.0,20.0,0.002558667572677143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1214,,,,,, +0,,395377.0,,,,,,,,1214,7.917013168334961,0.15669776499271393,15.669776916503906,99.0,20.0,0.002558667572677143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1215,,,,,, +0,,395377.0,,,,,,,,1215,7.969430923461914,0.1587795615196228,15.87795639038086,99.0,20.0,0.002558667572677143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1216,,,,,, +0,,395377.0,,,,,,,,1216,9.932418823242188,1.8057717361191628,24.663236618041992,99.0,20.0,0.002558667572677143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1217,,,,,, +0,,395377.0,,,,,,,,1217,3.493107318878174,1.4788224785346389,3.0504496097564697,99.0,20.0,0.002558667572677143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1218,,,,,, +0,,395377.0,,,,,,,,1218,5.138486385345459,0.06601010262966156,6.601010322570801,99.0,20.0,0.002558667572677143 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1219,,,,,, +0,,395377.0,,,,,,,,1219,7.312504768371582,8.391790286729286,13.368181228637695,99.0,20.0,0.004986555258965152 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1220,,,,,, +0,,395377.0,,,,,,,,1220,9.495184898376465,7.749031024234643,22.539634704589844,99.0,20.0,0.00456475966148404 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1221,,,,,, +0,,395377.0,,,,,,,,1221,7.775411128997803,0.15114255249500275,15.11425495147705,99.0,20.0,0.00456475966148404 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1222,,,,,, +0,,395377.0,,,,,,,,1222,5.224407196044922,5.134496266872639,6.823607444763184,99.0,20.0,0.0031906027799157946 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1223,,,,,, +0,,395377.0,,,,,,,,1223,6.7642693519592285,2.5916882345571848,11.438835144042969,99.0,20.0,0.0031906027799157946 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1224,,,,,, +0,,395377.0,,,,,,,,1224,7.604001045227051,4.33068188903092,14.455207824707031,99.0,20.0,0.0027887219568642763 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1225,,,,,, +0,,395377.0,,,,,,,,1225,8.991668701171875,4.4991561752849725,20.212528228759766,99.0,20.0,0.0028648130806711086 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1226,,,,,, +0,,395377.0,,,,,,,,1226,6.2557477951049805,4.338912458477959,9.783595085144043,99.0,20.0,0.00277202634406879 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1227,,,,,, +0,,395377.0,,,,,,,,1227,7.1526265144348145,0.127900168299675,12.790017127990723,99.0,20.0,0.00277202634406879 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1228,,,,,, +0,,395377.0,,,,,,,,1228,5.729160785675049,4.288736522611286,8.20582103729248,99.0,20.0,0.002674431749687411 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1229,,,,,, +0,,395377.0,,,,,,,,1229,5.346154689788818,2.1482725174017556,7.145342826843262,99.0,20.0,0.002674431749687411 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1230,,,,,, +0,,395377.0,,,,,,,,1230,7.398749828338623,1.9362023594857611,13.685376167297363,99.0,20.0,0.002674431749687411 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1231,,,,,, +0,,395377.0,,,,,,,,1231,7.496912479400635,1.6280510179886851,14.050922393798828,99.0,20.0,0.002674431749687411 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1232,,,,,, +0,,395377.0,,,,,,,,1232,7.687591075897217,4.832994709042122,14.774765014648438,99.0,20.0,0.003055606431495514 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1233,,,,,, +0,,395377.0,,,,,,,,1233,7.430464744567871,4.427413309247969,13.80295181274414,99.0,20.0,0.002807898028859547 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1234,,,,,, +0,,395377.0,,,,,,,,1234,9.40999984741211,0.22137022018432617,22.137022018432617,99.0,20.0,0.002807898028859547 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1235,,,,,, +0,,395377.0,,,,,,,,1235,3.2815170288085938,3.2740840702167007,2.6920886039733887,99.0,20.0,0.002807898028859547 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1236,,,,,, +0,,395377.0,,,,,,,,1236,8.860356330871582,3.077801342105532,19.626476287841797,99.0,20.0,0.002807898028859547 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1237,,,,,, +0,,395377.0,,,,,,,,1237,6.655468463897705,0.11073814332485199,11.073814392089844,99.0,20.0,0.002807898028859547 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1238,,,,,, +0,,395377.0,,,,,,,,1238,6.352784156799316,5.141184492676386,10.089466094970703,99.0,20.0,0.003246222649771579 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1239,,,,,, +0,,395377.0,,,,,,,,1239,6.310948371887207,4.781267602416115,9.957015991210938,99.0,20.0,0.0030519568310975903 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1240,,,,,, +0,,395377.0,,,,,,,,1240,6.972526550292969,3.952354966563439,12.154031753540039,99.0,20.0,0.002531760739433217 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1241,,,,,, +0,,395377.0,,,,,,,,1241,8.668383598327637,0.1878521740436554,18.78521728515625,99.0,20.0,0.002531760739433217 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1242,,,,,, +0,,395377.0,,,,,,,,1242,9.330699920654297,4.208477727737005,21.765491485595703,99.0,20.0,0.002629950599056753 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1243,,,,,, +0,,395377.0,,,,,,,,1243,7.398635387420654,4.328963331323264,13.684951782226562,99.0,20.0,0.00275380507752306 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1244,,,,,, +0,,395377.0,,,,,,,,1244,9.604515075683594,0.2306167483329773,23.061676025390625,99.0,20.0,0.00275380507752306 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1245,,,,,, +0,,395377.0,,,,,,,,1245,6.026605606079102,3.835942495002345,9.07999324798584,99.0,20.0,0.002485199209884028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1246,,,,,, +0,,395377.0,,,,,,,,1246,7.715221405029297,0.14881160855293274,14.881160736083984,99.0,20.0,0.002485199209884028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1247,,,,,, +0,,395377.0,,,,,,,,1247,5.361692428588867,0.07186935096979141,7.1869354248046875,99.0,20.0,0.002485199209884028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1248,,,,,, +0,,395377.0,,,,,,,,1248,4.842473983764648,7.601162864102186,5.862388610839844,99.0,20.0,0.002485199209884028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1249,,,,,, +0,,395377.0,,,,,,,,1249,6.0215911865234375,6.817030498167574,9.064889907836914,99.0,20.0,0.002485199209884028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1250,,,,,, +0,,395377.0,,,,,,,,1250,7.508674144744873,0.14095045626163483,14.095046043395996,99.0,20.0,0.002485199209884028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1251,,,,,, +0,,395377.0,,,,,,,,1251,5.6135478019714355,3.490269839799616,7.8779802322387695,99.0,20.0,0.002485199209884028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1252,,,,,, +0,,395377.0,,,,,,,,1252,6.5525898933410645,2.597863746436809,10.734107971191406,99.0,20.0,0.002485199209884028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1253,,,,,, +0,,395377.0,,,,,,,,1253,5.858471870422363,2.8634577962701755,8.580423355102539,99.0,20.0,0.002485199209884028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1254,,,,,, +0,,395377.0,,,,,,,,1254,6.894959926605225,0.1188511848449707,11.88511848449707,99.0,20.0,0.002485199209884028 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1255,,,,,, +0,,395377.0,,,,,,,,1255,7.500921249389648,7.837313314556255,14.06595516204834,99.0,20.0,0.004956467715757371 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1256,,,,,, +0,,395377.0,,,,,,,,1256,7.551799297332764,6.81206601864959,14.257417678833008,99.0,20.0,0.0043240296858938765 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1257,,,,,, +0,,395377.0,,,,,,,,1257,4.133078098297119,6.2131941451715536,4.2705841064453125,99.0,20.0,0.0043240296858938765 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1258,,,,,, +0,,395377.0,,,,,,,,1258,4.718894004821777,5.743460126374698,5.566989898681641,99.0,20.0,0.0043240296858938765 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1259,,,,,, +0,,395377.0,,,,,,,,1259,9.067947387695312,0.20556920766830444,20.556921005249023,99.0,20.0,0.0043240296858938765 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1260,,,,,, +0,,395377.0,,,,,,,,1260,6.990835189819336,4.917343933476422,12.217945098876953,99.0,20.0,0.0031454642049596948 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1261,,,,,, +0,,395377.0,,,,,,,,1261,7.038432598114014,0.1238488256931305,12.384882926940918,99.0,20.0,0.0031454642049596948 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1262,,,,,, +0,,395377.0,,,,,,,,1262,8.834137916564941,5.186259829295181,19.510496139526367,99.0,20.0,0.0032647858150909336 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1263,,,,,, +0,,395377.0,,,,,,,,1263,5.474198818206787,3.8739652526114092,7.491713523864746,99.0,20.0,0.0032647858150909336 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1264,,,,,, +0,,395377.0,,,,,,,,1264,8.912432670593262,3.712552857186248,19.857864379882812,99.0,20.0,0.0032647858150909336 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1265,,,,,, +0,,395377.0,,,,,,,,1265,5.397078514099121,4.012525810053141,7.282114028930664,99.0,20.0,0.0026377667557176327 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1266,,,,,, +0,,395377.0,,,,,,,,1266,6.034295082092285,2.8283007089430647,9.103179931640625,99.0,20.0,0.0026377667557176327 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1267,,,,,, +0,,395377.0,,,,,,,,1267,7.930370807647705,2.5628348939997334,15.722694396972656,99.0,20.0,0.0026377667557176327 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1268,,,,,, +0,,395377.0,,,,,,,,1268,9.972860336303711,4.1457802050515316,24.864484786987305,99.0,20.0,0.0025284763906987936 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1269,,,,,, +0,,395377.0,,,,,,,,1269,6.964410305023193,2.6151404373251355,12.125752449035645,99.0,20.0,0.0025284763906987936 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1270,,,,,, +0,,395377.0,,,,,,,,1270,6.118531227111816,0.09359105676412582,9.359106063842773,99.0,20.0,0.0025284763906987936 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1271,,,,,, +0,,395377.0,,,,,,,,1271,5.783232688903809,2.9309127776819865,8.361445426940918,99.0,20.0,0.0025284763906987936 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1272,,,,,, +0,,395377.0,,,,,,,,1272,7.237479209899902,4.096577171217723,13.09527587890625,99.0,20.0,0.0026678934462370414 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1273,,,,,, +0,,395377.0,,,,,,,,1273,6.236114501953125,3.6357588872508213,9.722280502319336,99.0,20.0,0.0024056327874987017 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1274,,,,,, +0,,395377.0,,,,,,,,1274,4.813889503479004,3.1650755661393064,5.79338264465332,99.0,20.0,0.00211746982909273 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1275,,,,,, +0,,395377.0,,,,,,,,1275,5.7597503662109375,2.7250221698618096,8.293681144714355,99.0,20.0,0.00211746982909273 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1276,,,,,, +0,,395377.0,,,,,,,,1276,8.625367164611816,0.18599238991737366,18.599239349365234,99.0,20.0,0.00211746982909273 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1277,,,,,, +0,,395377.0,,,,,,,,1277,7.33995246887207,2.4311675092183798,13.468725204467773,99.0,20.0,0.00211746982909273 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1278,,,,,, +0,,395377.0,,,,,,,,1278,6.469521999359131,5.278810272520842,10.463679313659668,99.0,20.0,0.003364651964412311 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1279,,,,,, +0,,395377.0,,,,,,,,1279,6.819578170776367,5.056315191370101,11.62666130065918,99.0,20.0,0.003273043475080111 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1280,,,,,, +0,,395377.0,,,,,,,,1280,8.090806007385254,2.1712687275536893,16.365285873413086,99.0,20.0,0.003273043475080111 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1281,,,,,, +0,,395377.0,,,,,,,,1281,10.279927253723145,2.0778376140276147,26.41922378540039,99.0,20.0,0.003273043475080111 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1282,,,,,, +0,,395377.0,,,,,,,,1282,7.482064247131348,3.812299212579394,13.995321273803711,99.0,20.0,0.0024291357410360106 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1283,,,,,, +0,,395377.0,,,,,,,,1283,7.749672889709473,0.1501435786485672,15.014357566833496,99.0,20.0,0.0024291357410360106 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1284,,,,,, +0,,395377.0,,,,,,,,1284,7.809959411621094,0.15248866379261017,15.24886703491211,99.0,20.0,0.0024291357410360106 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1285,,,,,, +0,,395377.0,,,,,,,,1285,8.757430076599121,4.6342226607398,19.173145294189453,99.0,20.0,0.0029777747433545193 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1286,,,,,, +0,,395377.0,,,,,,,,1286,11.300040245056152,3.340513146443925,31.922727584838867,99.0,20.0,0.0029777747433545193 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1287,,,,,, +0,,395377.0,,,,,,,,1287,6.27266788482666,0.0983659029006958,9.836590766906738,99.0,20.0,0.0029777747433545193 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1288,,,,,, +0,,395377.0,,,,,,,,1288,7.750091075897217,2.6575936693730684,15.01597785949707,99.0,20.0,0.0029777747433545193 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1289,,,,,, +0,,395377.0,,,,,,,,1289,9.304542541503906,0.21643628180027008,21.64362907409668,99.0,20.0,0.0029777747433545193 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1290,,,,,, +0,,395377.0,,,,,,,,1290,8.714675903320312,2.0585373822656474,18.986392974853516,99.0,20.0,0.0029777747433545193 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1291,,,,,, +0,,395377.0,,,,,,,,1291,8.297567367553711,0.17212405800819397,17.212406158447266,99.0,20.0,0.0029777747433545193 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1292,,,,,, +0,,395377.0,,,,,,,,1292,5.957876682281494,2.1757303949335327,8.874073028564453,99.0,20.0,0.0029777747433545193 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1293,,,,,, +0,,395377.0,,,,,,,,1293,9.323427200317383,5.089386758882226,21.731576919555664,99.0,20.0,0.0032517652866311513 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1294,,,,,, +0,,395377.0,,,,,,,,1294,8.537086486816406,2.1206897710261057,18.220462799072266,99.0,20.0,0.0032517652866311513 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1295,,,,,, +0,,395377.0,,,,,,,,1295,6.765126705169678,0.11441733688116074,11.441734313964844,99.0,20.0,0.0032517652866311513 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1296,,,,,, +0,,395377.0,,,,,,,,1296,6.933739185333252,1.8448982690099975,12.019184112548828,99.0,20.0,0.0032517652866311513 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1297,,,,,, +0,,395377.0,,,,,,,,1297,9.677780151367188,4.087142244090203,23.414857864379883,99.0,20.0,0.002683003456386732 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1298,,,,,, +0,,395377.0,,,,,,,,1298,8.28721809387207,3.7686618834688037,17.16949462890625,99.0,20.0,0.0025227829837807404 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1299,,,,,, +0,,395377.0,,,,,,,,1299,10.151373863220215,3.6074247795192873,25.762598037719727,99.0,20.0,0.002350862435891647 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1300,,,,,, +0,,395377.0,,,,,,,,1300,8.672521591186523,0.18803156912326813,18.803157806396484,99.0,20.0,0.002350862435891647 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1301,,,,,, +0,,395377.0,,,,,,,,1301,4.461913585662842,0.04977168142795563,4.977168083190918,99.0,20.0,0.002350862435891647 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1302,,,,,, +0,,395377.0,,,,,,,,1302,5.973174095153809,3.8850205066239676,8.919702529907227,99.0,20.0,0.0025703335988729314 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1303,,,,,, +0,,395377.0,,,,,,,,1303,10.113794326782227,0.2557220757007599,25.572208404541016,99.0,20.0,0.0025703335988729314 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1304,,,,,, +0,,395377.0,,,,,,,,1304,4.48670768737793,3.7081028222558183,5.0326361656188965,99.0,20.0,0.0024993624502784957 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1305,,,,,, +0,,395377.0,,,,,,,,1305,3.7007687091827393,0.03423922136425972,3.423922300338745,99.0,20.0,0.0024993624502784957 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1306,,,,,, +0,,395377.0,,,,,,,,1306,6.579951286315918,3.7076241593965653,10.82394027709961,99.0,20.0,0.0024993624502784957 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1307,,,,,, +0,,395377.0,,,,,,,,1307,4.317459583282471,3.504121681843703,4.660114765167236,99.0,20.0,0.002358014049388204 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1308,,,,,, +0,,395377.0,,,,,,,,1308,8.232217788696289,2.814114155067828,16.942354202270508,99.0,20.0,0.002358014049388204 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1309,,,,,, +0,,395377.0,,,,,,,,1309,7.400382995605469,2.173448986975729,13.691417694091797,99.0,20.0,0.002358014049388204 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1310,,,,,, +0,,395377.0,,,,,,,,1310,6.466541767120361,3.5083975692389533,10.45404052734375,99.0,20.0,0.0023489565621003248 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1311,,,,,, +0,,395377.0,,,,,,,,1311,7.218421459197998,0.1302640289068222,13.026403427124023,99.0,20.0,0.0023489565621003248 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1312,,,,,, +0,,395377.0,,,,,,,,1312,6.311142444610596,0.09957630187273026,9.957630157470703,99.0,20.0,0.0023489565621003248 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1313,,,,,, +0,,395377.0,,,,,,,,1313,7.49467134475708,3.4166280095478183,14.042524337768555,99.0,20.0,0.0022283293521294574 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1314,,,,,, +0,,395377.0,,,,,,,,1314,8.51953411102295,0.18145614862442017,18.145614624023438,99.0,20.0,0.0022283293521294574 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1315,,,,,, +0,,395377.0,,,,,,,,1315,5.228633403778076,3.214573773296089,6.834651947021484,99.0,20.0,0.0021701590976682867 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1316,,,,,, +0,,395377.0,,,,,,,,1316,9.406997680664062,3.172061211689847,22.122901916503906,99.0,20.0,0.0020182361347233333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1317,,,,,, +0,,395377.0,,,,,,,,1317,5.566006183624268,6.446013077839516,7.745105743408203,99.0,20.0,0.0020182361347233333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1318,,,,,, +0,,395377.0,,,,,,,,1318,7.082438945770264,0.12540234625339508,12.54023551940918,99.0,20.0,0.0020182361347233333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1319,,,,,, +0,,395377.0,,,,,,,,1319,7.023327350616455,0.12331780791282654,12.331781387329102,99.0,20.0,0.0020182361347233333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1320,,,,,, +0,,395377.0,,,,,,,,1320,6.963147163391113,5.682836896684113,12.121354103088379,99.0,20.0,0.0020182361347233333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1321,,,,,, +0,,395377.0,,,,,,,,1321,6.982409954071045,4.858640471189703,12.188512802124023,99.0,20.0,0.0020182361347233333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1322,,,,,, +0,,395377.0,,,,,,,,1322,7.590291976928711,3.7009224830300895,14.403133392333984,99.0,20.0,0.0020182361347233333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1323,,,,,, +0,,395377.0,,,,,,,,1323,5.209294319152832,2.608368093837773,6.784186840057373,99.0,20.0,0.0020182361347233333 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1324,,,,,, +0,,395377.0,,,,,,,,1324,7.124385833740234,4.833603322173009,12.689218521118164,99.0,20.0,0.003135433900135723 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1325,,,,,, +0,,395377.0,,,,,,,,1325,6.394855976104736,3.2483895447615745,10.22354507446289,99.0,20.0,0.003135433900135723 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1326,,,,,, +0,,395377.0,,,,,,,,1326,7.816845417022705,4.043206130140749,15.27576732635498,99.0,20.0,0.002651250774943711 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1327,,,,,, +0,,395377.0,,,,,,,,1327,8.279069900512695,4.502472590442129,17.13574981689453,99.0,20.0,0.002651250774943711 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1328,,,,,, +0,,395377.0,,,,,,,,1328,9.797607421875,3.608238003480296,23.99827766418457,99.0,20.0,0.0022905815548168685 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1329,,,,,, +0,,395377.0,,,,,,,,1329,7.03016471862793,4.477563938602367,12.355804443359375,99.0,20.0,0.0022905815548168685 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1330,,,,,, +0,,395377.0,,,,,,,,1330,7.687720775604248,4.06353367498401,14.775262832641602,99.0,20.0,0.0022905815548168685 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1331,,,,,, +0,,395377.0,,,,,,,,1331,8.139349937438965,3.2725839074808296,16.562255859375,99.0,20.0,0.0022905815548168685 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1332,,,,,, +0,,395377.0,,,,,,,,1332,9.614858627319336,2.528869572098003,23.111377716064453,99.0,20.0,0.0022905815548168685 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1333,,,,,, +0,,395377.0,,,,,,,,1333,7.142394065856934,2.1137380818454,12.753448486328125,99.0,20.0,0.0022905815548168685 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1334,,,,,, +0,,395377.0,,,,,,,,1334,6.88176965713501,0.11839687824249268,11.839688301086426,99.0,20.0,0.0022905815548168685 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1335,,,,,, +0,,395377.0,,,,,,,,1335,5.300880432128906,5.476098072755264,7.024833679199219,99.0,20.0,0.0036388594442779525 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1336,,,,,, +0,,395377.0,,,,,,,,1336,6.522071838378906,0.10634354501962662,10.634354591369629,99.0,20.0,0.0036388594442779525 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1337,,,,,, +0,,395377.0,,,,,,,,1337,4.623100757598877,4.737308047631918,5.343265533447266,99.0,20.0,0.0032308844055998695 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1338,,,,,, +0,,395377.0,,,,,,,,1338,3.7447221279144287,4.017102487651838,3.5057361125946045,99.0,20.0,0.002771388417662458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1339,,,,,, +0,,395377.0,,,,,,,,1339,8.861656188964844,6.281335102121917,19.63223648071289,99.0,20.0,0.002771388417662458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1340,,,,,, +0,,395377.0,,,,,,,,1340,6.816303253173828,6.181096488255408,11.615497589111328,99.0,20.0,0.002771388417662458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1341,,,,,, +0,,395377.0,,,,,,,,1341,8.471903800964355,0.17943286895751953,17.943286895751953,99.0,20.0,0.002771388417662458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1342,,,,,, +0,,395377.0,,,,,,,,1342,6.398747444152832,5.1643330839018375,10.235992431640625,99.0,20.0,0.002771388417662458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1343,,,,,, +0,,395377.0,,,,,,,,1343,7.242352485656738,0.1311291605234146,13.112916946411133,99.0,20.0,0.002771388417662458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1344,,,,,, +0,,395377.0,,,,,,,,1344,6.152405261993408,0.09463021904230118,9.463022232055664,99.0,20.0,0.002771388417662458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1345,,,,,, +0,,395377.0,,,,,,,,1345,4.837179660797119,0.058495767414569855,5.849576950073242,99.0,20.0,0.002771388417662458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1346,,,,,, +0,,395377.0,,,,,,,,1346,9.091816902160645,3.561958369080736,20.665283203125,99.0,20.0,0.002771388417662458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1347,,,,,, +0,,395377.0,,,,,,,,1347,8.492095947265625,9.033737887214379,18.0289249420166,99.0,20.0,0.005956560490244044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1348,,,,,, +0,,395377.0,,,,,,,,1348,8.817496299743652,0.19437061250209808,19.437061309814453,99.0,20.0,0.005956560490244044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1349,,,,,, +0,,395377.0,,,,,,,,1349,8.459214210510254,2.6741362068966854,17.889575958251953,99.0,20.0,0.005956560490244044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1350,,,,,, +0,,395377.0,,,,,,,,1350,9.995346069335938,2.559498178750525,24.976734161376953,99.0,20.0,0.005956560490244044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1351,,,,,, +0,,395377.0,,,,,,,,1351,4.654179096221924,2.1359194479218724,5.415346145629883,99.0,20.0,0.005956560490244044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1352,,,,,, +0,,395377.0,,,,,,,,1352,4.8358330726623535,1.9878702998324225,5.846320152282715,99.0,20.0,0.005956560490244044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1353,,,,,, +0,,395377.0,,,,,,,,1353,10.646214485168457,0.2833547294139862,28.335472106933594,99.0,20.0,0.005956560490244044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1354,,,,,, +0,,395377.0,,,,,,,,1354,5.057619571685791,2.191474066250055,6.394879341125488,99.0,20.0,0.005956560490244044 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1355,,,,,, +0,,395377.0,,,,,,,,1355,8.663640975952148,6.773085225344618,18.764667510986328,99.0,20.0,0.0044380998853400105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1356,,,,,, +0,,395377.0,,,,,,,,1356,5.790744304656982,0.0838317945599556,8.383179664611816,99.0,20.0,0.0044380998853400105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1357,,,,,, +0,,395377.0,,,,,,,,1357,7.2688889503479,0.13209186494350433,13.209186553955078,99.0,20.0,0.0044380998853400105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1358,,,,,, +0,,395377.0,,,,,,,,1358,3.6599278450012207,0.033487677574157715,3.3487677574157715,99.0,20.0,0.0044380998853400105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1359,,,,,, +0,,395377.0,,,,,,,,1359,5.876185417175293,2.9074128814103406,8.6323881149292,99.0,20.0,0.0044380998853400105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1360,,,,,, +0,,395377.0,,,,,,,,1360,5.716921806335449,2.761290709496615,8.170799255371094,99.0,20.0,0.0044380998853400105 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1361,,,,,, +0,,395377.0,,,,,,,,1361,4.874500274658203,4.730836733975589,5.940188407897949,99.0,20.0,0.003276436120950546 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1362,,,,,, +0,,395377.0,,,,,,,,1362,8.853392601013184,2.1964269014595725,19.595638275146484,99.0,20.0,0.003276436120950546 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1363,,,,,, +0,,395377.0,,,,,,,,1363,8.101362228393555,1.9524625189916611,16.408016204833984,99.0,20.0,0.003276436120950546 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1364,,,,,, +0,,395377.0,,,,,,,,1364,8.728338241577148,0.1904596984386444,19.045970916748047,99.0,20.0,0.003276436120950546 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1365,,,,,, +0,,395377.0,,,,,,,,1365,8.13968563079834,4.255802344174009,16.563623428344727,99.0,20.0,0.002907350888824917 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1366,,,,,, +0,,395377.0,,,,,,,,1366,3.3276329040527344,2.223248171724974,2.7682852745056152,99.0,20.0,0.002907350888824917 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1367,,,,,, +0,,395377.0,,,,,,,,1367,7.036080360412598,2.3222212198342693,12.376606941223145,99.0,20.0,0.002907350888824917 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1368,,,,,, +0,,395377.0,,,,,,,,1368,3.253323554992676,0.026460284367203712,2.646028518676758,99.0,20.0,0.002907350888824917 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1369,,,,,, +0,,395377.0,,,,,,,,1369,9.408613204956055,4.089712977123006,22.1304988861084,99.0,20.0,0.0027145969387275142 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1370,,,,,, +0,,395377.0,,,,,,,,1370,6.215090274810791,2.0900352220290754,9.65683650970459,99.0,20.0,0.0027145969387275142 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1371,,,,,, +0,,395377.0,,,,,,,,1371,3.6273672580718994,0.03289448842406273,3.2894487380981445,99.0,20.0,0.0027145969387275142 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1372,,,,,, +0,,395377.0,,,,,,,,1372,9.83547592163086,3.755425724238843,24.184146881103516,99.0,20.0,0.0024698897079207943 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1373,,,,,, +0,,395377.0,,,,,,,,1373,7.679704189300537,3.444895072930255,14.744464874267578,99.0,20.0,0.0023269939862977955 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1374,,,,,, +0,,395377.0,,,,,,,,1374,7.901647090911865,0.15609006583690643,15.609006881713867,99.0,20.0,0.0023269939862977955 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1375,,,,,, +0,,395377.0,,,,,,,,1375,10.204398155212402,3.409870043435312,26.032434463500977,99.0,20.0,0.0021931319681143127 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1376,,,,,, +0,,395377.0,,,,,,,,1376,7.447317123413086,2.557080653990585,13.865633010864258,99.0,20.0,0.0021931319681143127 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1377,,,,,, +0,,395377.0,,,,,,,,1377,8.88300895690918,0.19726963341236115,19.72696304321289,99.0,20.0,0.0021931319681143127 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1378,,,,,, +0,,395377.0,,,,,,,,1378,3.597520351409912,0.032355379313230515,3.2355380058288574,99.0,20.0,0.0021931319681143127 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1379,,,,,, +0,,395377.0,,,,,,,,1379,6.652658462524414,2.019668138837974,11.064467430114746,99.0,20.0,0.0021931319681143127 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1380,,,,,, +0,,395377.0,,,,,,,,1380,9.691004753112793,3.904806006636237,23.47889518737793,99.0,20.0,0.0025138700067989647 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1381,,,,,, +0,,395377.0,,,,,,,,1381,7.752589225769043,0.15025658905506134,15.025659561157227,99.0,20.0,0.0025138700067989647 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1382,,,,,, +0,,395377.0,,,,,,,,1382,5.243874549865723,0.06874555349349976,6.874555587768555,99.0,20.0,0.0025138700067989647 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1383,,,,,, +0,,395377.0,,,,,,,,1383,7.540468215942383,2.0044633741007116,14.214664459228516,99.0,20.0,0.0025138700067989647 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1384,,,,,, +0,,395377.0,,,,,,,,1384,7.401467800140381,3.5162024516636596,13.695430755615234,99.0,20.0,0.0023519592191647586 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1385,,,,,, +0,,395377.0,,,,,,,,1385,6.3272199630737305,3.2765751378632912,10.008427619934082,99.0,20.0,0.002224410500556868 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1386,,,,,, +0,,395377.0,,,,,,,,1386,9.975082397460938,0.24875567853450775,24.875568389892578,99.0,20.0,0.002224410500556868 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1387,,,,,, +0,,395377.0,,,,,,,,1387,3.6344780921936035,3.0983111833074966,3.3023579120635986,99.0,20.0,0.0021039578892785658 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1388,,,,,, +0,,395377.0,,,,,,,,1388,4.273162841796875,2.082156257034422,4.564980506896973,99.0,20.0,0.0021039578892785658 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1389,,,,,, +0,,395377.0,,,,,,,,1389,5.4521684646606445,1.788636129399889,7.431535720825195,99.0,20.0,0.0021039578892785658 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1390,,,,,, +0,,395377.0,,,,,,,,1390,6.05247688293457,0.09158118814229965,9.158119201660156,99.0,20.0,0.0021039578892785658 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1391,,,,,, +0,,395377.0,,,,,,,,1391,10.812732696533203,3.654933879680421,29.228797912597656,99.0,20.0,0.0022914719190045216 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1392,,,,,, +0,,395377.0,,,,,,,,1392,5.702741622924805,3.271396867183381,8.130315780639648,99.0,20.0,0.0021822303927278423 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1393,,,,,, +0,,395377.0,,,,,,,,1393,7.306750297546387,3.065120675676433,13.347149848937988,99.0,20.0,0.0020220494294683326 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1394,,,,,, +0,,395377.0,,,,,,,,1394,4.253384113311768,2.8804081710194396,4.522819519042969,99.0,20.0,0.0019419853908007629 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1395,,,,,, +0,,395377.0,,,,,,,,1395,8.240671157836914,3.0270722153766125,16.97716522216797,99.0,20.0,0.0019779608113515857 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1396,,,,,, +0,,395377.0,,,,,,,,1396,4.731019973754883,0.05595637857913971,5.595637798309326,99.0,20.0,0.0019779608113515857 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1397,,,,,, +0,,395377.0,,,,,,,,1397,5.084650039672852,4.816864527927122,6.463417053222656,99.0,20.0,0.0019779608113515857 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1398,,,,,, +0,,395377.0,,,,,,,,1398,6.906347274780273,0.11924409866333008,11.924409866333008,99.0,20.0,0.0019779608113515857 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1399,,,,,, +0,,395377.0,,,,,,,,1399,4.926151275634766,3.7773131235080317,6.066741943359375,99.0,20.0,0.00245725200460386 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1400,,,,,, +0,,395377.0,,,,,,,,1400,6.24785852432251,0.09758933633565903,9.758934020996094,99.0,20.0,0.00245725200460386 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1401,,,,,, +0,,395377.0,,,,,,,,1401,4.188945770263672,3.412755161584899,4.386816501617432,99.0,20.0,0.0022705665320467403 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1402,,,,,, +0,,395377.0,,,,,,,,1402,7.902798652648926,2.984000562971816,15.613556861877441,99.0,20.0,0.0022705665320467403 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1403,,,,,, +0,,395377.0,,,,,,,,1403,9.792479515075684,0.23973162472248077,23.973163604736328,99.0,20.0,0.0022705665320467403 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1404,,,,,, +0,,395377.0,,,,,,,,1404,4.480233669281006,2.1064939402437552,5.018123149871826,99.0,20.0,0.0022705665320467403 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1405,,,,,, +0,,395377.0,,,,,,,,1405,6.536327362060547,0.10680893808603287,10.680893898010254,99.0,20.0,0.0022705665320467403 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1406,,,,,, +0,,395377.0,,,,,,,,1406,7.032850742340088,0.12365247309207916,12.36524772644043,99.0,20.0,0.0022705665320467403 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1407,,,,,, +0,,395377.0,,,,,,,,1407,6.970682621002197,2.3847834287609686,12.147603988647461,99.0,20.0,0.0022705665320467403 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1408,,,,,, +0,,395377.0,,,,,,,,1408,8.598934173583984,0.1848541796207428,18.48541831970215,99.0,20.0,0.0022705665320467403 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1409,,,,,, +0,,395377.0,,,,,,,,1409,8.518720626831055,5.0896342793198235,18.142148971557617,99.0,20.0,0.003031886215845936 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1410,,,,,, +0,,395377.0,,,,,,,,1410,7.5390944480896,2.3377417292264986,14.20948600769043,99.0,20.0,0.003031886215845936 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1411,,,,,, +0,,395377.0,,,,,,,,1411,9.307312965393066,0.21656519174575806,21.656518936157227,99.0,20.0,0.003031886215845936 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1412,,,,,, +0,,395377.0,,,,,,,,1412,9.891820907592773,3.936427749395931,24.46202850341797,99.0,20.0,0.002459024371477366 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1413,,,,,, +0,,395377.0,,,,,,,,1413,6.818098545074463,1.899364956678433,11.62161636352539,99.0,20.0,0.002459024371477366 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1414,,,,,, +0,,395377.0,,,,,,,,1414,5.573885917663574,3.4816400162234933,7.767050266265869,99.0,20.0,0.002308309960349633 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1415,,,,,, +0,,395377.0,,,,,,,,1415,9.64108943939209,1.889371492675003,23.237648010253906,99.0,20.0,0.002308309960349633 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1416,,,,,, +0,,395377.0,,,,,,,,1416,10.23881721496582,1.75611082731047,26.20834732055664,99.0,20.0,0.002308309960349633 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1417,,,,,, +0,,395377.0,,,,,,,,1417,6.200931549072266,1.5288503532801427,9.612887382507324,99.0,20.0,0.002308309960349633 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1418,,,,,, +0,,395377.0,,,,,,,,1418,9.931567192077637,1.6915467875596768,24.659006118774414,99.0,20.0,0.002308309960349633 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1419,,,,,, +0,,395377.0,,,,,,,,1419,8.404570579528809,0.17659202218055725,17.659202575683594,99.0,20.0,0.002308309960349633 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1420,,,,,, +0,,395377.0,,,,,,,,1420,7.920625686645508,4.7701572951982,15.684078216552734,99.0,20.0,0.0028786532034195456 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1421,,,,,, +0,,395377.0,,,,,,,,1421,9.658150672912598,4.389691479510555,23.319969177246094,99.0,20.0,0.00266809353024922 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1422,,,,,, +0,,395377.0,,,,,,,,1422,5.61738395690918,2.080755951409536,7.888750076293945,99.0,20.0,0.00266809353024922 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1423,,,,,, +0,,395377.0,,,,,,,,1423,5.939775466918945,3.4569131601106706,8.820233345031738,99.0,20.0,0.0022448119793737186 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1424,,,,,, +0,,395377.0,,,,,,,,1424,6.241877555847168,3.4600758036564354,9.740259170532227,99.0,20.0,0.002223761328707851 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1425,,,,,, +0,,395377.0,,,,,,,,1425,5.2502288818359375,3.2513648094002385,6.891225814819336,99.0,20.0,0.002117237499861203 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1426,,,,,, +0,,395377.0,,,,,,,,1426,6.7466559410095215,0.11379341036081314,11.379341125488281,99.0,20.0,0.002117237499861203 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1427,,,,,, +0,,395377.0,,,,,,,,1427,5.309537887573242,3.4520978600814605,7.047798156738281,99.0,20.0,0.0021945636224933313 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1428,,,,,, +0,,395377.0,,,,,,,,1428,4.854349136352539,1.9155187322543878,5.891176223754883,99.0,20.0,0.0021945636224933313 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1429,,,,,, +0,,395377.0,,,,,,,,1429,5.200904369354248,3.456960672314222,6.762351036071777,99.0,20.0,0.002178750757687458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1430,,,,,, +0,,395377.0,,,,,,,,1430,9.8624267578125,0.2431686520576477,24.316865921020508,99.0,20.0,0.002178750757687458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1431,,,,,, +0,,395377.0,,,,,,,,1431,7.975826263427734,1.7526065543534908,15.903450965881348,99.0,20.0,0.002178750757687458 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1432,,,,,, +0,,395377.0,,,,,,,,1432,8.095705032348633,3.428545081792783,16.385108947753906,99.0,20.0,0.0021389866360976583 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1433,,,,,, +0,,395377.0,,,,,,,,1433,7.72434663772583,1.582474206304555,14.9163818359375,99.0,20.0,0.0021389866360976583 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1434,,,,,, +0,,395377.0,,,,,,,,1434,9.631332397460938,0.23190638422966003,23.19063949584961,99.0,20.0,0.0021389866360976583 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1435,,,,,, +0,,395377.0,,,,,,,,1435,6.930038928985596,3.2686406446258047,12.006359100341797,99.0,20.0,0.00209333538560027 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1436,,,,,, +0,,395377.0,,,,,,,,1436,6.32115364074707,3.036386594002673,9.989246368408203,99.0,20.0,0.0019789039586026206 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1437,,,,,, +0,,395377.0,,,,,,,,1437,4.708419322967529,0.055423036217689514,5.542303562164307,99.0,20.0,0.0019789039586026206 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1438,,,,,, +0,,395377.0,,,,,,,,1438,8.889695167541504,3.223451086450787,19.756671905517578,99.0,20.0,0.002036128827579892 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1439,,,,,, +0,,395377.0,,,,,,,,1439,3.3084676265716553,2.9617266991007023,2.736489772796631,99.0,20.0,0.001963132534759468 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1440,,,,,, +0,,395377.0,,,,,,,,1440,6.6096320152282715,2.4252558106731374,10.921808242797852,99.0,20.0,0.001963132534759468 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1441,,,,,, +0,,395377.0,,,,,,,,1441,9.585501670837402,0.2297045886516571,22.970458984375,99.0,20.0,0.001963132534759468 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1442,,,,,, +0,,395377.0,,,,,,,,1442,5.116703987121582,3.3693121431761357,6.545165061950684,99.0,20.0,0.0022131995217482756 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1443,,,,,, +0,,395377.0,,,,,,,,1443,6.41908073425293,3.2956696231956,10.301149368286133,99.0,20.0,0.0021585676630779356 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1444,,,,,, +0,,395377.0,,,,,,,,1444,9.577744483947754,2.1687382693339816,22.93329620361328,99.0,20.0,0.0021585676630779356 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1445,,,,,, +0,,395377.0,,,,,,,,1445,8.484529495239258,3.1798665593093376,17.996810913085938,99.0,20.0,0.0020215062021357363 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1446,,,,,, +0,,395377.0,,,,,,,,1446,5.264101982116699,0.06927692145109177,6.92769193649292,99.0,20.0,0.0020215062021357363 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1447,,,,,, +0,,395377.0,,,,,,,,1447,8.827790260314941,1.7628537380891103,19.48246955871582,99.0,20.0,0.0020215062021357363 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1448,,,,,, +0,,395377.0,,,,,,,,1448,11.037151336669922,1.7109295864232938,30.45467758178711,99.0,20.0,0.0020215062021357363 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1449,,,,,, +0,,395377.0,,,,,,,,1449,7.249740123748779,3.85136962856288,13.13968276977539,99.0,20.0,0.0025219478814825177 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1450,,,,,, +0,,395377.0,,,,,,,,1450,7.6988091468811035,3.398942865686207,14.817914962768555,99.0,20.0,0.0022121474558698916 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1451,,,,,, +0,,395377.0,,,,,,,,1451,7.879860877990723,1.984834847372916,15.523052215576172,99.0,20.0,0.0022121474558698916 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1452,,,,,, +0,,395377.0,,,,,,,,1452,7.665485858917236,0.14689919352531433,14.689919471740723,99.0,20.0,0.0022121474558698916 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1453,,,,,, +0,,395377.0,,,,,,,,1453,6.874858856201172,4.385580730579267,11.815919876098633,99.0,20.0,0.0028897906413076465 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1454,,,,,, +0,,395377.0,,,,,,,,1454,5.1501030921936035,0.0663089007139206,6.630889892578125,99.0,20.0,0.0028897906413076465 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1455,,,,,, +0,,395377.0,,,,,,,,1455,10.009805679321289,1.9514443422923107,25.049053192138672,99.0,20.0,0.0028897906413076465 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1456,,,,,, +0,,395377.0,,,,,,,,1456,7.733042240142822,1.677691971306012,14.94998550415039,99.0,20.0,0.0028897906413076465 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1457,,,,,, +0,,395377.0,,,,,,,,1457,8.62074089050293,0.1857929229736328,18.57929229736328,99.0,20.0,0.0028897906413076465 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1458,,,,,, +0,,395377.0,,,,,,,,1458,8.304742813110352,4.228009963152516,17.2421875,99.0,20.0,0.002724809507019583 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1459,,,,,, +0,,395377.0,,,,,,,,1459,3.1928083896636963,0.025485064834356308,2.548506498336792,99.0,20.0,0.002724809507019583 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1460,,,,,, +0,,395377.0,,,,,,,,1460,10.365307807922363,3.5724601657599355,26.859899520874023,99.0,20.0,0.002267972902480283 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1461,,,,,, +0,,395377.0,,,,,,,,1461,7.308567523956299,3.2030752077524176,13.353789329528809,99.0,20.0,0.002125333449719576 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1462,,,,,, +0,,395377.0,,,,,,,,1462,7.979933738708496,0.1591983586549759,15.919836044311523,99.0,20.0,0.002125333449719576 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1463,,,,,, +0,,395377.0,,,,,,,,1463,8.006609916687012,3.5433859415355773,16.02644920349121,99.0,20.0,0.0022967262725899946 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1464,,,,,, +0,,395377.0,,,,,,,,1464,7.894370079040527,3.394551623274426,15.580268859863281,99.0,20.0,0.0022967262725899946 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1465,,,,,, +0,,395377.0,,,,,,,,1465,7.317740440368652,3.4426334914481487,13.387331008911133,99.0,20.0,0.0022410433200632743 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1466,,,,,, +0,,395377.0,,,,,,,,1466,8.468149185180664,0.17927390336990356,17.927391052246094,99.0,20.0,0.0022410433200632743 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1467,,,,,, +0,,395377.0,,,,,,,,1467,10.501425743103027,0.27569982409477234,27.569984436035156,99.0,20.0,0.0022410433200632743 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1468,,,,,, +0,,395377.0,,,,,,,,1468,4.492881774902344,3.2375042616128447,5.046496868133545,99.0,20.0,0.0021704644741124077 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1469,,,,,, +0,,395377.0,,,,,,,,1469,5.756043910980225,3.1148346154544564,8.283010482788086,99.0,20.0,0.002076496208380271 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1470,,,,,, +0,,395377.0,,,,,,,,1470,10.135359764099121,0.25681379437446594,25.681381225585938,99.0,20.0,0.002076496208380271 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1471,,,,,, +0,,395377.0,,,,,,,,1471,7.252829074859619,0.13150882720947266,13.150882720947266,99.0,20.0,0.002076496208380271 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1472,,,,,, +0,,395377.0,,,,,,,,1472,6.695786952972412,3.348794629712878,11.208391189575195,99.0,20.0,0.002153639825815431 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1473,,,,,, +0,,395377.0,,,,,,,,1473,6.830346584320068,2.7079118804917583,11.663407325744629,99.0,20.0,0.002153639825815431 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1474,,,,,, +0,,395377.0,,,,,,,,1474,3.8094584941864014,0.036279939115047455,3.6279940605163574,99.0,20.0,0.002153639825815431 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1475,,,,,, +0,,395377.0,,,,,,,,1475,6.822644233703613,0.11637119203805923,11.63711929321289,99.0,20.0,0.002153639825815431 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1476,,,,,, +0,,395377.0,,,,,,,,1476,6.127584934234619,0.09386824071407318,9.386824607849121,99.0,20.0,0.002153639825815431 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1477,,,,,, +0,,395377.0,,,,,,,,1477,6.139942646026611,1.8903802780551053,9.424724578857422,99.0,20.0,0.002153639825815431 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1478,,,,,, +0,,395377.0,,,,,,,,1478,10.545598030090332,1.9301795333532579,27.802412033081055,99.0,20.0,0.002153639825815431 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1479,,,,,, +0,,395377.0,,,,,,,,1479,8.844314575195312,0.19555476307868958,19.555477142333984,99.0,20.0,0.002153639825815431 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1480,,,,,, +0,,395377.0,,,,,,,,1480,9.396111488342285,0.22071725130081177,22.071725845336914,99.0,20.0,0.002153639825815431 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1481,,,,,, +0,,395377.0,,,,,,,,1481,6.884027481079102,0.1184745728969574,11.847457885742188,99.0,20.0,0.002153639825815431 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1482,,,,,, +0,,395377.0,,,,,,,,1482,8.143258094787598,2.7375047285888985,16.578163146972656,99.0,20.0,0.002153639825815431 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1483,,,,,, +0,,395377.0,,,,,,,,1483,4.546665668487549,4.486953840445405,5.1680426597595215,99.0,20.0,0.002944694877865749 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1484,,,,,, +0,,395377.0,,,,,,,,1484,9.043562889099121,0.20446506142616272,20.44650650024414,99.0,20.0,0.002944694877865749 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1485,,,,,, +0,,395377.0,,,,,,,,1485,3.294192314147949,0.02712925896048546,2.712925910949707,99.0,20.0,0.002944694877865749 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1486,,,,,, +0,,395377.0,,,,,,,,1486,8.167858123779297,3.7573556766245626,16.678478240966797,99.0,20.0,0.0024945736125274796 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1487,,,,,, +0,,395377.0,,,,,,,,1487,8.279038429260254,3.490627367544871,17.135618209838867,99.0,20.0,0.0023184945210844464 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1488,,,,,, +0,,395377.0,,,,,,,,1488,5.093579292297363,3.112710855389673,6.4861369132995605,99.0,20.0,0.002135024183278472 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1489,,,,,, +0,,395377.0,,,,,,,,1489,8.912799835205078,0.19859500229358673,19.859500885009766,99.0,20.0,0.002135024183278472 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1490,,,,,, +0,,395377.0,,,,,,,,1490,7.232842445373535,0.1307850182056427,13.078502655029297,99.0,20.0,0.002135024183278472 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1491,,,,,, +0,,395377.0,,,,,,,,1491,6.391499042510986,3.6251423669557514,10.212814331054688,99.0,20.0,0.0023264429206866434 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1492,,,,,, +0,,395377.0,,,,,,,,1492,6.371307373046875,0.10148388892412186,10.148388862609863,99.0,20.0,0.0023264429206866434 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1493,,,,,, +0,,395377.0,,,,,,,,1493,9.389887809753418,2.8007385305852863,22.04250144958496,99.0,20.0,0.0023264429206866434 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1494,,,,,, +0,,395377.0,,,,,,,,1494,8.068795204162598,0.16276362538337708,16.276363372802734,99.0,20.0,0.0023264429206866434 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1495,,,,,, +0,,395377.0,,,,,,,,1495,8.053143501281738,3.8408409252850624,16.213279724121094,99.0,20.0,0.002475449482358676 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1496,,,,,, +0,,395377.0,,,,,,,,1496,7.963698387145996,0.15855123102664948,15.855123519897461,99.0,20.0,0.002475449482358676 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1497,,,,,, +0,,395377.0,,,,,,,,1497,6.6448493003845215,0.11038505285978317,11.038505554199219,99.0,20.0,0.002475449482358676 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1498,,,,,, +0,,395377.0,,,,,,,,1498,9.475667953491211,2.0802169167201234,22.447071075439453,99.0,20.0,0.002475449482358676 +,,,0.01,0.0005,0.0003,0.05,0.005,0.05,4.3734482957731104e-05,1499,,,,,, +0,,395377.0,,,,,,,,1499,8.84677505493164,3.5901719567919423,19.566356658935547,99.0,20.0,0.002324174586337251 +0,369498.71875,,,,,,,,,1499,,,,,, diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/mask_0.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/mask_0.png new file mode 100644 index 0000000000000000000000000000000000000000..6b55de44332f3ce858ae242d596b673ad3ca7c04 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/mask_0.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/mask_1.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/mask_1.png new file mode 100644 index 0000000000000000000000000000000000000000..611877004f1dc424c15c2e89517cc9856d526967 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/mask_1.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/outpaint_0.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/outpaint_0.png new file mode 100644 index 0000000000000000000000000000000000000000..9d2fe6df581b24aa857dcee82a1ad77280c0d717 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/outpaint_0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca5f6a7fa799bee6afdf8ed157f99b2d255afb7f91d9006072d6372041ab566d +size 339409 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/outpaint_1.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/outpaint_1.png new file mode 100644 index 0000000000000000000000000000000000000000..304623263ded4007a5d1ffa936d676bbc4991ce4 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/outpaint_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4d4d4b532ae4d4898d2404b3c921bbd71f96318eef7e2a42f6968df858c0957 +size 359948 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it0-val.mp4 b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it0-val.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4ceb96d166b156b36e9fe12f2271d3946623b980 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it0-val.mp4 differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1000-val.mp4 b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1000-val.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..981e2f536b511cc087b317be2132c53bc4bccfd3 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1000-val.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfdec36adaf80878fa939502c1b6c33ad15efa3cf96ce2391c71385864d6ef34 +size 212880 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test.mp4 b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..524fef1aa562ec51e485a670fc2962c3d7b47acb --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40f323ee52781294b02f67acb3f8cd00bb040d82b3afb7d6c9ecd21a7702c579 +size 445311 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/0.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/0.png new file mode 100644 index 0000000000000000000000000000000000000000..b35437eac7debd25d05951aff0158e6df26b0372 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2282a1482bea612ab7cc87598e342a4a193c7c146b502f7b6c2684cd55918422 +size 751094 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/1.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/1.png new file mode 100644 index 0000000000000000000000000000000000000000..80861759fd59c0e2fbf24f1f66d6be61f072c137 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:511067d48e9b81bbdb7dfd23317c0fadf81033de773fe64af78dafbeb8cb3374 +size 750436 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/10.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/10.png new file mode 100644 index 0000000000000000000000000000000000000000..2baca96970bd044300215dbc7bafaabe8b057dee --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/10.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76f60709e8407e6418847c312f0ab17dbbc529c712112702cb9f7c442be7af00 +size 759287 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/100.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/100.png new file mode 100644 index 0000000000000000000000000000000000000000..3a06e76d160ce0af4944509e0020ed8e44534d05 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/100.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1762e71dd2838313fd3a49895dfb3876d8028ca668a9673909f4ded170826d0 +size 817642 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/101.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/101.png new file mode 100644 index 0000000000000000000000000000000000000000..d04d65f9be92e5d068e42ce35a998de33b5ef3a6 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/101.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81dc7233d4a25605a01be3d9ea793f0d6a412d39c0909e90ddd3bf9c41b11c2c +size 819537 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/102.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/102.png new file mode 100644 index 0000000000000000000000000000000000000000..58ab318a26d2921ce4b47041c2e5390a74bdb71b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/102.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29ccbeb1cf1516bd49c1f139314e0024be105c20f84ae3df44301e3fd1fca064 +size 817320 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/103.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/103.png new file mode 100644 index 0000000000000000000000000000000000000000..8cf6ccf9e130bd335638dce4a6f5e64650258493 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/103.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:999cb90ff577254cd6c2780a2d4c8197ebcf14a5ea4eca9aba4430bd014e8db9 +size 816642 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/104.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/104.png new file mode 100644 index 0000000000000000000000000000000000000000..84492f9d70f226484b005b1f3d0c57f29f3bd2b1 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/104.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:571a4d1019279f5e20df50535d99248ad450de01229af3220f06934f40b741c7 +size 818074 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/105.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/105.png new file mode 100644 index 0000000000000000000000000000000000000000..8a93be43c6f758793cc3051db26f3c7de4a88b08 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/105.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3fb3ca6cea74f8dbcb39c2cc7ec81e2ace88a497bc6a945a86232af9ea883e9 +size 818105 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/106.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/106.png new file mode 100644 index 0000000000000000000000000000000000000000..c6fdf204520c6213b94291798d158c8d49e89926 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/106.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:203198ec6a13fff1b18220834c6c8b40d731f26d228efcad4664f3b725b1ded0 +size 819586 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/107.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/107.png new file mode 100644 index 0000000000000000000000000000000000000000..941ff802354a686933473ba81821958498a4b556 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/107.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6de28c638eef01c804f0f090dac4eb9509a500c826fedc6a0c53408de9402996 +size 820638 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/108.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/108.png new file mode 100644 index 0000000000000000000000000000000000000000..d81ceb8b2a833202ccc6d128b02d189f6fe4aaf8 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/108.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a598fb6fb9a7c2b685490f2f160142aa11ecccd5709d7c2c75d2d5973475633 +size 819634 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/109.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/109.png new file mode 100644 index 0000000000000000000000000000000000000000..a04fa2ef67a7358754c079d9c39349cbf1ae343c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/109.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e08cc1b370b8af5abb3fb28546352d84c7f22c1c631454561dd7e5f47f96fd88 +size 820937 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/11.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/11.png new file mode 100644 index 0000000000000000000000000000000000000000..daa832bb2824ea5d597dd803c55c7fcddd568bcc --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/11.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:650c3946a3b4cee4aad4f6c19148470ed71e825af94e7769fb1f6bd63b49f3c5 +size 757901 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/110.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/110.png new file mode 100644 index 0000000000000000000000000000000000000000..57445dd076fc1289c249ccbf52a39df92bac6bf9 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/110.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0329672f835037947c0bd1d61020c02324510b89de6e0628ed5b63d498e15a1 +size 816751 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/111.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/111.png new file mode 100644 index 0000000000000000000000000000000000000000..0e2254210d0bc331fa00a88cc7d6343f9e4c46ee --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/111.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ff63fa3d09760dde0128dcb140617fd2ee86eb9712645132c95a29f161078fb +size 821131 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/112.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/112.png new file mode 100644 index 0000000000000000000000000000000000000000..220aecad69fccb0ae003e34cb23e8d7116cdf5dc --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/112.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d218fa2d73a52ea3abbbab77413b75638ba3803820459239ea584af670e57a09 +size 818045 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/113.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/113.png new file mode 100644 index 0000000000000000000000000000000000000000..cea37f9ba8419716ffdc58008eeb9a14ec30514a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/113.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d42bcaa8e4b9dc0797ea5aed256c8bd05e8bf9334eb44f91416bc860af4a1e7 +size 818594 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/114.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/114.png new file mode 100644 index 0000000000000000000000000000000000000000..811b82ad5e27b40fdfe9e93a9112136e14186ad1 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/114.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e619a6346108b4b898b2e873654f0378811b7fae9b37e37c839b5a20af22fc76 +size 820125 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/115.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/115.png new file mode 100644 index 0000000000000000000000000000000000000000..9098d1b2992c88295af9c8fe9d8da5d0e77e6727 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/115.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:751bcb2515e333a7d905a8371292da126252846c23aec28ea70574aee2b4b58f +size 820913 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/116.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/116.png new file mode 100644 index 0000000000000000000000000000000000000000..06ae4019b3d2aa7259cbeaee92f4108cb3a7d285 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/116.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d61ba5eaa4fa65eaf999b330ae198ac797fad0f56d6b9cf582b7c5ed6e49aa52 +size 820668 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/117.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/117.png new file mode 100644 index 0000000000000000000000000000000000000000..cecaf017260b759f5c156160a34c488a4273ad7c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/117.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30b21cf86f39830cea5a6a83f7e9f02002786fa24a85eb0b1d5d1cba752ebf20 +size 816085 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/118.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/118.png new file mode 100644 index 0000000000000000000000000000000000000000..08f6538e097d636580de4972bfd9180b630e26af --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/118.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0df7b8391db745b5b21ca05694d035fd99707cbaf1889d2cbb5ea34f8c6db1ad +size 820506 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/119.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/119.png new file mode 100644 index 0000000000000000000000000000000000000000..92835494affcc1d563c82445440556fbf88c5aaf --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/119.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b1e03ec57c69fb94d3760373cfe229d5e02a010b061d58e59583127dfa1961f +size 824032 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/12.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/12.png new file mode 100644 index 0000000000000000000000000000000000000000..c1fb77b380329b369f116a704ce52ebf839e26ca --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/12.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17a0d89b588134c6ccffc44794ef4ea7446b2e3f74069094624913def1db3f8a +size 758569 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/13.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/13.png new file mode 100644 index 0000000000000000000000000000000000000000..29c8365e3786576dc38a985c97f595aee903162e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/13.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:823e17b6c64f9cfbeeaa5a871027e069a792dde686dda0f775e496de477ed42e +size 759831 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/14.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/14.png new file mode 100644 index 0000000000000000000000000000000000000000..9f2a71c649b8bd1b2e93a7b9a3bcfeb71b3f4f61 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/14.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e04da3ddd802cd536d3bab82c00f9a3eeef55378f289817b54e4e84e1cd7aefd +size 763162 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/15.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/15.png new file mode 100644 index 0000000000000000000000000000000000000000..23f1dd6e0a04e0bed0747ad2b45b033119d25e26 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/15.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c2a6873f0462fc9c428c38f33cbf18f2f67246494e35e49b0a46e547b3dbeef +size 762596 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/16.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/16.png new file mode 100644 index 0000000000000000000000000000000000000000..65e192ed8d388b25b03afadb16f59ede249f3ba0 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/16.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:392ab0ebecc1dea51cc7fa6a111e18d4de498f2e150fb12c5e8304a8cbcd67c6 +size 763427 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/17.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/17.png new file mode 100644 index 0000000000000000000000000000000000000000..bccd2a3f4eab66a034f1b4a8c7345417ac7f099c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/17.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1ebd36eeddf9e1a979496d5c0886d1b7b62fb7a7d90fe97afe041b403ebc1ff +size 766044 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/18.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/18.png new file mode 100644 index 0000000000000000000000000000000000000000..59b11bf8a598f709da36cd6bdbfa395c226a2c40 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/18.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1207e00787d1b40c0066128949c459ca4a3b198a3ad8d8e24514b739ef2afde +size 768479 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/19.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/19.png new file mode 100644 index 0000000000000000000000000000000000000000..cd5ecca9f98d0f7da5070f30bdab0b16e04075f1 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/19.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a62114571c493d029467a2a3d4b70f43688129074334e1d5a3f6c75dc6dc61e +size 769566 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/2.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/2.png new file mode 100644 index 0000000000000000000000000000000000000000..3d121c01c8309539c03b98d494c253e19981725c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deb560ea6cd3db05086049e253ee542e2f585d260f570dd9ab2ce201c1cae0a9 +size 751810 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/20.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/20.png new file mode 100644 index 0000000000000000000000000000000000000000..0be741d98578963f6ffd9499ead84ae2254f66d7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/20.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b01689b78695e11e2e24592eedc9e6ee84b058e1746dc94cfdf3d7a32373a44 +size 768012 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/21.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/21.png new file mode 100644 index 0000000000000000000000000000000000000000..3398c340f280e54fae0a7356d73ce09c26e7c06e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/21.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4059f4371b443f6722ffcddf74a2d094a6e26552d987eec1746870d450ba3516 +size 768651 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/22.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/22.png new file mode 100644 index 0000000000000000000000000000000000000000..4ecdcdf2f25829c8ceb0239078d38c4239ecbf99 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/22.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff38824c409b141c88cb59a02488cb6021c2b3aab70f569e529799efe9705108 +size 769224 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/23.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/23.png new file mode 100644 index 0000000000000000000000000000000000000000..15cb97e1145110eceacf6cefbbc1d74eb3324da7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/23.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bec91542e298393df7a50264ca88a1d02cc6cdca53dfabc5cc978384c015adc +size 769321 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/24.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/24.png new file mode 100644 index 0000000000000000000000000000000000000000..559545c5271ffcf1ccf34d679e00823eb550763b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/24.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4262b7a587a2679ef0cc19f31f7370ebfc79e19cc6f099fac989095675c8d81a +size 769084 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/25.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/25.png new file mode 100644 index 0000000000000000000000000000000000000000..2881615aea44a86a0d44b6b78f38e1c4a2fa05d0 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/25.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a95ba146135d9eb14588790ebe9da7fb70e1dad738925177c3ce42b298fe6273 +size 769862 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/26.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/26.png new file mode 100644 index 0000000000000000000000000000000000000000..f371148375549eec844114daf3bfd027a4326e66 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/26.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1859834ed5b55328267a1140204a09a70da8e9f4113c05e90b0b4f240d34fb7d +size 769138 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/27.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/27.png new file mode 100644 index 0000000000000000000000000000000000000000..681f5d8dbe095b103b21a06b5d9c64016edbc5c0 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/27.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dfacb7f89c222bef4de21cdc07b180cd28f6cd207da807bf6579b0ac69a5f41 +size 772971 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/28.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/28.png new file mode 100644 index 0000000000000000000000000000000000000000..565aba9be7900b99788c75b8d58d54b8c438220b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/28.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2986b6bb22d0b6a5be86d3188f80414af56cd6ed849530592d38693ffa06fae1 +size 775186 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/29.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/29.png new file mode 100644 index 0000000000000000000000000000000000000000..988fae65c036ffa30f061114577c10bbbdaf375c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/29.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c4a848081c33ba655b8a67577bf0baad09249f1b30ca6b374b697ddbd3e6a4 +size 776649 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/3.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/3.png new file mode 100644 index 0000000000000000000000000000000000000000..6422b76bec7fd19e30021b35ebd9aa4bcbacdb4c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbe25242c9ee76fe1c702319997e53e2662a008715a9bd716a55074d180a78dc +size 752525 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/30.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/30.png new file mode 100644 index 0000000000000000000000000000000000000000..a5c54a43cfd3d902e2b8274a4e183da2948c8b89 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/30.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ceedf56062e5aa3d1e1dc44c1bd07e636d516560b7a1d514ac4d51170f86f47d +size 774260 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/31.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/31.png new file mode 100644 index 0000000000000000000000000000000000000000..15f692016cbfe78261d5dedb3e52322f287ae2e3 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/31.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1af120030110699fcbda56944cabcf767785ee4e3ae6b9623f7032344a3d47e6 +size 774712 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/32.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/32.png new file mode 100644 index 0000000000000000000000000000000000000000..69577b981bdc26c6dc62d1cc4195051b4359c308 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/32.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:836fbff90bb15ad78c15dc90123220ec5818ef63a9d3cbbfdfb9a6e621d3d850 +size 778079 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/33.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/33.png new file mode 100644 index 0000000000000000000000000000000000000000..005ae41793a1be95a1d5f6de213e62734f39dd6f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/33.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8155cbc801145750aeae76cb56632ab3da7ba3b75241c695492992fecb098c6f +size 778715 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/34.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/34.png new file mode 100644 index 0000000000000000000000000000000000000000..8b652e33e0f2d7eaa1880261db96c983cf0c9cbb --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/34.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75f7f2573ecd9eefdfd1d22da70b3c0a28a101cc23e8f086e09f7277b603fc08 +size 783959 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/35.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/35.png new file mode 100644 index 0000000000000000000000000000000000000000..01f24383d2c366a0c32a5a6fedd88859a93c9438 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/35.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d54c1ce22f99e87cbcc660789331562439805899f535b45596cc908f91454486 +size 783920 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/36.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/36.png new file mode 100644 index 0000000000000000000000000000000000000000..1eb448ffad98f674c1f21bf7fddae5c776b14aaf --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/36.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f568eae3eb1bdaf1deb76d51f82c8213efac6e385ac7ad794baafcb45b0c3ab +size 783451 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/37.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/37.png new file mode 100644 index 0000000000000000000000000000000000000000..45fd5c1e2462a474d76c6e476921b4ac6d121062 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/37.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3ff8e3c08048cc46be4245321b0b342d20935d27e1e3c42818387e2ff5ef000 +size 786511 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/38.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/38.png new file mode 100644 index 0000000000000000000000000000000000000000..05763ae30a5b5dfe40622ff5193e7214da286e27 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/38.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af651c5138f84fb5499515fd074a79c6e2ae5c7d9c97cd7c11b13fa8193e743d +size 786461 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/39.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/39.png new file mode 100644 index 0000000000000000000000000000000000000000..29eaf0062b671911fc0cdcaea8af650e5f6bf65f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/39.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b054f56e7311a618b23672f06b1f25dd3cd4ff78cb8957b0e489a9cc7d8144bd +size 789403 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/4.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/4.png new file mode 100644 index 0000000000000000000000000000000000000000..9f38876d414f32ce790ec75123a6dec715b7366b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7266814c3b10147138705d0d417fe3f7771b9b2df6fa86c7e0643b5401b6ee5 +size 753209 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/40.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/40.png new file mode 100644 index 0000000000000000000000000000000000000000..db3cf20da4494768d6cd2373abd1ad3ee15f0ae4 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/40.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e1e5501b74cfafa0b6ef59f3a0946417f263299768ff6e0e8e58187825e28e7 +size 792945 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/41.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/41.png new file mode 100644 index 0000000000000000000000000000000000000000..4729f673427e3517c77c14c4df69f003c7200465 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/41.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e75093d8d27a7318b161816b3a2999978bfbe78d3b76bf80b121465d2a8e4d6b +size 793166 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/42.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/42.png new file mode 100644 index 0000000000000000000000000000000000000000..20b9f52f8ce5bcdc29fc0122ed03f33fb449057a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/42.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:804965f0fae9644c1b638f3ca652efb02b9495cb7f4c1e7951136c0231cfeeee +size 793692 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/43.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/43.png new file mode 100644 index 0000000000000000000000000000000000000000..4d6906aa72ef27c0a1f5703d4be551bb115a3680 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/43.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74e7c6bcfe003e28683a1de18a88af3e5da11398a4cc4f9bfab9e71110bd4b8c +size 797198 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/44.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/44.png new file mode 100644 index 0000000000000000000000000000000000000000..cccd675c796dcf8068d06fa8c2def269094bc590 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/44.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e14f1a8e5803f2182f84eeb885c6cda50eb9f3d6e731c900e94a0ff3c22f0e6 +size 795486 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/45.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/45.png new file mode 100644 index 0000000000000000000000000000000000000000..e0ce501887308b9c4564984b04496203e5610b46 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/45.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0f751b98f489dd5b7bfcc4017cd4d2e262ca70c89f218ccef14e3e5249ad858 +size 797111 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/46.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/46.png new file mode 100644 index 0000000000000000000000000000000000000000..c61c6e4df4319533a541f48aef91c8db859eccdf --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/46.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39b64a21a63286eab8dd88e82f88227a42257c95de8977d96c7c8c01d33152ec +size 801442 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/47.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/47.png new file mode 100644 index 0000000000000000000000000000000000000000..593dcce7f2ac19c4bed1e9856b3e3491f39b12c5 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/47.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f8f38c0764d9c2f54fd7e350fffff10577a0ff350ed7c69e0f4588c94f3fff6 +size 804216 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/48.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/48.png new file mode 100644 index 0000000000000000000000000000000000000000..c4bc07264027db93709d53d5b03eabacf52dee62 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/48.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42e6656e3986ab6eb2ec35c02679cb2963ecd7312622ff1a088f5216df1116e6 +size 805130 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/49.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/49.png new file mode 100644 index 0000000000000000000000000000000000000000..5b9fc79db3acd20c4847511390350e645366ade9 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/49.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56fde5062fd039ccdb980bc2f18bc45bac9e66f06a0e36bc29abe3bf7ca88ade +size 803499 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/5.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/5.png new file mode 100644 index 0000000000000000000000000000000000000000..31ffb80a95aab7ab19777ef24598f6ab799ed389 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/5.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:574dc1f6ab989629bd3e61cb212aa014be86422947c5806a8cfb709d9ed3232c +size 753670 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/50.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/50.png new file mode 100644 index 0000000000000000000000000000000000000000..fc396857d5f2c05ea2ee79ba007aac2d5155c5ff --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/50.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fe5ff69ec326672232986d3328833572b003a574c803a41a5679aedadf95ca6 +size 808066 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/51.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/51.png new file mode 100644 index 0000000000000000000000000000000000000000..a03cc0b637dc47b80e320eeefda25bc281a061f9 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/51.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b2731f82fc3e72cf0774347b17984f8abaa956bc94071cb5b3e45e0d72ec2e5 +size 808468 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/52.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/52.png new file mode 100644 index 0000000000000000000000000000000000000000..50f11df0cf55a880f71c1ee924c779e10e06747f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/52.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbe179b90697fba020f18bad45f0b401abd1fac76da5ab174ab8d9427a7ac287 +size 808916 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/53.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/53.png new file mode 100644 index 0000000000000000000000000000000000000000..2b9362871248e7ac0b8ddbe1fa111484f2d46533 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/53.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1f87061655ece1d0be9d46370ace7e1e2eb0d5943e8216b2a117ca484eaec54 +size 808000 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/54.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/54.png new file mode 100644 index 0000000000000000000000000000000000000000..5f470ad03552886b24691d7b880ad9b4530a03ba --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/54.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc8aab613d550ed8297125e1e05d9defdbc977b11b00c6cd33015abad32e3dbe +size 811023 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/55.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/55.png new file mode 100644 index 0000000000000000000000000000000000000000..f546bf6397f504c7a81efe577eda14020814e7e4 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/55.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7aaa9c58f17a198675c84d4729faf72115b1bee581795d0a9cd994de7d31e38 +size 811579 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/56.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/56.png new file mode 100644 index 0000000000000000000000000000000000000000..166000c5abe3f7c291e2b2f514bc284981287b56 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/56.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1bc323f6c6392bb87d9b8c9e259d2d6af9dc99e3a03e263dc8abefc06f22103 +size 809309 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/57.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/57.png new file mode 100644 index 0000000000000000000000000000000000000000..6449ab304b7f02b58c177d2557ca2d1b20aafa53 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/57.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d7ed10de205c1f6e78e400879ab502f8fddf50579d5804122957e1697a03588 +size 811012 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/58.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/58.png new file mode 100644 index 0000000000000000000000000000000000000000..6e557fcf8d6ac9803c498c548c91d47981537c76 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/58.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f0526940b8e305beeb73f3ba6f80e3190a4e97f05fa62bffb527609dffbfa49 +size 814477 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/59.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/59.png new file mode 100644 index 0000000000000000000000000000000000000000..2859f5c9e11e56913338454e36b00a12db869c01 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/59.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a29f570fc66c74401d2ce25e3d9a65a212823180af9f68091352c94bf923c8ba +size 812308 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/6.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/6.png new file mode 100644 index 0000000000000000000000000000000000000000..5101a86a1487f03e7924f063a999cb500006733a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/6.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:525d12349f53f37559ff7481a82534e959169bd1a15a43ff32e25cc6b98b5029 +size 755440 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/60.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/60.png new file mode 100644 index 0000000000000000000000000000000000000000..99d70e33727a2f1d1f6dc222aff4c1aff3760a22 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/60.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a1aba984be66d4eb84816975e56f670126c633a5a102539837d1b193c3d8c8f +size 812657 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/61.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/61.png new file mode 100644 index 0000000000000000000000000000000000000000..fede8275c1a2e66f9951eb0bde435fa566efa251 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/61.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a166ad18115db96335ed811ca80ef1866b05ae069cdc3c0db7301eebb2b99021 +size 815317 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/62.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/62.png new file mode 100644 index 0000000000000000000000000000000000000000..3129bfc4cda94dac870e80b97fdd1408b91fc840 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/62.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01b2cb6d3e47efae7f6adcc9a735278454ea4bb2dc4904ae5931dbd052374ca5 +size 815062 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/63.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/63.png new file mode 100644 index 0000000000000000000000000000000000000000..2b265c767607fd125d067773efead7a81b769c97 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/63.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f060663fcac8835dd135524d7836df1430cd16ef69cd7e8ed3bd9d7acccc6a5d +size 815168 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/64.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/64.png new file mode 100644 index 0000000000000000000000000000000000000000..7090f600110a4102e807bab10f8686993b868141 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/64.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6cb29ffe78675777ca9bb991ff892b123dc519093a232d1603be854b43925c3 +size 817188 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/65.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/65.png new file mode 100644 index 0000000000000000000000000000000000000000..7cfe7fd7eb9d4380689ac7972bcee449d776567d --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/65.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d372bf5e34ae7422e1d33795ea2582a0b81f45264f1de308ee763d4797d1531 +size 814984 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/66.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/66.png new file mode 100644 index 0000000000000000000000000000000000000000..af8c23405ce86adbc92dde844ff8031b296ac689 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/66.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f1169fb6fc1f21d802f253d3374aacf5c56c71ecef1f128a74d6ea150ae42bc +size 814371 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/67.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/67.png new file mode 100644 index 0000000000000000000000000000000000000000..56dee3407a4ce7f70d5c131f6140e27bb1f0169d --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/67.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e8e07c0d060eda464a6ffa110cdbdbe2cf821b78fa6bfc4491158bd7e611196 +size 815015 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/68.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/68.png new file mode 100644 index 0000000000000000000000000000000000000000..56fb4780739ef057efe6fd40945ba65e71f3962b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/68.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46e4498e69a6c8b88b6b1c577d6eac620d55e90b54ba53918422bfe80072899a +size 815628 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/69.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/69.png new file mode 100644 index 0000000000000000000000000000000000000000..4660ad8e5c626d60b1c8bd67bfccaed1f7b1966d --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/69.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcb4f2e93d2c181f5f0531f44d9f36224e7c8eb21404cb3d666d1e1b30111466 +size 815384 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/7.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/7.png new file mode 100644 index 0000000000000000000000000000000000000000..bea8a0151216282568edf9e8b62ba251716dd947 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/7.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab74ecc44d09ee13ef29cdc8108f7d34319567927d84bda1f7c4a5cdf46d09dc +size 755957 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/70.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/70.png new file mode 100644 index 0000000000000000000000000000000000000000..b0798d3c52db2ae4ae6309610f62a1643493048e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/70.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c6f02882c2bfe7f3ddf904f44fd7f6901fb9452d1347c4da20ac8dc2d57ef44 +size 814835 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/71.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/71.png new file mode 100644 index 0000000000000000000000000000000000000000..128ba00a5052d060182a29a8c828b6dfaf13aa84 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/71.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73fa6df61428c130a45ab898f8b9bc7d16c6651e974f10f3d52cef02782fcedf +size 815424 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/72.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/72.png new file mode 100644 index 0000000000000000000000000000000000000000..22deeb63bace66b0ccb9fd61dc89ebdee8278a07 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/72.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e821cd44436be9363deb97b49c031b5d450edd66da142fda37fa324a5341cdef +size 817197 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/73.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/73.png new file mode 100644 index 0000000000000000000000000000000000000000..877b0423ab495a3c72ff47471e94687c85ccb61e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/73.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50e88aba0ae2834bb87d9b1606f31c657ab29e300ed0c3ed3f695b63d00f79b4 +size 815602 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/74.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/74.png new file mode 100644 index 0000000000000000000000000000000000000000..3417f745a864233a316010448c3a3ca01061bade --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/74.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:496f948b8e67344df1bcc521b5e38741d190f53ebfdad0eae16d3c7208994baa +size 816536 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/75.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/75.png new file mode 100644 index 0000000000000000000000000000000000000000..8f916ddd6c473feeecac49e1163a41328913445e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/75.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e331ea2aaf7e2d08947537f2a63bfd9357612aca5e91f1c88cb77a84b8cf9efb +size 818602 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/76.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/76.png new file mode 100644 index 0000000000000000000000000000000000000000..cc581457f9592f285d93e39abaf51ab7c97aa1f0 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/76.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74cef3b586ce61ce5a7cfdae4f155c68f66a0a3b57ff787c7e587806f9fb98c7 +size 819966 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/77.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/77.png new file mode 100644 index 0000000000000000000000000000000000000000..7aede77533ac20274a50e6d9544c59d8be95011c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/77.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab769536226b9d1327afedb57bc6f622afed9d5c84166026e62e366e9080fdd1 +size 819792 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/78.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/78.png new file mode 100644 index 0000000000000000000000000000000000000000..7a9ca5ac23b59e86b3f3746232b8ac4401e56a72 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/78.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8811807f47d7f0e84a1d547da607ea8a92e6c15a79c4b1b1b66cac2789c6d984 +size 819829 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/79.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/79.png new file mode 100644 index 0000000000000000000000000000000000000000..fe7196281126d5b1a4c1b9d631cc2d822a796f27 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/79.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd68344f7b4e4f1f0f7cd300a0faf1a64add38420559fe63c6f5593c61d53d86 +size 820707 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/8.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/8.png new file mode 100644 index 0000000000000000000000000000000000000000..557dcd9a6caf3054fc530bff864b1724d26eed49 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/8.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ae9307b33bb56adebff3aad8ce55b4b4b07ec38b3bb80741e44694334fcc14f +size 754635 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/80.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/80.png new file mode 100644 index 0000000000000000000000000000000000000000..8802c384634ae8ebdc9f9477c8145d74a820f471 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-test/80.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a85e76d4c8d11271cb6475c089ca35d31cd72fc7651a5831fb04d39382779d1c +size 819206 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-val.mp4 b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-val.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7705544588d01b8d56a9b293ac614c06b9fc7b18 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it1500-val.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b709c2b4aa8d9b9d8eee7f6c85b3ed19ceaa512bbe9cb615a43e15cb3944e79 +size 215844 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it500-val.mp4 b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it500-val.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..117470276b54d46db9806b217a518fdf519f558e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it500-val.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f6362ca1c3d4de55ed8535c0f9eab51ee405d661173aa36284767c40c91902f +size 208062 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae.mp4 b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad5e2fd1020e92a6742e5b92d7c1a16e24f18f84 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae.mp4 differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/0.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/0.png new file mode 100644 index 0000000000000000000000000000000000000000..b18b7cb815470308cb74dae2a2d3f53cf5e5c115 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/0.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/10.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/10.png new file mode 100644 index 0000000000000000000000000000000000000000..c068337b11116bf1215a294ec3787acc91a72f00 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/10.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/100.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/100.png new file mode 100644 index 0000000000000000000000000000000000000000..af8c94fb0f5684303fc8affc9e2a0d15ce949d17 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/100.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/102.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/102.png new file mode 100644 index 0000000000000000000000000000000000000000..d275a1f0277b4b7ce5d936e1b78026c3470a3979 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/102.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/104.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/104.png new file mode 100644 index 0000000000000000000000000000000000000000..9732e6d26c4022307d743b547f7515cb451ee2aa Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/104.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/106.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/106.png new file mode 100644 index 0000000000000000000000000000000000000000..a795a7a78408e2048e4cd2ef88cb7b5974c0bf4a Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/106.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/108.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/108.png new file mode 100644 index 0000000000000000000000000000000000000000..d2980c2d2deedb85894d246f54446f3cf2a67e2b Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/108.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/110.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/110.png new file mode 100644 index 0000000000000000000000000000000000000000..a46c9c39ad749e6526b27babe4f4f4e055a06f74 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/110.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/112.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/112.png new file mode 100644 index 0000000000000000000000000000000000000000..f18f89770be288edba96ccd183d83b8b7495effc Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/112.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/114.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/114.png new file mode 100644 index 0000000000000000000000000000000000000000..00a9ec531c74ab316026f054e2524ce9055c0c03 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/114.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/116.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/116.png new file mode 100644 index 0000000000000000000000000000000000000000..bb8df72644749f73d54cc5012ad388b63eec034b Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/116.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/118.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/118.png new file mode 100644 index 0000000000000000000000000000000000000000..03942616a67887e8afd1cb0fd430b6528b3cbe89 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/118.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/12.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/12.png new file mode 100644 index 0000000000000000000000000000000000000000..1d4e0333b04dfaffa3acaa89571c93ed6dfaf4de Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/12.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/14.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/14.png new file mode 100644 index 0000000000000000000000000000000000000000..af716dbde44dc3cdd8a2081c05777b30d0953299 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/14.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/16.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/16.png new file mode 100644 index 0000000000000000000000000000000000000000..9d6c93fc778a34f2ed77f0923d7ae73907c9a59b Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/16.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/18.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/18.png new file mode 100644 index 0000000000000000000000000000000000000000..222405bf040a92cd749373c576cd3293db2a7fd9 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/18.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/2.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/2.png new file mode 100644 index 0000000000000000000000000000000000000000..f60d5a5119a82ed83401ddd9ca8c6cb546309e71 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/2.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/20.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/20.png new file mode 100644 index 0000000000000000000000000000000000000000..ad3062a3cfd73225e9e4012a36994c4b6fd9fb53 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/20.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/22.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/22.png new file mode 100644 index 0000000000000000000000000000000000000000..27bd86bf5ddbf93d66d3c3ca5946a10fb00a01ab Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/22.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/24.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/24.png new file mode 100644 index 0000000000000000000000000000000000000000..74b143e4efb642e33a3f123262b1d59b3baf270d Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/24.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/26.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/26.png new file mode 100644 index 0000000000000000000000000000000000000000..7d8813d108c9b960360647f02c28915b49a56553 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/26.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/28.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/28.png new file mode 100644 index 0000000000000000000000000000000000000000..285d5aa061321860809774fdccd89e3a70fc6a95 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/28.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/30.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/30.png new file mode 100644 index 0000000000000000000000000000000000000000..59ce377f3c231bc215c66b12d9834a0a2d9d65ed Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/30.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/32.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/32.png new file mode 100644 index 0000000000000000000000000000000000000000..a743ad64d984556594461486e0e1138f3b464585 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/32.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/34.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/34.png new file mode 100644 index 0000000000000000000000000000000000000000..fab5fc90a54b0bd914c167f7456118f232cd5bca Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/34.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/36.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/36.png new file mode 100644 index 0000000000000000000000000000000000000000..700f35a33345f09c0e9983529e1f90220ff48b7e Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/36.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/38.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/38.png new file mode 100644 index 0000000000000000000000000000000000000000..9775149b8bc6df22aff21f7e2992360efcaaf6cc Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/38.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/4.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/4.png new file mode 100644 index 0000000000000000000000000000000000000000..acb6f494a4c31005e8be8caa5a1ff7965b839b9a Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/4.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/40.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/40.png new file mode 100644 index 0000000000000000000000000000000000000000..e165a26f3c7837212816698d77bd24adbae7c46d Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/40.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/42.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/42.png new file mode 100644 index 0000000000000000000000000000000000000000..14fc0e5f9b68074f7eb05b50f13c3235635efaa8 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/42.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/44.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/44.png new file mode 100644 index 0000000000000000000000000000000000000000..f03b167467c9be4cd171fb8c5c2bacd0feef6c24 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/44.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/46.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/46.png new file mode 100644 index 0000000000000000000000000000000000000000..91cb3b0c0e172ce1e332867d7bb5de89c3e9d9a5 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/46.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/48.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/48.png new file mode 100644 index 0000000000000000000000000000000000000000..c7b29c832e7fd692bb649a22b10036783c3c16ec Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/48.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/50.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/50.png new file mode 100644 index 0000000000000000000000000000000000000000..b6352b6c7736ad34133bb98e581c30a56026795b Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/50.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/52.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/52.png new file mode 100644 index 0000000000000000000000000000000000000000..753a746df7a07ba00d359b4e36bab256ac69e027 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/52.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/54.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/54.png new file mode 100644 index 0000000000000000000000000000000000000000..2f1df28e16b3fc8ebce21f4c3facf24277ed938e Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/54.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/56.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/56.png new file mode 100644 index 0000000000000000000000000000000000000000..3059668d873511efa8b0bc301242fe1ec4c14ea5 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/56.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/58.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/58.png new file mode 100644 index 0000000000000000000000000000000000000000..8425c1b239ea61a0f77e2aafc472f3676d6c20f0 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/58.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/6.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/6.png new file mode 100644 index 0000000000000000000000000000000000000000..51c615cc4802bfe2ad329bc7ea39c6658ca8ff31 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/6.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/60.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/60.png new file mode 100644 index 0000000000000000000000000000000000000000..51304e76c62a6d31960bb455a3ec8f62d02ccab3 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/60.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/62.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/62.png new file mode 100644 index 0000000000000000000000000000000000000000..b35827345a6f370f8c7e40fa0d2f3d70a2491136 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/62.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/64.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/64.png new file mode 100644 index 0000000000000000000000000000000000000000..823e9ea5fc9ea35148a2421f8b75876adbb7308a Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/64.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/66.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/66.png new file mode 100644 index 0000000000000000000000000000000000000000..a56836131b1793e1f6cd34cc656b2279143ce330 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/66.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/68.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/68.png new file mode 100644 index 0000000000000000000000000000000000000000..2f97ec977b6d5059cddc7bbdba1856f00ec093c8 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/68.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/70.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/70.png new file mode 100644 index 0000000000000000000000000000000000000000..70b6b52e8187ff2024b647b15518acd3d699c105 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/70.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/72.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/72.png new file mode 100644 index 0000000000000000000000000000000000000000..8c5910fb1d69525e17c12335875b878877951570 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/72.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/74.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/74.png new file mode 100644 index 0000000000000000000000000000000000000000..eed84597cedf819fd9655f9ee9f045074f3a3f0e Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/74.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/76.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/76.png new file mode 100644 index 0000000000000000000000000000000000000000..07d23a0012babd8991f0b95ec7c853faa3bd98c6 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/76.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/78.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/78.png new file mode 100644 index 0000000000000000000000000000000000000000..151b1287979b351454a276f6cee793d876988ff1 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/78.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/8.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/8.png new file mode 100644 index 0000000000000000000000000000000000000000..48f8d725d3c5de88e220d78db922368c0f7bf194 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/8.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/80.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/80.png new file mode 100644 index 0000000000000000000000000000000000000000..3a990da86600aad74ebf8377bd4b9d5704c16528 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/80.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/82.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/82.png new file mode 100644 index 0000000000000000000000000000000000000000..b773dc4c58bab440d4a483778389082001106f62 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/82.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/84.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/84.png new file mode 100644 index 0000000000000000000000000000000000000000..0643dbe90a6debb1a7375bbef26725dd17baaa4a Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/84.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/86.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/86.png new file mode 100644 index 0000000000000000000000000000000000000000..2d27db41d838508eb60a9c85e0f5c90833e4ae41 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/86.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/88.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/88.png new file mode 100644 index 0000000000000000000000000000000000000000..e1c5c8e50f62ae8c259be2eac1db354c0c88f211 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/88.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/90.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/90.png new file mode 100644 index 0000000000000000000000000000000000000000..b555e102724c80493c9b38f39fdd4747279dcb7a Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/90.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/92.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/92.png new file mode 100644 index 0000000000000000000000000000000000000000..33cffeb87d2cf50212b75f77a76e797cd039d338 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/92.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/94.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/94.png new file mode 100644 index 0000000000000000000000000000000000000000..f0211d05f0f5a956c41afdef39873644ca393ccf Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/94.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/96.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/96.png new file mode 100644 index 0000000000000000000000000000000000000000..847e095fb754b9b3b29da5bca39308cc62322690 Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/96.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/98.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/98.png new file mode 100644 index 0000000000000000000000000000000000000000..c012f332f4559fc950203711dffa7bf732f667af Binary files /dev/null and b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-ae/98.png differ diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat.mp4 b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e789cdfada57b922f61308abf132412f7823ace0 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61ada7de6d93460c233021665cec47464926ea9ed136b961045b6d0cf863d117 +size 175977 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/0.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/0.png new file mode 100644 index 0000000000000000000000000000000000000000..16fe2afd46c07de62bd991842d417082ac933c91 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/0.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64e73684176b0559ac137b04f693843338814f0beffad00a4b608d24df8a3d39 +size 318452 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/1.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/1.png new file mode 100644 index 0000000000000000000000000000000000000000..891e032efd220263c158546085776f24a585ea57 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78aa65491020ab5e9bb7706bfc3b9d9c3b3d0f4f10fa000c0aabc19f6afcf8b3 +size 318671 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/10.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/10.png new file mode 100644 index 0000000000000000000000000000000000000000..568370f35754215ea4132743145098ddc86f711b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/10.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afc4507611ed3f27aa5cb936f826ece5256b7910d19a62ac6cd6a9c3b4ca74ef +size 324129 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/100.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/100.png new file mode 100644 index 0000000000000000000000000000000000000000..2ec3dc09d48e069e6da2753baa769514df414c5d --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/100.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0662989e398c79368c5a5b18dbbbea3e47f4d68e3912c17e5de6f358eb90652e +size 340238 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/101.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/101.png new file mode 100644 index 0000000000000000000000000000000000000000..2ddce85b0ac59a147029c1a99acdb6c6e075fbc7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/101.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24a7300026bcde9cc32df4a2c32550e3e02e7817e23038e35945b6bcd4d18bcd +size 339535 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/102.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/102.png new file mode 100644 index 0000000000000000000000000000000000000000..dc5502c4ae5d410df13a45f305e703147bb755b0 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/102.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10007e476f46d868541e551d8854697b54a3a98116b2257cdcc5ce074a788045 +size 338662 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/103.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/103.png new file mode 100644 index 0000000000000000000000000000000000000000..b209b1e97fe641bf8f7515493442da0e4ae410d5 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/103.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cfcbbf83d054638ada6c89988eeea146cb80a8b4ced93d342eea4e405cdc5cd +size 338074 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/104.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/104.png new file mode 100644 index 0000000000000000000000000000000000000000..2fa626f27db069fce1d0213c6ec06d1a574051c4 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/104.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dad4bf1da3cec30d95ee9e977abdc46186fde28171d3ea3066615ca10f1e8527 +size 337306 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/105.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/105.png new file mode 100644 index 0000000000000000000000000000000000000000..cff2c0b5daeb793abb1da0091385d45e9d8cabff --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/105.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aed97744bea4a31d412c9c5b5c43c3a4cf519b29b2c7a530c82a4ff08b83c18 +size 337056 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/106.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/106.png new file mode 100644 index 0000000000000000000000000000000000000000..44548df3d871f1881a64bac00f738de1fcb3ac23 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/106.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9a38248eb6cc2b41f042ad1d801ba17912497973811ee0e9835047b2cea1f41 +size 336017 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/107.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/107.png new file mode 100644 index 0000000000000000000000000000000000000000..cf6ab3b44eaefac4aa5d366521029da466ea64d9 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/107.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d55acde6c7dccc95d4c1f6f144d0ec909cfacdb56d11a526726507a6174c4897 +size 335686 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/108.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/108.png new file mode 100644 index 0000000000000000000000000000000000000000..0660b69dc403104aa5fbe667d16a464bcbe9d950 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/108.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05d27f62c7ff21deb5874e5809714e09d0ee8d6f7899593cb886d6e12fdadc67 +size 335208 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/109.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/109.png new file mode 100644 index 0000000000000000000000000000000000000000..f7559a4b13880c060d8059fa91cf30230d9f2e89 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/109.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8f58d60d498149e41a966e158bea01508168aa40931e82e9bcf7e47b3a62290 +size 334841 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/11.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/11.png new file mode 100644 index 0000000000000000000000000000000000000000..029258fc89ac11afeb2c48e3eb8ddbc3ffc286e1 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/11.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2f872821bf2779c9fbfcf930b1feb9dd601797cabfcce2f736b0c95fb0cc289 +size 324594 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/110.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/110.png new file mode 100644 index 0000000000000000000000000000000000000000..9569347c04f14852d754d660299e269bb7427622 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/110.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1175d8ca4505115b05a6bdff6c570c9ad3460f8a9bece01a0e0d006c8c45183a +size 334349 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/111.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/111.png new file mode 100644 index 0000000000000000000000000000000000000000..77f5ee4a168acfcbe03c1414e462735b332b1a2e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/111.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2c758ecc4b9d6d1b545f369e45fb6049f182ec0a4918e59a8c22a9e1c5188f3 +size 333910 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/112.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/112.png new file mode 100644 index 0000000000000000000000000000000000000000..9345cb0f67b6e4893fba82005eb532b816999345 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/112.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd77d3facbd1d19809048a94c23047dcc4e393ab5be6caf25c123d049682afaa +size 333574 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/113.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/113.png new file mode 100644 index 0000000000000000000000000000000000000000..fc79c1f6f74051ab801171a116d9e6f76e62e758 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/113.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3365c126d7b017d6b85c8a4be7bc3687f5d4a697cfc1d6e33a75ac65c3c556f4 +size 333225 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/114.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/114.png new file mode 100644 index 0000000000000000000000000000000000000000..9c80942c68edfa652e6af1191131e729c654c0b1 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/114.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09f0a90d3e8a4cf4a9cf102292a4241ca27d1d8788923675b15465e119eee635 +size 333024 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/115.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/115.png new file mode 100644 index 0000000000000000000000000000000000000000..166fc4a437d8d46b7e25f99b432047ad5fc296ef --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/115.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9be7e2f94bbd3020f97c1cfd803cf69365a83db1036c563982c33fdc1b479578 +size 333041 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/116.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/116.png new file mode 100644 index 0000000000000000000000000000000000000000..c36afc69aaea7870e77c5131b888f0cfc1d072d1 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/116.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f05c4d09e28a1dbf070d3156290d89adf05aac97066ec734b134b5491eebde3 +size 332980 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/117.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/117.png new file mode 100644 index 0000000000000000000000000000000000000000..415b32e8c2e68b69398ebc2244006ca5c12e91b8 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/117.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e09af448fae927517533fd62aed93670e599432bec0a0628b855380622a7b07 +size 333262 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/118.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/118.png new file mode 100644 index 0000000000000000000000000000000000000000..b73ff15bf3a0fa8881dc65aa75b3b81a6e2b8231 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/118.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34dbafc4d290fe1031cf5b65b0fe3ba6fbd02101feecbc172f53d5e75e2d0014 +size 333079 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/119.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/119.png new file mode 100644 index 0000000000000000000000000000000000000000..3b904984a9deafbf9b1d79592ef20e225be592d7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/119.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf3ede74dffbe5c966765efef8cb426d3d14e25499a68351270b8ef7bc204f25 +size 335078 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/12.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/12.png new file mode 100644 index 0000000000000000000000000000000000000000..0413919313ad6673f2762fbb0ca299083307efd3 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/12.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5e756d0087fbf9beb7d11b51913770395f112a883a3fecedbc1c9c7d9707370 +size 325291 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/13.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/13.png new file mode 100644 index 0000000000000000000000000000000000000000..6bf3610af7f27ca7794207903471fe0c28568029 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/13.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65721269fbd9faca1713b097366042e4a96635c025ac8098febc64976816999c +size 325873 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/14.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/14.png new file mode 100644 index 0000000000000000000000000000000000000000..c438c466d054a48b76745566d396e5ac81ae8204 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/14.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:872bd1a87e1fa694b9823996eb8e6fa2d30aaceb3f74f23c48c6d77a1608642d +size 326286 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/15.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/15.png new file mode 100644 index 0000000000000000000000000000000000000000..0cf5fc579cae9c9da1699cc17c6f334961f14a4f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/15.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea0bba3e2af8a4f51c146e75e4fc42e9d2cf1a461ae51614f05a3d6f9b7cb4ea +size 326765 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/16.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/16.png new file mode 100644 index 0000000000000000000000000000000000000000..5dccd265e979e4fc7c55c21f0a90af698c89d110 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/16.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bccb380d81928c4ce664e5b7270607879b12220da4e13867c6c5d11aba517ab +size 327601 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/17.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/17.png new file mode 100644 index 0000000000000000000000000000000000000000..84a8cce6f171f9c668f7e7ae6b1174971d160f12 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/17.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:409742aa0146d52b487248c9aefc1167f2a129555d95605db294c5287ecedaf3 +size 328263 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/18.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/18.png new file mode 100644 index 0000000000000000000000000000000000000000..94be61152fca7e9e72f01d5957e70733311011c6 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/18.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d18eab7ea1739779394e393e8a8963eb99c27949eacfd658bf1b39852d3bc18 +size 328537 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/19.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/19.png new file mode 100644 index 0000000000000000000000000000000000000000..c3c60b845e31a82e1b6f7a8c7ac427cf4d61f9e6 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/19.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b1275554837f7de22ab0243fba773bc0379ab72ca86931c86b8b9e4e989c28a +size 329147 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/2.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/2.png new file mode 100644 index 0000000000000000000000000000000000000000..ed84478add3af0faa2630a74323ccc991103b2b7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cc8603bff7bae5cf2e401776160c2aa457b975a2afaca503fbefa8a80919231 +size 319044 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/20.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/20.png new file mode 100644 index 0000000000000000000000000000000000000000..7ade47831fe4c3bb56c3720f5594c3f3ee5e4151 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/20.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d7f766bc4e1213e50c9a2ce0550979fcaf8fd357b6599eb9138963e777c853 +size 329636 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/21.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/21.png new file mode 100644 index 0000000000000000000000000000000000000000..13abf0900f560f59d02e922616a1aa9bcde9f575 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/21.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a42a64581b9bb2ad23b2f00e4666553f83b5f78d88d50f609072be56c14cdff +size 330542 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/22.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/22.png new file mode 100644 index 0000000000000000000000000000000000000000..a5664493b5eb8574cb833ff730f12c5325ca4295 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/22.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:950b2324a11f8204d7fe4f39d489c029bd2b1ef840f819c45d1291e65ec5c119 +size 330851 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/23.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/23.png new file mode 100644 index 0000000000000000000000000000000000000000..b8aebf55887be35c5c6341016cdc84e17102823f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/23.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19b53d5df6d6e8982c4318f72c29a74ff183301eab4fcc2c7ec86a744c7bfd18 +size 331503 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/24.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/24.png new file mode 100644 index 0000000000000000000000000000000000000000..3f7890417ce85fdcd7a14eb8d510c10585c53c48 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/24.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:814688f28f8a9b1c913c24d6bb7525e17edb7398c7f6c5b888f91a01f9171268 +size 332089 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/25.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/25.png new file mode 100644 index 0000000000000000000000000000000000000000..5179877b223fe5844a9739771c61cd79c55fb830 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/25.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dddcac0411de50025d9cba6c688f799f14089058901dff5baad97141f77a23e +size 332231 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/26.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/26.png new file mode 100644 index 0000000000000000000000000000000000000000..9745b6c9ce3c545702e83079f80879c67d39eedb --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/26.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38e3715d57a6ce8b7babc66a83f6f558ed6e8c5fb1c69c976f6459762fa1ab4f +size 333062 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/27.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/27.png new file mode 100644 index 0000000000000000000000000000000000000000..6841773ed8575a38d5627951ac877d929cf65f6c --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/27.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af37898c3c2234a67283a4c72877a0eac5b570e9cbe00fa8abff35b226469e1c +size 333706 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/28.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/28.png new file mode 100644 index 0000000000000000000000000000000000000000..caec8d87a2ab9d52d98f47037eb18bb76a47d343 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/28.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d12bc6b0738a29dd6b7a668d482c65b7128f37a6c52561068b7876593c3e35e0 +size 333733 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/29.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/29.png new file mode 100644 index 0000000000000000000000000000000000000000..c918c40b12a06d4138a95fc9e3b2b811aafa77ce --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/29.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67f8926c46e4183435a26a7ea2a1d000986116b8c1f92c445d1efd55c15bab90 +size 334373 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/3.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/3.png new file mode 100644 index 0000000000000000000000000000000000000000..ce32d3f44bbcc47bd3f75023a74900665416394f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e837dac283fc2b7ca9fea2b58373c3bfca4b87a46458f5cae2d6b3a138b2f8dd +size 318999 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/30.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/30.png new file mode 100644 index 0000000000000000000000000000000000000000..affaee2f6d828c73c63bc84ca08e2dc4ea9a1fc6 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/30.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b82a8048f8900e2f95416d55f85ebc6f7aa5879f33faa903a5f25d146ff39c65 +size 334996 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/31.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/31.png new file mode 100644 index 0000000000000000000000000000000000000000..bea6f25913483e0292910ce963e9cad5a4ebd134 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/31.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d4cf41ab482e2e72a1efcb7d9b389c8640558103d9191cacaa470694762b705 +size 335965 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/32.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/32.png new file mode 100644 index 0000000000000000000000000000000000000000..1e1b493b0594eab1903b788ccb1f8970e17aebd9 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/32.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:680b383e3a34df878f4132bac176f1a639d903b1fc742ef8c1d5aa9178ec08a0 +size 336891 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/33.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/33.png new file mode 100644 index 0000000000000000000000000000000000000000..2beefb6bd764f39df97cc0cdd2a9137638b4a2ec --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/33.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f0ecb92a7bca93d08713e2f6ab5663a2db6de8e65f8b435df28e7eca9b992b6 +size 337747 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/34.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/34.png new file mode 100644 index 0000000000000000000000000000000000000000..59dc81f839d21c11220d367a21b39859a1878ca1 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/34.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdf894357326479ff707355569dcf85c6d719a0593e0b0f8ce950b73f8847cd0 +size 338685 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/35.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/35.png new file mode 100644 index 0000000000000000000000000000000000000000..301b8ce34e635a7840dffa29390d2368d8dbac23 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/35.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36d28dbd42563e5dd9c69aa966eec33de7cb9052700647ae0e90a993b0aba5ef +size 339565 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/36.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/36.png new file mode 100644 index 0000000000000000000000000000000000000000..c3fcf31b5d2e327fcaf21d6d6113cf1d3feb1d06 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/36.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ceb5008d44d44ecda7667610bb32db6fce54435ee565eeaa1a1ec45b712624eb +size 340829 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/37.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/37.png new file mode 100644 index 0000000000000000000000000000000000000000..71c8317b904bedeb77ce956a71222ed042155f6e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/37.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a911f70170ea0763b561a9105dead49eb2ed8b6bd6457c9c279bf6492356f728 +size 341722 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/38.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/38.png new file mode 100644 index 0000000000000000000000000000000000000000..eaa633612acc72bdd392f28b7580b58cb189d310 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/38.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0055a099011b318c4deb03ae0e9a471a350f37cc520656babda3fa5abc6176af +size 342682 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/39.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/39.png new file mode 100644 index 0000000000000000000000000000000000000000..dee3bea84a1447f947f4260bbf583fcbfe27c96b --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/39.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8088e599ffec6ff396acbc74e1787ea8e8cb578fe9373b0c95df92c176db051 +size 343443 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/4.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/4.png new file mode 100644 index 0000000000000000000000000000000000000000..b2fdd99125402c1113bca3c4e6d63f4f01d8bff6 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c33be0e15f471e77185498e0a6d828544e99325cfb97c789bbe09f5c5c43e787 +size 319415 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/40.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/40.png new file mode 100644 index 0000000000000000000000000000000000000000..971a9f5d4c7c0d9f73416a05cdc16b2e2f18721a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/40.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79fab7fafeb67e3826817848a7e901ecd2a2381349b1125ab90499e25f2759f9 +size 344164 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/41.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/41.png new file mode 100644 index 0000000000000000000000000000000000000000..b44a69821aa5dd58083a680c9a3e155b0c82c769 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/41.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d541248bc21bdae25a8a5b7793523aacb8d69f253d85fb3a91ed4b03f438983c +size 345046 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/42.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/42.png new file mode 100644 index 0000000000000000000000000000000000000000..a47c4c999f94550e796637ef16acb6d91d57a044 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/42.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1600e8edf93b44e7da38c7ff1c9680e6c46647f0643049a83f6db88d56e01acf +size 345555 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/43.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/43.png new file mode 100644 index 0000000000000000000000000000000000000000..a25646957b91f5176bbc293934de3c0f5c375600 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/43.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82bbc8309c5d1868b73a85af40769bf4e661c648178943d4b15578ce79e17933 +size 346365 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/44.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/44.png new file mode 100644 index 0000000000000000000000000000000000000000..4ae27bfbdec53b8e9532820d776640b49069dc96 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/44.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f4ea82e79222c616e0a54ab1263a7372b82f8408540a37af0318ff98509f88f +size 347311 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/45.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/45.png new file mode 100644 index 0000000000000000000000000000000000000000..a1a1f010e33f78857d454caba78d3c4801f686eb --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/45.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0894f7077d566357656ba95eb9d34e2963485bcacf3a74069cc51bde3e505b6b +size 348084 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/46.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/46.png new file mode 100644 index 0000000000000000000000000000000000000000..41bbcc9789358b6ebc02dbdea02a4275d4a49ef9 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/46.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21dd19a9a7b2948b71feea13221922f41dce89d1e3e22c68e4b5c4e21f51820c +size 348711 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/47.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/47.png new file mode 100644 index 0000000000000000000000000000000000000000..9c737988ed63f3e8e01981f989b63b8d3a234337 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/47.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dbe43e316f9f16582e226f38e8311a4a48c069577fc089ef944a6c34f45d3f8 +size 349518 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/48.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/48.png new file mode 100644 index 0000000000000000000000000000000000000000..604e24d4eb481acbda7a1582592cf07c33698419 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/48.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82851c01a9176d6ed8ffe23b8c8792d4659be1f654b037231dd9dc590c540bca +size 350143 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/49.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/49.png new file mode 100644 index 0000000000000000000000000000000000000000..b85534ed7addc7668ff8527e4b63ee2a6e1082a7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/49.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c68d87da3c1516f35f02b1fde4b87e71879cf7cd846eabad30c86ea861ff3ac +size 350656 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/5.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/5.png new file mode 100644 index 0000000000000000000000000000000000000000..401fae864ec2b480be1947b83730d5ca74eb902a --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/5.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0cc37dd0f660d6c89a167251dca9214bea20ce7cc74b4806b4f8f49dc7cf49f +size 319866 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/50.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/50.png new file mode 100644 index 0000000000000000000000000000000000000000..033b83d06515eda6b903274a6a981a7cd34af6fc --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/50.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9cbd9501339e02d75a1bfe7bb73eb6382d88752ae2edc810496d45c85eb376f +size 351042 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/51.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/51.png new file mode 100644 index 0000000000000000000000000000000000000000..858dcafb72ac9f92442d71ee81370e3e53980f93 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/51.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78f706f8933de0a84f1a1bd4ef1d1f0fb5b2a96c934ad33b148139bfde33ff4f +size 351487 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/52.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/52.png new file mode 100644 index 0000000000000000000000000000000000000000..eb7e5b2050159f68567b9b7320d24b4faeb163dc --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/52.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f1aaf92cbb56549841ca2bb977acc9341f3fdab68c9e781e8a71fd93f5f974b +size 351584 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/53.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/53.png new file mode 100644 index 0000000000000000000000000000000000000000..edc7227cca40389e05c57855aa41e9e2b7943e5e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/53.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f8c03ff846b666202f4712211a1684c3f0c8464c3d8e42939fbf4da69e65d50 +size 352160 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/54.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/54.png new file mode 100644 index 0000000000000000000000000000000000000000..f4419ce96b077385306d9b0a2e369bf1cfa50024 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/54.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32ee3434e5d52d95d8a2d49e46300acaee70ae8426739bf57f148a6224d9b85f +size 352203 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/55.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/55.png new file mode 100644 index 0000000000000000000000000000000000000000..3932a8773d060711ab9ac3ea83613aee868bdcbc --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/55.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0236b9bfab14100f1935e8e9abd3657e0be01da1365e56db648e53e6ad4316e9 +size 352356 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/56.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/56.png new file mode 100644 index 0000000000000000000000000000000000000000..2b15bf89c143001e2000d4b224e1788b6881d4e6 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/56.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af61d064d90a1f3a8fcf24b6f0218f6ab2742a9186d4469fab1c47dc724b145e +size 352782 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/57.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/57.png new file mode 100644 index 0000000000000000000000000000000000000000..f9b226d5dd3fc9eed3a1f9140bba7906dcdf60cb --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/57.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88810b5785d017651fd34d7966aa2005db590c711cd780e0c7f1adce81b976c4 +size 352858 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/58.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/58.png new file mode 100644 index 0000000000000000000000000000000000000000..909d740c9bcbfd5b8ff9513c340876e9dd127f75 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/58.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b795f82b8d4229c48ee30b1be573c2719dbe9311bdec6e42c77c4d711fa55ff0 +size 352712 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/59.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/59.png new file mode 100644 index 0000000000000000000000000000000000000000..e8a679f4a80a5b7a862e3d045198163a73e49385 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/59.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:170b1527a646209579826bfe5415b42120cc3f58610aaccddeab1aef222a041a +size 352842 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/6.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/6.png new file mode 100644 index 0000000000000000000000000000000000000000..999a852bfc53aa068cc529042dfa682b1819a09f --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/6.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f40058dc9f9a73433aed23965b483a9b4d9c0e582e7f16cf5361068f41ca3ca +size 320157 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/60.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/60.png new file mode 100644 index 0000000000000000000000000000000000000000..a63e9f2d86939b0e459b1877c19de516347edbc8 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/60.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:248685d7208bf3af6bbf68d2ded1491ec9257a9cbbe9c1a2606bd1a5cecd8abb +size 352917 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/61.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/61.png new file mode 100644 index 0000000000000000000000000000000000000000..5b1975b924a6170524fc9dca5e30e29e8ada6cc9 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/61.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fcd46ff10c58b7a6afcfbed8d8217898414c7694bb77f47bbf25c355bd36232 +size 352687 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/62.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/62.png new file mode 100644 index 0000000000000000000000000000000000000000..c4b12cce499e58504b57051b71f9e1c3cb66b904 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/62.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45f798c96d4328fdd6a01d88b3eebf96b4be1800443bd63d021eae4675b626af +size 352619 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/63.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/63.png new file mode 100644 index 0000000000000000000000000000000000000000..2f15d6e9233d2f00ec783660bb2814e17bd6faeb --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/63.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12d5afd8a1cece660fb61237a8337fa5683066ba86ead4cc307a0672f084cb09 +size 352480 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/64.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/64.png new file mode 100644 index 0000000000000000000000000000000000000000..ae254590746a5fab61aad7a208e4b29be6e25426 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/64.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64e58a8b26535e6888bf382bbc2fd75e592c9f99c1578e03f22a5cd2af5d5cc7 +size 352409 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/65.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/65.png new file mode 100644 index 0000000000000000000000000000000000000000..efd373dfcb27aabf1c7a621490d2a5941256d63e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/65.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb01a34bc11adb00b13077b1e9a7d13820feda50679b697d8208c3f937289c39 +size 352501 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/66.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/66.png new file mode 100644 index 0000000000000000000000000000000000000000..afa42f484bea3be4b11da56da33367d335931f26 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/66.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:023a25240b7601ad28ebb7f71af39d662b29557a5fb331137a37804a8ef4c134 +size 352028 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/67.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/67.png new file mode 100644 index 0000000000000000000000000000000000000000..39ad8f2593bf2be41759b570738983a2905efd26 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/67.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66835b066e53744899889e648198cbf19f33a14aaceef61c6f366f47f0c6a304 +size 351582 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/68.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/68.png new file mode 100644 index 0000000000000000000000000000000000000000..5954cf65f9d466df0a92d28da577710001350fe7 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/68.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:578c4192be1942def58a5bd61277ff88ac5968f0c9c1d777d8bf3b5203758b1b +size 351695 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/69.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/69.png new file mode 100644 index 0000000000000000000000000000000000000000..a13e0a83f2ada7c6e6cabb28dfcbfe013a6cba9e --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/69.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12797ddfb89d272e629b4300180885c10450587e7b86e840687be9f2706f9150 +size 351521 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/7.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/7.png new file mode 100644 index 0000000000000000000000000000000000000000..d1fc89107870274c61a6ca30c37ab5f1528bc984 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/7.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d07c8b337fc14e33732f8e4d4267a6638e1b7179418570c7cd2acd00deca2ad3 +size 321160 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/70.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/70.png new file mode 100644 index 0000000000000000000000000000000000000000..f1da2a01cb180f08ea20bc26727ba64bd81f1adf --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/70.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:537fe1e8ada147268fd91fad926dc3fa5fa307a6db0ca8ee38e2e796cb4cc996 +size 351114 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/71.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/71.png new file mode 100644 index 0000000000000000000000000000000000000000..1dbb4775de01da135e812946727eacb2b028e406 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/71.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31929088c694481b69b2fbf6acc0420a6820c3df95ea997058eeba91f0b2b8d4 +size 351270 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/72.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/72.png new file mode 100644 index 0000000000000000000000000000000000000000..7e988a987a9b6c6f100e6fa581e642bb8bdfbbe5 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/72.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a1426e25b632069a99f645d98d91edbdd22cf73c373e42f3ecc53fc6a368137 +size 351054 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/73.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/73.png new file mode 100644 index 0000000000000000000000000000000000000000..905e004d7d049a0927fef9732eaf97d0953832d5 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/73.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dee4657ad10a3a93682aa47f5be95b20e5d764500f0bb8666ea4130a50e6c4c0 +size 351051 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/74.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/74.png new file mode 100644 index 0000000000000000000000000000000000000000..1876d60d8c640db1cafec279af019d0ab6cbe044 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/74.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22b6a0e8599f40f16e1022b767cb512d8ce365cfc75cce676cacb20f6d02b403 +size 350751 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/75.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/75.png new file mode 100644 index 0000000000000000000000000000000000000000..f164794a14a0d2954013392fe654f453a8178f45 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/75.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dbefdc21e779a94153c72653198ef43102151ac9e57c4a69c3ec775127b2165 +size 351034 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/76.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/76.png new file mode 100644 index 0000000000000000000000000000000000000000..74140fe32e617ce81f5cb26a8e3eb757b495b606 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/76.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e071fac1034f8dcea13559ffba7143a75db893cf9fc09e3ded491978836c975 +size 350684 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/77.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/77.png new file mode 100644 index 0000000000000000000000000000000000000000..45db8cd72687d840da20461ce234f69143443e47 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/77.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9db1731a2a087833f5cb31d0d898fc2f7423421839c877ca98d0fd822787f312 +size 351013 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/78.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/78.png new file mode 100644 index 0000000000000000000000000000000000000000..057ee04afd58c382a45ae6a81e5fe61b265a91f8 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/78.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c97eaec2a1af5bed5ed72491819e0e2c86f47af88df08f6beeb768c849cb1c03 +size 350697 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/79.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/79.png new file mode 100644 index 0000000000000000000000000000000000000000..96becc42f93b02bf99bde9ba611be069944bdb81 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/79.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2ac737f3587ea742d0e5819dfa688947b9e44e7184aa9e2e78c2291ebba0385 +size 350730 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/8.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/8.png new file mode 100644 index 0000000000000000000000000000000000000000..16fc5c7c10be5d7dbaa09d48abe5e7f99db42a0d --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/8.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cbcf69898819fa78408651e4723d62c9c0ac57eb96a197bb93c1b33be17d2b8 +size 322084 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/80.png b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/80.png new file mode 100644 index 0000000000000000000000000000000000000000..65c73c232b29817ea892f09d147ed533c15b2c16 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/it800-feat/80.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c046cf54c58d7b3dca5e8aea53e363fcfa1639944bfb4a87a679f3016f442dd +size 350479 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/point_cloud.ply b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/point_cloud.ply new file mode 100644 index 0000000000000000000000000000000000000000..037c41e1dffe40c15c3c375ca03b7e54d3fccf19 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/save/point_cloud.ply @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b44cff47adc2bb96367a6a715e07603510ca2dc22e734efbdc67986fd0c78a56 +size 26886052 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/tb_logs/version_0/events.out.tfevents.1738864353.delta4xa6000ada1.2277571.0 b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/tb_logs/version_0/events.out.tfevents.1738864353.delta4xa6000ada1.2277571.0 new file mode 100644 index 0000000000000000000000000000000000000000..f22f7b944147c4c4df3f6420b445c68828cdc9ca --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/tb_logs/version_0/events.out.tfevents.1738864353.delta4xa6000ada1.2277571.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8678619d09e711d782f040c12e6054f76d39ab5d216c85918bad9887ced9d260 +size 1198236 diff --git a/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/tb_logs/version_0/hparams.yaml b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/tb_logs/version_0/hparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0967ef424bce6791893e9a57bb952f80fd536e93 --- /dev/null +++ b/000000000017.1/gs-sds-generation/3DitScene@20250207-015119/tb_logs/version_0/hparams.yaml @@ -0,0 +1 @@ +{} diff --git a/3DitScene_060220251352.zip b/3DitScene_060220251352.zip new file mode 100644 index 0000000000000000000000000000000000000000..29b7b50b2a3b3a2ca0ea23bdf4c8ac5d8cb55dac --- /dev/null +++ b/3DitScene_060220251352.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c17d9c3093ebeb63156d2fa5484722bc87b606b75823dde32a167fad5279b042 +size 430673482