Hugging Face's logo

Spaces:
sshuair
/
dinov3-demo-multi
Running

App Files Community
sshuair commited on
Commit
c27e2f5
·
verified ·
1 Parent(s): 24b1cbe

Upload index.html

Browse files
Files changed (1) hide show
  1. index.html +490 -420
index.html CHANGED
@@ -1,435 +1,505 @@
1
- <!DOCTYPE html>
2
- <html>
3
- <head>
4
- <meta charset="utf-8">
5
- <meta name="description"
6
- content="Deformable Neural Radiance Fields creates free-viewpoint portraits (nerfies) from casually captured videos.">
7
- <meta name="keywords" content="Nerfies, D-NeRF, NeRF">
8
- <meta name="viewport" content="width=device-width, initial-scale=1">
9
- <title>Nerfies: Deformable Neural Radiance Fields</title>
10
-
11
- <link href="https://fonts.googleapis.com/css?family=Google+Sans|Noto+Sans|Castoro"
12
- rel="stylesheet">
13
-
14
- <link rel="stylesheet" href="./static/css/bulma.min.css">
15
- <link rel="stylesheet" href="./static/css/bulma-carousel.min.css">
16
- <link rel="stylesheet" href="./static/css/bulma-slider.min.css">
17
- <link rel="stylesheet" href="./static/css/fontawesome.all.min.css">
18
- <link rel="stylesheet"
19
- href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css">
20
- <link rel="stylesheet" href="./static/css/index.css">
21
- <link rel="icon" href="./static/images/favicon.svg">
22
-
23
- <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
24
- <script defer src="./static/js/fontawesome.all.min.js"></script>
25
- <script src="./static/js/bulma-carousel.min.js"></script>
26
- <script src="./static/js/bulma-slider.min.js"></script>
27
- <script src="./static/js/index.js"></script>
28
- </head>
29
- <body>
30
-
31
- <section class="hero">
32
- <div class="hero-body">
33
- <div class="container is-max-desktop">
34
- <div class="columns is-centered">
35
- <div class="column has-text-centered">
36
- <h1 class="title is-1 publication-title">Nerfies: Deformable Neural Radiance Fields</h1>
37
- <div class="is-size-5 publication-authors">
38
- <span class="author-block">
39
- <a href="https://keunhong.com" target="_blank">Keunhong Park</a><sup>1</sup>,</span>
40
- <span class="author-block">
41
- <a href="https://utkarshsinha.com" target="_blank">Utkarsh Sinha</a><sup>2</sup>,</span>
42
- <span class="author-block">
43
- <a href="https://jonbarron.info" target="_blank">Jonathan T. Barron</a><sup>2</sup>,
44
- </span>
45
- <span class="author-block">
46
- <a href="http://sofienbouaziz.com" target="_blank">Sofien Bouaziz</a><sup>2</sup>,
47
- </span>
48
- <span class="author-block">
49
- <a href="https://www.danbgoldman.com" target="_blank">Dan B Goldman</a><sup>2</sup>,
50
- </span>
51
- <span class="author-block">
52
- <a href="https://homes.cs.washington.edu/~seitz/" target="_blank">Steven M. Seitz</a><sup>1,2</sup>,
53
- </span>
54
- <span class="author-block">
55
- <a href="http://www.ricardomartinbrualla.com" target="_blank">Ricardo Martin-Brualla</a><sup>2</sup>
56
- </span>
57
- </div>
58
-
59
- <div class="is-size-5 publication-authors">
60
- <span class="author-block"><sup>1</sup>University of Washington,</span>
61
- <span class="author-block"><sup>2</sup>Google Research</span>
62
- </div>
63
-
64
- <div class="column has-text-centered">
65
- <div class="publication-links">
66
- <!-- PDF Link. -->
67
- <span class="link-block">
68
- <a href="https://arxiv.org/pdf/2011.12948" target="_blank"
69
- class="external-link button is-normal is-rounded is-dark">
70
- <span class="icon">
71
- <i class="fas fa-file-pdf"></i>
72
- </span>
73
- <span>Paper</span>
74
- </a>
75
- </span>
76
- <span class="link-block">
77
- <a href="https://arxiv.org/abs/2011.12948" target="_blank"
78
- class="external-link button is-normal is-rounded is-dark">
79
- <span class="icon">
80
- <i class="ai ai-arxiv"></i>
81
- </span>
82
- <span>arXiv</span>
83
- </a>
84
- </span>
85
- <!-- Video Link. -->
86
- <span class="link-block">
87
- <a href="https://www.youtube.com/watch?v=MrKrnHhk8IA" target="_blank"
88
- class="external-link button is-normal is-rounded is-dark">
89
- <span class="icon">
90
- <i class="fab fa-youtube"></i>
91
- </span>
92
- <span>Video</span>
93
- </a>
94
- </span>
95
- <!-- Code Link. -->
96
- <span class="link-block">
97
- <a href="https://github.com/google/nerfies" target="_blank"
98
- class="external-link button is-normal is-rounded is-dark">
99
- <span class="icon">
100
- <i class="fab fa-github"></i>
101
- </span>
102
- <span>Code</span>
103
- </a>
104
- </span>
105
- <!-- Dataset Link. -->
106
- <span class="link-block">
107
- <a href="https://github.com/google/nerfies/releases/tag/0.1" target="_blank"
108
- class="external-link button is-normal is-rounded is-dark">
109
- <span class="icon">
110
- <i class="far fa-images"></i>
111
- </span>
112
- <span>Data</span>
113
- </a>
114
- </div>
115
-
116
- </div>
117
- </div>
118
  </div>
119
- </div>
120
- </div>
121
- </section>
122
-
123
- <section class="hero teaser">
124
- <div class="container is-max-desktop">
125
- <div class="hero-body">
126
- <video id="teaser" autoplay muted loop playsinline height="100%">
127
- <source src="./static/videos/teaser.mp4"
128
- type="video/mp4">
129
- </video>
130
- <h2 class="subtitle has-text-centered">
131
- <span class="dnerf">Nerfies</span> turns selfie videos from your phone into
132
- free-viewpoint
133
- portraits.
134
- </h2>
135
- </div>
136
- </div>
137
- </section>
138
-
139
-
140
- <section class="hero is-light is-small">
141
- <div class="hero-body">
142
- <div class="container">
143
- <div id="results-carousel" class="carousel results-carousel">
144
- <div class="item item-steve">
145
- <video poster="" id="steve" autoplay controls muted loop playsinline height="100%">
146
- <source src="./static/videos/steve.mp4"
147
- type="video/mp4">
148
- </video>
149
- </div>
150
- <div class="item item-chair-tp">
151
- <video poster="" id="chair-tp" autoplay controls muted loop playsinline height="100%">
152
- <source src="./static/videos/chair-tp.mp4"
153
- type="video/mp4">
154
- </video>
155
- </div>
156
- <div class="item item-shiba">
157
- <video poster="" id="shiba" autoplay controls muted loop playsinline height="100%">
158
- <source src="./static/videos/shiba.mp4"
159
- type="video/mp4">
160
- </video>
161
- </div>
162
- <div class="item item-fullbody">
163
- <video poster="" id="fullbody" autoplay controls muted loop playsinline height="100%">
164
- <source src="./static/videos/fullbody.mp4"
165
- type="video/mp4">
166
- </video>
167
- </div>
168
- <div class="item item-blueshirt">
169
- <video poster="" id="blueshirt" autoplay controls muted loop playsinline height="100%">
170
- <source src="./static/videos/blueshirt.mp4"
171
- type="video/mp4">
172
- </video>
173
- </div>
174
- <div class="item item-mask">
175
- <video poster="" id="mask" autoplay controls muted loop playsinline height="100%">
176
- <source src="./static/videos/mask.mp4"
177
- type="video/mp4">
178
- </video>
179
- </div>
180
- <div class="item item-coffee">
181
- <video poster="" id="coffee" autoplay controls muted loop playsinline height="100%">
182
- <source src="./static/videos/coffee.mp4"
183
- type="video/mp4">
184
- </video>
185
- </div>
186
- <div class="item item-toby">
187
- <video poster="" id="toby" autoplay controls muted loop playsinline height="100%">
188
- <source src="./static/videos/toby2.mp4"
189
- type="video/mp4">
190
- </video>
191
- </div>
192
- </div>
193
- </div>
194
- </div>
195
- </section>
196
-
197
-
198
- <section class="section">
199
- <div class="container is-max-desktop">
200
- <!-- Abstract. -->
201
- <div class="columns is-centered has-text-centered">
202
- <div class="column is-four-fifths">
203
- <h2 class="title is-3">Abstract</h2>
204
- <div class="content has-text-justified">
205
- <p>
206
- We present the first method capable of photorealistically reconstructing a non-rigidly
207
- deforming scene using photos/videos captured casually from mobile phones.
208
- </p>
209
- <p>
210
- Our approach augments neural radiance fields
211
- (NeRF) by optimizing an
212
- additional continuous volumetric deformation field that warps each observed point into a
213
- canonical 5D NeRF.
214
- We observe that these NeRF-like deformation fields are prone to local minima, and
215
- propose a coarse-to-fine optimization method for coordinate-based models that allows for
216
- more robust optimization.
217
- By adapting principles from geometry processing and physical simulation to NeRF-like
218
- models, we propose an elastic regularization of the deformation field that further
219
- improves robustness.
220
- </p>
221
- <p>
222
- We show that <span class="dnerf">Nerfies</span> can turn casually captured selfie
223
- photos/videos into deformable NeRF
224
- models that allow for photorealistic renderings of the subject from arbitrary
225
- viewpoints, which we dub <i>"nerfies"</i>. We evaluate our method by collecting data
226
- using a
227
- rig with two mobile phones that take time-synchronized photos, yielding train/validation
228
- images of the same pose at different viewpoints. We show that our method faithfully
229
- reconstructs non-rigidly deforming scenes and reproduces unseen views with high
230
- fidelity.
231
- </p>
232
- </div>
233
- </div>
234
- </div>
235
- <!--/ Abstract. -->
236
-
237
- <!-- Paper video. -->
238
- <div class="columns is-centered has-text-centered">
239
- <div class="column is-four-fifths">
240
- <h2 class="title is-3">Video</h2>
241
- <div class="publication-video">
242
- <iframe src="https://www.youtube.com/embed/MrKrnHhk8IA?rel=0&amp;showinfo=0"
243
- frameborder="0" allow="autoplay; encrypted-media" allowfullscreen></iframe>
244
- </div>
245
- </div>
246
- </div>
247
- <!--/ Paper video. -->
248
- </div>
249
- </section>
250
 
251
-
252
- <section class="section">
253
- <div class="container is-max-desktop">
254
-
255
- <div class="columns is-centered">
256
-
257
- <!-- Visual Effects. -->
258
- <div class="column">
259
- <div class="content">
260
- <h2 class="title is-3">Visual Effects</h2>
261
- <p>
262
- Using <i>nerfies</i> you can create fun visual effects. This Dolly zoom effect
263
- would be impossible without nerfies since it would require going through a wall.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
264
  </p>
265
- <video id="dollyzoom" autoplay controls muted loop playsinline height="100%">
266
- <source src="./static/videos/dollyzoom-stacked.mp4"
267
- type="video/mp4">
268
- </video>
269
- </div>
270
- </div>
271
- <!--/ Visual Effects. -->
272
-
273
- <!-- Matting. -->
274
- <div class="column">
275
- <h2 class="title is-3">Matting</h2>
276
- <div class="columns is-centered">
277
- <div class="column content">
278
- <p>
279
- As a byproduct of our method, we can also solve the matting problem by ignoring
280
- samples that fall outside of a bounding box during rendering.
281
- </p>
282
- <video id="matting-video" controls playsinline height="100%">
283
- <source src="./static/videos/matting.mp4"
284
- type="video/mp4">
285
- </video>
286
- </div>
287
-
288
  </div>
289
- </div>
290
- </div>
291
- <!--/ Matting. -->
292
 
293
- <!-- Animation. -->
294
- <div class="columns is-centered">
295
- <div class="column is-full-width">
296
- <h2 class="title is-3">Animation</h2>
297
-
298
- <!-- Interpolating. -->
299
- <h3 class="title is-4">Interpolating states</h3>
300
- <div class="content has-text-justified">
301
- <p>
302
- We can also animate the scene by interpolating the deformation latent codes of two input
303
- frames. Use the slider here to linearly interpolate between the left frame and the right
304
- frame.
305
- </p>
306
- </div>
307
- <div class="columns is-vcentered interpolation-panel">
308
- <div class="column is-3 has-text-centered">
309
- <img src="./static/images/interpolate_start.jpg"
310
- class="interpolation-image"
311
- alt="Interpolate start reference image."/>
312
- <p>Start Frame</p>
313
- </div>
314
- <div class="column interpolation-video-column">
315
- <div id="interpolation-image-wrapper">
316
- Loading...
317
  </div>
318
- <input class="slider is-fullwidth is-large is-info"
319
- id="interpolation-slider"
320
- step="1" min="0" max="100" value="0" type="range">
321
- </div>
322
- <div class="column is-3 has-text-centered">
323
- <img src="./static/images/interpolate_end.jpg"
324
- class="interpolation-image"
325
- alt="Interpolation end reference image."/>
326
- <p class="is-bold">End Frame</p>
327
  </div>
328
  </div>
329
- <br/>
330
- <!--/ Interpolating. -->
331
 
332
- <!-- Re-rendering. -->
333
- <h3 class="title is-4">Re-rendering the input video</h3>
334
- <div class="content has-text-justified">
335
- <p>
336
- Using <span class="dnerf">Nerfies</span>, you can re-render a video from a novel
337
- viewpoint such as a stabilized camera by playing back the training deformations.
338
- </p>
339
- </div>
340
- <div class="content has-text-centered">
341
- <video id="replay-video"
342
- controls
343
- muted
344
- preload
345
- playsinline
346
- width="75%">
347
- <source src="./static/videos/replay.mp4"
348
- type="video/mp4">
349
- </video>
350
  </div>
351
- <!--/ Re-rendering. -->
352
 
353
- </div>
354
- </div>
355
- <!--/ Animation. -->
356
-
357
-
358
- <!-- Concurrent Work. -->
359
- <div class="columns is-centered">
360
- <div class="column is-full-width">
361
- <h2 class="title is-3">Related Links</h2>
362
-
363
- <div class="content has-text-justified">
364
- <p>
365
- There's a lot of excellent work that was introduced around the same time as ours.
366
- </p>
367
- <p>
368
- <a href="https://arxiv.org/abs/2104.09125" target="_blank">Progressive Encoding for Neural Optimization</a> introduces an idea similar to our windowed position encoding for coarse-to-fine optimization.
369
- </p>
370
- <p>
371
- <a href="https://www.albertpumarola.com/research/D-NeRF/index.html" target="_blank">D-NeRF</a> and <a href="https://gvv.mpi-inf.mpg.de/projects/nonrigid_nerf/" target="_blank">NR-NeRF</a>
372
- both use deformation fields to model non-rigid scenes.
373
- </p>
374
- <p>
375
- Some works model videos with a NeRF by directly modulating the density, such as <a href="https://video-nerf.github.io/" target="_blank">Video-NeRF</a>, <a href="https://www.cs.cornell.edu/~zl548/NSFF/" target="_blank">NSFF</a>, and <a href="https://neural-3d-video.github.io/" target="_blank">DyNeRF</a>
376
- </p>
377
- <p>
378
- There are probably many more by the time you are reading this. Check out <a href="https://dellaert.github.io/NeRF/" target="_blank">Frank Dellart's survey on recent NeRF papers</a>, and <a href="https://github.com/yenchenlin/awesome-NeRF" target="_blank">Yen-Chen Lin's curated list of NeRF papers</a>.
379
- </p>
380
- </div>
381
- </div>
382
- </div>
383
- <!--/ Concurrent Work. -->
384
-
385
- </div>
386
- </section>
387
-
388
-
389
- <section class="section" id="BibTeX">
390
- <div class="container is-max-desktop content">
391
- <h2 class="title">BibTeX</h2>
392
- <pre><code>@article{park2021nerfies,
393
- author = {Park, Keunhong and Sinha, Utkarsh and Barron, Jonathan T. and Bouaziz, Sofien and Goldman, Dan B and Seitz, Steven M. and Martin-Brualla, Ricardo},
394
- title = {Nerfies: Deformable Neural Radiance Fields},
395
- journal = {ICCV},
396
- year = {2021},
397
- }</code></pre>
398
- </div>
399
- </section>
400
-
401
-
402
- <footer class="footer">
403
- <div class="container">
404
- <div class="content has-text-centered">
405
- <a class="icon-link" target="_blank"
406
- href="./static/videos/nerfies_paper.pdf">
407
- <i class="fas fa-file-pdf"></i>
408
- </a>
409
- <a class="icon-link" href="https://github.com/keunhong" target="_blank" class="external-link" disabled>
410
- <i class="fab fa-github"></i>
411
- </a>
412
- </div>
413
- <div class="columns is-centered">
414
- <div class="column is-8">
415
- <div class="content">
416
- <p>
417
- This website is licensed under a <a rel="license" target="_blank"
418
- href="http://creativecommons.org/licenses/by-sa/4.0/">Creative
419
- Commons Attribution-ShareAlike 4.0 International License</a>.
420
- </p>
421
- <p>
422
- This means you are free to borrow the <a target="_blank"
423
- href="https://github.com/nerfies/nerfies.github.io">source code</a> of this website,
424
- we just ask that you link back to this page in the footer.
425
- Please remember to remove the analytics code included in the header of the website which
426
- you do not want on your website.
427
- </p>
428
  </div>
429
  </div>
430
  </div>
431
- </div>
432
- </footer>
433
 
434
- </body>
435
- </html>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8" />
5
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
6
+ <title>DINOv3 Web</title>
7
+ <script src="https://cdn.tailwindcss.com"></script>
8
+ <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet" />
9
+ <style>
10
+ body {
11
+ font-family:
12
+ "Inter",
13
+ -apple-system,
14
+ BlinkMacSystemFont,
15
+ "Segoe UI",
16
+ Roboto,
17
+ Helvetica,
18
+ Arial,
19
+ sans-serif;
20
+ }
21
+ /* Custom styles for the range slider */
22
+ input[type="range"] {
23
+ -webkit-appearance: none;
24
+ appearance: none;
25
+ width: 100%;
26
+ height: 0.5rem;
27
+ background: #4a5568; /* gray-700 */
28
+ border-radius: 0.25rem;
29
+ outline: none;
30
+ opacity: 0.7;
31
+ transition: opacity 0.2s;
32
+ }
33
+ input[type="range"]:hover {
34
+ opacity: 1;
35
+ }
36
+ input[type="range"]::-webkit-slider-thumb {
37
+ -webkit-appearance: none;
38
+ appearance: none;
39
+ width: 1.25rem;
40
+ height: 1.25rem;
41
+ background: #90cdf4; /* blue-300 */
42
+ cursor: pointer;
43
+ border-radius: 50%;
44
+ }
45
+ input[type="range"]::-moz-range-thumb {
46
+ width: 1.25rem;
47
+ height: 1.25rem;
48
+ background: #90cdf4; /* blue-300 */
49
+ cursor: pointer;
50
+ border-radius: 50%;
51
+ }
52
+ /* Additional styles for the toggle switch */
53
+ #modeToggle:checked ~ .dot {
54
+ transform: translateX(1.5rem); /* 24px */
55
+ }
56
+ #modeToggle:checked ~ .block {
57
+ background-color: #3b82f6; /* blue-500 */
58
+ }
59
+ </style>
60
+ </head>
61
+ <body class="bg-gray-900 text-gray-300 flex flex-col items-center justify-center min-h-screen p-4 sm:p-6 lg:p-8">
62
+ <div
63
+ class="w-full max-w-3xl bg-gray-800/50 backdrop-blur-sm rounded-2xl shadow-2xl shadow-black/30 border border-gray-700 p-6 sm:p-8 text-center"
64
+ >
65
+ <h1
66
+ class="text-3xl sm:text-4xl font-bold text-transparent bg-clip-text bg-gradient-to-r from-blue-400 to-purple-500 mb-2"
67
+ >
68
+ DINOv3 Web
69
+ </h1>
70
+ <p class="text-gray-400 mb-8 max-w-xl mx-auto">
71
+ Visualize rich, dense image features 100% locally in your browser.
72
+ </p>
73
+
74
+ <!-- Model Selection -->
75
+ <div class="mb-6 p-4 bg-gray-900/50 rounded-xl border border-gray-700">
76
+ <label for="modelSelect" class="block text-sm font-medium text-gray-400 mb-2 text-left">Select Model:</label>
77
+ <select id="modelSelect" class="w-full bg-gray-800 border border-gray-600 rounded-lg px-3 py-2 text-gray-300 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent">
78
+ <option value="onnx-community/dinov3-vits16-pretrain-lvd1689m-ONNX">ViT-S/16 distilled (21M) - LVD-1689M</option>
79
+ <option value="onnx-community/dinov3-vits16plus-pretrain-lvd1689m-ONNX">ViT-S+/16 distilled (29M) - LVD-1689M</option>
80
+ <option value="onnx-community/dinov3-vitb16-pretrain-lvd1689m-ONNX">ViT-B/16 distilled (86M) - LVD-1689M</option>
81
+ <option value="onnx-community/dinov3-vitl16-pretrain-lvd1689m-ONNX">ViT-L/16 distilled (300M) - LVD-1689M</option>
82
+ <option value="onnx-community/dinov3-vith16plus-pretrain-lvd1689m-ONNX">ViT-H+/16 distilled (840M) - LVD-1689M</option>
83
+ <option value="onnx-community/dinov3-convnext-tiny-pretrain-lvd1689m-ONNX">ConvNeXt Tiny (29M) - LVD-1689M</option>
84
+ <option value="onnx-community/dinov3-convnext-small-pretrain-lvd1689m-ONNX">ConvNeXt Small (50M) - LVD-1689M</option>
85
+ <option value="onnx-community/dinov3-convnext-base-pretrain-lvd1689m-ONNX">ConvNeXt Base (89M) - LVD-1689M</option>
86
+ <option value="onnx-community/dinov3-convnext-large-pretrain-lvd1689m-ONNX">ConvNeXt Large (198M) - LVD-1689M</option>
87
+ <option value="onnx-community/dinov3-vitl16-pretrain-sat493m-ONNX">ViT-L/16 distilled (300M) - SAT-493M</option>
88
+ </select>
89
+ <p class="text-xs text-gray-500 mt-2 text-left">Choose a model based on your needs. Larger models provide better features but require more processing time.</p>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
90
  </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
 
92
+ <div class="space-y-6">
93
+ <div
94
+ id="dropZone"
95
+ class="relative flex flex-col items-center justify-center bg-gray-900/50 border-2 border-dashed border-gray-600 rounded-xl p-6 text-center group hover:border-blue-500 transition-colors duration-300"
96
+ >
97
+ <svg
98
+ class="w-12 h-12 mb-4 text-gray-500 group-hover:text-blue-500 transition-colors duration-300"
99
+ aria-hidden="true"
100
+ xmlns="http://www.w3.org/2000/svg"
101
+ fill="none"
102
+ viewBox="0 0 20 16"
103
+ >
104
+ <path
105
+ stroke="currentColor"
106
+ stroke-linecap="round"
107
+ stroke-linejoin="round"
108
+ stroke-width="1.5"
109
+ d="M13 13h3a3 3 0 0 0 0-6h-.025A5.56 5.56 0 0 0 16 6.5 5.5 5.5 0 0 0 5.207 5.021C5.137 5.017 5.071 5 5 5a4 4 0 0 0 0 8h2.167M10 15V6m0 0L8 8m2-2 2 2"
110
+ />
111
+ </svg>
112
+ <p class="font-semibold text-gray-300">Click to upload or drag & drop</p>
113
+ <p class="text-xs text-gray-500 mb-2">PNG, JPG, or other image formats</p>
114
+ <p class="text-sm text-gray-400">
115
+ Or
116
+ <button
117
+ id="exampleBtn"
118
+ class="relative z-10 text-blue-400 hover:text-blue-300 font-semibold underline bg-transparent border-none cursor-pointer p-0"
119
+ >
120
+ try an example</button
121
+ >.
122
  </p>
123
+ <label for="imageLoader" class="absolute inset-0 cursor-pointer z-0"></label>
124
+ <input type="file" id="imageLoader" accept="image/*" class="hidden" />
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
125
  </div>
 
 
 
126
 
127
+ <div class="bg-gray-900/50 p-4 rounded-xl border border-gray-700 space-y-4">
128
+ <div class="grid grid-cols-1 sm:grid-cols-2 gap-4 items-center">
129
+ <div class="flex items-center justify-center w-full space-x-3">
130
+ <label for="scaleSlider" class="text-sm font-medium text-gray-400 whitespace-nowrap">Scale:</label>
131
+ <input id="scaleSlider" type="range" min="0.25" max="4" step="0.25" value="1" class="w-full" />
132
+ <span id="scaleValue" class="text-sm font-medium text-gray-400 w-12 text-right">1.00x</span>
133
+ </div>
134
+ <div class="flex items-center justify-center space-x-3">
135
+ <span class="text-sm font-medium text-gray-400">Overlay</span>
136
+ <label for="modeToggle" class="flex items-center cursor-pointer">
137
+ <div class="relative">
138
+ <input type="checkbox" id="modeToggle" class="sr-only" />
139
+ <div class="block bg-gray-600 w-14 h-8 rounded-full"></div>
140
+ <div class="dot absolute left-1 top-1 bg-white w-6 h-6 rounded-full transition transform"></div>
141
+ </div>
142
+ </label>
143
+ <span class="text-sm font-medium text-gray-400">Heatmap</span>
 
 
 
 
 
 
 
144
  </div>
 
 
 
 
 
 
 
 
 
145
  </div>
146
  </div>
 
 
147
 
148
+ <div id="status" class="flex items-center justify-center w-full font-medium text-gray-400 h-6">
149
+ <svg
150
+ id="spinner"
151
+ class="animate-spin mr-3 h-5 w-5 text-blue-400 hidden"
152
+ xmlns="http://www.w3.org/2000/svg"
153
+ fill="none"
154
+ viewBox="0 0 24 24"
155
+ >
156
+ <circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
157
+ <path
158
+ class="opacity-75"
159
+ fill="currentColor"
160
+ d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
161
+ ></path>
162
+ </svg>
163
+ <span id="statusText"></span>
 
 
164
  </div>
 
165
 
166
+ <div
167
+ id="canvasContainer"
168
+ class="w-full bg-gray-900/50 rounded-lg border border-gray-700 shadow-inner overflow-hidden min-h-[250px] flex items-center justify-center p-2"
169
+ >
170
+ <canvas id="imageCanvas" class="hidden rounded-lg cursor-crosshair block max-w-full h-auto"></canvas>
171
+ <div id="canvasPlaceholder" class="text-gray-500">Your image will appear here</div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
172
  </div>
173
  </div>
174
  </div>
 
 
175
 
176
+ <script type="module">
177
+ import { pipeline, RawImage, matmul } from "https://cdn.jsdelivr.net/npm/@huggingface/[email protected]";
178
+ // --- 1. Configuration & Global Variables ---
179
+ let MODEL_ID = "onnx-community/dinov3-vits16-pretrain-lvd1689m-ONNX";
180
+ const EXAMPLE_IMAGE_URL =
181
+ "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/cats.png";
182
+ // DOM Elements
183
+ const imageLoader = document.getElementById("imageLoader");
184
+ const exampleBtn = document.getElementById("exampleBtn");
185
+ const imageCanvas = document.getElementById("imageCanvas");
186
+ const ctx = imageCanvas.getContext("2d");
187
+ const spinner = document.getElementById("spinner");
188
+ const statusText = document.getElementById("statusText");
189
+ const canvasContainer = document.getElementById("canvasContainer");
190
+ const canvasPlaceholder = document.getElementById("canvasPlaceholder");
191
+ const dropZone = document.getElementById("dropZone");
192
+ const modeToggle = document.getElementById("modeToggle");
193
+ const scaleSlider = document.getElementById("scaleSlider");
194
+ const scaleValue = document.getElementById("scaleValue");
195
+ const modelSelect = document.getElementById("modelSelect");
196
+ // Application State
197
+ let extractor = null;
198
+ let similarityScores = null;
199
+ let originalImage = null;
200
+ let currentImageUrl = null;
201
+ let patchSize = null;
202
+ let isOverlayMode = true;
203
+ let lastHoverData = null;
204
+ let imageScale = 1.0;
205
+ let animationFrameId = null;
206
+ let lastMouseEvent = null;
207
+ let maxPixels = null;
208
+ // --- 2. Core Application Logic ---
209
+ function updateStatus(text, isLoading = false) {
210
+ statusText.textContent = text;
211
+ spinner.style.display = isLoading ? "block" : "none";
212
+ }
213
+ async function initialize() {
214
+ // Reset state when switching models
215
+ extractor = null;
216
+ similarityScores = null;
217
+ lastHoverData = null;
218
+ currentImageUrl = null;
219
+ patchSize = null;
220
+
221
+ // Clear canvas
222
+ if (imageCanvas.style.display !== "none") {
223
+ imageCanvas.style.display = "none";
224
+ canvasPlaceholder.style.display = "block";
225
+ canvasPlaceholder.textContent = "Please select an image after model loads";
226
+ }
227
+
228
+ const isWebGpuSupported = !!navigator.gpu;
229
+ const isMobile = /Mobi|Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(
230
+ navigator.userAgent,
231
+ );
232
+ maxPixels = isMobile ? 1048576 : 2097152;
233
+ const device = isWebGpuSupported ? "webgpu" : "wasm";
234
+ const dtype = isWebGpuSupported ? "q4" : "q8";
235
+ let statusMessage = `Loading model ${MODEL_ID.split('/').pop()} (${device.toUpperCase()})`;
236
+ if (isMobile) statusMessage += ". Mobile Detected.";
237
+ updateStatus(statusMessage, true);
238
+ try {
239
+ extractor = await pipeline("image-feature-extraction", MODEL_ID, {
240
+ device,
241
+ dtype,
242
+ });
243
+ extractor.processor.image_processor.do_resize = false;
244
+ patchSize = extractor.model.config.patch_size;
245
+ updateStatus("Ready. Please select an image.");
246
+ } catch (error) {
247
+ updateStatus("Failed to load the model. Please refresh.");
248
+ console.error("Model loading error:", error);
249
+ }
250
+ imageLoader.addEventListener("change", handleImageUpload);
251
+ exampleBtn.addEventListener("click", handleExample);
252
+ imageCanvas.addEventListener("mousemove", handleMouseMove);
253
+ imageCanvas.addEventListener("mouseleave", clearHighlights);
254
+ imageCanvas.addEventListener("touchmove", handleTouchMove);
255
+ imageCanvas.addEventListener("touchend", clearHighlights);
256
+ dropZone.addEventListener("dragover", handleDragOver);
257
+ dropZone.addEventListener("dragleave", handleDragLeave);
258
+ dropZone.addEventListener("drop", handleDrop);
259
+ modeToggle.addEventListener("change", handleModeChange);
260
+ scaleSlider.addEventListener("input", handleSliderInput);
261
+ scaleSlider.addEventListener("change", handleSliderChange);
262
+ modelSelect.addEventListener("change", handleModelChange);
263
+ }
264
+ async function handleExample() {
265
+ updateStatus("Loading example image...", true);
266
+ try {
267
+ const response = await fetch(EXAMPLE_IMAGE_URL);
268
+ const blob = await response.blob();
269
+ loadImageOntoCanvas(URL.createObjectURL(blob));
270
+ } catch (error) {
271
+ updateStatus("Failed to load example image.");
272
+ console.error("Example load error:", error);
273
+ }
274
+ }
275
+ function handleImageUpload(event) {
276
+ const file = event.target.files[0];
277
+ if (file) loadImageOntoCanvas(URL.createObjectURL(file));
278
+ }
279
+ function handleDragOver(event) {
280
+ event.preventDefault();
281
+ dropZone.classList.add("border-blue-500", "bg-gray-800");
282
+ }
283
+ function handleDragLeave(event) {
284
+ event.preventDefault();
285
+ dropZone.classList.remove("border-blue-500", "bg-gray-800");
286
+ }
287
+ function handleDrop(event) {
288
+ event.preventDefault();
289
+ dropZone.classList.remove("border-blue-500", "bg-gray-800");
290
+ const file = event.dataTransfer.files[0];
291
+ if (file && file.type.startsWith("image/")) {
292
+ // The label covers the whole dropzone, so we need to make sure the button can be clicked.
293
+ if (event.target.id === "exampleBtn") return;
294
+ loadImageOntoCanvas(URL.createObjectURL(file));
295
+ } else {
296
+ updateStatus("Please drop an image file.");
297
+ }
298
+ }
299
+ function handleModeChange(event) {
300
+ isOverlayMode = !event.target.checked;
301
+ if (lastHoverData) {
302
+ drawHighlights(lastHoverData.queryIndex, lastHoverData.allPatches);
303
+ } else {
304
+ clearHighlights();
305
+ }
306
+ }
307
+ function handleSliderInput(event) {
308
+ imageScale = parseFloat(event.target.value);
309
+ scaleValue.textContent = `${imageScale.toFixed(2)}x`;
310
+ }
311
+ function handleSliderChange() {
312
+ if (currentImageUrl) {
313
+ loadImageOntoCanvas(currentImageUrl);
314
+ }
315
+ }
316
+
317
+ async function handleModelChange() {
318
+ const newModelId = modelSelect.value;
319
+ if (newModelId !== MODEL_ID) {
320
+ MODEL_ID = newModelId;
321
+ await initialize();
322
+ }
323
+ }
324
+
325
+ function loadImageOntoCanvas(imageUrl) {
326
+ currentImageUrl = imageUrl;
327
+ originalImage = new Image();
328
+ originalImage.onload = async () => {
329
+ if (!patchSize) {
330
+ updateStatus("Error: Model not ready, patch size is unknown.");
331
+ return;
332
+ }
333
+ canvasPlaceholder.style.display = "none";
334
+ imageCanvas.style.display = "block";
335
+ let newWidth = originalImage.naturalWidth * imageScale;
336
+ let newHeight = originalImage.naturalHeight * imageScale;
337
+ const numPixels = newWidth * newHeight;
338
+ if (numPixels > maxPixels) {
339
+ const scaleRatio = Math.sqrt(maxPixels / numPixels);
340
+ newWidth *= scaleRatio;
341
+ newHeight *= scaleRatio;
342
+ }
343
+ const croppedWidth = Math.floor(newWidth / patchSize) * patchSize;
344
+ const croppedHeight = Math.floor(newHeight / patchSize) * patchSize;
345
+ if (croppedWidth < patchSize || croppedHeight < patchSize) {
346
+ updateStatus("Scaled image is too small to process.");
347
+ imageCanvas.style.display = "none";
348
+ canvasPlaceholder.style.display = "block";
349
+ canvasPlaceholder.textContent = "Scaled image is too small.";
350
+ return;
351
+ }
352
+ imageCanvas.width = croppedWidth;
353
+ imageCanvas.height = croppedHeight;
354
+ ctx.drawImage(originalImage, 0, 0, croppedWidth, croppedHeight);
355
+ await processImage();
356
+ setTimeout(() => {
357
+ canvasContainer.scrollIntoView({ behavior: "smooth", block: "center" });
358
+ }, 100);
359
+ };
360
+ originalImage.onerror = () => {
361
+ updateStatus("Failed to load the selected image.");
362
+ canvasPlaceholder.style.display = "block";
363
+ imageCanvas.style.display = "none";
364
+ };
365
+ originalImage.src = imageUrl;
366
+ }
367
+ async function processImage() {
368
+ if (!extractor) return;
369
+ updateStatus("Analyzing image... 🧠", true);
370
+ similarityScores = null;
371
+ lastHoverData = null;
372
+ try {
373
+ const imageData = await RawImage.fromCanvas(imageCanvas);
374
+ const features = await extractor(imageData, { pooling: "none" });
375
+ const numRegisterTokens = extractor.model.config.num_register_tokens ?? 0;
376
+ const startIndex = 1 + numRegisterTokens;
377
+ const patchFeatures = features.slice(null, [startIndex, null]);
378
+ const normalizedFeatures = patchFeatures.normalize(2, -1);
379
+ const scores = await matmul(normalizedFeatures, normalizedFeatures.permute(0, 2, 1));
380
+ similarityScores = (await scores.tolist())[0];
381
+ updateStatus(
382
+ `Image processed (${imageCanvas.width}x${imageCanvas.height}). Move over the image to explore features. ✨`,
383
+ );
384
+ } catch (error) {
385
+ updateStatus("An error occurred during image processing.");
386
+ console.error("Processing error:", error);
387
+ }
388
+ }
389
+ function handleTouchMove(event) {
390
+ event.preventDefault();
391
+ if (event.touches.length > 0) {
392
+ handleMouseMove(event.touches[0]);
393
+ }
394
+ }
395
+ function handleMouseMove(event) {
396
+ lastMouseEvent = event;
397
+ if (!animationFrameId) {
398
+ animationFrameId = requestAnimationFrame(drawLoop);
399
+ }
400
+ }
401
+ function drawLoop() {
402
+ if (!lastMouseEvent || !similarityScores || !originalImage) {
403
+ animationFrameId = null;
404
+ return;
405
+ }
406
+ const event = lastMouseEvent;
407
+ const rect = imageCanvas.getBoundingClientRect();
408
+ const scaleX = imageCanvas.width / rect.width;
409
+ const scaleY = imageCanvas.height / rect.height;
410
+ const x = (event.clientX - rect.left) * scaleX;
411
+ const y = (event.clientY - rect.top) * scaleY;
412
+ if (x < 0 || x >= imageCanvas.width || y < 0 || y >= imageCanvas.height) {
413
+ animationFrameId = null;
414
+ return;
415
+ }
416
+ const patchesPerRow = imageCanvas.width / patchSize;
417
+ const patchX = Math.floor(x / patchSize);
418
+ const patchY = Math.floor(y / patchSize);
419
+ const queryPatchIndex = patchY * patchesPerRow + patchX;
420
+ if (queryPatchIndex < 0 || queryPatchIndex >= similarityScores.length || !similarityScores[queryPatchIndex]) {
421
+ animationFrameId = null;
422
+ return;
423
+ }
424
+ const allPatches = Array.from(similarityScores[queryPatchIndex]).map((score, index) => ({ score, index }));
425
+ lastHoverData = { queryIndex: queryPatchIndex, allPatches };
426
+ drawHighlights(queryPatchIndex, allPatches);
427
+ animationFrameId = null;
428
+ }
429
+ const INFERNO_COLORMAP = [
430
+ [0.0, [0, 0, 4]],
431
+ [0.1, [39, 12, 69]],
432
+ [0.2, [84, 15, 104]],
433
+ [0.3, [128, 31, 103]],
434
+ [0.4, [170, 48, 88]],
435
+ [0.5, [209, 70, 68]],
436
+ [0.6, [240, 97, 47]],
437
+ [0.7, [253, 138, 28]],
438
+ [0.8, [252, 185, 26]],
439
+ [0.9, [240, 231, 56]],
440
+ [1.0, [252, 255, 160]],
441
+ ];
442
+ function getInfernoColor(t) {
443
+ for (let i = 1; i < INFERNO_COLORMAP.length; i++) {
444
+ const [t_prev, c_prev] = INFERNO_COLORMAP[i - 1];
445
+ const [t_curr, c_curr] = INFERNO_COLORMAP[i];
446
+ if (t <= t_curr) {
447
+ const t_interp = (t - t_prev) / (t_curr - t_prev);
448
+ const r = c_prev[0] + t_interp * (c_curr[0] - c_prev[0]);
449
+ const g = c_prev[1] + t_interp * (c_curr[1] - c_prev[1]);
450
+ const b = c_prev[2] + t_interp * (c_curr[2] - c_prev[2]);
451
+ return `rgb(${Math.round(r)}, ${Math.round(g)}, ${Math.round(b)})`;
452
+ }
453
+ }
454
+ return `rgb(${INFERNO_COLORMAP[INFERNO_COLORMAP.length - 1][1].join(",")})`;
455
+ }
456
+ function drawHighlights(queryIndex, allPatches) {
457
+ const patchesPerRow = imageCanvas.width / patchSize;
458
+ if (isOverlayMode) {
459
+ ctx.drawImage(originalImage, 0, 0, imageCanvas.width, imageCanvas.height);
460
+ ctx.fillStyle = "rgba(0, 0, 0, 0.6)";
461
+ ctx.fillRect(0, 0, imageCanvas.width, imageCanvas.height);
462
+ } else {
463
+ ctx.fillStyle = getInfernoColor(0);
464
+ ctx.fillRect(0, 0, imageCanvas.width, imageCanvas.height);
465
+ }
466
+ if (allPatches.length > 0) {
467
+ const scores = allPatches.map((p) => p.score);
468
+ const minScore = Math.min(...scores);
469
+ const maxScore = Math.max(...scores);
470
+ const scoreRange = maxScore - minScore;
471
+ for (const patch of allPatches) {
472
+ if (patch.index === queryIndex) continue;
473
+ const normalizedScore = scoreRange > 0.0001 ? (patch.score - minScore) / scoreRange : 1;
474
+ const patchY = Math.floor(patch.index / patchesPerRow);
475
+ const patchX = patch.index % patchesPerRow;
476
+ if (isOverlayMode) {
477
+ const brightness = Math.pow(normalizedScore, 2) * 0.8;
478
+ ctx.fillStyle = `rgba(255, 255, 255, ${brightness})`;
479
+ } else {
480
+ ctx.fillStyle = getInfernoColor(normalizedScore);
481
+ }
482
+ ctx.fillRect(patchX * patchSize, patchY * patchSize, patchSize, patchSize);
483
+ }
484
+ }
485
+ const queryY = Math.floor(queryIndex / patchesPerRow);
486
+ const queryX = queryIndex % patchesPerRow;
487
+ ctx.strokeStyle = isOverlayMode ? "rgba(129, 188, 255, 0.9)" : "cyan";
488
+ ctx.lineWidth = 2;
489
+ ctx.strokeRect(queryX * patchSize, queryY * patchSize, patchSize, patchSize);
490
+ }
491
+ function clearHighlights() {
492
+ if (animationFrameId) {
493
+ cancelAnimationFrame(animationFrameId);
494
+ animationFrameId = null;
495
+ }
496
+ lastMouseEvent = null;
497
+ lastHoverData = null;
498
+ if (originalImage) {
499
+ ctx.drawImage(originalImage, 0, 0, imageCanvas.width, imageCanvas.height);
500
+ }
501
+ }
502
+ initialize();
503
+ </script>
504
+ </body>
505
+ </html>