Spaces:
Running
Running
Commit
·
c9731fb
1
Parent(s):
bdef08e
added gaussian splatting viewer and scenes
Browse files- viewer/src/lib/data/models.json +8 -8
- viewer/src/lib/data/scenes.json +100 -10
- viewer/src/routes/viewer/[slug]/SplatViewer.ts +37 -5
- viewer/static/thumbnails/3dgs-bicycle.png +0 -0
- viewer/static/thumbnails/3dgs-bonsai.png +0 -0
- viewer/static/thumbnails/3dgs-counter.png +0 -0
- viewer/static/thumbnails/3dgs-flowers.png +0 -0
- viewer/static/thumbnails/3dgs-garden.png +0 -0
- viewer/static/thumbnails/3dgs-kitchen.png +0 -0
- viewer/static/thumbnails/3dgs-playroom.png +0 -0
- viewer/static/thumbnails/3dgs-room.png +0 -0
- viewer/static/thumbnails/3dgs-stump.png +0 -0
- viewer/static/thumbnails/3dgs-treehill.png +0 -0
viewer/src/lib/data/models.json
CHANGED
@@ -1,4 +1,12 @@
|
|
1 |
[
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2 |
{
|
3 |
"slug": "sync-dreamer",
|
4 |
"title": "SyncDreamer",
|
@@ -21,13 +29,5 @@
|
|
21 |
"project": "https://mv-dream.github.io/",
|
22 |
"code": "https://github.com/bytedance/MVDream",
|
23 |
"spaces": []
|
24 |
-
},
|
25 |
-
{
|
26 |
-
"slug": "3dgs",
|
27 |
-
"title": "3D Gaussian Splatting",
|
28 |
-
"paper": "https://huggingface.co/papers/2308.04079",
|
29 |
-
"project": "https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/",
|
30 |
-
"code": "https://github.com/graphdeco-inria/gaussian-splatting",
|
31 |
-
"spaces": []
|
32 |
}
|
33 |
]
|
|
|
1 |
[
|
2 |
+
{
|
3 |
+
"slug": "3dgs",
|
4 |
+
"title": "3D Gaussian Splatting",
|
5 |
+
"paper": "https://huggingface.co/papers/2308.04079",
|
6 |
+
"project": "https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/",
|
7 |
+
"code": "https://github.com/graphdeco-inria/gaussian-splatting",
|
8 |
+
"spaces": []
|
9 |
+
},
|
10 |
{
|
11 |
"slug": "sync-dreamer",
|
12 |
"title": "SyncDreamer",
|
|
|
29 |
"project": "https://mv-dream.github.io/",
|
30 |
"code": "https://github.com/bytedance/MVDream",
|
31 |
"spaces": []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
32 |
}
|
33 |
]
|
viewer/src/lib/data/scenes.json
CHANGED
@@ -1,4 +1,104 @@
|
|
1 |
[
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2 |
{
|
3 |
"slug": "sync-dreamer-armor",
|
4 |
"model": "sync-dreamer",
|
@@ -171,15 +271,5 @@
|
|
171 |
"Multi-view Diffusion",
|
172 |
"Marching Cubes"
|
173 |
]
|
174 |
-
},
|
175 |
-
{
|
176 |
-
"slug": "3dgs-stump",
|
177 |
-
"model": "3dgs",
|
178 |
-
"title": "Stump",
|
179 |
-
"type": "splat",
|
180 |
-
"url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/stump/stump-7k.splat",
|
181 |
-
"pipeline": [
|
182 |
-
"Gaussian Splatting"
|
183 |
-
]
|
184 |
}
|
185 |
]
|
|
|
1 |
[
|
2 |
+
{
|
3 |
+
"slug": "3dgs-bicycle",
|
4 |
+
"model": "3dgs",
|
5 |
+
"title": "Bicycle",
|
6 |
+
"type": "splat",
|
7 |
+
"url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bicycle/bicycle-7k.splat",
|
8 |
+
"pipeline": [
|
9 |
+
"Gaussian Splatting"
|
10 |
+
]
|
11 |
+
},
|
12 |
+
{
|
13 |
+
"slug": "3dgs-bonsai",
|
14 |
+
"model": "3dgs",
|
15 |
+
"title": "Bonsai",
|
16 |
+
"type": "splat",
|
17 |
+
"url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bonsai/bonsai-7k.splat",
|
18 |
+
"pipeline": [
|
19 |
+
"Gaussian Splatting"
|
20 |
+
]
|
21 |
+
},
|
22 |
+
{
|
23 |
+
"slug": "3dgs-counter",
|
24 |
+
"model": "3dgs",
|
25 |
+
"title": "Counter",
|
26 |
+
"type": "splat",
|
27 |
+
"url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/counter/counter-7k.splat",
|
28 |
+
"pipeline": [
|
29 |
+
"Gaussian Splatting"
|
30 |
+
]
|
31 |
+
},
|
32 |
+
{
|
33 |
+
"slug": "3dgs-flowers",
|
34 |
+
"model": "3dgs",
|
35 |
+
"title": "Flowers",
|
36 |
+
"type": "splat",
|
37 |
+
"url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/flowers/flowers-7k.splat",
|
38 |
+
"pipeline": [
|
39 |
+
"Gaussian Splatting"
|
40 |
+
]
|
41 |
+
},
|
42 |
+
{
|
43 |
+
"slug": "3dgs-garden",
|
44 |
+
"model": "3dgs",
|
45 |
+
"title": "Garden",
|
46 |
+
"type": "splat",
|
47 |
+
"url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/garden/garden-7k.splat",
|
48 |
+
"pipeline": [
|
49 |
+
"Gaussian Splatting"
|
50 |
+
]
|
51 |
+
},
|
52 |
+
{
|
53 |
+
"slug": "3dgs-kitchen",
|
54 |
+
"model": "3dgs",
|
55 |
+
"title": "Kitchen",
|
56 |
+
"type": "splat",
|
57 |
+
"url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/kitchen/kitchen-7k.splat",
|
58 |
+
"pipeline": [
|
59 |
+
"Gaussian Splatting"
|
60 |
+
]
|
61 |
+
},
|
62 |
+
{
|
63 |
+
"slug": "3dgs-playroom",
|
64 |
+
"model": "3dgs",
|
65 |
+
"title": "Playroom",
|
66 |
+
"type": "splat",
|
67 |
+
"url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/playroom/playroom-7k.splat",
|
68 |
+
"pipeline": [
|
69 |
+
"Gaussian Splatting"
|
70 |
+
]
|
71 |
+
},
|
72 |
+
{
|
73 |
+
"slug": "3dgs-room",
|
74 |
+
"model": "3dgs",
|
75 |
+
"title": "Room",
|
76 |
+
"type": "splat",
|
77 |
+
"url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/room/room-7k.splat",
|
78 |
+
"pipeline": [
|
79 |
+
"Gaussian Splatting"
|
80 |
+
]
|
81 |
+
},
|
82 |
+
{
|
83 |
+
"slug": "3dgs-stump",
|
84 |
+
"model": "3dgs",
|
85 |
+
"title": "Stump",
|
86 |
+
"type": "splat",
|
87 |
+
"url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/stump/stump-7k.splat",
|
88 |
+
"pipeline": [
|
89 |
+
"Gaussian Splatting"
|
90 |
+
]
|
91 |
+
},
|
92 |
+
{
|
93 |
+
"slug": "3dgs-treehill",
|
94 |
+
"model": "3dgs",
|
95 |
+
"title": "Treehill",
|
96 |
+
"type": "splat",
|
97 |
+
"url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/treehill/treehill-7k.splat",
|
98 |
+
"pipeline": [
|
99 |
+
"Gaussian Splatting"
|
100 |
+
]
|
101 |
+
},
|
102 |
{
|
103 |
"slug": "sync-dreamer-armor",
|
104 |
"model": "sync-dreamer",
|
|
|
271 |
"Multi-view Diffusion",
|
272 |
"Marching Cubes"
|
273 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
274 |
}
|
275 |
]
|
viewer/src/routes/viewer/[slug]/SplatViewer.ts
CHANGED
@@ -548,6 +548,7 @@ export class SplatViewer implements IViewer {
|
|
548 |
vertexCount: number;
|
549 |
worker: Worker;
|
550 |
|
|
|
551 |
vertexShader: WebGLShader;
|
552 |
fragmentShader: WebGLShader;
|
553 |
program: WebGLProgram;
|
@@ -592,7 +593,7 @@ export class SplatViewer implements IViewer {
|
|
592 |
this.canvas.height = innerHeight;
|
593 |
this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
|
594 |
|
595 |
-
|
596 |
this.camera.fx,
|
597 |
this.camera.fy,
|
598 |
this.canvas.width,
|
@@ -643,7 +644,7 @@ export class SplatViewer implements IViewer {
|
|
643 |
|
644 |
// projection
|
645 |
const u_projection = this.gl.getUniformLocation(this.program, "projection");
|
646 |
-
this.gl.uniformMatrix4fv(u_projection, false, projectionMatrix);
|
647 |
|
648 |
// viewport
|
649 |
const u_viewport = this.gl.getUniformLocation(this.program, "viewport");
|
@@ -736,7 +737,7 @@ export class SplatViewer implements IViewer {
|
|
736 |
this.camera.update();
|
737 |
viewMatrix = getViewMatrix(this.camera);
|
738 |
|
739 |
-
const viewProj = multiply4(projectionMatrix, viewMatrix);
|
740 |
this.worker.postMessage({ view: viewProj });
|
741 |
|
742 |
if (vertexCount > 0) {
|
@@ -888,6 +889,20 @@ export class SplatViewer implements IViewer {
|
|
888 |
this.canvas.width = innerWidth;
|
889 |
this.canvas.height = innerHeight;
|
890 |
this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
891 |
}
|
892 |
|
893 |
dispose() {
|
@@ -940,8 +955,25 @@ export class SplatViewer implements IViewer {
|
|
940 |
window.removeEventListener("resize", this.handleResize);
|
941 |
}
|
942 |
|
943 |
-
async capture() {
|
944 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
945 |
}
|
946 |
|
947 |
getStats(): { name: string, value: any }[] {
|
|
|
548 |
vertexCount: number;
|
549 |
worker: Worker;
|
550 |
|
551 |
+
projectionMatrix: number[];
|
552 |
vertexShader: WebGLShader;
|
553 |
fragmentShader: WebGLShader;
|
554 |
program: WebGLProgram;
|
|
|
593 |
this.canvas.height = innerHeight;
|
594 |
this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
|
595 |
|
596 |
+
this.projectionMatrix = getProjectionMatrix(
|
597 |
this.camera.fx,
|
598 |
this.camera.fy,
|
599 |
this.canvas.width,
|
|
|
644 |
|
645 |
// projection
|
646 |
const u_projection = this.gl.getUniformLocation(this.program, "projection");
|
647 |
+
this.gl.uniformMatrix4fv(u_projection, false, this.projectionMatrix);
|
648 |
|
649 |
// viewport
|
650 |
const u_viewport = this.gl.getUniformLocation(this.program, "viewport");
|
|
|
737 |
this.camera.update();
|
738 |
viewMatrix = getViewMatrix(this.camera);
|
739 |
|
740 |
+
const viewProj = multiply4(this.projectionMatrix, viewMatrix);
|
741 |
this.worker.postMessage({ view: viewProj });
|
742 |
|
743 |
if (vertexCount > 0) {
|
|
|
889 |
this.canvas.width = innerWidth;
|
890 |
this.canvas.height = innerHeight;
|
891 |
this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
|
892 |
+
|
893 |
+
const aspectRatio = this.canvas.width / this.canvas.height;
|
894 |
+
this.projectionMatrix = getProjectionMatrix(
|
895 |
+
this.camera.fx,
|
896 |
+
this.camera.fy,
|
897 |
+
this.canvas.width,
|
898 |
+
this.canvas.height,
|
899 |
+
);
|
900 |
+
|
901 |
+
const u_projection = this.gl.getUniformLocation(this.program, "projection");
|
902 |
+
this.gl.uniformMatrix4fv(u_projection, false, this.projectionMatrix);
|
903 |
+
|
904 |
+
const u_viewport = this.gl.getUniformLocation(this.program, "viewport");
|
905 |
+
this.gl.uniform2fv(u_viewport, new Float32Array([this.canvas.width, this.canvas.height]));
|
906 |
}
|
907 |
|
908 |
dispose() {
|
|
|
955 |
window.removeEventListener("resize", this.handleResize);
|
956 |
}
|
957 |
|
958 |
+
async capture(): Promise<string | null> {
|
959 |
+
return new Promise((resolve) => {
|
960 |
+
requestAnimationFrame(() => {
|
961 |
+
const offscreenCanvas = document.createElement("canvas");
|
962 |
+
offscreenCanvas.width = 512;
|
963 |
+
offscreenCanvas.height = 512;
|
964 |
+
const offscreenContext = offscreenCanvas.getContext("2d")!;
|
965 |
+
|
966 |
+
const x = (this.canvas.width - offscreenCanvas.width) / 2;
|
967 |
+
const y = (this.canvas.height - offscreenCanvas.height) / 2;
|
968 |
+
|
969 |
+
offscreenContext.drawImage(this.canvas, x, y, offscreenCanvas.width, offscreenCanvas.height, 0, 0, offscreenCanvas.width, offscreenCanvas.height);
|
970 |
+
const dataURL = offscreenCanvas.toDataURL("image/png");
|
971 |
+
offscreenCanvas.remove();
|
972 |
+
|
973 |
+
resolve(dataURL);
|
974 |
+
});
|
975 |
+
});
|
976 |
+
|
977 |
}
|
978 |
|
979 |
getStats(): { name: string, value: any }[] {
|
viewer/static/thumbnails/3dgs-bicycle.png
ADDED
viewer/static/thumbnails/3dgs-bonsai.png
ADDED
viewer/static/thumbnails/3dgs-counter.png
ADDED
viewer/static/thumbnails/3dgs-flowers.png
ADDED
viewer/static/thumbnails/3dgs-garden.png
ADDED
viewer/static/thumbnails/3dgs-kitchen.png
ADDED
viewer/static/thumbnails/3dgs-playroom.png
ADDED
viewer/static/thumbnails/3dgs-room.png
ADDED
viewer/static/thumbnails/3dgs-stump.png
CHANGED
viewer/static/thumbnails/3dgs-treehill.png
ADDED