dylanebert HF staff commited on
Commit
c9731fb
·
1 Parent(s): bdef08e

added gaussian splatting viewer and scenes

Browse files
viewer/src/lib/data/models.json CHANGED
@@ -1,4 +1,12 @@
1
  [
 
 
 
 
 
 
 
 
2
  {
3
  "slug": "sync-dreamer",
4
  "title": "SyncDreamer",
@@ -21,13 +29,5 @@
21
  "project": "https://mv-dream.github.io/",
22
  "code": "https://github.com/bytedance/MVDream",
23
  "spaces": []
24
- },
25
- {
26
- "slug": "3dgs",
27
- "title": "3D Gaussian Splatting",
28
- "paper": "https://huggingface.co/papers/2308.04079",
29
- "project": "https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/",
30
- "code": "https://github.com/graphdeco-inria/gaussian-splatting",
31
- "spaces": []
32
  }
33
  ]
 
1
  [
2
+ {
3
+ "slug": "3dgs",
4
+ "title": "3D Gaussian Splatting",
5
+ "paper": "https://huggingface.co/papers/2308.04079",
6
+ "project": "https://repo-sam.inria.fr/fungraph/3d-gaussian-splatting/",
7
+ "code": "https://github.com/graphdeco-inria/gaussian-splatting",
8
+ "spaces": []
9
+ },
10
  {
11
  "slug": "sync-dreamer",
12
  "title": "SyncDreamer",
 
29
  "project": "https://mv-dream.github.io/",
30
  "code": "https://github.com/bytedance/MVDream",
31
  "spaces": []
 
 
 
 
 
 
 
 
32
  }
33
  ]
viewer/src/lib/data/scenes.json CHANGED
@@ -1,4 +1,104 @@
1
  [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  {
3
  "slug": "sync-dreamer-armor",
4
  "model": "sync-dreamer",
@@ -171,15 +271,5 @@
171
  "Multi-view Diffusion",
172
  "Marching Cubes"
173
  ]
174
- },
175
- {
176
- "slug": "3dgs-stump",
177
- "model": "3dgs",
178
- "title": "Stump",
179
- "type": "splat",
180
- "url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/stump/stump-7k.splat",
181
- "pipeline": [
182
- "Gaussian Splatting"
183
- ]
184
  }
185
  ]
 
1
  [
2
+ {
3
+ "slug": "3dgs-bicycle",
4
+ "model": "3dgs",
5
+ "title": "Bicycle",
6
+ "type": "splat",
7
+ "url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bicycle/bicycle-7k.splat",
8
+ "pipeline": [
9
+ "Gaussian Splatting"
10
+ ]
11
+ },
12
+ {
13
+ "slug": "3dgs-bonsai",
14
+ "model": "3dgs",
15
+ "title": "Bonsai",
16
+ "type": "splat",
17
+ "url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/bonsai/bonsai-7k.splat",
18
+ "pipeline": [
19
+ "Gaussian Splatting"
20
+ ]
21
+ },
22
+ {
23
+ "slug": "3dgs-counter",
24
+ "model": "3dgs",
25
+ "title": "Counter",
26
+ "type": "splat",
27
+ "url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/counter/counter-7k.splat",
28
+ "pipeline": [
29
+ "Gaussian Splatting"
30
+ ]
31
+ },
32
+ {
33
+ "slug": "3dgs-flowers",
34
+ "model": "3dgs",
35
+ "title": "Flowers",
36
+ "type": "splat",
37
+ "url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/flowers/flowers-7k.splat",
38
+ "pipeline": [
39
+ "Gaussian Splatting"
40
+ ]
41
+ },
42
+ {
43
+ "slug": "3dgs-garden",
44
+ "model": "3dgs",
45
+ "title": "Garden",
46
+ "type": "splat",
47
+ "url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/garden/garden-7k.splat",
48
+ "pipeline": [
49
+ "Gaussian Splatting"
50
+ ]
51
+ },
52
+ {
53
+ "slug": "3dgs-kitchen",
54
+ "model": "3dgs",
55
+ "title": "Kitchen",
56
+ "type": "splat",
57
+ "url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/kitchen/kitchen-7k.splat",
58
+ "pipeline": [
59
+ "Gaussian Splatting"
60
+ ]
61
+ },
62
+ {
63
+ "slug": "3dgs-playroom",
64
+ "model": "3dgs",
65
+ "title": "Playroom",
66
+ "type": "splat",
67
+ "url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/playroom/playroom-7k.splat",
68
+ "pipeline": [
69
+ "Gaussian Splatting"
70
+ ]
71
+ },
72
+ {
73
+ "slug": "3dgs-room",
74
+ "model": "3dgs",
75
+ "title": "Room",
76
+ "type": "splat",
77
+ "url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/room/room-7k.splat",
78
+ "pipeline": [
79
+ "Gaussian Splatting"
80
+ ]
81
+ },
82
+ {
83
+ "slug": "3dgs-stump",
84
+ "model": "3dgs",
85
+ "title": "Stump",
86
+ "type": "splat",
87
+ "url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/stump/stump-7k.splat",
88
+ "pipeline": [
89
+ "Gaussian Splatting"
90
+ ]
91
+ },
92
+ {
93
+ "slug": "3dgs-treehill",
94
+ "model": "3dgs",
95
+ "title": "Treehill",
96
+ "type": "splat",
97
+ "url": "https://huggingface.co/datasets/dylanebert/3dgs/resolve/main/treehill/treehill-7k.splat",
98
+ "pipeline": [
99
+ "Gaussian Splatting"
100
+ ]
101
+ },
102
  {
103
  "slug": "sync-dreamer-armor",
104
  "model": "sync-dreamer",
 
271
  "Multi-view Diffusion",
272
  "Marching Cubes"
273
  ]
 
 
 
 
 
 
 
 
 
 
274
  }
275
  ]
viewer/src/routes/viewer/[slug]/SplatViewer.ts CHANGED
@@ -548,6 +548,7 @@ export class SplatViewer implements IViewer {
548
  vertexCount: number;
549
  worker: Worker;
550
 
 
551
  vertexShader: WebGLShader;
552
  fragmentShader: WebGLShader;
553
  program: WebGLProgram;
@@ -592,7 +593,7 @@ export class SplatViewer implements IViewer {
592
  this.canvas.height = innerHeight;
593
  this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
594
 
595
- let projectionMatrix = getProjectionMatrix(
596
  this.camera.fx,
597
  this.camera.fy,
598
  this.canvas.width,
@@ -643,7 +644,7 @@ export class SplatViewer implements IViewer {
643
 
644
  // projection
645
  const u_projection = this.gl.getUniformLocation(this.program, "projection");
646
- this.gl.uniformMatrix4fv(u_projection, false, projectionMatrix);
647
 
648
  // viewport
649
  const u_viewport = this.gl.getUniformLocation(this.program, "viewport");
@@ -736,7 +737,7 @@ export class SplatViewer implements IViewer {
736
  this.camera.update();
737
  viewMatrix = getViewMatrix(this.camera);
738
 
739
- const viewProj = multiply4(projectionMatrix, viewMatrix);
740
  this.worker.postMessage({ view: viewProj });
741
 
742
  if (vertexCount > 0) {
@@ -888,6 +889,20 @@ export class SplatViewer implements IViewer {
888
  this.canvas.width = innerWidth;
889
  this.canvas.height = innerHeight;
890
  this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
891
  }
892
 
893
  dispose() {
@@ -940,8 +955,25 @@ export class SplatViewer implements IViewer {
940
  window.removeEventListener("resize", this.handleResize);
941
  }
942
 
943
- async capture() {
944
- return null;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
945
  }
946
 
947
  getStats(): { name: string, value: any }[] {
 
548
  vertexCount: number;
549
  worker: Worker;
550
 
551
+ projectionMatrix: number[];
552
  vertexShader: WebGLShader;
553
  fragmentShader: WebGLShader;
554
  program: WebGLProgram;
 
593
  this.canvas.height = innerHeight;
594
  this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
595
 
596
+ this.projectionMatrix = getProjectionMatrix(
597
  this.camera.fx,
598
  this.camera.fy,
599
  this.canvas.width,
 
644
 
645
  // projection
646
  const u_projection = this.gl.getUniformLocation(this.program, "projection");
647
+ this.gl.uniformMatrix4fv(u_projection, false, this.projectionMatrix);
648
 
649
  // viewport
650
  const u_viewport = this.gl.getUniformLocation(this.program, "viewport");
 
737
  this.camera.update();
738
  viewMatrix = getViewMatrix(this.camera);
739
 
740
+ const viewProj = multiply4(this.projectionMatrix, viewMatrix);
741
  this.worker.postMessage({ view: viewProj });
742
 
743
  if (vertexCount > 0) {
 
889
  this.canvas.width = innerWidth;
890
  this.canvas.height = innerHeight;
891
  this.gl.viewport(0, 0, this.canvas.width, this.canvas.height);
892
+
893
+ const aspectRatio = this.canvas.width / this.canvas.height;
894
+ this.projectionMatrix = getProjectionMatrix(
895
+ this.camera.fx,
896
+ this.camera.fy,
897
+ this.canvas.width,
898
+ this.canvas.height,
899
+ );
900
+
901
+ const u_projection = this.gl.getUniformLocation(this.program, "projection");
902
+ this.gl.uniformMatrix4fv(u_projection, false, this.projectionMatrix);
903
+
904
+ const u_viewport = this.gl.getUniformLocation(this.program, "viewport");
905
+ this.gl.uniform2fv(u_viewport, new Float32Array([this.canvas.width, this.canvas.height]));
906
  }
907
 
908
  dispose() {
 
955
  window.removeEventListener("resize", this.handleResize);
956
  }
957
 
958
+ async capture(): Promise<string | null> {
959
+ return new Promise((resolve) => {
960
+ requestAnimationFrame(() => {
961
+ const offscreenCanvas = document.createElement("canvas");
962
+ offscreenCanvas.width = 512;
963
+ offscreenCanvas.height = 512;
964
+ const offscreenContext = offscreenCanvas.getContext("2d")!;
965
+
966
+ const x = (this.canvas.width - offscreenCanvas.width) / 2;
967
+ const y = (this.canvas.height - offscreenCanvas.height) / 2;
968
+
969
+ offscreenContext.drawImage(this.canvas, x, y, offscreenCanvas.width, offscreenCanvas.height, 0, 0, offscreenCanvas.width, offscreenCanvas.height);
970
+ const dataURL = offscreenCanvas.toDataURL("image/png");
971
+ offscreenCanvas.remove();
972
+
973
+ resolve(dataURL);
974
+ });
975
+ });
976
+
977
  }
978
 
979
  getStats(): { name: string, value: any }[] {
viewer/static/thumbnails/3dgs-bicycle.png ADDED
viewer/static/thumbnails/3dgs-bonsai.png ADDED
viewer/static/thumbnails/3dgs-counter.png ADDED
viewer/static/thumbnails/3dgs-flowers.png ADDED
viewer/static/thumbnails/3dgs-garden.png ADDED
viewer/static/thumbnails/3dgs-kitchen.png ADDED
viewer/static/thumbnails/3dgs-playroom.png ADDED
viewer/static/thumbnails/3dgs-room.png ADDED
viewer/static/thumbnails/3dgs-stump.png CHANGED
viewer/static/thumbnails/3dgs-treehill.png ADDED