Skip to content

Commit cf7cdad

Browse files
AntonioMacaronioAntonioMacaronio
committed
WE GOT IT WORKINGgit add ../../src/viser/client/src/Splatting/git add ../../src/viser/client/src/Splatting/ OMG OMG OMG!
1 parent e097109 commit cf7cdad

File tree

2 files changed

+193
-6
lines changed

2 files changed

+193
-6
lines changed

‎src/viser/client/src/Splatting/GaussianSplats.tsx

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ const GaussianSplatMaterial = /* @__PURE__ */ shaderMaterial(
9494
transparent: true,
9595
sh_degree: 0,
9696
textureBuffer: null,
97+
shTextureBuffer: null,
9798
textureT_camera_groups: null,
9899
transitionInState: 0.0,
99100
},
@@ -180,8 +181,7 @@ const GaussianSplatMaterial = /* @__PURE__ */ shaderMaterial(
180181
vec2 chol45 = unpackHalf2x16(intBufferData.z);
181182
182183
// Get spherical harmonic terms from the buffer, there are 48 coeffecients per vertex
183-
// uint shTexStart = sortedIndex * 6u;
184-
uint shTexStart = sortedIndex * 0u;
184+
uint shTexStart = sortedIndex * 6u;
185185
ivec2 shTexSize = textureSize(shTextureBuffer, 0);
186186
float sh_coeffs_unpacked[48];
187187
for (int i = 0; i < 6; i++) {
@@ -280,9 +280,15 @@ const GaussianSplatMaterial = /* @__PURE__ */ shaderMaterial(
280280
281281
vec3 rgb = C0 * sh_coeffs[0];
282282
vec3 pointFive = vec3(0.5, 0.5, 0.5);
283+
283284
vRgba = vec4(rgb + pointFive, float(rgbaUint32 >> uint(24)) / 255.0);
284-
// vRgba = vec4(sh_coeffs[0], 1.0); // im using this to see what the sh_coeffs actually are
285+
//vRgba = vec4(sh_coeffs[0], 1.0); // im using this to see what the sh_coeffs actually are
285286
287+
// if (sh_coeffs[0].x > 1.0 && sh_coeffs[0].x < 1.6) {
288+
// vRgba = vec4(1.0, 0.0, 0.0, 1.0);
289+
// } else {
290+
// vRgba = vec4(1.0, 1.0, 1.0, 1.0);
291+
// }
286292
287293
// rgb = rgb -
288294
// C1 * y * sh_coeffs[1] +
@@ -409,6 +415,7 @@ function SplatRenderer() {
409415
if (!initializedBufferTexture) {
410416
meshProps.material.uniforms.numGaussians.value = merged.numGaussians;
411417
meshProps.textureBuffer.needsUpdate = true;
418+
meshProps.shTextureBuffer.needsUpdate = true;
412419
initializedBufferTexture = true;
413420
}
414421
};
@@ -425,6 +432,7 @@ function SplatRenderer() {
425432
React.useEffect(() => {
426433
return () => {
427434
meshProps.textureBuffer.dispose();
435+
meshProps.shTextureBuffer.dispose();
428436
meshProps.geometry.dispose();
429437
meshProps.material.dispose();
430438
postToWorker({ close: true });
@@ -554,6 +562,7 @@ function mergeGaussianGroups(groupBufferFromName: {
554562
const groupBufferFromNameFiltered = Object.fromEntries(
555563
Object.entries(groupBufferFromName).filter(([key]) => !key.startsWith("sh_buffer_"))
556564
);
565+
// groupBufferFromNameFiltered only contains buffers for the gaussians (not sh buffers)
557566
for (const buffer of Object.values(groupBufferFromNameFiltered)) {
558567
totalBufferLength += buffer.length;
559568
}
@@ -599,6 +608,7 @@ function mergeGaussianGroups(groupBufferFromName: {
599608
combinedSHBuffer.set(sh_buffer, sh_offset)
600609
sh_offset += sh_buffer.length;
601610
}
611+
console.log(combinedSHBuffer)
602612

603613
const numGroups = Object.keys(groupBufferFromNameFiltered).length;
604614
return { numGaussians, gaussianBuffer, numGroups, groupIndices, combinedSHBuffer};
@@ -695,6 +705,7 @@ function useGaussianMeshProps(gaussianBuffer: Uint32Array, combinedSHBuffer: Uin
695705
geometry,
696706
material,
697707
textureBuffer,
708+
shTextureBuffer,
698709
sortedIndexAttribute,
699710
textureT_camera_groups,
700711
rowMajorT_camera_groups,

‎src/viser/client/src/Splatting/spherical_harmonics_testing.ipynb

Lines changed: 179 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"cells": [
33
{
44
"cell_type": "code",
5-
"execution_count": 2,
5+
"execution_count": 1,
66
"metadata": {},
77
"outputs": [],
88
"source": [
@@ -128,14 +128,14 @@
128128
},
129129
{
130130
"cell_type": "code",
131-
"execution_count": 5,
131+
"execution_count": 2,
132132
"metadata": {},
133133
"outputs": [
134134
{
135135
"name": "stdout",
136136
"output_type": "stream",
137137
"text": [
138-
"[-0.18104345 0.54372042 -0.14186212]\n"
138+
"[0.70985258 0.20630107 0.06283242]\n"
139139
]
140140
}
141141
],
@@ -147,6 +147,182 @@
147147
"print(eval_sh(deg, sh, dirs))\n"
148148
]
149149
},
150+
{
151+
"cell_type": "code",
152+
"execution_count": 3,
153+
"metadata": {},
154+
"outputs": [],
155+
"source": [
156+
"import struct\n",
157+
"import numpy as np\n",
158+
"\n",
159+
"def unpackHalf2x16(value):\n",
160+
" \"\"\"The first float is the least significant 16bits, the second is the most significant 16bits.\"\"\"\n",
161+
" # Convert int32 to its binary representation\n",
162+
" binary = format(value, '032b')\n",
163+
" \n",
164+
" # Split the binary string into two 16-bit parts\n",
165+
" binary1 = binary[:16]\n",
166+
" binary2 = binary[16:]\n",
167+
" \n",
168+
" # Convert each 16-bit binary string to an integer\n",
169+
" int1 = int(binary1, 2)\n",
170+
" int2 = int(binary2, 2)\n",
171+
" \n",
172+
" # Use numpy to convert uint16 to float16\n",
173+
" float1 = np.frombuffer(struct.pack('H', int1), dtype=np.float16)[0]\n",
174+
" float2 = np.frombuffer(struct.pack('H', int2), dtype=np.float16)[0]\n",
175+
" \n",
176+
" return float1, float2\n",
177+
"\n",
178+
"def int32_to_rgba(value):\n",
179+
" # Ensure the input is a 32-bit integer\n",
180+
" value = int(value) & 0xFFFFFFFF\n",
181+
" \n",
182+
" # Extract each 8-bit piece\n",
183+
" r = (value >> 24) & 0xFF\n",
184+
" g = (value >> 16) & 0xFF\n",
185+
" b = (value >> 8) & 0xFF\n",
186+
" a = value & 0xFF\n",
187+
" \n",
188+
" return r / 255.0, g / 255.0, b / 255.0, a / 255.0"
189+
]
190+
},
191+
{
192+
"cell_type": "code",
193+
"execution_count": 4,
194+
"metadata": {},
195+
"outputs": [
196+
{
197+
"data": {
198+
"text/plain": [
199+
"(0.803, 0.8213)"
200+
]
201+
},
202+
"execution_count": 4,
203+
"metadata": {},
204+
"output_type": "execute_result"
205+
}
206+
],
207+
"source": [
208+
"unpackHalf2x16(980236946)"
209+
]
210+
},
211+
{
212+
"cell_type": "code",
213+
"execution_count": 5,
214+
"metadata": {},
215+
"outputs": [
216+
{
217+
"data": {
218+
"text/plain": [
219+
"(-0.00319, 0.786)"
220+
]
221+
},
222+
"execution_count": 5,
223+
"metadata": {},
224+
"output_type": "execute_result"
225+
}
226+
],
227+
"source": [
228+
"unpackHalf2x16(2592619082)"
229+
]
230+
},
231+
{
232+
"cell_type": "code",
233+
"execution_count": 4,
234+
"metadata": {},
235+
"outputs": [
236+
{
237+
"data": {
238+
"text/plain": [
239+
"(0.3686274509803922,\n",
240+
" 0.7215686274509804,\n",
241+
" 0.7254901960784313,\n",
242+
" 0.7294117647058823)"
243+
]
244+
},
245+
"execution_count": 4,
246+
"metadata": {},
247+
"output_type": "execute_result"
248+
}
249+
],
250+
"source": [
251+
"int32_to_rgba(1589164474)"
252+
]
253+
},
254+
{
255+
"cell_type": "code",
256+
"execution_count": 5,
257+
"metadata": {},
258+
"outputs": [
259+
{
260+
"data": {
261+
"application/vnd.jupyter.widget-view+json": {
262+
"model_id": "c3317264494f4ea997d61191fa24a3c5",
263+
"version_major": 2,
264+
"version_minor": 0
265+
},
266+
"text/plain": [
267+
"VBox(children=(FloatSlider(value=0.8212358, description='Red:', max=1.0, step=0.01), FloatSlider(value=0.80300…"
268+
]
269+
},
270+
"metadata": {},
271+
"output_type": "display_data"
272+
}
273+
],
274+
"source": [
275+
"import ipywidgets as widgets\n",
276+
"from IPython.display import display, HTML\n",
277+
"import matplotlib.colors as mcolors\n",
278+
"\n",
279+
"def color_picker(r=0.5, g=0.5, b=0.5):\n",
280+
" def update_color(r, g, b):\n",
281+
" color = mcolors.to_hex([r, g, b])\n",
282+
" color_display.value = f'<div style=\"width: 100px; height: 100px; background-color: {color};\"></div>'\n",
283+
" rgb_display.value = f'RGB: ({r:.2f}, {g:.2f}, {b:.2f})'\n",
284+
" hex_display.value = f'Hex: {color}'\n",
285+
"\n",
286+
" r_slider = widgets.FloatSlider(value=r, min=0, max=1, step=0.01, description='Red:')\n",
287+
" g_slider = widgets.FloatSlider(value=g, min=0, max=1, step=0.01, description='Green:')\n",
288+
" b_slider = widgets.FloatSlider(value=b, min=0, max=1, step=0.01, description='Blue:')\n",
289+
"\n",
290+
" color_display = widgets.HTML()\n",
291+
" rgb_display = widgets.Label()\n",
292+
" hex_display = widgets.Label()\n",
293+
"\n",
294+
" widgets.interactive(update_color, r=r_slider, g=g_slider, b=b_slider)\n",
295+
"\n",
296+
" display(widgets.VBox([r_slider, g_slider, b_slider, color_display, rgb_display, hex_display]))\n",
297+
"\n",
298+
"# Usage\n",
299+
"color_picker(0.8212358, 0.8030037, 0.78623223) # You can change these initial values"
300+
]
301+
},
302+
{
303+
"cell_type": "code",
304+
"execution_count": 6,
305+
"metadata": {},
306+
"outputs": [
307+
{
308+
"data": {
309+
"application/vnd.jupyter.widget-view+json": {
310+
"model_id": "49ebcdf479814f618fab56eb4f17427b",
311+
"version_major": 2,
312+
"version_minor": 0
313+
},
314+
"text/plain": [
315+
"VBox(children=(FloatSlider(value=0.3686274509803922, description='Red:', max=1.0, step=0.01), FloatSlider(valu…"
316+
]
317+
},
318+
"metadata": {},
319+
"output_type": "display_data"
320+
}
321+
],
322+
"source": [
323+
"color_picker(0.3686274509803922, 0.7215686274509804, 0.7254901960784313)"
324+
]
325+
},
150326
{
151327
"cell_type": "code",
152328
"execution_count": null,

0 commit comments

Comments
 (0)