Skip to content
This repository was archived by the owner on Jan 7, 2023. It is now read-only.

Commit abdd324

Browse files
astojiljastojilj
committed
Added depth_camera.js: camera parameters extracted from pointcloud demo.
Using it simplifies the code here.
1 parent a095001 commit abdd324

File tree

3 files changed

+264
-63
lines changed

3 files changed

+264
-63
lines changed

‎README.md

+8-4
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,19 @@
33
![Alt text](/how_the_demo_looks.gif?raw=true "Depth camera capture demo")
44

55
To capture and manipulate depth camera stream in HTML5, you'll need:
6-
* Latest Chrome browser (no need for additional extensions),
7-
* Intel Realsense camera plugged to USB 3.0 port of a
6+
* Chrome browser version 58 or later (no need for additional extensions),
7+
* Version 58 is currently available [from Beta channel](http://www.chromium.org/getting-involved/dev-channel),
8+
* Intel® RealSense™ 3D camera plugged to USB 3.0 port
9+
* SR300 (and related cameras like Razer Stargazer or Creative BlasterX
10+
Senz3D) or R200,
811
* Windows, Linux or ChromeOS PC.
912

1013
These are the constraints of current implementation. The plan is to support other depth cameras and OSX and Android, too.
1114

12-
We are going to use camera's 16-bit depth stream in example code. Some of the use cases, like 3D pointcloud rendering, 3D scannning or object recognition, require higher data precision than 8-bit precision provided through ImageData that is most commonly used to access color video pixels data. http://www.w3schools.com/tags/canvas_getimagedata.asp
15+
An explanation on how to use the depth camera is in the article
16+
[Depth Camera Capture in HTML5](https://01.org/chromium/blogs/astojilj/2017/depth-camera-capture-html5).
1317

14-
The example code here covers
18+
The example code here covers:
1519
* Displaying depth video stream in <video> element
1620
* uploading depth frame data to WebGL texture,
1721
* enumerating depth frame data pixel values in loop.

‎depth-camera.js

+220
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
/*jshint esversion: 6 */
2+
3+
// Copyright 2017 Intel Corporation.
4+
//
5+
// Licensed under the Apache License, Version 2.0 (the "License");
6+
// you may not use this file except in compliance with the License.
7+
// You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing, software
12+
// distributed under the License is distributed on an "AS IS" BASIS,
13+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
// See the License for the specific language governing permissions and
15+
// limitations under the License.
16+
17+
DepthCamera.getDepthStream = function () {
18+
if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices ||
19+
!navigator.mediaDevices.getUserMedia)
20+
return Promise.reject("Your browser doesn't support the mediaDevices API.");
21+
22+
return new Promise(async function(resolve, reject) {
23+
try {
24+
const constraints = {
25+
audio: false,
26+
video:{
27+
// We don't use videoKind as it is still under development.
28+
// videoKind: {exact:"depth"},
29+
// R200 related hack: prefer depth (width = 628) to IR (width = 641)
30+
// stream.
31+
width: {ideal:628, max:640},
32+
33+
// SR300 depth camera enables capture at 110 frames per second.
34+
frameRate: {ideal:110},
35+
}
36+
}
37+
const stream = await navigator.mediaDevices.getUserMedia(constraints);
38+
const track = stream.getVideoTracks()[0];
39+
if (track.label.indexOf("RealSense") == -1) {
40+
41+
function getChromeVersion () {
42+
var raw = navigator.userAgent.match(/Chrom(e|ium)\/([0-9]+)\./);
43+
return raw ? parseInt(raw[2], 10) : false;
44+
}
45+
46+
return reject(getChromeVersion() < 58 ?
47+
"Your browser version is too old. Get Chrome version 58 or later." :
48+
"No RealSense camera connected.");
49+
}
50+
return resolve(stream);
51+
} catch(e) {
52+
return reject("Error getting the depth camera stream:" + e);
53+
}
54+
});
55+
}
56+
57+
// Call the method after getting depth_stream using getDepthStream.
58+
DepthCamera.getColorStreamForDepthStream = function(depth_stream,
59+
ideal_width = undefined) {
60+
// To get color stream from the same physical device providing the depth
61+
// stream, we will use groupId, once it is implemented:
62+
// See https://crbug.com/627793
63+
// For now, enumerate devices based on label.
64+
// Note: depth_stream is not used, for now, but deliberatelly added as a
65+
// parameter to mandate the need for previous call to getDepthStream.
66+
return new Promise(function(resolve, reject) {
67+
navigator.mediaDevices.enumerateDevices()
68+
.then(function(all_devices) {
69+
70+
let depth_device_id = null;
71+
const depth = depth_stream.getVideoTracks()[0];
72+
// Chrome, starting with version 59, implements getSettings() API.
73+
if (depth.getSettings) {
74+
depth_device_id = depth.getSettings().deviceId;
75+
} else if (ideal_width) {
76+
console.warn("Not able to set ideal width for color video as \
77+
MediaStreamTrack getSettings() API is not available. Try with \
78+
Chromium version > 59.");
79+
}
80+
81+
82+
const devices = all_devices
83+
.filter((device) => (device.kind == "videoinput" &&
84+
device.label.indexOf("RealSense") !== -1 &&
85+
device.deviceId != depth_device_id));
86+
if (devices.length < 1) {
87+
return reject("No RealSense camera connected.");
88+
}
89+
90+
// Select streams from these ids, so that some other camera doesn't get
91+
// selected (e.g. if the user has another rgb camera).
92+
const ids = devices.map((device) => device.deviceId);
93+
94+
// Select color stream.
95+
// Color stream tracks have larger resolution than depth stream tracks.
96+
// If we cannot use deviceId to select, for now, we need to misuse width.
97+
ideal_width = ids.length == 1 ? ideal_width : 1280;
98+
const constraints = ideal_width ?
99+
{
100+
video: {
101+
width: {ideal:ideal_width},
102+
deviceId: {exact: ids},
103+
},
104+
} : {
105+
video: {
106+
deviceId: {exact: ids},
107+
},
108+
}
109+
navigator.mediaDevices.getUserMedia(constraints)
110+
.then(function(stream) {
111+
return resolve(stream);
112+
})
113+
.catch(function(error) {
114+
return reject(error);
115+
});
116+
})
117+
.catch(function(error) {
118+
return reject(error);
119+
})
120+
});
121+
}
122+
123+
// Figure out the camera intristics and extrinsics based on the depth stream
124+
// camera model.
125+
//
126+
// This should be rewritten once the MediaCapture-Depth API works - don't
127+
// hardcode the values based on camera model, but query it from the API.
128+
//
129+
// See the documentation at
130+
// https://w3c.github.io/mediacapture-depth/#synchronizing-depth-and-color-video-rendering
131+
DepthCamera.getCameraCalibration = function(depth_stream) {
132+
const label = depth_stream.getVideoTracks()[0].label;
133+
const cameraName = label.includes("R200") ? "R200"
134+
: (label.includes("Camera S") || label.includes("SR300")) ? "SR300"
135+
: label;
136+
137+
var distortionModels = {
138+
NONE: 0,
139+
MODIFIED_BROWN_CONRADY: 1,
140+
INVERSE_BROWN_CONRADY: 2,
141+
};
142+
var result;
143+
if (cameraName === "R200") {
144+
result = {
145+
depthScale: 0.001,
146+
depthOffset: new Float32Array(
147+
[ 233.3975067138671875, 179.2618865966796875 ]
148+
),
149+
depthFocalLength: new Float32Array(
150+
[ 447.320953369140625, 447.320953369140625 ]
151+
),
152+
colorOffset: new Float32Array(
153+
[ 311.841033935546875, 229.7513275146484375 ]
154+
),
155+
colorFocalLength: new Float32Array(
156+
[ 627.9630126953125, 634.02410888671875 ]
157+
),
158+
depthToColor: [
159+
0.99998325109481811523, 0.002231199527159333229, 0.00533978315070271492, 0,
160+
-0.0021383403800427913666, 0.99984747171401977539, -0.017333013936877250671, 0,
161+
-0.0053776423446834087372, 0.017321307212114334106, 0.99983555078506469727, 0,
162+
-0.058898702263832092285, -0.00020283895719330757856, -0.0001998419174924492836, 1
163+
],
164+
depthDistortionModel: distortionModels.NONE,
165+
depthDistortioncoeffs: [ 0, 0, 0, 0, 0 ],
166+
colorDistortionModel: distortionModels.MODIFIED_BROWN_CONRADY,
167+
colorDistortioncoeffs: [
168+
-0.078357703983783721924,
169+
0.041351985186338424683,
170+
-0.00025565386749804019928,
171+
0.0012357287341728806496,
172+
0
173+
],
174+
};
175+
} else if (cameraName === "SR300") {
176+
result = {
177+
depthScale: 0.0001249866472790017724,
178+
depthOffset: new Float32Array(
179+
[ 310.743988037109375, 245.1811676025390625 ]
180+
),
181+
depthFocalLength: new Float32Array(
182+
[ 475.900726318359375, 475.900726318359375]
183+
),
184+
colorOffset: new Float32Array(
185+
[ 312.073974609375, 241.969329833984375 ]
186+
),
187+
colorFocalLength: new Float32Array(
188+
[ 617.65087890625, 617.65093994140625 ]
189+
),
190+
depthToColor: [
191+
0.99998641014099121094, -0.0051436689682304859161, 0.00084982655243948101997, 0,
192+
0.0051483912393450737, 0.99997079372406005859, -0.005651625804603099823, 0,
193+
-0.00082073162775486707687, 0.0056559243239462375641, 0.99998366832733154297, 0,
194+
0.025699997320771217346, -0.00073326355777680873871, 0.0039400043897330760956, 1
195+
],
196+
depthDistortionModel: distortionModels.INVERSE_BROWN_CONRADY,
197+
depthDistortioncoeffs: [
198+
0.14655706286430358887,
199+
0.078352205455303192139,
200+
0.0026113723870366811752,
201+
0.0029218809213489294052,
202+
0.066788062453269958496,
203+
],
204+
colorDistortionModel: distortionModels.NONE,
205+
colorDistortioncoeffs: [ 0, 0, 0, 0, 0 ],
206+
};
207+
} else {
208+
throw {
209+
name: "CameraNotSupported",
210+
message: "Sorry, your camera '" + cameraName + "' is not supported",
211+
};
212+
}
213+
// This also de-normalizes the depth value (it's originally a 16-bit
214+
// integer normalized into a float between 0 and 1).
215+
result.depthScale = result.depthScale * 65535;
216+
return result;
217+
}
218+
219+
function DepthCamera() {
220+
}

‎depthdemo.html

+36-59
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
</head>
44
<body onload="onLoad()">
55
<h2>HTML5 Depth Capture Demo</h2>
6+
<div id="errormessages" style="color: red; font-size: 150%;">
7+
<!-- Print error messages here. -->
8+
</div>
9+
610
<div class="select">
711
<label for="selectVideoDevice">Video capture device used for depth stream: </label><select id="selectVideoDevice"></select>
812
</div>
@@ -16,8 +20,20 @@ <h2>HTML5 Depth Capture Demo</h2>
1620
</div>
1721
</body>
1822

23+
<script type="text/javascript" src="depth-camera.js"></script>
1924
<script type="text/javascript">
2025

26+
function showErrorToUser(message) {
27+
var div = document.getElementById("errormessages");
28+
div.innerHTML += message + "</br>";
29+
}
30+
31+
function handleError(error) {
32+
console.error(error);
33+
showErrorToUser("Error:" + (error.name ? (error.name + ": " + error.message)
34+
: error));
35+
}
36+
2137
// Creates WebGL/WebGL2 context used to upload depth video to texture,
2238
// read the pixels to Float buffer and optionally render the texture.
2339
function GL(canvasId) {
@@ -82,58 +98,10 @@ <h2>HTML5 Depth Capture Demo</h2>
8298
return gl;
8399
}
84100

85-
function getDepthDeviceLabelBasedConstraints() {
86-
return new Promise(function(resolve, reject) {
87-
navigator.mediaDevices.enumerateDevices()
88-
.then(function(devices) {
89-
for (var i = 0; i < devices.length; ++i) {
90-
if (devices[i].label.indexOf("Depth") != -1)
91-
return resolve({video:{deviceId: {exact: devices[i].deviceId}}});
92-
}
93-
for (var i = 0; i < devices.length; ++i) {
94-
if (devices[i].label.indexOf("RealSense") != -1 && devices[i].kind === 'videoinput')
95-
return resolve({video:{deviceId: {exact: devices[i].deviceId}}});
96-
}
97-
return reject("No depth capture device found");
98-
})
99-
});
100-
}
101-
102-
function getDepthVideoKindStream() {
103-
var constraints = {
104-
video:{
105-
videoKind: {exact:"depth"},
106-
// R200 related hack to prefer depth (width = 628) to IR (width = 641)
107-
// stream. Seems harmless with other devices.
108-
width: {ideal:628}
109-
}
110-
}
111-
return navigator.mediaDevices.getUserMedia(constraints);
112-
}
113-
114-
function getDepthStream() {
115-
return new Promise(function(resolve, reject) {
116-
getDepthVideoKindStream()
117-
.then(function(stream) {
118-
return resolve(stream);
119-
}, function() {
120-
// Fallback to use label.
121-
getDepthDeviceLabelBasedConstraints()
122-
.then(function(constraints) {
123-
return navigator.mediaDevices.getUserMedia(constraints);
124-
}).then(function(stream) {
125-
return resolve(stream);
126-
}).catch(function(error) {
127-
alert(error);
128-
return reject("error getting the stream");
129-
});
130-
})
131-
});
132-
}
133-
134101
var gl;
135102
var video_frame_available = false;
136103
var read_buffer;
104+
var depth_device_id;
137105

138106
var read_format;
139107
var canvas_2d = document.getElementById("canvas2D");
@@ -245,16 +213,18 @@ <h2>HTML5 Depth Capture Demo</h2>
245213
select_device.value = selected;
246214
} else if (!selected) {
247215
// If no other device is selected, set the initial selection to depth device.
248-
getDepthDeviceLabelBasedConstraints().then(function(constraints) {
249-
select_device.value = constraints.video.deviceId.exact;
250-
}).catch(function(error) {
251-
console.log("Set initial selection error:" + error);
252-
});
216+
if (depth_device_id)
217+
select_device.value = depth_device_id;
253218
}
254219
}
255220

256221
function streamOpened(stream) {
257222
video.srcObject = stream;
223+
// Chrome, starting with version 59, implements getSettings() API.
224+
var track = stream.getVideoTracks()[0];
225+
if (track.getSettings)
226+
depth_device_id = track.getSettings().deviceId;
227+
258228
// Get all devices to drop-down list.
259229
return navigator.mediaDevices.enumerateDevices();
260230
}
@@ -268,12 +238,19 @@ <h2>HTML5 Depth Capture Demo</h2>
268238

269239
// If the item in drop-down list is selected, use it.
270240
var openStreamFunction = select_device.value ? function() {
271-
return navigator.mediaDevices.getUserMedia({video: {deviceId: {exact: select_device.value}}});
272-
} : getDepthStream;
241+
return navigator.mediaDevices.getUserMedia({
242+
video: {
243+
deviceId: {
244+
exact: select_device.value
245+
}
246+
}
247+
});
248+
} : DepthCamera.getDepthStream;
273249

274-
openStreamFunction().then(streamOpened).then(allDevicesEnumerated).catch(function(error) {
275-
console.log(error);
276-
});
250+
openStreamFunction()
251+
.then(streamOpened)
252+
.then(allDevicesEnumerated)
253+
.catch(handleError);
277254
}
278255

279256
function onLoad() {

0 commit comments

Comments
 (0)