getUserMedia compatiblity with Twilio Video #497
Comments
|
Thank you @sboudouk for this detailed issue. I will see if I can solve the problem. |
|
Yeah twilio video is much needed I would appreciate if it is fixed! |
|
After some debugging, I found that Twilio is keeping internal reference to MediaStream and MediaStreamTrack, the problem is that if you load TwilioVideo before calling iosrtc <!DOCTYPE HTML>
<html>
<head>
<title>
Twilio Video Room
</title>
<meta http-equiv="Content-Security-Policy" content="default-src 'self' data: gap: wss://* https://* 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; media-src *">
<base href="../">
</head>
<body>
<div id="local-media"></div>
<div id="remote-media"></div>
<script type="text/javascript" src="cordova.js"></script>
<!--<script src="https://media.twiliocdn.com/sdk/js/video/releases/2.0.0/twilio-video.js"></script>-->
<script type="text/javascript">
function connect() {
var result = {
token: 'YOUR_TWILIO_TOKEN',
room: 'test'
};
// Patch MediaStreamTrack with clone
MediaStreamTrack.prototype.clone = function () {
return new MediaStreamTrack(this);
};
loadScript('https://media.twiliocdn.com/sdk/js/video/releases/2.0.0/twilio-video.js').then(function () {
joinRoom(result);
});
}
var scriptUrls = [];
function loadScript(scriptUrl) {
if (scriptUrls[scriptUrl]) {
return Promise.resolve(scriptUrl);
}
return new Promise(function(resolve, reject) {
// load adapter.js
var script = document.createElement("script");
script.type = "text/javascript";
script.src = scriptUrl;
script.async = false;
document.getElementsByTagName("head")[0].appendChild(script);
script.onload = function() {
scriptUrls[scriptUrl] = true;
console.debug('loadScript.loaded', script.src);
resolve(scriptUrl);
};
});
}
function joinRoom(result) {
const Video = Twilio.Video;
Video.createLocalVideoTrack().then(track => {
const localMediaContainer = document.getElementById('local-media');
localMediaContainer.appendChild(track.attach());
});
Video.connect(result.token, {
name: result.room
}).then(room => {
console.log(`Successfully joined a Room: ${room}`);
// Attach the Tracks of the Room's Participants.
var remoteMediaContainer = document.getElementById('remote-media');
room.participants.forEach(function(participant) {
console.log("Already in Room: '" + participant.identity + "'");
participantConnected(participant, remoteMediaContainer);
});
room.on('participantConnected', participant => {
console.log(`A remote Participant connected: ${participant}`);
participantConnected(participant);
});
room.on('participantDisconnected', participant => {
console.log(`A remote Participant connected: ${participant}`);
participantDisconnected(participant);
});
}, error => {
console.error(`Unable to connect to Room: ${error.message}`);
});
function participantConnected(participant) {
console.log('Participant "%s" connected', participant.identity);
const div = document.createElement('div');
div.id = participant.sid;
participant.on('trackSubscribed', track => trackSubscribed(div, track));
participant.on('trackUnsubscribed', trackUnsubscribed);
participant.tracks.forEach(publication => {
if (publication.isSubscribed) {
trackSubscribed(div, publication.track);
}
});
document.getElementById('remote-media').appendChild(div);
}
function participantDisconnected(participant) {
console.log('Participant "%s" disconnected', participant.identity);
var div = document.getElementById(participant.sid)
if (div) {
div.remove();
}
}
function trackSubscribed(div, track) {
div.appendChild(track.attach());
}
function trackUnsubscribed(track) {
track.detach().forEach(element => element.remove());
}
}
document.addEventListener('deviceready', function () {
// Note: This allow this sample to run on any Browser
var cordova = window.cordova;
if (cordova && cordova.plugins && cordova.plugins.iosrtc) {
// Expose WebRTC and GetUserMedia SHIM as Globals (Optional)
// Alternatively WebRTC API will be inside cordova.plugins.iosrtc namespace
cordova.plugins.iosrtc.registerGlobals();
// Enable iosrtc debug (Optional)
cordova.plugins.iosrtc.debug.enable('*', true);
}
connect();
}, false);
</script>
</body>
</html>But even with loading TwilioVideo after iosrtc
|
It's not an issue with iosrtc but an issue with the way TwilioVideo is written. |
|
To solve the error "The order of m-lines in answer doesn't match order in offer. Rejecting answer" cause by twilio non detection of Options Twilio.Video.connect(result.token, {
name: result.room,
sdpSemantics: 'plan-b',
bundlePolicy: 'max-compat'
})See Updated sample: <!DOCTYPE HTML>
<html>
<head>
<title>
Twilio Video Room
</title>
<meta http-equiv="Content-Security-Policy" content="default-src 'self' data: gap: wss://* https://* 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; media-src *">
<base href="../">
</head>
<body>
<div id="local-media"></div>
<div id="remote-media"></div>
<script type="text/javascript" src="cordova.js"></script>
<!--<script src="https://media.twiliocdn.com/sdk/js/video/releases/2.0.0/twilio-video.js"></script>-->
<script type="text/javascript">
function connect() {
var result = {
token: 'YOUR_TWILIO_TOKEN',
room: 'test'
};
// Patch MediaStreamTrack with clone
MediaStreamTrack.prototype.clone = function () {
return this;
};
loadScript('https://media.twiliocdn.com/sdk/js/video/releases/2.0.0/twilio-video.js').then(function () {
joinRoom(result);
});
}
var scriptUrls = [];
function loadScript(scriptUrl) {
if (scriptUrls[scriptUrl]) {
return Promise.resolve(scriptUrl);
}
return new Promise(function(resolve, reject) {
// load adapter.js
var script = document.createElement("script");
script.type = "text/javascript";
script.src = scriptUrl;
script.async = false;
document.getElementsByTagName("head")[0].appendChild(script);
script.onload = function() {
scriptUrls[scriptUrl] = true;
console.debug('loadScript.loaded', script.src);
resolve(scriptUrl);
};
});
}
function joinRoom(result) {
const Video = Twilio.Video;
Video.createLocalVideoTrack().then(track => {
const localMediaContainer = document.getElementById('local-media');
localMediaContainer.appendChild(track.attach());
});
Video.connect(result.token, {
name: result.room,
sdpSemantics: 'plan-b',
bundlePolicy: 'max-compat'
}).then(room => {
console.log(`Successfully joined a Room: ${room}`);
// Attach the Tracks of the Room's Participants.
var remoteMediaContainer = document.getElementById('remote-media');
room.participants.forEach(function(participant) {
console.log("Already in Room: '" + participant.identity + "'");
participantConnected(participant, remoteMediaContainer);
});
room.on('participantConnected', participant => {
console.log(`A remote Participant connected: ${participant}`);
participantConnected(participant);
});
room.on('participantDisconnected', participant => {
console.log(`A remote Participant connected: ${participant}`);
participantDisconnected(participant);
});
}, error => {
console.error(`Unable to connect to Room: ${error.message}`);
});
function participantConnected(participant) {
console.log('Participant "%s" connected', participant.identity);
const div = document.createElement('div');
div.id = participant.sid;
participant.on('trackSubscribed', (track) => {
trackSubscribed(div, track)
});
participant.on('trackUnsubscribed', trackUnsubscribed);
participant.tracks.forEach(publication => {
if (publication.isSubscribed) {
trackSubscribed(div, publication.track);
}
});
document.getElementById('remote-media').appendChild(div);
}
function participantDisconnected(participant) {
console.log('Participant "%s" disconnected', participant.identity);
var div = document.getElementById(participant.sid)
if (div) {
div.remove();
}
}
function trackSubscribed(div, track) {
div.appendChild(track.attach());
}
function trackUnsubscribed(track) {
track.detach().forEach(element => element.remove());
}
}
document.addEventListener('deviceready', function () {
// Note: This allow this sample to run on any Browser
var cordova = window.cordova;
if (cordova && cordova.plugins && cordova.plugins.iosrtc) {
// Expose WebRTC and GetUserMedia SHIM as Globals (Optional)
// Alternatively WebRTC API will be inside cordova.plugins.iosrtc namespace
cordova.plugins.iosrtc.registerGlobals();
// Enable iosrtc debug (Optional)
cordova.plugins.iosrtc.debug.enable('*', true);
}
connect();
}, false);
</script>
</body>
</html>This result in video been received on browser from iosrtc but not on iosRTC from browser, more debugging is required to understand why |
|
I dont want to spent to much Twilio credits, if you want me to debug more I will need that someone give me access to a twilio |
|
I got you @hthetiot I'll give you mine by mail. |
|
Thank you @sboudouk |
|
I need to release 6.0.12 before but I have found some way to make it works. This may be on 6.0.1 because I had to SHAM MediaStreamTrack better and also override existing MediaStreamTrack and MediaStream prototype (I know it's dirty) to allow loading Twlio before iosrtc and still use iosRTC shim, this may cause issue with webrtc-adapter i need more testing. |
|
Hello ! Do you have any news on that ? Thanks ! |
|
No update yet, the fix will be a combination of #497 (comment) and #508 Also IF there were update you would see update here. If you want you can also contribute that free software, therefore it's done only on my spare time, I'm not your employee or at your service. |
|
Seriously ? Oo I would be very happy to contribute if i find the solution... ><' |
|
|
First off thank you for this amazing plugin! I just tried the fix you suggest but I got I can give you access to my Twilio account Cheers! |
|
@hthetiot wups the |
|
ok - figured out the black video, this was the cause twilio/video-quickstart-js#116 (comment). Now the local participant shows the video (the cordova app running on ios)! There is another issue. If i now open a browser on a computer, I can see and hear the iOS video, but on the device itself I can't see any audio or video from the computer. If I try it on iOS using the safari browser instead of the cordova app, everything works just fine. I see the iOS local video and the connected computers audio and video. |
|
iOS cordova is attaching the audio (I can hear it) of the computer, but just not the video. |
|
So I think I may have found the reason why the computer video does not show up in iOS cordova when someone joins. https://gist.github.com/cj/3226ecd8a4acdf616d66360565967dd3#file-gistfile1-txt-L33
Seems like https://github.com/cordova-rtc/cordova-plugin-iosrtc/blob/master/src/PluginMediaStream.swift#L24 might not be working correctly. Doing something hacky like |
|
#508 is now ready and should fix the issue with Twilio. |
|
@matijagrcic try to use aee2b39 commit, it works for me, when 6.0.12 don't, you should give it a try. |
|
Thanks, how would i install this this commit? |
|
|
Thanks, wasn't sure do i just reference some branch like mentioned here #508 or the commit like you did. |
|
|
Yep, all good, already trying that now. |
|
I'm still seeing the following
full log below: // require Twilio after we have registered iosrtc
const Video = require("twilio-video");
Video.createLocalVideoTrack({ facingMode: "user" }).then((track) => {
this.localPreviewVideoElement.nativeElement.appendChild(track.attach());
});Any pointers as it seems to get the video properly but then fails? |
|
@matijagrcic can you provide full code please ? With Twilio and iOS version ? |
|
All info below, anything else needed please let me know. iOS version: 12+ export enum CameraDirection {
Front,
Rear,
}
type VideoConfiguration = {
[key in CameraDirection]: string;
};
export const VideoConfigurations: VideoConfiguration = {
[CameraDirection.Front]: 'user',
[CameraDirection.Rear]: 'environment',
};
function registerWebRTCForiOS() {
var cordova = window.cordova;
if (cordova && cordova.plugins && cordova.plugins.iosrtc) {
cordova.plugins.iosrtc.registerGlobals();
// patch MediaStreamTrack with clone
MediaStreamTrack.prototype.clone = function () {
return this;
};
cordova.plugins.iosrtc.debug.enable('*', true);
}
}
function showLocalVideoPreview(cameraDirection: CameraDirection) {
registerWebRTCForiOS();
// we need to require Twilio after we have registered iosrtc
const Video = require('twilio-video');
Video.createLocalVideoTrack({
facingMode: VideoConfigurations[cameraDirection],
}).then((track) => {
this.localPreviewVideoElement.nativeElement.appendChild(track.attach());
});
}
showLocalVideoPreview(CameraDirection.Front); |
|
You should make sure that it's iosrtc MediaStreamTrack that is used before calling Video, maybe in the React's useEffect function: useEffect(() => {
const cordova = window.cordova
if (cordova && cordova.plugins && cordova.plugins.iosrtc) {
// Expose WebRTC and GetUserMedia SHIM as Globals (Optional)
// Alternatively WebRTC API will be inside cordova.plugins.iosrtc namespace
cordova.plugins.iosrtc.registerGlobals()
// Patch MediaStreamTrack with clone
MediaStreamTrack.prototype.clone = function () {
return this
}
// Enable iosrtc debug (Optional)
cordova.plugins.iosrtc.debug.enable('*', false)
}
MediaStreamTrack.prototype.clone = function () {
return this
}
connect(name, room)
// eslint-disable-next-line
}, [])Also, while running on device, put a breakpoint before calling "Video.createLocalVideoTrack" and in Safari's console, you can try to log MediaStreamTrack. Even if I think that's it's not the issue, because you would get a different error, so it might be an issue with iOS 12. Can you test this on iOS 13+ ? |
|
Thank you @sboudouk. Yeah it's strange, as per the output log all seems fine (stream is acquired) but then it errors out and so the output isn't displayed. Will try running it under iOS 13+ and will follow up with the result but this might not be feasible from the business perspective. |
|
I've managed to get it working on iOS 12+ (no getting I've added full log below, based on searching thru this repo I've tried setting width/height but I've also tried without them. Reference: #375 (comment) Would you have any clue of what can be wrong here? |
|
Actually ignore the above, it works fine, using createLocalVideoTrack_ on devices running iOS12 and iOS13. Summary: Ionic Angular config.xml <plugin name="cordova-plugin-inappbrowser" spec="^3.2.0" />
<plugin name="cordova-plugin-ionic" spec="^5.4.6">
<plugin name="cordova-plugin-ionic-webview" spec="^4.1.3" />
<plugin name="cordova-plugin-wkwebviewxhrfix" spec="https://github.com/TheMattRay/cordova-plugin-wkwebviewxhrfix" />package.json "cordova-plugin-ionic-webview": "^5.0.0",
"cordova-plugin-iosrtc": "git+https://github.com/cordova-rtc/cordova-plugin-iosrtc.git#aee2b39b2ccebd3dc01019d7c8ebdf2bd4c90600",
"twilio-video": "^2.5.0",I'll follow up when the Twilio P2P streaming starts working on iOS, works flawlessly on Android. |
|
Both caller and participant video/audio is working on iOS in Twilio P2P room, but the video size is small. I've tried adding video constraints in terms of sizes but nothing changes the size of the video. Do you happen to know anything else I can try? I didn't specify any height/width in the scss for the div or the video element. Is this mandatory on iOS? On Android the video size is full height. References: #332 |
Layout warning can be ignored see #422 |
Try using with/height css to set video size, 100%, flexbox or other should works. |
|
Alternatively you may want to call https://github.com/cordova-rtc/cordova-plugin-iosrtc/blob/master/docs/iosrtc.md#iosrtcrefreshvideos |
|
Nothing to do with #332 or #375 (comment) It's the video tag size refresh most likely the issue not the video stream. |
This should not be needed cause already on latest and master. |
|
Tried both #497 (comment) and #497 (comment) and video element is still small, log and scss below. Works great in Android so not sure what's missing. .main-video {
display: flex;
width: 100%;
justify-content: center;
align-items: center;
video {
height: auto;
width: 100%;
object-fit: fill;
}
}Anything else I can try? |
|
Try height 100% or with hardcoded size in pixel to debug, I think we need
to create a separate issue because this is not the same topic of the
original issues.
Also you must likely have a simple CSS or rendering issue that has nothing
bro do with Twillio.
I would recommend you look at the sample application we made:
- https://github.com/cordova-rtc/cordova-plugin-iosrtc-sample
…On Thu, Jun 18, 2020, 18:03 Matija Grcic ***@***.***> wrote:
Tried both #497 (comment)
<#497 (comment)>
and #497 (comment)
<#497 (comment)>
and video element is still small, log and scss below.
2020-06-18 16:44:43.768322+0100 App[63129:774260] iosrtc:MediaStreamRenderer refresh() | [data:{"elementLeft":0,"elementTop":119,"elementWidth":414,"elementHeight":207,"videoViewWidth":414,"videoViewHeight":207,"visible":true,"opacity":1,"zIndex":0,"mirrored":false,"objectFit":"fill","clip":true,"borderRadius":0}] +0ms
2020-06-18 16:44:43.768683+0100 App[63129:774260] iosrtcPlugin#MediaStreamRenderer_refresh()
2020-06-18 16:44:43.768953+0100 App[63129:774260] PluginMediaStreamRenderer#refresh() [elementLeft:0.0, elementTop:119.0, elementWidth:414.0, elementHeight:207.0, videoViewWidth:414.0, videoViewHeight:207.0, visible:true, opacity:1.0, zIndex:0.0, mirrored:false, clip:true, borderRadius:0.0]
2020-06-18 16:44:43.769354+0100 App[63129:774260] iosrtc:MediaStreamRenderer render() [stream:{"_listeners":{},"_id":"3027821557-3741128866-2149914973-1061480553","_active":true,"connected":true,"_audioTracks":{},"_videoTracks":{"51B6C4DF-F7E4-41E8-8A75-38EF886814BC":{"_listeners":{"ended":[null,null,null,null]},"id":"51B6C4DF-F7E4-41E8-8A75-38EF886814BC","kind":"video","muted":false,"readyState":"live","_enabled":true,"_ended":false}},"_blobId":"MediaStream_3027821557-3741128866-2149914973-1061480553"}] +1ms
.main-video {
display: flex;
width: 100%;
justify-content: center;
align-items: center;
video {
height: auto;
width: 100%;
object-fit: fill;
}
}
Anything else I can try?
—
You are receiving this because you modified the open/close state.
Reply to this email directly, view it on GitHub
<#497 (comment)>,
or unsubscribe
<https://github.com/notifications/unsubscribe-auth/AAACDO26UWT3ARZ3LEYRD5TRXI3G5ANCNFSM4MPLPNZA>
.
|
|
Thanks, I already looked at the sample application. Don't think it's a CSS issue rather a combination of Ionic (Angular) cordova plugins and iosrtc given it works fine on Android and also on Safari just not in this case. The height never gets properly set whatever css combination I use while width is set properly, tried several approaches. |
|
The dimensions is calculated on JavaScript side using Nothing special about it, what append when you force the size like 640x480 and don't use object-fit? Please provide capture or full example no reproduce you issue Inna separate issue, this issue should not be dedicated to getUserMedia and RTCPeerConnection Twilii support |
|
@sboudouk send me an email if you want me to reopen the issue for me it look like you solved your initial problem. @matijagrcic please create separate issue with issue instructions requirements. Thank you for you understanding. |
|
For master issue with twilio #541 |
Uh oh!
There was an error while loading. Please reload this page.
Expected behavior
Since
twilio-videousesgetUserMediato retrieve user's Video & Audio tracks,twilio-videoshould work as expected.Observed behavior
TypeError: track must be a LocalAudioTrack, LocalVideoTrack, LocalDataTrack, or MediaStreamTrackSteps to reproduce the problem
Platform information
Cordova version: 9.0.0 ([email protected])
Plugin version: cordova-plugin-iosrtc 6.0.10 "iosrtc"
iOS version: 13.4
Xcode version: 11.4.1 (11E503a)
twilio-video version: 2.3.0 (same issues with 1.XX tested with 1.19)
It would be great if that the MediaStreamTrack cake would match with
twilio-videosince It's a very popular package.Thanks & keep up the good work.
The text was updated successfully, but these errors were encountered: