Skip to content

Commit eca5d6a

Browse files
xBasovxBasov
authored
RTT (real-time-text) implementation. Re-Invite advanced scenarios development. (#1356)
* temp rtt commit. * Remove the hacky logic from GetMediaStream(). (#1314) Fix GetRTPChannel exception. Implement adding media (video) stream to an existing audio call ,and removing media (video) stream from an existing video call. ( #1307) * Real-time-text implementation (text-stream). * Implement adding media (text) stream to an existing audio call ,and removing media (text) stream from an existing RTT call. ( #1307) * Summary, comment fixes. Update Abstractions NuGet to latest version. * Final merge duplication fixes. * Implement requested logging changes.
1 parent 5cb5238 commit eca5d6a

14 files changed

+1045
-158
lines changed

‎src/SIPSorcery.csproj

100755100644
+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
<PackageReference Include="Portable.BouncyCastle" Version="1.9.0" />
2222
<PackageReference Include="DnsClient" Version="1.8.0" />
2323
<PackageReference Include="SIPSorcery.WebSocketSharp" Version="0.0.1" />
24-
<PackageReference Include="SIPSorceryMedia.Abstractions" Version="8.0.7" />
24+
<PackageReference Include="SIPSorceryMedia.Abstractions" Version="8.0.10" />
2525
<PackageReference Include="System.Net.Security" Version="4.3.2" />
2626
<PackageReference Include="System.Net.WebSockets.Client" Version="4.3.2" />
2727
</ItemGroup>

‎src/SIPSorcery.sln

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11

22
Microsoft Visual Studio Solution File, Format Version 12.00
3-
# Visual Studio Version 16
4-
VisualStudioVersion = 16.0.29230.61
3+
# Visual Studio Version 17
4+
VisualStudioVersion = 17.9.34701.34
55
MinimumVisualStudioVersion = 10.0.40219.1
66
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Test", "Test", "{5A855FE1-8B93-429D-95A5-B8215CF62A9D}"
77
EndProject

‎src/app/Media/AudioSendOnlyMediaSession.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ private void AudioFormatsNegotiated(List<AudioFormat> audioFormats)
4141

4242
public async override Task Start()
4343
{
44-
if (!base.IsStarted)
44+
if (!base.IsAudioStarted)
4545
{
4646
await base.Start().ConfigureAwait(false);
4747
await AudioExtrasSource.StartAudio().ConfigureAwait(false);

‎src/app/Media/IMediaSession.cs

+5-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
//-----------------------------------------------------------------------------
1515

1616
using System;
17-
using System.Collections.Generic;
1817
using System.Net;
1918
using System.Threading;
2019
using System.Threading.Tasks;
@@ -41,6 +40,11 @@ public enum SdpType
4140
/// </summary>
4241
public interface IMediaSession
4342
{
43+
/// <summary>
44+
/// Indicates wheter the session supports real time text.
45+
/// </summary>
46+
bool HasText { get; }
47+
4448
/// <summary>
4549
/// Indicates whether the session supports audio.
4650
/// </summary>

‎src/app/Media/VoIPMediaSession.cs

100755100644
+97-33
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,13 @@ public VoIPMediaSession(VoIPMediaSessionConfig config)
147147
}
148148
}
149149

150+
if (Media.TextSource != null)
151+
{
152+
var textTrack = new MediaStreamTrack(config.MediaEndPoint.TextSource.GetTextSourceFormat());
153+
base.addTrack(textTrack);
154+
Media.TextSource.OnTextSourceEncodedSample += base.SendText;
155+
}
156+
150157
if (Media.VideoSink != null)
151158
{
152159
Media.VideoSink.OnVideoSinkDecodedSample += VideoSinkSampleReady;
@@ -158,8 +165,14 @@ public VoIPMediaSession(VoIPMediaSessionConfig config)
158165
base.OnRtpPacketReceived += RtpMediaPacketReceived;
159166
}
160167

168+
if (Media.TextSink != null)
169+
{
170+
base.OnRtpPacketReceived += RtpMediaPacketReceived;
171+
}
172+
161173
base.OnAudioFormatsNegotiated += AudioFormatsNegotiated;
162174
base.OnVideoFormatsNegotiated += VideoFormatsNegotiated;
175+
base.OnTextFormatsNegotiated += TextFormatsNegotiated;
163176
}
164177

165178
private async void VideoSource_OnVideoSourceError(string errorMessage)
@@ -170,8 +183,11 @@ private async void VideoSource_OnVideoSourceError(string errorMessage)
170183

171184
logger.LogWarning("Video source for capture device failure. {ErrorMessage}", errorMessage);
172185

173-
// Can't use the webcam, switch to the test pattern source.
174-
await _videoTestPatternSource.StartVideo().ConfigureAwait(false);
186+
if (_videoTestPatternSource != null)
187+
{
188+
// Can't use the webcam, switch to the test pattern source.
189+
await _videoTestPatternSource.StartVideo().ConfigureAwait(false);
190+
}
175191
}
176192
}
177193

@@ -206,42 +222,56 @@ private void VideoFormatsNegotiated(List<VideoFormat> videoFormats)
206222
_videoTestPatternSource?.SetVideoSourceFormat(videoFormat);
207223
}
208224

225+
private void TextFormatsNegotiated(List<TextFormat> textFormats)
226+
{
227+
var textFormat = textFormats.First();
228+
logger.LogDebug("Setting text sink and source format to {TextFormatID}:{TextCodec}.", textFormat.FormatID, textFormat.Codec);
229+
Media.TextSource?.SetTextSourceFormat(textFormat);
230+
}
231+
209232
public async override Task Start()
210233
{
211-
if (!base.IsStarted)
234+
await base.Start().ConfigureAwait(false);
235+
236+
if (HasAudio)
212237
{
213-
await base.Start().ConfigureAwait(false);
238+
if (Media.AudioSource != null)
239+
{
240+
await Media.AudioSource.StartAudio().ConfigureAwait(false);
241+
}
242+
if (Media.AudioSink != null)
243+
{
244+
await Media.AudioSink.StartAudioSink().ConfigureAwait(false);
245+
}
246+
}
214247

215-
if (HasAudio)
248+
if (HasVideo && Media.VideoSource != null)
249+
{
250+
if (!_videoCaptureDeviceFailed)
216251
{
217-
if (Media.AudioSource != null)
218-
{
219-
await Media.AudioSource.StartAudio().ConfigureAwait(false);
220-
}
221-
if (Media.AudioSink != null)
222-
{
223-
await Media.AudioSink.StartAudioSink().ConfigureAwait(false);
224-
}
252+
await Media.VideoSource.StartVideo().ConfigureAwait(false);
225253
}
254+
else
255+
{
256+
logger.LogWarning("Webcam video source failed before start, switching to test pattern source.");
226257

227-
if (HasVideo)
258+
// The webcam source failed to start. Switch to a test pattern source.
259+
await _videoTestPatternSource.StartVideo().ConfigureAwait(false);
260+
}
261+
}
262+
263+
if (HasText)
264+
{
265+
if (Media.TextSource != null)
228266
{
229-
if (Media.VideoSource != null)
230-
{
231-
if (!_videoCaptureDeviceFailed)
232-
{
233-
await Media.VideoSource.StartVideo().ConfigureAwait(false);
234-
}
235-
else
236-
{
237-
logger.LogWarning("Webcam video source failed before start, switching to test pattern source.");
238-
239-
// The webcam source failed to start. Switch to a test pattern source.
240-
await _videoTestPatternSource.StartVideo().ConfigureAwait(false);
241-
}
242-
}
267+
await Media.TextSource.StartText().ConfigureAwait(false);
268+
}
269+
if (Media.TextSink != null)
270+
{
271+
await Media.TextSink.StartTextSink().ConfigureAwait(false);
243272
}
244273
}
274+
245275
}
246276

247277
public async override void Close(string reason)
@@ -282,6 +312,15 @@ public async override void Close(string reason)
282312
Media.VideoSink.OnVideoSinkDecodedSample -= VideoSinkSampleReady;
283313
base.OnVideoFrameReceived -= Media.VideoSink.GotVideoFrame;
284314
}
315+
316+
if (Media.TextSource != null)
317+
{
318+
await Media.TextSource.CloseText().ConfigureAwait(false);
319+
}
320+
if (Media.TextSink != null)
321+
{
322+
await Media.TextSink.CloseTextSink().ConfigureAwait(false);
323+
}
285324
}
286325
}
287326

@@ -301,6 +340,12 @@ protected void RtpMediaPacketReceived(IPEndPoint remoteEndPoint, SDPMediaTypesEn
301340

302341
Media.AudioSink.GotAudioRtp(remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType, marker, rtpPacket.Payload);
303342
}
343+
else if (mediaType == SDPMediaTypesEnum.text && Media.TextSink != null)
344+
{
345+
logger.LogTrace(nameof(RtpMediaPacketReceived) + " text RTP packet received from {RemoteEndPoint} ssrc {SyncSource} seqnum {SequenceNumber} timestamp {Timestamp} payload type {PayloadType}.", remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType);
346+
347+
Media.TextSink.GotTextRtp(remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType, hdr.MarkerBit, rtpPacket.Payload);
348+
}
304349
}
305350

306351
public async Task PutOnHold()
@@ -316,10 +361,18 @@ public async Task PutOnHold()
316361
await Media.VideoSource.PauseVideo().ConfigureAwait(false);
317362

318363
//_videoTestPatternSource.SetEmbeddedTestPatternPath(VideoTestPatternSource.TEST_PATTERN_INVERTED_RESOURCE_PATH);
319-
_videoTestPatternSource.SetFrameRate(TEST_PATTERN_ONHOLD_FPS);
364+
_videoTestPatternSource?.SetFrameRate(TEST_PATTERN_ONHOLD_FPS);
320365

321366
Media.VideoSource.ForceKeyFrame();
322-
await _videoTestPatternSource.ResumeVideo().ConfigureAwait(false);
367+
if (_videoTestPatternSource != null)
368+
{
369+
await _videoTestPatternSource.ResumeVideo().ConfigureAwait(false);
370+
}
371+
}
372+
373+
if (HasText)
374+
{
375+
// TODO can be put on / taken off hold?
323376
}
324377
}
325378

@@ -333,10 +386,13 @@ public async void TakeOffHold()
333386

334387
if (HasVideo)
335388
{
336-
await _videoTestPatternSource.PauseVideo().ConfigureAwait(false);
389+
if (_videoTestPatternSource != null)
390+
{
391+
await _videoTestPatternSource.PauseVideo().ConfigureAwait(false);
392+
}
337393

338394
//_videoTestPatternSource.SetEmbeddedTestPatternPath(VideoTestPatternSource.TEST_PATTERN_RESOURCE_PATH);
339-
_videoTestPatternSource.SetFrameRate(TEST_PATTERN_FPS);
395+
_videoTestPatternSource?.SetFrameRate(TEST_PATTERN_FPS);
340396

341397
Media.VideoSource.ForceKeyFrame();
342398

@@ -346,9 +402,17 @@ public async void TakeOffHold()
346402
}
347403
else
348404
{
349-
await _videoTestPatternSource.ResumeVideo().ConfigureAwait(false);
405+
if (_videoTestPatternSource != null)
406+
{
407+
await _videoTestPatternSource.ResumeVideo().ConfigureAwait(false);
408+
}
350409
}
351410
}
411+
412+
if (HasText)
413+
{
414+
// TODO can be put on / taken off hold?
415+
}
352416
}
353417
}
354418
}

0 commit comments

Comments
 (0)