move jitter buffer before mux (#327)

* move jitter buffer before mux

* update test client audio

Author: frostbyte73

pkg/sip/media_port.go

@@ -239,9 +239,7 @@ func (p *MediaPort) setupInput() {
 	// Decoding pipeline (SIP RTP -> LK PCM)
 	audioHandler := p.conf.Audio.Codec.DecodeRTP(p.audioIn, p.conf.Audio.Type)
 	p.audioInHandler = audioHandler
-	if p.jitterEnabled {
-		audioHandler = rtp.HandleJitter(p.conf.Audio.Codec.Info().RTPClockRate, audioHandler)
-	}
+
 	mux := rtp.NewMux(nil)
 	mux.SetDefault(newRTPStatsHandler(p.mon, "", nil))
 	mux.Register(p.conf.Audio.Type, newRTPStatsHandler(p.mon, p.conf.Audio.Codec.Info().SDPName, audioHandler))
@@ -258,7 +256,12 @@ func (p *MediaPort) setupInput() {
 			return nil
 		})))
 	}
-	p.conn.OnRTP(mux)
+
+	if p.jitterEnabled {
+		p.conn.OnRTP(rtp.HandleJitter(p.conf.Audio.Codec.Info().RTPClockRate, mux))
+	} else {
+		p.conn.OnRTP(mux)
+	}
 }
 
 // SetDTMFAudio forces SIP to generate audio dTMF tones in addition to digital signals.

pkg/siptest/client.go

@@ -37,6 +37,7 @@ import (
 	"github.com/icholy/digest"
 	"github.com/pion/sdp/v3"
 
+	"github.com/livekit/sip/pkg/mixer"
 	"github.com/livekit/sipgo"
 	"github.com/livekit/sipgo/sip"
 
@@ -106,7 +107,7 @@ func NewClient(id string, conf ClientConfig) (*Client, error) {
 	cli.media = rtp.NewSeqWriter(cli.mediaConn)
 	cli.mediaAudio = cli.media.NewStream(cli.audioType, codec.Info().RTPClockRate)
 	cli.mediaDTMF = cli.media.NewStream(101, dtmf.SampleRate)
-	cli.audioOut = cli.audioCodec.EncodeRTP(cli.mediaAudio)
+	cli.audioOut = mixer.NewMixer(cli.audioCodec.EncodeRTP(cli.mediaAudio), rtp.DefFrameDur)
 
 	cli.setupRTPReceiver()
 
@@ -174,7 +175,7 @@ type Client struct {
 	media         *rtp.SeqWriter
 	mediaAudio    *rtp.Stream
 	mediaDTMF     *rtp.Stream
-	audioOut      media.PCM16Writer
+	audioOut      *mixer.Mixer
 	sipClient     *sipgo.Client
 	sipServer     *sipgo.Server
 	inviteReq     *sip.Request
@@ -399,7 +400,9 @@ func (c *Client) sendBye() {
 
 func (c *Client) SendDTMF(digits string) error {
 	c.log.Debug("sending dtmf", "str", digits)
-	return dtmf.Write(context.Background(), c.audioOut, c.mediaDTMF, c.mediaAudio.GetCurrentTimestamp(), digits)
+	w := c.audioOut.NewInput()
+	defer w.Close()
+	return dtmf.Write(context.Background(), w, c.mediaDTMF, c.mediaAudio.GetCurrentTimestamp(), digits)
 }
 
 func (c *Client) SendNotify(eventReq *sip.Request, notifyStatus string) error {
@@ -568,7 +571,9 @@ func (c *Client) SendAudio(path string) error {
 	}
 
 	i := 0
-	w := c.audioOut
+	w := c.audioOut.NewInput()
+	defer w.Close()
+
 	for range time.NewTicker(rtp.DefFrameDur).C {
 		if i >= len(audioFrames) {
 			break
@@ -584,7 +589,8 @@ func (c *Client) SendAudio(path string) error {
 func (c *Client) SendSilence(ctx context.Context) error {
 	const framesPerSec = int(time.Second / rtp.DefFrameDur)
 	buf := make(media.PCM16Sample, c.audioCodec.Info().SampleRate/framesPerSec)
-	wr := c.audioOut
+	wr := c.audioOut.NewInput()
+	defer wr.Close()
 
 	ticker := time.NewTicker(rtp.DefFrameDur)
 	defer ticker.Stop()
@@ -612,7 +618,9 @@ func (c *Client) SendSignal(ctx context.Context, n int, val int) error {
 	const framesPerSec = int(time.Second / rtp.DefFrameDur)
 	signal := make(media.PCM16Sample, c.audioCodec.Info().SampleRate/framesPerSec)
 	audiotest.GenSignal(signal, []audiotest.Wave{{Ind: val, Amp: signalAmp}})
-	wr := c.audioOut
+	wr := c.audioOut.NewInput()
+	defer wr.Close()
+
 	c.log.Info("sending signal", "len", len(signal), "n", n, "sig", val)
 
 	ticker := time.NewTicker(rtp.DefFrameDur)

test/client/README.md

@@ -6,10 +6,12 @@ This is a SIP client that sends an INVITE and then sends PCMU audio for a local
 
 At this time no options exist, `go run .` is all that is required. It connects to a SIP server running on the local machine and will send audio.
 
-This program expects a `audio.mkv` to be in the directory. The file should contain PCMU in 20ms samples with a samplerate of 8000 and one channel.
-You can use the command below to convert a file, or use [this](https://siobud.com/audio.mkv) pre-encoded one.
+This program expects a `audio.mkv` to be in the directory. The file should contain PCM s16le in 20ms samples with a sample rate of 8000 and one channel.
+You can use the command below to convert a file, or convert [this](https://siobud.com/audio.mkv) one.
 
 
-### Creating a `audio.mkv` with GStreamer
+### Converting an audio file
 
-`gst-launch-1.0 filesrc location=$AUDIO_FILE ! decodebin ! audiomixer output-buffer-duration=20000000 ! audiorate ! audioresample ! audioconvert ! audio/x-raw, rate=8000, channels=1 ! mulawenc ! matroskamux ! filesink location=audio.mkv`
+`ffmpeg -i $AUDIO_FILE -c:a pcm_s16le audio.mkv`
+
+`gst-launch-1.0 filesrc location=$AUDIO_FILE ! decodebin ! audiomixer output-buffer-duration=20000000 ! audiorate ! audioresample ! audioconvert ! audio/x-raw, format=S16LE, rate=8000, channels=1 ! matroskamux ! filesink location=audio.mkv`

test/cloud/integration_test.go

@@ -45,7 +45,7 @@ func TestSIP(t *testing.T) {
 	defer cancel()
 	go b.SendSilence(ctx)
 
-	go a.SendAudio("audio-pcm.mkv")
+	go a.SendAudio("audio.mkv")
 
 	time.Sleep(time.Second * 5)
 	_ = a.SendDTMF("2345")