Skip to content

Commit 6f93b03

Browse files
GiviMADGiviMAD
authored
[Audio] Fix PCM format and use PipedAudioStream in sources (openhab#16111)
* [Audio] Fix pcm format and use PipedAudioStream * fix rustpotter format changes --------- Signed-off-by: Miguel Álvarez <[email protected]>
1 parent 1d3e7c8 commit 6f93b03

File tree

6 files changed

+82
-168
lines changed

6 files changed

+82
-168
lines changed

‎bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/PulseAudioAudioSource.java

+20-132
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,8 @@
1414

1515
import java.io.IOException;
1616
import java.io.InputStream;
17-
import java.io.InterruptedIOException;
18-
import java.io.PipedInputStream;
19-
import java.io.PipedOutputStream;
2017
import java.net.Socket;
21-
import java.util.HashSet;
2218
import java.util.Set;
23-
import java.util.concurrent.ConcurrentLinkedQueue;
2419
import java.util.concurrent.Future;
2520
import java.util.concurrent.ScheduledExecutorService;
2621

@@ -31,6 +26,7 @@
3126
import org.openhab.core.audio.AudioFormat;
3227
import org.openhab.core.audio.AudioSource;
3328
import org.openhab.core.audio.AudioStream;
29+
import org.openhab.core.audio.PipedAudioStream;
3430
import org.openhab.core.common.ThreadPoolManager;
3531
import org.slf4j.Logger;
3632
import org.slf4j.LoggerFactory;
@@ -45,25 +41,23 @@
4541
public class PulseAudioAudioSource extends PulseaudioSimpleProtocolStream implements AudioSource {
4642

4743
private final Logger logger = LoggerFactory.getLogger(PulseAudioAudioSource.class);
48-
private final ConcurrentLinkedQueue<PipedOutputStream> pipeOutputs = new ConcurrentLinkedQueue<>();
44+
private final PipedAudioStream.Group streamGroup;
4945
private final ScheduledExecutorService executor;
46+
private final AudioFormat streamFormat;
5047

5148
private @Nullable Future<?> pipeWriteTask;
5249

5350
public PulseAudioAudioSource(PulseaudioHandler pulseaudioHandler, ScheduledExecutorService scheduler) {
5451
super(pulseaudioHandler, scheduler);
52+
streamFormat = pulseaudioHandler.getSourceAudioFormat();
5553
executor = ThreadPoolManager
5654
.getScheduledPool("OH-binding-" + pulseaudioHandler.getThing().getUID() + "-source");
55+
streamGroup = PipedAudioStream.newGroup(streamFormat);
5756
}
5857

5958
@Override
6059
public Set<AudioFormat> getSupportedFormats() {
61-
var supportedFormats = new HashSet<AudioFormat>();
62-
var audioFormat = pulseaudioHandler.getSourceAudioFormat();
63-
if (audioFormat != null) {
64-
supportedFormats.add(audioFormat);
65-
}
66-
return supportedFormats;
60+
return Set.of(streamFormat);
6761
}
6862

6963
@Override
@@ -76,27 +70,18 @@ public AudioStream getInputStream(AudioFormat audioFormat) throws AudioException
7670
if (clientSocketLocal == null) {
7771
break;
7872
}
79-
var sourceFormat = pulseaudioHandler.getSourceAudioFormat();
80-
if (sourceFormat == null) {
81-
throw new AudioException("Unable to get source audio format");
82-
}
83-
if (!audioFormat.isCompatible(sourceFormat)) {
73+
if (!audioFormat.isCompatible(streamFormat)) {
8474
throw new AudioException("Incompatible audio format requested");
8575
}
86-
var pipeOutput = new PipedOutputStream();
87-
var pipeInput = new PipedInputStream(pipeOutput, 1024 * 10) {
88-
@Override
89-
public void close() throws IOException {
90-
unregisterPipe(pipeOutput);
91-
super.close();
92-
}
93-
};
94-
registerPipe(pipeOutput);
95-
// get raw audio from the pulse audio socket
96-
return new PulseAudioStream(sourceFormat, pipeInput, () -> {
97-
// ensure pipe is writing
98-
startPipeWrite();
76+
var audioStream = streamGroup.getAudioStreamInGroup();
77+
audioStream.onClose(() -> {
78+
minusClientCount();
79+
stopPipeWriteTask();
9980
});
81+
addClientCount();
82+
startPipeWrite();
83+
// get raw audio from the pulse audio socket
84+
return audioStream;
10085
} catch (IOException e) {
10186
disconnect(); // disconnect to force clear connection in case of socket not cleanly shutdown
10287
if (countAttempt == 2) { // we won't retry : log and quit
@@ -120,14 +105,6 @@ public void close() throws IOException {
120105
throw new AudioException("Unable to create input stream");
121106
}
122107

123-
private synchronized void registerPipe(PipedOutputStream pipeOutput) {
124-
boolean isAdded = this.pipeOutputs.add(pipeOutput);
125-
if (isAdded) {
126-
addClientCount();
127-
}
128-
startPipeWrite();
129-
}
130-
131108
/**
132109
* As startPipeWrite is called for every chunk read,
133110
* this wrapper method make the test before effectively
@@ -143,35 +120,16 @@ private synchronized void startPipeWriteSynchronized() {
143120
if (this.pipeWriteTask == null) {
144121
this.pipeWriteTask = executor.submit(() -> {
145122
int lengthRead;
146-
byte[] buffer = new byte[1024];
123+
byte[] buffer = new byte[1200];
147124
int readRetries = 3;
148-
while (!pipeOutputs.isEmpty()) {
125+
while (!streamGroup.isEmpty()) {
149126
var stream = getSourceInputStream();
150127
if (stream != null) {
151128
try {
152129
lengthRead = stream.read(buffer);
153130
readRetries = 3;
154-
for (var output : pipeOutputs) {
155-
try {
156-
output.write(buffer, 0, lengthRead);
157-
if (pipeOutputs.contains(output)) {
158-
output.flush();
159-
}
160-
} catch (InterruptedIOException e) {
161-
if (pipeOutputs.isEmpty()) {
162-
// task has been ended while writing
163-
return;
164-
}
165-
logger.warn("InterruptedIOException while writing from pulse source to pipe: {}",
166-
getExceptionMessage(e));
167-
} catch (IOException e) {
168-
logger.warn("IOException while writing from pulse source to pipe: {}",
169-
getExceptionMessage(e));
170-
} catch (RuntimeException e) {
171-
logger.warn("RuntimeException while writing from pulse source to pipe: {}",
172-
getExceptionMessage(e));
173-
}
174-
}
131+
streamGroup.write(buffer, 0, lengthRead);
132+
streamGroup.flush();
175133
} catch (IOException e) {
176134
logger.warn("IOException while reading from pulse source: {}", getExceptionMessage(e));
177135
if (readRetries == 0) {
@@ -192,25 +150,9 @@ private synchronized void startPipeWriteSynchronized() {
192150
}
193151
}
194152

195-
private synchronized void unregisterPipe(PipedOutputStream pipeOutput) {
196-
boolean isRemoved = this.pipeOutputs.remove(pipeOutput);
197-
if (isRemoved) {
198-
minusClientCount();
199-
}
200-
try {
201-
Thread.sleep(0);
202-
} catch (InterruptedException ignored) {
203-
}
204-
stopPipeWriteTask();
205-
try {
206-
pipeOutput.close();
207-
} catch (IOException ignored) {
208-
}
209-
}
210-
211153
private synchronized void stopPipeWriteTask() {
212154
var pipeWriteTask = this.pipeWriteTask;
213-
if (pipeOutputs.isEmpty() && pipeWriteTask != null) {
155+
if (streamGroup.isEmpty() && pipeWriteTask != null) {
214156
pipeWriteTask.cancel(true);
215157
this.pipeWriteTask = null;
216158
}
@@ -243,58 +185,4 @@ public void disconnect() {
243185
stopPipeWriteTask();
244186
super.disconnect();
245187
}
246-
247-
static class PulseAudioStream extends AudioStream {
248-
private final Logger logger = LoggerFactory.getLogger(PulseAudioAudioSource.class);
249-
private final AudioFormat format;
250-
private final InputStream input;
251-
private final Runnable activity;
252-
private boolean closed = false;
253-
254-
public PulseAudioStream(AudioFormat format, InputStream input, Runnable activity) {
255-
this.input = input;
256-
this.format = format;
257-
this.activity = activity;
258-
}
259-
260-
@Override
261-
public AudioFormat getFormat() {
262-
return format;
263-
}
264-
265-
@Override
266-
public int read() throws IOException {
267-
byte[] b = new byte[1];
268-
int bytesRead = read(b);
269-
if (-1 == bytesRead) {
270-
return bytesRead;
271-
}
272-
Byte bb = Byte.valueOf(b[0]);
273-
return bb.intValue();
274-
}
275-
276-
@Override
277-
public int read(byte @Nullable [] b) throws IOException {
278-
return read(b, 0, b == null ? 0 : b.length);
279-
}
280-
281-
@Override
282-
public int read(byte @Nullable [] b, int off, int len) throws IOException {
283-
if (b == null) {
284-
throw new IOException("Buffer is null");
285-
}
286-
logger.trace("reading from pulseaudio stream");
287-
if (closed) {
288-
throw new IOException("Stream is closed");
289-
}
290-
activity.run();
291-
return input.read(b, off, len);
292-
}
293-
294-
@Override
295-
public void close() throws IOException {
296-
closed = true;
297-
input.close();
298-
}
299-
}
300188
}

‎bundles/org.openhab.binding.pulseaudio/src/main/java/org/openhab/binding/pulseaudio/internal/handler/PulseaudioHandler.java

+36-25
Original file line numberDiff line numberDiff line change
@@ -469,39 +469,50 @@ public int getSimpleTcpPortAndLoadModuleIfNecessary() throws IOException, Interr
469469
.orElse(simpleTcpPort);
470470
}
471471

472-
public @Nullable AudioFormat getSourceAudioFormat() {
472+
public AudioFormat getSourceAudioFormat() {
473473
String simpleFormat = ((String) getThing().getConfiguration().get(DEVICE_PARAMETER_AUDIO_SOURCE_FORMAT));
474474
BigDecimal simpleRate = ((BigDecimal) getThing().getConfiguration().get(DEVICE_PARAMETER_AUDIO_SOURCE_RATE));
475475
BigDecimal simpleChannels = ((BigDecimal) getThing().getConfiguration()
476476
.get(DEVICE_PARAMETER_AUDIO_SOURCE_CHANNELS));
477+
AudioFormat fallback = new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, false, 16,
478+
16 * 16000, 16000L, 1);
477479
if (simpleFormat == null || simpleRate == null || simpleChannels == null) {
478-
return null;
480+
return fallback;
479481
}
482+
int sampleRateAllChannels = simpleRate.intValue() * simpleChannels.intValue();
480483
switch (simpleFormat) {
481-
case "u8":
482-
return new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_UNSIGNED, null, 8, 1,
483-
simpleRate.longValue(), simpleChannels.intValue());
484-
case "s16le":
485-
return new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, 16, 1,
486-
simpleRate.longValue(), simpleChannels.intValue());
487-
case "s16be":
488-
return new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, true, 16, 1,
489-
simpleRate.longValue(), simpleChannels.intValue());
490-
case "s24le":
491-
return new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, 24, 1,
492-
simpleRate.longValue(), simpleChannels.intValue());
493-
case "s24be":
494-
return new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, true, 24, 1,
495-
simpleRate.longValue(), simpleChannels.intValue());
496-
case "s32le":
497-
return new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, 32, 1,
498-
simpleRate.longValue(), simpleChannels.intValue());
499-
case "s32be":
500-
return new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, true, 32, 1,
501-
simpleRate.longValue(), simpleChannels.intValue());
502-
default:
484+
case "u8" -> {
485+
return new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_UNSIGNED, null, 8,
486+
8 * sampleRateAllChannels, simpleRate.longValue(), simpleChannels.intValue());
487+
}
488+
case "s16le" -> {
489+
return new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, false, 16,
490+
16 * sampleRateAllChannels, simpleRate.longValue(), simpleChannels.intValue());
491+
}
492+
case "s16be" -> {
493+
return new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, true, 16,
494+
16 * sampleRateAllChannels, simpleRate.longValue(), simpleChannels.intValue());
495+
}
496+
case "s24le" -> {
497+
return new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, false, 24,
498+
24 * sampleRateAllChannels, simpleRate.longValue(), simpleChannels.intValue());
499+
}
500+
case "s24be" -> {
501+
return new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, true, 24,
502+
24 * sampleRateAllChannels, simpleRate.longValue(), simpleChannels.intValue());
503+
}
504+
case "s32le" -> {
505+
return new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, false, 32,
506+
32 * sampleRateAllChannels, simpleRate.longValue(), simpleChannels.intValue());
507+
}
508+
case "s32be" -> {
509+
return new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, true, 32,
510+
32 * sampleRateAllChannels, simpleRate.longValue(), simpleChannels.intValue());
511+
}
512+
default -> {
503513
logger.warn("unsupported format {}", simpleFormat);
504-
return null;
514+
return fallback;
515+
}
505516
}
506517
}
507518

‎bundles/org.openhab.voice.googlestt/src/main/java/org/openhab/voice/googlestt/internal/GoogleSTTService.java

+6-8
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.eclipse.jdt.annotation.Nullable;
3030
import org.openhab.core.audio.AudioFormat;
3131
import org.openhab.core.audio.AudioStream;
32+
import org.openhab.core.audio.utils.AudioWaveUtils;
3233
import org.openhab.core.auth.client.oauth2.AccessTokenResponse;
3334
import org.openhab.core.auth.client.oauth2.OAuthClientService;
3435
import org.openhab.core.auth.client.oauth2.OAuthException;
@@ -144,12 +145,8 @@ public Set<Locale> getSupportedLocales() {
144145
@Override
145146
public Set<AudioFormat> getSupportedFormats() {
146147
return Set.of(
147-
new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, 16, null, 16000L),
148-
new AudioFormat(AudioFormat.CONTAINER_OGG, "OPUS", null, null, null, 8000L),
149-
new AudioFormat(AudioFormat.CONTAINER_OGG, "OPUS", null, null, null, 12000L),
150-
new AudioFormat(AudioFormat.CONTAINER_OGG, "OPUS", null, null, null, 16000L),
151-
new AudioFormat(AudioFormat.CONTAINER_OGG, "OPUS", null, null, null, 24000L),
152-
new AudioFormat(AudioFormat.CONTAINER_OGG, "OPUS", null, null, null, 48000L));
148+
new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, false, 16, null, 16000L),
149+
new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, 16, null, 16000L));
153150
}
154151

155152
@Override
@@ -248,8 +245,6 @@ private void streamAudio(ClientStream<StreamingRecognizeRequest> clientStream, A
248245
RecognitionConfig.AudioEncoding streamEncoding;
249246
if (AudioFormat.WAV.isCompatible(streamFormat)) {
250247
streamEncoding = RecognitionConfig.AudioEncoding.LINEAR16;
251-
} else if (AudioFormat.OGG.isCompatible(streamFormat)) {
252-
streamEncoding = RecognitionConfig.AudioEncoding.OGG_OPUS;
253248
} else {
254249
logger.debug("Unsupported format {}", streamFormat);
255250
return;
@@ -271,6 +266,9 @@ private void streamAudio(ClientStream<StreamingRecognizeRequest> clientStream, A
271266
final int bufferSize = 6400;
272267
int numBytesRead;
273268
int remaining = bufferSize;
269+
if (AudioFormat.CONTAINER_WAVE.equals(streamFormat.getContainer())) {
270+
AudioWaveUtils.removeFMT(audioStream);
271+
}
274272
byte[] audioBuffer = new byte[bufferSize];
275273
while (!aborted.get() && !responseObserver.isDone()) {
276274
numBytesRead = audioStream.read(audioBuffer, bufferSize - remaining, remaining);

‎bundles/org.openhab.voice.rustpotterks/src/main/java/org/openhab/voice/rustpotterks/internal/RustpotterKSService.java

+3
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,9 @@ public Set<Locale> getSupportedLocales() {
109109
@Override
110110
public Set<AudioFormat> getSupportedFormats() {
111111
return Set.of(
112+
new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, false, 16, null, 16000L),
113+
new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, null, 16, null, null),
114+
new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, null, 32, null, null),
112115
new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, 16, null, 16000L),
113116
new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, null, 16, null, null),
114117
new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, null, 32, null, null));

‎bundles/org.openhab.voice.voskstt/src/main/java/org/openhab/voice/voskstt/internal/VoskSTTService.java

+7-1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.openhab.core.OpenHAB;
3131
import org.openhab.core.audio.AudioFormat;
3232
import org.openhab.core.audio.AudioStream;
33+
import org.openhab.core.audio.utils.AudioWaveUtils;
3334
import org.openhab.core.common.ThreadPoolManager;
3435
import org.openhab.core.config.core.ConfigurableService;
3536
import org.openhab.core.config.core.Configuration;
@@ -159,6 +160,7 @@ public Set<Locale> getSupportedLocales() {
159160
@Override
160161
public Set<AudioFormat> getSupportedFormats() {
161162
return Set.of(
163+
new AudioFormat(AudioFormat.CONTAINER_NONE, AudioFormat.CODEC_PCM_SIGNED, false, null, null, 16000L),
162164
new AudioFormat(AudioFormat.CONTAINER_WAVE, AudioFormat.CODEC_PCM_SIGNED, false, null, null, 16000L));
163165
}
164166

@@ -167,10 +169,14 @@ public STTServiceHandle recognize(STTListener sttListener, AudioStream audioStre
167169
throws STTException {
168170
AtomicBoolean aborted = new AtomicBoolean(false);
169171
try {
170-
var frequency = audioStream.getFormat().getFrequency();
172+
AudioFormat format = audioStream.getFormat();
173+
var frequency = format.getFrequency();
171174
if (frequency == null) {
172175
throw new IOException("missing audio stream frequency");
173176
}
177+
if (AudioFormat.CONTAINER_WAVE.equals(format.getContainer())) {
178+
AudioWaveUtils.removeFMT(audioStream);
179+
}
174180
backgroundRecognize(sttListener, audioStream, frequency, aborted);
175181
} catch (IOException e) {
176182
throw new STTException(e);

0 commit comments

Comments
 (0)