diff options
| -rw-r--r-- | Makefile.am | 21 | ||||
| -rw-r--r-- | apple/Info.plist | 34 | ||||
| -rw-r--r-- | configure.ac | 53 | ||||
| -rw-r--r-- | pica-client/audiodevice.cpp | 783 | ||||
| -rw-r--r-- | pica-client/audiodevice.h | 86 | ||||
| -rw-r--r-- | pica-client/audioring.h | 181 | ||||
| -rw-r--r-- | pica-client/audiovideocallcontroller.cpp | 28 | ||||
| -rw-r--r-- | pica-client/audiovideocallcontroller.h | 5 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.cpp | 149 | ||||
| -rw-r--r-- | pica-client/dialogs/settingsdialog.h | 24 | ||||
| -rw-r--r-- | pica-client/echocanceller.cpp | 247 | ||||
| -rw-r--r-- | pica-client/echocanceller.h | 125 | ||||
| -rw-r--r-- | pica-client/main.cpp | 10 | ||||
| -rw-r--r-- | pica-client/nativeaudio.cpp | 77 | ||||
| -rw-r--r-- | pica-client/nativeaudio.h | 111 | ||||
| -rw-r--r-- | pica-client/nativeaudio_coreaudio.cpp | 831 | ||||
| -rw-r--r-- | pica-client/nativeaudio_wasapi.cpp | 968 | ||||
| -rw-r--r-- | pica-client/pica-client.pro | 26 | ||||
| -rw-r--r-- | pica-client/toneplayer/toneplayer.cpp | 136 | ||||
| -rw-r--r-- | pica-client/toneplayer/toneplayer.h | 13 |
20 files changed, 3823 insertions, 85 deletions
diff --git a/Makefile.am b/Makefile.am index bc36bdb..16f6363 100644 --- a/Makefile.am +++ b/Makefile.am @@ -103,6 +103,13 @@ pica-client/mediadevice.cpp \ pica-client/mediadevice.h \ pica-client/audiodevice.cpp \ pica-client/audiodevice.h \ +pica-client/audioring.h \ +pica-client/echocanceller.cpp \ +pica-client/echocanceller.h \ +pica-client/nativeaudio.cpp \ +pica-client/nativeaudio.h \ +pica-client/nativeaudio_wasapi.cpp \ +pica-client/nativeaudio_coreaudio.cpp \ pica-client/videodevice.cpp \ pica-client/videodevice.h \ \ @@ -185,8 +192,18 @@ ui_%.h: pica-client/%.ui $(UIC) $< -o $@ -pica_client_pica_client_CPPFLAGS = -DPICA_INSTALLPREFIX=\"$(prefix)\" $(QT_DEPENDENCIES_CFLAGS) $(FFMPEG_CFLAGS) $(PLATFORM_DEPS_CFLAGS) -pica_client_pica_client_LDADD = $(QT_DEPENDENCIES_LIBS) $(FFMPEG_LIBS) $(PLATFORM_DEPS_LIBS) +pica_client_pica_client_CPPFLAGS = -DPICA_INSTALLPREFIX=\"$(prefix)\" $(QT_DEPENDENCIES_CFLAGS) $(FFMPEG_CFLAGS) $(PLATFORM_DEPS_CFLAGS) $(SPEEXDSP_CFLAGS) +pica_client_pica_client_LDADD = $(QT_DEPENDENCIES_LIBS) $(FFMPEG_LIBS) $(PLATFORM_DEPS_LIBS) $(SPEEXDSP_LIBS) + +# Listing PulseAudio sources and sinks in the settings dialog. The two +# native backend sources above are compiled unconditionally and are empty +# preprocessor-wise on the platforms they are not for, but libpulse is a +# real dependency and only linked when configure found it. +if PICA_PULSE +pica_client_pica_client_CPPFLAGS += $(PULSE_CFLAGS) +pica_client_pica_client_LDADD += $(PULSE_LIBS) +MOCFLAGS += -DHAVE_LIBPULSE +endif # VAAPI accelerated encoding, decoding and rendering. Everything below is left # out of the build entirely when configure did not find libva/EGL or was told diff --git a/apple/Info.plist b/apple/Info.plist index b259c6a..c9aeb5a 100644 --- a/apple/Info.plist +++ b/apple/Info.plist @@ -5,7 +5,7 @@ <key>CFBundleName</key> <string>Pica Pica Messenger</string> - + <key>CFBundleDisplayName</key> <string>Pica Pica Messenger</string> @@ -33,5 +33,37 @@ <key>CFBundleIdentifier</key> <string>im.picapica.pica-client</string> + <!-- + Without these two, macOS 10.14 and later kills the process the moment it + opens the microphone or the camera rather than asking the user - so audio + and video calls do not merely fail, they take the application with them. + The strings are what the permission prompt shows as the reason. + + NSMicrophoneUsageDescription covers the capture side of a call, which on + this platform means the VoiceProcessingIO audio unit (see + nativeaudio_coreaudio.cpp). Note that the unit captures and renders + together, so this is also needed for call playback - though not for + ringtones, which deliberately use a plain output unit. + --> + <key>NSMicrophoneUsageDescription</key> + <string>Pica Pica needs the microphone so the person you are calling can hear you.</string> + + <key>NSCameraUsageDescription</key> + <string>Pica Pica needs the camera so the person you are calling can see you.</string> + + <!-- + Qt draws through its own windows rather than the system's, so the app has + to say it is high resolution capable or macOS scales a low resolution + backing store up and everything looks soft on a Retina display. + --> + <key>NSHighResolutionCapable</key> + <true/> + + <key>LSMinimumSystemVersion</key> + <string>10.13</string> + + <key>NSPrincipalClass</key> + <string>NSApplication</string> + </dict> </plist> diff --git a/configure.ac b/configure.ac index 2db202c..aee28ce 100644 --- a/configure.ac +++ b/configure.ac @@ -92,6 +92,14 @@ AC_ARG_ENABLE([vaapi], AS_HELP_STRING([--disable-vaapi], [Disable VAAPI accelerated video encoding, decoding and rendering])) +#checking if PulseAudio device enumeration is wanted. Only affects what the +#settings dialog can list; see the check further down. +AC_ARG_ENABLE([pulse], + AS_HELP_STRING([--enable-pulse], + [Enable listing of PulseAudio/PipeWire capture and playback devices (default: autodetect)]) + AS_HELP_STRING([--disable-pulse], + [Disable listing of PulseAudio/PipeWire devices])) + #checking if the X11 rendering path is wanted #Qt gives widgets a GLX context by default in an X11 session, and a GLX #context has no way to import the DMA-BUF handles a VA surface is exported @@ -134,7 +142,21 @@ AS_CASE([$host], [ PKG_CHECK_MODULES([PLATFORM_DEPS],["alsa"]) ], - [*-w64-mingw*], [QT_DEPENDENCIES_LIBS="$QT_DEPENDENCIES_LIBS -mwindows"], + [*-w64-mingw*], + [ + QT_DEPENDENCIES_LIBS="$QT_DEPENDENCIES_LIBS -mwindows" + #the WASAPI backend is COM: ole32 for CoCreateInstance and the + #task allocator, oleaut32 for PROPVARIANT. + PLATFORM_DEPS_LIBS="$PLATFORM_DEPS_LIBS -lole32 -loleaut32" + ], + [*-apple-darwin*], + [ + #the CoreAudio backend: the HAL for device enumeration, + #AudioUnit/AudioToolbox for the VoiceProcessingIO unit that does + #the echo cancellation, CoreFoundation for the CFString handling + #the HAL's names come back as. + PLATFORM_DEPS_LIBS="$PLATFORM_DEPS_LIBS -framework CoreAudio -framework AudioUnit -framework AudioToolbox -framework CoreFoundation" + ], [] ) AC_SUBST([QT_DEPENDENCIES_CFLAGS]) @@ -155,6 +177,35 @@ PKG_CHECK_MODULES([FFMPEG],["libavcodec libavdevice libavformat libavutil libswr AC_SUBST([FFMPEG_CFLAGS]) AC_SUBST([FFMPEG_LIBS]) +#check for speexdsp, which provides the acoustic echo cancellation used in +#calls. Not optional: on Linux there is no other cancellation in the picture +#at all, and on the platforms that have their own it is still the fallback for +#the endpoints and drivers that turn out not to. +PKG_CHECK_MODULES([SPEEXDSP],[speexdsp]) +AC_SUBST([SPEEXDSP_CFLAGS]) +AC_SUBST([SPEEXDSP_LIBS]) + +#check for libpulse, used only to list PulseAudio sources and sinks in the +#settings dialog - playing and capturing through PulseAudio goes via FFmpeg's +#own "pulse" device and needs nothing from us. Without it the "PulseAudio / +#PipeWire" driver still works, but offers only the server's default device. +AS_IF([test "x$enable_pulse" != "xno"], [ + PKG_CHECK_MODULES([PULSE], [libpulse], + [ + PICA_PULSE_ENABLED=yes + AC_DEFINE([HAVE_LIBPULSE], [1], [Define to 1 if PulseAudio device enumeration is available]) + AC_SUBST([PULSE_CFLAGS]) + AC_SUBST([PULSE_LIBS]) + ], + [ + AS_IF([test "x$enable_pulse" = "xyes"], + [AC_MSG_ERROR(["PulseAudio support was requested but libpulse was not found"])], + [AC_MSG_WARN(["libpulse not found - the PulseAudio driver will only offer the default device"])]) + ]) +]) + +AM_CONDITIONAL([PICA_PULSE], [test "x$PICA_PULSE_ENABLED" = "xyes"]) + #check for VAAPI libraries, unless explicitly disabled #libva is the accelerator itself, libva-drm opens it through a DRM render node, #and EGL is what lets a decoded surface be handed to OpenGL without a copy. diff --git a/pica-client/audiodevice.cpp b/pica-client/audiodevice.cpp index 28f5d0a..f0238e4 100644 --- a/pica-client/audiodevice.cpp +++ b/pica-client/audiodevice.cpp @@ -15,15 +15,21 @@ along with this program. If not, see <https://www.gnu.org/licenses/>. */ #include "audiodevice.h" +#include "nativeaudio.h" #include <QDebug> #include <QMutexLocker> +#include <QVector> #include <cstring> #ifdef Q_OS_LINUX #include <alsa/asoundlib.h> #endif +#ifdef HAVE_LIBPULSE +#include <pulse/pulseaudio.h> +#endif + // FFmpeg headers extern "C" { #include <libavformat/avformat.h> @@ -48,6 +54,15 @@ extern "C" { # define AD_HAVE_CH_LAYOUT 0 #endif +// FFmpeg 7.1 (libavcodec 61.13.100) added avcodec_get_supported_config() and +// deprecated the AVCodec fields listing what a codec supports; FFmpeg 9.0 +// removed those fields. See chooseEncoderSampleFmt(). +#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(61, 13, 100) +# define AD_HAVE_SUPPORTED_CONFIG 1 +#else +# define AD_HAVE_SUPPORTED_CONFIG 0 +#endif + // Every call this codebase currently negotiates is mono - matches // TonePlayer's design and keeps the resampling/framing code below simple. static const int kChannels = 1; @@ -56,6 +71,14 @@ static const int kChannels = 1; // a reasonable fixed default for a single speech channel. static const int64_t kOpusBitrate = 32000; +// The FFmpeg driver name used on Linux. Kept in a process wide variable +// rather than read from the settings database on demand, because the audio +// loops run on their own threads and QSqlDatabase connections belong to the +// thread that opened them. SetLinuxDriverName() is called from the GUI +// thread; see main.cpp and SettingsDialog::storeSettings(). +static QMutex s_driverMutex; +static QString s_linuxDriver = QStringLiteral("alsa"); + static QString ff_errstr(int err) { char buf[256] = {0}; @@ -109,6 +132,47 @@ static SwrContext *makeMonoResampler(AVSampleFormat inFmt, int inRate, AVSampleF return swr; } +// Which sample format to run the Opus encoder in. Interleaved 16 bit is +// preferred over whatever the encoder lists first, because that is the format +// the echo canceller works in - taking it means the microphone samples can go +// through the canceller in place on their way to the encoder rather than +// being converted twice around it. libopus offers it; the native "opus" +// encoder does not, and on that one the AEC has to convert. +static AVSampleFormat chooseEncoderSampleFmt(const AVCodec *enc) +{ + if (!enc) + return AV_SAMPLE_FMT_S16; + + const enum AVSampleFormat *fmts = nullptr; + +#if AD_HAVE_SUPPORTED_CONFIG + // FFmpeg 7.1 (libavcodec 61.13.100) replaced AVCodec's arrays of + // supported formats with this query and deprecated the fields; 9.0 + // removed the fields outright, which is what MXE builds against. + const void *configs = nullptr; + if (avcodec_get_supported_config(nullptr, enc, AV_CODEC_CONFIG_SAMPLE_FORMAT, + 0, &configs, nullptr) < 0) + return AV_SAMPLE_FMT_S16; + + fmts = (const enum AVSampleFormat *)configs; +#else + fmts = enc->sample_fmts; +#endif + + // No list at all means the encoder takes whatever it is given - so it + // takes the one we would rather have. + if (!fmts || fmts[0] == AV_SAMPLE_FMT_NONE) + return AV_SAMPLE_FMT_S16; + + for (int i = 0; fmts[i] != AV_SAMPLE_FMT_NONE; i++) + { + if (fmts[i] == AV_SAMPLE_FMT_S16) + return AV_SAMPLE_FMT_S16; + } + + return fmts[0]; +} + AudioDevice::AudioDevice(QObject *parent) : QObject(parent), m_sampleRate(48000), m_abort(0), m_lastPlayedSeq(0), m_havePlayedSeq(false) @@ -134,6 +198,11 @@ void AudioDevice::configurePlayback(QString deviceName, QString codec, int sampl m_sampleRate = sampleRate; } +void AudioDevice::setEchoCanceller(EchoCancellerPtr ec) +{ + m_echoCanceller = ec; +} + // Difference between two sequence numbers, correct across the point where // they wrap around: positive if a comes after b. static inline int seq_diff(quint16 a, quint16 b) @@ -192,10 +261,62 @@ void AudioDevice::Close() m_queueCond.wakeAll(); } +void AudioDevice::waitForPreroll() +{ + // Preroll: wait for a small cushion of buffered packets before writing + // the first one, so ordinary arrival jitter doesn't immediately drain + // the queue and starve the device buffer (heard as crackling/xruns). + QMutexLocker locker(&m_queueMutex); + + while (m_playQueue.size() < kPrerollDepth && !m_abort.loadRelaxed()) + m_queueCond.wait(&m_queueMutex, 100); +} + +bool AudioDevice::nextPacket(QByteArray *out) +{ + QMutexLocker locker(&m_queueMutex); + + while (m_playQueue.isEmpty() && !m_abort.loadRelaxed()) + m_queueCond.wait(&m_queueMutex, 100); + + if (m_playQueue.isEmpty()) + return !m_abort.loadRelaxed(); + + PlaybackPacket p = m_playQueue.takeFirst(); + + *out = p.data; + m_lastPlayedSeq = p.seq; + m_havePlayedSeq = true; + + return true; +} + void AudioDevice::Capture() { m_abort.storeRelaxed(0); + const QString driver = AudioDevice::PlatformDriverName(CAPTURE); + + if (AudioDevice::IsPlatformDriver(driver)) + captureNative(AudioDevice::PlatformApiName(driver)); + else + captureFFmpeg(driver); +} + +void AudioDevice::Play() +{ + m_abort.storeRelaxed(0); + + const QString driver = AudioDevice::PlatformDriverName(PLAYBACK); + + if (AudioDevice::IsPlatformDriver(driver)) + playNative(AudioDevice::PlatformApiName(driver)); + else + playFFmpeg(driver); +} + +void AudioDevice::captureFFmpeg(const QString &driver) +{ avdevice_register_all(); #if LIBAVFORMAT_VERSION_MAJOR < 58 av_register_all(); @@ -203,11 +324,11 @@ void AudioDevice::Capture() #endif const QByteArray deviceUtf8 = m_deviceName.toUtf8(); - const QByteArray driverUtf8 = AudioDevice::PlatformDriverName().toUtf8(); + const QByteArray driverUtf8 = driver.toUtf8(); const AVInputFormat *ifmt = av_find_input_format(driverUtf8.constData()); if (!ifmt) { - QString msg = QString("Could not find '%1' input driver").arg(AudioDevice::PlatformDriverName()); + QString msg = QString("Could not find '%1' input driver").arg(driver); qWarning() << msg; emit errorOccurred(msg); return; @@ -238,6 +359,18 @@ void AudioDevice::Capture() } AVStream *in_st = ifmt_ctx->streams[0]; + + emit deviceFormatInUse(QString("%1 in %2 Hz %3 ch %4 -> %5 Hz %6 ch") + .arg(driver) + .arg(in_st->codecpar->sample_rate) +#if AD_HAVE_CH_LAYOUT + .arg(in_st->codecpar->ch_layout.nb_channels) +#else + .arg(in_st->codecpar->channels) +#endif + .arg(QLatin1String(avcodec_get_name(in_st->codecpar->codec_id))) + .arg(m_sampleRate).arg(kChannels)); + const AVCodec *pcm_dec = avcodec_find_decoder(in_st->codecpar->codec_id); AVCodecContext *dec_ctx = pcm_dec ? avcodec_alloc_context3(pcm_dec) : nullptr; if (!pcm_dec || !dec_ctx || @@ -267,7 +400,7 @@ void AudioDevice::Capture() } enc_ctx->sample_rate = m_sampleRate; - enc_ctx->sample_fmt = enc->sample_fmts ? enc->sample_fmts[0] : AV_SAMPLE_FMT_S16; + enc_ctx->sample_fmt = chooseEncoderSampleFmt(enc); enc_ctx->bit_rate = kOpusBitrate; enc_ctx->time_base = AVRational{1, m_sampleRate}; applyMonoLayout(enc_ctx); @@ -304,6 +437,20 @@ void AudioDevice::Capture() AVAudioFifo *fifo = av_audio_fifo_alloc(enc_ctx->sample_fmt, kChannels, 1); int frame_size = enc_ctx->frame_size > 0 ? enc_ctx->frame_size : 960; + // The canceller only gets to work in place when the encoder is taking + // interleaved 16 bit, which libopus does. On the native encoder, whose + // only format is planar float, cancelling would mean converting to 16 bit + // and back around every frame; not worth it for a fallback path. + EchoCancellerPtr ec = m_echoCanceller; + if (ec && (!ec->isValid() || enc_ctx->sample_fmt != AV_SAMPLE_FMT_S16 || + ec->sampleRate() != m_sampleRate)) + { + if (ec->isValid()) + qWarning() << "Echo cancellation disabled: encoder format" << av_get_sample_fmt_name(enc_ctx->sample_fmt) + << "at" << m_sampleRate << "Hz is not what the canceller was built for"; + ec.clear(); + } + AVPacket *in_pkt = av_packet_alloc(); AVFrame *dec_frame = av_frame_alloc(); AVPacket *out_pkt = av_packet_alloc(); @@ -371,6 +518,13 @@ void AudioDevice::Capture() av_frame_get_buffer(enc_frame, 0); av_audio_fifo_read(fifo, (void **)enc_frame->data, frame_size); + + // Take the echo of what the playback direction has been + // feeding the speaker back out of the microphone signal, + // before it is encoded and sent. + if (ec) + ec->processNearEnd((qint16 *)enc_frame->data[0], frame_size); + enc_frame->pts = pts; pts += frame_size; @@ -402,10 +556,189 @@ void AudioDevice::Capture() avformat_close_input(&ifmt_ctx); } -void AudioDevice::Play() +void AudioDevice::captureNative(const QString &api) { - m_abort.storeRelaxed(0); + NativeAudioBackend *backend = NativeAudioBackend::create(api); + if (!backend) + { + QString msg = QString("No '%1' audio backend in this build").arg(api); + qWarning() << msg; + emit errorOccurred(msg); + return; + } + + QString err; + if (!backend->openCapture(m_deviceName, m_sampleRate, kChannels, NativeAudioVoiceCall, &err)) + { + qWarning() << err; + emit errorOccurred(err); + delete backend; + return; + } + + emit deviceFormatInUse(backend->formatDescription()); + const AVCodec *enc = avcodec_find_encoder_by_name("libopus"); + if (!enc) + enc = avcodec_find_encoder(AV_CODEC_ID_OPUS); + AVCodecContext *enc_ctx = enc ? avcodec_alloc_context3(enc) : nullptr; + if (!enc || !enc_ctx) + { + QString msg = "Opus encoder not available"; + qWarning() << msg; + emit errorOccurred(msg); + if (enc_ctx) avcodec_free_context(&enc_ctx); + backend->close(); + delete backend; + return; + } + + enc_ctx->sample_rate = m_sampleRate; + enc_ctx->sample_fmt = chooseEncoderSampleFmt(enc); + enc_ctx->bit_rate = kOpusBitrate; + enc_ctx->time_base = AVRational{1, m_sampleRate}; + applyMonoLayout(enc_ctx); + av_opt_set(enc_ctx->priv_data, "application", "voip", 0); + + int ret = avcodec_open2(enc_ctx, enc, nullptr); + if (ret < 0) + { + QString msg = QString("Could not open Opus encoder: %1").arg(ff_errstr(ret)); + qWarning() << msg; + emit errorOccurred(msg); + avcodec_free_context(&enc_ctx); + backend->close(); + delete backend; + return; + } + + // The backend delivers interleaved 16 bit at our rate. Anything but an + // encoder that takes the same needs a converter in between. + SwrContext *swr = nullptr; + if (enc_ctx->sample_fmt != AV_SAMPLE_FMT_S16) + { + swr = makeMonoResampler(AV_SAMPLE_FMT_S16, m_sampleRate, enc_ctx->sample_fmt, m_sampleRate); + if (!swr) + { + QString msg = "Could not set up capture converter"; + qWarning() << msg; + emit errorOccurred(msg); + avcodec_free_context(&enc_ctx); + backend->close(); + delete backend; + return; + } + } + + // Nothing for us to do when the platform has already taken the echo out + // below us - running a second canceller over an already cancelled signal + // only gives it a near silent reference to chase and can chew holes in + // the speech. + EchoCancellerPtr ec = m_echoCanceller; + if (backend->hasPlatformEchoCancellation()) + { + if (ec) + qDebug() << api << "provides echo cancellation - not running ours as well"; + ec.clear(); + } + else if (ec && (!ec->isValid() || ec->sampleRate() != m_sampleRate)) + { + ec.clear(); + } + + const int frame_size = enc_ctx->frame_size > 0 ? enc_ctx->frame_size : 960; + + QVector<qint16> pending; + QVector<qint16> scratch(frame_size * 4); + QVector<qint16> block(frame_size); + + AVPacket *out_pkt = av_packet_alloc(); + int64_t pts = 0; + bool loggedEncodeError = false; + + while (!m_abort.loadRelaxed()) + { + int got = backend->read(scratch.data(), scratch.size(), 100); + if (got < 0) + { + qWarning() << "Native capture read failed"; + break; + } + if (got == 0) + continue; + + int base = pending.size(); + pending.resize(base + got); + memcpy(pending.data() + base, scratch.constData(), got * sizeof(qint16)); + + int consumed = 0; + while (pending.size() - consumed >= frame_size) + { + memcpy(block.data(), pending.constData() + consumed, frame_size * sizeof(qint16)); + consumed += frame_size; + + if (ec) + ec->processNearEnd(block.data(), frame_size); + + AVFrame *enc_frame = av_frame_alloc(); + enc_frame->format = enc_ctx->sample_fmt; + enc_frame->sample_rate = enc_ctx->sample_rate; + enc_frame->nb_samples = frame_size; + applyMonoLayout(enc_frame); + + if (av_frame_get_buffer(enc_frame, 0) < 0) + { + av_frame_free(&enc_frame); + continue; + } + + if (swr) + { + const uint8_t *in[1] = { (const uint8_t *)block.constData() }; + if (swr_convert(swr, enc_frame->data, frame_size, in, frame_size) <= 0) + { + av_frame_free(&enc_frame); + continue; + } + } + else + { + memcpy(enc_frame->data[0], block.constData(), frame_size * sizeof(qint16)); + } + + enc_frame->pts = pts; + pts += frame_size; + + if (avcodec_send_frame(enc_ctx, enc_frame) == 0) + { + while (avcodec_receive_packet(enc_ctx, out_pkt) == 0) + { + emit packetReady(QByteArray((const char *)out_pkt->data, out_pkt->size)); + av_packet_unref(out_pkt); + } + } + else if (!loggedEncodeError) + { + qWarning() << "Opus encode failed"; + loggedEncodeError = true; + } + + av_frame_free(&enc_frame); + } + + if (consumed > 0) + pending.remove(0, consumed); + } + + av_packet_free(&out_pkt); + if (swr) swr_free(&swr); + avcodec_free_context(&enc_ctx); + backend->close(); + delete backend; +} + +void AudioDevice::playFFmpeg(const QString &driver) +{ avdevice_register_all(); #if LIBAVFORMAT_VERSION_MAJOR < 58 av_register_all(); @@ -414,12 +747,12 @@ void AudioDevice::Play() AVFormatContext *oc = nullptr; const QByteArray deviceUtf8 = m_deviceName.toUtf8(); - const QByteArray driverUtf8 = AudioDevice::PlatformDriverName().toUtf8(); + const QByteArray driverUtf8 = driver.toUtf8(); int ret = avformat_alloc_output_context2(&oc, nullptr, driverUtf8.constData(), deviceUtf8.constData()); if (ret < 0 || !oc) { - QString msg = QString("Could not allocate '%1' output context: %2").arg(AudioDevice::PlatformDriverName(), ff_errstr(ret)); + QString msg = QString("Could not allocate '%1' output context: %2").arg(driver, ff_errstr(ret)); qWarning() << msg; emit errorOccurred(msg); return; @@ -467,7 +800,7 @@ void AudioDevice::Play() { if ((ret = avio_open(&oc->pb, deviceUtf8.constData(), AVIO_FLAG_WRITE)) < 0) { - QString msg = QString("Could not open '%1' device '%2': %3").arg(AudioDevice::PlatformDriverName(), m_deviceName, ff_errstr(ret)); + QString msg = QString("Could not open '%1' device '%2': %3").arg(driver, m_deviceName, ff_errstr(ret)); qWarning() << msg; emit errorOccurred(msg); avcodec_free_context(&out_ctx); @@ -508,6 +841,10 @@ void AudioDevice::Play() return; } + EchoCancellerPtr ec = m_echoCanceller; + if (ec && (!ec->isValid() || ec->sampleRate() != m_sampleRate)) + ec.clear(); + SwrContext *swr = nullptr; AVPacket *out_pkt = av_packet_alloc(); AVFrame *dec_frame = av_frame_alloc(); @@ -516,35 +853,15 @@ void AudioDevice::Play() bool loggedResamplerError = false; bool loggedEncodeError = false; - // Preroll: wait for a small cushion of buffered packets before writing - // the first one, so ordinary arrival jitter doesn't immediately drain - // the queue and starve the ALSA buffer (heard as crackling/xruns). - { - QMutexLocker locker(&m_queueMutex); - while (m_playQueue.size() < kPrerollDepth && !m_abort.loadRelaxed()) - m_queueCond.wait(&m_queueMutex, 100); - } + waitForPreroll(); while (true) { QByteArray pktData; - { - QMutexLocker locker(&m_queueMutex); - while (m_playQueue.isEmpty() && !m_abort.loadRelaxed()) - m_queueCond.wait(&m_queueMutex, 100); - - if (m_abort.loadRelaxed() && m_playQueue.isEmpty()) - break; - - if (m_playQueue.isEmpty()) - continue; - - PlaybackPacket p = m_playQueue.takeFirst(); - - pktData = p.data; - m_lastPlayedSeq = p.seq; - m_havePlayedSeq = true; - } + if (!nextPacket(&pktData)) + break; + if (pktData.isEmpty()) + continue; AVPacket *in_pkt = av_packet_alloc(); if (av_new_packet(in_pkt, pktData.size()) < 0) @@ -602,6 +919,12 @@ void AudioDevice::Play() out_frame->pts = pts; pts += converted_samples; + // Hand the canceller the far end signal at the same + // moment the sound card gets it, so the delay it has to + // model is only the device buffer plus the room. + if (ec) + ec->pushFarEnd((const qint16 *)out_frame->data[0], converted_samples); + if (avcodec_send_frame(out_ctx, out_frame) == 0) { while (avcodec_receive_packet(out_ctx, out_pkt) == 0) @@ -634,12 +957,321 @@ void AudioDevice::Play() avformat_free_context(oc); } -QList<MediaDeviceInfo> AudioDevice::Enumerate(enum MediaDeviceStreamDirection dir) +void AudioDevice::playNative(const QString &api) +{ + NativeAudioBackend *backend = NativeAudioBackend::create(api); + if (!backend) + { + QString msg = QString("No '%1' audio backend in this build").arg(api); + qWarning() << msg; + emit errorOccurred(msg); + return; + } + + QString err; + if (!backend->openPlayback(m_deviceName, m_sampleRate, kChannels, NativeAudioVoiceCall, &err)) + { + qWarning() << err; + emit errorOccurred(err); + delete backend; + return; + } + + emit deviceFormatInUse(backend->formatDescription()); + + const AVCodec *in_codec = avcodec_find_decoder(AV_CODEC_ID_OPUS); + AVCodecContext *dec_ctx = in_codec ? avcodec_alloc_context3(in_codec) : nullptr; + if (dec_ctx) + { + dec_ctx->sample_rate = m_sampleRate; + applyMonoLayout(dec_ctx); + } + if (m_codec != QLatin1String("opus") || !in_codec || !dec_ctx || avcodec_open2(dec_ctx, in_codec, nullptr) < 0) + { + QString msg = QString("Unsupported call audio codec: %1").arg(m_codec); + qWarning() << msg; + emit errorOccurred(msg); + if (dec_ctx) avcodec_free_context(&dec_ctx); + backend->close(); + delete backend; + return; + } + + // On a platform that cancels for us, the far end signal it needs is the + // one it is rendering itself - there is nothing to hand to ours, and + // nothing for ours to do. + EchoCancellerPtr ec = m_echoCanceller; + if (ec && (!ec->isValid() || ec->sampleRate() != m_sampleRate)) + ec.clear(); + + SwrContext *swr = nullptr; + AVFrame *dec_frame = av_frame_alloc(); + QVector<qint16> out; + bool loggedDecodeError = false; + bool loggedResamplerError = false; + + waitForPreroll(); + + while (true) + { + QByteArray pktData; + if (!nextPacket(&pktData)) + break; + if (pktData.isEmpty()) + continue; + + AVPacket *in_pkt = av_packet_alloc(); + if (av_new_packet(in_pkt, pktData.size()) < 0) + { + av_packet_free(&in_pkt); + continue; + } + memcpy(in_pkt->data, pktData.constData(), pktData.size()); + + int ret = avcodec_send_packet(dec_ctx, in_pkt); + av_packet_free(&in_pkt); + if (ret < 0) + { + if (!loggedDecodeError) + { + qWarning() << QString("Opus decode failed: %1").arg(ff_errstr(ret)); + loggedDecodeError = true; + } + continue; + } + + while (avcodec_receive_frame(dec_ctx, dec_frame) == 0) + { + if (!swr) + { + swr = makeMonoResampler((AVSampleFormat)dec_frame->format, dec_frame->sample_rate, + AV_SAMPLE_FMT_S16, m_sampleRate); + if (!swr) + { + if (!loggedResamplerError) + { + qWarning() << "Could not set up playback resampler"; + loggedResamplerError = true; + } + av_frame_unref(dec_frame); + continue; + } + } + + int out_samples = (int)av_rescale_rnd(swr_get_delay(swr, dec_frame->sample_rate) + dec_frame->nb_samples, + m_sampleRate, dec_frame->sample_rate, AV_ROUND_UP); + if (out_samples > 0) + { + out.resize(out_samples * kChannels); + + uint8_t *outPlanes[1] = { (uint8_t *)out.data() }; + int converted = swr_convert(swr, outPlanes, out_samples, + (const uint8_t **)dec_frame->data, dec_frame->nb_samples); + if (converted > 0) + { + if (ec) + ec->pushFarEnd(out.constData(), converted * kChannels); + + if (backend->write(out.constData(), converted * kChannels, 100) < 0) + { + qWarning() << "Native playback write failed"; + av_frame_unref(dec_frame); + goto done; + } + } + } + + av_frame_unref(dec_frame); + } + } + +done: + av_frame_free(&dec_frame); + if (swr) swr_free(&swr); + avcodec_free_context(&dec_ctx); + backend->close(); + delete backend; +} + +#ifdef HAVE_LIBPULSE + +// PulseAudio's introspection API is asynchronous, so a one-shot enumeration +// means running a private mainloop until the callbacks below have said they +// are done. Doing it on the caller's thread with its own context keeps this +// self contained - nothing else in pica-client talks to PulseAudio. +struct PulseEnumState +{ + QList<MediaDeviceInfo> *result; + int *index; + // The introspection callback has reported the end of the list, so + // whatever happens to the connection afterwards is just teardown. + bool completed; + bool done; + bool failed; +}; + +static void pulse_source_cb(pa_context *, const pa_source_info *info, int eol, void *userdata) +{ + PulseEnumState *st = (PulseEnumState *)userdata; + + if (eol) + { + st->completed = true; + st->done = true; + return; + } + + if (!info) + return; + + // Monitor sources are the loopback of an output, not a microphone; they + // would show up in the capture list as duplicates of every speaker. + if (info->monitor_of_sink != PA_INVALID_INDEX) + return; + + MediaDeviceInfo d; + d.device = QString::fromUtf8(info->name); + d.humanReadable = QString::fromUtf8(info->description); + d.index = (*st->index)++; + *st->result << d; +} + +static void pulse_sink_cb(pa_context *, const pa_sink_info *info, int eol, void *userdata) +{ + PulseEnumState *st = (PulseEnumState *)userdata; + + if (eol) + { + st->completed = true; + st->done = true; + return; + } + + if (!info) + return; + + MediaDeviceInfo d; + d.device = QString::fromUtf8(info->name); + d.humanReadable = QString::fromUtf8(info->description); + d.index = (*st->index)++; + *st->result << d; +} + +static void pulse_state_cb(pa_context *ctx, void *userdata) +{ + PulseEnumState *st = (PulseEnumState *)userdata; + + // READY is handled by the loop in enumeratePulse(), which is where the + // introspection request gets issued; only the ways this can end early + // need noticing here. + switch (pa_context_get_state(ctx)) + { + case PA_CONTEXT_FAILED: + case PA_CONTEXT_TERMINATED: + if (!st->completed) + st->failed = true; + st->done = true; + break; + + default: + break; + } +} + +static QList<MediaDeviceInfo> enumeratePulse(enum MediaDeviceStreamDirection dir) { QList<MediaDeviceInfo> result; int index = 0; + // PulseAudio's own idea of the default, which follows the user moving + // their default device in the desktop's sound settings. + { + MediaDeviceInfo d; + d.device = QStringLiteral("default"); + d.humanReadable = (dir == PLAYBACK) + ? QObject::tr("Default output (PulseAudio)") + : QObject::tr("Default input (PulseAudio)"); + d.index = index++; + result << d; + } + + pa_mainloop *mainloop = pa_mainloop_new(); + if (!mainloop) + return result; + + pa_context *ctx = pa_context_new(pa_mainloop_get_api(mainloop), "pica-client"); + if (!ctx) + { + pa_mainloop_free(mainloop); + return result; + } + + PulseEnumState st; + st.result = &result; + st.index = &index; + st.completed = false; + st.done = false; + st.failed = false; + + pa_context_set_state_callback(ctx, pulse_state_cb, &st); + + if (pa_context_connect(ctx, nullptr, PA_CONTEXT_NOFLAGS, nullptr) < 0) + { + pa_context_unref(ctx); + pa_mainloop_free(mainloop); + return result; + } + + pa_operation *op = nullptr; + bool requested = false; + + // Bounded so a server that never answers cannot hang the settings dialog. + // One iteration is at most the 50ms below, so this is a 5 second ceiling. + for (int iterations = 0; iterations < 100 && !st.done; iterations++) + { + if (pa_mainloop_prepare(mainloop, 50 * 1000) < 0) + break; + if (pa_mainloop_poll(mainloop) < 0) + break; + if (pa_mainloop_dispatch(mainloop) < 0) + break; + + if (!requested && pa_context_get_state(ctx) == PA_CONTEXT_READY) + { + if (dir == PLAYBACK) + op = pa_context_get_sink_info_list(ctx, pulse_sink_cb, &st); + else + op = pa_context_get_source_info_list(ctx, pulse_source_cb, &st); + + requested = true; + + if (!op) + break; + } + } + + if (op) + pa_operation_unref(op); + + pa_context_disconnect(ctx); + pa_context_unref(ctx); + pa_mainloop_free(mainloop); + + if (st.failed) + qWarning() << "Could not enumerate PulseAudio devices - is a sound server running?"; + + return result; +} + +#endif // HAVE_LIBPULSE + #ifdef Q_OS_LINUX + +static QList<MediaDeviceInfo> enumerateAlsa(enum MediaDeviceStreamDirection dir) +{ + QList<MediaDeviceInfo> result; + int index = 0; + void **hints, **n; char *name = NULL, *descr = NULL, *io = NULL; const char *direction = (dir == PLAYBACK) ? "Output" : "Input"; @@ -673,19 +1305,94 @@ QList<MediaDeviceInfo> AudioDevice::Enumerate(enum MediaDeviceStreamDirection di n++; } snd_device_name_free_hint(hints); + + return result; +} + +#endif // Q_OS_LINUX + +QList<MediaDeviceInfo> AudioDevice::Enumerate(enum MediaDeviceStreamDirection dir) +{ + const QString driver = AudioDevice::PlatformDriverName(dir); + + if (AudioDevice::IsPlatformDriver(driver)) + return NativeAudioBackend::enumerate(AudioDevice::PlatformApiName(driver), dir); + +#ifdef HAVE_LIBPULSE + if (driver == QLatin1String("pulse")) + return enumeratePulse(dir); +#endif + +#ifdef Q_OS_LINUX + if (driver == QLatin1String("alsa")) + return enumerateAlsa(dir); #endif + + // Anything else - "oss", or "pulse" in a build without libpulse - has no + // enumeration of its own worth writing, so the caller gets the one name + // that always works. + QList<MediaDeviceInfo> result; + MediaDeviceInfo d; + d.device = QStringLiteral("default"); + d.humanReadable = QObject::tr("Default device"); + d.index = 0; + result << d; + return result; } -QString AudioDevice::PlatformDriverName() +void AudioDevice::SetLinuxDriverName(const QString &driver) +{ + QMutexLocker locker(&s_driverMutex); + + if (driver == QLatin1String("pulse")) + s_linuxDriver = QStringLiteral("pulse"); + else + s_linuxDriver = QStringLiteral("alsa"); +} + +QString AudioDevice::LinuxDriverName() { + QMutexLocker locker(&s_driverMutex); + return s_linuxDriver; +} + +QString AudioDevice::PlatformDriverName(enum MediaDeviceStreamDirection dir) +{ + Q_UNUSED(dir) + #if defined(Q_OS_LINUX) - return QStringLiteral("alsa"); + // Both directions come from the same setting. ALSA is the default because + // it is the one interface guaranteed to be there; "pulse" is worth + // choosing on a desktop, where it (or PipeWire wearing its face) is what + // actually owns the sound card, and where module-echo-cancel can take the + // job off our hands entirely. + return AudioDevice::LinuxDriverName(); #elif defined(Q_OS_MACOS) || defined(Q_OS_MAC) - return QStringLiteral("audiotoolbox"); + // Not libavdevice's "audiotoolbox". CoreAudio's echo cancellation lives + // in the VoiceProcessingIO audio unit, which renders and captures + // together, and there is no way to reach it through a demuxer and a muxer + // that know nothing about each other. + return QStringLiteral("platform-coreaudio"); #elif defined(Q_OS_WIN) - return QStringLiteral("dsound"); + // DirectSound is a compatibility shim over + // WASAPI these days and cannot mark a stream as belonging to a call, + // which is the only way to get Windows' own echo cancellation. + return QStringLiteral("platform-wasapi"); #else return QStringLiteral("oss"); #endif } + +bool AudioDevice::IsPlatformDriver(const QString &driver) +{ + return driver.startsWith(QLatin1String("platform-")); +} + +QString AudioDevice::PlatformApiName(const QString &driver) +{ + if (!IsPlatformDriver(driver)) + return QString(); + + return driver.mid(sizeof("platform-") - 1); +} diff --git a/pica-client/audiodevice.h b/pica-client/audiodevice.h index 216657a..988438c 100644 --- a/pica-client/audiodevice.h +++ b/pica-client/audiodevice.h @@ -18,6 +18,7 @@ #define AUDIODEVICE_H #include "mediadevice.h" +#include "echocanceller.h" #include <QObject> #include <QString> #include <QByteArray> @@ -26,6 +27,8 @@ #include <QWaitCondition> #include <QAtomicInt> +class NativeAudioBackend; + // A single AudioDevice instance is dedicated to one direction - either // capture+encode (via Capture(), fed from a local hardware input device) or // decode+playback (via Play(), fed from the network via enqueuePacket()). @@ -41,11 +44,22 @@ public: ~AudioDevice(); // Configure before invoking Capture()/Play(). deviceName is a platform - // device identifier as accepted by the FFmpeg driver named by - // PlatformDriverName() (e.g. an ALSA PCM name), or "default". + // device identifier as accepted by the driver named by + // PlatformDriverName() (e.g. an ALSA PCM name, a PulseAudio source name, + // a WASAPI endpoint id, a CoreAudio device UID), or "default". Q_INVOKABLE void configureCapture(QString deviceName, QString codec, int sampleRate); Q_INVOKABLE void configurePlayback(QString deviceName, QString codec, int sampleRate); + // Give this device the call's echo canceller. The capture direction runs + // the microphone signal through it, the playback direction feeds it the + // far end signal; both directions of one call must be given the same + // object or there is nothing to cancel against. + // + // Ignored when the platform is doing its own cancellation - see + // AudioVideoCallController::startAudioPipeline(), which is where that is + // decided. + void setEchoCanceller(EchoCancellerPtr ec); + // Push a codec packet received from the network into the playback // jitter buffer. Safe to call from any thread. Drops the oldest queued // packet if the buffer is full, favoring low latency over completeness. @@ -81,14 +95,53 @@ signals: void errorOccurred(QString message); + // Emitted once a device is open, describing the format it settled on. + // Counterpart of VideoDevice::accelerationInUse(). + void deviceFormatInUse(QString description); + public: virtual QList<MediaDeviceInfo> Enumerate(enum MediaDeviceStreamDirection dir); - // Name of the FFmpeg libavdevice muxer/demuxer for the host platform's - // native audio API (e.g. "alsa" on Linux, "audiotoolbox" on macOS, - // "dsound" on Windows). Shared by every component that opens an audio - // device through FFmpeg (capture, playback, TonePlayer). - static QString PlatformDriverName(); + // Name of the driver to use for one direction of audio. + // + // Two kinds of name come back. A bare name - "alsa", "pulse", "oss" - is + // an FFmpeg libavdevice muxer/demuxer, opened through libavformat the way + // the rest of this codebase opens things. A name starting with + // "platform-" is not an FFmpeg driver at all: what follows the prefix + // names a platform audio API that the caller has to drive itself through + // NativeAudioBackend, because libavdevice cannot express what we need + // from it. + // + // That is how Windows and macOS are handled. Both provide acoustic echo + // cancellation, but only to streams that declare themselves part of a + // voice call - AudioCategory_Communications on WASAPI, the + // VoiceProcessingIO audio unit on CoreAudio - and libavdevice's + // and "audiotoolbox" wrappers have no way to ask for either. Rather than + // open the sound card the plain way and then cancel the echo ourselves on + // platforms that would have done a better job of it below us, we talk to + // those APIs directly. + // + // The direction matters because the answer can differ between them: it is + // the capture side that Windows attaches its processing to, and on Linux + // the two directions can sit on different servers. + static QString PlatformDriverName(enum MediaDeviceStreamDirection dir); + + // True if the driver name is a "platform-<api>" one rather than an FFmpeg + // driver, and the API name out of such a string. + static bool IsPlatformDriver(const QString &driver); + static QString PlatformApiName(const QString &driver); + + // The FFmpeg driver Linux should use, "alsa" or "pulse", from the + // "audio.driver" setting. Meaningless on the other platforms, which have + // exactly one audio API worth using. + // + // The value is cached in a process wide variable rather than read from + // the settings database on demand, because Capture() and Play() run on + // their own threads and a QSqlDatabase connection belongs to the thread + // that opened it. Call the setter from the GUI thread at startup and + // whenever the setting changes. + static void SetLinuxDriverName(const QString &driver); + static QString LinuxDriverName(); private: QString m_deviceName; @@ -97,6 +150,18 @@ private: QAtomicInt m_abort; + // Shared with the other direction of the same call, and held by shared + // pointer because AudioVideoCallController tears a call down without + // joining either thread. + EchoCancellerPtr m_echoCanceller; + + // The FFmpeg and the native halves of each direction. Which one runs is + // decided by PlatformDriverName(). + void captureFFmpeg(const QString &driver); + void captureNative(const QString &api); + void playFFmpeg(const QString &driver); + void playNative(const QString &api); + // Jitter buffer for the playback direction: Play() blocks pulling from // here, enqueuePacket() (called from the controller on delivery of an // incoming network packet) pushes into it. @@ -116,6 +181,13 @@ private: quint16 m_lastPlayedSeq; bool m_havePlayedSeq; + // Shared by both playback paths. waitForPreroll() blocks until the jitter + // buffer has a cushion or the call ends; nextPacket() pops the next + // packet, returning false once Close() has been called and nothing is + // left. A true return with an empty *out is a spurious wakeup - carry on. + void waitForPreroll(); + bool nextPacket(QByteArray *out); + static const int kMaxQueueDepth = 8; // Number of packets Play() waits to have buffered before writing the diff --git a/pica-client/audioring.h b/pica-client/audioring.h new file mode 100644 index 0000000..cbc987f --- /dev/null +++ b/pica-client/audioring.h @@ -0,0 +1,181 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#ifndef AUDIORING_H +#define AUDIORING_H + +#include <QVector> +#include <QAtomicInt> +#include <cstring> + +// Lock-free single-producer / single-consumer ring of 16 bit samples. +// +// The native audio backends (WASAPI, CoreAudio) deliver and ask for audio +// from a callback running on a real time thread owned by the OS. That thread +// must never block, so it cannot take the same mutex as the AudioDevice +// capture/playback loop it is exchanging samples with. Only two threads ever +// touch one ring - the callback on one end, the AudioDevice loop on the +// other - which is exactly the case this structure is safe for. +// +// Overflow (producer faster than consumer) and underflow (the reverse) are +// both normal in an audio path that has just started or has been starved, so +// they are counted rather than treated as errors: the caller decides whether +// to pad with silence or drop. +class AudioRing +{ +public: + // capacity is in samples and is rounded up to a power of two, so the + // index arithmetic below is a mask rather than a modulo. + explicit AudioRing(int capacity = 48000) + : m_read(0), m_write(0), m_overruns(0), m_underruns(0) + { + int cap = 1; + while (cap < capacity) + cap <<= 1; + m_buf.resize(cap); + m_mask = cap - 1; + } + + void clear() + { + m_read.storeRelease(0); + m_write.storeRelease(0); + m_overruns.storeRelaxed(0); + m_underruns.storeRelaxed(0); + } + + int capacity() const { return m_buf.size(); } + + // Samples waiting to be read. Safe to call from either side; the answer + // is a lower bound for the consumer and an upper bound for the producer, + // which is the direction each of them needs it to be conservative in. + int available() const + { + return m_write.loadAcquire() - m_read.loadAcquire(); + } + + int freeSpace() const { return m_buf.size() - available(); } + + int overruns() const { return m_overruns.loadRelaxed(); } + int underruns() const { return m_underruns.loadRelaxed(); } + + // Producer side. Writes as much as fits and drops the rest, counting one + // overrun if anything had to be dropped. Returns the number written. + int write(const qint16 *src, int nsamples) + { + const int w = m_write.loadRelaxed(); + int space = m_buf.size() - (w - m_read.loadAcquire()); + int n = nsamples < space ? nsamples : space; + + if (n < nsamples) + m_overruns.fetchAndAddRelaxed(1); + + for (int i = 0; i < n; ) + { + int idx = (w + i) & m_mask; + int chunk = m_buf.size() - idx; + if (chunk > n - i) + chunk = n - i; + memcpy(m_buf.data() + idx, src + i, chunk * sizeof(qint16)); + i += chunk; + } + + m_write.storeRelease(w + n); + return n; + } + + // Producer side, for a playback ring the render callback drains: fill + // with nsamples of silence, used to keep the timeline moving when there + // is nothing to play. + int writeSilence(int nsamples) + { + const int w = m_write.loadRelaxed(); + int space = m_buf.size() - (w - m_read.loadAcquire()); + int n = nsamples < space ? nsamples : space; + + for (int i = 0; i < n; ) + { + int idx = (w + i) & m_mask; + int chunk = m_buf.size() - idx; + if (chunk > n - i) + chunk = n - i; + memset(m_buf.data() + idx, 0, chunk * sizeof(qint16)); + i += chunk; + } + + m_write.storeRelease(w + n); + return n; + } + + // Consumer side. Returns the number of samples actually read; the caller + // gets no padding, see readOrSilence() for that. + int read(qint16 *dst, int nsamples) + { + const int r = m_read.loadRelaxed(); + int avail = m_write.loadAcquire() - r; + int n = nsamples < avail ? nsamples : avail; + + if (n < nsamples) + m_underruns.fetchAndAddRelaxed(1); + + for (int i = 0; i < n; ) + { + int idx = (r + i) & m_mask; + int chunk = m_buf.size() - idx; + if (chunk > n - i) + chunk = n - i; + memcpy(dst + i, m_buf.constData() + idx, chunk * sizeof(qint16)); + i += chunk; + } + + m_read.storeRelease(r + n); + return n; + } + + // Consumer side. Always fills the whole buffer, padding the tail with + // silence if the ring ran dry - what an output callback needs, since it + // has to hand back a full buffer no matter what. + void readOrSilence(qint16 *dst, int nsamples) + { + int got = read(dst, nsamples); + if (got < nsamples) + memset(dst + got, 0, (nsamples - got) * sizeof(qint16)); + } + + // Consumer side. Throws away everything older than the newest nsamples, + // used to bound the latency of a ring whose producer has outrun it. + void trimTo(int nsamples) + { + int avail = available(); + if (avail > nsamples) + m_read.fetchAndAddRelease(avail - nsamples); + } + +private: + QVector<qint16> m_buf; + int m_mask; + + // Free running counts of samples read and written. They only ever grow, + // so the difference is the fill level; wrapping at 2^31 samples is a bit + // over 12 hours of a 48kHz call and would cost one bad read at worst. + QAtomicInt m_read; + QAtomicInt m_write; + + QAtomicInt m_overruns; + QAtomicInt m_underruns; +}; + +#endif // AUDIORING_H diff --git a/pica-client/audiovideocallcontroller.cpp b/pica-client/audiovideocallcontroller.cpp index 4e91507..1f6e5ce 100644 --- a/pica-client/audiovideocallcontroller.cpp +++ b/pica-client/audiovideocallcontroller.cpp @@ -155,6 +155,29 @@ void AudioVideoCallController::startAudioPipeline() m_audioSeq = 0; m_callClock.start(); + // One canceller for the call, shared by both directions: the playback + // side feeds it what it is about to play, the capture side subtracts the + // echo of that from the microphone. Ownership is shared because + // stopAudioPipeline() does not join either thread, so this has to outlive + // whichever of them returns last. + // + // It is created even on the platforms that cancel for us, because whether + // they really will is not known until the capture stream has been opened + // - see AudioDevice::captureNative(), which drops it once the backend + // says the platform has the job in hand. + if (st.loadValue("audio.echo_cancel", 1).toBool()) + m_echoCanceller = EchoCancellerPtr(new EchoCanceller(48000)); + else + m_echoCanceller.clear(); + + if (m_echoCanceller && !m_echoCanceller->isValid()) + m_echoCanceller.clear(); + + // Set before the blocking loops are started, so the queued Capture()/ + // Play() calls below are what publishes it to those threads. + microphone->setEchoCanceller(m_echoCanceller); + output->setEchoCanceller(m_echoCanceller); + // Declare our outgoing codec to the peer, then start capturing and // encoding right away - the peer's decoder is only ready once it has // processed this 0x74 message, but since the call is carried over TCP, @@ -182,6 +205,11 @@ void AudioVideoCallController::stopAudioPipeline() // exit after a call). microphone->Close(); output->Close(); + + // Drop our reference. The two audio threads still hold theirs until they + // come out of Capture()/Play(), and the canceller goes away with the last + // of them. + m_echoCanceller.clear(); } void AudioVideoCallController::startVideoPipeline() diff --git a/pica-client/audiovideocallcontroller.h b/pica-client/audiovideocallcontroller.h index 2a9a0fa..1dd9f20 100644 --- a/pica-client/audiovideocallcontroller.h +++ b/pica-client/audiovideocallcontroller.h @@ -73,6 +73,11 @@ private: QThread cam_thread; QThread remotevideo_thread; + // Acoustic echo cancellation for the current call, shared by microphone + // and output. Null when it is switched off in the settings, when + // speexdsp could not start one, or between calls. + EchoCancellerPtr m_echoCanceller; + // Sequence number / capture-clock timestamp stamped onto outgoing // 0x76 audio packets (see PICA_PROTO_CALL_PACKET_HDRSIZE). quint16 m_audioSeq; diff --git a/pica-client/dialogs/settingsdialog.cpp b/pica-client/dialogs/settingsdialog.cpp index b950ad4..a319f12 100644 --- a/pica-client/dialogs/settingsdialog.cpp +++ b/pica-client/dialogs/settingsdialog.cpp @@ -44,6 +44,13 @@ static const int kAudioTestSampleRate = 48000; // replays the recording at its natural rate. static const int kAudioTestPacketMs = 20; +// How long to wait for the microphone to produce its first packet before +// giving up on it. Opening a capture device is not instant - on Windows, +// COM plus WASAPI endpoint activation can take the better part of a second - +// so the recording window below is only started once audio is actually +// arriving, and this is the separate limit on getting that far at all. +static const int kAudioTestStartTimeoutMs = 4000; + // Grace period after the last packet has been handed over, for the jitter // buffer and the sound card's own buffer to drain before closing the device. static const int kAudioTestDrainMs = 500; @@ -141,6 +148,31 @@ SettingsDialog::SettingsDialog(QWidget *parent) : multilogintab->setLayout(multiloginlayout); // Audio Devices + audioDriver = nullptr; +#ifdef Q_OS_LINUX + QLabel *lbAudioDriver = new QLabel(tr("Audio system")); + + // ALSA is always there; PulseAudio is what a desktop session normally + // runs (PipeWire included, through its PulseAudio interface) and is the + // one that can hand us a device with the echo already cancelled. + audioDriver = new QComboBox(this); + audioDriver->addItem(tr("ALSA"), QStringLiteral("alsa")); + audioDriver->addItem(tr("PulseAudio / PipeWire"), QStringLiteral("pulse")); + + // Set before the device lists are filled, since Enumerate() asks it which + // driver's names to go looking for. + { + Settings drvst(config_dbname); + QString drv = drvst.loadValue("audio.driver", "alsa").toString(); + AudioDevice::SetLinuxDriverName(drv); + int drvItem = audioDriver->findData(drv); + if (drvItem >= 0) + audioDriver->setCurrentIndex(drvItem); + } + + connect(audioDriver, SIGNAL(currentIndexChanged(int)), this, SLOT(audioDriverChanged(int))); +#endif + QLabel *lbAudioCaptureDev = new QLabel(tr("Microphone Device 🎙️")); QLabel *lbAudioPlaybackDev = new QLabel(tr("Playback device 🎧")); QLabel *lbAudioRingDev = new QLabel(tr("Ring device 🔔☎️")); @@ -163,6 +195,12 @@ SettingsDialog::SettingsDialog(QWidget *parent) : audioTestStatus = new QLabel(this); audioTestStatus->setAlignment(Qt::AlignCenter); + cbEchoCancel = new QCheckBox(tr("Cancel acoustic echo"), this); + cbEchoCancel->setToolTip(tr("Removes the sound of the other person coming back out of your " + "microphone, which is what makes them hear themselves when you " + "are on speakers. Has no effect where the operating system " + "already does it.")); + btRingTest = new QPushButton(tr("Test ring 🔔"), this); connect(btRingTest, SIGNAL(clicked()), this, SLOT(toggleRingTest())); @@ -171,10 +209,15 @@ SettingsDialog::SettingsDialog(QWidget *parent) : // The microphone test covers the capture and playback devices, so it sits // with them, above the unrelated ring device below. +#ifdef Q_OS_LINUX + audiodevlayout->addWidget(lbAudioDriver); + audiodevlayout->addWidget(audioDriver); +#endif audiodevlayout->addWidget(lbAudioCaptureDev); audiodevlayout->addWidget(audioCaptureDev); audiodevlayout->addWidget(lbAudioPlaybackDev); audiodevlayout->addWidget(audioPlaybackDev); + audiodevlayout->addWidget(cbEchoCancel); audiodevlayout->addWidget(btAudioTest); audiodevlayout->addWidget(audioTestStatus); audiodevlayout->addWidget(lbAudioRingDev); @@ -196,6 +239,12 @@ SettingsDialog::SettingsDialog(QWidget *parent) : connect(audioTestRecordTimer, SIGNAL(timeout()), this, SLOT(audioTestRecordingFinished())); audioTestPlaybackTimer = new QTimer(this); + // A coarse timer is no use at this interval. Windows' default system + // timer granularity is 15.6 ms, so a coarse 20 ms timer really fires + // about every 31 ms; PreciseTimer asks for a multimedia timer instead. + // audioTestFeedPacket() does not rely on this being exact, but the closer + // it is the smoother the playback. + audioTestPlaybackTimer->setTimerType(Qt::PreciseTimer); connect(audioTestPlaybackTimer, SIGNAL(timeout()), this, SLOT(audioTestFeedPacket())); audioTestDrainTimer = new QTimer(this); @@ -206,11 +255,13 @@ SettingsDialog::SettingsDialog(QWidget *parent) : testMic->moveToThread(&testMicThread); connect(testMic, SIGNAL(packetReady(QByteArray)), this, SLOT(audioTestPacket(QByteArray))); connect(testMic, SIGNAL(errorOccurred(QString)), this, SLOT(audioTestError(QString))); + connect(testMic, SIGNAL(deviceFormatInUse(QString)), this, SLOT(audioTestPath(QString))); testMicThread.start(); testSpeaker = new AudioDevice(); testSpeaker->moveToThread(&testSpeakerThread); connect(testSpeaker, SIGNAL(errorOccurred(QString)), this, SLOT(audioTestError(QString))); + connect(testSpeaker, SIGNAL(deviceFormatInUse(QString)), this, SLOT(audioTestPath(QString))); testSpeakerThread.start(); ringTestRunning = false; @@ -428,6 +479,7 @@ void SettingsDialog::toggleAudioTest() audioTestPackets.clear(); audioTestPlaybackPos = 0; + audioTestPathReport.clear(); audioTestState = AudioTestRecording; btAudioTest->setText(tr("Stop ⏹")); audioTestStatus->setText(tr("Recording, say something...")); @@ -437,7 +489,11 @@ void SettingsDialog::toggleAudioTest() Q_ARG(int, kAudioTestSampleRate)); QMetaObject::invokeMethod(testMic, "Capture", Qt::QueuedConnection); - audioTestRecordTimer->start(kAudioTestRecordMs); + // Only a watchdog for the device failing to open at all. The recording + // window proper starts in audioTestPacket(), when the first packet shows + // up - otherwise the device's startup time comes out of the five seconds + // and the user gets a noticeably short recording. + audioTestRecordTimer->start(kAudioTestStartTimeoutMs); } void SettingsDialog::audioTestPacket(QByteArray data) @@ -447,6 +503,11 @@ void SettingsDialog::audioTestPacket(QByteArray data) if (audioTestState != AudioTestRecording) return; + // First packet: the microphone is live, so start the recording window now + // rather than when the device was merely asked to open. + if (audioTestPackets.isEmpty()) + audioTestRecordTimer->start(kAudioTestRecordMs); + audioTestPackets.append(data); } @@ -484,7 +545,22 @@ void SettingsDialog::startAudioTestPlayback() audioTestState = AudioTestPlaying; audioTestPlaybackPos = 0; - audioTestStatus->setText(tr("Playing back...")); + + // The packet count is worth showing, not just decoration. Capture emits + // one packet per kAudioTestPacketMs of audio it actually received, so a + // count well under kAudioTestRecordMs/kAudioTestPacketMs means the + // recording lost samples on the way in, and anything wrong with the sound + // is a capture problem. A full count with bad sound means the samples all + // arrived and something downstream is mangling them. + QString summary = tr("Playing back %1 of %2 packets (%3 s)...") + .arg(audioTestPackets.size()) + .arg(kAudioTestRecordMs / kAudioTestPacketMs) + .arg(audioTestPackets.size() * kAudioTestPacketMs / 1000.0, 0, 'f', 1); + + if (!audioTestPathReport.isEmpty()) + summary += QLatin1String("\n") + audioTestPathReport; + + audioTestStatus->setText(summary); QMetaObject::invokeMethod(testSpeaker, "configurePlayback", Qt::QueuedConnection, Q_ARG(QString, dev), Q_ARG(QString, QStringLiteral("opus")), @@ -495,6 +571,7 @@ void SettingsDialog::startAudioTestPlayback() // once: AudioDevice's jitter buffer holds only a few packets and drops // the oldest when full, so dumping the whole recording into it would // discard everything but the tail. + audioTestPlaybackClock.start(); audioTestPlaybackTimer->start(kAudioTestPacketMs); } @@ -503,21 +580,37 @@ void SettingsDialog::audioTestFeedPacket() if (audioTestState != AudioTestPlaying) return; + // Pace against the clock, not against the number of times this has been + // called. A QTimer asked for 20 ms does not necessarily deliver every + // 20 ms - on Windows the system timer granularity is 15.6 ms, so even a + // precise timer can run late - and handing over one packet per tick + // regardless would play the recording back at whatever rate the timer + // happened to run at. Late ticks stretched it out and left the sound card + // with nothing to play in between, which sounded like the recording was + // both slowed down and chopped up. + // + // The real call path has no equivalent problem: there the packets are + // paced by the peer sending them, not by a timer here. + const qint64 due = audioTestPlaybackClock.elapsed() / kAudioTestPacketMs + 1; + + while (audioTestPlaybackPos < audioTestPackets.size() && + (qint64)audioTestPlaybackPos < due) + { + // Direct call, for the same reason as in the call controller. Nothing + // here loses or reorders packets, so the sequence numbers are simply + // the order they were recorded in. + testSpeaker->enqueuePacket((quint16)audioTestPlaybackPos, + audioTestPackets.at(audioTestPlaybackPos)); + audioTestPlaybackPos++; + } + if (audioTestPlaybackPos >= audioTestPackets.size()) { audioTestPlaybackTimer->stop(); // Let what is already queued, plus the sound card's own buffer, play // out before tearing the playback device down. audioTestDrainTimer->start(kAudioTestDrainMs); - return; } - - // Direct call, for the same reason as in the call controller. Nothing - // here loses or reorders packets, so the sequence numbers are simply the - // order they were recorded in. - testSpeaker->enqueuePacket((quint16)audioTestPlaybackPos, - audioTestPackets.at(audioTestPlaybackPos)); - audioTestPlaybackPos++; } void SettingsDialog::audioTestPlaybackFinished() @@ -547,6 +640,16 @@ void SettingsDialog::stopAudioTest() audioTestStatus->clear(); } +void SettingsDialog::audioTestPath(QString description) +{ + if (description.isEmpty()) + return; + + if (!audioTestPathReport.isEmpty()) + audioTestPathReport += QLatin1String("\n"); + audioTestPathReport += description; +} + void SettingsDialog::audioTestError(QString message) { if (audioTestState == AudioTestIdle) @@ -783,6 +886,22 @@ void SettingsDialog::fillVideoDevices() fillDevicesComboBox(videoDev, &vd, CAPTURE); } +void SettingsDialog::audioDriverChanged(int index) +{ + Q_UNUSED(index) + + if (!audioDriver) + return; + + // Take effect straight away rather than on OK, so the device lists below + // show the right names and the Test button uses them. + AudioDevice::SetLinuxDriverName(audioDriver->itemData(audioDriver->currentIndex()).toString()); + + fillAudioCaptureDevices(); + fillAudioPlaybackDevices(); + fillAudioRingDevices(); +} + void SettingsDialog::fillAudioCaptureDevices() { AudioDevice acd; @@ -934,6 +1053,8 @@ void SettingsDialog::loadSettings() if (audioPlaybackDevItem >= 0) audioPlaybackDev->setCurrentIndex(audioPlaybackDevItem); + cbEchoCancel->setChecked(st.loadValue("audio.echo_cancel", 1).toBool()); + QString audioRingDevVal = st.loadValue("audio.ring_device", "default").toString(); int audioRingDevItem = audioRingDev->findData(audioRingDevVal); if (audioRingDevItem >= 0) @@ -986,4 +1107,12 @@ void SettingsDialog::storeSettings() st.storeValue("audio.capture_device", audioCaptureDev->itemData(audioCaptureDev->currentIndex()).toString()); st.storeValue("audio.playback_device", audioPlaybackDev->itemData(audioPlaybackDev->currentIndex()).toString()); st.storeValue("audio.ring_device", audioRingDev->itemData(audioRingDev->currentIndex()).toString()); + st.storeValue("audio.echo_cancel", cbEchoCancel->isChecked() ? "1" : "0"); + + if (audioDriver) + { + QString drv = audioDriver->itemData(audioDriver->currentIndex()).toString(); + st.storeValue("audio.driver", drv); + AudioDevice::SetLinuxDriverName(drv); + } } diff --git a/pica-client/dialogs/settingsdialog.h b/pica-client/dialogs/settingsdialog.h index e9ad558..424d43f 100644 --- a/pica-client/dialogs/settingsdialog.h +++ b/pica-client/dialogs/settingsdialog.h @@ -29,6 +29,7 @@ #include <QImage> #include <QList> #include <QTimer> +#include <QElapsedTimer> #include "../mediadevice.h" #include "../audiodevice.h" #include "../videodevice.h" @@ -68,6 +69,19 @@ private: QComboBox *audioCaptureDev; QComboBox *audioPlaybackDev; QComboBox *audioRingDev; + + // Which audio API to go through. Only Linux has a choice worth offering + // (ALSA or PulseAudio, the latter covering PipeWire's PulseAudio + // interface too), so elsewhere this stays null and no widget is built. + // The member and its slot are declared unconditionally all the same: + // moc runs its own preprocessor and would not see a platform #ifdef, so + // a slot hidden behind one ends up declared but never generated. + QComboBox *audioDriver; + + // Whether to run our own acoustic echo cancellation. Ignored when the + // platform provides one - see AudioDevice::PlatformDriverName(). + QCheckBox *cbEchoCancel; + QComboBox *videoDev; QPushButton *videoDevRefresh; @@ -92,6 +106,9 @@ private: }; AudioTestState audioTestState; + // What the microphone and playback devices reported negotiating, shown + // with the result. Same idea as videoTestPathReport. + QString audioTestPathReport; // The recording, as encoded packets, and how far playback has fed it. QList<QByteArray> audioTestPackets; int audioTestPlaybackPos; @@ -100,6 +117,9 @@ private: // fire into the next one. QTimer *audioTestRecordTimer; QTimer *audioTestPlaybackTimer; + // Playback is paced against this rather than against the number of times + // the timer has fired - see audioTestFeedPacket(). + QElapsedTimer audioTestPlaybackClock; QTimer *audioTestDrainTimer; void stopAudioTest(); @@ -161,6 +181,9 @@ private slots: void toggleIncomingConnections(bool checked); void toggleMultipleLogins(bool checked); void fillVideoDevices(); + // Device names are driver specific, so all three lists have to be filled + // again when the driver changes. Only ever connected on Linux. + void audioDriverChanged(int index); void fillAudioCaptureDevices(); void fillAudioPlaybackDevices(); void fillAudioRingDevices(); @@ -175,6 +198,7 @@ private slots: void audioTestFeedPacket(); void audioTestPlaybackFinished(); void audioTestError(QString message); + void audioTestPath(QString description); void toggleVideoTest(); void videoTestCaptureStarted(QString codec, int width, int height); diff --git a/pica-client/echocanceller.cpp b/pica-client/echocanceller.cpp new file mode 100644 index 0000000..60b427d --- /dev/null +++ b/pica-client/echocanceller.cpp @@ -0,0 +1,247 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#include "echocanceller.h" + +#include <QDebug> +#include <QMutexLocker> +#include <cmath> +#include <cstring> + +extern "C" { +#include <speex/speex_echo.h> +#include <speex/speex_preprocess.h> +} + +EchoCanceller::EchoCanceller(int sampleRate, int frameMs, int tailMs) + : m_sampleRate(sampleRate > 0 ? sampleRate : 48000), + m_frameSize(0), + m_filterLength(0), + m_echo(nullptr), + m_preprocess(nullptr), + m_farHead(0), + m_farCapacity(0), + m_nearEnergy(0.0), + m_outEnergy(0.0) +{ + if (frameMs <= 0) + frameMs = 10; + if (tailMs < frameMs) + tailMs = frameMs; + + m_frameSize = (m_sampleRate * frameMs) / 1000; + m_filterLength = (m_sampleRate * tailMs) / 1000; + + // speexdsp's MDF works in blocks of frame_size and wants the filter to + // be a whole number of them. + if (m_frameSize <= 0) + m_frameSize = 480; + m_filterLength = ((m_filterLength + m_frameSize - 1) / m_frameSize) * m_frameSize; + + // Room for the sound card's output buffer plus a healthy margin. See the + // comment on m_farCapacity in the header for what overflowing it means. + m_farCapacity = m_sampleRate; // one second + + SpeexEchoState *echo = speex_echo_state_init(m_frameSize, m_filterLength); + if (!echo) + { + qWarning() << "Could not create speex echo canceller state"; + return; + } + + int rate = m_sampleRate; + speex_echo_ctl(echo, SPEEX_ECHO_SET_SAMPLING_RATE, &rate); + + SpeexPreprocessState *pre = speex_preprocess_state_init(m_frameSize, m_sampleRate); + if (pre) + { + // The adaptive filter removes the part of the echo it can model; what + // is left over - non-linearities in a cheap speaker, the tail of the + // room past m_filterLength - has to be suppressed statistically, and + // that is what the preprocessor's echo suppressor does with the + // residual estimate the echo state hands it. + speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_STATE, echo); + + int suppress = -40; + speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_SUPPRESS, &suppress); + int suppressActive = -15; + speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_ECHO_SUPPRESS_ACTIVE, &suppressActive); + + // Leave denoise on - it costs little next to the filter and keeps the + // residual suppressor's noise estimate honest. AGC stays off: the + // call has no level control anywhere else, and having one end quietly + // ride the gain makes the echo path non-stationary, which is the one + // thing an adaptive filter cannot follow. + int on = 1, off = 0; + speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_DENOISE, &on); + speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_AGC, &off); + speex_preprocess_ctl(pre, SPEEX_PREPROCESS_SET_VAD, &off); + } + else + { + qWarning() << "Could not create speex preprocessor - running without residual echo suppression"; + } + + m_echo = reinterpret_cast<SpeexEchoState_ *>(echo); + m_preprocess = reinterpret_cast<SpeexPreprocessState_ *>(pre); + + m_farBlock.resize(m_frameSize); + m_farFifo.reserve(m_farCapacity); + + qDebug() << "Echo canceller:" << m_sampleRate << "Hz, frame" << m_frameSize + << "samples, tail" << m_filterLength << "samples (" + << (m_filterLength * 1000 / m_sampleRate) << "ms )"; +} + +EchoCanceller::~EchoCanceller() +{ + // Order matters: the preprocessor holds a pointer to the echo state. + if (m_preprocess) + speex_preprocess_state_destroy(reinterpret_cast<SpeexPreprocessState *>(m_preprocess)); + if (m_echo) + speex_echo_state_destroy(reinterpret_cast<SpeexEchoState *>(m_echo)); +} + +void EchoCanceller::reset() +{ + { + QMutexLocker locker(&m_procMutex); + if (m_echo) + speex_echo_state_reset(reinterpret_cast<SpeexEchoState *>(m_echo)); + } + + { + QMutexLocker locker(&m_farMutex); + m_farFifo.clear(); + m_farHead = 0; + } + + { + QMutexLocker locker(&m_statMutex); + m_nearEnergy = 0.0; + m_outEnergy = 0.0; + } +} + +void EchoCanceller::pushFarEnd(const qint16 *pcm, int nsamples) +{ + if (!m_echo || !pcm || nsamples <= 0) + return; + + QMutexLocker locker(&m_farMutex); + + // Reclaim the space already consumed rather than growing without bound; + // doing it only once the head has run past half the buffer keeps this + // from turning into a memmove on every call. + if (m_farHead > 0 && m_farHead >= m_farFifo.size() / 2) + { + m_farFifo.remove(0, m_farHead); + m_farHead = 0; + } + + int oldSize = m_farFifo.size(); + m_farFifo.resize(oldSize + nsamples); + memcpy(m_farFifo.data() + oldSize, pcm, nsamples * sizeof(qint16)); + + int pending = m_farFifo.size() - m_farHead; + if (pending > m_farCapacity) + { + m_farHead += pending - m_farCapacity; + if (m_farHead >= m_farFifo.size()) + { + m_farFifo.clear(); + m_farHead = 0; + } + } +} + +bool EchoCanceller::popFarEnd(qint16 *dst, int nsamples) +{ + QMutexLocker locker(&m_farMutex); + + int pending = m_farFifo.size() - m_farHead; + if (pending <= 0) + return false; + + int n = pending < nsamples ? pending : nsamples; + memcpy(dst, m_farFifo.constData() + m_farHead, n * sizeof(qint16)); + m_farHead += n; + + if (n < nsamples) + memset(dst + n, 0, (nsamples - n) * sizeof(qint16)); + + return true; +} + +void EchoCanceller::processNearEnd(qint16 *pcm, int nsamples) +{ + if (!m_echo || !pcm || nsamples < m_frameSize) + return; + + QMutexLocker locker(&m_procMutex); + + SpeexEchoState *echo = reinterpret_cast<SpeexEchoState *>(m_echo); + SpeexPreprocessState *pre = reinterpret_cast<SpeexPreprocessState *>(m_preprocess); + + double nearEnergy = 0.0; + double outEnergy = 0.0; + + int blocks = nsamples / m_frameSize; + for (int b = 0; b < blocks; b++) + { + // Not called "near" - windef.h still defines that as an empty macro + // on the mingw-w64 builds. + qint16 *nearBlock = pcm + b * m_frameSize; + + for (int i = 0; i < m_frameSize; i++) + nearEnergy += double(nearBlock[i]) * double(nearBlock[i]); + + if (!popFarEnd(m_farBlock.data(), m_frameSize)) + { + // Nothing is being played, so there is no echo to remove. Still + // run the frame through, otherwise the filter's idea of the far + // end signal silently runs ahead of the microphone. + memset(m_farBlock.data(), 0, m_frameSize * sizeof(qint16)); + } + + // speex_echo_cancellation() is happy to work in place on the near end + // buffer, which is what lets this sit directly in the capture path + // without another copy. + speex_echo_cancellation(echo, nearBlock, m_farBlock.constData(), nearBlock); + + if (pre) + speex_preprocess_run(pre, nearBlock); + + for (int i = 0; i < m_frameSize; i++) + outEnergy += double(nearBlock[i]) * double(nearBlock[i]); + } + + QMutexLocker statLocker(&m_statMutex); + // Exponential forgetting, so erle() reflects the last few seconds rather + // than the whole call. + m_nearEnergy = m_nearEnergy * 0.95 + nearEnergy * 0.05; + m_outEnergy = m_outEnergy * 0.95 + outEnergy * 0.05; +} + +double EchoCanceller::erle() +{ + QMutexLocker locker(&m_statMutex); + + if (m_nearEnergy <= 0.0 || m_outEnergy <= 0.0) + return 0.0; + + return 10.0 * log10(m_nearEnergy / m_outEnergy); +} diff --git a/pica-client/echocanceller.h b/pica-client/echocanceller.h new file mode 100644 index 0000000..476e88e --- /dev/null +++ b/pica-client/echocanceller.h @@ -0,0 +1,125 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#ifndef ECHOCANCELLER_H +#define ECHOCANCELLER_H + +#include <QString> +#include <QMutex> +#include <QVector> +#include <QSharedPointer> + +struct SpeexEchoState_; +struct SpeexPreprocessState_; + +// Acoustic echo cancellation for a call, using speexdsp's frequency domain +// adaptive filter (MDF). +// +// One instance is shared by the two AudioDevice objects of a call - the +// playback one feeds it the far end signal it is about to hand to the sound +// card via pushFarEnd(), the capture one runs the microphone signal through +// processNearEnd() before encoding. Both run on their own thread, so every +// entry point here takes a lock. +// +// Because AudioVideoCallController tears a call down without joining those +// two threads (see stopAudioPipeline()), ownership is shared: hold this +// through a QSharedPointer so the object outlives whichever thread leaves +// last. +// +// Alignment between the two signals is left to the FIFO: in the steady state +// the playback side pushes samples at the same rate the capture side pulls +// them, and the backlog that builds up in the FIFO settles at exactly the +// sound card's buffer depth - which is also how far ahead of the speaker the +// playback side is running. What remains is the acoustic flight time from +// speaker to microphone plus any capture side buffering, and that is what the +// filter tail is for. +class EchoCanceller +{ +public: + // frameMs is the block the filter works on and must divide the audio + // handed to processNearEnd(). tailMs is how much echo delay the filter + // can model - it has to cover the sound card's output buffer plus the + // room, and costs CPU and convergence time in proportion. + EchoCanceller(int sampleRate, int frameMs = 10, int tailMs = 150); + ~EchoCanceller(); + + bool isValid() const { return m_echo != nullptr; } + int sampleRate() const { return m_sampleRate; } + int frameSize() const { return m_frameSize; } + + // Forget the adapted filter and drop any buffered far end audio. Called + // when a call starts, since the previous call's room response is not + // worth keeping and a stale FIFO would start it off misaligned. + void reset(); + + // Playback side: the audio about to be written to the output device. + void pushFarEnd(const qint16 *pcm, int nsamples); + + // Capture side: removes the echo of what pushFarEnd() was given, in + // place. nsamples should be a whole number of frameSize() blocks; a + // partial trailing block is passed through untouched. + void processNearEnd(qint16 *pcm, int nsamples); + + // Rough measure of how well it is working, in dB - how much quieter the + // output is than the microphone input while the far end is active. Zero + // until enough far end audio has gone through to mean anything. + double erle(); + +private: + Q_DISABLE_COPY(EchoCanceller) + + int m_sampleRate; + int m_frameSize; + int m_filterLength; + + // speexdsp state. The echo canceller proper, plus a preprocessor whose + // only job here is the residual echo suppression that the adaptive + // filter alone cannot do (SPEEX_PREPROCESS_SET_ECHO_STATE). + SpeexEchoState_ *m_echo; + SpeexPreprocessState_ *m_preprocess; + + // Guards the speex state, which both directions reach into: the capture + // thread through speex_echo_cancellation(), and reset() from whichever + // thread starts the call. + QMutex m_procMutex; + + // Far end FIFO, written by the playback thread and drained by the + // capture thread. + QMutex m_farMutex; + QVector<qint16> m_farFifo; + int m_farHead; + + // Upper bound on the FIFO, in samples. Reached only if the two sound + // cards' clocks drift apart or playback outruns capture; past this the + // oldest audio is dropped, since keeping it would mean the reference + // signal lagging further and further behind the echo it is meant to + // explain. + int m_farCapacity; + + // Running sums of near end and output energy, for erle(). + double m_nearEnergy; + double m_outEnergy; + QMutex m_statMutex; + + // Scratch far end block, reused by processNearEnd(). + QVector<qint16> m_farBlock; + + bool popFarEnd(qint16 *dst, int nsamples); +}; + +typedef QSharedPointer<EchoCanceller> EchoCancellerPtr; + +#endif // ECHOCANCELLER_H diff --git a/pica-client/main.cpp b/pica-client/main.cpp index f10353e..330f487 100644 --- a/pica-client/main.cpp +++ b/pica-client/main.cpp @@ -37,6 +37,7 @@ #include "dhparam.h" #include "settings.h" #include "dialogs/settingsdialog.h" +#include "audiodevice.h" //globals @@ -447,6 +448,15 @@ static bool create_config_dir() update_database(); QFile::setPermissions(config_dbname, QFile::ReadOwner | QFile::WriteOwner); + + // Publish the audio driver choice to the audio threads while we are still + // the only thread and the settings database is reachable - see + // AudioDevice::SetLinuxDriverName() for why it cannot be read on demand. + { + Settings audiost(config_dbname); + AudioDevice::SetLinuxDriverName(audiost.loadValue("audio.driver", "alsa").toString()); + } + /*if (!QFile::exists(config_dir + QDir::separator() + PICA_CLIENT_DHPARAMFILE)) { OpenSSLTool osslt; diff --git a/pica-client/nativeaudio.cpp b/pica-client/nativeaudio.cpp new file mode 100644 index 0000000..499aad6 --- /dev/null +++ b/pica-client/nativeaudio.cpp @@ -0,0 +1,77 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#include "nativeaudio.h" + +#include <QtGlobal> + +// Per platform factories, defined next to the backends themselves. +#ifdef Q_OS_WIN +NativeAudioBackend *pica_create_wasapi_backend(); +QList<MediaDeviceInfo> pica_enumerate_wasapi(enum MediaDeviceStreamDirection dir); +#endif + +#if defined(Q_OS_MACOS) || defined(Q_OS_MAC) +NativeAudioBackend *pica_create_coreaudio_backend(); +QList<MediaDeviceInfo> pica_enumerate_coreaudio(enum MediaDeviceStreamDirection dir); +#endif + +NativeAudioBackend::~NativeAudioBackend() +{ +} + +bool NativeAudioBackend::isAvailable(const QString &api) +{ +#ifdef Q_OS_WIN + if (api == QLatin1String("wasapi")) + return true; +#endif +#if defined(Q_OS_MACOS) || defined(Q_OS_MAC) + if (api == QLatin1String("coreaudio")) + return true; +#endif + Q_UNUSED(api) + return false; +} + +NativeAudioBackend *NativeAudioBackend::create(const QString &api) +{ +#ifdef Q_OS_WIN + if (api == QLatin1String("wasapi")) + return pica_create_wasapi_backend(); +#endif +#if defined(Q_OS_MACOS) || defined(Q_OS_MAC) + if (api == QLatin1String("coreaudio")) + return pica_create_coreaudio_backend(); +#endif + Q_UNUSED(api) + return nullptr; +} + +QList<MediaDeviceInfo> NativeAudioBackend::enumerate(const QString &api, enum MediaDeviceStreamDirection dir) +{ +#ifdef Q_OS_WIN + if (api == QLatin1String("wasapi")) + return pica_enumerate_wasapi(dir); +#endif +#if defined(Q_OS_MACOS) || defined(Q_OS_MAC) + if (api == QLatin1String("coreaudio")) + return pica_enumerate_coreaudio(dir); +#endif + Q_UNUSED(api) + Q_UNUSED(dir) + return QList<MediaDeviceInfo>(); +} diff --git a/pica-client/nativeaudio.h b/pica-client/nativeaudio.h new file mode 100644 index 0000000..fa3cccb --- /dev/null +++ b/pica-client/nativeaudio.h @@ -0,0 +1,111 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#ifndef NATIVEAUDIO_H +#define NATIVEAUDIO_H + +#include "mediadevice.h" +#include <QString> +#include <QList> + +// Audio input/output through a platform API directly, bypassing FFmpeg's +// libavdevice. +// +// FFmpeg's audio device wrappers open the sound card the plain way, which on +// Windows and macOS means no acoustic echo cancellation: both platforms only +// run their own AEC for streams that were opened declaring themselves part of +// a voice call, and libavdevice has no way to say that. Since that AEC is +// better than anything we can do in process - it sits below us in the audio +// stack and can see the real speaker signal and the real clock - it is worth +// talking to those APIs ourselves. +// +// AudioDevice picks between this and the FFmpeg path on the strength of the +// driver string PlatformDriverName() returns: a name prefixed with +// "platform-" means the rest of it is an apiName() below. +// +// One instance drives one direction. Whether the two directions of a call end +// up sharing anything underneath is the backend's business - on macOS they +// have to, because the AEC there lives in a single audio unit that renders +// and captures at once. + +// What a stream is going to be used for. +// +// A call stream asks the platform for its voice processing, echo cancellation +// above all. A plain one does not, and must not: on macOS the echo canceller +// is an audio unit that captures as well as renders, so asking for it to play +// a ringtone would open the microphone, prompt for permission and light the +// recording indicator, all to make a telephone noise. +enum NativeAudioMode +{ + NativeAudioVoiceCall, + NativeAudioPlain +}; + +class NativeAudioBackend +{ +public: + virtual ~NativeAudioBackend(); + + // The platform API this backend speaks, matching what follows the + // "platform-" prefix in a driver name. + virtual QString apiName() const = 0; + + // device is one of the MediaDeviceInfo::device strings enumerate() + // returned, or "default". On failure *error, if given, is filled in with + // something worth showing the user. + virtual bool openCapture(const QString &device, int sampleRate, int channels, + NativeAudioMode mode, QString *error) = 0; + virtual bool openPlayback(const QString &device, int sampleRate, int channels, + NativeAudioMode mode, QString *error) = 0; + + // Reads up to maxSamples interleaved 16 bit samples, waiting at most + // timeoutMs for the first of them. Returns the number of samples read, + // which may be zero if the timeout ran out, or negative on error. + virtual int read(qint16 *buf, int maxSamples, int timeoutMs) = 0; + + // Hands nsamples interleaved 16 bit samples to the device, waiting up to + // timeoutMs for room. Returns the number accepted - short writes happen + // when the timeout expires with the device buffer still full - or + // negative on error. + virtual int write(const qint16 *buf, int nsamples, int timeoutMs) = 0; + + virtual void close() = 0; + + // One line describing what the stream actually negotiated, for the + // settings dialog to show next to the test result. What a device really + // gave us, as opposed to what we asked it for, is otherwise invisible + // from outside - and it is the first thing worth knowing when a recording + // comes out wrong. Empty when a backend has nothing to add. + virtual QString formatDescription() const { return QString(); } + + // Whether the stream was opened in a mode the platform documents as + // running its own acoustic echo cancellation, in which case the caller + // should not add ours on top. Note this reports what we asked for and + // were granted, which on Windows is not the same as a promise that the + // endpoint's driver actually implements the effect. + virtual bool hasPlatformEchoCancellation() const = 0; + + // True if this build knows the named API at all. + static bool isAvailable(const QString &api); + + // Caller owns the result. Returns null for an API this build has no + // implementation of. + static NativeAudioBackend *create(const QString &api); + + static QList<MediaDeviceInfo> enumerate(const QString &api, enum MediaDeviceStreamDirection dir); +}; + +#endif // NATIVEAUDIO_H diff --git a/pica-client/nativeaudio_coreaudio.cpp b/pica-client/nativeaudio_coreaudio.cpp new file mode 100644 index 0000000..be0d604 --- /dev/null +++ b/pica-client/nativeaudio_coreaudio.cpp @@ -0,0 +1,831 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#include "nativeaudio.h" + +#if defined(Q_OS_MACOS) || defined(Q_OS_MAC) + +#include "audioring.h" + +#include <QDebug> +#include <QObject> +#include <QMutex> +#include <QMutexLocker> +#include <QVector> + +#include <time.h> +#include <string.h> + +#include <CoreAudio/CoreAudio.h> +#include <AudioUnit/AudioUnit.h> +#include <AudioToolbox/AudioToolbox.h> +#include <CoreFoundation/CoreFoundation.h> + +// Bus numbering for an output-type audio unit, which is what both the +// VoiceProcessingIO and the plain HAL output unit are: bus 0 faces the +// speaker, bus 1 the microphone. +static const AudioUnitElement kOutputBus = 0; +static const AudioUnitElement kInputBus = 1; + +// Largest slice a unit will ever hand us. The default is 1156 frames; ask for +// a round number above it and size the scratch buffers to match. +static const UInt32 kMaxFramesPerSlice = 4096; + +static QString cfStringToQString(CFStringRef s) +{ + if (!s) + return QString(); + + CFIndex length = CFStringGetLength(s); + CFIndex maxSize = CFStringGetMaximumSizeForEncoding(length, kCFStringEncodingUTF8) + 1; + QVector<char> buf(maxSize); + + if (!CFStringGetCString(s, buf.data(), maxSize, kCFStringEncodingUTF8)) + return QString(); + + return QString::fromUtf8(buf.constData()); +} + +static QString osStatusString(OSStatus err) +{ + return QString::number((int)err); +} + +// Number of channels a device offers on one scope. Zero means it does not do +// that direction at all, which is how an input-only or output-only device is +// told apart. +static int deviceChannelCount(AudioDeviceID dev, bool input) +{ + AudioObjectPropertyAddress addr; + addr.mSelector = kAudioDevicePropertyStreamConfiguration; + addr.mScope = input ? kAudioDevicePropertyScopeInput : kAudioDevicePropertyScopeOutput; + addr.mElement = kAudioObjectPropertyElementMaster; + + UInt32 size = 0; + if (AudioObjectGetPropertyDataSize(dev, &addr, 0, nullptr, &size) != noErr || size == 0) + return 0; + + QVector<char> raw(size); + AudioBufferList *list = (AudioBufferList *)raw.data(); + if (AudioObjectGetPropertyData(dev, &addr, 0, nullptr, &size, list) != noErr) + return 0; + + int channels = 0; + for (UInt32 i = 0; i < list->mNumberBuffers; i++) + channels += list->mBuffers[i].mNumberChannels; + + return channels; +} + +static QString deviceUid(AudioDeviceID dev) +{ + AudioObjectPropertyAddress addr; + addr.mSelector = kAudioDevicePropertyDeviceUID; + addr.mScope = kAudioObjectPropertyScopeGlobal; + addr.mElement = kAudioObjectPropertyElementMaster; + + CFStringRef uid = nullptr; + UInt32 size = sizeof(uid); + if (AudioObjectGetPropertyData(dev, &addr, 0, nullptr, &size, &uid) != noErr || !uid) + return QString(); + + QString result = cfStringToQString(uid); + CFRelease(uid); + return result; +} + +static QString deviceName(AudioDeviceID dev) +{ + AudioObjectPropertyAddress addr; + addr.mSelector = kAudioObjectPropertyName; + addr.mScope = kAudioObjectPropertyScopeGlobal; + addr.mElement = kAudioObjectPropertyElementMaster; + + CFStringRef name = nullptr; + UInt32 size = sizeof(name); + if (AudioObjectGetPropertyData(dev, &addr, 0, nullptr, &size, &name) != noErr || !name) + return QString(); + + QString result = cfStringToQString(name); + CFRelease(name); + return result; +} + +static QVector<AudioDeviceID> allDevices() +{ + QVector<AudioDeviceID> devices; + + AudioObjectPropertyAddress addr; + addr.mSelector = kAudioHardwarePropertyDevices; + addr.mScope = kAudioObjectPropertyScopeGlobal; + addr.mElement = kAudioObjectPropertyElementMaster; + + UInt32 size = 0; + if (AudioObjectGetPropertyDataSize(kAudioObjectSystemObject, &addr, 0, nullptr, &size) != noErr) + return devices; + + devices.resize(size / sizeof(AudioDeviceID)); + if (devices.isEmpty()) + return devices; + + if (AudioObjectGetPropertyData(kAudioObjectSystemObject, &addr, 0, nullptr, &size, devices.data()) != noErr) + devices.clear(); + + return devices; +} + +// Looks up a device by the UID string enumerate() handed out. +static AudioDeviceID deviceForUid(const QString &uid, bool input) +{ + QVector<AudioDeviceID> devices = allDevices(); + + for (int i = 0; i < devices.size(); i++) + { + if (deviceChannelCount(devices[i], input) <= 0) + continue; + if (deviceUid(devices[i]) == uid) + return devices[i]; + } + + return kAudioObjectUnknown; +} + +// An output-type audio unit and the rings that connect its real time callbacks +// to the AudioDevice loops. +// +// Two shapes of it exist, and the difference is the whole reason this file is +// here. A voice unit is kAudioUnitSubType_VoiceProcessingIO: it renders and +// captures at once and subtracts what it is about to play from what it just +// heard, which is macOS's acoustic echo canceller and is better than anything +// we could run in process. Because that only works when both directions go +// through the same unit, and AudioDevice keeps one object per direction on its +// own thread, the voice unit is a refcounted singleton the two of them share. +// +// A plain unit is kAudioUnitSubType_HALOutput, output only, no voice +// processing, one per caller. Ringtones and earpiece tones use it: they are +// not part of a call, and opening a voice unit to play one would switch the +// microphone on, prompt for permission and light the recording indicator. +// +// One consequence of the singleton worth knowing about: an audio unit is +// bound to one device, so capture and playback cannot be on different sound +// cards while the AEC is in use. When the two selections disagree, the one +// that got there first wins and a warning is logged. +class CoreAudioUnit +{ +public: + // The shared voice unit. forCapture says which direction is attaching; + // release() must later be called with the same value. + static CoreAudioUnit *acquireVoice(const QString &device, int sampleRate, int channels, + bool forCapture, QString *error); + + // A private output only unit with no voice processing. The caller owns + // it and releases it the same way. + static CoreAudioUnit *createPlain(const QString &device, int sampleRate, int channels, + QString *error); + + void release(bool forCapture); + + AudioRing captureRing; + AudioRing playbackRing; + + bool voiceProcessing() const { return m_voiceProcessing; } + +private: + CoreAudioUnit(int sampleRate, int channels, bool voiceProcessing, + bool wantInput, bool wantOutput) + : captureRing(sampleRate * 2), playbackRing(sampleRate * 2), + m_unit(nullptr), m_sampleRate(sampleRate), m_channels(channels), + m_voiceProcessing(voiceProcessing), m_wantInput(wantInput), + m_wantOutput(wantOutput), m_captureRefs(0), m_playbackRefs(0), + m_started(false) + { + } + + ~CoreAudioUnit(); + + bool setUp(const QString &device, QString *error); + void tearDown(); + + static OSStatus inputCallback(void *inRefCon, AudioUnitRenderActionFlags *ioActionFlags, + const AudioTimeStamp *inTimeStamp, UInt32 inBusNumber, + UInt32 inNumberFrames, AudioBufferList *ioData); + static OSStatus renderCallback(void *inRefCon, AudioUnitRenderActionFlags *ioActionFlags, + const AudioTimeStamp *inTimeStamp, UInt32 inBusNumber, + UInt32 inNumberFrames, AudioBufferList *ioData); + + AudioUnit m_unit; + int m_sampleRate; + int m_channels; + bool m_voiceProcessing; + bool m_wantInput; + bool m_wantOutput; + + QString m_device; + + int m_captureRefs; + int m_playbackRefs; + bool m_started; + + // Scratch buffers used only from the audio callbacks, sized once at set + // up time so the real time threads never allocate. + QVector<float> m_inputScratch; + QVector<qint16> m_pcmScratch; + QVector<char> m_inputListStorage; + + // Guards the singleton below and the two refcounts. Plain units are + // reached by one thread only but take it too, which costs nothing. + static QMutex s_mutex; + static CoreAudioUnit *s_voiceInstance; +}; + +QMutex CoreAudioUnit::s_mutex; +CoreAudioUnit *CoreAudioUnit::s_voiceInstance = nullptr; + +CoreAudioUnit::~CoreAudioUnit() +{ + tearDown(); +} + +bool CoreAudioUnit::setUp(const QString &device, QString *error) +{ + AudioComponentDescription desc; + desc.componentType = kAudioUnitType_Output; + desc.componentSubType = m_voiceProcessing ? kAudioUnitSubType_VoiceProcessingIO + : kAudioUnitSubType_HALOutput; + desc.componentManufacturer = kAudioUnitManufacturer_Apple; + desc.componentFlags = 0; + desc.componentFlagsMask = 0; + + AudioComponent comp = AudioComponentFindNext(nullptr, &desc); + if (!comp) + { + if (error) + *error = m_voiceProcessing + ? QStringLiteral("The VoiceProcessingIO audio unit is not available") + : QStringLiteral("The HAL output audio unit is not available"); + return false; + } + + OSStatus err = AudioComponentInstanceNew(comp, &m_unit); + if (err != noErr || !m_unit) + { + if (error) + *error = QString("Could not create the audio unit: %1").arg(osStatusString(err)); + m_unit = nullptr; + return false; + } + + UInt32 one = 1; + UInt32 zero = 0; + + err = AudioUnitSetProperty(m_unit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Input, + kInputBus, m_wantInput ? &one : &zero, sizeof(one)); + if (err != noErr && m_wantInput) + { + if (error) + *error = QString("Could not enable audio input: %1").arg(osStatusString(err)); + return false; + } + + err = AudioUnitSetProperty(m_unit, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Output, + kOutputBus, m_wantOutput ? &one : &zero, sizeof(one)); + if (err != noErr && m_wantOutput) + { + if (error) + *error = QString("Could not enable audio output: %1").arg(osStatusString(err)); + return false; + } + + // Bind to a specific device when one was asked for. Left alone, the unit + // follows the system default input and output, which is what most users + // want and what keeps working when they unplug a headset mid-call. + if (!device.isEmpty() && device != QLatin1String("default")) + { + AudioDeviceID devId = deviceForUid(device, m_wantInput); + if (devId == kAudioObjectUnknown) + devId = deviceForUid(device, !m_wantInput); + + if (devId != kAudioObjectUnknown) + { + err = AudioUnitSetProperty(m_unit, kAudioOutputUnitProperty_CurrentDevice, + kAudioUnitScope_Global, kOutputBus, &devId, sizeof(devId)); + if (err != noErr) + qWarning() << "CoreAudio: could not bind the audio unit to device" << device + << osStatusString(err) << "- using the system default"; + } + else + { + qWarning() << "CoreAudio: device" << device << "not found - using the system default"; + } + } + m_device = device; + + UInt32 maxFrames = kMaxFramesPerSlice; + AudioUnitSetProperty(m_unit, kAudioUnitProperty_MaximumFramesPerSlice, kAudioUnitScope_Global, + 0, &maxFrames, sizeof(maxFrames)); + + // The client side format on both buses. The unit's own converters deal + // with whatever the hardware is really running at, including the sample + // rate, so we can ask for the 48 kHz mono the rest of the call uses. + // Float32 because that is what these units work in natively; going + // through Int16 here would only add a conversion the unit undoes. + AudioStreamBasicDescription fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.mSampleRate = m_sampleRate; + fmt.mFormatID = kAudioFormatLinearPCM; + fmt.mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked; + fmt.mFramesPerPacket = 1; + fmt.mChannelsPerFrame = m_channels; + fmt.mBitsPerChannel = 32; + fmt.mBytesPerFrame = 4 * m_channels; + fmt.mBytesPerPacket = fmt.mBytesPerFrame; + + if (m_wantInput) + { + // Output scope of the input bus is what AudioUnitRender() gives us. + err = AudioUnitSetProperty(m_unit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, + kInputBus, &fmt, sizeof(fmt)); + if (err != noErr) + { + if (error) + *error = QString("Could not set the capture format: %1").arg(osStatusString(err)); + return false; + } + } + + if (m_wantOutput) + { + // Input scope of the output bus is what our render callback supplies. + err = AudioUnitSetProperty(m_unit, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, + kOutputBus, &fmt, sizeof(fmt)); + if (err != noErr) + { + if (error) + *error = QString("Could not set the playback format: %1").arg(osStatusString(err)); + return false; + } + } + + if (m_voiceProcessing) + { + // Make sure the echo cancellation and the rest of the voice + // processing is actually switched on. It is by default, but a bypass + // left set by an earlier instance of the unit would silently cost us + // the whole point of using it. + UInt32 bypass = 0; + AudioUnitSetProperty(m_unit, kAUVoiceIOProperty_BypassVoiceProcessing, + kAudioUnitScope_Global, 0, &bypass, sizeof(bypass)); + + // Leave the unit's automatic gain control off, for the same reason + // the speexdsp path leaves speex's off: a moving gain in front of a + // call that has no level control anywhere else is more trouble than + // it is worth. + UInt32 agc = 0; + AudioUnitSetProperty(m_unit, kAUVoiceIOProperty_VoiceProcessingEnableAGC, + kAudioUnitScope_Global, 0, &agc, sizeof(agc)); + } + + AURenderCallbackStruct cb; + + if (m_wantInput) + { + cb.inputProc = &CoreAudioUnit::inputCallback; + cb.inputProcRefCon = this; + err = AudioUnitSetProperty(m_unit, kAudioOutputUnitProperty_SetInputCallback, + kAudioUnitScope_Global, kInputBus, &cb, sizeof(cb)); + if (err != noErr) + { + if (error) + *error = QString("Could not install the capture callback: %1").arg(osStatusString(err)); + return false; + } + } + + if (m_wantOutput) + { + cb.inputProc = &CoreAudioUnit::renderCallback; + cb.inputProcRefCon = this; + err = AudioUnitSetProperty(m_unit, kAudioUnitProperty_SetRenderCallback, + kAudioUnitScope_Input, kOutputBus, &cb, sizeof(cb)); + if (err != noErr) + { + if (error) + *error = QString("Could not install the playback callback: %1").arg(osStatusString(err)); + return false; + } + } + + m_inputScratch.resize(kMaxFramesPerSlice * m_channels); + m_pcmScratch.resize(kMaxFramesPerSlice * m_channels); + m_inputListStorage.resize(sizeof(AudioBufferList) + sizeof(AudioBuffer)); + + err = AudioUnitInitialize(m_unit); + if (err != noErr) + { + if (error) + *error = QString("Could not initialise the audio unit: %1").arg(osStatusString(err)); + return false; + } + + err = AudioOutputUnitStart(m_unit); + if (err != noErr) + { + if (error) + *error = QString("Could not start the audio unit: %1").arg(osStatusString(err)); + return false; + } + m_started = true; + + qDebug() << "CoreAudio:" << (m_voiceProcessing ? "VoiceProcessingIO" : "HALOutput") + << "started on" << (m_device.isEmpty() ? QStringLiteral("default") : m_device) + << m_sampleRate << "Hz" << m_channels << "ch"; + + return true; +} + +void CoreAudioUnit::tearDown() +{ + if (!m_unit) + return; + + if (m_started) + { + AudioOutputUnitStop(m_unit); + m_started = false; + } + + AudioUnitUninitialize(m_unit); + AudioComponentInstanceDispose(m_unit); + m_unit = nullptr; +} + +OSStatus CoreAudioUnit::inputCallback(void *inRefCon, AudioUnitRenderActionFlags *ioActionFlags, + const AudioTimeStamp *inTimeStamp, UInt32 inBusNumber, + UInt32 inNumberFrames, AudioBufferList *ioData) +{ + Q_UNUSED(ioData) + + CoreAudioUnit *self = (CoreAudioUnit *)inRefCon; + if (!self->m_unit || inNumberFrames == 0 || inNumberFrames > kMaxFramesPerSlice) + return noErr; + + AudioBufferList *list = (AudioBufferList *)self->m_inputListStorage.data(); + list->mNumberBuffers = 1; + list->mBuffers[0].mNumberChannels = self->m_channels; + list->mBuffers[0].mDataByteSize = inNumberFrames * self->m_channels * sizeof(float); + list->mBuffers[0].mData = self->m_inputScratch.data(); + + OSStatus err = AudioUnitRender(self->m_unit, ioActionFlags, inTimeStamp, + inBusNumber, inNumberFrames, list); + if (err != noErr) + return err; + + const int samples = inNumberFrames * self->m_channels; + const float *src = self->m_inputScratch.constData(); + qint16 *dst = self->m_pcmScratch.data(); + + for (int i = 0; i < samples; i++) + { + float v = src[i] * 32767.0f; + if (v > 32767.0f) v = 32767.0f; + else if (v < -32768.0f) v = -32768.0f; + dst[i] = (qint16)v; + } + + // A full ring means the capture loop has stalled; the ring counts the + // overrun and drops what does not fit, which is the right trade for a + // live call. + self->captureRing.write(dst, samples); + + return noErr; +} + +OSStatus CoreAudioUnit::renderCallback(void *inRefCon, AudioUnitRenderActionFlags *ioActionFlags, + const AudioTimeStamp *inTimeStamp, UInt32 inBusNumber, + UInt32 inNumberFrames, AudioBufferList *ioData) +{ + Q_UNUSED(ioActionFlags) + Q_UNUSED(inTimeStamp) + Q_UNUSED(inBusNumber) + + CoreAudioUnit *self = (CoreAudioUnit *)inRefCon; + if (!ioData || ioData->mNumberBuffers < 1) + return noErr; + + const int samples = inNumberFrames * self->m_channels; + if (samples > self->m_pcmScratch.size()) + { + for (UInt32 b = 0; b < ioData->mNumberBuffers; b++) + memset(ioData->mBuffers[b].mData, 0, ioData->mBuffers[b].mDataByteSize); + return noErr; + } + + // Silence for anything the playback loop has not produced yet. Between + // calls, and whenever the peer is quiet, this is all the callback does - + // and a voice unit still wants it, because its echo canceller needs to + // know that nothing is being played. + qint16 *pcm = self->m_pcmScratch.data(); + self->playbackRing.readOrSilence(pcm, samples); + + float *out = (float *)ioData->mBuffers[0].mData; + for (int i = 0; i < samples; i++) + out[i] = pcm[i] / 32768.0f; + + ioData->mBuffers[0].mDataByteSize = samples * sizeof(float); + + // A non-interleaved layout would give us one buffer per channel; the + // format set in setUp() is packed, so anything past the first is not ours + // to fill and is zeroed to be safe. + for (UInt32 b = 1; b < ioData->mNumberBuffers; b++) + memset(ioData->mBuffers[b].mData, 0, ioData->mBuffers[b].mDataByteSize); + + return noErr; +} + +CoreAudioUnit *CoreAudioUnit::acquireVoice(const QString &device, int sampleRate, int channels, + bool forCapture, QString *error) +{ + QMutexLocker locker(&s_mutex); + + if (!s_voiceInstance) + { + CoreAudioUnit *unit = new CoreAudioUnit(sampleRate, channels, true, true, true); + if (!unit->setUp(device, error)) + { + delete unit; + return nullptr; + } + s_voiceInstance = unit; + } + else + { + if (s_voiceInstance->m_sampleRate != sampleRate || s_voiceInstance->m_channels != channels) + { + // Both directions of a call are negotiated together, so this + // would mean two calls at once, which nothing above supports. + if (error) + *error = QStringLiteral("The voice audio unit is already running at another format"); + return nullptr; + } + + if (s_voiceInstance->m_device != device) + { + qWarning() << "CoreAudio: the voice unit is bound to" << s_voiceInstance->m_device + << "so" << device << "cannot also be used - see the note on CoreAudioUnit"; + } + } + + if (forCapture) + { + s_voiceInstance->captureRing.clear(); + s_voiceInstance->m_captureRefs++; + } + else + { + s_voiceInstance->playbackRing.clear(); + s_voiceInstance->m_playbackRefs++; + } + + return s_voiceInstance; +} + +CoreAudioUnit *CoreAudioUnit::createPlain(const QString &device, int sampleRate, int channels, + QString *error) +{ + CoreAudioUnit *unit = new CoreAudioUnit(sampleRate, channels, false, false, true); + if (!unit->setUp(device, error)) + { + delete unit; + return nullptr; + } + + unit->m_playbackRefs = 1; + return unit; +} + +void CoreAudioUnit::release(bool forCapture) +{ + QMutexLocker locker(&s_mutex); + + if (forCapture) + { + if (m_captureRefs > 0) + m_captureRefs--; + } + else + { + if (m_playbackRefs > 0) + m_playbackRefs--; + } + + if (m_captureRefs == 0 && m_playbackRefs == 0) + { + if (s_voiceInstance == this) + s_voiceInstance = nullptr; + delete this; + } +} + +// Waits a millisecond. The units fill and drain their rings from threads we +// do not own and have nothing to signal on, so there is nothing to block on +// but the clock. +static void sleepOneMs() +{ + struct timespec ts = { 0, 1000000 }; + nanosleep(&ts, nullptr); +} + +class CoreAudioBackend : public NativeAudioBackend +{ +public: + CoreAudioBackend() + : m_unit(nullptr), m_isCapture(false), m_channels(1), m_platformAec(false) + { + } + + ~CoreAudioBackend() override + { + close(); + } + + QString apiName() const override { return QStringLiteral("coreaudio"); } + + bool openCapture(const QString &device, int sampleRate, int channels, + NativeAudioMode mode, QString *error) override + { + close(); + + // There is no plain capture path: the only reason this backend + // captures at all is the call, and the call always wants the voice + // unit. + Q_UNUSED(mode) + + m_unit = CoreAudioUnit::acquireVoice(device, sampleRate, channels, true, error); + m_isCapture = true; + m_channels = channels; + m_platformAec = (m_unit != nullptr); + + return m_unit != nullptr; + } + + bool openPlayback(const QString &device, int sampleRate, int channels, + NativeAudioMode mode, QString *error) override + { + close(); + + if (mode == NativeAudioVoiceCall) + m_unit = CoreAudioUnit::acquireVoice(device, sampleRate, channels, false, error); + else + m_unit = CoreAudioUnit::createPlain(device, sampleRate, channels, error); + + m_isCapture = false; + m_channels = channels; + m_platformAec = false; + + return m_unit != nullptr; + } + + int read(qint16 *buf, int maxSamples, int timeoutMs) override + { + if (!m_unit || !m_isCapture) + return -1; + + int waited = 0; + for (;;) + { + int avail = m_unit->captureRing.available(); + if (avail > 0) + { + int n = avail < maxSamples ? avail : maxSamples; + return m_unit->captureRing.read(buf, n); + } + + if (timeoutMs >= 0 && waited >= timeoutMs) + return 0; + + sleepOneMs(); + waited++; + } + } + + int write(const qint16 *buf, int nsamples, int timeoutMs) override + { + if (!m_unit || m_isCapture) + return -1; + + int done = 0; + int waited = 0; + + while (done < nsamples) + { + done += m_unit->playbackRing.write(buf + done, nsamples - done); + + if (done >= nsamples) + break; + + if (timeoutMs >= 0 && waited >= timeoutMs) + break; + + sleepOneMs(); + waited++; + } + + return done; + } + + void close() override + { + if (m_unit) + { + m_unit->release(m_isCapture); + m_unit = nullptr; + } + m_platformAec = false; + } + + // The whole reason this backend exists. The voice unit's canceller is + // unconditional once the unit is running and not bypassed. + bool hasPlatformEchoCancellation() const override { return m_platformAec; } + + QString formatDescription() const override + { + if (!m_unit) + return QString(); + + return QString("CoreAudio %1 %2, %3 ch%4") + .arg(m_isCapture ? "in" : "out") + .arg(m_unit->voiceProcessing() ? "VoiceProcessingIO" : "HALOutput") + .arg(m_channels) + .arg(m_platformAec ? ", platform AEC" : ""); + } + +private: + CoreAudioUnit *m_unit; + bool m_isCapture; + int m_channels; + bool m_platformAec; +}; + +NativeAudioBackend *pica_create_coreaudio_backend() +{ + return new CoreAudioBackend(); +} + +QList<MediaDeviceInfo> pica_enumerate_coreaudio(enum MediaDeviceStreamDirection dir) +{ + QList<MediaDeviceInfo> result; + int index = 0; + + // The system default first, under the same "default" name every other + // driver in this codebase uses for it. On macOS it is also the only + // selection that lets the voice unit follow the user switching headsets + // during a call. + { + MediaDeviceInfo d; + d.device = QStringLiteral("default"); + d.humanReadable = (dir == PLAYBACK) + ? QObject::tr("System default output") + : QObject::tr("System default input"); + d.index = index++; + result << d; + } + + const bool wantInput = (dir != PLAYBACK); + QVector<AudioDeviceID> devices = allDevices(); + + for (int i = 0; i < devices.size(); i++) + { + if (deviceChannelCount(devices[i], wantInput) <= 0) + continue; + + QString uid = deviceUid(devices[i]); + if (uid.isEmpty()) + continue; + + MediaDeviceInfo d; + d.device = uid; + d.humanReadable = deviceName(devices[i]); + if (d.humanReadable.isEmpty()) + d.humanReadable = uid; + d.index = index++; + + result << d; + } + + return result; +} + +#endif // Q_OS_MACOS diff --git a/pica-client/nativeaudio_wasapi.cpp b/pica-client/nativeaudio_wasapi.cpp new file mode 100644 index 0000000..449dcd7 --- /dev/null +++ b/pica-client/nativeaudio_wasapi.cpp @@ -0,0 +1,968 @@ +/* + (c) Copyright 2012 - 2026 Anton Sviridenko + https://picapica.im + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, version 3. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. +*/ +#include "nativeaudio.h" + +#ifdef Q_OS_WIN + +#include <QDebug> +#include <QObject> +#include <QVector> +#include <cstring> + +// INITGUID has to come first so the CLSIDs and IIDs used below get defined +// here rather than expecting a separate GUID library at link time. +#define INITGUID +#include <initguid.h> + +#include <windows.h> +#include <objbase.h> +#include <mmreg.h> +#include <mmdeviceapi.h> +#include <audioclient.h> +#include <functiondiscoverykeys_devpkey.h> + +extern "C" { +#include <libavutil/channel_layout.h> +#include <libavutil/mathematics.h> +#include <libavutil/samplefmt.h> +#include <libswresample/swresample.h> +} + +#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(57, 24, 100) +# define NA_HAVE_CH_LAYOUT 1 +#else +# define NA_HAVE_CH_LAYOUT 0 +#endif + +// How much of a buffer to ask WASAPI for. Shared mode rounds this up to the +// engine period anyway; 40 ms is two of our 20 ms call packets, enough that a +// scheduling hiccup does not turn into a glitch without adding audible delay. +static const REFERENCE_TIME kBufferDuration = 400000; // in 100ns units + +static QString hrString(HRESULT hr) +{ + return QString("0x%1").arg((quint32)hr, 8, 16, QLatin1Char('0')); +} + +static QString wideToQString(LPCWSTR s) +{ + return s ? QString::fromWCharArray(s) : QString(); +} + +// Recovers the WAVE_FORMAT_* tag from a WAVEFORMATEXTENSIBLE subformat GUID. +// +// The subformats a shared mode mix format uses are the generic ones derived +// from a wave format tag: the tag sits in Data1 and everything after it is a +// fixed suffix. Reading it out that way rather than comparing against +// KSDATAFORMAT_SUBTYPE_PCM and KSDATAFORMAT_SUBTYPE_IEEE_FLOAT is deliberate - +// mingw-w64 declares those two but does not define them, the definitions +// being in ksuser, and INITGUID does not help because they are spelled with +// DEFINE_GUIDSTRUCT rather than DEFINE_GUID. Not worth a link dependency for +// two constants whose value is this predictable. +// +// Returns false for a subformat that is not one of the generic family, which +// is a codec (AC-3 and such) rather than something we could hand PCM to. +static bool ksSubFormatTag(const GUID *g, WORD *tag) +{ + static const unsigned char suffix[8] = + { 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71 }; + + if (g->Data2 != 0x0000 || g->Data3 != 0x0010) + return false; + + if (memcmp(g->Data4, suffix, sizeof(suffix)) != 0) + return false; + + if (g->Data1 > 0xFFFF) + return false; + + *tag = (WORD)g->Data1; + return true; +} + +// Works out which AVSampleFormat a WASAPI mix format corresponds to. +// Shared mode engines are float32 in practice, but integer formats turn up on +// some drivers and cost nothing to support. Returns AV_SAMPLE_FMT_NONE for +// anything we cannot describe, which sends the caller back to reporting a +// plain unsupported-format failure. +static AVSampleFormat waveFormatToAv(const WAVEFORMATEX *wfx) +{ + WORD tag = wfx->wFormatTag; + + if (tag == WAVE_FORMAT_EXTENSIBLE && wfx->cbSize >= 22) + { + const WAVEFORMATEXTENSIBLE *ext = (const WAVEFORMATEXTENSIBLE *)wfx; + if (!ksSubFormatTag(&ext->SubFormat, &tag)) + return AV_SAMPLE_FMT_NONE; + } + + if (tag == WAVE_FORMAT_IEEE_FLOAT) + return wfx->wBitsPerSample == 64 ? AV_SAMPLE_FMT_DBL : AV_SAMPLE_FMT_FLT; + + if (tag == WAVE_FORMAT_PCM) + { + switch (wfx->wBitsPerSample) + { + case 16: return AV_SAMPLE_FMT_S16; + case 32: return AV_SAMPLE_FMT_S32; + default: return AV_SAMPLE_FMT_NONE; + } + } + + return AV_SAMPLE_FMT_NONE; +} + +static SwrContext *makeResampler(AVSampleFormat inFmt, int inRate, int inCh, + AVSampleFormat outFmt, int outRate, int outCh) +{ + SwrContext *swr = nullptr; + +#if NA_HAVE_CH_LAYOUT + AVChannelLayout inLayout, outLayout; + av_channel_layout_default(&inLayout, inCh); + av_channel_layout_default(&outLayout, outCh); + int ret = swr_alloc_set_opts2(&swr, &outLayout, outFmt, outRate, + &inLayout, inFmt, inRate, 0, nullptr); + av_channel_layout_uninit(&inLayout); + av_channel_layout_uninit(&outLayout); + if (ret < 0) + return nullptr; +#else + swr = swr_alloc_set_opts(nullptr, + av_get_default_channel_layout(outCh), outFmt, outRate, + av_get_default_channel_layout(inCh), inFmt, inRate, + 0, nullptr); + if (!swr) + return nullptr; +#endif + + if (swr_init(swr) < 0) + { + swr_free(&swr); + return nullptr; + } + + return swr; +} + +class WasapiBackend : public NativeAudioBackend +{ +public: + WasapiBackend() + : m_comInitialized(false), m_client(nullptr), m_capture(nullptr), + m_render(nullptr), m_event(nullptr), m_mixFormat(nullptr), + m_isCapture(false), m_sampleRate(48000), m_channels(1), + m_devSampleRate(48000), m_devChannels(1), m_devFormat(AV_SAMPLE_FMT_S16), + m_devBytesPerFrame(2), m_swr(nullptr), m_bufferFrames(0), + m_platformAec(false), m_started(false), m_spillOffset(0) + { + } + + ~WasapiBackend() override + { + close(); + } + + QString apiName() const override { return QStringLiteral("wasapi"); } + + bool openCapture(const QString &device, int sampleRate, int channels, + NativeAudioMode mode, QString *error) override + { + return open(device, sampleRate, channels, true, mode, error); + } + + bool openPlayback(const QString &device, int sampleRate, int channels, + NativeAudioMode mode, QString *error) override + { + return open(device, sampleRate, channels, false, mode, error); + } + + int read(qint16 *buf, int maxSamples, int timeoutMs) override; + int write(const qint16 *buf, int nsamples, int timeoutMs) override; + void close() override; + + bool hasPlatformEchoCancellation() const override { return m_platformAec; } + + QString formatDescription() const override + { + if (!m_client) + return QString(); + + return QString("WASAPI %1 %2 Hz %3 ch %4 -> %5 Hz %6 ch s16, buffer %7 fr%8") + .arg(m_isCapture ? "in" : "out") + .arg(m_devSampleRate).arg(m_devChannels) + .arg(QLatin1String(av_get_sample_fmt_name(m_devFormat))) + .arg(m_sampleRate).arg(m_channels) + .arg(m_bufferFrames) + .arg(m_platformAec ? ", platform AEC" : ""); + } + +private: + bool open(const QString &device, int sampleRate, int channels, bool isCapture, + NativeAudioMode mode, QString *error); + bool initCom(); + IMMDevice *resolveDevice(const QString &device, bool isCapture, NativeAudioMode mode, QString *error); + void requestCommunicationsCategory(); + int appendConverted(const BYTE *deviceData, int deviceFrames, bool silent, + qint16 *buf, int maxSamples, int written); + + bool m_comInitialized; + IAudioClient *m_client; + IAudioCaptureClient *m_capture; + IAudioRenderClient *m_render; + HANDLE m_event; + WAVEFORMATEX *m_mixFormat; + + bool m_isCapture; + + // What our own pipeline speaks: interleaved 16 bit, m_channels wide. + int m_sampleRate; + int m_channels; + + // What the stream was actually opened with. Equal to the above whenever + // WASAPI accepted our format directly, which is the normal case; when it + // did not, m_swr bridges the two. + int m_devSampleRate; + int m_devChannels; + AVSampleFormat m_devFormat; + int m_devBytesPerFrame; + SwrContext *m_swr; + + UINT32 m_bufferFrames; + bool m_platformAec; + bool m_started; + + // WASAPI hands capture audio over one packet at a time and will not let a + // packet be released partially, so whatever the caller did not have room + // for is kept here until the next read(). + QVector<qint16> m_spill; + int m_spillOffset; + + // Scratch space for format conversion, kept between calls so the audio + // loop does not allocate. + QVector<qint16> m_convIn; + QVector<quint8> m_convOut; +}; + +bool WasapiBackend::initCom() +{ + // Every thread that touches COM has to initialise it, and Capture() and + // Play() each run on their own QThread. MULTITHREADED is what the WASAPI + // samples use and what suits a worker thread with no message loop. + HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + if (hr == RPC_E_CHANGED_MODE) + { + // Somebody else already put this thread in a single threaded + // apartment. That is still usable, but it is not ours to undo. + return true; + } + if (FAILED(hr)) + return false; + + m_comInitialized = true; + return true; +} + +IMMDevice *WasapiBackend::resolveDevice(const QString &device, bool isCapture, + NativeAudioMode mode, QString *error) +{ + IMMDeviceEnumerator *devEnum = nullptr; + HRESULT hr = CoCreateInstance(CLSID_MMDeviceEnumerator, nullptr, CLSCTX_ALL, + IID_IMMDeviceEnumerator, (void **)&devEnum); + if (FAILED(hr) || !devEnum) + { + if (error) + *error = QString("Could not create the audio endpoint enumerator: %1").arg(hrString(hr)); + return nullptr; + } + + IMMDevice *dev = nullptr; + if (device.isEmpty() || device == QLatin1String("default")) + { + // eCommunications rather than eConsole for a call: this is the + // endpoint the user nominated for calls, and asking for it by that + // role is part of what tells Windows to treat the stream as voice. + // A ringtone is not a call and belongs on the console endpoint, so + // that it is heard on the speakers rather than in a headset the user + // is not wearing yet. + const ERole role = (mode == NativeAudioVoiceCall) ? eCommunications : eConsole; + hr = devEnum->GetDefaultAudioEndpoint(isCapture ? eCapture : eRender, role, &dev); + } + else + { + hr = devEnum->GetDevice((LPCWSTR)device.utf16(), &dev); + } + + devEnum->Release(); + + if (FAILED(hr) || !dev) + { + if (error) + *error = QString("Could not open audio endpoint '%1': %2").arg(device, hrString(hr)); + return nullptr; + } + + return dev; +} + +void WasapiBackend::requestCommunicationsCategory() +{ +#ifdef __IAudioClient2_INTERFACE_DEFINED__ + // This is the part that buys us the platform's echo cancellation. Marking + // the stream AudioCategory_Communications before Initialize() puts it + // through the communications signal processing chain, which is where the + // AEC, noise suppression and gain control that Windows (or the endpoint's + // own APO) provides live. Without it we get the raw capture stream and + // have to cancel the echo ourselves. + // + // Whether an AEC is actually in that chain depends on the driver, so this + // is a request, not a guarantee - see hasPlatformEchoCancellation(). + IAudioClient2 *client2 = nullptr; + if (SUCCEEDED(m_client->QueryInterface(IID_IAudioClient2, (void **)&client2)) && client2) + { + AudioClientProperties props; + // ZeroMemory also covers Options, which only exists from the Windows + // 8.1 SDK on and whose "none" value is zero in every version that has + // it - so this compiles against either header. + ZeroMemory(&props, sizeof(props)); + props.cbSize = sizeof(props); + props.bIsOffload = FALSE; + props.eCategory = AudioCategory_Communications; + + HRESULT phr = client2->SetClientProperties(&props); + if (SUCCEEDED(phr)) + m_platformAec = true; + else + qWarning() << "WASAPI: SetClientProperties failed" << hrString(phr) + << "- no platform echo cancellation on this stream"; + + client2->Release(); + } + else + { + qWarning() << "WASAPI: IAudioClient2 not available - no platform echo cancellation on this stream"; + } +#else + qWarning() << "WASAPI: built without IAudioClient2 - no platform echo cancellation on this stream"; +#endif +} + +bool WasapiBackend::open(const QString &device, int sampleRate, int channels, bool isCapture, + NativeAudioMode mode, QString *error) +{ + close(); + + if (!initCom()) + { + if (error) + *error = QStringLiteral("Could not initialise COM on the audio thread"); + return false; + } + + m_isCapture = isCapture; + m_sampleRate = sampleRate; + m_channels = channels; + + IMMDevice *dev = resolveDevice(device, isCapture, mode, error); + if (!dev) + { + close(); + return false; + } + + HRESULT hr = dev->Activate(IID_IAudioClient, CLSCTX_ALL, nullptr, (void **)&m_client); + dev->Release(); + if (FAILED(hr) || !m_client) + { + if (error) + *error = QString("Could not activate the audio client: %1").arg(hrString(hr)); + close(); + return false; + } + + // Capture only, deliberately. + // + // On Windows the echo canceller lives in the capture endpoint's + // processing chain and takes its reference from the render endpoint + // itself, not from any particular render stream - so a render stream + // gains nothing from the communications category, while still paying for + // whatever the endpoint's communications APO does to it. Which endpoint a + // call plays to is a separate question, settled by the eCommunications + // role in resolveDevice(). + // + // Asking for it on render as well is what made the settings dialog's + // microphone test come out chopped while the ring test, which opens a + // plain stream, was clean. + if (mode == NativeAudioVoiceCall && isCapture) + requestCommunicationsCategory(); + + // Always run on the engine's own mix format and convert on this side, + // rather than asking WASAPI to hand us mono 16 bit through + // AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM. + // + // The flag is documented to insert a channel matrixer and a rate + // converter, and Initialize() reports success when it is set - but there + // is no way afterwards to ask what format the stream really ended up in, + // so a driver that quietly ignores it leaves us reading a stereo buffer + // as if it were mono. That takes half the samples of every packet, + // discards the rest with ReleaseBuffer(), and renders interleaved L,R + // pairs as consecutive mono samples: audio that is both chopped and + // running at half speed, which is exactly what the microphone test did. + // + // The mix format is the one format shared mode is guaranteed to accept, + // GetMixFormat() tells us exactly what it is, and libswresample is + // already linked and already doing this work on every other platform. + // One code path, no trust required. + hr = m_client->GetMixFormat(&m_mixFormat); + if (FAILED(hr) || !m_mixFormat) + { + if (error) + *error = QString("Could not query the audio engine mix format: %1").arg(hrString(hr)); + close(); + return false; + } + + m_devFormat = waveFormatToAv(m_mixFormat); + if (m_devFormat == AV_SAMPLE_FMT_NONE) + { + if (error) + *error = QString("The audio engine mix format is not one we can convert (tag %1, %2 bits)") + .arg(m_mixFormat->wFormatTag).arg(m_mixFormat->wBitsPerSample); + close(); + return false; + } + + m_devSampleRate = (int)m_mixFormat->nSamplesPerSec; + m_devChannels = (int)m_mixFormat->nChannels; + m_devBytesPerFrame = (int)m_mixFormat->nBlockAlign; + + hr = m_client->Initialize(AUDCLNT_SHAREMODE_SHARED, AUDCLNT_STREAMFLAGS_EVENTCALLBACK, + kBufferDuration, 0, m_mixFormat, nullptr); + if (FAILED(hr)) + { + if (error) + *error = QString("Could not initialise the audio client: %1").arg(hrString(hr)); + close(); + return false; + } + + if (isCapture) + m_swr = makeResampler(m_devFormat, m_devSampleRate, m_devChannels, + AV_SAMPLE_FMT_S16, m_sampleRate, m_channels); + else + m_swr = makeResampler(AV_SAMPLE_FMT_S16, m_sampleRate, m_channels, + m_devFormat, m_devSampleRate, m_devChannels); + + if (!m_swr) + { + if (error) + *error = QStringLiteral("Could not set up the audio format converter"); + close(); + return false; + } + + m_event = CreateEventW(nullptr, FALSE, FALSE, nullptr); + if (!m_event) + { + if (error) + *error = QStringLiteral("Could not create the audio event"); + close(); + return false; + } + + hr = m_client->SetEventHandle(m_event); + if (FAILED(hr)) + { + if (error) + *error = QString("Could not attach the audio event: %1").arg(hrString(hr)); + close(); + return false; + } + + hr = m_client->GetBufferSize(&m_bufferFrames); + if (FAILED(hr)) + { + if (error) + *error = QString("Could not query the audio buffer size: %1").arg(hrString(hr)); + close(); + return false; + } + + if (isCapture) + hr = m_client->GetService(IID_IAudioCaptureClient, (void **)&m_capture); + else + hr = m_client->GetService(IID_IAudioRenderClient, (void **)&m_render); + + if (FAILED(hr)) + { + if (error) + *error = QString("Could not get the audio %1 service: %2") + .arg(isCapture ? "capture" : "render", hrString(hr)); + close(); + return false; + } + + // Deliberately not pre-filling a render stream with silence here. It + // would buy nothing - the engine plays silence for an empty buffer by + // itself, and for a call the jitter buffer's own preroll has emptied it + // again long before the first packet arrives - while costing a whole + // buffer of added latency on every stream, and making the first write of + // a tone wait for a buffer of silence to drain before anything is heard. + + hr = m_client->Start(); + if (FAILED(hr)) + { + if (error) + *error = QString("Could not start the audio stream: %1").arg(hrString(hr)); + close(); + return false; + } + m_started = true; + + qDebug() << "WASAPI:" << (isCapture ? "capture" : "playback") << "on" << device + << "- engine" << m_devSampleRate << "Hz" << m_devChannels << "ch" + << av_get_sample_fmt_name(m_devFormat) + << "<->" << m_sampleRate << "Hz" << m_channels << "ch s16" + << ", buffer" << m_bufferFrames << "frames" + << (m_platformAec ? ", platform echo cancellation requested" : ""); + + return true; +} + +// Turns one captured packet into our own format and distributes it between +// the caller's buffer and the spill buffer. Returns the new written count. +int WasapiBackend::appendConverted(const BYTE *deviceData, int deviceFrames, bool silent, + qint16 *buf, int maxSamples, int written) +{ + const qint16 *src = nullptr; + int samples = 0; + + if (!m_swr) + { + samples = deviceFrames * m_channels; + if (silent) + { + m_convIn.resize(samples); + memset(m_convIn.data(), 0, samples * sizeof(qint16)); + src = m_convIn.constData(); + } + else + { + src = (const qint16 *)deviceData; + } + } + else + { + // Worst case output length, allowing for whatever the resampler is + // still holding on to. + int outFrames = (int)av_rescale_rnd(swr_get_delay(m_swr, m_devSampleRate) + deviceFrames, + m_sampleRate, m_devSampleRate, AV_ROUND_UP); + if (outFrames <= 0) + return written; + + m_convIn.resize(outFrames * m_channels); + + QVector<quint8> silence; + const quint8 *in = deviceData; + if (silent) + { + silence.resize(deviceFrames * m_devBytesPerFrame); + memset(silence.data(), 0, silence.size()); + in = silence.constData(); + } + + uint8_t *outPlanes[1] = { (uint8_t *)m_convIn.data() }; + const uint8_t *inPlanes[1] = { (const uint8_t *)in }; + + int got = swr_convert(m_swr, outPlanes, outFrames, inPlanes, deviceFrames); + if (got <= 0) + return written; + + samples = got * m_channels; + src = m_convIn.constData(); + } + + const int room = maxSamples - written; + const int n = samples < room ? samples : room; + + if (n > 0) + memcpy(buf + written, src, n * sizeof(qint16)); + + if (samples > n) + { + int extra = samples - n; + int base = m_spill.size(); + m_spill.resize(base + extra); + memcpy(m_spill.data() + base, src + n, extra * sizeof(qint16)); + } + + return written + n; +} + +int WasapiBackend::read(qint16 *buf, int maxSamples, int timeoutMs) +{ + if (!m_capture || !m_event) + return -1; + + int written = 0; + + // Anything a previous read() could not fit goes out first. + if (m_spillOffset < m_spill.size()) + { + int avail = m_spill.size() - m_spillOffset; + int n = avail < maxSamples ? avail : maxSamples; + memcpy(buf, m_spill.constData() + m_spillOffset, n * sizeof(qint16)); + m_spillOffset += n; + written += n; + + if (m_spillOffset >= m_spill.size()) + { + m_spill.clear(); + m_spillOffset = 0; + } + + if (written >= maxSamples) + return written; + } + + DWORD wait = WaitForSingleObject(m_event, timeoutMs < 0 ? INFINITE : (DWORD)timeoutMs); + if (wait == WAIT_TIMEOUT) + return written; + if (wait != WAIT_OBJECT_0) + return written > 0 ? written : -1; + + for (;;) + { + UINT32 packetFrames = 0; + HRESULT hr = m_capture->GetNextPacketSize(&packetFrames); + if (FAILED(hr)) + return written > 0 ? written : -1; + if (packetFrames == 0) + break; + + BYTE *data = nullptr; + UINT32 frames = 0; + DWORD flags = 0; + hr = m_capture->GetBuffer(&data, &frames, &flags, nullptr, nullptr); + if (hr == AUDCLNT_S_BUFFER_EMPTY) + { + // A success code, so the buffer is still ours to give back even + // though it holds nothing. Walking away without releasing it + // makes every later GetBuffer fail with AUDCLNT_E_OUT_OF_ORDER, + // which would kill capture for the rest of the call. + m_capture->ReleaseBuffer(0); + break; + } + if (FAILED(hr)) + return written > 0 ? written : -1; + + written = appendConverted(data, (int)frames, (flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0, + buf, maxSamples, written); + + // The packet has to be released whole, which is why anything that did + // not fit went to the spill buffer rather than being left behind. + m_capture->ReleaseBuffer(frames); + + if (written >= maxSamples) + break; + } + + return written; +} + +int WasapiBackend::write(const qint16 *buf, int nsamples, int timeoutMs) +{ + if (!m_render || !m_client || !m_event) + return -1; + + const BYTE *deviceBytes = nullptr; + int framesWanted = 0; + + if (!m_swr) + { + deviceBytes = (const BYTE *)buf; + framesWanted = nsamples / m_channels; + } + else + { + int inFrames = nsamples / m_channels; + int outFrames = (int)av_rescale_rnd(swr_get_delay(m_swr, m_sampleRate) + inFrames, + m_devSampleRate, m_sampleRate, AV_ROUND_UP); + if (outFrames <= 0) + return nsamples; + + m_convOut.resize(outFrames * m_devBytesPerFrame); + + uint8_t *outPlanes[1] = { (uint8_t *)m_convOut.data() }; + const uint8_t *inPlanes[1] = { (const uint8_t *)buf }; + + int got = swr_convert(m_swr, outPlanes, outFrames, inPlanes, inFrames); + if (got <= 0) + return nsamples; + + deviceBytes = (const BYTE *)m_convOut.constData(); + framesWanted = got; + } + + int framesDone = 0; + + // Deadline rather than a per-wait timeout. One WaitForSingleObject() + // returns after a single device period, which is smaller than the block + // the caller hands us, so filling a block normally takes several waits - + // an earlier version of this allowed only one and silently dropped the + // remainder of every block, which sounds like the tone being chopped up. + // Subtracting tick counts is wrap safe as long as it is done in DWORD. + const DWORD start = GetTickCount(); + + while (framesDone < framesWanted) + { + UINT32 padding = 0; + HRESULT hr = m_client->GetCurrentPadding(&padding); + if (FAILED(hr)) + return framesDone > 0 ? nsamples : -1; + + UINT32 freeFrames = m_bufferFrames > padding ? m_bufferFrames - padding : 0; + if (freeFrames == 0) + { + DWORD remaining; + + if (timeoutMs < 0) + { + remaining = INFINITE; + } + else + { + DWORD elapsed = GetTickCount() - start; + if (elapsed >= (DWORD)timeoutMs) + break; + remaining = (DWORD)timeoutMs - elapsed; + } + + DWORD wait = WaitForSingleObject(m_event, remaining); + if (wait == WAIT_TIMEOUT) + break; + if (wait != WAIT_OBJECT_0) + return framesDone > 0 ? nsamples : -1; + continue; + } + + UINT32 chunk = (UINT32)(framesWanted - framesDone); + if (chunk > freeFrames) + chunk = freeFrames; + + BYTE *data = nullptr; + hr = m_render->GetBuffer(chunk, &data); + if (FAILED(hr) || !data) + return framesDone > 0 ? nsamples : -1; + + memcpy(data, deviceBytes + (size_t)framesDone * m_devBytesPerFrame, + (size_t)chunk * m_devBytesPerFrame); + m_render->ReleaseBuffer(chunk, 0); + + framesDone += (int)chunk; + } + + // Report against what the caller gave us, not against the post-conversion + // frame count, since a short write here is a dropped tail either way. + if (framesDone >= framesWanted) + return nsamples; + + return (int)((qint64)nsamples * framesDone / (framesWanted > 0 ? framesWanted : 1)); +} + +void WasapiBackend::close() +{ + if (m_client && m_started) + { + // Stop() cuts off whatever is still in the buffer, which for a tone + // means losing its last few tens of milliseconds. Give the device a + // chance to play out what it has first - bounded, because a stream + // that has stopped advancing must not hold up the call teardown. + if (m_render && m_event) + { + const DWORD start = GetTickCount(); + const DWORD limit = 300; + + for (;;) + { + UINT32 padding = 0; + if (FAILED(m_client->GetCurrentPadding(&padding)) || padding == 0) + break; + + DWORD elapsed = GetTickCount() - start; + if (elapsed >= limit) + break; + + if (WaitForSingleObject(m_event, limit - elapsed) != WAIT_OBJECT_0) + break; + } + } + + m_client->Stop(); + m_started = false; + } + + if (m_capture) + { + m_capture->Release(); + m_capture = nullptr; + } + if (m_render) + { + m_render->Release(); + m_render = nullptr; + } + if (m_client) + { + m_client->Release(); + m_client = nullptr; + } + if (m_event) + { + CloseHandle(m_event); + m_event = nullptr; + } + if (m_mixFormat) + { + CoTaskMemFree(m_mixFormat); + m_mixFormat = nullptr; + } + if (m_swr) + { + swr_free(&m_swr); + m_swr = nullptr; + } + + m_spill.clear(); + m_spillOffset = 0; + m_convIn.clear(); + m_convOut.clear(); + m_platformAec = false; + + if (m_comInitialized) + { + CoUninitialize(); + m_comInitialized = false; + } +} + +NativeAudioBackend *pica_create_wasapi_backend() +{ + return new WasapiBackend(); +} + +QList<MediaDeviceInfo> pica_enumerate_wasapi(enum MediaDeviceStreamDirection dir) +{ + QList<MediaDeviceInfo> result; + + // The settings dialog calls this from the GUI thread, which Qt has + // already put into an apartment; the extra reference is still ours to + // balance, hence the matching CoUninitialize() below. + bool didInit = false; + HRESULT hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + if (SUCCEEDED(hr)) + didInit = true; + else if (hr != RPC_E_CHANGED_MODE) + return result; + + IMMDeviceEnumerator *devEnum = nullptr; + hr = CoCreateInstance(CLSID_MMDeviceEnumerator, nullptr, CLSCTX_ALL, + IID_IMMDeviceEnumerator, (void **)&devEnum); + if (FAILED(hr) || !devEnum) + { + qWarning() << "WASAPI: could not create the endpoint enumerator" << hrString(hr); + if (didInit) + CoUninitialize(); + return result; + } + + int index = 0; + + // The system default first, under the same "default" name every other + // driver in this codebase uses for it. + // + // Deliberately not named after a particular endpoint role. The same list + // fills the call devices and the ring device, and "default" resolves to a + // different endpoint for each: a call takes the communications one, a + // ringtone the console one, so that the bell is heard on the speakers + // rather than in a headset nobody is wearing yet. See resolveDevice(). + { + MediaDeviceInfo d; + d.device = QStringLiteral("default"); + d.humanReadable = (dir == PLAYBACK) + ? QObject::tr("System default output") + : QObject::tr("System default input"); + d.index = index++; + result << d; + } + + IMMDeviceCollection *collection = nullptr; + hr = devEnum->EnumAudioEndpoints(dir == PLAYBACK ? eRender : eCapture, + DEVICE_STATE_ACTIVE, &collection); + if (SUCCEEDED(hr) && collection) + { + UINT count = 0; + collection->GetCount(&count); + + for (UINT i = 0; i < count; i++) + { + IMMDevice *dev = nullptr; + if (FAILED(collection->Item(i, &dev)) || !dev) + continue; + + LPWSTR id = nullptr; + if (SUCCEEDED(dev->GetId(&id)) && id) + { + MediaDeviceInfo d; + d.device = wideToQString(id); + d.humanReadable = d.device; + + IPropertyStore *props = nullptr; + if (SUCCEEDED(dev->OpenPropertyStore(STGM_READ, &props)) && props) + { + PROPVARIANT name; + PropVariantInit(&name); + if (SUCCEEDED(props->GetValue(PKEY_Device_FriendlyName, &name)) && + name.vt == VT_LPWSTR) + { + d.humanReadable = wideToQString(name.pwszVal); + } + PropVariantClear(&name); + props->Release(); + } + + d.index = index++; + result << d; + + CoTaskMemFree(id); + } + + dev->Release(); + } + + collection->Release(); + } + + devEnum->Release(); + + if (didInit) + CoUninitialize(); + + return result; +} + +#endif // Q_OS_WIN diff --git a/pica-client/pica-client.pro b/pica-client/pica-client.pro index 43111d2..4973c2e 100644 --- a/pica-client/pica-client.pro +++ b/pica-client/pica-client.pro @@ -12,6 +12,10 @@ TEMPLATE = app SOURCES += main.cpp\ audiodevice.cpp \ + echocanceller.cpp \ + nativeaudio.cpp \ + nativeaudio_wasapi.cpp \ + nativeaudio_coreaudio.cpp \ audiovideocallcontroller.cpp \ mainwindow.cpp \ chatwindow.cpp \ @@ -51,6 +55,9 @@ SOURCES += main.cpp\ HEADERS += mainwindow.h \ audiodevice.h \ + audioring.h \ + echocanceller.h \ + nativeaudio.h \ audiovideocallcontroller.h \ chatwindow.h \ callwindow.h \ @@ -106,3 +113,22 @@ unix: CONFIG += link_pkgconfig unix: PKGCONFIG += libcrypto unix|win32: LIBS += -lminiupnpc + +# Acoustic echo cancellation. Required everywhere: on Linux there is no other +# cancellation in the picture, and on the platforms that have their own it is +# still the fallback for the endpoints and drivers that turn out not to. +unix:!macx: CONFIG += link_pkgconfig +unix:!macx: PKGCONFIG += speexdsp + +# Listing PulseAudio sources and sinks in the settings dialog. Optional - +# playing and capturing through PulseAudio goes via FFmpeg's own "pulse" +# device and needs nothing from us. +unix:!macx: PKGCONFIG += libpulse +unix:!macx: DEFINES += HAVE_LIBPULSE + +# The native audio backends. Both source files compile to nothing on the +# platforms they are not for, so they can stay in SOURCES unconditionally; +# what they link against cannot. +win32: LIBS += -lole32 -loleaut32 +macx: LIBS += -framework CoreAudio -framework AudioUnit -framework AudioToolbox -framework CoreFoundation +macx: LIBS += -lspeexdsp diff --git a/pica-client/toneplayer/toneplayer.cpp b/pica-client/toneplayer/toneplayer.cpp index d40e1e9..298686a 100644 --- a/pica-client/toneplayer/toneplayer.cpp +++ b/pica-client/toneplayer/toneplayer.cpp @@ -16,8 +16,10 @@ */ #include "toneplayer.h" #include "../audiodevice.h" +#include "../nativeaudio.h" #include <QDebug> +#include <QVector> #include <cmath> // FFmpeg headers @@ -153,6 +155,29 @@ static QString ff_errstr(int err) return QString::fromLocal8Bit(buf); } +void TonePlayer::renderBlock(const Tone &t, double &phase, int nb_samples, int channels, qint16 *out) +{ + const double two_pi = 2.0 * M_PI; + double phase_inc = 0.0; + + if (t.frequency > 0.0) + phase_inc = two_pi * t.frequency / double(m_sampleRate); + + for (int i = 0; i < nb_samples; ++i) { + double sampleVal = 0.0; + if (t.frequency > 0.0) { + sampleVal = sin(phase) * m_volume; + phase += phase_inc; + if (phase > 1e9 || phase < -1e9) phase = fmod(phase, two_pi); + } + + int16_t ival = static_cast<int16_t>(qBound(-1.0, sampleVal, 1.0) * 32767.0); + for (int ch = 0; ch < channels; ++ch) { + out[i * channels + ch] = ival; + } + } +} + void TonePlayer::play() { if (m_sequence.isEmpty()) { @@ -160,6 +185,87 @@ void TonePlayer::play() return; } + const QString driver = AudioDevice::PlatformDriverName(PLAYBACK); + + if (AudioDevice::IsPlatformDriver(driver)) + playNative(AudioDevice::PlatformApiName(driver)); + else + playFFmpeg(driver); +} + +void TonePlayer::playNative(const QString &api) +{ + NativeAudioBackend *backend = NativeAudioBackend::create(api); + if (!backend) { + QString msg = QString("No '%1' audio backend in this build").arg(api); + qWarning() << msg; + emit errorOccured(msg); + emit finishedPlaying(); + return; + } + + QString err; + if (!backend->openPlayback(m_deviceName, m_sampleRate, m_channels, NativeAudioPlain, &err)) { + qWarning() << err; + emit errorOccured(err); + delete backend; + emit finishedPlaying(); + return; + } + + // 20 ms a time, so stop() is acted on promptly without the write loop + // spinning. + const int blockFrames = m_sampleRate / 50; + QVector<qint16> block(blockFrames * m_channels); + double phase = 0.0; + + for (int idx = 0; idx < m_sequence.size(); ++idx) { + if (m_abort.loadRelaxed()) break; + + const Tone &t = m_sequence[idx]; + if (t.frequency == 0.0 && t.duration_ms == 0) + break; + if (t.duration_ms <= 0) + continue; + + int64_t total_samples = (int64_t)qRound((m_sampleRate * t.duration_ms) / 1000.0); + int64_t samples_written = 0; + + while (samples_written < total_samples) { + if (m_abort.loadRelaxed()) break; + + int nb_samples = blockFrames; + if (nb_samples > (int)(total_samples - samples_written)) + nb_samples = int(total_samples - samples_written); + + renderBlock(t, phase, nb_samples, m_channels, block.data()); + + if (backend->write(block.constData(), nb_samples * m_channels, 200) < 0) { + qWarning() << "Native tone playback write failed"; + samples_written = total_samples; + idx = m_sequence.size(); + break; + } + + samples_written += nb_samples; + } + } + + // Let what is still sitting in the device buffer be heard rather than + // being cut off by close(). One buffer's worth is the most it can be. + if (!m_abort.loadRelaxed()) { + memset(block.data(), 0, block.size() * sizeof(qint16)); + backend->write(block.constData(), block.size(), 200); + } + + backend->close(); + delete backend; + + emit finishedPlaying(); +} + +void TonePlayer::playFFmpeg(const QString &driver) +{ avdevice_register_all(); #if LIBAVFORMAT_VERSION_MAJOR < 58 av_register_all(); @@ -169,10 +275,10 @@ void TonePlayer::play() AVFormatContext *oc = nullptr; const QByteArray deviceUtf8 = m_deviceName.toUtf8(); const char *device = deviceUtf8.constData(); - const QByteArray driverUtf8 = AudioDevice::PlatformDriverName().toUtf8(); - const char *driver = driverUtf8.constData(); + const QByteArray driverUtf8 = driver.toUtf8(); + const char *driverName = driverUtf8.constData(); - int ret = avformat_alloc_output_context2(&oc, nullptr, driver, device); + int ret = avformat_alloc_output_context2(&oc, nullptr, driverName, device); if (ret < 0 || !oc) { QString msg = QString("Could not allocate '%1' output context: %2").arg(driver, ff_errstr(ret)); qWarning() << msg; @@ -306,7 +412,6 @@ void TonePlayer::play() int64_t pts = 0; double phase = 0.0; - const double two_pi = 2.0 * M_PI; #if TP_HAVE_CH_LAYOUT int chs = codecCtx->ch_layout.nb_channels; @@ -328,11 +433,6 @@ void TonePlayer::play() int64_t total_samples = (int64_t)qRound((m_sampleRate * t.duration_ms) / 1000.0); int64_t samples_written = 0; - double phase_inc = 0.0; - if (t.frequency > 0.0) { - phase_inc = two_pi * t.frequency / double(m_sampleRate); - } - while (samples_written < total_samples) { if (m_abort.loadRelaxed()) break; @@ -348,23 +448,7 @@ void TonePlayer::play() } frame->nb_samples = nb_samples; - int16_t *out = reinterpret_cast<int16_t*>(frame->data[0]); - - for (int i = 0; i < nb_samples; ++i) { - double sampleVal = 0.0; - if (t.frequency > 0.0) { - sampleVal = sin(phase) * m_volume; - phase += phase_inc; - if (phase > 1e9 || phase < -1e9) phase = fmod(phase, two_pi); - } else { - sampleVal = 0.0; - } - - int16_t ival = static_cast<int16_t>(qBound(-1.0, sampleVal, 1.0) * 32767.0); - for (int ch = 0; ch < chs; ++ch) { - out[i * chs + ch] = ival; - } - } + renderBlock(t, phase, nb_samples, chs, reinterpret_cast<int16_t*>(frame->data[0])); frame->pts = pts; pts += frame->nb_samples; diff --git a/pica-client/toneplayer/toneplayer.h b/pica-client/toneplayer/toneplayer.h index b3e237d..5c70e15 100644 --- a/pica-client/toneplayer/toneplayer.h +++ b/pica-client/toneplayer/toneplayer.h @@ -72,6 +72,19 @@ private: // Set the sequence, clear any pending stop request and play it. void playSequence(const QVector<Tone> &sequence); + // The two ways of getting samples to the speaker, picked apart by + // AudioDevice::PlatformDriverName(PLAYBACK) - see the comment on it. + // Tones always ask for a plain stream rather than a call one: a ringtone + // is not a call, and on macOS asking for the voice processing would open + // the microphone to play it. + void playFFmpeg(const QString &driver); + void playNative(const QString &api); + + // Fills out with nb_samples frames of a sine at the tone's frequency + // (silence when it is zero), carrying phase across calls so consecutive + // blocks of one tone do not click at the seam. + void renderBlock(const Tone &t, double &phase, int nb_samples, int channels, qint16 *out); + QVector<Tone> m_sequence; QAtomicInt m_abort; QString m_deviceName; |